Compare commits

..

1 Commits

Author SHA1 Message Date
agent_coder 4b2af3d34a refactor(mcp): дедупликация конвертер-смежных хелперов (node-ops форк, footnote-*, parse-node-arg)
Аудит при подготовке #413 нашёл дрейфующие дубли между packages/mcp и
packages/prosemirror-markdown. Четыре дедупа (поведение тулов не меняется):

1. node-ops: форк ~960 строк сведён в ОДНУ копию в prosemirror-markdown (живая
   mcp-версия — строгое надмножество замороженного #293-seed'а пакета; сверено по
   git-истории, новая пакет-копия байт-в-байт == прежней mcp-копии). Barrel-экспорт
   полной поверхности; mcp/client.ts/page-search.ts/transforms.ts/collaboration.ts
   импортируют из пакета; тесты переехали. node-ops тянет stripInlineMarkdown ->
   пакет-локальная text-normalize.ts несёт только этот примитив (mcp-версия —
   домен #408; заголовок документирует дубликацию + источник истины).
2. footnote-lex/footnote-analyze (vestigial legacy [^id]: диагностика): сведены к
   одному fence-aware предупреждению 'reference-style footnotes -> use ^[...]'
   (полезно для класса #410); footnote-lex удалён.
3. footnote-authoring -> примитивы (footnoteContentKey/makeFootnoteDefinition/
   generateFootnoteId) перенесены в пакетный footnote.ts, одна реализация конвенции.
4. parse-node-arg -> перенесён в prosemirror-markdown (не mcp: сервер CommonJS не
   импортирует ESM-only @docmost/mcp, но нативно импортирует пакет), обе копии
   удалены, консьюмеры перенаправлены.

canonicalizeFootnotes/ENFORCEMENT RULE #228 и comment-anchor/json-edit/text-normalize
(mcp) не тронуты. API-поверхность node-ops оставлена чистой для #409/#413.

closes #414

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-10 01:06:31 +03:00
32 changed files with 849 additions and 2547 deletions
@@ -17,7 +17,7 @@ import {
resolveCurrentPageResult, resolveCurrentPageResult,
type SelectionContext, type SelectionContext,
} from './current-page.util'; } from './current-page.util';
import { parseNodeArg } from './parse-node-arg'; import { parseNodeArg } from '@docmost/prosemirror-markdown';
import { modelFriendlyInput } from './model-friendly-input'; import { modelFriendlyInput } from './model-friendly-input';
import { SandboxStore } from '../../../integrations/sandbox/sandbox.store'; import { SandboxStore } from '../../../integrations/sandbox/sandbox.store';
import { import {
@@ -1,10 +1,10 @@
import { parseNodeArg } from './parse-node-arg'; import { parseNodeArg } from '@docmost/prosemirror-markdown';
/** /**
* Unit tests for the in-app `parseNodeArg` helper. It mirrors the standalone * Unit tests for the shared `parseNodeArg` helper (#414: now the single copy in
* MCP helper (packages/mcp/src/lib/parse-node-arg.ts) and is used by the * `@docmost/prosemirror-markdown`, imported by both the server tool adapters and
* patchNode / insertNode / updatePageJson tool adapters. Behavior must be * `@docmost/mcp`). Used by the patchNode / insertNode / updatePageJson adapters.
* byte-identical: object passthrough, valid-string parse, invalid-string throw. * Behavior: object passthrough, valid-string parse, invalid-string throw.
*/ */
describe('parseNodeArg', () => { describe('parseNodeArg', () => {
it('passes an object through unchanged', () => { it('passes an object through unchanged', () => {
@@ -1,26 +0,0 @@
// The model sometimes serializes a ProseMirror node arg as a JSON string
// instead of an object. Normalize: parse a string to an object (throwing on
// invalid JSON), pass an object through unchanged. Shared by patchNode /
// insertNode (and the analogous updatePageJson content parsing).
//
// This is behaviorally identical to `packages/mcp/src/lib/parse-node-arg.ts`
// (the function logic, default/explicit throw messages and branch order match;
// only comments and quote style differ). We cannot import that helper here:
// `@docmost/mcp` is ESM-only and this server
// compiles with module:commonjs, so it is loaded at runtime via the
// `new Function('import()')` trick (see docmost-client.loader.ts). Sharing
// runtime code across that ESM/CJS boundary by a normal import is impossible,
// hence the mirrored copy.
export function parseNodeArg(
node: unknown,
errMsg = 'node was a string but not valid JSON',
): unknown {
if (typeof node === 'string') {
try {
return JSON.parse(node);
} catch {
throw new Error(errMsg);
}
}
return node;
}
+173 -23
View File
@@ -8,6 +8,10 @@ import {
filterComment, filterComment,
filterSearchResult, filterSearchResult,
} from "./lib/filters.js"; } from "./lib/filters.js";
import { HocuspocusProvider } from "@hocuspocus/provider";
import { TiptapTransformer } from "@hocuspocus/transformer";
import * as Y from "yjs";
import WebSocket from "ws";
import { convertProseMirrorToMarkdown } from "./lib/markdown-converter.js"; import { convertProseMirrorToMarkdown } from "./lib/markdown-converter.js";
import { import {
collectInternalFileNodes, collectInternalFileNodes,
@@ -20,10 +24,11 @@ import {
markdownToProseMirror, markdownToProseMirror,
markdownToProseMirrorCanonical, markdownToProseMirrorCanonical,
mutatePageContent, mutatePageContent,
buildCollabWsUrl,
assertYjsEncodable, assertYjsEncodable,
applyDocToFragment,
MutationResult, MutationResult,
} from "./lib/collaboration.js"; } from "./lib/collaboration.js";
import { acquireCollabSession } from "./lib/collab-session.js";
import { footnoteWarningsField } from "./lib/footnote-analyze.js"; import { footnoteWarningsField } from "./lib/footnote-analyze.js";
import { buildPageTree } from "./lib/tree.js"; import { buildPageTree } from "./lib/tree.js";
import { import {
@@ -41,7 +46,7 @@ import {
insertTableRow, insertTableRow,
deleteTableRow, deleteTableRow,
updateTableCell, updateTableCell,
} from "./lib/node-ops.js"; } from "@docmost/prosemirror-markdown";
import { searchInDoc, SearchOptions } from "./lib/page-search.js"; import { searchInDoc, SearchOptions } from "./lib/page-search.js";
import { withPageLock } from "./lib/page-lock.js"; import { withPageLock } from "./lib/page-lock.js";
import { import {
@@ -412,32 +417,177 @@ export class DocmostClient {
* change report. The report is computed AFTER the atomic read->write and * change report. The report is computed AFTER the atomic read->write and
* never throws. * never throws.
*/ */
private async mutateLiveContentUnlocked( private mutateLiveContentUnlocked(
pageId: string, pageId: string,
collabToken: string, collabToken: string,
transform: (liveDoc: any) => any | null, transform: (liveDoc: any) => any | null,
): Promise<MutationResult> { ): Promise<MutationResult> {
// Reuse a live CollabSession for the page (issue #400) instead of opening a const CONNECT_TIMEOUT_MS = 25000;
// fresh provider per op. acquireCollabSession does NOT take the per-page const PERSIST_TIMEOUT_MS = 20000;
// lock — the caller (replaceImage) already holds ONE withPageLock across its const ydoc = new Y.Doc();
// scan -> upload -> write sequence, and the mutex is not reentrant, so const wsUrl = buildCollabWsUrl(this.apiUrl);
// taking it here would deadlock. The synchronous read->write section and the
// unsyncedChanges/connectionLost ack logic live in CollabSession.mutate, return new Promise<MutationResult>((resolve, reject) => {
// preserved verbatim from the old inline machine (incl. the #152 structural let provider: HocuspocusProvider | undefined;
// diff that keeps a live editor's cursor anchored). let applied = false; // onSynced may fire again on reconnect — apply once.
const session = await acquireCollabSession(pageId, collabToken, this.apiUrl, { let settled = false;
// Only the actual 25s collab connect timeout emits this — the connect-vs- let connectionLost = false;
// unload signal; the other failure paths must NOT emit it. let connectTimer: ReturnType<typeof setTimeout> | undefined;
onConnectTimeout: () => let persistTimer: ReturnType<typeof setTimeout> | undefined;
this.onMetricFn?.("collab_connect_timeouts_total", 1), let unsyncedHandler: ((data: { number: number }) => void) | undefined;
// The verifiable result resolved on every success/abort path. Set on abort
// (no-op report) and after a real write (computed change report).
let mutationResult: MutationResult;
const cleanup = () => {
if (connectTimer) clearTimeout(connectTimer);
if (persistTimer) clearTimeout(persistTimer);
if (provider) {
if (unsyncedHandler) {
try {
provider.off("unsyncedChanges", unsyncedHandler);
} catch (err) {}
}
try {
provider.destroy();
} catch (err) {}
}
};
const finish = (err: Error | null, value?: MutationResult) => {
if (settled) return;
settled = true;
cleanup();
if (err) reject(err);
else resolve(value as MutationResult);
};
connectTimer = setTimeout(() => {
// Only the actual 25s collab connect timeout fires here — the agent's
// collab connection to the server never became ready. This is the
// connect-vs-unload signal; the other finish() paths must NOT emit it.
this.onMetricFn?.("collab_connect_timeouts_total", 1);
finish(new Error("Connection timeout to collaboration server"));
}, CONNECT_TIMEOUT_MS);
const waitForPersistence = () => {
if (settled) return;
if (!provider) {
finish(new Error("collab provider gone before persistence"));
return;
}
if (provider.unsyncedChanges === 0) {
finish(null, mutationResult);
return;
}
persistTimer = setTimeout(() => {
finish(
new Error(
"Timeout waiting for collaboration server to persist the update",
),
);
}, PERSIST_TIMEOUT_MS);
unsyncedHandler = (data: { number: number }) => {
if (data.number === 0 && !connectionLost) {
finish(null, mutationResult);
}
};
provider.on("unsyncedChanges", unsyncedHandler);
};
provider = new HocuspocusProvider({
url: wsUrl,
name: `page.${pageId}`,
document: ydoc,
token: collabToken,
// @ts-ignore - Required for Node.js environment
WebSocketPolyfill: WebSocket,
onDisconnect: () => {
connectionLost = true;
finish(
new Error(
"Collaboration connection closed before the update was persisted/synced",
),
);
},
onClose: () => {
connectionLost = true;
finish(
new Error(
"Collaboration connection closed before the update was persisted/synced",
),
);
},
onSynced: () => {
if (applied || settled) return;
applied = true;
// CRITICAL: keep everything between reading and writing the live doc
// synchronous (no await) so no remote update can interleave.
let newDoc: any;
let beforeDoc: any;
try {
let liveDoc = TiptapTransformer.fromYdoc(ydoc, "default");
if (
!liveDoc ||
typeof liveDoc !== "object" ||
!Array.isArray(liveDoc.content)
) {
liveDoc = { type: "doc", content: [] };
}
// Snapshot the before-doc for the change report (safe deep clone).
beforeDoc = JSON.parse(JSON.stringify(liveDoc));
newDoc = transform(liveDoc);
if (newDoc == null) {
// Transform aborted — write nothing, return the live doc with a
// no-op change report.
mutationResult = {
doc: liveDoc,
verify: {
changed: false,
textInserted: 0,
textDeleted: 0,
blocksChanged: 0,
marks: {},
summary: "no changes (transform aborted)",
},
};
finish(null, mutationResult);
return;
}
// Structural diff into the live fragment (issue #152), mirroring
// the main write path: preserves the Yjs ids of unchanged nodes so
// an open editor's cursor is not yanked to the end of the document.
// The previous destructive rewrite (delete-all + applyUpdate of a
// fresh Y.Doc) discarded every node id, so replaceImage — the only
// caller of this method — still reproduced the #152 cursor jump
// (#164). applyDocToFragment runs its own atomic `transact`.
applyDocToFragment(ydoc, newDoc);
} catch (e) {
finish(e instanceof Error ? e : new Error(String(e)));
return;
}
// Compute the verifiable change report AFTER the transact write: it
// only needs the JSON before/after, so it cannot affect the atomic
// read->write window, and summarizeChange never throws.
mutationResult = {
doc: newDoc,
verify: summarizeChange(beforeDoc, newDoc),
};
waitForPersistence();
},
onAuthenticationFailed: () => {
finish(
new Error("Authentication failed for collaboration connection"),
);
},
});
}); });
try {
return await session.mutate(transform);
} catch (e) {
// Drop the session on any failure so the next call reconnects fresh.
session.destroy("mutate failed");
throw e;
}
} }
/** /**
+1 -6
View File
@@ -4,7 +4,7 @@ import { readFileSync } from "fs";
import { fileURLToPath } from "url"; import { fileURLToPath } from "url";
import { dirname, join } from "path"; import { dirname, join } from "path";
import { DocmostClient, DocmostMcpConfig } from "./client.js"; import { DocmostClient, DocmostMcpConfig } from "./client.js";
import { parseNodeArg } from "./lib/parse-node-arg.js"; import { parseNodeArg } from "@docmost/prosemirror-markdown";
import { SHARED_TOOL_SPECS, SharedToolSpec } from "./tool-specs.js"; import { SHARED_TOOL_SPECS, SharedToolSpec } from "./tool-specs.js";
// Re-export the client and its config type so embedding hosts (e.g. the gitmost // Re-export the client and its config type so embedding hosts (e.g. the gitmost
@@ -13,11 +13,6 @@ import { SHARED_TOOL_SPECS, SharedToolSpec } from "./tool-specs.js";
export { DocmostClient } from "./client.js"; export { DocmostClient } from "./client.js";
export type { DocmostMcpConfig } from "./client.js"; export type { DocmostMcpConfig } from "./client.js";
// Teardown for the live per-page CollabSession cache (issue #400). An embedding
// HTTP host (the gitmost NestJS server) should call this from its own shutdown
// hook so no cached collab provider outlives the process.
export { destroyAllSessions } from "./lib/collab-session.js";
// Re-export the zod-agnostic shared tool-spec registry so the in-app AI-SDK // Re-export the zod-agnostic shared tool-spec registry so the in-app AI-SDK
// service can read it off the loaded module (it cannot import the ESM package's // service can read it off the loaded module (it cannot import the ESM package's
// internals directly; it goes through loadDocmostMcp()). // internals directly; it goes through loadDocmostMcp()).
-668
View File
@@ -1,668 +0,0 @@
import { HocuspocusProvider } from "@hocuspocus/provider";
import { TiptapTransformer } from "@hocuspocus/transformer";
import * as Y from "yjs";
import WebSocket from "ws";
import {
buildCollabWsUrl,
applyDocToFragment,
MutationResult,
} from "./collaboration.js";
import { summarizeChange } from "./diff.js";
/**
* Live per-page collaboration session cache (issue #400).
*
* The one-shot write path (collaboration.mutatePageContent /
* client.mutateLiveContentUnlocked) used to open a NEW HocuspocusProvider, run
* the full connect -> auth -> onLoadDocument -> initial-sync handshake, apply a
* single edit, wait for persistence, and then `provider.destroy()` — for EVERY
* content mutation. Disconnecting after every edit means that once the pause
* between calls exceeds the server's write debounce, the server does a full
* store -> unload -> reload per cell, causing 25s connect timeouts and
* event-loop lag under a burst of edits on one page.
*
* This module keeps ONE live provider + ydoc per (wsUrl, pageId, token) alive
* across a SERIES of edits. While the provider stays connected the server never
* enters store -> unload -> reload, its debounce coalesces N writes into 1-2
* stores, and the repeated auth/load/initial-sync disappears.
*
* The synchronous read -> transform -> write section and the per-edit
* persistence-ack logic are preserved VERBATIM from the one-shot machine — the
* only change is that they run on a persistent provider instead of a throwaway
* one. See CollabSession.mutate.
*/
/** Time we wait for the initial handshake/sync before giving up. */
const CONNECT_TIMEOUT_MS = 25000;
/** Time we wait for the server to acknowledge our write before giving up. */
const PERSIST_TIMEOUT_MS = 20000;
/**
* Tunables, read fresh from the environment on every acquire so tests (and a
* live rollback) can change them without reloading the module. Mirrors how
* http.ts parses MCP_SESSION_IDLE_MS.
* - MCP_COLLAB_SESSION_IDLE_MS: idle TTL, reset after every op. Default 60s.
* 0 (or negative) DISABLES the cache — every op opens its own provider and
* destroys it after the op, i.e. the exact legacy per-op-provider behavior
* (the rollback path).
* - MCP_COLLAB_SESSION_MAX_AGE_MS: hard lifetime checked at acquire; bounds
* the permission-staleness window. Default 10 min.
* - MCP_COLLAB_SESSION_MAX_ENTRIES: registry cap; the least-recently-used
* session is destroy-evicted when the cap is reached. Default 32.
*/
interface SessionConfig {
idleMs: number;
maxAgeMs: number;
maxEntries: number;
}
function parseEnvInt(value: string | undefined, fallback: number): number {
const parsed = parseInt(value ?? "", 10);
return Number.isFinite(parsed) ? parsed : fallback;
}
function readConfig(): SessionConfig {
// idleMs: allow 0 (disable). A malformed value falls back to the default.
const idleRaw = parseInt(process.env.MCP_COLLAB_SESSION_IDLE_MS ?? "", 10);
const idleMs = Number.isFinite(idleRaw) ? Math.max(0, idleRaw) : 60 * 1000;
const maxAgeMs = Math.max(
0,
parseEnvInt(process.env.MCP_COLLAB_SESSION_MAX_AGE_MS, 10 * 60 * 1000),
);
const maxEntriesRaw = parseEnvInt(
process.env.MCP_COLLAB_SESSION_MAX_ENTRIES,
32,
);
const maxEntries = maxEntriesRaw > 0 ? maxEntriesRaw : 32;
return { idleMs, maxAgeMs, maxEntries };
}
/**
* The subset of HocuspocusProvider this module depends on, so the provider can
* be replaced with a fake in unit tests (there is no server in the test env).
*/
export interface CollabProviderLike {
synced: boolean;
unsyncedChanges: number;
destroy(): void;
on(event: "unsyncedChanges", handler: (data: { number: number }) => void): void;
off(event: "unsyncedChanges", handler: (data: { number: number }) => void): void;
}
/** The configuration object passed to the provider factory. */
export interface CollabProviderConfig {
url: string;
name: string;
document: Y.Doc;
token: string;
WebSocketPolyfill: unknown;
onConnect: () => void;
onSynced: () => void;
onDisconnect: () => void;
onClose: () => void;
onAuthenticationFailed: () => void;
}
export type CollabProviderFactory = (
config: CollabProviderConfig,
) => CollabProviderLike;
const defaultProviderFactory: CollabProviderFactory = (config) =>
// @ts-ignore - WebSocketPolyfill is required for the Node.js environment.
new HocuspocusProvider(config) as unknown as CollabProviderLike;
let providerFactory: CollabProviderFactory = defaultProviderFactory;
/**
* TEST SEAM: swap the provider factory (pass null to restore the real one).
* Not part of the public API — used only by the unit tests, which cannot reach
* a real collaboration server.
*/
export function __setCollabProviderFactory(
factory: CollabProviderFactory | null,
): void {
providerFactory = factory ?? defaultProviderFactory;
}
/** Optional per-acquire hooks (metrics), passed through from the call site. */
export interface AcquireOptions {
/** Invoked when the initial connect handshake times out (CONNECT_TIMEOUT_MS). */
onConnectTimeout?: () => void;
}
type SessionState = "connecting" | "ready" | "dead";
/**
* One live provider + ydoc for a single (wsUrl, pageId, token) triple.
*
* Lifecycle: connecting -> ready -> dead. A session becomes `dead` on the first
* disconnect/close/auth-failure at ANY time, on an idle/eviction/max-age
* teardown, or on an explicit destroy(); death is terminal and removes the
* session from the registry so the next acquire opens a fresh one. We never use
* the provider's auto-reconnect — destroying on the first disconnect closes the
* "reconnect drove unsyncedChanges to 0 without retransmitting our write" class
* of false success.
*/
export class CollabSession {
readonly key: string;
readonly pageId: string;
readonly wsUrl: string;
readonly token: string;
readonly createdAt: number;
state: SessionState = "connecting";
/**
* Set true on disconnect/close/auth-failure so a reconnect-driven
* unsyncedChanges->0 cannot be mistaken for a successful persist of our
* write (preserved verbatim from the one-shot machine).
*/
connectionLost = false;
provider: CollabProviderLike | undefined;
private readonly ydoc: Y.Doc;
private readonly cfg: SessionConfig;
/**
* Ephemeral sessions (cache disabled, MCP_COLLAB_SESSION_IDLE_MS<=0) are never
* registered and self-destroy after their single op — the legacy
* provider-per-op behavior.
*/
private readonly ephemeral: boolean;
private readonly opts: AcquireOptions | undefined;
private dead = false;
private connectTimer: ReturnType<typeof setTimeout> | undefined;
private idleTimer: ReturnType<typeof setTimeout> | undefined;
private openPromise: Promise<void> | undefined;
private openResolve: (() => void) | undefined;
private openReject: ((err: Error) => void) | undefined;
private openSettled = false;
/**
* The rejector of the CURRENT in-flight mutate, if any. A disconnect/close/
* auth-failure or timeout at ANY time rejects the in-flight op through this
* with the SAME error text the one-shot machine emitted.
*/
private inflightReject: ((err: Error) => void) | undefined;
constructor(
key: string,
pageId: string,
wsUrl: string,
token: string,
cfg: SessionConfig,
ephemeral: boolean,
opts: AcquireOptions | undefined,
) {
this.key = key;
this.pageId = pageId;
this.wsUrl = wsUrl;
this.token = token;
this.cfg = cfg;
this.ephemeral = ephemeral;
this.opts = opts;
this.createdAt = Date.now();
this.ydoc = new Y.Doc();
}
/**
* A cached session may be reused only when it is fully ready, still synced,
* has not lost its connection, and has not exceeded its max age (invariant 5
* "validate on reuse" + the max-age acquire check).
*/
isReusable(): boolean {
return (
!this.dead &&
this.state === "ready" &&
!this.connectionLost &&
!!this.provider &&
this.provider.synced === true &&
Date.now() - this.createdAt < this.cfg.maxAgeMs
);
}
/**
* Connect and wait for the initial sync (onSynced) within CONNECT_TIMEOUT_MS.
* Idempotent: repeated calls return the same in-flight/settled promise.
*/
open(): Promise<void> {
if (this.openPromise) return this.openPromise;
this.openPromise = new Promise<void>((resolve, reject) => {
this.openResolve = resolve;
this.openReject = reject;
this.connectTimer = setTimeout(() => {
// The 25s connect timeout: the collab connection never became ready.
this.opts?.onConnectTimeout?.();
this.teardown(
new Error("Connection timeout to collaboration server"),
false,
);
}, CONNECT_TIMEOUT_MS);
if (process.env.DEBUG)
console.error(`Connecting to WebSocket: ${this.wsUrl}`);
this.provider = providerFactory({
url: this.wsUrl,
name: `page.${this.pageId}`,
document: this.ydoc,
token: this.token,
WebSocketPolyfill: WebSocket,
onConnect: () => {
if (process.env.DEBUG) console.error("WS Connect");
},
// An unexpected disconnect/close at ANY time (during the connect-wait,
// between edits, or during a persistence wait) makes the session dead:
// surface it now instead of hanging, reject any in-flight op with the
// same error text as the one-shot machine, and remove ourselves from
// the registry so the next acquire opens fresh. `teardown` is idempotent
// so the onClose our own destroy() triggers is a harmless no-op.
onDisconnect: () => {
if (process.env.DEBUG) console.error("WS Disconnect");
this.teardown(
new Error(
"Collaboration connection closed before the update was persisted/synced",
),
true,
);
},
onClose: () => {
if (process.env.DEBUG) console.error("WS Close");
this.teardown(
new Error(
"Collaboration connection closed before the update was persisted/synced",
),
true,
);
},
onSynced: () => {
if (this.dead || this.openSettled) return;
if (process.env.DEBUG) console.error("Connected and synced!");
if (this.connectTimer) {
clearTimeout(this.connectTimer);
this.connectTimer = undefined;
}
this.state = "ready";
this.openSettled = true;
this.openResolve?.();
},
onAuthenticationFailed: () => {
this.teardown(
new Error("Authentication failed for collaboration connection"),
true,
);
},
});
});
return this.openPromise;
}
/**
* Run one atomic read -> transform -> write against the LIVE doc and wait for
* the server to acknowledge the write.
*
* INVARIANT 1 (read->write atomicity): between `TiptapTransformer.fromYdoc`
* and `applyDocToFragment` there is NO `await`. Yjs applies remote updates
* only when the event loop yields, so this synchronous block sees a consistent
* live doc and no concurrent human edit can interleave and be clobbered —
* exactly as in the one-shot onSynced code, just on a persistent provider.
*
* INVARIANT 2 (per-edit ack): after the write, resolve immediately if
* unsyncedChanges is already 0, else wait for the unsyncedChanges->0 event
* (PERSIST_TIMEOUT_MS), guarded by connectionLost so a reconnect handshake
* cannot report a false success.
*
* CONCURRENCY: not safe to invoke concurrently on ONE session — the caller
* MUST serialize (hold the per-page lock), mirroring acquireCollabSession.
* The in-flight op is tracked in a single `inflightReject` field, so an
* overlapping second call would clobber the first's rejector and leave it
* hanging on disconnect. A fail-fast guard below rejects the overlap instead.
* Sequential (awaited) mutates are fine: localFinish clears inflightReject
* before the promise settles, so the guard is clear by the time the next runs.
*/
mutate(
transform: (liveDoc: any) => any | null,
): Promise<MutationResult> {
// Belt-and-suspenders (acquire already validated): refuse to write on a
// session that is not in a live, synced, ready state.
if (
this.dead ||
this.state !== "ready" ||
this.connectionLost ||
!this.provider ||
this.provider.synced !== true
) {
return Promise.reject(
new Error("Collaboration session is not in a ready state"),
);
}
// Fail-fast on concurrent use: a second overlapping mutate would overwrite
// the first's inflightReject, so a disconnect would only reject the second
// and hang the first until PERSIST_TIMEOUT_MS. Reject the overlap WITHOUT
// touching the in-flight op's state (no localFinish/teardown here).
if (this.inflightReject) {
return Promise.reject(
new Error(
"mutate already in-flight; caller must serialize (hold the page lock)",
),
);
}
return new Promise<MutationResult>((resolve, reject) => {
let settled = false;
let persistTimer: ReturnType<typeof setTimeout> | undefined;
let unsyncedHandler:
| ((data: { number: number }) => void)
| undefined;
// The verifiable result resolved on every success/abort path. Set on
// abort (no-op report) and after a real write (computed change report).
let mutationResult: MutationResult;
const localFinish = (err: Error | null, value?: MutationResult) => {
if (settled) return;
settled = true;
if (persistTimer) clearTimeout(persistTimer);
if (unsyncedHandler && this.provider) {
try {
this.provider.off("unsyncedChanges", unsyncedHandler);
} catch (e) {}
}
this.inflightReject = undefined;
if (err) reject(err);
else resolve(value as MutationResult);
// Post-settle lifecycle: an ephemeral (cache-disabled) session dies with
// its single op; a cached session that is still alive re-arms its idle
// TTL so the clock starts from the LAST op.
if (this.ephemeral) {
this.destroy("ephemeral op complete");
} else if (!this.dead) {
this.armIdle();
}
};
// Register so a disconnect/close/auth-failure/teardown rejects THIS op
// with the connection-loss error text. localFinish's `settled` guard makes
// a racing teardown + normal resolve safe (first one wins).
this.inflightReject = (e: Error) => localFinish(e);
// Resolve once the server acknowledges our update: the provider increments
// unsyncedChanges when the local update is sent and decrements it on the
// server's SyncStatus(applied=true); reaching 0 means the authoritative
// in-memory ydoc on the server now contains our write.
const waitForPersistence = () => {
if (settled) return;
// A missing provider is a failure, not a success: without it the write
// can never have been acknowledged.
if (!this.provider) {
localFinish(new Error("collab provider gone before persistence"));
return;
}
if (this.provider.unsyncedChanges === 0) {
localFinish(null, mutationResult);
return;
}
persistTimer = setTimeout(() => {
localFinish(
new Error(
"Timeout waiting for collaboration server to persist the update",
),
);
}, PERSIST_TIMEOUT_MS);
unsyncedHandler = (data: { number: number }) => {
// Only treat unsyncedChanges->0 as success when the connection is
// still up. A transient disconnect + reconnect handshake can drive the
// counter back to 0 without our write being re-transmitted; in that
// case let the disconnect/close error win instead.
if (data.number === 0 && !this.connectionLost) {
localFinish(null, mutationResult);
}
};
this.provider.on("unsyncedChanges", unsyncedHandler);
};
// CRITICAL: everything between reading the live doc and writing it back
// must stay synchronous (no await). While the JS event loop is not
// yielded, no incoming remote update can interleave, so any already-synced
// concurrent edits are preserved in liveDoc.
let newDoc: any;
let beforeDoc: any;
try {
let liveDoc = TiptapTransformer.fromYdoc(this.ydoc, "default");
if (
!liveDoc ||
typeof liveDoc !== "object" ||
!Array.isArray(liveDoc.content)
) {
liveDoc = { type: "doc", content: [] };
}
// Snapshot the before-doc for the change report. Docs are
// JSON-serializable, so this is a safe deep clone.
beforeDoc = JSON.parse(JSON.stringify(liveDoc));
newDoc = transform(liveDoc);
if (newDoc == null) {
// Transform aborted — write nothing, return the live doc with a no-op
// change report.
mutationResult = {
doc: liveDoc,
verify: {
changed: false,
textInserted: 0,
textDeleted: 0,
blocksChanged: 0,
marks: {},
summary: "no changes (transform aborted)",
},
};
localFinish(null, mutationResult);
return;
}
// Structural diff into the live fragment (issue #152): preserves the Yjs
// ids of unchanged nodes, so an open editor's cursor is not yanked to the
// end of the document on every agent write.
applyDocToFragment(this.ydoc, newDoc);
} catch (e) {
// Includes errors thrown by transform (e.g. "afterText not found",
// "text not found"): propagate them verbatim to the caller.
localFinish(e instanceof Error ? e : new Error(String(e)));
return;
}
// Compute the verifiable change report AFTER the transact write: it only
// needs the JSON before/after, so it cannot affect the atomic read->write
// window, and summarizeChange never throws.
mutationResult = {
doc: newDoc,
verify: summarizeChange(beforeDoc, newDoc),
};
if (process.env.DEBUG)
console.error("Content written, waiting for server to persist...");
waitForPersistence();
});
}
/** (Re)arm the idle TTL so the clock starts from the most recent activity. */
armIdle(): void {
if (this.dead || this.ephemeral) return;
if (this.idleTimer) clearTimeout(this.idleTimer);
if (this.cfg.idleMs > 0) {
this.idleTimer = setTimeout(() => {
this.destroy("idle timeout");
}, this.cfg.idleMs);
// Never let the idle timer keep the process alive.
(this.idleTimer as any).unref?.();
}
}
/**
* Idempotent teardown: mark dead, clear timers, remove from the registry, fail
* any pending open/in-flight op, and destroy the provider. `inflightError` is
* the error a pending open or in-flight op is rejected with; `connectionLoss`
* marks the session as connection-lost so the ack guard cannot report a false
* success on a racing unsyncedChanges->0.
*/
private teardown(inflightError: Error | null, connectionLoss: boolean): void {
if (this.dead) return;
this.dead = true;
this.state = "dead";
if (connectionLoss) this.connectionLost = true;
if (this.connectTimer) {
clearTimeout(this.connectTimer);
this.connectTimer = undefined;
}
if (this.idleTimer) {
clearTimeout(this.idleTimer);
this.idleTimer = undefined;
}
// Remove ourselves from the registry (only if we are still the live entry —
// a re-open under the same key must not be evicted by our teardown).
if (sessions.get(this.key) === this) {
sessions.delete(this.key);
}
// Fail a pending open() and any in-flight mutate with the terminal error.
if (!this.openSettled) {
this.openSettled = true;
this.openReject?.(
inflightError ?? new Error("Collaboration session destroyed"),
);
}
if (this.inflightReject) {
const rej = this.inflightReject;
this.inflightReject = undefined;
rej(inflightError ?? new Error("Collaboration session destroyed"));
}
if (this.provider) {
try {
this.provider.destroy();
} catch (e) {}
this.provider = undefined;
}
}
/**
* Public idempotent teardown used by the acquire/eviction paths and by a
* caller that wants the session dropped after a failed op ("next call
* reconnects fresh").
*/
destroy(reason: string): void {
if (this.dead) return;
if (process.env.DEBUG)
console.error(`Destroying collab session ${this.pageId}: ${reason}`);
this.teardown(new Error(`Collaboration session destroyed: ${reason}`), false);
}
}
/** key = wsUrl + pageId + collabToken (identity isolation: invariant 4). */
const sessions = new Map<string, CollabSession>();
function sessionKey(wsUrl: string, pageId: string, token: string): string {
// The token is part of the key so sessions are NEVER shared between different
// users' MCP sessions (HTTP mode), and a token rotation makes a new entry
// while the old one idles out.
return `${wsUrl}${pageId}${token}`;
}
/**
* Get a live, synced CollabSession for a page, reusing a cached one when it is
* still valid or opening a fresh one otherwise. Does NOT take the per-page lock
* — the caller MUST already hold it (both call sites run inside withPageLock,
* which is not reentrant, so acquiring the lock here would deadlock
* mutateLiveContentUnlocked).
*/
export async function acquireCollabSession(
pageId: string,
collabToken: string,
baseUrl: string,
opts?: AcquireOptions,
): Promise<CollabSession> {
const cfg = readConfig();
const wsUrl = buildCollabWsUrl(baseUrl);
// Cache disabled (rollback path): open an unregistered ephemeral session that
// self-destroys after its single op — the exact legacy per-op-provider flow.
if (cfg.idleMs <= 0) {
const session = new CollabSession(
sessionKey(wsUrl, pageId, collabToken),
pageId,
wsUrl,
collabToken,
cfg,
true,
opts,
);
await session.open();
return session;
}
const key = sessionKey(wsUrl, pageId, collabToken);
const existing = sessions.get(key);
if (existing) {
if (existing.isReusable()) {
// Reuse. Refresh LRU order (re-insert = most recently used) and re-arm the
// idle TTL so the reuse counts as activity.
sessions.delete(key);
sessions.set(key, existing);
existing.armIdle();
if (process.env.DEBUG)
console.error(`Reusing collab session for page ${pageId}`);
return existing;
}
// Stale (not synced / past max age / lost): drop it and open fresh.
existing.destroy("stale on reuse");
}
// Enforce the registry cap before inserting: destroy-evict the least recently
// used (the first entry in insertion order) until there is room.
while (sessions.size >= cfg.maxEntries) {
const oldestKey: string | undefined = sessions.keys().next().value;
if (oldestKey === undefined) break;
const victim = sessions.get(oldestKey);
if (victim) victim.destroy("evicted (LRU cap)");
// destroy() removes it from the map; guard against a no-op destroy.
if (sessions.has(oldestKey)) sessions.delete(oldestKey);
}
const session = new CollabSession(
key,
pageId,
wsUrl,
collabToken,
cfg,
false,
opts,
);
sessions.set(key, session);
try {
await session.open();
} catch (e) {
// Failed connect/sync: make sure it is not left cached.
session.destroy("open failed");
throw e;
}
session.armIdle();
if (process.env.DEBUG)
console.error(`Opened new collab session for page ${pageId}`);
return session;
}
/**
* Destroy every cached session. Wired into the process shutdown so a hanging
* session does not keep a doc loaded on the server past exit.
*/
export function destroyAllSessions(): void {
for (const session of [...sessions.values()]) {
session.destroy("process shutdown");
}
sessions.clear();
}
/** TEST-ONLY: number of currently cached sessions. */
export function __sessionCountForTests(): number {
return sessions.size;
}
+211 -25
View File
@@ -1,3 +1,4 @@
import { HocuspocusProvider } from "@hocuspocus/provider";
import { TiptapTransformer } from "@hocuspocus/transformer"; import { TiptapTransformer } from "@hocuspocus/transformer";
import * as Y from "yjs"; import * as Y from "yjs";
import WebSocket from "ws"; import WebSocket from "ws";
@@ -13,10 +14,9 @@ import { JSDOM } from "jsdom";
import { markdownToProseMirror } from "@docmost/prosemirror-markdown"; import { markdownToProseMirror } from "@docmost/prosemirror-markdown";
import { docmostExtensions, docmostSchema } from "./docmost-schema.js"; import { docmostExtensions, docmostSchema } from "./docmost-schema.js";
import { withPageLock } from "./page-lock.js"; import { withPageLock } from "./page-lock.js";
import { sanitizeForYjs, findUnstorableAttr } from "./node-ops.js"; import { sanitizeForYjs, findUnstorableAttr } from "@docmost/prosemirror-markdown";
import { canonicalizeFootnotes } from "./footnote-canonicalize.js"; import { canonicalizeFootnotes } from "./footnote-canonicalize.js";
import { VerifyReport } from "./diff.js"; import { summarizeChange, VerifyReport } from "./diff.js";
import { acquireCollabSession } from "./collab-session.js";
export { markdownToProseMirror }; export { markdownToProseMirror };
@@ -194,27 +194,26 @@ export function assertYjsEncodable(doc: any): void {
} }
} }
/** Time we wait for the initial handshake/sync before giving up. */
const CONNECT_TIMEOUT_MS = 25000;
/** Time we wait for the server to acknowledge our write before giving up. */
const PERSIST_TIMEOUT_MS = 20000;
/** /**
* Safely mutate the live content of a page over the collaboration websocket. * Safely mutate the live content of a page over the collaboration websocket.
* *
* This is the single safe write path for every MCP content mutation. It: * This is the single safe write path for every MCP content mutation. It:
* 1. serializes per-page writes through withPageLock (no two MCP writes on * 1. serializes per-page writes through withPageLock (no two MCP writes on
* the same page overlap); * the same page overlap);
* 2. acquires a LIVE, synced CollabSession for the page (issue #400) — a * 2. connects to Hocuspocus and waits for the initial sync so the local ydoc
* cached provider whose local ydoc mirrors the authoritative server doc * mirrors the authoritative server doc — INCLUDING edits/comments/images
* (INCLUDING edits/comments/images not yet in the debounced REST snapshot), * that are not yet in the debounced REST snapshot;
* reused across a series of edits instead of a fresh connect/auth/sync per * 3. inside onSynced, SYNCHRONOUSLY reads the live doc, runs `transform`, and
* call; * writes the result back — with no `await` between read and write so no
* 3. SYNCHRONOUSLY reads the live doc, runs `transform`, and writes the result * remote update can interleave and clobber concurrent human edits;
* back — with no `await` between read and write so no remote update can
* interleave and clobber concurrent human edits (CollabSession.mutate);
* 4. waits for the server to acknowledge the write (unsyncedChanges -> 0) * 4. waits for the server to acknowledge the write (unsyncedChanges -> 0)
* before resolving, so the next operation observes our change. * before resolving, so the next operation observes our change.
* *
* On any mutate failure the session is destroyed so the next call reconnects
* fresh; the page lock is held for the whole acquire+mutate so the session's
* synchronous read->write window never overlaps another MCP write on the page.
*
* `transform` receives the live ProseMirror doc and returns the NEW full * `transform` receives the live ProseMirror doc and returns the NEW full
* ProseMirror doc to write, or `null` to abort with no write (a no-op). If * ProseMirror doc to write, or `null` to abort with no write (a no-op). If
* `transform` throws, the error is propagated to the caller (not swallowed). * `transform` throws, the error is propagated to the caller (not swallowed).
@@ -231,7 +230,7 @@ export async function mutatePageContent(
baseUrl: string, baseUrl: string,
transform: (liveDoc: any) => any | null, transform: (liveDoc: any) => any | null,
): Promise<MutationResult> { ): Promise<MutationResult> {
return withPageLock(pageId, async () => { return withPageLock(pageId, () => {
if (process.env.DEBUG) { if (process.env.DEBUG) {
console.error(`Starting realtime content mutate for page ${pageId}`); console.error(`Starting realtime content mutate for page ${pageId}`);
// Token prefix is sensitive; only log it under DEBUG. // Token prefix is sensitive; only log it under DEBUG.
@@ -240,15 +239,202 @@ export async function mutatePageContent(
); );
} }
const session = await acquireCollabSession(pageId, collabToken, baseUrl); const ydoc = new Y.Doc();
try { const wsUrl = buildCollabWsUrl(baseUrl);
return await session.mutate(transform); if (process.env.DEBUG) console.error(`Connecting to WebSocket: ${wsUrl}`);
} catch (e) {
// Drop the session on any failure so the next call reconnects fresh (this return new Promise<MutationResult>((resolve, reject) => {
// also closes the "reconnect drove the counter to 0" false-success class). let provider: HocuspocusProvider | undefined;
session.destroy("mutate failed"); let applied = false; // onSynced may fire again on reconnect — apply once.
throw e; let settled = false;
} // Set true on disconnect/close so a reconnect-driven unsyncedChanges->0
// cannot be mistaken for a successful persist of our write.
let connectionLost = false;
let connectTimer: ReturnType<typeof setTimeout> | undefined;
let persistTimer: ReturnType<typeof setTimeout> | undefined;
let unsyncedHandler: ((data: { number: number }) => void) | undefined;
const cleanup = () => {
if (connectTimer) clearTimeout(connectTimer);
if (persistTimer) clearTimeout(persistTimer);
if (provider) {
if (unsyncedHandler) {
try {
provider.off("unsyncedChanges", unsyncedHandler);
} catch (err) {}
}
try {
provider.destroy();
} catch (err) {}
}
};
const finish = (err: Error | null, value?: MutationResult) => {
if (settled) return;
settled = true;
cleanup();
if (err) reject(err);
else resolve(value as MutationResult);
};
connectTimer = setTimeout(() => {
finish(new Error("Connection timeout to collaboration server"));
}, CONNECT_TIMEOUT_MS);
// Resolve once the server has acknowledged our update. The provider
// increments unsyncedChanges when our local update is sent and
// decrements it when the server replies with a SyncStatus(applied=true);
// reaching 0 means the authoritative in-memory ydoc on the server now
// contains our write.
const waitForPersistence = () => {
if (settled) return;
// A missing provider is a failure, not a success: without it the write
// can never have been acknowledged. Only an actual unsyncedChanges===0
// on a live provider counts as persisted.
if (!provider) {
finish(new Error("collab provider gone before persistence"));
return;
}
if (provider.unsyncedChanges === 0) {
finish(null, mutationResult);
return;
}
persistTimer = setTimeout(() => {
finish(
new Error(
"Timeout waiting for collaboration server to persist the update",
),
);
}, PERSIST_TIMEOUT_MS);
unsyncedHandler = (data: { number: number }) => {
// Only treat unsyncedChanges->0 as success when the connection is
// still up. A transient disconnect + reconnect handshake can drive
// the counter back to 0 without our write being re-transmitted; in
// that case let the disconnect/close error win instead.
if (data.number === 0 && !connectionLost) {
finish(null, mutationResult);
}
};
provider.on("unsyncedChanges", unsyncedHandler);
};
// The verifiable result resolved on every success/abort path. Set on
// abort (no-op report) and after a real write (computed change report).
let mutationResult: MutationResult;
provider = new HocuspocusProvider({
url: wsUrl,
name: `page.${pageId}`,
document: ydoc,
token: collabToken,
// @ts-ignore - Required for Node.js environment
WebSocketPolyfill: WebSocket,
onConnect: () => {
if (process.env.DEBUG) console.error("WS Connect");
},
// An unexpected disconnect/close while we are still waiting (during the
// connect-wait before onSynced, or during the persistence wait after the
// write) means the update will never be acknowledged — surface it now
// instead of hanging until the connect/persist timeout fires. `finish`
// is idempotent via the `settled` flag, so the onClose that our own
// cleanup()->provider.destroy() triggers (after settled=true is set) is
// a harmless no-op and cannot cause a double-resolve.
onDisconnect: () => {
if (process.env.DEBUG) console.error("WS Disconnect");
// Mark BEFORE finish so the unsyncedChanges handler (if it races)
// sees the connection as lost and won't report a false success.
connectionLost = true;
finish(
new Error(
"Collaboration connection closed before the update was persisted/synced",
),
);
},
onClose: () => {
if (process.env.DEBUG) console.error("WS Close");
// Mark BEFORE finish so the unsyncedChanges handler (if it races)
// sees the connection as lost and won't report a false success.
connectionLost = true;
finish(
new Error(
"Collaboration connection closed before the update was persisted/synced",
),
);
},
onSynced: () => {
if (applied || settled) return;
applied = true;
if (process.env.DEBUG) console.error("Connected and synced!");
// CRITICAL: everything between reading the live doc and writing it
// back must stay synchronous (no await). While the JS event loop is
// not yielded, no incoming remote update can interleave, so any
// already-synced concurrent edits are preserved in liveDoc.
let newDoc: any;
let beforeDoc: any;
try {
let liveDoc = TiptapTransformer.fromYdoc(ydoc, "default");
if (
!liveDoc ||
typeof liveDoc !== "object" ||
!Array.isArray(liveDoc.content)
) {
liveDoc = { type: "doc", content: [] };
}
// Snapshot the before-doc for the change report. Docs are
// JSON-serializable, so this is a safe deep clone.
beforeDoc = JSON.parse(JSON.stringify(liveDoc));
newDoc = transform(liveDoc);
if (newDoc == null) {
// Transform aborted — write nothing, return the live doc with a
// no-op change report.
mutationResult = {
doc: liveDoc,
verify: {
changed: false,
textInserted: 0,
textDeleted: 0,
blocksChanged: 0,
marks: {},
summary: "no changes (transform aborted)",
},
};
finish(null, mutationResult);
return;
}
// Structural diff into the live fragment (issue #152): preserves
// the Yjs ids of unchanged nodes, so an open editor's cursor is not
// yanked to the end of the document on every agent write.
applyDocToFragment(ydoc, newDoc);
} catch (e) {
// Includes errors thrown by transform (e.g. "afterText not found",
// "text not found"): propagate them verbatim to the caller.
finish(e instanceof Error ? e : new Error(String(e)));
return;
}
// Compute the verifiable change report AFTER the transact write: it
// only needs the JSON before/after, so it cannot affect the atomic
// read->write window, and summarizeChange never throws.
mutationResult = {
doc: newDoc,
verify: summarizeChange(beforeDoc, newDoc),
};
if (process.env.DEBUG)
console.error("Content written, waiting for server to persist...");
waitForPersistence();
},
onAuthenticationFailed: () => {
finish(
new Error("Authentication failed for collaboration connection"),
);
},
});
});
}); });
} }
+41 -115
View File
@@ -1,136 +1,62 @@
/** /**
* Legacy footnote diagnostics for imported Markdown (issue #166). * Legacy footnote advisory for imported Markdown (issue #166, reduced in #414).
* *
* A PURE, fence-aware text scan (independent of the Markdown->ProseMirror * Since #293 STEP 5 the canonical import form is inline `^[body]` footnotes
* conversion path, so it reports the same problems for `create_page`, * (handled by `@docmost/prosemirror-markdown`). LEGACY reference-style
* `update_page` and `import_page_markdown`). It never changes the document — the * `[^id]: …` definition markup is now INERT on import — the importer leaves it as
* importer still creates the page; this only surfaces footnote problems to the * literal text — so authoring it silently produces broken footnotes (the #410
* caller so an agent can fix its own markup instead of shipping broken footnotes. * incident class). Rather than the old, elaborate diagnostics of every problem
* SHAPE (dangling/duplicate/empty/in-table) that no longer describe what the
* importer builds, this module surfaces ONE advisory warning whenever legacy
* reference-style definition syntax is present, nudging the author to the inline
* form. It never changes the document — the importer still creates the page.
* *
* SCOPE after #293 STEP 5: the canonical import form is now inline `^[body]` * The scan is fence-aware: a `[^id]:` line inside a ``` / ~~~ code block is
* footnotes (handled by `@docmost/prosemirror-markdown`), where these problems * example text, not markup, so it never triggers the warning.
* cannot arise. This scan therefore targets the LEGACY reference-style
* (`[^id]` / `[^id]:`) markup, which is now inert on import (left as literal
* text). The warnings remain useful as an advisory nudge when an agent still
* authors the old syntax, but they no longer describe what the importer builds.
*
* Detected problems:
* - danglingReferences: a `[^id]` reference with no `[^id]:` definition.
* - emptyDefinitions: a `[^id]:` whose (kept) text is empty/whitespace.
* - duplicateDefinitions: an id defined by two or more `[^id]:` lines (only the
* first would have been kept under the old first-wins import).
* - referencesInTables: a `[^id]` marker found in a GFM table row (heuristic:
* the line, trimmed, starts with `|`) — footnotes in table cells often do not
* render as expected.
*/ */
import { /** A legacy footnote DEFINITION line: `[^id]:` at the start of a (non-fenced) line. */
lexFootnoteLines, const FOOTNOTE_DEF_RE = /^\[\^[^\]\s]+\]:/;
forEachFootnoteReference, /** Opening/closing code fence marker (``` or ~~~). */
} from "./footnote-lex.js"; const FENCE_RE = /^\s*(`{3,}|~{3,})/;
export interface FootnoteDiagnostics { /** The single advisory shown when legacy reference-style footnotes are present. */
/** Reference ids (distinct, document order) with no matching definition. */ export const LEGACY_FOOTNOTE_WARNING =
danglingReferences: string[]; "Reference-style footnotes (`[^id]: …`) are not parsed on import and will " +
/** Definition ids whose first (kept) text is empty/whitespace. */ "appear as literal text. Use inline footnotes instead: `^[footnote text]`.";
emptyDefinitions: string[];
/** Ids defined by two or more `[^id]:` lines (only the first is kept). */
duplicateDefinitions: string[];
/** Reference ids found inside a GFM table row (heuristic). */
referencesInTables: string[];
/** Human-readable warning lines for the tool result (one per problem class). */
warnings: string[];
}
/** /**
* Analyze the footnotes in a Markdown string. Pure; safe to call on any body. * True when `markdown` contains a legacy `[^id]:` definition line OUTSIDE any
* code fence. Pure; safe to call on any body.
*/ */
export function analyzeFootnotes(markdown: string): FootnoteDiagnostics { export function hasLegacyFootnoteDefinition(markdown: string): boolean {
// Distinct reference ids in first-appearance order, plus the set of ids seen if (typeof markdown !== "string" || !markdown.includes("[^")) return false;
// inside a table row. let fence: string | null = null;
const refIds: string[] = []; for (const line of markdown.split("\n")) {
const refIdSet = new Set<string>(); const fenceMatch = FENCE_RE.exec(line);
const referencesInTables = new Set<string>(); if (fenceMatch) {
const addRef = (id: string, inTable: boolean) => { const marker = fenceMatch[1][0];
if (!refIdSet.has(id)) { if (fence === null) fence = marker; // opening fence
refIdSet.add(id); else if (marker === fence) fence = null; // matching closing fence
refIds.push(id);
}
if (inTable) referencesInTables.add(id);
};
// Definition texts per id, in first-appearance order of the id.
const defTextsById = new Map<string, string[]>();
// Same lexer the importer uses, so the analysis matches exactly what import
// keeps/strips (#166): fenced lines are inert, definition lines are pulled.
for (const tok of lexFootnoteLines(markdown)) {
if (tok.inFence) continue;
if (tok.definition) {
const { id, text } = tok.definition;
const arr = defTextsById.get(id);
if (arr) arr.push(text);
else defTextsById.set(id, [text]);
// A definition's TEXT can itself reference another footnote (`[^a]: see
// [^b]`); count those so such a `[^b]` is not falsely reported dangling.
forEachFootnoteReference(text, (rid) => addRef(rid, false));
continue; continue;
} }
const inTable = tok.line.trimStart().startsWith("|"); if (fence !== null) continue; // inside a fence: inert example text
forEachFootnoteReference(tok.line, (id) => addRef(id, inTable)); if (FOOTNOTE_DEF_RE.test(line)) return true;
} }
return false;
const danglingReferences = refIds.filter((id) => !defTextsById.has(id));
const duplicateDefinitions: string[] = [];
const emptyDefinitions: string[] = [];
for (const [id, texts] of defTextsById) {
if (texts.length >= 2) duplicateDefinitions.push(id);
// First-wins: the kept definition is the first one; flag it if it is blank.
if ((texts[0] ?? "").trim().length === 0) emptyDefinitions.push(id);
}
const tableRefs = [...referencesInTables];
const warnings: string[] = [];
const list = (ids: string[]) => ids.map((id) => `[^${id}]`).join(", ");
if (danglingReferences.length > 0) {
warnings.push(
`Footnote reference(s) with no matching definition: ${list(danglingReferences)} (each will render as an empty footnote in the editor).`,
);
}
if (emptyDefinitions.length > 0) {
warnings.push(
`Footnote definition(s) with empty text: ${list(emptyDefinitions)}.`,
);
}
if (duplicateDefinitions.length > 0) {
warnings.push(
`Footnote id(s) defined more than once (only the first definition was kept): ${list(duplicateDefinitions)}.`,
);
}
if (tableRefs.length > 0) {
warnings.push(
`Footnote marker(s) inside a table row (footnotes in table cells may not render as expected): ${list(tableRefs)}.`,
);
}
return {
danglingReferences,
emptyDefinitions,
duplicateDefinitions,
referencesInTables: tableRefs,
warnings,
};
} }
/** /**
* The optional `footnoteWarnings` field for a page-write tool result: present * The optional `footnoteWarnings` field for a page-write tool result: present
* (with the warning lines) only when `markdown` has footnote problems, omitted * (with the single advisory) only when `markdown` uses legacy reference-style
* otherwise. One helper so all three call sites (create/update/import) attach the * footnote syntax, omitted otherwise. One helper so all three call sites
* field identically. Spread into the result: `{ ...result, ...footnoteWarningsField(text) }`. * (create/update/import) attach the field identically. Spread into the result:
* `{ ...result, ...footnoteWarningsField(text) }`.
*/ */
export function footnoteWarningsField(markdown: string): { export function footnoteWarningsField(markdown: string): {
footnoteWarnings?: string[]; footnoteWarnings?: string[];
} { } {
const { warnings } = analyzeFootnotes(markdown); return hasLegacyFootnoteDefinition(markdown)
return warnings.length > 0 ? { footnoteWarnings: warnings } : {}; ? { footnoteWarnings: [LEGACY_FOOTNOTE_WARNING] }
: {};
} }
@@ -1,91 +0,0 @@
/**
* Inline-authoring helpers for footnotes (MCP).
*
* These build/identify footnote DEFINITION nodes for the author-inline tool
* (`insertInlineFootnote` in transforms.ts): a content key to de-duplicate notes
* by text, a definition-node factory, and a fresh uuidv7-style id generator.
*
* Split out of `footnote-canonicalize.ts` so that module stays a pure MIRROR of
* the editor-ext canonicalizer (compositionally symmetric to the editor-ext
* copy, which keeps its authoring helpers in `footnote-util.ts`). The pure
* canonicalizer has no dependency on these.
*/
const FOOTNOTE_DEFINITION_NAME = "footnoteDefinition";
function cloneJson<T>(v: T): T {
if (typeof structuredClone === "function") return structuredClone(v);
return JSON.parse(JSON.stringify(v)) as T;
}
/**
* Normalized content key for de-duplicating footnote DEFINITIONS by their text.
*
* Two definitions with the same key are the SAME footnote — so the inline
* authoring tool reuses one id (one number, one definition, several references)
* instead of minting a second definition. Key = plaintext (whitespace-collapsed,
* trimmed) PLUS a signature of the inline mark types in order, so two notes that
* read the same but differ in formatting (one bold, one plain) are NOT merged.
* Conservative: only an exact match merges.
*/
export function footnoteContentKey(defNode: any): string {
const parts: string[] = [];
const visit = (n: any): void => {
if (!n || typeof n !== "object") return;
if (n.type === "text" && typeof n.text === "string") {
const marks = Array.isArray(n.marks)
? n.marks.map((m: any) => m?.type).filter(Boolean).sort().join(",")
: "";
parts.push(`${n.text}${marks}`);
}
if (Array.isArray(n.content)) for (const c of n.content) visit(c);
};
visit(defNode);
// Collapse the assembled text's whitespace and trim, keeping the mark
// signature attached so formatting differences still distinguish notes.
return parts
.join("")
.replace(/[ \t\r\n]+/g, " ")
.trim();
}
/**
* Build a footnoteDefinition node from inline ProseMirror nodes, keyed by id.
*/
export function makeFootnoteDefinition(id: string, inlineNodes: any[]): any {
const content = Array.isArray(inlineNodes) ? cloneJson(inlineNodes) : [];
return {
type: FOOTNOTE_DEFINITION_NAME,
attrs: { id },
content: [{ type: "paragraph", content }],
};
}
/**
* Generate a uuidv7-style id (time-ordered), matching editor-ext's
* `generateFootnoteId`. Used for a genuinely-new inline footnote id.
*/
export function generateFootnoteId(): string {
const now = Date.now();
const timeHex = now.toString(16).padStart(12, "0");
const rand = (length: number) => {
let s = "";
for (let i = 0; i < length; i++)
s += Math.floor(Math.random() * 16).toString(16);
return s;
};
const versioned = "7" + rand(3);
const variantNibble = (8 + Math.floor(Math.random() * 4)).toString(16);
const variant = variantNibble + rand(3);
return (
timeHex.slice(0, 8) +
"-" +
timeHex.slice(8, 12) +
"-" +
versioned +
"-" +
variant +
"-" +
rand(12)
);
}
@@ -4,8 +4,8 @@
* `canonicalizeFootnotes(doc)` is a pure ProseMirror-JSON port of the editor's * `canonicalizeFootnotes(doc)` is a pure ProseMirror-JSON port of the editor's
* `footnoteSyncPlugin` end-state, identical in behaviour to * `footnoteSyncPlugin` end-state, identical in behaviour to
* `@docmost/editor-ext`'s `canonicalizeFootnotes`. It is mirrored here — rather * `@docmost/editor-ext`'s `canonicalizeFootnotes`. It is mirrored here — rather
* than imported from editor-ext — for the SAME reason `footnote-lex.ts` and the * than imported from editor-ext — for the SAME reason the `docmost-schema.ts`
* `docmost-schema.ts` nodes are mirrored: the MCP package is deliberately * nodes are mirrored: the MCP package is deliberately
* decoupled from the browser/React-heavy editor barrel and operates on plain * decoupled from the browser/React-heavy editor barrel and operates on plain
* JSON. The editor-ext copy owns the golden test against the live plugin; this * JSON. The editor-ext copy owns the golden test against the live plugin; this
* copy must stay behaviourally identical (a SHARED golden corpus, exercised by * copy must stay behaviourally identical (a SHARED golden corpus, exercised by
@@ -13,8 +13,8 @@
* *
* This module is the pure MIRROR only. The inline-authoring helpers * This module is the pure MIRROR only. The inline-authoring helpers
* (`footnoteContentKey`, `makeFootnoteDefinition`, `generateFootnoteId`) used by * (`footnoteContentKey`, `makeFootnoteDefinition`, `generateFootnoteId`) used by
* `insertInlineFootnote` live in the sibling `footnote-authoring.ts`, so this * `insertInlineFootnote` live in `@docmost/prosemirror-markdown` (next to the
* file is compositionally symmetric to the editor-ext copy. * importer's `assembleFootnotes`, #414), so this file stays a pure mirror.
* *
* Why it exists: every NON-editor write path (markdown import, update_page_json, * Why it exists: every NON-editor write path (markdown import, update_page_json,
* docmost_transform, insert_footnote) builds ProseMirror JSON directly, so the * docmost_transform, insert_footnote) builds ProseMirror JSON directly, so the
-73
View File
@@ -1,73 +0,0 @@
/**
* Shared, fence-aware line lexer for legacy footnote markdown (MCP-internal).
*
* Since #293 STEP 5 the markdown -> ProseMirror IMPORT path lives in the shared
* `@docmost/prosemirror-markdown` package (inline `^[body]` footnotes), so this
* lexer no longer backs an mcp importer. It now backs ONLY the import-time
* diagnostics (`analyzeFootnotes` in footnote-analyze.ts), which still scan the
* raw markdown for legacy reference-style `[^id]:` definition lines and surface
* advisory warnings (duplicate/orphan definitions) about content that is now
* inert on import. Fence-awareness (a `[^id]:` line inside a ``` / ~~~ block is
* NOT a definition) is the property the analyzer relies on.
*
* NOTE: this is deliberately NOT shared with editor-ext's
* `extractFootnoteDefinitions` — that lives in a different package and the
* decoupling between the editor and the MCP mirror is intentional.
*/
/** A footnote DEFINITION line: `[^id]: text` (id + text captured). */
export const FOOTNOTE_DEF_RE = /^\[\^([^\]\s]+)\]:[ \t]*(.*)$/;
/** Every footnote REFERENCE `[^id]` in a line (global; id captured). */
export const FOOTNOTE_REF_RE_G = /\[\^([^\]\s]+)\]/g;
/** Opening/closing code fence marker (``` or ~~~). */
const FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
export interface FootnoteLine {
/** The raw line, verbatim. */
line: string;
/**
* True for a code-fence marker line AND every line inside a fence — footnote
* syntax on such lines is inert (example text, not real markup). The importer
* keeps these in the body; the analyzer skips them.
*/
inFence: boolean;
/** The parsed definition, when this is a `[^id]: text` line OUTSIDE any fence. */
definition: { id: string; text: string } | null;
}
/** Classify every line of `markdown`, tracking fenced-code state. Pure. */
export function lexFootnoteLines(markdown: string): FootnoteLine[] {
const out: FootnoteLine[] = [];
let fence: string | null = null;
for (const line of markdown.split("\n")) {
const fenceMatch = FENCE_RE.exec(line);
if (fenceMatch) {
const marker = fenceMatch[2][0];
if (fence === null) fence = marker; // opening fence
else if (marker === fence) fence = null; // matching closing fence
out.push({ line, inFence: true, definition: null });
continue;
}
if (fence !== null) {
out.push({ line, inFence: true, definition: null });
continue;
}
const m = FOOTNOTE_DEF_RE.exec(line);
out.push({
line,
inFence: false,
definition: m ? { id: m[1], text: m[2] } : null,
});
}
return out;
}
/** Scan a line for every `[^id]` reference, invoking `onRef(id)` for each. */
export function forEachFootnoteReference(
line: string,
onRef: (id: string) => void,
): void {
FOOTNOTE_REF_RE_G.lastIndex = 0;
let m: RegExpExecArray | null;
while ((m = FOOTNOTE_REF_RE_G.exec(line)) !== null) onRef(m[1]);
}
-963
View File
@@ -1,963 +0,0 @@
/**
* Pure, network-free helpers for manipulating a ProseMirror/TipTap document
* tree by node id.
*
* A ProseMirror node here is a plain JSON object of the shape produced by
* Docmost: `{ type, attrs?, content?, text?, marks? }`. Children live in the
* `content` array; a node carries a stable id in `attrs.id`. Callouts and
* table cells hold their children in `content` just like any other block, so a
* single recursive walk reaches them all.
*
* Every exported function operates on a DEEP CLONE of the input document and
* returns the new document. The input doc and any `newNode`/`node` argument are
* never mutated. All functions are defensively null-safe: missing/!Array
* `content`, non-object nodes, and absent `attrs` are tolerated.
*/
import { stripInlineMarkdown } from "./text-normalize.js";
/** Deep-clone a JSON-serializable value without mutating the original. */
function clone<T>(value: T): T {
if (typeof structuredClone === "function") {
return structuredClone(value);
}
// Fallback for environments without structuredClone.
return JSON.parse(JSON.stringify(value)) as T;
}
/** True if `value` is a non-null object (and not an array). */
function isObject(value: any): value is Record<string, any> {
return value != null && typeof value === "object" && !Array.isArray(value);
}
/** True if `node` carries the given id in `node.attrs.id`. */
function matchesId(node: any, nodeId: string): boolean {
return isObject(node) && isObject(node.attrs) && node.attrs.id === nodeId;
}
/**
* Recursively concatenate all text contained in a node.
*
* Text nodes contribute their `text` string; container nodes contribute the
* joined `blockPlainText` of their `content` children. Returns "" for nullish
* or non-object inputs.
*/
export function blockPlainText(node: any): string {
if (!isObject(node)) return "";
let out = "";
if (typeof node.text === "string") {
out += node.text;
}
if (Array.isArray(node.content)) {
for (const child of node.content) {
out += blockPlainText(child);
}
}
return out;
}
/** Truncate `text` to at most `n` chars, appending an ellipsis when cut. */
function truncate(text: string, n: number): string {
return text.length > n ? text.slice(0, n) + "…" : text;
}
/** One compact outline entry for a single top-level block. */
export interface OutlineEntry {
index: number;
type: string | undefined;
id: string | null;
firstText: string;
/** Present for headings only. */
level?: number | null;
/** Present for tables only. */
rows?: number;
cols?: number;
header?: string[];
/** Present for list blocks only (bulletList/orderedList/taskList). */
items?: number;
}
/**
* Build a COMPACT outline of the TOP-LEVEL blocks of `doc` (the entries in
* `doc.content`). Deliberately does NOT recurse into paragraphs, list items, or
* table cells — compactness is the point; use `getNodeByRef` to drill into a
* specific block.
*
* Each entry carries `{ index, type, id, firstText }`, plus type-specific
* extras: headings add `level`; tables add `rows`/`cols` and the first row's
* cell texts as `header`; list blocks (types ending in "List") add `items`.
* `firstText` is the block's plain text truncated to 100 chars. Null-safe:
* a missing or non-object doc/content yields `[]`.
*/
export function buildOutline(doc: any): OutlineEntry[] {
if (!isObject(doc) || !Array.isArray(doc.content)) return [];
const out: OutlineEntry[] = [];
for (let i = 0; i < doc.content.length; i++) {
const block = doc.content[i];
const type = isObject(block) ? block.type : undefined;
const entry: OutlineEntry = {
index: i,
type,
id:
isObject(block) && isObject(block.attrs)
? (block.attrs.id ?? null)
: null,
firstText: truncate(blockPlainText(block), 100),
};
if (type === "heading") {
entry.level = isObject(block.attrs) ? (block.attrs.level ?? null) : null;
} else if (type === "table") {
const headerRow = block.content?.[0]?.content ?? [];
entry.rows = block.content?.length ?? 0;
entry.cols = block.content?.[0]?.content?.length ?? 0;
entry.header = headerRow.map((cell: any) =>
truncate(blockPlainText(cell), 40),
);
} else if (typeof type === "string" && type.endsWith("List")) {
entry.items = block.content?.length ?? 0;
}
out.push(entry);
}
return out;
}
/**
* Resolve a single node by reference and return `{ node, path, type }`, or
* `null` when nothing matches.
*
* - `ref` of the form `#<n>` (e.g. `#2`) selects the TOP-LEVEL block at index
* `n` in `doc.content`. This is the only way to address table/tableRow/
* tableCell nodes, which carry no `attrs.id`.
* - Otherwise `ref` is treated as a block id: the FIRST node anywhere in the
* tree with `attrs.id === ref` is returned.
*
* `path` is the array of child indices from the doc root down to the node
* (so a top-level block is `[index]`). The returned `node` is a DEEP CLONE,
* so callers can mutate it without touching the input doc. Null-safe.
*/
export function getNodeByRef(
doc: any,
ref: string,
): { node: any; path: number[]; type: string | undefined } | null {
if (!isObject(doc)) return null;
// "#<n>": index into the top-level content array.
const indexMatch = typeof ref === "string" ? ref.match(/^#(\d+)$/) : null;
if (indexMatch) {
const index = Number(indexMatch[1]);
const block = Array.isArray(doc.content) ? doc.content[index] : undefined;
if (!isObject(block)) return null;
return { node: clone(block), path: [index], type: block.type };
}
// Otherwise: depth-first search for the first node with attrs.id === ref.
const search = (
node: any,
trail: number[],
): { node: any; path: number[]; type: string } | null => {
if (!isObject(node)) return null;
if (Array.isArray(node.content)) {
for (let i = 0; i < node.content.length; i++) {
const child = node.content[i];
const path = [...trail, i];
if (matchesId(child, ref)) {
return { node: clone(child), path, type: child.type };
}
const hit = search(child, path);
if (hit != null) return hit;
}
}
return null;
};
return search(doc, []);
}
/**
* Replace EVERY node whose `attrs.id === nodeId` with a deep clone of
* `newNode`, anywhere in the tree (including inside callouts and table cells).
*
* Operates on a clone of `doc`; returns `{ doc, replaced }` where `replaced`
* is the number of nodes substituted. A fresh clone of `newNode` is used for
* each match so they do not share references.
*/
export function replaceNodeById(
doc: any,
nodeId: string,
newNode: any,
): { doc: any; replaced: number } {
const out = clone(doc);
let replaced = 0;
// Walk a content array, replacing direct matches and recursing into the
// (possibly new) children of non-matching nodes.
const walkContent = (content: any[]): void => {
for (let i = 0; i < content.length; i++) {
const child = content[i];
if (matchesId(child, nodeId)) {
content[i] = clone(newNode);
replaced++;
// Do not recurse into a freshly substituted node.
continue;
}
if (isObject(child) && Array.isArray(child.content)) {
walkContent(child.content);
}
}
};
if (isObject(out) && Array.isArray(out.content)) {
walkContent(out.content);
}
return { doc: out, replaced };
}
/**
* Remove EVERY node whose `attrs.id === nodeId` from its parent `content`
* array, anywhere in the tree (recursive, including callouts and tables).
*
* Operates on a clone of `doc`; returns `{ doc, deleted }` where `deleted` is
* the number of nodes removed.
*/
export function deleteNodeById(
doc: any,
nodeId: string,
): { doc: any; deleted: number } {
const out = clone(doc);
let deleted = 0;
// Filter a content array in place, dropping matches and recursing into the
// surviving children.
const walkContent = (content: any[]): any[] => {
const kept: any[] = [];
for (const child of content) {
if (matchesId(child, nodeId)) {
deleted++;
continue;
}
if (isObject(child) && Array.isArray(child.content)) {
child.content = walkContent(child.content);
}
kept.push(child);
}
return kept;
};
if (isObject(out) && Array.isArray(out.content)) {
out.content = walkContent(out.content);
}
return { doc: out, deleted };
}
/**
* Throw a clear, model-actionable error when a node-id write op did NOT match
* exactly one node (#159). `count === 0` -> "no node found"; `count > 1` ->
* "ambiguous, refused" — Docmost duplicates block ids on copy/paste, so a write
* by id could clobber/remove EVERY duplicate. The caller skips the write for any
* `count !== 1` (the transform returns null), so this only REPORTS; nothing was
* changed. No-op for the unambiguous single-match case.
*/
export function assertUnambiguousMatch(
op: "patch_node" | "delete_node",
verb: "replace" | "delete",
count: number,
nodeId: string,
pageId: string,
): void {
if (count === 0) {
throw new Error(
`${op}: no node with id "${nodeId}" found on page ${pageId}`,
);
}
if (count > 1) {
throw new Error(
`${op}: id "${nodeId}" is ambiguous — ${count} nodes on page ${pageId} share it (block ids are duplicated on copy/paste). Refusing to ${verb} all of them; nothing was changed. Re-target with a more specific anchor.`,
);
}
}
/**
* Deep-clone `doc` and strip every node/mark attribute whose value is strictly
* `undefined`, so the result is safe to hand to Yjs (which throws an opaque
* "Unexpected content type" when asked to store an `undefined` attribute value).
*
* Only `undefined` keys are removed; `null`, `false`, `0`, and `""` are all
* legitimate JSON-storable values and are preserved. Operates on a clone and
* returns it; the input is never mutated. Defensively null-safe like the rest
* of the file.
*/
export function sanitizeForYjs(doc: any): any {
const out = clone(doc);
// Drop every key whose value is strictly `undefined` from an attrs object.
const stripUndefined = (attrs: any): void => {
if (!isObject(attrs)) return;
for (const key of Object.keys(attrs)) {
if (attrs[key] === undefined) {
delete attrs[key];
}
}
};
const walk = (node: any): void => {
if (!isObject(node)) return;
stripUndefined(node.attrs);
if (Array.isArray(node.marks)) {
for (const mark of node.marks) {
if (isObject(mark)) stripUndefined(mark.attrs);
}
}
if (Array.isArray(node.content)) {
for (const child of node.content) {
walk(child);
}
}
};
walk(out);
return out;
}
/**
* Diagnostics helper: walk the tree and return a human-readable path string for
* the FIRST attribute value (in any `node.attrs` or `mark.attrs`) that Yjs
* cannot store — i.e. `undefined`, a `function`, a `symbol`, or a `bigint`
* (e.g. `content[3].content[0].attrs.indent (undefined)`). Returns `null` when
* every attribute is storable. Null-safe.
*/
export function findUnstorableAttr(doc: any): string | null {
const isUnstorable = (value: any): string | null => {
if (value === undefined) return "undefined";
const t = typeof value;
if (t === "function") return "function";
if (t === "symbol") return "symbol";
if (t === "bigint") return "bigint";
return null;
};
// Check an attrs object; return the offending sub-path or null.
const checkAttrs = (attrs: any, basePath: string): string | null => {
if (!isObject(attrs)) return null;
for (const key of Object.keys(attrs)) {
const kind = isUnstorable(attrs[key]);
if (kind != null) return `${basePath}.${key} (${kind})`;
}
return null;
};
const walk = (node: any, path: string): string | null => {
if (!isObject(node)) return null;
const attrHit = checkAttrs(node.attrs, `${path}.attrs`);
if (attrHit != null) return attrHit;
if (Array.isArray(node.marks)) {
for (let i = 0; i < node.marks.length; i++) {
const markHit = checkAttrs(
node.marks[i]?.attrs,
`${path}.marks[${i}].attrs`,
);
if (markHit != null) return markHit;
}
}
if (Array.isArray(node.content)) {
for (let i = 0; i < node.content.length; i++) {
const childHit = walk(node.content[i], `${path}.content[${i}]`);
if (childHit != null) return childHit;
}
}
return null;
};
// The root doc node carries no useful index, so start the path at "doc".
if (!isObject(doc)) return null;
const attrHit = checkAttrs(doc.attrs, "attrs");
if (attrHit != null) return attrHit;
if (Array.isArray(doc.content)) {
for (let i = 0; i < doc.content.length; i++) {
const childHit = walk(doc.content[i], `content[${i}]`);
if (childHit != null) return childHit;
}
}
return null;
}
/**
* Table structural node types and the container each must live directly inside.
* Used by `insertNodeRelative` to splice rows/cells into the correct ancestor
* rather than blindly into the anchor's direct parent (which would corrupt the
* table's nesting).
*/
const STRUCTURAL_TYPES = new Set(["tableRow", "tableCell", "tableHeader"]);
const REQUIRED_CONTAINER: Record<string, string> = {
tableRow: "table",
tableCell: "tableRow",
tableHeader: "tableRow",
};
/**
* Find the index of the first TOP-LEVEL block whose plain text includes the
* anchor, with a markdown-stripping FALLBACK. Returns -1 when none matches.
*
* Two passes preserve "exact wins globally":
* - Pass 1: first block containing the verbatim `anchorText`.
* - Pass 2 (only if pass 1 found nothing): first block containing the
* markdown-stripped anchor, when stripping actually changed it.
*/
function findAnchorTextIndex(content: any[], anchorText: string): number {
if (!Array.isArray(content)) return -1;
// Pass 1: exact.
for (let i = 0; i < content.length; i++) {
if (blockPlainText(content[i]).includes(anchorText)) return i;
}
// Pass 2: markdown-stripped fallback.
const a = stripInlineMarkdown(anchorText);
if (a !== anchorText && a.length > 0) {
for (let i = 0; i < content.length; i++) {
if (blockPlainText(content[i]).includes(a)) return i;
}
}
return -1;
}
/**
* Locate an anchor and return its ancestor chain (from `doc` down to and
* including the matched node). Each chain entry is `{ node, index }` where
* `index` is the node's position inside its parent's `content` array (the root
* doc has index -1). Returns `null` when the anchor cannot be resolved.
*/
function findAnchorChain(
doc: any,
opts: InsertOptions,
): { node: any; index: number }[] | null {
if (!isObject(doc)) return null;
// DFS by id anywhere in the tree, accumulating the path.
if (opts.anchorNodeId != null) {
const targetId = opts.anchorNodeId;
const search = (
node: any,
index: number,
trail: { node: any; index: number }[],
): { node: any; index: number }[] | null => {
if (!isObject(node)) return null;
const here = [...trail, { node, index }];
if (matchesId(node, targetId)) return here;
if (Array.isArray(node.content)) {
for (let i = 0; i < node.content.length; i++) {
const hit = search(node.content[i], i, here);
if (hit != null) return hit;
}
}
return null;
};
return search(doc, -1, []);
}
// By text: only top-level blocks are scanned (same rule as the JSON path).
// Exact match wins; a markdown-stripped fallback is tried only on a miss.
if (opts.anchorText != null && Array.isArray(doc.content)) {
const i = findAnchorTextIndex(doc.content, opts.anchorText);
if (i !== -1) {
return [
{ node: doc, index: -1 },
{ node: doc.content[i], index: i },
];
}
}
return null;
}
/** Options controlling where `insertNodeRelative` places the new node. */
export interface InsertOptions {
position: "before" | "after" | "append";
/** Resolve the anchor by node id anywhere in the tree (preferred). */
anchorNodeId?: string;
/** Fallback: first TOP-LEVEL block whose plain text includes this string. */
anchorText?: string;
}
/**
* Insert a deep clone of `node` relative to an anchor.
*
* - position "append": push the node onto the top-level `doc.content`.
* - position "before"/"after": locate the anchor and splice the node into the
* anchor's parent `content` array immediately before / after it.
*
* Anchor resolution for before/after:
* - if `anchorNodeId` is given, find the node with `attrs.id === anchorNodeId`
* anywhere in the tree (recursive);
* - otherwise, if `anchorText` is given, scan only TOP-LEVEL `doc.content`
* blocks and pick the first whose `blockPlainText` includes `anchorText`.
*
* Operates on a clone of `doc`; returns `{ doc, inserted }`. `inserted` is
* false when the anchor could not be resolved (the doc is returned unchanged
* apart from being cloned).
*/
export function insertNodeRelative(
doc: any,
node: any,
opts: InsertOptions,
): { doc: any; inserted: boolean } {
const out = clone(doc);
const fresh = clone(node);
// Defensive: stay null-safe like the other exports — a missing opts means
// there is nothing actionable to do.
if (!isObject(opts)) return { doc: out, inserted: false };
const isStructural = isObject(node) && STRUCTURAL_TYPES.has(node.type);
// "append": top-level push.
if (opts.position === "append") {
// Structural table nodes (tableRow/tableCell/tableHeader) cannot live at the
// top level — appending one would produce invalid nesting.
if (isStructural) {
throw new Error(
`insert_node: cannot append a ${node.type} at the top level; use ` +
`position before/after with an anchor inside the target table`,
);
}
if (isObject(out)) {
if (!Array.isArray(out.content)) out.content = [];
out.content.push(fresh);
return { doc: out, inserted: true };
}
return { doc: out, inserted: false };
}
const offset = opts.position === "after" ? 1 : 0;
// Structural insert (before/after a tableRow/tableCell/tableHeader): splice
// into the nearest enclosing table/tableRow rather than the anchor's direct
// parent, so the row/cell lands at the correct level of the table.
if (isStructural) {
const containerType = REQUIRED_CONTAINER[node.type];
const chain = findAnchorChain(out, opts);
// Anchor not resolved at all — keep the existing "anchor not found" path.
if (chain == null) return { doc: out, inserted: false };
// Find the DEEPEST ancestor (including the anchor itself) of the required
// container type.
let containerIdx = -1;
for (let i = chain.length - 1; i >= 0; i--) {
if (isObject(chain[i].node) && chain[i].node.type === containerType) {
containerIdx = i;
break;
}
}
if (containerIdx === -1) {
throw new Error(
`insert_node: cannot insert a ${node.type} here — the anchor is not ` +
`inside a ${containerType}. Anchor on a cell's text or a block id ` +
`that lives inside the target table.`,
);
}
const container = chain[containerIdx].node;
if (!Array.isArray(container.content)) container.content = [];
if (containerIdx === chain.length - 1) {
// The matched container IS the anchor node itself (e.g. anchorText
// resolved to the table block): append/prepend within it.
const at = opts.position === "after" ? container.content.length : 0;
container.content.splice(at, 0, fresh);
} else {
// The immediate child on the path leading to the anchor is the row/cell
// to splice next to.
const enclosingChildIndex = chain[containerIdx + 1].index;
container.content.splice(enclosingChildIndex + offset, 0, fresh);
}
return { doc: out, inserted: true };
}
// Resolve by id anywhere in the tree: splice into the parent content array.
if (opts.anchorNodeId != null) {
let inserted = false;
const walkContent = (content: any[]): void => {
for (let i = 0; i < content.length; i++) {
const child = content[i];
if (matchesId(child, opts.anchorNodeId as string)) {
content.splice(i + offset, 0, fresh);
inserted = true;
return;
}
if (isObject(child) && Array.isArray(child.content)) {
walkContent(child.content);
if (inserted) return;
}
}
};
if (isObject(out) && Array.isArray(out.content)) {
walkContent(out.content);
}
return { doc: out, inserted };
}
// Resolve by text: only top-level doc.content blocks are scanned. Exact
// match wins; a markdown-stripped fallback is tried only on a miss.
if (opts.anchorText != null && isObject(out) && Array.isArray(out.content)) {
const i = findAnchorTextIndex(out.content, opts.anchorText);
if (i !== -1) {
out.content.splice(i + offset, 0, fresh);
return { doc: out, inserted: true };
}
}
return { doc: out, inserted: false };
}
// ===========================================================================
// Table editing helpers
//
// A Docmost table is a ProseMirror subtree with NO ids on the structural nodes:
// table -> { type:"table", content:[tableRow...] }
// row -> { type:"tableRow", content:[tableCell|tableHeader...] }
// cell -> { type:"tableCell"|"tableHeader", attrs:{colspan,rowspan,colwidth},
// content:[paragraph...] }
// para -> { type:"paragraph", attrs:{id,indent}, content:[textNode...] }
// Only paragraphs/headings carry an `attrs.id`, so a cell is addressed via the
// id of the paragraph inside it. The helpers below all operate on a DEEP CLONE
// of the input doc (via `clone`) and never mutate their inputs.
// ===========================================================================
/**
* Collect EVERY `attrs.id` present anywhere in `node` into `used`. Used to seed
* `makeFreshId` so generated paragraph ids never collide with existing ones.
*/
function collectIds(node: any, used: Set<string>): void {
if (!isObject(node)) return;
if (isObject(node.attrs) && typeof node.attrs.id === "string") {
used.add(node.attrs.id);
}
if (Array.isArray(node.content)) {
for (const child of node.content) collectIds(child, used);
}
}
/**
* Fresh-id generator: returns a random Docmost-style id (12 chars from
* lowercase `a-z0-9`) that is not already in `used`, and records it. On the
* rare collision the id is regenerated. Callers rely on uniqueness, not on the
* exact string, so randomness is fine — and unlike a module-local counter it
* needs no reset and cannot become predictable across calls.
*/
function makeFreshId(used: Set<string>): string {
const alphabet = "abcdefghijklmnopqrstuvwxyz0123456789";
let id: string;
do {
id = "";
for (let i = 0; i < 12; i++) {
id += alphabet[Math.floor(Math.random() * alphabet.length)];
}
} while (used.has(id) || id === "");
used.add(id);
return id;
}
/**
* Resolve a table reference against an ALREADY-CLONED doc and return the LIVE
* table node (a reference inside `rootClone`, so the caller may mutate it) plus
* its index path. Returns null when no table matches.
*
* - `#<n>`: the top-level block at index `n`, only if its `type === "table"`.
* - otherwise: DFS for the node with `attrs.id === tableRef`, then walk UP its
* ancestor chain to the nearest `type === "table"` ancestor.
*/
function locateTable(
rootClone: any,
tableRef: string,
): { table: any; path: number[] } | null {
if (!isObject(rootClone)) return null;
// "#<n>": index into the top-level content array; must be a table.
const indexMatch =
typeof tableRef === "string" ? tableRef.match(/^#(\d+)$/) : null;
if (indexMatch) {
const index = Number(indexMatch[1]);
const block = Array.isArray(rootClone.content)
? rootClone.content[index]
: undefined;
if (isObject(block) && block.type === "table") {
return { table: block, path: [index] };
}
return null;
}
// Otherwise: DFS for attrs.id === tableRef, tracking the ancestor chain, then
// climb to the nearest enclosing table.
const search = (
node: any,
trail: { node: any; index: number }[],
): { table: any; path: number[] } | null => {
if (!isObject(node)) return null;
if (Array.isArray(node.content)) {
for (let i = 0; i < node.content.length; i++) {
const child = node.content[i];
const here = [...trail, { node: child, index: i }];
if (matchesId(child, tableRef)) {
// Walk UP to the nearest table ancestor (including the match itself).
for (let j = here.length - 1; j >= 0; j--) {
if (isObject(here[j].node) && here[j].node.type === "table") {
return {
table: here[j].node,
path: here.slice(0, j + 1).map((e) => e.index),
};
}
}
return null; // id found but no enclosing table
}
const hit = search(child, here);
if (hit != null) return hit;
}
}
return null;
};
return search(rootClone, []);
}
/** Build the plain-text → single-paragraph cell content used by all writers. */
function makeCellParagraph(id: string, text: string): any {
return {
type: "paragraph",
attrs: { id, indent: 0 },
// Empty string → a paragraph with an empty content array.
content: text ? [{ type: "text", text }] : [],
};
}
/**
* Read a table as a matrix. Returns null when `tableRef` resolves to no table.
*
* - `rows`/`cols`: the table's row count and the column count of its FIRST row.
* Tables may be ragged (rows of differing length), so `cols` reflects only
* row 0; use the per-row length of `cells`/`cellIds` for each row's actual
* width.
* - `cells`: `string[][]` of each cell's `blockPlainText`.
* - `cellIds`: `(string|null)[][]` of each cell's FIRST paragraph id (or null),
* so callers can `patch_node` a cell for rich-formatted edits.
* - `path`: index path of the table within the doc.
*/
export function readTable(
doc: any,
tableRef: string,
): {
rows: number;
cols: number;
cells: string[][];
cellIds: (string | null)[][];
path: number[];
} | null {
const root = clone(doc);
const located = locateTable(root, tableRef);
if (located == null) return null;
const { table, path } = located;
const rowNodes = Array.isArray(table.content) ? table.content : [];
const rows = rowNodes.length;
const cols = rowNodes[0]?.content?.length ?? 0;
const cells: string[][] = [];
const cellIds: (string | null)[][] = [];
for (const rowNode of rowNodes) {
const cellNodes = Array.isArray(rowNode?.content) ? rowNode.content : [];
const rowText: string[] = [];
const rowIds: (string | null)[] = [];
for (const cellNode of cellNodes) {
rowText.push(blockPlainText(cellNode));
// The cell's first paragraph carries the id used for patch_node.
const firstPara = Array.isArray(cellNode?.content)
? cellNode.content[0]
: undefined;
const id =
isObject(firstPara) && isObject(firstPara.attrs)
? (firstPara.attrs.id ?? null)
: null;
rowIds.push(id);
}
cells.push(rowText);
cellIds.push(rowIds);
}
return { rows, cols, cells, cellIds, path };
}
/**
* Insert a row of plain-text cells into a table. Returns `{ doc, inserted }`.
*
* The row is padded to the table's column count (`cells[i] ?? ""`); supplying
* MORE cells than columns throws. Each new cell copies `colwidth` for its
* column from the header row when present, gets a fresh-id paragraph, and a
* `colspan:1, rowspan:1` attrs. `index` (when an integer in `[0, rows]`) splices
* the row there; otherwise the row is appended at the end.
*/
export function insertTableRow(
doc: any,
tableRef: string,
cells: string[],
index?: number,
): { doc: any; inserted: boolean } {
const out = clone(doc);
const located = locateTable(out, tableRef);
if (located == null) return { doc: out, inserted: false };
const { table } = located;
if (!Array.isArray(table.content)) table.content = [];
const rows = table.content.length;
const headerRow = table.content[0];
const headerCells = Array.isArray(headerRow?.content)
? headerRow.content
: [];
// Column count is the WIDEST existing row, so the guard below stays
// meaningful for ragged tables and the new row matches the table's width.
// Fall back to the supplied cell count only when the table has no rows.
let colCount = 0;
for (const r of table.content) {
if (isObject(r) && Array.isArray(r.content))
colCount = Math.max(colCount, r.content.length);
}
if (colCount === 0) colCount = Array.isArray(cells) ? cells.length : 0;
if (Array.isArray(cells) && cells.length > colCount) {
throw new Error(
`table_insert_row: got ${cells.length} cell(s) but the table has ${colCount} column(s)`,
);
}
// Resolve the landing index up front so the cell-type decision and the splice
// below agree: a valid integer in [0, rows] splices there, else we append.
const landingIndex =
typeof index === "number" &&
Number.isInteger(index) &&
index >= 0 &&
index <= rows
? index
: rows;
// Seed the id generator with every id already in the doc so the new cell
// paragraph ids are unique within the whole document.
const used = new Set<string>();
collectIds(out, used);
const newCells: any[] = [];
for (let i = 0; i < colCount; i++) {
const text = (Array.isArray(cells) ? cells[i] : undefined) ?? "";
const attrs: Record<string, any> = { colspan: 1, rowspan: 1 };
// Copy this column's colwidth from the header row's cell when present.
const colwidth = headerCells[i]?.attrs?.colwidth;
if (colwidth !== undefined) attrs.colwidth = colwidth;
// A row landing at index 0 becomes the new header row, so inherit the
// current header cell's type per column (Docmost uses "tableHeader" there);
// every other position is a plain data cell.
const cellType =
landingIndex === 0 ? (headerCells[i]?.type ?? "tableCell") : "tableCell";
newCells.push({
type: cellType,
attrs,
content: [makeCellParagraph(makeFreshId(used), text)],
});
}
const newRow = { type: "tableRow", content: newCells };
// Splice at the resolved landing index (append when index was omitted/invalid).
table.content.splice(landingIndex, 0, newRow);
return { doc: out, inserted: true };
}
/**
* Delete the row at 0-based `index` from a table. Returns `{ doc, deleted }`.
* `deleted` is false only when the table cannot be located. Throws on an
* out-of-range index, and refuses to delete the table's only row.
*/
export function deleteTableRow(
doc: any,
tableRef: string,
index: number,
): { doc: any; deleted: boolean } {
const out = clone(doc);
const located = locateTable(out, tableRef);
if (located == null) return { doc: out, deleted: false };
const { table } = located;
if (!Array.isArray(table.content)) table.content = [];
const rows = table.content.length;
if (!Number.isInteger(index) || index < 0 || index >= rows) {
throw new Error(
`table_delete_row: row index ${index} out of range (table has ${rows} row(s))`,
);
}
if (rows <= 1) {
throw new Error(
"table_delete_row: refusing to delete the only row of the table",
);
}
table.content.splice(index, 1);
return { doc: out, deleted: true };
}
/**
* Set the plain-text content of cell `[row, col]` (0-based) to `text`. Returns
* `{ doc, updated }`; `updated` is false only when the table cannot be located.
* Throws when `row`/`col` is out of range. The cell's own attrs (colspan/
* rowspan/colwidth) are preserved; its content becomes a single text paragraph
* that reuses the cell's existing first-paragraph id when present, else a fresh
* one.
*/
export function updateTableCell(
doc: any,
tableRef: string,
row: number,
col: number,
text: string,
): { doc: any; updated: boolean } {
const out = clone(doc);
const located = locateTable(out, tableRef);
if (located == null) return { doc: out, updated: false };
const { table } = located;
const rowNodes = Array.isArray(table.content) ? table.content : [];
const rows = rowNodes.length;
const rowNode = rowNodes[row];
const cols =
isObject(rowNode) && Array.isArray(rowNode.content)
? rowNode.content.length
: 0;
if (
!Number.isInteger(row) ||
row < 0 ||
row >= rows ||
!Number.isInteger(col) ||
col < 0 ||
col >= cols
) {
throw new Error(`table_update_cell: cell [${row},${col}] out of range`);
}
const cellNode = rowNode.content[col];
// Reuse the cell's existing first-paragraph id, or mint a fresh unique one.
const existingPara = Array.isArray(cellNode?.content)
? cellNode.content[0]
: undefined;
let id =
isObject(existingPara) && isObject(existingPara.attrs)
? existingPara.attrs.id
: undefined;
if (typeof id !== "string" || id.length === 0) {
const used = new Set<string>();
collectIds(out, used);
id = makeFreshId(used);
}
cellNode.content = [makeCellParagraph(id, text)];
return { doc: out, updated: true };
}
+1 -1
View File
@@ -33,7 +33,7 @@
import RE2 from "re2"; import RE2 from "re2";
import { blockPlainText } from "./node-ops.js"; import { blockPlainText } from "@docmost/prosemirror-markdown";
/** An RE2 regex instance (RE2 extends `RegExp`, so it is usable as one). */ /** An RE2 regex instance (RE2 extends `RegExp`, so it is usable as one). */
type Re2Regex = InstanceType<typeof RE2>; type Re2Regex = InstanceType<typeof RE2>;
+4 -4
View File
@@ -14,13 +14,13 @@
* - `marks` arrays are preserved verbatim when fragments are split/reordered. * - `marks` arrays are preserved verbatim when fragments are split/reordered.
*/ */
import { blockPlainText } from "./node-ops.js";
import { canonicalizeFootnotes } from "./footnote-canonicalize.js";
import { import {
blockPlainText,
footnoteContentKey, footnoteContentKey,
makeFootnoteDefinition, makeFootnoteDefinition,
generateFootnoteId, generateFootnoteId,
} from "./footnote-authoring.js"; } from "@docmost/prosemirror-markdown";
import { canonicalizeFootnotes } from "./footnote-canonicalize.js";
export { canonicalizeFootnotes } from "./footnote-canonicalize.js"; export { canonicalizeFootnotes } from "./footnote-canonicalize.js";
@@ -365,7 +365,7 @@ export function noteItem(inlineNodes: any[]): any {
* { type:"footnoteDefinition", attrs:{id}, content:[{ type:"paragraph", content }] } * { type:"footnoteDefinition", attrs:{id}, content:[{ type:"paragraph", content }] }
* (mirrors the editor-ext / docmost-schema FootnoteDefinition node). * (mirrors the editor-ext / docmost-schema FootnoteDefinition node).
* *
* Built on the shared `makeFootnoteDefinition` factory (footnote-authoring.ts); * Built on the shared `makeFootnoteDefinition` factory (`@docmost/prosemirror-markdown`);
* the only extra is a fresh block id on the inner paragraph (Docmost stamps one, * the only extra is a fresh block id on the inner paragraph (Docmost stamps one,
* and the canonicalizer preserves attrs as-is). Single factory, one place to * and the canonicalizer preserves attrs as-is). Single factory, one place to
* change the definition shape. * change the definition shape.
-15
View File
@@ -1,7 +1,6 @@
#!/usr/bin/env node #!/usr/bin/env node
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
import { createDocmostMcpServer } from "./index.js"; import { createDocmostMcpServer } from "./index.js";
import { destroyAllSessions } from "./lib/collab-session.js";
// Standalone stdio entrypoint. This restores the original behavior of the // Standalone stdio entrypoint. This restores the original behavior of the
// package when run as a CLI (`docmost-mcp`): it reads credentials from the // package when run as a CLI (`docmost-mcp`): it reads credentials from the
@@ -34,20 +33,6 @@ async function run() {
console.error("Uncaught exception:", error); console.error("Uncaught exception:", error);
}); });
// Teardown hook (issue #400): destroy every cached live CollabSession on exit
// so a hanging session does not keep a doc loaded on the server (which would
// also defer the server's afterUnloadDocument cleanup). `exit` runs the
// synchronous idempotent teardown; SIGINT/SIGTERM also run it, then exit.
process.on("exit", () => {
destroyAllSessions();
});
for (const signal of ["SIGINT", "SIGTERM"] as const) {
process.on(signal, () => {
destroyAllSessions();
process.exit(0);
});
}
const server = createDocmostMcpServer({ const server = createDocmostMcpServer({
apiUrl: API_URL!, apiUrl: API_URL!,
email: EMAIL!, email: EMAIL!,
@@ -1,7 +1,7 @@
// Mock-HTTP test for the footnoteWarnings plumbing (#166). createPage is the // Mock-HTTP test for the footnoteWarnings plumbing (#166). createPage is the
// representative path that is fully plain-HTTP (import + getPage) and so is // representative path that is fully plain-HTTP (import + getPage) and so is
// mockable here; updatePage / importPageMarkdown attach footnoteWarnings with the // mockable here; updatePage / importPageMarkdown attach footnoteWarnings with the
// IDENTICAL wiring (`analyzeFootnotes(...)` + spread-when-non-empty) but run their // IDENTICAL wiring (`footnoteWarningsField(...)` spread-when-non-empty) but run their
// mutation over the Hocuspocus collab WebSocket, which this plain-HTTP harness // mutation over the Hocuspocus collab WebSocket, which this plain-HTTP harness
// does not stand up. The analyzer itself is unit-tested in footnote-analyze.test. // does not stand up. The analyzer itself is unit-tested in footnote-analyze.test.
import { test, after } from "node:test"; import { test, after } from "node:test";
@@ -76,35 +76,29 @@ function pageHandler() {
}; };
} }
test("createPage attaches footnoteWarnings when the content has footnote problems", async () => { test("createPage attaches footnoteWarnings when the content uses legacy footnote syntax", async () => {
const baseURL = await spawn(pageHandler()); const baseURL = await spawn(pageHandler());
const client = new DocmostClient(baseURL, "user@example.com", "pw"); const client = new DocmostClient(baseURL, "user@example.com", "pw");
// A dangling reference + a duplicate definition + a table marker. // Legacy reference-style `[^id]:` definitions — inert on import since #293.
const content = [ const content = ["Intro[^a].", "", "[^a]: a definition"].join("\n");
"Intro[^missing] and| cell[^t] |.",
"",
"[^d]: one",
"[^d]: two",
"[^t]: in table",
].join("\n");
const result = await client.createPage("T", content, "sp-1"); const result = await client.createPage("T", content, "sp-1");
assert.ok(Array.isArray(result.footnoteWarnings), "footnoteWarnings present"); assert.ok(Array.isArray(result.footnoteWarnings), "footnoteWarnings present");
const joined = result.footnoteWarnings.join("\n"); const joined = result.footnoteWarnings.join("\n");
assert.match(joined, /no matching definition/); // dangling [^missing] assert.match(joined, /reference-style footnotes/i);
assert.match(joined, /defined more than once/); // duplicate [^d] assert.match(joined, /\^\[footnote text\]/); // nudge to the inline form
// The page itself is still returned. // The page itself is still returned.
assert.equal(result.success, true); assert.equal(result.success, true);
}); });
test("createPage omits footnoteWarnings when the content is clean", async () => { test("createPage omits footnoteWarnings when the content uses the inline form", async () => {
const baseURL = await spawn(pageHandler()); const baseURL = await spawn(pageHandler());
const client = new DocmostClient(baseURL, "user@example.com", "pw"); const client = new DocmostClient(baseURL, "user@example.com", "pw");
const content = ["A[^a] and reuse[^a].", "", "[^a]: fine"].join("\n"); const content = "A note.^[the body] and reuse.^[the body]";
const result = await client.createPage("T", content, "sp-1"); const result = await client.createPage("T", content, "sp-1");
assert.equal( assert.equal(
"footnoteWarnings" in result, "footnoteWarnings" in result,
false, false,
"no footnoteWarnings field on clean input", "no footnoteWarnings field on inline-footnote input",
); );
assert.equal(result.success, true); assert.equal(result.success, true);
}); });
@@ -19,7 +19,6 @@ import { WebSocketServer } from "ws";
import { Hocuspocus } from "@hocuspocus/server"; import { Hocuspocus } from "@hocuspocus/server";
import { DocmostClient } from "../../build/client.js"; import { DocmostClient } from "../../build/client.js";
import { buildYDoc } from "../../build/lib/collaboration.js"; import { buildYDoc } from "../../build/lib/collaboration.js";
import { destroyAllSessions } from "../../build/lib/collab-session.js";
// Import the SAME page-lock module instance that build/client.js imports. ESM // Import the SAME page-lock module instance that build/client.js imports. ESM
// caches modules by resolved URL, so this `withPageLock` shares the very // caches modules by resolved URL, so this `withPageLock` shares the very
// per-page mutex map (`chains`) the client uses — letting the replaceImage test // per-page mutex map (`chains`) the client uses — letting the replaceImage test
@@ -189,10 +188,6 @@ async function spawnCollabStack(opts = {}) {
const openStacks = []; const openStacks = [];
after(async () => { after(async () => {
// #400: tests now leave a cached live CollabSession per page. Destroy them
// first (closes the client ws) so the server.close() below is not racing an
// open collab connection.
destroyAllSessions();
await Promise.all( await Promise.all(
openStacks.map( openStacks.map(
({ server, hocuspocus }) => ({ server, hocuspocus }) =>
@@ -275,23 +270,17 @@ test("a UUID input is passed through unchanged and triggers NO /pages/info fetch
); );
}); });
test("repeated slugId edits reuse ONE live collab session and resolve the UUID only once (#400 cache)", async () => { test("a repeated slugId edit resolves the UUID only once (cache)", async () => {
const { state, baseURL } = await spawnCollabStack(); const { state, baseURL } = await spawnCollabStack();
const client = new DocmostClient(baseURL, "user@example.com", "pw"); const client = new DocmostClient(baseURL, "user@example.com", "pw");
// #400: a series of edits on the same page reuses ONE live CollabSession, so // Each mock connection re-seeds a fresh "hello world" doc (the mock does not
// the connect/handshake happens once and the collab doc is OPENED a single // persist across connects), so both edits target "hello". The cache assertion
// time (not per edit). The live ydoc persists between edits (the whole point), // only concerns the slugId->uuid resolution, not the document content.
// so the second edit sees the first edit's result: after "hello" -> "hi world"
// it targets the still-present "world".
await client.editPageText(SLUG, [{ find: "hello", replace: "hi" }]); await client.editPageText(SLUG, [{ find: "hello", replace: "hi" }]);
await client.editPageText(SLUG, [{ find: "world", replace: "planet" }]); await client.editPageText(SLUG, [{ find: "hello", replace: "hey" }]);
assert.deepEqual( assert.deepEqual(state.docNames, [`page.${UUID}`, `page.${UUID}`]);
state.docNames,
[`page.${UUID}`],
"the two edits must reuse one live collab session -> a single collab-doc open (#400)",
);
assert.equal( assert.equal(
state.pagesInfoCalls.length, state.pagesInfoCalls.length,
1, 1,
@@ -336,9 +325,8 @@ test("replaceImage opens by the resolved UUID AND keys its page lock by that UUI
await uploadStarted; // deterministic: replaceImage now holds its page lock. await uploadStarted; // deterministic: replaceImage now holds its page lock.
// (a) OPEN BY UUID: the only collab doc opened so far (the scan pass) used the // (a) OPEN BY UUID: the only collab doc opened so far (the scan pass) used the
// canonical UUID, never the slugId. (#400: the write pass will REUSE this same // canonical UUID, never the slugId. (The write pass opens a second time after
// live session rather than reopen, so docNames stays a single entry — asserted // we release the gate; asserted at the end.)
// at the end.)
assert.deepEqual( assert.deepEqual(
state.docNames, state.docNames,
[`page.${UUID}`], [`page.${UUID}`],
@@ -390,9 +378,8 @@ test("replaceImage opens by the resolved UUID AND keys its page lock by that UUI
assert.equal(res.success, true); assert.equal(res.success, true);
assert.equal(res.replaced, 1, "the one seeded image must be repointed"); assert.equal(res.replaced, 1, "the one seeded image must be repointed");
// #400: the write pass REUSES the scan pass's live session, so the collab doc // Both opens (scan pass + write pass) used the UUID; the slugId never appears.
// is opened ONCE across both passes (never reopened, never by the slugId). assert.deepEqual(state.docNames, [`page.${UUID}`, `page.${UUID}`]);
assert.deepEqual(state.docNames, [`page.${UUID}`]);
assert.ok( assert.ok(
!state.docNames.includes(`page.${SLUG}`), !state.docNames.includes(`page.${SLUG}`),
"replaceImage must NEVER open the collab doc by the slugId (the #260 bug)", "replaceImage must NEVER open the collab doc by the slugId (the #260 bug)",
@@ -1,348 +0,0 @@
import { test, beforeEach, afterEach, mock } from "node:test";
import assert from "node:assert/strict";
import { EventEmitter } from "node:events";
import {
acquireCollabSession,
destroyAllSessions,
__setCollabProviderFactory,
__sessionCountForTests,
} from "../../build/lib/collab-session.js";
import { withPageLock } from "../../build/lib/page-lock.js";
// A stand-in for HocuspocusProvider: it shares the ydoc (so the real yjs
// read/transform/write in CollabSession.mutate runs unchanged), auto-completes
// the connect+sync handshake on a microtask (unaffected by mock timers), and
// exposes hooks to drive disconnect/close/auth-failure and the unsyncedChanges
// ack. There is no collaboration server in the test env, so every test drives
// the provider through this fake.
class FakeProvider extends EventEmitter {
static instances = [];
static connectCount = 0;
static reset() {
FakeProvider.instances = [];
FakeProvider.connectCount = 0;
}
static last() {
return FakeProvider.instances[FakeProvider.instances.length - 1];
}
constructor(config, opts = {}) {
super();
this.config = config;
this.ydoc = config.document;
this.synced = false;
this.unsyncedChanges = opts.unsynced ?? 0;
this.destroyed = false;
FakeProvider.instances.push(this);
FakeProvider.connectCount += 1;
if (opts.autoSync !== false) {
// Real HocuspocusProvider fires onSynced asynchronously after the
// handshake; a microtask reproduces that without depending on timers.
queueMicrotask(() => {
if (this.destroyed) return;
this.config.onConnect?.();
this.synced = true;
this.config.onSynced?.();
});
}
}
destroy() {
this.destroyed = true;
}
// --- test drivers ---
_disconnect() {
this.config.onDisconnect?.();
}
_close() {
this.config.onClose?.();
}
_authFail() {
this.config.onAuthenticationFailed?.();
}
_ack() {
this.unsyncedChanges = 0;
this.emit("unsyncedChanges", { number: 0 });
}
}
/** Build a provider factory that stamps every provider with the given opts. */
function factory(opts = {}) {
return (config) => new FakeProvider(config, opts);
}
/** A minimal, schema-valid ProseMirror doc for a write. */
function docWith(text) {
return {
type: "doc",
content: [
{ type: "paragraph", content: [{ type: "text", text: String(text) }] },
],
};
}
const ENV_KEYS = [
"MCP_COLLAB_SESSION_IDLE_MS",
"MCP_COLLAB_SESSION_MAX_AGE_MS",
"MCP_COLLAB_SESSION_MAX_ENTRIES",
];
let savedEnv;
beforeEach(() => {
savedEnv = {};
for (const k of ENV_KEYS) savedEnv[k] = process.env[k];
FakeProvider.reset();
__setCollabProviderFactory(factory());
});
afterEach(() => {
destroyAllSessions();
__setCollabProviderFactory(null);
mock.timers.reset();
for (const k of ENV_KEYS) {
if (savedEnv[k] === undefined) delete process.env[k];
else process.env[k] = savedEnv[k];
}
});
test("acquire opens one provider; reuse returns the SAME live session", async () => {
const a = await acquireCollabSession("page-1", "tok", "http://h/api");
const b = await acquireCollabSession("page-1", "tok", "http://h/api");
assert.equal(a, b, "same (wsUrl,page,token) must reuse the session");
assert.equal(FakeProvider.connectCount, 1, "exactly one connect/sync");
assert.equal(__sessionCountForTests(), 1);
});
test("N mutates on one page open the provider ONCE (coalesced series)", async () => {
const session = await acquireCollabSession("page-1", "tok", "http://h/api");
for (let i = 0; i < 20; i++) {
const r = await session.mutate(() => docWith(`edit ${i}`));
assert.ok(r && r.doc, "each mutate resolves a MutationResult");
}
assert.equal(
FakeProvider.connectCount,
1,
"20 mutates must not reconnect — one live provider for the whole series",
);
});
test("mutate preserves the transform-abort no-op report (null transform)", async () => {
const session = await acquireCollabSession("page-1", "tok", "http://h/api");
const r = await session.mutate(() => null);
assert.equal(r.verify.changed, false);
assert.equal(r.verify.summary, "no changes (transform aborted)");
});
test("registry key includes the token: different tokens => different sessions", async () => {
const a = await acquireCollabSession("page-1", "tok-A", "http://h/api");
const b = await acquireCollabSession("page-1", "tok-B", "http://h/api");
assert.notEqual(a, b, "different tokens must never share a session");
assert.equal(FakeProvider.connectCount, 2);
assert.equal(__sessionCountForTests(), 2);
// Same token reuses.
const a2 = await acquireCollabSession("page-1", "tok-A", "http://h/api");
assert.equal(a, a2);
assert.equal(FakeProvider.connectCount, 2);
});
test("disconnect at any time kills the session and removes it from the registry", async () => {
const a = await acquireCollabSession("page-1", "tok", "http://h/api");
assert.equal(__sessionCountForTests(), 1);
FakeProvider.last()._disconnect();
assert.equal(__sessionCountForTests(), 0, "dead session is deregistered");
// Re-acquire opens a brand new provider.
const b = await acquireCollabSession("page-1", "tok", "http://h/api");
assert.notEqual(a, b);
assert.equal(FakeProvider.connectCount, 2);
});
test("an in-flight mutate rejects with the connection-closed text on disconnect", async () => {
__setCollabProviderFactory(factory({ unsynced: 1 })); // stay pending after write
const session = await acquireCollabSession("page-1", "tok", "http://h/api");
const p = session.mutate(() => docWith("x"));
// The write happened synchronously; persistence is pending. Now the socket drops.
FakeProvider.last()._disconnect();
await assert.rejects(
p,
/Collaboration connection closed before the update was persisted\/synced/,
);
});
test("auth failure rejects the pending open with the auth error text", async () => {
__setCollabProviderFactory(factory({ autoSync: false }));
const p = acquireCollabSession("page-1", "tok", "http://h/api");
// Let the provider be constructed, then fail auth.
await Promise.resolve();
FakeProvider.last()._authFail();
await assert.rejects(
p,
/Authentication failed for collaboration connection/,
);
assert.equal(__sessionCountForTests(), 0);
});
test("mutate-error invalidates the session (caller destroys; re-acquire is fresh)", async () => {
const session = await acquireCollabSession("page-1", "tok", "http://h/api");
await assert.rejects(
session.mutate(() => {
throw new Error("afterText not found");
}),
/afterText not found/,
);
// The production caller destroys on failure; mirror that here.
session.destroy("mutate failed");
assert.equal(__sessionCountForTests(), 0);
const fresh = await acquireCollabSession("page-1", "tok", "http://h/api");
assert.notEqual(fresh, session);
assert.equal(FakeProvider.connectCount, 2);
});
test("a pending write resolves when the server acks (unsyncedChanges -> 0)", async () => {
__setCollabProviderFactory(factory({ unsynced: 1 }));
const session = await acquireCollabSession("page-1", "tok", "http://h/api");
const p = session.mutate(() => docWith("y"));
FakeProvider.last()._ack();
const r = await p;
assert.ok(r.doc);
});
test("concurrent mutate on one session: the second rejects, the first is unaffected", async () => {
__setCollabProviderFactory(factory({ unsynced: 1 })); // first stays pending after write
const session = await acquireCollabSession("page-1", "tok", "http://h/api");
// First mutate: write happens synchronously, persistence ack still pending.
const first = session.mutate(() => docWith("first"));
// Second overlapping mutate on the SAME session must fail fast.
await assert.rejects(
session.mutate(() => docWith("second")),
/mutate already in-flight; caller must serialize \(hold the page lock\)/,
);
// The first op is untouched: its rejector was not clobbered. Ack it now.
FakeProvider.last()._ack();
const r = await first;
assert.ok(r.doc, "the first in-flight mutate still resolves on its own ack");
assert.equal(FakeProvider.connectCount, 1, "no reconnect from the rejected overlap");
});
test("sequential mutates on one session both succeed (the guard doesn't break serialized use)", async () => {
const session = await acquireCollabSession("page-1", "tok", "http://h/api");
// Await the first fully, then run the second: inflightReject was cleared by
// localFinish before the first settled, so the guard is clear for the second.
const r1 = await session.mutate(() => docWith("one"));
assert.ok(r1.doc, "first sequential mutate resolves");
const r2 = await session.mutate(() => docWith("two"));
assert.ok(r2.doc, "second sequential mutate resolves — guard not tripped");
assert.equal(FakeProvider.connectCount, 1, "sequential mutates reuse one provider");
});
test("connect timeout rejects with the connect-timeout text and fires the metric hook", async () => {
mock.timers.enable({ apis: ["setTimeout"] });
__setCollabProviderFactory(factory({ autoSync: false }));
let metricFired = 0;
const p = acquireCollabSession("page-1", "tok", "http://h/api", {
onConnectTimeout: () => {
metricFired += 1;
},
});
mock.timers.tick(25000);
await assert.rejects(p, /Connection timeout to collaboration server/);
assert.equal(metricFired, 1);
assert.equal(__sessionCountForTests(), 0);
});
test("idle TTL destroys the session; re-acquire reconnects", async () => {
mock.timers.enable({ apis: ["setTimeout"] });
process.env.MCP_COLLAB_SESSION_IDLE_MS = "1000";
const a = await acquireCollabSession("page-1", "tok", "http://h/api");
assert.equal(__sessionCountForTests(), 1);
mock.timers.tick(1000); // idle fires
assert.equal(__sessionCountForTests(), 0, "idle timeout destroyed it");
const b = await acquireCollabSession("page-1", "tok", "http://h/api");
assert.notEqual(a, b);
assert.equal(FakeProvider.connectCount, 2);
});
test("max age is enforced at acquire (destroy + open fresh), idle held off", async () => {
mock.timers.enable({ apis: ["setTimeout", "Date"] });
process.env.MCP_COLLAB_SESSION_MAX_AGE_MS = "1000";
process.env.MCP_COLLAB_SESSION_IDLE_MS = "10000000"; // never fires in this test
const a = await acquireCollabSession("page-1", "tok", "http://h/api");
mock.timers.tick(2000); // past max age, below idle
const b = await acquireCollabSession("page-1", "tok", "http://h/api");
assert.notEqual(a, b, "a session past its max age must be replaced");
assert.equal(FakeProvider.connectCount, 2);
});
test("registry cap: least-recently-used session is destroy-evicted", async () => {
process.env.MCP_COLLAB_SESSION_MAX_ENTRIES = "2";
const s1 = await acquireCollabSession("page-1", "tok", "http://h/api");
const p1prov = FakeProvider.last();
const s2 = await acquireCollabSession("page-2", "tok", "http://h/api");
const p2prov = FakeProvider.last();
assert.equal(__sessionCountForTests(), 2);
// Touch page-1 so page-2 becomes the least-recently-used entry.
assert.equal(await acquireCollabSession("page-1", "tok", "http://h/api"), s1);
assert.equal(FakeProvider.connectCount, 2, "reuse must not reconnect");
// A third distinct page (cap 2) must evict the LRU (page-2), not page-1.
const s3 = await acquireCollabSession("page-3", "tok", "http://h/api");
assert.equal(__sessionCountForTests(), 2);
assert.equal(FakeProvider.connectCount, 3);
assert.equal(p2prov.destroyed, true, "the LRU provider was destroy-evicted");
assert.equal(p1prov.destroyed, false, "the MRU page-1 survived");
// page-1 still reused (survived), page-3 still reused.
assert.equal(await acquireCollabSession("page-1", "tok", "http://h/api"), s1);
assert.equal(await acquireCollabSession("page-3", "tok", "http://h/api"), s3);
assert.equal(FakeProvider.connectCount, 3, "no extra reconnects");
});
test("MCP_COLLAB_SESSION_IDLE_MS=0 disables the cache (legacy provider-per-op)", async () => {
process.env.MCP_COLLAB_SESSION_IDLE_MS = "0";
const s1 = await acquireCollabSession("page-1", "tok", "http://h/api");
// Never registered (ephemeral).
assert.equal(__sessionCountForTests(), 0);
const r1 = await s1.mutate(() => docWith("a"));
assert.ok(r1.doc);
// After its single op the ephemeral session self-destroyed.
assert.equal(FakeProvider.instances[0].destroyed, true);
// A second op opens a BRAND NEW provider (no reuse).
const s2 = await acquireCollabSession("page-1", "tok", "http://h/api");
assert.notEqual(s1, s2);
assert.equal(FakeProvider.connectCount, 2);
});
test("replaceImage-shaped flow: acquire under an EXTERNAL page lock does not deadlock and reuses one session", async () => {
const pageId = "page-lock";
// Mirror replaceImage: hold ONE withPageLock across scan (read-only) + write,
// each going through the non-locking acquireCollabSession.
const result = await withPageLock(pageId, async () => {
// pass 1: read-only scan (transform returns null -> no write)
const scan = await acquireCollabSession(pageId, "tok", "http://h/api");
const scanRes = await scan.mutate(() => null);
assert.equal(scanRes.verify.changed, false);
// pass 2: the actual write, same held lock
const write = await acquireCollabSession(pageId, "tok", "http://h/api");
return write.mutate(() => docWith("repointed"));
});
assert.ok(result.doc, "the locked scan+write completed without deadlock");
assert.equal(
FakeProvider.connectCount,
1,
"both passes under the held lock reuse ONE live session",
);
});
test("destroyAllSessions tears down every cached session", async () => {
await acquireCollabSession("page-1", "tok", "http://h/api");
await acquireCollabSession("page-2", "tok", "http://h/api");
assert.equal(__sessionCountForTests(), 2);
const provs = [...FakeProvider.instances];
destroyAllSessions();
assert.equal(__sessionCountForTests(), 0);
assert.ok(provs.every((p) => p.destroyed), "all providers destroyed");
});
@@ -1,64 +1,45 @@
import { test } from "node:test"; import { test } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { analyzeFootnotes } from "../../build/lib/footnote-analyze.js"; import {
footnoteWarningsField,
hasLegacyFootnoteDefinition,
} from "../../build/lib/footnote-analyze.js";
test("clean footnotes produce no diagnostics", () => { // #414: the legacy footnote diagnostics were reduced to ONE advisory that fires
const md = ["A[^a] and B[^b].", "", "[^a]: first", "[^b]: second"].join("\n"); // on the PRESENCE of legacy reference-style `[^id]:` definition syntax (inert on
const d = analyzeFootnotes(md); // import since #293), nudging the author to inline `^[...]` footnotes.
assert.deepEqual(d.danglingReferences, []);
assert.deepEqual(d.emptyDefinitions, []); test("inline `^[...]` footnotes produce no warning", () => {
assert.deepEqual(d.duplicateDefinitions, []); const md = "A note here.^[the body] and reuse elsewhere.^[the body]";
assert.deepEqual(d.referencesInTables, []); assert.equal(hasLegacyFootnoteDefinition(md), false);
assert.deepEqual(d.warnings, []); assert.deepEqual(footnoteWarningsField(md), {});
}); });
test("reuse (repeated references to one definition) is NOT a warning", () => { test("no footnotes at all produce no warning", () => {
const md = ["A[^a] B[^a] C[^a].", "", "[^a]: shared"].join("\n"); const md = "Just a paragraph with [a link](https://x) and no footnotes.";
const d = analyzeFootnotes(md); assert.equal(hasLegacyFootnoteDefinition(md), false);
assert.deepEqual(d.danglingReferences, []); assert.deepEqual(footnoteWarningsField(md), {});
assert.deepEqual(d.warnings, []);
}); });
test("dangling reference (no definition) is reported", () => { test("a legacy `[^id]:` definition triggers the single advisory", () => {
const md = ["See[^missing] and[^a].", "", "[^a]: defined"].join("\n"); const md = ["See[^a].", "", "[^a]: defined"].join("\n");
const d = analyzeFootnotes(md); assert.equal(hasLegacyFootnoteDefinition(md), true);
assert.deepEqual(d.danglingReferences, ["missing"]); const field = footnoteWarningsField(md);
assert.equal(d.warnings.length, 1); assert.equal(field.footnoteWarnings.length, 1);
assert.match(d.warnings[0], /no matching definition/); assert.match(field.footnoteWarnings[0], /reference-style footnotes/i);
assert.match(d.warnings[0], /\[\^missing\]/); assert.match(field.footnoteWarnings[0], /\^\[footnote text\]/);
}); });
test("empty definition text is reported", () => { test("a bare `[^id]` reference (no definition line) is not flagged", () => {
const md = ["See[^a].", "", "[^a]: "].join("\n"); // Only the definition syntax `[^id]:` is a reliable signal of legacy authoring;
const d = analyzeFootnotes(md); // a lone `[^x]` in prose is too ambiguous to warn on.
assert.deepEqual(d.emptyDefinitions, ["a"]); const md = "A sentence mentioning [^x] with no definition.";
assert.match(d.warnings.join("\n"), /empty text/); assert.equal(hasLegacyFootnoteDefinition(md), false);
assert.deepEqual(footnoteWarningsField(md), {});
}); });
test("duplicate definition id is reported (first-wins)", () => { test("legacy syntax inside a code fence is ignored (fence-aware)", () => {
const md = ["See[^d].", "", "[^d]: first", "[^d]: second"].join("\n");
const d = analyzeFootnotes(md);
assert.deepEqual(d.duplicateDefinitions, ["d"]);
assert.match(d.warnings.join("\n"), /defined more than once/);
});
test("reference inside a GFM table row is reported (heuristic)", () => {
const md = [
"| Col |",
"| --- |",
"| cell[^t] |",
"",
"[^t]: table note",
].join("\n");
const d = analyzeFootnotes(md);
assert.deepEqual(d.referencesInTables, ["t"]);
assert.match(d.warnings.join("\n"), /table/);
// It is defined, so it is NOT also dangling.
assert.deepEqual(d.danglingReferences, []);
});
test("footnote syntax inside a code fence is ignored", () => {
const md = [ const md = [
"Intro.", "Intro.",
"", "",
@@ -67,40 +48,22 @@ test("footnote syntax inside a code fence is ignored", () => {
"[^demo]: not a real definition", "[^demo]: not a real definition",
"```", "```",
"", "",
"Outro[^a].", "Outro with an inline note.^[real]",
"",
"[^a]: real",
].join("\n"); ].join("\n");
const d = analyzeFootnotes(md); assert.equal(hasLegacyFootnoteDefinition(md), false);
// `[^demo]` lives only in the fenced block, so it is neither a reference nor a assert.deepEqual(footnoteWarningsField(md), {});
// dangling one, and `[^demo]:` is not counted as a definition.
assert.deepEqual(d.danglingReferences, []);
assert.deepEqual(d.duplicateDefinitions, []);
assert.deepEqual(d.warnings, []);
}); });
test("a reference that only appears inside a definition's text is not dangling", () => { test("a legacy definition OUTSIDE a fence still warns even with a fenced sample", () => {
// `[^b]` is referenced from within [^a]'s text and has its own definition.
const md = ["See[^a].", "", "[^a]: see also [^b]", "[^b]: the other"].join(
"\n",
);
const d = analyzeFootnotes(md);
assert.deepEqual(d.danglingReferences, []);
});
test("multiple problem classes accumulate distinct warnings", () => {
const md = [ const md = [
"Ref[^x] and[^dup].", "```",
"[^demo]: example inside a fence",
"```",
"", "",
"[^dup]: one", "See[^a].",
"[^dup]: two", "",
"[^empty]:", "[^a]: real definition outside the fence",
].join("\n"); ].join("\n");
const d = analyzeFootnotes(md); assert.equal(hasLegacyFootnoteDefinition(md), true);
// x has no definition; dup is defined twice; empty is empty AND has no ref. assert.equal(footnoteWarningsField(md).footnoteWarnings.length, 1);
assert.ok(d.danglingReferences.includes("x"));
assert.deepEqual(d.duplicateDefinitions, ["dup"]);
assert.deepEqual(d.emptyDefinitions, ["empty"]);
// One warning line per problem class present.
assert.ok(d.warnings.length >= 3);
}); });
@@ -5,7 +5,7 @@ import { canonicalizeFootnotes } from "../../build/lib/footnote-canonicalize.js"
import { import {
footnoteContentKey, footnoteContentKey,
generateFootnoteId, generateFootnoteId,
} from "../../build/lib/footnote-authoring.js"; } from "@docmost/prosemirror-markdown";
import { insertInlineFootnote } from "../../build/lib/transforms.js"; import { insertInlineFootnote } from "../../build/lib/transforms.js";
import { markdownToProseMirrorCanonical } from "../../build/lib/collaboration.js"; import { markdownToProseMirrorCanonical } from "../../build/lib/collaboration.js";
@@ -1,39 +1,37 @@
import { test } from "node:test"; import { test } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { import { footnoteWarningsField } from "../../build/lib/footnote-analyze.js";
analyzeFootnotes,
footnoteWarningsField,
} from "../../build/lib/footnote-analyze.js";
import { import {
serializeDocmostMarkdown, serializeDocmostMarkdown,
parseDocmostMarkdown, parseDocmostMarkdown,
} from "../../build/lib/markdown-document.js"; } from "../../build/lib/markdown-document.js";
// Pins the footnoteWarnings PLUMBING contract (#169 review): the field is // Pins the footnoteWarnings PLUMBING contract (#169 review; reduced in #414): the
// present only on problems and omitted on clean input, AND `import_page_markdown` // field is present only when legacy reference-style `[^id]:` syntax is used and
// analyzes the BODY (after the docmost:meta / docmost:comments blocks) — so a // omitted otherwise, AND `import_page_markdown` analyzes the BODY (after the
// footnote-like token inside those JSON blocks never warns, while a real marker // docmost:meta / docmost:comments blocks) — so a footnote-like token inside those
// in the body does. importPageMarkdown does exactly // JSON blocks never warns, while a real definition in the body does.
// `footnoteWarningsField(parseDocmostMarkdown(full).body)` over a collab socket // importPageMarkdown does exactly `footnoteWarningsField(parseDocmostMarkdown(full).body)`
// this harness does not stand up, so we test the same pure composition directly. // over a collab socket this harness does not stand up, so we test the same pure
// composition directly.
test("footnoteWarningsField is present on problems and omitted on clean input", () => { test("footnoteWarningsField is present on legacy syntax and omitted on the inline form", () => {
const problem = footnoteWarningsField("See[^missing].\n\n[^a]: defined"); const legacy = footnoteWarningsField("See[^a].\n\n[^a]: defined");
assert.ok(Array.isArray(problem.footnoteWarnings)); assert.ok(Array.isArray(legacy.footnoteWarnings));
assert.match(problem.footnoteWarnings.join("\n"), /no matching definition/); assert.match(legacy.footnoteWarnings.join("\n"), /reference-style footnotes/i);
const clean = footnoteWarningsField("A[^a] and reuse[^a].\n\n[^a]: fine"); const inline = footnoteWarningsField("A note.^[the body] reused.^[the body]");
assert.deepEqual(clean, {}); // no key at all on clean input assert.deepEqual(inline, {}); // no key at all on inline-footnote input
}); });
test("import analyzes the BODY only — tokens inside meta/comments never warn", () => { test("import analyzes the BODY only — tokens inside meta/comments never warn", () => {
// meta + comments JSON carry `[^metaonly]` / `[^commentonly]`-looking text; the // meta + comments JSON carry `[^metaonly]:` / `[^commentonly]:`-looking text;
// BODY has a genuinely dangling `[^bodyref]`. // the BODY has a genuine legacy `[^bodyref]:` definition.
const full = serializeDocmostMarkdown( const full = serializeDocmostMarkdown(
{ pageId: "p1", note: "front-matter mentions [^metaonly] in text" }, { pageId: "p1", note: "front-matter mentions [^metaonly]: in text" },
"Body with a dangling[^bodyref] marker.", "Body with a legacy[^bodyref] marker.\n\n[^bodyref]: the definition",
[{ id: "c1", content: "a comment that says [^commentonly]" }], [{ id: "c1", content: "a comment that says [^commentonly]: text" }],
); );
const { body } = parseDocmostMarkdown(full); const { body } = parseDocmostMarkdown(full);
@@ -42,20 +40,19 @@ test("import analyzes the BODY only — tokens inside meta/comments never warn",
assert.ok(!body.includes("[^commentonly]")); assert.ok(!body.includes("[^commentonly]"));
const field = footnoteWarningsField(body); const field = footnoteWarningsField(body);
const joined = (field.footnoteWarnings ?? []).join("\n"); // ONLY the body's legacy definition triggers the advisory.
// ONLY the body's dangling reference is flagged. assert.ok(Array.isArray(field.footnoteWarnings));
assert.match(joined, /\[\^bodyref\]/); assert.match(field.footnoteWarnings.join("\n"), /reference-style footnotes/i);
assert.ok(!joined.includes("metaonly"));
assert.ok(!joined.includes("commentonly"));
// Cross-check against analyzeFootnotes directly (same composition the importer uses). // The meta/comments tokens, analyzed on their own, would NOT have warned in a
assert.deepEqual(analyzeFootnotes(body).danglingReferences, ["bodyref"]); // way that leaks here — the field is computed over the body only.
assert.deepEqual(footnoteWarningsField("front-matter mentions text"), {});
}); });
test("import on a clean body yields no footnoteWarnings field", () => { test("import on an inline-footnote body yields no footnoteWarnings field", () => {
const full = serializeDocmostMarkdown( const full = serializeDocmostMarkdown(
{ pageId: "p1" }, { pageId: "p1" },
"Clean body[^a] reusing[^a].\n\n[^a]: ok", "Clean body.^[a note] reusing.^[a note]",
[], [],
); );
const { body } = parseDocmostMarkdown(full); const { body } = parseDocmostMarkdown(full);
@@ -5,7 +5,7 @@ import {
insertNodeRelative, insertNodeRelative,
sanitizeForYjs, sanitizeForYjs,
findUnstorableAttr, findUnstorableAttr,
} from "../../build/lib/node-ops.js"; } from "@docmost/prosemirror-markdown";
// ProseMirror builders. Blocks carry a stable id in attrs.id. // ProseMirror builders. Blocks carry a stable id in attrs.id.
const textNode = (text) => ({ type: "text", text }); const textNode = (text) => ({ type: "text", text });
+1 -1
View File
@@ -7,7 +7,7 @@ import {
deleteNodeById, deleteNodeById,
assertUnambiguousMatch, assertUnambiguousMatch,
insertNodeRelative, insertNodeRelative,
} from "../../build/lib/node-ops.js"; } from "@docmost/prosemirror-markdown";
// ProseMirror builders. Blocks carry a stable id in attrs.id. // ProseMirror builders. Blocks carry a stable id in attrs.id.
const textNode = (text) => ({ type: "text", text }); const textNode = (text) => ({ type: "text", text });
+1 -1
View File
@@ -1,7 +1,7 @@
import { test } from "node:test"; import { test } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { buildOutline, getNodeByRef } from "../../build/lib/node-ops.js"; import { buildOutline, getNodeByRef } from "@docmost/prosemirror-markdown";
// Helpers to build the small fixture doc. // Helpers to build the small fixture doc.
const textNode = (text) => ({ type: "text", text }); const textNode = (text) => ({ type: "text", text });
+1 -1
View File
@@ -2,7 +2,7 @@ import { test } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { searchInDoc } from "../../build/lib/page-search.js"; import { searchInDoc } from "../../build/lib/page-search.js";
import { getNodeByRef } from "../../build/lib/node-ops.js"; import { getNodeByRef } from "@docmost/prosemirror-markdown";
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Document builders. Mirror the Docmost ProseMirror shape: paragraphs/headings // Document builders. Mirror the Docmost ProseMirror shape: paragraphs/headings
@@ -1,7 +1,7 @@
import { test } from "node:test"; import { test } from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { parseNodeArg } from "../../build/lib/parse-node-arg.js"; import { parseNodeArg } from "@docmost/prosemirror-markdown";
test("parseNodeArg passes an object through unchanged", () => { test("parseNodeArg passes an object through unchanged", () => {
const obj = { type: "paragraph", content: [] }; const obj = { type: "paragraph", content: [] };
+1 -1
View File
@@ -6,7 +6,7 @@ import {
insertTableRow, insertTableRow,
deleteTableRow, deleteTableRow,
updateTableCell, updateTableCell,
} from "../../build/lib/node-ops.js"; } from "@docmost/prosemirror-markdown";
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Builders. Tables/rows/cells carry NO attrs.id — only the paragraph inside a // Builders. Tables/rows/cells carry NO attrs.id — only the paragraph inside a
@@ -59,3 +59,89 @@ export function splitFootnoteParagraphs(encoded: string): string[] {
paragraphs.push(current); paragraphs.push(current);
return paragraphs; return paragraphs;
} }
// ---------------------------------------------------------------------------
// Inline-authoring helpers (#414: moved here from the mcp `footnote-authoring.ts`
// fork so the dedup convention — content-key + definition factory + id gen —
// has ONE home next to the importer that shares the convention). Used by the
// mcp author-inline tool (`insertInlineFootnote` in transforms.ts).
// ---------------------------------------------------------------------------
const FOOTNOTE_DEFINITION_NAME = "footnoteDefinition";
function cloneJson<T>(v: T): T {
if (typeof structuredClone === "function") return structuredClone(v);
return JSON.parse(JSON.stringify(v)) as T;
}
/**
* Normalized content key for de-duplicating footnote DEFINITIONS by their text.
*
* Two definitions with the same key are the SAME footnote — so the inline
* authoring tool reuses one id (one number, one definition, several references)
* instead of minting a second definition. Key = plaintext (whitespace-collapsed,
* trimmed) PLUS a signature of the inline mark types in order, so two notes that
* read the same but differ in formatting (one bold, one plain) are NOT merged.
* Conservative: only an exact match merges.
*/
export function footnoteContentKey(defNode: any): string {
const parts: string[] = [];
const visit = (n: any): void => {
if (!n || typeof n !== "object") return;
if (n.type === "text" && typeof n.text === "string") {
const marks = Array.isArray(n.marks)
? n.marks.map((m: any) => m?.type).filter(Boolean).sort().join(",")
: "";
parts.push(`${n.text}${marks}`);
}
if (Array.isArray(n.content)) for (const c of n.content) visit(c);
};
visit(defNode);
// Collapse the assembled text's whitespace and trim, keeping the mark
// signature attached so formatting differences still distinguish notes.
return parts
.join("")
.replace(/[ \t\r\n]+/g, " ")
.trim();
}
/**
* Build a footnoteDefinition node from inline ProseMirror nodes, keyed by id.
*/
export function makeFootnoteDefinition(id: string, inlineNodes: any[]): any {
const content = Array.isArray(inlineNodes) ? cloneJson(inlineNodes) : [];
return {
type: FOOTNOTE_DEFINITION_NAME,
attrs: { id },
content: [{ type: "paragraph", content }],
};
}
/**
* Generate a uuidv7-style id (time-ordered), matching editor-ext's
* `generateFootnoteId`. Used for a genuinely-new inline footnote id.
*/
export function generateFootnoteId(): string {
const now = Date.now();
const timeHex = now.toString(16).padStart(12, "0");
const rand = (length: number) => {
let s = "";
for (let i = 0; i < length; i++)
s += Math.floor(Math.random() * 16).toString(16);
return s;
};
const versioned = "7" + rand(3);
const variantNibble = (8 + Math.floor(Math.random() * 4)).toString(16);
const variant = variantNibble + rand(3);
return (
timeHex.slice(0, 8) +
"-" +
timeHex.slice(8, 12) +
"-" +
versioned +
"-" +
variant +
"-" +
rand(12)
);
}
@@ -44,3 +44,35 @@ export {
docsCanonicallyEqual, docsCanonicallyEqual,
} from "./canonicalize.js"; } from "./canonicalize.js";
export { parsePageFile, serializePageFile } from "./page-file.js"; export { parsePageFile, serializePageFile } from "./page-file.js";
// Pure, network-free helpers for manipulating a ProseMirror/TipTap document
// tree by node id (#414: the single canonical copy, formerly forked into mcp).
// Consumed by `@docmost/mcp` (patch/insert/delete node, table tools, outline).
export {
blockPlainText,
buildOutline,
getNodeByRef,
replaceNodeById,
deleteNodeById,
sanitizeForYjs,
findUnstorableAttr,
insertNodeRelative,
readTable,
insertTableRow,
deleteTableRow,
updateTableCell,
assertUnambiguousMatch,
} from "./node-ops.js";
export type { OutlineEntry } from "./node-ops.js";
// Normalize a ProseMirror node arg that the model may have serialized as a JSON
// string (#414: single copy shared by mcp and the CommonJS server app).
export { parseNodeArg } from "./parse-node-arg.js";
// Inline-footnote authoring convention (#414: single copy, formerly the mcp
// `footnote-authoring.ts` fork), shared with the importer's `assembleFootnotes`.
export {
footnoteContentKey,
makeFootnoteDefinition,
generateFootnoteId,
} from "./footnote.js";
@@ -14,6 +14,8 @@
* `content`, non-object nodes, and absent `attrs` are tolerated. * `content`, non-object nodes, and absent `attrs` are tolerated.
*/ */
import { stripInlineMarkdown } from "./text-normalize.js";
/** Deep-clone a JSON-serializable value without mutating the original. */ /** Deep-clone a JSON-serializable value without mutating the original. */
function clone<T>(value: T): T { function clone<T>(value: T): T {
if (typeof structuredClone === "function") { if (typeof structuredClone === "function") {
@@ -97,12 +99,15 @@ export function buildOutline(doc: any): OutlineEntry[] {
const entry: OutlineEntry = { const entry: OutlineEntry = {
index: i, index: i,
type, type,
id: isObject(block) && isObject(block.attrs) ? block.attrs.id ?? null : null, id:
isObject(block) && isObject(block.attrs)
? (block.attrs.id ?? null)
: null,
firstText: truncate(blockPlainText(block), 100), firstText: truncate(blockPlainText(block), 100),
}; };
if (type === "heading") { if (type === "heading") {
entry.level = isObject(block.attrs) ? block.attrs.level ?? null : null; entry.level = isObject(block.attrs) ? (block.attrs.level ?? null) : null;
} else if (type === "table") { } else if (type === "table") {
const headerRow = block.content?.[0]?.content ?? []; const headerRow = block.content?.[0]?.content ?? [];
entry.rows = block.content?.length ?? 0; entry.rows = block.content?.length ?? 0;
@@ -247,6 +252,33 @@ export function deleteNodeById(
return { doc: out, deleted }; return { doc: out, deleted };
} }
/**
* Throw a clear, model-actionable error when a node-id write op did NOT match
* exactly one node (#159). `count === 0` -> "no node found"; `count > 1` ->
* "ambiguous, refused" — Docmost duplicates block ids on copy/paste, so a write
* by id could clobber/remove EVERY duplicate. The caller skips the write for any
* `count !== 1` (the transform returns null), so this only REPORTS; nothing was
* changed. No-op for the unambiguous single-match case.
*/
export function assertUnambiguousMatch(
op: "patch_node" | "delete_node",
verb: "replace" | "delete",
count: number,
nodeId: string,
pageId: string,
): void {
if (count === 0) {
throw new Error(
`${op}: no node with id "${nodeId}" found on page ${pageId}`,
);
}
if (count > 1) {
throw new Error(
`${op}: id "${nodeId}" is ambiguous — ${count} nodes on page ${pageId} share it (block ids are duplicated on copy/paste). Refusing to ${verb} all of them; nothing was changed. Re-target with a more specific anchor.`,
);
}
}
/** /**
* Deep-clone `doc` and strip every node/mark attribute whose value is strictly * Deep-clone `doc` and strip every node/mark attribute whose value is strictly
* `undefined`, so the result is safe to hand to Yjs (which throws an opaque * `undefined`, so the result is safe to hand to Yjs (which throws an opaque
@@ -364,6 +396,31 @@ const REQUIRED_CONTAINER: Record<string, string> = {
tableHeader: "tableRow", tableHeader: "tableRow",
}; };
/**
* Find the index of the first TOP-LEVEL block whose plain text includes the
* anchor, with a markdown-stripping FALLBACK. Returns -1 when none matches.
*
* Two passes preserve "exact wins globally":
* - Pass 1: first block containing the verbatim `anchorText`.
* - Pass 2 (only if pass 1 found nothing): first block containing the
* markdown-stripped anchor, when stripping actually changed it.
*/
function findAnchorTextIndex(content: any[], anchorText: string): number {
if (!Array.isArray(content)) return -1;
// Pass 1: exact.
for (let i = 0; i < content.length; i++) {
if (blockPlainText(content[i]).includes(anchorText)) return i;
}
// Pass 2: markdown-stripped fallback.
const a = stripInlineMarkdown(anchorText);
if (a !== anchorText && a.length > 0) {
for (let i = 0; i < content.length; i++) {
if (blockPlainText(content[i]).includes(a)) return i;
}
}
return -1;
}
/** /**
* Locate an anchor and return its ancestor chain (from `doc` down to and * Locate an anchor and return its ancestor chain (from `doc` down to and
* including the matched node). Each chain entry is `{ node, index }` where * including the matched node). Each chain entry is `{ node, index }` where
@@ -399,14 +456,14 @@ function findAnchorChain(
} }
// By text: only top-level blocks are scanned (same rule as the JSON path). // By text: only top-level blocks are scanned (same rule as the JSON path).
// Exact match wins; a markdown-stripped fallback is tried only on a miss.
if (opts.anchorText != null && Array.isArray(doc.content)) { if (opts.anchorText != null && Array.isArray(doc.content)) {
for (let i = 0; i < doc.content.length; i++) { const i = findAnchorTextIndex(doc.content, opts.anchorText);
if (blockPlainText(doc.content[i]).includes(opts.anchorText)) { if (i !== -1) {
return [ return [
{ node: doc, index: -1 }, { node: doc, index: -1 },
{ node: doc.content[i], index: i }, { node: doc.content[i], index: i },
]; ];
}
} }
} }
@@ -540,13 +597,13 @@ export function insertNodeRelative(
return { doc: out, inserted }; return { doc: out, inserted };
} }
// Resolve by text: only top-level doc.content blocks are scanned. // Resolve by text: only top-level doc.content blocks are scanned. Exact
// match wins; a markdown-stripped fallback is tried only on a miss.
if (opts.anchorText != null && isObject(out) && Array.isArray(out.content)) { if (opts.anchorText != null && isObject(out) && Array.isArray(out.content)) {
for (let i = 0; i < out.content.length; i++) { const i = findAnchorTextIndex(out.content, opts.anchorText);
if (blockPlainText(out.content[i]).includes(opts.anchorText)) { if (i !== -1) {
out.content.splice(i + offset, 0, fresh); out.content.splice(i + offset, 0, fresh);
return { doc: out, inserted: true }; return { doc: out, inserted: true };
}
} }
} }
@@ -617,7 +674,8 @@ function locateTable(
if (!isObject(rootClone)) return null; if (!isObject(rootClone)) return null;
// "#<n>": index into the top-level content array; must be a table. // "#<n>": index into the top-level content array; must be a table.
const indexMatch = typeof tableRef === "string" ? tableRef.match(/^#(\d+)$/) : null; const indexMatch =
typeof tableRef === "string" ? tableRef.match(/^#(\d+)$/) : null;
if (indexMatch) { if (indexMatch) {
const index = Number(indexMatch[1]); const index = Number(indexMatch[1]);
const block = Array.isArray(rootClone.content) const block = Array.isArray(rootClone.content)
@@ -717,7 +775,7 @@ export function readTable(
: undefined; : undefined;
const id = const id =
isObject(firstPara) && isObject(firstPara.attrs) isObject(firstPara) && isObject(firstPara.attrs)
? firstPara.attrs.id ?? null ? (firstPara.attrs.id ?? null)
: null; : null;
rowIds.push(id); rowIds.push(id);
} }
@@ -751,14 +809,17 @@ export function insertTableRow(
if (!Array.isArray(table.content)) table.content = []; if (!Array.isArray(table.content)) table.content = [];
const rows = table.content.length; const rows = table.content.length;
const headerRow = table.content[0]; const headerRow = table.content[0];
const headerCells = Array.isArray(headerRow?.content) ? headerRow.content : []; const headerCells = Array.isArray(headerRow?.content)
? headerRow.content
: [];
// Column count is the WIDEST existing row, so the guard below stays // Column count is the WIDEST existing row, so the guard below stays
// meaningful for ragged tables and the new row matches the table's width. // meaningful for ragged tables and the new row matches the table's width.
// Fall back to the supplied cell count only when the table has no rows. // Fall back to the supplied cell count only when the table has no rows.
let colCount = 0; let colCount = 0;
for (const r of table.content) { for (const r of table.content) {
if (isObject(r) && Array.isArray(r.content)) colCount = Math.max(colCount, r.content.length); if (isObject(r) && Array.isArray(r.content))
colCount = Math.max(colCount, r.content.length);
} }
if (colCount === 0) colCount = Array.isArray(cells) ? cells.length : 0; if (colCount === 0) colCount = Array.isArray(cells) ? cells.length : 0;
@@ -771,7 +832,10 @@ export function insertTableRow(
// Resolve the landing index up front so the cell-type decision and the splice // Resolve the landing index up front so the cell-type decision and the splice
// below agree: a valid integer in [0, rows] splices there, else we append. // below agree: a valid integer in [0, rows] splices there, else we append.
const landingIndex = const landingIndex =
typeof index === "number" && Number.isInteger(index) && index >= 0 && index <= rows typeof index === "number" &&
Number.isInteger(index) &&
index >= 0 &&
index <= rows
? index ? index
: rows; : rows;
@@ -790,7 +854,8 @@ export function insertTableRow(
// A row landing at index 0 becomes the new header row, so inherit the // A row landing at index 0 becomes the new header row, so inherit the
// current header cell's type per column (Docmost uses "tableHeader" there); // current header cell's type per column (Docmost uses "tableHeader" there);
// every other position is a plain data cell. // every other position is a plain data cell.
const cellType = landingIndex === 0 ? headerCells[i]?.type ?? "tableCell" : "tableCell"; const cellType =
landingIndex === 0 ? (headerCells[i]?.type ?? "tableCell") : "tableCell";
newCells.push({ newCells.push({
type: cellType, type: cellType,
attrs, attrs,
@@ -862,9 +927,10 @@ export function updateTableCell(
const rowNodes = Array.isArray(table.content) ? table.content : []; const rowNodes = Array.isArray(table.content) ? table.content : [];
const rows = rowNodes.length; const rows = rowNodes.length;
const rowNode = rowNodes[row]; const rowNode = rowNodes[row];
const cols = isObject(rowNode) && Array.isArray(rowNode.content) const cols =
? rowNode.content.length isObject(rowNode) && Array.isArray(rowNode.content)
: 0; ? rowNode.content.length
: 0;
if ( if (
!Number.isInteger(row) || !Number.isInteger(row) ||
@@ -2,6 +2,11 @@
// instead of an object. Normalize: parse a string to an object (throwing on // instead of an object. Normalize: parse a string to an object (throwing on
// invalid JSON), pass an object through unchanged. Shared by patch_node / // invalid JSON), pass an object through unchanged. Shared by patch_node /
// insert_node (and the analogous update_page_json content parsing). // insert_node (and the analogous update_page_json content parsing).
//
// This lives in the converter package (#414) so BOTH consumers import the ONE
// copy: `@docmost/mcp` (ESM) and the CommonJS server app. The server cannot
// import `@docmost/mcp` directly (ESM-only, no declaration files), but it does
// import `@docmost/prosemirror-markdown` natively — so this is the shared home.
export function parseNodeArg( export function parseNodeArg(
node: unknown, node: unknown,
errMsg = "node was a string but not valid JSON", errMsg = "node was a string but not valid JSON",
@@ -0,0 +1,99 @@
/**
* Locator normalization: strip inline markdown wrappers and trailing
* decoration from a LOCATOR string so a find/anchor that the model wrote with
* markdown (or a stray emoji) can still match the document's plain text.
*
* This is used ONLY as a fallback for LOCATING (after an exact match fails);
* it is never applied to replacement text or inserted node content, so no
* formatting is ever lost.
*
* Scope note (#414): this package-local copy exists so `node-ops.ts` — which
* lives here now (the single canonical copy) — can resolve its markdown-tolerant
* anchor fallback without a circular dependency back on `@docmost/mcp`. It
* intentionally carries ONLY `stripInlineMarkdown` (the primitive `node-ops`
* needs); the mcp-side `text-normalize.ts` (which additionally serves
* `json-edit.ts` via `stripBalancedWrappers`) is the subject of a separate
* dedup task and is left untouched here.
*/
/** Maximum unwrap passes, so pathological/nested input cannot loop forever. */
const MAX_PASSES = 8;
/**
* Inline emphasis/code/strikethrough wrappers, strong BEFORE emphasis so
* `**x**` collapses to `x` rather than leaving a stray `*x*`. Each pattern is
* non-greedy and capture group 1 is the inner text. Applied repeatedly until
* the string stops changing (nested wrappers like `**_x_**`).
*/
const WRAPPER_PATTERNS: RegExp[] = [
/\*\*([^*]+?)\*\*/g, // **x**
/__([^_]+?)__/g, // __x__
/~~([^~]+?)~~/g, // ~~x~~
/\*([^*]+?)\*/g, // *x*
/_([^_]+?)_/g, // _x_
/``([^`]+?)``/g, // ``x``
/`([^`]+?)`/g, // `x`
];
/** Links/images -> their visible text. `!?` covers both `[t](u)` and `![a](s)`. */
const LINK_IMAGE_RE = /!?\[([^\]]*)\]\([^)]*\)/g;
/**
* Apply the two balanced/link passes: first collapse links/images to their
* visible text, then collapse balanced inline wrappers repeatedly until stable.
* Does NOT trim decoration, does NOT guard against an empty result — it returns
* exactly the transformed string.
*/
function stripWrappersAndLinks(s: string): string {
// 1. Links/images -> their visible text.
let out = s.replace(LINK_IMAGE_RE, "$1");
// 2. Strip balanced wrappers, repeating until the string is stable so nested
// wrappers (`**_x_**`) and adjacent runs both collapse.
for (let pass = 0; pass < MAX_PASSES; pass++) {
const before = out;
for (const re of WRAPPER_PATTERNS) {
out = out.replace(re, "$1");
}
if (out === before) break;
}
return out;
}
/**
* Conservatively strip inline markdown from a locator string.
*
* Deterministic, order-fixed steps:
* 1. Links/images: `[text](url)` -> `text`, `![alt](src)` -> `alt`.
* 2. Balanced inline wrappers (strong before emphasis, code, strikethrough),
* applied repeatedly until stable for nested cases.
* 3. Trim leading/trailing decoration only: whitespace, leftover marker chars
* (`* _ ~ \``) and emoji. Letters/digits and sentence punctuation (`.`/`,`
* etc.) are NEVER trimmed.
*
* If the result is empty (e.g. the input was only markers like `***`), the
* ORIGINAL string is returned so a locator can never normalize down to "" and
* match everything.
*/
export function stripInlineMarkdown(s: string): string {
if (typeof s !== "string" || s.length === 0) return s;
// 1 + 2. Shared link/image and balanced-wrapper passes.
let out = stripWrappersAndLinks(s);
// 3. Trim leading/trailing decoration: whitespace, leftover markdown markers,
// and emoji (Extended_Pictographic plus the VS16 / ZWJ joiners, plus the
// regional-indicator range U+1F1E6–U+1F1FF for flag emoji, which are NOT
// Extended_Pictographic). The `u` flag enables the Unicode property escape.
// Anchored runs only — interior text and sentence punctuation are untouched.
const DECORATION =
"[\\s*_~\\x60\\p{Extended_Pictographic}\\u{1F1E6}-\\u{1F1FF}\\u{FE0F}\\u{200D}]+";
out = out
.replace(new RegExp("^" + DECORATION, "u"), "")
.replace(new RegExp(DECORATION + "$", "u"), "");
// 4. Never normalize a locator down to nothing.
if (out.length === 0) return s;
return out;
}