diff --git a/apps/server/src/collaboration/collaboration.util.ts b/apps/server/src/collaboration/collaboration.util.ts index 50e961ce..6f6d04c4 100644 --- a/apps/server/src/collaboration/collaboration.util.ts +++ b/apps/server/src/collaboration/collaboration.util.ts @@ -141,7 +141,57 @@ export function htmlToJson(html: string) { } } -export function jsonToText(tiptapJson: JSONContent) { +/** + * Deterministic text-serializer overrides for the `format:"text"` page read + * (#502). Non-text nodes render to a STABLE placeholder instead of their + * (structure-dependent) inner text, so a machine diff of two text reads is + * driven only by the page's actual prose — output stability across package + * versions IS the contract (pinned by a snapshot test). Returning a string from + * a `textSerializer` also stops `generateText` descending into the node, so a + * table renders as ONE token rather than its flattened cell text. + * + * Only nodes with no meaningful flat-text form are overridden; every other node + * (paragraph/heading/list/code/blockquote/callout/…) keeps its natural text so + * a config written as markdown reads back byte-identical. + */ +const TEXT_READ_SERIALIZERS: Record string> = + { + // Image atom: no inner text -> a fixed placeholder. + image: () => '[image]', + // Table: `[table RxC]` where R = row count, C = the first row's cell count + // (a table's columns are uniform per the schema). Computed from the PM node, + // so it is independent of cell contents. + table: ({ node }) => { + const rows = node?.childCount ?? 0; + const cols = rows > 0 ? (node.child(0)?.childCount ?? 0) : 0; + return `[table ${rows}x${cols}]`; + }, + }; + +/** + * Serialize a ProseMirror/TipTap document to plain text. + * + * Default (no options): the long-standing search-index behavior — bare + * concatenated node text with `generateText`'s default `\n\n` block separator. + * This feeds the page `textContent` tsvector and MUST NOT change. + * + * `deterministic:true` (#502 `format:"text"` page read): a flat, machine-diffable + * rendering — one line per block (`\n` block separator; `hardBreak` already + * serializes to `\n`), inline marks/anchors/autoformat dropped, and non-text + * nodes replaced by the stable placeholders above (`[image]`, `[table RxC]`). + */ +export function jsonToText( + // `any` (like jsonToHtml/jsonToMarkdown) so a loosely-typed DB `page.content` + // (JsonValue) can be passed straight through, as the controller does. + tiptapJson: any, + options?: { deterministic?: boolean }, +) { + if (options?.deterministic) { + return generateText(tiptapJson, tiptapExtensions, { + blockSeparator: '\n', + textSerializers: TEXT_READ_SERIALIZERS, + }); + } return generateText(tiptapJson, tiptapExtensions); } diff --git a/apps/server/src/collaboration/json-to-text-deterministic.spec.ts b/apps/server/src/collaboration/json-to-text-deterministic.spec.ts new file mode 100644 index 00000000..4a563758 --- /dev/null +++ b/apps/server/src/collaboration/json-to-text-deterministic.spec.ts @@ -0,0 +1,116 @@ +import { jsonToText } from './collaboration.util'; + +// #502 READ contract: `jsonToText(json, { deterministic: true })` — the flat, +// machine-diffable text rendering behind getPage `format:"text"`. Block-per-line +// (`\n` separator), inline marks/anchors dropped, hardBreak -> `\n`, and non-text +// nodes replaced by STABLE placeholders (`[image]`, `[table RxC]`). Output +// stability across package versions IS the contract, so it is pinned by a +// snapshot below. The DEFAULT (no options) path is the search-index serializer +// and MUST be unchanged — asserted separately. + +const doc = (...content: any[]) => ({ type: 'doc', content }); +const para = (...content: any[]) => ({ type: 'paragraph', content }); +const text = (t: string, marks?: any[]) => + marks ? { type: 'text', text: t, marks } : { type: 'text', text: t }; + +describe('jsonToText — default (search index) behavior is unchanged', () => { + it('uses the `\\n\\n` block separator and drops non-text nodes to empty', () => { + const d = doc(para(text('alpha')), para(text('beta'))); + expect(jsonToText(d)).toBe('alpha\n\nbeta'); + }); + + it('an image contributes no text in the default (tsvector) mode', () => { + const d = doc(para(text('cap')), { type: 'image', attrs: { src: 's' } }); + // No `[image]` placeholder leaks into the search index. + expect(jsonToText(d)).not.toContain('[image]'); + }); +}); + +describe('jsonToText — deterministic:true (getPage format:"text")', () => { + it('renders one line per block with `\\n` separators, marks dropped', () => { + const d = doc( + { type: 'heading', attrs: { level: 2 }, content: [text('Title')] }, + para( + text('hello ', [{ type: 'bold' }]), + text('world', [{ type: 'italic' }]), + ), + ); + expect(jsonToText(d, { deterministic: true })).toBe('Title\nhello world'); + }); + + it('a hardBreak renders as a newline', () => { + const d = doc(para(text('a'), { type: 'hardBreak' }, text('b'))); + expect(jsonToText(d, { deterministic: true })).toBe('a\nb'); + }); + + it('an image node -> the stable `[image]` placeholder', () => { + const d = doc(para(text('before')), { type: 'image', attrs: { src: 's' } }); + expect(jsonToText(d, { deterministic: true })).toBe('before\n[image]'); + }); + + it('a table -> `[table RxC]` (rows x columns) and its cell text is NOT flattened in', () => { + const cell = (t: string) => ({ + type: 'tableCell', + content: [para(text(t))], + }); + const row = (...cells: any[]) => ({ type: 'tableRow', content: cells }); + const table = { + type: 'table', + content: [ + row(cell('CELLONE'), cell('CELLTWO'), cell('CELLTHREE')), + row(cell('CELLFOUR'), cell('CELLFIVE'), cell('CELLSIX')), + ], + }; + const d = doc(para(text('grid:')), table); + const out = jsonToText(d, { deterministic: true }); + expect(out).toBe('grid:\n[table 2x3]'); + expect(out).not.toContain('CELL'); // cell text is not flattened into the read + }); + + it('SNAPSHOT: a mixed document renders to a stable, deterministic string', () => { + const cell = (t: string) => ({ + type: 'tableCell', + content: [para(text(t))], + }); + const d = doc( + { type: 'heading', attrs: { level: 1 }, content: [text('Config')] }, + para(text('key = ', [{ type: 'bold' }]), text('value')), + { + type: 'bulletList', + content: [ + { type: 'listItem', content: [para(text('one'))] }, + { type: 'listItem', content: [para(text('two'))] }, + ], + }, + { type: 'image', attrs: { src: 's' } }, + { + type: 'table', + content: [{ type: 'tableRow', content: [cell('x'), cell('y')] }], + }, + ); + expect(jsonToText(d, { deterministic: true })).toMatchInlineSnapshot(` + "Config + key = value + + + one + + two + [image] + [table 1x2]" + `); + }); + + it('SCENARIO: a config written as a code block reads back byte-identical (diff empty)', () => { + const config = + 'export TICKET_LIFETIME=$2592000\nservers:\n - www.internal.host'; + const d = doc({ + type: 'codeBlock', + attrs: { language: 'yaml' }, + content: [text(config)], + }); + // The code block is one block; its text (dollars, bare domain, newlines) is + // preserved verbatim, so a read-as-text of a stored config diffs empty. + expect(jsonToText(d, { deterministic: true })).toBe(config); + }); +}); diff --git a/apps/server/src/core/page/dto/create-page.dto.ts b/apps/server/src/core/page/dto/create-page.dto.ts index 81b15903..7f0757b2 100644 --- a/apps/server/src/core/page/dto/create-page.dto.ts +++ b/apps/server/src/core/page/dto/create-page.dto.ts @@ -10,6 +10,12 @@ import { Transform } from 'class-transformer'; export type ContentFormat = 'json' | 'markdown' | 'html'; +// READ-only rendering formats for `getPage` (#502). A superset of the writable +// `ContentFormat` with `text` — a flat, deterministic, machine-diffable text +// rendering — added. Kept SEPARATE from `ContentFormat` so the write path +// (createPage/updatePage `parseProsemirrorContent`) can never be handed `text`. +export type PageReadFormat = ContentFormat | 'text'; + export class CreatePageDto { @IsOptional() @IsString() diff --git a/apps/server/src/core/page/dto/page.dto.ts b/apps/server/src/core/page/dto/page.dto.ts index c3b64b23..861587da 100644 --- a/apps/server/src/core/page/dto/page.dto.ts +++ b/apps/server/src/core/page/dto/page.dto.ts @@ -8,7 +8,7 @@ import { } from 'class-validator'; import { Transform } from 'class-transformer'; -import { ContentFormat } from './create-page.dto'; +import { PageReadFormat } from './create-page.dto'; import { IsPageIdOrSlugId } from './page-identity.validator'; export class PageIdDto { @@ -43,8 +43,8 @@ export class PageInfoDto extends PageIdDto { @IsOptional() @Transform(({ value }) => value?.toLowerCase()) - @IsIn(['json', 'markdown', 'html']) - format?: ContentFormat; + @IsIn(['json', 'markdown', 'html', 'text']) + format?: PageReadFormat; } export class DeletePageDto extends PageIdDto { diff --git a/apps/server/src/core/page/page.controller.getpage-format.spec.ts b/apps/server/src/core/page/page.controller.getpage-format.spec.ts new file mode 100644 index 00000000..32e8cd25 --- /dev/null +++ b/apps/server/src/core/page/page.controller.getpage-format.spec.ts @@ -0,0 +1,76 @@ +import { NotFoundException } from '@nestjs/common'; +import { PageController } from './page.controller'; +import { jsonToText, jsonToMarkdown } from '../../collaboration/collaboration.util'; + +// #502 READ: getPage `format:"text"` routes through the deterministic jsonToText +// path (placeholders for non-text nodes, block-per-line), while `format:"json"` +// (or none) returns the raw content and `format:"markdown"` still converts. This +// pins the CONTROLLER wiring with lightweight mocks (no DB needed — the jsonToText +// output contract itself is pinned in json-to-text-deterministic.spec.ts). + +const CONTENT = { + type: 'doc', + content: [ + { type: 'heading', attrs: { level: 1 }, content: [{ type: 'text', text: 'Config' }] }, + { + type: 'paragraph', + content: [ + { type: 'text', text: 'key=', marks: [{ type: 'bold' }] }, + { type: 'text', text: 'value' }, + ], + }, + { type: 'image', attrs: { src: 's' } }, + ], +}; + +function makeController(page: any): PageController { + const pageRepo = { findById: jest.fn().mockResolvedValue(page) } as any; + const pageAccessService = { + validateCanViewWithPermissions: jest + .fn() + .mockResolvedValue({ canEdit: true, hasRestriction: false }), + } as any; + // Only pageRepo + pageAccessService are exercised by getPage; the rest are + // never touched on this path, so undefined placeholders are fine. + return new PageController( + undefined as any, // pageService + pageRepo, + undefined as any, // pageHistoryService + undefined as any, // spaceAbility + pageAccessService, + undefined as any, // backlinkService + undefined as any, // labelService + undefined as any, // auditService + ); +} + +const user = { id: 'u1' } as any; + +describe('PageController.getPage — format:"text" (#502)', () => { + it('returns deterministic flat text with placeholders for non-text nodes', async () => { + const controller = makeController({ id: 'p1', content: CONTENT }); + const res: any = await controller.getPage({ pageId: 'p1', format: 'text' } as any, user); + expect(res.content).toBe(jsonToText(CONTENT, { deterministic: true })); + expect(res.content).toBe('Config\nkey=value\n[image]'); + expect(res.permissions).toEqual({ canEdit: true, hasRestriction: false }); + }); + + it('markdown format still converts to markdown (unchanged)', async () => { + const controller = makeController({ id: 'p1', content: CONTENT }); + const res: any = await controller.getPage({ pageId: 'p1', format: 'markdown' } as any, user); + expect(res.content).toBe(jsonToMarkdown(CONTENT)); + }); + + it('json / no format returns the raw ProseMirror content object', async () => { + const controller = makeController({ id: 'p1', content: CONTENT }); + const res: any = await controller.getPage({ pageId: 'p1' } as any, user); + expect(res.content).toEqual(CONTENT); // untouched object, not a string + }); + + it('missing page -> NotFoundException', async () => { + const controller = makeController(null); + await expect( + controller.getPage({ pageId: 'nope', format: 'text' } as any, user), + ).rejects.toBeInstanceOf(NotFoundException); + }); +}); diff --git a/apps/server/src/core/page/page.controller.ts b/apps/server/src/core/page/page.controller.ts index 6355b1d3..481177ce 100644 --- a/apps/server/src/core/page/page.controller.ts +++ b/apps/server/src/core/page/page.controller.ts @@ -49,6 +49,7 @@ import { AddLabelsDto, RemoveLabelDto } from '../label/dto/label.dto'; import { jsonToHtml, jsonToMarkdown, + jsonToText, } from '../../collaboration/collaboration.util'; import { AuditEvent, AuditResource } from '../../common/events/audit-events'; import { @@ -93,10 +94,16 @@ export class PageController { const permissions = { canEdit, hasRestriction }; if (dto.format && dto.format !== 'json' && page.content) { - const contentOutput = - dto.format === 'markdown' - ? jsonToMarkdown(page.content) - : jsonToHtml(page.content); + let contentOutput: string; + if (dto.format === 'markdown') { + contentOutput = jsonToMarkdown(page.content); + } else if (dto.format === 'text') { + // #502: flat, deterministic, machine-diffable text (block-per-line, + // inline marks/anchors dropped, non-text nodes -> stable placeholders). + contentOutput = jsonToText(page.content, { deterministic: true }); + } else { + contentOutput = jsonToHtml(page.content); + } return { ...page, content: contentOutput, diff --git a/apps/server/src/integrations/import/import.controller.ts b/apps/server/src/integrations/import/import.controller.ts index c47e87ea..d3121d2b 100644 --- a/apps/server/src/integrations/import/import.controller.ts +++ b/apps/server/src/integrations/import/import.controller.ts @@ -85,6 +85,14 @@ export class ImportController { throw new BadRequestException('spaceId is required'); } + // #502: optional multipart field. Only the MCP agent `createPage` path sends + // `disableMarkdownExtensions=true` (its body is agent-authored plain prose / + // config, so a `$…$` span must stay literal and a bare `www.host` must not + // autolink). A HUMAN file upload omits the field, so it stays false and math + // + autolink remain ON for human imports. Settable ONLY via this API param. + const disableMarkdownExtensions = + file.fields?.disableMarkdownExtensions?.value === 'true'; + const ability = await this.spaceAbility.createForUser(user, spaceId); if (ability.cannot(SpaceCaslAction.Edit, SpaceCaslSubject.Page)) { throw new ForbiddenException(); @@ -95,6 +103,7 @@ export class ImportController { user.id, spaceId, workspace.id, + disableMarkdownExtensions, ); const ext = path.extname(file.filename).toLowerCase(); diff --git a/apps/server/src/integrations/import/services/import.service.processMarkdown.spec.ts b/apps/server/src/integrations/import/services/import.service.processMarkdown.spec.ts new file mode 100644 index 00000000..9a0bd646 --- /dev/null +++ b/apps/server/src/integrations/import/services/import.service.processMarkdown.spec.ts @@ -0,0 +1,67 @@ +// Importing ImportService transitively loads import-formatter.ts, which imports +// the ESM-only @sindresorhus/slugify (not in jest's transform allowlist). It is +// irrelevant to this path, so mock it to keep the module graph loadable (mirrors +// the sibling import.service specs). +jest.mock('@sindresorhus/slugify', () => ({ + __esModule: true, + default: (input: string) => String(input), +})); + +import { ImportService } from './import.service'; + +// #502 BLOCKER 1: the server markdown import path (`/pages/import`) now accepts an +// optional `disableMarkdownExtensions` flag threaded into `processMarkdown`. +// - MCP agent `createPage` sends it TRUE -> extensions OFF (a `$…$` config span +// stays literal, a schemeless `www.host` is not autolinked). +// - a HUMAN file upload omits it (default FALSE) -> extensions ON, so a real +// `$x^2$` still becomes a formula (human imports unaffected). +// `processMarkdown` only uses the imported converter (no injected deps on this +// path), so the service is constructed with null deps for this focused unit test. + +function makeService(): ImportService { + return new ImportService(null as any, null as any, null as any, null as any); +} + +function findAll(node: any, type: string, acc: any[] = []): any[] { + if (!node || typeof node !== 'object') return acc; + if (node.type === type) acc.push(node); + if (Array.isArray(node.content)) for (const c of node.content) findAll(c, type, acc); + return acc; +} +function hasLink(node: any): boolean { + return findAll(node, 'text').some((t: any) => + t.marks?.some((m: any) => m.type === 'link'), + ); +} + +describe('ImportService.processMarkdown — #502 disableMarkdownExtensions', () => { + it('disableMarkdownExtensions=true (MCP createPage): `$…$` stays literal, no math', async () => { + const svc = makeService(); + const doc = await svc.processMarkdown('export A=$FOO and B=$BAR done', true); + expect(findAll(doc, 'mathInline')).toHaveLength(0); + }); + + it('disableMarkdownExtensions=true: a schemeless www is NOT autolinked', async () => { + const svc = makeService(); + const doc = await svc.processMarkdown('see www.example.com here', true); + expect(hasLink(doc)).toBe(false); + }); + + it('disableMarkdownExtensions=true: an explicit https:// STILL links', async () => { + const svc = makeService(); + const doc = await svc.processMarkdown('see https://example.com here', true); + expect(hasLink(doc)).toBe(true); + }); + + it('DEFAULT (human upload): a real `$x^2$` DOES become a math node', async () => { + const svc = makeService(); + const doc = await svc.processMarkdown('$x^2$'); + expect(findAll(doc, 'mathInline')).toHaveLength(1); + }); + + it('DEFAULT (human upload): a schemeless www IS autolinked', async () => { + const svc = makeService(); + const doc = await svc.processMarkdown('see www.example.com here'); + expect(hasLink(doc)).toBe(true); + }); +}); diff --git a/apps/server/src/integrations/import/services/import.service.ts b/apps/server/src/integrations/import/services/import.service.ts index 7c480ef2..cd4fc1c3 100644 --- a/apps/server/src/integrations/import/services/import.service.ts +++ b/apps/server/src/integrations/import/services/import.service.ts @@ -52,6 +52,12 @@ export class ImportService { userId: string, spaceId: string, workspaceId: string, + // #502: when true, the markdown importer runs with the two layered + // extensions OFF (a `$…$` span stays literal text; a schemeless `www.host` is + // NOT autolinked). ONLY the MCP agent `createPage` path sets this; a HUMAN + // file upload never passes it, so it defaults false and math/autolink stay ON + // for human imports (their `$x^2$` still becomes a formula). + disableMarkdownExtensions = false, ) { const file = await filePromise; const fileBuffer = await file.toBuffer(); @@ -66,7 +72,10 @@ export class ImportService { try { if (fileExtension.endsWith('.md')) { - prosemirrorState = await this.processMarkdown(fileContent); + prosemirrorState = await this.processMarkdown( + fileContent, + disableMarkdownExtensions, + ); } else if (fileExtension.endsWith('.html')) { prosemirrorState = await this.processHTML(fileContent); } @@ -138,7 +147,12 @@ export class ImportService { return createdPage; } - async processMarkdown(markdownInput: string): Promise { + async processMarkdown( + markdownInput: string, + // #502: forwarded to the importer. DEFAULT false keeps math + fuzzy autolink + // ON (human uploads unaffected); the MCP agent `createPage` path passes true. + disableMarkdownExtensions = false, + ): Promise { // Canonical markdown -> ProseMirror JSON directly via // `@docmost/prosemirror-markdown` (issue #345) — no HTML intermediate and no // second editor-ext markdown layer. Foreign markdown surfaces the strict @@ -147,7 +161,12 @@ export class ImportService { // The HTML-cleanup pass (`normalizeImportHtml`) is intentionally skipped here: // it targets foreign *HTML* (Notion/XWiki), which only ever arrives on the // `.html` path (`processHTML`), never as canonical markdown. - return markdownToProseMirror(normalizeForeignMarkdown(markdownInput)); + return markdownToProseMirror( + normalizeForeignMarkdown(markdownInput), + disableMarkdownExtensions + ? { parseMath: false, fuzzyLinkify: false } + : undefined, + ); } async processHTML(htmlInput: string): Promise { diff --git a/packages/mcp/README.md b/packages/mcp/README.md index e6c93341..d60409a4 100644 --- a/packages/mcp/README.md +++ b/packages/mcp/README.md @@ -118,7 +118,10 @@ All 41 tools, grouped by what you'd reach for them. - **`getPage`** — A page's content as clean **Markdown** (canonical for text; drops only block ids, resolved-comment anchors, and a fixed no-Markdown-representation attr set — table spans/colwidth/background, indent, `callout.icon`, `orderedList.type`, and link - `internal`/`target`/`rel`/`class`; use `getPageJson` when you need those). + `internal`/`target`/`rel`/`class`; use `getPageJson` when you need those). Pass + `format:"text"` for a flat, deterministic plain-text rendering (one line per block, + marks dropped, stable `[image]`/`[table RxC]` placeholders) to machine-diff what you + wrote against what was stored. - **`getPageJson`** — A page's **lossless ProseMirror/TipTap JSON**, including every block's `attrs.id` and the `slugId` used in URLs. This is what the per-block editing tools consume. diff --git a/packages/mcp/README.ru.md b/packages/mcp/README.ru.md index 3b3de41d..574839d6 100644 --- a/packages/mcp/README.ru.md +++ b/packages/mcp/README.ru.md @@ -123,7 +123,10 @@ Docmost-MCP не сочетают: лишь id блоков, якоря разрешённых комментариев и фиксированный набор атрибутов без markdown-представления — спаны/colwidth/фон ячеек таблиц, отступы (indent), `callout.icon`, `orderedList.type` и `internal`/`target`/`rel`/`class` у ссылок; - используйте `getPageJson`, когда они нужны). + используйте `getPageJson`, когда они нужны). Передайте `format:"text"` для плоского + детерминированного plain-text рендера (одна строка на блок, маркировка убрана, + стабильные плейсхолдеры `[image]`/`[table RxC]`) — чтобы machine-diff'ом сверить то, + что вы записали, с тем, что сохранилось. - **`getPageJson`** — **Lossless ProseMirror/TipTap JSON** страницы, включая `attrs.id` каждого блока и `slugId`, используемый в URL. Именно его потребляют инструменты поблочного редактирования. diff --git a/packages/mcp/src/client/context.ts b/packages/mcp/src/client/context.ts index 8b15a1ac..95c6de85 100644 --- a/packages/mcp/src/client/context.ts +++ b/packages/mcp/src/client/context.ts @@ -911,10 +911,20 @@ export abstract class DocmostClientContext { } - /** Raw page info including the ProseMirror JSON content and slugId. */ - async getPageRaw(pageId: string) { + /** + * Raw page info including the ProseMirror JSON content and slugId. + * + * With `format:"text"` (#502) the server instead renders `content` as a flat, + * deterministic text string (its `jsonToText` path — the SAME serializer that + * feeds search), so the MCP text read reuses the server's ONE serializer + * rather than shipping a second one. Every other caller omits `format` and + * gets the JSON content unchanged. + */ + async getPageRaw(pageId: string, format?: "text") { await this.ensureAuthenticated(); - const response = await this.client.post("/pages/info", { pageId }); + const body: Record = { pageId }; + if (format) body.format = format; + const response = await this.client.post("/pages/info", body); return response.data?.data ?? response.data; } diff --git a/packages/mcp/src/client/pages.ts b/packages/mcp/src/client/pages.ts index 8d69603b..d3520040 100644 --- a/packages/mcp/src/client/pages.ts +++ b/packages/mcp/src/client/pages.ts @@ -93,6 +93,13 @@ export function PagesMixin>(Bas const buildForm = () => { const form = new FormData(); form.append("spaceId", spaceId); + // #502: this is an AGENT-authored body (plain prose / config), so tell the + // server import path to run the markdown importer with the two layered + // extensions OFF — a `$…$` span stays literal text (real math via + // `update_page_json`) and a schemeless `www.host`/email is not autolinked + // (an explicit `https://…` still links). A human file upload never sends + // this field, so human imports keep math + autolink ON. + form.append("disableMarkdownExtensions", "true"); form.append("file", fileContent, { filename: `${title || "import"}.md`, contentType: "text/markdown", diff --git a/packages/mcp/src/client/read.ts b/packages/mcp/src/client/read.ts index 85fb9fa6..9d3cfb20 100644 --- a/packages/mcp/src/client/read.ts +++ b/packages/mcp/src/client/read.ts @@ -72,7 +72,7 @@ export interface IReadMixin { getTree(spaceId: string, rootPageId?: string, maxDepth?: number): any; getPageContext(pageId: string): any; listSidebarPages(spaceId: string, pageId?: string): any; - getPage(pageId: string): any; + getPage(pageId: string, format?: "markdown" | "text"): any; getPageJson(pageId: string): any; getOutline(pageId: string): any; getNode(pageId: string, nodeId: string, format?: "markdown" | "json"): any; @@ -440,8 +440,34 @@ export function ReadMixin>(Base return convertProseMirrorToMarkdown(content, options); } - async getPage(pageId: string) { + async getPage(pageId: string, format: "markdown" | "text" = "markdown") { await this.ensureAuthenticated(); + + // #502 `format:"text"`: a flat, deterministic, machine-diffable rendering + // (block-per-line, inline marks/comment anchors dropped, non-text nodes -> + // stable placeholders like `[image]` / `[table RxC]`). The server produces + // it via its `jsonToText` path (the SAME serializer that feeds search), so + // there is no second serializer here — we just request it and pass the + // string through. Distinct from the markdown default: no PM->markdown walk, + // no markdown conversion cache, and no `{{SUBPAGES}}` substitution (the text + // renderer emits no such placeholder). Use it to diff a page you wrote as a + // config/prose against what was stored. + if (format === "text") { + const textData = await this.getPageRaw(pageId, "text"); + let subpages: any[] = []; + try { + subpages = await this.listSidebarPages(textData.spaceId, textData.id); + } catch (e: any) { + console.warn("Failed to fetch subpages:", e); + } + const textContent = + typeof textData.content === "string" ? textData.content : ""; + return { + data: filterPage(textData, textContent, subpages), + success: true, + }; + } + const resultData = await this.getPageRaw(pageId); // Agent read: hide resolved-comment anchors so the agent sees only active diff --git a/packages/mcp/src/lib/collaboration.ts b/packages/mcp/src/lib/collaboration.ts index 0de4e497..379c99dc 100644 --- a/packages/mcp/src/lib/collaboration.ts +++ b/packages/mcp/src/lib/collaboration.ts @@ -14,6 +14,7 @@ import { markdownToProseMirror, normalizeAgentMarkdown, } from "@docmost/prosemirror-markdown"; +import type { MarkdownImportOptions } from "@docmost/prosemirror-markdown"; import { docmostExtensions, docmostSchema } from "./docmost-schema.js"; import { withPageLock } from "./page-lock.js"; import type { PageId } from "./page-id.js"; @@ -111,16 +112,31 @@ global.WebSocket = WebSocket; * horizontalRule the serializer emitted, and stripping it would silently drop the * page's leading content (#493 review). The front-matter strip stays on the * server FILE-import boundary only (`normalizeForeignMarkdown`). + * + * #502 IMPORTANT — the two layered markdown extensions (`$…$` math, schemeless + * fuzzy autolink) are NOT decided here; they are the CALLER's choice via + * `options`, because the two callers of this wrapper need OPPOSITE behavior: + * - AGENT-authored plain markdown (`updatePageMarkdown`) → both OFF, so a + * `$…$` config span stays literal and a bare `www.host` is not autolinked + * (real math is authored via `update_page_json`). + * - FULL-FILE round-trip import (`import_page_markdown`, #328 lossless) → + * DEFAULTS (both ON), because the exporter serializes a math node as readable + * `$x^2$`; re-importing with math OFF would degrade it to literal text and + * BREAK the lossless export→import pair. + * So `options` DEFAULTS to `undefined` → the package importer's defaults (ON), + * which is the safe round-trip behavior; the agent-write caller opts OUT + * explicitly. (The fragment path `importMarkdownFragment` opts out on its own.) */ export async function markdownToProseMirrorCanonical( markdownContent: string, + options?: MarkdownImportOptions, ): Promise { // #419: normalize + merge glyph-forked footnote definitions BEFORE // canonicalizing, so the canonicalizer re-hangs references and drops the // now-orphaned duplicate definitions. return canonicalizeFootnotes( normalizeAndMergeFootnotes( - await markdownToProseMirror(normalizeAgentMarkdown(markdownContent)), + await markdownToProseMirror(normalizeAgentMarkdown(markdownContent), options), ), ); } @@ -358,7 +374,17 @@ export async function updatePageContentRealtime( ): Promise { // PAGE write: canonicalize footnotes (markdown import builds the bottom list in // definition order; numbering is reference-ordered). - const tiptapJson = await markdownToProseMirrorCanonical(markdownContent); + // + // #502: this is the AGENT-authored `updatePageMarkdown` body — plain prose / + // config — so the two layered markdown extensions are turned OFF: a `$…$` span + // stays literal text (real math via `update_page_json`) and a SCHEMELESS + // `www.host`/email is not autolinked (an explicit `https://…` still links). + // Contrast `import_page_markdown`, which keeps DEFAULTS for the #328 lossless + // round-trip. + const tiptapJson = await markdownToProseMirrorCanonical(markdownContent, { + parseMath: false, + fuzzyLinkify: false, + }); return await mutatePageContent( pageId, collabToken, diff --git a/packages/mcp/src/lib/markdown-fragment.ts b/packages/mcp/src/lib/markdown-fragment.ts index 79796212..c24862e8 100644 --- a/packages/mcp/src/lib/markdown-fragment.ts +++ b/packages/mcp/src/lib/markdown-fragment.ts @@ -135,7 +135,17 @@ export interface MarkdownFragment { export async function importMarkdownFragment( markdown: string, ): Promise { - const doc = await markdownToProseMirror(markdown); + // #502: the fragment path is an MCP agent WRITE (patch_node/insert_node + // markdown), so it uses the SAME extensions-OFF importer options as the + // full-page write (markdownToProseMirrorCanonical): a `$…$` span stays literal + // and a schemeless domain/email is not autolinked. This keeps a block written + // via markdown canonically identical to the same content in a full-page write + // (no "second canon"). Explicit `https://…` links and block structure are + // unaffected. + const doc = await markdownToProseMirror(markdown, { + parseMath: false, + fuzzyLinkify: false, + }); const content: any[] = Array.isArray(doc?.content) ? doc.content : []; const blocks: any[] = []; diff --git a/packages/mcp/src/tool-specs.ts b/packages/mcp/src/tool-specs.ts index f3f0a2fe..c94eba12 100644 --- a/packages/mcp/src/tool-specs.ts +++ b/packages/mcp/src/tool-specs.ts @@ -469,7 +469,11 @@ export const SHARED_TOOL_SPECS = { '(markdown). The fragment may be SEVERAL blocks (a "1 → N" splice: rewrite a ' + 'whole section in one call) — the first block inherits this block id, the ' + 'rest get fresh ids. `^[...]` footnotes are supported (their definitions ' + - "merge into the page's footnote list). REJECTED when the target is a table " + + "merge into the page's footnote list). Markdown is taken LITERALLY — " + + '`$...$`/`$$...$$` are NOT parsed as math and schemeless `www.host` / bare ' + + 'emails are NOT auto-linked (an explicit `https://` URL still links); for a ' + + 'real formula pass a `mathInline`/`mathBlock` ProseMirror node via `node` ' + + '(or updatePageJson). REJECTED when the target is a table ' + 'cell with attributes markdown cannot represent (merged/colored/fixed-width) ' + '— use the table tools or `node`. ' + '`node` (for precise attr/mark work): a raw ProseMirror node, e.g. a ' + @@ -535,6 +539,10 @@ export const SHARED_TOOL_SPECS = { 'Provide EXACTLY ONE of `markdown` or `node`. ' + '`markdown` (RECOMMENDED): a canonical markdown fragment — may be SEVERAL ' + 'blocks, inserted in order at the anchor; `^[...]` footnotes supported. ' + + 'Markdown is taken LITERALLY — `$...$`/`$$...$$` are NOT parsed as math and ' + + 'schemeless `www.host` / bare emails are NOT auto-linked (an explicit ' + + '`https://` URL still links); for a real formula pass a ' + + '`mathInline`/`mathBlock` ProseMirror node via `node` (or updatePageJson). ' + '`node` (for precise attr/mark work OR table structure): a raw ProseMirror ' + 'node. Table structure is JSON-only (not expressible in markdown): to add a ' + 'tableRow, pass a tableRow node with position before/after and anchor INSIDE ' + @@ -913,28 +921,46 @@ export const SHARED_TOOL_SPECS = { inAppKey: 'getPage', writeClass: 'readOnly', description: - 'Fetch a single page as Markdown by its id. Returns the page title and ' + - 'its Markdown content. The converter is canonical (round-trips text and ' + - 'block structure), so this is sufficient for text edits; use the ' + - 'page-JSON read tool only when you need what Markdown cannot carry. The ' + - 'Markdown drops exactly: (1) block ids (not visible in Markdown); ' + - '(2) resolved-comment anchors (hidden here; only active anchors remain); (3) a fixed set of attributes with no ' + - 'Markdown representation — table-cell colspan/rowspan/colwidth/' + - 'backgroundColor/backgroundColorName, heading/paragraph indent, ' + - 'callout.icon, orderedList.type, and link internal/target/rel/class. ' + - 'Inline tags in the markdown are comment highlight ' + - 'anchors — treat them as markup, not page text.', + 'Fetch a single page by its id. Returns the page title and its content. ' + + 'format:"markdown" (DEFAULT) returns canonical Markdown (round-trips text ' + + 'and block structure), sufficient for text edits; use the page-JSON read ' + + 'tool only when you need what Markdown cannot carry. The Markdown drops ' + + 'exactly: (1) block ids (not visible in Markdown); (2) resolved-comment ' + + 'anchors (hidden here; only active anchors remain); ' + + '(3) a fixed set of attributes with no Markdown representation — table-cell ' + + 'colspan/rowspan/colwidth/backgroundColor/backgroundColorName, ' + + 'heading/paragraph indent, callout.icon, orderedList.type, and link ' + + 'internal/target/rel/class. Inline tags in the ' + + 'markdown are comment highlight anchors — treat them as markup, not page ' + + 'text. format:"text" returns a FLAT, DETERMINISTIC plain-text rendering ' + + 'for machine diffing: one line per block, ALL inline marks/formatting and ' + + 'comment anchors dropped, a hardBreak is a newline, and non-text nodes ' + + 'become STABLE placeholders — an image is "[image]" and a table is ' + + '"[table RxC]" (R rows x C columns). This output is stable across versions ' + + '(pinned by a snapshot test); use it to diff a config/prose you wrote ' + + 'against what was stored (an empty diff means it was stored verbatim).', tier: 'core', - catalogLine: 'getPage — fetch a page as Markdown by its id.', + catalogLine: + 'getPage — fetch a page by its id (format:"markdown" default, or "text" for a flat machine-diffable read).', // Reconciled: MCP's stricter .min(1) kept; in-app's more-informative // "(or slugId)" describe kept. buildShape: (z) => ({ pageId: z.string().min(1).describe('The id (or slugId) of the page.'), + format: z + .enum(['markdown', 'text']) + .optional() + .describe( + 'Output format: "markdown" (default, canonical round-trippable) or ' + + '"text" (flat deterministic plain text for machine diffing).', + ), }), // MCP wraps the raw `{ data, success }` as JSON. The in-app host instead // projects a token-efficient `{ title, markdown }` (its long-standing shape). - execute: (client, { pageId }) => client.getPage(pageId as string), + execute: (client, { pageId, format }) => + client.getPage( + pageId as string, + (format as 'markdown' | 'text' | undefined) ?? 'markdown', + ), inAppExecute: async (client, { pageId }) => { // getPage(pageId) -> { data: filterPage(page, markdown), success }. const result = (await client.getPage(pageId as string)) as { @@ -1077,8 +1103,12 @@ export const SHARED_TOOL_SPECS = { description: 'Create a new page with a Markdown body in a space, optionally under a ' + 'parent page (omit parentPageId to create at the space root). Returns ' + - 'the new page id and title. Reversible: a page can be moved to trash ' + - 'later.', + 'the new page id and title. Body text is taken LITERALLY — ' + + '`$...$`/`$$...$$` are NOT parsed into a math formula and schemeless ' + + '`www.host` / bare emails are NOT auto-linked (an explicit `https://` URL ' + + 'still links); for a real formula use updatePageJson with ' + + '`mathInline`/`mathBlock` nodes instead. Reversible: a page can be moved ' + + 'to trash later.', tier: 'deferred', catalogLine: 'createPage — create a new page with a Markdown body in a space.', // Reconciled schema DRIFT: the MCP copy pinned `content` to .min(1) while @@ -1337,7 +1367,11 @@ export const SHARED_TOOL_SPECS = { 'title). The whole body is re-imported from the markdown (block ids ' + 'regenerate — for surgical or id-preserving edits use the find/replace, ' + 'node-patch or page-JSON tools instead). Docmost-flavoured markdown is ' + - 'parsed, including `^[...]` inline footnotes. Reversible: the previous ' + + 'parsed, including `^[...]` inline footnotes. Text is taken LITERALLY — ' + + '`$...$`/`$$...$$` are NOT parsed into a math formula and schemeless ' + + '`www.host` / bare emails are NOT auto-linked (an explicit `https://` URL ' + + 'still links); for a real formula use updatePageJson with ' + + '`mathInline`/`mathBlock` nodes instead. Reversible: the previous ' + 'version is kept in page history.', tier: 'deferred', catalogLine: diff --git a/packages/mcp/test/mock/createpage-import-flag.test.mjs b/packages/mcp/test/mock/createpage-import-flag.test.mjs new file mode 100644 index 00000000..c1067849 --- /dev/null +++ b/packages/mcp/test/mock/createpage-import-flag.test.mjs @@ -0,0 +1,83 @@ +// #502 BLOCKER 1 (client half): the MCP `createPage` tool builds its body as an +// AGENT-authored markdown file and POSTs it to the server `/pages/import` +// endpoint. It must send the `disableMarkdownExtensions=true` multipart field so +// the SERVER importer runs with math + fuzzy-autolink OFF (a human file upload +// omits the field and keeps them ON). This mock http server captures the raw +// multipart body and asserts the field is present. +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { DocmostClient } from "../../build/client.js"; + +function sendJson(res, status, obj, extra = {}) { + res.writeHead(status, { "Content-Type": "application/json", ...extra }); + res.end(JSON.stringify(obj)); +} +function readRaw(req) { + return new Promise((resolve) => { + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8"))); + }); +} + +const openServers = []; +after(async () => { + await Promise.all(openServers.map((s) => new Promise((r) => s.close(r)))); +}); + +const NEW_ID = "00000000-0000-4000-8000-000000000042"; +const SPACE = "00000000-0000-4000-8000-0000000000aa"; + +function spawn(state) { + return new Promise((resolve) => { + const server = http.createServer(async (req, res) => { + const raw = await readRaw(req); + if (req.url === "/api/auth/login") { + return sendJson(res, 200, { success: true }, { "Set-Cookie": "authToken=t; Path=/; HttpOnly" }); + } + if (req.url === "/api/pages/import") { + state.importBody = raw; // the raw multipart payload + return sendJson(res, 200, { data: { id: NEW_ID } }); + } + if (req.url === "/api/pages/update") { + return sendJson(res, 200, { data: { id: NEW_ID } }); + } + if (req.url === "/api/pages/info") { + return sendJson(res, 200, { + data: { + id: NEW_ID, + slugId: "slugnew1234", + title: "T", + spaceId: SPACE, + updatedAt: "2026-01-01T00:00:00Z", + content: { type: "doc", content: [{ type: "paragraph", content: [{ type: "text", text: "x" }] }] }, + }, + }); + } + if (req.url === "/api/pages/sidebar-pages") { + return sendJson(res, 200, { data: { items: [], meta: { hasNextPage: false, nextCursor: null } } }); + } + return sendJson(res, 404, { message: "not found" }); + }); + server.listen(0, "127.0.0.1", () => { + openServers.push(server); + resolve(`http://127.0.0.1:${server.address().port}/api`); + }); + }); +} + +test("createPage sends disableMarkdownExtensions=true in the /pages/import multipart", async () => { + const state = {}; + const baseURL = await spawn(state); + const client = new DocmostClient(baseURL, "e@x.com", "pw"); + + await client.createPage("My Config", "ticket $x=1$ at www.host.com", SPACE); + + assert.ok(state.importBody, "the import endpoint received a body"); + // The multipart payload carries the field name and its "true" value. + assert.match(state.importBody, /name="disableMarkdownExtensions"/); + assert.match(state.importBody, /name="disableMarkdownExtensions"[\s\S]*?\r?\n\r?\ntrue\r?\n/); + // The agent body itself is still sent as the file part. + assert.match(state.importBody, /ticket \$x=1\$ at www\.host\.com/); +}); diff --git a/packages/mcp/test/mock/getpage-text-format.test.mjs b/packages/mcp/test/mock/getpage-text-format.test.mjs new file mode 100644 index 00000000..958ffefe --- /dev/null +++ b/packages/mcp/test/mock/getpage-text-format.test.mjs @@ -0,0 +1,116 @@ +// #502 READ: the getPage tool gains a `format:"text"` mode. It requests the +// server's flat, deterministic text rendering (the server's jsonToText path — the +// SAME serializer that feeds search), passing it through unchanged. This mock +// stands up a local http server (same harness style as getpage-conversion-cache) +// and asserts end-to-end that: +// - format:"text" sends `format:"text"` in the /pages/info body and returns the +// server's text string verbatim (no client-side markdown conversion); +// - the DEFAULT (no format) still returns markdown-converted content; +// - the text read resolves page + subpages like the markdown read. +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { DocmostClient } from "../../build/client.js"; + +function readBody(req) { + return new Promise((resolve) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => resolve(raw)); + }); +} +function sendJson(res, status, obj, extra = {}) { + res.writeHead(status, { "Content-Type": "application/json", ...extra }); + res.end(JSON.stringify(obj)); +} + +const openServers = []; +after(async () => { + await Promise.all(openServers.map((s) => new Promise((r) => s.close(r)))); +}); + +const PAGE_UUID = "00000000-0000-4000-8000-000000000010"; +const SPACE_UUID = "00000000-0000-4000-8000-0000000000aa"; +const CHILD_UUID = "00000000-0000-4000-8000-0000000000bb"; + +// The deterministic text the SERVER's jsonToText would produce for this page. +const SERVER_TEXT = "Title line\nsecond block\n[image]\n[table 2x3]"; + +function makeDoc(text) { + return { type: "doc", content: [{ type: "paragraph", content: [{ type: "text", text }] }] }; +} + +function spawn(state) { + return new Promise((resolve) => { + const server = http.createServer(async (req, res) => { + const body = await readBody(req); + if (req.url === "/api/auth/login") { + return sendJson(res, 200, { success: true }, { "Set-Cookie": "authToken=t; Path=/; HttpOnly" }); + } + if (req.url === "/api/pages/info") { + const parsed = body ? JSON.parse(body) : {}; + state.lastInfoBody = parsed; + // Emulate the server: when format:"text" is requested, `content` is the + // flat text string; otherwise it is the raw ProseMirror JSON. + const content = parsed.format === "text" ? SERVER_TEXT : makeDoc("Title line"); + return sendJson(res, 200, { + success: true, + data: { + id: PAGE_UUID, + slugId: "slug123456", + title: "Text Page", + parentPageId: null, + spaceId: SPACE_UUID, + updatedAt: "2026-01-01T00:00:00Z", + content, + }, + }); + } + if (req.url === "/api/pages/sidebar-pages") { + return sendJson(res, 200, { + success: true, + data: { + items: [{ id: CHILD_UUID, title: "Child", hasChildren: false }], + meta: { hasNextPage: false, nextCursor: null }, + }, + }); + } + return sendJson(res, 404, { message: "not found" }); + }); + server.listen(0, "127.0.0.1", () => { + openServers.push(server); + resolve(`http://127.0.0.1:${server.address().port}/api`); + }); + }); +} + +function makeClient(baseURL) { + return new DocmostClient({ apiUrl: baseURL, getToken: async () => "access" }); +} + +test('getPage(format:"text") requests text and returns the server text verbatim', async () => { + const state = {}; + const client = makeClient(await spawn(state)); + + const result = await client.getPage(PAGE_UUID, "text"); + assert.equal(state.lastInfoBody.format, "text", "the info request carried format:text"); + assert.equal(result.success, true); + assert.equal(result.data.content, SERVER_TEXT, "returns the server's flat text unchanged"); + // Deterministic placeholders are surfaced to the agent. + assert.ok(result.data.content.includes("[image]")); + assert.ok(result.data.content.includes("[table 2x3]")); + // No client-side markdown artifacts leaked in. + assert.ok(!result.data.content.includes("{{SUBPAGES}}")); + // Subpages still resolve for context. + assert.deepEqual(result.data.subpages, [{ id: CHILD_UUID, title: "Child" }]); +}); + +test('getPage default (no format) still returns markdown, not text', async () => { + const state = {}; + const client = makeClient(await spawn(state)); + + const result = await client.getPage(PAGE_UUID); + assert.equal(state.lastInfoBody.format, undefined, "default read sends no format"); + assert.ok(result.data.content.includes("Title line"), "markdown content converted client-side"); + assert.notEqual(result.data.content, SERVER_TEXT); +}); diff --git a/packages/mcp/test/mock/write-path-extensions-wiring.test.mjs b/packages/mcp/test/mock/write-path-extensions-wiring.test.mjs new file mode 100644 index 00000000..ed9a7e22 --- /dev/null +++ b/packages/mcp/test/mock/write-path-extensions-wiring.test.mjs @@ -0,0 +1,181 @@ +// #502 caller-WIRING: end-to-end through a live Hocuspocus collab stack (same +// harness style as markdown-patch-insert), driving the REAL client methods and +// reading the persisted document back, so the test proves each write tool passes +// the RIGHT importer options — not just that the shared wrapper can: +// - updatePageMarkdown (client.updatePage) -> extensions OFF (`$…$` literal, www not linked) +// - import_page_markdown (client.importPageMarkdown) -> DEFAULTS (`$x^2$` -> math node, #328) +// Mutating either caller's option flips the matching assertion (see the coder's +// mutation note), so this file guards the wiring, not only the wrapper. +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { WebSocketServer } from "ws"; +import { Hocuspocus } from "@hocuspocus/server"; +import { DocmostClient } from "../../build/client.js"; +import { buildYDoc } from "../../build/lib/collaboration.js"; +import { serializeDocmostMarkdown } from "@docmost/prosemirror-markdown"; + +const PAGE = "11111111-1111-4111-8111-111111111111"; + +function findAll(node, type, acc = []) { + if (!node || typeof node !== "object") return acc; + if (node.type === type) acc.push(node); + if (Array.isArray(node.content)) for (const c of node.content) findAll(c, type, acc); + return acc; +} +function allText(node, acc = []) { + if (!node || typeof node !== "object") return acc.join(""); + if (node.type === "text" && typeof node.text === "string") acc.push(node.text); + if (Array.isArray(node.content)) for (const c of node.content) allText(c, acc); + return acc.join(""); +} +function hasLink(node) { + return findAll(node, "text").some((t) => t.marks?.some((m) => m.type === "link")); +} + +function fragmentToJson(frag) { + const decodeNode = (el) => { + if (el.constructor.name === "YXmlText") { + const delta = el.toDelta(); + return delta.map((d) => { + const node = { type: "text", text: d.insert }; + if (d.attributes && Object.keys(d.attributes).length) { + node.marks = Object.entries(d.attributes).map(([type, attrs]) => + attrs && typeof attrs === "object" && Object.keys(attrs).length + ? { type, attrs } + : { type }, + ); + } + return node; + }); + } + const node = { type: el.nodeName }; + const attrs = el.getAttributes(); + if (attrs && Object.keys(attrs).length) node.attrs = attrs; + const children = []; + for (const child of el.toArray()) { + const decoded = decodeNode(child); + if (Array.isArray(decoded)) children.push(...decoded); + else children.push(decoded); + } + if (children.length) node.content = children; + return node; + }; + const content = []; + for (const child of frag.toArray()) content.push(decodeNode(child)); + return { type: "doc", content }; +} + +const openStacks = []; +after(async () => { + await Promise.all( + openStacks.map( + ({ server, hocuspocus }) => + new Promise((resolve) => { + server.close(() => { + Promise.resolve(hocuspocus.destroy?.()).finally(resolve); + }); + }), + ), + ); +}); + +async function spawnCollabStack(seedDoc) { + const state = { lastDoc: null }; + const hocuspocus = new Hocuspocus({ + quiet: true, + async onLoadDocument() { + return buildYDoc(seedDoc); + }, + async onChange(data) { + try { + state.lastDoc = fragmentToJson(data.document.getXmlFragment("default")); + } catch { + /* ignore teardown-race decode errors */ + } + }, + }); + const wss = new WebSocketServer({ noServer: true }); + const server = http.createServer((req, res) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => { + if (req.url === "/api/auth/login") { + res.writeHead(200, { + "Content-Type": "application/json", + "Set-Cookie": "authToken=t; Path=/; HttpOnly", + }); + res.end(JSON.stringify({ success: true })); + return; + } + if (req.url === "/api/auth/collab-token") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ data: { token: "collab-jwt" } })); + return; + } + res.writeHead(404, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ message: "not found" })); + }); + }); + server.on("upgrade", (request, socket, head) => { + if (!request.url || !request.url.startsWith("/collab")) { + socket.destroy(); + return; + } + wss.handleUpgrade(request, socket, head, (ws) => { + hocuspocus.handleConnection(ws, request); + }); + }); + const baseURL = await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => { + resolve(`http://127.0.0.1:${server.address().port}/api`); + }); + }); + openStacks.push({ server, hocuspocus }); + return { state, baseURL }; +} + +function seed() { + return { + type: "doc", + content: [ + { type: "paragraph", attrs: { id: "p-id" }, content: [{ type: "text", text: "seed" }] }, + ], + }; +} + +test("updatePageMarkdown wiring: extensions OFF — `$…$` literal, www not linked, https links", async () => { + const { state, baseURL } = await spawnCollabStack(seed()); + const client = new DocmostClient(baseURL, "e@x.com", "pw"); + + await client.updatePage(PAGE, "cfg $x=1$ and www.host.com and https://ex.com"); + + assert.ok(state.lastDoc, "a document was persisted"); + assert.equal(findAll(state.lastDoc, "mathInline").length, 0, "no phantom math from an agent write"); + assert.ok(allText(state.lastDoc).includes("$x=1$"), "literal dollars preserved"); + assert.ok(allText(state.lastDoc).includes("www.host.com"), "bare domain preserved as text"); + // The explicit https URL still links (only the schemeless autolink is off). + const links = findAll(state.lastDoc, "text").filter((t) => + t.marks?.some((m) => m.type === "link"), + ); + assert.ok(links.some((t) => t.text?.includes("ex.com")), "explicit https still links"); +}); + +test("import_page_markdown wiring: DEFAULTS — exported `$x^2$` re-imports AS a math node (#328)", async () => { + const { state, baseURL } = await spawnCollabStack(seed()); + const client = new DocmostClient(baseURL, "e@x.com", "pw"); + + // A self-contained docmost markdown file whose body carries a math span (as the + // exporter emits it). import_page_markdown must import it with math ON. + const meta = { version: 1, pageId: PAGE, slugId: "s", title: "T", spaceId: "sp", parentPageId: null }; + const fullMd = serializeDocmostMarkdown(meta, "energy is $x^2$ here", []); + + await client.importPageMarkdown(PAGE, fullMd); + + assert.ok(state.lastDoc, "a document was persisted"); + assert.equal( + findAll(state.lastDoc, "mathInline").length, + 1, + "the lossless round-trip is intact: math survives import_page_markdown", + ); +}); diff --git a/packages/mcp/test/unit/mcp-write-extensions-off.test.mjs b/packages/mcp/test/unit/mcp-write-extensions-off.test.mjs new file mode 100644 index 00000000..2c4928c7 --- /dev/null +++ b/packages/mcp/test/unit/mcp-write-extensions-off.test.mjs @@ -0,0 +1,108 @@ +// #502: the two layered markdown extensions (`$…$` math, schemeless fuzzy +// autolink) are decided PER CALLER of the shared write importer, not hardcoded in +// the wrapper — because the callers need OPPOSITE behavior: +// - AGENT-authored plain markdown (updatePageMarkdown, patch_node/insert_node) +// -> extensions OFF: a `$…$` config span stays literal, a bare `www.host` is +// not autolinked, an explicit `https://…` still links. +// - FULL-FILE round-trip import (import_page_markdown, #328 lossless) -> +// DEFAULTS (extensions ON): an exported math node's `$x^2$` re-imports AS a +// math node, so the export→import pair is NOT broken. +// This unit file pins the importer contract at each of those semantics. The +// caller-WIRING (that each client method passes the right options) is pinned by +// the collab-backed test in mock/write-path-extensions-wiring.test.mjs. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { markdownToProseMirrorCanonical } from "../../build/lib/collaboration.js"; +import { importMarkdownFragment } from "../../build/lib/markdown-fragment.js"; +import { + markdownToProseMirror, + convertProseMirrorToMarkdown, +} from "@docmost/prosemirror-markdown"; + +const OFF = { parseMath: false, fuzzyLinkify: false }; + +function findAll(node, type, acc = []) { + if (!node || typeof node !== "object") return acc; + if (node.type === type) acc.push(node); + if (Array.isArray(node.content)) for (const c of node.content) findAll(c, type, acc); + return acc; +} +function allText(node, acc = []) { + if (!node || typeof node !== "object") return acc.join(""); + if (node.type === "text" && typeof node.text === "string") acc.push(node.text); + if (Array.isArray(node.content)) for (const c of node.content) allText(c, acc); + return acc.join(""); +} +function hasLink(node) { + return findAll(node, "text").some((t) => t.marks?.some((m) => m.type === "link")); +} + +// --- AGENT-write semantics (updatePageMarkdown): extensions OFF ----------------- + +test("agent-write importer (OFF): `$…$` config stays literal, no math node", async () => { + const doc = await markdownToProseMirrorCanonical("export A=$FOO and B=$BAR done", OFF); + assert.equal(findAll(doc, "mathInline").length, 0); + assert.equal(allText(doc), "export A=$FOO and B=$BAR done"); +}); + +test("agent-write importer (OFF): schemeless www NOT linked; explicit https STILL linked", async () => { + const bare = await markdownToProseMirrorCanonical("see www.example.com here", OFF); + assert.equal(hasLink(bare), false); + assert.equal(allText(bare), "see www.example.com here"); + + const explicit = await markdownToProseMirrorCanonical("see https://example.com here", OFF); + assert.equal(hasLink(explicit), true); +}); + +test("agent-write importer (OFF): heading + list structure preserved", async () => { + const doc = await markdownToProseMirrorCanonical("## Heading\n\n- one\n- two", OFF); + assert.equal(findAll(doc, "heading").length, 1); + assert.equal(findAll(doc, "bulletList").length, 1); +}); + +test("fragment importer (patch_node/insert_node) is OFF: `$x=1$` literal, https links", async () => { + const { blocks } = await importMarkdownFragment("cfg $x=1$ and https://ex.com"); + const doc = { type: "doc", content: blocks }; + assert.equal(findAll(doc, "mathInline").length, 0); + assert.equal(hasLink(doc), true); + assert.ok(allText(doc).includes("$x=1$")); +}); + +// --- FULL-FILE import semantics (import_page_markdown): DEFAULTS (math ON) ------ + +test("import_page_markdown importer (DEFAULTS): `$x^2$` DOES create a math node", async () => { + // markdownToProseMirrorCanonical WITHOUT options == what import_page_markdown + // passes. Math must survive (this is the REAL importer, not the package default). + const doc = await markdownToProseMirrorCanonical("$x^2$"); + assert.equal(findAll(doc, "mathInline").length, 1); + assert.equal(findAll(doc, "mathInline")[0].attrs.text, "x^2"); +}); + +test("import_page_markdown (DEFAULTS): #328 lossless export->import keeps math (round-trip)", async () => { + // The exporter serializes a math node as readable `$x^2$`; re-importing through + // the REAL import_page_markdown importer (markdownToProseMirrorCanonical, no + // options) must yield a math node again and be byte-stable. Under the BUGGY code + // (canonical hardcoded parseMath:false) doc2 would be literal text -> this test + // would REDDEN. + const source = { + type: "doc", + content: [{ type: "paragraph", content: [{ type: "mathInline", attrs: { text: "x^2" } }] }], + }; + const md1 = convertProseMirrorToMarkdown(source); + const doc2 = await markdownToProseMirrorCanonical(md1); // real import path, defaults + assert.equal(findAll(doc2, "mathInline").length, 1, "math survives the round-trip import"); + const md2 = convertProseMirrorToMarkdown(doc2); + assert.equal(md2, md1, "export is byte-stable across the round-trip"); +}); + +test("import_page_markdown (DEFAULTS): a schemeless www IS autolinked", async () => { + const doc = await markdownToProseMirrorCanonical("see www.example.com here"); + assert.equal(hasLink(doc), true); +}); + +// --- Package default importer (editor/file-import/git-sync) is UNCHANGED -------- + +test("PACKAGE default importer keeps math ON (editor/file/git-sync path)", async () => { + const doc = await markdownToProseMirror("$x^2$"); + assert.equal(findAll(doc, "mathInline").length, 1); +}); diff --git a/packages/prosemirror-markdown/src/lib/index.ts b/packages/prosemirror-markdown/src/lib/index.ts index 6bcd3ef0..f5080ef3 100644 --- a/packages/prosemirror-markdown/src/lib/index.ts +++ b/packages/prosemirror-markdown/src/lib/index.ts @@ -25,6 +25,7 @@ export { markdownToProseMirror, markdownToProseMirrorSync, } from "./markdown-to-prosemirror.js"; +export type { MarkdownImportOptions } from "./markdown-to-prosemirror.js"; // Foreign-markdown normalizer (#493): the input-liberal pre-pass that rewrites // GFM `[^id]` reference footnotes to canonical inline `^[body]`. Two variants: diff --git a/packages/prosemirror-markdown/src/lib/markdown-to-prosemirror.ts b/packages/prosemirror-markdown/src/lib/markdown-to-prosemirror.ts index f6ce4696..8755198f 100644 --- a/packages/prosemirror-markdown/src/lib/markdown-to-prosemirror.ts +++ b/packages/prosemirror-markdown/src/lib/markdown-to-prosemirror.ts @@ -7,7 +7,7 @@ * natively through the collab gateway, so no websocket/Yjs write-path lives * here. */ -import { Marked } from "marked"; +import { Marked, Tokenizer } from "marked"; import { parseHtmlDocument, generateJsonWith } from "./dom-parser.js"; import type { TokenizerExtension, RendererExtension } from "marked"; import { docmostExtensions } from "./docmost-schema.js"; @@ -231,19 +231,88 @@ function escapeFootnoteAttr(value: string): string { return String(value).replace(/&/g, "&").replace(/"/g, """); } -// Dedicated marked instance: default (GFM) options plus the `==` highlight -// inline extension, the `$…$` / `$$…$$` math extensions (#293 canon #6), and the -// `^[…]` inline-footnote extension (#293 canon #2). Constructed once at module -// load so the extensions are registered exactly once and never mutate the global -// `marked` singleton. -const markedInstance = new Marked().use({ - extensions: [ +/** + * Options controlling which of the two *layered* markdown extensions the + * canonical importer applies. Both default to `true`, so the human editor, + * file-import and git-sync paths keep their existing behavior byte-for-byte; + * ONLY the MCP markdown-write path opts OUT (see #502). + * + * The extensions are optional because they are the SOURCE of two silent + * corruptions when an AGENT writes plain prose/config as markdown: + * - `parseMath`: a `$…$` span becomes a `mathInline` node. An agent writing a + * config like `export A=$FOO and B=$BAR` gets `$FOO and B=$` silently turned + * into a formula. With `parseMath:false` the `$` stays literal text (real + * formulas go through `update_page_json` with `mathInline`/`mathBlock`). + * - `fuzzyLinkify`: marked's GFM autolinker turns a SCHEMELESS `www.foo.com` + * (and email) into a link. With `fuzzyLinkify:false` a schemeless domain + * stays literal text; an EXPLICIT `https://…` STILL becomes a link (only the + * fuzzy, schemeless autolink is suppressed). + */ +export interface MarkdownImportOptions { + /** Apply the `$…$` / `$$…$$` math extensions (default true). */ + parseMath?: boolean; + /** Apply marked's GFM schemeless (fuzzy) autolinker (default true). */ + fuzzyLinkify?: boolean; +} + +/** + * `fuzzyLinkify:false` override of marked's built-in GFM `url` inline tokenizer. + * + * The stock tokenizer autolinks THREE shapes: a schemeless `www.host` domain, a + * bare email, and an EXPLICIT `scheme://…` URL. #502 wants only the last kept — + * a schemeless domain/email an agent typed as prose must stay literal text, but + * a deliberate `https://…` still links. We delegate to the original tokenizer + * and, when it matched, DROP the token (returning `undefined`, so the run stays + * literal text) unless the matched RAW text carries an explicit `scheme:` prefix. + * `www.`/email matches have no scheme in their raw text, so they are dropped; + * `https://…`/`ftp://…` keep their link. + */ +const SCHEME_PREFIX_RE = /^[a-zA-Z][a-zA-Z0-9+.-]*:/; +const noFuzzyUrlTokenizer = { + url(this: any, src: string) { + const token = Tokenizer.prototype.url.call(this, src) as any; + if (!token) return token; + // Keep only explicit-scheme URLs; schemeless `www.`/email matches -> literal. + return SCHEME_PREFIX_RE.test(token.raw) ? token : undefined; + }, +}; + +/** + * Build a dedicated `marked` instance for a given extension combination: default + * (GFM) options plus the `==` highlight and `^[…]` footnote inline extensions + * ALWAYS, the `$…$`/`$$…$$` math extensions only when `parseMath`, and the + * schemeless-autolink suppressor only when `!fuzzyLinkify`. Built on a private + * `Marked` instance so nothing leaks into the global `marked` singleton. + */ +function buildMarkedInstance(parseMath: boolean, fuzzyLinkify: boolean): Marked { + const extensions: (TokenizerExtension & RendererExtension)[] = [ highlightMarkExtension, - mathInlineExtension, - mathBlockExtension, footnoteInlineExtension, - ], -}); + ]; + if (parseMath) { + extensions.push(mathInlineExtension, mathBlockExtension); + } + const instance = new Marked().use({ extensions }); + if (!fuzzyLinkify) { + instance.use({ tokenizer: noFuzzyUrlTokenizer as any }); + } + return instance; +} + +// Memoize one instance per (parseMath, fuzzyLinkify) combination so the +// extensions are registered exactly once per combo (never on the global +// singleton). The default `(true, true)` instance preserves the pre-#502 +// behavior exactly for the editor/file-import/git-sync paths. +const markedInstanceCache = new Map(); +function getMarkedInstance(parseMath: boolean, fuzzyLinkify: boolean): Marked { + const key = `${parseMath}:${fuzzyLinkify}`; + let instance = markedInstanceCache.get(key); + if (!instance) { + instance = buildMarkedInstance(parseMath, fuzzyLinkify); + markedInstanceCache.set(key, instance); + } + return instance; +} // NOTE: this module no longer installs a module-level `global.window`/`document` // jsdom shim. The HTML->DOM passes below (bridgeTaskLists / applyCommentDirectives @@ -302,7 +371,7 @@ const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/; // preprocess is sync. Keeping it sync lets a sync converter entry // (`markdownToProseMirrorSync`, used by the client's chat renderer which must // stay synchronous) share this exact logic with the async entry. -function preprocessCallouts(markdown: string): string { +function preprocessCallouts(markdown: string, markedInstance: Marked): string { // Defensive cap: skip preprocessing for pathologically large inputs. if (markdown.length > MAX_CALLOUT_PREPROCESS_BYTES) { return markdown; @@ -956,7 +1025,7 @@ function applyCommentDirectives(html: string): string { // sups stay inert) rather than hang. const MAX_FOOTNOTE_ROUNDS = 10000; -function assembleFootnotes(html: string): string { +function assembleFootnotes(html: string, markedInstance: Marked): string { // Cheap early-out: nothing carries a footnote body -> nothing to assemble. if (!html.includes("data-fn-text")) return html; const document = parseHtmlDocument(html); @@ -1082,8 +1151,18 @@ function stripEmptyParagraphs(node: any): any { * for every existing Node consumer). A sync entry is REQUIRED by the client's * chat renderer, which runs inside a React render/useMemo and cannot await. */ -export function markdownToProseMirrorSync(markdownContent: string): any { - const withCallouts = preprocessCallouts(markdownContent); +export function markdownToProseMirrorSync( + markdownContent: string, + options?: MarkdownImportOptions, +): any { + // Select the marked instance for this call's extension combination. Defaults + // (math + fuzzy autolink ON) preserve the editor/file-import/git-sync paths; + // the MCP markdown-write path passes both false (#502). + const markedInstance = getMarkedInstance( + options?.parseMath ?? true, + options?.fuzzyLinkify ?? true, + ); + const withCallouts = preprocessCallouts(markdownContent, markedInstance); const html = markedInstance.parse(withCallouts) as string; // Materialize comment directives (#293 #9 attached textAlign; #5 standalone // subpages/pageBreak) while the comment nodes still exist, before generateJSON @@ -1092,7 +1171,7 @@ export function markdownToProseMirrorSync(markdownContent: string): any { // #293 canon #2: assemble the doc-level footnote list from the `` markers (from `^[…]` or the raw-HTML column form) before // generateJSON, so references + definitions materialize into the schema model. - const withFootnotes = assembleFootnotes(withAttrs); + const withFootnotes = assembleFootnotes(withAttrs, markedInstance); const bridged = bridgeTaskLists(withFootnotes); const doc = generateJsonWith(bridged, docmostExtensions); // Promote unambiguously-internal wiki-page links (`[t](/s//p/)`) @@ -1110,6 +1189,7 @@ export function markdownToProseMirrorSync(markdownContent: string): any { */ export async function markdownToProseMirror( markdownContent: string, + options?: MarkdownImportOptions, ): Promise { - return markdownToProseMirrorSync(markdownContent); + return markdownToProseMirrorSync(markdownContent, options); } diff --git a/packages/prosemirror-markdown/test/markdown-import-options.test.ts b/packages/prosemirror-markdown/test/markdown-import-options.test.ts new file mode 100644 index 00000000..33700764 --- /dev/null +++ b/packages/prosemirror-markdown/test/markdown-import-options.test.ts @@ -0,0 +1,170 @@ +import { describe, expect, it } from 'vitest'; +// Import DIRECTLY from src (like math.test.ts) so we exercise the real +// converter and its module-load jsdom setup. +import { convertProseMirrorToMarkdown } from '../src/lib/markdown-converter.js'; +import { markdownToProseMirror } from '../src/lib/markdown-to-prosemirror.js'; + +// --------------------------------------------------------------------------- +// #502: the canonical importer is parameterized with `{ parseMath, fuzzyLinkify }` +// (both DEFAULT true). The MCP markdown-WRITE path passes both false so an agent's +// plain prose/config is imported LITERALLY: +// - parseMath:false -> a `$…$` span stays literal text (no phantom mathInline) +// - fuzzyLinkify:false -> a SCHEMELESS `www.host`/email stays literal text; an +// EXPLICIT `https://…` STILL links (only the fuzzy autolink is suppressed). +// The DEFAULTS (editor/file-import/git-sync) keep math + fuzzy autolink ON, so +// this file also pins that the defaults are UNCHANGED. +// --------------------------------------------------------------------------- + +const OFF = { parseMath: false, fuzzyLinkify: false } as const; + +// Collect the first paragraph's inline children (the common assertion target). +function firstParaInline(doc: any): any[] { + const p = doc.content?.find((n: any) => n.type === 'paragraph'); + return p?.content ?? []; +} + +function findAll(node: any, type: string, acc: any[] = []): any[] { + if (!node || typeof node !== 'object') return acc; + if (node.type === type) acc.push(node); + if (Array.isArray(node.content)) { + for (const c of node.content) findAll(c, type, acc); + } + return acc; +} + +// Flatten every text node's text (ignoring marks/structure) into one string. +function allText(node: any, acc: string[] = []): string { + if (!node || typeof node !== 'object') return acc.join(''); + if (node.type === 'text' && typeof node.text === 'string') acc.push(node.text); + if (Array.isArray(node.content)) { + for (const c of node.content) allText(c, acc); + } + return acc.join(''); +} + +describe('#502 importer options — extensions OFF (MCP write path)', () => { + it('a `$…$` config span stays literal text (no mathInline node)', async () => { + const md = 'export A=$FOO and B=$BAR done'; + const doc = await markdownToProseMirror(md, OFF); + expect(findAll(doc, 'mathInline')).toHaveLength(0); + expect(allText(doc)).toBe('export A=$FOO and B=$BAR done'); + }); + + it('a real-looking `$x=1$` span stays literal text', async () => { + const doc = await markdownToProseMirror('$x=1$', OFF); + expect(findAll(doc, 'mathInline')).toHaveLength(0); + expect(allText(doc)).toBe('$x=1$'); + }); + + it('the reported `($ticket_lifetime=2592000)` config stays literal', async () => { + const doc = await markdownToProseMirror('($ticket_lifetime=2592000)', OFF); + expect(findAll(doc, 'mathInline')).toHaveLength(0); + expect(allText(doc)).toBe('($ticket_lifetime=2592000)'); + }); + + it('a `$$…$$` block stays literal (no mathBlock node)', async () => { + const doc = await markdownToProseMirror('$$\nx^2\n$$', OFF); + expect(findAll(doc, 'mathBlock')).toHaveLength(0); + expect(allText(doc)).toContain('x^2'); + }); + + it('a SCHEMELESS `www.example.com` is NOT autolinked', async () => { + const doc = await markdownToProseMirror('see www.example.com here', OFF); + const inline = firstParaInline(doc); + expect(inline.some((n: any) => n.marks?.some((m: any) => m.type === 'link'))).toBe(false); + expect(allText(doc)).toBe('see www.example.com here'); + }); + + it('a bare dotted domain `gitea.vvzvlad.xyz` stays literal text', async () => { + const doc = await markdownToProseMirror('see gitea.vvzvlad.xyz here', OFF); + expect(findAll(doc, 'text').every((t: any) => !t.marks?.some((m: any) => m.type === 'link'))).toBe(true); + }); + + it('an EXPLICIT `https://…` STILL becomes a link', async () => { + const doc = await markdownToProseMirror('see https://example.com here', OFF); + const linked = firstParaInline(doc).find((n: any) => + n.marks?.some((m: any) => m.type === 'link'), + ); + expect(linked?.text).toBe('https://example.com'); + const link = linked.marks.find((m: any) => m.type === 'link'); + expect(link.attrs.href).toBe('https://example.com'); + }); + + it('`foo_bar_baz` is not italicized (CommonMark, unaffected by options)', async () => { + const doc = await markdownToProseMirror('foo_bar_baz', OFF); + expect(findAll(doc, 'text').some((t: any) => t.marks?.some((m: any) => m.type === 'italic' || m.type === 'em'))).toBe(false); + expect(allText(doc)).toBe('foo_bar_baz'); + }); + + it('block STRUCTURE (headings, lists, code fence) is preserved with extensions off', async () => { + const md = '## Heading\n\n- one\n- two\n\n```\ncode $x$ here\n```'; + const doc = await markdownToProseMirror(md, OFF); + expect(findAll(doc, 'heading')).toHaveLength(1); + expect(findAll(doc, 'bulletList')).toHaveLength(1); + expect(findAll(doc, 'codeBlock')).toHaveLength(1); + // The `$x$` inside the code fence never becomes math regardless. + expect(findAll(doc, 'mathInline')).toHaveLength(0); + }); +}); + +describe('#502 importer options — DEFAULTS unchanged (editor/file/git-sync)', () => { + it('DEFAULT: `$x^2$` DOES create a mathInline node (file-import unaffected)', async () => { + const doc = await markdownToProseMirror('$x^2$'); + expect(findAll(doc, 'mathInline')).toHaveLength(1); + expect(findAll(doc, 'mathInline')[0].attrs.text).toBe('x^2'); + }); + + it('DEFAULT: a schemeless `www.example.com` IS autolinked', async () => { + const doc = await markdownToProseMirror('see www.example.com here'); + const linked = firstParaInline(doc).find((n: any) => + n.marks?.some((m: any) => m.type === 'link'), + ); + expect(linked?.text).toBe('www.example.com'); + }); + + it('DEFAULT: explicitly passing {parseMath:true, fuzzyLinkify:true} equals no-options', async () => { + const a = await markdownToProseMirror('$x^2$ and www.foo.com'); + const b = await markdownToProseMirror('$x^2$ and www.foo.com', { + parseMath: true, + fuzzyLinkify: true, + }); + expect(JSON.stringify(b)).toBe(JSON.stringify(a)); + }); + + it('round-trip export->import at the PACKAGE-DEFAULT layer keeps math (file-import / #328)', async () => { + // The package DEFAULT importer is what the server file-import path uses. A + // page holding real math is exported, then re-imported with DEFAULTS (math + // ON): the mathInline survives. (The tool-level import_page_markdown round- + // trip, which goes through mcp's markdownToProseMirrorCanonical, is pinned + // authoritatively in @docmost/mcp's mcp-write-extensions-off test.) + const source = { + type: 'doc', + content: [ + { type: 'paragraph', content: [{ type: 'mathInline', attrs: { text: 'x^2' } }] }, + ], + }; + const md1 = convertProseMirrorToMarkdown(source); + const doc2 = await markdownToProseMirror(md1); // DEFAULTS -> math on + expect(findAll(doc2, 'mathInline')).toHaveLength(1); + const md2 = convertProseMirrorToMarkdown(doc2); + expect(md2).toBe(md1); // byte-stable + }); +}); + +describe('#502 mutation guard', () => { + // If a future change silently flipped the MCP write path back to parseMath:true, + // the OFF assertion below would go RED — this pins the discriminating behavior. + it('with parseMath:true the same span DOES become math (proves the flag drives it)', async () => { + const on = await markdownToProseMirror('$x=1$', { parseMath: true, fuzzyLinkify: false }); + expect(findAll(on, 'mathInline')).toHaveLength(1); + const off = await markdownToProseMirror('$x=1$', OFF); + expect(findAll(off, 'mathInline')).toHaveLength(0); + }); + + it('with fuzzyLinkify:true the same www domain DOES link (proves the flag drives it)', async () => { + const on = await markdownToProseMirror('www.example.com', { parseMath: false, fuzzyLinkify: true }); + expect(findAll(on, 'text').some((t: any) => t.marks?.some((m: any) => m.type === 'link'))).toBe(true); + const off = await markdownToProseMirror('www.example.com', OFF); + expect(findAll(off, 'text').some((t: any) => t.marks?.some((m: any) => m.type === 'link'))).toBe(false); + }); +});