Merge pull request 'feat(mcp): буквальная запись markdown (без math/autolink) + getPage format:text (#502)' (#552) from feat/502-mcp-md-modes into develop

Reviewed-on: #552

apps/server/src/collaboration/collaboration.util.ts

@@ -141,7 +141,57 @@ export function htmlToJson(html: string) {
   }
 }
 
-export function jsonToText(tiptapJson: JSONContent) {
+/**
+ * Deterministic text-serializer overrides for the `format:"text"` page read
+ * (#502). Non-text nodes render to a STABLE placeholder instead of their
+ * (structure-dependent) inner text, so a machine diff of two text reads is
+ * driven only by the page's actual prose — output stability across package
+ * versions IS the contract (pinned by a snapshot test). Returning a string from
+ * a `textSerializer` also stops `generateText` descending into the node, so a
+ * table renders as ONE token rather than its flattened cell text.
+ *
+ * Only nodes with no meaningful flat-text form are overridden; every other node
+ * (paragraph/heading/list/code/blockquote/callout/…) keeps its natural text so
+ * a config written as markdown reads back byte-identical.
+ */
+const TEXT_READ_SERIALIZERS: Record<string, (props: { node: any }) => string> =
+  {
+    // Image atom: no inner text -> a fixed placeholder.
+    image: () => '[image]',
+    // Table: `[table RxC]` where R = row count, C = the first row's cell count
+    // (a table's columns are uniform per the schema). Computed from the PM node,
+    // so it is independent of cell contents.
+    table: ({ node }) => {
+      const rows = node?.childCount ?? 0;
+      const cols = rows > 0 ? (node.child(0)?.childCount ?? 0) : 0;
+      return `[table ${rows}x${cols}]`;
+    },
+  };
+
+/**
+ * Serialize a ProseMirror/TipTap document to plain text.
+ *
+ * Default (no options): the long-standing search-index behavior — bare
+ * concatenated node text with `generateText`'s default `\n\n` block separator.
+ * This feeds the page `textContent` tsvector and MUST NOT change.
+ *
+ * `deterministic:true` (#502 `format:"text"` page read): a flat, machine-diffable
+ * rendering — one line per block (`\n` block separator; `hardBreak` already
+ * serializes to `\n`), inline marks/anchors/autoformat dropped, and non-text
+ * nodes replaced by the stable placeholders above (`[image]`, `[table RxC]`).
+ */
+export function jsonToText(
+  // `any` (like jsonToHtml/jsonToMarkdown) so a loosely-typed DB `page.content`
+  // (JsonValue) can be passed straight through, as the controller does.
+  tiptapJson: any,
+  options?: { deterministic?: boolean },
+) {
+  if (options?.deterministic) {
+    return generateText(tiptapJson, tiptapExtensions, {
+      blockSeparator: '\n',
+      textSerializers: TEXT_READ_SERIALIZERS,
+    });
+  }
   return generateText(tiptapJson, tiptapExtensions);
 }
 

apps/server/src/collaboration/json-to-text-deterministic.spec.ts

@@ -0,0 +1,116 @@
+import { jsonToText } from './collaboration.util';
+
+// #502 READ contract: `jsonToText(json, { deterministic: true })` — the flat,
+// machine-diffable text rendering behind getPage `format:"text"`. Block-per-line
+// (`\n` separator), inline marks/anchors dropped, hardBreak -> `\n`, and non-text
+// nodes replaced by STABLE placeholders (`[image]`, `[table RxC]`). Output
+// stability across package versions IS the contract, so it is pinned by a
+// snapshot below. The DEFAULT (no options) path is the search-index serializer
+// and MUST be unchanged — asserted separately.
+
+const doc = (...content: any[]) => ({ type: 'doc', content });
+const para = (...content: any[]) => ({ type: 'paragraph', content });
+const text = (t: string, marks?: any[]) =>
+  marks ? { type: 'text', text: t, marks } : { type: 'text', text: t };
+
+describe('jsonToText — default (search index) behavior is unchanged', () => {
+  it('uses the `\\n\\n` block separator and drops non-text nodes to empty', () => {
+    const d = doc(para(text('alpha')), para(text('beta')));
+    expect(jsonToText(d)).toBe('alpha\n\nbeta');
+  });
+
+  it('an image contributes no text in the default (tsvector) mode', () => {
+    const d = doc(para(text('cap')), { type: 'image', attrs: { src: 's' } });
+    // No `[image]` placeholder leaks into the search index.
+    expect(jsonToText(d)).not.toContain('[image]');
+  });
+});
+
+describe('jsonToText — deterministic:true (getPage format:"text")', () => {
+  it('renders one line per block with `\\n` separators, marks dropped', () => {
+    const d = doc(
+      { type: 'heading', attrs: { level: 2 }, content: [text('Title')] },
+      para(
+        text('hello ', [{ type: 'bold' }]),
+        text('world', [{ type: 'italic' }]),
+      ),
+    );
+    expect(jsonToText(d, { deterministic: true })).toBe('Title\nhello world');
+  });
+
+  it('a hardBreak renders as a newline', () => {
+    const d = doc(para(text('a'), { type: 'hardBreak' }, text('b')));
+    expect(jsonToText(d, { deterministic: true })).toBe('a\nb');
+  });
+
+  it('an image node -> the stable `[image]` placeholder', () => {
+    const d = doc(para(text('before')), { type: 'image', attrs: { src: 's' } });
+    expect(jsonToText(d, { deterministic: true })).toBe('before\n[image]');
+  });
+
+  it('a table -> `[table RxC]` (rows x columns) and its cell text is NOT flattened in', () => {
+    const cell = (t: string) => ({
+      type: 'tableCell',
+      content: [para(text(t))],
+    });
+    const row = (...cells: any[]) => ({ type: 'tableRow', content: cells });
+    const table = {
+      type: 'table',
+      content: [
+        row(cell('CELLONE'), cell('CELLTWO'), cell('CELLTHREE')),
+        row(cell('CELLFOUR'), cell('CELLFIVE'), cell('CELLSIX')),
+      ],
+    };
+    const d = doc(para(text('grid:')), table);
+    const out = jsonToText(d, { deterministic: true });
+    expect(out).toBe('grid:\n[table 2x3]');
+    expect(out).not.toContain('CELL'); // cell text is not flattened into the read
+  });
+
+  it('SNAPSHOT: a mixed document renders to a stable, deterministic string', () => {
+    const cell = (t: string) => ({
+      type: 'tableCell',
+      content: [para(text(t))],
+    });
+    const d = doc(
+      { type: 'heading', attrs: { level: 1 }, content: [text('Config')] },
+      para(text('key = ', [{ type: 'bold' }]), text('value')),
+      {
+        type: 'bulletList',
+        content: [
+          { type: 'listItem', content: [para(text('one'))] },
+          { type: 'listItem', content: [para(text('two'))] },
+        ],
+      },
+      { type: 'image', attrs: { src: 's' } },
+      {
+        type: 'table',
+        content: [{ type: 'tableRow', content: [cell('x'), cell('y')] }],
+      },
+    );
+    expect(jsonToText(d, { deterministic: true })).toMatchInlineSnapshot(`
+     "Config
+     key = value
+
+
+     one
+
+     two
+     [image]
+     [table 1x2]"
+    `);
+  });
+
+  it('SCENARIO: a config written as a code block reads back byte-identical (diff empty)', () => {
+    const config =
+      'export TICKET_LIFETIME=$2592000\nservers:\n  - www.internal.host';
+    const d = doc({
+      type: 'codeBlock',
+      attrs: { language: 'yaml' },
+      content: [text(config)],
+    });
+    // The code block is one block; its text (dollars, bare domain, newlines) is
+    // preserved verbatim, so a read-as-text of a stored config diffs empty.
+    expect(jsonToText(d, { deterministic: true })).toBe(config);
+  });
+});

apps/server/src/core/page/dto/create-page.dto.ts

@@ -10,6 +10,12 @@ import { Transform } from 'class-transformer';
 
 export type ContentFormat = 'json' | 'markdown' | 'html';
 
+// READ-only rendering formats for `getPage` (#502). A superset of the writable
+// `ContentFormat` with `text` — a flat, deterministic, machine-diffable text
+// rendering — added. Kept SEPARATE from `ContentFormat` so the write path
+// (createPage/updatePage `parseProsemirrorContent`) can never be handed `text`.
+export type PageReadFormat = ContentFormat | 'text';
+
 export class CreatePageDto {
   @IsOptional()
   @IsString()

apps/server/src/core/page/dto/page.dto.ts

@@ -8,7 +8,7 @@ import {
 } from 'class-validator';
 import { Transform } from 'class-transformer';
 
-import { ContentFormat } from './create-page.dto';
+import { PageReadFormat } from './create-page.dto';
 import { IsPageIdOrSlugId } from './page-identity.validator';
 
 export class PageIdDto {
@@ -43,8 +43,8 @@ export class PageInfoDto extends PageIdDto {
 
   @IsOptional()
   @Transform(({ value }) => value?.toLowerCase())
-  @IsIn(['json', 'markdown', 'html'])
-  format?: ContentFormat;
+  @IsIn(['json', 'markdown', 'html', 'text'])
+  format?: PageReadFormat;
 }
 
 export class DeletePageDto extends PageIdDto {

apps/server/src/core/page/page.controller.getpage-format.spec.ts

@@ -0,0 +1,76 @@
+import { NotFoundException } from '@nestjs/common';
+import { PageController } from './page.controller';
+import { jsonToText, jsonToMarkdown } from '../../collaboration/collaboration.util';
+
+// #502 READ: getPage `format:"text"` routes through the deterministic jsonToText
+// path (placeholders for non-text nodes, block-per-line), while `format:"json"`
+// (or none) returns the raw content and `format:"markdown"` still converts. This
+// pins the CONTROLLER wiring with lightweight mocks (no DB needed — the jsonToText
+// output contract itself is pinned in json-to-text-deterministic.spec.ts).
+
+const CONTENT = {
+  type: 'doc',
+  content: [
+    { type: 'heading', attrs: { level: 1 }, content: [{ type: 'text', text: 'Config' }] },
+    {
+      type: 'paragraph',
+      content: [
+        { type: 'text', text: 'key=', marks: [{ type: 'bold' }] },
+        { type: 'text', text: 'value' },
+      ],
+    },
+    { type: 'image', attrs: { src: 's' } },
+  ],
+};
+
+function makeController(page: any): PageController {
+  const pageRepo = { findById: jest.fn().mockResolvedValue(page) } as any;
+  const pageAccessService = {
+    validateCanViewWithPermissions: jest
+      .fn()
+      .mockResolvedValue({ canEdit: true, hasRestriction: false }),
+  } as any;
+  // Only pageRepo + pageAccessService are exercised by getPage; the rest are
+  // never touched on this path, so undefined placeholders are fine.
+  return new PageController(
+    undefined as any, // pageService
+    pageRepo,
+    undefined as any, // pageHistoryService
+    undefined as any, // spaceAbility
+    pageAccessService,
+    undefined as any, // backlinkService
+    undefined as any, // labelService
+    undefined as any, // auditService
+  );
+}
+
+const user = { id: 'u1' } as any;
+
+describe('PageController.getPage — format:"text" (#502)', () => {
+  it('returns deterministic flat text with placeholders for non-text nodes', async () => {
+    const controller = makeController({ id: 'p1', content: CONTENT });
+    const res: any = await controller.getPage({ pageId: 'p1', format: 'text' } as any, user);
+    expect(res.content).toBe(jsonToText(CONTENT, { deterministic: true }));
+    expect(res.content).toBe('Config\nkey=value\n[image]');
+    expect(res.permissions).toEqual({ canEdit: true, hasRestriction: false });
+  });
+
+  it('markdown format still converts to markdown (unchanged)', async () => {
+    const controller = makeController({ id: 'p1', content: CONTENT });
+    const res: any = await controller.getPage({ pageId: 'p1', format: 'markdown' } as any, user);
+    expect(res.content).toBe(jsonToMarkdown(CONTENT));
+  });
+
+  it('json / no format returns the raw ProseMirror content object', async () => {
+    const controller = makeController({ id: 'p1', content: CONTENT });
+    const res: any = await controller.getPage({ pageId: 'p1' } as any, user);
+    expect(res.content).toEqual(CONTENT); // untouched object, not a string
+  });
+
+  it('missing page -> NotFoundException', async () => {
+    const controller = makeController(null);
+    await expect(
+      controller.getPage({ pageId: 'nope', format: 'text' } as any, user),
+    ).rejects.toBeInstanceOf(NotFoundException);
+  });
+});

apps/server/src/core/page/page.controller.ts

@@ -49,6 +49,7 @@ import { AddLabelsDto, RemoveLabelDto } from '../label/dto/label.dto';
 import {
   jsonToHtml,
   jsonToMarkdown,
+  jsonToText,
 } from '../../collaboration/collaboration.util';
 import { AuditEvent, AuditResource } from '../../common/events/audit-events';
 import {
@@ -93,10 +94,16 @@ export class PageController {
     const permissions = { canEdit, hasRestriction };
 
     if (dto.format && dto.format !== 'json' && page.content) {
-      const contentOutput =
-        dto.format === 'markdown'
-          ? jsonToMarkdown(page.content)
-          : jsonToHtml(page.content);
+      let contentOutput: string;
+      if (dto.format === 'markdown') {
+        contentOutput = jsonToMarkdown(page.content);
+      } else if (dto.format === 'text') {
+        // #502: flat, deterministic, machine-diffable text (block-per-line,
+        // inline marks/anchors dropped, non-text nodes -> stable placeholders).
+        contentOutput = jsonToText(page.content, { deterministic: true });
+      } else {
+        contentOutput = jsonToHtml(page.content);
+      }
       return {
         ...page,
         content: contentOutput,

apps/server/src/integrations/import/import.controller.ts

@@ -85,6 +85,14 @@ export class ImportController {
       throw new BadRequestException('spaceId is required');
     }
 
+    // #502: optional multipart field. Only the MCP agent `createPage` path sends
+    // `disableMarkdownExtensions=true` (its body is agent-authored plain prose /
+    // config, so a `$…$` span must stay literal and a bare `www.host` must not
+    // autolink). A HUMAN file upload omits the field, so it stays false and math
+    // + autolink remain ON for human imports. Settable ONLY via this API param.
+    const disableMarkdownExtensions =
+      file.fields?.disableMarkdownExtensions?.value === 'true';
+
     const ability = await this.spaceAbility.createForUser(user, spaceId);
     if (ability.cannot(SpaceCaslAction.Edit, SpaceCaslSubject.Page)) {
       throw new ForbiddenException();
@@ -95,6 +103,7 @@ export class ImportController {
       user.id,
       spaceId,
       workspace.id,
+      disableMarkdownExtensions,
     );
 
     const ext = path.extname(file.filename).toLowerCase();

apps/server/src/integrations/import/services/import.service.processMarkdown.spec.ts

@@ -0,0 +1,67 @@
+// Importing ImportService transitively loads import-formatter.ts, which imports
+// the ESM-only @sindresorhus/slugify (not in jest's transform allowlist). It is
+// irrelevant to this path, so mock it to keep the module graph loadable (mirrors
+// the sibling import.service specs).
+jest.mock('@sindresorhus/slugify', () => ({
+  __esModule: true,
+  default: (input: string) => String(input),
+}));
+
+import { ImportService } from './import.service';
+
+// #502 BLOCKER 1: the server markdown import path (`/pages/import`) now accepts an
+// optional `disableMarkdownExtensions` flag threaded into `processMarkdown`.
+//   - MCP agent `createPage` sends it TRUE  -> extensions OFF (a `$…$` config span
+//     stays literal, a schemeless `www.host` is not autolinked).
+//   - a HUMAN file upload omits it (default FALSE) -> extensions ON, so a real
+//     `$x^2$` still becomes a formula (human imports unaffected).
+// `processMarkdown` only uses the imported converter (no injected deps on this
+// path), so the service is constructed with null deps for this focused unit test.
+
+function makeService(): ImportService {
+  return new ImportService(null as any, null as any, null as any, null as any);
+}
+
+function findAll(node: any, type: string, acc: any[] = []): any[] {
+  if (!node || typeof node !== 'object') return acc;
+  if (node.type === type) acc.push(node);
+  if (Array.isArray(node.content)) for (const c of node.content) findAll(c, type, acc);
+  return acc;
+}
+function hasLink(node: any): boolean {
+  return findAll(node, 'text').some((t: any) =>
+    t.marks?.some((m: any) => m.type === 'link'),
+  );
+}
+
+describe('ImportService.processMarkdown — #502 disableMarkdownExtensions', () => {
+  it('disableMarkdownExtensions=true (MCP createPage): `$…$` stays literal, no math', async () => {
+    const svc = makeService();
+    const doc = await svc.processMarkdown('export A=$FOO and B=$BAR done', true);
+    expect(findAll(doc, 'mathInline')).toHaveLength(0);
+  });
+
+  it('disableMarkdownExtensions=true: a schemeless www is NOT autolinked', async () => {
+    const svc = makeService();
+    const doc = await svc.processMarkdown('see www.example.com here', true);
+    expect(hasLink(doc)).toBe(false);
+  });
+
+  it('disableMarkdownExtensions=true: an explicit https:// STILL links', async () => {
+    const svc = makeService();
+    const doc = await svc.processMarkdown('see https://example.com here', true);
+    expect(hasLink(doc)).toBe(true);
+  });
+
+  it('DEFAULT (human upload): a real `$x^2$` DOES become a math node', async () => {
+    const svc = makeService();
+    const doc = await svc.processMarkdown('$x^2$');
+    expect(findAll(doc, 'mathInline')).toHaveLength(1);
+  });
+
+  it('DEFAULT (human upload): a schemeless www IS autolinked', async () => {
+    const svc = makeService();
+    const doc = await svc.processMarkdown('see www.example.com here');
+    expect(hasLink(doc)).toBe(true);
+  });
+});

apps/server/src/integrations/import/services/import.service.ts

@@ -52,6 +52,12 @@ export class ImportService {
     userId: string,
     spaceId: string,
     workspaceId: string,
+    // #502: when true, the markdown importer runs with the two layered
+    // extensions OFF (a `$…$` span stays literal text; a schemeless `www.host` is
+    // NOT autolinked). ONLY the MCP agent `createPage` path sets this; a HUMAN
+    // file upload never passes it, so it defaults false and math/autolink stay ON
+    // for human imports (their `$x^2$` still becomes a formula).
+    disableMarkdownExtensions = false,
   ) {
     const file = await filePromise;
     const fileBuffer = await file.toBuffer();
@@ -66,7 +72,10 @@ export class ImportService {
 
     try {
       if (fileExtension.endsWith('.md')) {
-        prosemirrorState = await this.processMarkdown(fileContent);
+        prosemirrorState = await this.processMarkdown(
+          fileContent,
+          disableMarkdownExtensions,
+        );
       } else if (fileExtension.endsWith('.html')) {
         prosemirrorState = await this.processHTML(fileContent);
       }
@@ -138,7 +147,12 @@ export class ImportService {
     return createdPage;
   }
 
-  async processMarkdown(markdownInput: string): Promise<any> {
+  async processMarkdown(
+    markdownInput: string,
+    // #502: forwarded to the importer. DEFAULT false keeps math + fuzzy autolink
+    // ON (human uploads unaffected); the MCP agent `createPage` path passes true.
+    disableMarkdownExtensions = false,
+  ): Promise<any> {
     // Canonical markdown -> ProseMirror JSON directly via
     // `@docmost/prosemirror-markdown` (issue #345) — no HTML intermediate and no
     // second editor-ext markdown layer. Foreign markdown surfaces the strict
@@ -147,7 +161,12 @@ export class ImportService {
     // The HTML-cleanup pass (`normalizeImportHtml`) is intentionally skipped here:
     // it targets foreign *HTML* (Notion/XWiki), which only ever arrives on the
     // `.html` path (`processHTML`), never as canonical markdown.
-    return markdownToProseMirror(normalizeForeignMarkdown(markdownInput));
+    return markdownToProseMirror(
+      normalizeForeignMarkdown(markdownInput),
+      disableMarkdownExtensions
+        ? { parseMath: false, fuzzyLinkify: false }
+        : undefined,
+    );
   }
 
   async processHTML(htmlInput: string): Promise<any> {