diff --git a/AGENTS.md b/AGENTS.md index a68d5d7a..4093d15c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -201,7 +201,7 @@ pnpm workspace (`pnpm@10.4.0`) orchestrated by **Nx**. Four workspace packages: | `apps/client` | `client` | React 18 + Vite + Mantine 8 + TanStack Query + Jotai | SPA frontend | | `packages/editor-ext` | `@docmost/editor-ext` | Tiptap/ProseMirror | Shared Tiptap node/mark extensions, imported by both the client and the server | | `packages/mcp` | `@docmost/mcp` | MCP SDK, Tiptap, Yjs | Standalone MCP server, also bundled into the server at `/mcp`. Consumes the shared converter/schema from `@docmost/prosemirror-markdown` (#293) — it no longer carries its own vendored converter/schema copy | -| `packages/prosemirror-markdown` | `@docmost/prosemirror-markdown` | Tiptap, marked, jsdom | The single, canonical ProseMirror↔Markdown converter + Docmost schema mirror (#293). Consumed by `mcp` and `git-sync`; there is exactly ONE copy of the converter now | +| `packages/prosemirror-markdown` | `@docmost/prosemirror-markdown` | Tiptap, marked, jsdom | The single, canonical ProseMirror↔Markdown converter + Docmost schema mirror (#293). Consumed by `mcp`, `git-sync`, AND `apps/server` (server-side markdown import/export, #345); there is exactly ONE copy of the converter now | `build` targets are Nx-cached and dependency-ordered (`dependsOn: ["^build"]`), so `editor-ext` builds before the apps. `nx.json` sets `affected.defaultBase: main`. @@ -284,7 +284,7 @@ The API server is a Fastify app with a global `/api` prefix (`main.ts` excludes ### Client structure Vite SPA. Code is organized by feature under `apps/client/src/features/*` (mirrors the server domains: `page`, `space`, `comment`, `ai-chat`, `editor`, …). Conventions: - **TanStack Query** for server state (one `queries/` file per feature), **Jotai** atoms for local/shared UI state, **Mantine 8** + CSS modules (`*.module.css`) + `postcss-preset-mantine` for UI. -- The editor is Tiptap; shared node/mark extensions live in `packages/editor-ext` and are imported by **both the client and the server** (collaboration, import/export) — editor schema changes often need to be made in `editor-ext`, not just the client. The ProseMirror↔Markdown converter and its Docmost schema mirror now live in a SINGLE package, `@docmost/prosemirror-markdown` (#293), consumed by both `mcp` and `git-sync` — do NOT reintroduce a per-package copy. `editor-ext` is the upstream source of the Tiptap schema; the package's `docmost-schema.ts` mirrors it and a serializer-contract test (`packages/prosemirror-markdown/test/serializer-contract.test.ts`) guards the boundary (every schema node must have a converter case), so a drift surfaces as a failing test rather than silent divergence. +- The editor is Tiptap; shared node/mark extensions live in `packages/editor-ext` and are imported by **both the client and the server** (collaboration, schema, `canonicalizeFootnotes`) — editor schema changes often need to be made in `editor-ext`, not just the client. Server-side markdown import/export no longer lives in `editor-ext`: it goes through the canonical converter (#345, see below). The ProseMirror↔Markdown converter and its Docmost schema mirror now live in a SINGLE package, `@docmost/prosemirror-markdown` (#293), consumed by `mcp`, `git-sync`, and `apps/server` (#345) — do NOT reintroduce a per-package copy. `editor-ext` is the upstream source of the Tiptap schema; the package's `docmost-schema.ts` mirrors it and a serializer-contract test (`packages/prosemirror-markdown/test/serializer-contract.test.ts`) guards the boundary (every schema node must have a converter case), so a drift surfaces as a failing test rather than silent divergence. - API access goes through `apps/client/src/lib/api-client.ts` (axios). The `@` alias maps to `apps/client/src`. - Runtime config is injected at build time by `vite.config.ts` via `define` (`APP_URL`, `COLLAB_URL`, `APP_VERSION`, …) — these come from the root `.env`, not from `import.meta.env`. diff --git a/apps/server/package.json b/apps/server/package.json index 517f1b3d..f12c1b2d 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -23,7 +23,7 @@ "migration:reset": "tsx src/database/migrate.ts down-to NO_MIGRATIONS", "migration:codegen": "kysely-codegen --dialect=postgres --camel-case --env-file=../../.env --out-file=./src/database/types/db.d.ts", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", - "pretest": "pnpm --filter @docmost/editor-ext build", + "pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build", "test": "jest", "test:int": "jest --config test/jest-integration.json", "test:watch": "jest --watch", @@ -43,6 +43,7 @@ "@clickhouse/client": "^1.18.2", "@docmost/mcp": "workspace:*", "@docmost/pdf-inspector": "1.9.6", + "@docmost/prosemirror-markdown": "workspace:*", "@fastify/cookie": "^11.0.2", "@fastify/multipart": "^10.0.0", "@fastify/static": "^9.1.3", @@ -175,7 +176,7 @@ "/node_modules/" ], "transform": { - "happy-dom.+\\.js$": [ + "(happy-dom.+|prosemirror-markdown/build/.+)\\.js$": [ "babel-jest", { "presets": [ @@ -193,7 +194,7 @@ "^.+\\.(t|j)sx?$": "ts-jest" }, "transformIgnorePatterns": [ - "/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom|lib0)(@|/))" + "/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom|lib0|@docmost/prosemirror-markdown)(@|/))" ], "collectCoverageFrom": [ "**/*.(t|j)s" @@ -204,7 +205,8 @@ "^@docmost/db/(.*)$": "/database/$1", "^@docmost/transactional/(.*)$": "/integrations/transactional/$1", "^@docmost/ee/(.*)$": "/ee/$1", - "^src/(.*)$": "/$1" + "^src/(.*)$": "/$1", + "^@tiptap/react$": "/../test/stubs/tiptap-react.js" } } } diff --git a/apps/server/src/collaboration/collaboration.util.ts b/apps/server/src/collaboration/collaboration.util.ts index 7970051b..50e961ce 100644 --- a/apps/server/src/collaboration/collaboration.util.ts +++ b/apps/server/src/collaboration/collaboration.util.ts @@ -43,7 +43,6 @@ import { Column, Status, addUniqueIdsToDoc, - htmlToMarkdown, TransclusionSource, TransclusionReference, FootnoteReference, @@ -51,6 +50,7 @@ import { FootnoteDefinition, PageEmbed, } from '@docmost/editor-ext'; +import { convertProseMirrorToMarkdown } from '@docmost/prosemirror-markdown'; import { generateText, getSchema, JSONContent } from '@tiptap/core'; import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html'; // @tiptap/html library works best for generating prosemirror json state but not HTML @@ -239,6 +239,10 @@ export function prosemirrorNodeToYElement(node: any): Y.XmlElement | Y.XmlText { } export function jsonToMarkdown(tiptapJson: any): string { - const html = jsonToHtml(tiptapJson); - return htmlToMarkdown(html); + // Direct ProseMirror JSON -> Markdown via the canonical converter + // (`@docmost/prosemirror-markdown`) — no HTML intermediate, no second + // editor-ext markdown layer. Same serializer as the page/space export and the + // git-sync vault writer, so every server PM->MD path emits identical canonical + // markdown (issue #345). + return convertProseMirrorToMarkdown(tiptapJson); } diff --git a/apps/server/src/core/page/services/page.service.ts b/apps/server/src/core/page/services/page.service.ts index c6ee150d..ae20d277 100644 --- a/apps/server/src/core/page/services/page.service.ts +++ b/apps/server/src/core/page/services/page.service.ts @@ -52,7 +52,9 @@ import { INTERNAL_LINK_REGEX, extractPageSlugId, } from '../../../integrations/export/utils'; -import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext'; +import { canonicalizeFootnotes } from '@docmost/editor-ext'; +import { markdownToProseMirror } from '@docmost/prosemirror-markdown'; +import { normalizeForeignMarkdown } from '../../../integrations/import/utils/foreign-markdown'; import { WatcherService } from '../../watcher/watcher.service'; import { sql } from 'kysely'; import { TransclusionService } from '../transclusion/transclusion.service'; @@ -1301,8 +1303,14 @@ export class PageService { switch (format) { case 'markdown': { - const html = await markdownToHtml(content as string); - prosemirrorJson = htmlToJson(html as string); + // Canonical markdown -> ProseMirror JSON directly via + // `@docmost/prosemirror-markdown` (issue #345) — no HTML intermediate, + // no editor-ext markdown layer. Foreign markdown surfaces the strict + // parser rejects (GFM `[^id]` reference footnotes) are normalized to the + // canonical inline form first. + prosemirrorJson = await markdownToProseMirror( + normalizeForeignMarkdown(content as string), + ); break; } case 'html': { diff --git a/apps/server/src/integrations/export/export-markdown.spec.ts b/apps/server/src/integrations/export/export-markdown.spec.ts new file mode 100644 index 00000000..a242e6cb --- /dev/null +++ b/apps/server/src/integrations/export/export-markdown.spec.ts @@ -0,0 +1,145 @@ +// export.service.ts imports the ESM-only @sindresorhus/slugify (not in jest's +// transform allowlist). It is irrelevant to the markdown-serialization path under +// test (only used for page-mention link slugs on the DB path), so it is mocked +// out to keep the module graph loadable under ts-jest (mirrors the import specs). +jest.mock('@sindresorhus/slugify', () => ({ + __esModule: true, + default: (input: string) => String(input), +})); + +import { convertProseMirrorToMarkdown } from '@docmost/prosemirror-markdown'; +import { ExportService } from './export.service'; +import { ExportFormat } from './dto/export-dto'; + +/** + * STEP 1 golden test for issue #345: server MARKDOWN export runs DIRECTLY through + * the canonical converter (`convertProseMirrorToMarkdown`) — no HTML intermediate + * and no `@docmost/editor-ext` markdown layer — so the emitted markdown is in the + * canonical package forms and is byte-identical to the git-sync vault body. + * + * These are the goldens the swap has to satisfy: they assert the CANONICAL + * surface (callout `> [!type]`, inline footnote `^[…]`, lossless image + * ``) rather than the old editor-ext forms (`:::type`, `[^id]`, + * lossy `![alt](src)`). + * + * `exportPage(..., singlePage=false)` takes no DB path (no mention rewriting), so + * the service is constructed with null collaborators and only the pure + * PM -> Markdown path is exercised. + */ + +function makeService(): ExportService { + return new ExportService( + null as any, // pageRepo + null as any, // pagePermissionRepo + null as any, // db + null as any, // storageService + null as any, // environmentService + null as any, // domainService + ); +} + +// A representative page exercising the node types whose canonical markdown form +// changed with the move off the editor-ext layer: callout, inline footnote, and a +// lossless image carrying width/align attrs that the old layer dropped. +const REPRESENTATIVE_DOC = { + type: 'doc', + content: [ + { + type: 'paragraph', + content: [ + { type: 'text', text: 'Body ' }, + { type: 'footnoteReference', attrs: { id: 'fn-1' } }, + { type: 'text', text: ' end.' }, + ], + }, + { + type: 'callout', + attrs: { type: 'info', icon: null }, + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'Heads up' }], + }, + ], + }, + { + type: 'image', + attrs: { + src: '/files/pic.png', + alt: 'Pic', + width: 320, + align: 'left', + }, + }, + { + type: 'footnotesList', + content: [ + { + type: 'footnoteDefinition', + attrs: { id: 'fn-1' }, + content: [ + { + type: 'paragraph', + content: [{ type: 'text', text: 'the note' }], + }, + ], + }, + ], + }, + ], +}; + +describe('ExportService — markdown export via the canonical converter (#345)', () => { + it('emits canonical callout, inline footnote and lossless image forms', async () => { + const service = makeService(); + const md = (await service.exportPage(ExportFormat.Markdown, { + title: '', + content: REPRESENTATIVE_DOC, + } as any)) as string; + + // Callout: Obsidian `> [!type]`, NOT the legacy `:::type`. + expect(md).toContain('> [!info]'); + expect(md).not.toContain(':::'); + + // Inline footnote: `^[…]`, NOT the reference `[^id]` form. + expect(md).toContain('^[the note]'); + expect(md).not.toMatch(/\[\^/); + + // Lossless image: trailing `` carrying the dropped attrs. + expect(md).toContain('![Pic](/files/pic.png)'); + expect(md).toContain('` comment; a callout carries its + // `type`. This asserts those survive the PM->HTML->PM hop — the one hop the + // package's PM<->MD suite does not exercise. + it('preserves image width/align and callout type through the PM->HTML->PM hop', async () => { + const md = [ + '# Doc', + '', + '![a picture](https://example.com/i.png) ', + '', + ':::warning', + 'Careful now.', + ':::', + ].join('\n'); - const importAttachmentService = { - processAttachments: async ({ html }: any) => html, - }; - const backlinkRepo = { insertBacklink: jest.fn() }; - const eventEmitter = { emit: jest.fn() }; - const auditService = { logBatchWithContext: jest.fn() }; + const content = await runZipImport(md); - const pageService = { nextPagePosition: async () => 'a0' }; + const image = findFirst(content, 'image'); + expect(image).toBeTruthy(); + // The lossless sizing/alignment must survive the HTML hop. + expect(String(image.attrs?.width)).toBe('320'); + expect(image.attrs?.align).toBe('left'); - const service = new FileImportTaskService( - {} as any, // storageService - importService as any, - pageService as any, - backlinkRepo as any, - db, - importAttachmentService as any, - eventEmitter as any, - auditService as any, - ); - - const fileTask: any = { - id: 'task-1', - source: 'generic', - spaceId: 'space-1', - workspaceId: 'ws-1', - creatorId: 'user-1', - }; - - try { - await service.processGenericImport({ extractDir, fileTask }); - - expect(captured).toBeTruthy(); - const content = captured.content; - // Reference order is c, a, b (NOT the markdown definition order a, b, c). - expect(footnoteListIds(content)).toEqual(['c', 'a', 'b']); - // Orphan [^z] dropped; reused [^a] collapses to one definition; one list. - expect(footnoteListIds(content)).not.toContain('z'); - const lists = (content.content ?? []).filter( - (n: any) => n.type === 'footnotesList', - ); - expect(lists).toHaveLength(1); - expect(footnoteListIds(content).filter((id) => id === 'a')).toHaveLength(1); - } finally { - await fs.rm(extractDir, { recursive: true, force: true }); - } + const callout = findFirst(content, 'callout'); + expect(callout).toBeTruthy(); + expect(callout.attrs?.type).toBe('warning'); }); }); diff --git a/apps/server/src/integrations/import/services/file-import-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts index 5ec2fe8d..a5115c43 100644 --- a/apps/server/src/integrations/import/services/file-import-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -1,6 +1,9 @@ import { Inject, Injectable, Logger } from '@nestjs/common'; import * as path from 'path'; -import { jsonToText } from '../../../collaboration/collaboration.util'; +import { + jsonToHtml, + jsonToText, +} from '../../../collaboration/collaboration.util'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB } from '@docmost/db/types/kysely.types'; import { @@ -18,9 +21,11 @@ import { generateSlugId } from '../../../common/helpers'; import { v7 } from 'uuid'; import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; import { FileTask, InsertablePage } from '@docmost/db/types/entity.types'; -import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext'; +import { canonicalizeFootnotes } from '@docmost/editor-ext'; +import { markdownToProseMirror } from '@docmost/prosemirror-markdown'; import { getProsemirrorContent } from '../../../common/helpers/prosemirror/utils'; import { formatImportHtml } from '../utils/import-formatter'; +import { normalizeForeignMarkdown } from '../utils/foreign-markdown'; import { buildAttachmentCandidates, collectMarkdownAndHtmlFiles, @@ -461,7 +466,18 @@ export class FileImportTaskService { content = await fs.readFile(absPath, 'utf-8'); if (page.fileExtension.toLowerCase() === '.md') { - content = await markdownToHtml(content); + // Parse markdown with the single canonical converter + // (`@docmost/prosemirror-markdown`), after normalizing foreign + // reference footnotes, then serialize to HTML so the shared HTML + // pipeline below (processAttachments + formatImportHtml + + // processHTML) keeps handling `.md` and `.html` imports + // uniformly. The markdown PARSE no longer goes through the + // editor-ext markdown layer (issue #345) — the drift source is + // gone. The PM -> HTML -> PM hop that follows is lossless + // plumbing for attachment/link resolution, NOT a second parse. + content = jsonToHtml( + await markdownToProseMirror(normalizeForeignMarkdown(content)), + ); } } catch (err: any) { if (err?.code === 'ENOENT') { @@ -500,10 +516,12 @@ export class FileImportTaskService { this.importService.extractTitleAndRemoveHeading(pmState); // Canonicalize footnote topology on this non-editor write path - // (markdownToHtml/processHTML never runs footnoteSyncPlugin), so a - // zip-imported page's footnotes are reference-ordered, deduped, and + // (the HTML pipeline's processHTML never runs footnoteSyncPlugin), so + // a zip-imported page's footnotes are reference-ordered, deduped, and // orphan-free like the editor's invariant (issue #228). Pure + - // idempotent + shape-safe; a footnote-free doc is unchanged. + // idempotent + shape-safe; a footnote-free doc is unchanged. (For a + // `.md` file the package parser already yields canonical footnotes, + // so this is a no-op there.) // (Future consolidation, architecture B: like import.service, this // path persists directly rather than via PageService — a shared // "prepare JSON for persist" helper would centralize this call.) diff --git a/apps/server/src/integrations/import/services/import.service.footnote-canonicalize.spec.ts b/apps/server/src/integrations/import/services/import.service.footnote-canonicalize.spec.ts index e53b17a1..40972b10 100644 --- a/apps/server/src/integrations/import/services/import.service.footnote-canonicalize.spec.ts +++ b/apps/server/src/integrations/import/services/import.service.footnote-canonicalize.spec.ts @@ -12,13 +12,19 @@ import { canonicalizeFootnotes } from '@docmost/editor-ext'; /** * Integration-ish test for the USER-FACING markdown import path - * (`ImportService.importPage`). It exercises the REAL markdown -> HTML -> JSON - * conversion and asserts that the stored page content has its footnotes - * canonicalized — the gap that issue #228 fixes: the import path builds - * ProseMirror JSON directly (never running the editor's footnoteSyncPlugin), so - * before this wiring the stored footnotes kept the markdown's physical - * definition order (out of order vs. references), retained orphan definitions, - * and did not collapse reused references. + * (`ImportService.importPage`). It exercises the REAL markdown -> ProseMirror + * conversion and asserts the stored page's footnotes are canonical: ordered by + * FIRST REFERENCE (not markdown definition order), reused references deduped to a + * single definition, and orphan definitions dropped. + * + * Since #345 the markdown parse runs through the canonical package + * (`normalizeForeignMarkdown` -> `markdownToProseMirror`), which owns this + * canonicalization: the input's GFM `[^id]` reference footnotes are normalized to + * inline `^[…]`, and the parser assigns fresh sequential ids (`fn-*`) in + * reference order while merging identical bodies — so we assert by definition + * BODY order, not by the source labels. `canonicalizeFootnotes` remains wired as + * an idempotent safety net (issue #228) and is a no-op on this already-canonical + * output. * * The DB/ydoc side-effects are stubbed: `getNewPagePosition` (DB query) and * `createYdoc` (Yjs encode) are spied, and `pageRepo.insertPage` captures the @@ -67,24 +73,14 @@ function makeService() { } /** List the footnote-definition ids of the (single) footnotesList, in order. */ -function footnoteListIds(content: any): string[] { +/** Definition body texts of the (single) footnotesList, in list order. */ +function footnoteListBodies(content: any): string[] { const list = (content.content ?? []).find( (n: any) => n.type === 'footnotesList', ); - if (!list) return []; - return (list.content ?? []) + return (list?.content ?? []) .filter((n: any) => n.type === 'footnoteDefinition') - .map((n: any) => n.attrs?.id); -} - -function definitionText(content: any, id: string): string | undefined { - const list = (content.content ?? []).find( - (n: any) => n.type === 'footnotesList', - ); - const def = (list?.content ?? []).find( - (n: any) => n.type === 'footnoteDefinition' && n.attrs?.id === id, - ); - return def?.content?.[0]?.content?.[0]?.text; + .map((n: any) => n.content?.[0]?.content?.[0]?.text); } describe('ImportService.importPage — footnote canonicalization (#228)', () => { @@ -101,23 +97,23 @@ describe('ImportService.importPage — footnote canonicalization (#228)', () => const content = getCaptured().content; expect(content).toBeTruthy(); - // Reference order is c, a, b (NOT the markdown definition order a, b, c). - expect(footnoteListIds(content)).toEqual(['c', 'a', 'b']); - - // Definitions preserved and attached to the right ids. - expect(definitionText(content, 'c')).toBe('note C'); - expect(definitionText(content, 'a')).toBe('note A'); - expect(definitionText(content, 'b')).toBe('note B'); + // Definitions ordered by FIRST REFERENCE (C, A, B) — NOT the markdown + // definition order (A, B, C) — with the orphan [^z] dropped and the reused + // [^a] collapsed to a single definition. (Ids are the parser's fresh `fn-*`, + // so we pin the BODIES.) + expect(footnoteListBodies(content)).toEqual(['note C', 'note A', 'note B']); // Orphan definition [^z] is dropped. - expect(footnoteListIds(content)).not.toContain('z'); + expect(footnoteListBodies(content)).not.toContain('orphan note'); // Reused [^a] yields exactly ONE definition, and exactly one list. const lists = (content.content ?? []).filter( (n: any) => n.type === 'footnotesList', ); expect(lists).toHaveLength(1); - expect(footnoteListIds(content).filter((id) => id === 'a')).toHaveLength(1); + expect( + footnoteListBodies(content).filter((b) => b === 'note A'), + ).toHaveLength(1); }); it('is idempotent: canonicalizing the stored output again is a no-op', async () => { @@ -134,6 +130,6 @@ describe('ImportService.importPage — footnote canonicalization (#228)', () => // time must not change it (safe to wire into every write path). const second = canonicalizeFootnotes(stored); expect(second).toEqual(stored); - expect(footnoteListIds(second)).toEqual(['c', 'a', 'b']); + expect(footnoteListBodies(second)).toEqual(['note C', 'note A', 'note B']); }); }); diff --git a/apps/server/src/integrations/import/services/import.service.ts b/apps/server/src/integrations/import/services/import.service.ts index 75418e55..dd86d71e 100644 --- a/apps/server/src/integrations/import/services/import.service.ts +++ b/apps/server/src/integrations/import/services/import.service.ts @@ -17,7 +17,9 @@ import { import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; import { TiptapTransformer } from '@hocuspocus/transformer'; import * as Y from 'yjs'; -import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext'; +import { canonicalizeFootnotes } from '@docmost/editor-ext'; +import { markdownToProseMirror } from '@docmost/prosemirror-markdown'; +import { normalizeForeignMarkdown } from '../utils/foreign-markdown'; import { FileTaskStatus, FileTaskType, @@ -85,11 +87,13 @@ export class ImportService { const extracted = this.extractTitleAndRemoveHeading(prosemirrorState); const title = extracted.title; - // Imported markdown/HTML is built via markdownToHtml -> htmlToJson, which - // never runs the editor's footnoteSyncPlugin, so the footnote topology keeps - // the source's PHYSICAL definition order (out of order vs. references), - // retains orphan definitions, and is not deduped. Canonicalize before - // persisting so the stored page matches the editor's invariant (issue #228). + // The markdown path now canonicalizes footnotes itself (the package parser), + // but the HTML path (processHTML -> htmlToJson) does NOT run the editor's + // footnoteSyncPlugin, so an imported HTML doc can keep its source's PHYSICAL + // definition order (out of order vs. references), retain orphan definitions, + // and not be deduped. Canonicalize before persisting so the stored page + // matches the editor's invariant (issue #228); it is an idempotent no-op on + // the already-canonical markdown output. // Pure + idempotent + shape-safe: a doc with no footnotes is unchanged. // (Future consolidation, architecture B: this import path persists directly // via pageRepo.insertPage rather than through PageService.createPage, so the @@ -133,12 +137,15 @@ export class ImportService { } async processMarkdown(markdownInput: string): Promise { - try { - const html = await markdownToHtml(markdownInput); - return this.processHTML(html); - } catch (err) { - throw err; - } + // Canonical markdown -> ProseMirror JSON directly via + // `@docmost/prosemirror-markdown` (issue #345) — no HTML intermediate and no + // second editor-ext markdown layer. Foreign markdown surfaces the strict + // canonical parser does not accept (GFM `[^id]` reference footnotes) are + // rewritten to the canonical inline form by `normalizeForeignMarkdown` first. + // The HTML-cleanup pass (`normalizeImportHtml`) is intentionally skipped here: + // it targets foreign *HTML* (Notion/XWiki), which only ever arrives on the + // `.html` path (`processHTML`), never as canonical markdown. + return markdownToProseMirror(normalizeForeignMarkdown(markdownInput)); } async processHTML(htmlInput: string): Promise { diff --git a/apps/server/src/integrations/import/utils/foreign-markdown.spec.ts b/apps/server/src/integrations/import/utils/foreign-markdown.spec.ts new file mode 100644 index 00000000..8a43fa20 --- /dev/null +++ b/apps/server/src/integrations/import/utils/foreign-markdown.spec.ts @@ -0,0 +1,218 @@ +import { + convertProseMirrorToMarkdown, + markdownToProseMirror, +} from '@docmost/prosemirror-markdown'; +import { normalizeForeignMarkdown } from './foreign-markdown'; + +/** + * STEP 2 goldens for issue #345: the foreign-markdown normalizer that runs at the + * import boundary BEFORE the strict canonical parser (`markdownToProseMirror`). + * + * Two layers: + * 1. PURE string→string cases pinning the normalizer's own behavior (GFM + * reference footnotes → inline `^[…]`). + * 2. END-TO-END acceptance: for a foreign corpus, `normalizeForeignMarkdown` + * then `markdownToProseMirror` then `convertProseMirrorToMarkdown` must leave + * NO literal `[^id]` / `:::` garbage in the document and must re-export in the + * canonical forms. + */ + +describe('normalizeForeignMarkdown — GFM reference footnotes', () => { + it('inlines a single-line reference footnote and drops its definition', () => { + const out = normalizeForeignMarkdown( + 'A note[^1] here.\n\n[^1]: The definition.', + ); + expect(out).toBe('A note^[The definition.] here.\n'); + }); + + it('inlines every reference to a reused id (downstream dedups)', () => { + const out = normalizeForeignMarkdown( + 'X[^a] and Y[^a].\n\n[^a]: shared.', + ); + expect(out).toBe('X^[shared.] and Y^[shared.].\n'); + }); + + it('joins indented continuation lines of a definition with a space', () => { + const out = normalizeForeignMarkdown( + 'See[^n].\n\n[^n]: line one\n line two', + ); + expect(out).toBe('See^[line one line two].\n'); + }); + + it('never rewrites a reference inside a fenced code block', () => { + const out = normalizeForeignMarkdown( + '```\ncode[^1] here\n```\n\n[^1]: def.', + ); + expect(out).toContain('code[^1] here'); + // The (now orphaned) definition line is still removed. + expect(out).not.toContain('[^1]: def.'); + }); + + it('never rewrites a reference inside an INLINE-code span (backticks)', () => { + // The `[^1]` inside backticks is literal code and must survive verbatim; + // the one outside is rewritten. (Bug #1: only fenced blocks were protected.) + const out = normalizeForeignMarkdown( + 'Use `arr[^1]` in code but note[^1] in prose.\n\n[^1]: def.', + ); + expect(out).toBe('Use `arr[^1]` in code but note^[def.] in prose.\n'); + }); + + it('escapes brackets in a body so an unbalanced ] cannot truncate the footnote', () => { + // A foreign definition body with a stray `]` would, unescaped, close the + // canonical `^[...]` early and leak the tail as text (bug #2). The body's + // brackets are backslash-escaped so the footnote stays whole. + const out = normalizeForeignMarkdown( + 'Ref[^1] here.\n\n[^1]: see item ] and [more] later', + ); + expect(out).toBe('Ref^[see item \\] and \\[more\\] later] here.\n'); + // The tokenizer must see exactly one unescaped closing bracket (our own). + expect(out.match(/(? { + const out = normalizeForeignMarkdown('Dangling[^x] ref.'); + expect(out).toBe('Dangling[^x] ref.'); + }); + + it('returns the input unchanged when there are no reference footnotes', () => { + const md = '# Title\n\nJust text with `inline code` and a [link](/x).'; + expect(normalizeForeignMarkdown(md)).toBe(md); + }); + + it('does NOT touch callout surfaces — the canonical parser handles them', () => { + const callouts = ':::info\nHi\n:::\n\n> [!warning]\n> Careful'; + expect(normalizeForeignMarkdown(callouts)).toBe(callouts); + }); + + it('strips a leading YAML front-matter block (Obsidian/Hugo/git-sync files)', () => { + const out = normalizeForeignMarkdown( + '---\ntitle: My Page\ntags: [a, b]\n---\n\n# Heading\n\nBody.', + ); + expect(out).toBe('# Heading\n\nBody.'); + // The front-matter must not leak into the body as a setext heading. + expect(out).not.toContain('title: My Page'); + expect(out).not.toContain('---'); + }); + + it('does not strip a horizontal rule that is not leading front-matter', () => { + const md = 'Intro paragraph.\n\n---\n\nAfter the rule.'; + expect(normalizeForeignMarkdown(md)).toBe(md); + }); + + it('is linear on a document with thousands of definitions (no quadratic blowup)', () => { + // F2(a): the pass-2 rewrite must be O(text), not O(text × defs). Build a + // pathological doc (many defs + many plain text lines) and assert it + // completes well under a second — a quadratic implementation took ~14s. + const N = 4000; + const refs = Array.from({ length: N }, (_, i) => `line ${i} plain text`).join('\n'); + const defs = Array.from({ length: N }, (_, i) => `[^n${i}]: def ${i}`).join('\n'); + const doc = `start[^n0] and[^n${N - 1}] end\n\n${refs}\n\n${defs}`; + const t0 = Date.now(); + const out = normalizeForeignMarkdown(doc); + const elapsed = Date.now() - t0; + expect(elapsed).toBeLessThan(2000); + // Sanity: the two real references were still inlined. + expect(out).toContain('^[def 0]'); + expect(out).toContain(`^[def ${N - 1}]`); + }); + + it('is bounded on a long unclosed backtick run (no inline-split ReDoS)', () => { + // F2(b): a huge unterminated backtick run must not cause quadratic + // backtracking in the inline-code split. Oversized lines skip the split + // entirely (left untouched), so this returns promptly. + const line = 'x' + '`'.repeat(200000); + const doc = `${line}\n\n[^1]: def`; + const t0 = Date.now(); + normalizeForeignMarkdown(doc); + expect(Date.now() - t0).toBeLessThan(2000); + }); + + it('does not crash or slow down on thousands of prefix-chain definition ids', () => { + // F7: the rewrite must use a FIXED generic scanner, not an alternation built + // from the ids. A `(a|aa|aaa|…)` alternation over prefix-chain ids blows the + // V8 regex compiler (FATAL RegExpCompiler Allocation failed — uncatchable, + // kills the process). A fixed scanner has no id-dependent compilation cost. + const N = 4000; + const ids = Array.from({ length: N }, (_, i) => 'a'.repeat(i + 1)); + const defs = ids.map((id) => `[^${id}]: body ${id.length}`).join('\n'); + const doc = `ref[^${ids[0]}] and[^${ids[N - 1]}] end\n\n${defs}`; + const t0 = Date.now(); + const out = normalizeForeignMarkdown(doc); + expect(Date.now() - t0).toBeLessThan(2000); + // Prefix disambiguation is correct: [^a] and [^aaaa...] inline their OWN body. + expect(out).toContain('^[body 1]'); + expect(out).toContain(`^[body ${N}]`); + }); + + it('strips a CRLF (Windows) front-matter block, not just LF', () => { + // F9: the line-anchored regex needs LF after the opening `---`, so a Windows + // file (`---\r\n…`) would slip past the strip and leak the front-matter into + // the body. normalizeForeignMarkdown normalizes CRLF -> LF first. + const out = normalizeForeignMarkdown( + '---\r\ntitle: Foo\r\ntags: [a]\r\n---\r\n\r\n# Heading\r\n\r\nBody.', + ); + expect(out).toBe('# Heading\n\nBody.'); + expect(out).not.toContain('title: Foo'); + expect(out).not.toContain('---'); + }); + + it('strips front-matter whose value contains a triple-dash (line-anchored)', () => { + // F8: the block must close only on a `\n---` LINE, not the first inline + // `---`. A value like `title: Q1 --- Q2` must not truncate the front-matter + // and leak the rest (author/closing ---) into the body. + const out = normalizeForeignMarkdown( + '---\ntitle: Q1 --- Q2 results\nauthor: bob\n---\n\nReal body.', + ); + expect(out).toBe('Real body.'); + expect(out).not.toContain('author: bob'); + expect(out).not.toContain('Q2 results'); + }); +}); + +describe('foreign markdown import acceptance (normalizer + canonical parser)', () => { + const FOREIGN = [ + '# Doc', + '', + 'Body refs [^c] and [^a] and [^b] and again [^a].', + '', + ':::info', + 'A legacy callout.', + ':::', + '', + '| h1 | h2 |', + '| --- | --- |', + '| 1 | 2 |', + '', + '[^a]: note A', + '[^b]: note B', + '[^c]: note C', + '[^z]: orphan note', + ].join('\n'); + + it('leaves no literal [^id] or ::: in the imported doc and re-exports canonically', async () => { + const normalized = normalizeForeignMarkdown(FOREIGN); + const doc = await markdownToProseMirror(normalized); + const reexport = convertProseMirrorToMarkdown(doc); + + // No foreign garbage leaks into the document. + expect(reexport).not.toMatch(/\[\^/); // no reference footnote refs/defs + expect(reexport).not.toContain(':::'); // no legacy callout fences + + // Canonical forms are present. + expect(reexport).toContain('^[note C]'); + expect(reexport).toContain('> [!info]'); + expect(reexport).toContain('| h1 | h2 |'); + + // Footnotes: ordered by first reference (C, A, B), reused [^a] deduped to one, + // orphan [^z] dropped (it had no reference after normalization). + const list = doc.content.find((n: any) => n.type === 'footnotesList'); + const bodies = list.content.map( + (d: any) => d.content[0].content[0].text, + ); + expect(bodies).toEqual(['note C', 'note A', 'note B']); + expect(bodies).not.toContain('orphan note'); + expect( + doc.content.filter((n: any) => n.type === 'footnotesList'), + ).toHaveLength(1); + }); +}); diff --git a/apps/server/src/integrations/import/utils/foreign-markdown.ts b/apps/server/src/integrations/import/utils/foreign-markdown.ts new file mode 100644 index 00000000..2173bfdc --- /dev/null +++ b/apps/server/src/integrations/import/utils/foreign-markdown.ts @@ -0,0 +1,265 @@ +/** + * Foreign-markdown normalizer — an input-liberal / output-canonical adapter that + * runs at the IMPORT boundary, BEFORE the canonical parser + * (`markdownToProseMirror` from `@docmost/prosemirror-markdown`). + * + * The canonical parser is deliberately STRICT: it only understands Docmost's + * canonical markdown surface (Obsidian-style `> [!type]` callouts, Pandoc/Obsidian + * inline footnotes `^[body]`, lossless `![alt](src) ` images, …). + * Import, however, ingests FOREIGN files (GitHub/GFM, Notion, old Docmost + * exports). Those use surfaces the canonical parser does not accept, most notably + * GitHub-flavoured *reference* footnotes: + * + * Text with a note[^1] and another[^long]. + * + * [^1]: The first definition. + * [^long]: A second one. + * + * Left untouched, the parser does NOT recognise `[^id]` (it only parses `^[body]`), + * so the reference leaks as literal text — and worse, the trailing `[^id]: def` + * line is a valid CommonMark *link-reference definition*, so `[^id]` is silently + * rendered as a bogus link. This normalizer rewrites reference footnotes into the + * canonical inline form so the parser materialises real footnote nodes. + * + * This is a TEXT pre-pass, NOT a second parser fork: it does not re-implement any + * converter logic. Callout surfaces (`:::type` and `> [!type]`) are intentionally + * NOT touched here — the canonical parser already accepts BOTH natively (its + * `preprocessCallouts` pass), so normalizing them would be redundant and would + * only risk degrading the parser's nesting/code-fence-aware handling. + */ + +/** Matches a fenced code block delimiter (``` or ~~~), capturing the marker run. */ +const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/; + +/** + * Matches a GFM footnote DEFINITION line: `[^id]: body`. The id is any run of + * non-`]` characters; the body is the remainder of the line (possibly empty). + */ +const FOOTNOTE_DEF_RE = /^\[\^([^\]]+)\]:[ \t]?(.*)$/; + +/** True when a line is a code-fence delimiter that toggles fenced-code state. */ +function fenceMarker(line: string): string | null { + const m = line.match(CODE_FENCE_RE); + return m ? m[2] : null; +} + +/** True when a line is indented (leading space/tab) and not blank — a continuation. */ +function isIndentedContinuation(line: string): boolean { + return /^[ \t]+\S/.test(line); +} + +function escapeRegExp(value: string): string { + return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** + * Backslash-escape any square bracket in a footnote body before it is wrapped in + * `^[...]`. The canonical inline-footnote tokenizer scans the body with bracket + * balancing and closes on the first UNMATCHED `]`, so an unbalanced bracket in a + * foreign definition (e.g. `[^1]: see item ] later`) would otherwise truncate the + * footnote and leak the tail as literal text. Escaping every `[`/`]` makes the + * body an inert run of characters — the tokenizer then closes only on our own + * closing `]`. (A balanced `[link](url)` inside a body still round-trips because + * the escaped form renders the literal brackets, which is the safe reading for a + * footnote body; the alternative — brittle balance tracking — risks worse.) + */ +function escapeFootnoteBody(body: string): string { + return body.replace(/[[\]]/g, '\\$&'); +} + +/** + * Rewrite every `[^id]` reference on a line to its `^[body]` form, but ONLY in the + * text OUTSIDE inline-code spans. A `[^id]` inside backticks is literal code + * content and must be preserved verbatim (a footnote ref never lives inside code). + * We split the line on inline-code spans (paired backtick runs) and rewrite only + * the non-code segments. + */ +// Above this length a single line is not split into inline-code spans (see +// below). A genuine markdown line carrying a footnote reference is never tens of +// KB; the cap only bypasses the inline-code protection for pathological lines. +const INLINE_SPLIT_MAX_LINE = 8192; + +function rewriteRefsOutsideInlineCode( + line: string, + replace: (text: string) => string, +): string { + // The inline-code split alternation `(`+)(?:(?!\1)[\s\S])*\1` backtracks + // quadratically on a long UNCLOSED backtick run (its middle can consume the + // rest of the line, then fail to find a closing run and retry from each + // position). On an untrusted import this is a request-thread ReDoS. A real + // footnote line is short, so for an oversized line we skip the inline-code + // protection entirely and leave the line UNTOUCHED (rewriting it wholesale + // could corrupt a `[^id]` that legitimately lives inside inline code). This is + // a conservative bypass: an over-8KB line simply does not get its reference + // footnotes inlined — acceptable for a pathological input. + if (line.length > INLINE_SPLIT_MAX_LINE) return line; + + // Alternation: an inline-code span (one or more backticks, then anything up to + // the SAME run of backticks) OR a run of non-backtick text. Unterminated + // backticks fall through as ordinary text (matched by the second branch on the + // leftover), so a stray backtick never swallows the rest of the line. + const parts = line.match(/(`+)(?:(?!\1)[\s\S])*\1|[^`]+|`+/g); + if (!parts) return line; + return parts + .map((seg) => (seg.startsWith('`') ? seg : replace(seg))) + .join(''); +} + +/** + * Convert GFM reference footnotes (`[^id]` + `[^id]: def`) into canonical inline + * footnotes (`^[def]`). + * + * - Definitions are collected first (a leading `[^id]: text` line plus any + * immediately-following indented continuation lines, joined with a space) and + * removed from the output. + * - Each in-text reference `[^id]` for which a definition was found is replaced by + * `^[def]`. References with no matching definition are left literal (there is no + * body to inline; the parser fails them open the same way). + * - Code is respected on both passes: `[^id]` inside a fenced ``` / ~~~ block is + * never rewritten and a `[^id]:` line inside a fence is never a definition; and + * on the rewrite pass a `[^id]` inside an INLINE-code span (backticks) is left + * literal too. + * - The inlined body is bracket-escaped so an unbalanced `[`/`]` in a foreign + * definition cannot truncate the resulting `^[...]` footnote. + * + * Deduplication / reference-ordering / orphan-dropping of the resulting footnotes + * is handled downstream by the canonical parser (`assembleFootnotes`); this pass + * only changes the surface syntax. + */ +function convertReferenceFootnotes(markdown: string): string { + const lines = markdown.split('\n'); + + // Pass 1: collect definitions and mark their lines for removal. + const defs = new Map(); + const dropped = new Array(lines.length).fill(false); + let inFence = false; + let fence = ''; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const marker = fenceMarker(line); + if (inFence) { + if (marker && marker[0] === fence[0] && marker.length >= fence.length) { + inFence = false; + fence = ''; + } + continue; + } + if (marker) { + inFence = true; + fence = marker; + continue; + } + + const def = line.match(FOOTNOTE_DEF_RE); + if (!def) continue; + + const id = def[1]; + const body: string[] = [def[2].trim()]; + dropped[i] = true; + + // Consume immediately-following indented continuation lines (GFM lazy + // continuation is not supported by design — keep it simple and predictable). + let j = i + 1; + while (j < lines.length && isIndentedContinuation(lines[j])) { + body.push(lines[j].trim()); + dropped[j] = true; + j++; + } + i = j - 1; + + // Last definition wins for a duplicated id (matches CommonMark link-ref + // semantics closely enough for a foreign-input adapter). + defs.set(id, body.filter((s) => s.length > 0).join(' ')); + } + + if (defs.size === 0) { + return markdown; + } + + // ONE fixed, generic scanner regex — NOT one built from the definition ids. + // It matches ANY `[^id]` shape, and the replacer decides per match via a map + // lookup whether that id is a real definition (replace) or not (leave as-is). + // This is genuinely O(total text) with no per-document regex compilation. + // + // Do NOT rebuild this as an alternation over `[...defs.keys()]`: a giant + // `(id1|id2|...)` alternation over thousands of ids can blow the V8 regex + // compiler's stack — a fatal, UNCATCHABLE "RegExpCompiler Allocation failed" + // on prefix-chain ids (`a`, `aa`, `aaa`, ...) that kills the whole process + // (worse than the earlier per-def thread-hang). A fixed scanner has no + // id-dependent compilation cost and cannot blow up. + const refRe = /\[\^([^\]]+)\]/g; + const rewriteSegment = (segment: string): string => + segment.replace(refRe, (whole, id: string) => { + const body = defs.get(id); + // Only real definitions are inlined; an unknown id is left literal (same as + // the old per-def loop, which simply never matched it). + return body === undefined ? whole : `^[${escapeFootnoteBody(body)}]`; + }); + + // Pass 2: rewrite in-text references, skipping fenced code and dropped lines. + const out: string[] = []; + inFence = false; + fence = ''; + for (let i = 0; i < lines.length; i++) { + if (dropped[i]) continue; + let line = lines[i]; + + const marker = fenceMarker(line); + if (inFence) { + out.push(line); + if (marker && marker[0] === fence[0] && marker.length >= fence.length) { + inFence = false; + fence = ''; + } + continue; + } + if (marker) { + inFence = true; + fence = marker; + out.push(line); + continue; + } + + line = rewriteRefsOutsideInlineCode(line, rewriteSegment); + out.push(line); + } + + return out.join('\n'); +} + +/** + * Strip a single leading YAML front-matter block (`---\n…\n---`). Foreign files + * from Obsidian / Hugo / Jekyll / Notion — and Docmost's OWN git-sync page files + * — open with front-matter that the canonical parser does not consume, so + * without this it leaks into the body (and `title: Foo` above the closing `---` + * renders as a setext `

` that `extractTitleAndRemoveHeading` can hijack as + * the page title). It is a no-op for front-matter-free input. + * + * LINE-ANCHORED (the same shape the canonical parser uses in + * prosemirror-markdown/page-file.ts): the block opens only on `---\n` at the + * very start and closes only on a `\n---` line. The retired `markdownToHtml` + * strip closed on the FIRST `---` ANYWHERE (an unanchored close), so a value + * containing a triple-dash (e.g. `title: Q1 --- Q2`) truncated the front-matter + * and leaked the rest into the body. An optional leading BOM is tolerated. + */ +const YAML_FRONT_MATTER_RE = /^\uFEFF?---\n[\s\S]*?\n---\n?/; + +/** + * Normalize a foreign markdown string into Docmost's canonical markdown surface + * so the strict canonical parser accepts it losslessly: normalize line endings, + * strip a leading YAML front-matter block, then rewrite GFM reference footnotes + * into inline footnotes. Add further fixture-driven foreign-surface cases here as + * they are found. + */ +export function normalizeForeignMarkdown(markdown: string): string { + if (!markdown) return markdown; + // Normalize CRLF -> LF FIRST. The line-anchored front-matter regex requires a + // bare `\n` after the opening `---`, and convertReferenceFootnotes splits on + // `\n`; a Windows/CRLF foreign file (`---\r\n…`) would otherwise slip past the + // front-matter strip and leak into the body. The canonical parser + // (page-file.ts parsePageFile) normalizes the same way before its FRONTMATTER_RE. + const src = markdown.replace(/\r\n/g, '\n'); + const withoutFrontMatter = src.replace(YAML_FRONT_MATTER_RE, '').trimStart(); + return convertReferenceFootnotes(withoutFrontMatter); +} diff --git a/apps/server/test/stubs/tiptap-react.js b/apps/server/test/stubs/tiptap-react.js new file mode 100644 index 00000000..edd98738 --- /dev/null +++ b/apps/server/test/stubs/tiptap-react.js @@ -0,0 +1,23 @@ +// Jest stub for @tiptap/react. +// +// The server export/import code paths transitively import editor-ext, whose node +// extensions import from `@tiptap/react`. The real module re-exports all of +// `@tiptap/core` (headless, safe under node) AND adds React view helpers +// (`ReactNodeViewRenderer`, …) that eagerly pull in react-dom — which throws +// `navigator is not defined` under jest's node environment. +// +// So this stub DELEGATES to the real `@tiptap/core` (keeping `mergeAttributes`, +// `Node`, `Mark`, `nodeInputRule`, … working — they are used by +// `jsonToHtml`/`htmlToJson` on the server) and overrides ONLY the React view +// helpers with no-ops. Those helpers are referenced solely inside `addNodeView()` +// — code that runs only in a live browser editor, never on the server; if any +// were actually invoked here it would (correctly) surface as a test failure. +const core = require('@tiptap/core'); + +module.exports = { + ...core, + ReactNodeViewRenderer: () => () => ({}), + NodeViewWrapper: () => null, + NodeViewContent: () => null, + ReactRenderer: class {}, +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0d99a612..f70155a5 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -543,6 +543,9 @@ importers: '@docmost/pdf-inspector': specifier: 1.9.6 version: 1.9.6 + '@docmost/prosemirror-markdown': + specifier: workspace:* + version: link:../../packages/prosemirror-markdown '@fastify/cookie': specifier: ^11.0.2 version: 11.0.2