refactor(#345): серверный экспорт/импорт markdown через @docmost/prosemirror-markdown #369
@@ -201,7 +201,7 @@ pnpm workspace (`pnpm@10.4.0`) orchestrated by **Nx**. Four workspace packages:
|
||||
| `apps/client` | `client` | React 18 + Vite + Mantine 8 + TanStack Query + Jotai | SPA frontend |
|
||||
| `packages/editor-ext` | `@docmost/editor-ext` | Tiptap/ProseMirror | Shared Tiptap node/mark extensions, imported by both the client and the server |
|
||||
| `packages/mcp` | `@docmost/mcp` | MCP SDK, Tiptap, Yjs | Standalone MCP server, also bundled into the server at `/mcp`. Consumes the shared converter/schema from `@docmost/prosemirror-markdown` (#293) — it no longer carries its own vendored converter/schema copy |
|
||||
| `packages/prosemirror-markdown` | `@docmost/prosemirror-markdown` | Tiptap, marked, jsdom | The single, canonical ProseMirror↔Markdown converter + Docmost schema mirror (#293). Consumed by `mcp` and `git-sync`; there is exactly ONE copy of the converter now |
|
||||
| `packages/prosemirror-markdown` | `@docmost/prosemirror-markdown` | Tiptap, marked, jsdom | The single, canonical ProseMirror↔Markdown converter + Docmost schema mirror (#293). Consumed by `mcp`, `git-sync`, AND `apps/server` (server-side markdown import/export, #345); there is exactly ONE copy of the converter now |
|
||||
|
||||
`build` targets are Nx-cached and dependency-ordered (`dependsOn: ["^build"]`), so `editor-ext` builds before the apps. `nx.json` sets `affected.defaultBase: main`.
|
||||
|
||||
@@ -284,7 +284,7 @@ The API server is a Fastify app with a global `/api` prefix (`main.ts` excludes
|
||||
### Client structure
|
||||
Vite SPA. Code is organized by feature under `apps/client/src/features/*` (mirrors the server domains: `page`, `space`, `comment`, `ai-chat`, `editor`, …). Conventions:
|
||||
- **TanStack Query** for server state (one `queries/` file per feature), **Jotai** atoms for local/shared UI state, **Mantine 8** + CSS modules (`*.module.css`) + `postcss-preset-mantine` for UI.
|
||||
- The editor is Tiptap; shared node/mark extensions live in `packages/editor-ext` and are imported by **both the client and the server** (collaboration, import/export) — editor schema changes often need to be made in `editor-ext`, not just the client. The ProseMirror↔Markdown converter and its Docmost schema mirror now live in a SINGLE package, `@docmost/prosemirror-markdown` (#293), consumed by both `mcp` and `git-sync` — do NOT reintroduce a per-package copy. `editor-ext` is the upstream source of the Tiptap schema; the package's `docmost-schema.ts` mirrors it and a serializer-contract test (`packages/prosemirror-markdown/test/serializer-contract.test.ts`) guards the boundary (every schema node must have a converter case), so a drift surfaces as a failing test rather than silent divergence.
|
||||
- The editor is Tiptap; shared node/mark extensions live in `packages/editor-ext` and are imported by **both the client and the server** (collaboration, schema, `canonicalizeFootnotes`) — editor schema changes often need to be made in `editor-ext`, not just the client. Server-side markdown import/export no longer lives in `editor-ext`: it goes through the canonical converter (#345, see below). The ProseMirror↔Markdown converter and its Docmost schema mirror now live in a SINGLE package, `@docmost/prosemirror-markdown` (#293), consumed by `mcp`, `git-sync`, and `apps/server` (#345) — do NOT reintroduce a per-package copy. `editor-ext` is the upstream source of the Tiptap schema; the package's `docmost-schema.ts` mirrors it and a serializer-contract test (`packages/prosemirror-markdown/test/serializer-contract.test.ts`) guards the boundary (every schema node must have a converter case), so a drift surfaces as a failing test rather than silent divergence.
|
||||
- API access goes through `apps/client/src/lib/api-client.ts` (axios). The `@` alias maps to `apps/client/src`.
|
||||
- Runtime config is injected at build time by `vite.config.ts` via `define` (`APP_URL`, `COLLAB_URL`, `APP_VERSION`, …) — these come from the root `.env`, not from `import.meta.env`.
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"migration:reset": "tsx src/database/migrate.ts down-to NO_MIGRATIONS",
|
||||
"migration:codegen": "kysely-codegen --dialect=postgres --camel-case --env-file=../../.env --out-file=./src/database/types/db.d.ts",
|
||||
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
||||
"pretest": "pnpm --filter @docmost/editor-ext build",
|
||||
"pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build",
|
||||
"test": "jest",
|
||||
"test:int": "jest --config test/jest-integration.json",
|
||||
"test:watch": "jest --watch",
|
||||
@@ -43,6 +43,7 @@
|
||||
"@clickhouse/client": "^1.18.2",
|
||||
"@docmost/mcp": "workspace:*",
|
||||
"@docmost/pdf-inspector": "1.9.6",
|
||||
"@docmost/prosemirror-markdown": "workspace:*",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/multipart": "^10.0.0",
|
||||
"@fastify/static": "^9.1.3",
|
||||
@@ -175,7 +176,7 @@
|
||||
"/node_modules/"
|
||||
],
|
||||
"transform": {
|
||||
"happy-dom.+\\.js$": [
|
||||
"(happy-dom.+|prosemirror-markdown/build/.+)\\.js$": [
|
||||
"babel-jest",
|
||||
{
|
||||
"presets": [
|
||||
@@ -193,7 +194,7 @@
|
||||
"^.+\\.(t|j)sx?$": "ts-jest"
|
||||
},
|
||||
"transformIgnorePatterns": [
|
||||
"/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom|lib0)(@|/))"
|
||||
"/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom|lib0|@docmost/prosemirror-markdown)(@|/))"
|
||||
],
|
||||
"collectCoverageFrom": [
|
||||
"**/*.(t|j)s"
|
||||
@@ -204,7 +205,8 @@
|
||||
"^@docmost/db/(.*)$": "<rootDir>/database/$1",
|
||||
"^@docmost/transactional/(.*)$": "<rootDir>/integrations/transactional/$1",
|
||||
"^@docmost/ee/(.*)$": "<rootDir>/ee/$1",
|
||||
"^src/(.*)$": "<rootDir>/$1"
|
||||
"^src/(.*)$": "<rootDir>/$1",
|
||||
"^@tiptap/react$": "<rootDir>/../test/stubs/tiptap-react.js"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,7 +43,6 @@ import {
|
||||
Column,
|
||||
Status,
|
||||
addUniqueIdsToDoc,
|
||||
htmlToMarkdown,
|
||||
TransclusionSource,
|
||||
TransclusionReference,
|
||||
FootnoteReference,
|
||||
@@ -51,6 +50,7 @@ import {
|
||||
FootnoteDefinition,
|
||||
PageEmbed,
|
||||
} from '@docmost/editor-ext';
|
||||
import { convertProseMirrorToMarkdown } from '@docmost/prosemirror-markdown';
|
||||
import { generateText, getSchema, JSONContent } from '@tiptap/core';
|
||||
import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
|
||||
// @tiptap/html library works best for generating prosemirror json state but not HTML
|
||||
@@ -239,6 +239,10 @@ export function prosemirrorNodeToYElement(node: any): Y.XmlElement | Y.XmlText {
|
||||
}
|
||||
|
||||
export function jsonToMarkdown(tiptapJson: any): string {
|
||||
const html = jsonToHtml(tiptapJson);
|
||||
return htmlToMarkdown(html);
|
||||
// Direct ProseMirror JSON -> Markdown via the canonical converter
|
||||
// (`@docmost/prosemirror-markdown`) — no HTML intermediate, no second
|
||||
// editor-ext markdown layer. Same serializer as the page/space export and the
|
||||
// git-sync vault writer, so every server PM->MD path emits identical canonical
|
||||
// markdown (issue #345).
|
||||
return convertProseMirrorToMarkdown(tiptapJson);
|
||||
}
|
||||
|
||||
@@ -52,7 +52,9 @@ import {
|
||||
INTERNAL_LINK_REGEX,
|
||||
extractPageSlugId,
|
||||
} from '../../../integrations/export/utils';
|
||||
import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext';
|
||||
import { canonicalizeFootnotes } from '@docmost/editor-ext';
|
||||
import { markdownToProseMirror } from '@docmost/prosemirror-markdown';
|
||||
import { normalizeForeignMarkdown } from '../../../integrations/import/utils/foreign-markdown';
|
||||
import { WatcherService } from '../../watcher/watcher.service';
|
||||
import { sql } from 'kysely';
|
||||
import { TransclusionService } from '../transclusion/transclusion.service';
|
||||
@@ -1301,8 +1303,14 @@ export class PageService {
|
||||
|
||||
switch (format) {
|
||||
case 'markdown': {
|
||||
const html = await markdownToHtml(content as string);
|
||||
prosemirrorJson = htmlToJson(html as string);
|
||||
// Canonical markdown -> ProseMirror JSON directly via
|
||||
// `@docmost/prosemirror-markdown` (issue #345) — no HTML intermediate,
|
||||
// no editor-ext markdown layer. Foreign markdown surfaces the strict
|
||||
// parser rejects (GFM `[^id]` reference footnotes) are normalized to the
|
||||
// canonical inline form first.
|
||||
prosemirrorJson = await markdownToProseMirror(
|
||||
normalizeForeignMarkdown(content as string),
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 'html': {
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
// export.service.ts imports the ESM-only @sindresorhus/slugify (not in jest's
|
||||
// transform allowlist). It is irrelevant to the markdown-serialization path under
|
||||
// test (only used for page-mention link slugs on the DB path), so it is mocked
|
||||
// out to keep the module graph loadable under ts-jest (mirrors the import specs).
|
||||
jest.mock('@sindresorhus/slugify', () => ({
|
||||
__esModule: true,
|
||||
default: (input: string) => String(input),
|
||||
}));
|
||||
|
||||
import { convertProseMirrorToMarkdown } from '@docmost/prosemirror-markdown';
|
||||
import { ExportService } from './export.service';
|
||||
import { ExportFormat } from './dto/export-dto';
|
||||
|
||||
/**
|
||||
* STEP 1 golden test for issue #345: server MARKDOWN export runs DIRECTLY through
|
||||
* the canonical converter (`convertProseMirrorToMarkdown`) — no HTML intermediate
|
||||
* and no `@docmost/editor-ext` markdown layer — so the emitted markdown is in the
|
||||
* canonical package forms and is byte-identical to the git-sync vault body.
|
||||
*
|
||||
* These are the goldens the swap has to satisfy: they assert the CANONICAL
|
||||
* surface (callout `> [!type]`, inline footnote `^[…]`, lossless image
|
||||
* `<!--img …-->`) rather than the old editor-ext forms (`:::type`, `[^id]`,
|
||||
* lossy ``).
|
||||
*
|
||||
* `exportPage(..., singlePage=false)` takes no DB path (no mention rewriting), so
|
||||
* the service is constructed with null collaborators and only the pure
|
||||
* PM -> Markdown path is exercised.
|
||||
*/
|
||||
|
||||
function makeService(): ExportService {
|
||||
return new ExportService(
|
||||
null as any, // pageRepo
|
||||
null as any, // pagePermissionRepo
|
||||
null as any, // db
|
||||
null as any, // storageService
|
||||
null as any, // environmentService
|
||||
null as any, // domainService
|
||||
);
|
||||
}
|
||||
|
||||
// A representative page exercising the node types whose canonical markdown form
|
||||
// changed with the move off the editor-ext layer: callout, inline footnote, and a
|
||||
// lossless image carrying width/align attrs that the old layer dropped.
|
||||
const REPRESENTATIVE_DOC = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{ type: 'text', text: 'Body ' },
|
||||
{ type: 'footnoteReference', attrs: { id: 'fn-1' } },
|
||||
{ type: 'text', text: ' end.' },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'callout',
|
||||
attrs: { type: 'info', icon: null },
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [{ type: 'text', text: 'Heads up' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'image',
|
||||
attrs: {
|
||||
src: '/files/pic.png',
|
||||
alt: 'Pic',
|
||||
width: 320,
|
||||
align: 'left',
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'footnotesList',
|
||||
content: [
|
||||
{
|
||||
type: 'footnoteDefinition',
|
||||
attrs: { id: 'fn-1' },
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [{ type: 'text', text: 'the note' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe('ExportService — markdown export via the canonical converter (#345)', () => {
|
||||
it('emits canonical callout, inline footnote and lossless image forms', async () => {
|
||||
const service = makeService();
|
||||
const md = (await service.exportPage(ExportFormat.Markdown, {
|
||||
title: '',
|
||||
content: REPRESENTATIVE_DOC,
|
||||
} as any)) as string;
|
||||
|
||||
// Callout: Obsidian `> [!type]`, NOT the legacy `:::type`.
|
||||
expect(md).toContain('> [!info]');
|
||||
expect(md).not.toContain(':::');
|
||||
|
||||
// Inline footnote: `^[…]`, NOT the reference `[^id]` form.
|
||||
expect(md).toContain('^[the note]');
|
||||
expect(md).not.toMatch(/\[\^/);
|
||||
|
||||
// Lossless image: trailing `<!--img …-->` carrying the dropped attrs.
|
||||
expect(md).toContain('');
|
||||
expect(md).toContain('<!--img');
|
||||
expect(md).toContain('"width":"320"');
|
||||
expect(md).toContain('"align":"left"');
|
||||
});
|
||||
|
||||
it('export body is byte-identical to the git-sync vault serializer (export == vault)', async () => {
|
||||
const service = makeService();
|
||||
// A title-less page: exportPage prepends NO heading, so the whole output is
|
||||
// the page BODY — exactly what git-sync serializes (git-sync stores the title
|
||||
// in frontmatter / the filename, never as an in-body H1).
|
||||
const exported = (await service.exportPage(ExportFormat.Markdown, {
|
||||
title: '',
|
||||
content: REPRESENTATIVE_DOC,
|
||||
} as any)) as string;
|
||||
|
||||
// The git-sync vault writer feeds this SAME converter (git-sync
|
||||
// `stabilizePageBody` = convertProseMirrorToMarkdown(content) at the
|
||||
// fixpoint). For an already-stable doc the single pass IS the fixpoint, so
|
||||
// the two are byte-identical by construction — assert it.
|
||||
const vaultBody = convertProseMirrorToMarkdown(REPRESENTATIVE_DOC);
|
||||
expect(exported).toBe(vaultBody);
|
||||
});
|
||||
|
||||
it('prepends the page title as an H1 heading (the one documented export/vault delta)', async () => {
|
||||
const service = makeService();
|
||||
const md = (await service.exportPage(ExportFormat.Markdown, {
|
||||
title: 'My Page',
|
||||
content: { type: 'doc', content: [] },
|
||||
} as any)) as string;
|
||||
|
||||
// Export makes standalone files, so it prepends the title as an H1. This is
|
||||
// the ONE deliberate difference from the vault body (which carries the title
|
||||
// in frontmatter). The body below the heading still serializes canonically.
|
||||
expect(md.startsWith('# My Page')).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -37,7 +37,7 @@ import {
|
||||
getAttachmentIds,
|
||||
getProsemirrorContent,
|
||||
} from '../../common/helpers/prosemirror/utils';
|
||||
import { htmlToMarkdown } from '@docmost/editor-ext';
|
||||
import { convertProseMirrorToMarkdown } from '@docmost/prosemirror-markdown';
|
||||
|
||||
type AllowedAttachment = { id: string; fileName: string; filePath: string };
|
||||
|
||||
@@ -79,9 +79,8 @@ export class ExportService {
|
||||
prosemirrorJson.content.unshift(titleNode);
|
||||
}
|
||||
|
||||
const pageHtml = jsonToHtml(prosemirrorJson);
|
||||
|
||||
if (format === ExportFormat.HTML) {
|
||||
const pageHtml = jsonToHtml(prosemirrorJson);
|
||||
return `<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
@@ -92,11 +91,14 @@ export class ExportService {
|
||||
}
|
||||
|
||||
if (format === ExportFormat.Markdown) {
|
||||
const newPageHtml = pageHtml.replace(
|
||||
/<colgroup[^>]*>[\s\S]*?<\/colgroup>/gim,
|
||||
'',
|
||||
);
|
||||
return htmlToMarkdown(newPageHtml);
|
||||
// Direct ProseMirror JSON -> Markdown via the canonical converter
|
||||
// (`@docmost/prosemirror-markdown`). This is the SAME serializer the
|
||||
// git-sync vault writer feeds (see git-sync `stabilizePageBody`), so an
|
||||
// exported page body is byte-identical to its vault representation — no
|
||||
// HTML intermediate, no second markdown layer, no format drift (issue
|
||||
// #345). The old `<colgroup>` scrub is gone with the HTML step: the
|
||||
// converter emits GFM tables directly and never produces `<colgroup>`.
|
||||
return convertProseMirrorToMarkdown(prosemirrorJson);
|
||||
}
|
||||
|
||||
return;
|
||||
|
||||
+144
-77
@@ -17,6 +17,22 @@ jest.mock('image-dimensions', () => ({
|
||||
__esModule: true,
|
||||
imageDimensionsFromData: () => undefined,
|
||||
}));
|
||||
// FileImportTaskService -> PageService -> collaboration.gateway ->
|
||||
// metrics.registry imports `prom-client`, which is not resolvable in this
|
||||
// workspace's node_modules (types-only stub, no runtime entry). Metrics are
|
||||
// disabled on this path, so a virtual no-op mock keeps the module graph loadable.
|
||||
jest.mock(
|
||||
'prom-client',
|
||||
() => ({
|
||||
collectDefaultMetrics: () => undefined,
|
||||
Registry: class {},
|
||||
Histogram: class {},
|
||||
Gauge: class {},
|
||||
Counter: class {},
|
||||
Summary: class {},
|
||||
}),
|
||||
{ virtual: true },
|
||||
);
|
||||
|
||||
import { promises as fs } from 'fs';
|
||||
import * as os from 'os';
|
||||
@@ -26,14 +42,17 @@ import { ImportService } from './import.service';
|
||||
|
||||
/**
|
||||
* Binding test for issue #228 / review #5: FileImportTaskService.processGenericImport
|
||||
* is a NON-editor write path (markdownToHtml -> processHTML -> JSON, never runs
|
||||
* footnoteSyncPlugin), so it canonicalizes footnotes before persisting. This pins
|
||||
* that binding — the same one import.service has a spec for — which previously had
|
||||
* NO spec at all.
|
||||
* is a NON-editor write path, so a zip-imported `.md` page ends up with canonical
|
||||
* footnotes before persisting: ordered by first reference, reused refs deduped,
|
||||
* orphan definitions dropped.
|
||||
*
|
||||
* The markdown -> HTML -> ProseMirror conversion is REAL (a real ImportService,
|
||||
* its createYdoc stubbed); the filesystem is a real temp dir with one .md file;
|
||||
* the DB transaction is stubbed to capture the persisted page content.
|
||||
* Since #345 the `.md` parse runs `normalizeForeignMarkdown` ->
|
||||
* `markdownToProseMirror` -> `jsonToHtml` (feeding the shared HTML attachment /
|
||||
* link pipeline) -> `processHTML` -> `canonicalizeFootnotes`. The parser assigns
|
||||
* fresh `fn-*` ids, so we assert by definition BODY order rather than the source
|
||||
* labels. The conversion is REAL (a real ImportService, its createYdoc stubbed);
|
||||
* the filesystem is a real temp dir with one .md file; the DB transaction is
|
||||
* stubbed to capture the persisted page content.
|
||||
*/
|
||||
|
||||
// Out-of-order references (c, a, b), a REUSED reference ([^a] twice), and an
|
||||
@@ -49,13 +68,14 @@ const MARKDOWN = [
|
||||
'[^z]: orphan note',
|
||||
].join('\n');
|
||||
|
||||
function footnoteListIds(content: any): string[] {
|
||||
/** Definition body texts of the (single) footnotesList, in list order. */
|
||||
function footnoteListBodies(content: any): string[] {
|
||||
const list = (content?.content ?? []).find(
|
||||
(n: any) => n.type === 'footnotesList',
|
||||
);
|
||||
return (list?.content ?? [])
|
||||
.filter((n: any) => n.type === 'footnoteDefinition')
|
||||
.map((n: any) => n.attrs?.id);
|
||||
.map((n: any) => n.content?.[0]?.content?.[0]?.text);
|
||||
}
|
||||
|
||||
// A permissive chainable stub for the spaces lookup (selectFrom(...).select(...)
|
||||
@@ -71,80 +91,127 @@ function chainable(result: any): any {
|
||||
return proxy;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one markdown file through the REAL zip-import pipeline
|
||||
* (`processGenericImport` -> `markdownToProseMirror` -> `jsonToHtml` ->
|
||||
* `processHTML`/`htmlToJson`) and return the persisted page `content`. This is
|
||||
* the server-specific PM->HTML->PM hop that the package's own PM<->MD tests do
|
||||
* NOT cover.
|
||||
*/
|
||||
async function runZipImport(markdown: string): Promise<any> {
|
||||
const extractDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fit-canon-'));
|
||||
await fs.writeFile(path.join(extractDir, 'note.md'), markdown, 'utf-8');
|
||||
|
||||
const importService = new ImportService(
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
);
|
||||
jest
|
||||
.spyOn(importService as any, 'createYdoc')
|
||||
.mockResolvedValue(Buffer.from([]) as any);
|
||||
|
||||
let captured: any = null;
|
||||
const trx = {
|
||||
insertInto: (table: string) => ({
|
||||
values: (v: any) => {
|
||||
if (table === 'pages') captured = v;
|
||||
return { execute: async () => {} };
|
||||
},
|
||||
}),
|
||||
};
|
||||
const db: any = {
|
||||
selectFrom: () => chainable({ slug: 'space-slug' }),
|
||||
transaction: () => ({ execute: (fn: any) => fn(trx) }),
|
||||
};
|
||||
|
||||
const importAttachmentService = {
|
||||
processAttachments: async ({ html }: any) => html,
|
||||
};
|
||||
const service = new FileImportTaskService(
|
||||
{} as any, // storageService
|
||||
importService as any,
|
||||
{ nextPagePosition: async () => 'a0' } as any,
|
||||
{ insertBacklink: jest.fn() } as any,
|
||||
db,
|
||||
importAttachmentService as any,
|
||||
{ emit: jest.fn() } as any,
|
||||
{ logBatchWithContext: jest.fn() } as any,
|
||||
);
|
||||
|
||||
const fileTask: any = {
|
||||
id: 'task-1',
|
||||
source: 'generic',
|
||||
spaceId: 'space-1',
|
||||
workspaceId: 'ws-1',
|
||||
creatorId: 'user-1',
|
||||
};
|
||||
|
||||
try {
|
||||
await service.processGenericImport({ extractDir, fileTask });
|
||||
expect(captured).toBeTruthy();
|
||||
return captured.content;
|
||||
} finally {
|
||||
await fs.rm(extractDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Find the first node of a given type anywhere in a PM content tree. */
|
||||
function findFirst(node: any, type: string): any {
|
||||
if (!node || typeof node !== 'object') return null;
|
||||
if (node.type === type) return node;
|
||||
for (const child of node.content ?? []) {
|
||||
const hit = findFirst(child, type);
|
||||
if (hit) return hit;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
describe('FileImportTaskService.processGenericImport — footnote canonicalization (#228)', () => {
|
||||
it('orders footnotes by first reference, dedupes reuse, and drops orphans on zip import', async () => {
|
||||
const extractDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fit-canon-'));
|
||||
await fs.writeFile(path.join(extractDir, 'note.md'), MARKDOWN, 'utf-8');
|
||||
|
||||
// Real ImportService for the html -> JSON conversion; stub the yjs encode.
|
||||
const importService = new ImportService(
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
const content = await runZipImport(MARKDOWN);
|
||||
// Definitions ordered by FIRST REFERENCE (C, A, B), NOT the markdown
|
||||
// definition order (A, B, C). Ids are the parser's fresh `fn-*`, so pin
|
||||
// the BODIES.
|
||||
expect(footnoteListBodies(content)).toEqual(['note C', 'note A', 'note B']);
|
||||
// Orphan [^z] dropped; reused [^a] collapses to one definition; one list.
|
||||
expect(footnoteListBodies(content)).not.toContain('orphan note');
|
||||
const lists = (content.content ?? []).filter(
|
||||
(n: any) => n.type === 'footnotesList',
|
||||
);
|
||||
jest
|
||||
.spyOn(importService as any, 'createYdoc')
|
||||
.mockResolvedValue(Buffer.from([]) as any);
|
||||
expect(lists).toHaveLength(1);
|
||||
expect(
|
||||
footnoteListBodies(content).filter((b) => b === 'note A'),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
let captured: any = null;
|
||||
const trx = {
|
||||
insertInto: (table: string) => ({
|
||||
values: (v: any) => {
|
||||
if (table === 'pages') captured = v;
|
||||
return { execute: async () => {} };
|
||||
},
|
||||
}),
|
||||
};
|
||||
const db: any = {
|
||||
selectFrom: () => chainable({ slug: 'space-slug' }),
|
||||
transaction: () => ({ execute: (fn: any) => fn(trx) }),
|
||||
};
|
||||
// #345 F4: the zip path routes markdown through jsonToHtml -> processHTML ->
|
||||
// htmlToJson (the shared HTML attachment pipeline). #345's headline is LOSSLESS
|
||||
// image width/align via the `<!--img {...}-->` comment; a callout carries its
|
||||
// `type`. This asserts those survive the PM->HTML->PM hop — the one hop the
|
||||
// package's PM<->MD suite does not exercise.
|
||||
it('preserves image width/align and callout type through the PM->HTML->PM hop', async () => {
|
||||
const md = [
|
||||
'# Doc',
|
||||
'',
|
||||
' <!--img {"width":"320","align":"left"}-->',
|
||||
'',
|
||||
':::warning',
|
||||
'Careful now.',
|
||||
':::',
|
||||
].join('\n');
|
||||
|
||||
const importAttachmentService = {
|
||||
processAttachments: async ({ html }: any) => html,
|
||||
};
|
||||
const backlinkRepo = { insertBacklink: jest.fn() };
|
||||
const eventEmitter = { emit: jest.fn() };
|
||||
const auditService = { logBatchWithContext: jest.fn() };
|
||||
const content = await runZipImport(md);
|
||||
|
||||
const pageService = { nextPagePosition: async () => 'a0' };
|
||||
const image = findFirst(content, 'image');
|
||||
expect(image).toBeTruthy();
|
||||
// The lossless sizing/alignment must survive the HTML hop.
|
||||
expect(String(image.attrs?.width)).toBe('320');
|
||||
expect(image.attrs?.align).toBe('left');
|
||||
|
||||
const service = new FileImportTaskService(
|
||||
{} as any, // storageService
|
||||
importService as any,
|
||||
pageService as any,
|
||||
backlinkRepo as any,
|
||||
db,
|
||||
importAttachmentService as any,
|
||||
eventEmitter as any,
|
||||
auditService as any,
|
||||
);
|
||||
|
||||
const fileTask: any = {
|
||||
id: 'task-1',
|
||||
source: 'generic',
|
||||
spaceId: 'space-1',
|
||||
workspaceId: 'ws-1',
|
||||
creatorId: 'user-1',
|
||||
};
|
||||
|
||||
try {
|
||||
await service.processGenericImport({ extractDir, fileTask });
|
||||
|
||||
expect(captured).toBeTruthy();
|
||||
const content = captured.content;
|
||||
// Reference order is c, a, b (NOT the markdown definition order a, b, c).
|
||||
expect(footnoteListIds(content)).toEqual(['c', 'a', 'b']);
|
||||
// Orphan [^z] dropped; reused [^a] collapses to one definition; one list.
|
||||
expect(footnoteListIds(content)).not.toContain('z');
|
||||
const lists = (content.content ?? []).filter(
|
||||
(n: any) => n.type === 'footnotesList',
|
||||
);
|
||||
expect(lists).toHaveLength(1);
|
||||
expect(footnoteListIds(content).filter((id) => id === 'a')).toHaveLength(1);
|
||||
} finally {
|
||||
await fs.rm(extractDir, { recursive: true, force: true });
|
||||
}
|
||||
const callout = findFirst(content, 'callout');
|
||||
expect(callout).toBeTruthy();
|
||||
expect(callout.attrs?.type).toBe('warning');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import * as path from 'path';
|
||||
import { jsonToText } from '../../../collaboration/collaboration.util';
|
||||
import {
|
||||
jsonToHtml,
|
||||
jsonToText,
|
||||
} from '../../../collaboration/collaboration.util';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import {
|
||||
@@ -18,9 +21,11 @@ import { generateSlugId } from '../../../common/helpers';
|
||||
import { v7 } from 'uuid';
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { FileTask, InsertablePage } from '@docmost/db/types/entity.types';
|
||||
import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext';
|
||||
import { canonicalizeFootnotes } from '@docmost/editor-ext';
|
||||
import { markdownToProseMirror } from '@docmost/prosemirror-markdown';
|
||||
import { getProsemirrorContent } from '../../../common/helpers/prosemirror/utils';
|
||||
import { formatImportHtml } from '../utils/import-formatter';
|
||||
import { normalizeForeignMarkdown } from '../utils/foreign-markdown';
|
||||
import {
|
||||
buildAttachmentCandidates,
|
||||
collectMarkdownAndHtmlFiles,
|
||||
@@ -461,7 +466,18 @@ export class FileImportTaskService {
|
||||
content = await fs.readFile(absPath, 'utf-8');
|
||||
|
||||
if (page.fileExtension.toLowerCase() === '.md') {
|
||||
content = await markdownToHtml(content);
|
||||
// Parse markdown with the single canonical converter
|
||||
// (`@docmost/prosemirror-markdown`), after normalizing foreign
|
||||
// reference footnotes, then serialize to HTML so the shared HTML
|
||||
// pipeline below (processAttachments + formatImportHtml +
|
||||
// processHTML) keeps handling `.md` and `.html` imports
|
||||
// uniformly. The markdown PARSE no longer goes through the
|
||||
// editor-ext markdown layer (issue #345) — the drift source is
|
||||
// gone. The PM -> HTML -> PM hop that follows is lossless
|
||||
// plumbing for attachment/link resolution, NOT a second parse.
|
||||
content = jsonToHtml(
|
||||
await markdownToProseMirror(normalizeForeignMarkdown(content)),
|
||||
);
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err?.code === 'ENOENT') {
|
||||
@@ -500,10 +516,12 @@ export class FileImportTaskService {
|
||||
this.importService.extractTitleAndRemoveHeading(pmState);
|
||||
|
||||
// Canonicalize footnote topology on this non-editor write path
|
||||
// (markdownToHtml/processHTML never runs footnoteSyncPlugin), so a
|
||||
// zip-imported page's footnotes are reference-ordered, deduped, and
|
||||
// (the HTML pipeline's processHTML never runs footnoteSyncPlugin), so
|
||||
// a zip-imported page's footnotes are reference-ordered, deduped, and
|
||||
// orphan-free like the editor's invariant (issue #228). Pure +
|
||||
// idempotent + shape-safe; a footnote-free doc is unchanged.
|
||||
// idempotent + shape-safe; a footnote-free doc is unchanged. (For a
|
||||
// `.md` file the package parser already yields canonical footnotes,
|
||||
// so this is a no-op there.)
|
||||
// (Future consolidation, architecture B: like import.service, this
|
||||
// path persists directly rather than via PageService — a shared
|
||||
// "prepare JSON for persist" helper would centralize this call.)
|
||||
|
||||
+27
-31
@@ -12,13 +12,19 @@ import { canonicalizeFootnotes } from '@docmost/editor-ext';
|
||||
|
||||
/**
|
||||
* Integration-ish test for the USER-FACING markdown import path
|
||||
* (`ImportService.importPage`). It exercises the REAL markdown -> HTML -> JSON
|
||||
* conversion and asserts that the stored page content has its footnotes
|
||||
* canonicalized — the gap that issue #228 fixes: the import path builds
|
||||
* ProseMirror JSON directly (never running the editor's footnoteSyncPlugin), so
|
||||
* before this wiring the stored footnotes kept the markdown's physical
|
||||
* definition order (out of order vs. references), retained orphan definitions,
|
||||
* and did not collapse reused references.
|
||||
* (`ImportService.importPage`). It exercises the REAL markdown -> ProseMirror
|
||||
* conversion and asserts the stored page's footnotes are canonical: ordered by
|
||||
* FIRST REFERENCE (not markdown definition order), reused references deduped to a
|
||||
* single definition, and orphan definitions dropped.
|
||||
*
|
||||
* Since #345 the markdown parse runs through the canonical package
|
||||
* (`normalizeForeignMarkdown` -> `markdownToProseMirror`), which owns this
|
||||
* canonicalization: the input's GFM `[^id]` reference footnotes are normalized to
|
||||
* inline `^[…]`, and the parser assigns fresh sequential ids (`fn-*`) in
|
||||
* reference order while merging identical bodies — so we assert by definition
|
||||
* BODY order, not by the source labels. `canonicalizeFootnotes` remains wired as
|
||||
* an idempotent safety net (issue #228) and is a no-op on this already-canonical
|
||||
* output.
|
||||
*
|
||||
* The DB/ydoc side-effects are stubbed: `getNewPagePosition` (DB query) and
|
||||
* `createYdoc` (Yjs encode) are spied, and `pageRepo.insertPage` captures the
|
||||
@@ -67,24 +73,14 @@ function makeService() {
|
||||
}
|
||||
|
||||
/** List the footnote-definition ids of the (single) footnotesList, in order. */
|
||||
function footnoteListIds(content: any): string[] {
|
||||
/** Definition body texts of the (single) footnotesList, in list order. */
|
||||
function footnoteListBodies(content: any): string[] {
|
||||
const list = (content.content ?? []).find(
|
||||
(n: any) => n.type === 'footnotesList',
|
||||
);
|
||||
if (!list) return [];
|
||||
return (list.content ?? [])
|
||||
return (list?.content ?? [])
|
||||
.filter((n: any) => n.type === 'footnoteDefinition')
|
||||
.map((n: any) => n.attrs?.id);
|
||||
}
|
||||
|
||||
function definitionText(content: any, id: string): string | undefined {
|
||||
const list = (content.content ?? []).find(
|
||||
(n: any) => n.type === 'footnotesList',
|
||||
);
|
||||
const def = (list?.content ?? []).find(
|
||||
(n: any) => n.type === 'footnoteDefinition' && n.attrs?.id === id,
|
||||
);
|
||||
return def?.content?.[0]?.content?.[0]?.text;
|
||||
.map((n: any) => n.content?.[0]?.content?.[0]?.text);
|
||||
}
|
||||
|
||||
describe('ImportService.importPage — footnote canonicalization (#228)', () => {
|
||||
@@ -101,23 +97,23 @@ describe('ImportService.importPage — footnote canonicalization (#228)', () =>
|
||||
const content = getCaptured().content;
|
||||
expect(content).toBeTruthy();
|
||||
|
||||
// Reference order is c, a, b (NOT the markdown definition order a, b, c).
|
||||
expect(footnoteListIds(content)).toEqual(['c', 'a', 'b']);
|
||||
|
||||
// Definitions preserved and attached to the right ids.
|
||||
expect(definitionText(content, 'c')).toBe('note C');
|
||||
expect(definitionText(content, 'a')).toBe('note A');
|
||||
expect(definitionText(content, 'b')).toBe('note B');
|
||||
// Definitions ordered by FIRST REFERENCE (C, A, B) — NOT the markdown
|
||||
// definition order (A, B, C) — with the orphan [^z] dropped and the reused
|
||||
// [^a] collapsed to a single definition. (Ids are the parser's fresh `fn-*`,
|
||||
// so we pin the BODIES.)
|
||||
expect(footnoteListBodies(content)).toEqual(['note C', 'note A', 'note B']);
|
||||
|
||||
// Orphan definition [^z] is dropped.
|
||||
expect(footnoteListIds(content)).not.toContain('z');
|
||||
expect(footnoteListBodies(content)).not.toContain('orphan note');
|
||||
|
||||
// Reused [^a] yields exactly ONE definition, and exactly one list.
|
||||
const lists = (content.content ?? []).filter(
|
||||
(n: any) => n.type === 'footnotesList',
|
||||
);
|
||||
expect(lists).toHaveLength(1);
|
||||
expect(footnoteListIds(content).filter((id) => id === 'a')).toHaveLength(1);
|
||||
expect(
|
||||
footnoteListBodies(content).filter((b) => b === 'note A'),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('is idempotent: canonicalizing the stored output again is a no-op', async () => {
|
||||
@@ -134,6 +130,6 @@ describe('ImportService.importPage — footnote canonicalization (#228)', () =>
|
||||
// time must not change it (safe to wire into every write path).
|
||||
const second = canonicalizeFootnotes(stored);
|
||||
expect(second).toEqual(stored);
|
||||
expect(footnoteListIds(second)).toEqual(['c', 'a', 'b']);
|
||||
expect(footnoteListBodies(second)).toEqual(['note C', 'note A', 'note B']);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -17,7 +17,9 @@ import {
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { TiptapTransformer } from '@hocuspocus/transformer';
|
||||
import * as Y from 'yjs';
|
||||
import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext';
|
||||
import { canonicalizeFootnotes } from '@docmost/editor-ext';
|
||||
import { markdownToProseMirror } from '@docmost/prosemirror-markdown';
|
||||
import { normalizeForeignMarkdown } from '../utils/foreign-markdown';
|
||||
import {
|
||||
FileTaskStatus,
|
||||
FileTaskType,
|
||||
@@ -85,11 +87,13 @@ export class ImportService {
|
||||
|
||||
const extracted = this.extractTitleAndRemoveHeading(prosemirrorState);
|
||||
const title = extracted.title;
|
||||
// Imported markdown/HTML is built via markdownToHtml -> htmlToJson, which
|
||||
// never runs the editor's footnoteSyncPlugin, so the footnote topology keeps
|
||||
// the source's PHYSICAL definition order (out of order vs. references),
|
||||
// retains orphan definitions, and is not deduped. Canonicalize before
|
||||
// persisting so the stored page matches the editor's invariant (issue #228).
|
||||
// The markdown path now canonicalizes footnotes itself (the package parser),
|
||||
// but the HTML path (processHTML -> htmlToJson) does NOT run the editor's
|
||||
// footnoteSyncPlugin, so an imported HTML doc can keep its source's PHYSICAL
|
||||
// definition order (out of order vs. references), retain orphan definitions,
|
||||
// and not be deduped. Canonicalize before persisting so the stored page
|
||||
// matches the editor's invariant (issue #228); it is an idempotent no-op on
|
||||
// the already-canonical markdown output.
|
||||
// Pure + idempotent + shape-safe: a doc with no footnotes is unchanged.
|
||||
// (Future consolidation, architecture B: this import path persists directly
|
||||
// via pageRepo.insertPage rather than through PageService.createPage, so the
|
||||
@@ -133,12 +137,15 @@ export class ImportService {
|
||||
}
|
||||
|
||||
async processMarkdown(markdownInput: string): Promise<any> {
|
||||
try {
|
||||
const html = await markdownToHtml(markdownInput);
|
||||
return this.processHTML(html);
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
// Canonical markdown -> ProseMirror JSON directly via
|
||||
// `@docmost/prosemirror-markdown` (issue #345) — no HTML intermediate and no
|
||||
// second editor-ext markdown layer. Foreign markdown surfaces the strict
|
||||
// canonical parser does not accept (GFM `[^id]` reference footnotes) are
|
||||
// rewritten to the canonical inline form by `normalizeForeignMarkdown` first.
|
||||
// The HTML-cleanup pass (`normalizeImportHtml`) is intentionally skipped here:
|
||||
// it targets foreign *HTML* (Notion/XWiki), which only ever arrives on the
|
||||
// `.html` path (`processHTML`), never as canonical markdown.
|
||||
return markdownToProseMirror(normalizeForeignMarkdown(markdownInput));
|
||||
}
|
||||
|
||||
async processHTML(htmlInput: string): Promise<any> {
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
import {
|
||||
convertProseMirrorToMarkdown,
|
||||
markdownToProseMirror,
|
||||
} from '@docmost/prosemirror-markdown';
|
||||
import { normalizeForeignMarkdown } from './foreign-markdown';
|
||||
|
||||
/**
|
||||
* STEP 2 goldens for issue #345: the foreign-markdown normalizer that runs at the
|
||||
* import boundary BEFORE the strict canonical parser (`markdownToProseMirror`).
|
||||
*
|
||||
* Two layers:
|
||||
* 1. PURE string→string cases pinning the normalizer's own behavior (GFM
|
||||
* reference footnotes → inline `^[…]`).
|
||||
* 2. END-TO-END acceptance: for a foreign corpus, `normalizeForeignMarkdown`
|
||||
* then `markdownToProseMirror` then `convertProseMirrorToMarkdown` must leave
|
||||
* NO literal `[^id]` / `:::` garbage in the document and must re-export in the
|
||||
* canonical forms.
|
||||
*/
|
||||
|
||||
describe('normalizeForeignMarkdown — GFM reference footnotes', () => {
|
||||
it('inlines a single-line reference footnote and drops its definition', () => {
|
||||
const out = normalizeForeignMarkdown(
|
||||
'A note[^1] here.\n\n[^1]: The definition.',
|
||||
);
|
||||
expect(out).toBe('A note^[The definition.] here.\n');
|
||||
});
|
||||
|
||||
it('inlines every reference to a reused id (downstream dedups)', () => {
|
||||
const out = normalizeForeignMarkdown(
|
||||
'X[^a] and Y[^a].\n\n[^a]: shared.',
|
||||
);
|
||||
expect(out).toBe('X^[shared.] and Y^[shared.].\n');
|
||||
});
|
||||
|
||||
it('joins indented continuation lines of a definition with a space', () => {
|
||||
const out = normalizeForeignMarkdown(
|
||||
'See[^n].\n\n[^n]: line one\n line two',
|
||||
);
|
||||
expect(out).toBe('See^[line one line two].\n');
|
||||
});
|
||||
|
||||
it('never rewrites a reference inside a fenced code block', () => {
|
||||
const out = normalizeForeignMarkdown(
|
||||
'```\ncode[^1] here\n```\n\n[^1]: def.',
|
||||
);
|
||||
expect(out).toContain('code[^1] here');
|
||||
// The (now orphaned) definition line is still removed.
|
||||
expect(out).not.toContain('[^1]: def.');
|
||||
});
|
||||
|
||||
it('never rewrites a reference inside an INLINE-code span (backticks)', () => {
|
||||
// The `[^1]` inside backticks is literal code and must survive verbatim;
|
||||
// the one outside is rewritten. (Bug #1: only fenced blocks were protected.)
|
||||
const out = normalizeForeignMarkdown(
|
||||
'Use `arr[^1]` in code but note[^1] in prose.\n\n[^1]: def.',
|
||||
);
|
||||
expect(out).toBe('Use `arr[^1]` in code but note^[def.] in prose.\n');
|
||||
});
|
||||
|
||||
it('escapes brackets in a body so an unbalanced ] cannot truncate the footnote', () => {
|
||||
// A foreign definition body with a stray `]` would, unescaped, close the
|
||||
// canonical `^[...]` early and leak the tail as text (bug #2). The body's
|
||||
// brackets are backslash-escaped so the footnote stays whole.
|
||||
const out = normalizeForeignMarkdown(
|
||||
'Ref[^1] here.\n\n[^1]: see item ] and [more] later',
|
||||
);
|
||||
expect(out).toBe('Ref^[see item \\] and \\[more\\] later] here.\n');
|
||||
// The tokenizer must see exactly one unescaped closing bracket (our own).
|
||||
expect(out.match(/(?<!\\)\]/g)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('leaves a reference with no matching definition literal (no body to inline)', () => {
|
||||
const out = normalizeForeignMarkdown('Dangling[^x] ref.');
|
||||
expect(out).toBe('Dangling[^x] ref.');
|
||||
});
|
||||
|
||||
it('returns the input unchanged when there are no reference footnotes', () => {
|
||||
const md = '# Title\n\nJust text with `inline code` and a [link](/x).';
|
||||
expect(normalizeForeignMarkdown(md)).toBe(md);
|
||||
});
|
||||
|
||||
it('does NOT touch callout surfaces — the canonical parser handles them', () => {
|
||||
const callouts = ':::info\nHi\n:::\n\n> [!warning]\n> Careful';
|
||||
expect(normalizeForeignMarkdown(callouts)).toBe(callouts);
|
||||
});
|
||||
|
||||
it('strips a leading YAML front-matter block (Obsidian/Hugo/git-sync files)', () => {
|
||||
const out = normalizeForeignMarkdown(
|
||||
'---\ntitle: My Page\ntags: [a, b]\n---\n\n# Heading\n\nBody.',
|
||||
);
|
||||
expect(out).toBe('# Heading\n\nBody.');
|
||||
// The front-matter must not leak into the body as a setext heading.
|
||||
expect(out).not.toContain('title: My Page');
|
||||
expect(out).not.toContain('---');
|
||||
});
|
||||
|
||||
it('does not strip a horizontal rule that is not leading front-matter', () => {
|
||||
const md = 'Intro paragraph.\n\n---\n\nAfter the rule.';
|
||||
expect(normalizeForeignMarkdown(md)).toBe(md);
|
||||
});
|
||||
|
||||
it('is linear on a document with thousands of definitions (no quadratic blowup)', () => {
|
||||
// F2(a): the pass-2 rewrite must be O(text), not O(text × defs). Build a
|
||||
// pathological doc (many defs + many plain text lines) and assert it
|
||||
// completes well under a second — a quadratic implementation took ~14s.
|
||||
const N = 4000;
|
||||
const refs = Array.from({ length: N }, (_, i) => `line ${i} plain text`).join('\n');
|
||||
const defs = Array.from({ length: N }, (_, i) => `[^n${i}]: def ${i}`).join('\n');
|
||||
const doc = `start[^n0] and[^n${N - 1}] end\n\n${refs}\n\n${defs}`;
|
||||
const t0 = Date.now();
|
||||
const out = normalizeForeignMarkdown(doc);
|
||||
const elapsed = Date.now() - t0;
|
||||
expect(elapsed).toBeLessThan(2000);
|
||||
// Sanity: the two real references were still inlined.
|
||||
expect(out).toContain('^[def 0]');
|
||||
expect(out).toContain(`^[def ${N - 1}]`);
|
||||
});
|
||||
|
||||
it('is bounded on a long unclosed backtick run (no inline-split ReDoS)', () => {
|
||||
// F2(b): a huge unterminated backtick run must not cause quadratic
|
||||
// backtracking in the inline-code split. Oversized lines skip the split
|
||||
// entirely (left untouched), so this returns promptly.
|
||||
const line = 'x' + '`'.repeat(200000);
|
||||
const doc = `${line}\n\n[^1]: def`;
|
||||
const t0 = Date.now();
|
||||
normalizeForeignMarkdown(doc);
|
||||
expect(Date.now() - t0).toBeLessThan(2000);
|
||||
});
|
||||
|
||||
it('does not crash or slow down on thousands of prefix-chain definition ids', () => {
|
||||
// F7: the rewrite must use a FIXED generic scanner, not an alternation built
|
||||
// from the ids. A `(a|aa|aaa|…)` alternation over prefix-chain ids blows the
|
||||
// V8 regex compiler (FATAL RegExpCompiler Allocation failed — uncatchable,
|
||||
// kills the process). A fixed scanner has no id-dependent compilation cost.
|
||||
const N = 4000;
|
||||
const ids = Array.from({ length: N }, (_, i) => 'a'.repeat(i + 1));
|
||||
const defs = ids.map((id) => `[^${id}]: body ${id.length}`).join('\n');
|
||||
const doc = `ref[^${ids[0]}] and[^${ids[N - 1]}] end\n\n${defs}`;
|
||||
const t0 = Date.now();
|
||||
const out = normalizeForeignMarkdown(doc);
|
||||
expect(Date.now() - t0).toBeLessThan(2000);
|
||||
// Prefix disambiguation is correct: [^a] and [^aaaa...] inline their OWN body.
|
||||
expect(out).toContain('^[body 1]');
|
||||
expect(out).toContain(`^[body ${N}]`);
|
||||
});
|
||||
|
||||
it('strips a CRLF (Windows) front-matter block, not just LF', () => {
|
||||
// F9: the line-anchored regex needs LF after the opening `---`, so a Windows
|
||||
// file (`---\r\n…`) would slip past the strip and leak the front-matter into
|
||||
// the body. normalizeForeignMarkdown normalizes CRLF -> LF first.
|
||||
const out = normalizeForeignMarkdown(
|
||||
'---\r\ntitle: Foo\r\ntags: [a]\r\n---\r\n\r\n# Heading\r\n\r\nBody.',
|
||||
);
|
||||
expect(out).toBe('# Heading\n\nBody.');
|
||||
expect(out).not.toContain('title: Foo');
|
||||
expect(out).not.toContain('---');
|
||||
});
|
||||
|
||||
it('strips front-matter whose value contains a triple-dash (line-anchored)', () => {
|
||||
// F8: the block must close only on a `\n---` LINE, not the first inline
|
||||
// `---`. A value like `title: Q1 --- Q2` must not truncate the front-matter
|
||||
// and leak the rest (author/closing ---) into the body.
|
||||
const out = normalizeForeignMarkdown(
|
||||
'---\ntitle: Q1 --- Q2 results\nauthor: bob\n---\n\nReal body.',
|
||||
);
|
||||
expect(out).toBe('Real body.');
|
||||
expect(out).not.toContain('author: bob');
|
||||
expect(out).not.toContain('Q2 results');
|
||||
});
|
||||
});
|
||||
|
||||
describe('foreign markdown import acceptance (normalizer + canonical parser)', () => {
|
||||
const FOREIGN = [
|
||||
'# Doc',
|
||||
'',
|
||||
'Body refs [^c] and [^a] and [^b] and again [^a].',
|
||||
'',
|
||||
':::info',
|
||||
'A legacy callout.',
|
||||
':::',
|
||||
'',
|
||||
'| h1 | h2 |',
|
||||
'| --- | --- |',
|
||||
'| 1 | 2 |',
|
||||
'',
|
||||
'[^a]: note A',
|
||||
'[^b]: note B',
|
||||
'[^c]: note C',
|
||||
'[^z]: orphan note',
|
||||
].join('\n');
|
||||
|
||||
it('leaves no literal [^id] or ::: in the imported doc and re-exports canonically', async () => {
|
||||
const normalized = normalizeForeignMarkdown(FOREIGN);
|
||||
const doc = await markdownToProseMirror(normalized);
|
||||
const reexport = convertProseMirrorToMarkdown(doc);
|
||||
|
||||
// No foreign garbage leaks into the document.
|
||||
expect(reexport).not.toMatch(/\[\^/); // no reference footnote refs/defs
|
||||
expect(reexport).not.toContain(':::'); // no legacy callout fences
|
||||
|
||||
// Canonical forms are present.
|
||||
expect(reexport).toContain('^[note C]');
|
||||
expect(reexport).toContain('> [!info]');
|
||||
expect(reexport).toContain('| h1 | h2 |');
|
||||
|
||||
// Footnotes: ordered by first reference (C, A, B), reused [^a] deduped to one,
|
||||
// orphan [^z] dropped (it had no reference after normalization).
|
||||
const list = doc.content.find((n: any) => n.type === 'footnotesList');
|
||||
const bodies = list.content.map(
|
||||
(d: any) => d.content[0].content[0].text,
|
||||
);
|
||||
expect(bodies).toEqual(['note C', 'note A', 'note B']);
|
||||
expect(bodies).not.toContain('orphan note');
|
||||
expect(
|
||||
doc.content.filter((n: any) => n.type === 'footnotesList'),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,265 @@
|
||||
/**
|
||||
* Foreign-markdown normalizer — an input-liberal / output-canonical adapter that
|
||||
* runs at the IMPORT boundary, BEFORE the canonical parser
|
||||
* (`markdownToProseMirror` from `@docmost/prosemirror-markdown`).
|
||||
*
|
||||
* The canonical parser is deliberately STRICT: it only understands Docmost's
|
||||
* canonical markdown surface (Obsidian-style `> [!type]` callouts, Pandoc/Obsidian
|
||||
* inline footnotes `^[body]`, lossless ` <!--img {...}-->` images, …).
|
||||
* Import, however, ingests FOREIGN files (GitHub/GFM, Notion, old Docmost
|
||||
* exports). Those use surfaces the canonical parser does not accept, most notably
|
||||
* GitHub-flavoured *reference* footnotes:
|
||||
*
|
||||
* Text with a note[^1] and another[^long].
|
||||
*
|
||||
* [^1]: The first definition.
|
||||
* [^long]: A second one.
|
||||
*
|
||||
* Left untouched, the parser does NOT recognise `[^id]` (it only parses `^[body]`),
|
||||
* so the reference leaks as literal text — and worse, the trailing `[^id]: def`
|
||||
* line is a valid CommonMark *link-reference definition*, so `[^id]` is silently
|
||||
* rendered as a bogus link. This normalizer rewrites reference footnotes into the
|
||||
* canonical inline form so the parser materialises real footnote nodes.
|
||||
*
|
||||
* This is a TEXT pre-pass, NOT a second parser fork: it does not re-implement any
|
||||
* converter logic. Callout surfaces (`:::type` and `> [!type]`) are intentionally
|
||||
* NOT touched here — the canonical parser already accepts BOTH natively (its
|
||||
* `preprocessCallouts` pass), so normalizing them would be redundant and would
|
||||
* only risk degrading the parser's nesting/code-fence-aware handling.
|
||||
*/
|
||||
|
||||
/** Matches a fenced code block delimiter (``` or ~~~), capturing the marker run. */
|
||||
const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
|
||||
|
||||
/**
|
||||
* Matches a GFM footnote DEFINITION line: `[^id]: body`. The id is any run of
|
||||
* non-`]` characters; the body is the remainder of the line (possibly empty).
|
||||
*/
|
||||
const FOOTNOTE_DEF_RE = /^\[\^([^\]]+)\]:[ \t]?(.*)$/;
|
||||
|
||||
/** True when a line is a code-fence delimiter that toggles fenced-code state. */
|
||||
function fenceMarker(line: string): string | null {
|
||||
const m = line.match(CODE_FENCE_RE);
|
||||
return m ? m[2] : null;
|
||||
}
|
||||
|
||||
/** True when a line is indented (leading space/tab) and not blank — a continuation. */
|
||||
function isIndentedContinuation(line: string): boolean {
|
||||
return /^[ \t]+\S/.test(line);
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
/**
|
||||
* Backslash-escape any square bracket in a footnote body before it is wrapped in
|
||||
* `^[...]`. The canonical inline-footnote tokenizer scans the body with bracket
|
||||
* balancing and closes on the first UNMATCHED `]`, so an unbalanced bracket in a
|
||||
* foreign definition (e.g. `[^1]: see item ] later`) would otherwise truncate the
|
||||
* footnote and leak the tail as literal text. Escaping every `[`/`]` makes the
|
||||
* body an inert run of characters — the tokenizer then closes only on our own
|
||||
* closing `]`. (A balanced `[link](url)` inside a body still round-trips because
|
||||
* the escaped form renders the literal brackets, which is the safe reading for a
|
||||
* footnote body; the alternative — brittle balance tracking — risks worse.)
|
||||
*/
|
||||
function escapeFootnoteBody(body: string): string {
|
||||
return body.replace(/[[\]]/g, '\\$&');
|
||||
}
|
||||
|
||||
/**
|
||||
* Rewrite every `[^id]` reference on a line to its `^[body]` form, but ONLY in the
|
||||
* text OUTSIDE inline-code spans. A `[^id]` inside backticks is literal code
|
||||
* content and must be preserved verbatim (a footnote ref never lives inside code).
|
||||
* We split the line on inline-code spans (paired backtick runs) and rewrite only
|
||||
* the non-code segments.
|
||||
*/
|
||||
// Above this length a single line is not split into inline-code spans (see
|
||||
// below). A genuine markdown line carrying a footnote reference is never tens of
|
||||
// KB; the cap only bypasses the inline-code protection for pathological lines.
|
||||
const INLINE_SPLIT_MAX_LINE = 8192;
|
||||
|
||||
function rewriteRefsOutsideInlineCode(
|
||||
line: string,
|
||||
replace: (text: string) => string,
|
||||
): string {
|
||||
// The inline-code split alternation `(`+)(?:(?!\1)[\s\S])*\1` backtracks
|
||||
// quadratically on a long UNCLOSED backtick run (its middle can consume the
|
||||
// rest of the line, then fail to find a closing run and retry from each
|
||||
// position). On an untrusted import this is a request-thread ReDoS. A real
|
||||
// footnote line is short, so for an oversized line we skip the inline-code
|
||||
// protection entirely and leave the line UNTOUCHED (rewriting it wholesale
|
||||
// could corrupt a `[^id]` that legitimately lives inside inline code). This is
|
||||
// a conservative bypass: an over-8KB line simply does not get its reference
|
||||
// footnotes inlined — acceptable for a pathological input.
|
||||
if (line.length > INLINE_SPLIT_MAX_LINE) return line;
|
||||
|
||||
// Alternation: an inline-code span (one or more backticks, then anything up to
|
||||
// the SAME run of backticks) OR a run of non-backtick text. Unterminated
|
||||
// backticks fall through as ordinary text (matched by the second branch on the
|
||||
// leftover), so a stray backtick never swallows the rest of the line.
|
||||
const parts = line.match(/(`+)(?:(?!\1)[\s\S])*\1|[^`]+|`+/g);
|
||||
if (!parts) return line;
|
||||
return parts
|
||||
.map((seg) => (seg.startsWith('`') ? seg : replace(seg)))
|
||||
.join('');
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert GFM reference footnotes (`[^id]` + `[^id]: def`) into canonical inline
|
||||
* footnotes (`^[def]`).
|
||||
*
|
||||
* - Definitions are collected first (a leading `[^id]: text` line plus any
|
||||
* immediately-following indented continuation lines, joined with a space) and
|
||||
* removed from the output.
|
||||
* - Each in-text reference `[^id]` for which a definition was found is replaced by
|
||||
* `^[def]`. References with no matching definition are left literal (there is no
|
||||
* body to inline; the parser fails them open the same way).
|
||||
* - Code is respected on both passes: `[^id]` inside a fenced ``` / ~~~ block is
|
||||
* never rewritten and a `[^id]:` line inside a fence is never a definition; and
|
||||
* on the rewrite pass a `[^id]` inside an INLINE-code span (backticks) is left
|
||||
* literal too.
|
||||
* - The inlined body is bracket-escaped so an unbalanced `[`/`]` in a foreign
|
||||
* definition cannot truncate the resulting `^[...]` footnote.
|
||||
*
|
||||
* Deduplication / reference-ordering / orphan-dropping of the resulting footnotes
|
||||
* is handled downstream by the canonical parser (`assembleFootnotes`); this pass
|
||||
* only changes the surface syntax.
|
||||
*/
|
||||
function convertReferenceFootnotes(markdown: string): string {
|
||||
const lines = markdown.split('\n');
|
||||
|
||||
// Pass 1: collect definitions and mark their lines for removal.
|
||||
const defs = new Map<string, string>();
|
||||
const dropped = new Array<boolean>(lines.length).fill(false);
|
||||
let inFence = false;
|
||||
let fence = '';
|
||||
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
const marker = fenceMarker(line);
|
||||
if (inFence) {
|
||||
if (marker && marker[0] === fence[0] && marker.length >= fence.length) {
|
||||
inFence = false;
|
||||
fence = '';
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (marker) {
|
||||
inFence = true;
|
||||
fence = marker;
|
||||
continue;
|
||||
}
|
||||
|
||||
const def = line.match(FOOTNOTE_DEF_RE);
|
||||
if (!def) continue;
|
||||
|
||||
const id = def[1];
|
||||
const body: string[] = [def[2].trim()];
|
||||
dropped[i] = true;
|
||||
|
||||
// Consume immediately-following indented continuation lines (GFM lazy
|
||||
// continuation is not supported by design — keep it simple and predictable).
|
||||
let j = i + 1;
|
||||
while (j < lines.length && isIndentedContinuation(lines[j])) {
|
||||
body.push(lines[j].trim());
|
||||
dropped[j] = true;
|
||||
j++;
|
||||
}
|
||||
i = j - 1;
|
||||
|
||||
// Last definition wins for a duplicated id (matches CommonMark link-ref
|
||||
// semantics closely enough for a foreign-input adapter).
|
||||
defs.set(id, body.filter((s) => s.length > 0).join(' '));
|
||||
}
|
||||
|
||||
if (defs.size === 0) {
|
||||
return markdown;
|
||||
}
|
||||
|
||||
// ONE fixed, generic scanner regex — NOT one built from the definition ids.
|
||||
// It matches ANY `[^id]` shape, and the replacer decides per match via a map
|
||||
// lookup whether that id is a real definition (replace) or not (leave as-is).
|
||||
// This is genuinely O(total text) with no per-document regex compilation.
|
||||
//
|
||||
// Do NOT rebuild this as an alternation over `[...defs.keys()]`: a giant
|
||||
// `(id1|id2|...)` alternation over thousands of ids can blow the V8 regex
|
||||
// compiler's stack — a fatal, UNCATCHABLE "RegExpCompiler Allocation failed"
|
||||
// on prefix-chain ids (`a`, `aa`, `aaa`, ...) that kills the whole process
|
||||
// (worse than the earlier per-def thread-hang). A fixed scanner has no
|
||||
// id-dependent compilation cost and cannot blow up.
|
||||
const refRe = /\[\^([^\]]+)\]/g;
|
||||
const rewriteSegment = (segment: string): string =>
|
||||
segment.replace(refRe, (whole, id: string) => {
|
||||
const body = defs.get(id);
|
||||
// Only real definitions are inlined; an unknown id is left literal (same as
|
||||
// the old per-def loop, which simply never matched it).
|
||||
return body === undefined ? whole : `^[${escapeFootnoteBody(body)}]`;
|
||||
});
|
||||
|
||||
// Pass 2: rewrite in-text references, skipping fenced code and dropped lines.
|
||||
const out: string[] = [];
|
||||
inFence = false;
|
||||
fence = '';
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
if (dropped[i]) continue;
|
||||
let line = lines[i];
|
||||
|
||||
const marker = fenceMarker(line);
|
||||
if (inFence) {
|
||||
out.push(line);
|
||||
if (marker && marker[0] === fence[0] && marker.length >= fence.length) {
|
||||
inFence = false;
|
||||
fence = '';
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (marker) {
|
||||
inFence = true;
|
||||
fence = marker;
|
||||
out.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
line = rewriteRefsOutsideInlineCode(line, rewriteSegment);
|
||||
out.push(line);
|
||||
}
|
||||
|
||||
return out.join('\n');
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip a single leading YAML front-matter block (`---\n…\n---`). Foreign files
|
||||
* from Obsidian / Hugo / Jekyll / Notion — and Docmost's OWN git-sync page files
|
||||
* — open with front-matter that the canonical parser does not consume, so
|
||||
* without this it leaks into the body (and `title: Foo` above the closing `---`
|
||||
* renders as a setext `<h2>` that `extractTitleAndRemoveHeading` can hijack as
|
||||
* the page title). It is a no-op for front-matter-free input.
|
||||
*
|
||||
* LINE-ANCHORED (the same shape the canonical parser uses in
|
||||
* prosemirror-markdown/page-file.ts): the block opens only on `---\n` at the
|
||||
* very start and closes only on a `\n---` line. The retired `markdownToHtml`
|
||||
* strip closed on the FIRST `---` ANYWHERE (an unanchored close), so a value
|
||||
* containing a triple-dash (e.g. `title: Q1 --- Q2`) truncated the front-matter
|
||||
* and leaked the rest into the body. An optional leading BOM is tolerated.
|
||||
*/
|
||||
const YAML_FRONT_MATTER_RE = /^\uFEFF?---\n[\s\S]*?\n---\n?/;
|
||||
|
||||
/**
|
||||
* Normalize a foreign markdown string into Docmost's canonical markdown surface
|
||||
* so the strict canonical parser accepts it losslessly: normalize line endings,
|
||||
* strip a leading YAML front-matter block, then rewrite GFM reference footnotes
|
||||
* into inline footnotes. Add further fixture-driven foreign-surface cases here as
|
||||
* they are found.
|
||||
*/
|
||||
export function normalizeForeignMarkdown(markdown: string): string {
|
||||
if (!markdown) return markdown;
|
||||
// Normalize CRLF -> LF FIRST. The line-anchored front-matter regex requires a
|
||||
// bare `\n` after the opening `---`, and convertReferenceFootnotes splits on
|
||||
// `\n`; a Windows/CRLF foreign file (`---\r\n…`) would otherwise slip past the
|
||||
// front-matter strip and leak into the body. The canonical parser
|
||||
// (page-file.ts parsePageFile) normalizes the same way before its FRONTMATTER_RE.
|
||||
const src = markdown.replace(/\r\n/g, '\n');
|
||||
const withoutFrontMatter = src.replace(YAML_FRONT_MATTER_RE, '').trimStart();
|
||||
return convertReferenceFootnotes(withoutFrontMatter);
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Jest stub for @tiptap/react.
|
||||
//
|
||||
// The server export/import code paths transitively import editor-ext, whose node
|
||||
// extensions import from `@tiptap/react`. The real module re-exports all of
|
||||
// `@tiptap/core` (headless, safe under node) AND adds React view helpers
|
||||
// (`ReactNodeViewRenderer`, …) that eagerly pull in react-dom — which throws
|
||||
// `navigator is not defined` under jest's node environment.
|
||||
//
|
||||
// So this stub DELEGATES to the real `@tiptap/core` (keeping `mergeAttributes`,
|
||||
// `Node`, `Mark`, `nodeInputRule`, … working — they are used by
|
||||
// `jsonToHtml`/`htmlToJson` on the server) and overrides ONLY the React view
|
||||
// helpers with no-ops. Those helpers are referenced solely inside `addNodeView()`
|
||||
// — code that runs only in a live browser editor, never on the server; if any
|
||||
// were actually invoked here it would (correctly) surface as a test failure.
|
||||
const core = require('@tiptap/core');
|
||||
|
||||
module.exports = {
|
||||
...core,
|
||||
ReactNodeViewRenderer: () => () => ({}),
|
||||
NodeViewWrapper: () => null,
|
||||
NodeViewContent: () => null,
|
||||
ReactRenderer: class {},
|
||||
};
|
||||
Generated
+3
@@ -543,6 +543,9 @@ importers:
|
||||
'@docmost/pdf-inspector':
|
||||
specifier: 1.9.6
|
||||
version: 1.9.6
|
||||
'@docmost/prosemirror-markdown':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/prosemirror-markdown
|
||||
'@fastify/cookie':
|
||||
specifier: ^11.0.2
|
||||
version: 11.0.2
|
||||
|
||||
Reference in New Issue
Block a user