diff --git a/packages/prosemirror-markdown/src/lib/markdown-converter.ts b/packages/prosemirror-markdown/src/lib/markdown-converter.ts index 2d173e52..481c3691 100644 --- a/packages/prosemirror-markdown/src/lib/markdown-converter.ts +++ b/packages/prosemirror-markdown/src/lib/markdown-converter.ts @@ -48,6 +48,52 @@ export interface ConvertProseMirrorToMarkdownOptions { dropResolvedCommentAnchors?: boolean; } +/** + * Adjacent sibling lists that share a markdown MARKER FAMILY re-parse as ONE + * merged list — bulletList and taskList both emit `- ` markers (→ a single + * `
tags), so it
// must escape < > & — NOT the attribute escaper. The language rides in
// a class ATTRIBUTE, so it uses escapeAttr.
+ //
+ // Read the child text RAW (as `case "codeBlock"` does) and keep it
+ // VERBATIM — do NOT strip the trailing newline. Unlike the markdown fence
+ // path (which strips then relies on marked re-adding one `\n`), the schema
+ // codeBlock parseHTML reads the `` text content back byte-for-byte,
+ // so stripping here would drop a trailing newline the node legitimately
+ // carries and break the round trip inside a column/cell.
const code = escapeHtmlText(
children
- .map(processNode)
- .join("")
- .replace(/\n+$/, ""),
+ .map((child: any) =>
+ typeof child?.text === "string" ? child.text : "",
+ )
+ .join(""),
);
const cls = lang ? ` class="language-${escapeAttr(lang)}"` : "";
return `${code}
`;
@@ -1484,6 +1564,13 @@ export function convertProseMirrorToMarkdown(
const indent = " ".repeat(indentWidth);
const lines: string[] = [];
childStrings.forEach((child, childIndex) => {
+ // Separate consecutive block children with a BLANK line so the item is a
+ // CommonMark "loose" list item and each block stays its own node. Without
+ // it, a second paragraph (`- a\n b`) is re-parsed as a lazy continuation
+ // of the first and the two merge into one paragraph — silent data loss.
+ // The blank line still sits INSIDE the item (the following block keeps the
+ // continuation indent), so nested lists/code blocks remain nested.
+ if (childIndex > 0) lines.push("");
child.split("\n").forEach((line, lineIndex) => {
if (childIndex === 0 && lineIndex === 0) {
// First physical line of the first block gets the marker.
@@ -1500,7 +1587,9 @@ export function convertProseMirrorToMarkdown(
const processListItem = (item: any, prefix: string): string => {
const itemContent = item.content || [];
- const childStrings = itemContent.map(processNode);
+ // A `` entry separates two adjacent same-family sublists inside the
+ // item so they do not merge on re-parse (see renderBlockChildren).
+ const childStrings = renderBlockChildren(itemContent, processNode);
if (childStrings.length === 0) return prefix;
// The rendered marker is `${prefix} ` (prefix + one space), so its width —
// and thus the continuation indent — is prefix.length + 1. This is correct
@@ -1514,7 +1603,9 @@ export function convertProseMirrorToMarkdown(
const checkbox = checked ? "[x]" : "[ ]";
const prefix = `- ${checkbox}`;
const itemContent = item.content || [];
- const childStrings = itemContent.map(processNode);
+ // A `` entry separates two adjacent same-family sublists inside the
+ // item so they do not merge on re-parse (see renderBlockChildren).
+ const childStrings = renderBlockChildren(itemContent, processNode);
// An empty task item still needs its checkbox marker; without this guard
// the indent below produces "" and the "- [ ]"/"- [x]" row disappears.
if (childStrings.length === 0) return prefix;
diff --git a/packages/prosemirror-markdown/src/lib/markdown-to-prosemirror.ts b/packages/prosemirror-markdown/src/lib/markdown-to-prosemirror.ts
index b5e0f2d7..f482ed5e 100644
--- a/packages/prosemirror-markdown/src/lib/markdown-to-prosemirror.ts
+++ b/packages/prosemirror-markdown/src/lib/markdown-to-prosemirror.ts
@@ -270,9 +270,10 @@ const CALLOUT_CLOSE_RE = /^:::\s*$/;
* optional title after the type is allowed but ignored (the Docmost callout
* schema has no title). The body is the following contiguous blockquote lines.
*/
-const CALLOUT_BQ_OPEN_RE = /^>\s*\[!(\w+)\]/;
-/** Matches any blockquote continuation line (`>` … ). */
-const BLOCKQUOTE_LINE_RE = /^>/;
+// The callout's own `>` marker may be preceded by an ENCLOSING container prefix:
+// list-item indentation (` `) and/or blockquote markers (`> `). Group 1 captures
+// that prefix (lazily, so the LAST `>` before `[!type]` is the callout's own).
+const CALLOUT_BQ_OPEN_RE = /^([>\s]*?)>\s*\[!(\w+)\]/;
/** Matches the start/end of a code fence (``` or ~~~), capturing the marker. */
const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
@@ -402,20 +403,47 @@ async function preprocessCallouts(markdown: string): Promise {
// recurse so nested callouts (`> > [!type]`) are handled, then emit the same
// callout div the `:::` path produces. A normal blockquote (no `[!type]` on
// its first line) does not match and stays a blockquote.
+ //
+ // PREFIX-aware: a callout nested inside a list item and/or a blockquote is
+ // serialized with the enclosing container prefix in front of its own `>`
+ // marker — ` > [!type]` (list indent) or `> > [!type]` (blockquote). We
+ // capture that prefix, take only continuation lines carrying `prefix>`, strip
+ // it, and re-apply the prefix to the emitted HTML block so the callout div
+ // stays WITHIN its container (an unprefixed div would escape and re-parse as
+ // a top-level callout / plain blockquote — silent structure loss).
const bqOpen = line.match(CALLOUT_BQ_OPEN_RE);
if (bqOpen) {
- const type = bqOpen[1].toLowerCase();
+ const prefix = bqOpen[1];
+ const type = bqOpen[2].toLowerCase();
+ const cont = prefix + ">"; // a body line = prefix + the callout's own `>`
const bodyLines: string[] = [];
let j = i + 1;
for (; j < lines.length; j++) {
- if (!BLOCKQUOTE_LINE_RE.test(lines[j])) break;
- bodyLines.push(lines[j].replace(/^>\s?/, ""));
+ if (!lines[j].startsWith(cont)) break;
+ // Drop the prefix + `>` + one optional space, leaving the body content.
+ bodyLines.push(lines[j].slice(prefix.length).replace(/^>\s?/, ""));
}
const inner = await transform(bodyLines);
const renderedInner = await markedInstance.parse(inner);
- out.push(
- `\n${renderedInner}\n`,
- );
+ const block = `${renderedInner}`;
+ if (prefix.length === 0) {
+ // Top-level callout: blank lines isolate the HTML block.
+ out.push(`\n${block}\n`);
+ } else if (prefix.includes(">")) {
+ // Enclosing BLOCKQUOTE: prefix every line and add NO surrounding blank
+ // lines — a blank line would terminate the blockquote and split the
+ // callout out of it.
+ out.push(block.split("\n").map((l) => prefix + l).join("\n"));
+ } else {
+ // Pure LIST-ITEM indentation: re-indent and keep the blank-line
+ // separators (a loose list item), so the div sits at the marker column.
+ out.push(
+ `\n${block
+ .split("\n")
+ .map((l) => (l.length ? prefix + l : l))
+ .join("\n")}\n`,
+ );
+ }
i = j;
continue;
}
@@ -597,6 +625,40 @@ function bridgeTaskLists(html: string): string {
* (null from parseAttachedComment), an unknown name, a wrong-position comment, or
* an unknown/empty attr value is ignored.
*/
+/**
+ * A directive comment is in ATTACHED position when it sits inside a ``/``
+ * textblock — bound to that block's text (the `attrs`/`img` conventions). Every
+ * other parent (body, document level, a block container like blockquote/details/
+ * li/column div) is STANDALONE position, where a lone-block directive
+ * (subpages/pagebreak/pageembed/transclusion) is materialized. Broadening
+ * standalone beyond body/document is what lets these nodes survive NESTED inside
+ * a blockquote/callout/details/list item (previously dropped -> silent data loss).
+ */
+function isAttachedPosition(tag: string): boolean {
+ return tag === "p" || /^h[1-6]$/.test(tag);
+}
+
+/**
+ * Place a materialized standalone-directive element in the DOM: replace the
+ * comment IN PLACE when it has a real element parent inside (body itself
+ * or a nested block container), preserving document order; queue it as a leading
+ * div only when the comment is at document level (no parentElement) or directly
+ * under `` (outside , which `document.body.innerHTML` would drop).
+ */
+function placeStandalone(
+ comment: any,
+ el: any,
+ tag: string,
+ leadingDivs: any[],
+): void {
+ if (comment.parentElement && tag !== "html") {
+ comment.replaceWith(el);
+ } else {
+ comment.remove();
+ leadingDivs.push(el);
+ }
+}
+
function applyCommentDirectives(html: string): string {
// Cheap early-out: no comments at all -> nothing to intercept.
if (!html.includes("` separator between
+// such adjacent lists (markdown-converter.ts renderBlockChildren), so they stay
+// distinct and round-trip; the generator therefore emits them freely again.
+describe('#351 nested generative round-trip — properties', () => {
+ it('P1 — semantic round-trip: docsCanonicallyEqual(mdToPm(pmToMd(d)), d)', async () => {
+ await fc.assert(
+ fc.asyncProperty(docArb, async (doc) => {
+ const { doc2 } = await roundTrip(doc);
+ if (!docsCanonicallyEqual(doc2, doc)) {
+ const div = firstDivergence(
+ JSON.parse(JSON.stringify(canonicalizeContent(doc2))),
+ JSON.parse(JSON.stringify(canonicalizeContent(doc))),
+ );
+ throw new Error(
+ `P1 divergence @ ${div?.path}: got=${JSON.stringify(div?.a)} want=${JSON.stringify(div?.b)}`,
+ );
+ }
+ }),
+ { numRuns: NUM_RUNS, seed: SEED },
+ );
+ });
+
+ it('P2 — byte fixpoint: pmToMd(mdToPm(pmToMd(d))) === pmToMd(d)', async () => {
+ await fc.assert(
+ fc.asyncProperty(docArb, async (doc) => {
+ const { md1, md2 } = await roundTrip(doc);
+ expect(md2).toBe(md1);
+ }),
+ { numRuns: NUM_RUNS, seed: SEED },
+ );
+ });
+
+ it('P3 — totality: neither converter throws', async () => {
+ await fc.assert(
+ fc.asyncProperty(docArb, async (doc) => {
+ await roundTrip(doc);
+ }),
+ { numRuns: NUM_RUNS, seed: SEED },
+ );
+ });
+});
+
+// ---------------------------------------------------------------------------
+// P4 — parser fuzz. Independent of the doc generator: for ANY input string the
+// PARSER (markdownToProseMirror) must be TOTAL — never throw — and must always
+// return a schema-valid document. The corpus mixes raw unicode strings with
+// strings assembled from markdown-significant fragments (headings, list bullets,
+// fences, pipes, thematic breaks, HTML-ish snippets) to probe the block/inline
+// parsers on hostile but plausible input.
+// ---------------------------------------------------------------------------
+
+const mdFragmentArb: fc.Arbitrary = fc.constantFrom(
+ '# ', '## ', '### ###', '- ', '* ', '+ ', '1. ', '> ', '>> ',
+ '```', '```js', '~~~', '---', '***', '___', '| a | b |', '|---|---|',
+ '[link](http://x)', '', '**', '__', '~~', '`code`',
+ '', '', '', '', '', '
', '&',
+ '\t', '\n', ' ', '\\', '^[fn]', '[^1]:', '- [ ] ', '- [x] ',
+ '$$', '$x$', ':::', '{.class}', '\u0000', '\uFEFF', '😀', 'مرحبا',
+);
+
+// Full-unicode strings (fast-check v4 replaced fullUnicodeString with the
+// `unit: 'binary'` string option, which draws over the whole code-point range).
+const fullUnicodeStringArb = (max?: number) =>
+ fc.string({ unit: 'binary', ...(max !== undefined ? { maxLength: max } : {}) });
+
+const assembledMarkdownArb: fc.Arbitrary = fc
+ .array(fc.oneof(mdFragmentArb, fc.string(), fullUnicodeStringArb(8)), {
+ minLength: 1,
+ maxLength: 12,
+ })
+ .map((parts) => parts.join(''));
+
+const parserInputArb: fc.Arbitrary = fc.oneof(
+ { weight: 2, arbitrary: fc.string() },
+ { weight: 2, arbitrary: fullUnicodeStringArb() },
+ { weight: 3, arbitrary: assembledMarkdownArb },
+ { weight: 1, arbitrary: fc.array(mdFragmentArb, { minLength: 1, maxLength: 8 }).map((p) => p.join('\n')) },
+);
+
+describe('#351 parser fuzz — totality on arbitrary input (P4)', () => {
+ it('P4 — markdownToProseMirror never throws and always returns a schema-valid doc', async () => {
+ await fc.assert(
+ fc.asyncProperty(parserInputArb, async (s) => {
+ let result: any;
+ try {
+ result = await mdToPm(s);
+ } catch (e: any) {
+ throw new Error(`P4 parser THREW on input ${JSON.stringify(s)}: ${e?.message ?? e}`);
+ }
+ try {
+ schema.nodeFromJSON(result).check();
+ } catch (e: any) {
+ throw new Error(
+ `P4 parser produced an INVALID doc for input ${JSON.stringify(s)}: ${e?.message ?? e}\n` +
+ `doc=${JSON.stringify(result).slice(0, 600)}`,
+ );
+ }
+ }),
+ { numRuns: NUM_RUNS * 2, seed: SEED },
+ );
+ });
+});
diff --git a/packages/prosemirror-markdown/test/markdown-converter-gaps.test.ts b/packages/prosemirror-markdown/test/markdown-converter-gaps.test.ts
index 4aae6b3b..4f864fea 100644
--- a/packages/prosemirror-markdown/test/markdown-converter-gaps.test.ts
+++ b/packages/prosemirror-markdown/test/markdown-converter-gaps.test.ts
@@ -374,7 +374,9 @@ describe('converter gap coverage — emission branches (specs 1–11)', () => {
],
}),
);
- expect(out).toBe('- [ ] top\n - child');
+ // Block children of a task item are blank-line separated (loose list) per the
+ // #351 fix; the sublist stays at the fixed 2-column continuation indent.
+ expect(out).toBe('- [ ] top\n\n - child');
});
// 10. A bulletList inside a blockquote: each list line independently prefixed.
diff --git a/packages/prosemirror-markdown/test/markdown-converter.test.ts b/packages/prosemirror-markdown/test/markdown-converter.test.ts
index 2a39de6c..bd4e734b 100644
--- a/packages/prosemirror-markdown/test/markdown-converter.test.ts
+++ b/packages/prosemirror-markdown/test/markdown-converter.test.ts
@@ -198,7 +198,11 @@ describe('convertProseMirrorToMarkdown', () => {
}),
);
// First line carries the marker; the nested list is indented 2 columns.
- expect(out).toBe('- parent\n - child');
+ // Block children of a list item are separated by a BLANK line (loose list):
+ // this is the #351 fix — a single "\n" let a following block merge into the
+ // first paragraph on re-parse (silent content loss). The blank line stays
+ // inside the item, so the sublist remains nested at the 2-col marker column.
+ expect(out).toBe('- parent\n\n - child');
});
it('nested ordered list indents by the wider 3-col marker width', () => {
@@ -219,8 +223,9 @@ describe('convertProseMirrorToMarkdown', () => {
],
}),
);
- // "1. " is 3 columns wide, so the continuation indent is 3 spaces.
- expect(out).toBe('1. parent\n 1. child');
+ // "1. " is 3 columns wide, so the continuation indent is 3 spaces. Block
+ // children are blank-line separated (loose list) per the #351 fix.
+ expect(out).toBe('1. parent\n\n 1. child');
});
});
@@ -539,11 +544,12 @@ describe('convertProseMirrorToMarkdown', () => {
);
// The 10th marker is the 4-column "10. "; the nested sublist line must be
- // indented exactly 4 spaces (prefix.length 3 + 1), NOT 3.
- expect(out).toContain('10. j\n 1. x');
+ // indented exactly 4 spaces (prefix.length 3 + 1), NOT 3. Block children are
+ // blank-line separated (loose list) per the #351 fix.
+ expect(out).toContain('10. j\n\n 1. x');
// Guard against the off-by-one (3-space) regression that would re-parse
// the sublist as loose/sibling content on import.
- expect(out).not.toContain('10. j\n 1. x');
+ expect(out).not.toContain('10. j\n\n 1. x');
// And the single-digit items keep the narrower 3-column marker (no body
// continuation here, but the marker itself must stay "1. ".."9. ").
expect(out.startsWith('1. a\n2. b\n')).toBe(true);
@@ -580,17 +586,18 @@ describe('convertProseMirrorToMarkdown', () => {
content: [para(text('line1')), para(text('line2'))],
}),
);
- // NOTE(review): the spec predicted ':::warning\nline1\n\nline2\n:::' (a
- // The converter joins the callout's rendered children with a single '\n'
- // and emits an Obsidian-native callout: a `> [!type]` opener plus one
- // `>`-prefixed body line per content line. We pin the lowercasing
- // (WARNING -> warning) and the multi-child join.
- expect(out).toBe('> [!warning]\n> line1\n> line2');
+ // The converter emits an Obsidian-native callout: a `> [!type]` opener plus
+ // one `>`-prefixed body line per content line. Block children are separated
+ // by a blank `>` line (#351 fix): a single '\n' let the two paragraphs merge
+ // into one on re-parse. We pin the lowercasing (WARNING -> warning) and the
+ // blank-line-separated multi-child join.
+ expect(out).toBe('> [!warning]\n> line1\n>\n> line2');
// The type is lowercased (an uppercase `[!WARNING]` would not re-import).
expect(out.startsWith('> [!warning]\n')).toBe(true);
expect(out).not.toContain('[!WARNING]');
- // Both paragraph children are present, each blockquote-prefixed.
- expect(out).toContain('> line1\n> line2');
+ // Both paragraph children are present, each blockquote-prefixed, blank-`>`
+ // separated so they stay distinct paragraphs on re-parse.
+ expect(out).toContain('> line1\n>\n> line2');
});
// Spec 4 — blockquote per-line prefixer over a multi-line nested callout.
@@ -607,13 +614,11 @@ describe('convertProseMirrorToMarkdown', () => {
],
}),
);
- // NOTE(review): the spec predicted '> :::info\n> a\n>\n> b\n> :::',
- // assuming the nested callout body contains a blank line between 'a' and
- // The nested callout renders as an Obsidian callout '> [!info]\n> a\n> b'
- // (single-'\n' join, no blank line). The outer blockquote prefixer then
- // prefixes each of those lines with '> ' again, yielding a doubly-nested
- // blockquote — the realistic per-line-prefix loop over a multi-line child.
- expect(out).toBe('> > [!info]\n> > a\n> > b');
+ // The nested callout renders as an Obsidian callout '> [!info]\n> a\n>\n> b'
+ // (blank-`>` separated children per the #351 fix). The outer blockquote
+ // prefixer then prefixes each of those lines with '> ' again, yielding a
+ // doubly-nested blockquote — the per-line-prefix loop over a multi-line child.
+ expect(out).toBe('> > [!info]\n> > a\n> >\n> > b');
// Every produced line carries the '> ' prefix (no line escapes to col 0).
for (const line of out.split('\n')) {
expect(line.startsWith('>')).toBe(true);