Merge pull request 'feat(footnotes): author-inline footnotes + deterministic server canonicalization (#228)' (#232) from feat/228-inline-footnotes into develop

Reviewed-on: #232

apps/client/src/features/editor/extensions/markdown-clipboard.canonicalize.test.ts

@@ -0,0 +1,168 @@
+import { describe, it, expect } from "vitest";
+import { Editor } from "@tiptap/core";
+import { Document } from "@tiptap/extension-document";
+import { Paragraph } from "@tiptap/extension-paragraph";
+import { Text } from "@tiptap/extension-text";
+import { Node as PMNode, Fragment, Slice } from "@tiptap/pm/model";
+import {
+  FootnoteReference,
+  FootnotesList,
+  FootnoteDefinition,
+  FOOTNOTE_REFERENCE_NAME,
+  FOOTNOTE_DEFINITION_NAME,
+  FOOTNOTES_LIST_NAME,
+} from "@docmost/editor-ext";
+import { canonicalizePastedFootnotes } from "./markdown-clipboard";
+
+/**
+ * A markdown paste builds its ProseMirror fragment via DOM -> parseSlice and is
+ * applied with a manual transaction (handlePaste returns true), so it bypasses
+ * the editor's footnoteSyncPlugin — which never reorders an existing list. These
+ * tests pin canonicalizePastedFootnotes, the focused hook that makes a pasted
+ * out-of-order markdown footnote block come out canonical (issue #228).
+ */
+
+const extensions = [
+  Document,
+  Paragraph,
+  Text,
+  FootnoteReference,
+  FootnotesList,
+  FootnoteDefinition,
+];
+
+function makeSchema() {
+  const editor = new Editor({ extensions, content: { type: "doc", content: [] } });
+  const { schema } = editor;
+  return { editor, schema };
+}
+
+/** List footnote def ids of the (single) footnotesList in a slice, in order. */
+function listIds(slice: Slice): string[] {
+  const out: string[] = [];
+  slice.content.forEach((node: PMNode) => {
+    if (node.type.name === FOOTNOTES_LIST_NAME) {
+      node.content.forEach((def: PMNode) => {
+        if (def.type.name === FOOTNOTE_DEFINITION_NAME) out.push(def.attrs.id);
+      });
+    }
+  });
+  return out;
+}
+
+function hasList(slice: Slice): boolean {
+  let found = false;
+  slice.content.forEach((n: PMNode) => {
+    if (n.type.name === FOOTNOTES_LIST_NAME) found = true;
+  });
+  return found;
+}
+
+describe("canonicalizePastedFootnotes", () => {
+  it("reorders a pasted block to reference order, dedups reuse, drops orphans", () => {
+    const { editor, schema } = makeSchema();
+    // Body references c, a, b (and again a => reuse); definitions a, b, c, z
+    // (z is an orphan) — the exact shape a markdown paste produces.
+    const slice = new Slice(
+      Fragment.fromArray([
+        schema.nodes.paragraph.create(null, [
+          schema.text("body "),
+          schema.nodes[FOOTNOTE_REFERENCE_NAME].create({ id: "c" }),
+          schema.nodes[FOOTNOTE_REFERENCE_NAME].create({ id: "a" }),
+          schema.nodes[FOOTNOTE_REFERENCE_NAME].create({ id: "b" }),
+          schema.nodes[FOOTNOTE_REFERENCE_NAME].create({ id: "a" }),
+        ]),
+        schema.nodes[FOOTNOTES_LIST_NAME].create(null, [
+          schema.nodes[FOOTNOTE_DEFINITION_NAME].create({ id: "a" }, [
+            schema.nodes.paragraph.create(null, [schema.text("note A")]),
+          ]),
+          schema.nodes[FOOTNOTE_DEFINITION_NAME].create({ id: "b" }, [
+            schema.nodes.paragraph.create(null, [schema.text("note B")]),
+          ]),
+          schema.nodes[FOOTNOTE_DEFINITION_NAME].create({ id: "c" }, [
+            schema.nodes.paragraph.create(null, [schema.text("note C")]),
+          ]),
+          schema.nodes[FOOTNOTE_DEFINITION_NAME].create({ id: "z" }, [
+            schema.nodes.paragraph.create(null, [schema.text("orphan")]),
+          ]),
+        ]),
+      ]),
+      0,
+      0,
+    );
+
+    const out = canonicalizePastedFootnotes(slice, schema);
+    // Reference order, orphan z dropped, reused a appears once.
+    expect(listIds(out)).toEqual(["c", "a", "b"]);
+    editor.destroy();
+  });
+
+  it("leaves a reference-ONLY paste untouched (no synthesized definitions)", () => {
+    // A paste that reuses an id defined in the TARGET doc must NOT gain a
+    // synthesized empty definition here — it carries no footnotesList of its own.
+    const { editor, schema } = makeSchema();
+    const slice = new Slice(
+      Fragment.from(
+        schema.nodes.paragraph.create(null, [
+          schema.text("see "),
+          schema.nodes[FOOTNOTE_REFERENCE_NAME].create({ id: "a" }),
+        ]),
+      ),
+      0,
+      0,
+    );
+    const out = canonicalizePastedFootnotes(slice, schema);
+    expect(hasList(out)).toBe(false);
+    expect(out).toBe(slice); // returned unchanged (same reference)
+    editor.destroy();
+  });
+
+  it("leaves a definitions-ONLY paste untouched (no references -> no empty paste)", () => {
+    // A whole-block paste of ONLY definitions (a footnotesList with no matching
+    // footnoteReference anywhere in the selection). Canonicalizing it would strip
+    // the reference-less list -> an EMPTY paste, losing the pasted text. The hook
+    // must leave such a block untouched.
+    const { editor, schema } = makeSchema();
+    const slice = new Slice(
+      Fragment.fromArray([
+        schema.nodes[FOOTNOTES_LIST_NAME].create(null, [
+          schema.nodes[FOOTNOTE_DEFINITION_NAME].create({ id: "a" }, [
+            schema.nodes.paragraph.create(null, [schema.text("note A")]),
+          ]),
+          schema.nodes[FOOTNOTE_DEFINITION_NAME].create({ id: "b" }, [
+            schema.nodes.paragraph.create(null, [schema.text("note B")]),
+          ]),
+        ]),
+      ]),
+      0,
+      0,
+    );
+    const out = canonicalizePastedFootnotes(slice, schema);
+    expect(out).toBe(slice); // returned unchanged (same reference, content kept)
+    expect(listIds(out)).toEqual(["a", "b"]);
+    editor.destroy();
+  });
+
+  it("leaves an open (partial) slice untouched even if it carries a list", () => {
+    // An open slice (openStart/openEnd > 0) is a partial selection, not a
+    // standalone block, so it is returned as-is BEFORE any footnote handling.
+    const { editor, schema } = makeSchema();
+    const slice = new Slice(
+      Fragment.fromArray([
+        schema.nodes.paragraph.create(null, [
+          schema.nodes[FOOTNOTE_REFERENCE_NAME].create({ id: "a" }),
+        ]),
+        schema.nodes[FOOTNOTES_LIST_NAME].create(null, [
+          schema.nodes[FOOTNOTE_DEFINITION_NAME].create({ id: "a" }, [
+            schema.nodes.paragraph.create(null, [schema.text("A")]),
+          ]),
+        ]),
+      ]),
+      1,
+      1,
+    );
+    const out = canonicalizePastedFootnotes(slice, schema);
+    expect(out).toBe(slice);
+    editor.destroy();
+  });
+});

apps/client/src/features/editor/extensions/markdown-clipboard.ts

@@ -3,7 +3,14 @@ import { Extension } from "@tiptap/core";
 import { Plugin, PluginKey, TextSelection } from "@tiptap/pm/state";
 import { DOMParser, DOMSerializer, Fragment, Slice } from "@tiptap/pm/model";
 import { find } from "linkifyjs";
-import { markdownToHtml, htmlToMarkdown } from "@docmost/editor-ext";
+import {
+  markdownToHtml,
+  htmlToMarkdown,
+  canonicalizeFootnotes,
+  FOOTNOTES_LIST_NAME,
+  FOOTNOTE_REFERENCE_NAME,
+} from "@docmost/editor-ext";
+import type { Schema } from "@tiptap/pm/model";
 
 export const MarkdownClipboard = Extension.create({
   name: "markdownClipboard",
@@ -83,12 +90,25 @@ export const MarkdownClipboard = Extension.create({
             const body = elementFromString(parsed);
             normalizeTableColumnWidths(body);
 
-            const contentNodes = DOMParser.fromSchema(
+            const parsedSlice = DOMParser.fromSchema(
               this.editor.schema,
             ).parseSlice(body, {
               preserveWhitespace: true,
             });
 
+            // A markdown paste builds its ProseMirror fragment directly (DOM ->
+            // parseSlice), bypassing the editor's footnoteSyncPlugin, which never
+            // reorders an existing list. So a pasted markdown block whose footnote
+            // definitions are out of order (or contains orphan defs) would be
+            // stored out of order. Canonicalize the self-contained pasted block so
+            // its footnotes come out reference-ordered, deduped and orphan-free
+            // (issue #228). See canonicalizePastedFootnotes for why this is scoped
+            // to whole-block pastes that carry their own footnotesList.
+            const contentNodes = canonicalizePastedFootnotes(
+              parsedSlice,
+              this.editor.schema,
+            );
+
             tr.replaceRange(from, to, contentNodes);
             const insertEnd = tr.mapping.map(from, 1);
             tr.setSelection(TextSelection.near(tr.doc.resolve(Math.max(from, insertEnd - 2)), -1));
@@ -133,6 +153,54 @@ export const MarkdownClipboard = Extension.create({
   },
 });
 
+/**
+ * Reorder/dedup the footnotes of a SELF-CONTAINED pasted markdown block to the
+ * canonical invariant (the live footnoteSyncPlugin never reorders an existing
+ * list, so an out-of-order pasted block would otherwise persist out of order).
+ *
+ * Scoped deliberately to whole-block pastes (openStart/openEnd === 0) that carry
+ * their OWN footnotesList: canonicalizeFootnotes would synthesize empty
+ * definitions for any reference lacking a definition, which is correct for a
+ * standalone block but would be wrong for a reference-only paste that REUSES a
+ * footnote already defined in the target document — so those are left untouched
+ * for the paste/sync plugins to merge. Residual: when the pasted block is merged
+ * into a doc that already has footnotes, ordering RELATIVE to the pre-existing
+ * footnotes is still governed by the sync plugin (which does not reorder).
+ *
+ * Also requires at least one footnoteReference in the selection: a definitions-ONLY
+ * paste (`[^a]: …` with no `[^a]` reference in the same block) has no references,
+ * so canonicalizeFootnotes would drop the whole list and the paste would come out
+ * EMPTY — losing the pasted text. Such a block is left as-is for the sync plugin.
+ */
+export function canonicalizePastedFootnotes(slice: Slice, schema: Schema): Slice {
+  if (slice.openStart !== 0 || slice.openEnd !== 0) return slice;
+
+  let hasFootnotesList = false;
+  let hasReference = false;
+  slice.content.forEach((node) => {
+    if (node.type.name === FOOTNOTES_LIST_NAME) hasFootnotesList = true;
+    // footnoteReference is an inline atom, never a top-level slice child here
+    // (this function early-returns for open slices, so children are whole
+    // blocks), so it is only reachable by descending.
+    node.descendants((child) => {
+      if (child.type.name === FOOTNOTE_REFERENCE_NAME) hasReference = true;
+    });
+  });
+  if (!hasFootnotesList) return slice;
+  // No reference anywhere -> a definitions-only paste; canonicalizing would strip
+  // the reference-less list (empty paste). Leave it untouched.
+  if (!hasReference) return slice;
+
+  const content = slice.content.toJSON();
+  if (!Array.isArray(content)) return slice;
+
+  const canonical = canonicalizeFootnotes({ type: "doc", content }) as {
+    content?: unknown[];
+  };
+  const fragment = Fragment.fromJSON(schema, canonical.content ?? []);
+  return new Slice(fragment, 0, 0);
+}
+
 function elementFromString(value) {
   // add a wrapper to preserve leading and trailing whitespace
   const wrappedValue = `<body>${value}</body>`;

apps/server/src/core/page/services/page.service.footnote-canonicalize.spec.ts

@@ -0,0 +1,153 @@
+// Binding test for issue #228 must-fix #1 / test-coverage #12: footnote
+// canonicalization moved OUT of parseProsemirrorContent and is now applied only
+// on FULL-document writes (createPage, and updatePageContent with operation
+// 'replace'), NEVER on an append/prepend FRAGMENT.
+//
+// The Yjs encode / plain-text extract are stubbed (partial module mock keeps the
+// REAL canonicalizeFootnotes) and parseProsemirrorContent is spied to return the
+// raw fixture, so the test isolates the canonicalize BINDING from schema/Yjs.
+jest.mock('@docmost/editor-ext', () => {
+  const actual = jest.requireActual('@docmost/editor-ext');
+  return {
+    ...actual,
+    createYdocFromJson: jest.fn(() => Buffer.from([])),
+    jsonToText: jest.fn(() => ''),
+  };
+});
+
+import { PageService } from './page.service';
+
+const refNode = (id: string) => ({ type: 'footnoteReference', attrs: { id } });
+const defNode = (id: string, text: string) => ({
+  type: 'footnoteDefinition',
+  attrs: { id },
+  content: [{ type: 'paragraph', content: [{ type: 'text', text }] }],
+});
+const doc = (...content: any[]) => ({ type: 'doc', content });
+
+/** A full doc whose footnote definitions are OUT of reference order (b,a refs;
+ *  a,b defs) — canonicalization must reorder the definitions to [b, a]. */
+const outOfOrderFull = () =>
+  doc(
+    { type: 'paragraph', content: [{ type: 'text', text: 'x' }, refNode('b'), refNode('a')] },
+    { type: 'footnotesList', content: [defNode('a', 'A'), defNode('b', 'B')] },
+  );
+
+/** A definition-ONLY fragment (no references): canonicalizing it would drop the
+ *  whole footnotesList (referenceIds is empty) — i.e. LOSE the footnote. */
+const defOnlyFragment = () =>
+  doc({ type: 'footnotesList', content: [defNode('a', 'appended note')] });
+
+/** A reference-only fragment that REUSES an id defined elsewhere in the live
+ *  doc: canonicalizing it would synthesize a bogus empty footnotesList/def. */
+const refReuseFragment = () =>
+  doc({ type: 'paragraph', content: [{ type: 'text', text: 'more' }, refNode('a')] });
+
+function listDefIds(content: any): string[] {
+  const list = (content.content ?? []).find((n: any) => n.type === 'footnotesList');
+  return (list?.content ?? [])
+    .filter((n: any) => n.type === 'footnoteDefinition')
+    .map((n: any) => n.attrs?.id);
+}
+function hasFootnotesList(content: any): boolean {
+  return (content.content ?? []).some((n: any) => n.type === 'footnotesList');
+}
+
+describe('PageService footnote canonicalization binding (#228)', () => {
+  function makeService() {
+    let insertedContent: any = null;
+    let yjsPayload: any = null;
+
+    const pageRepo = {
+      insertPage: jest.fn(async (values: any) => {
+        insertedContent = values.content;
+        return { id: 'page-id', slugId: 'slug-id' };
+      }),
+    };
+    const generalQueue = { add: jest.fn().mockReturnValue({ catch: jest.fn() }) };
+    const collaborationGateway = {
+      handleYjsEvent: jest.fn(async (_evt: string, _name: string, payload: any) => {
+        yjsPayload = payload;
+      }),
+    };
+
+    const service = new PageService(
+      pageRepo as any,
+      {} as any, // pagePermissionRepo
+      {} as any, // attachmentRepo
+      {} as any, // db
+      {} as any, // storageService
+      {} as any, // attachmentQueue
+      {} as any, // aiQueue
+      generalQueue as any,
+      {} as any, // eventEmitter
+      collaborationGateway as any,
+      {} as any, // watcherService
+      {} as any, // transclusionService
+    );
+    // Isolate the canonicalize BINDING: return the raw fixture (a deep clone so
+    // canonicalize never mutates the caller's object) instead of running the
+    // real markdown/HTML/JSON parse + schema validation.
+    jest
+      .spyOn(service as any, 'parseProsemirrorContent')
+      .mockImplementation(async (content: any) => structuredClone(content));
+    jest.spyOn(service as any, 'nextPagePosition').mockResolvedValue('a0');
+
+    return { service, getInsertedContent: () => insertedContent, getYjsPayload: () => yjsPayload };
+  }
+
+  it('createPage (full write) canonicalizes footnotes into reference order', async () => {
+    const { service, getInsertedContent } = makeService();
+    await service.create('user-id', 'workspace-id', {
+      spaceId: 'space-id',
+      content: outOfOrderFull(),
+      format: 'json',
+    } as any);
+    // Definitions reordered to reference order [b, a].
+    expect(listDefIds(getInsertedContent())).toEqual(['b', 'a']);
+  });
+
+  it("updatePageContent operation 'replace' canonicalizes footnotes", async () => {
+    const { service, getYjsPayload } = makeService();
+    await service.updatePageContent(
+      'page-id',
+      outOfOrderFull(),
+      'replace' as any,
+      'json' as any,
+      { id: 'user-id' } as any,
+    );
+    expect(getYjsPayload().operation).toBe('replace');
+    expect(listDefIds(getYjsPayload().prosemirrorJson)).toEqual(['b', 'a']);
+  });
+
+  it("append of a definition-only fragment is NOT canonicalized (footnote preserved, not dropped)", async () => {
+    const { service, getYjsPayload } = makeService();
+    await service.updatePageContent(
+      'page-id',
+      defOnlyFragment(),
+      'append' as any,
+      'json' as any,
+      { id: 'user-id' } as any,
+    );
+    // Canonicalizing a reference-less fragment would DROP the whole list; the
+    // fragment must pass through untouched so the merge keeps the definition.
+    expect(getYjsPayload().operation).toBe('append');
+    expect(hasFootnotesList(getYjsPayload().prosemirrorJson)).toBe(true);
+    expect(listDefIds(getYjsPayload().prosemirrorJson)).toEqual(['a']);
+  });
+
+  it('prepend of a reference-reuse fragment is NOT canonicalized (no synthesized garbage list)', async () => {
+    const { service, getYjsPayload } = makeService();
+    await service.updatePageContent(
+      'page-id',
+      refReuseFragment(),
+      'prepend' as any,
+      'json' as any,
+      { id: 'user-id' } as any,
+    );
+    // Canonicalizing would synthesize a bogus empty footnotesList for the reused
+    // reference; the fragment must pass through with no list at all.
+    expect(getYjsPayload().operation).toBe('prepend');
+    expect(hasFootnotesList(getYjsPayload().prosemirrorJson)).toBe(false);
+  });
+});

apps/server/src/core/page/services/page.service.ts

@@ -52,7 +52,7 @@ import {
   INTERNAL_LINK_REGEX,
   extractPageSlugId,
 } from '../../../integrations/export/utils';
-import { markdownToHtml } from '@docmost/editor-ext';
+import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext';
 import { WatcherService } from '../../watcher/watcher.service';
 import { sql } from 'kysely';
 import { TransclusionService } from '../transclusion/transclusion.service';
@@ -160,9 +160,14 @@ export class PageService {
     let ydoc = undefined;
 
     if (createPageDto?.content && createPageDto?.format) {
-      const prosemirrorJson = await this.parseProsemirrorContent(
-        createPageDto.content,
-        createPageDto.format,
+      // createPage always writes a FULL document, so canonicalize footnotes to
+      // the editor's invariant before persisting (issue #228). Pure + idempotent
+      // + shape-safe: a doc with no footnotes is returned unchanged.
+      const prosemirrorJson = canonicalizeFootnotes(
+        await this.parseProsemirrorContent(
+          createPageDto.content,
+          createPageDto.format,
+        ),
       );
 
       content = prosemirrorJson;
@@ -343,7 +348,17 @@ export class PageService {
     format: ContentFormat,
     user: User,
   ): Promise<void> {
-    const prosemirrorJson = await this.parseProsemirrorContent(content, format);
+    let prosemirrorJson = await this.parseProsemirrorContent(content, format);
+
+    // Canonicalize footnotes ONLY for a full-document write ('replace'). For an
+    // append/prepend FRAGMENT, canonicalizing is semantically wrong (it would
+    // drop a definition-only fragment's list, or synthesize a duplicate empty
+    // definition for a fragment reusing an existing id) — the fragment merges
+    // into the live doc where the editor's footnoteSyncPlugin keeps the invariant
+    // (issue #228, must-fix #1).
+    if (operation === 'replace') {
+      prosemirrorJson = canonicalizeFootnotes(prosemirrorJson);
+    }
 
     const documentName = `page.${pageId}`;
     await this.collaborationGateway.handleYjsEvent(
@@ -1301,6 +1316,24 @@ export class PageService {
       }
     }
 
+    // NOTE: footnote canonicalization is intentionally NOT done here. This
+    // method serves BOTH full writes (createPage / updatePageContent with
+    // operation 'replace') AND fragment writes (append / prepend). Canonicalizing
+    // a FRAGMENT is semantically wrong — e.g. a definition-only fragment has no
+    // references, so the canonicalizer would drop its whole footnotesList (lost
+    // footnotes), and a fragment reusing an existing id would synthesize an empty
+    // duplicate definition. The canonicalizer therefore runs only at the
+    // FULL-DOCUMENT callers (createPage, and updatePageContent for 'replace'),
+    // never on a fragment (issue #228, must-fix #1).
+    // (Future consolidation, architecture B: the import services persist via a
+    // different path; folding all of these into one "prepare JSON for persist"
+    // helper would centralize the canonicalize call — left as follow-up.)
+    //
+    // ENFORCEMENT RULE (#228): any NEW FULL-document persist path MUST call
+    // `canonicalizeFootnotes(json)` before writing (see createPage and
+    // updatePageContent 'replace'); append/prepend FRAGMENT writes MUST NOT (it
+    // would drop or duplicate footnotes — that is exactly why this is per-call-site
+    // rather than a single wrapper here).
     try {
       jsonToNode(prosemirrorJson);
     } catch (err) {

apps/server/src/integrations/import/services/file-import-task.service.footnote-canonicalize.spec.ts

@@ -0,0 +1,150 @@
+// Importing FileImportTaskService transitively loads import-formatter.ts, which
+// imports the ESM-only @sindresorhus/slugify package (not in jest's transform
+// allowlist). slugify is irrelevant to the path under test, so it is mocked out
+// to keep the module graph loadable under ts-jest (mirrors the import.service spec).
+jest.mock('@sindresorhus/slugify', () => ({
+  __esModule: true,
+  default: (input: string) => String(input),
+}));
+// import-attachment.service.ts (loaded transitively for DI typing) imports the
+// ESM-only `p-limit` / `image-dimensions`; neither is exercised on the path under
+// test, so stub them so the module graph loads under ts-jest.
+jest.mock('p-limit', () => ({
+  __esModule: true,
+  default: () => (fn: any) => fn(),
+}));
+jest.mock('image-dimensions', () => ({
+  __esModule: true,
+  imageDimensionsFromData: () => undefined,
+}));
+
+import { promises as fs } from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { FileImportTaskService } from './file-import-task.service';
+import { ImportService } from './import.service';
+
+/**
+ * Binding test for issue #228 / review #5: FileImportTaskService.processGenericImport
+ * is a NON-editor write path (markdownToHtml -> processHTML -> JSON, never runs
+ * footnoteSyncPlugin), so it canonicalizes footnotes before persisting. This pins
+ * that binding — the same one import.service has a spec for — which previously had
+ * NO spec at all.
+ *
+ * The markdown -> HTML -> ProseMirror conversion is REAL (a real ImportService,
+ * its createYdoc stubbed); the filesystem is a real temp dir with one .md file;
+ * the DB transaction is stubbed to capture the persisted page content.
+ */
+
+// Out-of-order references (c, a, b), a REUSED reference ([^a] twice), and an
+// ORPHAN definition ([^z], never referenced).
+const MARKDOWN = [
+  '# Title',
+  '',
+  'Body refs [^c] and [^a] and [^b] and again [^a].',
+  '',
+  '[^a]: note A',
+  '[^b]: note B',
+  '[^c]: note C',
+  '[^z]: orphan note',
+].join('\n');
+
+function footnoteListIds(content: any): string[] {
+  const list = (content?.content ?? []).find(
+    (n: any) => n.type === 'footnotesList',
+  );
+  return (list?.content ?? [])
+    .filter((n: any) => n.type === 'footnoteDefinition')
+    .map((n: any) => n.attrs?.id);
+}
+
+// A permissive chainable stub for the spaces lookup (selectFrom(...).select(...)
+// .where(...).executeTakeFirst()).
+function chainable(result: any): any {
+  const proxy: any = new Proxy(function () {}, {
+    get: (_t, prop) => {
+      if (prop === 'executeTakeFirst') return async () => result;
+      if (prop === 'execute') return async () => [];
+      return () => proxy;
+    },
+  });
+  return proxy;
+}
+
+describe('FileImportTaskService.processGenericImport — footnote canonicalization (#228)', () => {
+  it('orders footnotes by first reference, dedupes reuse, and drops orphans on zip import', async () => {
+    const extractDir = await fs.mkdtemp(path.join(os.tmpdir(), 'fit-canon-'));
+    await fs.writeFile(path.join(extractDir, 'note.md'), MARKDOWN, 'utf-8');
+
+    // Real ImportService for the html -> JSON conversion; stub the yjs encode.
+    const importService = new ImportService(
+      {} as any,
+      {} as any,
+      {} as any,
+      {} as any,
+    );
+    jest
+      .spyOn(importService as any, 'createYdoc')
+      .mockResolvedValue(Buffer.from([]) as any);
+
+    let captured: any = null;
+    const trx = {
+      insertInto: (table: string) => ({
+        values: (v: any) => {
+          if (table === 'pages') captured = v;
+          return { execute: async () => {} };
+        },
+      }),
+    };
+    const db: any = {
+      selectFrom: () => chainable({ slug: 'space-slug' }),
+      transaction: () => ({ execute: (fn: any) => fn(trx) }),
+    };
+
+    const importAttachmentService = {
+      processAttachments: async ({ html }: any) => html,
+    };
+    const backlinkRepo = { insertBacklink: jest.fn() };
+    const eventEmitter = { emit: jest.fn() };
+    const auditService = { logBatchWithContext: jest.fn() };
+
+    const pageService = { nextPagePosition: async () => 'a0' };
+
+    const service = new FileImportTaskService(
+      {} as any, // storageService
+      importService as any,
+      pageService as any,
+      backlinkRepo as any,
+      db,
+      importAttachmentService as any,
+      eventEmitter as any,
+      auditService as any,
+    );
+
+    const fileTask: any = {
+      id: 'task-1',
+      source: 'generic',
+      spaceId: 'space-1',
+      workspaceId: 'ws-1',
+      creatorId: 'user-1',
+    };
+
+    try {
+      await service.processGenericImport({ extractDir, fileTask });
+
+      expect(captured).toBeTruthy();
+      const content = captured.content;
+      // Reference order is c, a, b (NOT the markdown definition order a, b, c).
+      expect(footnoteListIds(content)).toEqual(['c', 'a', 'b']);
+      // Orphan [^z] dropped; reused [^a] collapses to one definition; one list.
+      expect(footnoteListIds(content)).not.toContain('z');
+      const lists = (content.content ?? []).filter(
+        (n: any) => n.type === 'footnotesList',
+      );
+      expect(lists).toHaveLength(1);
+      expect(footnoteListIds(content).filter((id) => id === 'a')).toHaveLength(1);
+    } finally {
+      await fs.rm(extractDir, { recursive: true, force: true });
+    }
+  });
+});

apps/server/src/integrations/import/services/file-import-task.service.ts

@@ -18,7 +18,7 @@ import { generateSlugId } from '../../../common/helpers';
 import { v7 } from 'uuid';
 import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
 import { FileTask, InsertablePage } from '@docmost/db/types/entity.types';
-import { markdownToHtml } from '@docmost/editor-ext';
+import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext';
 import { getProsemirrorContent } from '../../../common/helpers/prosemirror/utils';
 import { formatImportHtml } from '../utils/import-formatter';
 import {
@@ -496,9 +496,19 @@ export class FileImportTaskService {
               await this.importService.processHTML(html),
             );
 
-            const { title, prosemirrorJson } =
+            const { title, prosemirrorJson: extractedJson } =
               this.importService.extractTitleAndRemoveHeading(pmState);
 
+            // Canonicalize footnote topology on this non-editor write path
+            // (markdownToHtml/processHTML never runs footnoteSyncPlugin), so a
+            // zip-imported page's footnotes are reference-ordered, deduped, and
+            // orphan-free like the editor's invariant (issue #228). Pure +
+            // idempotent + shape-safe; a footnote-free doc is unchanged.
+            // (Future consolidation, architecture B: like import.service, this
+            // path persists directly rather than via PageService — a shared
+            // "prepare JSON for persist" helper would centralize this call.)
+            const prosemirrorJson = canonicalizeFootnotes(extractedJson);
+
             const insertablePage: InsertablePage = {
               id: page.id,
               slugId: page.slugId,

apps/server/src/integrations/import/services/import.service.footnote-canonicalize.spec.ts

@@ -0,0 +1,139 @@
+// Importing ImportService transitively loads import-formatter.ts, which imports
+// the ESM-only @sindresorhus/slugify package (not in jest's transform
+// allowlist). slugify is irrelevant to the path under test, so it is mocked out
+// to keep the module graph loadable under ts-jest.
+jest.mock('@sindresorhus/slugify', () => ({
+  __esModule: true,
+  default: (input: string) => String(input),
+}));
+
+import { ImportService } from './import.service';
+import { canonicalizeFootnotes } from '@docmost/editor-ext';
+
+/**
+ * Integration-ish test for the USER-FACING markdown import path
+ * (`ImportService.importPage`). It exercises the REAL markdown -> HTML -> JSON
+ * conversion and asserts that the stored page content has its footnotes
+ * canonicalized — the gap that issue #228 fixes: the import path builds
+ * ProseMirror JSON directly (never running the editor's footnoteSyncPlugin), so
+ * before this wiring the stored footnotes kept the markdown's physical
+ * definition order (out of order vs. references), retained orphan definitions,
+ * and did not collapse reused references.
+ *
+ * The DB/ydoc side-effects are stubbed: `getNewPagePosition` (DB query) and
+ * `createYdoc` (Yjs encode) are spied, and `pageRepo.insertPage` captures the
+ * persisted `content`. Everything between markdown and persistence is REAL.
+ */
+
+// Out-of-order references (c, a, b), a REUSED reference ([^a] twice -> one
+// footnote), and an ORPHAN definition ([^z], never referenced).
+const MARKDOWN = [
+  '# Title',
+  '',
+  'Body refs [^c] and [^a] and [^b] and again [^a].',
+  '',
+  '[^a]: note A',
+  '[^b]: note B',
+  '[^c]: note C',
+  '[^z]: orphan note',
+].join('\n');
+
+function makeFile(filename: string, contents: string) {
+  return {
+    filename,
+    toBuffer: async () => Buffer.from(contents),
+  } as any;
+}
+
+function makeService() {
+  let captured: any = null;
+  const pageRepo = {
+    insertPage: jest.fn(async (values: any) => {
+      captured = values;
+      return { id: 'page-id', slugId: 'slug-id' };
+    }),
+  };
+  const service = new ImportService(
+    pageRepo as any,
+    {} as any,
+    {} as any,
+    {} as any,
+  );
+  jest.spyOn(service as any, 'getNewPagePosition').mockResolvedValue('a0');
+  jest
+    .spyOn(service as any, 'createYdoc')
+    .mockResolvedValue(Buffer.from([]) as any);
+  return { service, pageRepo, getCaptured: () => captured };
+}
+
+/** List the footnote-definition ids of the (single) footnotesList, in order. */
+function footnoteListIds(content: any): string[] {
+  const list = (content.content ?? []).find(
+    (n: any) => n.type === 'footnotesList',
+  );
+  if (!list) return [];
+  return (list.content ?? [])
+    .filter((n: any) => n.type === 'footnoteDefinition')
+    .map((n: any) => n.attrs?.id);
+}
+
+function definitionText(content: any, id: string): string | undefined {
+  const list = (content.content ?? []).find(
+    (n: any) => n.type === 'footnotesList',
+  );
+  const def = (list?.content ?? []).find(
+    (n: any) => n.type === 'footnoteDefinition' && n.attrs?.id === id,
+  );
+  return def?.content?.[0]?.content?.[0]?.text;
+}
+
+describe('ImportService.importPage — footnote canonicalization (#228)', () => {
+  it('orders footnotes by first reference, dedupes reuse, and drops orphans', async () => {
+    const { service, getCaptured } = makeService();
+
+    await service.importPage(
+      Promise.resolve(makeFile('note.md', MARKDOWN)),
+      'user-id',
+      'space-id',
+      'workspace-id',
+    );
+
+    const content = getCaptured().content;
+    expect(content).toBeTruthy();
+
+    // Reference order is c, a, b (NOT the markdown definition order a, b, c).
+    expect(footnoteListIds(content)).toEqual(['c', 'a', 'b']);
+
+    // Definitions preserved and attached to the right ids.
+    expect(definitionText(content, 'c')).toBe('note C');
+    expect(definitionText(content, 'a')).toBe('note A');
+    expect(definitionText(content, 'b')).toBe('note B');
+
+    // Orphan definition [^z] is dropped.
+    expect(footnoteListIds(content)).not.toContain('z');
+
+    // Reused [^a] yields exactly ONE definition, and exactly one list.
+    const lists = (content.content ?? []).filter(
+      (n: any) => n.type === 'footnotesList',
+    );
+    expect(lists).toHaveLength(1);
+    expect(footnoteListIds(content).filter((id) => id === 'a')).toHaveLength(1);
+  });
+
+  it('is idempotent: canonicalizing the stored output again is a no-op', async () => {
+    const { service, getCaptured } = makeService();
+    await service.importPage(
+      Promise.resolve(makeFile('note.md', MARKDOWN)),
+      'user-id',
+      'space-id',
+      'workspace-id',
+    );
+    const stored = getCaptured().content;
+
+    // The stored content is already canonical; running the canonicalizer a second
+    // time must not change it (safe to wire into every write path).
+    const second = canonicalizeFootnotes(stored);
+    expect(second).toEqual(stored);
+    expect(footnoteListIds(second)).toEqual(['c', 'a', 'b']);
+  });
+});

apps/server/src/integrations/import/services/import.service.ts

@@ -17,7 +17,7 @@ import {
 import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
 import { TiptapTransformer } from '@hocuspocus/transformer';
 import * as Y from 'yjs';
-import { markdownToHtml } from '@docmost/editor-ext';
+import { markdownToHtml, canonicalizeFootnotes } from '@docmost/editor-ext';
 import {
   FileTaskStatus,
   FileTaskType,
@@ -85,7 +85,17 @@ export class ImportService {
 
     const extracted = this.extractTitleAndRemoveHeading(prosemirrorState);
     const title = extracted.title;
-    const prosemirrorJson = extracted.prosemirrorJson;
+    // Imported markdown/HTML is built via markdownToHtml -> htmlToJson, which
+    // never runs the editor's footnoteSyncPlugin, so the footnote topology keeps
+    // the source's PHYSICAL definition order (out of order vs. references),
+    // retains orphan definitions, and is not deduped. Canonicalize before
+    // persisting so the stored page matches the editor's invariant (issue #228).
+    // Pure + idempotent + shape-safe: a doc with no footnotes is unchanged.
+    // (Future consolidation, architecture B: this import path persists directly
+    // via pageRepo.insertPage rather than through PageService.createPage, so the
+    // canonicalize call lives here; folding both into one "prepare JSON for
+    // persist" helper is a sensible follow-up.)
+    const prosemirrorJson = canonicalizeFootnotes(extracted.prosemirrorJson);
 
     const pageTitle = title || fileName;