feat(footnotes): inline authoring + deterministic server-side canonicalization

Make footnotes author-inline: the agent/tool inserts a footnote at its point
of use (anchor + text) and the numbering plus the bottom list are DERIVED
deterministically server-side. The agent has no access to footnotesList and
cannot desync — out-of-order lists, orphan definitions, and raw trailing
[^id] blocks become structurally impossible.

editor-ext:
- canonicalizeFootnotes(docJSON) -> docJSON: a pure, EditorView-free port of
  footnoteSyncPlugin's end-state. Distinct reference ids in document order are
  the source of truth; exactly one trailing footnotesList holds one definition
  per referenced id in reference order (reusing the existing node or
  synthesizing an empty one); orphans dropped; duplicate definitions resolved
  deterministically (first wins, never lost); idempotent.
- Unit tests + a golden parity suite: on every editor-reachable steady state
  the live footnoteSyncPlugin's JSON is a canonicalize no-op (byte-for-byte
  parity), and the canonicalizer additionally repairs the out-of-order list a
  non-editor write produces.

mcp:
- footnote-canonicalize.ts: behavioural mirror of the editor-ext canonicalizer
  (the MCP package is intentionally decoupled from the editor barrel, like
  footnote-lex/docmost-schema), plus footnoteContentKey for content dedup.
- Auto-canonicalize on EVERY write path: markdownToProseMirror (fixes import
  ordering), update_page_json, and after every docmost_transform. Idempotent,
  so it is a no-op when footnotes are already canonical.
- insert_footnote tool + insertInlineFootnote: anchor + markdown text -> a
  mark-safe footnoteReference and a content-dedup'd definition; the list and
  numbering are derived. Same-content footnotes reuse one number/definition.
- canonicalizeFootnotes + insertInlineFootnote exposed as docmost_transform
  sandbox helpers.

Tests: editor-ext 157 green; MCP 325 green; server + client tsc clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
claude code agent 227
2026-06-27 06:35:25 +03:00
parent 904f7b4303
commit 30cb9d293c
14 changed files with 1685 additions and 17 deletions

View File

@@ -0,0 +1,327 @@
import { describe, it, expect } from 'vitest';
import { Editor, getSchema } from '@tiptap/core';
import { Document } from '@tiptap/extension-document';
import { Paragraph } from '@tiptap/extension-paragraph';
import { Text } from '@tiptap/extension-text';
import { FootnoteReference } from './footnote-reference';
import { FootnotesList } from './footnotes-list';
import { FootnoteDefinition } from './footnote-definition';
import { canonicalizeFootnotes } from './footnote-canonicalize';
import {
collectReferenceIds,
computeFootnoteNumbers,
FOOTNOTE_REFERENCE_NAME,
FOOTNOTES_LIST_NAME,
FOOTNOTE_DEFINITION_NAME,
} from './footnote-util';
import { Node as PMNode } from '@tiptap/pm/model';
const extensions = [
Document,
Paragraph,
Text,
FootnoteReference,
FootnotesList,
FootnoteDefinition,
];
const ref = (id: string) => ({ type: FOOTNOTE_REFERENCE_NAME, attrs: { id } });
const def = (id: string, text?: string) => ({
type: FOOTNOTE_DEFINITION_NAME,
attrs: { id },
content: [
text
? { type: 'paragraph', content: [{ type: 'text', text }] }
: { type: 'paragraph' },
],
});
const list = (...defs: any[]) => ({ type: FOOTNOTES_LIST_NAME, content: defs });
const para = (...inline: any[]) => ({ type: 'paragraph', content: inline });
/** Find every node of `type`, document order. */
function findAll(node: any, type: string, acc: any[] = []): any[] {
if (!node || typeof node !== 'object') return acc;
if (node.type === type) acc.push(node);
if (Array.isArray(node.content)) {
for (const c of node.content) findAll(c, type, acc);
}
return acc;
}
/** Physical id order of the definitions in the (single) footnotesList. */
function defOrder(doc: any): string[] {
return findAll(doc, FOOTNOTE_DEFINITION_NAME).map((d) => d.attrs.id);
}
const schema = getSchema(extensions);
/** Reference order (distinct, document order) computed via the shared util. */
function refOrder(doc: any): string[] {
return collectReferenceIds(PMNode.fromJSON(schema, doc));
}
describe('canonicalizeFootnotes (pure JSON)', () => {
it('orders definitions by FIRST reference (out-of-order list -> 1..N)', () => {
// References appear b, a, d, c; the bottom list is in a different (import)
// order. The canonical list must follow reference order so reading it top to
// bottom yields numbers 1..N.
const doc = {
type: 'doc',
content: [
para(
{ type: 'text', text: 'x' },
ref('b'),
ref('a'),
ref('d'),
ref('c'),
),
list(def('a', 'A'), def('c', 'C'), def('b', 'B'), def('d', 'D')),
],
};
const out = canonicalizeFootnotes(doc);
expect(defOrder(out)).toEqual(['b', 'a', 'd', 'c']);
// The physical definition order now matches reference order, so the derived
// numbers (1..N) run sequentially down the list.
expect(refOrder(out)).toEqual(['b', 'a', 'd', 'c']);
const numbers = computeFootnoteNumbers(PMNode.fromJSON(schema, out));
expect(numbers.get('b')).toBe(1);
expect(numbers.get('a')).toBe(2);
expect(numbers.get('d')).toBe(3);
expect(numbers.get('c')).toBe(4);
});
it('numbers run 1..N down the canonical list', () => {
const doc = {
type: 'doc',
content: [
para({ type: 'text', text: 'x' }, ref('b'), ref('a'), ref('c')),
list(def('a', 'A'), def('c', 'C'), def('b', 'B')),
],
};
const out = canonicalizeFootnotes(doc);
// Definition order == reference order == 1,2,3 reading down.
expect(defOrder(out)).toEqual(['b', 'a', 'c']);
});
it('drops an orphan definition (no matching reference)', () => {
const doc = {
type: 'doc',
content: [
para({ type: 'text', text: 'x' }, ref('a')),
list(def('a', 'A'), def('orphan', 'O')),
],
};
const out = canonicalizeFootnotes(doc);
expect(defOrder(out)).toEqual(['a']);
expect(findAll(out, FOOTNOTE_DEFINITION_NAME)).toHaveLength(1);
});
it('with NO references, removes the footnotesList entirely', () => {
const doc = {
type: 'doc',
content: [
para({ type: 'text', text: 'plain' }),
list(def('orphan', 'O')),
],
};
const out = canonicalizeFootnotes(doc);
expect(findAll(out, FOOTNOTES_LIST_NAME)).toHaveLength(0);
expect(findAll(out, FOOTNOTE_DEFINITION_NAME)).toHaveLength(0);
});
it('reuse: repeated references collapse to ONE definition/number', () => {
const doc = {
type: 'doc',
content: [
para(ref('d'), { type: 'text', text: ' a ' }, ref('d'), ref('d')),
list(def('d', 'shared')),
],
};
const out = canonicalizeFootnotes(doc);
// One definition; the three references keep id "d".
expect(defOrder(out)).toEqual(['d']);
expect(
findAll(out, FOOTNOTE_REFERENCE_NAME).map((r) => r.attrs.id),
).toEqual(['d', 'd', 'd']);
});
it('duplicate definitions: first wins, the rest are dropped (never resurface as orphans)', () => {
const doc = {
type: 'doc',
content: [
para({ type: 'text', text: 'x' }, ref('d')),
list(def('d', 'first'), def('d', 'second'), def('d', 'third')),
],
};
const out = canonicalizeFootnotes(doc);
const defs = findAll(out, FOOTNOTE_DEFINITION_NAME);
expect(defs.map((d) => d.attrs.id)).toEqual(['d']);
expect(defs[0].content[0].content[0].text).toBe('first');
});
it('synthesizes an empty definition for a reference that has none', () => {
const doc = {
type: 'doc',
content: [para({ type: 'text', text: 'x' }, ref('missing'))],
};
const out = canonicalizeFootnotes(doc);
expect(defOrder(out)).toEqual(['missing']);
const list0 = findAll(out, FOOTNOTES_LIST_NAME);
expect(list0).toHaveLength(1);
});
it('merges multiple footnotesList nodes into one', () => {
const doc = {
type: 'doc',
content: [
para({ type: 'text', text: 'a' }, ref('x'), ref('y')),
list(def('x', 'X')),
para({ type: 'text', text: 'tail' }),
list(def('y', 'Y')),
],
};
const out = canonicalizeFootnotes(doc);
expect(findAll(out, FOOTNOTES_LIST_NAME)).toHaveLength(1);
expect(defOrder(out)).toEqual(['x', 'y']);
});
it('places the single list before trailing empty paragraphs', () => {
const doc = {
type: 'doc',
content: [
para({ type: 'text', text: 'x' }, ref('a')),
list(def('a', 'A')),
{ type: 'paragraph' },
],
};
const out = canonicalizeFootnotes(doc);
const last = out.content[out.content.length - 1];
expect(last.type).toBe('paragraph');
expect(out.content[out.content.length - 2].type).toBe(FOOTNOTES_LIST_NAME);
});
it('is idempotent: canonicalize(canonicalize(x)) === canonicalize(x)', () => {
const doc = {
type: 'doc',
content: [
para({ type: 'text', text: 'x' }, ref('b'), ref('a')),
list(def('a', 'A'), def('b', 'B'), def('orphan', 'O')),
],
};
const once = canonicalizeFootnotes(doc);
const twice = canonicalizeFootnotes(once);
expect(twice).toEqual(once);
});
it('does not mutate its input', () => {
const doc = {
type: 'doc',
content: [
para({ type: 'text', text: 'x' }, ref('a')),
list(def('orphan', 'O')),
],
};
const snapshot = JSON.parse(JSON.stringify(doc));
canonicalizeFootnotes(doc);
expect(doc).toEqual(snapshot);
});
});
/**
* GOLDEN PARITY against the live `footnoteSyncPlugin`. The server canonicalizer
* must produce EXACTLY what the editor keeps. For every editor-reachable steady
* state (the list is already reference-ordered there), driving a real editor to
* convergence and then running `canonicalizeFootnotes` on its JSON must be a
* byte-for-byte no-op — proving the server output is identical to the editor's.
*/
describe('canonicalizeFootnotes golden parity with footnoteSyncPlugin', () => {
function makeEditor(content: any) {
return new Editor({ extensions, content });
}
/** Load `content`, fire one local edit so the sync plugin converges, return JSON. */
function pluginSteadyState(content: any): any {
const editor = makeEditor(content);
// A local doc change triggers footnoteSyncPlugin.appendTransaction.
editor.commands.insertContentAt(1, ' ');
const json = editor.state.doc.toJSON();
editor.destroy();
return json;
}
const corpus: Array<{ name: string; content: any }> = [
{
name: 'plain ref + def',
content: {
type: 'doc',
content: [para({ type: 'text', text: 'a' }, ref('x')), list(def('x', 'X'))],
},
},
{
name: 'two refs, two defs in reference order',
content: {
type: 'doc',
content: [
para({ type: 'text', text: 'a' }, ref('x'), { type: 'text', text: 'b' }, ref('y')),
list(def('x', 'X'), def('y', 'Y')),
],
},
},
{
name: 'orphan definition gets removed',
content: {
type: 'doc',
content: [para({ type: 'text', text: 'a' }, ref('x')), list(def('x', 'X'), def('orphan', 'O'))],
},
},
{
name: 'reference missing its definition (synth empty)',
content: {
type: 'doc',
content: [para({ type: 'text', text: 'a' }, ref('x'))],
},
},
{
name: 'reuse: repeated references, one definition',
content: {
type: 'doc',
content: [
para(ref('d'), { type: 'text', text: ' a ' }, ref('d'), ref('d')),
list(def('d', 'shared')),
],
},
},
{
name: 'no footnotes at all',
content: {
type: 'doc',
content: [para({ type: 'text', text: 'just text' })],
},
},
];
for (const { name, content } of corpus) {
it(`steady state is a canonicalize no-op: ${name}`, () => {
const steady = pluginSteadyState(content);
expect(canonicalizeFootnotes(steady)).toEqual(steady);
});
}
it('the canonicalizer and the editor agree on reference order and definition set', () => {
const content = {
type: 'doc',
content: [
para({ type: 'text', text: 'a' }, ref('x'), { type: 'text', text: 'b' }, ref('y')),
list(def('y', 'Y'), def('x', 'X')), // physically reversed
],
};
const steady = pluginSteadyState(content);
const canon = canonicalizeFootnotes(content);
// Same reference order and same DEFINITION SET (ids) in both, even though the
// physical list order may differ (the plugin preserves node identity, the
// canonicalizer reorders). Numbering — derived from reference order — matches.
expect(refOrder(steady)).toEqual(['x', 'y']);
expect(defOrder(canon)).toEqual(['x', 'y']);
expect(new Set(defOrder(steady))).toEqual(new Set(defOrder(canon)));
});
});

View File

@@ -0,0 +1,190 @@
import {
FOOTNOTE_REFERENCE_NAME,
FOOTNOTES_LIST_NAME,
FOOTNOTE_DEFINITION_NAME,
deriveFootnoteId,
} from './footnote-util';
/**
* Server-side, EditorView-free port of the footnote integrity invariant that
* `footnoteSyncPlugin` maintains in the live editor. Where the plugin is an
* `appendTransaction` that only runs inside a ProseMirror `EditorView`, this is
* a PURE function over ProseMirror JSON: `canonicalizeFootnotes(doc) -> doc`.
*
* It exists because every NON-editor write path (the MCP `markdownToProseMirror`
* importer, `update_page_json`, `docmost_transform`, the future git-sync writer)
* builds ProseMirror JSON directly via `TiptapTransformer`/`updateYFragment`,
* which NEVER runs the editor's plugins — so the canonical footnote topology was
* never enforced on those writes. That is the root cause of the symptom in the
* issue: footnotes rendered out of order (`1, 4, 2, 3, …`), a raw trailing
* `[^id]: …` block, and orphan definitions, all of which are simply the result
* of content written PAST the canonicalizer.
*
* The desired end-state (identical to the plugin's) is:
*
* 1. Reference ids in DOCUMENT ORDER are the single source of truth for which
* definitions exist and in what order (numbering is derived from this, see
* `computeFootnoteNumbers`). Repeated references that share an id are REUSE
* (one footnote, one number, one definition) — never re-id'd.
* 2. Exactly ONE `footnotesList`, holding one definition per referenced id in
* REFERENCE order, reusing the existing definition node (content preserved)
* or synthesizing an empty one when missing. The list sits after the last
* meaningful block (only trailing empty paragraphs may follow it).
* 3. Orphan definitions (no matching reference) are dropped.
* 4. Duplicate DEFINITIONS (two nodes sharing an id) are resolved
* deterministically: the first keeps the id; each later duplicate is re-id'd
* via `deriveFootnoteId` (never random) so it is never silently lost — and,
* lacking a matching reference, it then falls under the orphan policy and is
* dropped. This matches the editor's never-lose-by-collision rule and the
* importer's first-wins rule (both converge to "one definition per id").
* 5. Idempotent: a document that already satisfies the invariant is returned
* structurally unchanged (the existing definition/list nodes are reused
* verbatim), so re-running the canonicalizer — or running it on a write that
* the editor already canonicalized — is a no-op. This is what makes it safe
* to wire into EVERY write path without spurious mutations / git-sync churn.
*
* Divergence from the live plugin (intentional): the plugin preserves the
* PHYSICAL order of existing definition nodes to keep their Yjs/CRDT subtree
* identity stable across collaborators (numbering is decoration-derived, so the
* displayed numbers are correct regardless of physical order). This function has
* no live CRDT to protect, so it physically REORDERS the list into reference
* order — which is exactly the repair the out-of-order import needs. On every
* editor-reachable steady state (where the list is already reference-ordered) the
* two agree byte-for-byte; see the golden test.
*
* Pure: deep-clones its input, never mutates the caller's object, and is
* deterministic (no `Math.random`/`Date.now`).
*/
export function canonicalizeFootnotes<T = any>(doc: T): T {
if (
doc == null ||
typeof doc !== 'object' ||
!Array.isArray((doc as any).content)
) {
return doc;
}
const out = cloneJson(doc) as any;
// 1) Distinct reference ids in document order (deep — references can live in
// callouts, tables, list items, ...). This is the ordering/numbering truth.
const referenceIds: string[] = [];
const seenRefIds = new Set<string>();
collectReferenceIds(out, referenceIds, seenRefIds);
// 2) Every definition node in document order (deep — defs normally live inside
// one or more `footnotesList` blocks, but we tolerate stray placements).
const defNodes: any[] = [];
collectDefinitions(out, defNodes);
// 3) Resolve the id topology deterministically. The first definition for an id
// keeps it; a later duplicate is re-id'd to a fresh derived id (never lost),
// which — having no matching reference — is dropped as an orphan in step 4.
const taken = new Set<string>(referenceIds);
for (const d of defNodes) {
const id = d?.attrs?.id;
if (id) taken.add(id);
}
const occurrenceOf = new Map<string, number>();
const seenDefIds = new Set<string>();
// finalId -> definition node (the node reference inside `out`).
const defByFinalId = new Map<string, any>();
for (const d of defNodes) {
const origId = d?.attrs?.id;
if (!origId) continue;
if (!seenDefIds.has(origId)) {
seenDefIds.add(origId);
defByFinalId.set(origId, d);
} else {
const next = (occurrenceOf.get(origId) ?? 1) + 1;
occurrenceOf.set(origId, next);
const newId = deriveFootnoteId(origId, next, taken);
taken.add(newId);
defByFinalId.set(newId, d);
}
}
// 4) Build the ordered definition list: one per referenced id, in REFERENCE
// order, reusing the existing node (content preserved, id normalized) or
// synthesizing an empty definition. Definitions whose final id is NOT
// referenced are orphans and are simply never added.
const orderedDefs: any[] = [];
for (const id of referenceIds) {
const existing = defByFinalId.get(id);
if (existing) {
const node = cloneJson(existing);
node.attrs = { ...(node.attrs ?? {}), id };
orderedDefs.push(node);
} else {
orderedDefs.push(emptyDefinition(id));
}
}
// 5) Strip every existing top-level footnotesList; we rebuild a single one.
const top: any[] = out.content.filter(
(n: any) => !(n && n.type === FOOTNOTES_LIST_NAME),
);
// 6) No references -> there must be NO list at all.
if (referenceIds.length === 0) {
out.content = top;
return out;
}
// 7) Insert exactly one footnotesList after the last meaningful (non-empty
// paragraph) block, so it coexists with a trailing-node empty paragraph.
let insertAt = top.length;
while (insertAt > 0 && isEmptyParagraph(top[insertAt - 1])) insertAt--;
top.splice(insertAt, 0, { type: FOOTNOTES_LIST_NAME, content: orderedDefs });
out.content = top;
return out;
}
/** A fresh empty definition node for a referenced id with no definition. */
function emptyDefinition(id: string): any {
return {
type: FOOTNOTE_DEFINITION_NAME,
attrs: { id },
content: [{ type: 'paragraph' }],
};
}
function isEmptyParagraph(node: any): boolean {
return (
!!node &&
node.type === 'paragraph' &&
(!Array.isArray(node.content) || node.content.length === 0)
);
}
/** Collect DISTINCT footnoteReference ids in document order (first appearance). */
function collectReferenceIds(
node: any,
out: string[],
seen: Set<string>,
): void {
if (!node || typeof node !== 'object') return;
if (node.type === FOOTNOTE_REFERENCE_NAME) {
const id = node?.attrs?.id;
if (id && !seen.has(id)) {
seen.add(id);
out.push(id);
}
}
if (Array.isArray(node.content)) {
for (const child of node.content) collectReferenceIds(child, out, seen);
}
}
/** Collect every footnoteDefinition node in document order. */
function collectDefinitions(node: any, out: any[]): void {
if (!node || typeof node !== 'object') return;
if (node.type === FOOTNOTE_DEFINITION_NAME) out.push(node);
if (Array.isArray(node.content)) {
for (const child of node.content) collectDefinitions(child, out);
}
}
function cloneJson<T>(v: T): T {
if (typeof structuredClone === 'function') return structuredClone(v);
return JSON.parse(JSON.stringify(v)) as T;
}

View File

@@ -4,3 +4,4 @@ export * from "./footnotes-list";
export * from "./footnote-definition";
export * from "./footnote-numbering";
export * from "./footnote-sync";
export * from "./footnote-canonicalize";