feat(git-sync): двусторонний Docmost↔git синк на унифицированном конвертере (#359)
Схлопнутая дельта feat/git-sync-2 поверх актуального develop (3085ec1b). Почему squash, а не буквальный rebase: в ветке 113 коммитов, из которых develop уже впитал ~80 в курированном виде при унификации конвертера (#326/#293) — но не patch-identical, поэтому они не отваливаются сами; а коммит, УДАЛЯЮЩИЙ вендоренную копию конвертера, апстримный и в replay-набор не входит, так что наивный replay заново добавил бы вендоренные копии. Поэтому корректный итог — «develop + чистая net-дельта ветки», сведённая 3-way мержом (merge-base5336f06d). Net-дельта: серверный git-sync модуль (GitSyncModule/orchestrator/HTTP), движок git-sync (layout/reconcile/pull/push/stabilize + QA), два фикса round-trip/data-loss конвертера поверх УНИФИЦИРОВАННОГО @docmost/prosemirror-markdown (вендоренной копии больше нет — git-sync целиком на унифицированном конвертере, поведение унифицированного принято за эталон), e2e-скрипты, доки. Конфликты (4) слиты объединением: main.ts (метрики + GitHttpService), apps/server/package.json (pretest-суперсет; moduleNameMapper: git-sync→src и .js-strip оставлены, а bare-specifier @docmost/prosemirror-markdown→src УБРАН — серверный jest выровнен на develop-подход #345 со сборённым пакетом), .env.example (метрики + GIT_SYNC блоки), AGENTS.md (строки про пакеты; устаревшая заметка про «три hand-synced копии схемы» в git-sync поправлена — схема теперь только в @docmost/prosemirror-markdown). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -28,6 +28,7 @@ import { ClsModule } from 'nestjs-cls';
|
||||
import { NoopAuditModule } from './integrations/audit/audit.module';
|
||||
import { ThrottleModule } from './integrations/throttle/throttle.module';
|
||||
import { McpModule } from './integrations/mcp/mcp.module';
|
||||
import { GitSyncModule } from './integrations/git-sync/git-sync.module';
|
||||
import { SandboxModule } from './integrations/sandbox/sandbox.module';
|
||||
import { AiModule } from './integrations/ai/ai.module';
|
||||
import { AiChatModule } from './core/ai-chat/ai-chat.module';
|
||||
@@ -92,6 +93,7 @@ try {
|
||||
TelemetryModule,
|
||||
ThrottleModule,
|
||||
McpModule,
|
||||
GitSyncModule,
|
||||
SandboxModule,
|
||||
AiModule,
|
||||
AiChatModule,
|
||||
|
||||
@@ -188,6 +188,45 @@ export class CollaborationGateway {
|
||||
return this.hocuspocus.openDirectConnection(documentName, context);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a git-originated body into a page, applying the merge on the instance
|
||||
* that OWNS the live Y.Doc so a connected editor CONVERGES on the change.
|
||||
*
|
||||
* git-sync must NOT use openDirectConnection directly for this: that opens the
|
||||
* document on whichever instance/process runs git-sync (the API/worker). When
|
||||
* an editor is connected to a DIFFERENT collab instance/process, that is a
|
||||
* SEPARATE, detached Y.Doc — the merge lands in the detached doc and the DB,
|
||||
* but the live editor never receives the Yjs update; its next debounced
|
||||
* autosave then overwrites the DB with its stale state and SILENTLY REVERTS
|
||||
* the git change (the data-loss bug). Routing through the custom-event channel
|
||||
* runs the merge on the owning instance's shared Document, whose update is
|
||||
* broadcast to every connection (handleUpdate), so the editor's CRDT converges
|
||||
* on the merged result.
|
||||
*
|
||||
* Without redis there is a single instance, so the write runs locally — which
|
||||
* is already the owning (and only) instance the editor is connected to.
|
||||
*/
|
||||
async writePageBody(
|
||||
documentName: string,
|
||||
payload: {
|
||||
prosemirrorJson: unknown;
|
||||
baseProsemirrorJson?: unknown;
|
||||
userId: string;
|
||||
},
|
||||
): Promise<void> {
|
||||
if (this.redisSync) {
|
||||
await this.handleYjsEvent(
|
||||
'gitSyncWriteBody',
|
||||
documentName,
|
||||
payload as any,
|
||||
);
|
||||
return;
|
||||
}
|
||||
await this.collabEventsService
|
||||
.getHandlers(this.hocuspocus)
|
||||
.gitSyncWriteBody(documentName, payload as any);
|
||||
}
|
||||
|
||||
/*
|
||||
*Can be used before calling openDirectConnection directly
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
// Exercises the REAL `gitSyncWriteBody` collab handler (the owner-routed body
|
||||
// write the data-loss fix introduces). The handler imports the editor graph via
|
||||
// collaboration.util / yjs.util (tiptapExtensions -> editor-ext -> react-dom,
|
||||
// unloadable under jest's node env, same coupling noted in
|
||||
// gitmost-datasource.service.spec.ts), so we stub those + the transformer. The
|
||||
// stubbed toYdoc builds paragraph blocks straight from the ProseMirror JSON so
|
||||
// we can assert convergence on real text.
|
||||
jest.mock('./collaboration.util', () => ({
|
||||
tiptapExtensions: [],
|
||||
getPageId: (name: string) => name.replace(/^page\./, ''),
|
||||
prosemirrorNodeToYElement: jest.fn(),
|
||||
}));
|
||||
jest.mock('./yjs.util', () => ({
|
||||
setYjsMark: jest.fn(),
|
||||
updateYjsMarkAttribute: jest.fn(),
|
||||
}));
|
||||
jest.mock('@hocuspocus/transformer', () => {
|
||||
const Yjs = require('yjs');
|
||||
return {
|
||||
TiptapTransformer: {
|
||||
toYdoc: (json: any) => {
|
||||
if (json?.__throw) throw new Error('boom: malformed doc');
|
||||
const d = new Yjs.Doc();
|
||||
const frag = d.getXmlFragment('default');
|
||||
const blocks = (json?.content ?? []).map((node: any) => {
|
||||
const el = new Yjs.XmlElement(node.type || 'paragraph');
|
||||
const text = (node.content ?? [])
|
||||
.map((t: any) => t.text ?? '')
|
||||
.join('');
|
||||
const t = new Yjs.XmlText();
|
||||
if (text) t.insert(0, text);
|
||||
el.insert(0, [t]);
|
||||
return el;
|
||||
});
|
||||
if (blocks.length) frag.insert(0, blocks);
|
||||
return d;
|
||||
},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import * as Y from 'yjs';
|
||||
import { CollaborationHandler } from './collaboration.handler';
|
||||
|
||||
const pmDoc = (...paras: string[]) => ({
|
||||
type: 'doc',
|
||||
content: paras.map((text) => ({
|
||||
type: 'paragraph',
|
||||
content: text ? [{ type: 'text', text }] : [],
|
||||
})),
|
||||
});
|
||||
|
||||
const texts = (frag: Y.XmlFragment): string[] =>
|
||||
frag.toArray().map((el) =>
|
||||
(el as Y.XmlElement)
|
||||
.toArray()
|
||||
.map((c) => (c as Y.XmlText).toString())
|
||||
.join(''),
|
||||
);
|
||||
|
||||
// Build a fake Hocuspocus whose openDirectConnection yields a DirectConnection
|
||||
// over a REAL shared Document, with a connected "editor" doc that receives the
|
||||
// shared doc's updates (modelling Document.handleUpdate's broadcast on the
|
||||
// OWNING instance). Initial content carries live block ids; the editor starts
|
||||
// fully synced with the shared doc.
|
||||
function fakeHocuspocus(initial: { text: string; id: string }[]) {
|
||||
const shared = new Y.Doc();
|
||||
const frag = shared.getXmlFragment('default');
|
||||
shared.transact(() => {
|
||||
frag.insert(
|
||||
0,
|
||||
initial.map((s) => {
|
||||
const el = new Y.XmlElement('paragraph');
|
||||
el.setAttribute('id', s.id);
|
||||
const t = new Y.XmlText();
|
||||
if (s.text) t.insert(0, s.text);
|
||||
el.insert(0, [t]);
|
||||
return el;
|
||||
}),
|
||||
);
|
||||
});
|
||||
const editor = new Y.Doc();
|
||||
Y.applyUpdate(editor, Y.encodeStateAsUpdate(shared));
|
||||
// Broadcast relay: server-originated updates flow to the connected editor.
|
||||
shared.on('update', (u: Uint8Array, origin: any) => {
|
||||
if (origin !== 'editor') Y.applyUpdate(editor, u, 'server');
|
||||
});
|
||||
|
||||
const openDirectConnection = jest.fn(async () => ({
|
||||
// DirectConnection.transact runs the fn directly against the Document (no
|
||||
// wrapping Y transaction), exactly like @hocuspocus/server.
|
||||
transact: async (fn: (doc: Y.Doc) => void) => fn(shared),
|
||||
disconnect: jest.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
return { hocuspocus: { openDirectConnection } as any, shared, editor };
|
||||
}
|
||||
|
||||
describe('CollaborationHandler.gitSyncWriteBody (owner-routed body write)', () => {
|
||||
it('converges a connected editor on the git change (no silent revert)', async () => {
|
||||
const { hocuspocus, shared, editor } = fakeHocuspocus([
|
||||
{ text: 'alpha', id: 'p1' },
|
||||
{ text: 'beta', id: 'p2' },
|
||||
]);
|
||||
const handler = new CollaborationHandler();
|
||||
const handlers = handler.getHandlers(hocuspocus);
|
||||
|
||||
// git changed block 1 beta -> beta2; base is the pre-change content.
|
||||
await handlers.gitSyncWriteBody('page.x', {
|
||||
prosemirrorJson: pmDoc('alpha', 'beta2'),
|
||||
baseProsemirrorJson: pmDoc('alpha', 'beta'),
|
||||
userId: 'svc-user',
|
||||
});
|
||||
|
||||
// The shared (owning-instance) doc holds the merge...
|
||||
expect(texts(shared.getXmlFragment('default'))).toEqual(['alpha', 'beta2']);
|
||||
// ...and the connected editor CONVERGED via the broadcast (the bug would
|
||||
// leave it on 'beta' and revert the page on its next autosave).
|
||||
expect(texts(editor.getXmlFragment('default'))).toEqual(['alpha', 'beta2']);
|
||||
});
|
||||
|
||||
it('preserves a concurrent edit to a DIFFERENT block (3-way, finding #2)', async () => {
|
||||
const { hocuspocus, shared, editor } = fakeHocuspocus([
|
||||
{ text: 'alpha', id: 'p1' },
|
||||
{ text: 'beta', id: 'p2' },
|
||||
]);
|
||||
// The editor is actively editing block 0 while the push arrives.
|
||||
const eFrag = editor.getXmlFragment('default');
|
||||
editor.transact(
|
||||
() => (eFrag.get(0) as Y.XmlElement).get(0) instanceof Y.XmlText &&
|
||||
((eFrag.get(0) as Y.XmlElement).get(0) as Y.XmlText).insert(5, ' EDIT'),
|
||||
'editor',
|
||||
);
|
||||
Y.applyUpdate(shared, Y.encodeStateAsUpdate(editor), 'editor');
|
||||
|
||||
const handler = new CollaborationHandler();
|
||||
await handler.getHandlers(hocuspocus).gitSyncWriteBody('page.x', {
|
||||
prosemirrorJson: pmDoc('alpha', 'beta2'),
|
||||
baseProsemirrorJson: pmDoc('alpha', 'beta'),
|
||||
userId: 'svc-user',
|
||||
});
|
||||
|
||||
// Human's block-0 edit AND git's block-1 change both survive on the editor.
|
||||
expect(texts(editor.getXmlFragment('default'))).toEqual([
|
||||
'alpha EDIT',
|
||||
'beta2',
|
||||
]);
|
||||
});
|
||||
|
||||
it('FLUSHES the pending debounced store BEFORE merging so an in-flight edit survives (finding #2)', async () => {
|
||||
// QA #119 finding #2: the 3-way merge must run against the latest live-doc
|
||||
// state. A concurrent UI edit that is still in-flight (the store is debounced)
|
||||
// must be drained into the live doc BEFORE git merges, or git clean-applies and
|
||||
// the edit is silently dropped — even on a DIFFERENT block. Model the drain via
|
||||
// the pending-store flush: when it runs, the in-flight block-0 edit lands.
|
||||
const shared = new Y.Doc();
|
||||
const frag = shared.getXmlFragment('default');
|
||||
shared.transact(() => {
|
||||
frag.insert(
|
||||
0,
|
||||
[
|
||||
{ text: 'alpha', id: 'p1' },
|
||||
{ text: 'beta', id: 'p2' },
|
||||
].map((s) => {
|
||||
const el = new Y.XmlElement('paragraph');
|
||||
el.setAttribute('id', s.id);
|
||||
const t = new Y.XmlText();
|
||||
t.insert(0, s.text);
|
||||
el.insert(0, [t]);
|
||||
return el;
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
const order: string[] = [];
|
||||
const debouncer = {
|
||||
isDebounced: jest.fn(() => true),
|
||||
executeNow: jest.fn(async () => {
|
||||
order.push('flush');
|
||||
// The in-flight client edit to block 0 only lands once the pending store
|
||||
// is flushed (i.e. the event loop is drained) — BEFORE the merge.
|
||||
shared.transact(() =>
|
||||
((frag.get(0) as Y.XmlElement).get(0) as Y.XmlText).insert(5, ' EDIT'),
|
||||
);
|
||||
}),
|
||||
};
|
||||
const openDirectConnection = jest.fn(async () => ({
|
||||
transact: async (fn: (doc: Y.Doc) => void) => {
|
||||
order.push('merge');
|
||||
fn(shared);
|
||||
},
|
||||
disconnect: jest.fn(async () => undefined),
|
||||
}));
|
||||
const hocuspocus = { openDirectConnection, debouncer } as any;
|
||||
|
||||
const handler = new CollaborationHandler();
|
||||
await handler.getHandlers(hocuspocus).gitSyncWriteBody('page.x', {
|
||||
prosemirrorJson: pmDoc('alpha', 'beta2'), // git changes block 1
|
||||
baseProsemirrorJson: pmDoc('alpha', 'beta'),
|
||||
userId: 'svc-user',
|
||||
});
|
||||
|
||||
// The flush ran, and it ran BEFORE the merge transaction.
|
||||
expect(debouncer.executeNow).toHaveBeenCalledTimes(1);
|
||||
expect(order).toEqual(['flush', 'merge']);
|
||||
// Both the in-flight block-0 edit and git's block-1 change survive — the
|
||||
// pre-flush bug would have produced ['alpha', 'beta2'] (UI edit dropped).
|
||||
expect(texts(shared.getXmlFragment('default'))).toEqual([
|
||||
'alpha EDIT',
|
||||
'beta2',
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not flush when no store is pending (isDebounced false)', async () => {
|
||||
const { hocuspocus, shared } = fakeHocuspocus([{ text: 'a', id: 'p1' }]);
|
||||
const executeNow = jest.fn();
|
||||
(hocuspocus as any).debouncer = {
|
||||
isDebounced: jest.fn(() => false),
|
||||
executeNow,
|
||||
};
|
||||
const handler = new CollaborationHandler();
|
||||
await handler.getHandlers(hocuspocus).gitSyncWriteBody('page.x', {
|
||||
prosemirrorJson: pmDoc('a', 'b'),
|
||||
userId: 'svc-user',
|
||||
});
|
||||
expect(executeNow).not.toHaveBeenCalled();
|
||||
expect(texts(shared.getXmlFragment('default'))).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
it('crash-safe: a transform failure never opens the connection or mutates the live doc', async () => {
|
||||
const { hocuspocus, shared } = fakeHocuspocus([{ text: 'alpha', id: 'p1' }]);
|
||||
const before = texts(shared.getXmlFragment('default'));
|
||||
const handler = new CollaborationHandler();
|
||||
|
||||
await expect(
|
||||
handler.getHandlers(hocuspocus).gitSyncWriteBody('page.x', {
|
||||
prosemirrorJson: { __throw: true } as any,
|
||||
userId: 'svc-user',
|
||||
}),
|
||||
).rejects.toThrow('boom');
|
||||
|
||||
// The incoming doc is built BEFORE opening the connection, so the throw
|
||||
// happens first: the live doc is untouched and no connection was opened.
|
||||
expect(hocuspocus.openDirectConnection).not.toHaveBeenCalled();
|
||||
expect(texts(shared.getXmlFragment('default'))).toEqual(before);
|
||||
});
|
||||
|
||||
it('falls back to a 2-way merge when no base is supplied', async () => {
|
||||
const { hocuspocus, shared, editor } = fakeHocuspocus([
|
||||
{ text: 'alpha', id: 'p1' },
|
||||
]);
|
||||
const handler = new CollaborationHandler();
|
||||
|
||||
await handler.getHandlers(hocuspocus).gitSyncWriteBody('page.x', {
|
||||
prosemirrorJson: pmDoc('alpha', 'gamma'),
|
||||
userId: 'svc-user',
|
||||
});
|
||||
|
||||
expect(texts(shared.getXmlFragment('default'))).toEqual(['alpha', 'gamma']);
|
||||
expect(texts(editor.getXmlFragment('default'))).toEqual(['alpha', 'gamma']);
|
||||
});
|
||||
});
|
||||
@@ -14,6 +14,10 @@ import {
|
||||
} from './yjs.util';
|
||||
import * as Y from 'yjs';
|
||||
import { User } from '@docmost/db/types/entity.types';
|
||||
import {
|
||||
mergeXmlFragments,
|
||||
mergeXmlFragments3WayWithStats,
|
||||
} from './merge/yjs-body-merge';
|
||||
|
||||
export type CollabEventHandlers = ReturnType<
|
||||
CollaborationHandler['getHandlers']
|
||||
@@ -181,9 +185,130 @@ export class CollaborationHandler {
|
||||
},
|
||||
);
|
||||
},
|
||||
/**
|
||||
* Git-sync body write, applied as a block-level MERGE into the LIVE doc on
|
||||
* the instance that OWNS it (routed here via the custom-event channel —
|
||||
* see CollaborationGateway.writePageBody). Running on the owning instance
|
||||
* is what makes a connected editor CONVERGE: the merge mutates the shared
|
||||
* Document, whose update is broadcast to every connection, so the editor's
|
||||
* CRDT applies the git change instead of silently reverting it on its next
|
||||
* autosave (the data-loss bug this fixes).
|
||||
*
|
||||
* With a `baseProsemirrorJson` (the last-synced common ancestor) it does a
|
||||
* THREE-WAY merge — a block only the human changed is kept, a block only
|
||||
* git changed is taken (conflicts -> git). Without a base it falls back to
|
||||
* the 2-way merge.
|
||||
*/
|
||||
gitSyncWriteBody: async (
|
||||
documentName: string,
|
||||
payload: {
|
||||
prosemirrorJson: any;
|
||||
baseProsemirrorJson?: any;
|
||||
userId: string;
|
||||
},
|
||||
) => {
|
||||
const { prosemirrorJson, baseProsemirrorJson, userId } = payload;
|
||||
|
||||
// Build the incoming (and base) Yjs docs BEFORE opening the connection /
|
||||
// touching the live doc. If a transform throws (a malformed/unsupported
|
||||
// doc) we must NOT have mutated the live body — otherwise a conversion
|
||||
// failure could leave the page empty (crash-safe conversion).
|
||||
const targetDoc = TiptapTransformer.toYdoc(
|
||||
prosemirrorJson,
|
||||
'default',
|
||||
tiptapExtensions,
|
||||
);
|
||||
const baseDoc =
|
||||
baseProsemirrorJson != null
|
||||
? TiptapTransformer.toYdoc(
|
||||
baseProsemirrorJson,
|
||||
'default',
|
||||
tiptapExtensions,
|
||||
)
|
||||
: null;
|
||||
|
||||
// CONCURRENT-EDIT FLUSH (QA #119, finding #2). The 3-way merge below runs
|
||||
// against the LIVE Y.Doc, so a concurrent UI edit is only preserved if it
|
||||
// is already part of that doc. A user's edit is debounced before it lands
|
||||
// (the editor batches; the collab store is debounced up to 10s), so the
|
||||
// merge could otherwise run against a PRE-EDIT doc: git would then
|
||||
// clean-apply (no same-block conflict detected) and the in-flight UI edit
|
||||
// — even on a DIFFERENT block — would be silently dropped.
|
||||
//
|
||||
// Flushing the pending debounced store here (a) drains the event loop so a
|
||||
// just-arrived client Yjs update is applied to the live doc BEFORE we
|
||||
// merge, and (b) persists the live doc so the merge baseline is current
|
||||
// even on the doc-reload-from-DB path. After the flush the merge sees the
|
||||
// latest state, so an edit on a different block is MERGED (not overwritten)
|
||||
// and a genuine same-block edit is detected as a conflict -> the
|
||||
// boundary-snapshot in PersistenceExtension pins it to page history
|
||||
// (recoverable) instead of vanishing silently.
|
||||
await this.flushPendingStore(hocuspocus, documentName);
|
||||
|
||||
// actor:'git-sync' + the service user flow into PersistenceExtension
|
||||
// (lastUpdatedSource='git-sync', lastUpdatedById=userId).
|
||||
await this.withYdocConnection(
|
||||
hocuspocus,
|
||||
documentName,
|
||||
{ actor: 'git-sync', user: { id: userId } },
|
||||
(doc) => {
|
||||
const liveFrag = doc.getXmlFragment('default');
|
||||
const targetFrag = targetDoc.getXmlFragment('default');
|
||||
if (baseDoc) {
|
||||
const { conflicts } = mergeXmlFragments3WayWithStats(
|
||||
liveFrag,
|
||||
targetFrag,
|
||||
baseDoc.getXmlFragment('default'),
|
||||
);
|
||||
// SAME-BLOCK conflict contract (SPEC §9): a block both the human
|
||||
// and git changed resolves to GIT (deterministic). Make that
|
||||
// OBSERVABLE rather than silent — log it. The losing human content
|
||||
// is NOT destroyed: the persistence extension's boundary snapshot
|
||||
// pins the pre-merge page state to history on this user->git-sync
|
||||
// transition, so it stays recoverable.
|
||||
if (conflicts > 0) {
|
||||
this.logger.warn(
|
||||
`git-sync merge for ${documentName}: ${conflicts} same-block ` +
|
||||
`conflict(s) resolved to the git version; the prior page ` +
|
||||
`state is preserved in page history (recoverable).`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
mergeXmlFragments(liveFrag, targetFrag);
|
||||
}
|
||||
},
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush any pending DEBOUNCED store for `documentName` so the live Y.Doc and the
|
||||
* DB are current BEFORE a git-sync merge reads them (QA #119, finding #2 —
|
||||
* concurrent UI edit silently lost). Mirrors the PersistenceExtension.onDisconnect
|
||||
* flush: only acts when a store is actually pending (`isDebounced`), runs the
|
||||
* SAME scheduled payload (`executeNow`, preserving the edit's context/actor), and
|
||||
* never throws — a flush failure must not abort the git-sync write. Awaiting it
|
||||
* also drains the event loop, so a client Yjs update sitting in the socket buffer
|
||||
* is applied to the live doc before the merge transaction runs.
|
||||
*/
|
||||
private async flushPendingStore(
|
||||
hocuspocus: Hocuspocus,
|
||||
documentName: string,
|
||||
): Promise<void> {
|
||||
const debounceId = `onStoreDocument-${documentName}`;
|
||||
try {
|
||||
const debouncer = (hocuspocus as any)?.debouncer;
|
||||
if (!debouncer?.isDebounced?.(debounceId)) return;
|
||||
await debouncer.executeNow(debounceId);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`git-sync pre-merge flush failed for ${documentName}: ` +
|
||||
(err instanceof Error ? err.message : String(err)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async withYdocConnection<T>(
|
||||
hocuspocus: Hocuspocus,
|
||||
documentName: string,
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { PersistenceExtension } from './persistence.extension';
|
||||
|
||||
/**
|
||||
* Regression for the QA #119 "loss-on-fast-close" data loss: editing a page then
|
||||
* closing the tab within the collab debounce window (~3-18s) lost the edit
|
||||
* because, with `unloadImmediately: false`, Hocuspocus does NOT flush the
|
||||
* debounced onStoreDocument on a last-client disconnect. PersistenceExtension
|
||||
* now flushes the pending store on the LAST disconnect (and only then).
|
||||
*/
|
||||
describe('PersistenceExtension.onDisconnect flush (loss-on-fast-close)', () => {
|
||||
function makeExt(): PersistenceExtension {
|
||||
// onDisconnect touches none of the injected deps; pass casts.
|
||||
return new PersistenceExtension(
|
||||
null as any,
|
||||
null as any,
|
||||
null as any,
|
||||
null as any,
|
||||
null as any,
|
||||
null as any,
|
||||
null as any,
|
||||
null as any,
|
||||
);
|
||||
}
|
||||
|
||||
function makeData(opts: {
|
||||
clientsCount: number;
|
||||
isDebounced: boolean;
|
||||
isLoading?: boolean;
|
||||
}) {
|
||||
const executeNow = jest.fn(async () => undefined);
|
||||
const isDebounced = jest.fn(() => opts.isDebounced);
|
||||
return {
|
||||
executeNow,
|
||||
isDebounced,
|
||||
payload: {
|
||||
clientsCount: opts.clientsCount,
|
||||
context: {},
|
||||
document: { isLoading: opts.isLoading ?? false } as any,
|
||||
documentName: 'page.abc',
|
||||
instance: { debouncer: { isDebounced, executeNow } } as any,
|
||||
requestHeaders: {},
|
||||
requestParameters: new URLSearchParams(),
|
||||
socketId: 's',
|
||||
} as any,
|
||||
};
|
||||
}
|
||||
|
||||
it('flushes the pending store when the LAST client disconnects', async () => {
|
||||
const ext = makeExt();
|
||||
const { executeNow, payload } = makeData({
|
||||
clientsCount: 0,
|
||||
isDebounced: true,
|
||||
});
|
||||
await ext.onDisconnect(payload);
|
||||
expect(executeNow).toHaveBeenCalledTimes(1);
|
||||
expect(executeNow).toHaveBeenCalledWith('onStoreDocument-page.abc');
|
||||
});
|
||||
|
||||
it('does NOT flush while other editors remain connected', async () => {
|
||||
const ext = makeExt();
|
||||
const { executeNow, payload } = makeData({
|
||||
clientsCount: 2,
|
||||
isDebounced: true,
|
||||
});
|
||||
await ext.onDisconnect(payload);
|
||||
expect(executeNow).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does NOT write when nothing is pending (already persisted)', async () => {
|
||||
const ext = makeExt();
|
||||
const { executeNow, payload } = makeData({
|
||||
clientsCount: 0,
|
||||
isDebounced: false,
|
||||
});
|
||||
await ext.onDisconnect(payload);
|
||||
expect(executeNow).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does NOT flush a doc that is still loading (load error guard)', async () => {
|
||||
const ext = makeExt();
|
||||
const { executeNow, payload } = makeData({
|
||||
clientsCount: 0,
|
||||
isDebounced: true,
|
||||
isLoading: true,
|
||||
});
|
||||
await ext.onDisconnect(payload);
|
||||
expect(executeNow).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,223 @@
|
||||
// Stub collaboration.util so importing the extension does not drag in the
|
||||
// editor-ext -> @tiptap/react -> react-dom graph (unloadable under jest's node
|
||||
// env, same coupling the gitmost-datasource / mcp specs document). The
|
||||
// extension only calls getPageId, jsonToText and isEmptyParagraphDoc from it on
|
||||
// the store path; tiptapExtensions is unused by onStoreDocument.
|
||||
jest.mock('../collaboration.util', () => ({
|
||||
tiptapExtensions: [],
|
||||
getPageId: (name: string) => name.replace(/^page\./, ''),
|
||||
jsonToText: () => 'text',
|
||||
isEmptyParagraphDoc: () => false,
|
||||
// The post-write mention extraction walks the doc via jsonToNode().descendants;
|
||||
// return a node-like stub with no descendants so no mentions are produced
|
||||
// (mention handling is out of scope here — we only assert provenance).
|
||||
jsonToNode: () => ({ descendants: () => undefined }),
|
||||
}));
|
||||
|
||||
// Control the Yjs<->JSON bridge: fromYdoc returns the "incoming" doc the writer
|
||||
// is storing. We keep it distinct from the page's persisted content so the
|
||||
// no-op guard (isDeepStrictEqual) never short-circuits the write.
|
||||
const INCOMING_JSON = { type: 'doc', content: [{ type: 'paragraph' }, { t: 1 }] };
|
||||
jest.mock('@hocuspocus/transformer', () => ({
|
||||
TiptapTransformer: {
|
||||
fromYdoc: jest.fn(() => INCOMING_JSON),
|
||||
toYdoc: jest.fn(),
|
||||
},
|
||||
}));
|
||||
|
||||
// Run the executeTx callback inline with a passthrough trx.
|
||||
jest.mock('@docmost/db/utils', () => ({
|
||||
executeTx: jest.fn(async (_db: any, cb: any) => cb({} as any)),
|
||||
}));
|
||||
|
||||
import * as Y from 'yjs';
|
||||
import { PersistenceExtension } from './persistence.extension';
|
||||
import {
|
||||
onChangePayload,
|
||||
onStoreDocumentPayload,
|
||||
} from '@hocuspocus/server';
|
||||
|
||||
/**
|
||||
* Provenance-precedence coverage for PersistenceExtension.onStoreDocument
|
||||
* (test-strategy Module 4 / item #2): the contract `agent > git-sync > user`,
|
||||
* plus the negative that a git-sync store does NOT pin a boundary history
|
||||
* snapshot. We drive the precedence through the real public method (onChange to
|
||||
* arm the sticky agent marker, then onStoreDocument), mocking the repos / db /
|
||||
* Yjs bridge so no real database or collab server is needed. The store's
|
||||
* persisted `lastUpdatedSource` and the saveHistory call are the observable
|
||||
* outputs.
|
||||
*/
|
||||
describe('PersistenceExtension.onStoreDocument — provenance precedence (#2)', () => {
|
||||
const DOCUMENT_NAME = 'page.page-1';
|
||||
const PAGE_ID = 'page-1';
|
||||
|
||||
// `page.content` differs from INCOMING_JSON so the write is never skipped.
|
||||
const persistedPage = (overrides?: { lastUpdatedSource?: string }) => ({
|
||||
id: PAGE_ID,
|
||||
slugId: 'slug-1',
|
||||
spaceId: 'space-1',
|
||||
workspaceId: 'ws-1',
|
||||
creatorId: 'creator-1',
|
||||
contributorIds: ['creator-1'],
|
||||
content: { type: 'doc', content: [{ type: 'paragraph', content: [] }] },
|
||||
lastUpdatedSource: overrides?.lastUpdatedSource ?? 'user',
|
||||
createdAt: new Date(),
|
||||
});
|
||||
|
||||
const build = (pageOverrides?: { lastUpdatedSource?: string }) => {
|
||||
const pageRepo = {
|
||||
findById: jest.fn().mockResolvedValue(persistedPage(pageOverrides)),
|
||||
updatePage: jest.fn().mockResolvedValue({ numUpdatedRows: 1n }),
|
||||
};
|
||||
const pageHistoryRepo = {
|
||||
// No prior snapshot -> humanBaselineMissing is true, so the ONLY thing
|
||||
// gating the boundary snapshot in these tests is the source precedence.
|
||||
findPageLastHistory: jest.fn().mockResolvedValue(null),
|
||||
saveHistory: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const aiQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
const historyQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
const notificationQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
const collabHistory = {
|
||||
addContributors: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const transclusionService = {
|
||||
syncPageTransclusions: jest.fn().mockResolvedValue(undefined),
|
||||
syncPageReferences: jest.fn().mockResolvedValue(undefined),
|
||||
syncPageTemplateReferences: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
const ext = new PersistenceExtension(
|
||||
pageRepo as any,
|
||||
pageHistoryRepo as any,
|
||||
{} as any, // db
|
||||
aiQueue as any,
|
||||
historyQueue as any,
|
||||
notificationQueue as any,
|
||||
collabHistory as any,
|
||||
transclusionService as any,
|
||||
);
|
||||
|
||||
return { ext, pageRepo, pageHistoryRepo, historyQueue };
|
||||
};
|
||||
|
||||
// A real Y.Doc is required for Y.encodeStateAsUpdate(document); broadcastStateless
|
||||
// is a no-op spy. The fromYdoc bridge is mocked, so the doc's contents are
|
||||
// irrelevant to the JSON path.
|
||||
const makeStorePayload = (context: any): onStoreDocumentPayload =>
|
||||
({
|
||||
documentName: DOCUMENT_NAME,
|
||||
document: Object.assign(new Y.Doc(), {
|
||||
broadcastStateless: jest.fn(),
|
||||
}),
|
||||
context,
|
||||
}) as any;
|
||||
|
||||
const makeChangePayload = (actor: string): onChangePayload =>
|
||||
({
|
||||
documentName: DOCUMENT_NAME,
|
||||
context: { user: { id: 'user-1' }, actor },
|
||||
}) as any;
|
||||
|
||||
const sourceOf = (pageRepo: { updatePage: jest.Mock }) =>
|
||||
pageRepo.updatePage.mock.calls[0][0].lastUpdatedSource;
|
||||
|
||||
it("tags 'user' for a plain write (no agent touch, no git-sync actor)", async () => {
|
||||
const { ext, pageRepo } = build();
|
||||
|
||||
await ext.onStoreDocument(
|
||||
makeStorePayload({ user: { id: 'user-1' }, actor: 'user' }),
|
||||
);
|
||||
|
||||
expect(sourceOf(pageRepo)).toBe('user');
|
||||
});
|
||||
|
||||
it("tags 'git-sync' when the writer's actor is 'git-sync' and no agent touched the window", async () => {
|
||||
const { ext, pageRepo } = build();
|
||||
|
||||
await ext.onStoreDocument(
|
||||
makeStorePayload({ user: { id: 'svc-user' }, actor: 'git-sync' }),
|
||||
);
|
||||
|
||||
expect(sourceOf(pageRepo)).toBe('git-sync');
|
||||
});
|
||||
|
||||
it("keeps 'git-sync' for an explicit git-sync store even with a sticky agent marker (#14 loop-guard)", async () => {
|
||||
const { ext, pageRepo } = build();
|
||||
|
||||
// An agent edit landed earlier in the coalescing window (sticky marker),
|
||||
// then a git-sync writer performs the store. Red-team finding #14: an
|
||||
// EXPLICIT current-write actor is authoritative for THIS write, so the
|
||||
// store must stay 'git-sync' — otherwise the PageChangeListener loop-guard
|
||||
// (keyed on lastUpdatedSource === 'git-sync') fails to recognize git-sync's
|
||||
// own write and re-exports it. Explicit 'agent' still wins (see below); the
|
||||
// sticky marker only promotes a plain human writer to 'agent'.
|
||||
await ext.onChange(makeChangePayload('agent'));
|
||||
await ext.onStoreDocument(
|
||||
makeStorePayload({ user: { id: 'svc-user' }, actor: 'git-sync' }),
|
||||
);
|
||||
|
||||
expect(sourceOf(pageRepo)).toBe('git-sync');
|
||||
});
|
||||
|
||||
it("tags 'agent' when the storing writer itself is the agent (no prior onChange)", async () => {
|
||||
const { ext, pageRepo } = build();
|
||||
|
||||
await ext.onStoreDocument(
|
||||
makeStorePayload({ user: { id: 'agent-user' }, actor: 'agent' }),
|
||||
);
|
||||
|
||||
expect(sourceOf(pageRepo)).toBe('agent');
|
||||
});
|
||||
|
||||
// --- boundary snapshot for a git-sync store over a HUMAN baseline -----------
|
||||
// SPEC §9 observable-loss guard (bug #2): a git-sync body write is a block-level
|
||||
// 3-way merge whose same-block rule is "git wins". To keep a concurrent human
|
||||
// edit RECOVERABLE rather than silently overwritten, a git-sync store over a
|
||||
// prior NON-git-sync baseline pins that prior state to page history first —
|
||||
// exactly like the agent path. So saveHistory MUST be called here.
|
||||
it('DOES pin a boundary snapshot for a git-sync store over a prior human state', async () => {
|
||||
const { ext, pageHistoryRepo } = build({ lastUpdatedSource: 'user' });
|
||||
|
||||
await ext.onStoreDocument(
|
||||
makeStorePayload({ user: { id: 'svc-user' }, actor: 'git-sync' }),
|
||||
);
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
// --- negative: a git-sync store over a git-sync baseline does NOT re-pin -----
|
||||
// The boundary is pinned once on the transition INTO git-sync; a subsequent
|
||||
// git-sync store over an already-git-sync baseline must not churn history.
|
||||
it('does NOT re-pin a boundary snapshot for a git-sync store over a git-sync baseline', async () => {
|
||||
const { ext, pageHistoryRepo } = build({ lastUpdatedSource: 'git-sync' });
|
||||
|
||||
await ext.onStoreDocument(
|
||||
makeStorePayload({ user: { id: 'svc-user' }, actor: 'git-sync' }),
|
||||
);
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('DOES pin a boundary snapshot for an agent store over a prior human state (control)', async () => {
|
||||
// Confirms the negative above is meaningful: under the SAME mocks, an agent
|
||||
// store over a 'user' baseline DOES trigger the boundary snapshot.
|
||||
const { ext, pageHistoryRepo } = build({ lastUpdatedSource: 'user' });
|
||||
|
||||
await ext.onStoreDocument(
|
||||
makeStorePayload({ user: { id: 'agent-user' }, actor: 'agent' }),
|
||||
);
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('does NOT pin a boundary snapshot for a plain user store', async () => {
|
||||
const { ext, pageHistoryRepo } = build({ lastUpdatedSource: 'user' });
|
||||
|
||||
await ext.onStoreDocument(
|
||||
makeStorePayload({ user: { id: 'user-1' }, actor: 'user' }),
|
||||
);
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
afterUnloadDocumentPayload,
|
||||
Extension,
|
||||
onChangePayload,
|
||||
onDisconnectPayload,
|
||||
onLoadDocumentPayload,
|
||||
onStatelessPayload,
|
||||
onStoreDocumentPayload,
|
||||
@@ -83,7 +84,17 @@ export function resolveSource(
|
||||
stickyTouched: boolean,
|
||||
contextActor?: string,
|
||||
): ProvenanceSource {
|
||||
return stickyTouched || contextActor === 'agent' ? 'agent' : 'user';
|
||||
// An EXPLICIT current-write actor is authoritative for THIS write and wins
|
||||
// over the sticky-agent fallback. Order: explicit 'agent' > explicit
|
||||
// 'git-sync' > sticky agent marker > plain human 'user'. The git-sync case
|
||||
// must NOT be masked by the sticky marker, or the PageChangeListener
|
||||
// loop-guard (which keys on lastUpdatedSource === 'git-sync') would re-export
|
||||
// git-sync's own writes (#14). Explicit agent still wins so a window that
|
||||
// mixed an agent edit stays tagged 'agent'.
|
||||
if (contextActor === 'agent') return 'agent';
|
||||
if (contextActor === 'git-sync') return 'git-sync';
|
||||
if (stickyTouched) return 'agent';
|
||||
return 'user';
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -192,6 +203,40 @@ export class PersistenceExtension implements Extension {
|
||||
return new Y.Doc();
|
||||
}
|
||||
|
||||
/**
|
||||
* LOSS-ON-FAST-CLOSE FIX (QA #119). When the LAST editor disconnects, FLUSH any
|
||||
* pending (debounced) store to the DB IMMEDIATELY instead of waiting out the
|
||||
* up-to-10s `debounce` window.
|
||||
*
|
||||
* The collab server runs with `unloadImmediately: false` (collaboration.gateway),
|
||||
* so on a last-client disconnect Hocuspocus does NOT flush the debounced
|
||||
* onStoreDocument — it relies on the timer firing later. A quick edit-then-close
|
||||
* (closing the tab within the debounce window, ~3-18s) therefore left the edit
|
||||
* only in the soon-to-be-unloaded in-memory Y.Doc; meanwhile git-sync mirrored
|
||||
* the STALE/empty DB body to the vault (the reported "59-byte frontmatter-only"
|
||||
* data loss). Running the already-scheduled store now closes that window.
|
||||
*
|
||||
* Gated tightly so it never adds a redundant write: only on the LAST disconnect
|
||||
* (`clientsCount === 0`), only for a fully-loaded doc, and only when a store is
|
||||
* actually pending (`isDebounced`). `executeNow` runs the SAME payload Hocuspocus
|
||||
* scheduled (preserving the edit's context/actor) and clears the timer.
|
||||
*/
|
||||
async onDisconnect(data: onDisconnectPayload) {
|
||||
const { instance, document, documentName, clientsCount } = data;
|
||||
if (clientsCount > 0) return;
|
||||
if (!document || document.isLoading) return;
|
||||
const debounceId = `onStoreDocument-${documentName}`;
|
||||
if (!instance?.debouncer?.isDebounced(debounceId)) return;
|
||||
try {
|
||||
await instance.debouncer.executeNow(debounceId);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`onDisconnect flush failed for ${documentName}: ` +
|
||||
(err instanceof Error ? err.message : String(err)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async onStoreDocument(data: onStoreDocumentPayload) {
|
||||
// #355 — time the full store (persist + post-store side effects) into
|
||||
// collab_store_duration_seconds. No-op when METRICS_PORT is unset.
|
||||
@@ -225,6 +270,11 @@ export class PersistenceExtension implements Extension {
|
||||
// Sticky agent marker: 'agent' if any agent edit landed in this window, OR
|
||||
// if the current writer is the agent (covers a store with no prior onChange
|
||||
// agent event in the same window). §15 H2.
|
||||
// Provenance precedence: agent > git-sync > user (see resolveSource). A
|
||||
// 'git-sync' store is NOT given an immediate history snapshot — it is
|
||||
// debounced like a human edit (a git-sync write is a block-level merge into
|
||||
// the live doc, so it reads like an incremental human edit, not a bulk
|
||||
// import that would warrant its own immediate snapshot).
|
||||
const lastUpdatedSource = resolveSource(
|
||||
this.consumeAgentTouched(documentName),
|
||||
context?.actor,
|
||||
@@ -291,17 +341,25 @@ export class PersistenceExtension implements Extension {
|
||||
// flag via that same hoisted consume (a "cleared then retyped"
|
||||
// sequence can't leave a usable one behind).
|
||||
const incomingEmpty = isEmptyParagraphDoc(tiptapJson as any);
|
||||
// A git-sync write is authoritative and its content IS the vault file:
|
||||
// an empty incoming doc there means the user DELIBERATELY cleared the
|
||||
// page's markdown in git (there is no "transient glitch empty" for a
|
||||
// file-sourced write). Honor it, otherwise the empty-guard rejects the
|
||||
// clear, the vault ref has already advanced past the empty commit, and
|
||||
// vault<->Docmost diverge permanently (review warning). This mirrors the
|
||||
// #251 intentional-clear allowance for a different authoritative source.
|
||||
const gitSyncClear = lastUpdatedSource === 'git-sync';
|
||||
if (
|
||||
incomingEmpty &&
|
||||
page.content &&
|
||||
!isEmptyParagraphDoc(page.content as any)
|
||||
) {
|
||||
if (allowIntentionalClear) {
|
||||
if (allowIntentionalClear || gitSyncClear) {
|
||||
this.logger.debug(
|
||||
`Intentional clear for ${pageId}: persisting empty doc over ` +
|
||||
`non-empty content (user-signalled)`,
|
||||
`non-empty content (${gitSyncClear ? 'git-sync' : 'user-signalled'})`,
|
||||
);
|
||||
// fall through — the empty write is allowed exactly once.
|
||||
// fall through — the empty write is allowed.
|
||||
} else {
|
||||
this.logger.warn(
|
||||
`Skipping store for ${pageId}: empty live doc would overwrite ` +
|
||||
@@ -326,21 +384,30 @@ export class PersistenceExtension implements Extension {
|
||||
//this.logger.debug('Contributors error:' + err?.['message']);
|
||||
}
|
||||
|
||||
// Approach A — boundary snapshot before the agent's first edit.
|
||||
// When this store is the agent's and the page's currently persisted
|
||||
// state was authored by a human, pin that human state as its own
|
||||
// history version BEFORE the agent overwrites it. `page` still holds
|
||||
// the OLD content/provenance here, so saveHistory(page) captures the
|
||||
// pre-agent state tagged 'user'. The agent's new content is
|
||||
// snapshotted later by the debounced PAGE_HISTORY job ('agent'). Skip
|
||||
// if the prior state is already agent-authored (boundary already
|
||||
// pinned on the user->agent transition), if the page is effectively
|
||||
// empty, or if the latest existing snapshot already equals this human
|
||||
// state (avoid duplicates).
|
||||
if (
|
||||
lastUpdatedSource === 'agent' &&
|
||||
page.lastUpdatedSource !== 'agent'
|
||||
) {
|
||||
// Approach A — boundary snapshot before a MACHINE write overwrites a
|
||||
// human (or other-source) baseline. When this store is from a machine
|
||||
// source — the AGENT or GIT-SYNC — and the page's currently persisted
|
||||
// state was authored by a DIFFERENT source, pin that prior state as its
|
||||
// own history version BEFORE the machine write overwrites it. `page`
|
||||
// still holds the OLD content/provenance here, so saveHistory(page)
|
||||
// captures the pre-write state. The machine's new content is snapshotted
|
||||
// later by the debounced PAGE_HISTORY job.
|
||||
//
|
||||
// For GIT-SYNC this is the OBSERVABLE-LOSS guard (SPEC §9 conflict
|
||||
// contract): a git-sync body write is a block-level 3-way merge whose
|
||||
// same-block rule is "git wins". Without this pin, a concurrent human
|
||||
// edit to a block git also changed would be overwritten with NO trace.
|
||||
// Pinning the pre-merge state here means the human's content is always
|
||||
// RECOVERABLE via page history rather than silently lost — git still
|
||||
// wins the live doc deterministically, but nothing is destroyed.
|
||||
//
|
||||
// Skip if the prior state was already authored by THIS machine source
|
||||
// (boundary already pinned on the transition into it), if the page is
|
||||
// effectively empty, or if the latest existing snapshot already equals
|
||||
// the prior state (avoid duplicates).
|
||||
const isMachineWrite =
|
||||
lastUpdatedSource === 'agent' || lastUpdatedSource === 'git-sync';
|
||||
if (isMachineWrite && page.lastUpdatedSource !== lastUpdatedSource) {
|
||||
// pageHistory.pageId is uuid-typed; use page.id (never the doc-name
|
||||
// slugId) so a `page.<slugId>` doc cannot throw 22P02 here (#260).
|
||||
const lastHistory = await this.pageHistoryRepo.findPageLastHistory(
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
// Regression coverage for the custom-event request/reply protocol in the
|
||||
// RedisSyncExtension. git-sync routes its body write through a custom event
|
||||
// (`gitSyncWriteBody`) which, when the target doc is owned by a DIFFERENT collab
|
||||
// instance, runs REMOTELY inside `handleRedisMessage` on the owning instance. The
|
||||
// remote handler can THROW (markdown->ProseMirror transform on a malformed body).
|
||||
//
|
||||
// Before the fix the throw was uncaught: (1) no `customEventComplete` reply was
|
||||
// published, so the origin's awaiting promise only rejected after `customEventTTL`
|
||||
// (~30s) as a generic 'TIMEOUT', and (2) an unhandledRejection escaped the async
|
||||
// `messageBuffer` listener on the owning instance. These tests assert the throw is
|
||||
// turned into an error-carrying reply that rejects the origin PROMPTLY with the
|
||||
// real message, with the no-throw and local paths unchanged.
|
||||
|
||||
import { RedisSyncExtension } from './redis-sync.extension';
|
||||
|
||||
type Listener = (channel: Buffer, message: Buffer) => unknown;
|
||||
|
||||
// Minimal in-memory pub/sub + lock store shared across FakeRedis duplicates,
|
||||
// modelling the two-instance topology (origin + owner) over one Redis.
|
||||
class FakeRedisBus {
|
||||
instances: FakeRedis[] = [];
|
||||
locks = new Map<string, string>();
|
||||
published: { channel: string; message: Buffer }[] = [];
|
||||
|
||||
register(inst: FakeRedis) {
|
||||
this.instances.push(inst);
|
||||
}
|
||||
|
||||
publish(channel: string, message: Buffer) {
|
||||
this.published.push({ channel, message });
|
||||
for (const inst of this.instances) {
|
||||
if (!inst.subscribed.has(channel)) continue;
|
||||
for (const listener of inst.messageListeners) {
|
||||
// ioredis delivers async; `void` mirrors the production listener
|
||||
// registration (`sub.on('messageBuffer', ...)`), whose rejection would
|
||||
// surface as an unhandledRejection if the handler did not catch.
|
||||
void listener(Buffer.from(channel), message);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class FakeRedis {
|
||||
subscribed = new Set<string>();
|
||||
messageListeners: Listener[] = [];
|
||||
|
||||
constructor(private bus: FakeRedisBus) {
|
||||
bus.register(this);
|
||||
}
|
||||
|
||||
duplicate() {
|
||||
return new FakeRedis(this.bus);
|
||||
}
|
||||
|
||||
subscribe(...channels: string[]) {
|
||||
for (const c of channels) this.subscribed.add(c);
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
on(event: string, cb: any) {
|
||||
if (event === 'messageBuffer') this.messageListeners.push(cb as Listener);
|
||||
return this;
|
||||
}
|
||||
|
||||
publish(channel: string, message: Buffer) {
|
||||
this.bus.publish(channel, message);
|
||||
return Promise.resolve(1);
|
||||
}
|
||||
|
||||
// Models `SET key val PX ttl NX GET`: only writes when absent (NX); returns the
|
||||
// previous value (GET) so the origin observes the owner already holding the lock.
|
||||
set(key: string, val: string, ...args: any[]) {
|
||||
const hasNX = args.includes('NX');
|
||||
const hasGET = args.includes('GET');
|
||||
const old = this.bus.locks.get(key) ?? null;
|
||||
if (!hasNX || old === null) this.bus.locks.set(key, val);
|
||||
return Promise.resolve(hasGET ? old : 'OK');
|
||||
}
|
||||
|
||||
del(key: string) {
|
||||
this.bus.locks.delete(key);
|
||||
return Promise.resolve(1);
|
||||
}
|
||||
|
||||
disconnect() {}
|
||||
}
|
||||
|
||||
const pack = (m: any) => Buffer.from(JSON.stringify(m));
|
||||
const unpack = (b: Buffer) => JSON.parse(b.toString());
|
||||
|
||||
function makeExtension(
|
||||
bus: FakeRedisBus,
|
||||
serverId: string,
|
||||
customEvents: Record<string, (doc: string, payload: any) => Promise<any>>,
|
||||
) {
|
||||
const ext = new RedisSyncExtension({
|
||||
redis: new FakeRedis(bus) as any,
|
||||
pack: pack as any,
|
||||
unpack: unpack as any,
|
||||
serverId,
|
||||
customEvents: customEvents as any,
|
||||
customEventTTL: 30_000,
|
||||
});
|
||||
// Doc is NOT loaded on this instance -> handleEvent takes the remote/proxy path.
|
||||
(ext as any).instance = { documents: new Map() };
|
||||
return ext;
|
||||
}
|
||||
|
||||
describe('RedisSyncExtension custom-event error propagation', () => {
|
||||
let unhandled: unknown[];
|
||||
let onUnhandled: (e: unknown) => void;
|
||||
|
||||
beforeEach(() => {
|
||||
// Fake timers so the 30s TTL fallback timer never fires (and never dangles).
|
||||
jest.useFakeTimers();
|
||||
unhandled = [];
|
||||
onUnhandled = (e) => unhandled.push(e);
|
||||
process.on('unhandledRejection', onUnhandled);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
process.off('unhandledRejection', onUnhandled);
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
const flush = async () => {
|
||||
for (let i = 0; i < 10; i++) await Promise.resolve();
|
||||
};
|
||||
|
||||
it('owner publishes an error-carrying reply (no unhandledRejection) when the remote handler throws', async () => {
|
||||
const bus = new FakeRedisBus();
|
||||
const owner = makeExtension(bus, 'owner', {
|
||||
boom: async () => {
|
||||
throw new Error('kaboom');
|
||||
},
|
||||
});
|
||||
|
||||
// Drive the remote branch directly, as if the origin's customEventStart arrived.
|
||||
await (owner as any).handleRedisMessage(
|
||||
Buffer.from('collabMsg:owner'),
|
||||
pack({
|
||||
type: 'customEventStart',
|
||||
documentName: 'page.x',
|
||||
eventName: 'boom',
|
||||
payload: {},
|
||||
replyTo: 'collabMsg:origin',
|
||||
replyId: 7,
|
||||
}),
|
||||
);
|
||||
await flush();
|
||||
|
||||
const replies = bus.published
|
||||
.filter((p) => p.channel === 'collabMsg:origin')
|
||||
.map((p) => unpack(p.message));
|
||||
expect(replies).toHaveLength(1);
|
||||
expect(replies[0]).toMatchObject({
|
||||
type: 'customEventComplete',
|
||||
replyId: 7,
|
||||
error: 'kaboom',
|
||||
});
|
||||
expect(unhandled).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('origin rejects PROMPTLY with the real error (not a TTL TIMEOUT) when the remote handler throws', async () => {
|
||||
const bus = new FakeRedisBus();
|
||||
// Owner already holds the document lock.
|
||||
bus.locks.set('collabLock:page.x', 'owner');
|
||||
makeExtension(bus, 'owner', {
|
||||
boom: async () => {
|
||||
throw new Error('kaboom');
|
||||
},
|
||||
});
|
||||
const origin = makeExtension(bus, 'origin', {
|
||||
boom: async () => undefined,
|
||||
});
|
||||
|
||||
const promise = (origin as any).handleEvent('boom', 'page.x', { foo: 1 });
|
||||
// Attach a catch immediately so a rejection is never momentarily unhandled.
|
||||
const settled = promise.then(
|
||||
() => ({ ok: true as const }),
|
||||
(e: unknown) => ({ ok: false as const, error: e }),
|
||||
);
|
||||
|
||||
await flush();
|
||||
// Resolves WITHOUT advancing any timer -> the 30s TIMEOUT fallback did not fire.
|
||||
const result = await settled;
|
||||
expect(result.ok).toBe(false);
|
||||
expect((result as any).error).toBeInstanceOf(Error);
|
||||
expect(((result as any).error as Error).message).toBe('kaboom');
|
||||
expect(unhandled).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('origin resolves with the payload when the remote handler succeeds (unchanged behavior)', async () => {
|
||||
const bus = new FakeRedisBus();
|
||||
bus.locks.set('collabLock:page.x', 'owner');
|
||||
makeExtension(bus, 'owner', {
|
||||
ok: async (_doc: string, payload: any) => ({ echoed: payload }),
|
||||
});
|
||||
const origin = makeExtension(bus, 'origin', {
|
||||
ok: async () => undefined,
|
||||
});
|
||||
|
||||
const promise = (origin as any).handleEvent('ok', 'page.x', { foo: 1 });
|
||||
await flush();
|
||||
await expect(promise).resolves.toEqual({ echoed: { foo: 1 } });
|
||||
expect(unhandled).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -51,9 +51,15 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
private instance!: Hocuspocus;
|
||||
private readonly customEvents: TCE;
|
||||
private replyIdCounter: number = 0;
|
||||
// @ts-ignore
|
||||
private pendingReplies: Record<number, PromiseWithResolvers<any>['resolve']> =
|
||||
{};
|
||||
private pendingReplies: Record<
|
||||
number,
|
||||
{
|
||||
// @ts-ignore
|
||||
resolve: PromiseWithResolvers<any>['resolve'];
|
||||
// @ts-ignore
|
||||
reject: PromiseWithResolvers<any>['reject'];
|
||||
}
|
||||
> = {};
|
||||
|
||||
constructor(configuration: Configuration<TCE>) {
|
||||
const {
|
||||
@@ -176,25 +182,45 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
}
|
||||
if (type === 'customEventStart') {
|
||||
const { documentName, eventName, payload, replyTo, replyId } = msg;
|
||||
const res = await this.handleEventLocally(
|
||||
eventName as Extract<keyof TCE, string>,
|
||||
documentName,
|
||||
payload,
|
||||
);
|
||||
const reply: RSAMessageCustomEventComplete = {
|
||||
type: 'customEventComplete',
|
||||
replyId,
|
||||
payload: res,
|
||||
};
|
||||
let reply: RSAMessageCustomEventComplete;
|
||||
try {
|
||||
const res = await this.handleEventLocally(
|
||||
eventName as Extract<keyof TCE, string>,
|
||||
documentName,
|
||||
payload,
|
||||
);
|
||||
reply = {
|
||||
type: 'customEventComplete',
|
||||
replyId,
|
||||
payload: res,
|
||||
};
|
||||
} catch (err) {
|
||||
// The remote handler threw (e.g. the markdown->ProseMirror transform in
|
||||
// gitSyncWriteBody can throw on a malformed body). Reply with the error on
|
||||
// the SAME correlation channel so the origin rejects promptly with the real
|
||||
// message instead of waiting out customEventTTL as a generic 'TIMEOUT'.
|
||||
// Catching here also keeps the throw from escaping this async messageBuffer
|
||||
// listener as an unhandledRejection on the owning instance.
|
||||
reply = {
|
||||
type: 'customEventComplete',
|
||||
replyId,
|
||||
payload: undefined,
|
||||
error: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
this.pub.publish(`${replyTo}`, this.pack(reply));
|
||||
return;
|
||||
}
|
||||
if (type === 'customEventComplete') {
|
||||
const { replyId, payload } = msg;
|
||||
const resolveFn = this.pendingReplies[replyId];
|
||||
if (!resolveFn) return;
|
||||
const { replyId, payload, error } = msg;
|
||||
const pending = this.pendingReplies[replyId];
|
||||
if (!pending) return;
|
||||
delete this.pendingReplies[replyId];
|
||||
resolveFn(payload);
|
||||
if (error !== undefined) {
|
||||
pending.reject(new Error(error));
|
||||
} else {
|
||||
pending.resolve(payload);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const { socketId } = msg;
|
||||
@@ -273,11 +299,22 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
};
|
||||
const msg = this.pack(proxyMessage);
|
||||
this.pub.publish(`${this.msgChannel}:${proxyTo}`, msg);
|
||||
// @ts-ignore
|
||||
const { promise, resolve, reject } = Promise.withResolvers();
|
||||
this.pendingReplies[replyId] = resolve;
|
||||
// Manual deferred (no Promise.withResolvers) so this runs on Node < 22 too.
|
||||
let resolve!: (v: unknown) => void;
|
||||
let reject!: (e: unknown) => void;
|
||||
const promise = new Promise((res, rej) => {
|
||||
resolve = res;
|
||||
reject = rej;
|
||||
});
|
||||
this.pendingReplies[replyId] = { resolve, reject };
|
||||
setTimeout(() => {
|
||||
reject('TIMEOUT');
|
||||
// Fallback for a genuinely lost reply. A handler that threw now rejects
|
||||
// promptly via the error-carrying customEventComplete above; this TIMEOUT
|
||||
// only fires when no reply ever comes back.
|
||||
if (this.pendingReplies[replyId]) {
|
||||
delete this.pendingReplies[replyId];
|
||||
reject('TIMEOUT');
|
||||
}
|
||||
}, this.customEventTTL);
|
||||
return promise as Promise<ReturnType<TCE[TName]>>;
|
||||
}
|
||||
|
||||
@@ -72,6 +72,10 @@ export type RSAMessageCustomEventComplete = {
|
||||
type: 'customEventComplete';
|
||||
replyId: number;
|
||||
payload: unknown;
|
||||
// When the remote handler THREW, the owner sends back the error message here
|
||||
// instead of a payload, so the origin can reject its awaiting promise promptly
|
||||
// (with the real error) rather than waiting out the customEventTTL timeout.
|
||||
error?: string;
|
||||
};
|
||||
|
||||
export type RSAMessage =
|
||||
|
||||
@@ -0,0 +1,587 @@
|
||||
/**
|
||||
* JEST CONFIG NOTE (#119 ESM refactor): this is the one spec that needs the REAL
|
||||
* `@docmost/git-sync` converter (not a mock). The package is now ESM, which jest
|
||||
* cannot `require()` nor `import()` without --experimental-vm-modules, so the
|
||||
* server jest config `moduleNameMapper`s `@docmost/git-sync` to its TS SOURCE and
|
||||
* strips the ESM `.js` import suffixes. ts-jest then type-checks that source under
|
||||
* the server's (looser) tsconfig and trips a benign narrowing; the global
|
||||
* `isolatedModules: true` on the ts-jest transform (apps/server/package.json)
|
||||
* makes it transpile-only so this spec loads. Full type-checking of the package
|
||||
* is still enforced by its own `tsc`/vitest gates and the server `tsc --noEmit`.
|
||||
*
|
||||
* §13.1 IDEMPOTENCY GATE — the blocking gate for git-sync Phase B.
|
||||
*
|
||||
* Proves the `@docmost/git-sync` pure converter is schema-compatible
|
||||
* with the server's REAL editor-ext document schema: a representative corpus of
|
||||
* editor-ext ProseMirror documents must survive a full round trip through the
|
||||
* actual server write path without losing any node / mark / attribute.
|
||||
*
|
||||
* Pipeline per document (issue #194 §13.1):
|
||||
* 1. md = convertProseMirrorToMarkdown(content) // git-sync export
|
||||
* 2. doc = await markdownToProseMirror(md) // git-sync import
|
||||
* 3. push `doc` through the REAL editor-ext Yjs write path the server uses:
|
||||
* ydoc = TiptapTransformer.toYdoc(doc, 'default', tiptapExtensions)
|
||||
* normalized = TiptapTransformer.fromYdoc(ydoc, 'default')
|
||||
* This is exactly what PersistenceExtension does on store
|
||||
* (apps/server/src/collaboration/extensions/persistence.extension.ts:96/115)
|
||||
* with the same `tiptapExtensions` (collaboration.util.ts) and the same
|
||||
* `@hocuspocus/transformer`, so the gate exercises the real schema
|
||||
* validation that runs on a git-sync write (issue #194 §3.3).
|
||||
* 4. assert docsCanonicallyEqual(canon(original), canon(normalized)) === true
|
||||
*
|
||||
* Any node / mark / attr that editor-ext drops (because the git-sync
|
||||
* docmost-schema named it differently, or declares a different default) makes
|
||||
* the gate FAIL for that document — exactly the schema-divergence issue #194 §3.3 /
|
||||
* §13.1 warn about. Genuine, irreducible divergences are isolated into the
|
||||
* clearly-named `KNOWN DIVERGENCE` block at the bottom (never silently hidden).
|
||||
*
|
||||
* Requires the workspace packages built first:
|
||||
* pnpm --filter @docmost/editor-ext build
|
||||
* pnpm --filter @docmost/git-sync build
|
||||
*/
|
||||
import { TiptapTransformer } from '@hocuspocus/transformer';
|
||||
// Import the server's real schema FIRST so `@docmost/editor-ext` resolves to its
|
||||
// built CJS `dist` (its `main`). The ESM-only `@docmost/git-sync` package is
|
||||
// mapped to its TS SOURCE by the jest `moduleNameMapper` (the built ESM cannot
|
||||
// be `require()`d nor dynamically `import()`ed under jest's node VM), so ts-jest
|
||||
// transpiles the real converter to CJS here — exercising the actual converter
|
||||
// the server ships, not a stub.
|
||||
import { tiptapExtensions } from './collaboration.util';
|
||||
import {
|
||||
convertProseMirrorToMarkdown,
|
||||
markdownToProseMirror,
|
||||
canonicalizeContent,
|
||||
docsCanonicallyEqual,
|
||||
} from '@docmost/git-sync';
|
||||
|
||||
/**
|
||||
* Run a single editor-ext document through the full gate pipeline and return
|
||||
* the canonical original vs the canonical doc as it lands after the real Yjs
|
||||
* write path, plus the intermediate markdown for diagnostics.
|
||||
*/
|
||||
async function runGate(original: any): Promise<{
|
||||
md: string;
|
||||
imported: any;
|
||||
normalized: any;
|
||||
canonOriginal: any;
|
||||
canonNormalized: any;
|
||||
}> {
|
||||
// 1) editor-ext JSON -> markdown (git-sync export).
|
||||
const md = convertProseMirrorToMarkdown(original);
|
||||
|
||||
// 2) markdown -> ProseMirror JSON (git-sync import, docmost-schema).
|
||||
const imported = await markdownToProseMirror(md);
|
||||
|
||||
// 3) push through the REAL editor-ext schema via the server's Yjs write path.
|
||||
// toYdoc validates `imported` against tiptapExtensions (throws on an
|
||||
// unknown node, drops unknown attrs); fromYdoc reads it back as the
|
||||
// normalized editor-ext JSON the server would persist.
|
||||
const ydoc = TiptapTransformer.toYdoc(imported, 'default', tiptapExtensions);
|
||||
const normalized = TiptapTransformer.fromYdoc(ydoc, 'default');
|
||||
|
||||
return {
|
||||
md,
|
||||
imported,
|
||||
normalized,
|
||||
canonOriginal: canonicalizeContent(original),
|
||||
canonNormalized: canonicalizeContent(normalized),
|
||||
};
|
||||
}
|
||||
|
||||
const doc = (...content: any[]) => ({ type: 'doc', content });
|
||||
const text = (t: string, marks?: any[]) =>
|
||||
marks ? { type: 'text', text: t, marks } : { type: 'text', text: t };
|
||||
const para = (...content: any[]) => ({ type: 'paragraph', content });
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Corpus: editor-ext ProseMirror documents covering the common node/mark types.
|
||||
// Node / mark / attr names and DEFAULTS are taken from the real schema —
|
||||
// editor-ext (packages/editor-ext/src) + the server's tiptapExtensions
|
||||
// (collaboration.util.ts) — NOT guessed. Where editor-ext materializes a
|
||||
// non-null default on import (e.g. image.align="center", callout.type, list
|
||||
// start) the fixture pre-authors that materialized value so the round trip is
|
||||
// already at its fixpoint (matches how the engine normalizes-on-write, SPEC §11).
|
||||
// ---------------------------------------------------------------------------
|
||||
const CORPUS: Record<string, any> = {
|
||||
'paragraphs + headings (h1-h3)': doc(
|
||||
{ type: 'heading', attrs: { level: 1 }, content: [text('Heading one')] },
|
||||
{ type: 'heading', attrs: { level: 2 }, content: [text('Heading two')] },
|
||||
{ type: 'heading', attrs: { level: 3 }, content: [text('Heading three')] },
|
||||
para(text('A plain paragraph of text.')),
|
||||
para(text('Second paragraph.')),
|
||||
),
|
||||
|
||||
// A non-default paragraph alignment now round-trips (item #7 fix): it exports
|
||||
// as `<p style="text-align:center">` and the schema's paragraph parseHTML
|
||||
// reads `style="text-align"` back onto `textAlign` on import, so the alignment
|
||||
// survives the full editor-ext write path. Promoted from the old KNOWN
|
||||
// DIVERGENCE block (which only heading alignment still occupies).
|
||||
'aligned paragraph (textAlign center)': doc({
|
||||
type: 'paragraph',
|
||||
attrs: { textAlign: 'center' },
|
||||
content: [text('centered')],
|
||||
}),
|
||||
|
||||
'inline marks (bold/italic/strike/code)': doc(
|
||||
para(
|
||||
text('normal '),
|
||||
text('bold', [{ type: 'bold' }]),
|
||||
text(' '),
|
||||
text('italic', [{ type: 'italic' }]),
|
||||
text(' '),
|
||||
text('struck', [{ type: 'strike' }]),
|
||||
text(' '),
|
||||
text('code', [{ type: 'code' }]),
|
||||
),
|
||||
),
|
||||
|
||||
'links': doc(
|
||||
para(
|
||||
text('see '),
|
||||
text('the site', [
|
||||
{ type: 'link', attrs: { href: 'https://example.com' } },
|
||||
]),
|
||||
text(' for more'),
|
||||
),
|
||||
),
|
||||
|
||||
'bullet list': doc({
|
||||
type: 'bulletList',
|
||||
content: [
|
||||
{ type: 'listItem', content: [para(text('first'))] },
|
||||
{ type: 'listItem', content: [para(text('second'))] },
|
||||
{ type: 'listItem', content: [para(text('third'))] },
|
||||
],
|
||||
}),
|
||||
|
||||
'ordered list': doc({
|
||||
type: 'orderedList',
|
||||
attrs: { start: 1 },
|
||||
content: [
|
||||
{ type: 'listItem', content: [para(text('one'))] },
|
||||
{ type: 'listItem', content: [para(text('two'))] },
|
||||
],
|
||||
}),
|
||||
|
||||
'task list (checkbox)': doc({
|
||||
type: 'taskList',
|
||||
content: [
|
||||
{
|
||||
type: 'taskItem',
|
||||
attrs: { checked: true },
|
||||
content: [para(text('done item'))],
|
||||
},
|
||||
{
|
||||
type: 'taskItem',
|
||||
attrs: { checked: false },
|
||||
content: [para(text('todo item'))],
|
||||
},
|
||||
],
|
||||
}),
|
||||
|
||||
'blockquote': doc({
|
||||
type: 'blockquote',
|
||||
content: [para(text('a quoted line')), para(text('second quoted line'))],
|
||||
}),
|
||||
|
||||
'callout (info)': doc({
|
||||
type: 'callout',
|
||||
attrs: { type: 'info' },
|
||||
content: [para(text('an informational callout'))],
|
||||
}),
|
||||
|
||||
'callout (warning)': doc({
|
||||
type: 'callout',
|
||||
attrs: { type: 'warning' },
|
||||
content: [para(text('a warning callout'))],
|
||||
}),
|
||||
|
||||
'code block (with language)': doc({
|
||||
type: 'codeBlock',
|
||||
attrs: { language: 'typescript' },
|
||||
// A fenced code block's body is stored with a trailing newline (the form a
|
||||
// markdown ``` fence round-trips to: marked normalizes the code text to end
|
||||
// in "\n"). Authoring the fixture at that fixpoint mirrors how the engine
|
||||
// normalizes-on-write (SPEC §11): codeBlock + `language` round-trip exactly.
|
||||
content: [text('const a: number = 1;\nconsole.log(a);\n')],
|
||||
}),
|
||||
|
||||
'horizontal rule': doc(
|
||||
para(text('before')),
|
||||
{ type: 'horizontalRule' },
|
||||
para(text('after')),
|
||||
),
|
||||
|
||||
'table (header row + cells)': doc({
|
||||
type: 'table',
|
||||
content: [
|
||||
{
|
||||
type: 'tableRow',
|
||||
content: [
|
||||
{
|
||||
type: 'tableHeader',
|
||||
attrs: { colspan: 1, rowspan: 1, colwidth: null },
|
||||
content: [para(text('Name'))],
|
||||
},
|
||||
{
|
||||
type: 'tableHeader',
|
||||
attrs: { colspan: 1, rowspan: 1, colwidth: null },
|
||||
content: [para(text('Value'))],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'tableRow',
|
||||
content: [
|
||||
{
|
||||
type: 'tableCell',
|
||||
attrs: { colspan: 1, rowspan: 1, colwidth: null },
|
||||
content: [para(text('alpha'))],
|
||||
},
|
||||
{
|
||||
type: 'tableCell',
|
||||
attrs: { colspan: 1, rowspan: 1, colwidth: null },
|
||||
content: [para(text('1'))],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
|
||||
// #8 — a table with a MULTI-BLOCK cell (two paragraphs). A GFM pipe table
|
||||
// cannot hold two blocks without flattening them; the converter emits a
|
||||
// lossless HTML <table> instead, and the two blocks must survive the round trip.
|
||||
'table (multi-block cell, #8)': doc({
|
||||
type: 'table',
|
||||
content: [
|
||||
{
|
||||
type: 'tableRow',
|
||||
content: [
|
||||
{
|
||||
type: 'tableHeader',
|
||||
attrs: { colspan: 1, rowspan: 1, colwidth: null },
|
||||
content: [para(text('H'))],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'tableRow',
|
||||
content: [
|
||||
{
|
||||
type: 'tableCell',
|
||||
attrs: { colspan: 1, rowspan: 1, colwidth: null },
|
||||
content: [para(text('first')), para(text('second'))],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
|
||||
// #7 — a table nested inside a column. Columns render as HTML containers, and a
|
||||
// table inside one must stay an HTML <table> (a GFM pipe table cannot live
|
||||
// inside an HTML block), round-tripping without being unwrapped or lost.
|
||||
// `widthMode` is pre-authored at its materialized `normal` default (SPEC §11).
|
||||
'table inside a column (#7)': doc({
|
||||
type: 'columns',
|
||||
attrs: { layout: 'two', widthMode: 'normal' },
|
||||
content: [
|
||||
{
|
||||
type: 'column',
|
||||
content: [
|
||||
{
|
||||
type: 'table',
|
||||
content: [
|
||||
{
|
||||
type: 'tableRow',
|
||||
content: [
|
||||
{
|
||||
type: 'tableHeader',
|
||||
attrs: { colspan: 1, rowspan: 1, colwidth: null },
|
||||
content: [para(text('C7'))],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{ type: 'column', content: [para(text('right'))] },
|
||||
],
|
||||
}),
|
||||
|
||||
// --- editor-ext nodes/marks beyond the original corpus (item #7) ----------
|
||||
// Each of these was verified to round-trip CLEANLY through the real gate
|
||||
// (export -> markdown -> import -> editor-ext Yjs write path). Fixtures are
|
||||
// pre-authored at the engine's normalize-on-write fixpoint (SPEC §11), e.g.
|
||||
// details carries the materialized `open:false`, and color marks use the
|
||||
// `rgb(...)` form the HTML re-parser normalizes to.
|
||||
|
||||
'mention (user)': doc(
|
||||
para(
|
||||
text('hi '),
|
||||
{
|
||||
type: 'mention',
|
||||
attrs: {
|
||||
id: 'user-123',
|
||||
label: 'Alice',
|
||||
entityType: 'user',
|
||||
entityId: 'user-123',
|
||||
creatorId: 'creator-1',
|
||||
},
|
||||
},
|
||||
text(' there'),
|
||||
),
|
||||
),
|
||||
|
||||
'inline math': doc(
|
||||
para(
|
||||
text('inline '),
|
||||
{ type: 'mathInline', attrs: { text: 'x^2' } },
|
||||
text(' math'),
|
||||
),
|
||||
),
|
||||
|
||||
'block math': doc({ type: 'mathBlock', attrs: { text: 'x^2 + y^2 = z^2' } }),
|
||||
|
||||
'details (collapsible)': doc({
|
||||
type: 'details',
|
||||
// `open:false` is the value editor-ext materializes on import; pre-authoring
|
||||
// it puts the fixture at its round-trip fixpoint.
|
||||
attrs: { open: false },
|
||||
content: [
|
||||
{ type: 'detailsSummary', content: [text('Summary line')] },
|
||||
{ type: 'detailsContent', content: [para(text('hidden body'))] },
|
||||
],
|
||||
}),
|
||||
|
||||
'highlight (mark, no color)': doc(
|
||||
para(
|
||||
text('a '),
|
||||
text('highlighted', [{ type: 'highlight' }]),
|
||||
text(' word'),
|
||||
),
|
||||
),
|
||||
|
||||
'highlight (mark, with color)': doc(
|
||||
para(
|
||||
text('a '),
|
||||
text('red', [{ type: 'highlight', attrs: { color: 'rgb(255, 0, 0)' } }]),
|
||||
text(' word'),
|
||||
),
|
||||
),
|
||||
|
||||
'subscript': doc(
|
||||
para(text('H'), text('2', [{ type: 'subscript' }]), text('O')),
|
||||
),
|
||||
|
||||
'superscript': doc(
|
||||
para(text('E=mc'), text('2', [{ type: 'superscript' }])),
|
||||
),
|
||||
|
||||
'text color (textStyle)': doc(
|
||||
// The HTML re-parser normalizes CSS colors to the `rgb(...)` form, so the
|
||||
// fixture pre-authors that form; a `#hex` color would round-trip to the
|
||||
// equivalent rgb() and is therefore a value-normalization divergence (see
|
||||
// the KNOWN DIVERGENCE block below).
|
||||
para(text('green', [{ type: 'textStyle', attrs: { color: 'rgb(0, 255, 0)' } }])),
|
||||
),
|
||||
|
||||
'nested / mixed document': doc(
|
||||
{ type: 'heading', attrs: { level: 1 }, content: [text('Mixed')] },
|
||||
para(
|
||||
text('intro with '),
|
||||
text('bold', [{ type: 'bold' }]),
|
||||
text(' and a '),
|
||||
text('link', [{ type: 'link', attrs: { href: 'https://example.com' } }]),
|
||||
text('.'),
|
||||
),
|
||||
{
|
||||
type: 'bulletList',
|
||||
content: [
|
||||
{
|
||||
type: 'listItem',
|
||||
content: [
|
||||
para(text('item with '), text('code', [{ type: 'code' }])),
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'listItem',
|
||||
content: [
|
||||
para(text('item with sublist')),
|
||||
{
|
||||
type: 'bulletList',
|
||||
content: [
|
||||
{ type: 'listItem', content: [para(text('nested a'))] },
|
||||
{ type: 'listItem', content: [para(text('nested b'))] },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'callout',
|
||||
attrs: { type: 'success' },
|
||||
content: [
|
||||
para(text('callout body')),
|
||||
{ type: 'codeBlock', attrs: { language: 'bash' }, content: [text('echo hi\n')] },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'blockquote',
|
||||
content: [para(text('quote at the end'))],
|
||||
},
|
||||
),
|
||||
|
||||
// Atom embeds that carry no inline text: they must round-trip via their
|
||||
// schema-matching HTML (data-type div), NOT a literal that re-imports as plain
|
||||
// text. `subpages` used to export as the literal "{{SUBPAGES}}" and came back
|
||||
// as visible text on the page (red-team round-trip data loss) — this locks it.
|
||||
// editor-ext materializes the `recursive: false` default on import, so the
|
||||
// fixture pre-authors it to sit at the round-trip fixpoint (matches the other
|
||||
// default-materializing fixtures above).
|
||||
'subpages embed': doc({ type: 'subpages', attrs: { recursive: false } }),
|
||||
};
|
||||
|
||||
describe('git-sync converter §13.1 idempotency gate (editor-ext schema)', () => {
|
||||
for (const [name, original] of Object.entries(CORPUS)) {
|
||||
it(`round-trips losslessly: ${name}`, async () => {
|
||||
const { md, canonOriginal, canonNormalized } = await runGate(original);
|
||||
|
||||
const equal = docsCanonicallyEqual(original, canonNormalized);
|
||||
if (!equal) {
|
||||
// Surface a readable diff so a real divergence is actionable.
|
||||
// eslint-disable-next-line no-console
|
||||
console.error(
|
||||
`\n[GATE FAIL] ${name}\n--- markdown ---\n${md}\n` +
|
||||
`--- canonical original ---\n${JSON.stringify(canonOriginal, null, 2)}\n` +
|
||||
`--- canonical round-tripped ---\n${JSON.stringify(canonNormalized, null, 2)}\n`,
|
||||
);
|
||||
}
|
||||
expect(equal).toBe(true);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Image layout attrs preserved by canon #4 (width/height/align all round-trip).
|
||||
//
|
||||
// The `image` NODE round-trips through editor-ext fine. Plain markdown ``
|
||||
// has no way to express layout attrs, so the canonical converter (#293/#326
|
||||
// canon decision #4) appends a machine comment `<!--img {...}-->` carrying the
|
||||
// NON-DEFAULT attrs, and re-parses it on import — the same trailing-comment
|
||||
// pattern used for media/textAlign.
|
||||
//
|
||||
// About `align` — DO NOT repeat an earlier false diagnosis: align is NOT lost.
|
||||
// `center` is the schema default, so the emitter omits it (only left/right go
|
||||
// into the comment) and the importer restores it via the image `align` default
|
||||
// ("center"). `canonNormalized.align` reads `undefined` here ONLY because
|
||||
// `canonicalizeContent` normalizes the "center" default away SYMMETRICALLY (from
|
||||
// both the original and the round trip), so docsCanonicallyEqual is unaffected —
|
||||
// this is canonical-form normalization, not a divergence. Verified empirically:
|
||||
// left/right survive the raw round trip; center is restored on import then
|
||||
// canon-stripped on both sides. The real round-trip instability in this family
|
||||
// was the empty-string-vs-absent class (image.alt `absent -> ""`), fixed
|
||||
// parse-side in the converter PACKAGE on develop (PR #350). This branch absorbs
|
||||
// that fix via the next develop merge; nothing to flip here for align.
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('git-sync converter §13.1 image layout attrs round-trip (align via schema default)', () => {
|
||||
const imageDoc = doc({
|
||||
type: 'image',
|
||||
attrs: {
|
||||
src: 'https://example.com/pic.png',
|
||||
width: 640,
|
||||
height: 480,
|
||||
align: 'center',
|
||||
},
|
||||
});
|
||||
|
||||
it('preserves width/height via the canon `<!--img {...}-->` comment; center align is the default', async () => {
|
||||
const { md, canonNormalized } = await runGate(imageDoc);
|
||||
|
||||
// Canon #4: bare `` plus a trailing `<!--img {...}-->` comment that
|
||||
// carries the non-default layout attrs. `center` is the default, so it is
|
||||
// correctly OMITTED from the comment (only width/height appear here).
|
||||
expect(md.trim()).toBe(
|
||||
' <!--img {"width":"640","height":"480"}-->',
|
||||
);
|
||||
|
||||
// The round-tripped image keeps src + width/height. width/height are
|
||||
// re-imported as strings (matching the video/audio/pdf string convention),
|
||||
// so assert the values rather than the JS type.
|
||||
const imgAttrs = (canonNormalized as any).content[0].attrs;
|
||||
expect((canonNormalized as any).content[0].type).toBe('image');
|
||||
expect(imgAttrs.src).toBe('https://example.com/pic.png');
|
||||
expect(String(imgAttrs.width)).toBe('640');
|
||||
expect(String(imgAttrs.height)).toBe('480');
|
||||
// `align` is NOT lost — see the block comment above: `center` is the schema
|
||||
// default, restored on import then normalized away symmetrically by
|
||||
// canonicalize, so it reads `undefined` on the CANONICAL form (not a loss).
|
||||
expect(imgAttrs.align).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// HEADING text alignment — now round-trips (item A1; formerly a KNOWN DIVERGENCE).
|
||||
// Symmetric with the paragraph fix: a heading's non-default `textAlign` is
|
||||
// exported as a styled `<hN style="text-align:…">` (was a bare ATX `## text`
|
||||
// that dropped it) and re-parsed by the heading + textAlign parseHTML on import,
|
||||
// so a non-default heading alignment SURVIVES a full round trip.
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('git-sync converter §13.1 heading text alignment round-trips', () => {
|
||||
it('preserves a heading textAlign across the markdown round trip', async () => {
|
||||
const alignedHeading = doc({
|
||||
type: 'heading',
|
||||
attrs: { level: 2, textAlign: 'center' },
|
||||
content: [text('centered heading')],
|
||||
});
|
||||
|
||||
const { md, canonNormalized } = await runGate(alignedHeading);
|
||||
|
||||
// Canon #9: ATX heading plus a trailing `<!--attrs {...}-->` comment carrying
|
||||
// the non-default textAlign (was a lossy bare `## centered heading`).
|
||||
expect(md.trim()).toBe(
|
||||
'## centered heading <!--attrs {"textAlign":"center"}-->',
|
||||
);
|
||||
expect(docsCanonicallyEqual(alignedHeading, canonNormalized)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// KNOWN DIVERGENCE — textStyle color is VALUE-NORMALIZED, not lost (item #7).
|
||||
//
|
||||
// The textStyle/color mark itself round-trips (the green CORPUS has the rgb()
|
||||
// form). But a `#hex` color is normalized to the equivalent `rgb(...)` string
|
||||
// by the HTML re-parser on import, and canonicalize.ts does NOT normalize color
|
||||
// formats — so a `#hex` original is not STRING-identical to its round trip even
|
||||
// though the color is semantically preserved. Locked here so the boundary is
|
||||
// explicit: author color fixtures in rgb() form to stay in the green corpus.
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('git-sync converter §13.1 KNOWN DIVERGENCE (textStyle color #hex -> rgb)', () => {
|
||||
it('normalizes a #hex text color to rgb() (semantically preserved, string-divergent)', async () => {
|
||||
const hexDoc = doc(
|
||||
para(text('green', [{ type: 'textStyle', attrs: { color: '#00ff00' } }])),
|
||||
);
|
||||
|
||||
const { canonNormalized } = await runGate(hexDoc);
|
||||
|
||||
// Color survives, but as the normalized rgb() string.
|
||||
expect(canonNormalized).toEqual({
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'green',
|
||||
marks: [{ type: 'textStyle', attrs: { color: 'rgb(0, 255, 0)' } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
// Not string-identical to the #hex original.
|
||||
expect(docsCanonicallyEqual(hexDoc, canonNormalized)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
/**
|
||||
* Backward-filled LCS length table for sequences `a` and `b`: `dp[i][j]` is the
|
||||
* length of the longest common subsequence of the suffixes `a[i:]` and `b[j:]`.
|
||||
* O(n*m) time/space — fine for page block counts.
|
||||
*
|
||||
* Shared by the two-way block diff (`yjs-body-merge.diffBlocks`) and the
|
||||
* three-way merge planner (`three-way-merge.lcsPairs`) so the (identical) table
|
||||
* construction lives in ONE place; each caller does its own traceback over the
|
||||
* returned table.
|
||||
*/
|
||||
export function buildLcsTable(a: string[], b: string[]): number[][] {
|
||||
const n = a.length;
|
||||
const m = b.length;
|
||||
const dp: number[][] = Array.from({ length: n + 1 }, () =>
|
||||
new Array(m + 1).fill(0),
|
||||
);
|
||||
for (let i = n - 1; i >= 0; i--) {
|
||||
for (let j = m - 1; j >= 0; j--) {
|
||||
dp[i][j] =
|
||||
a[i] === b[j]
|
||||
? dp[i + 1][j + 1] + 1
|
||||
: Math.max(dp[i + 1][j], dp[i][j + 1]);
|
||||
}
|
||||
}
|
||||
return dp;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
import { diff3Plan, type Pick } from './three-way-merge';
|
||||
|
||||
// Materialize a plan into the merged key sequence for assertion.
|
||||
function apply(plan: Pick[], live: string[], target: string[]): string[] {
|
||||
return plan.map((p) => (p.src === 'live' ? live[p.index] : target[p.index]));
|
||||
}
|
||||
|
||||
const merge = (o: string[], a: string[], b: string[]): string[] =>
|
||||
apply(diff3Plan(o, a, b), a, b);
|
||||
|
||||
describe('diff3Plan red-team #9 (human edit + adjacent git insert)', () => {
|
||||
it('keeps human block-2 edit AND applies git insert of 2.5', () => {
|
||||
// base: 1 2 3
|
||||
// live: 1 H 3 (human rewrote block 2)
|
||||
// target: 1 2 2.5 3 (git inserted 2.5 after block 2)
|
||||
expect(
|
||||
merge(['1', '2', '3'], ['1', 'H', '3'], ['1', '2', '2.5', '3']),
|
||||
).toEqual(['1', 'H', '2.5', '3']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,159 @@
|
||||
import {
|
||||
diff3Plan,
|
||||
diff3PlanWithConflicts,
|
||||
type Pick,
|
||||
} from './three-way-merge';
|
||||
|
||||
// Materialize a plan into the merged key sequence for assertion.
|
||||
function apply(plan: Pick[], live: string[], target: string[]): string[] {
|
||||
return plan.map((p) => (p.src === 'live' ? live[p.index] : target[p.index]));
|
||||
}
|
||||
|
||||
const merge = (o: string[], a: string[], b: string[]): string[] =>
|
||||
apply(diff3Plan(o, a, b), a, b);
|
||||
|
||||
describe('diff3Plan (block-level three-way merge)', () => {
|
||||
it('identical on all three sides -> unchanged (all from live)', () => {
|
||||
const plan = diff3Plan(['1', '2', '3'], ['1', '2', '3'], ['1', '2', '3']);
|
||||
expect(plan.every((p) => p.src === 'live')).toBe(true);
|
||||
expect(apply(plan, ['1', '2', '3'], ['1', '2', '3'])).toEqual(['1', '2', '3']);
|
||||
});
|
||||
|
||||
it('git changed a block the human did not -> takes git', () => {
|
||||
expect(merge(['1', '2', '3'], ['1', '2', '3'], ['1', '9', '3'])).toEqual([
|
||||
'1',
|
||||
'9',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
it('human changed a block git did not -> KEEPS the human edit (the core 3-way win)', () => {
|
||||
expect(merge(['1', '2', '3'], ['1', 'H', '3'], ['1', '2', '3'])).toEqual([
|
||||
'1',
|
||||
'H',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
// Bug #2 observability: diff3PlanWithConflicts reports SAME-BLOCK conflicts so
|
||||
// the caller can surface the "git wins" loss (log + history pin) instead of
|
||||
// dropping the human side silently.
|
||||
describe('diff3PlanWithConflicts (same-block conflict reporting)', () => {
|
||||
it('reports 0 conflicts when sides changed DIFFERENT blocks (clean merge)', () => {
|
||||
const r = diff3PlanWithConflicts(
|
||||
['1', '2', '3'],
|
||||
['H', '2', '3'],
|
||||
['1', '2', 'G'],
|
||||
);
|
||||
expect(r.conflicts).toBe(0);
|
||||
expect(apply(r.picks, ['H', '2', '3'], ['1', '2', 'G'])).toEqual([
|
||||
'H',
|
||||
'2',
|
||||
'G',
|
||||
]);
|
||||
});
|
||||
|
||||
it('reports 1 conflict and git wins when BOTH rewrote the SAME block', () => {
|
||||
const r = diff3PlanWithConflicts(
|
||||
['1', '2', '3'],
|
||||
['1', 'H', '3'], // human rewrote block 2
|
||||
['1', 'G', '3'], // git rewrote block 2
|
||||
);
|
||||
expect(r.conflicts).toBe(1);
|
||||
// Git wins the contested block; the human 'H' is NOT in the picks.
|
||||
expect(apply(r.picks, ['1', 'H', '3'], ['1', 'G', '3'])).toEqual([
|
||||
'1',
|
||||
'G',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
it('does NOT count a git-only region (no human content to lose) as a conflict', () => {
|
||||
const r = diff3PlanWithConflicts(
|
||||
['1', '2', '3'],
|
||||
['1', '2', '3'], // human unchanged
|
||||
['1', '9', '3'], // git rewrote block 2
|
||||
);
|
||||
expect(r.conflicts).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
it('human and git changed DIFFERENT blocks -> both preserved', () => {
|
||||
// human rewrote block 1, git rewrote block 3.
|
||||
expect(merge(['1', '2', '3'], ['H', '2', '3'], ['1', '2', 'G'])).toEqual([
|
||||
'H',
|
||||
'2',
|
||||
'G',
|
||||
]);
|
||||
});
|
||||
|
||||
it('human inserted a block AND git changed a different block -> both preserved', () => {
|
||||
expect(
|
||||
merge(['1', '2', '3'], ['1', '1.5', '2', '3'], ['1', '2', 'G']),
|
||||
).toEqual(['1', '1.5', '2', 'G']);
|
||||
});
|
||||
|
||||
it('both changed the SAME block -> conflict resolves to git', () => {
|
||||
expect(merge(['1', '2', '3'], ['1', 'H', '3'], ['1', 'G', '3'])).toEqual([
|
||||
'1',
|
||||
'G',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
it('both made the SAME edit -> that edit (no duplication)', () => {
|
||||
expect(merge(['1', '2', '3'], ['1', 'X', '3'], ['1', 'X', '3'])).toEqual([
|
||||
'1',
|
||||
'X',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
it('human deleted a block git left alone -> deletion preserved', () => {
|
||||
expect(merge(['1', '2', '3'], ['1', '3'], ['1', '2', '3'])).toEqual([
|
||||
'1',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
it('git deleted a block the human left alone -> deletion applied', () => {
|
||||
expect(merge(['1', '2', '3'], ['1', '2', '3'], ['1', '3'])).toEqual([
|
||||
'1',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
it('both deleted the same block -> gone (no conflict)', () => {
|
||||
expect(merge(['1', '2', '3'], ['1', '3'], ['1', '3'])).toEqual(['1', '3']);
|
||||
});
|
||||
|
||||
it('git appended a trailing block -> appended', () => {
|
||||
expect(merge(['1', '2'], ['1', '2'], ['1', '2', '3'])).toEqual([
|
||||
'1',
|
||||
'2',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
it('human appended a trailing block git did not -> kept', () => {
|
||||
expect(merge(['1', '2'], ['1', '2', '3'], ['1', '2'])).toEqual([
|
||||
'1',
|
||||
'2',
|
||||
'3',
|
||||
]);
|
||||
});
|
||||
|
||||
it('empty base, git provides content (brand-new page body) -> git content', () => {
|
||||
expect(merge([], [], ['1', '2'])).toEqual(['1', '2']);
|
||||
});
|
||||
|
||||
it('git changed block 1, human edited block 3, far apart -> both kept', () => {
|
||||
expect(
|
||||
merge(
|
||||
['a', 'b', 'c', 'd', 'e'],
|
||||
['a', 'b', 'c', 'd', 'E'],
|
||||
['A', 'b', 'c', 'd', 'e'],
|
||||
),
|
||||
).toEqual(['A', 'b', 'c', 'd', 'E']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,284 @@
|
||||
/**
|
||||
* Pure block-level THREE-WAY merge planner (diff3) over arrays of opaque block
|
||||
* keys. Used by the git-sync body write to merge an incoming git body into the
|
||||
* live page using the last-synced version as the common ancestor (review #5):
|
||||
*
|
||||
* - a block only the human changed (live != base, git == base) -> keep LIVE
|
||||
* - a block only git changed (git != base, live == base) -> take GIT
|
||||
* - a block both sides changed (a real conflict) -> GIT wins
|
||||
* - inserts/deletes from either side are preserved when unambiguous
|
||||
*
|
||||
* Content-agnostic: it works on string keys and returns the merged block order as
|
||||
* picks ({ src: 'live'|'target', index }) — the caller (the Yjs applier)
|
||||
* materializes them — so the whole algorithm is unit-testable on plain arrays.
|
||||
*
|
||||
* Algorithm: anchor on base blocks present (unchanged) in BOTH live and target
|
||||
* (their LCS-with-base intersection). Between consecutive anchors lies one region
|
||||
* the human and/or git rewrote; resolve each region three-way. Stable anchor
|
||||
* blocks are emitted from LIVE so the applier keeps the existing Yjs block
|
||||
* instances (and the human's in-flight edits) in place.
|
||||
*
|
||||
* LOCATION (deferred): this and its `lcs.ts` sibling are pure, framework-free and
|
||||
* could conceptually live in `packages/git-sync` (the engine). They are kept in
|
||||
* the server integration on purpose: `packages/git-sync` is a VENDORED engine
|
||||
* (pinned upstream, manually re-synced), so adding first-party files there
|
||||
* complicates the re-sync story, and the only consumer today is the server. Move
|
||||
* them into the engine only once the vendoring re-sync story is settled.
|
||||
*/
|
||||
|
||||
import { buildLcsTable } from './lcs';
|
||||
|
||||
/** Matched index pairs of the longest common subsequence of `a` and `b`. */
|
||||
function lcsPairs(a: string[], b: string[]): Array<[number, number]> {
|
||||
const n = a.length;
|
||||
const m = b.length;
|
||||
const dp = buildLcsTable(a, b);
|
||||
const pairs: Array<[number, number]> = [];
|
||||
let i = 0;
|
||||
let j = 0;
|
||||
while (i < n && j < m) {
|
||||
if (a[i] === b[j]) {
|
||||
pairs.push([i, j]);
|
||||
i++;
|
||||
j++;
|
||||
} else if (dp[i + 1][j] >= dp[i][j + 1]) {
|
||||
i++;
|
||||
} else {
|
||||
j++;
|
||||
}
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
/** o-index -> matched index in the other side (only for LCS-matched blocks). */
|
||||
function matchMap(pairs: Array<[number, number]>): Map<number, number> {
|
||||
const m = new Map<number, number>();
|
||||
for (const [o, x] of pairs) m.set(o, x);
|
||||
return m;
|
||||
}
|
||||
|
||||
/**
|
||||
* One change `side` made to `base` within a region: base blocks `[oStart,oEnd)`
|
||||
* were replaced by the side's blocks listed in `content` (region-local indices).
|
||||
* A pure insert has `oStart === oEnd`; a pure delete has empty `content`.
|
||||
*/
|
||||
interface Hunk {
|
||||
oStart: number;
|
||||
oEnd: number;
|
||||
content: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Diff `o` against one side as a list of non-overlapping hunks (the base spans
|
||||
* the side rewrote/inserted/deleted), derived from their LCS alignment.
|
||||
*/
|
||||
function buildHunks(o: string[], side: string[]): Hunk[] {
|
||||
const pairs = lcsPairs(o, side); // [oIdx, sideIdx] kept (unchanged) blocks
|
||||
const hunks: Hunk[] = [];
|
||||
let prevO = -1;
|
||||
let prevS = -1;
|
||||
const flush = (curO: number, curS: number): void => {
|
||||
const oStart = prevO + 1;
|
||||
const oEnd = curO;
|
||||
const content: number[] = [];
|
||||
for (let s = prevS + 1; s < curS; s++) content.push(s);
|
||||
if (oEnd > oStart || content.length > 0) hunks.push({ oStart, oEnd, content });
|
||||
};
|
||||
for (const [oIdx, sIdx] of pairs) {
|
||||
flush(oIdx, sIdx);
|
||||
prevO = oIdx;
|
||||
prevS = sIdx;
|
||||
}
|
||||
flush(o.length, side.length);
|
||||
return hunks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Do two hunks (one per side) touch the same base region? Pure inserts only
|
||||
* collide when nested strictly inside the other hunk's base span (or, for two
|
||||
* inserts, at the same gap); changes sitting at a shared boundary do not.
|
||||
*/
|
||||
function hunksOverlap(a: Hunk, b: Hunk): boolean {
|
||||
const aIns = a.oStart === a.oEnd;
|
||||
const bIns = b.oStart === b.oEnd;
|
||||
if (aIns && bIns) return a.oStart === b.oStart;
|
||||
if (aIns) return b.oStart < a.oStart && a.oStart < b.oEnd;
|
||||
if (bIns) return a.oStart < b.oStart && b.oStart < a.oEnd;
|
||||
return Math.max(a.oStart, b.oStart) < Math.min(a.oEnd, b.oEnd);
|
||||
}
|
||||
|
||||
interface LocalPick {
|
||||
src: 'live' | 'target';
|
||||
local: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fine-grained three-way merge of ONE inter-anchor region. Combines the human's
|
||||
* and git's NON-overlapping hunks (e.g. a human edit to one block plus a git
|
||||
* insert/delete of OTHER blocks in the same region) so neither change is lost.
|
||||
* Returns the merged region as region-local picks, or `null` when the two sides
|
||||
* changed the SAME base block — a genuine conflict the caller resolves by the
|
||||
* original all-or-nothing rule (git wins the whole region).
|
||||
*/
|
||||
function tryMergeRegion(
|
||||
o: string[],
|
||||
a: string[],
|
||||
b: string[],
|
||||
): LocalPick[] | null {
|
||||
// Agreement short-circuit (review #11). When live (a) and target (b) are
|
||||
// identical, both sides converged on the SAME result — diff3 "agreement", NOT
|
||||
// a conflict. This is the dominant echo case (live == target != base) that
|
||||
// otherwise trips the overlap check below and is logged as a false "N same-block
|
||||
// conflict(s) resolved to the git version", masking REAL data-loss signals.
|
||||
// Emit the region straight from live (which equals target); no conflict.
|
||||
if (a.length === b.length && a.every((v, i) => v === b[i])) {
|
||||
return a.map((_v, i) => ({ src: 'live', local: i }) as LocalPick);
|
||||
}
|
||||
|
||||
const aHunks = buildHunks(o, a);
|
||||
const bHunks = buildHunks(o, b);
|
||||
|
||||
// Any overlap between a human hunk and a git hunk is a real conflict; bail so
|
||||
// the caller falls back to git-wins (preserving the original behavior).
|
||||
for (const ah of aHunks) {
|
||||
for (const bh of bHunks) {
|
||||
if (hunksOverlap(ah, bh)) return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Disjoint: live index of each base block that BOTH sides kept (stable).
|
||||
const aKept = matchMap(lcsPairs(o, a)); // base index -> live index
|
||||
|
||||
const out: LocalPick[] = [];
|
||||
let pa = 0;
|
||||
let pb = 0;
|
||||
let oi = 0;
|
||||
while (oi < o.length || pa < aHunks.length || pb < bHunks.length) {
|
||||
const ah = pa < aHunks.length ? aHunks[pa] : null;
|
||||
const bh = pb < bHunks.length ? bHunks[pb] : null;
|
||||
const nextStart = Math.min(
|
||||
ah ? ah.oStart : o.length,
|
||||
bh ? bh.oStart : o.length,
|
||||
);
|
||||
|
||||
// Emit stable base blocks (kept by both) until the next hunk, from LIVE.
|
||||
while (oi < nextStart) {
|
||||
out.push({ src: 'live', local: aKept.get(oi) as number });
|
||||
oi++;
|
||||
}
|
||||
if (!ah && !bh) break;
|
||||
|
||||
// Apply the hunk at oi. When both sides act here they are disjoint, so the
|
||||
// pure-insert (oEnd === oi) is emitted before the side that consumes base oi.
|
||||
const aHere = ah !== null && ah.oStart === oi;
|
||||
const bHere = bh !== null && bh.oStart === oi;
|
||||
let useA: boolean;
|
||||
if (aHere && bHere) {
|
||||
useA = ah!.oEnd === oi; // insert side first; otherwise either order is fine
|
||||
} else {
|
||||
useA = aHere;
|
||||
}
|
||||
const h = (useA ? ah : bh) as Hunk;
|
||||
const src: 'live' | 'target' = useA ? 'live' : 'target';
|
||||
for (const idx of h.content) out.push({ src, local: idx });
|
||||
oi = h.oEnd;
|
||||
if (useA) pa++;
|
||||
else pb++;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface Pick {
|
||||
src: 'live' | 'target';
|
||||
index: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The merged block order PLUS how many regions resolved as a genuine SAME-BLOCK
|
||||
* conflict (both sides rewrote the same base block — `tryMergeRegion` returned
|
||||
* null and git won the whole region, so the live/human version of those blocks
|
||||
* is NOT in `picks`). `conflicts > 0` is the OBSERVABLE signal the caller uses to
|
||||
* surface "git won a concurrent same-block edit" (log it + pin the human
|
||||
* baseline to page history) instead of dropping the human side silently.
|
||||
*/
|
||||
export interface Diff3Result {
|
||||
picks: Pick[];
|
||||
conflicts: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Three-way merge of base `o`, live `a`, target `b` (arrays of block keys).
|
||||
* Returns the merged block order as picks from live/target. Thin wrapper over
|
||||
* `diff3PlanWithConflicts` (kept for the existing pure-array callers/tests).
|
||||
*/
|
||||
export function diff3Plan(o: string[], a: string[], b: string[]): Pick[] {
|
||||
return diff3PlanWithConflicts(o, a, b).picks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Like `diff3Plan` but also reports the SAME-BLOCK conflict count (see
|
||||
* `Diff3Result`). A region where both the human and git rewrote the same base
|
||||
* block cannot be merged automatically; the rule is deterministic — GIT WINS the
|
||||
* whole region — but the human's version of those blocks is then absent from the
|
||||
* picks, so we count it so the caller can make the loss observable/recoverable
|
||||
* rather than silent (the documented conflict contract).
|
||||
*/
|
||||
export function diff3PlanWithConflicts(
|
||||
o: string[],
|
||||
a: string[],
|
||||
b: string[],
|
||||
): Diff3Result {
|
||||
const oToA = matchMap(lcsPairs(o, a));
|
||||
const oToB = matchMap(lcsPairs(o, b));
|
||||
|
||||
const res: Pick[] = [];
|
||||
let conflicts = 0;
|
||||
let oi = 0;
|
||||
let ai = 0;
|
||||
let bi = 0;
|
||||
|
||||
for (;;) {
|
||||
// Next anchor: a base block present (unchanged) in BOTH live and target.
|
||||
let anchor = oi;
|
||||
while (anchor < o.length && !(oToA.has(anchor) && oToB.has(anchor))) {
|
||||
anchor++;
|
||||
}
|
||||
const aEnd = anchor < o.length ? (oToA.get(anchor) as number) : a.length;
|
||||
const bEnd = anchor < o.length ? (oToB.get(anchor) as number) : b.length;
|
||||
|
||||
// Resolve the region [oi,anchor) that one or both sides rewrote/inserted.
|
||||
// Try a fine-grained three-way merge first so a human block-edit survives a
|
||||
// git insert/delete of OTHER blocks in the same region; only a genuine
|
||||
// same-block conflict (null) falls back to the original git-wins rule.
|
||||
const merged = tryMergeRegion(
|
||||
o.slice(oi, anchor),
|
||||
a.slice(ai, aEnd),
|
||||
b.slice(bi, bEnd),
|
||||
);
|
||||
if (merged) {
|
||||
for (const p of merged) {
|
||||
res.push(
|
||||
p.src === 'live'
|
||||
? { src: 'live', index: ai + p.local }
|
||||
: { src: 'target', index: bi + p.local },
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// SAME-BLOCK CONFLICT: count it ONLY when the human side actually had
|
||||
// content in this region that git's win discards (live region non-empty).
|
||||
// A region only git rewrote (live region empty) is not a human loss.
|
||||
if (aEnd > ai) conflicts++;
|
||||
for (let k = bi; k < bEnd; k++) res.push({ src: 'target', index: k });
|
||||
}
|
||||
|
||||
if (anchor >= o.length) break;
|
||||
|
||||
// Emit the stable anchor block from LIVE, then advance past it on all sides.
|
||||
res.push({ src: 'live', index: aEnd });
|
||||
ai = aEnd + 1;
|
||||
bi = bEnd + 1;
|
||||
oi = anchor + 1;
|
||||
}
|
||||
|
||||
return { picks: res, conflicts };
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
import { TiptapTransformer } from '@hocuspocus/transformer';
|
||||
import * as Y from 'yjs';
|
||||
import {
|
||||
markdownToProseMirror,
|
||||
convertProseMirrorToMarkdown,
|
||||
} from '@docmost/git-sync';
|
||||
|
||||
import { tiptapExtensions } from '../collaboration.util';
|
||||
import { mergeXmlFragments, mergeXmlFragments3Way } from './yjs-body-merge';
|
||||
|
||||
/**
|
||||
* Regression for the QA #119 callout findings (body-duplication re-verify +
|
||||
* "callout strips the whole body"). These reproduce the ACTUAL live merge path:
|
||||
*
|
||||
* live = TiptapTransformer.toYdoc(editor JSON, tiptapExtensions) (the
|
||||
* collaboration server's materialization — schema defaults stamped)
|
||||
* git = toYdoc(markdownToProseMirror(convertProseMirrorToMarkdown(editor)))
|
||||
* (the engine round-trip the push side feeds into writePageBody)
|
||||
*
|
||||
* A page containing a callout (with a neighbouring heading + paragraphs) must:
|
||||
* - merge with ZERO ops on an unchanged resync (no duplication — bug #1), and
|
||||
* - NEVER lose blocks / collapse to empty (no strip — bug #2),
|
||||
* across repeated cycles, for every editor-canonical callout type.
|
||||
*/
|
||||
|
||||
const toYdoc = (content: unknown[]) =>
|
||||
TiptapTransformer.toYdoc(
|
||||
{ type: 'doc', content },
|
||||
'default',
|
||||
tiptapExtensions as any,
|
||||
);
|
||||
|
||||
const blockTypes = (f: Y.XmlFragment) =>
|
||||
f.toArray().map((n: any) => n.nodeName);
|
||||
|
||||
function editorPage(calloutType: string) {
|
||||
return [
|
||||
{
|
||||
type: 'heading',
|
||||
attrs: { id: 'h1', level: 1 },
|
||||
content: [{ type: 'text', text: 'Title here' }],
|
||||
},
|
||||
{
|
||||
type: 'paragraph',
|
||||
attrs: { id: 'p1' },
|
||||
content: [{ type: 'text', text: 'Para before callout' }],
|
||||
},
|
||||
{
|
||||
type: 'callout',
|
||||
attrs: { type: calloutType },
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
attrs: { id: 'pc' },
|
||||
content: [{ type: 'text', text: 'Inside the callout' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'paragraph',
|
||||
attrs: { id: 'p2' },
|
||||
content: [{ type: 'text', text: 'Para after callout' }],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
async function gitRoundTrip(content: unknown[]): Promise<any[]> {
|
||||
const md = await convertProseMirrorToMarkdown({ type: 'doc', content });
|
||||
const json = await markdownToProseMirror(md);
|
||||
return json.content;
|
||||
}
|
||||
|
||||
describe('git-sync callout merge is idempotent + non-destructive (QA #119)', () => {
|
||||
for (const type of ['info', 'note', 'warning', 'danger', 'success', 'default']) {
|
||||
it(`callout(${type}) resyncs with 0 ops and never strips the body`, async () => {
|
||||
const editor = editorPage(type);
|
||||
const gitContent = await gitRoundTrip(editor);
|
||||
|
||||
const liveDoc = toYdoc(editor);
|
||||
const live = liveDoc.getXmlFragment('default');
|
||||
const before = live.toArray().length;
|
||||
expect(before).toBe(4);
|
||||
|
||||
// 2-way: live vs the git round-trip -> no-op (no dup, no strip).
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments(live, toYdoc(gitContent).getXmlFragment('default'));
|
||||
});
|
||||
expect(applied).toBe(0);
|
||||
expect(live.toArray().length).toBe(before);
|
||||
|
||||
// 3-way across 4 cycles with base == git (the steady-state) -> stable.
|
||||
for (let cycle = 0; cycle < 4; cycle++) {
|
||||
let a = -1;
|
||||
liveDoc.transact(() => {
|
||||
a = mergeXmlFragments3Way(
|
||||
live,
|
||||
toYdoc(gitContent).getXmlFragment('default'),
|
||||
toYdoc(gitContent).getXmlFragment('default'),
|
||||
);
|
||||
});
|
||||
expect(a).toBe(0);
|
||||
expect(live.toArray().length).toBe(before);
|
||||
expect(blockTypes(live)).toEqual([
|
||||
'heading',
|
||||
'paragraph',
|
||||
'callout',
|
||||
'paragraph',
|
||||
]);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
it('3-way with a stale base (callout JUST added) keeps the callout + neighbours', async () => {
|
||||
// base = the previously-synced version WITHOUT the callout (git round-trip);
|
||||
// the human just inserted the callout -> the merge must KEEP everything.
|
||||
const prev = [
|
||||
{ type: 'heading', attrs: { id: 'h1', level: 1 }, content: [{ type: 'text', text: 'Title here' }] },
|
||||
{ type: 'paragraph', attrs: { id: 'p1' }, content: [{ type: 'text', text: 'Para before callout' }] },
|
||||
{ type: 'paragraph', attrs: { id: 'p2' }, content: [{ type: 'text', text: 'Para after callout' }] },
|
||||
];
|
||||
const editor = editorPage('info');
|
||||
const baseContent = await gitRoundTrip(prev);
|
||||
const gitContent = await gitRoundTrip(editor);
|
||||
|
||||
const liveDoc = toYdoc(editor);
|
||||
const live = liveDoc.getXmlFragment('default');
|
||||
liveDoc.transact(() => {
|
||||
mergeXmlFragments3Way(
|
||||
live,
|
||||
toYdoc(gitContent).getXmlFragment('default'),
|
||||
toYdoc(baseContent).getXmlFragment('default'),
|
||||
);
|
||||
});
|
||||
// Body survives in full — NOT stripped to empty / a lone paragraph.
|
||||
expect(blockTypes(live)).toEqual([
|
||||
'heading',
|
||||
'paragraph',
|
||||
'callout',
|
||||
'paragraph',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('git-sync callout type fidelity (QA "callout type -> [!info]")', () => {
|
||||
for (const type of ['info', 'note', 'warning', 'danger', 'success', 'default']) {
|
||||
it(`preserves callout type "${type}" across the engine round-trip`, async () => {
|
||||
const content = editorPage(type);
|
||||
const gitContent = await gitRoundTrip(content);
|
||||
const co = gitContent.find((b: any) => b.type === 'callout');
|
||||
expect(co?.attrs?.type).toBe(type);
|
||||
});
|
||||
}
|
||||
|
||||
it('maps a known GitHub/Obsidian alias to the editor banner (tip -> success)', async () => {
|
||||
// `tip` is not a schema callout type — it is an input alias the editor itself
|
||||
// maps onto the supported set (GITHUB_ALERT_TYPE_MAP: tip -> success). git-sync
|
||||
// mirrors that so the ingest lands on the closest banner instead of flatly info.
|
||||
const content = editorPage('tip');
|
||||
const gitContent = await gitRoundTrip(content);
|
||||
const co = gitContent.find((b: any) => b.type === 'callout');
|
||||
expect(co?.attrs?.type).toBe('success');
|
||||
});
|
||||
|
||||
it('flattens a genuinely unknown callout type to info', async () => {
|
||||
const content = editorPage('banana'); // not a type and not a known alias
|
||||
const gitContent = await gitRoundTrip(content);
|
||||
const co = gitContent.find((b: any) => b.type === 'callout');
|
||||
expect(co?.attrs?.type).toBe('info');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,198 @@
|
||||
import * as Y from 'yjs';
|
||||
|
||||
import { mergeXmlFragments, mergeXmlFragments3Way } from './yjs-body-merge';
|
||||
|
||||
/**
|
||||
* Regression for the HIGH-severity runaway whole-body duplication: a page body
|
||||
* was RE-APPENDED in full on every git-sync reconcile cycle, unbounded, with NO
|
||||
* client connected.
|
||||
*
|
||||
* ROOT CAUSE (confirmed in-process against the real failing page): the LIVE Yjs
|
||||
* document materializes the editor-schema default `indent: 0` on every
|
||||
* paragraph/heading (and on the paragraph inside every list item, callout, and
|
||||
* table cell), but a body re-imported from git — parsed from clean markdown —
|
||||
* carries NO indent attribute. So every live block's comparison key differed from
|
||||
* the same block coming back from git; the three-way merge could anchor on
|
||||
* NOTHING, and the trailing unit that git's export already contained (but the
|
||||
* merge could not match against the byte-identical live tail) was re-appended
|
||||
* each cycle. Each grown export then diverged from the last-pushed base by one
|
||||
* more unit — a self-sustaining loop.
|
||||
*
|
||||
* The fix normalizes the materialized default (`indent: 0`) out of the block key
|
||||
* (the schema-derived `serializeXmlNode` normalization in yjs-body-merge.ts drops
|
||||
* every attr equal to its ProseMirror-schema default; `indent: 0` is one such),
|
||||
* so a live block compares equal to its git-round-tripped twin and the resync is
|
||||
* a true no-op. The sibling `yjs-body-merge.schema-defaults.spec.ts` covers the
|
||||
* rest of the bug class (image.align, link mark internal, …).
|
||||
*
|
||||
* These tests model that EXACTLY at the Yjs level: a LIVE fragment whose blocks
|
||||
* carry `indent: 0` + block ids, versus a git-derived fragment of the SAME
|
||||
* content with neither — for a body built from BYTE-IDENTICAL units that each
|
||||
* contain a heading, a paragraph, a callout, and a table with empty cells (the
|
||||
* trigger). RED before the fix (the merge applies > 0 ops and the body grows),
|
||||
* GREEN after (0 ops, no growth).
|
||||
*/
|
||||
|
||||
type Attrs = Record<string, string | number>;
|
||||
|
||||
function el(
|
||||
name: string,
|
||||
attrs: Attrs,
|
||||
children: (Y.XmlElement | Y.XmlText)[],
|
||||
) {
|
||||
const e = new Y.XmlElement(name);
|
||||
for (const [k, v] of Object.entries(attrs)) e.setAttribute(k, v as string);
|
||||
if (children.length) e.insert(0, children);
|
||||
return e;
|
||||
}
|
||||
|
||||
function text(s: string): Y.XmlText {
|
||||
const t = new Y.XmlText();
|
||||
if (s) t.insert(0, s);
|
||||
return t;
|
||||
}
|
||||
|
||||
/**
|
||||
* One byte-identical content unit (heading / paragraph / callout / table-with-
|
||||
* empty-cells). `live` toggles the two things that exist ONLY in the live Yjs
|
||||
* doc and NOT in a git round-trip: the materialized `indent: 0` default and the
|
||||
* per-block `id`. `n` makes each unit's ids unique (as the editor would stamp)
|
||||
* while keeping the visible CONTENT byte-identical across units.
|
||||
*/
|
||||
function unit(
|
||||
live: boolean,
|
||||
n: number,
|
||||
headingText = 'Big Heading',
|
||||
): Y.XmlElement[] {
|
||||
const ind: Attrs = live ? { indent: 0 } : {};
|
||||
const id = (base: string): Attrs => (live ? { id: `${base}${n}` } : {});
|
||||
const para = (attrs: Attrs, s: string) =>
|
||||
el('paragraph', { ...attrs, ...ind }, [text(s)]);
|
||||
|
||||
const cell = (name: string) =>
|
||||
el(name, { colspan: 1, rowspan: 1 }, [para({}, '')]);
|
||||
|
||||
return [
|
||||
el('heading', { ...id('h'), level: 1, ...ind }, [text(headingText)]),
|
||||
para(id('p'), 'Para with the same words'),
|
||||
el('callout', { type: 'info' }, [para(id('c'), 'CalloutText here')]),
|
||||
el('table', {}, [
|
||||
el('tableRow', {}, [cell('tableHeader'), cell('tableHeader')]),
|
||||
el('tableRow', {}, [cell('tableCell'), cell('tableCell')]),
|
||||
]),
|
||||
];
|
||||
}
|
||||
|
||||
function fragmentOf(units: Y.XmlElement[][]): {
|
||||
doc: Y.Doc;
|
||||
frag: Y.XmlFragment;
|
||||
} {
|
||||
const doc = new Y.Doc();
|
||||
const frag = doc.getXmlFragment('default');
|
||||
const blocks = units.flat();
|
||||
if (blocks.length) frag.insert(0, blocks);
|
||||
return { doc, frag };
|
||||
}
|
||||
|
||||
const blockCount = (frag: Y.XmlFragment): number => frag.toArray().length;
|
||||
|
||||
describe('git-sync reconcile import is idempotent (no whole-body duplication)', () => {
|
||||
const UNITS = 3;
|
||||
|
||||
it('3-way: identical content, live carries indent:0, base stale-by-one -> 0 ops, no growth', () => {
|
||||
// LIVE: the editor-stamped Yjs doc (indent:0 + ids on every block).
|
||||
const { doc: liveDoc, frag: live } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => unit(true, i)),
|
||||
);
|
||||
// INCOMING (git export -> re-import): same content, NO indent / ids.
|
||||
const { frag: incoming } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => unit(false, i)),
|
||||
);
|
||||
// BASE = last-pushed file, lagging by ONE unit (the realistic divergence
|
||||
// that drives the trailing insert-vs-insert).
|
||||
const { frag: base } = fragmentOf(
|
||||
Array.from({ length: UNITS - 1 }, (_, i) => unit(false, i)),
|
||||
);
|
||||
|
||||
const before = blockCount(live);
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments3Way(live, incoming, base);
|
||||
});
|
||||
|
||||
expect(applied).toBe(0);
|
||||
expect(blockCount(live)).toBe(before);
|
||||
});
|
||||
|
||||
it('3-way is a fixpoint across repeated cycles (does not grow)', () => {
|
||||
const { doc: liveDoc, frag: live } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => unit(true, i)),
|
||||
);
|
||||
const incomingUnits = () =>
|
||||
fragmentOf(Array.from({ length: UNITS }, (_, i) => unit(false, i))).frag;
|
||||
const baseUnits = () =>
|
||||
fragmentOf(Array.from({ length: UNITS - 1 }, (_, i) => unit(false, i)))
|
||||
.frag;
|
||||
|
||||
const before = blockCount(live);
|
||||
for (let cycle = 0; cycle < 5; cycle++) {
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments3Way(live, incomingUnits(), baseUnits());
|
||||
});
|
||||
expect(applied).toBe(0);
|
||||
expect(blockCount(live)).toBe(before);
|
||||
}
|
||||
});
|
||||
|
||||
it('2-way: identical content, live carries indent:0 -> 0 ops, no growth', () => {
|
||||
const { doc: liveDoc, frag: live } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => unit(true, i)),
|
||||
);
|
||||
const { frag: incoming } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => unit(false, i)),
|
||||
);
|
||||
|
||||
const before = blockCount(live);
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments(live, incoming);
|
||||
});
|
||||
|
||||
expect(applied).toBe(0);
|
||||
expect(blockCount(live)).toBe(before);
|
||||
});
|
||||
|
||||
it('does NOT regress real edits: a git change to one block still lands', () => {
|
||||
const { doc: liveDoc, frag: live } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => unit(true, i)),
|
||||
);
|
||||
const base = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => unit(false, i)),
|
||||
).frag;
|
||||
// git edits the heading text of the LAST unit.
|
||||
const incoming = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) =>
|
||||
unit(false, i, i === UNITS - 1 ? 'EDITED Heading' : 'Big Heading'),
|
||||
),
|
||||
).frag;
|
||||
|
||||
const before = blockCount(live);
|
||||
liveDoc.transact(() => {
|
||||
mergeXmlFragments3Way(live, incoming, base);
|
||||
});
|
||||
|
||||
// The edit landed, and the body did NOT grow (one block changed in place).
|
||||
const headings = live
|
||||
.toArray()
|
||||
.filter((b) => (b as Y.XmlElement).nodeName === 'heading')
|
||||
.map((b) =>
|
||||
(b as Y.XmlElement)
|
||||
.toArray()
|
||||
.map((c) => (c as Y.XmlText).toString())
|
||||
.join(''),
|
||||
);
|
||||
expect(headings).toContain('EDITED Heading');
|
||||
expect(blockCount(live)).toBe(before);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,316 @@
|
||||
import { TiptapTransformer } from '@hocuspocus/transformer';
|
||||
import * as Y from 'yjs';
|
||||
|
||||
import { tiptapExtensions } from '../collaboration.util';
|
||||
import { mergeXmlFragments, mergeXmlFragments3Way } from './yjs-body-merge';
|
||||
|
||||
/**
|
||||
* Regression for the BUG CLASS behind the runaway whole-body duplication: the
|
||||
* point-fix (7a7b840e) only normalized `indent: 0`, but the SAME divergence
|
||||
* recurs for every attribute whose editor-ext (server) schema default the live
|
||||
* Yjs doc MATERIALIZES while the git round-trip — which comes through the engine
|
||||
* schema (different, usually null, defaults) plus `y-prosemirror`'s null-attr
|
||||
* dropping — does NOT carry. Confirmed triggers beyond `indent`:
|
||||
*
|
||||
* - `image.align` : editor-ext default "center" (materialized) vs engine
|
||||
* default null (dropped) -> element-attr divergence.
|
||||
* - link mark `internal`: editor-ext default false (materialized) vs engine
|
||||
* default null -> MARK-attr divergence (the prior denylist
|
||||
* could not reach marks at all — they are serialized raw in
|
||||
* the XmlText delta).
|
||||
*
|
||||
* `highlight.colorName` is normalized too (defense-in-depth); it is NOT a strong
|
||||
* real-world trigger because BOTH schemas default it to null, but the schema-
|
||||
* derived normalization handles it for free and stays idempotent.
|
||||
*
|
||||
* The fix derives the defaults from the ACTUAL ProseMirror schema (getSchema of
|
||||
* the server tiptapExtensions) and drops any element- OR mark-attribute equal to
|
||||
* its schema default (or null/undefined) from the block comparison key — so a
|
||||
* live block compares equal to its git-round-tripped twin and an unchanged
|
||||
* resync applies 0 ops. RED before the fix (keys diverge -> ops > 0 / growth),
|
||||
* GREEN after.
|
||||
*/
|
||||
|
||||
type Attrs = Record<string, unknown>;
|
||||
|
||||
function el(
|
||||
name: string,
|
||||
attrs: Attrs,
|
||||
children: (Y.XmlElement | Y.XmlText)[],
|
||||
): Y.XmlElement {
|
||||
const e = new Y.XmlElement(name);
|
||||
for (const [k, v] of Object.entries(attrs)) e.setAttribute(k, v as string);
|
||||
if (children.length) e.insert(0, children);
|
||||
return e;
|
||||
}
|
||||
|
||||
/** Text carrying marks, as the live Yjs doc stores them (XmlText format ops). */
|
||||
function markedText(s: string, marks: Record<string, unknown>): Y.XmlText {
|
||||
const t = new Y.XmlText();
|
||||
t.insert(0, s, marks);
|
||||
return t;
|
||||
}
|
||||
|
||||
/**
|
||||
* One byte-identical RICH unit: a paragraph with a LINK, a top-level IMAGE, and
|
||||
* a paragraph with a HIGHLIGHT. `live` toggles exactly what the editor
|
||||
* materializes but a git round-trip does not: block `id`, `indent: 0`,
|
||||
* `image.align: "center"`, the link mark's `internal: false`, and the
|
||||
* highlight's `colorName: null`.
|
||||
*/
|
||||
function richUnit(live: boolean, n: number): Y.XmlElement[] {
|
||||
const ind: Attrs = live ? { indent: 0 } : {};
|
||||
const id = (base: string): Attrs => (live ? { id: `${base}${n}` } : {});
|
||||
|
||||
const linkMarks = live
|
||||
? {
|
||||
link: {
|
||||
href: 'https://example.com',
|
||||
target: '_blank',
|
||||
rel: 'noopener noreferrer nofollow',
|
||||
class: null,
|
||||
title: null,
|
||||
internal: false, // editor-ext default, materialized
|
||||
},
|
||||
}
|
||||
: {
|
||||
link: {
|
||||
href: 'https://example.com',
|
||||
target: '_blank',
|
||||
rel: 'noopener noreferrer nofollow',
|
||||
internal: null, // engine default
|
||||
},
|
||||
};
|
||||
|
||||
const hlMarks = live
|
||||
? { highlight: { color: '#ffd43b', colorName: null } }
|
||||
: { highlight: { color: '#ffd43b' } };
|
||||
|
||||
const imageAttrs: Attrs = live
|
||||
? { src: 'https://img.example.com/a.png', align: 'center' } // materialized
|
||||
: { src: 'https://img.example.com/a.png' }; // align:null dropped on git side
|
||||
|
||||
return [
|
||||
el('paragraph', { ...id('lp'), ...ind }, [
|
||||
markedText('click here', linkMarks),
|
||||
]),
|
||||
el('image', imageAttrs, []),
|
||||
el('paragraph', { ...id('hp'), ...ind }, [markedText('hot', hlMarks)]),
|
||||
];
|
||||
}
|
||||
|
||||
function fragmentOf(units: Y.XmlElement[][]): {
|
||||
doc: Y.Doc;
|
||||
frag: Y.XmlFragment;
|
||||
} {
|
||||
const doc = new Y.Doc();
|
||||
const frag = doc.getXmlFragment('default');
|
||||
const blocks = units.flat();
|
||||
if (blocks.length) frag.insert(0, blocks);
|
||||
return { doc, frag };
|
||||
}
|
||||
|
||||
const blockCount = (frag: Y.XmlFragment): number => frag.toArray().length;
|
||||
|
||||
describe('git-sync reconcile is idempotent for schema-default attrs (image/link/highlight)', () => {
|
||||
const UNITS = 3;
|
||||
|
||||
it('3-way: live carries image.align/link.internal/indent defaults, base stale-by-one -> 0 ops', () => {
|
||||
const { doc: liveDoc, frag: live } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => richUnit(true, i)),
|
||||
);
|
||||
const { frag: incoming } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => richUnit(false, i)),
|
||||
);
|
||||
const { frag: base } = fragmentOf(
|
||||
Array.from({ length: UNITS - 1 }, (_, i) => richUnit(false, i)),
|
||||
);
|
||||
|
||||
const before = blockCount(live);
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments3Way(live, incoming, base);
|
||||
});
|
||||
|
||||
expect(applied).toBe(0);
|
||||
expect(blockCount(live)).toBe(before);
|
||||
});
|
||||
|
||||
it('2-way: live carries the materialized defaults -> 0 ops, no growth', () => {
|
||||
const { doc: liveDoc, frag: live } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => richUnit(true, i)),
|
||||
);
|
||||
const { frag: incoming } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => richUnit(false, i)),
|
||||
);
|
||||
|
||||
const before = blockCount(live);
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments(live, incoming);
|
||||
});
|
||||
|
||||
expect(applied).toBe(0);
|
||||
expect(blockCount(live)).toBe(before);
|
||||
});
|
||||
|
||||
it('is a fixpoint across repeated cycles (does not grow)', () => {
|
||||
const { doc: liveDoc, frag: live } = fragmentOf(
|
||||
Array.from({ length: UNITS }, (_, i) => richUnit(true, i)),
|
||||
);
|
||||
const incoming = () =>
|
||||
fragmentOf(Array.from({ length: UNITS }, (_, i) => richUnit(false, i)))
|
||||
.frag;
|
||||
const base = () =>
|
||||
fragmentOf(
|
||||
Array.from({ length: UNITS - 1 }, (_, i) => richUnit(false, i)),
|
||||
).frag;
|
||||
|
||||
const before = blockCount(live);
|
||||
for (let cycle = 0; cycle < 5; cycle++) {
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments3Way(live, incoming(), base());
|
||||
});
|
||||
expect(applied).toBe(0);
|
||||
expect(blockCount(live)).toBe(before);
|
||||
}
|
||||
});
|
||||
|
||||
it('does NOT regress a genuine non-default value (a real link.href / image.align:left still diffs)', () => {
|
||||
const { doc: liveDoc, frag: live } = fragmentOf([richUnit(true, 0)]);
|
||||
const base = fragmentOf([richUnit(false, 0)]).frag;
|
||||
// git genuinely changes the image alignment to a NON-default value.
|
||||
const incomingUnit = richUnit(false, 0);
|
||||
(incomingUnit[1] as Y.XmlElement).setAttribute('align', 'left');
|
||||
const incoming = fragmentOf([incomingUnit]).frag;
|
||||
|
||||
liveDoc.transact(() => {
|
||||
mergeXmlFragments3Way(live, incoming, base);
|
||||
});
|
||||
|
||||
const img = live
|
||||
.toArray()
|
||||
.find((b) => (b as Y.XmlElement).nodeName === 'image') as Y.XmlElement;
|
||||
expect(img.getAttribute('align')).toBe('left');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* FAITHFUL end-to-end proof through the REAL server transformer: build the live
|
||||
* doc the way the collaboration server does (defaults omitted in the JSON ->
|
||||
* TiptapTransformer.toYdoc MATERIALIZES image.align:"center", link.internal:false,
|
||||
* indent:0) versus the git-derived doc (engine-style: defaults emitted as
|
||||
* explicit null, no block ids). An unchanged resync must apply 0 ops.
|
||||
*/
|
||||
describe('git-sync reconcile is idempotent through the real toYdoc materialization', () => {
|
||||
const liveContent = [
|
||||
{
|
||||
type: 'paragraph',
|
||||
attrs: { id: 'p1' },
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'click here',
|
||||
marks: [{ type: 'link', attrs: { href: 'https://example.com' } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
{ type: 'image', attrs: { src: 'https://img.example.com/a.png' } },
|
||||
{
|
||||
type: 'paragraph',
|
||||
attrs: { id: 'p2' },
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'hot',
|
||||
marks: [{ type: 'highlight', attrs: { color: '#ffd43b' } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
// git/engine-style: explicit nulls for the engine-default attrs, no ids.
|
||||
const gitContent = [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'click here',
|
||||
marks: [
|
||||
{
|
||||
type: 'link',
|
||||
attrs: {
|
||||
href: 'https://example.com',
|
||||
target: '_blank',
|
||||
rel: 'noopener noreferrer nofollow',
|
||||
class: null,
|
||||
title: null,
|
||||
internal: null,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'image',
|
||||
attrs: { src: 'https://img.example.com/a.png', align: null },
|
||||
},
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'hot',
|
||||
marks: [
|
||||
{ type: 'highlight', attrs: { color: '#ffd43b', colorName: null } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const toYdoc = (content: unknown[]) =>
|
||||
TiptapTransformer.toYdoc(
|
||||
{ type: 'doc', content },
|
||||
'default',
|
||||
tiptapExtensions as any,
|
||||
);
|
||||
|
||||
it('3-way: materialized-default live vs engine-style git, base stale-by-one -> 0 ops', () => {
|
||||
const liveDoc = toYdoc(liveContent);
|
||||
const targetDoc = toYdoc(gitContent);
|
||||
const baseDoc = toYdoc(gitContent.slice(0, gitContent.length - 1));
|
||||
|
||||
const live = liveDoc.getXmlFragment('default');
|
||||
const before = live.toArray().length;
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments3Way(
|
||||
live,
|
||||
targetDoc.getXmlFragment('default'),
|
||||
baseDoc.getXmlFragment('default'),
|
||||
);
|
||||
});
|
||||
|
||||
expect(applied).toBe(0);
|
||||
expect(live.toArray().length).toBe(before);
|
||||
});
|
||||
|
||||
it('2-way: materialized-default live vs engine-style git -> 0 ops', () => {
|
||||
const liveDoc = toYdoc(liveContent);
|
||||
const targetDoc = toYdoc(gitContent);
|
||||
|
||||
const live = liveDoc.getXmlFragment('default');
|
||||
const before = live.toArray().length;
|
||||
let applied = -1;
|
||||
liveDoc.transact(() => {
|
||||
applied = mergeXmlFragments(live, targetDoc.getXmlFragment('default'));
|
||||
});
|
||||
|
||||
expect(applied).toBe(0);
|
||||
expect(live.toArray().length).toBe(before);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,373 @@
|
||||
import * as Y from 'yjs';
|
||||
|
||||
import {
|
||||
mergeXmlFragments,
|
||||
mergeXmlFragments3Way,
|
||||
mergeXmlFragments3WayWithStats,
|
||||
cloneXmlNode,
|
||||
diffBlocks,
|
||||
} from './yjs-body-merge';
|
||||
|
||||
// Build a Y.XmlFragment('default') in `doc` from a list of paragraph specs.
|
||||
// Each spec is the paragraph's plain text (a single XmlText child).
|
||||
function buildFragment(doc: Y.Doc, paragraphs: string[]): Y.XmlFragment {
|
||||
const frag = doc.getXmlFragment('default');
|
||||
const blocks = paragraphs.map((text) => {
|
||||
const el = new Y.XmlElement('paragraph');
|
||||
const t = new Y.XmlText();
|
||||
if (text) t.insert(0, text);
|
||||
el.insert(0, [t]);
|
||||
return el;
|
||||
});
|
||||
if (blocks.length) frag.insert(0, blocks);
|
||||
return frag;
|
||||
}
|
||||
|
||||
function texts(frag: Y.XmlFragment): string[] {
|
||||
return frag.toArray().map((el) => (el as Y.XmlElement).toArray()
|
||||
.map((c) => (c as Y.XmlText).toString())
|
||||
.join(''));
|
||||
}
|
||||
|
||||
describe('yjs-body-merge', () => {
|
||||
describe('diffBlocks (LCS edit script)', () => {
|
||||
it('identical sequences produce only keeps (no edits)', () => {
|
||||
const ops = diffBlocks(['a', 'b', 'c'], ['a', 'b', 'c']);
|
||||
expect(ops.every((o) => o.op === 'keep')).toBe(true);
|
||||
});
|
||||
|
||||
it('a single changed middle element is one del + one ins', () => {
|
||||
const ops = diffBlocks(['a', 'b', 'c'], ['a', 'B', 'c']);
|
||||
expect(ops.filter((o) => o.op === 'del')).toHaveLength(1);
|
||||
expect(ops.filter((o) => o.op === 'ins')).toHaveLength(1);
|
||||
expect(ops.filter((o) => o.op === 'keep')).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeXmlFragments', () => {
|
||||
it('identical content is a complete no-op (0 ops) — never clobbers an unchanged resync', () => {
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const liveFrag = buildFragment(live, ['one', 'two', 'three']);
|
||||
const targetFrag = buildFragment(target, ['one', 'two', 'three']);
|
||||
|
||||
// Capture block identities to prove they are left untouched.
|
||||
const before = liveFrag.toArray();
|
||||
let applied = -1;
|
||||
live.transact(() => {
|
||||
applied = mergeXmlFragments(liveFrag, targetFrag);
|
||||
});
|
||||
|
||||
expect(applied).toBe(0);
|
||||
// Same Y.XmlElement instances — nothing was deleted/recreated.
|
||||
expect(liveFrag.toArray()).toEqual(before);
|
||||
expect(texts(liveFrag)).toEqual(['one', 'two', 'three']);
|
||||
});
|
||||
|
||||
it('a human edit to one block survives a git change to a DIFFERENT block', () => {
|
||||
// Live: the human has the doc open; block 0 holds their edit. Git changed
|
||||
// only block 2. The merge must touch ONLY block 2 and leave block 0 (and
|
||||
// its in-flight edit) exactly as-is.
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const liveFrag = buildFragment(live, ['HUMAN EDIT', 'shared', 'old tail']);
|
||||
const targetFrag = buildFragment(target, [
|
||||
'HUMAN EDIT',
|
||||
'shared',
|
||||
'new tail from git',
|
||||
]);
|
||||
|
||||
const block0Before = liveFrag.get(0); // the human's block instance
|
||||
const block1Before = liveFrag.get(1);
|
||||
|
||||
let applied = -1;
|
||||
live.transact(() => {
|
||||
applied = mergeXmlFragments(liveFrag, targetFrag);
|
||||
});
|
||||
|
||||
// Only block 2 was replaced: one del + one ins.
|
||||
expect(applied).toBe(2);
|
||||
// The human's block and the shared block are the SAME instances (untouched).
|
||||
expect(liveFrag.get(0)).toBe(block0Before);
|
||||
expect(liveFrag.get(1)).toBe(block1Before);
|
||||
// Block 2 now carries git's content.
|
||||
expect(texts(liveFrag)).toEqual([
|
||||
'HUMAN EDIT',
|
||||
'shared',
|
||||
'new tail from git',
|
||||
]);
|
||||
});
|
||||
|
||||
it('appends a new trailing block without disturbing existing ones', () => {
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const liveFrag = buildFragment(live, ['a', 'b']);
|
||||
const targetFrag = buildFragment(target, ['a', 'b', 'c']);
|
||||
const a = liveFrag.get(0);
|
||||
const b = liveFrag.get(1);
|
||||
|
||||
let applied = -1;
|
||||
live.transact(() => {
|
||||
applied = mergeXmlFragments(liveFrag, targetFrag);
|
||||
});
|
||||
|
||||
expect(applied).toBe(1); // single insert
|
||||
expect(liveFrag.get(0)).toBe(a);
|
||||
expect(liveFrag.get(1)).toBe(b);
|
||||
expect(texts(liveFrag)).toEqual(['a', 'b', 'c']);
|
||||
});
|
||||
|
||||
it('deletes a removed block, keeping its neighbours', () => {
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const liveFrag = buildFragment(live, ['a', 'b', 'c']);
|
||||
const targetFrag = buildFragment(target, ['a', 'c']);
|
||||
const a = liveFrag.get(0);
|
||||
|
||||
let applied = -1;
|
||||
live.transact(() => {
|
||||
applied = mergeXmlFragments(liveFrag, targetFrag);
|
||||
});
|
||||
|
||||
expect(applied).toBe(1); // single delete
|
||||
expect(liveFrag.get(0)).toBe(a);
|
||||
expect(texts(liveFrag)).toEqual(['a', 'c']);
|
||||
});
|
||||
|
||||
it('a fully different body is replaced (and stays valid)', () => {
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const liveFrag = buildFragment(live, ['x', 'y']);
|
||||
const targetFrag = buildFragment(target, ['p', 'q', 'r']);
|
||||
live.transact(() => mergeXmlFragments(liveFrag, targetFrag));
|
||||
expect(texts(liveFrag)).toEqual(['p', 'q', 'r']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeXmlFragments3Way', () => {
|
||||
it('keeps a human edit to one block while applying a git change to another (3-way)', () => {
|
||||
// base (last synced): [a, b, c]. Human edited block 0 in the live doc; git
|
||||
// changed block 2 in the incoming file. 3-way must keep BOTH — the 2-way
|
||||
// merge would instead revert the human's block 0 to git's stale version.
|
||||
const base = new Y.Doc();
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const baseFrag = buildFragment(base, ['a', 'b', 'c']);
|
||||
const liveFrag = buildFragment(live, ['HUMAN', 'b', 'c']);
|
||||
const targetFrag = buildFragment(target, ['a', 'b', 'GIT']);
|
||||
|
||||
const humanBlock = liveFrag.get(0); // the human's live instance
|
||||
live.transact(() =>
|
||||
mergeXmlFragments3Way(liveFrag, targetFrag, baseFrag),
|
||||
);
|
||||
|
||||
// Human's block preserved as the SAME instance; git's change applied.
|
||||
expect(liveFrag.get(0)).toBe(humanBlock);
|
||||
expect(texts(liveFrag)).toEqual(['HUMAN', 'b', 'GIT']);
|
||||
});
|
||||
|
||||
it('a block both sides changed resolves to git (conflict policy)', () => {
|
||||
const base = new Y.Doc();
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const baseFrag = buildFragment(base, ['a', 'b', 'c']);
|
||||
const liveFrag = buildFragment(live, ['a', 'HUMAN', 'c']);
|
||||
const targetFrag = buildFragment(target, ['a', 'GIT', 'c']);
|
||||
|
||||
live.transact(() =>
|
||||
mergeXmlFragments3Way(liveFrag, targetFrag, baseFrag),
|
||||
);
|
||||
expect(texts(liveFrag)).toEqual(['a', 'GIT', 'c']);
|
||||
});
|
||||
|
||||
// Bug #2 observability: the stats variant reports the same-block conflict so
|
||||
// the handler can log it + the persistence layer can pin the human baseline.
|
||||
it('reports the same-block conflict count via mergeXmlFragments3WayWithStats', () => {
|
||||
const base = new Y.Doc();
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const baseFrag = buildFragment(base, ['a', 'b', 'c']);
|
||||
const liveFrag = buildFragment(live, ['a', 'HUMAN', 'c']);
|
||||
const targetFrag = buildFragment(target, ['a', 'GIT', 'c']);
|
||||
|
||||
let result!: { applied: number; conflicts: number };
|
||||
live.transact(() => {
|
||||
result = mergeXmlFragments3WayWithStats(liveFrag, targetFrag, baseFrag);
|
||||
});
|
||||
expect(result.conflicts).toBe(1);
|
||||
expect(texts(liveFrag)).toEqual(['a', 'GIT', 'c']);
|
||||
});
|
||||
|
||||
it('reports 0 conflicts for a clean different-block 3-way merge', () => {
|
||||
const base = new Y.Doc();
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const baseFrag = buildFragment(base, ['a', 'b', 'c']);
|
||||
const liveFrag = buildFragment(live, ['HUMAN', 'b', 'c']);
|
||||
const targetFrag = buildFragment(target, ['a', 'b', 'GIT']);
|
||||
|
||||
let result!: { applied: number; conflicts: number };
|
||||
live.transact(() => {
|
||||
result = mergeXmlFragments3WayWithStats(liveFrag, targetFrag, baseFrag);
|
||||
});
|
||||
expect(result.conflicts).toBe(0);
|
||||
expect(texts(liveFrag)).toEqual(['HUMAN', 'b', 'GIT']);
|
||||
});
|
||||
|
||||
it('git change with no concurrent human edit (live == base) applies cleanly', () => {
|
||||
const base = new Y.Doc();
|
||||
const live = new Y.Doc();
|
||||
const target = new Y.Doc();
|
||||
const baseFrag = buildFragment(base, ['a', 'b']);
|
||||
const liveFrag = buildFragment(live, ['a', 'b']);
|
||||
const targetFrag = buildFragment(target, ['a', 'B2']);
|
||||
|
||||
live.transact(() =>
|
||||
mergeXmlFragments3Way(liveFrag, targetFrag, baseFrag),
|
||||
);
|
||||
expect(texts(liveFrag)).toEqual(['a', 'B2']);
|
||||
});
|
||||
});
|
||||
|
||||
// Regression: start-of-document content duplicating on every two-way sync.
|
||||
//
|
||||
// The LIVE Docmost doc stamps a per-block UniqueID on every heading/paragraph;
|
||||
// a body arriving FROM git is parsed from clean markdown and carries NO block
|
||||
// ids. If the merge comparison key includes that `id`, an unchanged live block
|
||||
// never matches the SAME block coming from git, so the three-way merge cannot
|
||||
// anchor on it — and an incoming block with no anchor (content inserted at the
|
||||
// TOP of the page) is RE-ADDED on every cycle, an unbounded duplication loop.
|
||||
// These tests model that exact id-asymmetry and assert the reconciliation is
|
||||
// IDEMPOTENT (no block growth). They are RED before excluding `id` from the
|
||||
// key in `serializeXmlNode`.
|
||||
describe('idempotent reconciliation with live block ids (start-of-doc dup)', () => {
|
||||
// Build a fragment from block specs. `id` is set only when provided, mirroring
|
||||
// the live doc (ids present) vs a git-parsed body (ids absent).
|
||||
type Spec = { tag: 'heading' | 'paragraph'; text: string; id?: string };
|
||||
function buildDoc(doc: Y.Doc, specs: Spec[]): Y.XmlFragment {
|
||||
const frag = doc.getXmlFragment('default');
|
||||
const blocks = specs.map((s) => {
|
||||
const el = new Y.XmlElement(s.tag);
|
||||
if (s.id) el.setAttribute('id', s.id);
|
||||
if (s.tag === 'heading') el.setAttribute('level', '2');
|
||||
const t = new Y.XmlText();
|
||||
if (s.text) t.insert(0, s.text);
|
||||
el.insert(0, [t]);
|
||||
return el;
|
||||
});
|
||||
if (blocks.length) frag.insert(0, blocks);
|
||||
return frag;
|
||||
}
|
||||
const textsOf = (frag: Y.XmlFragment): string[] =>
|
||||
frag.toArray().map((el) =>
|
||||
(el as Y.XmlElement)
|
||||
.toArray()
|
||||
.map((c) => (c as Y.XmlText).toString())
|
||||
.join(''),
|
||||
);
|
||||
|
||||
it('re-merging the SAME git body does NOT re-add the top block (idempotent)', () => {
|
||||
// last-synced base (from git markdown): NO block ids.
|
||||
const base = new Y.Doc();
|
||||
const baseFrag = buildDoc(base, [
|
||||
{ tag: 'heading', text: 'Title' },
|
||||
{ tag: 'paragraph', text: 'Some paragraph.' },
|
||||
{ tag: 'paragraph', text: 'End block.' },
|
||||
]);
|
||||
// live Docmost doc: SAME content, but every block carries a UniqueID.
|
||||
const live = new Y.Doc();
|
||||
const liveFrag = buildDoc(live, [
|
||||
{ tag: 'heading', text: 'Title', id: 'ida' },
|
||||
{ tag: 'paragraph', text: 'Some paragraph.', id: 'idb' },
|
||||
{ tag: 'paragraph', text: 'End block.', id: 'idc' },
|
||||
]);
|
||||
// incoming git body: the user inserted a heading at the very TOP.
|
||||
const buildTarget = (): Y.XmlFragment =>
|
||||
buildDoc(new Y.Doc(), [
|
||||
{ tag: 'heading', text: 'TOPDUP' },
|
||||
{ tag: 'heading', text: 'Title' },
|
||||
{ tag: 'paragraph', text: 'Some paragraph.' },
|
||||
{ tag: 'paragraph', text: 'End block.' },
|
||||
]);
|
||||
|
||||
// First sync: the top block is added once.
|
||||
live.transact(() =>
|
||||
mergeXmlFragments3Way(liveFrag, buildTarget(), baseFrag),
|
||||
);
|
||||
expect(textsOf(liveFrag)).toEqual([
|
||||
'TOPDUP',
|
||||
'Title',
|
||||
'Some paragraph.',
|
||||
'End block.',
|
||||
]);
|
||||
|
||||
// Subsequent sync of the SAME git body against the SAME base must be a
|
||||
// NO-OP — not a second copy of the top block. Before the fix this re-adds
|
||||
// 'TOPDUP', growing the doc on every cycle.
|
||||
live.transact(() =>
|
||||
mergeXmlFragments3Way(liveFrag, buildTarget(), baseFrag),
|
||||
);
|
||||
expect(textsOf(liveFrag)).toEqual([
|
||||
'TOPDUP',
|
||||
'Title',
|
||||
'Some paragraph.',
|
||||
'End block.',
|
||||
]);
|
||||
expect(textsOf(liveFrag).filter((t) => t === 'TOPDUP')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('an unchanged git body (live ids, none in git) is a complete no-op', () => {
|
||||
// base == git body (no pending git change); live is the same content with
|
||||
// ids. With `id` in the key the whole body looks rewritten; the merge must
|
||||
// still leave live byte-identical (block instances untouched).
|
||||
const base = new Y.Doc();
|
||||
const baseFrag = buildDoc(base, [
|
||||
{ tag: 'heading', text: 'Title' },
|
||||
{ tag: 'paragraph', text: 'Body.' },
|
||||
]);
|
||||
const live = new Y.Doc();
|
||||
const liveFrag = buildDoc(live, [
|
||||
{ tag: 'heading', text: 'Title', id: 'ida' },
|
||||
{ tag: 'paragraph', text: 'Body.', id: 'idb' },
|
||||
]);
|
||||
const before = liveFrag.toArray();
|
||||
let applied = -1;
|
||||
live.transact(() => {
|
||||
applied = mergeXmlFragments3Way(
|
||||
liveFrag,
|
||||
buildDoc(new Y.Doc(), [
|
||||
{ tag: 'heading', text: 'Title' },
|
||||
{ tag: 'paragraph', text: 'Body.' },
|
||||
]),
|
||||
baseFrag,
|
||||
);
|
||||
});
|
||||
expect(applied).toBe(0);
|
||||
// Same live block instances (ids preserved) — nothing recreated.
|
||||
expect(liveFrag.toArray()).toEqual(before);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cloneXmlNode', () => {
|
||||
it('preserves text marks (XmlText delta) across docs', () => {
|
||||
const src = new Y.Doc();
|
||||
const srcFrag = src.getXmlFragment('default');
|
||||
const el = new Y.XmlElement('paragraph');
|
||||
const t = new Y.XmlText();
|
||||
t.insert(0, 'plain ');
|
||||
t.insert(6, 'bold', { bold: true });
|
||||
el.insert(0, [t]);
|
||||
srcFrag.insert(0, [el]);
|
||||
|
||||
const dst = new Y.Doc();
|
||||
const dstFrag = dst.getXmlFragment('default');
|
||||
dstFrag.insert(0, [cloneXmlNode(srcFrag.get(0) as Y.XmlElement)]);
|
||||
|
||||
const clonedText = (dstFrag.get(0) as Y.XmlElement).get(0) as Y.XmlText;
|
||||
expect(clonedText.toDelta()).toEqual([
|
||||
{ insert: 'plain ' },
|
||||
{ insert: 'bold', attributes: { bold: true } },
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,371 @@
|
||||
import * as Y from 'yjs';
|
||||
import { getSchema } from '@tiptap/core';
|
||||
import type { Schema } from '@tiptap/pm/model';
|
||||
|
||||
import { tiptapExtensions } from '../collaboration.util';
|
||||
import { diff3PlanWithConflicts } from './three-way-merge';
|
||||
import { buildLcsTable } from './lcs';
|
||||
|
||||
/**
|
||||
* Block-level merge of an incoming (git) page body into a LIVE Yjs document,
|
||||
* replacing the previous full-body "delete everything + re-insert" write that
|
||||
* clobbered concurrent human edits on every sync (review #5 — "do the write as a
|
||||
* merge").
|
||||
*
|
||||
* Strategy: diff the two documents at TOP-LEVEL BLOCK granularity (an LCS over a
|
||||
* canonical structural serialization of each block) and apply only the minimal
|
||||
* insert/delete operations. Blocks that are byte-identical on both sides are
|
||||
* left UNTOUCHED in the live doc — so a human editing one paragraph is unaffected
|
||||
* when git changes a different paragraph, and an unchanged re-sync is a complete
|
||||
* no-op (zero Yjs operations). Yjs then CRDT-merges the minimal ops with any
|
||||
* concurrent edits.
|
||||
*
|
||||
* Merge mode: a THREE-WAY merge (live vs incoming vs base) runs whenever the
|
||||
* engine plumbs the last-synced base (`baseMarkdown` from refs/docmost/last-pushed)
|
||||
* — which it now does end-to-end — so a block both sides changed is a genuine
|
||||
* conflict resolved deterministically (git wins that block; the prior state is
|
||||
* preserved in page history). Only when NO base is available (a brand-new file)
|
||||
* does it fall back to a 2-way merge (live vs incoming). Common cases — unchanged
|
||||
* resync and edits to DIFFERENT blocks — are lossless in both modes.
|
||||
*/
|
||||
|
||||
type XmlNode = Y.XmlElement | Y.XmlText | Y.XmlHook;
|
||||
|
||||
/**
|
||||
* Node attributes that are VOLATILE identity (not content) and so must be
|
||||
* excluded from the block comparison key.
|
||||
*
|
||||
* `id` is the per-block UniqueID the editor stamps on every heading/paragraph
|
||||
* (and transclusionSource). It exists ONLY in the live Yjs document — a body
|
||||
* arriving from git is parsed from clean markdown, which carries no block ids
|
||||
* (`markdownToProseMirror` materializes `id: null`, which the Yjs transform then
|
||||
* drops). If `id` were part of the key, an UNCHANGED live block (id "abc123")
|
||||
* would never match the SAME block coming from git (no id), so the three-way
|
||||
* merge's LCS could not anchor on it. The merge would then treat every live
|
||||
* block as deleted-and-reinserted and, when an incoming block has no matching
|
||||
* anchor (e.g. content inserted at the very TOP of the page), RE-ADD a copy of
|
||||
* it on every sync cycle — a non-convergent, unbounded duplication loop
|
||||
* (start-of-document content duplicating each push/pull cycle).
|
||||
*
|
||||
* Excluding `id` makes blocks compare by CONTENT, so an unchanged block matches
|
||||
* across the git round-trip and the reconciliation is idempotent. Block identity
|
||||
* is still preserved in the merged output: `diff3Plan` keeps the LIVE block
|
||||
* INSTANCE (with its id) for an anchor — picks are by index, not by key — so the
|
||||
* stable Yjs block (and any in-flight human edit on it) stays put. This mirrors
|
||||
* `canonicalize.ts`, which already strips the regenerated block `id` from the
|
||||
* round-trip idempotency comparison for exactly the same reason.
|
||||
*
|
||||
* Known limitation (accepted trade-off of content-based matching): two GENUINELY
|
||||
* DISTINCT blocks whose content is byte-identical now collapse to the same content
|
||||
* key, so when git deletes one of the duplicates the LCS may drop the OTHER live
|
||||
* instance instead. The visible result is identical (one copy removed, one kept),
|
||||
* but a concurrent in-flight human edit on the dropped instance could be lost.
|
||||
*/
|
||||
const VOLATILE_KEY_ATTRS = new Set(['id']);
|
||||
|
||||
/**
|
||||
* The editor (ProseMirror) schema, built ONCE from the same `tiptapExtensions`
|
||||
* the collaboration server uses to materialize Yjs docs. Memoized: building the
|
||||
* schema is non-trivial and the block key is computed per block per cycle.
|
||||
*
|
||||
* Why the schema (not a hardcoded denylist): the LIVE Yjs document is produced by
|
||||
* `TiptapTransformer.toYdoc(pm, 'default', tiptapExtensions)`, which STAMPS every
|
||||
* schema-default attribute onto every node and mark — `indent: 0` on every
|
||||
* paragraph/heading, `image.align: "center"`, the link mark's `internal: false`,
|
||||
* `highlight.colorName: null`, and so on for youtube/pdf/any future node. A body
|
||||
* re-imported from git comes through the engine's `markdownToProseMirror`, whose
|
||||
* schema declares those attrs with DIFFERENT (usually null) defaults; the
|
||||
* resulting null/absent element attrs are then DROPPED by `y-prosemirror`'s
|
||||
* toYdoc. So the SAME block carries materialized defaults on the live side and
|
||||
* nothing on the git side, its key diverges, the three-way merge anchors on
|
||||
* NOTHING, and the whole body is RE-APPENDED every reconcile cycle — an unbounded
|
||||
* duplication loop with no client connected.
|
||||
*
|
||||
* Deriving the defaults from the actual schema normalizes ALL such attributes
|
||||
* generally (it is not another per-attribute denylist): any attribute whose value
|
||||
* equals the schema default — or is null/undefined — is dropped from the key, on
|
||||
* BOTH element attributes and the mark attributes inside each XmlText delta, so a
|
||||
* live block compares equal to its git-round-tripped twin and an unchanged resync
|
||||
* applies zero ops. Genuinely non-default values (a real `indent: 2`, an
|
||||
* `align: "left"`, a real `link.href`, a real highlight color) are content and
|
||||
* stay in the key, so real edits still diff and land.
|
||||
*/
|
||||
let memoSchema: Schema | null = null;
|
||||
let memoSchemaTried = false;
|
||||
function getMergeSchema(): Schema | null {
|
||||
if (!memoSchemaTried) {
|
||||
memoSchemaTried = true;
|
||||
try {
|
||||
memoSchema = getSchema(tiptapExtensions as any);
|
||||
} catch {
|
||||
// Defensive: if the schema can't be built (e.g. a degenerate extension
|
||||
// set in a unit test that stubs `tiptapExtensions`), fall back to dropping
|
||||
// only null/undefined attrs. The real server always builds it fine.
|
||||
memoSchema = null;
|
||||
}
|
||||
}
|
||||
return memoSchema;
|
||||
}
|
||||
|
||||
/** True if `value` is the schema default for `attrName` of `attrSpecs`, or is
|
||||
* null/undefined (which a git round-trip drops). Such attributes are excluded
|
||||
* from the comparison key. `attrSpecs` is a ProseMirror node/mark spec attr map
|
||||
* (`{ [name]: { default } }`); a missing map (unknown node/mark) only drops
|
||||
* null/undefined. (A non-null value matching an attr declared without a default
|
||||
* cannot occur — `spec.default === value` is then `undefined === value`, false.) */
|
||||
function isDefaultAttr(
|
||||
attrSpecs: Record<string, any> | undefined | null,
|
||||
attrName: string,
|
||||
value: unknown,
|
||||
): boolean {
|
||||
if (value === null || value === undefined) return true;
|
||||
const spec = attrSpecs?.[attrName];
|
||||
return !!spec && spec.default === value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize one XmlText delta op's mark attributes: drop every mark-attr whose
|
||||
* value equals the mark's schema default (or is null/undefined), so the link
|
||||
* mark's materialized `internal: false`/`target: "_blank"` and a highlight's
|
||||
* `colorName: null` no longer diverge from a git round-trip that carries neither.
|
||||
* The text (op.insert) and genuinely-set mark attrs (a real `href`, a real
|
||||
* highlight color) are preserved verbatim. `attributes` maps markName -> mark
|
||||
* attrs object (or `true`/boolean for attr-less marks); each is handled safely.
|
||||
*/
|
||||
function normalizeDelta(delta: any[]): any[] {
|
||||
const schema = getMergeSchema();
|
||||
return delta.map((op) => {
|
||||
if (!op || op.attributes == null || typeof op.attributes !== 'object') {
|
||||
return op;
|
||||
}
|
||||
const marks: Record<string, unknown> = {};
|
||||
for (const markName of Object.keys(op.attributes).sort()) {
|
||||
const markVal = op.attributes[markName];
|
||||
if (markVal === null || markVal === undefined) continue;
|
||||
if (typeof markVal !== 'object') {
|
||||
// attr-less mark stored as a primitive (e.g. `true`) — keep as-is.
|
||||
marks[markName] = markVal;
|
||||
continue;
|
||||
}
|
||||
const markSpec = schema?.marks[markName]?.spec.attrs as
|
||||
| Record<string, any>
|
||||
| undefined;
|
||||
const cleaned: Record<string, unknown> = {};
|
||||
for (const ak of Object.keys(markVal as object).sort()) {
|
||||
const av = (markVal as Record<string, unknown>)[ak];
|
||||
if (isDefaultAttr(markSpec, ak, av)) continue;
|
||||
cleaned[ak] = av;
|
||||
}
|
||||
marks[markName] = cleaned;
|
||||
}
|
||||
return { ...op, attributes: marks };
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Canonical, comparable serialization of a Yjs XML node (structure + text +
|
||||
* marks + attributes), with attribute keys sorted so equal blocks always produce
|
||||
* an identical string regardless of attribute insertion order. The volatile
|
||||
* block `id` (see `VOLATILE_KEY_ATTRS`) and every schema-default attribute (see
|
||||
* `getMergeSchema`) are excluded at every level — on element attributes AND on
|
||||
* the mark attributes inside each XmlText delta — so a block compares equal by
|
||||
* CONTENT across the git round-trip (which materializes neither), keeping the
|
||||
* merge anchor-able and idempotent.
|
||||
*/
|
||||
export function serializeXmlNode(node: unknown): unknown {
|
||||
if (node instanceof Y.XmlText) {
|
||||
return { t: normalizeDelta(node.toDelta()) };
|
||||
}
|
||||
if (node instanceof Y.XmlElement) {
|
||||
const attrs = node.getAttributes() as Record<string, unknown>;
|
||||
const attrSpecs = getMergeSchema()?.nodes[node.nodeName]?.spec.attrs as
|
||||
| Record<string, any>
|
||||
| undefined;
|
||||
const sorted: Record<string, unknown> = {};
|
||||
for (const k of Object.keys(attrs).sort()) {
|
||||
if (VOLATILE_KEY_ATTRS.has(k)) continue;
|
||||
if (isDefaultAttr(attrSpecs, k, attrs[k])) continue;
|
||||
sorted[k] = attrs[k];
|
||||
}
|
||||
return {
|
||||
n: node.nodeName,
|
||||
a: sorted,
|
||||
c: node.toArray().map(serializeXmlNode),
|
||||
};
|
||||
}
|
||||
// XmlHook / unknown: fall back to a stable string so it compares by identity
|
||||
// of its serialized form (these do not occur in the Docmost block schema).
|
||||
return { u: String(node) };
|
||||
}
|
||||
|
||||
const key = (node: unknown): string => JSON.stringify(serializeXmlNode(node));
|
||||
|
||||
/**
|
||||
* Deep-clone a detached/owned Yjs XML node into a fresh node that can be inserted
|
||||
* into ANOTHER document (Yjs types are bound to their doc, so cross-doc moves are
|
||||
* impossible — we rebuild). Preserves nodeName, attributes, text+marks (via the
|
||||
* XmlText delta) and the full child subtree.
|
||||
*/
|
||||
export function cloneXmlNode(node: XmlNode): Y.XmlElement | Y.XmlText {
|
||||
if (node instanceof Y.XmlText) {
|
||||
const t = new Y.XmlText();
|
||||
const delta = node.toDelta();
|
||||
if (delta.length) t.applyDelta(delta);
|
||||
return t;
|
||||
}
|
||||
if (node instanceof Y.XmlElement) {
|
||||
const el = new Y.XmlElement(node.nodeName);
|
||||
const attrs = node.getAttributes() as Record<string, unknown>;
|
||||
for (const k of Object.keys(attrs)) el.setAttribute(k, attrs[k] as string);
|
||||
const kids = node.toArray().map((c) => cloneXmlNode(c as XmlNode));
|
||||
if (kids.length) el.insert(0, kids);
|
||||
return el;
|
||||
}
|
||||
// Best-effort for any other node type (XmlHook — does not occur in the
|
||||
// Docmost block schema): an empty paragraph so the merge never crashes.
|
||||
return new Y.XmlElement('paragraph');
|
||||
}
|
||||
|
||||
type Op = { op: 'keep' } | { op: 'del' } | { op: 'ins'; bi: number };
|
||||
|
||||
/**
|
||||
* LCS-based edit script turning sequence `a` (live block keys) into `b` (incoming
|
||||
* block keys): a run of keep/del/ins ops. O(n*m) table — fine for page block
|
||||
* counts.
|
||||
*/
|
||||
export function diffBlocks(a: string[], b: string[]): Op[] {
|
||||
const n = a.length;
|
||||
const m = b.length;
|
||||
const dp = buildLcsTable(a, b);
|
||||
const ops: Op[] = [];
|
||||
let i = 0;
|
||||
let j = 0;
|
||||
while (i < n && j < m) {
|
||||
if (a[i] === b[j]) {
|
||||
ops.push({ op: 'keep' });
|
||||
i++;
|
||||
j++;
|
||||
} else if (dp[i + 1][j] >= dp[i][j + 1]) {
|
||||
ops.push({ op: 'del' });
|
||||
i++;
|
||||
} else {
|
||||
ops.push({ op: 'ins', bi: j });
|
||||
j++;
|
||||
}
|
||||
}
|
||||
while (i < n) {
|
||||
ops.push({ op: 'del' });
|
||||
i++;
|
||||
}
|
||||
while (j < m) {
|
||||
ops.push({ op: 'ins', bi: j });
|
||||
j++;
|
||||
}
|
||||
return ops;
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge `target` block children into `live`, mutating `live` in place with the
|
||||
* minimal set of inserts/deletes. MUST be called inside a Yjs transaction.
|
||||
* Returns the number of block operations applied (0 == content already identical).
|
||||
*/
|
||||
export function mergeXmlFragments(
|
||||
live: Y.XmlFragment,
|
||||
target: Y.XmlFragment,
|
||||
): number {
|
||||
const liveKids = live.toArray();
|
||||
const targetKids = target.toArray();
|
||||
const liveKeys = liveKids.map(key);
|
||||
const targetKeys = targetKids.map(key);
|
||||
|
||||
const ops = diffBlocks(liveKeys, targetKeys);
|
||||
|
||||
let cursor = 0; // index into the LIVE fragment as we mutate it
|
||||
let applied = 0;
|
||||
for (const op of ops) {
|
||||
if (op.op === 'keep') {
|
||||
cursor++;
|
||||
} else if (op.op === 'del') {
|
||||
live.delete(cursor, 1); // remove the live block at the cursor; do not advance
|
||||
applied++;
|
||||
} else {
|
||||
live.insert(cursor, [cloneXmlNode(targetKids[op.bi] as XmlNode)]);
|
||||
cursor++;
|
||||
applied++;
|
||||
}
|
||||
}
|
||||
return applied;
|
||||
}
|
||||
|
||||
/** Outcome of a 3-way block merge: ops applied + same-block conflict count. */
|
||||
export interface Merge3WayResult {
|
||||
/** Number of block insert/delete operations spliced into `live`. */
|
||||
applied: number;
|
||||
/**
|
||||
* Regions where the human AND git rewrote the SAME base block. The rule is
|
||||
* deterministic (GIT WINS the region), so the human's version of those blocks
|
||||
* is dropped from the live doc. `conflicts > 0` is the OBSERVABLE signal the
|
||||
* caller uses to LOG the loss and pin the human baseline to page history (so it
|
||||
* is recoverable), instead of the edit vanishing silently.
|
||||
*/
|
||||
conflicts: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* THREE-WAY block merge: reconcile `live` toward `target` using `base` (the
|
||||
* last-synced common ancestor) so a block only the human changed is KEPT and a
|
||||
* block only git changed is taken — instead of git's version always winning
|
||||
* (review #5). Conflicts (both changed the same block) resolve to git.
|
||||
*
|
||||
* Implementation: diff3Plan computes the merged block ORDER (picks from live or
|
||||
* target); we materialize that as a virtual target fragment and reuse the 2-way
|
||||
* `mergeXmlFragments` to splice it into `live` minimally (so untouched live block
|
||||
* instances — and their in-flight edits — stay put). MUST be called inside a Yjs
|
||||
* transaction. Returns the number of block operations applied. (Use
|
||||
* `mergeXmlFragments3WayWithStats` when the SAME-BLOCK conflict count is needed.)
|
||||
*/
|
||||
export function mergeXmlFragments3Way(
|
||||
live: Y.XmlFragment,
|
||||
target: Y.XmlFragment,
|
||||
base: Y.XmlFragment,
|
||||
): number {
|
||||
return mergeXmlFragments3WayWithStats(live, target, base).applied;
|
||||
}
|
||||
|
||||
/**
|
||||
* As `mergeXmlFragments3Way`, but also returns the SAME-BLOCK conflict count so
|
||||
* the caller can make a "git won a concurrent same-block edit" event OBSERVABLE
|
||||
* (the documented conflict contract: git wins deterministically, but the losing
|
||||
* human content is never destroyed silently — it is logged and recoverable via
|
||||
* page history).
|
||||
*/
|
||||
export function mergeXmlFragments3WayWithStats(
|
||||
live: Y.XmlFragment,
|
||||
target: Y.XmlFragment,
|
||||
base: Y.XmlFragment,
|
||||
): Merge3WayResult {
|
||||
const liveKids = live.toArray();
|
||||
const targetKids = target.toArray();
|
||||
const liveKeys = liveKids.map(key);
|
||||
const targetKeys = targetKids.map(key);
|
||||
const baseKeys = base.toArray().map(key);
|
||||
|
||||
const { picks: plan, conflicts } = diff3PlanWithConflicts(
|
||||
baseKeys,
|
||||
liveKeys,
|
||||
targetKeys,
|
||||
);
|
||||
|
||||
// Build the merged block sequence in a throwaway doc, cloning from whichever
|
||||
// side each pick came from, then 2-way merge it back into the live fragment.
|
||||
const merged = new Y.Doc();
|
||||
const mergedFrag = merged.getXmlFragment('default');
|
||||
const nodes = plan.map((p) =>
|
||||
cloneXmlNode(
|
||||
(p.src === 'live' ? liveKids[p.index] : targetKids[p.index]) as XmlNode,
|
||||
),
|
||||
);
|
||||
if (nodes.length) mergedFrag.insert(0, nodes);
|
||||
|
||||
return { applied: mergeXmlFragments(live, mergedFrag), conflicts };
|
||||
}
|
||||
@@ -0,0 +1,144 @@
|
||||
import { getSchema } from '@tiptap/core';
|
||||
import { Schema } from '@tiptap/pm/model';
|
||||
import { tiptapExtensions } from './collaboration.util';
|
||||
// The canonical converter mirror's extension set. The schema mirror now lives in
|
||||
// the single `@docmost/prosemirror-markdown` package (#293); the server jest
|
||||
// config maps it to the package SOURCE (moduleNameMapper
|
||||
// `^@docmost/prosemirror-markdown$`), so this reads the real mirror, not a build.
|
||||
import { docmostExtensions as gitSyncExtensions } from '@docmost/prosemirror-markdown';
|
||||
|
||||
/**
|
||||
* ATTRIBUTE-LEVEL SCHEMA CONTRACT (review #293, variant A).
|
||||
*
|
||||
* The document schema exists as three hand-synced copies (editor-ext =
|
||||
* source-of-truth, plus the git-sync and mcp converter mirrors). The existing
|
||||
* `schema-editor-ext-contract.test.ts` compares only node/mark TYPE NAMES, so a
|
||||
* NEW ATTRIBUTE added to an existing node upstream slips through and its value is
|
||||
* silently dropped on every git-sync round trip. That is a repeatedly-hit
|
||||
* data-loss class (image caption #221, paragraph alignment #10, details `open`).
|
||||
*
|
||||
* This test closes the attribute gap MECHANICALLY: it builds the real canonical
|
||||
* schema from the server's `tiptapExtensions` (the same set the collab write path
|
||||
* uses) and the git-sync mirror schema, then asserts that for every node/mark the
|
||||
* two schemas share, their ATTRIBUTE-KEY sets are equal — minus a committed
|
||||
* allowlist of intentional, understood divergences. A forgotten attribute now
|
||||
* fails CI loudly instead of losing data in production.
|
||||
*
|
||||
* WHY THIS ISN'T THE "fragile attribute compare" the sibling name-level contract
|
||||
* (`packages/git-sync/test/schema-editor-ext-contract.test.ts`) deferred: that
|
||||
* concern was about comparing raw extension CONFIGS, where editor-ext spreads
|
||||
* global attributes (textAlign, id, …) across separate extensions and StarterKit
|
||||
* contributes types the mirror gets elsewhere. We instead compare the RESOLVED
|
||||
* ProseMirror `Schema` objects — `getSchema()` has already merged every
|
||||
* addGlobalAttributes spread into concrete per-node attrs on both sides — so the
|
||||
* compare is apples-to-apples (57 shared nodes/marks, only a handful of
|
||||
* documented divergences) rather than config-shape noise.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Intentional, understood attribute divergences between the canonical schema and
|
||||
* the git-sync mirror. Each entry MUST carry a reason. The test asserts the
|
||||
* allowlist is not stale (every listed attr is actually still divergent), so this
|
||||
* cannot rot into a silent escape hatch.
|
||||
*
|
||||
* Shape: { [nodeOrMarkName]: { canonicalOnly?: string[]; mirrorOnly?: string[] } }
|
||||
*/
|
||||
const ALLOWED_DIVERGENCES: Record<
|
||||
string,
|
||||
{ canonicalOnly?: string[]; mirrorOnly?: string[] }
|
||||
> = {
|
||||
// mirrorOnly: the converter mirror carries `align` on table cells/headers so a
|
||||
// GFM column-alignment marker (:--, :-:, --:) can be reconstructed on export;
|
||||
// editor-ext expresses cell alignment differently. Intentional, round-trip-used.
|
||||
tableCell: { mirrorOnly: ['align'] },
|
||||
tableHeader: { mirrorOnly: ['align'] },
|
||||
// youtube: the mirror adds `align` (media alignment it renders as data-align)
|
||||
// and does NOT carry editor-ext's `start` (video start-time). `start` is a
|
||||
// PRE-EXISTING gap (a youtube embed's start offset is not preserved across a
|
||||
// markdown round trip) — documented here so the contract is green for the known
|
||||
// state and RED for any NEW drift. Follow-up: carry `start` through the mirror.
|
||||
youtube: { mirrorOnly: ['align'], canonicalOnly: ['start'] },
|
||||
// image.title: the mirror carries a `title` attr (used to round-trip the
|
||||
// markdown image title ``) that editor-ext does not declare
|
||||
// on its image node. Mirror-only and round-trip-used, not data loss. Intentional.
|
||||
image: { mirrorOnly: ['title'] },
|
||||
// highlight.colorName (a named-color alias alongside the color value) is a
|
||||
// PRE-EXISTING mirror gap; the color value itself round-trips. Documented.
|
||||
highlight: { canonicalOnly: ['colorName'] },
|
||||
};
|
||||
|
||||
function attrKeys(schema: Schema): Map<string, Set<string>> {
|
||||
const out = new Map<string, Set<string>>();
|
||||
for (const [name, type] of Object.entries(schema.nodes)) {
|
||||
out.set(name, new Set(Object.keys((type.spec as any).attrs ?? {})));
|
||||
}
|
||||
for (const [name, type] of Object.entries(schema.marks)) {
|
||||
out.set(name, new Set(Object.keys((type.spec as any).attrs ?? {})));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function diff(a: Set<string>, b: Set<string>): string[] {
|
||||
return [...a].filter((x) => !b.has(x)).sort();
|
||||
}
|
||||
|
||||
describe('schema attribute contract: git-sync mirror vs canonical editor-ext', () => {
|
||||
const canonical = attrKeys(getSchema(tiptapExtensions as never));
|
||||
const mirror = attrKeys(getSchema(gitSyncExtensions as never));
|
||||
|
||||
it('builds meaningful schemas (guard against a vacuous pass)', () => {
|
||||
expect(canonical.size).toBeGreaterThan(10);
|
||||
expect(mirror.size).toBeGreaterThan(10);
|
||||
});
|
||||
|
||||
it('every shared node/mark has matching attribute keys (modulo the allowlist)', () => {
|
||||
const drift: string[] = [];
|
||||
for (const [name, canonAttrs] of canonical) {
|
||||
const mirrorAttrs = mirror.get(name);
|
||||
if (!mirrorAttrs) continue; // name-level gaps are the other test's job
|
||||
const allow = ALLOWED_DIVERGENCES[name] ?? {};
|
||||
const canonicalOnly = diff(canonAttrs, mirrorAttrs).filter(
|
||||
(k) => !(allow.canonicalOnly ?? []).includes(k),
|
||||
);
|
||||
const mirrorOnly = diff(mirrorAttrs, canonAttrs).filter(
|
||||
(k) => !(allow.mirrorOnly ?? []).includes(k),
|
||||
);
|
||||
if (canonicalOnly.length) {
|
||||
drift.push(
|
||||
`${name}: attrs in editor-ext but MISSING from git-sync mirror ` +
|
||||
`(silently dropped on round trip): ${canonicalOnly.join(', ')}`,
|
||||
);
|
||||
}
|
||||
if (mirrorOnly.length) {
|
||||
drift.push(
|
||||
`${name}: attrs in git-sync mirror but NOT in editor-ext ` +
|
||||
`(mirror invented an attribute): ${mirrorOnly.join(', ')}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
expect(drift).toEqual([]);
|
||||
});
|
||||
|
||||
it('the allowlist is not stale (every listed divergence is still real)', () => {
|
||||
const stale: string[] = [];
|
||||
for (const [name, allow] of Object.entries(ALLOWED_DIVERGENCES)) {
|
||||
const canonAttrs = canonical.get(name);
|
||||
const mirrorAttrs = mirror.get(name);
|
||||
if (!canonAttrs || !mirrorAttrs) {
|
||||
stale.push(`${name}: no longer a shared node/mark`);
|
||||
continue;
|
||||
}
|
||||
for (const k of allow.canonicalOnly ?? []) {
|
||||
if (!(canonAttrs.has(k) && !mirrorAttrs.has(k))) {
|
||||
stale.push(`${name}.canonicalOnly '${k}' is no longer divergent`);
|
||||
}
|
||||
}
|
||||
for (const k of allow.mirrorOnly ?? []) {
|
||||
if (!(mirrorAttrs.has(k) && !canonAttrs.has(k))) {
|
||||
stale.push(`${name}.mirrorOnly '${k}' is no longer divergent`);
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(stale).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -73,6 +73,32 @@ describe('agentSourceFields', () => {
|
||||
).toEqual({ lastUpdatedSource: 'agent', lastUpdatedAiChatId: null });
|
||||
});
|
||||
|
||||
it("stamps ONLY the source column 'git-sync' (no chat key) for a git-sync write", () => {
|
||||
// The git-sync data plane (issue #194 §8.1) has no internal ai_chats row, so
|
||||
// it stamps the *Source column 'git-sync' and OMITS the chat key entirely
|
||||
// (unlike the agent branch, which also writes aiChatId). Pinned directly here
|
||||
// because the page.service.spec only exercises it indirectly.
|
||||
expect(
|
||||
agentSourceFields(
|
||||
{ actor: 'git-sync', aiChatId: null },
|
||||
'lastUpdatedSource',
|
||||
'lastUpdatedAiChatId',
|
||||
),
|
||||
).toEqual({ lastUpdatedSource: 'git-sync' });
|
||||
});
|
||||
|
||||
it("ignores any aiChatId on a git-sync write (chat key never written)", () => {
|
||||
// Even if a non-null aiChatId is present, the git-sync branch must not emit
|
||||
// the chat key.
|
||||
expect(
|
||||
agentSourceFields(
|
||||
{ actor: 'git-sync', aiChatId: 'should-be-ignored' },
|
||||
'createdSource',
|
||||
'aiChatId',
|
||||
),
|
||||
).toEqual({ createdSource: 'git-sync' });
|
||||
});
|
||||
|
||||
it('returns {} for a user write so the column keeps its default', () => {
|
||||
expect(
|
||||
agentSourceFields(
|
||||
|
||||
@@ -9,6 +9,8 @@ import { ProvenanceSource } from '../../core/auth/dto/jwt-payload';
|
||||
* cannot fake an 'agent' marker.
|
||||
*/
|
||||
export interface AuthProvenanceData {
|
||||
// ProvenanceSource includes 'git-sync' — set by the in-process git-sync data
|
||||
// plane (issue #194 §8.1) when it drives PageService writes; never from a request token.
|
||||
actor: ProvenanceSource;
|
||||
aiChatId: string | null;
|
||||
}
|
||||
@@ -60,6 +62,14 @@ export function agentSourceFields<S extends string, C extends string>(
|
||||
sourceKey: S,
|
||||
chatKey: C,
|
||||
): Partial<Record<S, ProvenanceSource> & Record<C, string | null>> {
|
||||
// git-sync data-plane write (issue #194 §8.1): stamp the source 'git-sync' with NO
|
||||
// aiChatId (it has no internal ai_chats row). Mirrors the agent branch; each
|
||||
// write has a single actor, so precedence is irrelevant here.
|
||||
if (provenance?.actor === 'git-sync') {
|
||||
return { [sourceKey]: 'git-sync' } as Partial<
|
||||
Record<S, ProvenanceSource> & Record<C, string | null>
|
||||
>;
|
||||
}
|
||||
if (provenance?.actor !== 'agent') return {};
|
||||
return {
|
||||
[sourceKey]: 'agent',
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* Dynamic ESM import bridge for a CommonJS build.
|
||||
*
|
||||
* The server compiles with `module: commonjs`, and TypeScript downlevels a
|
||||
* literal `import()` expression to `require()` — which cannot load an ESM-only
|
||||
* package (`@docmost/mcp`, `@docmost/git-sync`). Indirecting through `new
|
||||
* Function` hides the `import()` from the TS downleveler so the REAL dynamic
|
||||
* `import()` survives to runtime and can load ESM from CommonJS.
|
||||
*
|
||||
* This is the single shared copy of that bridge. The per-package typed loaders
|
||||
* (git-sync.loader.ts, docmost-client.loader.ts, mcp.service.ts) import this and
|
||||
* keep their own typed `loadX()` wrappers (require.resolve + pathToFileURL +
|
||||
* memoization) on top.
|
||||
*/
|
||||
export const esmImport = new Function(
|
||||
'specifier',
|
||||
'return import(specifier)',
|
||||
) as (specifier: string) => Promise<unknown>;
|
||||
@@ -0,0 +1,71 @@
|
||||
import { resolveRequestWorkspace } from './resolve-request-workspace';
|
||||
|
||||
// Unit tests for the shared self-hosted/cloud workspace resolver deduplicated out
|
||||
// of DomainMiddleware + GitHttpService (architecture #11). They must behave
|
||||
// identically, so this pins the single source of truth.
|
||||
|
||||
type AnyMock = jest.Mock;
|
||||
|
||||
function build(opts: {
|
||||
selfHosted: boolean;
|
||||
first?: { id: string } | null;
|
||||
byHostname?: { id: string } | null;
|
||||
}) {
|
||||
const env = {
|
||||
isSelfHosted: jest.fn(() => opts.selfHosted),
|
||||
isCloud: jest.fn(() => !opts.selfHosted),
|
||||
};
|
||||
const repo = {
|
||||
findFirst: jest.fn(async () => opts.first ?? null) as AnyMock,
|
||||
findByHostname: jest.fn(async () => opts.byHostname ?? null) as AnyMock,
|
||||
};
|
||||
return { env, repo };
|
||||
}
|
||||
|
||||
describe('resolveRequestWorkspace', () => {
|
||||
it('self-hosted: returns the first/default workspace, ignoring the host', async () => {
|
||||
const { env, repo } = build({ selfHosted: true, first: { id: 'ws-1' } });
|
||||
const ws = await resolveRequestWorkspace(
|
||||
env as any,
|
||||
repo as any,
|
||||
'anything.example.com',
|
||||
);
|
||||
expect(ws).toEqual({ id: 'ws-1' });
|
||||
expect(repo.findFirst).toHaveBeenCalledTimes(1);
|
||||
expect(repo.findByHostname).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('self-hosted: returns null when no workspace is configured', async () => {
|
||||
const { env, repo } = build({ selfHosted: true, first: null });
|
||||
expect(await resolveRequestWorkspace(env as any, repo as any, 'h')).toBeNull();
|
||||
});
|
||||
|
||||
it('cloud: resolves by the host-header subdomain', async () => {
|
||||
const { env, repo } = build({
|
||||
selfHosted: false,
|
||||
byHostname: { id: 'ws-acme' },
|
||||
});
|
||||
const ws = await resolveRequestWorkspace(
|
||||
env as any,
|
||||
repo as any,
|
||||
'acme.example.com',
|
||||
);
|
||||
expect(ws).toEqual({ id: 'ws-acme' });
|
||||
expect(repo.findByHostname).toHaveBeenCalledWith('acme');
|
||||
expect(repo.findFirst).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('cloud: returns null for a blank/missing host (no throw)', async () => {
|
||||
const { env, repo } = build({ selfHosted: false, byHostname: { id: 'x' } });
|
||||
expect(await resolveRequestWorkspace(env as any, repo as any, undefined)).toBeNull();
|
||||
expect(await resolveRequestWorkspace(env as any, repo as any, '')).toBeNull();
|
||||
expect(repo.findByHostname).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('cloud: returns null when the subdomain matches no workspace', async () => {
|
||||
const { env, repo } = build({ selfHosted: false, byHostname: null });
|
||||
expect(
|
||||
await resolveRequestWorkspace(env as any, repo as any, 'ghost.example.com'),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
|
||||
import { Workspace } from '@docmost/db/types/entity.types';
|
||||
import { EnvironmentService } from '../../integrations/environment/environment.service';
|
||||
|
||||
/**
|
||||
* The ONE canonical way to resolve the workspace for an incoming request:
|
||||
* - self-hosted (single workspace) -> the first/default workspace;
|
||||
* - cloud (multi-tenant) -> resolved by the host-header subdomain.
|
||||
* Returns null when none resolves (no workspace configured, or a blank/unknown
|
||||
* subdomain on cloud). `isSelfHosted()` is `!isCloud()`, so exactly one branch is
|
||||
* always taken.
|
||||
*
|
||||
* Extracted so the self-hosted/cloud branch is not hand-duplicated. Shared by
|
||||
* `DomainMiddleware` (the normal /api request path) and `GitHttpService` (the raw
|
||||
* root-mounted /git smart-HTTP host, which Nest middleware does NOT run for) so
|
||||
* the two cannot drift.
|
||||
*
|
||||
* This helper does NOT catch DB errors — callers decide: DomainMiddleware lets a
|
||||
* throw bubble (as before); GitHttpService wraps it to log + treat as
|
||||
* unresolvable (-> 404). A blank/missing host on cloud resolves to null rather
|
||||
* than throwing.
|
||||
*/
|
||||
export async function resolveRequestWorkspace(
|
||||
environmentService: EnvironmentService,
|
||||
workspaceRepo: WorkspaceRepo,
|
||||
hostHeader: string | undefined,
|
||||
): Promise<Workspace | null> {
|
||||
if (environmentService.isSelfHosted()) {
|
||||
return (await workspaceRepo.findFirst()) ?? null;
|
||||
}
|
||||
// Cloud (isSelfHosted === !isCloud, so this is the only remaining branch).
|
||||
const subdomain = hostHeader ? hostHeader.split('.')[0] : '';
|
||||
if (!subdomain) return null;
|
||||
return (await workspaceRepo.findByHostname(subdomain)) ?? null;
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
import { Injectable, NestMiddleware, NotFoundException } from '@nestjs/common';
|
||||
import { Injectable, NestMiddleware } from '@nestjs/common';
|
||||
import { FastifyRequest, FastifyReply } from 'fastify';
|
||||
import { EnvironmentService } from '../../integrations/environment/environment.service';
|
||||
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
|
||||
import { resolveRequestWorkspace } from '../helpers/resolve-request-workspace';
|
||||
|
||||
@Injectable()
|
||||
export class DomainMiddleware implements NestMiddleware {
|
||||
@@ -14,30 +15,19 @@ export class DomainMiddleware implements NestMiddleware {
|
||||
res: FastifyReply['raw'],
|
||||
next: () => void,
|
||||
) {
|
||||
if (this.environmentService.isSelfHosted()) {
|
||||
const workspace = await this.workspaceRepo.findFirst();
|
||||
if (!workspace) {
|
||||
//throw new NotFoundException('Workspace not found');
|
||||
(req as any).workspaceId = null;
|
||||
return next();
|
||||
}
|
||||
|
||||
// TODO: unify
|
||||
(req as any).workspaceId = workspace.id;
|
||||
(req as any).workspace = workspace;
|
||||
} else if (this.environmentService.isCloud()) {
|
||||
const header = req.headers.host;
|
||||
const subdomain = header.split('.')[0];
|
||||
|
||||
const workspace = await this.workspaceRepo.findByHostname(subdomain);
|
||||
|
||||
if (!workspace) {
|
||||
(req as any).workspaceId = null;
|
||||
return next();
|
||||
}
|
||||
// Shared self-hosted/cloud resolution (the SAME branch the /git host uses),
|
||||
// so the logic cannot drift between the two.
|
||||
const workspace = await resolveRequestWorkspace(
|
||||
this.environmentService,
|
||||
this.workspaceRepo,
|
||||
req.headers.host,
|
||||
);
|
||||
|
||||
if (workspace) {
|
||||
(req as any).workspaceId = workspace.id;
|
||||
(req as any).workspace = workspace;
|
||||
} else {
|
||||
(req as any).workspaceId = null;
|
||||
}
|
||||
|
||||
next();
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { esmImport } from '../../../common/helpers/esm-import';
|
||||
|
||||
/**
|
||||
* Minimal structural type for the `DocmostClient` class we consume from the
|
||||
@@ -256,14 +257,8 @@ interface DocmostMcpModule {
|
||||
SHARED_TOOL_SPECS: Record<string, SharedToolSpec>;
|
||||
}
|
||||
|
||||
// TS with module:commonjs downlevels a literal `import()` to `require()`, which
|
||||
// cannot load the ESM-only `@docmost/mcp` package. Indirect through Function so
|
||||
// the real dynamic `import()` survives compilation and can load ESM from
|
||||
// CommonJS at runtime (same trick as integrations/mcp/mcp.service.ts).
|
||||
const esmImport = new Function(
|
||||
'specifier',
|
||||
'return import(specifier)',
|
||||
) as (specifier: string) => Promise<unknown>;
|
||||
// The CJS->ESM dynamic-import bridge lives in one shared helper
|
||||
// (common/helpers/esm-import.ts). The typed `loadDocmostMcp()` wrapper stays here.
|
||||
|
||||
// Memoize the in-flight/loaded module so the dynamic import runs at most once.
|
||||
let modulePromise: Promise<DocmostMcpModule> | null = null;
|
||||
|
||||
@@ -3,8 +3,12 @@
|
||||
* from the SIGNED token claim (never a request body), so 'agent' is unspoofable.
|
||||
* Single source of truth so a typo like 'agnet' can't slip through as a bare
|
||||
* string (#143 review). Distinct from `ActorType` (auth principal kind).
|
||||
*
|
||||
* 'git-sync' marks writes made by the git-sync data plane (issue #194 §8.1). It NEVER
|
||||
* travels in a user-facing token; it is set in-process on the collab connection
|
||||
* context by the native datasource, so it cannot be spoofed from a request.
|
||||
*/
|
||||
export type ProvenanceSource = 'user' | 'agent';
|
||||
export type ProvenanceSource = 'user' | 'agent' | 'git-sync';
|
||||
|
||||
export enum JwtType {
|
||||
ACCESS = 'access',
|
||||
@@ -26,7 +30,8 @@ export type JwtPayload = {
|
||||
// normal user token (treated as 'user'); set only when the internal agent
|
||||
// mints a provenance access token so REST writes (create/rename/move page,
|
||||
// comment create/resolve) record a non-spoofable 'agent' marker (§6.5 / §15
|
||||
// C3 / §14 N2).
|
||||
// C3 / §14 N2). (git-sync writes use the in-process actor, not a token — see
|
||||
// the ProvenanceSource note.)
|
||||
actor?: ProvenanceSource;
|
||||
// Nullable: an external MCP agent has no internal ai_chats row, so it carries
|
||||
// an 'agent' actor with a null aiChatId.
|
||||
@@ -39,7 +44,8 @@ export type JwtCollabPayload = {
|
||||
type: 'collab';
|
||||
// Optional agent-edit provenance, signed into the collab token. Absent for
|
||||
// the human collab path (treated as 'user'); set only when the internal agent
|
||||
// mints a provenance collab token (§6.6 / §15 C2).
|
||||
// mints a provenance collab token (§6.6 / §15 C2). 'git-sync' (in ProvenanceSource)
|
||||
// is accepted for type-compatibility with the in-process git-sync write path.
|
||||
actor?: ProvenanceSource;
|
||||
// Nullable: an external MCP agent has no internal ai_chats row, so it carries
|
||||
// an 'agent' actor with a null aiChatId.
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
import { BadRequestException } from '@nestjs/common';
|
||||
import { PageService } from './page.service';
|
||||
import { MovePageDto } from '../dto/move-page.dto';
|
||||
import { Page } from '@docmost/db/types/entity.types';
|
||||
import { CreatePageDto } from '../dto/create-page.dto';
|
||||
import { UpdatePageDto } from '../dto/update-page.dto';
|
||||
import { Page, User } from '@docmost/db/types/entity.types';
|
||||
import { DEFAULT_TEMPORARY_NOTE_HOURS } from '../constants/temporary-note.constants';
|
||||
import { AuthProvenanceData } from '../../../common/decorators/auth-provenance.decorator';
|
||||
|
||||
// Direct instantiation with stub deps. The Test.createTestingModule form failed
|
||||
// to resolve the @InjectKysely()/@InjectQueue() tokens at compile(), and this
|
||||
@@ -496,4 +499,295 @@ describe('PageService', () => {
|
||||
expect(db.selectFrom).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('git-sync provenance stamping (#1)', () => {
|
||||
const GIT_SYNC: AuthProvenanceData = { actor: 'git-sync', aiChatId: null };
|
||||
const USER_PROVENANCE: AuthProvenanceData = { actor: 'user', aiChatId: null };
|
||||
|
||||
describe('create()', () => {
|
||||
// Build a service whose insertPage/generalQueue are observable and whose
|
||||
// nextPagePosition (a DB query) is stubbed, so create() reaches insertPage
|
||||
// without a real database.
|
||||
const makeService = () => {
|
||||
const insertedPage = { id: 'page-1', slugId: 'slug-1' };
|
||||
const pageRepo = {
|
||||
insertPage: jest.fn().mockResolvedValue(insertedPage),
|
||||
};
|
||||
// add() is fire-and-forget (the service .catch()es it); resolve so no
|
||||
// unhandled rejection leaks.
|
||||
const generalQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
|
||||
const svc = new PageService(
|
||||
pageRepo as any, // pageRepo
|
||||
{} as any, // pagePermissionRepo
|
||||
{} as any, // attachmentRepo
|
||||
{} as any, // db
|
||||
{} as any, // storageService
|
||||
{} as any, // attachmentQueue
|
||||
{} as any, // aiQueue
|
||||
generalQueue as any, // generalQueue
|
||||
{} as any, // eventEmitter
|
||||
{} as any, // collaborationGateway
|
||||
{} as any, // watcherService
|
||||
{} as any, // transclusionService
|
||||
);
|
||||
|
||||
// nextPagePosition runs a kysely query; stub it so create() never hits
|
||||
// the db. No DTO content is provided, so parseProsemirrorContent is
|
||||
// skipped entirely (content/textContent/ydoc stay undefined).
|
||||
jest.spyOn(svc, 'nextPagePosition').mockResolvedValue('a0');
|
||||
|
||||
return { svc, pageRepo };
|
||||
};
|
||||
|
||||
const createDto: CreatePageDto = {
|
||||
title: 'New page',
|
||||
spaceId: 'space-1',
|
||||
} as any;
|
||||
|
||||
it("stamps lastUpdatedSource:'git-sync' on the insertPage payload", async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
|
||||
await svc.create('user-1', 'ws-1', createDto, GIT_SYNC);
|
||||
|
||||
expect(pageRepo.insertPage).toHaveBeenCalledTimes(1);
|
||||
expect(pageRepo.insertPage).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ lastUpdatedSource: 'git-sync' }),
|
||||
);
|
||||
// git-sync carries no aiChatId (unlike the agent branch).
|
||||
const payload = pageRepo.insertPage.mock.calls[0][0];
|
||||
expect(payload.lastUpdatedAiChatId).toBeUndefined();
|
||||
// The human stays the responsible author.
|
||||
expect(payload.creatorId).toBe('user-1');
|
||||
expect(payload.lastUpdatedById).toBe('user-1');
|
||||
});
|
||||
|
||||
it('leaves the source column unset for a plain user create', async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
|
||||
await svc.create('user-1', 'ws-1', createDto, USER_PROVENANCE);
|
||||
|
||||
const payload = pageRepo.insertPage.mock.calls[0][0];
|
||||
expect(payload.lastUpdatedSource).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('update() (rename)', () => {
|
||||
const makeService = () => {
|
||||
const pageRepo = {
|
||||
updatePage: jest.fn().mockResolvedValue({ numUpdatedRows: 1n }),
|
||||
// update() re-reads the row at the end to return the refreshed page.
|
||||
findById: jest.fn().mockResolvedValue({ id: 'page-1' }),
|
||||
};
|
||||
const generalQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
const aiQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
|
||||
const svc = new PageService(
|
||||
pageRepo as any, // pageRepo
|
||||
{} as any, // pagePermissionRepo
|
||||
{} as any, // attachmentRepo
|
||||
{} as any, // db
|
||||
{} as any, // storageService
|
||||
{} as any, // attachmentQueue
|
||||
aiQueue as any, // aiQueue
|
||||
generalQueue as any, // generalQueue
|
||||
{} as any, // eventEmitter
|
||||
{} as any, // collaborationGateway
|
||||
{} as any, // watcherService
|
||||
{} as any, // transclusionService
|
||||
);
|
||||
|
||||
return { svc, pageRepo };
|
||||
};
|
||||
|
||||
const page: Page = {
|
||||
id: 'page-1',
|
||||
slugId: 'slug-1',
|
||||
spaceId: 'space-1',
|
||||
workspaceId: 'ws-1',
|
||||
title: 'Old title',
|
||||
icon: null,
|
||||
parentPageId: null,
|
||||
contributorIds: [],
|
||||
} as any;
|
||||
|
||||
const user: User = { id: 'user-1' } as any;
|
||||
|
||||
it("stamps lastUpdatedSource:'git-sync' on the updatePage payload", async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
const dto: UpdatePageDto = { title: 'New title' } as any;
|
||||
|
||||
await svc.update(page, dto, user, GIT_SYNC);
|
||||
|
||||
expect(pageRepo.updatePage).toHaveBeenCalledTimes(1);
|
||||
const payload = pageRepo.updatePage.mock.calls[0][0];
|
||||
expect(payload.lastUpdatedSource).toBe('git-sync');
|
||||
expect(payload.lastUpdatedAiChatId).toBeUndefined();
|
||||
// The acting user stays the responsible author.
|
||||
expect(payload.lastUpdatedById).toBe('user-1');
|
||||
});
|
||||
|
||||
it('leaves the source column unset for a plain user rename', async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
const dto: UpdatePageDto = { title: 'New title' } as any;
|
||||
|
||||
await svc.update(page, dto, user, USER_PROVENANCE);
|
||||
|
||||
const payload = pageRepo.updatePage.mock.calls[0][0];
|
||||
expect(payload.lastUpdatedSource).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('movePage()', () => {
|
||||
const SPACE_ID = 'space-1';
|
||||
const VALID_POSITION = 'a0';
|
||||
|
||||
const makeService = () => {
|
||||
const pageRepo = {
|
||||
findById: jest.fn().mockResolvedValue({
|
||||
id: 'dest-parent',
|
||||
deletedAt: null,
|
||||
spaceId: SPACE_ID,
|
||||
}),
|
||||
updatePage: jest.fn().mockResolvedValue({ numUpdatedRows: 1n }),
|
||||
};
|
||||
const eventEmitter = { emit: jest.fn() };
|
||||
|
||||
// movePage now runs the cycle-check + UPDATE inside executeTx(this.db),
|
||||
// i.e. this.db.transaction().execute(fn => fn(trx)). A permissive
|
||||
// chainable Proxy stands in for the Kysely trx so the per-space
|
||||
// advisory-lock `sql``.execute(trx)` resolves and updatePage runs.
|
||||
const trxStub: any = new Proxy(function () {}, {
|
||||
get: (_t, p) =>
|
||||
p === 'then'
|
||||
? undefined
|
||||
: p === 'execute' || p === 'executeTakeFirst'
|
||||
? () => Promise.resolve([])
|
||||
: () => trxStub,
|
||||
});
|
||||
const db = {
|
||||
transaction: () => ({ execute: (fn: any) => fn(trxStub) }),
|
||||
};
|
||||
|
||||
const svc = new PageService(
|
||||
pageRepo as any, // pageRepo
|
||||
{} as any, // pagePermissionRepo
|
||||
{} as any, // attachmentRepo
|
||||
db as any, // db
|
||||
{} as any, // storageService
|
||||
{} as any, // attachmentQueue
|
||||
{} as any, // aiQueue
|
||||
{} as any, // generalQueue
|
||||
eventEmitter as any, // eventEmitter
|
||||
{} as any, // collaborationGateway
|
||||
{} as any, // watcherService
|
||||
{} as any, // transclusionService
|
||||
);
|
||||
|
||||
// No cycle: the destination's ancestor chain does not contain the moved
|
||||
// page, so movePage reaches updatePage.
|
||||
jest
|
||||
.spyOn(svc, 'getPageBreadCrumbs')
|
||||
.mockResolvedValue([{ id: 'dest-parent' }, { id: 'root' }] as any);
|
||||
|
||||
return { svc, pageRepo };
|
||||
};
|
||||
|
||||
const movedPage: Page = {
|
||||
id: 'page-1',
|
||||
parentPageId: 'old-parent',
|
||||
spaceId: SPACE_ID,
|
||||
workspaceId: 'ws-1',
|
||||
slugId: 'slug-1',
|
||||
title: 'Page 1',
|
||||
icon: null,
|
||||
} as any;
|
||||
|
||||
const dto: MovePageDto = {
|
||||
pageId: 'page-1',
|
||||
position: VALID_POSITION,
|
||||
parentPageId: 'dest-parent',
|
||||
};
|
||||
|
||||
it("stamps lastUpdatedSource:'git-sync' on the updatePage payload", async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
|
||||
await svc.movePage(dto, movedPage, GIT_SYNC);
|
||||
|
||||
expect(pageRepo.updatePage).toHaveBeenCalledTimes(1);
|
||||
const payload = pageRepo.updatePage.mock.calls[0][0];
|
||||
expect(payload.lastUpdatedSource).toBe('git-sync');
|
||||
expect(payload.lastUpdatedAiChatId).toBeUndefined();
|
||||
});
|
||||
|
||||
it('leaves the source column unset for a plain user move', async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
|
||||
await svc.movePage(dto, movedPage, USER_PROVENANCE);
|
||||
|
||||
const payload = pageRepo.updatePage.mock.calls[0][0];
|
||||
expect(payload.lastUpdatedSource).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('removePage()', () => {
|
||||
// removePage forwards a `source` 4th arg to pageRepo.removePage: 'git-sync'
|
||||
// for a git-sync-driven soft-delete (so the change-listener loop-guard skips
|
||||
// its own write), undefined otherwise.
|
||||
const makeService = () => {
|
||||
const pageRepo = {
|
||||
removePage: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
||||
const svc = new PageService(
|
||||
pageRepo as any, // pageRepo
|
||||
{} as any, // pagePermissionRepo
|
||||
{} as any, // attachmentRepo
|
||||
{} as any, // db
|
||||
{} as any, // storageService
|
||||
{} as any, // attachmentQueue
|
||||
{} as any, // aiQueue
|
||||
{} as any, // generalQueue
|
||||
{} as any, // eventEmitter
|
||||
{} as any, // collaborationGateway
|
||||
{} as any, // watcherService
|
||||
{} as any, // transclusionService
|
||||
);
|
||||
|
||||
return { svc, pageRepo };
|
||||
};
|
||||
|
||||
it("forwards 'git-sync' as the source for a git-sync soft-delete", async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
|
||||
await svc.removePage('page-1', 'user-1', 'ws-1', GIT_SYNC);
|
||||
|
||||
expect(pageRepo.removePage).toHaveBeenCalledTimes(1);
|
||||
const [pageId, userId, workspaceId, source] =
|
||||
pageRepo.removePage.mock.calls[0];
|
||||
expect(pageId).toBe('page-1');
|
||||
expect(userId).toBe('user-1');
|
||||
expect(workspaceId).toBe('ws-1');
|
||||
expect(source).toBe('git-sync');
|
||||
});
|
||||
|
||||
it('forwards undefined as the source for a plain user delete', async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
|
||||
await svc.removePage('page-1', 'user-1', 'ws-1', USER_PROVENANCE);
|
||||
|
||||
const [, , , source] = pageRepo.removePage.mock.calls[0];
|
||||
expect(source).toBeUndefined();
|
||||
});
|
||||
|
||||
it('forwards undefined as the source when no provenance is given', async () => {
|
||||
const { svc, pageRepo } = makeService();
|
||||
|
||||
await svc.removePage('page-1', 'user-1', 'ws-1');
|
||||
|
||||
const [, , , source] = pageRepo.removePage.mock.calls[0];
|
||||
expect(source).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -950,6 +950,12 @@ export class PageService {
|
||||
// Optional agent-edit provenance (from the signed access claim). Stamps the
|
||||
// source marker when the agent moves a page via REST (§6.6 REST path).
|
||||
provenance?: AuthProvenanceData,
|
||||
// Optional responsible author. When set (git-sync), the move is ATTRIBUTED
|
||||
// to that account via `lastUpdatedById` — parity with create/delete/rename,
|
||||
// which all stamp the service user. A normal user move omits it, leaving
|
||||
// `lastUpdatedById` untouched (a reparent is not a content edit, so the
|
||||
// existing author is preserved — unchanged behavior).
|
||||
actorUserId?: string,
|
||||
) {
|
||||
// validate position value by attempting to generate a key
|
||||
try {
|
||||
@@ -1019,6 +1025,9 @@ export class PageService {
|
||||
{
|
||||
position: dto.position,
|
||||
parentPageId: parentPageId,
|
||||
// Attribute a git-initiated move to the service account (parity with
|
||||
// create/delete/rename). Omitted for normal user moves -> unchanged.
|
||||
...(actorUserId ? { lastUpdatedById: actorUserId } : {}),
|
||||
// Agent-edit provenance: annotate the source on an agent move. A
|
||||
// normal user request leaves the existing source value unchanged.
|
||||
...agentSourceFields(
|
||||
@@ -1291,8 +1300,18 @@ export class PageService {
|
||||
pageId: string,
|
||||
userId: string,
|
||||
workspaceId: string,
|
||||
// Optional provenance. A git-sync-driven soft-delete stamps
|
||||
// `lastUpdatedSource = 'git-sync'` so the change-listener loop-guard skips
|
||||
// its own write (mirrors the create/update/move provenance branches above).
|
||||
provenance?: AuthProvenanceData,
|
||||
): Promise<void> {
|
||||
await this.pageRepo.removePage(pageId, userId, workspaceId);
|
||||
const isGitSync = provenance?.actor === 'git-sync';
|
||||
await this.pageRepo.removePage(
|
||||
pageId,
|
||||
userId,
|
||||
workspaceId,
|
||||
isGitSync ? 'git-sync' : undefined,
|
||||
);
|
||||
}
|
||||
|
||||
private async parseProsemirrorContent(
|
||||
|
||||
@@ -15,4 +15,12 @@ export class UpdateSpaceDto extends PartialType(CreateSpaceDto) {
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
allowViewerComments: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
gitSyncEnabled?: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
autoMergeConflicts?: boolean;
|
||||
}
|
||||
|
||||
@@ -22,4 +22,199 @@ describe('SpaceService', () => {
|
||||
it('should be defined', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
describe('updateSpace gitSyncEnabled', () => {
|
||||
const workspaceId = 'ws-1';
|
||||
const spaceId = 'space-1';
|
||||
|
||||
// executeTx runs the callback immediately with a passthrough trx so the
|
||||
// repo calls happen inline; mirrors how the sibling sharing/comments flags
|
||||
// are persisted.
|
||||
const buildService = (settingsBefore: Record<string, any>) => {
|
||||
const spaceRepo = {
|
||||
findById: jest.fn().mockResolvedValue({
|
||||
id: spaceId,
|
||||
name: 'Space',
|
||||
slug: 'space',
|
||||
description: '',
|
||||
settings: settingsBefore,
|
||||
}),
|
||||
updateGitSyncSettings: jest.fn().mockResolvedValue({}),
|
||||
updateSharingSettings: jest.fn().mockResolvedValue({}),
|
||||
updateCommentSettings: jest.fn().mockResolvedValue({}),
|
||||
updateSpace: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ id: spaceId, name: 'Space', slug: 'space' }),
|
||||
slugExists: jest.fn().mockResolvedValue(false),
|
||||
};
|
||||
const auditService = { log: jest.fn() };
|
||||
|
||||
const svc = new SpaceService(
|
||||
spaceRepo as any,
|
||||
{} as any, // spaceMemberService
|
||||
{} as any, // shareRepo
|
||||
{} as any, // workspaceRepo
|
||||
{} as any, // licenseCheckService
|
||||
{} as any, // db
|
||||
{} as any, // attachmentQueue
|
||||
auditService as any,
|
||||
);
|
||||
|
||||
// executeTx is invoked via the imported helper; patch it on the module.
|
||||
jest
|
||||
.spyOn(require('@docmost/db/utils'), 'executeTx')
|
||||
.mockImplementation(async (_db: any, cb: any) => cb({} as any));
|
||||
|
||||
return { svc, spaceRepo, auditService };
|
||||
};
|
||||
|
||||
it('persists gitSyncEnabled via updateGitSyncSettings(enabled)', async () => {
|
||||
const { svc, spaceRepo } = buildService({});
|
||||
|
||||
await svc.updateSpace(
|
||||
{ spaceId, gitSyncEnabled: true } as any,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
expect(spaceRepo.updateGitSyncSettings).toHaveBeenCalledWith(
|
||||
spaceId,
|
||||
workspaceId,
|
||||
'enabled',
|
||||
true,
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not call updateGitSyncSettings when flag is undefined', async () => {
|
||||
const { svc, spaceRepo } = buildService({});
|
||||
|
||||
await svc.updateSpace({ spaceId } as any, workspaceId);
|
||||
|
||||
expect(spaceRepo.updateGitSyncSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// --- audit delta on the git-sync toggle (test-strategy Module 4 / item #5)
|
||||
// updateSpace builds a before/after delta only when a flag's value actually
|
||||
// changes, and only logs an audit event when that delta is non-empty. These
|
||||
// assert that contract specifically for gitSyncEnabled.
|
||||
it('writes a SPACE_UPDATED audit delta on a REAL gitSyncEnabled change (false -> true)', async () => {
|
||||
// Prior persisted state: gitSync.enabled = false; the request flips it on.
|
||||
const { svc, auditService } = buildService({ gitSync: { enabled: false } });
|
||||
|
||||
await svc.updateSpace(
|
||||
{ spaceId, gitSyncEnabled: true } as any,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
expect(auditService.log).toHaveBeenCalledTimes(1);
|
||||
expect(auditService.log).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
resourceId: spaceId,
|
||||
spaceId,
|
||||
changes: {
|
||||
before: expect.objectContaining({ gitSyncEnabled: false }),
|
||||
after: expect.objectContaining({ gitSyncEnabled: true }),
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('also records the delta when no prior gitSync settings exist (undefined -> true defaults prev to false)', async () => {
|
||||
// No gitSync key at all: prev resolves to the `?? false` default, so
|
||||
// enabling it is still a real change and is audited.
|
||||
const { svc, auditService } = buildService({});
|
||||
|
||||
await svc.updateSpace(
|
||||
{ spaceId, gitSyncEnabled: true } as any,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
expect(auditService.log).toHaveBeenCalledTimes(1);
|
||||
const call = auditService.log.mock.calls[0][0];
|
||||
expect(call.changes.before.gitSyncEnabled).toBe(false);
|
||||
expect(call.changes.after.gitSyncEnabled).toBe(true);
|
||||
});
|
||||
|
||||
it('does NOT write an audit delta on a no-op gitSyncEnabled (same value true -> true)', async () => {
|
||||
// Prior persisted state already true; the request sets the same value.
|
||||
// updateGitSyncSettings still runs (idempotent persist), but nothing is
|
||||
// added to the before/after delta, so no audit event is emitted.
|
||||
const { svc, spaceRepo, auditService } = buildService({
|
||||
gitSync: { enabled: true },
|
||||
});
|
||||
|
||||
await svc.updateSpace(
|
||||
{ spaceId, gitSyncEnabled: true } as any,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
expect(spaceRepo.updateGitSyncSettings).toHaveBeenCalledTimes(1);
|
||||
expect(auditService.log).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// --- autoMergeConflicts: a SECOND key in the SAME `gitSync` jsonb object,
|
||||
// persisted the same way as `enabled` (the repo's jsonb-merge keeps siblings).
|
||||
it('persists autoMergeConflicts via updateGitSyncSettings(autoMergeConflicts)', async () => {
|
||||
const { svc, spaceRepo } = buildService({});
|
||||
|
||||
await svc.updateSpace(
|
||||
{ spaceId, autoMergeConflicts: true } as any,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
expect(spaceRepo.updateGitSyncSettings).toHaveBeenCalledWith(
|
||||
spaceId,
|
||||
workspaceId,
|
||||
'autoMergeConflicts',
|
||||
true,
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not call updateGitSyncSettings when autoMergeConflicts is undefined', async () => {
|
||||
const { svc, spaceRepo } = buildService({});
|
||||
|
||||
await svc.updateSpace({ spaceId } as any, workspaceId);
|
||||
|
||||
expect(spaceRepo.updateGitSyncSettings).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('writes a SPACE_UPDATED audit delta on a REAL autoMergeConflicts change (false -> true)', async () => {
|
||||
// Prior persisted state: gitSync.autoMergeConflicts = false; flip it on.
|
||||
const { svc, auditService } = buildService({
|
||||
gitSync: { autoMergeConflicts: false },
|
||||
});
|
||||
|
||||
await svc.updateSpace(
|
||||
{ spaceId, autoMergeConflicts: true } as any,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
expect(auditService.log).toHaveBeenCalledTimes(1);
|
||||
expect(auditService.log).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
resourceId: spaceId,
|
||||
spaceId,
|
||||
changes: {
|
||||
before: expect.objectContaining({ autoMergeConflicts: false }),
|
||||
after: expect.objectContaining({ autoMergeConflicts: true }),
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('does NOT write an audit delta on a no-op autoMergeConflicts (same value true -> true)', async () => {
|
||||
const { svc, spaceRepo, auditService } = buildService({
|
||||
gitSync: { autoMergeConflicts: true },
|
||||
});
|
||||
|
||||
await svc.updateSpace(
|
||||
{ spaceId, autoMergeConflicts: true } as any,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
expect(spaceRepo.updateGitSyncSettings).toHaveBeenCalledTimes(1);
|
||||
expect(auditService.log).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -213,6 +213,41 @@ export class SpaceService {
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof updateSpaceDto.gitSyncEnabled !== 'undefined') {
|
||||
const prev = settingsBefore?.gitSync?.enabled ?? false;
|
||||
if (prev !== updateSpaceDto.gitSyncEnabled) {
|
||||
before.gitSyncEnabled = prev;
|
||||
after.gitSyncEnabled = updateSpaceDto.gitSyncEnabled;
|
||||
}
|
||||
|
||||
await this.spaceRepo.updateGitSyncSettings(
|
||||
updateSpaceDto.spaceId,
|
||||
workspaceId,
|
||||
'enabled',
|
||||
updateSpaceDto.gitSyncEnabled,
|
||||
trx,
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof updateSpaceDto.autoMergeConflicts !== 'undefined') {
|
||||
const prev = settingsBefore?.gitSync?.autoMergeConflicts ?? false;
|
||||
if (prev !== updateSpaceDto.autoMergeConflicts) {
|
||||
before.autoMergeConflicts = prev;
|
||||
after.autoMergeConflicts = updateSpaceDto.autoMergeConflicts;
|
||||
}
|
||||
|
||||
// Merges into the SAME `gitSync` jsonb object as `enabled` (the repo's
|
||||
// jsonb-merge preserves sibling keys), so toggling one never clobbers the
|
||||
// other.
|
||||
await this.spaceRepo.updateGitSyncSettings(
|
||||
updateSpaceDto.spaceId,
|
||||
workspaceId,
|
||||
'autoMergeConflicts',
|
||||
updateSpaceDto.autoMergeConflicts,
|
||||
trx,
|
||||
);
|
||||
}
|
||||
|
||||
updatedSpace = await this.spaceRepo.updateSpace(
|
||||
{
|
||||
name: updateSpaceDto.name,
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
import {
|
||||
Kysely,
|
||||
CamelCasePlugin,
|
||||
DummyDriver,
|
||||
PostgresAdapter,
|
||||
PostgresIntrospector,
|
||||
PostgresQueryCompiler,
|
||||
CompiledQuery,
|
||||
} from 'kysely';
|
||||
import { PageRepo } from './page.repo';
|
||||
import type { KyselyDB } from '../../types/kysely.types';
|
||||
|
||||
/**
|
||||
* SQL-builder unit test for the git-sync provenance stamp on PageRepo's
|
||||
* soft-delete / restore paths (PR #119 review). Both `removePage` and
|
||||
* `restorePage` take an optional `lastUpdatedSource` arg and conditionally fold
|
||||
* it into the recursive-subtree `UPDATE pages SET ...` via
|
||||
* `...(lastUpdatedSource ? { lastUpdatedSource } : {})`. The change-listener
|
||||
* loop-guard reads `last_updated_source = 'git-sync'` to recognize git-sync's own
|
||||
* writes and skip the echo cycle; this test guards that the stamp is present when
|
||||
* the arg is supplied and ABSENT when it is omitted (an ordinary user delete must
|
||||
* not clobber the column).
|
||||
*
|
||||
* Harness: the same compile-only Kysely/DummyDriver pattern as
|
||||
* space.repo.spec.ts, plus the production `CamelCasePlugin` (so the compiled SQL
|
||||
* carries the real snake_case column names, e.g. `last_updated_source`) and a
|
||||
* thin driver that returns ONE fixed row for every query. The fixed row is what
|
||||
* lets the repo's guard reads (root snapshot / recursive descendants / restore
|
||||
* target) resolve non-empty so execution reaches the subtree UPDATE we assert on
|
||||
* — a bare DummyDriver returns no rows and both methods short-circuit before the
|
||||
* update. We never hit a real database; we capture each compiled statement via
|
||||
* Kysely's `log` hook and inspect the `update "pages" set ...` SQL.
|
||||
*/
|
||||
describe('PageRepo — git-sync provenance on soft-delete / restore SQL', () => {
|
||||
// A single row shaped to satisfy every column the repo reads off its guard
|
||||
// queries. `parentPageId: null` keeps restorePage on the simple path (no
|
||||
// parent-detach UPDATE), so the only `update "pages"` statement is the one we
|
||||
// assert on.
|
||||
const FIXED_ROW = {
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'Doc',
|
||||
icon: null,
|
||||
position: 'a0',
|
||||
spaceId: 'space-1',
|
||||
parentPageId: null,
|
||||
deletedAt: null,
|
||||
};
|
||||
|
||||
class FixedRowDriver extends DummyDriver {
|
||||
async acquireConnection(): Promise<any> {
|
||||
return {
|
||||
async executeQuery() {
|
||||
return { rows: [{ ...FIXED_ROW }] };
|
||||
},
|
||||
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
||||
async *streamQuery() {},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
interface Captured {
|
||||
sql: string;
|
||||
parameters: readonly unknown[];
|
||||
}
|
||||
|
||||
// Compile-only Kysely on the Postgres dialect (CamelCasePlugin for real column
|
||||
// names) whose `log` hook records every executed statement's compiled SQL.
|
||||
function makeRepoCapturingSql() {
|
||||
const captured: Captured[] = [];
|
||||
const db = new Kysely<any>({
|
||||
dialect: {
|
||||
createAdapter: () => new PostgresAdapter(),
|
||||
createDriver: () => new FixedRowDriver(),
|
||||
createIntrospector: (d) => new PostgresIntrospector(d),
|
||||
createQueryCompiler: () => new PostgresQueryCompiler(),
|
||||
},
|
||||
plugins: [new CamelCasePlugin()],
|
||||
log: (event) => {
|
||||
if (event.level === 'query') {
|
||||
const q = event.query as CompiledQuery;
|
||||
captured.push({ sql: q.sql, parameters: q.parameters });
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
const repo = new PageRepo(
|
||||
db as unknown as KyselyDB,
|
||||
{} as any,
|
||||
{ emit: jest.fn() } as any,
|
||||
);
|
||||
// Find the single subtree UPDATE on pages (collapse whitespace for matching).
|
||||
const getUpdatePagesSql = (): Captured | undefined =>
|
||||
captured
|
||||
.map((c) => ({ ...c, sql: c.sql.replace(/\s+/g, ' ') }))
|
||||
.find((c) => /update "pages" set/i.test(c.sql));
|
||||
return { repo, getUpdatePagesSql };
|
||||
}
|
||||
|
||||
describe('removePage', () => {
|
||||
it("stamps last_updated_source = 'git-sync' on the subtree soft-delete when the provenance arg is supplied", async () => {
|
||||
const { repo, getUpdatePagesSql } = makeRepoCapturingSql();
|
||||
|
||||
await repo.removePage('p1', 'user-1', 'ws-1', 'git-sync');
|
||||
|
||||
const update = getUpdatePagesSql();
|
||||
expect(update).toBeDefined();
|
||||
// The provenance column is in the UPDATE's SET clause...
|
||||
expect(update!.sql).toContain('"last_updated_source" =');
|
||||
// ...with the 'git-sync' marker as the bound value.
|
||||
expect(update!.parameters).toContain('git-sync');
|
||||
// Sanity: it is still the soft-delete UPDATE (sets deleted_at too).
|
||||
expect(update!.sql).toContain('"deleted_at" =');
|
||||
});
|
||||
|
||||
it('OMITS last_updated_source from the soft-delete when the provenance arg is undefined', async () => {
|
||||
const { repo, getUpdatePagesSql } = makeRepoCapturingSql();
|
||||
|
||||
await repo.removePage('p1', 'user-1', 'ws-1');
|
||||
|
||||
const update = getUpdatePagesSql();
|
||||
expect(update).toBeDefined();
|
||||
// Ordinary user delete: the column must NOT be touched (keeps prior value).
|
||||
expect(update!.sql).not.toContain('last_updated_source');
|
||||
expect(update!.parameters).not.toContain('git-sync');
|
||||
// It is still the soft-delete UPDATE.
|
||||
expect(update!.sql).toContain('"deleted_at" =');
|
||||
});
|
||||
});
|
||||
|
||||
describe('restorePage', () => {
|
||||
it("stamps last_updated_source = 'git-sync' on the subtree restore when the provenance arg is supplied", async () => {
|
||||
const { repo, getUpdatePagesSql } = makeRepoCapturingSql();
|
||||
|
||||
await repo.restorePage('p1', 'ws-1', 'git-sync');
|
||||
|
||||
const update = getUpdatePagesSql();
|
||||
expect(update).toBeDefined();
|
||||
expect(update!.sql).toContain('"last_updated_source" =');
|
||||
expect(update!.parameters).toContain('git-sync');
|
||||
// Sanity: it is the restore UPDATE (clears deleted_at).
|
||||
expect(update!.sql).toContain('"deleted_at" =');
|
||||
});
|
||||
|
||||
it('OMITS last_updated_source from the restore when the provenance arg is undefined', async () => {
|
||||
const { repo, getUpdatePagesSql } = makeRepoCapturingSql();
|
||||
|
||||
await repo.restorePage('p1', 'ws-1');
|
||||
|
||||
const update = getUpdatePagesSql();
|
||||
expect(update).toBeDefined();
|
||||
expect(update!.sql).not.toContain('last_updated_source');
|
||||
expect(update!.parameters).not.toContain('git-sync');
|
||||
expect(update!.sql).toContain('"deleted_at" =');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -349,6 +349,11 @@ export class PageRepo {
|
||||
pageId: string,
|
||||
deletedById: string,
|
||||
workspaceId: string,
|
||||
// Optional provenance marker. When the soft-delete is driven by an automated
|
||||
// data plane (e.g. git-sync), stamp `lastUpdatedSource` so the change-listener
|
||||
// loop-guard recognizes it as its own write and does not schedule an echo
|
||||
// cycle. Omitted for ordinary user deletes (column keeps its prior value).
|
||||
lastUpdatedSource?: string,
|
||||
): Promise<void> {
|
||||
const currentDate = new Date();
|
||||
|
||||
@@ -399,6 +404,7 @@ export class PageRepo {
|
||||
.set({
|
||||
deletedById: deletedById,
|
||||
deletedAt: currentDate,
|
||||
...(lastUpdatedSource ? { lastUpdatedSource } : {}),
|
||||
})
|
||||
.where('id', 'in', pageIds)
|
||||
.where('deletedAt', 'is', null)
|
||||
@@ -429,7 +435,14 @@ export class PageRepo {
|
||||
}
|
||||
}
|
||||
|
||||
async restorePage(pageId: string, workspaceId: string): Promise<void> {
|
||||
async restorePage(
|
||||
pageId: string,
|
||||
workspaceId: string,
|
||||
// See removePage: stamp `lastUpdatedSource` for automated (git-sync) restores
|
||||
// so the change-listener loop-guard skips the echo cycle. Omitted for
|
||||
// ordinary user restores.
|
||||
lastUpdatedSource?: string,
|
||||
): Promise<void> {
|
||||
// First, check if the page being restored has a deleted parent
|
||||
const pageToRestore = await this.db
|
||||
.selectFrom('pages')
|
||||
@@ -480,7 +493,12 @@ export class PageRepo {
|
||||
// On restore, disarm the death timer: pulling a note out of trash means
|
||||
// "keep it". Otherwise a deadline now in the past would re-trash it on the
|
||||
// next cleanup sweep.
|
||||
.set({ deletedById: null, deletedAt: null, temporaryExpiresAt: null })
|
||||
.set({
|
||||
deletedById: null,
|
||||
deletedAt: null,
|
||||
temporaryExpiresAt: null,
|
||||
...(lastUpdatedSource ? { lastUpdatedSource } : {}),
|
||||
})
|
||||
.where('id', 'in', pageIds)
|
||||
.execute();
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
import {
|
||||
Kysely,
|
||||
DummyDriver,
|
||||
PostgresAdapter,
|
||||
PostgresIntrospector,
|
||||
PostgresQueryCompiler,
|
||||
CompiledQuery,
|
||||
} from 'kysely';
|
||||
import { SpaceRepo } from './space.repo';
|
||||
import type { KyselyDB } from '../../types/kysely.types';
|
||||
|
||||
/**
|
||||
* SQL-builder unit test for the jsonb-merge invariant of
|
||||
* SpaceRepo.updateGitSyncSettings (review comment #694 / test-strategy item #6).
|
||||
*
|
||||
* The merge is RAW SQL, so a behavioural test would need a live Postgres — which
|
||||
* is intentionally out of scope here (the reviewer's own §13.3 was deferred for
|
||||
* the same reason). Instead we follow the existing repo-spec convention
|
||||
* (ai-agent-roles.repo.spec.ts) of NOT executing: we compile the query with a
|
||||
* DummyDriver Postgres dialect and assert the generated SQL preserves sibling
|
||||
* keys. The structural invariant the SQL must encode:
|
||||
*
|
||||
* settings := COALESCE(settings, '{}') || jsonb_build_object('gitSync', ...)
|
||||
* gitSync := COALESCE(settings->'gitSync', '{}') || jsonb_build_object(key, value)
|
||||
*
|
||||
* The OUTER `||` merges into the existing top-level `settings`, so a sibling
|
||||
* top-level key (e.g. `sharing`) is preserved. The INNER COALESCE merges into
|
||||
* the existing `gitSync` object, so a sibling key inside gitSync (e.g. `other`)
|
||||
* is preserved. A naive `set settings = jsonb_build_object('gitSync', ...)`
|
||||
* would clobber both — this test guards exactly that regression.
|
||||
*/
|
||||
describe('SpaceRepo.updateGitSyncSettings — jsonb merge SQL', () => {
|
||||
// A real Kysely on the Postgres dialect, but with a DummyDriver: it compiles
|
||||
// queries to real Postgres SQL without ever opening a connection.
|
||||
function makeCompileOnlyDb() {
|
||||
return new Kysely<any>({
|
||||
dialect: {
|
||||
createAdapter: () => new PostgresAdapter(),
|
||||
createDriver: () => new DummyDriver(),
|
||||
createIntrospector: (db) => new PostgresIntrospector(db),
|
||||
createQueryCompiler: () => new PostgresQueryCompiler(),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Build the repo over the compile-only db. The repo terminates the query with
|
||||
// `.executeTakeFirst()`, so we wrap every kysely builder in a Proxy: when the
|
||||
// repo finally calls `executeTakeFirst`, we `.compile()` that same builder
|
||||
// ourselves to capture the exact SQL it was about to run, then delegate.
|
||||
function makeRepoCapturingSql() {
|
||||
const db = makeCompileOnlyDb();
|
||||
let captured: CompiledQuery | undefined;
|
||||
|
||||
// kysely builders are immutable — each .set()/.where()/.returningAll()
|
||||
// returns a NEW builder — so re-wrap any chainable result.
|
||||
const wrap = (b: any): any =>
|
||||
new Proxy(b, {
|
||||
get(target, prop, receiver) {
|
||||
const value = Reflect.get(target, prop, receiver);
|
||||
if (typeof value !== 'function') return value;
|
||||
return (...callArgs: unknown[]) => {
|
||||
// Capture the SQL at the terminal execute call.
|
||||
if (
|
||||
(prop === 'executeTakeFirst' || prop === 'execute') &&
|
||||
typeof target.compile === 'function'
|
||||
) {
|
||||
captured = target.compile();
|
||||
}
|
||||
const result = value.apply(target, callArgs);
|
||||
if (
|
||||
result &&
|
||||
typeof result === 'object' &&
|
||||
typeof (result as any).compile === 'function'
|
||||
) {
|
||||
return wrap(result);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const originalUpdateTable = db.updateTable.bind(db);
|
||||
jest
|
||||
.spyOn(db, 'updateTable')
|
||||
.mockImplementation((...args: Parameters<typeof originalUpdateTable>) =>
|
||||
wrap(originalUpdateTable(...args)),
|
||||
);
|
||||
|
||||
const repo = new SpaceRepo(db as unknown as KyselyDB, {} as any);
|
||||
return { repo, getCaptured: () => captured };
|
||||
}
|
||||
|
||||
it("compiles a jsonb merge that preserves sibling top-level and gitSync keys", async () => {
|
||||
const { repo, getCaptured } = makeRepoCapturingSql();
|
||||
|
||||
// DummyDriver yields no rows; executeTakeFirst resolves to undefined. The
|
||||
// SQL is fully compiled by then, which is all we assert.
|
||||
await repo.updateGitSyncSettings('space-1', 'ws-1', 'enabled', true);
|
||||
|
||||
const compiled = getCaptured();
|
||||
expect(compiled).toBeDefined();
|
||||
// The raw SQL template carries newlines/indentation; collapse whitespace so
|
||||
// the structural assertions are not coupled to source formatting.
|
||||
const sql = compiled!.sql.replace(/\s+/g, ' ');
|
||||
|
||||
// OUTER merge into the existing settings object -> sibling top-level keys
|
||||
// (e.g. `sharing`) survive (NOT a bare jsonb_build_object assignment).
|
||||
expect(sql).toContain(`set "settings" = COALESCE(settings, '{}'::jsonb) ||`);
|
||||
// INNER merge into the existing gitSync object -> sibling gitSync keys
|
||||
// (e.g. `other`) survive.
|
||||
expect(sql).toContain(
|
||||
`jsonb_build_object('gitSync', COALESCE(settings->'gitSync', '{}'::jsonb) ||`,
|
||||
);
|
||||
// The pref key is set via jsonb_build_object on the inner object, with the
|
||||
// key as a BOUND, ::text-cast PARAMETER (not sql.raw) — security fix #5.
|
||||
expect(sql).toMatch(/jsonb_build_object\(\$\d+::text,/);
|
||||
// Scoped to the row + workspace.
|
||||
expect(sql).toContain(`where "id" =`);
|
||||
expect(sql).toContain(`and "workspaceId" =`);
|
||||
|
||||
// Sanity: this is NOT a clobbering assignment (no top-level
|
||||
// `set "settings" = jsonb_build_object(` without the COALESCE/merge).
|
||||
expect(sql).not.toContain(`set "settings" = jsonb_build_object(`);
|
||||
|
||||
// The pref VALUE stays inlined via sql.lit, but the KEY is now a bound
|
||||
// parameter, so id + workspaceId + the key are all bound (updatedAt is a Date).
|
||||
expect(compiled!.parameters).toContain('space-1');
|
||||
expect(compiled!.parameters).toContain('ws-1');
|
||||
expect(compiled!.parameters).toContain('enabled');
|
||||
});
|
||||
|
||||
it('binds the prefKey as a ::text parameter (no sql.raw splice) and inlines prefValue via sql.lit', async () => {
|
||||
const { repo, getCaptured } = makeRepoCapturingSql();
|
||||
|
||||
await repo.updateGitSyncSettings('space-1', 'ws-1', 'enabled', false);
|
||||
|
||||
const compiled = getCaptured()!;
|
||||
const sql = compiled.sql.replace(/\s+/g, ' ');
|
||||
// The key is a bound `$N::text` parameter; the value is the sql.lit literal.
|
||||
expect(sql).toMatch(/jsonb_build_object\(\$\d+::text, false\)/);
|
||||
// The literal key must NOT be spliced into the statement text (the footgun).
|
||||
expect(sql).not.toContain(`'enabled'`);
|
||||
// The key rides as a bound parameter instead.
|
||||
expect(compiled.parameters).toContain('enabled');
|
||||
});
|
||||
});
|
||||
@@ -111,6 +111,34 @@ export class SpaceRepo {
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
async updateGitSyncSettings(
|
||||
spaceId: string,
|
||||
workspaceId: string,
|
||||
prefKey: string,
|
||||
prefValue: string | boolean,
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.updateTable('spaces')
|
||||
.set({
|
||||
// The jsonb key is a BOUND PARAMETER (`${prefKey}::text`), not
|
||||
// `sql.raw(prefKey)`. The callers here only ever pass the literals
|
||||
// 'enabled' / 'autoMergeConflicts', but sql.raw would splice the string
|
||||
// straight into the statement — a latent SQL-injection footgun the moment
|
||||
// a future caller passes a request-derived key. Parameterizing closes it
|
||||
// with no behaviour change for the current literal callers.
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb)
|
||||
|| jsonb_build_object('gitSync', COALESCE(settings->'gitSync', '{}'::jsonb)
|
||||
|| jsonb_build_object(${prefKey}::text, ${sql.lit(prefValue)}))`,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where('id', '=', spaceId)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.returningAll()
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
async updateCommentSettings(
|
||||
spaceId: string,
|
||||
workspaceId: string,
|
||||
|
||||
@@ -15,6 +15,164 @@ describe('EnvironmentService', () => {
|
||||
expect(service).toBeDefined();
|
||||
});
|
||||
|
||||
describe('getGitSyncPollIntervalMs', () => {
|
||||
const withEnv = (value?: string) =>
|
||||
new EnvironmentService({
|
||||
get: (_key: string, fallback?: string) => value ?? fallback,
|
||||
} as any);
|
||||
|
||||
it('defaults to 15000 when unset', () => {
|
||||
expect(withEnv().getGitSyncPollIntervalMs()).toBe(15000);
|
||||
});
|
||||
|
||||
it('parses a valid positive int', () => {
|
||||
expect(withEnv('30000').getGitSyncPollIntervalMs()).toBe(30000);
|
||||
});
|
||||
|
||||
it('falls back to 15000 for non-positive or unparseable values', () => {
|
||||
expect(withEnv('0').getGitSyncPollIntervalMs()).toBe(15000);
|
||||
expect(withEnv('-100').getGitSyncPollIntervalMs()).toBe(15000);
|
||||
expect(withEnv('not-a-number').getGitSyncPollIntervalMs()).toBe(15000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getGitSyncDebounceMs', () => {
|
||||
const withEnv = (value?: string) =>
|
||||
new EnvironmentService({
|
||||
get: (_key: string, fallback?: string) => value ?? fallback,
|
||||
} as any);
|
||||
|
||||
it('defaults to 2000 when unset', () => {
|
||||
expect(withEnv().getGitSyncDebounceMs()).toBe(2000);
|
||||
});
|
||||
|
||||
it('parses a valid positive int', () => {
|
||||
expect(withEnv('500').getGitSyncDebounceMs()).toBe(500);
|
||||
});
|
||||
|
||||
it('falls back to 2000 for non-positive or unparseable values', () => {
|
||||
expect(withEnv('0').getGitSyncDebounceMs()).toBe(2000);
|
||||
expect(withEnv('-5').getGitSyncDebounceMs()).toBe(2000);
|
||||
expect(withEnv('not-a-number').getGitSyncDebounceMs()).toBe(2000);
|
||||
});
|
||||
});
|
||||
|
||||
// getGitSyncDataDir reads two distinct keys (GIT_SYNC_DATA_DIR and DATA_DIR),
|
||||
// so this builder maps each key to a supplied value (and honours the fallback
|
||||
// the getter passes for DATA_DIR's `|| './data'`).
|
||||
describe('getGitSyncDataDir', () => {
|
||||
const withEnv = (values: Record<string, string | undefined>) =>
|
||||
new EnvironmentService({
|
||||
get: (key: string, fallback?: string) => values[key] ?? fallback,
|
||||
} as any);
|
||||
|
||||
it("defaults to './data/git-sync' when neither key is set", () => {
|
||||
expect(withEnv({}).getGitSyncDataDir()).toBe('./data/git-sync');
|
||||
});
|
||||
|
||||
it('derives from DATA_DIR with the /git-sync suffix', () => {
|
||||
expect(
|
||||
withEnv({ DATA_DIR: '/var/lib/docmost' }).getGitSyncDataDir(),
|
||||
).toBe('/var/lib/docmost/git-sync');
|
||||
});
|
||||
|
||||
it('strips trailing slashes from DATA_DIR before appending', () => {
|
||||
expect(
|
||||
withEnv({ DATA_DIR: '/var/lib/docmost///' }).getGitSyncDataDir(),
|
||||
).toBe('/var/lib/docmost/git-sync');
|
||||
});
|
||||
|
||||
it('lets an explicit GIT_SYNC_DATA_DIR override the DATA_DIR derivation', () => {
|
||||
expect(
|
||||
withEnv({
|
||||
GIT_SYNC_DATA_DIR: '/custom/vault',
|
||||
DATA_DIR: '/var/lib/docmost',
|
||||
}).getGitSyncDataDir(),
|
||||
).toBe('/custom/vault');
|
||||
});
|
||||
|
||||
it('returns the explicit override verbatim (no /git-sync suffix, no slash strip)', () => {
|
||||
expect(
|
||||
withEnv({ GIT_SYNC_DATA_DIR: '/custom/vault/' }).getGitSyncDataDir(),
|
||||
).toBe('/custom/vault/');
|
||||
});
|
||||
});
|
||||
|
||||
// isGitSyncEnabled is the `.toLowerCase() === 'true'` contract: only a
|
||||
// case-insensitive "true" enables it; everything else (unset, "false",
|
||||
// garbage) is false.
|
||||
describe('isGitSyncEnabled', () => {
|
||||
const withEnv = (value?: string) =>
|
||||
new EnvironmentService({
|
||||
get: (_key: string, fallback?: string) => value ?? fallback,
|
||||
} as any);
|
||||
|
||||
it('is true for "true" and "TRUE" (case-insensitive)', () => {
|
||||
expect(withEnv('true').isGitSyncEnabled()).toBe(true);
|
||||
expect(withEnv('TRUE').isGitSyncEnabled()).toBe(true);
|
||||
});
|
||||
|
||||
it('is false when unset (defaults to "false")', () => {
|
||||
expect(withEnv().isGitSyncEnabled()).toBe(false);
|
||||
});
|
||||
|
||||
it('is false for "false" and garbage values', () => {
|
||||
expect(withEnv('false').isGitSyncEnabled()).toBe(false);
|
||||
expect(withEnv('maybe').isGitSyncEnabled()).toBe(false);
|
||||
expect(withEnv('1').isGitSyncEnabled()).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// isGitSyncHttpEnabled is the master gate of the /git smart-HTTP trust boundary.
|
||||
// When GIT_SYNC_HTTP_ENABLED is UNSET it FALLS BACK to isGitSyncEnabled(); when
|
||||
// set it is honored verbatim ('true' -> on, anything else -> off). The fallback
|
||||
// (default) branch is what these tests pin.
|
||||
describe('isGitSyncHttpEnabled', () => {
|
||||
const withEnv = (values: Record<string, string | undefined>) =>
|
||||
new EnvironmentService({
|
||||
get: (key: string, fallback?: string) => values[key] ?? fallback,
|
||||
} as any);
|
||||
|
||||
it('DEFAULT branch: unset -> falls back to isGitSyncEnabled() === true', () => {
|
||||
expect(
|
||||
withEnv({ GIT_SYNC_ENABLED: 'true' }).isGitSyncHttpEnabled(),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('DEFAULT branch: unset -> falls back to isGitSyncEnabled() === false', () => {
|
||||
// Neither key set: the fallback resolves to isGitSyncEnabled() which is
|
||||
// false by default.
|
||||
expect(withEnv({}).isGitSyncHttpEnabled()).toBe(false);
|
||||
expect(
|
||||
withEnv({ GIT_SYNC_ENABLED: 'false' }).isGitSyncHttpEnabled(),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('explicit "true" enables the host regardless of GIT_SYNC_ENABLED', () => {
|
||||
expect(
|
||||
withEnv({
|
||||
GIT_SYNC_HTTP_ENABLED: 'true',
|
||||
GIT_SYNC_ENABLED: 'false',
|
||||
}).isGitSyncHttpEnabled(),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('explicit non-"true" disables the host even when sync is enabled', () => {
|
||||
expect(
|
||||
withEnv({
|
||||
GIT_SYNC_HTTP_ENABLED: 'false',
|
||||
GIT_SYNC_ENABLED: 'true',
|
||||
}).isGitSyncHttpEnabled(),
|
||||
).toBe(false);
|
||||
expect(
|
||||
withEnv({
|
||||
GIT_SYNC_HTTP_ENABLED: 'maybe',
|
||||
GIT_SYNC_ENABLED: 'true',
|
||||
}).isGitSyncHttpEnabled(),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getSandboxTtlMs', () => {
|
||||
// ConfigService stub: get(key, def) returns the configured value for the key
|
||||
// (falling back to def), matching the @nestjs/config contract the service
|
||||
|
||||
@@ -370,6 +370,99 @@ export class EnvironmentService {
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
// --- git-sync (issue #194 §7.2) -------------------------------------------------
|
||||
|
||||
/** Global master switch for the git-sync control plane (default false). */
|
||||
isGitSyncEnabled(): boolean {
|
||||
return (
|
||||
this.configService.get<string>('GIT_SYNC_ENABLED', 'false').toLowerCase() ===
|
||||
'true'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether gitmost serves the per-space vaults over smart-HTTP (the /git host).
|
||||
* When GIT_SYNC_HTTP_ENABLED is UNSET it DEFAULTS to isGitSyncEnabled() — so
|
||||
* enabling sync also enables the host unless explicitly disabled. When set, it
|
||||
* is honored verbatim ('true' -> on, anything else -> off).
|
||||
*/
|
||||
isGitSyncHttpEnabled(): boolean {
|
||||
const raw = this.configService.get<string>('GIT_SYNC_HTTP_ENABLED');
|
||||
if (raw === undefined) return this.isGitSyncEnabled();
|
||||
return raw.toLowerCase() === 'true';
|
||||
}
|
||||
|
||||
/**
|
||||
* Root directory holding the per-space vault repos. Defaults to
|
||||
* `<DATA_DIR or ./data>/git-sync`. `DATA_DIR` is read directly (no dedicated
|
||||
* getter exists in this codebase) so the vault root tracks the data volume.
|
||||
*/
|
||||
getGitSyncDataDir(): string {
|
||||
const explicit = this.configService.get<string>('GIT_SYNC_DATA_DIR');
|
||||
if (explicit) return explicit;
|
||||
const dataDir = this.configService.get<string>('DATA_DIR') || './data';
|
||||
return `${dataDir.replace(/\/+$/, '')}/git-sync`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional remote template, e.g. `git@host:vault-{spaceId}.git` (`{spaceId}` is
|
||||
* substituted per-space in the orchestrator). SCAFFOLDING for the deferred
|
||||
* remote-push feature: the vendored engine has no remote-push path yet (SPEC
|
||||
* §7), so this value is currently inert — kept so the wiring is ready when the
|
||||
* engine grows a push path.
|
||||
*/
|
||||
getGitSyncRemoteTemplate(): string | undefined {
|
||||
return this.configService.get<string>('GIT_SYNC_REMOTE_TEMPLATE');
|
||||
}
|
||||
|
||||
/**
|
||||
* Poll-safety interval in ms (default 15000). A NaN / non-positive value falls
|
||||
* back to the default so a bad override can never disable or zero the poll loop.
|
||||
*/
|
||||
getGitSyncPollIntervalMs(): number {
|
||||
const parsed = parseInt(
|
||||
this.configService.get<string>('GIT_SYNC_POLL_INTERVAL_MS', '15000'),
|
||||
10,
|
||||
);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : 15000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Spawned `git http-backend` watchdog timeout in ms (default 120000). Bounds a
|
||||
* single smart-HTTP request so a stalled `git-receive-pack` cannot hold the
|
||||
* per-space lock forever (the child is killed and a 500 sent on expiry). A NaN /
|
||||
* non-positive value falls back to the default so a bad override can never
|
||||
* disable the watchdog.
|
||||
*/
|
||||
getGitSyncBackendTimeoutMs(): number {
|
||||
const v = parseInt(
|
||||
this.configService.get<string>('GIT_SYNC_BACKEND_TIMEOUT_MS', '120000'),
|
||||
10,
|
||||
);
|
||||
return Number.isFinite(v) && v > 0 ? v : 120000;
|
||||
}
|
||||
|
||||
/**
|
||||
* Event debounce window in ms (default 2000). A NaN / non-positive value falls
|
||||
* back to the default so a bad override can never disable the debounce.
|
||||
*/
|
||||
getGitSyncDebounceMs(): number {
|
||||
const parsed = parseInt(
|
||||
this.configService.get<string>('GIT_SYNC_DEBOUNCE_MS', '2000'),
|
||||
10,
|
||||
);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : 2000;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* The service user id git-sync writes are attributed to. Required when sync is
|
||||
* enabled (validated in environment.validation.ts); optional otherwise.
|
||||
*/
|
||||
getGitSyncServiceUserId(): string | undefined {
|
||||
return this.configService.get<string>('GIT_SYNC_SERVICE_USER_ID');
|
||||
}
|
||||
|
||||
// --- Blob sandbox (in-RAM ephemeral blob transfer; see SandboxModule) ---
|
||||
|
||||
// Base URL the sandbox `uri` is built from. It MUST be reachable over the
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
import { plainToInstance } from 'class-transformer';
|
||||
import { validateSync } from 'class-validator';
|
||||
import { EnvironmentVariables } from './environment.validation';
|
||||
|
||||
/**
|
||||
* Validation-layer coverage for the git-sync env contract (test-strategy Module
|
||||
* 4 / item #4). We drive the decorated class with `validateSync` directly — the
|
||||
* exported `validate()` helper calls `process.exit(1)` on failure and so cannot
|
||||
* be asserted in-process. We only assert the git-sync rules, providing the
|
||||
* minimal always-required fields so unrelated validators do not add noise.
|
||||
*/
|
||||
describe('EnvironmentVariables — git-sync validation', () => {
|
||||
// A baseline config that satisfies the unconditionally-required fields
|
||||
// (DATABASE_URL, REDIS_URL, APP_SECRET) so the only errors we ever see come
|
||||
// from the git-sync rules under test.
|
||||
const baseConfig = {
|
||||
DATABASE_URL: 'postgres://user:pass@localhost:5432/docmost',
|
||||
REDIS_URL: 'redis://localhost:6379',
|
||||
APP_SECRET: 'x'.repeat(32),
|
||||
};
|
||||
|
||||
const validate = (extra: Record<string, unknown>) => {
|
||||
const instance = plainToInstance(EnvironmentVariables, {
|
||||
...baseConfig,
|
||||
...extra,
|
||||
});
|
||||
return validateSync(instance);
|
||||
};
|
||||
|
||||
const errorFor = (errors: ReturnType<typeof validateSync>, property: string) =>
|
||||
errors.find((e) => e.property === property);
|
||||
|
||||
it('flags GIT_SYNC_SERVICE_USER_ID when GIT_SYNC_ENABLED="true" and the id is absent', () => {
|
||||
const errors = validate({ GIT_SYNC_ENABLED: 'true' });
|
||||
|
||||
const err = errorFor(errors, 'GIT_SYNC_SERVICE_USER_ID');
|
||||
expect(err).toBeDefined();
|
||||
// @IsNotEmpty is the failing constraint (sync is on but no attributable
|
||||
// author was configured).
|
||||
expect(err?.constraints).toHaveProperty('isNotEmpty');
|
||||
});
|
||||
|
||||
it('accepts GIT_SYNC_ENABLED="true" once GIT_SYNC_SERVICE_USER_ID is present', () => {
|
||||
const errors = validate({
|
||||
GIT_SYNC_ENABLED: 'true',
|
||||
GIT_SYNC_SERVICE_USER_ID: 'service-user-1',
|
||||
});
|
||||
|
||||
expect(errorFor(errors, 'GIT_SYNC_SERVICE_USER_ID')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not require the service user id when git-sync is disabled (unset)', () => {
|
||||
const errors = validate({});
|
||||
|
||||
// The @ValidateIf gate (GIT_SYNC_ENABLED === "true") is not met, so the
|
||||
// required-if-enabled rule is skipped entirely.
|
||||
expect(errorFor(errors, 'GIT_SYNC_SERVICE_USER_ID')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not require the service user id when git-sync is explicitly "false"', () => {
|
||||
const errors = validate({ GIT_SYNC_ENABLED: 'false' });
|
||||
|
||||
expect(errorFor(errors, 'GIT_SYNC_SERVICE_USER_ID')).toBeUndefined();
|
||||
expect(errorFor(errors, 'GIT_SYNC_ENABLED')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects a GIT_SYNC_ENABLED value outside the {true,false} set via @IsIn', () => {
|
||||
const errors = validate({ GIT_SYNC_ENABLED: 'maybe' });
|
||||
|
||||
const err = errorFor(errors, 'GIT_SYNC_ENABLED');
|
||||
expect(err).toBeDefined();
|
||||
expect(err?.constraints).toHaveProperty('isIn');
|
||||
});
|
||||
});
|
||||
@@ -172,6 +172,55 @@ export class EnvironmentVariables {
|
||||
)
|
||||
CLICKHOUSE_URL: string;
|
||||
|
||||
// --- git-sync (issue #194 §7.2) — all OPTIONAL. The master switch defaults off; a
|
||||
// required-if-enabled service user id is validated only when sync is on. ---
|
||||
|
||||
@IsOptional()
|
||||
@IsIn(['true', 'false'])
|
||||
@IsString()
|
||||
GIT_SYNC_ENABLED: string;
|
||||
|
||||
// Whether to serve the per-space vaults over smart-HTTP (the /git host).
|
||||
// When unset, defaults to GIT_SYNC_ENABLED (see isGitSyncHttpEnabled).
|
||||
@IsOptional()
|
||||
@IsIn(['true', 'false'])
|
||||
@IsString()
|
||||
GIT_SYNC_HTTP_ENABLED: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
GIT_SYNC_DATA_DIR: string;
|
||||
|
||||
// SCAFFOLDING for the deferred remote-push feature: the vendored engine does
|
||||
// not consume gitRemote yet (SPEC §7), so this is currently inert — validated
|
||||
// here so the wiring is ready when remote push lands.
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
GIT_SYNC_REMOTE_TEMPLATE: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
GIT_SYNC_POLL_INTERVAL_MS: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
GIT_SYNC_DEBOUNCE_MS: string;
|
||||
|
||||
// Watchdog timeout (ms) for the spawned `git http-backend` process (default
|
||||
// 120000): a stalled receive-pack is killed so it cannot hold the per-space
|
||||
// lock forever. Optional int (validated as a string env).
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
GIT_SYNC_BACKEND_TIMEOUT_MS: string;
|
||||
|
||||
|
||||
// Required when git-sync is enabled: the service user create/move/rename/delete
|
||||
// are attributed to (issue #194 §7.2). Optional otherwise.
|
||||
@ValidateIf((obj) => obj.GIT_SYNC_ENABLED === 'true')
|
||||
@IsNotEmpty()
|
||||
@IsString()
|
||||
GIT_SYNC_SERVICE_USER_ID: string;
|
||||
|
||||
// --- Blob sandbox (in-RAM ephemeral blob transfer; see SandboxModule) ---
|
||||
|
||||
@IsOptional()
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
/**
|
||||
* Git-sync control-plane constants.
|
||||
*
|
||||
* Event/job names are REUSED from the shared event contract (event.contants.ts)
|
||||
* so the listener subscribes to the exact names the rest of the server emits —
|
||||
* never a string literal that could drift. The Redis lock-key prefix + TTLs back
|
||||
* the single-writer leader lock (§9); the debounce default backs the per-space
|
||||
* event coalescing (§10).
|
||||
*/
|
||||
import { EventName } from '../../common/events/event.contants';
|
||||
|
||||
/**
|
||||
* The page lifecycle events the git-sync listener reacts to. A change
|
||||
* to any of these in an enabled space schedules a debounced sync cycle.
|
||||
* - PAGE_CREATED / PAGE_UPDATED / PAGE_MOVED — structural + content edits;
|
||||
* - PAGE_SOFT_DELETED / PAGE_RESTORED — Trash transitions (deletes are soft);
|
||||
* - PAGE_MOVED_TO_SPACE — cross-space move (cross-repo).
|
||||
*
|
||||
* NOTE: body edits arrive via PAGE_UPDATED (emitted from persistence.extension),
|
||||
* NOT via EventName.PAGE_CONTENT_UPDATED — that name is a BullMQ queue-job name,
|
||||
* not an EventEmitter2 event, so @OnEvent would never fire for it.
|
||||
*/
|
||||
export const GIT_SYNC_PAGE_EVENTS = [
|
||||
EventName.PAGE_CREATED,
|
||||
EventName.PAGE_UPDATED,
|
||||
EventName.PAGE_MOVED,
|
||||
EventName.PAGE_MOVED_TO_SPACE,
|
||||
EventName.PAGE_SOFT_DELETED,
|
||||
EventName.PAGE_RESTORED,
|
||||
] as const;
|
||||
|
||||
/** Redis key prefix for the per-space leader lock. */
|
||||
export const GIT_SYNC_LOCK_PREFIX = 'git-sync:lock:';
|
||||
|
||||
/**
|
||||
* Leader-lock TTL (ms). Must exceed the maximum expected cycle duration so the
|
||||
* lock is not lost mid-cycle; on a crash it expires on its own. The
|
||||
* in-process mutex (orchestrator) prevents overlapping cycles on one instance,
|
||||
* and the Redis lock prevents two instances racing the same space.
|
||||
*/
|
||||
export const GIT_SYNC_LOCK_TTL_MS = 5 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Bounded retry budget for ACQUIRING the per-space lock on the PUSH (external
|
||||
* receive-pack) path. The poll cycle holds the single-writer lock while it
|
||||
* processes a whole space, so a legitimate `git push` that arrives during a
|
||||
* cycle would otherwise IMMEDIATELY 503 (GitSyncLockHeldError) even though the
|
||||
* cycle is about to release the lock in well under a second for most spaces.
|
||||
* Under continuous polling that made a majority of pushes 503 non-
|
||||
* deterministically. So the push path retries the acquire with a small capped
|
||||
* backoff for up to ~`TOTAL_MS` BEFORE giving up — a transient overlap with a
|
||||
* cycle no longer fails the push, while a genuinely stuck/long cycle still
|
||||
* surfaces a 503 after the bound (git then retries the whole push, which is
|
||||
* safe: the receive-pack only runs ONCE the lock is held, so a 503 never leaves
|
||||
* a half-applied ref). The POLL cycle itself does NOT retry (it just skips and
|
||||
* the next tick reconciles), so this is push-only — the smaller blast radius.
|
||||
*/
|
||||
export const GIT_SYNC_PUSH_LOCK_RETRY_TOTAL_MS = 5_000;
|
||||
/** First backoff between push lock-acquire attempts (ms); doubles, capped. */
|
||||
export const GIT_SYNC_PUSH_LOCK_RETRY_BASE_MS = 100;
|
||||
/** Cap on the per-attempt push lock-acquire backoff (ms). */
|
||||
export const GIT_SYNC_PUSH_LOCK_RETRY_MAX_MS = 500;
|
||||
@@ -0,0 +1,138 @@
|
||||
// Unit tests for the ops/testing controller. The orchestrator, env,
|
||||
// and the workspace-ability factory are hand-built mocks. We assert the admin
|
||||
// guard (non-admin -> ForbiddenException, no orchestrator call), that trigger
|
||||
// uses the workspace from request context (never the body), and that status
|
||||
// returns the env-derived object.
|
||||
import { ForbiddenException, NotFoundException } from '@nestjs/common';
|
||||
import {
|
||||
WorkspaceCaslAction,
|
||||
WorkspaceCaslSubject,
|
||||
} from '../../core/casl/interfaces/workspace-ability.type';
|
||||
import { GitSyncController } from './git-sync.controller';
|
||||
|
||||
type AnyMock = jest.Mock;
|
||||
|
||||
interface Built {
|
||||
controller: GitSyncController;
|
||||
orchestrator: { runOnce: AnyMock };
|
||||
env: Record<string, AnyMock>;
|
||||
workspaceAbility: { createForUser: AnyMock };
|
||||
ability: { cannot: AnyMock };
|
||||
spaceRepo: { findById: AnyMock };
|
||||
}
|
||||
|
||||
function build(opts: { cannot?: boolean; spaceFound?: boolean } = {}): Built {
|
||||
const { cannot = false, spaceFound = true } = opts;
|
||||
const ability = { cannot: jest.fn(() => cannot) };
|
||||
const workspaceAbility = { createForUser: jest.fn(() => ability) };
|
||||
|
||||
const orchestrator = {
|
||||
runOnce: jest.fn(async () => ({ spaceId: 'space-1', ran: true })),
|
||||
};
|
||||
const env: Record<string, AnyMock> = {
|
||||
isGitSyncEnabled: jest.fn(() => true),
|
||||
getGitSyncDataDir: jest.fn(() => '/vaults'),
|
||||
getGitSyncPollIntervalMs: jest.fn(() => 15000),
|
||||
getGitSyncDebounceMs: jest.fn(() => 2000),
|
||||
getGitSyncServiceUserId: jest.fn(() => 'svc-user'),
|
||||
};
|
||||
const spaceRepo = {
|
||||
findById: jest.fn(async () => (spaceFound ? { id: 'space-1' } : undefined)),
|
||||
};
|
||||
|
||||
const controller = new GitSyncController(
|
||||
orchestrator as any,
|
||||
env as any,
|
||||
workspaceAbility as any,
|
||||
spaceRepo as any,
|
||||
);
|
||||
return { controller, orchestrator, env, workspaceAbility, ability, spaceRepo };
|
||||
}
|
||||
|
||||
const USER = { id: 'user-1' } as any;
|
||||
const WORKSPACE = { id: 'ctx-ws' } as any;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('GitSyncController', () => {
|
||||
describe('trigger', () => {
|
||||
it('blocks a non-admin: throws ForbiddenException and never calls runOnce', async () => {
|
||||
const { controller, orchestrator, ability } = build({ cannot: true });
|
||||
|
||||
await expect(
|
||||
controller.trigger({ spaceId: 'space-1' } as any, USER, WORKSPACE),
|
||||
).rejects.toBeInstanceOf(ForbiddenException);
|
||||
|
||||
expect(ability.cannot).toHaveBeenCalledWith(
|
||||
WorkspaceCaslAction.Manage,
|
||||
WorkspaceCaslSubject.Settings,
|
||||
);
|
||||
expect(orchestrator.runOnce).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('admin: calls runOnce(dto.spaceId, workspace.id) using the workspace from context', async () => {
|
||||
const { controller, orchestrator, spaceRepo } = build({ cannot: false });
|
||||
|
||||
// The body carries an attacker-controlled workspaceId that must be ignored.
|
||||
const res = await controller.trigger(
|
||||
{ spaceId: 'space-1', workspaceId: 'evil-ws' } as any,
|
||||
USER,
|
||||
WORKSPACE,
|
||||
);
|
||||
|
||||
// The space is resolved workspace-scoped (context workspace, not the body).
|
||||
expect(spaceRepo.findById).toHaveBeenCalledWith('space-1', 'ctx-ws');
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledWith('space-1', 'ctx-ws');
|
||||
expect(res).toEqual({ spaceId: 'space-1', ran: true });
|
||||
});
|
||||
|
||||
it('admin: 404s a spaceId that is not in the workspace and never calls runOnce', async () => {
|
||||
// A foreign/non-existent space must be rejected BEFORE buildSettings runs
|
||||
// (which would otherwise create an empty per-space vault directory).
|
||||
const { controller, orchestrator, spaceRepo } = build({
|
||||
cannot: false,
|
||||
spaceFound: false,
|
||||
});
|
||||
|
||||
await expect(
|
||||
controller.trigger({ spaceId: 'foreign' } as any, USER, WORKSPACE),
|
||||
).rejects.toBeInstanceOf(NotFoundException);
|
||||
|
||||
expect(spaceRepo.findById).toHaveBeenCalledWith('foreign', 'ctx-ws');
|
||||
expect(orchestrator.runOnce).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('status', () => {
|
||||
it('blocks a non-admin: throws ForbiddenException and never reads env', async () => {
|
||||
const { controller, env, ability } = build({ cannot: true });
|
||||
|
||||
await expect(controller.status(USER, WORKSPACE)).rejects.toBeInstanceOf(
|
||||
ForbiddenException,
|
||||
);
|
||||
|
||||
expect(ability.cannot).toHaveBeenCalledWith(
|
||||
WorkspaceCaslAction.Manage,
|
||||
WorkspaceCaslSubject.Settings,
|
||||
);
|
||||
// The admin guard short-circuits before the env-derived status is built.
|
||||
expect(env.isGitSyncEnabled).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('admin: returns the env-derived status object', async () => {
|
||||
const { controller } = build({ cannot: false });
|
||||
|
||||
const res = await controller.status(USER, WORKSPACE);
|
||||
|
||||
expect(res).toEqual({
|
||||
enabled: true,
|
||||
dataDir: '/vaults',
|
||||
pollIntervalMs: 15000,
|
||||
debounceMs: 2000,
|
||||
serviceUserConfigured: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
ForbiddenException,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
NotFoundException,
|
||||
Post,
|
||||
Get,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
|
||||
import { AuthUser } from '../../common/decorators/auth-user.decorator';
|
||||
import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
|
||||
import { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
|
||||
import WorkspaceAbilityFactory from '../../core/casl/abilities/workspace-ability.factory';
|
||||
import {
|
||||
WorkspaceCaslAction,
|
||||
WorkspaceCaslSubject,
|
||||
} from '../../core/casl/interfaces/workspace-ability.type';
|
||||
import { EnvironmentService } from '../environment/environment.service';
|
||||
import { IsUUID } from 'class-validator';
|
||||
import {
|
||||
GitSyncOrchestrator,
|
||||
GitSyncRunStatus,
|
||||
} from './services/git-sync.orchestrator';
|
||||
|
||||
/** Body for the manual one-shot trigger. */
|
||||
class TriggerGitSyncDto {
|
||||
// The global ValidationPipe runs with whitelist:true, which STRIPS any field
|
||||
// lacking a validation decorator — without this @IsUUID the spaceId would be
|
||||
// dropped and arrive as undefined.
|
||||
@IsUUID()
|
||||
spaceId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ops/testing endpoints for the git-sync control plane. Admin-guarded
|
||||
* (workspace Manage/Settings, mirroring WorkspaceController) so only workspace
|
||||
* admins can force a cycle. Mounted under the global `/api` prefix:
|
||||
* - POST /api/git-sync/trigger { spaceId } — run one cycle now (await result),
|
||||
* - GET /api/git-sync/status — report whether sync is enabled + config.
|
||||
*/
|
||||
@UseGuards(JwtAuthGuard)
|
||||
@Controller('git-sync')
|
||||
export class GitSyncController {
|
||||
constructor(
|
||||
private readonly orchestrator: GitSyncOrchestrator,
|
||||
private readonly environmentService: EnvironmentService,
|
||||
private readonly workspaceAbility: WorkspaceAbilityFactory,
|
||||
private readonly spaceRepo: SpaceRepo,
|
||||
) {}
|
||||
|
||||
/** Throw unless the caller is a workspace admin (Manage Settings). */
|
||||
private assertAdmin(user: User, workspace: Workspace): void {
|
||||
const ability = this.workspaceAbility.createForUser(user, workspace);
|
||||
if (
|
||||
ability.cannot(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Settings)
|
||||
) {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('trigger')
|
||||
async trigger(
|
||||
@Body() dto: TriggerGitSyncDto,
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<GitSyncRunStatus> {
|
||||
this.assertAdmin(user, workspace);
|
||||
// Verify the client-supplied spaceId BELONGS to this workspace before doing
|
||||
// any work (review): without this, `runOnce` -> `buildSettings` reads the
|
||||
// raw `spaces` row and creates an empty per-space vault directory for a
|
||||
// foreign/non-existent space before the content read finally 404s. Resolve
|
||||
// it workspace-scoped and 404 early.
|
||||
const space = await this.spaceRepo.findById(dto.spaceId, workspace.id);
|
||||
if (!space) {
|
||||
throw new NotFoundException('Space not found');
|
||||
}
|
||||
// Use the workspace from the request context (never client-supplied).
|
||||
return this.orchestrator.runOnce(dto.spaceId, workspace.id);
|
||||
}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Get('status')
|
||||
async status(
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<{
|
||||
enabled: boolean;
|
||||
dataDir: string;
|
||||
pollIntervalMs: number;
|
||||
debounceMs: number;
|
||||
serviceUserConfigured: boolean;
|
||||
}> {
|
||||
this.assertAdmin(user, workspace);
|
||||
return {
|
||||
enabled: this.environmentService.isGitSyncEnabled(),
|
||||
dataDir: this.environmentService.getGitSyncDataDir(),
|
||||
pollIntervalMs: this.environmentService.getGitSyncPollIntervalMs(),
|
||||
debounceMs: this.environmentService.getGitSyncDebounceMs(),
|
||||
serviceUserConfigured: Boolean(
|
||||
this.environmentService.getGitSyncServiceUserId(),
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { esmImport } from '../../common/helpers/esm-import';
|
||||
import type {
|
||||
VaultGit as VaultGitClass,
|
||||
vaultGitEnv as vaultGitEnvFn,
|
||||
runCycle as runCycleFn,
|
||||
parseDocmostMarkdown as parseDocmostMarkdownFn,
|
||||
markdownToProseMirror as markdownToProseMirrorFn,
|
||||
sanitizeTitle as sanitizeTitleFn,
|
||||
docsCanonicallyEqual as docsCanonicallyEqualFn,
|
||||
} from '@docmost/git-sync';
|
||||
|
||||
/**
|
||||
* Runtime value-export surface of the ESM-only `@docmost/git-sync` package that
|
||||
* the server consumes. Types are imported with `import type` (erased at compile,
|
||||
* no runtime require); only the VALUE exports below need the dynamic-load
|
||||
* treatment so a CJS `require()` of the ESM package never happens.
|
||||
*/
|
||||
interface GitSyncModule {
|
||||
VaultGit: typeof VaultGitClass;
|
||||
vaultGitEnv: typeof vaultGitEnvFn;
|
||||
runCycle: typeof runCycleFn;
|
||||
parseDocmostMarkdown: typeof parseDocmostMarkdownFn;
|
||||
markdownToProseMirror: typeof markdownToProseMirrorFn;
|
||||
sanitizeTitle: typeof sanitizeTitleFn;
|
||||
docsCanonicallyEqual: typeof docsCanonicallyEqualFn;
|
||||
}
|
||||
|
||||
// The CJS->ESM dynamic-import bridge lives in one shared helper
|
||||
// (common/helpers/esm-import.ts); see it for why `import()` must be hidden from
|
||||
// the TS commonjs downleveler. The typed `loadGitSync()` wrapper stays here.
|
||||
|
||||
// Memoize the in-flight/loaded module so the dynamic import runs at most once.
|
||||
let modulePromise: Promise<GitSyncModule> | null = null;
|
||||
|
||||
/**
|
||||
* Lazily load the ESM-only `@docmost/git-sync` package (cached). Resolves the
|
||||
* package entry to an absolute path, then imports it as a `file://` URL so the
|
||||
* package "exports" map is honoured without bare-specifier resolution-base
|
||||
* fragility.
|
||||
*/
|
||||
export async function loadGitSync(): Promise<GitSyncModule> {
|
||||
if (!modulePromise) {
|
||||
modulePromise = (async () => {
|
||||
const entry = require.resolve('@docmost/git-sync');
|
||||
const mod = (await esmImport(
|
||||
pathToFileURL(entry).href,
|
||||
)) as GitSyncModule;
|
||||
return mod;
|
||||
})().catch((err) => {
|
||||
// Do not cache a rejected import — allow the next call to retry.
|
||||
modulePromise = null;
|
||||
throw err;
|
||||
});
|
||||
}
|
||||
return modulePromise;
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ScheduleModule } from '@nestjs/schedule';
|
||||
import { DatabaseModule } from '@docmost/db/database.module';
|
||||
import { EnvironmentModule } from '../environment/environment.module';
|
||||
import { CollaborationModule } from '../../collaboration/collaboration.module';
|
||||
import { PageModule } from '../../core/page/page.module';
|
||||
import { AuthModule } from '../../core/auth/auth.module';
|
||||
import { GitmostDataSourceService } from './services/gitmost-datasource.service';
|
||||
import { GitSyncOrchestrator } from './services/git-sync.orchestrator';
|
||||
import { SpaceLockService } from './services/space-lock.service';
|
||||
import { VaultRegistryService } from './services/vault-registry.service';
|
||||
import { PageChangeListener } from './listeners/page-change.listener';
|
||||
import { GitSyncController } from './git-sync.controller';
|
||||
import { GitHttpBackendService } from './http/git-http-backend.service';
|
||||
import { GitHttpService } from './http/git-http.service';
|
||||
|
||||
/**
|
||||
* The git-sync control plane. Wires the native datasource, the
|
||||
* orchestrator (poll + leader-lock), the per-space vault registry, the
|
||||
* event-driven listener, and the admin trigger controller.
|
||||
*
|
||||
* Imports:
|
||||
* - DatabaseModule (global) — PageRepo / SpaceRepo / KyselyDB for the
|
||||
* datasource + orchestrator queries;
|
||||
* - EnvironmentModule (global) — EnvironmentService config;
|
||||
* - CollaborationModule — exports CollaborationGateway for native body writes;
|
||||
* - PageModule — exports PageService for structural mutations;
|
||||
* - ScheduleModule (NOT forRoot) — so SchedulerRegistry is injectable (the
|
||||
* orchestrator registers a DYNAMIC poll interval in onModuleInit). forRoot()
|
||||
* is already registered globally by TelemetryModule; importing the plain
|
||||
* module here avoids a duplicate scheduler registration.
|
||||
*
|
||||
* RedisService is provided by the global RedisModule (app.module) and CASL's
|
||||
* WorkspaceAbilityFactory by the global CaslModule — both resolve without an
|
||||
* explicit import here.
|
||||
*/
|
||||
@Module({
|
||||
imports: [
|
||||
DatabaseModule,
|
||||
EnvironmentModule,
|
||||
CollaborationModule,
|
||||
PageModule,
|
||||
// AuthModule exports AuthService (verifyUserCredentials for /git HTTP Basic).
|
||||
AuthModule,
|
||||
ScheduleModule,
|
||||
],
|
||||
controllers: [GitSyncController],
|
||||
providers: [
|
||||
GitmostDataSourceService,
|
||||
GitSyncOrchestrator,
|
||||
SpaceLockService,
|
||||
VaultRegistryService,
|
||||
PageChangeListener,
|
||||
// /git smart-HTTP host (the raw Fastify route in main.ts resolves these).
|
||||
GitHttpBackendService,
|
||||
GitHttpService,
|
||||
],
|
||||
// Exported so the raw Fastify route registered in main.ts can resolve the
|
||||
// handler from the Nest container (app.get(GitHttpService)).
|
||||
exports: [GitHttpService],
|
||||
})
|
||||
export class GitSyncModule {}
|
||||
@@ -0,0 +1,414 @@
|
||||
// Unit tests for the pure CGI-response helpers used by GitHttpBackendService.
|
||||
// The header/body split MUST treat the body as binary (Buffer) and never
|
||||
// stringify it; the Status: header sets the HTTP status (default 200).
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { spawn } from 'node:child_process';
|
||||
|
||||
// Mock the spawn boundary so run() never launches a real `git http-backend`; the
|
||||
// fake child lets us drive every stdout/stderr/error/close branch by hand.
|
||||
jest.mock('node:child_process', () => ({ spawn: jest.fn() }));
|
||||
// vaultGitEnv just builds the CGI env overlay; stub it to a passthrough so the
|
||||
// service runs without the real engine. The service loads it at runtime via the
|
||||
// `loadGitSync()` bridge (the ESM `@docmost/git-sync` package cannot be
|
||||
// `require()`d under jest), so we mock that loader rather than the package.
|
||||
jest.mock('../git-sync.loader', () => ({
|
||||
loadGitSync: jest.fn(async () => ({
|
||||
vaultGitEnv: (overlay: Record<string, string>) => overlay,
|
||||
})),
|
||||
}));
|
||||
|
||||
import {
|
||||
parseCgiResponse,
|
||||
splitCgiBuffer,
|
||||
buildGitBackendCgiEnv,
|
||||
GitHttpBackendService,
|
||||
} from './git-http-backend.service';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import type { GitHttpBackendRequest } from './git-http-backend.service';
|
||||
|
||||
const spawnMock = spawn as unknown as jest.Mock;
|
||||
|
||||
/** A fake `git http-backend` child: EventEmitter + stdout/stderr/stdin streams. */
|
||||
function fakeChild() {
|
||||
const child = new EventEmitter() as any;
|
||||
child.stdout = new EventEmitter();
|
||||
child.stderr = new EventEmitter();
|
||||
// stdin is written/ended/piped to; capture the calls, swallow nothing.
|
||||
child.stdin = Object.assign(new EventEmitter(), {
|
||||
end: jest.fn(),
|
||||
write: jest.fn(),
|
||||
});
|
||||
// The watchdog kills the child on timeout; capture the signal.
|
||||
child.kill = jest.fn();
|
||||
return child;
|
||||
}
|
||||
|
||||
/** A fake raw Node ServerResponse capturing status/headers/body/end. */
|
||||
function fakeRes() {
|
||||
const res: any = {
|
||||
headersSent: false,
|
||||
writableEnded: false,
|
||||
statusCode: 200,
|
||||
_headers: {} as Record<string, string>,
|
||||
_written: [] as Buffer[],
|
||||
setHeader: jest.fn((name: string, value: string) => {
|
||||
res._headers[name] = value;
|
||||
}),
|
||||
write: jest.fn((chunk: Buffer) => {
|
||||
res._written.push(chunk);
|
||||
return true;
|
||||
}),
|
||||
end: jest.fn((chunk?: Buffer | string) => {
|
||||
if (chunk !== undefined) res._written.push(chunk as Buffer);
|
||||
res.writableEnded = true;
|
||||
}),
|
||||
};
|
||||
return res;
|
||||
}
|
||||
|
||||
/** A fake raw Node IncomingMessage (GET => no body piped). */
|
||||
function fakeReq() {
|
||||
const req = new EventEmitter() as any;
|
||||
req.pipe = jest.fn();
|
||||
return req;
|
||||
}
|
||||
|
||||
const baseRequest: GitHttpBackendRequest = {
|
||||
spaceId: 'space-1',
|
||||
subpath: 'info/refs',
|
||||
method: 'GET',
|
||||
queryString: 'service=git-upload-pack',
|
||||
contentType: '',
|
||||
remoteUser: 'alice@example.com',
|
||||
};
|
||||
|
||||
function buildService(backendTimeoutMs = 120000) {
|
||||
const env = {
|
||||
getGitSyncDataDir: jest.fn(() => '/vaults'),
|
||||
// The watchdog timeout for the spawned git http-backend. Tests inject a tiny
|
||||
// value (or use fake timers) to drive the timeout branch.
|
||||
getGitSyncBackendTimeoutMs: jest.fn(() => backendTimeoutMs),
|
||||
};
|
||||
return new GitHttpBackendService(env as any);
|
||||
}
|
||||
|
||||
// `run()` now awaits the async `loadGitSync()` bridge before it spawns the
|
||||
// child, so the spawn (and its stream-handler wiring) happens one microtask
|
||||
// after `run()` is called. These tests drive the fake child synchronously, so
|
||||
// flush the microtask queue first to let `run()` reach the spawn.
|
||||
const flush = () => new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
describe('GitHttpBackendService.run', () => {
|
||||
beforeEach(() => {
|
||||
spawnMock.mockReset();
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
|
||||
});
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('(a) responds 500 when the child errors before any headers were written', async () => {
|
||||
const child = fakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
const service = buildService();
|
||||
const res = fakeRes();
|
||||
|
||||
const p = service.run(baseRequest, fakeReq(), res);
|
||||
await flush();
|
||||
// Emit a child 'error' before any stdout -> 500, headers not already sent.
|
||||
child.emit('error', new Error('ENOENT spawn git'));
|
||||
await p;
|
||||
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(res._headers['Content-Type']).toBe('text/plain');
|
||||
expect(res.end).toHaveBeenCalledWith('Internal server error');
|
||||
});
|
||||
|
||||
it('(a) responds 500 when the child closes before a complete CGI header block', async () => {
|
||||
const child = fakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
const service = buildService();
|
||||
const res = fakeRes();
|
||||
|
||||
const p = service.run(baseRequest, fakeReq(), res);
|
||||
await flush();
|
||||
// stderr diagnostics, then a close with no valid CGI output -> 500.
|
||||
child.stderr.emit('data', Buffer.from('fatal: boom'));
|
||||
child.emit('close', 128);
|
||||
await p;
|
||||
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(res.end).toHaveBeenCalledWith('Internal server error');
|
||||
});
|
||||
|
||||
it('(b) parses the CGI header block, sets status/headers, writes the body', async () => {
|
||||
const child = fakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
const service = buildService();
|
||||
const res = fakeRes();
|
||||
|
||||
const p = service.run(baseRequest, fakeReq(), res);
|
||||
await flush();
|
||||
// A full CGI response: status line + header + blank line + body.
|
||||
child.stdout.emit(
|
||||
'data',
|
||||
Buffer.from(
|
||||
'Status: 200 OK\r\nContent-Type: application/x-git-upload-pack-advertisement\r\n\r\nPACKBODY',
|
||||
'utf8',
|
||||
),
|
||||
);
|
||||
child.emit('close', 0);
|
||||
await p;
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res._headers['Content-Type']).toBe(
|
||||
'application/x-git-upload-pack-advertisement',
|
||||
);
|
||||
expect(Buffer.concat(res._written.map((c) => Buffer.from(c))).toString()).toContain(
|
||||
'PACKBODY',
|
||||
);
|
||||
expect(res.writableEnded).toBe(true);
|
||||
});
|
||||
|
||||
it('(c) swallows a stdout stream error (EPIPE) without throwing or 500ing', async () => {
|
||||
const child = fakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
const service = buildService();
|
||||
const res = fakeRes();
|
||||
const warnSpy = jest.spyOn(Logger.prototype, 'warn');
|
||||
|
||||
const p = service.run(baseRequest, fakeReq(), res);
|
||||
await flush();
|
||||
// The stdout 'error' handler must absorb this — no unhandled throw, no 500.
|
||||
expect(() => child.stdout.emit('error', new Error('EPIPE'))).not.toThrow();
|
||||
expect(() => child.stderr.emit('error', new Error('EPIPE'))).not.toThrow();
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
expect(res.statusCode).not.toBe(500);
|
||||
|
||||
// Let run() settle so the promise does not dangle.
|
||||
child.emit('close', 0);
|
||||
await p;
|
||||
});
|
||||
|
||||
it('(d) timeout: a child that never closes is killed and a 500 is sent', async () => {
|
||||
// The child never emits stdout/close (a stalled git-receive-pack). With a
|
||||
// tiny injected watchdog timeout the run() promise must still resolve: the
|
||||
// child is killed and a clean 500 is sent (no headers were sent yet).
|
||||
const child = fakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
const service = buildService(5); // 5ms watchdog
|
||||
const res = fakeRes();
|
||||
const warnSpy = jest.spyOn(Logger.prototype, 'warn');
|
||||
|
||||
// run() resolves only via the watchdog firing (no close/error emitted).
|
||||
await service.run(baseRequest, fakeReq(), res);
|
||||
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGTERM');
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(res.end).toHaveBeenCalledWith('Internal server error');
|
||||
});
|
||||
|
||||
it('(d) timeout watchdog is cleared on a normal close (no kill, no 500)', async () => {
|
||||
// A normal request that completes well within the watchdog window must NOT be
|
||||
// killed and must NOT trip the timeout 500 — the timer is cleared on close.
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const child = fakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
const service = buildService(120000);
|
||||
const res = fakeRes();
|
||||
|
||||
const p = service.run(baseRequest, fakeReq(), res);
|
||||
// loadGitSync resolves on a real microtask; advance it under fake timers.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
child.stdout.emit(
|
||||
'data',
|
||||
Buffer.from('Status: 200 OK\r\nContent-Type: text/plain\r\n\r\nOK', 'utf8'),
|
||||
);
|
||||
child.emit('close', 0);
|
||||
await p;
|
||||
|
||||
// The watchdog never fired even if we advance past its window.
|
||||
jest.advanceTimersByTime(200000);
|
||||
expect(child.kill).not.toHaveBeenCalled();
|
||||
expect(res.statusCode).toBe(200);
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('spawn throwing synchronously -> 500 (spawn-failed)', async () => {
|
||||
spawnMock.mockImplementation(() => {
|
||||
throw new Error('spawn EACCES');
|
||||
});
|
||||
const service = buildService();
|
||||
const res = fakeRes();
|
||||
|
||||
await service.run(baseRequest, fakeReq(), res);
|
||||
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(res.end).toHaveBeenCalledWith('Internal server error');
|
||||
});
|
||||
|
||||
it('(abort) an ALREADY-aborted signal -> no spawn, 500 lock-lost', async () => {
|
||||
// The per-space lock was already lost before run() reached the spawn: we must
|
||||
// NOT start writing the working tree after a possible lock takeover.
|
||||
const child = fakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
const service = buildService();
|
||||
const res = fakeRes();
|
||||
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await service.run(baseRequest, fakeReq(), res, controller.signal);
|
||||
|
||||
expect(spawnMock).not.toHaveBeenCalled();
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(res.end).toHaveBeenCalledWith('Internal server error');
|
||||
});
|
||||
|
||||
it('(abort) a live signal aborted mid-request -> child SIGTERM + response closed', async () => {
|
||||
// The lock lapses mid-push: the abort fires, the child is killed (SIGTERM,
|
||||
// then SIGKILL on escalation), and the response is finished.
|
||||
const child = fakeChild();
|
||||
spawnMock.mockReturnValue(child);
|
||||
const service = buildService();
|
||||
const res = fakeRes();
|
||||
const warnSpy = jest.spyOn(Logger.prototype, 'warn');
|
||||
|
||||
const controller = new AbortController();
|
||||
const p = service.run(baseRequest, fakeReq(), res, controller.signal);
|
||||
await flush(); // let run() reach the spawn + wire the abort listener
|
||||
controller.abort();
|
||||
await p;
|
||||
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGTERM');
|
||||
expect(warnSpy).toHaveBeenCalled();
|
||||
// No headers were sent before the abort -> a clean 500 is sent and ended.
|
||||
expect(res.statusCode).toBe(500);
|
||||
expect(res.writableEnded).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildGitBackendCgiEnv', () => {
|
||||
const base = {
|
||||
spaceId: 'space-1',
|
||||
subpath: 'info/refs',
|
||||
method: 'GET',
|
||||
queryString: 'service=git-upload-pack',
|
||||
contentType: '',
|
||||
remoteUser: 'alice@example.com',
|
||||
};
|
||||
|
||||
it('points PATH_INFO at the NON-bare repo dir (no .git suffix)', () => {
|
||||
// Regression guard: the vault lives at <root>/<spaceId> (a working repo), so
|
||||
// PATH_INFO must be /<spaceId>/<subpath>. A `.git` suffix made git
|
||||
// http-backend resolve <root>/<spaceId>.git and 404 every fetch/push.
|
||||
const env = buildGitBackendCgiEnv(base, '/vaults');
|
||||
expect(env.PATH_INFO).toBe('/space-1/info/refs');
|
||||
expect(env.PATH_INFO).not.toContain('.git');
|
||||
expect(env.GIT_PROJECT_ROOT).toBe('/vaults');
|
||||
});
|
||||
|
||||
it('forwards method/query/content-type/remote-user and exports all repos', () => {
|
||||
const env = buildGitBackendCgiEnv(
|
||||
{ ...base, method: 'POST', subpath: 'git-receive-pack', contentType: 'application/x-git-receive-pack-request', queryString: '' },
|
||||
'/vaults',
|
||||
);
|
||||
expect(env.REQUEST_METHOD).toBe('POST');
|
||||
expect(env.PATH_INFO).toBe('/space-1/git-receive-pack');
|
||||
expect(env.CONTENT_TYPE).toBe('application/x-git-receive-pack-request');
|
||||
expect(env.REMOTE_USER).toBe('alice@example.com');
|
||||
expect(env.GIT_HTTP_EXPORT_ALL).toBe('1');
|
||||
});
|
||||
|
||||
it('sets GIT_PROTOCOL only when the client sent the header', () => {
|
||||
expect(buildGitBackendCgiEnv(base, '/vaults').GIT_PROTOCOL).toBeUndefined();
|
||||
expect(
|
||||
buildGitBackendCgiEnv({ ...base, gitProtocol: 'version=2' }, '/vaults')
|
||||
.GIT_PROTOCOL,
|
||||
).toBe('version=2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseCgiResponse', () => {
|
||||
it('defaults to status 200 with no Status header', () => {
|
||||
const r = parseCgiResponse('Content-Type: application/x-git-upload-pack-result');
|
||||
expect(r.statusCode).toBe(200);
|
||||
expect(r.headers).toEqual([
|
||||
['Content-Type', 'application/x-git-upload-pack-result'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('honors a Status header and does not forward it', () => {
|
||||
const r = parseCgiResponse('Status: 404 Not Found\nContent-Type: text/plain');
|
||||
expect(r.statusCode).toBe(404);
|
||||
expect(r.headers).toEqual([['Content-Type', 'text/plain']]);
|
||||
});
|
||||
|
||||
it('parses multiple headers and trims whitespace', () => {
|
||||
const r = parseCgiResponse(
|
||||
'Status: 403 Forbidden\r\nContent-Type: text/plain \r\nX-Foo: bar ',
|
||||
);
|
||||
expect(r.statusCode).toBe(403);
|
||||
expect(r.headers).toEqual([
|
||||
['Content-Type', 'text/plain'],
|
||||
['X-Foo', 'bar'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('ignores malformed (colon-less) lines defensively', () => {
|
||||
const r = parseCgiResponse('Content-Type: text/plain\ngarbage-line\nX-A: b');
|
||||
expect(r.statusCode).toBe(200);
|
||||
expect(r.headers).toEqual([
|
||||
['Content-Type', 'text/plain'],
|
||||
['X-A', 'b'],
|
||||
]);
|
||||
});
|
||||
|
||||
it('ignores an out-of-range Status code and keeps the default', () => {
|
||||
const r = parseCgiResponse('Status: not-a-number\nContent-Type: text/plain');
|
||||
expect(r.statusCode).toBe(200);
|
||||
});
|
||||
|
||||
it('treats the Status header case-insensitively', () => {
|
||||
const r = parseCgiResponse('status: 500 Boom');
|
||||
expect(r.statusCode).toBe(500);
|
||||
expect(r.headers).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('splitCgiBuffer', () => {
|
||||
it('splits on a CRLF blank line and keeps the body as bytes', () => {
|
||||
const buf = Buffer.concat([
|
||||
Buffer.from('Status: 200 OK\r\nContent-Type: text/plain\r\n\r\n', 'utf8'),
|
||||
Buffer.from([0x00, 0x01, 0x02, 0xff]),
|
||||
]);
|
||||
const split = splitCgiBuffer(buf);
|
||||
expect(split).not.toBeNull();
|
||||
expect(split!.headerText).toBe('Status: 200 OK\r\nContent-Type: text/plain');
|
||||
expect(Array.from(split!.body)).toEqual([0x00, 0x01, 0x02, 0xff]);
|
||||
});
|
||||
|
||||
it('splits on a bare LF blank line', () => {
|
||||
const buf = Buffer.from('Content-Type: text/plain\n\nhello', 'utf8');
|
||||
const split = splitCgiBuffer(buf);
|
||||
expect(split).not.toBeNull();
|
||||
expect(split!.headerText).toBe('Content-Type: text/plain');
|
||||
expect(split!.body.toString('utf8')).toBe('hello');
|
||||
});
|
||||
|
||||
it('returns an empty body when nothing follows the separator', () => {
|
||||
const buf = Buffer.from('Content-Type: text/plain\r\n\r\n', 'utf8');
|
||||
const split = splitCgiBuffer(buf);
|
||||
expect(split).not.toBeNull();
|
||||
expect(split!.body.length).toBe(0);
|
||||
});
|
||||
|
||||
it('returns null when there is no blank-line separator yet', () => {
|
||||
const buf = Buffer.from('Content-Type: text/plain\r\nincomplete', 'utf8');
|
||||
expect(splitCgiBuffer(buf)).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,419 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { spawn } from 'node:child_process';
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http';
|
||||
import { loadGitSync } from '../git-sync.loader';
|
||||
import { EnvironmentService } from '../../environment/environment.service';
|
||||
|
||||
/** The parsed first part of a CGI response: the HTTP status + header pairs. */
|
||||
export interface ParsedCgiResponse {
|
||||
statusCode: number;
|
||||
/** Lower-cased? No — keep header names verbatim as git http-backend emits. */
|
||||
headers: Array<[string, string]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the CGI header block emitted by `git http-backend` into an HTTP status
|
||||
* and a list of header pairs. The input is ONLY the header text (everything up
|
||||
* to, but not including, the blank-line separator) — the binary body is split
|
||||
* off by the caller on the raw Buffer (never stringified).
|
||||
*
|
||||
* CGI semantics (RFC 3875 §6): a `Status: <code> <reason>` header sets the HTTP
|
||||
* status (default 200 when absent). Every other header is forwarded verbatim.
|
||||
* Header lines are `Name: value`; a line without a ':' is ignored defensively.
|
||||
*
|
||||
* Pure + framework-free so it is unit-testable in isolation.
|
||||
*/
|
||||
export function parseCgiResponse(headerBlock: string): ParsedCgiResponse {
|
||||
let statusCode = 200;
|
||||
const headers: Array<[string, string]> = [];
|
||||
|
||||
// Header lines may be separated by CRLF or LF; split on either.
|
||||
const lines = headerBlock.split(/\r?\n/);
|
||||
for (const line of lines) {
|
||||
if (line.length === 0) continue;
|
||||
const sep = line.indexOf(':');
|
||||
if (sep === -1) continue; // not a header line — ignore defensively
|
||||
const name = line.slice(0, sep).trim();
|
||||
const value = line.slice(sep + 1).trim();
|
||||
if (name.toLowerCase() === 'status') {
|
||||
// `Status: 404 Not Found` — the leading integer is the HTTP status code.
|
||||
const code = parseInt(value, 10);
|
||||
if (Number.isFinite(code) && code >= 100 && code <= 599) {
|
||||
statusCode = code;
|
||||
}
|
||||
continue; // never forward the CGI Status header itself
|
||||
}
|
||||
headers.push([name, value]);
|
||||
}
|
||||
|
||||
return { statusCode, headers };
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a raw CGI response buffer at the first blank-line boundary
|
||||
* (`\r\n\r\n` or `\n\n`). Returns the header text and the remaining body bytes.
|
||||
* Returns null when no blank-line separator is present (a malformed response).
|
||||
*
|
||||
* Pure (operates on Buffers, never stringifies the body) so it is testable.
|
||||
*/
|
||||
export function splitCgiBuffer(
|
||||
buf: Buffer,
|
||||
): { headerText: string; body: Buffer } | null {
|
||||
// Prefer the CRLF separator; fall back to bare LF.
|
||||
let idx = buf.indexOf('\r\n\r\n');
|
||||
let sepLen = 4;
|
||||
if (idx === -1) {
|
||||
idx = buf.indexOf('\n\n');
|
||||
sepLen = 2;
|
||||
}
|
||||
if (idx === -1) return null;
|
||||
const headerText = buf.subarray(0, idx).toString('utf8');
|
||||
const body = buf.subarray(idx + sepLen);
|
||||
return { headerText, body };
|
||||
}
|
||||
|
||||
/** A parsed git smart-HTTP request, resolved by the controller/handler. */
|
||||
export interface GitHttpBackendRequest {
|
||||
/** The space id (the on-disk vault dir name == GIT_PROJECT_ROOT child). */
|
||||
spaceId: string;
|
||||
/** The subpath after `<spaceId>.git/`, e.g. `info/refs` or `git-receive-pack`. */
|
||||
subpath: string;
|
||||
/** REQUEST_METHOD — `GET` or `POST`. */
|
||||
method: string;
|
||||
/** Raw query string WITHOUT the leading '?', e.g. `service=git-receive-pack`. */
|
||||
queryString: string;
|
||||
/** Content-Type header value (may be empty for GET). */
|
||||
contentType: string;
|
||||
/** The Git-Protocol request header value, or undefined when absent. */
|
||||
gitProtocol?: string;
|
||||
/** Content-Encoding request header (e.g. `gzip`), or undefined when absent.
|
||||
* git gzips RPC bodies >1KiB; http-backend only inflates when HTTP_CONTENT_ENCODING
|
||||
* is present, so it MUST be forwarded or a non-trivial `git pull` fails with
|
||||
* `fatal: expected 'packfile'` (review #4). */
|
||||
contentEncoding?: string;
|
||||
/** Authenticated user email — used as REMOTE_USER (reflog identity). */
|
||||
remoteUser: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bridges an HTTP git smart-protocol request to `git http-backend` (the CGI that
|
||||
* implements the entire smart-HTTP protocol: info/refs, upload-pack,
|
||||
* receive-pack, protocol v2, dumb fallback). We do NOT reimplement pkt-line.
|
||||
*
|
||||
* The Fastify reply is hijacked by the caller; this service streams the request
|
||||
* body to the child's stdin and writes the child's CGI response (status +
|
||||
* headers parsed from the leading header block, then the raw binary body) to the
|
||||
* Node response. Errors before any output produce a 500. Credentials are never
|
||||
* logged.
|
||||
*/
|
||||
/**
|
||||
* Build the `git http-backend` CGI environment overlay for one request (the
|
||||
* variables layered on top of `vaultGitEnv`'s cwd-isolated base). Pure so the
|
||||
* PATH_INFO / REMOTE_USER / conditional GIT_PROTOCOL wiring is unit-testable
|
||||
* without spawning git.
|
||||
*
|
||||
* PATH_INFO is the repo-relative CGI path. The vault is a NON-BARE working repo
|
||||
* on disk at `<dataDir>/<spaceId>` (the engine needs a working tree), so the
|
||||
* repo directory git http-backend must resolve is `<spaceId>` — NOT
|
||||
* `<spaceId>.git`. The URL carries the conventional `.git` suffix (stripped by
|
||||
* parseGitPath into `spaceId`); re-appending it here pointed the CGI at a
|
||||
* non-existent `<dataDir>/<spaceId>.git` and every fetch/push 404'd.
|
||||
*/
|
||||
export function buildGitBackendCgiEnv(
|
||||
parsed: GitHttpBackendRequest,
|
||||
projectRoot: string,
|
||||
): Record<string, string> {
|
||||
const cgiEnv: Record<string, string> = {
|
||||
GIT_PROJECT_ROOT: projectRoot,
|
||||
GIT_HTTP_EXPORT_ALL: '1', // authz is done by us; no git-daemon-export-ok file
|
||||
PATH_INFO: `/${parsed.spaceId}/${parsed.subpath}`,
|
||||
REQUEST_METHOD: parsed.method,
|
||||
QUERY_STRING: parsed.queryString,
|
||||
CONTENT_TYPE: parsed.contentType,
|
||||
REMOTE_USER: parsed.remoteUser,
|
||||
};
|
||||
// GIT_PROTOCOL is only set when the client sent the Git-Protocol header.
|
||||
if (parsed.gitProtocol) {
|
||||
cgiEnv.GIT_PROTOCOL = parsed.gitProtocol;
|
||||
}
|
||||
// HTTP_CONTENT_ENCODING must be forwarded so git http-backend inflates a
|
||||
// gzip'd RPC body (git compresses receive-pack/upload-pack bodies >1KiB).
|
||||
// Without it a non-trivial `git pull` negotiation fails deterministically with
|
||||
// `fatal: expected 'packfile'` (review #4). The body is piped to stdin as-is
|
||||
// (no upstream decompression), so the CGI must do the inflate.
|
||||
if (parsed.contentEncoding) {
|
||||
cgiEnv.HTTP_CONTENT_ENCODING = parsed.contentEncoding;
|
||||
}
|
||||
return cgiEnv;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class GitHttpBackendService {
|
||||
private readonly logger = new Logger(GitHttpBackendService.name);
|
||||
|
||||
constructor(private readonly environmentService: EnvironmentService) {}
|
||||
|
||||
/**
|
||||
* Spawn `git http-backend` for one request and bridge it to the raw Node
|
||||
* request/response. Resolves when the response has been fully written (the
|
||||
* child exited and its output was flushed), or after a 500 was sent on an
|
||||
* early failure. Never rejects — push ingestion relies on this resolving so
|
||||
* the lock-held cycle body can run afterwards.
|
||||
*
|
||||
* `signal` (optional) is the git-sync per-space lock's lost-lock abort signal.
|
||||
* A receive-pack writes `main`'s working tree, so if the lock lapses mid-push
|
||||
* (heartbeat CAS miss / Redis outage) the signal fires and we kill the child —
|
||||
* preventing it from continuing to write the working tree while another replica
|
||||
* may have taken over the lock and started a cycle (warning #3).
|
||||
*/
|
||||
async run(
|
||||
parsed: GitHttpBackendRequest,
|
||||
rawReq: IncomingMessage,
|
||||
rawRes: ServerResponse,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const { vaultGitEnv } = await loadGitSync();
|
||||
const projectRoot = this.environmentService.getGitSyncDataDir();
|
||||
// Build the CGI env from the engine's cwd-isolated base (strips GIT_DIR /
|
||||
// GIT_WORK_TREE), then layer the http-backend CGI variables. PATH is
|
||||
// preserved (vaultGitEnv already copies process.env, so PATH carries
|
||||
// through).
|
||||
const env = vaultGitEnv(buildGitBackendCgiEnv(parsed, projectRoot));
|
||||
|
||||
return new Promise<void>((resolve) => {
|
||||
let settled = false;
|
||||
// Set once the child exists so the abort handler can target it.
|
||||
let onAbort: (() => void) | null = null;
|
||||
// The watchdog timer; cleared centrally in done() so EVERY settle path
|
||||
// (close, error, timeout, abort) tears it down exactly once.
|
||||
let watchdogTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
const done = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (watchdogTimer) clearTimeout(watchdogTimer);
|
||||
// Detach the abort listener so a later lock loss does not fire into a
|
||||
// request that already finished.
|
||||
if (onAbort) {
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
onAbort = null;
|
||||
}
|
||||
resolve();
|
||||
};
|
||||
|
||||
// Reject early if the lock was already lost before we even spawned: do not
|
||||
// start writing the working tree after a possible lock takeover.
|
||||
if (signal?.aborted) {
|
||||
if (!rawRes.headersSent) this.send500(rawRes, 'lock-lost');
|
||||
else
|
||||
try {
|
||||
rawRes.end();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return done();
|
||||
}
|
||||
|
||||
let child: ReturnType<typeof spawn>;
|
||||
try {
|
||||
child = spawn('git', ['http-backend'], { env });
|
||||
} catch (err) {
|
||||
this.send500(rawRes, 'spawn-failed', err);
|
||||
return done();
|
||||
}
|
||||
|
||||
// Lost-lock abort: the per-space lock lapsed mid-request. Kill the child so
|
||||
// a receive-pack stops writing `main`'s working tree before another replica
|
||||
// (which may now hold the lock) starts a cycle. Same kill+finish path the
|
||||
// watchdog uses (extracted into terminateChild).
|
||||
onAbort = () => {
|
||||
this.terminateChild(
|
||||
child,
|
||||
rawRes,
|
||||
headerParsed,
|
||||
'lock-lost',
|
||||
'git http-backend aborted (git-sync lock lost mid-request); killing child',
|
||||
done,
|
||||
);
|
||||
};
|
||||
signal?.addEventListener('abort', onAbort);
|
||||
|
||||
// Watchdog: a client that opens git-receive-pack and stalls keeps the
|
||||
// child alive forever, so run() never resolves and (because this runs
|
||||
// inside withSpaceLock) the per-space lock is held + heartbeat-refreshed
|
||||
// indefinitely. Bound the request: on expiry kill the child, send a clean
|
||||
// 500 if nothing was sent yet, and settle the promise. `.unref()` so the
|
||||
// timer never keeps the event loop alive; ALWAYS cleared in done().
|
||||
watchdogTimer = setTimeout(() => {
|
||||
this.terminateChild(
|
||||
child,
|
||||
rawRes,
|
||||
headerParsed,
|
||||
'timeout',
|
||||
`git http-backend timed out after ` +
|
||||
`${this.environmentService.getGitSyncBackendTimeoutMs()}ms; killing child`,
|
||||
done,
|
||||
);
|
||||
}, this.environmentService.getGitSyncBackendTimeoutMs());
|
||||
watchdogTimer.unref?.();
|
||||
|
||||
// Accumulate stdout until we have the full CGI header block, then write the
|
||||
// parsed status/headers and start streaming the remaining body bytes.
|
||||
let headerParsed = false;
|
||||
let pending: Buffer = Buffer.alloc(0);
|
||||
|
||||
const flushHeadersAndBody = (chunk: Buffer): void => {
|
||||
pending = Buffer.concat([pending, chunk]);
|
||||
const split = splitCgiBuffer(pending);
|
||||
if (!split) return; // header block not complete yet
|
||||
headerParsed = true;
|
||||
const { statusCode, headers } = parseCgiResponse(split.headerText);
|
||||
rawRes.statusCode = statusCode;
|
||||
for (const [name, value] of headers) {
|
||||
rawRes.setHeader(name, value);
|
||||
}
|
||||
if (split.body.length > 0) rawRes.write(split.body);
|
||||
pending = Buffer.alloc(0);
|
||||
};
|
||||
|
||||
child.stdout?.on('data', (chunk: Buffer) => {
|
||||
if (headerParsed) {
|
||||
rawRes.write(chunk);
|
||||
} else {
|
||||
flushHeadersAndBody(chunk);
|
||||
}
|
||||
});
|
||||
// A stream 'error' (e.g. EPIPE when the client aborts mid-response) is an
|
||||
// EventEmitter 'error' with no listener -> Node rethrows it as an uncaught
|
||||
// exception and crashes the process. Swallow + log it (never echo to the
|
||||
// client); child.on('close')/'error' below drives the actual cleanup.
|
||||
child.stdout?.on('error', (err) => {
|
||||
this.logger.warn(`git http-backend stdout stream error: ${err.message}`);
|
||||
});
|
||||
|
||||
let stderr = '';
|
||||
child.stderr?.on('data', (chunk: Buffer) => {
|
||||
// Capture for diagnostics; never echo to the client. http-backend writes
|
||||
// CGI errors here. We do NOT log the request body or any credentials.
|
||||
if (stderr.length < 8192) stderr += chunk.toString('utf8');
|
||||
});
|
||||
child.stderr?.on('error', (err) => {
|
||||
this.logger.warn(`git http-backend stderr stream error: ${err.message}`);
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
// The watchdog timer is cleared centrally in done().
|
||||
if (!headerParsed && !rawRes.headersSent) {
|
||||
this.send500(rawRes, 'child-error', err);
|
||||
} else {
|
||||
// Output already started — we can only terminate the stream.
|
||||
try {
|
||||
rawRes.end();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
done();
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
// The watchdog timer is cleared centrally in done().
|
||||
if (!headerParsed && !rawRes.headersSent) {
|
||||
// The child exited before emitting a complete CGI header block.
|
||||
this.logger.error(
|
||||
`git http-backend produced no valid response (exit ${code}) for ` +
|
||||
`space; stderr: ${stderr.trim().slice(0, 500)}`,
|
||||
);
|
||||
this.send500(rawRes, 'no-output');
|
||||
} else {
|
||||
try {
|
||||
rawRes.end();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
done();
|
||||
});
|
||||
|
||||
// Pipe the request body to the child's stdin. For GET there is no body, so
|
||||
// end stdin immediately. We pipe `rawReq` (the raw Node stream) directly so
|
||||
// large pushes are streamed, not buffered.
|
||||
if (parsed.method === 'POST') {
|
||||
rawReq.pipe(child.stdin!);
|
||||
rawReq.on('error', () => {
|
||||
try {
|
||||
child.stdin?.end();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
});
|
||||
} else {
|
||||
child.stdin?.end();
|
||||
}
|
||||
// Swallow EPIPE etc. on the child's stdin so a client disconnect does not
|
||||
// crash the process.
|
||||
child.stdin?.on('error', () => {
|
||||
/* ignore broken-pipe on stdin */
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Kill the child (SIGTERM, then SIGKILL after a grace period if it ignores the
|
||||
* term) and finish the HTTP response cleanly, then settle. Shared by the two
|
||||
* forced-termination paths — the watchdog timeout and the lost-lock abort —
|
||||
* which differ ONLY by the log line and the send500 `reason`. If no response
|
||||
* has started a clean 500 is sent; otherwise the in-flight stream is just
|
||||
* ended. Never throws (a thrown kill/end would crash the request).
|
||||
*/
|
||||
private terminateChild(
|
||||
child: ReturnType<typeof spawn>,
|
||||
rawRes: ServerResponse,
|
||||
responseStarted: boolean,
|
||||
send500Reason: string,
|
||||
logMessage: string,
|
||||
done: () => void,
|
||||
): void {
|
||||
this.logger.warn(logMessage);
|
||||
try {
|
||||
child.kill('SIGTERM');
|
||||
// Escalate to SIGKILL shortly after in case SIGTERM is ignored.
|
||||
const sigkill = setTimeout(() => {
|
||||
try {
|
||||
child.kill('SIGKILL');
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}, 2000);
|
||||
sigkill.unref?.();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
if (!responseStarted && !rawRes.headersSent) {
|
||||
this.send500(rawRes, send500Reason);
|
||||
} else {
|
||||
try {
|
||||
rawRes.end();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
done();
|
||||
}
|
||||
|
||||
/** Send a clean 500 without leaking credentials or the request body. */
|
||||
private send500(rawRes: ServerResponse, reason: string, err?: unknown): void {
|
||||
const message = err instanceof Error ? err.message : undefined;
|
||||
this.logger.error(
|
||||
`git http-backend failed (${reason})${message ? `: ${message}` : ''}`,
|
||||
);
|
||||
try {
|
||||
if (!rawRes.headersSent) {
|
||||
rawRes.statusCode = 500;
|
||||
rawRes.setHeader('Content-Type', 'text/plain');
|
||||
}
|
||||
rawRes.end('Internal server error');
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,211 @@
|
||||
// Unit tests for the pure /git smart-HTTP helpers: URL parsing, service->kind
|
||||
// mapping (read vs write), and the gating/auth decision precedence.
|
||||
import {
|
||||
decideGitHttpGate,
|
||||
parseGitPath,
|
||||
resolveServiceKind,
|
||||
} from './git-http.helpers';
|
||||
|
||||
describe('parseGitPath', () => {
|
||||
it('parses spaceId + subpath, stripping the trailing .git', () => {
|
||||
expect(parseGitPath('abc123.git/info/refs')).toEqual({
|
||||
spaceId: 'abc123',
|
||||
subpath: 'info/refs',
|
||||
});
|
||||
});
|
||||
|
||||
it('tolerates a leading slash', () => {
|
||||
expect(parseGitPath('/abc.git/git-receive-pack')).toEqual({
|
||||
spaceId: 'abc',
|
||||
subpath: 'git-receive-pack',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns an empty subpath for the bare repo root', () => {
|
||||
expect(parseGitPath('abc.git')).toEqual({ spaceId: 'abc', subpath: '' });
|
||||
});
|
||||
|
||||
it('returns null when the first segment lacks .git', () => {
|
||||
expect(parseGitPath('abc/info/refs')).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null on an empty space id', () => {
|
||||
expect(parseGitPath('.git/info/refs')).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects path traversal', () => {
|
||||
expect(parseGitPath('abc.git/../../etc/passwd')).toBeNull();
|
||||
expect(parseGitPath('..git/x')).toBeNull();
|
||||
});
|
||||
|
||||
it('rejects percent-encoded dot/slash traversal in the subpath (case-insensitive)', () => {
|
||||
expect(parseGitPath('abc.git/%2e%2e%2fetc/passwd')).toBeNull();
|
||||
expect(parseGitPath('abc.git/%2E%2E/secret')).toBeNull();
|
||||
expect(parseGitPath('abc.git/objects/%2fabsolute')).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveServiceKind', () => {
|
||||
it('GET info/refs?service=git-upload-pack -> read', () => {
|
||||
expect(
|
||||
resolveServiceKind({
|
||||
method: 'GET',
|
||||
subpath: 'info/refs',
|
||||
service: 'git-upload-pack',
|
||||
}),
|
||||
).toBe('read');
|
||||
});
|
||||
|
||||
it('GET info/refs?service=git-receive-pack -> write', () => {
|
||||
expect(
|
||||
resolveServiceKind({
|
||||
method: 'GET',
|
||||
subpath: 'info/refs',
|
||||
service: 'git-receive-pack',
|
||||
}),
|
||||
).toBe('write');
|
||||
});
|
||||
|
||||
it('POST git-upload-pack -> read', () => {
|
||||
expect(
|
||||
resolveServiceKind({ method: 'POST', subpath: 'git-upload-pack' }),
|
||||
).toBe('read');
|
||||
});
|
||||
|
||||
it('POST git-receive-pack -> write', () => {
|
||||
expect(
|
||||
resolveServiceKind({ method: 'POST', subpath: 'git-receive-pack' }),
|
||||
).toBe('write');
|
||||
});
|
||||
|
||||
it('a dumb-protocol GET (HEAD / objects) -> read', () => {
|
||||
expect(resolveServiceKind({ method: 'GET', subpath: 'HEAD' })).toBe('read');
|
||||
expect(
|
||||
resolveServiceKind({ method: 'GET', subpath: 'objects/12/abcdef' }),
|
||||
).toBe('read');
|
||||
});
|
||||
|
||||
it('info/refs with no/unknown service -> read (dumb discovery)', () => {
|
||||
expect(resolveServiceKind({ method: 'GET', subpath: 'info/refs' })).toBe(
|
||||
'read',
|
||||
);
|
||||
});
|
||||
|
||||
it('an unknown POST endpoint -> null', () => {
|
||||
expect(resolveServiceKind({ method: 'POST', subpath: 'whatever' })).toBeNull();
|
||||
});
|
||||
|
||||
it('an unsupported method -> null', () => {
|
||||
expect(
|
||||
resolveServiceKind({ method: 'DELETE', subpath: 'git-receive-pack' }),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('decideGitHttpGate', () => {
|
||||
const base = {
|
||||
hasCredentials: true,
|
||||
credentialsValid: true,
|
||||
serviceKind: 'read' as const,
|
||||
gitSyncEnabled: true,
|
||||
gitHttpEnabled: true,
|
||||
spaceExists: true,
|
||||
spaceGitSyncEnabled: true,
|
||||
userIsSpaceMember: true,
|
||||
permissionGranted: true,
|
||||
};
|
||||
|
||||
it('proceeds on the happy path', () => {
|
||||
expect(decideGitHttpGate(base)).toEqual({ kind: 'proceed' });
|
||||
});
|
||||
|
||||
it('401 when credentials are missing (even for a valid space)', () => {
|
||||
expect(
|
||||
decideGitHttpGate({ ...base, hasCredentials: false }),
|
||||
).toEqual({ kind: 'unauthorized' });
|
||||
});
|
||||
|
||||
it('401 when credentials are present but invalid', () => {
|
||||
expect(
|
||||
decideGitHttpGate({ ...base, credentialsValid: false }),
|
||||
).toEqual({ kind: 'unauthorized' });
|
||||
});
|
||||
|
||||
it('400 on an unparseable service kind', () => {
|
||||
expect(decideGitHttpGate({ ...base, serviceKind: null })).toEqual({
|
||||
kind: 'bad-request',
|
||||
});
|
||||
});
|
||||
|
||||
it('404 when the space is not git-sync-enabled (never reveals existence)', () => {
|
||||
expect(
|
||||
decideGitHttpGate({ ...base, spaceGitSyncEnabled: false }),
|
||||
).toEqual({ kind: 'not-found' });
|
||||
});
|
||||
|
||||
it('404 when the space does not exist', () => {
|
||||
expect(decideGitHttpGate({ ...base, spaceExists: false })).toEqual({
|
||||
kind: 'not-found',
|
||||
});
|
||||
});
|
||||
|
||||
it('404 when git-sync is globally disabled', () => {
|
||||
expect(decideGitHttpGate({ ...base, gitSyncEnabled: false })).toEqual({
|
||||
kind: 'not-found',
|
||||
});
|
||||
});
|
||||
|
||||
it('404 when the git-http host is disabled', () => {
|
||||
expect(decideGitHttpGate({ ...base, gitHttpEnabled: false })).toEqual({
|
||||
kind: 'not-found',
|
||||
});
|
||||
});
|
||||
|
||||
it('403 when a MEMBER lacks the required permission (reader on write)', () => {
|
||||
// A member of the space (existence already known to them) who lacks the role:
|
||||
// 403 leaks nothing new.
|
||||
expect(
|
||||
decideGitHttpGate({
|
||||
...base,
|
||||
serviceKind: 'write',
|
||||
userIsSpaceMember: true,
|
||||
permissionGranted: false,
|
||||
}),
|
||||
).toEqual({ kind: 'forbidden' });
|
||||
});
|
||||
|
||||
it('404 (NOT 403) when an authenticated NON-member hits a git-sync space', () => {
|
||||
// SECURITY: a non-member must be indistinguishable from a missing/disabled
|
||||
// space. If this returned 403, the 403↔404 difference would let any
|
||||
// authenticated workspace user brute-force slugs to discover which spaces
|
||||
// exist and which have git-sync enabled.
|
||||
expect(
|
||||
decideGitHttpGate({
|
||||
...base,
|
||||
serviceKind: 'write',
|
||||
userIsSpaceMember: false,
|
||||
permissionGranted: false,
|
||||
}),
|
||||
).toEqual({ kind: 'not-found' });
|
||||
// Same for a read by a non-member.
|
||||
expect(
|
||||
decideGitHttpGate({
|
||||
...base,
|
||||
serviceKind: 'read',
|
||||
userIsSpaceMember: false,
|
||||
permissionGranted: false,
|
||||
}),
|
||||
).toEqual({ kind: 'not-found' });
|
||||
});
|
||||
|
||||
it('still 401 (not 404) for missing creds against a disabled space', () => {
|
||||
// Anonymous probe must always get 401 first, regardless of space state.
|
||||
expect(
|
||||
decideGitHttpGate({
|
||||
...base,
|
||||
hasCredentials: false,
|
||||
spaceGitSyncEnabled: false,
|
||||
}),
|
||||
).toEqual({ kind: 'unauthorized' });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,164 @@
|
||||
// Pure, framework-free helpers for the /git smart-HTTP host. They carry no Nest
|
||||
// / DI / concrete-service imports so the request parsing and the auth/authz
|
||||
// gating DECISION can be unit-tested in isolation, and nothing here ever logs a
|
||||
// password or the Authorization header.
|
||||
|
||||
/** The git operation a request maps to: a read (fetch/clone) or a write (push). */
|
||||
export type GitHttpServiceKind = 'read' | 'write';
|
||||
|
||||
/** A parsed `/git/<spaceId>.git/<subpath>` URL. */
|
||||
export interface ParsedGitPath {
|
||||
spaceId: string;
|
||||
/** The subpath after `<spaceId>.git/` (no leading slash), e.g. `info/refs`. */
|
||||
subpath: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the `<rest>` of a `/git/<rest>` URL path (no query string) into the
|
||||
* space id and the repo-relative subpath. The space id is the first path
|
||||
* segment with its trailing `.git` stripped. Returns null when the shape does
|
||||
* not match (missing `.git`, empty space id, traversal attempt).
|
||||
*
|
||||
* `rest` MUST already be URL-path-decoded of its query string by the caller
|
||||
* (pass the pathname only). We reject `..` segments defensively even though
|
||||
* http-backend resolves PATH_INFO against GIT_PROJECT_ROOT.
|
||||
*/
|
||||
export function parseGitPath(rest: string): ParsedGitPath | null {
|
||||
// Strip a leading slash, then take the first segment as `<spaceId>.git`.
|
||||
const clean = rest.replace(/^\/+/, '');
|
||||
const slash = clean.indexOf('/');
|
||||
const first = slash === -1 ? clean : clean.slice(0, slash);
|
||||
const subpath = slash === -1 ? '' : clean.slice(slash + 1);
|
||||
|
||||
if (!first.endsWith('.git')) return null;
|
||||
const spaceId = first.slice(0, -'.git'.length);
|
||||
if (!spaceId) return null;
|
||||
|
||||
// Reject path traversal / degenerate ids in either component.
|
||||
if (
|
||||
spaceId === '.' ||
|
||||
spaceId.includes('..') ||
|
||||
spaceId.includes('/') ||
|
||||
subpath.split('/').some((seg) => seg === '..')
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Defense-in-depth: reject percent-encoded dot/slash traversal (`%2e`, `%2f`,
|
||||
// case-insensitive) in the subpath BEFORE it is used to build PATH_INFO — a
|
||||
// decoder downstream could otherwise turn `%2e%2e%2f` back into `../`.
|
||||
if (/%2e|%2f/i.test(subpath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return { spaceId, subpath };
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a parsed git request (method + subpath + query) to the required operation
|
||||
* kind. The smart-HTTP shapes:
|
||||
* - GET info/refs?service=git-upload-pack -> read (fetch)
|
||||
* - GET info/refs?service=git-receive-pack -> write (push)
|
||||
* - POST git-upload-pack -> read (fetch)
|
||||
* - POST git-receive-pack -> write (push)
|
||||
* - any other dumb-protocol GET (HEAD, objects/…) -> read
|
||||
* Returns null for an unsupported shape (e.g. a POST that is neither pack
|
||||
* endpoint) so the caller can 403/404 rather than guess.
|
||||
*/
|
||||
export function resolveServiceKind(input: {
|
||||
method: string;
|
||||
subpath: string;
|
||||
service?: string;
|
||||
}): GitHttpServiceKind | null {
|
||||
const method = input.method.toUpperCase();
|
||||
const subpath = input.subpath;
|
||||
|
||||
if (method === 'GET') {
|
||||
if (subpath === 'info/refs') {
|
||||
if (input.service === 'git-receive-pack') return 'write';
|
||||
if (input.service === 'git-upload-pack') return 'read';
|
||||
// info/refs without a known service: dumb-protocol discovery — read.
|
||||
return 'read';
|
||||
}
|
||||
// Dumb-protocol object/ref fetches (HEAD, objects/…) are reads.
|
||||
return 'read';
|
||||
}
|
||||
|
||||
if (method === 'POST') {
|
||||
if (subpath === 'git-receive-pack') return 'write';
|
||||
if (subpath === 'git-upload-pack') return 'read';
|
||||
return null; // unknown POST endpoint
|
||||
}
|
||||
|
||||
return null; // unsupported method
|
||||
}
|
||||
|
||||
/** The outcome of the gating/auth decision the request handler must enforce. */
|
||||
export type GitHttpGateDecision =
|
||||
| { kind: 'unauthorized' } // 401 + WWW-Authenticate (missing/invalid creds)
|
||||
| { kind: 'not-found' } // 404 (space hidden / sync or http disabled)
|
||||
| { kind: 'forbidden' } // 403 (authenticated but lacks the permission)
|
||||
| { kind: 'bad-request' } // 400 (unparseable git request shape)
|
||||
| { kind: 'proceed' }; // run http-backend
|
||||
|
||||
/**
|
||||
* Pure gating decision, mirroring the handler precedence so it can be unit
|
||||
* tested without the DB / CASL graph. Inputs are the already-resolved booleans
|
||||
* the handler computes from EnvironmentService / SpaceRepo / SpaceAbilityFactory.
|
||||
*
|
||||
* Precedence (matches the spec):
|
||||
* 1. no/invalid Basic credentials -> 401 (regardless of space).
|
||||
* 2. credentials present but invalid -> 401.
|
||||
* 3. unparseable git request shape -> 400.
|
||||
* 4. git-sync globally disabled, or git-http disabled, or the space is missing
|
||||
* / not git-sync-enabled, OR the authenticated user is NOT a member of the
|
||||
* space (has no role at all) -> 404 (never reveal existence).
|
||||
* 5. a MEMBER of the space who lacks the required perm (e.g. a reader trying to
|
||||
* push) -> 403.
|
||||
* 6. otherwise -> proceed.
|
||||
*
|
||||
* Note (4) is checked AFTER (1)/(2): an anonymous probe always gets 401 first;
|
||||
* an authenticated user hitting a hidden/disabled space — OR a space they are not
|
||||
* a member of — gets 404 (not 403). Folding non-membership into the 404 branch is
|
||||
* a SECURITY requirement: if a non-member got 403 here (as a "permission denied")
|
||||
* while a non-existent / sync-disabled space got 404, the 403↔404 difference would
|
||||
* let any authenticated workspace user brute-force slugs to discover which spaces
|
||||
* exist and which have git-sync enabled — including spaces they cannot see. 403 is
|
||||
* therefore reserved for the one case where existence is ALREADY known to the
|
||||
* caller because they ARE a member (so it leaks nothing new): a member without the
|
||||
* required role. `userIsSpaceMember` is the resolved "the user has SOME role in
|
||||
* this space" boolean (false when SpaceAbilityFactory.createForUser throws
|
||||
* NotFound / the user has no role).
|
||||
*/
|
||||
export function decideGitHttpGate(input: {
|
||||
hasCredentials: boolean;
|
||||
credentialsValid: boolean;
|
||||
serviceKind: GitHttpServiceKind | null;
|
||||
gitSyncEnabled: boolean;
|
||||
gitHttpEnabled: boolean;
|
||||
spaceExists: boolean;
|
||||
spaceGitSyncEnabled: boolean;
|
||||
/** The user has SOME role in the space (false = non-member -> 404, not 403). */
|
||||
userIsSpaceMember: boolean;
|
||||
permissionGranted: boolean;
|
||||
}): GitHttpGateDecision {
|
||||
if (!input.hasCredentials) return { kind: 'unauthorized' };
|
||||
if (!input.credentialsValid) return { kind: 'unauthorized' };
|
||||
if (input.serviceKind === null) return { kind: 'bad-request' };
|
||||
|
||||
if (
|
||||
!input.gitSyncEnabled ||
|
||||
!input.gitHttpEnabled ||
|
||||
!input.spaceExists ||
|
||||
!input.spaceGitSyncEnabled ||
|
||||
// A non-member must be indistinguishable from a missing/disabled space: 404,
|
||||
// never 403 (otherwise the 403↔404 split leaks space existence — see above).
|
||||
!input.userIsSpaceMember
|
||||
) {
|
||||
return { kind: 'not-found' };
|
||||
}
|
||||
|
||||
if (!input.permissionGranted) return { kind: 'forbidden' };
|
||||
|
||||
return { kind: 'proceed' };
|
||||
}
|
||||
@@ -0,0 +1,643 @@
|
||||
// Unit tests for GitHttpService — the /git smart-HTTP handler. Everything it
|
||||
// depends on (backend, auth, repos, ability factory, env, orchestrator) is
|
||||
// mocked so we exercise ONLY the handler wiring: workspace resolution (which is
|
||||
// done HERE, not by DomainMiddleware — see FIX 1), the auth/gating precedence,
|
||||
// the read-vs-write dispatch, and that a fetch does NOT take the lock.
|
||||
//
|
||||
// These tests deliberately NEVER set `req.raw.workspaceId`: the workspace must
|
||||
// come from WorkspaceRepo. If the handler regressed to reading
|
||||
// `req.raw.workspaceId`, the happy-path fetch test below would fail (the repo
|
||||
// would not be consulted and the request would 401).
|
||||
import {
|
||||
Logger,
|
||||
NotFoundException,
|
||||
UnauthorizedException,
|
||||
} from '@nestjs/common';
|
||||
import { CREDENTIALS_MISMATCH_MESSAGE } from '../../../core/auth/auth.constants';
|
||||
import {
|
||||
SpaceCaslAction,
|
||||
SpaceCaslSubject,
|
||||
} from '../../../core/casl/interfaces/space-ability.type';
|
||||
import { GitHttpService } from './git-http.service';
|
||||
import { GitSyncLockHeldError } from '../services/git-sync.orchestrator';
|
||||
|
||||
type AnyMock = jest.Mock;
|
||||
|
||||
interface BuildOptions {
|
||||
selfHosted?: boolean;
|
||||
gitSyncEnabled?: boolean;
|
||||
gitHttpEnabled?: boolean;
|
||||
/** What workspaceRepo.findFirst() returns (self-hosted resolution). */
|
||||
workspace?: { id: string } | null;
|
||||
/** What spaceRepo.findById() returns. */
|
||||
space?: { id: string; settings?: unknown } | null;
|
||||
/** Result of authService.verifyUserCredentials: a user, or throw 401. */
|
||||
user?: { id: string; email: string } | null;
|
||||
/** Whether the created ability grants the requested action. */
|
||||
abilityCan?: boolean;
|
||||
}
|
||||
|
||||
interface Built {
|
||||
service: GitHttpService;
|
||||
env: Record<string, AnyMock>;
|
||||
authService: { verifyUserCredentials: AnyMock };
|
||||
spaceRepo: { findById: AnyMock };
|
||||
workspaceRepo: { findFirst: AnyMock; findByHostname: AnyMock };
|
||||
abilityFactory: { createForUser: AnyMock };
|
||||
abilityCan: AnyMock;
|
||||
vaultRegistry: { ensureServable: AnyMock };
|
||||
orchestrator: {
|
||||
ingestExternalPush: AnyMock;
|
||||
serveReadAdvertisement: AnyMock;
|
||||
};
|
||||
backend: { run: AnyMock };
|
||||
}
|
||||
|
||||
function build(opts: BuildOptions = {}): Built {
|
||||
const {
|
||||
selfHosted = true,
|
||||
gitSyncEnabled = true,
|
||||
gitHttpEnabled = true,
|
||||
workspace = { id: 'ws-1' },
|
||||
space = { id: 'space-1', settings: { gitSync: { enabled: true } } },
|
||||
user = { id: 'user-1', email: 'dev@example.com' },
|
||||
abilityCan = true,
|
||||
} = opts;
|
||||
|
||||
const env: Record<string, AnyMock> = {
|
||||
isSelfHosted: jest.fn(() => selfHosted),
|
||||
isCloud: jest.fn(() => !selfHosted),
|
||||
isGitSyncEnabled: jest.fn(() => gitSyncEnabled),
|
||||
isGitSyncHttpEnabled: jest.fn(() => gitHttpEnabled),
|
||||
};
|
||||
|
||||
const authService = {
|
||||
verifyUserCredentials: jest.fn(async () => {
|
||||
if (!user) throw new UnauthorizedException();
|
||||
return user;
|
||||
}),
|
||||
};
|
||||
|
||||
const spaceRepo = { findById: jest.fn(async () => space) };
|
||||
|
||||
const workspaceRepo = {
|
||||
findFirst: jest.fn(async () => workspace),
|
||||
findByHostname: jest.fn(async () => workspace),
|
||||
};
|
||||
|
||||
const abilityCanMock = jest.fn(() => abilityCan);
|
||||
const abilityFactory = {
|
||||
createForUser: jest.fn(async () => ({ can: abilityCanMock })),
|
||||
};
|
||||
|
||||
const vaultRegistry = { ensureServable: jest.fn(async () => undefined) };
|
||||
const orchestrator = {
|
||||
ingestExternalPush: jest.fn(async () => undefined),
|
||||
// The read-advertisement wrapper pins HEAD under the lock then serves; the
|
||||
// mock just runs the serve callback so the read path still hits backend.run.
|
||||
serveReadAdvertisement: jest.fn(
|
||||
async (_spaceId: string, serve: () => Promise<void>) => serve(),
|
||||
),
|
||||
};
|
||||
const backend = { run: jest.fn(async () => undefined) };
|
||||
|
||||
const service = new GitHttpService(
|
||||
env as any,
|
||||
authService as any,
|
||||
spaceRepo as any,
|
||||
workspaceRepo as any,
|
||||
abilityFactory as any,
|
||||
vaultRegistry as any,
|
||||
orchestrator as any,
|
||||
backend as any,
|
||||
);
|
||||
|
||||
return {
|
||||
service,
|
||||
env,
|
||||
authService,
|
||||
spaceRepo,
|
||||
workspaceRepo,
|
||||
abilityFactory,
|
||||
abilityCan: abilityCanMock,
|
||||
vaultRegistry,
|
||||
orchestrator,
|
||||
backend,
|
||||
};
|
||||
}
|
||||
|
||||
/** A fake Fastify reply capturing the terminal status/headers/body. */
|
||||
function fakeReply() {
|
||||
const state: {
|
||||
statusCode?: number;
|
||||
headers: Record<string, string>;
|
||||
body?: unknown;
|
||||
hijacked: boolean;
|
||||
sent: boolean;
|
||||
} = { headers: {}, hijacked: false, sent: false };
|
||||
|
||||
const reply: any = {
|
||||
header(name: string, value: string) {
|
||||
state.headers[name] = value;
|
||||
return reply;
|
||||
},
|
||||
status(code: number) {
|
||||
state.statusCode = code;
|
||||
return reply;
|
||||
},
|
||||
send(body: unknown) {
|
||||
state.body = body;
|
||||
state.sent = true;
|
||||
return reply;
|
||||
},
|
||||
hijack() {
|
||||
state.hijacked = true;
|
||||
},
|
||||
get sent() {
|
||||
return state.sent;
|
||||
},
|
||||
// The raw Node response — only touched on the streaming/error paths.
|
||||
raw: {
|
||||
headersSent: false,
|
||||
writableEnded: false,
|
||||
statusCode: 200,
|
||||
setHeader: jest.fn(),
|
||||
end: jest.fn(),
|
||||
},
|
||||
};
|
||||
return { reply, state };
|
||||
}
|
||||
|
||||
/** A fake Fastify request for a /git smart-HTTP call. */
|
||||
function fakeRequest(opts: {
|
||||
url: string;
|
||||
method?: string;
|
||||
authorization?: string;
|
||||
host?: string;
|
||||
}) {
|
||||
const { url, method = 'GET', authorization, host = 'docs.example.com' } = opts;
|
||||
const headers: Record<string, string> = { host };
|
||||
if (authorization) headers['authorization'] = authorization;
|
||||
// query is parsed by Fastify; mirror the `service` param when present.
|
||||
const qIdx = url.indexOf('?');
|
||||
const query: Record<string, string> = {};
|
||||
if (qIdx !== -1) {
|
||||
for (const pair of url.slice(qIdx + 1).split('&')) {
|
||||
const [k, v] = pair.split('=');
|
||||
if (k) query[k] = v ?? '';
|
||||
}
|
||||
}
|
||||
return {
|
||||
url,
|
||||
method,
|
||||
headers,
|
||||
query,
|
||||
// raw is intentionally WITHOUT workspaceId — the handler must resolve it
|
||||
// itself via WorkspaceRepo (a regression to req.raw.workspaceId would 401).
|
||||
raw: {},
|
||||
} as any;
|
||||
}
|
||||
|
||||
function basic(email: string, password: string): string {
|
||||
return 'Basic ' + Buffer.from(`${email}:${password}`).toString('base64');
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
// Silence the handler's logger.warn/error in negative-path tests.
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
|
||||
});
|
||||
|
||||
describe('GitHttpService.handle', () => {
|
||||
it('fetch with valid creds resolves the workspace via the repo and dispatches WITHOUT the lock', async () => {
|
||||
const built = build({ selfHosted: true });
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
// The workspace came from WorkspaceRepo, NOT req.raw.workspaceId.
|
||||
expect(built.workspaceRepo.findFirst).toHaveBeenCalledTimes(1);
|
||||
expect(built.authService.verifyUserCredentials).toHaveBeenCalledWith(
|
||||
{ email: 'dev@example.com', password: 'pw' },
|
||||
'ws-1',
|
||||
);
|
||||
expect(built.spaceRepo.findById).toHaveBeenCalledWith('space-1', 'ws-1');
|
||||
// Read ability was evaluated.
|
||||
expect(built.abilityCan).toHaveBeenCalledWith(
|
||||
SpaceCaslAction.Read,
|
||||
SpaceCaslSubject.Page,
|
||||
);
|
||||
// It proceeded: vault prepared, reply hijacked, backend ran directly.
|
||||
expect(built.vaultRegistry.ensureServable).toHaveBeenCalledWith('space-1');
|
||||
expect(state.hijacked).toBe(true);
|
||||
expect(built.backend.run).toHaveBeenCalledTimes(1);
|
||||
// A fetch must NOT take the push lock.
|
||||
expect(built.orchestrator.ingestExternalPush).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('upload-pack ref advertisement is served HEAD-pinned via serveReadAdvertisement (bug #3)', async () => {
|
||||
// GET info/refs?service=git-upload-pack carries the HEAD symref a clone reads
|
||||
// for its default branch, so it must be served with HEAD pinned to `main`
|
||||
// (under the lock) — not streamed raw — or a clone racing a mid-pull cycle
|
||||
// would default to the read-only `docmost` mirror.
|
||||
const built = build({ abilityCan: true });
|
||||
const { reply } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(built.orchestrator.serveReadAdvertisement).toHaveBeenCalledTimes(1);
|
||||
expect(built.orchestrator.serveReadAdvertisement.mock.calls[0][0]).toBe(
|
||||
'space-1',
|
||||
);
|
||||
// The wrapper still streams the backend (the mock runs the serve callback).
|
||||
expect(built.backend.run).toHaveBeenCalledTimes(1);
|
||||
expect(built.orchestrator.ingestExternalPush).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a POST git-upload-pack pack fetch streams directly (no HEAD-pin needed, resolved by SHA)', async () => {
|
||||
// The pack negotiation is object-SHA based; only the ref advertisement carries
|
||||
// the HEAD symref, so the pack POST streams the backend directly (no lock).
|
||||
const built = build({ abilityCan: true });
|
||||
const { reply } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/git-upload-pack',
|
||||
method: 'POST',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(built.orchestrator.serveReadAdvertisement).not.toHaveBeenCalled();
|
||||
expect(built.backend.run).toHaveBeenCalledTimes(1);
|
||||
expect(built.orchestrator.ingestExternalPush).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('cloud deployment resolves the workspace by the host subdomain', async () => {
|
||||
const built = build({ selfHosted: false });
|
||||
const { reply } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
host: 'acme.example.com',
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(built.workspaceRepo.findByHostname).toHaveBeenCalledWith('acme');
|
||||
expect(built.workspaceRepo.findFirst).not.toHaveBeenCalled();
|
||||
expect(built.backend.run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('missing Basic credentials -> 401 with WWW-Authenticate', async () => {
|
||||
const built = build();
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
// no Authorization header
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(state.statusCode).toBe(401);
|
||||
expect(state.headers['WWW-Authenticate']).toBe('Basic realm="gitmost"');
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
expect(built.authService.verifyUserCredentials).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('invalid Basic credentials -> 401 with WWW-Authenticate', async () => {
|
||||
const built = build({ user: null }); // verifyUserCredentials throws 401
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'wrong'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(state.statusCode).toBe(401);
|
||||
expect(state.headers['WWW-Authenticate']).toBe('Basic realm="gitmost"');
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a write by a Read-only user -> 403 (reader cannot push)', async () => {
|
||||
const built = build({ abilityCan: false });
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/git-receive-pack',
|
||||
method: 'POST',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
// The Manage ability was checked for a write and denied.
|
||||
expect(built.abilityCan).toHaveBeenCalledWith(
|
||||
SpaceCaslAction.Manage,
|
||||
SpaceCaslSubject.Page,
|
||||
);
|
||||
expect(state.statusCode).toBe(403);
|
||||
expect(built.orchestrator.ingestExternalPush).not.toHaveBeenCalled();
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('an authenticated NON-member of a git-sync space -> 404, NOT 403 (no existence leak)', async () => {
|
||||
// createForUser throws NotFound when the user holds no role in the space (a
|
||||
// non-member). The gate must return 404 — the SAME response a missing /
|
||||
// sync-disabled space gives — so a 403↔404 difference cannot be used to
|
||||
// brute-force which spaces exist / have git-sync enabled (the security fix).
|
||||
const built = build({ abilityCan: false });
|
||||
built.abilityFactory.createForUser.mockRejectedValue(
|
||||
new NotFoundException('Space permissions not found'),
|
||||
);
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/secret-space.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(built.abilityFactory.createForUser).toHaveBeenCalledTimes(1);
|
||||
expect(state.statusCode).toBe(404);
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
expect(built.orchestrator.ingestExternalPush).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a space that is not git-sync-enabled -> 404 (existence never revealed)', async () => {
|
||||
const built = build({
|
||||
space: { id: 'space-1', settings: { gitSync: { enabled: false } } },
|
||||
});
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(state.statusCode).toBe(404);
|
||||
// CASL is never even evaluated for a non-candidate space.
|
||||
expect(built.abilityFactory.createForUser).not.toHaveBeenCalled();
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('git-sync globally disabled -> 404 even with valid creds', async () => {
|
||||
const built = build({ gitSyncEnabled: false });
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(state.statusCode).toBe(404);
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('a valid write proceeds through the orchestrator (push takes the lock)', async () => {
|
||||
const built = build({ abilityCan: true });
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/git-receive-pack',
|
||||
method: 'POST',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
expect(built.abilityCan).toHaveBeenCalledWith(
|
||||
SpaceCaslAction.Manage,
|
||||
SpaceCaslSubject.Page,
|
||||
);
|
||||
expect(state.hijacked).toBe(true);
|
||||
expect(built.orchestrator.ingestExternalPush).toHaveBeenCalledTimes(1);
|
||||
const [spaceId, workspaceId] =
|
||||
built.orchestrator.ingestExternalPush.mock.calls[0];
|
||||
expect(spaceId).toBe('space-1');
|
||||
expect(workspaceId).toBe('ws-1');
|
||||
});
|
||||
|
||||
it('GET info/refs?service=git-receive-pack streams the backend WITHOUT a cycle/lock (so the follow-up POST never 503-collides)', async () => {
|
||||
// A push is a TWO-request exchange: GET info/refs?service=git-receive-pack
|
||||
// (ref advertisement) then POST git-receive-pack (the pack). The info/refs
|
||||
// request is write-AUTHORIZED (push perms needed to see those refs) but is
|
||||
// READ-ONLY — it must NOT run ingestExternalPush (a Docmost cycle under the
|
||||
// per-space lock), or the immediately-following POST collides with the still-
|
||||
// running cycle and deterministically 503s. It must just stream the backend.
|
||||
const built = build({ abilityCan: true });
|
||||
const { reply } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-receive-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
// Authorized as a write (Manage), but executed as a plain stream.
|
||||
expect(built.abilityCan).toHaveBeenCalledWith(
|
||||
SpaceCaslAction.Manage,
|
||||
SpaceCaslSubject.Page,
|
||||
);
|
||||
expect(built.orchestrator.ingestExternalPush).not.toHaveBeenCalled();
|
||||
expect(built.backend.run).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('a push that loses the lock -> 503 with Retry-After and a busy body (headers not written twice)', async () => {
|
||||
const built = build({ abilityCan: true });
|
||||
// The lock could not be acquired: the receive-pack closure never ran, so the
|
||||
// response is still unwritten and the handler must answer 503 itself.
|
||||
built.orchestrator.ingestExternalPush.mockRejectedValue(
|
||||
new GitSyncLockHeldError('space-1'),
|
||||
);
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/git-receive-pack',
|
||||
method: 'POST',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
// It hijacked and went through the orchestrator (write path), but the lock
|
||||
// was held so the backend never ran.
|
||||
expect(state.hijacked).toBe(true);
|
||||
expect(built.orchestrator.ingestExternalPush).toHaveBeenCalledTimes(1);
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
|
||||
// 503 + Retry-After were written on the raw response (headersSent was false).
|
||||
const raw = reply.raw as any;
|
||||
expect(raw.statusCode).toBe(503);
|
||||
expect(raw.setHeader).toHaveBeenCalledWith('Content-Type', 'text/plain');
|
||||
expect(raw.setHeader).toHaveBeenCalledWith('Retry-After', '1');
|
||||
// The body carries the busy/retry message and the response was ended once.
|
||||
expect(raw.end).toHaveBeenCalledTimes(1);
|
||||
expect(raw.end).toHaveBeenCalledWith('git-sync busy, retry');
|
||||
// Exactly the two headers above were set — no double write of headers.
|
||||
expect(raw.setHeader).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does NOT rewrite the 503 status/headers when the response is already sent', async () => {
|
||||
const built = build({ abilityCan: true });
|
||||
built.orchestrator.ingestExternalPush.mockRejectedValue(
|
||||
new GitSyncLockHeldError('space-1'),
|
||||
);
|
||||
const { reply } = fakeReply();
|
||||
// Simulate the (defensive) case where headers were already flushed: the
|
||||
// handler must skip statusCode/setHeader and only end() the socket.
|
||||
const raw = reply.raw as any;
|
||||
raw.headersSent = true;
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/git-receive-pack',
|
||||
method: 'POST',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
// No header writes when headersSent is already true (no "headers already
|
||||
// sent" double-write path), but the body/end still runs.
|
||||
expect(raw.setHeader).not.toHaveBeenCalled();
|
||||
expect(raw.statusCode).toBe(200); // untouched default from the fake
|
||||
expect(raw.end).toHaveBeenCalledTimes(1);
|
||||
expect(raw.end).toHaveBeenCalledWith('git-sync busy, retry');
|
||||
});
|
||||
|
||||
it('an unresolvable workspace -> 401 (credentials cannot be validated without one)', async () => {
|
||||
const built = build({ workspace: null });
|
||||
const { reply, state } = fakeReply();
|
||||
const req = fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'pw'),
|
||||
});
|
||||
|
||||
await built.service.handle(req, reply);
|
||||
|
||||
// Without a workspace we cannot run verifyUserCredentials, so credentials
|
||||
// are not validated -> 401 (the 401-before-404 ordering is preserved: an
|
||||
// unauthenticated request never reaches the space-existence 404).
|
||||
expect(built.workspaceRepo.findFirst).toHaveBeenCalledTimes(1);
|
||||
expect(built.authService.verifyUserCredentials).not.toHaveBeenCalled();
|
||||
expect(state.statusCode).toBe(401);
|
||||
expect(state.headers['WWW-Authenticate']).toBe('Basic realm="gitmost"');
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// --- brute-force throttle (must-fix #1, mirrors the /mcp Basic limiter) -----
|
||||
describe('HTTP-Basic brute-force throttle', () => {
|
||||
/** A request with wrong credentials for the given email. */
|
||||
const wrongCredReq = (email = 'dev@example.com') =>
|
||||
fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic(email, 'wrong'),
|
||||
});
|
||||
|
||||
it('rejects the (threshold+1)-th failed attempt with 429 BEFORE bcrypt', async () => {
|
||||
const built = build();
|
||||
// Realistic credential failure: verifyUserCredentials throws the SAME
|
||||
// UnauthorizedException(CREDENTIALS_MISMATCH_MESSAGE) production throws, so
|
||||
// isCredentialsFailure matches and the reservation is KEPT (counted).
|
||||
built.authService.verifyUserCredentials.mockRejectedValue(
|
||||
new UnauthorizedException(CREDENTIALS_MISMATCH_MESSAGE),
|
||||
);
|
||||
|
||||
// 5 failed attempts (threshold = 5): each runs the credential check -> 401.
|
||||
for (let i = 0; i < 5; i++) {
|
||||
const { reply, state } = fakeReply();
|
||||
await built.service.handle(wrongCredReq(), reply);
|
||||
expect(state.statusCode).toBe(401);
|
||||
}
|
||||
expect(built.authService.verifyUserCredentials).toHaveBeenCalledTimes(5);
|
||||
|
||||
// The 6th attempt is throttled: 429, Retry-After, and bcrypt is NOT run.
|
||||
const { reply, state } = fakeReply();
|
||||
await built.service.handle(wrongCredReq(), reply);
|
||||
expect(state.statusCode).toBe(429);
|
||||
expect(state.headers['Retry-After']).toBe('60');
|
||||
expect(state.headers['WWW-Authenticate']).toBe('Basic realm="gitmost"');
|
||||
// Still 5 — the 6th never reached verifyUserCredentials (pre-bcrypt reject).
|
||||
expect(built.authService.verifyUserCredentials).toHaveBeenCalledTimes(5);
|
||||
expect(built.backend.run).not.toHaveBeenCalled();
|
||||
|
||||
built.service.onModuleDestroy();
|
||||
});
|
||||
|
||||
it('a successful auth resets the limiter so later attempts are not throttled', async () => {
|
||||
const built = build();
|
||||
const verify = built.authService.verifyUserCredentials;
|
||||
// First 4 attempts fail (credential mismatch), then one SUCCEEDS.
|
||||
verify
|
||||
.mockRejectedValueOnce(new UnauthorizedException(CREDENTIALS_MISMATCH_MESSAGE))
|
||||
.mockRejectedValueOnce(new UnauthorizedException(CREDENTIALS_MISMATCH_MESSAGE))
|
||||
.mockRejectedValueOnce(new UnauthorizedException(CREDENTIALS_MISMATCH_MESSAGE))
|
||||
.mockRejectedValueOnce(new UnauthorizedException(CREDENTIALS_MISMATCH_MESSAGE))
|
||||
.mockResolvedValueOnce({ id: 'user-1', email: 'dev@example.com' });
|
||||
|
||||
for (let i = 0; i < 4; i++) {
|
||||
const { reply } = fakeReply();
|
||||
await built.service.handle(wrongCredReq(), reply);
|
||||
}
|
||||
// 5th attempt succeeds -> proceeds (not throttled) and clears the budget.
|
||||
const okReply = fakeReply();
|
||||
await built.service.handle(
|
||||
fakeRequest({
|
||||
url: '/git/space-1.git/info/refs?service=git-upload-pack',
|
||||
method: 'GET',
|
||||
authorization: basic('dev@example.com', 'right'),
|
||||
}),
|
||||
okReply.reply,
|
||||
);
|
||||
expect(okReply.state.hijacked).toBe(true); // proceeded to the backend
|
||||
|
||||
// After the reset, a fresh wrong attempt is evaluated (401), NOT a 429 —
|
||||
// proving the per-IP/per-IP+email budget was cleared by the success.
|
||||
verify.mockRejectedValueOnce(
|
||||
new UnauthorizedException(CREDENTIALS_MISMATCH_MESSAGE),
|
||||
);
|
||||
const { reply, state } = fakeReply();
|
||||
await built.service.handle(wrongCredReq(), reply);
|
||||
expect(state.statusCode).toBe(401);
|
||||
|
||||
built.service.onModuleDestroy();
|
||||
});
|
||||
|
||||
it('a non-credential error releases the reservation (does not burn the budget)', async () => {
|
||||
const built = build();
|
||||
// A DB error (not a credentials mismatch) must NOT count toward the limiter.
|
||||
built.authService.verifyUserCredentials.mockRejectedValue(
|
||||
new Error('db down'),
|
||||
);
|
||||
|
||||
// 10 such failures — far beyond the threshold — must all be 401, never 429,
|
||||
// because each releases its reservation.
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const { reply, state } = fakeReply();
|
||||
await built.service.handle(wrongCredReq(), reply);
|
||||
expect(state.statusCode).toBe(401);
|
||||
}
|
||||
expect(built.authService.verifyUserCredentials).toHaveBeenCalledTimes(10);
|
||||
|
||||
built.service.onModuleDestroy();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,488 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
UnauthorizedException,
|
||||
} from '@nestjs/common';
|
||||
import type { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { AuthService } from '../../../core/auth/services/auth.service';
|
||||
import SpaceAbilityFactory from '../../../core/casl/abilities/space-ability.factory';
|
||||
import {
|
||||
SpaceCaslAction,
|
||||
SpaceCaslSubject,
|
||||
} from '../../../core/casl/interfaces/space-ability.type';
|
||||
import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
|
||||
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
|
||||
import { User } from '@docmost/db/types/entity.types';
|
||||
import {
|
||||
parseBasicAuth,
|
||||
FailedLoginLimiter,
|
||||
clientIp,
|
||||
isCredentialsFailure,
|
||||
} from '../../mcp/mcp-auth.helpers';
|
||||
import { resolveRequestWorkspace } from '../../../common/helpers/resolve-request-workspace';
|
||||
import { EnvironmentService } from '../../environment/environment.service';
|
||||
import { VaultRegistryService } from '../services/vault-registry.service';
|
||||
import {
|
||||
GitSyncLockHeldError,
|
||||
GitSyncOrchestrator,
|
||||
} from '../services/git-sync.orchestrator';
|
||||
import { GitHttpBackendService } from './git-http-backend.service';
|
||||
import {
|
||||
decideGitHttpGate,
|
||||
parseGitPath,
|
||||
resolveServiceKind,
|
||||
GitHttpServiceKind,
|
||||
} from './git-http.helpers';
|
||||
|
||||
const WWW_AUTHENTICATE = 'Basic realm="gitmost"';
|
||||
|
||||
/**
|
||||
* The /git smart-HTTP host. Wires request parsing, the reused auth primitives
|
||||
* (HTTP Basic -> AuthService.verifyUserCredentials), per-space gating
|
||||
* (EnvironmentService flags + space.settings.gitSync.enabled), CASL authz
|
||||
* (SpaceAbilityFactory), and dispatch to `git http-backend`:
|
||||
* - fetch (read) -> ensureServable then stream http-backend directly (no lock).
|
||||
* - push (write) -> ensureServable then orchestrator.ingestExternalPush, which
|
||||
* runs the receive-pack under the space lock and then a Docmost cycle.
|
||||
*
|
||||
* Mounted at the ROOT (`/git/...`) by a raw Fastify route in main.ts (the global
|
||||
* `/api` prefix does not apply). Never logs the password or Authorization header.
|
||||
*/
|
||||
@Injectable()
|
||||
export class GitHttpService implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(GitHttpService.name);
|
||||
|
||||
/**
|
||||
* In-process brute-force speed bump for the /git HTTP-Basic path. The raw
|
||||
* `/git/*` Fastify route bypasses the Nest pipeline (so ThrottlerGuard, which is
|
||||
* only on controllers, never runs) and there is no fastify rate-limit plugin, so
|
||||
* without this `verifyUserCredentials` (bcrypt) would run unthrottled on every
|
||||
* request once GIT_SYNC_HTTP_ENABLED is on. Mirrors the /mcp Basic path EXACTLY
|
||||
* (FailedLoginLimiter, same 5/60s thresholds, the same per-IP / per-IP+email /
|
||||
* global-per-email keys) so the two auth seams cannot diverge. A speed bump, not
|
||||
* a hard boundary (in-process, per replica).
|
||||
*/
|
||||
private readonly failedLogins = new FailedLoginLimiter(5, 60_000);
|
||||
/** Periodic sweep to bound limiter memory (mirrors McpService / mcp http.ts). */
|
||||
private readonly sweepIntervalMs = 60_000;
|
||||
private readonly sweepTimer: NodeJS.Timeout;
|
||||
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
private readonly authService: AuthService,
|
||||
private readonly spaceRepo: SpaceRepo,
|
||||
private readonly workspaceRepo: WorkspaceRepo,
|
||||
private readonly spaceAbilityFactory: SpaceAbilityFactory,
|
||||
private readonly vaultRegistry: VaultRegistryService,
|
||||
private readonly orchestrator: GitSyncOrchestrator,
|
||||
private readonly backend: GitHttpBackendService,
|
||||
) {
|
||||
this.sweepTimer = setInterval(() => {
|
||||
try {
|
||||
this.failedLogins.sweep();
|
||||
} catch (err) {
|
||||
this.logger.error('git-http failed-login limiter sweep failed', err as Error);
|
||||
}
|
||||
}, this.sweepIntervalMs);
|
||||
// Never keep the event loop alive solely for the sweep timer.
|
||||
this.sweepTimer.unref?.();
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
clearInterval(this.sweepTimer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the workspace for a /git request the SAME way DomainMiddleware does,
|
||||
* because Nest middleware does NOT run for this raw root-mounted route (it is
|
||||
* registered under the global '/api' router), so `req.raw.workspaceId` is never
|
||||
* populated here. Delegates to the shared `resolveRequestWorkspace` helper (the
|
||||
* SAME self-hosted/cloud branch DomainMiddleware uses) and returns just the id:
|
||||
* - self-hosted (single workspace) -> workspaceRepo.findFirst();
|
||||
* - cloud (multi-tenant) -> resolve by the host-header subdomain.
|
||||
* Returns null when no workspace resolves; the gate then 404s (after the
|
||||
* 401-before-404 credential check encoded in decideGitHttpGate).
|
||||
*/
|
||||
private async resolveWorkspaceId(req: FastifyRequest): Promise<string | null> {
|
||||
try {
|
||||
// Same self-hosted/cloud resolution DomainMiddleware uses — shared so the
|
||||
// branch cannot drift between the two call sites.
|
||||
const workspace = await resolveRequestWorkspace(
|
||||
this.environmentService,
|
||||
this.workspaceRepo,
|
||||
this.headerValue(req.headers['host']),
|
||||
);
|
||||
return workspace?.id ?? null;
|
||||
} catch (err) {
|
||||
// A DB error resolving the workspace must not leak details; treat as
|
||||
// unresolvable (the gate will 404, unless creds are missing -> 401 first).
|
||||
this.logger.warn(
|
||||
`git-http: workspace resolution error: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle one `/git/<spaceId>.git/<subpath>` request. `rest` is the path AFTER
|
||||
* the `/git/` prefix (no query string). The Fastify reply is hijacked before
|
||||
* any streaming so the binary CGI body is written directly to the raw socket.
|
||||
*/
|
||||
async handle(req: FastifyRequest, reply: FastifyReply): Promise<void> {
|
||||
const rawReq = req.raw;
|
||||
const rawRes = reply.raw;
|
||||
|
||||
// --- parse the URL into spaceId + subpath -------------------------------
|
||||
const rest = this.extractRest(req.url);
|
||||
const parsedPath = rest === null ? null : parseGitPath(rest);
|
||||
|
||||
// --- resolve the requested git service kind (read vs write) -------------
|
||||
const service =
|
||||
typeof req.query === 'object' && req.query !== null
|
||||
? (req.query as Record<string, string | undefined>).service
|
||||
: undefined;
|
||||
const serviceKind: GitHttpServiceKind | null = parsedPath
|
||||
? resolveServiceKind({
|
||||
method: req.method,
|
||||
subpath: parsedPath.subpath,
|
||||
service,
|
||||
})
|
||||
: null;
|
||||
|
||||
// --- authenticate (HTTP Basic) ------------------------------------------
|
||||
const authHeader = req.headers['authorization'];
|
||||
const basic = parseBasicAuth(
|
||||
Array.isArray(authHeader) ? authHeader[0] : authHeader,
|
||||
);
|
||||
// Resolve the workspace ourselves — DomainMiddleware does NOT run for this
|
||||
// raw root route, so `req.raw.workspaceId` is never set (see resolver doc).
|
||||
const workspaceId: string | null = await this.resolveWorkspaceId(req);
|
||||
|
||||
let user: User | undefined;
|
||||
let credentialsValid = false;
|
||||
let throttled = false;
|
||||
if (basic && workspaceId) {
|
||||
// Brute-force speed bump, mirroring the /mcp Basic path EXACTLY. Reserve
|
||||
// ALL three keys ATOMICALLY and BEFORE bcrypt (tryReserve folds the check
|
||||
// and the increment into one synchronous step), so the (threshold+1)-th
|
||||
// attempt is rejected before verifyUserCredentials/bcrypt ever runs and
|
||||
// concurrent attempts for one email cannot all observe count=0. The
|
||||
// reservation IS the recorded failure: a genuine credential failure leaves
|
||||
// it in place, a SUCCESS clears it (reset), a non-credential error releases
|
||||
// it (so it cannot burn a victim's budget).
|
||||
const emailLc = basic.email.toLowerCase();
|
||||
const ip = clientIp(req);
|
||||
const ipKey = `ip:${ip}`;
|
||||
const ipEmailKey = `ip-email:${ip}:${emailLc}`;
|
||||
// GLOBAL per-email backstop (no IP): the only key that survives IP / XFF
|
||||
// rotation, so it is the real account-brute defense (see mcp-auth.helpers).
|
||||
const emailKey = `email:${emailLc}`;
|
||||
const ipOk = this.failedLogins.tryReserve(ipKey);
|
||||
const ipEmailOk = this.failedLogins.tryReserve(ipEmailKey);
|
||||
const emailOk = this.failedLogins.tryReserve(emailKey);
|
||||
if (!ipOk || !ipEmailOk || !emailOk) {
|
||||
// Blocked: release only the keys we actually reserved this call so an
|
||||
// already-throttled request does not over-charge keys still under budget
|
||||
// (matches the /mcp reserve model). Do NOT run bcrypt.
|
||||
if (ipOk) this.failedLogins.release(ipKey);
|
||||
if (ipEmailOk) this.failedLogins.release(ipEmailKey);
|
||||
if (emailOk) this.failedLogins.release(emailKey);
|
||||
throttled = true;
|
||||
} else {
|
||||
try {
|
||||
user = await this.authService.verifyUserCredentials(
|
||||
{ email: basic.email, password: basic.password },
|
||||
workspaceId,
|
||||
);
|
||||
credentialsValid = true;
|
||||
// Success: clear the per-IP and per-IP+email budgets fully; for the
|
||||
// GLOBAL per-email key only release the one increment THIS request took
|
||||
// (do not reset() it, or a victim's own success would wipe a parallel
|
||||
// attacker's accumulated failures for that email — same rule as /mcp).
|
||||
this.failedLogins.reset(ipKey);
|
||||
this.failedLogins.reset(ipEmailKey);
|
||||
this.failedLogins.release(emailKey);
|
||||
} catch (err) {
|
||||
// Only a genuine credentials failure (wrong email/password) keeps the
|
||||
// reservation (it IS the recorded failure). Any other error — DB error,
|
||||
// etc. — is NOT a password-guess signal, so release the reservation so
|
||||
// it cannot burn a victim's limiter budget. credentialsValid stays
|
||||
// false either way (the gate then 401s).
|
||||
if (!isCredentialsFailure(err)) {
|
||||
this.failedLogins.release(ipKey);
|
||||
this.failedLogins.release(ipEmailKey);
|
||||
this.failedLogins.release(emailKey);
|
||||
}
|
||||
if (!(err instanceof UnauthorizedException)) {
|
||||
// A non-credential failure (e.g. DB error): treat as invalid creds
|
||||
// for the gate (a 401), and log without leaking the password/header.
|
||||
this.logger.warn(
|
||||
`git-http: credential check error: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
credentialsValid = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Brute-force throttle tripped: reject BEFORE the gate (and before any space
|
||||
// lookup), so a throttled attacker gets a uniform 429 with no bcrypt and no
|
||||
// existence signal. WWW-Authenticate is still sent so a legitimate client
|
||||
// re-prompts after the window.
|
||||
if (throttled) {
|
||||
reply
|
||||
.header('WWW-Authenticate', WWW_AUTHENTICATE)
|
||||
.header('Retry-After', '60')
|
||||
.status(429)
|
||||
.send('Too many failed authentication attempts. Try again later.');
|
||||
return;
|
||||
}
|
||||
|
||||
// --- resolve the space + per-space gating + CASL ------------------------
|
||||
let spaceExists = false;
|
||||
let spaceGitSyncEnabled = false;
|
||||
let spaceId: string | undefined;
|
||||
// The user has SOME role in the space. SECURITY: a non-member must get the
|
||||
// SAME 404 a missing/disabled space gets — never a 403 — or the 403↔404 split
|
||||
// would let any authenticated user brute-force slugs to learn which spaces
|
||||
// exist / have sync enabled (the leak this gate's contract forbids). 403 is
|
||||
// reserved for a MEMBER who lacks the required role (existence already known).
|
||||
let userIsSpaceMember = false;
|
||||
let permissionGranted = false;
|
||||
if (credentialsValid && user && workspaceId && parsedPath && serviceKind) {
|
||||
const space = await this.spaceRepo.findById(
|
||||
parsedPath.spaceId,
|
||||
workspaceId,
|
||||
);
|
||||
if (space) {
|
||||
spaceExists = true;
|
||||
spaceId = space.id;
|
||||
spaceGitSyncEnabled =
|
||||
(space.settings as any)?.gitSync?.enabled === true;
|
||||
|
||||
// Only evaluate CASL when the space is actually a sync candidate — an
|
||||
// unrelated space stays a 404 (existence is never revealed).
|
||||
if (spaceGitSyncEnabled) {
|
||||
try {
|
||||
const ability = await this.spaceAbilityFactory.createForUser(
|
||||
user,
|
||||
space.id,
|
||||
);
|
||||
// createForUser RESOLVED -> the user holds a role in this space (it
|
||||
// throws NotFound for a non-member). Record membership BEFORE the
|
||||
// permission check: a member lacking the role -> 403; a non-member ->
|
||||
// 404 (handled by the gate via userIsSpaceMember=false below).
|
||||
userIsSpaceMember = true;
|
||||
const action =
|
||||
serviceKind === 'write'
|
||||
? SpaceCaslAction.Manage
|
||||
: SpaceCaslAction.Read;
|
||||
permissionGranted = ability.can(action, SpaceCaslSubject.Page);
|
||||
} catch {
|
||||
// createForUser throws NotFoundException when the user has no role in
|
||||
// the space (a non-member). Leave userIsSpaceMember=false so the gate
|
||||
// returns 404, NOT 403 — a non-member must not be able to tell this
|
||||
// space apart from a non-existent one. (Any other error also falls
|
||||
// here and is treated as non-member -> 404, the safe default that
|
||||
// never reveals existence.)
|
||||
userIsSpaceMember = false;
|
||||
permissionGranted = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- the gate decision (pure) -------------------------------------------
|
||||
const decision = decideGitHttpGate({
|
||||
hasCredentials: Boolean(basic),
|
||||
credentialsValid,
|
||||
serviceKind,
|
||||
gitSyncEnabled: this.environmentService.isGitSyncEnabled(),
|
||||
gitHttpEnabled: this.environmentService.isGitSyncHttpEnabled(),
|
||||
spaceExists,
|
||||
spaceGitSyncEnabled,
|
||||
userIsSpaceMember,
|
||||
permissionGranted,
|
||||
});
|
||||
|
||||
if (decision.kind === 'unauthorized') {
|
||||
reply
|
||||
.header('WWW-Authenticate', WWW_AUTHENTICATE)
|
||||
.status(401)
|
||||
.send('Authentication required');
|
||||
return;
|
||||
}
|
||||
if (decision.kind === 'bad-request') {
|
||||
reply.status(400).send('Bad request');
|
||||
return;
|
||||
}
|
||||
if (decision.kind === 'not-found') {
|
||||
reply.status(404).send('Not found');
|
||||
return;
|
||||
}
|
||||
if (decision.kind === 'forbidden') {
|
||||
reply.status(403).send('Forbidden');
|
||||
return;
|
||||
}
|
||||
|
||||
// decision.kind === 'proceed' — guaranteed below (narrowing for TS).
|
||||
if (!parsedPath || !serviceKind || !spaceId || !user || !workspaceId) {
|
||||
// Defensive: 'proceed' implies these are set, but keep TS + runtime safe.
|
||||
reply.status(500).send('Internal server error');
|
||||
return;
|
||||
}
|
||||
|
||||
// --- dispatch to git http-backend ---------------------------------------
|
||||
const backendRequest = {
|
||||
spaceId,
|
||||
subpath: parsedPath.subpath,
|
||||
method: req.method,
|
||||
queryString: this.extractQueryString(req.url),
|
||||
contentType: this.headerValue(req.headers['content-type']) ?? '',
|
||||
gitProtocol: this.headerValue(req.headers['git-protocol']),
|
||||
// Forward Content-Encoding so http-backend inflates gzip'd RPC bodies
|
||||
// (review #4) — else a non-trivial `git pull` fails with expected 'packfile'.
|
||||
contentEncoding: this.headerValue(req.headers['content-encoding']),
|
||||
remoteUser: user.email,
|
||||
};
|
||||
|
||||
try {
|
||||
// Idempotently make the vault servable (repo + receive/upload config).
|
||||
await this.vaultRegistry.ensureServable(spaceId);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`git-http: failed to prepare vault for space ${spaceId}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
if (!reply.sent) reply.status(500).send('Internal server error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Hijack the reply so the backend can stream the raw (possibly binary) CGI
|
||||
// response directly to the socket (mirrors the MCP transport pattern).
|
||||
reply.hijack();
|
||||
|
||||
// Only the ACTUAL pack-receiving write (POST git-receive-pack) runs under the
|
||||
// space lock + a Docmost cycle. Everything else streams the http-backend
|
||||
// directly with NO lock and NO cycle: a fetch/clone (read), AND the
|
||||
// write-AUTHORIZED but READ-ONLY ref advertisement
|
||||
// (GET info/refs?service=git-receive-pack). Running a cycle on info/refs is
|
||||
// both wasteful and HARMFUL — it holds the per-space lock, so the push's
|
||||
// immediately-following POST git-receive-pack collides with it and 503s
|
||||
// (a deterministic push failure). Authz already happened above via the gate.
|
||||
const isReceivePack =
|
||||
req.method === 'POST' && parsedPath.subpath === 'git-receive-pack';
|
||||
if (serviceKind === 'read' || !isReceivePack) {
|
||||
// The clone's default branch comes from the HEAD symref advertised by the
|
||||
// upload-pack ref advertisement (or a dumb `GET HEAD`). The engine
|
||||
// transiently checks out the read-only `docmost` mirror mid-cycle, so serve
|
||||
// THAT advertisement with HEAD pinned to `main` under the per-space lock so
|
||||
// a clone never defaults to `docmost` (bug #3). Pack streaming and every
|
||||
// other read are resolved by object SHA and need no pin, so they stream
|
||||
// directly (no lock) as before.
|
||||
const isReadAdvertise =
|
||||
req.method === 'GET' &&
|
||||
((parsedPath.subpath === 'info/refs' &&
|
||||
service === 'git-upload-pack') ||
|
||||
parsedPath.subpath === 'HEAD');
|
||||
if (isReadAdvertise) {
|
||||
// The read-advertise path runs under the space lock (to pin HEAD=main).
|
||||
// AFTER reply.hijack() Fastify no longer manages this response, so a
|
||||
// rejection here (e.g. SpaceLockService.acquire when Redis is down) would
|
||||
// otherwise leave the socket open forever — every clone/fetch hangs until
|
||||
// the client times out (review #5). Mirror the push branch: catch, answer
|
||||
// 500 if nothing was written yet, and always end the raw socket.
|
||||
try {
|
||||
await this.orchestrator.serveReadAdvertisement(spaceId, () =>
|
||||
this.backend.run(backendRequest, rawReq, rawRes),
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`git-sync: read advertisement failed for space ${spaceId}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
if (!rawRes.headersSent) {
|
||||
rawRes.statusCode = 500;
|
||||
rawRes.setHeader('Content-Type', 'text/plain');
|
||||
rawRes.end('Internal server error');
|
||||
} else if (!rawRes.writableEnded) {
|
||||
rawRes.end();
|
||||
}
|
||||
}
|
||||
} else {
|
||||
await this.backend.run(backendRequest, rawReq, rawRes);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Push: run the receive-pack under the space lock, then a Docmost cycle.
|
||||
try {
|
||||
await this.orchestrator.ingestExternalPush(
|
||||
spaceId,
|
||||
workspaceId,
|
||||
// The lock's lost-lock signal is threaded into the backend so the
|
||||
// receive-pack child is killed if the lock lapses mid-write (warning #3).
|
||||
(signal) => this.backend.run(backendRequest, rawReq, rawRes, signal),
|
||||
);
|
||||
} catch (err) {
|
||||
if (err instanceof GitSyncLockHeldError) {
|
||||
// The lock could not be acquired and the receive-pack never ran, so the
|
||||
// response is still unwritten — answer 503 so git retries.
|
||||
if (!rawRes.headersSent) {
|
||||
rawRes.statusCode = 503;
|
||||
rawRes.setHeader('Content-Type', 'text/plain');
|
||||
rawRes.setHeader('Retry-After', '1');
|
||||
}
|
||||
try {
|
||||
rawRes.end('git-sync busy, retry');
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Any other error: the receive-pack closure handles its own response, so
|
||||
// we only log here and make sure the socket is closed.
|
||||
this.logger.error(
|
||||
`git-http: push ingestion error for space ${spaceId}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
try {
|
||||
if (!rawRes.writableEnded) rawRes.end();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Normalise a possibly-array header value to its first string. */
|
||||
private headerValue(value: string | string[] | undefined): string | undefined {
|
||||
if (Array.isArray(value)) return value[0];
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the part of the URL AFTER `/git/` and BEFORE the query string.
|
||||
* Returns null when the URL is not under `/git/`.
|
||||
*/
|
||||
private extractRest(url: string): string | null {
|
||||
const qIdx = url.indexOf('?');
|
||||
const pathname = qIdx === -1 ? url : url.slice(0, qIdx);
|
||||
const prefix = '/git/';
|
||||
if (!pathname.startsWith(prefix)) return null;
|
||||
return pathname.slice(prefix.length);
|
||||
}
|
||||
|
||||
/** The raw query string without the leading '?', or '' when none. */
|
||||
private extractQueryString(url: string): string {
|
||||
const qIdx = url.indexOf('?');
|
||||
return qIdx === -1 ? '' : url.slice(qIdx + 1);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
// Unit tests for the event-driven git-sync trigger. The orchestrator
|
||||
// and page repo are hand-built mocks; the debounce coalescing is exercised with
|
||||
// jest fake timers. We assert the gate, the loop-guard (anti-echo), the
|
||||
// missing-page short-circuit, the heterogeneous event-shape id resolution, the
|
||||
// debounce collapse, and that errors are swallowed + logged.
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { PageChangeListener } from './page-change.listener';
|
||||
|
||||
type AnyMock = jest.Mock;
|
||||
|
||||
interface Built {
|
||||
listener: PageChangeListener;
|
||||
env: { isGitSyncEnabled: AnyMock; getGitSyncDebounceMs: AnyMock };
|
||||
orchestrator: { runOnce: AnyMock };
|
||||
pageRepo: { findById: AnyMock };
|
||||
}
|
||||
|
||||
function build(opts: { enabled?: boolean; debounceMs?: number } = {}): Built {
|
||||
const { enabled = true, debounceMs = 2000 } = opts;
|
||||
const env = {
|
||||
isGitSyncEnabled: jest.fn(() => enabled),
|
||||
getGitSyncDebounceMs: jest.fn(() => debounceMs),
|
||||
};
|
||||
const orchestrator = { runOnce: jest.fn(async () => undefined) };
|
||||
const pageRepo = { findById: jest.fn() };
|
||||
|
||||
const listener = new PageChangeListener(
|
||||
env as any,
|
||||
orchestrator as any,
|
||||
pageRepo as any,
|
||||
);
|
||||
return { listener, env, orchestrator, pageRepo };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('PageChangeListener', () => {
|
||||
describe('gate', () => {
|
||||
it('does nothing when git-sync is disabled (no findById, no schedule)', async () => {
|
||||
const { listener, orchestrator, pageRepo } = build({ enabled: false });
|
||||
await listener.handlePageEvent({ pageId: 'p1', workspaceId: 'ws-1' });
|
||||
expect(pageRepo.findById).not.toHaveBeenCalled();
|
||||
expect(orchestrator.runOnce).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('loop-guard (anti-echo)', () => {
|
||||
it("does NOT schedule a cycle when the page row's source is 'git-sync'", async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { listener, orchestrator, pageRepo } = build();
|
||||
pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
spaceId: 'space-1',
|
||||
workspaceId: 'ws-1',
|
||||
lastUpdatedSource: 'git-sync',
|
||||
});
|
||||
await listener.handlePageEvent({ pageId: 'p1', workspaceId: 'ws-1' });
|
||||
jest.runOnlyPendingTimers();
|
||||
expect(orchestrator.runOnce).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('schedules exactly one cycle for a normal (non-git-sync) source', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { listener, orchestrator, pageRepo } = build();
|
||||
pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
spaceId: 'space-1',
|
||||
workspaceId: 'ws-1',
|
||||
lastUpdatedSource: 'user',
|
||||
});
|
||||
await listener.handlePageEvent({ pageId: 'p1', workspaceId: 'ws-1' });
|
||||
jest.runOnlyPendingTimers();
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledTimes(1);
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledWith('space-1', 'ws-1');
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('missing page', () => {
|
||||
it('does not schedule when findById returns null/undefined', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { listener, orchestrator, pageRepo } = build();
|
||||
pageRepo.findById.mockResolvedValue(undefined);
|
||||
await listener.handlePageEvent({ pageId: 'p1', workspaceId: 'ws-1' });
|
||||
jest.runOnlyPendingTimers();
|
||||
expect(orchestrator.runOnce).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('spaceId/workspaceId resolution', () => {
|
||||
// The page row used to fill in any ids the event omits.
|
||||
const pageRow = {
|
||||
id: 'p1',
|
||||
spaceId: 'row-space',
|
||||
workspaceId: 'row-ws',
|
||||
lastUpdatedSource: 'user',
|
||||
};
|
||||
|
||||
async function resolve(event: Record<string, unknown>) {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { listener, orchestrator, pageRepo } = build();
|
||||
pageRepo.findById.mockResolvedValue(pageRow);
|
||||
await listener.handlePageEvent(event as any);
|
||||
jest.runOnlyPendingTimers();
|
||||
return { orchestrator, pageRepo };
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
}
|
||||
|
||||
it("resolves pageId + event.spaceId + event.workspaceId", async () => {
|
||||
const { orchestrator, pageRepo } = await resolve({
|
||||
pageId: 'p1',
|
||||
spaceId: 'evt-space',
|
||||
workspaceId: 'evt-ws',
|
||||
});
|
||||
expect(pageRepo.findById).toHaveBeenCalledWith('p1', { includeContent: false });
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledWith('evt-space', 'evt-ws');
|
||||
});
|
||||
|
||||
it('resolves pageId from pageIds[0]', async () => {
|
||||
const { orchestrator, pageRepo } = await resolve({
|
||||
pageIds: ['p1', 'p2'],
|
||||
spaceId: 'evt-space',
|
||||
workspaceId: 'evt-ws',
|
||||
});
|
||||
expect(pageRepo.findById).toHaveBeenCalledWith('p1', { includeContent: false });
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledWith('evt-space', 'evt-ws');
|
||||
});
|
||||
|
||||
it('resolves pageId + spaceId from pages[]', async () => {
|
||||
const { orchestrator } = await resolve({
|
||||
pages: [{ id: 'p1', spaceId: 'pages-space' }],
|
||||
workspaceId: 'evt-ws',
|
||||
});
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledWith('pages-space', 'evt-ws');
|
||||
});
|
||||
|
||||
it('resolves pageId + spaceId from node', async () => {
|
||||
const { orchestrator } = await resolve({
|
||||
node: { id: 'p1', spaceId: 'node-space' },
|
||||
workspaceId: 'evt-ws',
|
||||
});
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledWith('node-space', 'evt-ws');
|
||||
});
|
||||
|
||||
it('falls back to the fetched page row when the event omits spaceId/workspaceId', async () => {
|
||||
const { orchestrator } = await resolve({ pageId: 'p1' });
|
||||
// No spaceId/workspaceId on the event -> use the page row's values.
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledWith('row-space', 'row-ws');
|
||||
});
|
||||
});
|
||||
|
||||
describe('debounce coalescing', () => {
|
||||
it('collapses a burst of N events for one space into exactly one runOnce', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { listener, orchestrator, pageRepo } = build({ debounceMs: 500 });
|
||||
pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
spaceId: 'space-1',
|
||||
workspaceId: 'ws-1',
|
||||
lastUpdatedSource: 'user',
|
||||
});
|
||||
|
||||
// Fire a burst of 5 events; await each so its findById promise settles
|
||||
// and schedule() runs before the next event resets the timer.
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await listener.handlePageEvent({ pageId: 'p1', workspaceId: 'ws-1' });
|
||||
}
|
||||
|
||||
// Nothing fired yet (still within the debounce window).
|
||||
expect(orchestrator.runOnce).not.toHaveBeenCalled();
|
||||
|
||||
// Advance past the debounce window: the coalesced cycle fires once.
|
||||
jest.advanceTimersByTime(500);
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledTimes(1);
|
||||
expect(orchestrator.runOnce).toHaveBeenCalledWith('space-1', 'ws-1');
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('onModuleDestroy', () => {
|
||||
it('clears every pending debounce timer and empties the map', async () => {
|
||||
jest.useFakeTimers();
|
||||
const clearSpy = jest.spyOn(global, 'clearTimeout');
|
||||
try {
|
||||
const { listener, orchestrator, pageRepo } = build({ debounceMs: 500 });
|
||||
pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
spaceId: 'space-1',
|
||||
workspaceId: 'ws-1',
|
||||
lastUpdatedSource: 'user',
|
||||
});
|
||||
|
||||
// Schedule a pending cycle, then tear the module down before it fires.
|
||||
await listener.handlePageEvent({ pageId: 'p1', workspaceId: 'ws-1' });
|
||||
clearSpy.mockClear(); // ignore any clears done by schedule() itself
|
||||
|
||||
listener.onModuleDestroy();
|
||||
|
||||
// The pending timer was cleared and the map drained, so advancing past
|
||||
// the debounce window fires NO cycle.
|
||||
expect(clearSpy).toHaveBeenCalledTimes(1);
|
||||
expect((listener as any).debounce.size).toBe(0);
|
||||
jest.advanceTimersByTime(500);
|
||||
expect(orchestrator.runOnce).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
clearSpy.mockRestore();
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('error swallowing', () => {
|
||||
it('does not throw and logs a warning when findById throws', async () => {
|
||||
const warnSpy = jest
|
||||
.spyOn(Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
try {
|
||||
const { listener, orchestrator, pageRepo } = build();
|
||||
pageRepo.findById.mockRejectedValue(new Error('db down'));
|
||||
|
||||
await expect(
|
||||
listener.handlePageEvent({ pageId: 'p1', workspaceId: 'ws-1' }),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
expect(warnSpy).toHaveBeenCalledTimes(1);
|
||||
expect(String(warnSpy.mock.calls[0][0])).toContain('db down');
|
||||
expect(orchestrator.runOnce).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
warnSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,168 @@
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { OnEvent } from '@nestjs/event-emitter';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { EnvironmentService } from '../../environment/environment.service';
|
||||
import { GitSyncOrchestrator } from '../services/git-sync.orchestrator';
|
||||
import { GIT_SYNC_PAGE_EVENTS } from '../git-sync.constants';
|
||||
|
||||
/**
|
||||
* Shape of the page domain events the listener consumes. Different emit sites
|
||||
* carry different optional fields (page.repo `PageEvent`, `PageMovedEvent`,
|
||||
* etc.), so this is the intersection we read: a `pageIds` list / single `pageId`,
|
||||
* the `workspaceId`, and an OPTIONAL `spaceId` (present only on some events). When
|
||||
* `spaceId` is absent we resolve it from the page row.
|
||||
*/
|
||||
interface PageEventLike {
|
||||
pageIds?: string[];
|
||||
pageId?: string;
|
||||
workspaceId?: string;
|
||||
spaceId?: string;
|
||||
pages?: { id: string; spaceId: string }[];
|
||||
node?: { id: string; spaceId: string };
|
||||
}
|
||||
|
||||
/**
|
||||
* Event-driven trigger for the git-sync control plane. Subscribes to
|
||||
* the page lifecycle events and, for an enabled space, schedules a DEBOUNCED
|
||||
* `orchestrator.runOnce(spaceId, workspaceId)` — coalescing a burst of edits into
|
||||
* a single cycle per space.
|
||||
*
|
||||
* Loop-guard (best-effort): an event whose page row already reads
|
||||
* `lastUpdatedSource === 'git-sync'` is the orchestrator's OWN write, so we skip
|
||||
* it to avoid a write -> event -> sync echo. The guard ALWAYS runs (the page row
|
||||
* is fetched for every event, structural ones included). This is the cheap first
|
||||
* guard; the full bodyHash + updatedAt loop-guard (consuming the push side's
|
||||
* `PushedPageRecord`) is a later hardening step — noted, not built here.
|
||||
*
|
||||
* KNOWN OVER-SKIP (latency, NOT data loss): the guard keys ONLY on
|
||||
* `lastUpdatedSource`, and a user MOVE / RENAME / DELETE does NOT change that
|
||||
* column (only body writes stamp it). So a genuine user move/rename/delete of a
|
||||
* page whose BODY was last written by git-sync still reads
|
||||
* `lastUpdatedSource === 'git-sync'` and is dropped on this fast debounced path.
|
||||
* No change is lost: the poll-safety interval (~GIT_SYNC_POLL_INTERVAL_MS, default
|
||||
* 15s) re-enumerates the space and reconciles it — the only cost is up to one poll
|
||||
* interval of extra latency before that structural change reaches git. The
|
||||
* bodyHash+updatedAt loop-guard above would close this gap precisely.
|
||||
*/
|
||||
@Injectable()
|
||||
export class PageChangeListener implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(PageChangeListener.name);
|
||||
// spaceId -> pending debounce timer. The cycle closes over its own
|
||||
// workspaceId, so the timer handle is all the map needs to track.
|
||||
private readonly debounce = new Map<string, NodeJS.Timeout>();
|
||||
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
private readonly orchestrator: GitSyncOrchestrator,
|
||||
private readonly pageRepo: PageRepo,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* One handler bound to ALL git-sync page events (the array form of `@OnEvent`).
|
||||
* Fetches the page row once to apply the loop-guard (unconditionally) and to
|
||||
* resolve the page's space + workspace, then schedules the debounced cycle.
|
||||
*/
|
||||
@OnEvent(GIT_SYNC_PAGE_EVENTS as unknown as string[])
|
||||
async handlePageEvent(event: PageEventLike): Promise<void> {
|
||||
if (!this.environmentService.isGitSyncEnabled()) return;
|
||||
|
||||
try {
|
||||
const pageId = this.firstPageId(event);
|
||||
if (!pageId) return;
|
||||
|
||||
// The loop-guard MUST always run — even structural events that already
|
||||
// carry spaceId+workspaceId could be the orchestrator's OWN write (it stamps
|
||||
// lastUpdatedSource='git-sync' on create/update/move/rename + body writes).
|
||||
// So ALWAYS fetch the page row: it gives us the loop-guard source AND fills
|
||||
// in any missing space/workspace in a single read. A missing page
|
||||
// (hard-deleted) is ignored.
|
||||
const page = await this.pageRepo.findById(pageId, {
|
||||
includeContent: false,
|
||||
});
|
||||
if (!page) return;
|
||||
|
||||
// Loop-guard: skip our own writes to avoid a write -> event -> sync echo
|
||||
// (best-effort). Applies unconditionally now. NOTE this also over-skips a
|
||||
// user move/rename/delete of a page whose BODY was last written by git-sync
|
||||
// (those structural ops don't touch lastUpdatedSource) — that change is not
|
||||
// lost, just deferred to the ~15s poll backstop (see class docstring).
|
||||
if (page.lastUpdatedSource === 'git-sync') return;
|
||||
|
||||
// Prefer ids carried on the event; fall back to the row we already fetched.
|
||||
const spaceId = this.eventSpaceId(event, pageId) ?? page.spaceId;
|
||||
const workspaceId = event.workspaceId ?? page.workspaceId;
|
||||
|
||||
if (!spaceId || !workspaceId) return;
|
||||
this.schedule(spaceId, workspaceId);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`git-sync: failed to handle page event: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Pull the first affected pageId out of the heterogeneous event shapes. */
|
||||
private firstPageId(event: PageEventLike): string | undefined {
|
||||
return (
|
||||
event.pageId ??
|
||||
event.pageIds?.[0] ??
|
||||
event.pages?.[0]?.id ??
|
||||
event.node?.id
|
||||
);
|
||||
}
|
||||
|
||||
/** A spaceId carried directly on the event, for the given pageId if scoped. */
|
||||
private eventSpaceId(
|
||||
event: PageEventLike,
|
||||
pageId: string,
|
||||
): string | undefined {
|
||||
if (event.spaceId) return event.spaceId;
|
||||
const fromPages = event.pages?.find((p) => p.id === pageId)?.spaceId;
|
||||
if (fromPages) return fromPages;
|
||||
if (event.node?.id === pageId) return event.node.spaceId;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* On shutdown, clear every pending debounce timer so a not-yet-fired cycle does
|
||||
* not run against a tearing-down module. The timers are already `.unref()`'d (so
|
||||
* they never block process exit), but clearing them also drops the dangling
|
||||
* references and prevents a late `runOnce` from firing post-destroy.
|
||||
*/
|
||||
onModuleDestroy(): void {
|
||||
for (const timer of this.debounce.values()) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
this.debounce.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Debounce per space: a new event resets the timer so a burst collapses into a
|
||||
* single cycle. On fire, `runOnce` is enqueued (it internally serializes via the
|
||||
* in-process mutex + Redis lock, so a still-running cycle is simply skipped and
|
||||
* the next event reschedules).
|
||||
*/
|
||||
private schedule(spaceId: string, workspaceId: string): void {
|
||||
const existing = this.debounce.get(spaceId);
|
||||
if (existing) clearTimeout(existing);
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
this.debounce.delete(spaceId);
|
||||
void this.orchestrator
|
||||
.runOnce(spaceId, workspaceId)
|
||||
.catch((err) =>
|
||||
this.logger.error(
|
||||
`git-sync: debounced cycle for space ${spaceId} failed: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
),
|
||||
);
|
||||
}, this.environmentService.getGitSyncDebounceMs());
|
||||
|
||||
// Do not keep the event loop alive solely for a pending sync.
|
||||
timer.unref?.();
|
||||
this.debounce.set(spaceId, timer);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import * as Y from 'yjs';
|
||||
|
||||
import { mergeXmlFragments3Way } from '../../../collaboration/merge/yjs-body-merge';
|
||||
|
||||
/**
|
||||
* Convergence repro for the git-ingest "silent revert" data-loss bug.
|
||||
*
|
||||
* ROOT CAUSE (confirmed): the merge logic itself is correct, but the git-ingest
|
||||
* write was applied via `openDirectConnection` on whichever instance/process
|
||||
* runs git-sync (the api/worker). When an editor is connected to a DIFFERENT
|
||||
* collab instance/process, that opens a SEPARATE, detached Y.Doc. The merge
|
||||
* lands in that detached doc (and the DB), but the live editor's Y.Doc never
|
||||
* receives the Yjs update — so its next debounced autosave overwrites the DB
|
||||
* with its STALE state and silently reverts the git change.
|
||||
*
|
||||
* These tests reproduce the invariant deterministically at the Yjs level (two
|
||||
* Y.Docs exchanging updates), because the real failure is DISTRIBUTED — it only
|
||||
* manifests when the write and the editor live on different instances, which a
|
||||
* single in-process Hocuspocus cannot reproduce (in one process the direct
|
||||
* connection already shares the editor's doc). HONEST SCOPE: this models the two
|
||||
* outcomes; full cross-instance convergence is not (and cannot be) proven in a
|
||||
* unit test without a live multi-instance Hocuspocus + redis.
|
||||
*
|
||||
* PATH B (the BUG): the git update is NOT delivered to the editor's doc — the
|
||||
* editor's later autosave reverts the change. Asserts the LOSS.
|
||||
* PATH A (the FIX): the git update IS delivered to the editor's doc as a Yjs
|
||||
* update — which is exactly what running the merge on the OWNING instance's
|
||||
* shared Document does (its update is broadcast to every connection). The
|
||||
* editor's CRDT converges and a later autosave preserves the git change.
|
||||
*
|
||||
* The fix routes git-sync's body write through CollaborationGateway.writePageBody
|
||||
* (the custom-event channel) so it executes on the owning instance — turning
|
||||
* PATH B into PATH A.
|
||||
*/
|
||||
|
||||
type Spec = { text: string; id?: string };
|
||||
|
||||
// Build a Y.XmlFragment('default'). `id` is set only when provided, mirroring
|
||||
// the live doc (block UniqueIDs present) vs a git-parsed body (ids absent).
|
||||
function buildFragment(doc: Y.Doc, specs: Spec[]): Y.XmlFragment {
|
||||
const frag = doc.getXmlFragment('default');
|
||||
const blocks = specs.map((s) => {
|
||||
const el = new Y.XmlElement('paragraph');
|
||||
if (s.id) el.setAttribute('id', s.id);
|
||||
const t = new Y.XmlText();
|
||||
if (s.text) t.insert(0, s.text);
|
||||
el.insert(0, [t]);
|
||||
return el;
|
||||
});
|
||||
if (blocks.length) frag.insert(0, blocks);
|
||||
return frag;
|
||||
}
|
||||
|
||||
const texts = (frag: Y.XmlFragment): string[] =>
|
||||
frag.toArray().map((el) =>
|
||||
(el as Y.XmlElement)
|
||||
.toArray()
|
||||
.map((c) => (c as Y.XmlText).toString())
|
||||
.join(''),
|
||||
);
|
||||
|
||||
// Append '!' to the end of the given block's text — a tiny human edit that
|
||||
// stands in for a connected editor's autosave-triggering keystroke.
|
||||
function humanEdit(doc: Y.Doc, blockIndex: number, mark = '!'): void {
|
||||
const frag = doc.getXmlFragment('default');
|
||||
const el = frag.get(blockIndex) as Y.XmlElement;
|
||||
const t = el.get(0) as Y.XmlText;
|
||||
doc.transact(() => t.insert(t.length, mark));
|
||||
}
|
||||
|
||||
describe('git-ingest convergence with an open editor', () => {
|
||||
// Shared setup: the page is persisted with two blocks (live ids), and BOTH the
|
||||
// server-side ingest doc (S) and the connected editor's doc (C) load that same
|
||||
// state — they start fully synced, exactly like two instances that each loaded
|
||||
// the page from the DB.
|
||||
function setup() {
|
||||
const db = new Y.Doc();
|
||||
buildFragment(db, [
|
||||
{ text: 'alpha', id: 'p1' },
|
||||
{ text: 'beta', id: 'p2' },
|
||||
]);
|
||||
const state0 = Y.encodeStateAsUpdate(db);
|
||||
|
||||
const server = new Y.Doc(); // where the git merge is applied
|
||||
Y.applyUpdate(server, state0);
|
||||
const editor = new Y.Doc(); // the browser's live in-memory doc
|
||||
Y.applyUpdate(editor, state0);
|
||||
|
||||
// base (last-synced, from git markdown — no ids) == the pre-change content.
|
||||
const baseDoc = new Y.Doc();
|
||||
const baseFrag = buildFragment(baseDoc, [{ text: 'alpha' }, { text: 'beta' }]);
|
||||
return { state0, server, editor, baseFrag };
|
||||
}
|
||||
|
||||
// git changed the SECOND block alpha/beta -> beta2; the editor is idle on it.
|
||||
function applyGitMerge(server: Y.Doc, baseFrag: Y.XmlFragment): Uint8Array {
|
||||
const targetDoc = new Y.Doc();
|
||||
const targetFrag = buildFragment(targetDoc, [
|
||||
{ text: 'alpha' },
|
||||
{ text: 'beta2' },
|
||||
]);
|
||||
let captured: Uint8Array | null = null;
|
||||
const onUpdate = (u: Uint8Array) => {
|
||||
// Accumulate (the merge emits one update per op when unwrapped); here a
|
||||
// single transact yields one update covering the whole merge.
|
||||
captured = captured ? Y.mergeUpdates([captured, u]) : u;
|
||||
};
|
||||
server.on('update', onUpdate);
|
||||
server.transact(() =>
|
||||
mergeXmlFragments3Way(
|
||||
server.getXmlFragment('default'),
|
||||
targetFrag,
|
||||
baseFrag,
|
||||
),
|
||||
);
|
||||
server.off('update', onUpdate);
|
||||
return captured!;
|
||||
}
|
||||
|
||||
it('PATH B (the BUG): undelivered git update is reverted by the editor autosave — DATA LOSS', () => {
|
||||
const { server, editor, baseFrag } = setup();
|
||||
|
||||
// git merge lands on the server doc only.
|
||||
applyGitMerge(server, baseFrag);
|
||||
expect(texts(server.getXmlFragment('default'))).toEqual(['alpha', 'beta2']);
|
||||
|
||||
// The editor NEVER receives the update (detached doc on another instance).
|
||||
// It makes an unrelated edit on block 0 and autosaves its full state.
|
||||
humanEdit(editor, 0);
|
||||
const persisted = new Y.Doc();
|
||||
Y.applyUpdate(persisted, Y.encodeStateAsUpdate(editor));
|
||||
|
||||
// git's 'beta2' is gone — the page reverted to 'beta'. This is the bug.
|
||||
expect(texts(persisted.getXmlFragment('default'))).toEqual([
|
||||
'alpha!',
|
||||
'beta',
|
||||
]);
|
||||
});
|
||||
|
||||
it('PATH A (the FIX): delivering the git update to the editor converges — git change SURVIVES', () => {
|
||||
const { server, editor, baseFrag } = setup();
|
||||
|
||||
// git merge on the server doc, capturing the broadcastable Yjs update.
|
||||
const gitUpdate = applyGitMerge(server, baseFrag);
|
||||
|
||||
// Running on the OWNING instance broadcasts the update to the connected
|
||||
// editor (Document.handleUpdate). Model that: the editor applies it.
|
||||
Y.applyUpdate(editor, gitUpdate);
|
||||
expect(texts(editor.getXmlFragment('default'))).toEqual(['alpha', 'beta2']);
|
||||
|
||||
// The editor now autosaves (unrelated edit on block 0). Its full state still
|
||||
// carries git's change — no revert.
|
||||
humanEdit(editor, 0);
|
||||
const persisted = new Y.Doc();
|
||||
Y.applyUpdate(persisted, Y.encodeStateAsUpdate(editor));
|
||||
expect(texts(persisted.getXmlFragment('default'))).toEqual([
|
||||
'alpha!',
|
||||
'beta2',
|
||||
]);
|
||||
});
|
||||
|
||||
it('PATH A — concurrent edits to DIFFERENT paragraphs both survive (finding #2)', () => {
|
||||
const { server, editor, baseFrag } = setup();
|
||||
|
||||
// The editor is actively editing block 0 (concurrent with the push).
|
||||
humanEdit(editor, 0, ' EDIT');
|
||||
|
||||
// git changes block 1; merge on the server, broadcast to the editor.
|
||||
const gitUpdate = applyGitMerge(server, baseFrag);
|
||||
Y.applyUpdate(editor, gitUpdate);
|
||||
|
||||
// Both sides preserved: the human's block-0 edit AND git's block-1 change.
|
||||
const persisted = new Y.Doc();
|
||||
Y.applyUpdate(persisted, Y.encodeStateAsUpdate(editor));
|
||||
expect(texts(persisted.getXmlFragment('default'))).toEqual([
|
||||
'alpha EDIT',
|
||||
'beta2',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,666 @@
|
||||
// Unit tests for the git-sync control plane. The engine's `runCycle`
|
||||
// (which owns the PULL->PUSH branch choreography) is mocked so we exercise ONLY
|
||||
// the orchestrator's wiring: gating, the Redis leader lock + in-process mutex
|
||||
// (via SpaceLockService),
|
||||
// the remote-template substitution in the settings it hands the engine, the
|
||||
// external-push ingest, and the idempotent interval lifecycle. The cycle
|
||||
// mechanics themselves are covered by the engine's own cycle round-trip spec.
|
||||
//
|
||||
// The engine mock must be declared before importing the orchestrator so the
|
||||
// runtime `loadGitSync()` bridge resolves to the mocked `runCycle` (the ESM
|
||||
// `@docmost/git-sync` package cannot be `require()`d under jest). The `mock`
|
||||
// prefix lets the hoisted factory reference it.
|
||||
const mockRunCycle = jest.fn();
|
||||
|
||||
jest.mock('../git-sync.loader', () => ({
|
||||
loadGitSync: jest.fn(async () => ({
|
||||
runCycle: mockRunCycle,
|
||||
})),
|
||||
}));
|
||||
|
||||
import { Logger } from '@nestjs/common';
|
||||
import {
|
||||
Kysely,
|
||||
DummyDriver,
|
||||
PostgresAdapter,
|
||||
PostgresIntrospector,
|
||||
PostgresQueryCompiler,
|
||||
CompiledQuery,
|
||||
} from 'kysely';
|
||||
import {
|
||||
GitSyncOrchestrator,
|
||||
GitSyncLockHeldError,
|
||||
} from './git-sync.orchestrator';
|
||||
import { SpaceLockService } from './space-lock.service';
|
||||
|
||||
type AnyMock = jest.Mock;
|
||||
|
||||
const runCycleMock = mockRunCycle as unknown as AnyMock;
|
||||
|
||||
/** The default happy-path cycle result the engine returns. */
|
||||
const OK_CYCLE = {
|
||||
ran: true,
|
||||
pull: { written: 0, deleted: 0, conflict: false },
|
||||
push: { mode: 'apply', failures: 0 },
|
||||
};
|
||||
|
||||
interface BuildOptions {
|
||||
/** Env tunables (only the load-bearing ones are surfaced as overrides). */
|
||||
enabled?: boolean;
|
||||
serviceUserId?: string | undefined;
|
||||
remoteTemplate?: string | undefined;
|
||||
dataDir?: string;
|
||||
pollIntervalMs?: number;
|
||||
debounceMs?: number;
|
||||
/** A hook applied to the fake vault so a test can override its behaviour. */
|
||||
vaultOverrides?: Record<string, unknown>;
|
||||
/**
|
||||
* The row `buildSettings` reads for the per-space `autoMergeConflicts` flag
|
||||
* (`executeTakeFirst` on the `autoMergeConflicts`-aliased query). Default: the
|
||||
* SAFE off value. Pass `undefined` to model a missing row (no space / no
|
||||
* settings).
|
||||
*/
|
||||
settingsRow?: { autoMergeConflicts: boolean } | undefined;
|
||||
/**
|
||||
* The per-space opt-in flag `runOnce` reads via `isSpaceGitSyncEnabled`
|
||||
* (`executeTakeFirst` on the `enabled`-aliased query — a DIFFERENT query from
|
||||
* the `autoMergeConflicts` read above; commit c838fdee added it). Default
|
||||
* `true` so a build() space passes the per-space gate and the cycle proceeds.
|
||||
* Set `false` to model a space that did NOT opt in (skipped:'space-not-enabled').
|
||||
*/
|
||||
spaceEnabled?: boolean;
|
||||
}
|
||||
|
||||
interface Built {
|
||||
orchestrator: GitSyncOrchestrator;
|
||||
env: Record<string, AnyMock>;
|
||||
dataSource: { bind: AnyMock };
|
||||
client: Record<string, AnyMock>;
|
||||
vaultRegistry: { getVault: AnyMock; vaultPath: AnyMock };
|
||||
vault: Record<string, AnyMock>;
|
||||
scheduler: Record<string, AnyMock>;
|
||||
redis: { set: AnyMock; eval: AnyMock };
|
||||
redisService: { getOrThrow: AnyMock };
|
||||
db: unknown;
|
||||
}
|
||||
|
||||
function build(opts: BuildOptions = {}): Built {
|
||||
const {
|
||||
enabled = true,
|
||||
remoteTemplate = undefined,
|
||||
dataDir = '/vaults',
|
||||
pollIntervalMs = 15000,
|
||||
debounceMs = 2000,
|
||||
vaultOverrides = {},
|
||||
spaceEnabled = true,
|
||||
} = opts;
|
||||
// Distinguish "key omitted" (default off row) from "key present but undefined"
|
||||
// (a deliberately MISSING settings row).
|
||||
const settingsRow =
|
||||
'settingsRow' in opts ? opts.settingsRow : { autoMergeConflicts: false };
|
||||
// Distinguish "key omitted" (default to a valid id) from "key present but
|
||||
// undefined" (the no-service-user test deliberately sets it undefined).
|
||||
const serviceUserId = 'serviceUserId' in opts ? opts.serviceUserId : 'svc-user';
|
||||
|
||||
const env: Record<string, AnyMock> = {
|
||||
isGitSyncEnabled: jest.fn(() => enabled),
|
||||
getGitSyncServiceUserId: jest.fn(() => serviceUserId),
|
||||
getGitSyncRemoteTemplate: jest.fn(() => remoteTemplate),
|
||||
getGitSyncDataDir: jest.fn(() => dataDir),
|
||||
getGitSyncPollIntervalMs: jest.fn(() => pollIntervalMs),
|
||||
getGitSyncDebounceMs: jest.fn(() => debounceMs),
|
||||
};
|
||||
|
||||
// The read-side / write-side client the datasource hands back.
|
||||
const client: Record<string, AnyMock> = {
|
||||
listSpaceTree: jest.fn(async () => ({ pages: [], complete: true })),
|
||||
// Historical behavior: every candidate id is a real page row. Present so the
|
||||
// fake does not rely on `listTrackedFiles -> []` to avoid the ghost-guard
|
||||
// probe, and won't break if a future test seeds tracked files.
|
||||
pageIdsExist: jest.fn(async (ids: string[]) => ids),
|
||||
deletePage: jest.fn(async () => undefined),
|
||||
createPage: jest.fn(async () => undefined),
|
||||
updatePageBody: jest.fn(async () => undefined),
|
||||
};
|
||||
const dataSource = { bind: jest.fn(() => client) };
|
||||
|
||||
// The fake VaultGit: every method the orchestrator calls is a jest.fn.
|
||||
const vault: Record<string, AnyMock> = {
|
||||
assertGitAvailable: jest.fn(async () => undefined),
|
||||
ensureRepo: jest.fn(async () => undefined),
|
||||
isMergeInProgress: jest.fn(async () => false),
|
||||
ensureBranch: jest.fn(async () => undefined),
|
||||
checkout: jest.fn(async () => undefined),
|
||||
listTrackedFiles: jest.fn(async () => []),
|
||||
pinHeadToMain: jest.fn(async () => undefined),
|
||||
...(vaultOverrides as Record<string, AnyMock>),
|
||||
};
|
||||
const vaultRegistry = {
|
||||
getVault: jest.fn(async () => vault),
|
||||
vaultPath: jest.fn((spaceId: string) => `${dataDir}/${spaceId}`),
|
||||
};
|
||||
|
||||
const scheduler: Record<string, AnyMock> = {
|
||||
addInterval: jest.fn(),
|
||||
deleteInterval: jest.fn(),
|
||||
};
|
||||
|
||||
const redis = {
|
||||
// Default: lock acquired. Tests override per-case.
|
||||
set: jest.fn(async () => 'OK'),
|
||||
eval: jest.fn(async () => 1),
|
||||
};
|
||||
const redisService = { getOrThrow: jest.fn(() => redis) };
|
||||
|
||||
// Chainable Kysely stub. TWO distinct single-row queries run through the same
|
||||
// builder and both end in `executeTakeFirst`, so the stub must tell them apart
|
||||
// by the column ALIAS the real code selects (they are otherwise identical):
|
||||
// - `isSpaceGitSyncEnabled` (per-space opt-in gate in `runOnce`, added by
|
||||
// c838fdee): `.select(sql\`...->>'enabled' = 'true'\`.as('enabled'))` ->
|
||||
// the code reads `row?.enabled`. Return `{ enabled: spaceEnabled }`.
|
||||
// - `buildSettings`: `.select(sql\`...->>'autoMergeConflicts' = 'true'\`
|
||||
// .as('autoMergeConflicts'))` -> the code reads `row?.autoMergeConflicts`.
|
||||
// Return `settingsRow` (default SAFE off; `undefined` models a missing row).
|
||||
// `enabledSpaces` uses `.select([...strings]).execute()` (no alias, no
|
||||
// executeTakeFirst) and is stubbed/replaced in the tests that exercise it.
|
||||
const db = (() => {
|
||||
let lastAlias: string | undefined;
|
||||
const aliasOf = (sel: unknown): string | undefined => {
|
||||
try {
|
||||
const node = (sel as { toOperationNode?: () => any })?.toOperationNode?.();
|
||||
return node?.kind === 'AliasNode' ? node.alias?.name : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
const builder: any = {
|
||||
select: (sel: unknown) => {
|
||||
const alias = aliasOf(sel);
|
||||
if (alias) lastAlias = alias;
|
||||
return builder;
|
||||
},
|
||||
where: () => builder,
|
||||
executeTakeFirst: async () =>
|
||||
lastAlias === 'enabled' ? { enabled: spaceEnabled } : settingsRow,
|
||||
execute: async () => [],
|
||||
};
|
||||
return { selectFrom: () => builder };
|
||||
})();
|
||||
|
||||
// The REAL SpaceLockService, constructed against the mock redis above, so all
|
||||
// existing lock assertions (lock-held, in-progress, leader lock, release CAS,
|
||||
// heartbeat) still exercise the same `redis.set`/`redis.eval` mock unchanged.
|
||||
const spaceLock = new SpaceLockService(redisService as any);
|
||||
|
||||
const orchestrator = new GitSyncOrchestrator(
|
||||
env as any,
|
||||
dataSource as any,
|
||||
vaultRegistry as any,
|
||||
scheduler as any,
|
||||
spaceLock as any,
|
||||
db as any,
|
||||
);
|
||||
|
||||
return {
|
||||
orchestrator,
|
||||
env,
|
||||
dataSource,
|
||||
client,
|
||||
vaultRegistry,
|
||||
vault,
|
||||
scheduler,
|
||||
redis,
|
||||
redisService,
|
||||
db,
|
||||
};
|
||||
}
|
||||
|
||||
/** The engine runs a clean cycle by default. */
|
||||
function primeEngineHappyPath(): void {
|
||||
runCycleMock.mockResolvedValue(OK_CYCLE);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
primeEngineHappyPath();
|
||||
});
|
||||
|
||||
describe('GitSyncOrchestrator', () => {
|
||||
describe('runOnce gating', () => {
|
||||
it("short-circuits with skipped:'disabled' when git-sync is disabled", async () => {
|
||||
const { orchestrator, redis, vaultRegistry } = build({ enabled: false });
|
||||
const res = await orchestrator.runOnce('space-1', 'ws-1');
|
||||
expect(res).toEqual({ spaceId: 'space-1', ran: false, skipped: 'disabled' });
|
||||
// No lock, no vault work performed.
|
||||
expect(redis.set).not.toHaveBeenCalled();
|
||||
expect(vaultRegistry.getVault).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns skipped:'no-service-user' when the service user id is falsy", async () => {
|
||||
const { orchestrator, redis } = build({ serviceUserId: undefined });
|
||||
const res = await orchestrator.runOnce('space-1', 'ws-1');
|
||||
expect(res).toEqual({
|
||||
spaceId: 'space-1',
|
||||
ran: false,
|
||||
skipped: 'no-service-user',
|
||||
});
|
||||
expect(redis.set).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('in-process mutex', () => {
|
||||
it("a second runOnce while the first is in-flight returns skipped:'in-progress'", async () => {
|
||||
const built = build();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
// Hang the first cycle inside driveCycle by stalling getVault.
|
||||
built.vaultRegistry.getVault.mockImplementationOnce(async () => {
|
||||
await gate;
|
||||
return built.vault;
|
||||
});
|
||||
|
||||
const first = built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
// Let the first call enter the running set + acquire the lock.
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
|
||||
const second = await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
expect(second).toEqual({
|
||||
spaceId: 'space-1',
|
||||
ran: false,
|
||||
skipped: 'in-progress',
|
||||
});
|
||||
|
||||
release();
|
||||
await first;
|
||||
});
|
||||
});
|
||||
|
||||
describe('redis leader lock', () => {
|
||||
it("returns skipped:'lock-held' and cleans up the mutex when the lock is not acquired", async () => {
|
||||
const built = build();
|
||||
// First acquire fails (not 'OK'); a later acquire succeeds.
|
||||
built.redis.set
|
||||
.mockResolvedValueOnce(null)
|
||||
.mockResolvedValue('OK');
|
||||
|
||||
const res = await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
expect(res).toEqual({
|
||||
spaceId: 'space-1',
|
||||
ran: false,
|
||||
skipped: 'lock-held',
|
||||
});
|
||||
// The mutex must be clear: a subsequent call can acquire + run.
|
||||
const res2 = await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
expect(res2.ran).toBe(true);
|
||||
expect(res2.skipped).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('poisoned-space protection', () => {
|
||||
it('releases the lock and clears the mutex when the cycle throws, returning { error }', async () => {
|
||||
const built = build();
|
||||
jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
|
||||
runCycleMock.mockRejectedValueOnce(new Error('boom'));
|
||||
|
||||
const res = await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
expect(res.ran).toBe(false);
|
||||
expect(res.error).toBe('boom');
|
||||
// CAS release was invoked (eval) and the space is no longer "running":
|
||||
expect(built.redis.eval).toHaveBeenCalledTimes(1);
|
||||
|
||||
// A subsequent call can re-acquire (mutex cleared after the throw).
|
||||
runCycleMock.mockResolvedValue(OK_CYCLE);
|
||||
const res2 = await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
expect(res2.ran).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('cycle wiring', () => {
|
||||
it('drives runCycle with the space vault, the bound client, and settings', async () => {
|
||||
const built = build();
|
||||
await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
|
||||
expect(runCycleMock).toHaveBeenCalledTimes(1);
|
||||
const [deps] = runCycleMock.mock.calls[0];
|
||||
expect(deps.spaceId).toBe('space-1');
|
||||
expect(deps.vault).toBe(built.vault);
|
||||
expect(deps.client).toBe(built.client);
|
||||
expect(deps.settings.vaultPath).toBe('/vaults/space-1');
|
||||
// The bound datasource identity is the (workspace, service-user) pair,
|
||||
// plus the reconciling spaceId used by deletePage's cross-move guard.
|
||||
expect(built.dataSource.bind).toHaveBeenCalledWith({
|
||||
workspaceId: 'ws-1',
|
||||
userId: 'svc-user',
|
||||
spaceId: 'space-1',
|
||||
});
|
||||
});
|
||||
|
||||
it('threads autoMergeConflicts:true from the space settings row into the engine settings', async () => {
|
||||
const built = build({ settingsRow: { autoMergeConflicts: true } });
|
||||
await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
const [deps] = runCycleMock.mock.calls[0];
|
||||
expect(deps.settings.autoMergeConflicts).toBe(true);
|
||||
});
|
||||
|
||||
it('defaults autoMergeConflicts to false when the settings row is missing', async () => {
|
||||
const built = build({ settingsRow: undefined });
|
||||
await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
const [deps] = runCycleMock.mock.calls[0];
|
||||
expect(deps.settings.autoMergeConflicts).toBe(false);
|
||||
});
|
||||
|
||||
it("escalates a divergent-`docmost` push refusal to WARN and surfaces the flag in the status", async () => {
|
||||
const built = build();
|
||||
const warnSpy = jest
|
||||
.spyOn(Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
// The engine refused to fast-forward a divergent `docmost` mirror (§5).
|
||||
runCycleMock.mockResolvedValue({ ...OK_CYCLE, divergentDocmost: true });
|
||||
|
||||
const res = await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
|
||||
// The flag is surfaced in the returned status (consumable by /status).
|
||||
expect(res.divergentDocmost).toBe(true);
|
||||
// And escalated from the engine's info `log` to a WARN naming the space.
|
||||
expect(warnSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('DIVERGENT'),
|
||||
);
|
||||
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('space-1'));
|
||||
});
|
||||
|
||||
it("does NOT warn when the cycle is clean (divergentDocmost falsy)", async () => {
|
||||
const built = build();
|
||||
const warnSpy = jest
|
||||
.spyOn(Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
runCycleMock.mockResolvedValue(OK_CYCLE);
|
||||
|
||||
const res = await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
|
||||
expect(res.divergentDocmost).toBeUndefined();
|
||||
expect(warnSpy).not.toHaveBeenCalledWith(
|
||||
expect.stringContaining('DIVERGENT'),
|
||||
);
|
||||
});
|
||||
|
||||
it("surfaces the engine's skipped status (e.g. merge-in-progress) verbatim", async () => {
|
||||
const built = build();
|
||||
runCycleMock.mockResolvedValue({ ran: false, skipped: 'merge-in-progress' });
|
||||
|
||||
const res = await built.orchestrator.runOnce('space-1', 'ws-1');
|
||||
expect(res).toEqual({
|
||||
spaceId: 'space-1',
|
||||
ran: false,
|
||||
skipped: 'merge-in-progress',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('ingestExternalPush', () => {
|
||||
it('streams the receive-pack FIRST, then runs the Docmost cycle', async () => {
|
||||
const order: string[] = [];
|
||||
const built = build();
|
||||
runCycleMock.mockImplementation(async () => {
|
||||
order.push('cycle');
|
||||
return OK_CYCLE;
|
||||
});
|
||||
const runReceivePack = jest.fn(async () => {
|
||||
order.push('receive-pack');
|
||||
});
|
||||
|
||||
await built.orchestrator.ingestExternalPush('space-1', 'ws-1', runReceivePack);
|
||||
|
||||
expect(runReceivePack).toHaveBeenCalledTimes(1);
|
||||
// The cycle only runs AFTER the push commits land on main.
|
||||
expect(order).toEqual(['receive-pack', 'cycle']);
|
||||
});
|
||||
|
||||
// Explicit timeout: ingestExternalPush exhausts the full bounded
|
||||
// acquire-retry budget (GIT_SYNC_PUSH_LOCK_RETRY_TOTAL_MS = 5_000ms) before it
|
||||
// gives up and throws, which races jest's DEFAULT 5_000ms test timeout — flaky
|
||||
// on a loaded/slow runner. Give it headroom so it deterministically observes
|
||||
// the eventual LockHeldError instead of timing out first.
|
||||
it('throws GitSyncLockHeldError and does NOT run the receive-pack when the lock is held', async () => {
|
||||
const built = build();
|
||||
built.redis.set.mockResolvedValue(null); // acquire fails → lock-held
|
||||
const runReceivePack = jest.fn(async () => undefined);
|
||||
|
||||
await expect(
|
||||
built.orchestrator.ingestExternalPush('space-1', 'ws-1', runReceivePack),
|
||||
).rejects.toBeInstanceOf(GitSyncLockHeldError);
|
||||
|
||||
// We must never write to the working tree concurrently with a cycle.
|
||||
expect(runReceivePack).not.toHaveBeenCalled();
|
||||
expect(runCycleMock).not.toHaveBeenCalled();
|
||||
}, 15_000);
|
||||
|
||||
it('swallows a post-push cycle error (the push is durable; poll retries)', async () => {
|
||||
jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
|
||||
const built = build();
|
||||
// The cycle throws AFTER the receive-pack already succeeded.
|
||||
runCycleMock.mockRejectedValueOnce(new Error('cycle boom'));
|
||||
const runReceivePack = jest.fn(async () => undefined);
|
||||
|
||||
// Does NOT throw — the durable push must not be reported as failed.
|
||||
await expect(
|
||||
built.orchestrator.ingestExternalPush('space-1', 'ws-1', runReceivePack),
|
||||
).resolves.toBeUndefined();
|
||||
expect(runReceivePack).toHaveBeenCalledTimes(1);
|
||||
// Lock was still released (CAS eval) despite the cycle error.
|
||||
expect(built.redis.eval).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('runs the receive-pack but SKIPS the cycle when no service user is configured', async () => {
|
||||
jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
|
||||
const built = build({ serviceUserId: undefined });
|
||||
const runReceivePack = jest.fn(async () => undefined);
|
||||
|
||||
await expect(
|
||||
built.orchestrator.ingestExternalPush('space-1', 'ws-1', runReceivePack),
|
||||
).resolves.toBeUndefined();
|
||||
// The push is durable on main; the immediate cycle is skipped, not failed.
|
||||
expect(runReceivePack).toHaveBeenCalledTimes(1);
|
||||
expect(runCycleMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses (LockHeldError) and runs nothing when git-sync is globally disabled', async () => {
|
||||
const built = build({ enabled: false });
|
||||
const runReceivePack = jest.fn(async () => undefined);
|
||||
|
||||
await expect(
|
||||
built.orchestrator.ingestExternalPush('space-1', 'ws-1', runReceivePack),
|
||||
).rejects.toBeInstanceOf(GitSyncLockHeldError);
|
||||
expect(runReceivePack).not.toHaveBeenCalled();
|
||||
expect(built.redis.set).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('remote template substitution', () => {
|
||||
it('substitutes {spaceId} into the gitRemote settings handed to the engine', async () => {
|
||||
const built = build({ remoteTemplate: 'git@h:vault-{spaceId}.git' });
|
||||
await built.orchestrator.runOnce('space-42', 'ws-1');
|
||||
const [deps] = runCycleMock.mock.calls[0];
|
||||
expect(deps.settings.gitRemote).toBe('git@h:vault-space-42.git');
|
||||
});
|
||||
});
|
||||
|
||||
describe('serveReadAdvertisement (bug #3 — stable advertised HEAD)', () => {
|
||||
it('pins HEAD to main and serves under the space lock', async () => {
|
||||
const built = build();
|
||||
const serve = jest.fn(async () => undefined);
|
||||
|
||||
await built.orchestrator.serveReadAdvertisement('space-1', serve);
|
||||
|
||||
// The lock was taken (redis SET NX) and released (CAS eval).
|
||||
expect(built.redis.set).toHaveBeenCalledTimes(1);
|
||||
expect(built.redis.eval).toHaveBeenCalled();
|
||||
// HEAD pinned BEFORE serving, on the right vault.
|
||||
expect(built.vaultRegistry.getVault).toHaveBeenCalledWith('space-1');
|
||||
expect(built.vault.pinHeadToMain).toHaveBeenCalledTimes(1);
|
||||
expect(serve).toHaveBeenCalledTimes(1);
|
||||
const pinOrder = built.vault.pinHeadToMain.mock.invocationCallOrder[0];
|
||||
const serveOrder = serve.mock.invocationCallOrder[0];
|
||||
expect(pinOrder).toBeLessThan(serveOrder);
|
||||
});
|
||||
|
||||
it('serves WITHOUT a pin/lock when git-sync is globally disabled', async () => {
|
||||
const built = build({ enabled: false });
|
||||
const serve = jest.fn(async () => undefined);
|
||||
|
||||
await built.orchestrator.serveReadAdvertisement('space-1', serve);
|
||||
|
||||
expect(serve).toHaveBeenCalledTimes(1);
|
||||
expect(built.redis.set).not.toHaveBeenCalled();
|
||||
expect(built.vault.pinHeadToMain).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('module lifecycle', () => {
|
||||
it('registers exactly one interval on init and tears it down idempotently on destroy', () => {
|
||||
const built = build();
|
||||
jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined);
|
||||
|
||||
built.orchestrator.onModuleInit();
|
||||
expect(built.scheduler.addInterval).toHaveBeenCalledTimes(1);
|
||||
const [name] = built.scheduler.addInterval.mock.calls[0];
|
||||
|
||||
built.orchestrator.onModuleDestroy();
|
||||
expect(built.scheduler.deleteInterval).toHaveBeenCalledTimes(1);
|
||||
expect(built.scheduler.deleteInterval).toHaveBeenCalledWith(name);
|
||||
|
||||
// A second destroy is a no-op (guard against double-delete).
|
||||
built.orchestrator.onModuleDestroy();
|
||||
expect(built.scheduler.deleteInterval).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('registers nothing on init when git-sync is disabled', () => {
|
||||
const built = build({ enabled: false });
|
||||
built.orchestrator.onModuleInit();
|
||||
expect(built.scheduler.addInterval).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// The poll-safety backstop: each tick enumerates the STRICT opt-in spaces and
|
||||
// reconciles each one under its own lock. We drive the private `pollTick()`
|
||||
// directly and (separately) compile `enabledSpaces()` to assert its opt-in SQL.
|
||||
describe('pollTick + enabledSpaces (strict opt-in backstop)', () => {
|
||||
it('runs runOnce exactly once per enabled space, with the right (spaceId, workspaceId)', async () => {
|
||||
const built = build();
|
||||
// Isolate the tick wiring from the cycle machinery: stub the enumeration
|
||||
// and count runOnce (it never throws; here we don't exercise its body).
|
||||
const runOnce = jest
|
||||
.spyOn(built.orchestrator, 'runOnce')
|
||||
.mockResolvedValue({ spaceId: 'x', ran: true });
|
||||
jest
|
||||
.spyOn(built.orchestrator as any, 'enabledSpaces')
|
||||
.mockResolvedValue([
|
||||
{ spaceId: 'space-1', workspaceId: 'ws-1' },
|
||||
{ spaceId: 'space-2', workspaceId: 'ws-2' },
|
||||
]);
|
||||
|
||||
await (built.orchestrator as any).pollTick();
|
||||
|
||||
expect(runOnce).toHaveBeenCalledTimes(2);
|
||||
// Per-space isolation: each space is reconciled with its OWN workspace id.
|
||||
expect(runOnce).toHaveBeenNthCalledWith(1, 'space-1', 'ws-1');
|
||||
expect(runOnce).toHaveBeenNthCalledWith(2, 'space-2', 'ws-2');
|
||||
});
|
||||
|
||||
it('skips an overlapping tick while a previous pass is still in flight (re-entrancy guard)', async () => {
|
||||
const built = build();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
// Stall the first pass inside enabledSpaces so a second tick fires while it
|
||||
// is still running.
|
||||
const enabledSpy = jest
|
||||
.spyOn(built.orchestrator as any, 'enabledSpaces')
|
||||
.mockImplementation(async () => {
|
||||
await gate;
|
||||
return [{ spaceId: 'space-1', workspaceId: 'ws-1' }];
|
||||
});
|
||||
const runOnce = jest
|
||||
.spyOn(built.orchestrator, 'runOnce')
|
||||
.mockResolvedValue({ spaceId: 'space-1', ran: true });
|
||||
|
||||
const first = (built.orchestrator as any).pollTick();
|
||||
await Promise.resolve(); // let the first pass set polling=true + await gate
|
||||
|
||||
// A second tick during the first must be skipped: it never even enumerates.
|
||||
await (built.orchestrator as any).pollTick();
|
||||
expect(enabledSpy).toHaveBeenCalledTimes(1);
|
||||
|
||||
release();
|
||||
await first;
|
||||
expect(runOnce).toHaveBeenCalledTimes(1);
|
||||
|
||||
// After the first pass cleared the flag, a fresh tick runs normally.
|
||||
await (built.orchestrator as any).pollTick();
|
||||
expect(enabledSpy).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does NOT throw and runs nothing when the enabled-spaces query throws (try/catch backstop)', async () => {
|
||||
jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
|
||||
const built = build();
|
||||
const runOnce = jest.spyOn(built.orchestrator, 'runOnce');
|
||||
jest
|
||||
.spyOn(built.orchestrator as any, 'enabledSpaces')
|
||||
.mockRejectedValue(new Error('db down'));
|
||||
|
||||
// A failed enumeration must never break the interval — pollTick swallows it.
|
||||
await expect(
|
||||
(built.orchestrator as any).pollTick(),
|
||||
).resolves.toBeUndefined();
|
||||
expect(runOnce).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('early-returns (no enumeration, no runOnce) when git-sync is disabled', async () => {
|
||||
const built = build({ enabled: false });
|
||||
const enabled = jest.spyOn(built.orchestrator as any, 'enabledSpaces');
|
||||
const runOnce = jest.spyOn(built.orchestrator, 'runOnce');
|
||||
|
||||
await (built.orchestrator as any).pollTick();
|
||||
|
||||
// Gated on the master switch before any DB work.
|
||||
expect(enabled).not.toHaveBeenCalled();
|
||||
expect(runOnce).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('compiles the STRICT opt-in enumeration SQL (spaces, deletedAt is null, enabled flag)', async () => {
|
||||
// Inject a compile-only Kysely (DummyDriver) whose `log` hook captures the
|
||||
// exact SQL `enabledSpaces()` runs — no fake builder, the real query is
|
||||
// compiled. DummyDriver yields no rows; we only assert the SQL shape.
|
||||
const built = build();
|
||||
let captured: CompiledQuery | undefined;
|
||||
const compileDb = new Kysely<any>({
|
||||
dialect: {
|
||||
createAdapter: () => new PostgresAdapter(),
|
||||
createDriver: () => new DummyDriver(),
|
||||
createIntrospector: (d) => new PostgresIntrospector(d),
|
||||
createQueryCompiler: () => new PostgresQueryCompiler(),
|
||||
},
|
||||
log: (event) => {
|
||||
if (event.level === 'query') captured = event.query as CompiledQuery;
|
||||
},
|
||||
});
|
||||
// Swap the orchestrator's injected db for the compile-only instance.
|
||||
(built.orchestrator as any).db = compileDb;
|
||||
|
||||
const rows = await (built.orchestrator as any).enabledSpaces();
|
||||
// DummyDriver returns no rows -> empty opt-in list (the no-space default).
|
||||
expect(rows).toEqual([]);
|
||||
|
||||
expect(captured).toBeDefined();
|
||||
const sql = captured!.sql.replace(/\s+/g, ' ');
|
||||
expect(sql).toContain('from "spaces"');
|
||||
// deletedAt-is-null guard (live spaces only).
|
||||
expect(sql).toContain('"deletedAt" is null');
|
||||
// STRICT per-space opt-in: the raw jsonb flag predicate, verbatim.
|
||||
expect(sql).toContain(`settings->'gitSync'->>'enabled' = 'true'`);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,575 @@
|
||||
import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleDestroy,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { SchedulerRegistry } from '@nestjs/schedule';
|
||||
import {
|
||||
lstat,
|
||||
mkdir,
|
||||
readFile,
|
||||
realpath,
|
||||
rm,
|
||||
writeFile,
|
||||
} from 'node:fs/promises';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { sql } from 'kysely';
|
||||
import type { Settings } from '@docmost/git-sync';
|
||||
import { loadGitSync } from '../git-sync.loader';
|
||||
import { EnvironmentService } from '../../environment/environment.service';
|
||||
import { GitmostDataSourceService } from './gitmost-datasource.service';
|
||||
import { VaultRegistryService } from './vault-registry.service';
|
||||
import { SpaceLockService } from './space-lock.service';
|
||||
import {
|
||||
GIT_SYNC_PUSH_LOCK_RETRY_BASE_MS,
|
||||
GIT_SYNC_PUSH_LOCK_RETRY_MAX_MS,
|
||||
GIT_SYNC_PUSH_LOCK_RETRY_TOTAL_MS,
|
||||
} from '../git-sync.constants';
|
||||
|
||||
/** A space the poll loop should reconcile: its id + the workspace it lives in. */
|
||||
interface EnabledSpace {
|
||||
spaceId: string;
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thrown by `ingestExternalPush` when the per-space lock cannot be acquired (a
|
||||
* poll cycle is mid-flight on this or another replica). The /git HTTP handler
|
||||
* maps it to a 503 so the git client retries rather than racing a cycle's
|
||||
* working-tree checkout/merge.
|
||||
*/
|
||||
export class GitSyncLockHeldError extends Error {
|
||||
constructor(public readonly spaceId: string) {
|
||||
super(`git-sync: space ${spaceId} is busy (lock held); retry the push`);
|
||||
this.name = 'GitSyncLockHeldError';
|
||||
}
|
||||
}
|
||||
|
||||
/** Small status summary returned by `runOnce` (for the admin trigger + logs). */
|
||||
export interface GitSyncRunStatus {
|
||||
spaceId: string;
|
||||
ran: boolean;
|
||||
/** Why the cycle did not run (lock held elsewhere, busy, disabled, error). */
|
||||
skipped?:
|
||||
| 'lock-held'
|
||||
| 'in-progress'
|
||||
| 'disabled'
|
||||
| 'no-service-user'
|
||||
| 'space-not-enabled'
|
||||
| 'merge-in-progress';
|
||||
pull?: { written: number; deleted: number; conflict: boolean };
|
||||
push?: { mode: string; failures: number };
|
||||
/**
|
||||
* True when the push REFUSED to fast-forward a divergent `docmost` mirror
|
||||
* (invariant §5 broken — `docmost` no longer mirrors what Docmost contains).
|
||||
* Surfaced here (not just logged) so /status can report it. No data is lost,
|
||||
* but it signals an operator-visible drift that needs attention.
|
||||
*/
|
||||
divergentDocmost?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The git-sync control plane. Drives the vendored engine in
|
||||
* process: under a Redis leader lock (single-writer across replicas) plus an
|
||||
* in-process per-space mutex (no overlapping cycles on one instance), it runs a
|
||||
* PULL (Docmost -> vault) then a PUSH (vault -> Docmost) for a space.
|
||||
*
|
||||
* Enumeration of enabled spaces: STRICT opt-in. Only spaces whose
|
||||
* per-space flag `space.settings.gitSync.enabled === true` (written by the Phase-C
|
||||
* UI) are reconciled. There is intentionally NO all-spaces fallback: when no space
|
||||
* carries the flag, git-sync does NOTHING (an empty list) — flagging every space
|
||||
* the moment GIT_SYNC_ENABLED flips on is a safety hazard (it could mass-sync large
|
||||
* spaces). The whole loop is still gated on the GIT_SYNC_ENABLED master switch
|
||||
* first; per-space opt-in is now REQUIRED on top of it.
|
||||
*/
|
||||
@Injectable()
|
||||
export class GitSyncOrchestrator implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(GitSyncOrchestrator.name);
|
||||
/** The registered poll-interval name, or null when none is registered. */
|
||||
private pollIntervalName: string | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
private readonly dataSource: GitmostDataSourceService,
|
||||
private readonly vaultRegistry: VaultRegistryService,
|
||||
private readonly schedulerRegistry: SchedulerRegistry,
|
||||
private readonly spaceLock: SpaceLockService,
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
) {}
|
||||
|
||||
// --- enabled-space enumeration --------------------------------
|
||||
|
||||
/**
|
||||
* Enumerate the spaces the poll loop should reconcile. STRICT opt-in: ONLY
|
||||
* spaces carrying the Phase-C per-space flag (`settings->'gitSync'->>'enabled'
|
||||
* = 'true'`, written by the Phase-C UI) are returned. There is intentionally NO
|
||||
* fallback to "all live spaces" — when no space is flagged this returns an empty
|
||||
* list and git-sync does nothing (correct opt-in behavior). The GIT_SYNC_ENABLED
|
||||
* master switch gates whether the loop runs at all; this flag gates which spaces.
|
||||
*/
|
||||
private async enabledSpaces(): Promise<EnabledSpace[]> {
|
||||
return this.db
|
||||
.selectFrom('spaces')
|
||||
.select(['id as spaceId', 'workspaceId'])
|
||||
.where('deletedAt', 'is', null)
|
||||
.where(sql<boolean>`settings->'gitSync'->>'enabled' = 'true'`)
|
||||
.execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* Authoritative per-space opt-in check used by the EVENT path (runOnce), which
|
||||
* — unlike the poll path — is not pre-filtered by enabledSpaces(). Same STRICT
|
||||
* `settings.gitSync.enabled === 'true'` semantics; a missing/soft-deleted space
|
||||
* or any non-'true' value returns false (review #3).
|
||||
*/
|
||||
private async isSpaceGitSyncEnabled(spaceId: string): Promise<boolean> {
|
||||
const row = await this.db
|
||||
.selectFrom('spaces')
|
||||
.select(
|
||||
sql<boolean>`settings->'gitSync'->>'enabled' = 'true'`.as('enabled'),
|
||||
)
|
||||
.where('id', '=', spaceId)
|
||||
.where('deletedAt', 'is', null)
|
||||
.executeTakeFirst();
|
||||
return row?.enabled ?? false;
|
||||
}
|
||||
|
||||
// --- one sync cycle for a space -------------------------------
|
||||
|
||||
/**
|
||||
* Build the engine `Settings` for a space. The datasource writes in-process,
|
||||
* so only `vaultPath`, `docmostSpaceId` and the tunables are load-bearing; the
|
||||
* dead REST-era fields (docmostApiUrl/email/password) were removed (review).
|
||||
*
|
||||
* `gitRemote` is NOT yet consumed: the vendored engine has no remote-push path
|
||||
* (see engine/git.ts, engine/pull.ts, SPEC §7 — remote push is deferred), so
|
||||
* the GIT_SYNC_REMOTE_TEMPLATE env -> validation -> getter -> this field chain
|
||||
* is inert SCAFFOLDING kept in place for the future remote-push feature. It is
|
||||
* harmless (the engine ignores it) and removing it would only churn; we still
|
||||
* populate it so the wiring is ready when the engine grows a push path.
|
||||
*/
|
||||
private async buildSettings(spaceId: string): Promise<Settings> {
|
||||
// Scaffolding for the deferred remote-push feature — the engine does not read
|
||||
// `gitRemote` yet (see the docstring above). Substitute {spaceId} per-space so
|
||||
// the value is correct the moment the engine starts consuming it.
|
||||
const remoteTemplate = this.environmentService.getGitSyncRemoteTemplate();
|
||||
const gitRemote = remoteTemplate
|
||||
? remoteTemplate.replace(/\{spaceId\}/g, spaceId)
|
||||
: undefined;
|
||||
// Per-space PUSH policy for still-conflicted page bodies (SPEC §9): read the
|
||||
// `gitSync.autoMergeConflicts` flag from the space's jsonb settings. STRICT
|
||||
// opt-in like `enabled` — anything other than the literal 'true' (absent, null,
|
||||
// 'false') resolves to the SAFE default (skip a conflicted page, do not push).
|
||||
const row = await this.db
|
||||
.selectFrom('spaces')
|
||||
.select(
|
||||
sql<boolean>`settings->'gitSync'->>'autoMergeConflicts' = 'true'`.as(
|
||||
'autoMergeConflicts',
|
||||
),
|
||||
)
|
||||
.where('id', '=', spaceId)
|
||||
.executeTakeFirst();
|
||||
return {
|
||||
docmostSpaceId: spaceId,
|
||||
vaultPath: this.vaultRegistry.vaultPath(spaceId),
|
||||
gitRemote,
|
||||
pollIntervalMs: this.environmentService.getGitSyncPollIntervalMs(),
|
||||
debounceMs: this.environmentService.getGitSyncDebounceMs(),
|
||||
logLevel: 'info',
|
||||
autoMergeConflicts: row?.autoMergeConflicts ?? false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one full PULL + PUSH cycle for a space, under the Redis leader lock and
|
||||
* the in-process mutex. Never throws — per-space errors are caught, logged, and
|
||||
* returned in the status so a poll interval is never broken by one bad space.
|
||||
*/
|
||||
async runOnce(
|
||||
spaceId: string,
|
||||
workspaceId: string,
|
||||
): Promise<GitSyncRunStatus> {
|
||||
if (!this.environmentService.isGitSyncEnabled()) {
|
||||
return { spaceId, ran: false, skipped: 'disabled' };
|
||||
}
|
||||
const serviceUserId = this.environmentService.getGitSyncServiceUserId();
|
||||
if (!serviceUserId) {
|
||||
this.logger.error(
|
||||
'git-sync: GIT_SYNC_SERVICE_USER_ID is required when GIT_SYNC_ENABLED — skipping',
|
||||
);
|
||||
return { spaceId, ran: false, skipped: 'no-service-user' };
|
||||
}
|
||||
// Per-space opt-in gate (review #3). The global GIT_SYNC_ENABLED env switch is
|
||||
// NOT sufficient — sync must also be turned on for THIS space
|
||||
// (`space.settings.gitSync.enabled === true`). The poll path filters via
|
||||
// enabledSpaces(), but the event path (page-change listener) calls runOnce
|
||||
// directly; without this check an edit in ANY space would `git init` a vault
|
||||
// and export that space's whole history, violating the module's opt-in
|
||||
// contract (orchestrator docstring §79-86). STRICT: anything but 'true' skips.
|
||||
if (!(await this.isSpaceGitSyncEnabled(spaceId))) {
|
||||
return { spaceId, ran: false, skipped: 'space-not-enabled' };
|
||||
}
|
||||
|
||||
// Run the full cycle under the per-space lock. withSpaceLock owns the
|
||||
// in-process mutex (no overlapping cycles on this instance) AND the Redis
|
||||
// leader lock (single writer across replicas), and returns a skip sentinel
|
||||
// when it could not enter — surfaced here as the existing skipped:'in-progress'
|
||||
// / 'lock-held' status so runOnce's observable behavior is unchanged.
|
||||
try {
|
||||
const result = await this.spaceLock.withSpaceLock(spaceId, (signal) =>
|
||||
this.driveCycle(spaceId, workspaceId, serviceUserId, signal),
|
||||
);
|
||||
if ('skipped' in result && !('spaceId' in result)) {
|
||||
return { spaceId, ran: false, skipped: result.skipped };
|
||||
}
|
||||
return result;
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
this.logger.error(`git-sync: cycle failed for space ${spaceId}: ${message}`);
|
||||
return { spaceId, ran: false, error: message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingest a push that arrived over smart-HTTP (the /git host). Under the SAME
|
||||
* per-space lock the poll cycle uses, it:
|
||||
* 1. runs `runReceivePack()` — the closure that spawns `git http-backend` for
|
||||
* the receive-pack request and finishes streaming the HTTP response to the
|
||||
* client. The client's push result is determined here.
|
||||
* 2. THEN — still holding the lock — runs the full Docmost cycle (the same
|
||||
* `driveCycle` body `runOnce` uses) so the freshly received commits on
|
||||
* `main` flow back into Docmost pages.
|
||||
*
|
||||
* If the cycle body in step 2 throws, it is LOGGED but NOT rethrown: the push
|
||||
* already succeeded and the commits are durable on `main`, so the poll-interval
|
||||
* backstop will reconcile them on the next tick. The receive-pack itself is the
|
||||
* load-bearing step.
|
||||
*
|
||||
* Lock contention: if the lock cannot be acquired (a poll cycle is mid-flight),
|
||||
* this throws a `GitSyncLockHeldError`. The HTTP handler converts that to a 503
|
||||
* so git surfaces a retryable error to the user (chosen over blocking the
|
||||
* request behind a potentially long cycle). The receive-pack is NOT run when
|
||||
* the lock is held — we never write to the working tree concurrently with a
|
||||
* cycle.
|
||||
*
|
||||
* `runReceivePack` receives the per-space lock's lost-lock `AbortSignal`: a
|
||||
* receive-pack writes `main`'s working tree (receive.denyCurrentBranch=
|
||||
* updateInstead), so if the lock is lost mid-push (a long Redis outage drops the
|
||||
* heartbeat CAS) the signal fires and the receive-pack's `git http-backend`
|
||||
* child is killed — closing the window where another replica could grab the lock
|
||||
* and start a cycle while this child is still writing the working tree.
|
||||
*/
|
||||
async ingestExternalPush(
|
||||
spaceId: string,
|
||||
workspaceId: string,
|
||||
runReceivePack: (signal: AbortSignal) => Promise<void>,
|
||||
): Promise<void> {
|
||||
if (!this.environmentService.isGitSyncEnabled()) {
|
||||
// The HTTP gate already checks this, but be defensive: never run a cycle
|
||||
// when sync is globally off.
|
||||
throw new GitSyncLockHeldError(spaceId);
|
||||
}
|
||||
const serviceUserId = this.environmentService.getGitSyncServiceUserId();
|
||||
|
||||
const result = await this.spaceLock.withSpaceLock(
|
||||
spaceId,
|
||||
async (signal) => {
|
||||
// 1) Stream the receive-pack to the client (durable commits land on main).
|
||||
// Pass the lost-lock signal so the receive-pack child is killed if the lock
|
||||
// lapses mid-write (no concurrent working-tree writer across replicas).
|
||||
await runReceivePack(signal);
|
||||
|
||||
// 2) Reconcile the new commits into Docmost. A service user is required to
|
||||
// attribute the writes; without one we cannot run the cycle — the commits
|
||||
// are still durable and the poll backstop will pick them up once configured.
|
||||
if (!serviceUserId) {
|
||||
this.logger.error(
|
||||
'git-sync: GIT_SYNC_SERVICE_USER_ID is required to ingest an external ' +
|
||||
'push — the push is durable on main; skipping the immediate cycle.',
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await this.driveCycle(spaceId, workspaceId, serviceUserId, signal);
|
||||
} catch (err) {
|
||||
// Do NOT rethrow: the push succeeded and the commits are durable on main;
|
||||
// the poll-interval backstop retries the cycle. Log for visibility.
|
||||
this.logger.error(
|
||||
`git-sync: post-push cycle failed for space ${spaceId} (push is ` +
|
||||
`durable; poll will retry): ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
},
|
||||
// BOUNDED retry-acquire (push path only): a push that briefly overlaps a
|
||||
// poll cycle waits a moment (capped backoff up to the budget) instead of
|
||||
// immediately 503-ing — the cycle releases the lock in well under a second
|
||||
// for most spaces, so this turns a transient overlap into a SUCCESS rather
|
||||
// than a spurious failure. A genuinely long/stuck cycle still skips after
|
||||
// the bound -> GitSyncLockHeldError -> 503, and git retries the whole push
|
||||
// (the receive-pack only runs once the lock is held, so there is never a
|
||||
// half-applied ref on a 503).
|
||||
{
|
||||
acquireRetry: {
|
||||
timeoutMs: GIT_SYNC_PUSH_LOCK_RETRY_TOTAL_MS,
|
||||
baseMs: GIT_SYNC_PUSH_LOCK_RETRY_BASE_MS,
|
||||
maxMs: GIT_SYNC_PUSH_LOCK_RETRY_MAX_MS,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// The lock was held (in-progress or another replica) — surface to the caller
|
||||
// so the HTTP handler can answer 503 and let git retry.
|
||||
if (typeof result === 'object' && result !== null && 'skipped' in result) {
|
||||
throw new GitSyncLockHeldError(spaceId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Serve a git smart-HTTP READ ADVERTISEMENT (`GET info/refs?service=git-upload-pack`
|
||||
* or a dumb `GET HEAD`) with the repo's symbolic `HEAD` deterministically pinned
|
||||
* to `main` (bug #3). The advertised `HEAD` symref decides a clone's default
|
||||
* branch; the engine transiently checks out the read-only `docmost` mirror during
|
||||
* a cycle, so an unsynchronized advertisement could route a clone to `docmost`
|
||||
* (~1/4 of clones under continuous syncing).
|
||||
*
|
||||
* Running the pin + the advertisement under the SAME per-space lock the cycle
|
||||
* uses guarantees no cycle is mid-flight while we pin (HEAD cannot flap) and that
|
||||
* the pin never corrupts a cycle's checkout. The advertisement is cheap (a ref
|
||||
* listing, no pack stream), so holding the lock for it is fine. A bounded
|
||||
* retry-acquire absorbs a brief overlap with a cycle; if the lock still cannot be
|
||||
* taken (a long cycle), we fall back to serving WITHOUT the pin — the cycle's
|
||||
* finally-restore leaves HEAD on `main` between cycles, so the advertisement is
|
||||
* still almost always correct (degrades only under sustained contention).
|
||||
*/
|
||||
async serveReadAdvertisement(
|
||||
spaceId: string,
|
||||
serve: () => Promise<void>,
|
||||
): Promise<void> {
|
||||
if (!this.environmentService.isGitSyncEnabled()) {
|
||||
await serve();
|
||||
return;
|
||||
}
|
||||
const result = await this.spaceLock.withSpaceLock(
|
||||
spaceId,
|
||||
async () => {
|
||||
const vault = await this.vaultRegistry.getVault(spaceId);
|
||||
await vault.pinHeadToMain();
|
||||
await serve();
|
||||
},
|
||||
{
|
||||
acquireRetry: {
|
||||
timeoutMs: GIT_SYNC_PUSH_LOCK_RETRY_TOTAL_MS,
|
||||
baseMs: GIT_SYNC_PUSH_LOCK_RETRY_BASE_MS,
|
||||
maxMs: GIT_SYNC_PUSH_LOCK_RETRY_MAX_MS,
|
||||
},
|
||||
},
|
||||
);
|
||||
// Lock contended for the whole budget (in-progress / another replica): serve
|
||||
// anyway. `serve` (backend.run) never ran inside the lock in this case.
|
||||
if (typeof result === 'object' && result !== null && 'skipped' in result) {
|
||||
await serve();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drive ONE reconcile cycle for a space. The PULL->PUSH branch choreography
|
||||
* lives in the engine's `runCycle` (so it can never drift from the engine it
|
||||
* ships with); the orchestrator owns only the lock (its caller) and the
|
||||
* service binding. There is no delete cap — deletes apply unconditionally (they
|
||||
* are soft/reversible) and every cycle logs what it deleted via `log`.
|
||||
*/
|
||||
private async driveCycle(
|
||||
spaceId: string,
|
||||
workspaceId: string,
|
||||
serviceUserId: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<GitSyncRunStatus> {
|
||||
const { runCycle } = await loadGitSync();
|
||||
const settings = await this.buildSettings(spaceId);
|
||||
const vault = await this.vaultRegistry.getVault(spaceId);
|
||||
const client = this.dataSource.bind({
|
||||
workspaceId,
|
||||
userId: serviceUserId,
|
||||
spaceId,
|
||||
});
|
||||
|
||||
const result = await runCycle({
|
||||
// Cooperative-abort signal from the per-space lock: if a heartbeat refresh
|
||||
// cannot confirm the lock, the cycle bails before its next destructive
|
||||
// write phase instead of writing blind after a possible lock loss.
|
||||
signal,
|
||||
spaceId,
|
||||
client,
|
||||
vault,
|
||||
settings,
|
||||
// ABSOLUTE-path fs primitives the engine cycle injects (it stays IO-free).
|
||||
// `lstat`/`realpath` back the engine's symlink guard: both MUST yield
|
||||
// `null` on ENOENT (a not-yet-created file is the normal write case) so the
|
||||
// guard can tell "absent" (safe to create) from "is a symlink" (refuse).
|
||||
// `lstat` does NOT follow the final link; `realpath` resolves it.
|
||||
fs: {
|
||||
readFile: (absPath) => readFile(absPath, 'utf8'),
|
||||
writeFile: (absPath, text) => writeFile(absPath, text, 'utf8'),
|
||||
mkdir: (absDir) => mkdir(absDir, { recursive: true }).then(() => undefined),
|
||||
rm: (absPath) => rm(absPath, { force: true }),
|
||||
lstat: (absPath) =>
|
||||
lstat(absPath).then(
|
||||
(st) => ({ isSymbolicLink: st.isSymbolicLink() }),
|
||||
(err: NodeJS.ErrnoException) => {
|
||||
if (err && err.code === 'ENOENT') return null;
|
||||
throw err;
|
||||
},
|
||||
),
|
||||
realpath: (absPath) =>
|
||||
realpath(absPath).then(
|
||||
(p) => p,
|
||||
(err: NodeJS.ErrnoException) => {
|
||||
if (err && err.code === 'ENOENT') return null;
|
||||
throw err;
|
||||
},
|
||||
),
|
||||
},
|
||||
// Every cycle logs its full push plan + per-action lines + completion
|
||||
// counts (created/updated/deleted/skipped/failures) through this `log`, so
|
||||
// what was deleted (and what was not) is always recorded. There is no
|
||||
// delete cap: deletes are soft (Trash, reversible), so a blocking limit
|
||||
// only got in the way of legitimate deletes; engine correctness (covered by
|
||||
// the reconcile/layout tests) is what prevents phantom deletions.
|
||||
log: (line: string) => this.logger.log(`git-sync[${spaceId}] ${line}`),
|
||||
});
|
||||
|
||||
// §5 invariant breach: the push refused to fast-forward a divergent `docmost`
|
||||
// mirror. No data is lost (the refusal is the safety), but the mirror no
|
||||
// longer reflects Docmost and the next push will keep refusing until an
|
||||
// operator reconciles it — so escalate from the engine's info `log` to a
|
||||
// WARN with the spaceId, and surface the flag in the returned status (/status).
|
||||
if (result.divergentDocmost) {
|
||||
this.logger.warn(
|
||||
`git-sync[${spaceId}] push refused to fast-forward a DIVERGENT 'docmost' ` +
|
||||
`mirror (invariant §5 broken); manual reconciliation required`,
|
||||
);
|
||||
}
|
||||
|
||||
return { spaceId, ...result };
|
||||
}
|
||||
|
||||
// --- poll-safety interval -------------------------------------
|
||||
|
||||
/** Registered interval name (shared by registration + teardown). */
|
||||
private static readonly POLL_INTERVAL_NAME = 'git-sync-poll';
|
||||
|
||||
/**
|
||||
* Register the poll-safety interval DYNAMICALLY so it honors the configured
|
||||
* GIT_SYNC_POLL_INTERVAL_MS (a static `@Interval` decorator could only hardcode
|
||||
* a value at class-eval time, before config is readable — diverging from what
|
||||
* `/status` reports). When git-sync is disabled we register nothing.
|
||||
*
|
||||
* ScheduleModule: forRoot() is registered ONCE globally by TelemetryModule;
|
||||
* GitSyncModule imports the plain ScheduleModule so SchedulerRegistry is
|
||||
* injectable without a duplicate forRoot.
|
||||
*
|
||||
* KNOWN MULTI-REPLICA LIMITATION (deferred — do not silently lose this):
|
||||
* This is an IN-PROCESS `setInterval` running on EVERY replica. Cross-replica
|
||||
* single-writer safety currently rests on the per-space Redis lock
|
||||
* (SpaceLockService) plus best-effort abort-on-failed-heartbeat — NOT on true
|
||||
* fencing. Under an adversarial schedule (lock TTL lapse during a GC/IO pause)
|
||||
* two replicas could still briefly believe they hold a space's lock. The
|
||||
* intended future direction is to move this orchestration to a BullMQ queue
|
||||
* (one durable, deduplicated job per space instead of N independent interval
|
||||
* timers) and add FENCING TOKENS so a stale writer's writes are rejected by the
|
||||
* store. The author deferred fencing tokens; this comment is the breadcrumb so
|
||||
* the gap is tracked rather than forgotten. See SpaceLockService.liveLocks.
|
||||
*/
|
||||
onModuleInit(): void {
|
||||
if (!this.environmentService.isGitSyncEnabled()) return;
|
||||
|
||||
// GIT_SYNC_REMOTE_TEMPLATE is inert scaffolding: the vendored engine has no
|
||||
// remote-push path yet (SPEC §7), so setting it does nothing today. Warn once
|
||||
// at startup so an operator who configured it isn't left with a silent no-op
|
||||
// (review). Remove this warning when the engine grows a remote-push path.
|
||||
if (this.environmentService.getGitSyncRemoteTemplate()) {
|
||||
this.logger.warn(
|
||||
'git-sync: GIT_SYNC_REMOTE_TEMPLATE is set but NOT yet consumed — ' +
|
||||
'remote push is deferred (SPEC §7); this value currently has no effect.',
|
||||
);
|
||||
}
|
||||
|
||||
const ms = this.environmentService.getGitSyncPollIntervalMs();
|
||||
const handle = setInterval(() => {
|
||||
void this.pollTick();
|
||||
}, ms);
|
||||
// Do not keep the event loop alive solely for the poll timer.
|
||||
handle.unref?.();
|
||||
this.schedulerRegistry.addInterval(
|
||||
GitSyncOrchestrator.POLL_INTERVAL_NAME,
|
||||
handle,
|
||||
);
|
||||
this.pollIntervalName = GitSyncOrchestrator.POLL_INTERVAL_NAME;
|
||||
this.logger.log(`git-sync: poll interval registered (${ms}ms).`);
|
||||
}
|
||||
|
||||
/** Tear down the dynamic interval on shutdown (guard against double-delete). */
|
||||
onModuleDestroy(): void {
|
||||
if (!this.pollIntervalName) return;
|
||||
try {
|
||||
// deleteInterval clears the timer and removes it from the registry.
|
||||
this.schedulerRegistry.deleteInterval(this.pollIntervalName);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`git-sync: failed to delete poll interval: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
} finally {
|
||||
this.pollIntervalName = null;
|
||||
}
|
||||
}
|
||||
|
||||
/** True while a pollTick pass is in flight (re-entrancy guard). */
|
||||
private polling = false;
|
||||
|
||||
/**
|
||||
* One poll tick: catches events missed by the listener and reconciles after
|
||||
* downtime. Gated on GIT_SYNC_ENABLED (defensive — the interval is only
|
||||
* registered when enabled). Each enabled space runs under its own lock
|
||||
* (overlaps skipped). Never throws (runOnce swallows per-space errors).
|
||||
*
|
||||
* Re-entrancy guard: a batch of cycles can take LONGER than the poll interval
|
||||
* (many spaces, slow pushes), so the next interval tick could fire while this
|
||||
* pass is still running. The per-space lock already prevents overlapping cycles
|
||||
* for one space, but an overlapping tick still re-runs enabledSpaces() and
|
||||
* redundant per-space lock attempts for every space. The `polling` flag skips a
|
||||
* tick while one is already in flight; it is in-process only (each replica
|
||||
* guards its own ticks — cross-replica overlap is handled by the Redis lock).
|
||||
*/
|
||||
private async pollTick(): Promise<void> {
|
||||
if (!this.environmentService.isGitSyncEnabled()) return;
|
||||
if (this.polling) return;
|
||||
this.polling = true;
|
||||
try {
|
||||
let spaces: EnabledSpace[];
|
||||
try {
|
||||
spaces = await this.enabledSpaces();
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`git-sync: failed to enumerate enabled spaces: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
for (const { spaceId, workspaceId } of spaces) {
|
||||
// runOnce never throws; a per-space error is logged and returned in status.
|
||||
await this.runOnce(spaceId, workspaceId);
|
||||
}
|
||||
} finally {
|
||||
this.polling = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,967 @@
|
||||
// Stub the collab util so importing the service does not drag in the
|
||||
// editor-ext -> @tiptap/react -> react-dom graph (unloadable under jest's node
|
||||
// env, same coupling noted in mcp.service.spec.ts). The captured transact
|
||||
// callback is never executed in these unit tests, so the stub extensions array
|
||||
// is sufficient; the real collab write path is exercised by integration tests.
|
||||
jest.mock('../../../collaboration/collaboration.util', () => ({
|
||||
tiptapExtensions: [],
|
||||
getPageId: (name: string) => name.replace(/^page\./, ''),
|
||||
}));
|
||||
// writeBody now builds the replacement Yjs state eagerly (before clearing the
|
||||
// live doc), so TiptapTransformer.toYdoc runs in these unit tests. Real Tiptap
|
||||
// extensions are stubbed to [] above (they drag in the React graph), which can't
|
||||
// build a schema — so stub the transformer to return a small non-empty Y.Doc.
|
||||
// The real conversion is exercised by the @docmost/git-sync converter tests and
|
||||
// the integration tests.
|
||||
jest.mock('@hocuspocus/transformer', () => {
|
||||
const Yjs = require('yjs');
|
||||
return {
|
||||
TiptapTransformer: {
|
||||
toYdoc: jest.fn(() => {
|
||||
const d = new Yjs.Doc();
|
||||
d.getXmlFragment('default').insert(0, [new Yjs.XmlElement('paragraph')]);
|
||||
return d;
|
||||
}),
|
||||
},
|
||||
};
|
||||
});
|
||||
// PageService is only ever a mocked dependency here; stub the editor-ext entry
|
||||
// it imports so loading its module does not pull in the React graph either.
|
||||
jest.mock('@docmost/editor-ext', () => ({
|
||||
markdownToHtml: jest.fn(),
|
||||
}));
|
||||
// The service loads `parseDocmostMarkdown` / `markdownToProseMirror` at runtime
|
||||
// via the `loadGitSync()` bridge (the ESM `@docmost/git-sync` package cannot be
|
||||
// `require()`d under jest). Stub the loader: the real conversion is exercised by
|
||||
// the @docmost/git-sync converter tests and the converter gate; here the mocked
|
||||
// TiptapTransformer.toYdoc ignores the converted doc anyway, so a passthrough
|
||||
// body + a minimal ProseMirror doc is sufficient.
|
||||
jest.mock('../git-sync.loader', () => ({
|
||||
loadGitSync: jest.fn(async () => ({
|
||||
parseDocmostMarkdown: (md: string) => ({ meta: {}, body: md }),
|
||||
markdownToProseMirror: async () => ({
|
||||
type: 'doc',
|
||||
content: [{ type: 'paragraph' }],
|
||||
}),
|
||||
// renamePage funnels the current title through sanitizeTitle to detect the
|
||||
// sanitized-stem echo; identity is the correct default here (none of the
|
||||
// rename fixtures use filename-hostile chars, so the sanitized form equals
|
||||
// the input and the guard never fires).
|
||||
sanitizeTitle: (title: string) => title,
|
||||
// importPageMarkdown guard #2 uses docsCanonicallyEqual to skip a no-op
|
||||
// re-ingest. A key-order-insensitive JSON compare is a sufficient stand-in
|
||||
// for the unit tests (the real semantic equality is covered by the
|
||||
// @docmost/git-sync converter tests); returning false for genuinely
|
||||
// different docs lets the write paths under test proceed.
|
||||
docsCanonicallyEqual: (a: unknown, b: unknown) =>
|
||||
JSON.stringify(a) === JSON.stringify(b),
|
||||
})),
|
||||
}));
|
||||
|
||||
import { GitmostDataSourceService } from './gitmost-datasource.service';
|
||||
// The loader is mocked above; this binding is the hoisted jest.fn, so a single
|
||||
// test can swap the runtime bridge (e.g. a smarter `docsCanonicallyEqual`) via
|
||||
// `mockResolvedValueOnce` without perturbing the default used by every other test.
|
||||
import { loadGitSync } from '../git-sync.loader';
|
||||
|
||||
// Focused unit/contract test for the native GitSyncClient adapter.
|
||||
// No DB, no real collab server: the repos/services/gateway are mocked and we
|
||||
// assert the mapping logic + the provenance/soft-delete/position contracts.
|
||||
|
||||
type AnyMock = jest.Mock;
|
||||
|
||||
interface Mocks {
|
||||
pageRepo: {
|
||||
findById: AnyMock;
|
||||
getSpaceDescendants: AnyMock;
|
||||
restorePage: AnyMock;
|
||||
};
|
||||
spaceRepo: { findById: AnyMock };
|
||||
pageService: {
|
||||
create: AnyMock;
|
||||
update: AnyMock;
|
||||
movePage: AnyMock;
|
||||
removePage: AnyMock;
|
||||
};
|
||||
collabGateway: { writePageBody: AnyMock };
|
||||
// Minimal Kysely-ish chainable mock for the direct-query paths.
|
||||
db: any;
|
||||
}
|
||||
|
||||
function makeQueryBuilder(rows: any[]) {
|
||||
const qb: any = {};
|
||||
for (const m of ['select', 'where', 'orderBy', 'limit']) {
|
||||
qb[m] = jest.fn(() => qb);
|
||||
}
|
||||
qb.execute = jest.fn(async () => rows);
|
||||
qb.executeTakeFirst = jest.fn(async () => rows[0]);
|
||||
return qb;
|
||||
}
|
||||
|
||||
function build(rows: any[] = []): {
|
||||
service: GitmostDataSourceService;
|
||||
mocks: Mocks;
|
||||
} {
|
||||
const mocks: Mocks = {
|
||||
pageRepo: {
|
||||
findById: jest.fn(),
|
||||
getSpaceDescendants: jest.fn(),
|
||||
restorePage: jest.fn(async () => undefined),
|
||||
},
|
||||
spaceRepo: { findById: jest.fn(async () => ({ id: 'space-1' })) },
|
||||
pageService: {
|
||||
create: jest.fn(),
|
||||
update: jest.fn(async () => undefined),
|
||||
movePage: jest.fn(async () => undefined),
|
||||
removePage: jest.fn(async () => undefined),
|
||||
},
|
||||
collabGateway: {
|
||||
writePageBody: jest.fn(async () => undefined),
|
||||
},
|
||||
db: {
|
||||
selectFrom: jest.fn(() => makeQueryBuilder(rows)),
|
||||
},
|
||||
};
|
||||
|
||||
const service = new GitmostDataSourceService(
|
||||
mocks.pageRepo as any,
|
||||
mocks.spaceRepo as any,
|
||||
mocks.pageService as any,
|
||||
mocks.collabGateway as any,
|
||||
mocks.db as any,
|
||||
);
|
||||
|
||||
return { service, mocks };
|
||||
}
|
||||
|
||||
const CTX = { workspaceId: 'ws-1', userId: 'svc-user' };
|
||||
// A bound context that carries the reconciling spaceId, enabling deletePage's
|
||||
// cross-space MOVE guard (the `if (ctx.spaceId)` branch).
|
||||
const CTX_SPACE = { ...CTX, spaceId: 'space-1' };
|
||||
// A syntactically VALID parent uuid. createPage/movePage now coerce a malformed
|
||||
// (non-UUID) parentPageId to root (F1), so tests exercising the normal
|
||||
// pass-through parent path must use a real uuid, not an arbitrary token.
|
||||
const PARENT_UUID = '11111111-1111-4111-8111-111111111111';
|
||||
|
||||
describe('GitmostDataSourceService', () => {
|
||||
describe('listSpaceTree', () => {
|
||||
it('maps descendants to PageNode and is always complete:true', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.spaceRepo.findById.mockResolvedValue({ id: 'space-1' });
|
||||
mocks.pageRepo.getSpaceDescendants.mockResolvedValue([
|
||||
{
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'Root',
|
||||
parentPageId: null,
|
||||
position: 'a0',
|
||||
},
|
||||
{
|
||||
id: 'p2',
|
||||
slugId: 's2',
|
||||
title: 'Child',
|
||||
parentPageId: 'p1',
|
||||
position: 'a1',
|
||||
},
|
||||
]);
|
||||
|
||||
const client = service.bind(CTX);
|
||||
const res = await client.listSpaceTree('space-1');
|
||||
|
||||
expect(res.complete).toBe(true);
|
||||
expect(mocks.pageRepo.getSpaceDescendants).toHaveBeenCalledWith(
|
||||
'space-1',
|
||||
{ includeContent: false },
|
||||
);
|
||||
expect(res.pages).toEqual([
|
||||
{
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'Root',
|
||||
parentPageId: null,
|
||||
hasChildren: true, // p2's parent is p1
|
||||
position: 'a0',
|
||||
},
|
||||
{
|
||||
id: 'p2',
|
||||
slugId: 's2',
|
||||
title: 'Child',
|
||||
parentPageId: 'p1',
|
||||
hasChildren: false,
|
||||
position: 'a1',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('throws when the space is not found', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.spaceRepo.findById.mockResolvedValue(undefined);
|
||||
await expect(service.bind(CTX).listSpaceTree('nope')).rejects.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
// D-P3-1 ghost guard: the pull reconciler asks which candidate-delete ids are
|
||||
// REAL page rows before deleting a tracked vault file. Only rows returned here
|
||||
// may be absence-deleted; a ghost id (never a page) is preserved.
|
||||
describe('pageIdsExist (D-P3-1 ghost guard)', () => {
|
||||
const UUID_A = '11111111-1111-4111-8111-111111111111';
|
||||
const UUID_B = '22222222-2222-4222-8222-222222222222';
|
||||
|
||||
it('returns the subset that are real page rows, workspace-scoped (any space/state)', async () => {
|
||||
// The DB reports only UUID_A exists as a row.
|
||||
const { service, mocks } = build([{ id: UUID_A }]);
|
||||
const res = await service.bind(CTX).pageIdsExist([UUID_A, UUID_B]);
|
||||
expect(res).toEqual([UUID_A]);
|
||||
// The query is workspace-scoped and restricted to the candidate ids.
|
||||
// Crucially there is NO deletedAt / spaceId filter: a trashed page or a
|
||||
// page moved to another space still counts as "exists" (its file is
|
||||
// cleaned up), while a ghost has no row (its file is preserved).
|
||||
expect(mocks.db.selectFrom).toHaveBeenCalledWith('pages');
|
||||
const qb = mocks.db.selectFrom.mock.results[0].value;
|
||||
expect(qb.where).toHaveBeenCalledWith('workspaceId', '=', 'ws-1');
|
||||
expect(qb.where).toHaveBeenCalledWith('id', 'in', [UUID_A, UUID_B]);
|
||||
const filterCols = qb.where.mock.calls.map((c: any[]) => c[0]);
|
||||
expect(filterCols).not.toContain('deletedAt');
|
||||
expect(filterCols).not.toContain('spaceId');
|
||||
});
|
||||
|
||||
it('filters malformed non-UUID ids BEFORE the predicate (and skips the query when none remain)', async () => {
|
||||
// A mix of one valid uuid + malformed tokens: only the valid uuid reaches
|
||||
// the predicate, so a bad `gitmost_id` cannot trigger a 22P02 that would
|
||||
// reject the whole existence probe.
|
||||
const { service, mocks } = build([]);
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.pageIdsExist([UUID_A, 'not-a-uuid', '[unclosed']);
|
||||
const qb = mocks.db.selectFrom.mock.results[0].value;
|
||||
expect(qb.where).toHaveBeenCalledWith('id', 'in', [UUID_A]);
|
||||
expect(res).toEqual([]);
|
||||
|
||||
// When EVERY candidate is malformed, the query is skipped entirely.
|
||||
const { service: s2, mocks: m2 } = build([]);
|
||||
const res2 = await s2.bind(CTX).pageIdsExist(['not-a-uuid', '[unclosed']);
|
||||
expect(res2).toEqual([]);
|
||||
expect(m2.db.selectFrom).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('getPageJson', () => {
|
||||
it('returns the engine page shape with ISO updatedAt + content', async () => {
|
||||
const { service, mocks } = build();
|
||||
const updatedAt = new Date('2026-06-20T10:00:00.000Z');
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'Doc',
|
||||
parentPageId: null,
|
||||
spaceId: 'space-1',
|
||||
updatedAt,
|
||||
content: { type: 'doc', content: [] },
|
||||
});
|
||||
|
||||
const res = await service.bind(CTX).getPageJson('p1');
|
||||
expect(mocks.pageRepo.findById).toHaveBeenCalledWith('p1', {
|
||||
includeContent: true,
|
||||
});
|
||||
expect(res).toEqual({
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'Doc',
|
||||
parentPageId: null,
|
||||
spaceId: 'space-1',
|
||||
updatedAt: '2026-06-20T10:00:00.000Z',
|
||||
content: { type: 'doc', content: [] },
|
||||
});
|
||||
});
|
||||
|
||||
it('throws NotFound when the page does not exist', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue(undefined);
|
||||
await expect(service.bind(CTX).getPageJson('gone')).rejects.toThrow(
|
||||
/not found/i,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('importPageMarkdown', () => {
|
||||
it('parses md, converts to ProseMirror, and routes the body write to the owning instance', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
updatedAt: new Date('2026-06-20T11:00:00.000Z'),
|
||||
});
|
||||
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.importPageMarkdown('p1', '# Hello\n\nworld');
|
||||
|
||||
// writeBody routes through writePageBody (NOT openDirectConnection): the
|
||||
// merge must run on the instance that owns the live doc so a connected
|
||||
// editor converges instead of silently reverting the change. The service
|
||||
// user rides on the payload as the responsible author.
|
||||
expect(mocks.collabGateway.writePageBody).toHaveBeenCalledTimes(1);
|
||||
const [docName, payload] = mocks.collabGateway.writePageBody.mock.calls[0];
|
||||
expect(docName).toBe('page.p1');
|
||||
expect(payload.userId).toBe('svc-user');
|
||||
// A converted ProseMirror doc was passed; no base on a plain import.
|
||||
expect(payload.prosemirrorJson).toEqual(
|
||||
expect.objectContaining({ type: 'doc' }),
|
||||
);
|
||||
expect(payload.baseProsemirrorJson).toBeUndefined();
|
||||
|
||||
expect(res.updatedAt).toBe('2026-06-20T11:00:00.000Z');
|
||||
});
|
||||
|
||||
it('returns updatedAt:undefined when the page row is gone after the write (stale-read branch)', async () => {
|
||||
// The page EXISTS at import time (so the unknown-page guard N1-D1 does not
|
||||
// fire), writeBody succeeds, but the POST-write findById returns nothing (e.g.
|
||||
// the page was concurrently hard-deleted) -> the optional updatedAt is omitted.
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById
|
||||
.mockResolvedValueOnce({
|
||||
id: 'p1',
|
||||
updatedAt: new Date('2026-06-20T11:00:00.000Z'),
|
||||
}) // currentPage (import-time read) exists
|
||||
.mockResolvedValue(undefined); // post-write read: page is gone
|
||||
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.importPageMarkdown('p1', '# Hello\n\nworld');
|
||||
|
||||
expect(mocks.collabGateway.writePageBody).toHaveBeenCalledTimes(1);
|
||||
expect(res.updatedAt).toBeUndefined();
|
||||
});
|
||||
|
||||
it('skips (no writeBody) when the gitmost_id is a valid UUID matching NO page (bug N1-D1)', async () => {
|
||||
// A well-formed but stale/foreign id (restore-from-backup, copied file) must
|
||||
// NOT fall through to writeBody on a non-existent page (which throws "Page not
|
||||
// found" and wedges the space's sync loop). It is skipped as an inert no-op.
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue(undefined);
|
||||
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.importPageMarkdown('019f2500-face-7000-8000-000000000002', '# orphan');
|
||||
|
||||
expect(res).toEqual({});
|
||||
expect(mocks.collabGateway.writePageBody).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// F5 acceptance, criterion (b): guard #2 (docsCanonicallyEqual) must SKIP the
|
||||
// re-ingest when the freshly-parsed doc is canonically equal to the page's
|
||||
// current DB content even though the two are NOT byte-identical — the DB copy
|
||||
// carries the real per-block uuids (comments anchor to them) while a fresh
|
||||
// parse has none. Skipping is what protects a concurrent, not-yet-flushed
|
||||
// human edit from being clobbered by an idempotent poll re-ingest.
|
||||
//
|
||||
// NON-VACUITY: `currentContent` here differs from `doc` in raw JSON (it has an
|
||||
// `attrs.id` uuid `doc` lacks). The default mock `docsCanonicallyEqual` is a
|
||||
// plain `JSON.stringify` compare — it would return FALSE for these inputs, so
|
||||
// if guard #2 were removed (or reverted to a canonicalJsonEqual that does not
|
||||
// strip ids) writePageBody WOULD be called and this test would fail. We model
|
||||
// the package's authoritative id-stripping equality (returns TRUE here) by
|
||||
// swapping in a `docsCanonicallyEqual` that normalizes away block ids, proving
|
||||
// it is the SEMANTIC equality — not a byte compare — that suppresses the write.
|
||||
it('does NOT call writePageBody (guard #2) when content is canonically equal despite differing block ids (F5 criterion b)', async () => {
|
||||
const { service, mocks } = build();
|
||||
const realUuid = '11111111-1111-4111-8111-111111111111';
|
||||
// The DB row's content carries a REAL per-block uuid; a fresh parse does not.
|
||||
// These two docs are canonically equal (id-only difference) but NOT
|
||||
// byte-identical, so a naive JSON compare treats the page as "changed".
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
updatedAt: new Date('2026-06-20T11:00:00.000Z'),
|
||||
content: {
|
||||
type: 'doc',
|
||||
content: [{ type: 'paragraph', attrs: { id: realUuid } }],
|
||||
},
|
||||
});
|
||||
// Swap the runtime bridge for THIS call only: same passthrough parse/convert
|
||||
// as the default mock (so `doc` is the id-less `{type:'doc',[paragraph]}`),
|
||||
// but `docsCanonicallyEqual` strips block ids before comparing — the real
|
||||
// converter's semantics. It returns TRUE for (doc, currentContent) here.
|
||||
const stripIds = (node: any): any => {
|
||||
if (Array.isArray(node)) return node.map(stripIds);
|
||||
if (node && typeof node === 'object') {
|
||||
const out: any = {};
|
||||
for (const [k, v] of Object.entries(node)) {
|
||||
if (k === 'attrs' && v && typeof v === 'object') {
|
||||
const { id: _id, ...rest } = v as Record<string, unknown>;
|
||||
if (Object.keys(rest).length) out.attrs = stripIds(rest);
|
||||
} else {
|
||||
out[k] = stripIds(v);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return node;
|
||||
};
|
||||
(loadGitSync as jest.Mock).mockResolvedValueOnce({
|
||||
parseDocmostMarkdown: (md: string) => ({ meta: {}, body: md }),
|
||||
markdownToProseMirror: async () => ({
|
||||
type: 'doc',
|
||||
content: [{ type: 'paragraph' }],
|
||||
}),
|
||||
sanitizeTitle: (title: string) => title,
|
||||
docsCanonicallyEqual: (a: unknown, b: unknown) =>
|
||||
JSON.stringify(stripIds(a)) === JSON.stringify(stripIds(b)),
|
||||
});
|
||||
|
||||
// No baseMarkdown -> guard #1 (fullMarkdown === baseMarkdown) cannot fire, so
|
||||
// the outcome is decided purely by guard #2.
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.importPageMarkdown('p1', '# Hello\n\nworld');
|
||||
|
||||
// Guard #2 fired: the redundant re-ingest is skipped, so the concurrent
|
||||
// human edit in the live doc is NOT clobbered.
|
||||
expect(mocks.collabGateway.writePageBody).not.toHaveBeenCalled();
|
||||
// The unchanged page's updatedAt is still surfaced from the DB row.
|
||||
expect(res.updatedAt).toBe('2026-06-20T11:00:00.000Z');
|
||||
});
|
||||
|
||||
// The 2-way path (no base) is covered above; this exercises the THREE-WAY
|
||||
// branch that only fires when a `baseMarkdown` is supplied (review #5). The
|
||||
// merge dispatch itself now lives in the collab handler (gitSyncWriteBody);
|
||||
// here we assert the datasource forwards the base so the owning instance can
|
||||
// run the 3-way reconcile.
|
||||
describe('with a baseMarkdown (three-way merge)', () => {
|
||||
it('forwards the parsed base body so the owning instance can three-way merge', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
updatedAt: new Date('2026-06-20T11:00:00.000Z'),
|
||||
});
|
||||
|
||||
await service
|
||||
.bind(CTX)
|
||||
.importPageMarkdown('p1', '# Full\n\ngit', '# Base\n\nbase');
|
||||
|
||||
expect(mocks.collabGateway.writePageBody).toHaveBeenCalledTimes(1);
|
||||
const [, payload] = mocks.collabGateway.writePageBody.mock.calls[0];
|
||||
// Both the incoming body AND the last-synced base were converted and
|
||||
// forwarded — proof the 3-way common-ancestor is plumbed through.
|
||||
expect(payload.prosemirrorJson).toEqual(
|
||||
expect.objectContaining({ type: 'doc' }),
|
||||
);
|
||||
expect(payload.baseProsemirrorJson).toEqual(
|
||||
expect.objectContaining({ type: 'doc' }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// The cross-space confused-deputy guard (review S2) fires only when
|
||||
// ctx.spaceId is bound. A vault file in space A can carry space B's pageId;
|
||||
// without this check the reconciling space could overwrite B's page body — a
|
||||
// write it has no authority over. When the resolved page already lives in a
|
||||
// DIFFERENT space, the import is SKIPPED (writePageBody not called) and the
|
||||
// page's own updatedAt is returned unchanged.
|
||||
describe('cross-space guard (ctx.spaceId bound, review S2)', () => {
|
||||
it('does NOT call writePageBody when the target page lives in another space', async () => {
|
||||
const { service, mocks } = build();
|
||||
// The resolved page is in space-2, but the reconciling context is space-1.
|
||||
// Its content DIFFERS from the parsed doc, so guard #2 (docsCanonicallyEqual)
|
||||
// cannot be what suppresses the write — proving NON-VACUITY: without the S2
|
||||
// guard the flow would reach writeBody and writePageBody WOULD be called.
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
deletedAt: null,
|
||||
spaceId: 'space-2',
|
||||
updatedAt: new Date('2026-06-21T09:00:00.000Z'),
|
||||
content: {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{ type: 'paragraph', content: [{ type: 'text', text: 'B body' }] },
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
const res = await service
|
||||
.bind(CTX_SPACE)
|
||||
.importPageMarkdown('p1', '# Hello\n\nworld');
|
||||
|
||||
// The cross-space page is preserved: no body write happened.
|
||||
expect(mocks.collabGateway.writePageBody).not.toHaveBeenCalled();
|
||||
// Early-return shape: only the page's own updatedAt is surfaced.
|
||||
expect(res).toEqual({ updatedAt: '2026-06-21T09:00:00.000Z' });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Bug C9-D1: a vault file with a malformed (non-UUID) `gitmost_id` frontmatter
|
||||
// makes the id reach a Postgres `uuid` predicate, which throws error code
|
||||
// '22P02'. Left unhandled the push apply records it as a per-cycle failure that
|
||||
// never clears -> the whole space's sync loops forever. The bind() seam wraps the
|
||||
// id-scoped writes so exactly that error is swallowed as an inert no-op.
|
||||
describe('malformed-id guard (bug C9-D1: non-UUID gitmost_id must not wedge sync)', () => {
|
||||
const pgInvalidUuid = Object.assign(
|
||||
new Error('invalid input syntax for type uuid: "not-a-uuid"'),
|
||||
{ code: '22P02' },
|
||||
);
|
||||
|
||||
it('importPageMarkdown swallows a 22P02 and does NOT write the body', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockRejectedValue(pgInvalidUuid);
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.importPageMarkdown('not-a-uuid', '# x');
|
||||
expect(res).toEqual({}); // inert no-op, no throw
|
||||
expect(mocks.collabGateway.writePageBody).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletePage swallows the 22P02 thrown by the uuid predicate (no wedge)', async () => {
|
||||
const { service, mocks } = build();
|
||||
// The malformed id reaches removePage's `uuid` predicate, which throws 22P02.
|
||||
mocks.pageService.removePage.mockRejectedValue(pgInvalidUuid);
|
||||
await expect(
|
||||
service.bind(CTX).deletePage('not-a-uuid'),
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('re-throws a NON-22P02 error (does not mask real failures)', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockRejectedValue(new Error('db down'));
|
||||
await expect(
|
||||
service.bind(CTX).importPageMarkdown('not-a-uuid', '# x'),
|
||||
).rejects.toThrow('db down');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createPage', () => {
|
||||
it('creates the shell with git-sync provenance, writes body, returns id', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageService.create.mockResolvedValue({ id: 'new-id' });
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'new-id',
|
||||
updatedAt: new Date('2026-06-20T12:00:00.000Z'),
|
||||
});
|
||||
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.createPage('Title', 'body md', 'space-1', PARENT_UUID);
|
||||
|
||||
expect(mocks.pageService.create).toHaveBeenCalledWith(
|
||||
'svc-user',
|
||||
'ws-1',
|
||||
{ spaceId: 'space-1', title: 'Title', parentPageId: PARENT_UUID },
|
||||
{ actor: 'git-sync', aiChatId: null },
|
||||
);
|
||||
expect(mocks.collabGateway.writePageBody).toHaveBeenCalledWith(
|
||||
'page.new-id',
|
||||
expect.objectContaining({ userId: 'svc-user' }),
|
||||
);
|
||||
expect(res).toEqual({
|
||||
data: { id: 'new-id' },
|
||||
updatedAt: '2026-06-20T12:00:00.000Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns updatedAt:undefined when the fresh page row is missing after create', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageService.create.mockResolvedValue({ id: 'new-id' });
|
||||
// The post-create findById returns nothing -> the optional updatedAt is
|
||||
// omitted (the id is still returned from create()).
|
||||
mocks.pageRepo.findById.mockResolvedValue(undefined);
|
||||
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.createPage('Title', 'body md', 'space-1');
|
||||
|
||||
expect(res).toEqual({ data: { id: 'new-id' }, updatedAt: undefined });
|
||||
});
|
||||
|
||||
// F1 (bug C9-D1, parent-id variant): a parent folder-note carrying a broken
|
||||
// non-UUID `gitmost_id` makes the push planner hand createPage a malformed
|
||||
// parentPageId. Left as-is it flows into pageService.create -> findById
|
||||
// (slugId fallback -> no row) -> NotFoundException, a throw that never clears
|
||||
// the push `failures` set, so the WHOLE space wedges forever. The fix COERCES
|
||||
// the malformed parent to root (undefined) so the page is created at the space
|
||||
// root (self-heal) instead of being dropped — and never wedges.
|
||||
//
|
||||
// NON-VACUITY: the mocked pageService.create asserts it received
|
||||
// `parentPageId: undefined`. Against the UNcoerced createPage the malformed
|
||||
// string would flow straight through and this assertion would fail (create
|
||||
// would be called with parentPageId:'[unclosed-broken-id'), so the test
|
||||
// genuinely exercises the coercion.
|
||||
it('coerces a malformed (non-UUID) parentPageId to root and does NOT wedge (F1)', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageService.create.mockResolvedValue({ id: 'new-id' });
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'new-id',
|
||||
updatedAt: new Date('2026-06-20T12:00:00.000Z'),
|
||||
});
|
||||
|
||||
const res = await service
|
||||
.bind(CTX)
|
||||
.createPage('Title', 'body md', 'space-1', '[unclosed-broken-id');
|
||||
|
||||
// No throw propagated to `failures`; create ran with the parent coerced to
|
||||
// root (undefined), NOT the malformed string — so create's findById /
|
||||
// nextPagePosition never see a non-uuid and cannot 22P02 / NotFound.
|
||||
expect(mocks.pageService.create).toHaveBeenCalledWith(
|
||||
'svc-user',
|
||||
'ws-1',
|
||||
{ spaceId: 'space-1', title: 'Title', parentPageId: undefined },
|
||||
{ actor: 'git-sync', aiChatId: null },
|
||||
);
|
||||
expect(res).toEqual({
|
||||
data: { id: 'new-id' },
|
||||
updatedAt: '2026-06-20T12:00:00.000Z',
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves a VALID uuid parentPageId unchanged (only a malformed parent is coerced)', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageService.create.mockResolvedValue({ id: 'new-id' });
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'new-id',
|
||||
updatedAt: new Date('2026-06-20T12:00:00.000Z'),
|
||||
});
|
||||
|
||||
await service
|
||||
.bind(CTX)
|
||||
.createPage('Title', 'body md', 'space-1', PARENT_UUID);
|
||||
|
||||
// A real uuid passes the isValidUUID check and is forwarded untouched.
|
||||
expect(mocks.pageService.create).toHaveBeenCalledWith(
|
||||
'svc-user',
|
||||
'ws-1',
|
||||
{ spaceId: 'space-1', title: 'Title', parentPageId: PARENT_UUID },
|
||||
{ actor: 'git-sync', aiChatId: null },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('deletePage', () => {
|
||||
it('uses the soft-delete path (removePage), not a force delete', async () => {
|
||||
const { service, mocks } = build();
|
||||
await service.bind(CTX).deletePage('p1');
|
||||
// Passes git-sync provenance so the soft-delete stamps
|
||||
// lastUpdatedSource='git-sync' (loop-guard, PR #119 review).
|
||||
expect(mocks.pageService.removePage).toHaveBeenCalledWith(
|
||||
'p1',
|
||||
'svc-user',
|
||||
'ws-1',
|
||||
{ actor: 'git-sync', aiChatId: null },
|
||||
);
|
||||
// No forceDelete on the service surface used here.
|
||||
expect((mocks.pageService as any).forceDelete).toBeUndefined();
|
||||
});
|
||||
|
||||
// The cross-space MOVE guard fires only when ctx.spaceId is bound. It re-reads
|
||||
// the page and SKIPS the soft-delete when the page has already moved to a
|
||||
// DIFFERENT space (otherwise a move-out would trash the page from both vaults).
|
||||
describe('cross-space move guard (ctx.spaceId bound)', () => {
|
||||
it("skips removePage when the page moved to another space (move-out)", async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
deletedAt: null,
|
||||
spaceId: 'space-2',
|
||||
});
|
||||
|
||||
const res = await service.bind(CTX_SPACE).deletePage('p1');
|
||||
|
||||
// The page still lives in space-2 — it must NOT be trashed.
|
||||
expect(mocks.pageService.removePage).not.toHaveBeenCalled();
|
||||
expect(res).toEqual({ id: 'p1', skipped: 'moved-to-other-space' });
|
||||
});
|
||||
|
||||
it('soft-deletes when the page is still in the reconciling space (genuine delete)', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
deletedAt: null,
|
||||
spaceId: 'space-1',
|
||||
});
|
||||
|
||||
await service.bind(CTX_SPACE).deletePage('p1');
|
||||
|
||||
// Same space -> a real deletion; removePage runs with git-sync provenance.
|
||||
expect(mocks.pageService.removePage).toHaveBeenCalledWith(
|
||||
'p1',
|
||||
'svc-user',
|
||||
'ws-1',
|
||||
{ actor: 'git-sync', aiChatId: null },
|
||||
);
|
||||
});
|
||||
|
||||
it('soft-deletes when the page row is not found (guard must not swallow a real delete)', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue(undefined);
|
||||
|
||||
await service.bind(CTX_SPACE).deletePage('p1');
|
||||
|
||||
expect(mocks.pageService.removePage).toHaveBeenCalledWith(
|
||||
'p1',
|
||||
'svc-user',
|
||||
'ws-1',
|
||||
{ actor: 'git-sync', aiChatId: null },
|
||||
);
|
||||
});
|
||||
|
||||
it('soft-deletes when the page is already soft-deleted (deletedAt non-null)', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
deletedAt: new Date('2026-06-20T10:00:00.000Z'),
|
||||
spaceId: 'space-2',
|
||||
});
|
||||
|
||||
await service.bind(CTX_SPACE).deletePage('p1');
|
||||
|
||||
// deletedAt is non-null -> not treated as a live move-out; removePage runs.
|
||||
expect(mocks.pageService.removePage).toHaveBeenCalledWith(
|
||||
'p1',
|
||||
'svc-user',
|
||||
'ws-1',
|
||||
{ actor: 'git-sync', aiChatId: null },
|
||||
);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('movePage', () => {
|
||||
it('computes a fractional position when none is supplied', async () => {
|
||||
// db query returns a last sibling at 'a0' -> jittered key after it.
|
||||
const { service, mocks } = build([{ position: 'a0' }]);
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
spaceId: 'space-1',
|
||||
});
|
||||
|
||||
await service.bind(CTX).movePage('p1', PARENT_UUID);
|
||||
|
||||
expect(mocks.pageService.movePage).toHaveBeenCalledTimes(1);
|
||||
const [dto, page, provenance, actorUserId] =
|
||||
mocks.pageService.movePage.mock.calls[0];
|
||||
expect(dto.pageId).toBe('p1');
|
||||
expect(dto.parentPageId).toBe(PARENT_UUID);
|
||||
expect(typeof dto.position).toBe('string');
|
||||
expect(dto.position.length).toBeGreaterThan(0);
|
||||
expect(page).toEqual({ id: 'p1', spaceId: 'space-1' });
|
||||
expect(provenance).toEqual({ actor: 'git-sync', aiChatId: null });
|
||||
// The git-initiated move is attributed to the service user (lastUpdatedById
|
||||
// parity with create/delete/rename).
|
||||
expect(actorUserId).toBe('svc-user');
|
||||
});
|
||||
|
||||
it('passes through an explicit position unchanged', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
spaceId: 'space-1',
|
||||
});
|
||||
|
||||
await service.bind(CTX).movePage('p1', null, 'zz');
|
||||
const [dto] = mocks.pageService.movePage.mock.calls[0];
|
||||
expect(dto.position).toBe('zz');
|
||||
// db not consulted for a supplied position.
|
||||
expect(mocks.db.selectFrom).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('throws NotFound and moves nothing when the page does not exist', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue(undefined);
|
||||
await expect(
|
||||
service.bind(CTX).movePage('gone', 'parent-1'),
|
||||
).rejects.toThrow(/not found/i);
|
||||
expect(mocks.pageService.movePage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// F1 (parent-id variant), mirror of the createPage case. A malformed
|
||||
// (non-UUID) destination parentPageId would reach computeMovePosition's raw
|
||||
// uuid predicate (22P02, swallowed but mis-logged) or — when a position is
|
||||
// supplied — pageService.movePage's findById -> NotFoundException (NOT a
|
||||
// 22P02, so NOT swallowed -> the space wedges). The fix coerces it to root
|
||||
// (null) so the reparent targets root instead of wedging. The page here has a
|
||||
// current parent, so coerced-root != current parent and the echo-guard does
|
||||
// not short-circuit — the move proceeds against a null (root) parent.
|
||||
//
|
||||
// NON-VACUITY: the assertion is `dto.parentPageId === null`. Against the
|
||||
// UNcoerced movePage the malformed string would flow through to
|
||||
// pageService.movePage's dto and this would fail (it would be the raw
|
||||
// '[broken-parent' token).
|
||||
it('coerces a malformed (non-UUID) parentPageId to root and does NOT wedge (F1)', async () => {
|
||||
const { service, mocks } = build([]); // no siblings -> fresh position key
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
spaceId: 'space-1',
|
||||
parentPageId: '22222222-2222-4222-8222-222222222222',
|
||||
});
|
||||
|
||||
await service.bind(CTX).movePage('p1', '[broken-parent');
|
||||
|
||||
expect(mocks.pageService.movePage).toHaveBeenCalledTimes(1);
|
||||
const [dto] = mocks.pageService.movePage.mock.calls[0];
|
||||
expect(dto.pageId).toBe('p1');
|
||||
// Coerced to root: the malformed parent never reaches the uuid predicate.
|
||||
expect(dto.parentPageId).toBeNull();
|
||||
expect(typeof dto.position).toBe('string');
|
||||
});
|
||||
});
|
||||
|
||||
describe('renamePage', () => {
|
||||
it('updates only the title with git-sync provenance', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({ id: 'p1', title: 'old' });
|
||||
|
||||
await service.bind(CTX).renamePage('p1', 'new title');
|
||||
|
||||
const [page, dto, user, provenance] =
|
||||
mocks.pageService.update.mock.calls[0];
|
||||
expect(page).toEqual({ id: 'p1', title: 'old' });
|
||||
expect(dto.title).toBe('new title');
|
||||
expect(user).toEqual({ id: 'svc-user' });
|
||||
expect(provenance).toEqual({ actor: 'git-sync', aiChatId: null });
|
||||
});
|
||||
|
||||
it("strips the ` ~<slugId>` disambiguation suffix when it matches the page's OWN slugId", async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'old',
|
||||
});
|
||||
|
||||
await service.bind(CTX).renamePage('p1', 'Real Title ~s1');
|
||||
|
||||
const [, dto] = mocks.pageService.update.mock.calls[0];
|
||||
// The trailing ` ~s1` equals this page's own slugId -> a vault artifact; strip.
|
||||
expect(dto.title).toBe('Real Title');
|
||||
});
|
||||
|
||||
it('leaves a FOREIGN ` ~<slugId>` tail untouched (only the own slugId is stripped)', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue({
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'old',
|
||||
});
|
||||
|
||||
await service.bind(CTX).renamePage('p1', 'Real Title ~other');
|
||||
|
||||
const [, dto] = mocks.pageService.update.mock.calls[0];
|
||||
// ` ~other` is not this page's slugId -> a genuine title; must not be corrupted.
|
||||
expect(dto.title).toBe('Real Title ~other');
|
||||
});
|
||||
|
||||
it('throws NotFound and renames nothing when the page does not exist', async () => {
|
||||
const { service, mocks } = build();
|
||||
mocks.pageRepo.findById.mockResolvedValue(undefined);
|
||||
await expect(
|
||||
service.bind(CTX).renamePage('gone', 'whatever'),
|
||||
).rejects.toThrow(/not found/i);
|
||||
expect(mocks.pageService.update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('restorePage', () => {
|
||||
it('restores via the repo restore path scoped to the workspace', async () => {
|
||||
const { service, mocks } = build();
|
||||
const res = await service.bind(CTX).restorePage('p1');
|
||||
// Stamps lastUpdatedSource='git-sync' on restore (loop-guard, PR #119).
|
||||
expect(mocks.pageRepo.restorePage).toHaveBeenCalledWith(
|
||||
'p1',
|
||||
'ws-1',
|
||||
'git-sync',
|
||||
);
|
||||
expect(res).toEqual({ id: 'p1' });
|
||||
});
|
||||
});
|
||||
|
||||
// Phase-B+ continuous-sync methods: not yet called by the engine but wired into
|
||||
// the GitSyncClient seam (PR #119 review #5). Exercised via the bound client.
|
||||
describe('listRecentSince', () => {
|
||||
it('queries non-deleted pages newest-first and ISO-stringifies updatedAt', async () => {
|
||||
const rows = [
|
||||
{
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'A',
|
||||
parentPageId: null,
|
||||
spaceId: 'space-1',
|
||||
updatedAt: new Date('2026-06-20T10:00:00.000Z'),
|
||||
},
|
||||
];
|
||||
const { service, mocks } = build(rows);
|
||||
const qb = mocks.db.selectFrom.mock.results; // populated after the call
|
||||
|
||||
const out = (await service
|
||||
.bind(CTX)
|
||||
.listRecentSince('space-1', '2026-06-19T00:00:00.000Z', 100)) as any[];
|
||||
|
||||
// Query builder shaped against the `pages` table with the expected chain.
|
||||
expect(mocks.db.selectFrom).toHaveBeenCalledWith('pages');
|
||||
const builder = qb[0].value;
|
||||
expect(builder.select).toHaveBeenCalled();
|
||||
expect(builder.orderBy).toHaveBeenCalledWith('updatedAt', 'desc');
|
||||
// deletedAt is null + the conditional spaceId / since / cap clauses.
|
||||
const whereArgs = builder.where.mock.calls.map((c: any[]) => c[0]);
|
||||
expect(whereArgs).toContain('deletedAt');
|
||||
expect(whereArgs).toContain('spaceId');
|
||||
expect(whereArgs).toContain('updatedAt');
|
||||
expect(builder.limit).toHaveBeenCalledWith(100);
|
||||
|
||||
expect(out).toEqual([
|
||||
{
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'A',
|
||||
parentPageId: null,
|
||||
spaceId: 'space-1',
|
||||
updatedAt: '2026-06-20T10:00:00.000Z',
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('omits the spaceId / since / cap clauses when not supplied', async () => {
|
||||
const { service, mocks } = build([]);
|
||||
|
||||
await service.bind(CTX).listRecentSince(undefined, null);
|
||||
|
||||
const builder = mocks.db.selectFrom.mock.results[0].value;
|
||||
const whereArgs = builder.where.mock.calls.map((c: any[]) => c[0]);
|
||||
// Only the deletedAt-is-null guard; no spaceId / updatedAt> clauses.
|
||||
expect(whereArgs).toEqual(['deletedAt']);
|
||||
expect(builder.limit).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('listTrash', () => {
|
||||
it('queries soft-deleted pages and ISO-stringifies deletedAt (null stays null)', async () => {
|
||||
const rows = [
|
||||
{
|
||||
id: 'p1',
|
||||
slugId: 's1',
|
||||
title: 'Trashed',
|
||||
parentPageId: null,
|
||||
spaceId: 'space-1',
|
||||
deletedAt: new Date('2026-06-21T09:00:00.000Z'),
|
||||
},
|
||||
{
|
||||
id: 'p2',
|
||||
slugId: 's2',
|
||||
title: 'NoDate',
|
||||
parentPageId: null,
|
||||
spaceId: 'space-1',
|
||||
deletedAt: null,
|
||||
},
|
||||
];
|
||||
const { service, mocks } = build(rows);
|
||||
|
||||
const out = (await service.bind(CTX).listTrash('space-1')) as any[];
|
||||
|
||||
expect(mocks.db.selectFrom).toHaveBeenCalledWith('pages');
|
||||
const builder = mocks.db.selectFrom.mock.results[0].value;
|
||||
const whereCalls = builder.where.mock.calls;
|
||||
// deletedAt is-not null (the trash predicate) + spaceId filter.
|
||||
expect(whereCalls).toContainEqual(['deletedAt', 'is not', null]);
|
||||
expect(whereCalls).toContainEqual(['spaceId', '=', 'space-1']);
|
||||
expect(builder.orderBy).toHaveBeenCalledWith('deletedAt', 'desc');
|
||||
|
||||
expect(out[0].deletedAt).toBe('2026-06-21T09:00:00.000Z');
|
||||
expect(out[1].deletedAt).toBeNull();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,738 @@
|
||||
import { Injectable, Logger, NotFoundException } from '@nestjs/common';
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { validate as isValidUUID } from 'uuid';
|
||||
import type {
|
||||
GitSyncClient,
|
||||
GitSyncPageNodeLite,
|
||||
} from '@docmost/git-sync';
|
||||
import { loadGitSync } from '../git-sync.loader';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { PageService } from '../../../core/page/services/page.service';
|
||||
import { CollaborationGateway } from '../../../collaboration/collaboration.gateway';
|
||||
import { AuthProvenanceData } from '../../../common/decorators/auth-provenance.decorator';
|
||||
|
||||
/**
|
||||
* The acting context the orchestrator binds the datasource to. The datasource is
|
||||
* NOT a fixed-identity singleton: it operates on behalf of a (workspaceId,
|
||||
* userId) pair the orchestrator supplies per space. `userId` is the
|
||||
* git-sync service user — it stays the responsible author (creatorId /
|
||||
* lastUpdatedById) while the `'git-sync'` actor marks provenance.
|
||||
*/
|
||||
export interface GitSyncBindContext {
|
||||
workspaceId: string;
|
||||
userId: string;
|
||||
/**
|
||||
* The space this cycle reconciles. Used to distinguish a genuine page deletion
|
||||
* from a cross-space MOVE: when a page leaves space A, A's vault file is removed
|
||||
* and the push phase would otherwise soft-delete the page — but the page still
|
||||
* lives in space B. `deletePage` skips the soft-delete when the page's current
|
||||
* space differs from the reconciling space. Optional for back-compat.
|
||||
*/
|
||||
spaceId?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The git-sync provenance carried into PageService writes. PageService.create/
|
||||
* update/movePage honor this provenance and stamp `lastUpdatedSource = 'git-sync'`
|
||||
* on the page row when `provenance.actor === 'git-sync'`. Body writes (writeBody,
|
||||
* §3.3) likewise stamp 'git-sync' because the collab context's `actor: 'git-sync'`
|
||||
* flows into PersistenceExtension. So ALL git-sync structural + body writes mark
|
||||
* the row's source, which the listener's loop-guard reads to skip our own writes.
|
||||
*/
|
||||
const GIT_SYNC_PROVENANCE: AuthProvenanceData = {
|
||||
actor: 'git-sync',
|
||||
aiChatId: null,
|
||||
};
|
||||
|
||||
/**
|
||||
* Native, in-process implementation of the engine's `GitSyncClient` seam
|
||||
* Reads go through repositories (PageRepo/SpaceRepo); body writes go
|
||||
* through collab `openDirectConnection` (§3.3); structural mutations
|
||||
* (create/move/delete/rename) go through PageService.
|
||||
*
|
||||
* Shape: this is an `@Injectable()` holding the repos/services. The orchestrator
|
||||
* calls `bind({ workspaceId, userId })` to obtain a `GitSyncClient` bound to that
|
||||
* acting context. The bound object is a thin closure over `this` — no per-call
|
||||
* identity plumbing leaks into the engine.
|
||||
*/
|
||||
@Injectable()
|
||||
export class GitmostDataSourceService {
|
||||
private readonly logger = new Logger(GitmostDataSourceService.name);
|
||||
|
||||
constructor(
|
||||
private readonly pageRepo: PageRepo,
|
||||
private readonly spaceRepo: SpaceRepo,
|
||||
private readonly pageService: PageService,
|
||||
private readonly collabGateway: CollaborationGateway,
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Bind the datasource to an acting (workspaceId, userId) context and return a
|
||||
* `GitSyncClient` the engine can consume directly.
|
||||
*/
|
||||
bind(ctx: GitSyncBindContext): GitSyncClient {
|
||||
return {
|
||||
listSpaceTree: (spaceId, rootPageId) =>
|
||||
this.listSpaceTree(ctx, spaceId, rootPageId),
|
||||
pageIdsExist: (pageIds) => this.pageIdsExist(ctx, pageIds),
|
||||
getPageJson: (pageId) => this.getPageJson(ctx, pageId),
|
||||
// The id-scoped WRITE ops are wrapped so a malformed (non-UUID) `pageId`
|
||||
// from a broken vault `gitmost_id` frontmatter cannot wedge the space's sync
|
||||
// loop (bug C9-D1) — see `skipIfMalformedId`.
|
||||
importPageMarkdown: (pageId, fullMarkdown, baseMarkdown) =>
|
||||
this.skipIfMalformedId(
|
||||
'import',
|
||||
pageId,
|
||||
ctx,
|
||||
() => this.importPageMarkdown(ctx, pageId, fullMarkdown, baseMarkdown),
|
||||
{},
|
||||
),
|
||||
createPage: (title, content, spaceId, parentPageId) =>
|
||||
this.createPage(ctx, title, content, spaceId, parentPageId),
|
||||
deletePage: (pageId) =>
|
||||
this.skipIfMalformedId('delete', pageId, ctx, () =>
|
||||
this.deletePage(ctx, pageId),
|
||||
),
|
||||
movePage: (pageId, parentPageId, position) =>
|
||||
this.skipIfMalformedId('move', pageId, ctx, () =>
|
||||
this.movePage(ctx, pageId, parentPageId, position),
|
||||
),
|
||||
renamePage: (pageId, title) =>
|
||||
this.skipIfMalformedId('rename', pageId, ctx, () =>
|
||||
this.renamePage(ctx, pageId, title),
|
||||
),
|
||||
listRecentSince: (spaceId, sinceIso, hardPageCap) =>
|
||||
this.listRecentSince(spaceId, sinceIso, hardPageCap),
|
||||
listTrash: (spaceId) => this.listTrash(spaceId),
|
||||
restorePage: (pageId) =>
|
||||
this.skipIfMalformedId('restore', pageId, ctx, () =>
|
||||
this.restorePage(ctx, pageId),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Run an id-scoped write op; if `pageId` was a malformed NON-UUID token (a
|
||||
* broken/hand-edited vault `gitmost_id`, e.g. `gitmost_id: [unclosed`), Postgres
|
||||
* rejects it at the `uuid` predicate with error code `22P02`
|
||||
* ("invalid input syntax for type uuid"). Left unhandled, the push apply records
|
||||
* that throw as a per-cycle failure that NEVER clears — refs never advance, so
|
||||
* the WHOLE space's sync loops on the same failure indefinitely (bug C9-D1).
|
||||
* Swallow exactly that error as an inert no-op so the cycle succeeds and the rest
|
||||
* of the space keeps syncing; re-throw anything else. NOTE: `pageId` is NOT the
|
||||
* only user-influenced uuid in these ops — `movePage` (and `createPage`) also
|
||||
* carry a `parentPageId`, sourced from the PARENT folder-note's `gitmost_id`
|
||||
* frontmatter. So a 22P02 caught here can originate from a malformed PARENT id,
|
||||
* not the child `pageId`; the log message is therefore op-generic and does not
|
||||
* attribute the bad id to a specific field. (createPage/movePage additionally
|
||||
* COERCE a malformed `parentPageId` to root up-front — see those methods — so
|
||||
* that variant self-heals and never reaches this catch.)
|
||||
*/
|
||||
private async skipIfMalformedId<T>(
|
||||
op: string,
|
||||
pageId: string,
|
||||
ctx: GitSyncBindContext,
|
||||
run: () => Promise<T>,
|
||||
fallback?: T,
|
||||
): Promise<T | undefined> {
|
||||
try {
|
||||
return await run();
|
||||
} catch (err) {
|
||||
if ((err as { code?: string })?.code === '22P02') {
|
||||
this.logger.warn(
|
||||
`git-sync[${ctx.spaceId ?? '-'}] skip ${op}: malformed (non-UUID) id reached a uuid predicate; ignored (no wedge)`,
|
||||
);
|
||||
return fallback;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// --- reads (pull) ---------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Full page tree of a space mapped to the engine's `PageNode` shape. We read
|
||||
* the DB directly, so `complete` is ALWAYS `true` — the incomplete-fetch
|
||||
* suppression (SPEC §8) never fires natively.
|
||||
*/
|
||||
private async listSpaceTree(
|
||||
ctx: GitSyncBindContext,
|
||||
spaceId: string,
|
||||
_rootPageId?: string,
|
||||
): Promise<{ pages: GitSyncPageNodeLite[]; complete: boolean }> {
|
||||
const space = await this.spaceRepo.findById(spaceId, ctx.workspaceId);
|
||||
if (!space) {
|
||||
throw new NotFoundException(`Space ${spaceId} not found`);
|
||||
}
|
||||
|
||||
const rows = await this.pageRepo.getSpaceDescendants(space.id, {
|
||||
includeContent: false,
|
||||
});
|
||||
|
||||
// `getSpaceDescendants` does not select `hasChildren`; derive it from the
|
||||
// parent links present in the same result set.
|
||||
const parentIds = new Set<string>();
|
||||
for (const row of rows) {
|
||||
if (row.parentPageId) parentIds.add(row.parentPageId);
|
||||
}
|
||||
|
||||
const pages: GitSyncPageNodeLite[] = rows.map((row) => ({
|
||||
id: row.id,
|
||||
slugId: row.slugId,
|
||||
title: row.title,
|
||||
parentPageId: row.parentPageId ?? null,
|
||||
hasChildren: parentIds.has(row.id),
|
||||
position: row.position,
|
||||
}));
|
||||
|
||||
return { pages, complete: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Existence probe for the pull-side ghost guard (D-P3-1). Return the subset of
|
||||
* `pageIds` that correspond to a REAL page ROW in this workspace — INCLUDING
|
||||
* soft-deleted (trashed) rows and pages in ANY OTHER space (no `deletedAt` /
|
||||
* `spaceId` filter), i.e. any id that has ever been a page. The engine only
|
||||
* absence-deletes a tracked vault file whose id is returned here; a GHOST id
|
||||
* (a git file whose id was never a page) has no row and is preserved.
|
||||
*
|
||||
* Malformed (non-UUID) ids are filtered out BEFORE the `id = ANY(...)`
|
||||
* predicate: an invalid uuid literal would make Postgres reject the whole
|
||||
* query (error 22P02), and a ghost id is often exactly such a hand-edited
|
||||
* token. A filtered-out id is never returned, so its file is preserved.
|
||||
*/
|
||||
private async pageIdsExist(
|
||||
ctx: GitSyncBindContext,
|
||||
pageIds: string[],
|
||||
): Promise<string[]> {
|
||||
const validIds = pageIds.filter((id) => isValidUUID(id));
|
||||
if (validIds.length === 0) return [];
|
||||
|
||||
const rows = await this.db
|
||||
.selectFrom('pages')
|
||||
.select(['id'])
|
||||
.where('workspaceId', '=', ctx.workspaceId)
|
||||
.where('id', 'in', validIds)
|
||||
.execute();
|
||||
|
||||
return rows.map((row) => row.id);
|
||||
}
|
||||
|
||||
/**
|
||||
* One page WITH its ProseMirror body content (editor-ext schema). `updatedAt`
|
||||
* is serialized to an ISO string for the loop-guard.
|
||||
*/
|
||||
private async getPageJson(
|
||||
ctx: GitSyncBindContext,
|
||||
pageId: string,
|
||||
): Promise<{
|
||||
id: string;
|
||||
slugId: string;
|
||||
title: string;
|
||||
parentPageId: string | null;
|
||||
spaceId: string;
|
||||
updatedAt: string;
|
||||
content: unknown;
|
||||
}> {
|
||||
const page = await this.pageRepo.findById(pageId, { includeContent: true });
|
||||
if (!page) {
|
||||
throw new NotFoundException(`Page ${pageId} not found`);
|
||||
}
|
||||
|
||||
return {
|
||||
id: page.id,
|
||||
slugId: page.slugId,
|
||||
title: page.title,
|
||||
parentPageId: page.parentPageId ?? null,
|
||||
spaceId: page.spaceId,
|
||||
updatedAt: new Date(page.updatedAt).toISOString(),
|
||||
content: page.content,
|
||||
};
|
||||
}
|
||||
|
||||
// --- writes (push) --------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Merge a page's body from a self-contained markdown file: parse the meta+body
|
||||
* envelope, convert the body to ProseMirror, then merge it through collab
|
||||
* (§3.3). When `baseMarkdown` (the last-synced version of the file) is given,
|
||||
* the body write is a THREE-WAY merge against the live doc so concurrent human
|
||||
* edits survive (review #5); without it, a 2-way merge. Returns the fresh
|
||||
* page's `updatedAt` for the loop-guard.
|
||||
*/
|
||||
private async importPageMarkdown(
|
||||
ctx: GitSyncBindContext,
|
||||
pageId: string,
|
||||
fullMarkdown: string,
|
||||
baseMarkdown?: string | null,
|
||||
): Promise<{ updatedAt?: string }> {
|
||||
// Idempotency guard #1 (fixes GS-EDIT-REVERT + idle re-ingest churn). The
|
||||
// reconcile can call this every poll cycle for a page whose vault file did
|
||||
// NOT actually change since the last sync (non-idempotent change-detection
|
||||
// upstream). Each such call re-imports the SAME vault body into the live
|
||||
// collab doc — a no-op at idle, but it CLOBBERS a concurrent human edit that
|
||||
// is still in the (debounced, not-yet-flushed) Yjs doc, silently reverting
|
||||
// it within one poll. When `baseMarkdown` (the last-synced version) is
|
||||
// byte-identical to the current file, there is genuinely nothing to ingest.
|
||||
// A real git-side change makes the strings differ, so legitimate
|
||||
// git->Docmost ingests still proceed.
|
||||
const currentPage = await this.pageRepo.findById(pageId, {
|
||||
includeContent: true,
|
||||
});
|
||||
// Unknown-page guard (bug N1-D1). `importPageMarkdown` is only ever called for a
|
||||
// vault file that CARRIES a `gitmost_id`, so a null page means the id is a
|
||||
// well-formed UUID that matches NO page — a stale id from a restored-from-backup
|
||||
// file, or a copied/foreign id. Left unhandled it falls through to `writeBody()`
|
||||
// on a non-existent page, which throws "Page … not found"; the push apply records
|
||||
// that as a per-cycle failure that never clears, wedging the whole space's sync
|
||||
// loop (same user-visible impact as C9-D1, but the id is a VALID uuid so the
|
||||
// 22P02 guard does not catch it). Skip it as an inert no-op so the cycle succeeds
|
||||
// and the rest of the space keeps syncing. (ADOPTING such a file as a fresh page
|
||||
// — the restore-from-backup use case — is a separate design decision: the title
|
||||
// lives in the filename, which the engine classifier holds, not this method.)
|
||||
if (currentPage == null) {
|
||||
this.logger.warn(
|
||||
`git-sync[${ctx.spaceId ?? '-'}] skip import of page ${pageId}: no page with that id (stale/foreign gitmost_id; not adopted, no wedge)`,
|
||||
);
|
||||
return {};
|
||||
}
|
||||
// Cross-space confused-deputy guard (review S2). The target `pageId` comes
|
||||
// from THIS space's vault file frontmatter, but a file in space A could carry
|
||||
// space B's page id. Without this check that file could resurrect (via
|
||||
// restorePage), overwrite the body (writeBody), or clear the content of B's
|
||||
// page — a cross-space write the reconciling space has no authority over.
|
||||
// Mirror deletePage's guard (same `ctx.spaceId` source, same fail-safe
|
||||
// direction): when the reconciling space is known and the resolved page
|
||||
// already lives in a DIFFERENT space, skip — touch nothing. A null page is
|
||||
// already handled by the N1-D1 guard above, so here `currentPage` is
|
||||
// guaranteed non-null.
|
||||
if (ctx.spaceId && currentPage.spaceId !== ctx.spaceId) {
|
||||
this.logger.log(
|
||||
`git-sync[${ctx.spaceId}] skip import of page ${pageId}: page lives in space ${currentPage.spaceId} (cross-space vault reference; page preserved)`,
|
||||
);
|
||||
return {
|
||||
updatedAt: new Date(currentPage.updatedAt).toISOString(),
|
||||
};
|
||||
}
|
||||
// Revert-of-delete undelete (review warning). If the target page is currently
|
||||
// SOFT-DELETED, this ingest is a git-revert that re-added the page's file
|
||||
// (the push classifier saw an add carrying a known pageId -> UPDATE). Writing
|
||||
// the body to a trashed page leaves it in Trash, and the next pull re-deletes
|
||||
// the file — the git revert is silently nullified. Restore the page FIRST so
|
||||
// the revert actually brings it back, then apply the body below. (restorePage
|
||||
// stamps git-sync provenance so the loop-guard skips its own echo.)
|
||||
if (currentPage.deletedAt != null) {
|
||||
await this.restorePage(ctx, pageId);
|
||||
}
|
||||
// Skip the early no-op return when we just restored (the page must still get
|
||||
// its body write below); only short-circuit for a live, unchanged page.
|
||||
if (
|
||||
currentPage.deletedAt == null &&
|
||||
baseMarkdown != null &&
|
||||
fullMarkdown === baseMarkdown
|
||||
) {
|
||||
return {
|
||||
updatedAt: new Date(currentPage.updatedAt).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
const { parseDocmostMarkdown, markdownToProseMirror, docsCanonicallyEqual } =
|
||||
await loadGitSync();
|
||||
const { body } = parseDocmostMarkdown(fullMarkdown);
|
||||
const doc = await markdownToProseMirror(body);
|
||||
|
||||
// Idempotency guard #2 (defense-in-depth). Even when the vault file text
|
||||
// differs cosmetically, the PARSED body can be SEMANTICALLY identical to the
|
||||
// page's current Docmost content — the markdown<->ProseMirror round-trip is
|
||||
// not byte-stable, so upstream change-detection mis-flags such pages as
|
||||
// changed every cycle. But the divergence is NOT just JSON key order: a fresh
|
||||
// `markdownToProseMirror(doc)` carries new/null block ids and materialized
|
||||
// schema default attrs, whereas `currentContent` (from the DB) carries the
|
||||
// real per-block uuids (to which comments are anchored) and KNOWN_DEFAULTS.
|
||||
// A key-order-only compare therefore NEVER matches a real collab page, so use
|
||||
// the package's authoritative `docsCanonicallyEqual` — the same equality the
|
||||
// converter's round-trip losslessness tests use, which strips block ids and
|
||||
// normalizes KNOWN_DEFAULTS. If the incoming body already equals current
|
||||
// content, this ingest is a no-op — skip it so a concurrent live edit is
|
||||
// never clobbered and the vault never churns. A genuine content change is not
|
||||
// canonically equal, so it proceeds.
|
||||
const currentContent =
|
||||
typeof currentPage.content === 'string'
|
||||
? (() => {
|
||||
try {
|
||||
return JSON.parse(currentPage.content as unknown as string);
|
||||
} catch {
|
||||
return currentPage.content;
|
||||
}
|
||||
})()
|
||||
: currentPage.content;
|
||||
if (currentContent && docsCanonicallyEqual(doc, currentContent)) {
|
||||
return {
|
||||
updatedAt: new Date(currentPage.updatedAt).toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
let baseDoc: unknown;
|
||||
if (baseMarkdown != null) {
|
||||
const { body: baseBody } = parseDocmostMarkdown(baseMarkdown);
|
||||
baseDoc = await markdownToProseMirror(baseBody);
|
||||
}
|
||||
|
||||
await this.writeBody(pageId, doc, ctx.userId, baseDoc);
|
||||
|
||||
// CAVEAT: writeBody merges through collab, whose persistence is DEBOUNCED, so
|
||||
// this `updatedAt` read can be STALE — it may reflect the row BEFORE the
|
||||
// debounced flush lands. Currently harmless: the only consumer is the deferred
|
||||
// §10 loop-guard, which is not yet wired. When that loop-guard is implemented
|
||||
// it MUST NOT trust this timestamp as a read-after-write of the body change
|
||||
// (it would misfire on the pre-flush value); it needs a post-flush read (or to
|
||||
// key off the collab flush completion) instead.
|
||||
const page = await this.pageRepo.findById(pageId);
|
||||
return {
|
||||
updatedAt: page ? new Date(page.updatedAt).toISOString() : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a page shell via PageService, then write its body through collab.
|
||||
* Returns the assigned id (`data.id`) + the page's `updatedAt`.
|
||||
*/
|
||||
private async createPage(
|
||||
ctx: GitSyncBindContext,
|
||||
title: string,
|
||||
content: string,
|
||||
spaceId: string,
|
||||
parentPageId?: string,
|
||||
): Promise<{ data: { id: string }; updatedAt?: string }> {
|
||||
// F1 self-heal (bug C9-D1, parent-id variant). `parentPageId` is a SECOND
|
||||
// user-influenced uuid: the push planner derives it from the parent
|
||||
// folder-note's `gitmost_id` frontmatter (resolveParentPageIdViaTree). A
|
||||
// broken/hand-edited non-UUID value flows into pageService.create, whose
|
||||
// findById(parentPageId) falls back to a slugId lookup (no row) and throws
|
||||
// NotFoundException — a throw that lands in the push `failures` set. The
|
||||
// wedge-gate only advances refs when failures is empty, so the WHOLE space
|
||||
// loops forever re-attempting (same failure mode as the self-id C9-D1 bug,
|
||||
// inside the same threat model). Do NOT skip the create (that would DROP the
|
||||
// page); instead COERCE a malformed parent to root (undefined) so the page is
|
||||
// created at the space root and self-heals, never wedging. Reuses the shared
|
||||
// `uuid` validator — the same check pageRepo.findById uses to tell an id from
|
||||
// a slugId — so a VALID parentPageId is left untouched.
|
||||
if (parentPageId && !isValidUUID(parentPageId)) {
|
||||
this.logger.warn(
|
||||
`git-sync[${ctx.spaceId ?? '-'}] createPage: malformed (non-UUID) parentPageId '${parentPageId}' coerced to root (self-heal; no wedge)`,
|
||||
);
|
||||
parentPageId = undefined;
|
||||
}
|
||||
|
||||
const page = await this.pageService.create(
|
||||
ctx.userId,
|
||||
ctx.workspaceId,
|
||||
{ spaceId, title, parentPageId },
|
||||
GIT_SYNC_PROVENANCE,
|
||||
);
|
||||
|
||||
// The shell is created without body; push the markdown body through collab.
|
||||
const { parseDocmostMarkdown, markdownToProseMirror } = await loadGitSync();
|
||||
const { body } = parseDocmostMarkdown(content);
|
||||
const doc = await markdownToProseMirror(body);
|
||||
await this.writeBody(page.id, doc, ctx.userId);
|
||||
|
||||
const fresh = await this.pageRepo.findById(page.id);
|
||||
return {
|
||||
data: { id: page.id },
|
||||
updatedAt: fresh ? new Date(fresh.updatedAt).toISOString() : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Soft-delete the page to Trash (reversible). NOT a force delete — `restorePage`
|
||||
* can bring it back.
|
||||
*/
|
||||
private async deletePage(
|
||||
ctx: GitSyncBindContext,
|
||||
pageId: string,
|
||||
): Promise<unknown> {
|
||||
// Cross-space MOVE guard. A push-phase delete fires when a page's file
|
||||
// disappears from THIS space's vault. That happens for a genuine deletion —
|
||||
// but ALSO when the page was moved to another space (source vault file
|
||||
// removed, page recreated in the destination vault). In the move case the
|
||||
// page still exists and must NOT be trashed: soft-deleting it here loses the
|
||||
// page from BOTH vaults and dumps it in Trash (observed data-loss on
|
||||
// move-to-space with git-sync enabled). If the page's CURRENT space differs
|
||||
// from the space we're reconciling, this is a move-out — drop only the vault
|
||||
// file (already done by the engine), never the page.
|
||||
if (ctx.spaceId) {
|
||||
const page = await this.pageRepo.findById(pageId);
|
||||
if (page && page.deletedAt == null && page.spaceId !== ctx.spaceId) {
|
||||
this.logger.log(
|
||||
`git-sync[${ctx.spaceId}] skip delete of page ${pageId}: moved to space ${page.spaceId} (vault file removed, page preserved)`,
|
||||
);
|
||||
return { id: pageId, skipped: 'moved-to-other-space' };
|
||||
}
|
||||
}
|
||||
await this.pageService.removePage(
|
||||
pageId,
|
||||
ctx.userId,
|
||||
ctx.workspaceId,
|
||||
GIT_SYNC_PROVENANCE,
|
||||
);
|
||||
return { id: pageId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Reparent a page. Docmost-move REQUIRES a fractional-index `position`; when the
|
||||
* engine omits it, compute a key after the destination's last sibling (plan
|
||||
* §3.2 / §14.4).
|
||||
*/
|
||||
private async movePage(
|
||||
ctx: GitSyncBindContext,
|
||||
pageId: string,
|
||||
parentPageId: string | null,
|
||||
position?: string,
|
||||
): Promise<unknown> {
|
||||
const page = await this.pageRepo.findById(pageId);
|
||||
if (!page) {
|
||||
throw new NotFoundException(`Page ${pageId} not found`);
|
||||
}
|
||||
|
||||
// F1 self-heal (parent-id variant), mirror of createPage. `parentPageId` is a
|
||||
// second user-influenced uuid (the DESTINATION parent folder-note's
|
||||
// `gitmost_id`). A malformed value reaches `computeMovePosition`'s raw
|
||||
// `where('parentPageId','=',parentPageId)` uuid predicate (22P02 — swallowed
|
||||
// by skipIfMalformedId but MIS-attributed to the child `pageId`) when no
|
||||
// position is supplied; and when a position IS supplied it reaches
|
||||
// pageService.movePage's findById(parentPageId) -> NotFoundException, which is
|
||||
// NOT a 22P02 and so is NOT caught by skipIfMalformedId -> the space wedges.
|
||||
// Coerce a malformed parent to root (null) so the page becomes a root page
|
||||
// (self-heal) rather than wedging or being silently swallowed. Same shared
|
||||
// `uuid` validator as createPage; a valid parent is left untouched.
|
||||
if (parentPageId && !isValidUUID(parentPageId)) {
|
||||
this.logger.warn(
|
||||
`git-sync[${ctx.spaceId ?? '-'}] movePage: malformed (non-UUID) parentPageId '${parentPageId}' coerced to root (self-heal; no wedge)`,
|
||||
);
|
||||
parentPageId = null;
|
||||
}
|
||||
|
||||
// GS-MOVE-ECHO guard (review #6). A drag-move in Docmost echoes back through
|
||||
// git-sync as movePage(pageId, sameParent) WITHOUT a position. Recomputing a
|
||||
// position here would append the page to the end of its sibling list,
|
||||
// clobbering the position the user just chose. If the parent is unchanged and
|
||||
// no explicit position was provided, there is nothing to reparent — skip, so
|
||||
// the user's ordering is preserved. A real reparent (parent differs) or an
|
||||
// explicit position still proceeds.
|
||||
if (position == null && parentPageId === (page.parentPageId ?? null)) {
|
||||
return { id: pageId, skipped: 'no-op-move-echo' };
|
||||
}
|
||||
|
||||
const resolvedPosition =
|
||||
position ?? (await this.computeMovePosition(page.spaceId, parentPageId));
|
||||
|
||||
await this.pageService.movePage(
|
||||
{ pageId, parentPageId: parentPageId ?? null, position: resolvedPosition },
|
||||
page,
|
||||
GIT_SYNC_PROVENANCE,
|
||||
// Attribute the git-initiated move to the service user (lastUpdatedById),
|
||||
// matching create/delete/rename — the contract is "git-operations are
|
||||
// attributed to the service account".
|
||||
ctx.userId,
|
||||
);
|
||||
return { id: pageId };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute a fractional-index position AFTER the last sibling under
|
||||
* `parentPageId` (root pages when null) in the space, ordered by `position`
|
||||
* with the "C" collation Docmost uses. Falls back to a fresh key
|
||||
* when there are no siblings.
|
||||
*/
|
||||
private async computeMovePosition(
|
||||
spaceId: string,
|
||||
parentPageId: string | null,
|
||||
): Promise<string> {
|
||||
let query = this.db
|
||||
.selectFrom('pages')
|
||||
.select(['position'])
|
||||
.where('spaceId', '=', spaceId)
|
||||
.where('deletedAt', 'is', null)
|
||||
.orderBy('position', (ob) => ob.collate('C').desc())
|
||||
.limit(1);
|
||||
|
||||
query = parentPageId
|
||||
? query.where('parentPageId', '=', parentPageId)
|
||||
: query.where('parentPageId', 'is', null);
|
||||
|
||||
const lastSibling = await query.executeTakeFirst();
|
||||
return generateJitteredKeyBetween(lastSibling?.position ?? null, null);
|
||||
}
|
||||
|
||||
/** Change a page's title only (no body touch). */
|
||||
private async renamePage(
|
||||
ctx: GitSyncBindContext,
|
||||
pageId: string,
|
||||
title: string,
|
||||
): Promise<unknown> {
|
||||
const page = await this.pageRepo.findById(pageId);
|
||||
if (!page) {
|
||||
throw new NotFoundException(`Page ${pageId} not found`);
|
||||
}
|
||||
// Defensive de-pollution of the cosmetic ` ~<slugId>` disambiguation suffix.
|
||||
// When two sibling pages share a title, the vault layout appends ` ~<slugId>`
|
||||
// to the colliding file's stem (engine `disambiguate(name, slugId)` = exactly
|
||||
// `${name} ~${slugId}`) so two pages never map to one `.md`. That suffix is a
|
||||
// LOCAL filesystem artifact and must NEVER become the page's real Docmost
|
||||
// title. A filename-derived title can carry it back in on ingest (observed:
|
||||
// intermittent same-title collision left a page permanently titled
|
||||
// "Title ~<slugId>"). Strip it here on the RENAME path (this is where a
|
||||
// filename-derived title lands as a page's real title); other title-write
|
||||
// paths (e.g. createPage / importPageMarkdown) are separate and not covered
|
||||
// by this choke point. Strip ONLY when the trailing token equals THIS page's
|
||||
// own slugId, so a genuine user title that legitimately ends in ` ~token` is
|
||||
// never corrupted (slugId is a random nanoid; no real collision).
|
||||
const suffix = ` ~${page.slugId}`;
|
||||
const cleanTitle =
|
||||
page.slugId && title.endsWith(suffix)
|
||||
? title.slice(0, -suffix.length)
|
||||
: title;
|
||||
// GS-TITLE-SANITIZE guard (review Critical #2). A rename in Docmost to a
|
||||
// title with filename-hostile chars (`:` `/` `"` `|`, newlines, double
|
||||
// spaces, >120 chars) is pulled to a SANITIZED file stem; the same cycle then
|
||||
// sees an R(ename) line and would call renamePage with that sanitized stem,
|
||||
// PERMANENTLY replacing the real title with its sanitized form (e.g.
|
||||
// "Project: Plan" -> "Project- Plan"). In that echo the incoming title equals
|
||||
// `sanitizeTitle(current title)`, so skip the write — the sanitized stem is a
|
||||
// local filesystem artifact, never the page's real Docmost title. A genuine
|
||||
// retitle does NOT equal the sanitized current title, so it still applies.
|
||||
const { sanitizeTitle } = await loadGitSync();
|
||||
if (
|
||||
page.title &&
|
||||
cleanTitle !== page.title &&
|
||||
sanitizeTitle(page.title) === cleanTitle
|
||||
) {
|
||||
this.logger.log(
|
||||
`git-sync: skip rename of page ${pageId}: incoming title is the sanitized form of current title (filesystem artifact; real title preserved)`,
|
||||
);
|
||||
return { id: pageId };
|
||||
}
|
||||
// PageService.update takes a User; the git-sync service user is the
|
||||
// responsible author. Only the id is read off it for lastUpdatedById.
|
||||
// `pageId` satisfies the UpdatePageDto type; PageService.update reads the
|
||||
// page id off `page`, not the DTO. Only `title` is applied here.
|
||||
await this.pageService.update(
|
||||
page,
|
||||
{ pageId, title: cleanTitle },
|
||||
{ id: ctx.userId } as any,
|
||||
GIT_SYNC_PROVENANCE,
|
||||
);
|
||||
return { id: pageId };
|
||||
}
|
||||
|
||||
// --- continuous (phase B+) ------------------------------------------------
|
||||
|
||||
/**
|
||||
* Pages in the space updated since `sinceIso` (poll-safety reconciliation,
|
||||
* SPEC §8). `spaceId` undefined widens to all spaces; `hardPageCap` bounds the
|
||||
* result. Reads the DB directly (no cursor pagination needed here).
|
||||
*/
|
||||
private async listRecentSince(
|
||||
spaceId: string | undefined,
|
||||
sinceIso: string | null,
|
||||
hardPageCap?: number,
|
||||
): Promise<unknown[]> {
|
||||
let query = this.db
|
||||
.selectFrom('pages')
|
||||
.select([
|
||||
'id',
|
||||
'slugId',
|
||||
'title',
|
||||
'parentPageId',
|
||||
'spaceId',
|
||||
'updatedAt',
|
||||
])
|
||||
.where('deletedAt', 'is', null)
|
||||
.orderBy('updatedAt', 'desc');
|
||||
|
||||
if (spaceId) query = query.where('spaceId', '=', spaceId);
|
||||
if (sinceIso) query = query.where('updatedAt', '>', new Date(sinceIso));
|
||||
if (hardPageCap) query = query.limit(hardPageCap);
|
||||
|
||||
const rows = await query.execute();
|
||||
return rows.map((row) => ({
|
||||
...row,
|
||||
updatedAt: new Date(row.updatedAt).toISOString(),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Soft-deleted (trashed) pages for the space (deletion detection). */
|
||||
private async listTrash(spaceId: string): Promise<unknown[]> {
|
||||
const rows = await this.db
|
||||
.selectFrom('pages')
|
||||
.select(['id', 'slugId', 'title', 'parentPageId', 'spaceId', 'deletedAt'])
|
||||
.where('spaceId', '=', spaceId)
|
||||
.where('deletedAt', 'is not', null)
|
||||
.orderBy('deletedAt', 'desc')
|
||||
.execute();
|
||||
|
||||
return rows.map((row) => ({
|
||||
...row,
|
||||
deletedAt: row.deletedAt ? new Date(row.deletedAt).toISOString() : null,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Restore a soft-deleted page from Trash. */
|
||||
private async restorePage(
|
||||
ctx: GitSyncBindContext,
|
||||
pageId: string,
|
||||
): Promise<unknown> {
|
||||
// Stamp git-sync provenance so the change-listener loop-guard skips the
|
||||
// PAGE_RESTORED echo (mirrors deletePage / create / update / move).
|
||||
await this.pageRepo.restorePage(
|
||||
pageId,
|
||||
ctx.workspaceId,
|
||||
GIT_SYNC_PROVENANCE.actor,
|
||||
);
|
||||
return { id: pageId };
|
||||
}
|
||||
|
||||
// --- linchpin: native body write (§3.3) -----------------------------------
|
||||
|
||||
/**
|
||||
* In-process body write — no loopback websocket, no service-user token.
|
||||
*
|
||||
* Routes the write through `CollaborationGateway.writePageBody`, which applies
|
||||
* the block-level MERGE on the instance that OWNS the live Y.Doc (via the
|
||||
* custom-event channel) rather than opening a direct connection on this
|
||||
* (api/worker) instance. That distinction is load-bearing: when an editor is
|
||||
* connected to a different collab instance/process, a direct connection here
|
||||
* mutates a SEPARATE, detached doc the editor never sees — the editor's next
|
||||
* autosave then silently REVERTS the git change (data loss). Running on the
|
||||
* owning instance broadcasts the merge as a Yjs update so the editor converges
|
||||
* (see CollaborationGateway.writePageBody for the full rationale).
|
||||
*
|
||||
* The merge itself stays a block-level reconcile, not a full-body replace
|
||||
* (review #5): only changed blocks are touched, concurrently-edited blocks are
|
||||
* left untouched, and an unchanged resync is a 0-op write. With a `base` (the
|
||||
* last-synced version) it is a THREE-WAY merge so a block ONLY the human
|
||||
* changed is kept and a block ONLY git changed is taken (conflicts -> git);
|
||||
* without a base (e.g. createPage) it falls back to the 2-way merge. The
|
||||
* `{ actor: 'git-sync', user: { id: userId } }` context flows into
|
||||
* PersistenceExtension.onStoreDocument, which persists ydoc+content+textContent,
|
||||
* stamps `lastUpdatedSource = 'git-sync'`, and broadcasts `page.updated`.
|
||||
*/
|
||||
private async writeBody(
|
||||
pageId: string,
|
||||
prosemirrorJson: unknown,
|
||||
userId: string,
|
||||
baseProsemirrorJson?: unknown,
|
||||
): Promise<void> {
|
||||
const documentName = `page.${pageId}`;
|
||||
await this.collabGateway.writePageBody(documentName, {
|
||||
prosemirrorJson,
|
||||
baseProsemirrorJson,
|
||||
userId,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
// Red-team finding #10: single-writer guarantee across replicas must survive a
|
||||
// TTL lapse with a swallowed heartbeat refresh. Two SpaceLockService instances
|
||||
// (A, B) share ONE redis store. A holds 'X' and stays in-flight; the lock key
|
||||
// then disappears (TTL expiry while refreshLock silently failed). B must NOT be
|
||||
// able to acquire 'X' and run its fn concurrently with A — that would be two
|
||||
// writers racing the same working tree. This test asserts the DESIRED
|
||||
// single-writer behavior, so it FAILS today if the lapse lets B in.
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { SpaceLockService } from './space-lock.service';
|
||||
import { GIT_SYNC_LOCK_PREFIX } from '../git-sync.constants';
|
||||
|
||||
/**
|
||||
* Minimal shared fake redis honoring exactly the two primitives the lock uses:
|
||||
* - `SET key val PX ttl NX` → 'OK' only when the key is absent (NX semantics).
|
||||
* - `eval(<get/del CAS>|<get/pexpire CAS>, 1, key, instanceId[, ttl])` →
|
||||
* compares the stored value to ARGV[1] before del/pexpire (CAS).
|
||||
* TTL expiry is not time-driven here; tests simulate it by mutating `store`.
|
||||
*/
|
||||
function makeSharedRedis() {
|
||||
const store = new Map<string, string>();
|
||||
return {
|
||||
store,
|
||||
async set(key: string, val: string, _px: 'PX', _ttl: number, nx: 'NX') {
|
||||
if (nx === 'NX' && store.has(key)) return null;
|
||||
store.set(key, val);
|
||||
return 'OK';
|
||||
},
|
||||
async eval(lua: string, _numKeys: number, key: string, argInstanceId: string) {
|
||||
// Only act when WE still own the key (CAS), mirroring the Lua scripts.
|
||||
if (store.get(key) !== argInstanceId) return 0;
|
||||
if (lua.includes('del')) {
|
||||
store.delete(key);
|
||||
return 1;
|
||||
}
|
||||
// pexpire CAS refresh: value matches, "extend" is a no-op in the fake.
|
||||
return 1;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildInstance(redis: ReturnType<typeof makeSharedRedis>) {
|
||||
const redisService = { getOrThrow: jest.fn(() => redis) };
|
||||
return new SpaceLockService(redisService as any);
|
||||
}
|
||||
|
||||
async function flushMicrotasks(): Promise<void> {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
}
|
||||
|
||||
beforeAll(() => {
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
});
|
||||
|
||||
describe('SpaceLockService — finding #10 single-writer across TTL lapse', () => {
|
||||
it('B must not run its fn concurrently with an in-flight A after the lock key vanishes', async () => {
|
||||
const redis = makeSharedRedis();
|
||||
const A = buildInstance(redis);
|
||||
const B = buildInstance(redis);
|
||||
|
||||
let aRunning = false;
|
||||
let releaseA!: () => void;
|
||||
const gateA = new Promise<void>((resolve) => {
|
||||
releaseA = resolve;
|
||||
});
|
||||
|
||||
// A acquires 'X' and stays in-flight awaiting the gate.
|
||||
const aResult = A.withSpaceLock('X', async () => {
|
||||
aRunning = true;
|
||||
await gateA;
|
||||
aRunning = false;
|
||||
return 'A-done';
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
// Sanity: A is in-flight and owns the redis key.
|
||||
expect(aRunning).toBe(true);
|
||||
expect(redis.store.has(GIT_SYNC_LOCK_PREFIX + 'X')).toBe(true);
|
||||
|
||||
// Simulate TTL lapse with a swallowed heartbeat refresh: the lock key
|
||||
// disappears from the shared store while A is still running.
|
||||
redis.store.delete(GIT_SYNC_LOCK_PREFIX + 'X');
|
||||
|
||||
// Now B tries to take 'X'. Desired: rejected as 'lock-held' (single writer);
|
||||
// and under no circumstance may fn2 run while A is still in flight.
|
||||
let bRanWhileARunning = false;
|
||||
const bResult = await B.withSpaceLock('X', async () => {
|
||||
bRanWhileARunning = aRunning; // captures whether A was still in-flight
|
||||
return 'B-done';
|
||||
});
|
||||
|
||||
// Single-writer assertions: B did NOT execute concurrently with A.
|
||||
expect(bRanWhileARunning).toBe(false);
|
||||
expect(bResult).toEqual({ skipped: 'lock-held' });
|
||||
|
||||
// Cleanup: let A finish.
|
||||
releaseA();
|
||||
await expect(aResult).resolves.toBe('A-done');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,330 @@
|
||||
// Unit tests for SpaceLockService in ISOLATION. The lock is exercised against a
|
||||
// fake redis (mock `set`/`eval`) and we assert the exact ARGUMENTS passed to
|
||||
// redis — the test-coverage gap this refactor (PR #119 #2) closes: acquire uses
|
||||
// `SET ... PX <ttl> NX`, release uses a DEL-CAS Lua, and the heartbeat refresh
|
||||
// uses a PEXPIRE-CAS Lua, all keyed by the same private instanceId.
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { SpaceLockService } from './space-lock.service';
|
||||
import {
|
||||
GIT_SYNC_LOCK_PREFIX,
|
||||
GIT_SYNC_LOCK_TTL_MS,
|
||||
} from '../git-sync.constants';
|
||||
|
||||
type AnyMock = jest.Mock;
|
||||
|
||||
interface Built {
|
||||
service: SpaceLockService;
|
||||
redis: { set: AnyMock; eval: AnyMock };
|
||||
}
|
||||
|
||||
function build(): Built {
|
||||
const redis = {
|
||||
// Default: lock acquired. Tests override per-case.
|
||||
set: jest.fn(async () => 'OK'),
|
||||
eval: jest.fn(async () => 1),
|
||||
};
|
||||
const redisService = { getOrThrow: jest.fn(() => redis) };
|
||||
const service = new SpaceLockService(redisService as any);
|
||||
return { service, redis };
|
||||
}
|
||||
|
||||
/** Drain queued microtasks so awaited continuations inside the lock run. */
|
||||
async function flushMicrotasks(): Promise<void> {
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('SpaceLockService', () => {
|
||||
describe('acquire (SET NX/PX)', () => {
|
||||
it('calls redis.set with (prefix+spaceId, <instanceId>, PX, ttl, NX) and reuses the instanceId on release', async () => {
|
||||
const { service, redis } = build();
|
||||
|
||||
const result = await service.withSpaceLock('space-1', async () => 'ok');
|
||||
expect(result).toBe('ok');
|
||||
|
||||
// acquire arguments
|
||||
expect(redis.set).toHaveBeenCalledTimes(1);
|
||||
const [key, instanceId, px, ttl, nx] = redis.set.mock.calls[0];
|
||||
expect(key).toBe(GIT_SYNC_LOCK_PREFIX + 'space-1');
|
||||
expect(typeof instanceId).toBe('string');
|
||||
expect(instanceId.length).toBeGreaterThan(0);
|
||||
expect(px).toBe('PX');
|
||||
expect(ttl).toBe(GIT_SYNC_LOCK_TTL_MS);
|
||||
expect(nx).toBe('NX');
|
||||
|
||||
// release (eval) reuses the SAME instanceId as ARGV[1]
|
||||
expect(redis.eval).toHaveBeenCalledTimes(1);
|
||||
const [, , relKey, relInstanceId] = redis.eval.mock.calls[0];
|
||||
expect(relKey).toBe(GIT_SYNC_LOCK_PREFIX + 'space-1');
|
||||
expect(relInstanceId).toBe(instanceId);
|
||||
});
|
||||
});
|
||||
|
||||
describe('release (DEL-CAS Lua)', () => {
|
||||
it('returns the fn result and runs a get/del CAS-compared release in finally', async () => {
|
||||
const { service, redis } = build();
|
||||
|
||||
const result = await service.withSpaceLock('space-1', async () => 42);
|
||||
expect(result).toBe(42);
|
||||
|
||||
expect(redis.eval).toHaveBeenCalledTimes(1);
|
||||
const [lua, numKeys, key, instanceId] = redis.eval.mock.calls[0];
|
||||
expect(lua).toContain('get');
|
||||
expect(lua).toContain('del');
|
||||
expect(lua).toContain('== ARGV[1]');
|
||||
expect(numKeys).toBe(1);
|
||||
expect(key).toBe(GIT_SYNC_LOCK_PREFIX + 'space-1');
|
||||
expect(typeof instanceId).toBe('string');
|
||||
});
|
||||
});
|
||||
|
||||
describe('lock held by another replica', () => {
|
||||
it("returns { skipped: 'lock-held' } without running fn or releasing when set != 'OK'", async () => {
|
||||
const { service, redis } = build();
|
||||
redis.set.mockResolvedValueOnce(null);
|
||||
const fn = jest.fn(async () => 'ran');
|
||||
|
||||
const result = await service.withSpaceLock('space-1', fn);
|
||||
|
||||
expect(result).toEqual({ skipped: 'lock-held' });
|
||||
expect(fn).not.toHaveBeenCalled();
|
||||
// No release: we never acquired it.
|
||||
expect(redis.eval).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('in-process mutex', () => {
|
||||
it("a second withSpaceLock on the same space mid-flight returns { skipped: 'in-progress' } without a second set", async () => {
|
||||
const { service, redis } = build();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
|
||||
const first = service.withSpaceLock('space-1', async () => {
|
||||
await gate;
|
||||
return 'first';
|
||||
});
|
||||
// Let the first call acquire + enter the running set.
|
||||
await flushMicrotasks();
|
||||
|
||||
const second = await service.withSpaceLock('space-1', async () => 'second');
|
||||
expect(second).toEqual({ skipped: 'in-progress' });
|
||||
// Only the first call hit redis.set — the mutex short-circuits the second.
|
||||
expect(redis.set).toHaveBeenCalledTimes(1);
|
||||
|
||||
release();
|
||||
await expect(first).resolves.toBe('first');
|
||||
});
|
||||
});
|
||||
|
||||
// Bug #1 (push 503 starvation): the PUSH path passes a bounded acquireRetry so a
|
||||
// transient overlap with a poll cycle is retried (and succeeds) instead of an
|
||||
// immediate 503. A genuinely stuck lock still skips after the bound. The poll
|
||||
// cycle passes NO retry (immediate skip), so only the push path waits.
|
||||
describe('bounded acquire-retry (push path)', () => {
|
||||
const retry = { timeoutMs: 5_000, baseMs: 100, maxMs: 500 };
|
||||
|
||||
it('retries the acquire and SUCCEEDS when the lock is briefly held then released', async () => {
|
||||
const { service, redis } = build();
|
||||
// First acquire attempt fails (lock briefly held by a cycle), the next
|
||||
// succeeds — the bounded retry must turn this into a SUCCESS, not a skip.
|
||||
redis.set
|
||||
.mockResolvedValueOnce(null) // attempt 1: held
|
||||
.mockResolvedValueOnce(null) // attempt 2: still held
|
||||
.mockResolvedValue('OK'); // attempt 3+: released -> acquired
|
||||
const fn = jest.fn(async () => 'pushed');
|
||||
|
||||
const result = await service.withSpaceLock('space-1', fn, {
|
||||
acquireRetry: retry,
|
||||
});
|
||||
|
||||
expect(result).toBe('pushed');
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
expect(redis.set.mock.calls.length).toBeGreaterThanOrEqual(3);
|
||||
// The acquired lock is released in finally (DEL-CAS eval).
|
||||
expect(redis.eval).toHaveBeenCalledTimes(1);
|
||||
expect(redis.eval.mock.calls[0][0]).toContain('del');
|
||||
});
|
||||
|
||||
it('still skips (lock-held) after the bound when the lock stays stuck — and never runs fn', async () => {
|
||||
const { service, redis } = build();
|
||||
redis.set.mockResolvedValue(null); // permanently held
|
||||
const fn = jest.fn(async () => 'pushed');
|
||||
|
||||
const result = await service.withSpaceLock('space-1', fn, {
|
||||
acquireRetry: { timeoutMs: 300, baseMs: 50, maxMs: 100 },
|
||||
});
|
||||
|
||||
expect(result).toEqual({ skipped: 'lock-held' });
|
||||
expect(fn).not.toHaveBeenCalled();
|
||||
// It retried more than once before giving up (bound > one interval).
|
||||
expect(redis.set.mock.calls.length).toBeGreaterThan(1);
|
||||
// Never acquired -> never released.
|
||||
expect(redis.eval).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('without acquireRetry (poll path) a held lock skips IMMEDIATELY (single attempt)', async () => {
|
||||
const { service, redis } = build();
|
||||
redis.set.mockResolvedValue(null);
|
||||
const fn = jest.fn(async () => 'cycle');
|
||||
|
||||
const result = await service.withSpaceLock('space-1', fn);
|
||||
|
||||
expect(result).toEqual({ skipped: 'lock-held' });
|
||||
expect(redis.set).toHaveBeenCalledTimes(1); // no retry
|
||||
expect(fn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('fn throwing', () => {
|
||||
it('propagates the throw AND still releases (eval) in finally', async () => {
|
||||
const { service, redis } = build();
|
||||
const boom = new Error('boom');
|
||||
|
||||
await expect(
|
||||
service.withSpaceLock('space-1', async () => {
|
||||
throw boom;
|
||||
}),
|
||||
).rejects.toBe(boom);
|
||||
|
||||
// Release still ran despite the throw.
|
||||
expect(redis.eval).toHaveBeenCalledTimes(1);
|
||||
const [lua] = redis.eval.mock.calls[0];
|
||||
expect(lua).toContain('del');
|
||||
});
|
||||
});
|
||||
|
||||
describe('heartbeat refresh (PEXPIRE-CAS Lua)', () => {
|
||||
it('extends the lock via a pexpire CAS-Lua with the same instanceId while fn is in flight', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { service, redis } = build();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
|
||||
const run = service.withSpaceLock('space-1', async () => {
|
||||
await gate;
|
||||
return 'done';
|
||||
});
|
||||
// Let acquire resolve and the running.add + setInterval registration run.
|
||||
await flushMicrotasks();
|
||||
|
||||
// Capture the instanceId used on acquire so we can assert it is reused.
|
||||
const instanceId = redis.set.mock.calls[0][1];
|
||||
|
||||
// Advance past one heartbeat interval (≈ TTL/3) to fire refreshLock.
|
||||
jest.advanceTimersByTime(Math.floor(GIT_SYNC_LOCK_TTL_MS / 3));
|
||||
await flushMicrotasks();
|
||||
|
||||
// The refresh eval ran (release has not, fn still awaiting the gate).
|
||||
expect(redis.eval).toHaveBeenCalledTimes(1);
|
||||
const [lua, numKeys, key, argInstanceId, ttlArg] =
|
||||
redis.eval.mock.calls[0];
|
||||
expect(lua).toContain('pexpire');
|
||||
expect(lua).toContain('== ARGV[1]');
|
||||
expect(numKeys).toBe(1);
|
||||
expect(key).toBe(GIT_SYNC_LOCK_PREFIX + 'space-1');
|
||||
expect(argInstanceId).toBe(instanceId);
|
||||
expect(ttlArg).toBe(String(GIT_SYNC_LOCK_TTL_MS));
|
||||
|
||||
// Let fn finish; release runs in finally (second eval, the DEL-CAS).
|
||||
release();
|
||||
await flushMicrotasks();
|
||||
await expect(run).resolves.toBe('done');
|
||||
expect(redis.eval).toHaveBeenCalledTimes(2);
|
||||
expect(redis.eval.mock.calls[1][0]).toContain('del');
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// The lost-lock guard: a heartbeat refresh that cannot CONFIRM we still own the
|
||||
// lock (CAS miss, res !== 1) OR that throws (Redis error) aborts the supplied
|
||||
// controller so the in-flight protected fn stops instead of writing blind after
|
||||
// a possible lock takeover. `withSpaceLock` threads that signal into `fn`.
|
||||
describe('abort-on-lost-lock', () => {
|
||||
it('aborts the in-flight fn when the heartbeat refresh CAS-MISSES (eval -> 0)', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { service, redis } = build();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
let captured: AbortSignal | undefined;
|
||||
|
||||
const run = service.withSpaceLock('space-1', async (signal) => {
|
||||
captured = signal;
|
||||
await gate;
|
||||
return 'done';
|
||||
});
|
||||
// Let acquire resolve and the setInterval register.
|
||||
await flushMicrotasks();
|
||||
expect(captured).toBeDefined();
|
||||
expect(captured!.aborted).toBe(false);
|
||||
|
||||
// The refresh CAS-misses: the key no longer holds our instanceId.
|
||||
redis.eval.mockResolvedValue(0);
|
||||
jest.advanceTimersByTime(Math.floor(GIT_SYNC_LOCK_TTL_MS / 3));
|
||||
await flushMicrotasks();
|
||||
|
||||
// The lost lock aborted the protected fn's signal.
|
||||
expect(captured!.aborted).toBe(true);
|
||||
|
||||
release();
|
||||
await flushMicrotasks();
|
||||
await expect(run).resolves.toBe('done');
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it('aborts the in-flight fn when the heartbeat refresh THROWS (Redis error)', async () => {
|
||||
jest.useFakeTimers();
|
||||
try {
|
||||
const { service, redis } = build();
|
||||
let release!: () => void;
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
let captured: AbortSignal | undefined;
|
||||
|
||||
const run = service.withSpaceLock('space-1', async (signal) => {
|
||||
captured = signal;
|
||||
await gate;
|
||||
return 'done';
|
||||
});
|
||||
await flushMicrotasks();
|
||||
expect(captured!.aborted).toBe(false);
|
||||
|
||||
// The refresh eval rejects (Redis down). release() in finally must still
|
||||
// resolve, so only reject the NEXT (heartbeat) call, then go back to OK.
|
||||
redis.eval.mockRejectedValueOnce(new Error('redis down'));
|
||||
jest.advanceTimersByTime(Math.floor(GIT_SYNC_LOCK_TTL_MS / 3));
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(captured!.aborted).toBe(true);
|
||||
|
||||
release();
|
||||
await flushMicrotasks();
|
||||
await expect(run).resolves.toBe('done');
|
||||
} finally {
|
||||
jest.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// Silence the warn logger if a refresh/release path ever logs (defensive).
|
||||
beforeAll(() => {
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
});
|
||||
@@ -0,0 +1,251 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { RedisService } from '@nestjs-labs/nestjs-ioredis';
|
||||
import type { Redis } from 'ioredis';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import {
|
||||
GIT_SYNC_LOCK_PREFIX,
|
||||
GIT_SYNC_LOCK_TTL_MS,
|
||||
} from '../git-sync.constants';
|
||||
|
||||
/**
|
||||
* The per-space lock used by the git-sync control plane: an in-process per-space
|
||||
* mutex (no overlapping cycles on one instance) PLUS a Redis leader lock
|
||||
* (single writer across replicas). Extracted from `GitSyncOrchestrator` so the
|
||||
* locking primitive is a single reusable, independently testable unit
|
||||
* (PR #119 refactor #2).
|
||||
*/
|
||||
@Injectable()
|
||||
export class SpaceLockService {
|
||||
private readonly logger = new Logger(SpaceLockService.name);
|
||||
private readonly redis: Redis;
|
||||
/** Unique per process instance — the leader-lock value (CAS on release). */
|
||||
private readonly instanceId = randomUUID();
|
||||
/** In-process per-space mutex: spaceIds with a cycle currently running. */
|
||||
private readonly running = new Set<string>();
|
||||
/**
|
||||
* Process-wide single-writer guard: spaceId -> instanceId of the live holder.
|
||||
* Unlike `running` (scoped to ONE service instance), this is shared by every
|
||||
* SpaceLockService in the process, so even if the Redis lock key lapses
|
||||
* (swallowed heartbeat / TTL expiry) a SECOND holder in the same process
|
||||
* cannot start a concurrent cycle for the same space — it is rejected
|
||||
* 'lock-held'. The cross-PROCESS race is handled by the Redis lock plus
|
||||
* abort-on-refresh-failure (and, as a follow-up, fencing tokens).
|
||||
*/
|
||||
private static readonly liveLocks = new Map<string, string>();
|
||||
|
||||
constructor(redisService: RedisService) {
|
||||
this.redis = redisService.getOrThrow();
|
||||
}
|
||||
|
||||
// --- Redis leader lock -----------------------------------------
|
||||
|
||||
/**
|
||||
* Acquire per-space leadership: `SET <key> <instanceId> PX <ttl> NX` returns
|
||||
* 'OK' only when the key did not exist. Any other reply means another replica
|
||||
* holds it.
|
||||
*/
|
||||
private async acquire(spaceId: string): Promise<boolean> {
|
||||
const ok = await this.redis.set(
|
||||
GIT_SYNC_LOCK_PREFIX + spaceId,
|
||||
this.instanceId,
|
||||
'PX',
|
||||
GIT_SYNC_LOCK_TTL_MS,
|
||||
'NX',
|
||||
);
|
||||
return ok === 'OK';
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the lock with a CAS Lua so we only delete it when WE still hold it
|
||||
* (the value matches our instanceId) — never another replica's lock that took
|
||||
* over after our TTL expired.
|
||||
*/
|
||||
private async release(spaceId: string): Promise<void> {
|
||||
const lua =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("del", KEYS[1]) else return 0 end';
|
||||
try {
|
||||
await this.redis.eval(lua, 1, GIT_SYNC_LOCK_PREFIX + spaceId, this.instanceId);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`git-sync: failed to release lock for space ${spaceId}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* CAS-guarded TTL refresh: extend the lock's TTL ONLY while WE still own it
|
||||
* (the stored value matches our instanceId) — never extend another replica's
|
||||
* lock that took over after our TTL expired. Used by the heartbeat in
|
||||
* `withSpaceLock` so a long-running push (client-controlled receive-pack + the
|
||||
* Docmost cycle) cannot outlive the lock and let a concurrent cycle race the
|
||||
* working tree. Never throws (a thrown timer callback would crash the process),
|
||||
* but a refresh it cannot CONFIRM is treated as a LOST lock: it aborts the
|
||||
* supplied controller so the in-flight protected fn stops instead of writing
|
||||
* blind while another replica may already have taken over the lock.
|
||||
*/
|
||||
private async refreshLock(
|
||||
spaceId: string,
|
||||
controller?: AbortController,
|
||||
): Promise<void> {
|
||||
const lua =
|
||||
'if redis.call("get", KEYS[1]) == ARGV[1] then return redis.call("pexpire", KEYS[1], ARGV[2]) else return 0 end';
|
||||
try {
|
||||
const res = await this.redis.eval(
|
||||
lua,
|
||||
1,
|
||||
GIT_SYNC_LOCK_PREFIX + spaceId,
|
||||
this.instanceId,
|
||||
String(GIT_SYNC_LOCK_TTL_MS),
|
||||
);
|
||||
// CAS miss (res !== 1): we no longer own the key — our TTL lapsed and
|
||||
// another replica may hold it now. Abort the in-flight cycle rather than
|
||||
// swallowing the loss and racing the working tree.
|
||||
if (res !== 1) {
|
||||
this.logger.warn(
|
||||
`git-sync: lock for space ${spaceId} lost during refresh — aborting in-flight cycle`,
|
||||
);
|
||||
controller?.abort();
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`git-sync: failed to refresh lock for space ${spaceId}: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
// A refresh we cannot confirm means we may no longer hold the lock; abort.
|
||||
controller?.abort();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Options for `withSpaceLock`. `acquireRetry` (PUSH path only) bounds a
|
||||
* retry-acquire loop: if the lock cannot be entered on the first try, keep
|
||||
* retrying with a capped exponential backoff until `timeoutMs` elapses before
|
||||
* returning the skip sentinel. The poll cycle holds the lock while it
|
||||
* processes a whole space, so a legitimate external push that briefly overlaps
|
||||
* a cycle should WAIT a moment rather than immediately 503 (bug: ~60% of
|
||||
* pushes 503'd under continuous polling). The poll cycle passes NO retry (it
|
||||
* just skips and the next tick reconciles).
|
||||
*/
|
||||
async withSpaceLock<T>(
|
||||
spaceId: string,
|
||||
fn: (signal: AbortSignal) => Promise<T>,
|
||||
options?: {
|
||||
acquireRetry?: { timeoutMs: number; baseMs: number; maxMs: number };
|
||||
},
|
||||
): Promise<T | { skipped: 'lock-held' | 'in-progress' }> {
|
||||
const retry = options?.acquireRetry;
|
||||
// Deadline for the bounded retry-acquire (push path). `Date.now()` once so a
|
||||
// slow first attempt does not over-extend the budget.
|
||||
const deadline = retry ? Date.now() + retry.timeoutMs : 0;
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
// Reserve the in-process slot synchronously (before any await) so two
|
||||
// concurrent same-space calls on THIS instance cannot both pass the guard
|
||||
// and race acquire(). On any failure this is released before we retry/skip.
|
||||
const reservation = this.tryReserveInProcess(spaceId);
|
||||
if (reservation) {
|
||||
// Could not even reserve in-process (this instance mid-cycle, or another
|
||||
// live holder in the process). Retry within the bound, else skip.
|
||||
if (retry && Date.now() < deadline) {
|
||||
await this.sleep(this.nextBackoff(attempt++, retry, deadline));
|
||||
continue;
|
||||
}
|
||||
return reservation;
|
||||
}
|
||||
// Reserved in-process — now contend for the Redis leader lock. Release the
|
||||
// in-process slot on EVERY non-running path so a retry/skip leaves no leak.
|
||||
let acquired = false;
|
||||
try {
|
||||
acquired = await this.acquire(spaceId);
|
||||
} finally {
|
||||
if (!acquired) this.releaseInProcess(spaceId);
|
||||
}
|
||||
if (!acquired) {
|
||||
if (retry && Date.now() < deadline) {
|
||||
await this.sleep(this.nextBackoff(attempt++, retry, deadline));
|
||||
continue;
|
||||
}
|
||||
return { skipped: 'lock-held' };
|
||||
}
|
||||
// Both locks held — run `fn` under the heartbeat, releasing in `finally`.
|
||||
// Lost-lock signal: a failed/CAS-missed heartbeat refresh aborts this so the
|
||||
// protected fn can stop instead of writing blind after our lock lapsed.
|
||||
const controller = new AbortController();
|
||||
// Heartbeat: periodically (≈ TTL/3) extend the lock's TTL while `fn` runs so
|
||||
// a long push (client-controlled receive-pack + the Docmost cycle) cannot
|
||||
// outlive the fixed TTL and let a concurrent cycle race the working tree. The
|
||||
// refresh is CAS-guarded (only extends while WE own it). `.unref()` keeps the
|
||||
// timer from holding the event loop open; it is ALWAYS cleared in `finally`.
|
||||
const heartbeat = setInterval(() => {
|
||||
void this.refreshLock(spaceId, controller);
|
||||
}, Math.max(1, Math.floor(GIT_SYNC_LOCK_TTL_MS / 3)));
|
||||
heartbeat.unref?.();
|
||||
try {
|
||||
return await fn(controller.signal);
|
||||
} finally {
|
||||
clearInterval(heartbeat);
|
||||
await this.release(spaceId);
|
||||
this.releaseInProcess(spaceId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Synchronously try to reserve the in-process single-writer slot for a space.
|
||||
* Returns a skip sentinel when another holder is live (this instance mid-cycle
|
||||
* -> 'in-progress'; another SpaceLockService in this process -> 'lock-held'),
|
||||
* or `null` when the slot was reserved (caller MUST `releaseInProcess` later).
|
||||
* Both checks + the reservation happen with NO await between them so two
|
||||
* concurrent same-space calls cannot both pass.
|
||||
*/
|
||||
private tryReserveInProcess(
|
||||
spaceId: string,
|
||||
): { skipped: 'lock-held' | 'in-progress' } | null {
|
||||
if (this.running.has(spaceId)) {
|
||||
return { skipped: 'in-progress' };
|
||||
}
|
||||
// Cross-instance, same-process single-writer guard: another live holder (a
|
||||
// different SpaceLockService in this process) is mid-cycle for this space.
|
||||
// This survives a swallowed heartbeat / Redis TTL lapse, so a second writer
|
||||
// in the process cannot race the working tree — it is rejected 'lock-held'.
|
||||
if (SpaceLockService.liveLocks.has(spaceId)) {
|
||||
return { skipped: 'lock-held' };
|
||||
}
|
||||
this.running.add(spaceId);
|
||||
SpaceLockService.liveLocks.set(spaceId, this.instanceId);
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Release the in-process single-writer slot reserved by tryReserveInProcess. */
|
||||
private releaseInProcess(spaceId: string): void {
|
||||
this.running.delete(spaceId);
|
||||
SpaceLockService.liveLocks.delete(spaceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Backoff (ms) before the next push lock-acquire attempt: capped exponential
|
||||
* (`baseMs * 2^attempt`, ceilinged at `maxMs`) clamped so it never overshoots
|
||||
* the retry `deadline`. Deterministic (no jitter) so the bound is testable.
|
||||
*/
|
||||
private nextBackoff(
|
||||
attempt: number,
|
||||
retry: { baseMs: number; maxMs: number },
|
||||
deadline: number,
|
||||
): number {
|
||||
const exp = retry.baseMs * 2 ** attempt;
|
||||
const capped = Math.min(exp, retry.maxMs);
|
||||
const remaining = Math.max(0, deadline - Date.now());
|
||||
return Math.max(0, Math.min(capped, remaining));
|
||||
}
|
||||
|
||||
/** Promise-based delay (extracted so tests can reason about the retry loop). */
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const t = setTimeout(resolve, ms);
|
||||
t.unref?.();
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
// Unit tests for the per-space vault path resolver + lazy VaultGit cache
|
||||
// `mkdir` and the git-sync loader are mocked so construction is cheap and
|
||||
// no real filesystem / git work happens. We assert the path normalization
|
||||
// (trailing slash) and the one-VaultGit-per-space caching contract.
|
||||
//
|
||||
// The service loads `VaultGit` (and `vaultGitEnv`) at runtime via the
|
||||
// `loadGitSync()` bridge (the ESM `@docmost/git-sync` package cannot be
|
||||
// `require()`d under jest), so we mock that loader rather than the package.
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { loadGitSync } from '../git-sync.loader';
|
||||
|
||||
jest.mock('node:fs/promises', () => ({
|
||||
mkdir: jest.fn(async () => undefined),
|
||||
}));
|
||||
|
||||
// ensureServable shells out via `promisify(execFile)`; mock execFile with a
|
||||
// callback-style fn so promisify resolves. Each `git config <key> <value>` call
|
||||
// is recorded so the config writes (incl. the security-critical
|
||||
// receive.denyNonFastForwards=true and core.symlinks=false) can be asserted.
|
||||
jest.mock('node:child_process', () => ({
|
||||
execFile: jest.fn((_cmd: string, _args: string[], _opts: any, cb: any) =>
|
||||
cb(null, { stdout: '', stderr: '' }),
|
||||
),
|
||||
}));
|
||||
|
||||
// Cheap VaultGit stub: records the path it was constructed with; no shell-out.
|
||||
// `ensureRepo` is a resolved jest.fn so ensureServable can call it. Declared with
|
||||
// a `mock`-prefixed name so jest allows referencing it inside the hoisted
|
||||
// `jest.mock` factory below.
|
||||
const mockVaultGit = jest
|
||||
.fn()
|
||||
.mockImplementation((path: string) => ({
|
||||
path,
|
||||
ensureRepo: jest.fn().mockResolvedValue(undefined),
|
||||
}));
|
||||
|
||||
jest.mock('../git-sync.loader', () => ({
|
||||
loadGitSync: jest.fn(async () => ({
|
||||
VaultGit: mockVaultGit,
|
||||
vaultGitEnv: jest.fn(() => ({})),
|
||||
})),
|
||||
}));
|
||||
|
||||
import { VaultRegistryService } from './vault-registry.service';
|
||||
|
||||
type AnyMock = jest.Mock;
|
||||
|
||||
const mkdirMock = mkdir as unknown as AnyMock;
|
||||
const execFileMock = execFile as unknown as AnyMock;
|
||||
const VaultGitMock = mockVaultGit;
|
||||
void loadGitSync;
|
||||
|
||||
function build(dataDir: string): { service: VaultRegistryService } {
|
||||
const env = {
|
||||
getGitSyncDataDir: jest.fn(() => dataDir),
|
||||
getGitSyncBackendTimeoutMs: jest.fn(() => 120000),
|
||||
};
|
||||
const service = new VaultRegistryService(env as any);
|
||||
return { service };
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('VaultRegistryService', () => {
|
||||
describe('vaultPath', () => {
|
||||
it('normalizes a trailing slash in the data dir (no double slash)', () => {
|
||||
const { service } = build('/vaults/');
|
||||
expect(service.vaultPath('space-1')).toBe('/vaults/space-1');
|
||||
});
|
||||
|
||||
it('works without a trailing slash too', () => {
|
||||
const { service } = build('/vaults');
|
||||
expect(service.vaultPath('space-1')).toBe('/vaults/space-1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('getVault lazy cache', () => {
|
||||
it('returns the SAME instance on a second call (one VaultGit per space)', async () => {
|
||||
const { service } = build('/vaults');
|
||||
|
||||
const first = await service.getVault('space-1');
|
||||
const second = await service.getVault('space-1');
|
||||
|
||||
// Same cached instance, constructed exactly once.
|
||||
expect(second).toBe(first);
|
||||
expect(VaultGitMock).toHaveBeenCalledTimes(1);
|
||||
expect(VaultGitMock).toHaveBeenCalledWith('/vaults/space-1');
|
||||
// mkdir is only run on the first (cache-miss) construction.
|
||||
expect(mkdirMock).toHaveBeenCalledTimes(1);
|
||||
expect(mkdirMock).toHaveBeenCalledWith('/vaults/space-1', {
|
||||
recursive: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('ensureServable', () => {
|
||||
it('ensures the repo then writes the force-push-protection + symlink-guard git configs', async () => {
|
||||
const { service } = build('/vaults');
|
||||
|
||||
const path = await service.ensureServable('space-1');
|
||||
expect(path).toBe('/vaults/space-1');
|
||||
|
||||
// ensureRepo ran first on the cached vault.
|
||||
const vault = await service.getVault('space-1');
|
||||
expect((vault as any).ensureRepo).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Collect every `git config <key> <value>` write.
|
||||
const configWrites = execFileMock.mock.calls
|
||||
.filter(([cmd, args]) => cmd === 'git' && args[0] === 'config')
|
||||
.map(([, args]) => [args[1], args[2]]);
|
||||
|
||||
expect(configWrites).toEqual([
|
||||
['receive.denyCurrentBranch', 'updateInstead'],
|
||||
// Security-critical: blocks force-push / history rewrites on main.
|
||||
['receive.denyNonFastForwards', 'true'],
|
||||
['http.receivepack', 'true'],
|
||||
['http.uploadpack', 'true'],
|
||||
// Security-critical (PR #119 review): a pushed symlink is checked out as
|
||||
// a plain file, never a real link, so it cannot be followed to leak/
|
||||
// overwrite a file outside the vault.
|
||||
['core.symlinks', 'false'],
|
||||
]);
|
||||
|
||||
// Every config write targets THIS vault's cwd and is time-bounded so a
|
||||
// wedged git cannot hang the request path.
|
||||
for (const [cmd, args, opts] of execFileMock.mock.calls) {
|
||||
if (cmd === 'git' && args[0] === 'config') {
|
||||
expect(opts.cwd).toBe('/vaults/space-1');
|
||||
expect(opts.timeout).toBe(120000);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects (and writes no git config) when ensureRepo rejects', async () => {
|
||||
const { service } = build('/vaults');
|
||||
const vault = await service.getVault('space-1');
|
||||
(vault as any).ensureRepo.mockRejectedValueOnce(new Error('init failed'));
|
||||
|
||||
await expect(service.ensureServable('space-1')).rejects.toThrow(
|
||||
'init failed',
|
||||
);
|
||||
|
||||
const configWrites = execFileMock.mock.calls.filter(
|
||||
([cmd, args]) => cmd === 'git' && args[0] === 'config',
|
||||
);
|
||||
expect(configWrites).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { mkdir } from 'node:fs/promises';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import type { VaultGit } from '@docmost/git-sync';
|
||||
import { loadGitSync } from '../git-sync.loader';
|
||||
import { EnvironmentService } from '../../environment/environment.service';
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/**
|
||||
* Resolves the on-disk vault location per space and owns the (lazily created,
|
||||
* cached) `VaultGit` instance for each one.
|
||||
*
|
||||
* Topology: one git repo per enabled space, rooted at
|
||||
* `<GIT_SYNC_DATA_DIR>/<spaceId>`. A `VaultGit` is constructed at most once per
|
||||
* space and reused across cycles — it is a thin, stateless shell-out wrapper, so
|
||||
* caching it just avoids re-resolving the path and re-running `mkdir`.
|
||||
*/
|
||||
@Injectable()
|
||||
export class VaultRegistryService {
|
||||
private readonly logger = new Logger(VaultRegistryService.name);
|
||||
private readonly vaults = new Map<string, VaultGit>();
|
||||
|
||||
constructor(private readonly environmentService: EnvironmentService) {}
|
||||
|
||||
/** Absolute vault path for a space: `<GIT_SYNC_DATA_DIR>/<spaceId>`. */
|
||||
vaultPath(spaceId: string): string {
|
||||
const root = this.environmentService.getGitSyncDataDir().replace(/\/+$/, '');
|
||||
return `${root}/${spaceId}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get (or lazily construct + cache) the `VaultGit` for a space, ensuring its
|
||||
* directory exists. `VaultGit.ensureRepo()` is NOT called here — the engine's
|
||||
* pull/push paths call it (and the branch/ref setup) as their first step; this
|
||||
* only guarantees the parent dir exists so a fresh space does not ENOENT.
|
||||
*/
|
||||
async getVault(spaceId: string): Promise<VaultGit> {
|
||||
const cached = this.vaults.get(spaceId);
|
||||
if (cached) return cached;
|
||||
|
||||
const path = this.vaultPath(spaceId);
|
||||
await mkdir(path, { recursive: true });
|
||||
const { VaultGit } = await loadGitSync();
|
||||
const vault = new VaultGit(path);
|
||||
this.vaults.set(spaceId, vault);
|
||||
return vault;
|
||||
}
|
||||
|
||||
/**
|
||||
* Make a space's vault repo servable over smart-HTTP (the /git host). Ensures
|
||||
* the repo exists (engine `ensureRepo`: `git init -b main` + initial commit +
|
||||
* branches; idempotent), then sets the LOCAL git config a `git http-backend`
|
||||
* push needs:
|
||||
*
|
||||
* - receive.denyCurrentBranch=updateInstead — a push to the checked-out
|
||||
* `main` updates the working tree too (the engine's human-facing branch).
|
||||
* Requires a clean tree, which is guaranteed between cycles / under the
|
||||
* orchestrator lock that wraps an external push.
|
||||
* - receive.denyNonFastForwards=true — block force-push so a client cannot
|
||||
* rewrite the engine's history on `main`.
|
||||
* - http.receivepack=true / http.uploadpack=true — explicitly allow the
|
||||
* receive/upload services over HTTP.
|
||||
* - core.symlinks=false — SECURITY (PR #119 review). A writer could push a
|
||||
* `.md` entry that is a SYMLINK (e.g. `leak.md -> /etc/passwd` or
|
||||
* `-> .env`); with symlinks enabled `updateInstead` would materialize a
|
||||
* real link in the working tree, and the next push cycle would follow it
|
||||
* and PUBLISH the target's contents as a Docmost page (server-file
|
||||
* disclosure), or use a symlinked directory to write OUTSIDE the vault on
|
||||
* pull. With `core.symlinks=false` git checks out such a blob as a PLAIN
|
||||
* FILE containing the link text, never a real link, defusing the primitive
|
||||
* at the git layer. (The engine's per-access lstat/realpath guard is the
|
||||
* second layer — see path-guard.ts.)
|
||||
*
|
||||
* All are set idempotently (plain `git config` overwrites the local value).
|
||||
* Returns the absolute vault path. Idempotent and safe to call before every
|
||||
* request.
|
||||
*/
|
||||
async ensureServable(spaceId: string): Promise<string> {
|
||||
const { vaultGitEnv } = await loadGitSync();
|
||||
const vault = await this.getVault(spaceId);
|
||||
const path = this.vaultPath(spaceId);
|
||||
|
||||
// ensureRepo also verifies git is available on its first git call; it does
|
||||
// `git init -b main` + an initial commit + the engine branches. Idempotent.
|
||||
await vault.ensureRepo();
|
||||
|
||||
const configs: Array<[string, string]> = [
|
||||
['receive.denyCurrentBranch', 'updateInstead'],
|
||||
['receive.denyNonFastForwards', 'true'],
|
||||
['http.receivepack', 'true'],
|
||||
['http.uploadpack', 'true'],
|
||||
['core.symlinks', 'false'],
|
||||
];
|
||||
// Bound each `git config` (review suggestion): this runs in the request path
|
||||
// BEFORE the watchdog, so a wedged git (a stale `.git/config.lock`) would
|
||||
// otherwise hang the request indefinitely. Mirror the engine's GIT_EXEC
|
||||
// bound via the configured backend timeout.
|
||||
const timeout = this.environmentService.getGitSyncBackendTimeoutMs();
|
||||
for (const [key, value] of configs) {
|
||||
await execFileAsync('git', ['config', key, value], {
|
||||
cwd: path,
|
||||
// Use the engine's cwd-isolated env (strips GIT_DIR / GIT_WORK_TREE) so
|
||||
// the config is written to THIS vault's local config, nothing else.
|
||||
env: vaultGitEnv(),
|
||||
timeout,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
} from '@nestjs/common';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { esmImport } from '../../common/helpers/esm-import';
|
||||
import { IncomingMessage } from 'node:http';
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
|
||||
@@ -63,14 +64,9 @@ const MCP_RESOLVED = Symbol('mcpResolvedConfig');
|
||||
// (never the token value) so operators can migrate without log spam.
|
||||
let warnedLegacyMcpAuth = false;
|
||||
|
||||
// TS with module:commonjs downlevels a literal import() to require(), which
|
||||
// cannot load the ESM-only @docmost/mcp package. Indirect through Function so
|
||||
// the real dynamic import() survives compilation and can load ESM from
|
||||
// CommonJS at runtime.
|
||||
const esmImport = new Function(
|
||||
'specifier',
|
||||
'return import(specifier)',
|
||||
) as (specifier: string) => Promise<unknown>;
|
||||
// The CJS->ESM dynamic-import bridge lives in one shared helper
|
||||
// (common/helpers/esm-import.ts); see it for why import() must be hidden from the
|
||||
// TS commonjs downleveler.
|
||||
|
||||
@Injectable()
|
||||
export class McpService implements OnModuleDestroy {
|
||||
|
||||
@@ -19,6 +19,7 @@ import { resolveTrustProxy } from './integrations/environment/trust-proxy.util';
|
||||
import { isMetricsEnabled } from './integrations/metrics/metrics.registry';
|
||||
import { recordHttpResponse } from './integrations/metrics/http-metrics.hook';
|
||||
import { startMetricsServer } from './integrations/metrics/metrics.server';
|
||||
import { GitHttpService } from './integrations/git-sync/http/git-http.service';
|
||||
|
||||
async function bootstrap() {
|
||||
const app = await NestFactory.create<NestFastifyApplication>(
|
||||
@@ -123,6 +124,23 @@ async function bootstrap() {
|
||||
},
|
||||
);
|
||||
|
||||
// git smart-HTTP POST bodies use these media types. Register PASSTHROUGH
|
||||
// content-type parsers so Fastify does NOT buffer/parse them (it would
|
||||
// otherwise reject the unknown type with 415); the /git handler streams the
|
||||
// raw Node request (request.raw) to `git http-backend` stdin instead. A
|
||||
// passthrough parser also bypasses the bodyLimit, so large pushes are not
|
||||
// truncated (the bytes are never buffered by Fastify).
|
||||
app
|
||||
.getHttpAdapter()
|
||||
.getInstance()
|
||||
.addContentTypeParser(
|
||||
[
|
||||
'application/x-git-upload-pack-request',
|
||||
'application/x-git-receive-pack-request',
|
||||
],
|
||||
(_req, payload, done) => done(null, payload),
|
||||
);
|
||||
|
||||
app
|
||||
.getHttpAdapter()
|
||||
.getInstance()
|
||||
@@ -177,6 +195,25 @@ async function bootstrap() {
|
||||
app.useGlobalInterceptors(new TransformHttpResponseInterceptor(reflector));
|
||||
app.enableShutdownHooks();
|
||||
|
||||
// git smart-HTTP host (the /git/<spaceId>.git/... subtree). Registered as a
|
||||
// RAW Fastify route — NOT a Nest controller under the global '/api' prefix —
|
||||
// so it lives at the ROOT and a single wildcard reliably captures the whole
|
||||
// multi-segment subtree (avoiding the path-to-regexp v8 wildcard / global-
|
||||
// prefix-exclude ambiguity in NestJS v11). The handler is resolved from the
|
||||
// Nest container so all auth/authz/gating still runs. NOTE: Nest middleware
|
||||
// (DomainMiddleware) does NOT run for this raw root route — it is bound to the
|
||||
// Nest router under the global '/api' prefix — so request.raw.workspaceId is
|
||||
// NOT populated here; GitHttpService resolves the workspace itself (mirroring
|
||||
// DomainMiddleware). The Fastify wildcard '/git/*' captures the multi-segment
|
||||
// subpath; the handler re-parses req.url itself.
|
||||
const gitHttpService = app.get(GitHttpService);
|
||||
app
|
||||
.getHttpAdapter()
|
||||
.getInstance()
|
||||
.all('/git/*', async (request, reply) => {
|
||||
await gitHttpService.handle(request as any, reply as any);
|
||||
});
|
||||
|
||||
const logger = new Logger('NestApplication');
|
||||
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
|
||||
Reference in New Issue
Block a user