feat(sync): add git vault layer (§5) and the Docmost->vault pull cycle (§6)

Turn the read-only mirror into a git-backed pull cycle. Read-only toward Docmost.

- git.ts (VaultGit): system-git wrapper, all ops cwd=vaultPath (vault is its own
  repo under data/vault, never the source repo); ensureRepo/branches main+docmost,
  commit with provenance (author/committer identity + Docmost-Sync-Source trailer,
  §7.3), merge with conflict surfacing (no auto-resolve, §9), isMergeInProgress;
  GIT_DIR/GIT_WORK_TREE stripped from env (§12 cwd isolation)
- stabilize.ts: normalize-on-write (one export->import->export fixpoint pass, §11)
- reconcile.ts: pure planReconciliation (add/update/move/delete by pageId) +
  decideAbsenceDeletions gate
- pull.ts: write/commit on docmost -> merge into main; listSpaceTree completeness
  signal suppresses absence-deletions on a partial fetch (§8); mass-delete guard;
  merge-in-progress guard makes re-runs converge (§12); move old-path removal only
  on successful write
- docmost-client: listSpaceTree({pages, complete}) without touching the 1:1-copied
  enumerateSpacePages
- tests: reconcile planner + decideAbsenceDeletions, VaultGit incl. real temp-repo
  merge conflict, listSpaceTree completeness (586 green)

Push to a git remote and the FS->Docmost direction are deferred to the next increment.
This commit is contained in:
vvzvlad
2026-06-16 23:57:50 +03:00
parent 4b34f4d30a
commit 531b320776
8 changed files with 1594 additions and 47 deletions

View File

@@ -442,6 +442,105 @@ describe('checkNewComments', () => {
});
});
// ---------------------------------------------------------------------------
// listSpaceTree — completeness signal (SPEC §8)
// ---------------------------------------------------------------------------
describe('listSpaceTree (completeness)', () => {
// The walk seeds from /pages/sidebar-pages with only { spaceId } (roots), then
// fetches each hasChildren node's children with { spaceId, pageId }. We route
// by the presence of `pageId` in the request body.
it('returns complete:true and every node for a fully-fetched tree', async () => {
const client = new DocmostClient(BASE_URL, 'a@b.c', 'pw');
stubLoginSuccess(globalAxiosMock());
const imock = instanceMock(client);
imock.onPost('/pages/sidebar-pages').reply((config) => {
const body = JSON.parse(config.data);
if (!body.pageId) {
// Root level: one parent with children + one leaf.
return [
200,
{
data: {
items: [
{ id: 'root', title: 'Root', hasChildren: true },
{ id: 'leaf', title: 'Leaf', hasChildren: false },
],
},
},
];
}
if (body.pageId === 'root') {
return [
200,
{ data: { items: [{ id: 'child', title: 'Child', hasChildren: false }] } },
];
}
return [200, { data: { items: [] } }];
});
const { pages, complete } = await client.listSpaceTree('space-1');
expect(complete).toBe(true);
expect(new Set(pages.map((p: any) => p.id))).toEqual(
new Set(['root', 'leaf', 'child']),
);
});
it('returns complete:false but still the other nodes when a branch fetch THROWS', async () => {
const client = new DocmostClient(BASE_URL, 'a@b.c', 'pw');
stubLoginSuccess(globalAxiosMock());
const imock = instanceMock(client);
imock.onPost('/pages/sidebar-pages').reply((config) => {
const body = JSON.parse(config.data);
if (!body.pageId) {
// Two parents, both claim children; one of them will fail to expand.
return [
200,
{
data: {
items: [
{ id: 'ok', title: 'Ok', hasChildren: true },
{ id: 'boom', title: 'Boom', hasChildren: true },
],
},
},
];
}
if (body.pageId === 'ok') {
return [
200,
{ data: { items: [{ id: 'okchild', title: 'OkChild', hasChildren: false }] } },
];
}
// The 'boom' branch fails -> walk must continue, completeness must drop.
return [500, {}];
});
const { pages, complete } = await client.listSpaceTree('space-1');
// The failed branch flips completeness to false...
expect(complete).toBe(false);
// ...but the rest of the tree is still collected (no abort, no wipe signal).
expect(new Set(pages.map((p: any) => p.id))).toEqual(
new Set(['ok', 'boom', 'okchild']),
);
});
it('returns complete:false and no nodes when the seed (root) fetch fails', async () => {
const client = new DocmostClient(BASE_URL, 'a@b.c', 'pw');
stubLoginSuccess(globalAxiosMock());
const imock = instanceMock(client);
// Every sidebar-pages call fails -> listSidebarPages itself throws on the
// seed, so the walk returns empty + incomplete (never "0 pages, complete").
imock.onPost('/pages/sidebar-pages').reply(500, {});
const { pages, complete } = await client.listSpaceTree('space-1');
expect(complete).toBe(false);
expect(pages).toEqual([]);
});
});
// ---------------------------------------------------------------------------
// AUTH: 401 interceptor + re-login dedup + getCollabTokenWithReauth
// ---------------------------------------------------------------------------

292
test/git.test.ts Normal file
View File

@@ -0,0 +1,292 @@
import { execFile } from 'node:child_process';
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { promisify } from 'node:util';
import { afterEach, beforeAll, describe, expect, it } from 'vitest';
import {
VaultGit,
BOT_AUTHOR_NAME,
BOT_AUTHOR_EMAIL,
buildCommitMessage,
} from '../src/git.js';
const execFileAsync = promisify(execFile);
/** True if a usable `git` binary is on PATH (skip the suite otherwise). */
async function gitAvailable(): Promise<boolean> {
try {
await execFileAsync('git', ['--version']);
return true;
} catch {
return false;
}
}
/** Read the full commit message of HEAD (subject + body) in a repo dir. */
async function headMessage(dir: string): Promise<string> {
const { stdout } = await execFileAsync(
'git',
['--no-pager', 'log', '-1', '--pretty=%B'],
{ cwd: dir },
);
return stdout.trim();
}
/** Read the author "Name <email>" of HEAD in a repo dir. */
async function headAuthor(dir: string): Promise<string> {
const { stdout } = await execFileAsync(
'git',
['--no-pager', 'log', '-1', '--pretty=%an <%ae>'],
{ cwd: dir },
);
return stdout.trim();
}
describe('buildCommitMessage (pure)', () => {
it('returns the bare subject when there are no trailers', () => {
expect(buildCommitMessage('subject')).toBe('subject');
expect(buildCommitMessage('subject', [])).toBe('subject');
});
it('appends trailers separated from the subject by a blank line', () => {
expect(buildCommitMessage('subject', ['Docmost-Sync-Source: docmost'])).toBe(
'subject\n\nDocmost-Sync-Source: docmost',
);
});
});
describe('VaultGit (integration; temp repo)', () => {
let available = false;
let dir: string;
beforeAll(async () => {
available = await gitAvailable();
});
afterEach(async () => {
if (dir) {
await rm(dir, { recursive: true, force: true });
}
});
/** Make a fresh temp dir for one test (under the OS tmpdir, NOT the repo). */
async function freshDir(): Promise<string> {
dir = await mkdtemp(join(tmpdir(), 'docmost-vault-'));
return dir;
}
it('ensureRepo creates .git + main + an initial commit', async () => {
if (!available) return; // skip gracefully when git is unavailable
const vault = await freshDir();
const git = new VaultGit(vault);
await git.ensureRepo();
// It is a git work-tree now.
const { stdout: insideWt } = await execFileAsync(
'git',
['rev-parse', '--is-inside-work-tree'],
{ cwd: vault },
);
expect(insideWt.trim()).toBe('true');
// On `main`.
expect(await git.currentBranch()).toBe('main');
// Has the initial commit.
expect(await headMessage(vault)).toBe('init vault');
// Idempotent: calling again does not create a second commit.
await git.ensureRepo();
const { stdout: count } = await execFileAsync(
'git',
['rev-list', '--count', 'HEAD'],
{ cwd: vault },
);
expect(count.trim()).toBe('1');
});
it('ensureBranch creates the docmost branch from main', async () => {
if (!available) return;
const vault = await freshDir();
const git = new VaultGit(vault);
await git.ensureRepo();
expect(await git.branchExists('docmost')).toBe(false);
await git.ensureBranch('docmost', 'main');
expect(await git.branchExists('docmost')).toBe(true);
// Idempotent.
await git.ensureBranch('docmost', 'main');
expect(await git.branchExists('docmost')).toBe(true);
});
it('commit writes a commit with the provenance trailer and the bot identity', async () => {
if (!available) return;
const vault = await freshDir();
const git = new VaultGit(vault);
await git.ensureRepo();
await writeFile(join(vault, 'page.md'), 'hello\n', 'utf8');
await git.stageAll();
const made = await git.commit('docmost: sync 1 page(s)', {
authorName: BOT_AUTHOR_NAME,
authorEmail: BOT_AUTHOR_EMAIL,
trailers: ['Docmost-Sync-Source: docmost'],
});
expect(made).toBe(true);
const msg = await headMessage(vault);
expect(msg).toContain('docmost: sync 1 page(s)');
expect(msg).toContain('Docmost-Sync-Source: docmost');
const author = await headAuthor(vault);
expect(author).toBe(`${BOT_AUTHOR_NAME} <${BOT_AUTHOR_EMAIL}>`);
// The trailer is parseable by git itself.
const { stdout: trailers } = await execFileAsync(
'git',
['--no-pager', 'log', '-1', '--pretty=%(trailers:key=Docmost-Sync-Source,valueonly)'],
{ cwd: vault },
);
expect(trailers.trim()).toBe('docmost');
});
it('commit is a no-op when there is nothing to commit', async () => {
if (!available) return;
const vault = await freshDir();
const git = new VaultGit(vault);
await git.ensureRepo();
await git.stageAll(); // nothing changed since the init commit
const made = await git.commit('docmost: sync 0 page(s)', {
authorName: BOT_AUTHOR_NAME,
authorEmail: BOT_AUTHOR_EMAIL,
trailers: ['Docmost-Sync-Source: docmost'],
});
expect(made).toBe(false);
// Still exactly one commit (the init one).
const { stdout: count } = await execFileAsync(
'git',
['rev-list', '--count', 'HEAD'],
{ cwd: vault },
);
expect(count.trim()).toBe('1');
});
it('merge fast-forwards main to docmost', async () => {
if (!available) return;
const vault = await freshDir();
const git = new VaultGit(vault);
await git.ensureRepo();
await git.ensureBranch('docmost', 'main');
// Commit a file on docmost.
await git.checkout('docmost');
await writeFile(join(vault, 'a.md'), 'a\n', 'utf8');
await git.stageAll();
await git.commit('docmost: sync 1 page(s)', {
authorName: BOT_AUTHOR_NAME,
authorEmail: BOT_AUTHOR_EMAIL,
trailers: ['Docmost-Sync-Source: docmost'],
});
// main has not diverged, so the merge is a clean fast-forward.
await git.checkout('main');
const res = await git.merge('docmost');
expect(res.ok).toBe(true);
expect(res.conflict).toBe(false);
// main now contains the file and the docmost commit.
const tracked = await git.listTrackedFiles();
expect(tracked).toContain('a.md');
expect(await headMessage(vault)).toContain('docmost: sync 1 page(s)');
});
it('merge surfaces a conflict distinctly (no auto-resolve)', async () => {
if (!available) return;
const vault = await freshDir();
const git = new VaultGit(vault);
await git.ensureRepo();
await git.ensureBranch('docmost', 'main');
// Divergent edits to the SAME file on both branches -> real conflict.
await git.checkout('docmost');
await writeFile(join(vault, 'c.md'), 'from docmost\n', 'utf8');
await git.stageAll();
await git.commit('docmost edit', {
authorName: BOT_AUTHOR_NAME,
authorEmail: BOT_AUTHOR_EMAIL,
});
await git.checkout('main');
await writeFile(join(vault, 'c.md'), 'from main\n', 'utf8');
await git.stageAll();
await git.commit('main edit', {
authorName: 'Human',
authorEmail: 'human@local',
});
const res = await git.merge('docmost');
expect(res.ok).toBe(false);
expect(res.conflict).toBe(true);
});
it('isMergeInProgress is false on a clean repo and true mid-merge', async () => {
if (!available) return;
const vault = await freshDir();
const git = new VaultGit(vault);
await git.ensureRepo();
await git.ensureBranch('docmost', 'main');
// Clean repo, no merge in progress.
expect(await git.isMergeInProgress()).toBe(false);
// Create a REAL conflict: divergent edits to the same file on both branches.
await git.checkout('docmost');
await writeFile(join(vault, 'c.md'), 'from docmost\n', 'utf8');
await git.stageAll();
await git.commit('docmost edit', {
authorName: BOT_AUTHOR_NAME,
authorEmail: BOT_AUTHOR_EMAIL,
});
await git.checkout('main');
await writeFile(join(vault, 'c.md'), 'from main\n', 'utf8');
await git.stageAll();
await git.commit('main edit', {
authorName: 'Human',
authorEmail: 'human@local',
});
// Merge conflicts -> the repo is now left mid-merge.
const res = await git.merge('docmost');
expect(res.conflict).toBe(true);
expect(await git.isMergeInProgress()).toBe(true);
// Aborting the merge clears the in-progress state again.
await execFileAsync('git', ['--no-pager', 'merge', '--abort'], { cwd: vault });
expect(await git.isMergeInProgress()).toBe(false);
});
it('listTrackedFiles supports a glob and returns forward-slash paths', async () => {
if (!available) return;
const vault = await freshDir();
const git = new VaultGit(vault);
await git.ensureRepo();
await writeFile(join(vault, 'keep.md'), 'k\n', 'utf8');
await writeFile(join(vault, 'note.txt'), 't\n', 'utf8');
await git.stageAll();
await git.commit('add files', {
authorName: BOT_AUTHOR_NAME,
authorEmail: BOT_AUTHOR_EMAIL,
});
const md = await git.listTrackedFiles('*.md');
expect(md).toEqual(['keep.md']);
const all = await git.listTrackedFiles();
expect(new Set(all)).toEqual(new Set(['keep.md', 'note.txt']));
});
});

238
test/reconcile.test.ts Normal file
View File

@@ -0,0 +1,238 @@
import { describe, expect, it } from 'vitest';
import {
planReconciliation,
decideAbsenceDeletions,
type ExistingEntry,
type LiveEntry,
} from '../src/reconcile.js';
describe('planReconciliation', () => {
it('ADD: a new live page (not tracked) is written, nothing deleted', () => {
const live: LiveEntry[] = [{ pageId: 'p1', relPath: 'Space/New.md' }];
const existing: ExistingEntry[] = [];
const plan = planReconciliation(live, existing);
expect(plan.toWrite).toEqual([{ pageId: 'p1', relPath: 'Space/New.md' }]);
expect(plan.toDelete).toEqual([]);
expect(plan.moved).toEqual([]);
});
it('CONTENT-UPDATE: tracked page at the SAME path is rewritten, not moved/deleted', () => {
const live: LiveEntry[] = [{ pageId: 'p1', relPath: 'Space/Doc.md' }];
const existing: ExistingEntry[] = [{ pageId: 'p1', relPath: 'Space/Doc.md' }];
const plan = planReconciliation(live, existing);
// Still written (re-emitted; identical bytes => git no-op), no move/delete.
expect(plan.toWrite).toEqual([{ pageId: 'p1', relPath: 'Space/Doc.md' }]);
expect(plan.toDelete).toEqual([]);
expect(plan.moved).toEqual([]);
});
it('MOVE: same pageId, new path -> write new + recorded as moved (NOT in toDelete)', () => {
const live: LiveEntry[] = [{ pageId: 'p1', relPath: 'Space/NewParent/Doc.md' }];
const existing: ExistingEntry[] = [
{ pageId: 'p1', relPath: 'Space/OldParent/Doc.md' },
];
const plan = planReconciliation(live, existing);
expect(plan.toWrite).toEqual([
{ pageId: 'p1', relPath: 'Space/NewParent/Doc.md' },
]);
// The old path is a MOVE removal, NOT an absence delete -> not in toDelete.
expect(plan.toDelete).toEqual([]);
expect(plan.moved).toEqual([
{
pageId: 'p1',
fromRelPath: 'Space/OldParent/Doc.md',
toRelPath: 'Space/NewParent/Doc.md',
removeOldPath: true,
},
]);
});
it('DELETE: a tracked pageId gone from live -> its file is deleted', () => {
const live: LiveEntry[] = [{ pageId: 'p1', relPath: 'Space/Keep.md' }];
const existing: ExistingEntry[] = [
{ pageId: 'p1', relPath: 'Space/Keep.md' },
{ pageId: 'p2', relPath: 'Space/Gone.md' },
];
const plan = planReconciliation(live, existing);
expect(plan.toWrite).toEqual([{ pageId: 'p1', relPath: 'Space/Keep.md' }]);
expect(plan.toDelete).toEqual(['Space/Gone.md']);
expect(plan.moved).toEqual([]);
});
it('NO-OP: live and existing identical -> writes (re-emit) but no deletes/moves', () => {
const live: LiveEntry[] = [
{ pageId: 'p1', relPath: 'A.md' },
{ pageId: 'p2', relPath: 'B.md' },
];
const existing: ExistingEntry[] = [
{ pageId: 'p1', relPath: 'A.md' },
{ pageId: 'p2', relPath: 'B.md' },
];
const plan = planReconciliation(live, existing);
expect(plan.toWrite).toEqual(live);
expect(plan.toDelete).toEqual([]);
expect(plan.moved).toEqual([]);
});
it('does NOT delete an old path that another live page will write (path reuse)', () => {
// p1 moves from X.md to Y.md; p2 is a NEW page taking over X.md. The old
// X.md must NOT be deleted, because p2 writes it.
const live: LiveEntry[] = [
{ pageId: 'p1', relPath: 'Y.md' },
{ pageId: 'p2', relPath: 'X.md' },
];
const existing: ExistingEntry[] = [{ pageId: 'p1', relPath: 'X.md' }];
const plan = planReconciliation(live, existing);
expect(new Set(plan.toWrite)).toEqual(
new Set([
{ pageId: 'p1', relPath: 'Y.md' },
{ pageId: 'p2', relPath: 'X.md' },
]),
);
// X.md is a live target, so nothing is deleted.
expect(plan.toDelete).toEqual([]);
// The move is still recorded, but its old path is NOT removable (p2 writes
// X.md): removeOldPath:false protects the reused path from data loss.
expect(plan.moved).toEqual([
{ pageId: 'p1', fromRelPath: 'X.md', toRelPath: 'Y.md', removeOldPath: false },
]);
});
it('combines add + update + move + delete in one plan', () => {
const live: LiveEntry[] = [
{ pageId: 'keep', relPath: 'Keep.md' }, // update in place
{ pageId: 'mover', relPath: 'New/Moved.md' }, // moved
{ pageId: 'fresh', relPath: 'Fresh.md' }, // added
];
const existing: ExistingEntry[] = [
{ pageId: 'keep', relPath: 'Keep.md' },
{ pageId: 'mover', relPath: 'Old/Moved.md' },
{ pageId: 'dead', relPath: 'Dead.md' }, // deleted
];
const plan = planReconciliation(live, existing);
expect(plan.toWrite).toEqual(live);
expect(plan.moved).toEqual([
{
pageId: 'mover',
fromRelPath: 'Old/Moved.md',
toRelPath: 'New/Moved.md',
removeOldPath: true,
},
]);
// toDelete is ABSENCE-only now: the moved old path lives in `moved`, so only
// the genuinely-gone page (Dead.md) is here.
expect(plan.toDelete).toEqual(['Dead.md']);
});
it('records each duplicate tracked row of a present pageId as a removable move', () => {
// Two stray files both claim pageId "dup"; the live page lives elsewhere.
// Each stray is a MOVE (same pageId, different path) -> recorded in `moved`
// with removeOldPath:true, NOT in absence-based toDelete.
const live: LiveEntry[] = [{ pageId: 'dup', relPath: 'Canonical.md' }];
const existing: ExistingEntry[] = [
{ pageId: 'dup', relPath: 'StrayA.md' },
{ pageId: 'dup', relPath: 'StrayB.md' },
];
const plan = planReconciliation(live, existing);
expect(plan.toWrite).toEqual([{ pageId: 'dup', relPath: 'Canonical.md' }]);
expect(plan.toDelete).toEqual([]);
expect(plan.moved).toEqual([
{
pageId: 'dup',
fromRelPath: 'StrayA.md',
toRelPath: 'Canonical.md',
removeOldPath: true,
},
{
pageId: 'dup',
fromRelPath: 'StrayB.md',
toRelPath: 'Canonical.md',
removeOldPath: true,
},
]);
});
});
describe('decideAbsenceDeletions (SPEC §8)', () => {
it('APPLIES when the tree is complete and the delete count is modest', () => {
const d = decideAbsenceDeletions({
treeComplete: true,
liveCount: 10,
existingCount: 10,
deleteCount: 1,
});
expect(d).toEqual({ apply: true });
});
it('SUPPRESSES all absence deletions when the tree fetch is incomplete', () => {
// Even a single absence delete is suppressed on a partial tree (a missing
// pageId in a partial tree is NOT proof of deletion).
const d = decideAbsenceDeletions({
treeComplete: false,
liveCount: 9,
existingCount: 10,
deleteCount: 1,
});
expect(d).toEqual({ apply: false, reason: 'incomplete-fetch' });
});
it('SUPPRESSES when live returned 0 pages but files are tracked (complete flag aside)', () => {
const d = decideAbsenceDeletions({
treeComplete: true,
liveCount: 0,
existingCount: 5,
deleteCount: 5,
});
expect(d).toEqual({ apply: false, reason: 'empty-live' });
});
it('SUPPRESSES over the mass-delete guard (> 50% of a non-trivial vault)', () => {
const d = decideAbsenceDeletions({
treeComplete: true,
liveCount: 4,
existingCount: 10,
deleteCount: 6, // 60% > 50%
});
expect(d).toEqual({ apply: false, reason: 'mass-delete' });
});
it('does NOT apply the fraction guard for a tiny vault (below the floor)', () => {
// 1-of-2 is normal in a tiny vault; the fraction guard does not fire.
const d = decideAbsenceDeletions({
treeComplete: true,
liveCount: 1,
existingCount: 2,
deleteCount: 1,
});
expect(d).toEqual({ apply: true });
});
it('incomplete-fetch takes precedence over the mass-delete reason', () => {
const d = decideAbsenceDeletions({
treeComplete: false,
liveCount: 4,
existingCount: 10,
deleteCount: 6,
});
expect(d).toEqual({ apply: false, reason: 'incomplete-fetch' });
});
it('trivially applies when nothing is tracked or nothing would be deleted', () => {
expect(
decideAbsenceDeletions({
treeComplete: false,
liveCount: 0,
existingCount: 0,
deleteCount: 0,
}),
).toEqual({ apply: true });
expect(
decideAbsenceDeletions({
treeComplete: false,
liveCount: 5,
existingCount: 5,
deleteCount: 0,
}),
).toEqual({ apply: true });
});
});