Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 8503ff1f3d | |||
| ae3dfd8de6 | |||
| 03eafa6c68 | |||
| a42f1ead48 | |||
| b7a3ec227d | |||
| 846341d7d4 | |||
| 32e10ca6d3 | |||
| 74d212cfd3 | |||
| c96fafc4ad | |||
| 9a6389133b | |||
| 0b8497e496 | |||
| 433252bdb1 | |||
| 2f8c5d9a98 | |||
| 6f81067f4d | |||
| 0af7eb30c3 | |||
| 216499d57b | |||
| c765483947 | |||
| daf728676f | |||
| 1d8e3444f4 | |||
| c50f5b66bb | |||
| 51d44b6061 | |||
| 6247585b66 |
@@ -226,6 +226,13 @@ jobs:
|
||||
- name: Build mcp
|
||||
run: pnpm --filter @docmost/mcp build
|
||||
|
||||
# apps/server imports @docmost/token-estimate at runtime (history-budget.ts,
|
||||
# #490); its dist/ is gitignored and `test:e2e` type-checks + runs the code,
|
||||
# so build it here or tsc fails with TS2307 Cannot find module
|
||||
# '@docmost/token-estimate' (mirrors the editor-ext / mcp build steps above).
|
||||
- name: Build token-estimate
|
||||
run: pnpm --filter @docmost/token-estimate build
|
||||
|
||||
- name: Run migrations
|
||||
run: pnpm --filter ./apps/server migration:latest
|
||||
|
||||
|
||||
@@ -159,6 +159,14 @@ jobs:
|
||||
- name: Build prosemirror-markdown
|
||||
run: pnpm --filter @docmost/prosemirror-markdown build
|
||||
|
||||
# @docmost/token-estimate is a shared workspace package the client vitest
|
||||
# suite resolves via its dist build (main: ./dist/index.js); dist/ is
|
||||
# gitignored and `pnpm -r test` does NOT honour nx `dependsOn: ^build`, so
|
||||
# build it before the recursive test run or the client suite fails with
|
||||
# "Failed to resolve import '@docmost/token-estimate'" (#490).
|
||||
- name: Build token-estimate
|
||||
run: pnpm --filter @docmost/token-estimate build
|
||||
|
||||
- name: Run unit tests
|
||||
run: pnpm -r test
|
||||
|
||||
|
||||
@@ -135,6 +135,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
new resync path re-reads the live anchor so the suggestion applies against the
|
||||
current text, and orphaned anchors (whose marked run was deleted) are
|
||||
reconciled rather than left blocking. (#496)
|
||||
- **Save intentional page versions.** Press `Cmd/Ctrl+S` (or use the page menu)
|
||||
to save a named version of a page. The history panel now distinguishes
|
||||
intentional versions (a "Saved" / "Agent version" badge) from automatic
|
||||
snapshots, dims autosaves, and offers an "Only versions" filter. Automatic
|
||||
snapshots switched from a fixed interval to a trailing idle-flush with a
|
||||
max-wait ceiling, and a boundary snapshot is pinned whenever the editing source
|
||||
changes (e.g. a person's edits followed by the AI agent). (#370)
|
||||
|
||||
- **Place several images side by side in a row.** A new "Inline (side by
|
||||
side)" alignment mode in the image bubble menu renders consecutive inline
|
||||
@@ -385,6 +392,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
`@docmost/token-estimate` package) so they can never diverge. Deferred-tool
|
||||
activation is also cached in the chat metadata to avoid re-resolving it each
|
||||
turn. (#490)
|
||||
- **Cyrillic (and any non-ASCII) draw.io labels no longer turn into mojibake
|
||||
when a diagram is opened in the draw.io editor.** Agent-created diagrams
|
||||
(`drawioCreate`) and Confluence-imported diagrams stored their model in the
|
||||
SVG's `content=` attribute as base64; the draw.io editor decodes that via
|
||||
Latin-1 `atob` (no UTF-8 step), so every non-ASCII char (e.g. `Старт-бит`,
|
||||
`ё`, `—`) split into garbage and the editor's autosave then persisted the
|
||||
corrupted model, breaking the page preview too. Both write paths
|
||||
(`buildDrawioSvg`, the import service's `createDrawioSvg`) now write `content=`
|
||||
as XML-entity-escaped mxfile XML — draw.io's own native form, decoded by the
|
||||
DOM as UTF-8 — so labels open intact. The decoder reads both the new
|
||||
entity-encoded form and the old base64 form, so existing diagrams still open.
|
||||
*Healing pre-fix diagrams:* only a diagram that still holds its original
|
||||
(correct-UTF-8) base64 — i.e. one not yet opened/autosaved in the draw.io
|
||||
editor — can be repaired in place by `drawioGet` → `drawioUpdate` with the
|
||||
same XML (rewrites the attachment in the new form); no migration script is
|
||||
needed. A diagram that was already opened in the editor persisted the
|
||||
mojibake at rest, so `drawioGet` reads the already-corrupted text and
|
||||
`drawioUpdate` faithfully rewrites it — that text is lost and is not
|
||||
recoverable by a rewrite. (#507)
|
||||
- **A chat with one malformed message part no longer 500s on every turn, and a
|
||||
failed send no longer duplicates the user's message.** Incoming client parts
|
||||
are now whitelisted to `text` (a forged tool-result part can no longer reach
|
||||
|
||||
@@ -59,6 +59,14 @@ COPY --from=builder /app/packages/mcp/data /app/packages/mcp/data
|
||||
COPY --from=builder /app/packages/prosemirror-markdown/build /app/packages/prosemirror-markdown/build
|
||||
COPY --from=builder /app/packages/prosemirror-markdown/package.json /app/packages/prosemirror-markdown/package.json
|
||||
|
||||
# apps/server imports @docmost/token-estimate (workspace:*) at runtime
|
||||
# (history-budget.ts, #490). tsc emits only dist/ and dist/ is gitignored, so the
|
||||
# prod install would resolve a broken workspace symlink and the server would die
|
||||
# with ERR_MODULE_NOT_FOUND on the first history-budget call. Ship the built
|
||||
# package + its manifest, mirroring prosemirror-markdown above.
|
||||
COPY --from=builder /app/packages/token-estimate/dist /app/packages/token-estimate/dist
|
||||
COPY --from=builder /app/packages/token-estimate/package.json /app/packages/token-estimate/package.json
|
||||
|
||||
# Copy root package files
|
||||
COPY --from=builder /app/package.json /app/package.json
|
||||
COPY --from=builder /app/pnpm*.yaml /app/
|
||||
|
||||
@@ -1418,5 +1418,14 @@
|
||||
"The commented text changed since this suggestion was made; it was not applied.": "The commented text changed since this suggestion was made; it was not applied.",
|
||||
"Dismiss": "Dismiss",
|
||||
"Suggestion dismissed": "Suggestion dismissed",
|
||||
"Failed to dismiss suggestion": "Failed to dismiss suggestion"
|
||||
"Failed to dismiss suggestion": "Failed to dismiss suggestion",
|
||||
"Save version": "Save version",
|
||||
"Ctrl+S": "Ctrl+S",
|
||||
"Version saved": "Version saved",
|
||||
"Already saved as the latest version": "Already saved as the latest version",
|
||||
"Agent version": "Agent version",
|
||||
"Boundary": "Boundary",
|
||||
"Autosave": "Autosave",
|
||||
"Only versions": "Only versions",
|
||||
"No saved versions yet.": "No saved versions yet."
|
||||
}
|
||||
|
||||
@@ -1433,5 +1433,14 @@
|
||||
"The commented text changed since this suggestion was made; it was not applied.": "Прокомментированный текст изменился после создания предложения; оно не было применено.",
|
||||
"Dismiss": "Не применять",
|
||||
"Suggestion dismissed": "Предложение отклонено",
|
||||
"Failed to dismiss suggestion": "Не удалось отклонить предложение"
|
||||
"Failed to dismiss suggestion": "Не удалось отклонить предложение",
|
||||
"Save version": "Сохранить версию",
|
||||
"Ctrl+S": "Ctrl+S",
|
||||
"Version saved": "Версия сохранена",
|
||||
"Already saved as the latest version": "Уже сохранено как последняя версия",
|
||||
"Agent version": "Версия агента",
|
||||
"Boundary": "Граница",
|
||||
"Autosave": "Автосейв",
|
||||
"Only versions": "Только версии",
|
||||
"No saved versions yet.": "Пока нет сохранённых версий."
|
||||
}
|
||||
|
||||
@@ -27,6 +27,7 @@ vi.mock("@/features/ai-chat/utils/markdown.ts", async () => {
|
||||
|
||||
import MessageItem from "./message-item";
|
||||
import { messageSignature } from "@/features/ai-chat/utils/message-signature.ts";
|
||||
import { splitPlainChunks } from "./streaming-plain-text";
|
||||
|
||||
// matchMedia (read by MantineProvider) is stubbed globally in vitest.setup.ts.
|
||||
|
||||
@@ -114,3 +115,89 @@ describe("MessageItem markdown memoization", () => {
|
||||
expect(queryByText("streamed answer")).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// PERF SMOKE (#492): the whole point of the incremental streaming render is that
|
||||
// the ANSWER path costs O(number of markdown blocks), NOT O(number of throttled
|
||||
// ~20Hz ticks). Pre-#492 the finalized MarkdownPart re-parsed the WHOLE growing
|
||||
// answer on every delta — a synthetic ~100 KB stream measured 394 renderChatMarkdown
|
||||
// calls (one per tick). With the incremental render each STABILIZED block is parsed
|
||||
// exactly once (memoized in MarkdownChunk) and the live tail is cheap plain text, so
|
||||
// the call count collapses to ~= the block count regardless of tick granularity.
|
||||
describe("MessageItem streaming answer render is O(blocks), not O(ticks)", () => {
|
||||
// ~100 KB answer. Each section is a heading + a paragraph — TWO blank-line
|
||||
// delimited markdown blocks — so the safe-cut block count is ~2× the section
|
||||
// count. The perf claim is about the BLOCK count (the memoization granularity),
|
||||
// measured directly with splitPlainChunks below, not the section count.
|
||||
const buildAnswer = () => {
|
||||
const SECTIONS = 100;
|
||||
const paragraphs: string[] = [];
|
||||
for (let i = 0; i < SECTIONS; i++) {
|
||||
paragraphs.push(`## Section ${i}\n\n` + "lorem ipsum dolor ".repeat(55));
|
||||
}
|
||||
const full = paragraphs.join("\n\n");
|
||||
// The number of memoized markdown blocks the incremental render splits into
|
||||
// (all but the live tail are parsed once each).
|
||||
return { full, blocks: splitPlainChunks(full).length };
|
||||
};
|
||||
|
||||
const streamMsg = (text: string, state: "streaming" | "done"): UIMessage =>
|
||||
({
|
||||
id: "m1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text, state }],
|
||||
}) as UIMessage;
|
||||
|
||||
it("parses each block ~once over a 100KB stream (≈blocks, ≪ ticks)", () => {
|
||||
renderChatMarkdownSpy.mockClear();
|
||||
const { full, blocks } = buildAnswer();
|
||||
const CHUNK = 128; // a realistic ~20Hz throttled delta size
|
||||
const ticks = Math.ceil(full.length / CHUNK);
|
||||
|
||||
let msg = streamMsg(full.slice(0, CHUNK), "streaming");
|
||||
const { rerender } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg}
|
||||
signature={messageSignature(msg)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
for (let end = 2 * CHUNK; end < full.length; end += CHUNK) {
|
||||
msg = streamMsg(full.slice(0, end), "streaming");
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg}
|
||||
signature={messageSignature(msg)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
}
|
||||
// Finalize: the streaming→done flip renders the whole answer through ONE
|
||||
// canonical pass (visual parity), so the finished DOM matches the pre-#492
|
||||
// output. This is the single extra parse on top of the per-block ones.
|
||||
const done = streamMsg(full, "done");
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem message={done} signature={messageSignature(done)} />
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
const calls = renderChatMarkdownSpy.mock.calls.length;
|
||||
// Sanity: the stream really had far more ticks than blocks (else the test is
|
||||
// vacuous — the point is that calls scale with blocks, not ticks).
|
||||
expect(ticks).toBeGreaterThan(blocks * 3);
|
||||
// O(blocks): each stabilized block parsed once + the single final whole-text
|
||||
// parse. A small constant absorbs the finalize render and the live-tail block;
|
||||
// the load-bearing claim is the bound below.
|
||||
expect(calls).toBeLessThanOrEqual(blocks + 2);
|
||||
// ≪ ticks — and, non-vacuously, the blocks WERE parsed (not skipped entirely).
|
||||
expect(calls).toBeLessThan(ticks / 3);
|
||||
expect(calls).toBeGreaterThan(blocks / 2);
|
||||
// MUTATION-VERIFY (documented, not run here): dropping the `memo()` wrapper on
|
||||
// MarkdownChunk (so every stable block re-parses each tick) drives `calls`
|
||||
// toward `ticks` (~394), reddening both upper-bound assertions above.
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { render } from "@testing-library/react";
|
||||
import { MantineProvider } from "@mantine/core";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
|
||||
// Stub react-i18next (the component reads `useTranslation`). Mirrors the other
|
||||
// message-item specs.
|
||||
vi.mock("react-i18next", () => ({
|
||||
useTranslation: () => ({ t: (key: string) => key }),
|
||||
}));
|
||||
|
||||
import MessageItem from "./message-item";
|
||||
import { messageSignature } from "@/features/ai-chat/utils/message-signature.ts";
|
||||
// The REAL canonical renderer (NOT the spy the memo test installs): this file
|
||||
// exercises the actual markdown output so the visual-regression assertions below
|
||||
// compare against genuine HTML (incl. the schema's `<li><p>` wrappers).
|
||||
import { renderChatMarkdown } from "@/features/ai-chat/utils/markdown.ts";
|
||||
import classes from "./ai-chat.module.css";
|
||||
|
||||
const msg = (
|
||||
parts: UIMessage["parts"],
|
||||
extra?: Partial<UIMessage>,
|
||||
): UIMessage =>
|
||||
({ id: "m1", role: "assistant", parts, ...extra }) as UIMessage;
|
||||
|
||||
const renderRow = (message: UIMessage, turnStreaming = false) =>
|
||||
render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={message}
|
||||
signature={messageSignature(message)}
|
||||
turnStreaming={turnStreaming}
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
// A rich multi-block answer that exercises headings, a list (the `<li><p>` case
|
||||
// the scoped CSS tightens), inline emphasis, and multiple paragraphs.
|
||||
const ANSWER = [
|
||||
"# Заголовок",
|
||||
"",
|
||||
"Первый абзац с **жирным** и `кодом`.",
|
||||
"",
|
||||
"- пункт один",
|
||||
"- пункт два",
|
||||
"",
|
||||
"Второй абзац.",
|
||||
].join("\n");
|
||||
|
||||
describe("MessageItem final render — visual parity with the canonical pipeline", () => {
|
||||
it("a finalized text part renders exactly renderChatMarkdown(text)", () => {
|
||||
const { container } = renderRow(
|
||||
msg([{ type: "text", text: ANSWER, state: "done" }]),
|
||||
);
|
||||
const block = container.querySelector(`.${classes.markdown}`);
|
||||
expect(block).not.toBeNull();
|
||||
// Byte-for-byte the canonical output (the SAME whole-text pass the pre-#492
|
||||
// MarkdownPart produced), including `<li><p>…</p></li>` wrappers.
|
||||
expect(block!.innerHTML).toBe(renderChatMarkdown(ANSWER, {}));
|
||||
// The list wrapper is really present (guards against a vacuous empty render).
|
||||
expect(container.querySelectorAll("li p").length).toBe(2);
|
||||
});
|
||||
|
||||
it("the streaming incremental view CONVERGES to the canonical render on finish", () => {
|
||||
// Mount mid-stream (live tail) — the DOM here is the incremental view.
|
||||
const { container, rerender } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg([{ type: "text", text: ANSWER, state: "streaming" }])}
|
||||
signature={messageSignature(
|
||||
msg([{ type: "text", text: ANSWER, state: "streaming" }]),
|
||||
)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
// Finish the turn: state flips to done AND the turn is no longer streaming.
|
||||
const done = msg([{ type: "text", text: ANSWER, state: "done" }]);
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem message={done} signature={messageSignature(done)} />
|
||||
</MantineProvider>,
|
||||
);
|
||||
// After finish there is exactly ONE canonical markdown container whose HTML is
|
||||
// the whole-text render — identical to the non-streaming path above.
|
||||
const blocks = container.querySelectorAll(`.${classes.markdown}`);
|
||||
expect(blocks.length).toBe(1);
|
||||
expect(blocks[0].innerHTML).toBe(renderChatMarkdown(ANSWER, {}));
|
||||
});
|
||||
|
||||
it("neutralizeInternalLinks is honored on the finalized render", () => {
|
||||
const linkAnswer = "См. [страницу](/p/abc).";
|
||||
const { container } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg([{ type: "text", text: linkAnswer, state: "done" }])}
|
||||
signature={messageSignature(
|
||||
msg([{ type: "text", text: linkAnswer, state: "done" }]),
|
||||
)}
|
||||
neutralizeInternalLinks
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
const block = container.querySelector(`.${classes.markdown}`);
|
||||
expect(block!.innerHTML).toBe(
|
||||
renderChatMarkdown(linkAnswer, { neutralizeInternalLinks: true }),
|
||||
);
|
||||
// The internal link was made inert (no href) by the neutralization flag.
|
||||
const a = container.querySelector("a");
|
||||
expect(a?.hasAttribute("href")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import { useTranslation } from "react-i18next";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
import ToolCallCard from "@/features/ai-chat/components/tool-call-card.tsx";
|
||||
import ReasoningBlock from "@/features/ai-chat/components/reasoning-block.tsx";
|
||||
import { StreamingMarkdownText } from "@/features/ai-chat/components/streaming-markdown-text.tsx";
|
||||
import ChatErrorAlert from "@/features/ai-chat/components/chat-error-alert.tsx";
|
||||
import ChatStoppedNotice from "@/features/ai-chat/components/chat-stopped-notice.tsx";
|
||||
import { ToolUiPart, isToolPart } from "@/features/ai-chat/utils/tool-parts.tsx";
|
||||
@@ -86,17 +87,39 @@ interface MessageItemProps {
|
||||
* One assistant text part rendered as sanitized markdown. Memoized on its inputs
|
||||
* so a finalized text part is NOT re-parsed on every streamed delta: during a
|
||||
* turn only the actively-growing tail part changes its `text`, so every earlier
|
||||
* part hits the memo and skips the expensive marked + DOMPurify pass. Props are
|
||||
* primitives, so React.memo's default shallow compare is exactly right (the
|
||||
* `text` string is compared by value).
|
||||
* part hits the memo and skips the expensive canonical parse + DOMPurify pass.
|
||||
* Props are primitives, so React.memo's default shallow compare is exactly right
|
||||
* (the `text` string is compared by value).
|
||||
*
|
||||
* Streaming gate (#492) — mirrors ReasoningBlock:
|
||||
* - `streaming` (this is the live, actively-growing tail part of an in-flight
|
||||
* turn): render incrementally via StreamingMarkdownText — the stabilized blocks
|
||||
* go through the canonical pipeline (each parsed ONCE, memoized) and only the
|
||||
* live tail is cheap plain text. This makes the per-tick cost O(new blocks),
|
||||
* not the pre-#492 O(ticks) whole-answer re-parse on every ~20Hz delta.
|
||||
* - finalized (the common case, and the turn-end flip): render the WHOLE text
|
||||
* through ONE canonical pass — byte-identical to the pre-#492 output (visual
|
||||
* parity). The row re-renders on the streaming→done flip because
|
||||
* `messageSignature` tracks each part's `state` (and `turnStreaming` flips at
|
||||
* turn end), so the incremental view always converges to this single render.
|
||||
*/
|
||||
const MarkdownPart = memo(function MarkdownPart({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
streaming,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
streaming: boolean;
|
||||
}) {
|
||||
if (streaming) {
|
||||
return (
|
||||
<StreamingMarkdownText
|
||||
text={text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
);
|
||||
}
|
||||
const html = renderChatMarkdown(text, { neutralizeInternalLinks });
|
||||
if (html) {
|
||||
return (
|
||||
@@ -179,47 +202,10 @@ function MessageItem({
|
||||
{resolveAssistantName(assistantName) ?? t("AI agent")}
|
||||
</Text>
|
||||
{message.parts.map((part, index) => {
|
||||
if (part.type === "reasoning") {
|
||||
// Reasoning ("thinking") -> a collapsible block with its own token
|
||||
// count. Empty/whitespace reasoning with no authoritative count carries
|
||||
// nothing to show, so skip it (avoids an empty 0-token block).
|
||||
const text = (part as { text?: string }).text ?? "";
|
||||
if (!text.trim() && !(reasoningTokens && reasoningTokens > 0))
|
||||
return null;
|
||||
// Absent state (persisted rows) and "done" both mean finalized.
|
||||
// `messageSignature` already includes each part's `state`, so the
|
||||
// streaming→done flip changes the row signature and re-renders this
|
||||
// row — which is what lets ReasoningBlock switch from chunked plain
|
||||
// text to its one-time markdown parse (see reasoning-block.tsx).
|
||||
// ALSO require the turn to be live: a part stranded at
|
||||
// `state:"streaming"` after the turn ended (no `reasoning-end` — see
|
||||
// the `turnStreaming` prop doc) must still finalize and parse.
|
||||
const streaming =
|
||||
turnStreaming && (part as { state?: string }).state === "streaming";
|
||||
return (
|
||||
<ReasoningBlock
|
||||
key={index}
|
||||
text={text}
|
||||
tokens={reasoningTokens}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (part.type === "text") {
|
||||
// Skip empty/whitespace-only text parts (a streaming message often
|
||||
// starts with an empty text part before the first token arrives); the
|
||||
// typing indicator covers that gap until real content streams in.
|
||||
if (!part.text.trim()) return null;
|
||||
return (
|
||||
<MarkdownPart
|
||||
key={index}
|
||||
text={part.text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
// Tool parts (`tool-*` / `dynamic-tool`) are template-literal kinds, so
|
||||
// they cannot be a `switch` case; the runtime guard handles them, and the
|
||||
// switch below covers every CLOSED (literal-typed) part kind with a
|
||||
// compile-time exhaustiveness check in its default.
|
||||
if (isToolPart(part.type)) {
|
||||
return (
|
||||
<ToolCallCard
|
||||
@@ -232,7 +218,76 @@ function MessageItem({
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
switch (part.type) {
|
||||
case "reasoning": {
|
||||
// Reasoning ("thinking") -> a collapsible block with its own token
|
||||
// count. Empty/whitespace reasoning with no authoritative count
|
||||
// carries nothing to show, so skip it (avoids an empty 0-token block).
|
||||
const text = part.text ?? "";
|
||||
if (!text.trim() && !(reasoningTokens && reasoningTokens > 0))
|
||||
return null;
|
||||
// Absent state (persisted rows) and "done" both mean finalized.
|
||||
// `messageSignature` already includes each part's `state`, so the
|
||||
// streaming→done flip changes the row signature and re-renders this
|
||||
// row — which is what lets ReasoningBlock switch from chunked plain
|
||||
// text to its one-time markdown parse (see reasoning-block.tsx).
|
||||
// ALSO require the turn to be live: a part stranded at
|
||||
// `state:"streaming"` after the turn ended (no `reasoning-end` — see
|
||||
// the `turnStreaming` prop doc) must still finalize and parse.
|
||||
const streaming = turnStreaming && part.state === "streaming";
|
||||
return (
|
||||
<ReasoningBlock
|
||||
key={index}
|
||||
text={text}
|
||||
tokens={reasoningTokens}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
case "text": {
|
||||
// Skip empty/whitespace-only text parts (a streaming message often
|
||||
// starts with an empty text part before the first token arrives); the
|
||||
// typing indicator covers that gap until real content streams in.
|
||||
if (!part.text.trim()) return null;
|
||||
// The live, actively-growing tail part of the in-flight turn renders
|
||||
// incrementally (see MarkdownPart); a finalized part (persisted, or
|
||||
// the turn-end flip) renders the whole text through one canonical
|
||||
// pass. Same liveness rule as the reasoning branch above.
|
||||
const streaming = turnStreaming && part.state === "streaming";
|
||||
return (
|
||||
<MarkdownPart
|
||||
key={index}
|
||||
text={part.text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
case "source-url":
|
||||
case "source-document":
|
||||
case "file":
|
||||
case "step-start":
|
||||
// Not surfaced in the chat bubble (v1) — same as the pre-#492 default.
|
||||
return null;
|
||||
|
||||
default: {
|
||||
// Compile-time exhaustiveness over the CLOSED union members: every
|
||||
// literal-typed part kind is handled above, so the only kinds that
|
||||
// can reach here are the OPEN template-literal ones (`tool-*` — caught
|
||||
// by the guard at runtime — and `data-*`) plus `dynamic-tool`. Adding
|
||||
// a NEW closed part kind to UIMessagePart makes this assignment fail
|
||||
// to compile, forcing it to be handled instead of silently ignored
|
||||
// (this replaces the pre-#492 fall-through `return null` + WARNING).
|
||||
const _exhaustive:
|
||||
| `tool-${string}`
|
||||
| "dynamic-tool"
|
||||
| `data-${string}` = part.type;
|
||||
void _exhaustive;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
})}
|
||||
{/* A persisted turn error (server stored it in metadata.error). Rendered
|
||||
here so it survives a thread remount and shows in reopened history. */}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import { memo, useMemo } from "react";
|
||||
import { splitPlainChunks } from "@/features/ai-chat/components/streaming-plain-text.tsx";
|
||||
import { renderChatMarkdown } from "@/features/ai-chat/utils/markdown.ts";
|
||||
import classes from "@/features/ai-chat/components/ai-chat.module.css";
|
||||
|
||||
/**
|
||||
* One STABILIZED markdown block, rendered through the canonical pipeline and
|
||||
* memoized on its string prop. During streaming only the TAIL chunk grows (the
|
||||
* `splitPlainChunks` append-only invariant guarantees every earlier chunk is
|
||||
* byte-identical across deltas), so React skips every stable block and each one
|
||||
* is parsed by `renderChatMarkdown` EXACTLY ONCE — turning the pre-#492
|
||||
* "re-parse the whole accumulated answer on every ~20Hz tick" (O(ticks)) into
|
||||
* O(number of blocks). The markup is DOMPurify-sanitized inside renderChatMarkdown
|
||||
* before it reaches `dangerouslySetInnerHTML`.
|
||||
*
|
||||
* NOTE (transient streaming-only artifact): a safe cut is a blank-line boundary,
|
||||
* so a construct that legitimately contains a blank line (e.g. a fenced code block
|
||||
* with an empty line) can be split across chunks and render oddly WHILE it is still
|
||||
* streaming. This is cosmetic and self-heals: the moment the part finalizes,
|
||||
* MarkdownPart renders the WHOLE text through one canonical pass (visual parity
|
||||
* with the pre-#492 output). The reasoning path makes the same trade (plain text
|
||||
* while streaming, one markdown parse at the end).
|
||||
*/
|
||||
const MarkdownChunk = memo(function MarkdownChunk({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
}) {
|
||||
const html = renderChatMarkdown(text, { neutralizeInternalLinks });
|
||||
if (html) {
|
||||
return (
|
||||
<div
|
||||
className={classes.markdown}
|
||||
// Sanitized by renderChatMarkdown (DOMPurify) before insertion.
|
||||
dangerouslySetInnerHTML={{ __html: html }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
// Malformed/unsupported markdown could not render synchronously: raw text.
|
||||
return (
|
||||
<div className={classes.markdown} style={{ whiteSpace: "pre-wrap" }}>
|
||||
{text}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* The cheap streaming-time stand-in for the finalized answer's one-time markdown
|
||||
* parse (see MarkdownPart in message-item.tsx). Mirrors StreamingPlainText's
|
||||
* chunked-memo pattern but renders the STABILIZED prefix as real markdown (each
|
||||
* block parsed once, memoized) and only the LIVE tail as flat plain text — so the
|
||||
* user sees formatted output for everything up to the last safe cut, and the not-
|
||||
* yet-stable tail (which markdown-parsing every tick would make O(ticks)) stays a
|
||||
* single cheap escaped text node until it stabilizes into a new block.
|
||||
*
|
||||
* `splitPlainChunks` yields chunks where, under append-only growth, every chunk
|
||||
* except the LAST is immutable; the last chunk is the live tail. Index keys are
|
||||
* therefore stable (a given index never changes to a different chunk's content).
|
||||
*/
|
||||
export function StreamingMarkdownText({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
}) {
|
||||
const chunks = useMemo(() => splitPlainChunks(text), [text]);
|
||||
return (
|
||||
<>
|
||||
{chunks.map((chunk, index) =>
|
||||
index < chunks.length - 1 ? (
|
||||
<MarkdownChunk
|
||||
key={index}
|
||||
text={chunk}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
) : (
|
||||
// The live tail: flat, React-escaped plain text (no markdown parse, no
|
||||
// sanitizer, no innerHTML). `pre-wrap` preserves its newlines; trailing
|
||||
// separator newlines are dropped at display time so the block gap comes
|
||||
// from the markdown margins, not a doubled empty line (mirrors
|
||||
// PlainChunk in streaming-plain-text.tsx).
|
||||
<div
|
||||
key={index}
|
||||
className={classes.markdown}
|
||||
style={{ whiteSpace: "pre-wrap" }}
|
||||
>
|
||||
{chunk.replace(/\n+$/, "")}
|
||||
</div>
|
||||
),
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -3,11 +3,20 @@ import { atom } from "jotai";
|
||||
// import would drag the whole @tiptap/core engine into the eager graph of every
|
||||
// shell component that reads one of these atoms.
|
||||
import type { Editor } from "@tiptap/core";
|
||||
import type { HocuspocusProvider } from "@hocuspocus/provider";
|
||||
import { PageEditMode } from "@/features/user/types/user.types.ts";
|
||||
import type { DictationUnavailableReason } from "@/features/dictation/dictation-status";
|
||||
|
||||
export const pageEditorAtom = atom<Editor | null>(null);
|
||||
|
||||
// #370 — the active page's collab provider, published by the page editor so the
|
||||
// header menu can emit the "save-version" stateless signal (Cmd+S / button).
|
||||
// Null when the page is read-only / collab isn't connected. A typed initial
|
||||
// value (rather than an explicit generic) keeps jotai's overload resolution on
|
||||
// the writable PrimitiveAtom branch.
|
||||
const initialCollabProvider: HocuspocusProvider | null = null;
|
||||
export const collabProviderAtom = atom(initialCollabProvider);
|
||||
|
||||
export const titleEditorAtom = atom<Editor | null>(null);
|
||||
|
||||
export const readOnlyEditorAtom = atom<Editor | null>(null);
|
||||
|
||||
@@ -31,11 +31,18 @@ import { useAtom, useAtomValue, useSetAtom } from "jotai";
|
||||
import useCollaborationUrl from "@/features/editor/hooks/use-collaboration-url";
|
||||
import { currentUserAtom } from "@/features/user/atoms/current-user-atom";
|
||||
import {
|
||||
collabProviderAtom,
|
||||
currentPageEditModeAtom,
|
||||
dictationAvailabilityAtom,
|
||||
pageEditorAtom,
|
||||
yjsConnectionStatusAtom,
|
||||
} from "@/features/editor/atoms/editor-atoms";
|
||||
import { notifications } from "@mantine/notifications";
|
||||
import {
|
||||
VERSION_SAVED_MESSAGE_TYPE,
|
||||
type VersionSavedMessage,
|
||||
saveVersionPending,
|
||||
} from "@/features/page-history/version-messages";
|
||||
import { asideStateAtom } from "@/components/layouts/global/hooks/atoms/sidebar-atom";
|
||||
import {
|
||||
activeCommentIdAtom,
|
||||
@@ -124,6 +131,7 @@ export default function PageEditor({
|
||||
|
||||
const [currentUser] = useAtom(currentUserAtom);
|
||||
const [, setEditor] = useAtom(pageEditorAtom);
|
||||
const setCollabProvider = useSetAtom(collabProviderAtom);
|
||||
const [, setAsideState] = useAtom(asideStateAtom);
|
||||
const [, setActiveCommentId] = useAtom(activeCommentIdAtom);
|
||||
const [showCommentPopup, setShowCommentPopup] = useAtom(showCommentPopupAtom);
|
||||
@@ -181,6 +189,24 @@ export default function PageEditor({
|
||||
const onStatelessHandler = ({ payload }: onStatelessParameters) => {
|
||||
try {
|
||||
const message = JSON.parse(payload);
|
||||
// #370 — a version was saved somewhere; live-refresh the history panel
|
||||
// on every client. Only the client that pressed Save (tracked by the
|
||||
// module-level flag) shows the confirmation toast.
|
||||
if (message?.type === VERSION_SAVED_MESSAGE_TYPE) {
|
||||
const versionMsg = message as VersionSavedMessage;
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ["page-history-list"],
|
||||
});
|
||||
if (saveVersionPending.current) {
|
||||
saveVersionPending.current = false;
|
||||
notifications.show({
|
||||
message: versionMsg.alreadySaved
|
||||
? t("Already saved as the latest version")
|
||||
: t("Version saved"),
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (message?.type !== "page.updated" || !message.updatedAt) return;
|
||||
const pageData = queryClient.getQueryData<IPage>(["pages", slugId]);
|
||||
if (pageData) {
|
||||
@@ -238,12 +264,16 @@ export default function PageEditor({
|
||||
|
||||
local.on("synced", onLocalSyncedHandler);
|
||||
providersRef.current = { socket, local, remote };
|
||||
// #370 — publish the provider so the header menu can emit save-version.
|
||||
setCollabProvider(remote);
|
||||
setProvidersReady(true);
|
||||
} else {
|
||||
setCollabProvider(providersRef.current.remote);
|
||||
setProvidersReady(true);
|
||||
}
|
||||
// Only destroy on final unmount
|
||||
return () => {
|
||||
setCollabProvider(null);
|
||||
providersRef.current?.socket.destroy();
|
||||
providersRef.current?.remote.destroy();
|
||||
providersRef.current?.local.destroy();
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
import { Text, Group, UnstyledButton, Avatar, Tooltip } from "@mantine/core";
|
||||
import {
|
||||
Text,
|
||||
Group,
|
||||
UnstyledButton,
|
||||
Avatar,
|
||||
Tooltip,
|
||||
Badge,
|
||||
} from "@mantine/core";
|
||||
import { CustomAvatar } from "@/components/ui/custom-avatar.tsx";
|
||||
import { AgentAvatarStack } from "@/components/ui/agent-avatar-stack.tsx";
|
||||
import { formattedDate } from "@/lib/time";
|
||||
@@ -7,36 +14,59 @@ import clsx from "clsx";
|
||||
import { IPageHistory } from "@/features/page-history/types/page.types";
|
||||
import { memo, useCallback } from "react";
|
||||
import { useSetAtom } from "jotai";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { historyAtoms } from "@/features/page-history/atoms/history-atoms.ts";
|
||||
|
||||
const MAX_VISIBLE_AVATARS = 5;
|
||||
|
||||
/**
|
||||
* #370 — map a snapshot's intentionality tier to its badge. `version: true`
|
||||
* marks the intentional points (manual / agent); autosaves (boundary / idle /
|
||||
* legacy null) are non-versions and get dimmed in the list.
|
||||
*/
|
||||
type HistoryKindMeta = { labelKey: string; color: string; version: boolean };
|
||||
export function historyKindMeta(kind?: string | null): HistoryKindMeta {
|
||||
switch (kind) {
|
||||
case "manual":
|
||||
return { labelKey: "Saved", color: "blue", version: true };
|
||||
case "agent":
|
||||
return { labelKey: "Agent version", color: "violet", version: true };
|
||||
case "boundary":
|
||||
return { labelKey: "Boundary", color: "gray", version: false };
|
||||
default: // "idle" | null | undefined (legacy autosave)
|
||||
return { labelKey: "Autosave", color: "gray", version: false };
|
||||
}
|
||||
}
|
||||
|
||||
interface HistoryItemProps {
|
||||
historyItem: IPageHistory;
|
||||
index: number;
|
||||
onSelect: (id: string, index: number) => void;
|
||||
onHover?: (id: string, index: number) => void;
|
||||
// The previous snapshot for diff/restore is resolved by id from the FULL list
|
||||
// in the parent (resolvePrevSnapshotId), so the item only needs to report its
|
||||
// own id — never a list index (which would be the filtered-view index).
|
||||
onSelect: (id: string) => void;
|
||||
onHover?: (id: string) => void;
|
||||
onHoverEnd?: () => void;
|
||||
isActive: boolean;
|
||||
}
|
||||
|
||||
const HistoryItem = memo(function HistoryItem({
|
||||
historyItem,
|
||||
index,
|
||||
onSelect,
|
||||
onHover,
|
||||
onHoverEnd,
|
||||
isActive,
|
||||
}: HistoryItemProps) {
|
||||
const setHistoryModalOpen = useSetAtom(historyAtoms);
|
||||
const { t } = useTranslation();
|
||||
const kindMeta = historyKindMeta(historyItem.kind);
|
||||
|
||||
const handleClick = useCallback(() => {
|
||||
onSelect(historyItem.id, index);
|
||||
}, [onSelect, historyItem.id, index]);
|
||||
onSelect(historyItem.id);
|
||||
}, [onSelect, historyItem.id]);
|
||||
|
||||
const handleMouseEnter = useCallback(() => {
|
||||
onHover?.(historyItem.id, index);
|
||||
}, [onHover, historyItem.id, index]);
|
||||
onHover?.(historyItem.id);
|
||||
}, [onHover, historyItem.id]);
|
||||
|
||||
const contributors = historyItem.contributors;
|
||||
const hasContributors = contributors && contributors.length > 0;
|
||||
@@ -49,8 +79,20 @@ const HistoryItem = memo(function HistoryItem({
|
||||
onMouseEnter={handleMouseEnter}
|
||||
onMouseLeave={onHoverEnd}
|
||||
className={clsx(classes.history, { [classes.active]: isActive })}
|
||||
// #370 — dim autosnapshots so intentional versions stand out.
|
||||
style={{ opacity: kindMeta.version ? 1 : 0.55 }}
|
||||
>
|
||||
<Text size="sm">{formattedDate(new Date(historyItem.createdAt))}</Text>
|
||||
<Group gap={6} wrap="nowrap" justify="space-between">
|
||||
<Text size="sm">{formattedDate(new Date(historyItem.createdAt))}</Text>
|
||||
<Badge
|
||||
size="xs"
|
||||
radius="sm"
|
||||
variant={kindMeta.version ? "filled" : "light"}
|
||||
color={kindMeta.color}
|
||||
>
|
||||
{t(kindMeta.labelKey)}
|
||||
</Badge>
|
||||
</Group>
|
||||
|
||||
<Group gap={6} wrap="nowrap" mt={4}>
|
||||
{hasContributors ? (
|
||||
|
||||
@@ -2,14 +2,16 @@ import {
|
||||
usePageHistoryListQuery,
|
||||
prefetchPageHistory,
|
||||
} from "@/features/page-history/queries/page-history-query";
|
||||
import HistoryItem from "@/features/page-history/components/history-item";
|
||||
import HistoryItem, {
|
||||
historyKindMeta,
|
||||
} from "@/features/page-history/components/history-item";
|
||||
import {
|
||||
activeHistoryIdAtom,
|
||||
activeHistoryPrevIdAtom,
|
||||
historyAtoms,
|
||||
} from "@/features/page-history/atoms/history-atoms";
|
||||
import { useAtom, useSetAtom } from "jotai";
|
||||
import { useCallback, useEffect, useMemo, useRef } from "react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import {
|
||||
Button,
|
||||
ScrollArea,
|
||||
@@ -17,9 +19,12 @@ import {
|
||||
Divider,
|
||||
Loader,
|
||||
Center,
|
||||
Switch,
|
||||
Text,
|
||||
} from "@mantine/core";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { useHistoryRestore } from "@/features/page-history/hooks";
|
||||
import { resolvePrevSnapshotId } from "@/features/page-history/utils/resolve-prev-snapshot";
|
||||
|
||||
const PREFETCH_DELAY_MS = 150;
|
||||
|
||||
@@ -47,6 +52,22 @@ function HistoryList({ pageId }: Props) {
|
||||
[pageHistoryData],
|
||||
);
|
||||
|
||||
// #370 — "only versions" filter: hide autosnapshots (idle/boundary/legacy
|
||||
// null), keep only intentional points (manual/agent). Filtering is over the
|
||||
// already-loaded pages; the diff/restore still targets the true previous
|
||||
// snapshot, so items carry their index within the FULL list.
|
||||
const [onlyVersions, setOnlyVersions] = useState(false);
|
||||
// Reuse historyKindMeta().version — the SAME predicate the badge (HistoryItem)
|
||||
// uses to mark intentional points — so the "Only versions" filter and the badge
|
||||
// can never drift apart when a future intentional kind is added.
|
||||
const visibleItems = useMemo(
|
||||
() =>
|
||||
onlyVersions
|
||||
? historyItems.filter((item) => historyKindMeta(item.kind).version)
|
||||
: historyItems,
|
||||
[historyItems, onlyVersions],
|
||||
);
|
||||
|
||||
const loadMoreRef = useRef<HTMLDivElement>(null);
|
||||
const prefetchTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
@@ -60,11 +81,13 @@ function HistoryList({ pageId }: Props) {
|
||||
}, []);
|
||||
|
||||
const handleHover = useCallback(
|
||||
(historyId: string, index: number) => {
|
||||
(historyId: string) => {
|
||||
clearPrefetchTimeout();
|
||||
prefetchTimeoutRef.current = setTimeout(() => {
|
||||
prefetchPageHistory(historyId);
|
||||
const prevId = historyItems[index + 1]?.id;
|
||||
// The true previous snapshot in the FULL list (not the previous visible
|
||||
// one under the "only versions" filter).
|
||||
const prevId = resolvePrevSnapshotId(historyItems, historyId);
|
||||
if (prevId) {
|
||||
prefetchPageHistory(prevId);
|
||||
}
|
||||
@@ -78,9 +101,11 @@ function HistoryList({ pageId }: Props) {
|
||||
}, [clearPrefetchTimeout]);
|
||||
|
||||
const handleSelect = useCallback(
|
||||
(id: string, index: number) => {
|
||||
(id: string) => {
|
||||
setActiveHistoryId(id);
|
||||
setActiveHistoryPrevId(historyItems[index + 1]?.id ?? "");
|
||||
// Baseline = true previous snapshot in the FULL list, so the "only
|
||||
// versions" filter never diffs/restores against the wrong item.
|
||||
setActiveHistoryPrevId(resolvePrevSnapshotId(historyItems, id));
|
||||
},
|
||||
[historyItems, setActiveHistoryId, setActiveHistoryPrevId],
|
||||
);
|
||||
@@ -128,12 +153,27 @@ function HistoryList({ pageId }: Props) {
|
||||
|
||||
return (
|
||||
<div>
|
||||
<Group px="xs" py={6} justify="flex-end">
|
||||
<Switch
|
||||
size="xs"
|
||||
checked={onlyVersions}
|
||||
onChange={(e) => setOnlyVersions(e.currentTarget.checked)}
|
||||
label={t("Only versions")}
|
||||
/>
|
||||
</Group>
|
||||
|
||||
<ScrollArea h={620} w="100%" type="scroll" scrollbarSize={5}>
|
||||
{historyItems.map((historyItem, index) => (
|
||||
{onlyVersions && visibleItems.length === 0 && (
|
||||
<Center py="md">
|
||||
<Text size="sm" c="dimmed">
|
||||
{t("No saved versions yet.")}
|
||||
</Text>
|
||||
</Center>
|
||||
)}
|
||||
{visibleItems.map((historyItem) => (
|
||||
<HistoryItem
|
||||
key={historyItem.id}
|
||||
historyItem={historyItem}
|
||||
index={index}
|
||||
onSelect={handleSelect}
|
||||
onHover={handleHover}
|
||||
onHoverEnd={clearPrefetchTimeout}
|
||||
|
||||
@@ -24,6 +24,10 @@ export interface IPageHistory {
|
||||
updatedAt: string;
|
||||
lastUpdatedBy: IPageHistoryUser;
|
||||
contributors?: IPageHistoryUser[];
|
||||
// #370 — intentionality tier: 'manual'/'agent' are versions (intentional
|
||||
// points), 'idle'/'boundary' are autosnapshots; null/undefined = legacy
|
||||
// autosave. Derived server-side, drives the history badge + "versions" filter.
|
||||
kind?: "manual" | "agent" | "idle" | "boundary" | null;
|
||||
// Provenance markers copied off the page row when the snapshot was saved.
|
||||
// `'agent'` marks a version written by the AI agent; `lastUpdatedAiChatId`
|
||||
// (when present) deep-links to the chat that produced the edit.
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { resolvePrevSnapshotId } from "./resolve-prev-snapshot";
|
||||
|
||||
// #370 F4 — the risky client path: with the "only versions" filter active, diff
|
||||
// and restore must still baseline against the TRUE previous snapshot in the FULL
|
||||
// list, never the previous VISIBLE version (which would skip the autosnapshots
|
||||
// between two versions). These pin that the resolution is by FULL-list order.
|
||||
describe("resolvePrevSnapshotId", () => {
|
||||
// Newest-first, as the history list stores it: a version, then two autosaves,
|
||||
// then an older version.
|
||||
const full = [
|
||||
{ id: "v2", kind: "manual" },
|
||||
{ id: "a2", kind: "idle" },
|
||||
{ id: "a1", kind: "boundary" },
|
||||
{ id: "v1", kind: "manual" },
|
||||
{ id: "a0", kind: null },
|
||||
];
|
||||
|
||||
it("returns the immediate FULL-list successor, not the previous visible version", () => {
|
||||
// Selecting v2 while filtered to versions-only must baseline against a2 (the
|
||||
// real chronological predecessor), NOT v1 (the previous visible version).
|
||||
expect(resolvePrevSnapshotId(full, "v2")).toBe("a2");
|
||||
});
|
||||
|
||||
it("resolves an autosnapshot's predecessor by full-list order", () => {
|
||||
expect(resolvePrevSnapshotId(full, "a1")).toBe("v1");
|
||||
});
|
||||
|
||||
it("returns '' for the oldest item (no predecessor)", () => {
|
||||
expect(resolvePrevSnapshotId(full, "a0")).toBe("");
|
||||
});
|
||||
|
||||
it("returns '' for an id not in the list", () => {
|
||||
expect(resolvePrevSnapshotId(full, "missing")).toBe("");
|
||||
});
|
||||
|
||||
it("does not depend on a filtered subset — same result whatever is visible", () => {
|
||||
// The helper only ever sees the full list; a filtered view cannot change the
|
||||
// baseline it computes.
|
||||
expect(resolvePrevSnapshotId(full, "v1")).toBe("a0");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* #370 — resolve the TRUE previous snapshot for a history item.
|
||||
*
|
||||
* The history panel can be filtered to "only versions" (manual/agent), but diff
|
||||
* and restore must always compare against the immediately-preceding snapshot in
|
||||
* the FULL, unfiltered list — NOT the previous VISIBLE item. Comparing against
|
||||
* the previous visible version would silently skip the autosnapshots between two
|
||||
* versions and diff/restore the wrong baseline.
|
||||
*
|
||||
* Given the full (newest-first) list and an item id, this returns the id of the
|
||||
* item right after it in the full list (its chronological predecessor), or "" if
|
||||
* it is the oldest / not found. Pure and list-order-preserving so it can be unit
|
||||
* tested without mounting the component.
|
||||
*/
|
||||
export function resolvePrevSnapshotId(
|
||||
fullItems: ReadonlyArray<{ id: string }>,
|
||||
id: string,
|
||||
): string {
|
||||
const index = fullItems.findIndex((item) => item.id === id);
|
||||
if (index === -1) return "";
|
||||
return fullItems[index + 1]?.id ?? "";
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* #370 — page-version stateless wire formats. Kept in one place so the client
|
||||
* emitter (Save hotkey / button) and the client listener (page-editor) agree
|
||||
* with the server (PersistenceExtension) on the message shapes.
|
||||
*/
|
||||
|
||||
/** Client → server: "save a version now". The server derives the tier
|
||||
* (manual/agent) from the signed connection actor, never from this payload. */
|
||||
export const SAVE_VERSION_MESSAGE_TYPE = "save-version";
|
||||
|
||||
/** Server → all clients: a version was saved (or promoted / already existed). */
|
||||
export const VERSION_SAVED_MESSAGE_TYPE = "version.saved";
|
||||
|
||||
export interface VersionSavedMessage {
|
||||
type: typeof VERSION_SAVED_MESSAGE_TYPE;
|
||||
historyId: string;
|
||||
kind: "manual" | "agent";
|
||||
/** True when the latest snapshot was already a manual version (a no-op save). */
|
||||
alreadySaved: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cross-component coordination flag so only the client that pressed Save shows
|
||||
* the confirmation toast, while every other client silently refreshes its
|
||||
* history panel on the broadcast. A module-level ref avoids stale-closure
|
||||
* pitfalls in the editor's long-lived stateless handler.
|
||||
*/
|
||||
export const saveVersionPending = { current: false };
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
IconArrowRight,
|
||||
IconArrowsHorizontal,
|
||||
IconClockHour4,
|
||||
IconDeviceFloppy,
|
||||
IconDots,
|
||||
IconEye,
|
||||
IconEyeOff,
|
||||
@@ -17,7 +18,7 @@ import {
|
||||
IconTrash,
|
||||
IconWifiOff,
|
||||
} from "@tabler/icons-react";
|
||||
import React, { useEffect, useRef, useState } from "react";
|
||||
import React, { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { useAsideTriggerProps } from "@/hooks/use-toggle-aside.tsx";
|
||||
import { useAtom, useAtomValue } from "jotai";
|
||||
import { historyAtoms } from "@/features/page-history/atoms/history-atoms.ts";
|
||||
@@ -39,9 +40,14 @@ import { Trans, useTranslation } from "react-i18next";
|
||||
import ExportModal from "@/components/common/export-modal";
|
||||
import { convertProseMirrorToMarkdown } from "@docmost/prosemirror-markdown/browser";
|
||||
import {
|
||||
collabProviderAtom,
|
||||
pageEditorAtom,
|
||||
yjsConnectionStatusAtom,
|
||||
} from "@/features/editor/atoms/editor-atoms.ts";
|
||||
import {
|
||||
SAVE_VERSION_MESSAGE_TYPE,
|
||||
saveVersionPending,
|
||||
} from "@/features/page-history/version-messages.ts";
|
||||
import { formattedDate } from "@/lib/time.ts";
|
||||
import { PageEditModeToggle } from "@/features/user/components/page-state-pref.tsx";
|
||||
import MovePageModal from "@/features/page/components/move-page-modal.tsx";
|
||||
@@ -72,9 +78,34 @@ export default function PageHeaderMenu({ readOnly }: PageHeaderMenuProps) {
|
||||
});
|
||||
const isDeleted = !!page?.deletedAt;
|
||||
const [workspace] = useAtom(workspaceAtom);
|
||||
const collabProvider = useAtomValue(collabProviderAtom);
|
||||
// Community public-sharing entry point (replaces the removed EE PageShareModal)
|
||||
const workspaceSharingDisabled = workspace?.settings?.sharing?.disabled === true;
|
||||
|
||||
// #370 — explicit "save a version" (Cmd+S / Save button). One path for the
|
||||
// human; the server derives the tier from the signed actor. Readers can't save
|
||||
// (the button is hidden and the collab connection is read-only server-side).
|
||||
const handleSaveVersion = useCallback(() => {
|
||||
if (readOnly || !collabProvider) return;
|
||||
// Flag this client as the initiator so only it shows the confirmation toast;
|
||||
// a safety timeout clears it if no broadcast comes back (e.g. offline).
|
||||
saveVersionPending.current = true;
|
||||
window.setTimeout(() => {
|
||||
saveVersionPending.current = false;
|
||||
}, 5000);
|
||||
collabProvider.sendStateless(
|
||||
JSON.stringify({ type: SAVE_VERSION_MESSAGE_TYPE }),
|
||||
);
|
||||
}, [readOnly, collabProvider]);
|
||||
|
||||
// mod+S must also block the browser's "Save page" dialog. `triggerOnContent-
|
||||
// Editable` + empty ignore-list so it fires while typing in the editor/title.
|
||||
useHotkeys(
|
||||
[["mod+S", handleSaveVersion, { preventDefault: true }]],
|
||||
[],
|
||||
true,
|
||||
);
|
||||
|
||||
useHotkeys(
|
||||
[
|
||||
[
|
||||
@@ -133,15 +164,16 @@ export default function PageHeaderMenu({ readOnly }: PageHeaderMenuProps) {
|
||||
</ActionIcon>
|
||||
</Tooltip>
|
||||
|
||||
<PageActionMenu readOnly={readOnly} />
|
||||
<PageActionMenu readOnly={readOnly} onSaveVersion={handleSaveVersion} />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
interface PageActionMenuProps {
|
||||
readOnly?: boolean;
|
||||
onSaveVersion?: () => void;
|
||||
}
|
||||
function PageActionMenu({ readOnly }: PageActionMenuProps) {
|
||||
function PageActionMenu({ readOnly, onSaveVersion }: PageActionMenuProps) {
|
||||
const { t } = useTranslation();
|
||||
const [, setHistoryModalOpen] = useAtom(historyAtoms);
|
||||
const clipboard = useClipboard({ timeout: 500 });
|
||||
@@ -303,6 +335,20 @@ function PageActionMenu({ readOnly }: PageActionMenuProps) {
|
||||
</Group>
|
||||
</Menu.Item>
|
||||
|
||||
{!readOnly && (
|
||||
<Menu.Item
|
||||
leftSection={<IconDeviceFloppy size={16} />}
|
||||
onClick={onSaveVersion}
|
||||
rightSection={
|
||||
<Text size="xs" c="dimmed">
|
||||
{t("Ctrl+S")}
|
||||
</Text>
|
||||
}
|
||||
>
|
||||
{t("Save version")}
|
||||
</Menu.Item>
|
||||
)}
|
||||
|
||||
<Menu.Item
|
||||
leftSection={<IconHistory size={16} />}
|
||||
onClick={openHistoryModal}
|
||||
|
||||
@@ -281,10 +281,12 @@ const SpaceTree = forwardRef<SpaceTreeApi, SpaceTreeProps>(function SpaceTree(
|
||||
setOpenTreeNodes((prev) => ({ ...prev, [id]: isOpen }));
|
||||
if (isOpen) {
|
||||
const node = treeModel.find(data, id) as SpaceTreeNode | null;
|
||||
if (
|
||||
node?.hasChildren &&
|
||||
(!node.children || node.children.length === 0)
|
||||
) {
|
||||
// Same "unloaded branch" predicate the realtime insert paths use
|
||||
// (`isUnloadedBranch`) so the lazy-load gate and the realtime inserts
|
||||
// (`insertByPosition` / `placeByPosition`) can never disagree about what
|
||||
// counts as unloaded (#525). Note: local raw `insert` (DnD/create-page)
|
||||
// does not yet route through it — see #525 follow-up.
|
||||
if (treeModel.isUnloadedBranch(node)) {
|
||||
const fetched = await fetchAllAncestorChildren({
|
||||
pageId: id,
|
||||
spaceId: node.spaceId,
|
||||
|
||||
@@ -74,6 +74,48 @@ describe("treeModel.isDescendant", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// #525: the single "is this branch unloaded?" predicate shared by the lazy-load
|
||||
// gate and the insert paths. Unloaded == server says hasChildren but none are
|
||||
// present locally (canonical form `children: []`, also `undefined`). A parent
|
||||
// without hasChildren is genuinely empty, not unloaded.
|
||||
describe("treeModel.isUnloadedBranch", () => {
|
||||
type PH = TreeNode<{ name: string; hasChildren?: boolean }>;
|
||||
it("true for hasChildren + empty array (canonical unloaded form)", () => {
|
||||
const n: PH = { id: "p", name: "P", hasChildren: true, children: [] };
|
||||
expect(treeModel.isUnloadedBranch(n)).toBe(true);
|
||||
});
|
||||
it("true for hasChildren + undefined children", () => {
|
||||
const n: PH = { id: "p", name: "P", hasChildren: true };
|
||||
expect(treeModel.isUnloadedBranch(n)).toBe(true);
|
||||
});
|
||||
it("false for hasChildren + already-loaded children", () => {
|
||||
const n: PH = {
|
||||
id: "p",
|
||||
name: "P",
|
||||
hasChildren: true,
|
||||
children: [{ id: "c", name: "C" }],
|
||||
};
|
||||
expect(treeModel.isUnloadedBranch(n)).toBe(false);
|
||||
});
|
||||
it("false for a genuinely-empty parent (no hasChildren)", () => {
|
||||
expect(
|
||||
treeModel.isUnloadedBranch({
|
||||
id: "p",
|
||||
name: "P",
|
||||
hasChildren: false,
|
||||
children: [],
|
||||
} as PH),
|
||||
).toBe(false);
|
||||
expect(
|
||||
treeModel.isUnloadedBranch({ id: "p", name: "P" } as PH),
|
||||
).toBe(false);
|
||||
});
|
||||
it("false for null/undefined", () => {
|
||||
expect(treeModel.isUnloadedBranch(null)).toBe(false);
|
||||
expect(treeModel.isUnloadedBranch(undefined)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("treeModel.visible", () => {
|
||||
it("returns only root nodes when no openIds", () => {
|
||||
const v = treeModel.visible(fixture, new Set());
|
||||
@@ -197,43 +239,64 @@ describe("treeModel.insertByPosition", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
// #159 #1: inserting/moving a node under a parent whose children are NOT
|
||||
// loaded (`children === undefined`, e.g. a collapsed page) must NOT materialize
|
||||
// a partial `[node]` list — that would defeat the lazy-load gate and hide the
|
||||
// parent's other real children. The node is left to be lazy-loaded; only
|
||||
// `hasChildren` is flagged so the chevron appears.
|
||||
it("does NOT materialize a child under an UNLOADED parent (children undefined)", () => {
|
||||
type PH = TreeNode<{
|
||||
name: string;
|
||||
position?: string;
|
||||
hasChildren?: boolean;
|
||||
}>;
|
||||
type PH = TreeNode<{
|
||||
name: string;
|
||||
position?: string;
|
||||
hasChildren?: boolean;
|
||||
}>;
|
||||
|
||||
// #159 #1 / #525: inserting/moving a node under an UNLOADED parent must NOT
|
||||
// materialize a partial `[node]` list — that would defeat the lazy-load gate and
|
||||
// hide the parent's other real children. The canonical unloaded form here is
|
||||
// `children: []` + `hasChildren: true` (from `pageToTreeNode` /
|
||||
// `pruneCollapsedChildren`), which the pre-#525 `=== undefined` guard MISSED.
|
||||
// The node is left to be lazy-loaded; the chevron stays enabled.
|
||||
it("does NOT materialize a child under an UNLOADED parent (children: [], hasChildren: true)", () => {
|
||||
const tree: PH[] = [
|
||||
{ id: "p", name: "P", position: "a0", hasChildren: false }, // children: undefined
|
||||
{ id: "p", name: "P", position: "a0", hasChildren: true, children: [] },
|
||||
];
|
||||
const node: PH = { id: "x", name: "X", position: "a1" };
|
||||
const t = treeModel.insertByPosition(tree, "p", node);
|
||||
const parent = treeModel.find(t, "p");
|
||||
// The node was NOT inserted (children stay unloaded -> lazy-load fetches the
|
||||
// full set, including this node, on expand).
|
||||
expect(parent?.children).toBeUndefined();
|
||||
// full set, including this node, on expand). MUTATION: the pre-#525 predicate
|
||||
// `children === undefined` does not fire for `[]`, so it would insert `[x]`
|
||||
// here and reredden this expectation.
|
||||
expect(parent?.children).toEqual([]);
|
||||
expect(treeModel.find(t, "x")).toBeNull();
|
||||
// ...but the chevron is enabled so the user can expand to load it.
|
||||
// ...and the chevron stays enabled so the user can expand to load it.
|
||||
expect((parent as PH).hasChildren).toBe(true);
|
||||
});
|
||||
|
||||
it("DOES insert under a LOADED-but-empty parent (children: [])", () => {
|
||||
type PH = TreeNode<{
|
||||
name: string;
|
||||
position?: string;
|
||||
hasChildren?: boolean;
|
||||
}>;
|
||||
it("does NOT materialize a child under an UNLOADED parent (children undefined, hasChildren: true)", () => {
|
||||
const tree: PH[] = [
|
||||
{ id: "p", name: "P", position: "a0", hasChildren: true }, // children: undefined
|
||||
];
|
||||
const node: PH = { id: "x", name: "X", position: "a1" };
|
||||
const t = treeModel.insertByPosition(tree, "p", node);
|
||||
const parent = treeModel.find(t, "p");
|
||||
expect(parent?.children).toBeUndefined();
|
||||
expect(treeModel.find(t, "x")).toBeNull();
|
||||
expect((parent as PH).hasChildren).toBe(true);
|
||||
});
|
||||
|
||||
it("DOES insert under a genuinely-empty parent (children: [], hasChildren: false)", () => {
|
||||
const tree: PH[] = [
|
||||
{ id: "p", name: "P", position: "a0", hasChildren: false, children: [] },
|
||||
];
|
||||
const node: PH = { id: "x", name: "X", position: "a1" };
|
||||
const t = treeModel.insertByPosition(tree, "p", node);
|
||||
// A loaded (empty) child list is complete, so the node IS inserted.
|
||||
// No server children (`hasChildren: false`), so materializing the first child
|
||||
// is correct — nothing is hidden.
|
||||
expect(treeModel.find(t, "p")?.children?.map((n) => n.id)).toEqual(["x"]);
|
||||
});
|
||||
|
||||
it("DOES insert under a genuinely-empty parent (children undefined, hasChildren: false)", () => {
|
||||
const tree: PH[] = [
|
||||
{ id: "p", name: "P", position: "a0", hasChildren: false }, // children: undefined
|
||||
];
|
||||
const node: PH = { id: "x", name: "X", position: "a1" };
|
||||
const t = treeModel.insertByPosition(tree, "p", node);
|
||||
expect(treeModel.find(t, "p")?.children?.map((n) => n.id)).toEqual(["x"]);
|
||||
});
|
||||
|
||||
|
||||
@@ -43,6 +43,26 @@ export const treeModel = {
|
||||
};
|
||||
},
|
||||
|
||||
// A branch is "unloaded" when the server says it HAS children (`hasChildren`)
|
||||
// but none are present locally. The canonical unloaded form in this codebase
|
||||
// is `children: []` (produced by `pageToTreeNode` and by `pruneCollapsedChildren`
|
||||
// resetting collapsed branches), NOT `children: undefined` — so a predicate that
|
||||
// only checks `=== undefined` misses the real case and materializes a misleading
|
||||
// partial list (#525). This is the SINGLE source of truth for "should a
|
||||
// fetch/materialize be deferred?", shared by the lazy-load gate (`handleToggle`)
|
||||
// and the realtime insert paths (`insertByPosition` / `placeByPosition`), so they
|
||||
// can never drift apart again. (The local raw `insert` primitive and its DnD/
|
||||
// create-page callers do NOT yet route through this predicate — see #525
|
||||
// follow-up.) A parent WITHOUT `hasChildren` is genuinely empty
|
||||
// (no server children) — inserting its first child is correct, not deferred.
|
||||
isUnloadedBranch<T extends object>(
|
||||
node: TreeNode<T> | null | undefined,
|
||||
): boolean {
|
||||
if (!node) return false;
|
||||
const hasChildren = (node as { hasChildren?: boolean }).hasChildren === true;
|
||||
return hasChildren && (node.children == null || node.children.length === 0);
|
||||
},
|
||||
|
||||
isDescendant<T extends object>(
|
||||
tree: TreeNode<T>[],
|
||||
ancestorId: string,
|
||||
@@ -127,14 +147,15 @@ export const treeModel = {
|
||||
}
|
||||
const parent = treeModel.find(tree, parentId);
|
||||
// The parent is in the tree but its children have NOT been lazy-loaded yet
|
||||
// (`children === undefined`, distinct from a loaded-but-empty `[]`). Inserting
|
||||
// (`hasChildren` set + children absent/empty — see `isUnloadedBranch`; the
|
||||
// canonical unloaded form is `children: []`, NOT just `undefined`). Inserting
|
||||
// here would MATERIALIZE a misleading partial child list (`[node]`) that
|
||||
// defeats the lazy-load gate — which fetches only when children are
|
||||
// absent/empty — so the parent's OTHER real children would never load and the
|
||||
// moved/added node would be the only one shown (a silent data loss, #159 #1).
|
||||
// Instead, leave the children unloaded and just flag `hasChildren` so the
|
||||
// chevron appears; expanding fetches the FULL set (including this node).
|
||||
if (parent && parent.children === undefined) {
|
||||
if (parent && treeModel.isUnloadedBranch(parent)) {
|
||||
return treeModel.update(
|
||||
tree,
|
||||
parentId,
|
||||
|
||||
@@ -82,17 +82,19 @@ describe("applyMoveTreeNode", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("does NOT create a partial child list when the destination is loaded-but-collapsed (children unloaded) — keeps it lazy-loadable (#159)", () => {
|
||||
// `dstCollapsed` is in the tree but its children were never lazy-loaded
|
||||
// (children === undefined). The OLD behavior inserted `src` as the ONLY
|
||||
// child ([src]), which defeated the lazy-load gate and HID the parent's
|
||||
// other real children. Now the move leaves children unloaded (so expanding
|
||||
// fetches the FULL set, including src) and just flags hasChildren.
|
||||
it("does NOT create a partial child list when the destination is loaded-but-collapsed (children unloaded) — keeps it lazy-loadable (#159 #525)", () => {
|
||||
// `dstCollapsed` is in the tree but its children were never lazy-loaded. The
|
||||
// CANONICAL unloaded form here is `hasChildren: true` + `children: []` (from
|
||||
// `pageToTreeNode` / `pruneCollapsedChildren`), NOT `children: undefined`.
|
||||
// The pre-#525 predicate (`children === undefined`) missed this form and
|
||||
// inserted `src` as the ONLY child ([src]), defeating the lazy-load gate and
|
||||
// HIDING the parent's other real children. Now the move leaves children
|
||||
// unloaded (so expanding fetches the FULL set, including src).
|
||||
const tree: SpaceTreeNode[] = [
|
||||
node("dstCollapsed", {
|
||||
position: "a0",
|
||||
hasChildren: false,
|
||||
children: undefined as unknown as SpaceTreeNode[],
|
||||
hasChildren: true,
|
||||
children: [],
|
||||
}),
|
||||
node("src", { position: "a9" }),
|
||||
];
|
||||
@@ -105,9 +107,10 @@ describe("applyMoveTreeNode", () => {
|
||||
pageData: {},
|
||||
});
|
||||
const dst = treeModel.find(next, "dstCollapsed");
|
||||
// Children stay unloaded -> the lazy-load gate fetches the FULL set (incl.
|
||||
// src) on expand, rather than showing a misleading partial [src] list.
|
||||
expect(dst?.children).toBeUndefined();
|
||||
// Children stay unloaded ([] not materialized to [src]) -> the lazy-load gate
|
||||
// fetches the FULL set (incl. src) on expand. MUTATION: the pre-#525
|
||||
// `=== undefined` predicate would insert [src] here and redden this.
|
||||
expect(dst?.children).toEqual([]);
|
||||
expect(dst?.hasChildren).toBe(true);
|
||||
// src moved away from its old root slot (it lives under dstCollapsed
|
||||
// server-side and reappears when the parent is expanded/loaded).
|
||||
|
||||
@@ -1,10 +1,36 @@
|
||||
export const HISTORY_INTERVAL = 5 * 60 * 1000;
|
||||
export const HISTORY_FAST_INTERVAL = 60 * 1000;
|
||||
export const HISTORY_FAST_THRESHOLD = 5 * 60 * 1000;
|
||||
|
||||
// #348 — debounce window for the per-page RAG re-embed job. Repeated saves
|
||||
// within this window collapse to a single delayed job (coalesced by a stable
|
||||
// jobId), so active editing does not pile up expensive re-embeds (external API
|
||||
// + page_embeddings rewrite, concurrency 1). The worker reads the CURRENT page
|
||||
// state at run time, so the last content within the window wins.
|
||||
export const EMBED_DEBOUNCE_MS = 30 * 1000;
|
||||
|
||||
/**
|
||||
* #370 — page-history intentionality tiers. Domain of `page_history.kind`.
|
||||
* - 'manual' / 'agent' → Tier 1 versions (intentional points)
|
||||
* - 'idle' / 'boundary' → Tier 0 autosnapshots (safety net)
|
||||
* A legacy `null` kind is treated as an autosave.
|
||||
*/
|
||||
export type PageHistoryKind = 'manual' | 'agent' | 'idle' | 'boundary';
|
||||
|
||||
/**
|
||||
* #370 — trailing idle-flush windows. A page's pending idle snapshot is
|
||||
* re-armed on every store and fires this long after edits go quiet, so a burst
|
||||
* of edits collapses into a single autosnapshot instead of one-per-store. Human
|
||||
* sessions are noisier and less risky, so they flush less often than the agent.
|
||||
*/
|
||||
export const IDLE_INTERVAL_USER = 60 * 60 * 1000; // 60m
|
||||
export const IDLE_INTERVAL_AGENT = 15 * 60 * 1000; // 15m
|
||||
|
||||
/**
|
||||
* #370 — max-wait ceiling for the idle flush. Pure trailing debounce starves the
|
||||
* safety net: hocuspocus stores at least every ~45s, so a CONTINUOUS editing
|
||||
* session would re-arm the trailing timer forever and never take an idle
|
||||
* snapshot until edits finally go quiet (up to IDLE_INTERVAL_USER = 60m). This
|
||||
* ceiling bounds the actual wait from the FIRST edit of a burst, so an idle
|
||||
* snapshot fires at least this often during a long unbroken session — restoring
|
||||
* a recovery point cadence closer to the old heuristic without one-per-store
|
||||
* noise. Mirrors hocuspocus's own maxDebounce idea.
|
||||
*/
|
||||
export const IDLE_MAX_WAIT_USER = 10 * 60 * 1000; // 10m
|
||||
export const IDLE_MAX_WAIT_AGENT = 5 * 60 * 1000; // 5m
|
||||
|
||||
@@ -1,84 +1,93 @@
|
||||
import { computeHistoryJob, resolveSource } from './persistence.extension';
|
||||
import {
|
||||
computeHistoryJob,
|
||||
resolveSource,
|
||||
} from './persistence.extension';
|
||||
import {
|
||||
HISTORY_FAST_INTERVAL,
|
||||
HISTORY_FAST_THRESHOLD,
|
||||
HISTORY_INTERVAL,
|
||||
IDLE_INTERVAL_AGENT,
|
||||
IDLE_INTERVAL_USER,
|
||||
IDLE_MAX_WAIT_AGENT,
|
||||
IDLE_MAX_WAIT_USER,
|
||||
} from '../constants';
|
||||
|
||||
// A fixed clock + fixed createdAt make pageAge deterministic.
|
||||
const NOW = 1_700_000_000_000;
|
||||
const PAGE_ID = '550e8400-e29b-41d4-a716-446655440000';
|
||||
|
||||
// Build a minimal page whose age (NOW - createdAt) is exactly `ageMs`.
|
||||
const pageAged = (ageMs: number) => ({
|
||||
id: PAGE_ID,
|
||||
createdAt: new Date(NOW - ageMs),
|
||||
});
|
||||
const page = { id: PAGE_ID };
|
||||
|
||||
describe('computeHistoryJob', () => {
|
||||
it('agent edit → delay MUST be 0 and job id is source-keyed', () => {
|
||||
// INVARIANT (§15 H2 / persistence.extension): the agent delay MUST stay 0.
|
||||
// The worker re-reads the page row at run time, so any non-zero delay risks
|
||||
// snapshotting content a later human edit has already overwritten. This is
|
||||
// the load-bearing assertion of this spec — do not relax it.
|
||||
const { jobId, delay } = computeHistoryJob(pageAged(0), 'agent', NOW);
|
||||
expect(delay).toBe(0);
|
||||
expect(jobId).toBe(`${PAGE_ID}-agent`);
|
||||
});
|
||||
|
||||
it('agent edit on an OLD page is still delay 0 (age never applies to agents)', () => {
|
||||
// Even when the page is far older than the fast threshold, the agent path
|
||||
// must short-circuit to 0 — age-based debounce is a human-only concern.
|
||||
const { jobId, delay } = computeHistoryJob(
|
||||
pageAged(HISTORY_FAST_THRESHOLD + 60_000),
|
||||
'agent',
|
||||
NOW,
|
||||
);
|
||||
expect(delay).toBe(0);
|
||||
expect(jobId).toBe(`${PAGE_ID}-agent`);
|
||||
});
|
||||
|
||||
it('human edit on a YOUNG page (age < threshold) → fast interval, bare job id', () => {
|
||||
const { jobId, delay } = computeHistoryJob(
|
||||
pageAged(HISTORY_FAST_THRESHOLD - 1),
|
||||
'user',
|
||||
NOW,
|
||||
);
|
||||
expect(delay).toBe(HISTORY_FAST_INTERVAL);
|
||||
describe('computeHistoryJob (#370 — shared trailing idle pipeline)', () => {
|
||||
it('human edit → user idle window, bare page.id job', () => {
|
||||
// Humans and the agent now share ONE idle job per page (jobId = page.id).
|
||||
// The agent's old delay=0 fast path is GONE — intentional agent points now
|
||||
// arrive via the explicit save-version signal, not a zero-delay snapshot.
|
||||
const { jobId, delay } = computeHistoryJob(page, 'user');
|
||||
expect(delay).toBe(IDLE_INTERVAL_USER);
|
||||
expect(jobId).toBe(PAGE_ID);
|
||||
});
|
||||
|
||||
it('human edit on an OLD page (age > threshold) → standard interval', () => {
|
||||
const { jobId, delay } = computeHistoryJob(
|
||||
pageAged(HISTORY_FAST_THRESHOLD + 1),
|
||||
'user',
|
||||
NOW,
|
||||
);
|
||||
expect(delay).toBe(HISTORY_INTERVAL);
|
||||
it('agent edit → agent idle window (shorter), still the bare page.id job', () => {
|
||||
const { jobId, delay } = computeHistoryJob(page, 'agent');
|
||||
expect(delay).toBe(IDLE_INTERVAL_AGENT);
|
||||
// No `-agent` suffix anymore: the agent joins the common idle pipeline.
|
||||
expect(jobId).toBe(PAGE_ID);
|
||||
});
|
||||
|
||||
it('boundary: pageAge EXACTLY === threshold takes the slow branch (the `<` is strict)', () => {
|
||||
// Off-by-one guard: the condition is `pageAge < HISTORY_FAST_THRESHOLD`, so
|
||||
// an age of exactly the threshold is NOT "fast" — it must use HISTORY_INTERVAL.
|
||||
const { delay } = computeHistoryJob(
|
||||
pageAged(HISTORY_FAST_THRESHOLD),
|
||||
'user',
|
||||
NOW,
|
||||
);
|
||||
expect(delay).toBe(HISTORY_INTERVAL);
|
||||
it('agent flushes sooner than a human', () => {
|
||||
expect(IDLE_INTERVAL_AGENT).toBeLessThan(IDLE_INTERVAL_USER);
|
||||
});
|
||||
|
||||
it('treats any non-"agent" source string as human', () => {
|
||||
// resolveSource only ever yields 'agent' | 'user', but guard the contract:
|
||||
// the agent branch keys strictly on === 'agent'.
|
||||
const { jobId, delay } = computeHistoryJob(pageAged(0), 'user', NOW);
|
||||
expect(delay).toBe(HISTORY_FAST_INTERVAL);
|
||||
it('treats any non-"agent" source string as human (keys strictly on === agent)', () => {
|
||||
const { jobId, delay } = computeHistoryJob(page, 'user');
|
||||
expect(delay).toBe(IDLE_INTERVAL_USER);
|
||||
expect(jobId).toBe(PAGE_ID);
|
||||
});
|
||||
|
||||
// #370 review round-1 WARNING: the max-wait ceiling prevents autosnapshot
|
||||
// starvation during a continuous editing session (the trailing timer would
|
||||
// otherwise re-arm forever and never fire).
|
||||
describe('max-wait ceiling', () => {
|
||||
const T0 = 1_000_000; // arbitrary fixed epoch for deterministic tests
|
||||
|
||||
it('once a burst is armed, delay clamps to the remaining max-wait budget', () => {
|
||||
// 1 minute into the burst the USER interval (60m) far exceeds the remaining
|
||||
// max-wait budget (10m - 1m = 9m), so the delay is clamped DOWN to that
|
||||
// remaining budget — the full interval is NOT used once a ceiling applies.
|
||||
const { delay } = computeHistoryJob(page, 'user', T0, T0 + 60_000);
|
||||
expect(delay).toBe(IDLE_MAX_WAIT_USER - 60_000);
|
||||
});
|
||||
|
||||
it('never waits longer than the max-wait budget from the burst start', () => {
|
||||
// A store arriving right at the ceiling → delay 0 (fire promptly).
|
||||
const { delay } = computeHistoryJob(
|
||||
page,
|
||||
'user',
|
||||
T0,
|
||||
T0 + IDLE_MAX_WAIT_USER,
|
||||
);
|
||||
expect(delay).toBe(0);
|
||||
});
|
||||
|
||||
it('past the ceiling never returns a negative delay', () => {
|
||||
const { delay } = computeHistoryJob(
|
||||
page,
|
||||
'user',
|
||||
T0,
|
||||
T0 + IDLE_MAX_WAIT_USER + 5 * 60_000,
|
||||
);
|
||||
expect(delay).toBe(0);
|
||||
});
|
||||
|
||||
it('the agent ceiling is shorter than the user ceiling', () => {
|
||||
expect(IDLE_MAX_WAIT_AGENT).toBeLessThan(IDLE_MAX_WAIT_USER);
|
||||
const { delay } = computeHistoryJob(
|
||||
page,
|
||||
'agent',
|
||||
T0,
|
||||
T0 + IDLE_MAX_WAIT_AGENT,
|
||||
);
|
||||
expect(delay).toBe(0);
|
||||
});
|
||||
|
||||
it('without a burstStart there is no ceiling (backward-compatible)', () => {
|
||||
expect(computeHistoryJob(page, 'user').delay).toBe(IDLE_INTERVAL_USER);
|
||||
expect(computeHistoryJob(page, 'agent').delay).toBe(IDLE_INTERVAL_AGENT);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveSource (truth table)', () => {
|
||||
|
||||
@@ -40,11 +40,12 @@ describe('PersistenceExtension.onStoreDocument — Approach-A boundary snapshot'
|
||||
let pageHistoryRepo: {
|
||||
saveHistory: jest.Mock;
|
||||
findPageLastHistory: jest.Mock;
|
||||
updateHistoryKind: jest.Mock;
|
||||
};
|
||||
let aiQueue: { add: jest.Mock };
|
||||
let historyQueue: { add: jest.Mock };
|
||||
let historyQueue: { add: jest.Mock; remove: jest.Mock };
|
||||
let notificationQueue: { add: jest.Mock };
|
||||
let collabHistory: { addContributors: jest.Mock };
|
||||
let collabHistory: { addContributors: jest.Mock; popContributors: jest.Mock };
|
||||
let transclusionService: {
|
||||
syncPageTransclusions: jest.Mock;
|
||||
syncPageReferences: jest.Mock;
|
||||
@@ -93,13 +94,22 @@ describe('PersistenceExtension.onStoreDocument — Approach-A boundary snapshot'
|
||||
pageHistoryRepo = {
|
||||
saveHistory: jest.fn().mockImplementation(async () => {
|
||||
callOrder.push('saveHistory');
|
||||
return { id: 'history-1' };
|
||||
}),
|
||||
findPageLastHistory: jest.fn().mockResolvedValue(null),
|
||||
updateHistoryKind: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
aiQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
historyQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
historyQueue = {
|
||||
add: jest.fn().mockResolvedValue(undefined),
|
||||
// #370 — enqueuePageHistory now removes any pending idle job before re-adding.
|
||||
remove: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
notificationQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
collabHistory = { addContributors: jest.fn().mockResolvedValue(undefined) };
|
||||
collabHistory = {
|
||||
addContributors: jest.fn().mockResolvedValue(undefined),
|
||||
popContributors: jest.fn().mockResolvedValue([]),
|
||||
};
|
||||
transclusionService = {
|
||||
syncPageTransclusions: jest.fn().mockResolvedValue(undefined),
|
||||
syncPageReferences: jest.fn().mockResolvedValue(undefined),
|
||||
@@ -165,6 +175,50 @@ describe('PersistenceExtension.onStoreDocument — Approach-A boundary snapshot'
|
||||
expect(pageRepo.updatePage.mock.calls[0][0].lastUpdatedSource).toBe('user');
|
||||
});
|
||||
|
||||
// #370 review round-1 SUGGESTION: the boundary was GENERALIZED from a
|
||||
// user→agent special-case to ANY lastUpdatedSource transition. These pin the
|
||||
// generalized behaviour it was rebuilt for.
|
||||
describe('generalized boundary — any source transition', () => {
|
||||
// Same persisted page but with an explicit prior source.
|
||||
const pageWithPriorSource = (prior: string | null) => ({
|
||||
...persistedHumanPage('NEW CONTENT'),
|
||||
lastUpdatedSource: prior,
|
||||
});
|
||||
|
||||
it('agent→user transition fires the boundary (pins the prior agent revision)', async () => {
|
||||
const document = ydocFor(doc('NEW CONTENT'));
|
||||
pageRepo.findById.mockResolvedValue(pageWithPriorSource('agent'));
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue(null);
|
||||
|
||||
await ext.onStoreDocument(buildData(document, 'user') as any);
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).toHaveBeenCalledTimes(1);
|
||||
expect(callOrder).toEqual(['saveHistory', 'updatePage']);
|
||||
expect(pageRepo.updatePage.mock.calls[0][0].lastUpdatedSource).toBe('user');
|
||||
});
|
||||
|
||||
it('git→user transition fires the boundary (git-sync overwrite is a source change)', async () => {
|
||||
const document = ydocFor(doc('NEW CONTENT'));
|
||||
pageRepo.findById.mockResolvedValue(pageWithPriorSource('git'));
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue(null);
|
||||
|
||||
await ext.onStoreDocument(buildData(document, 'user') as any);
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).toHaveBeenCalledTimes(1);
|
||||
expect(callOrder).toEqual(['saveHistory', 'updatePage']);
|
||||
});
|
||||
|
||||
it('a null prior source (first-ever edit) does NOT fire the boundary', async () => {
|
||||
const document = ydocFor(doc('NEW CONTENT'));
|
||||
pageRepo.findById.mockResolvedValue(pageWithPriorSource(null));
|
||||
|
||||
await ext.onStoreDocument(buildData(document, 'agent') as any);
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).not.toHaveBeenCalled();
|
||||
expect(pageRepo.updatePage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
it('idempotency: unchanged content → no updatePage, no history, no queues', async () => {
|
||||
// The Y.Doc content equals the persisted content deeply → early skip.
|
||||
// A Y.Doc round-trip normalizes attrs (e.g. paragraph indent), so derive
|
||||
@@ -479,4 +533,231 @@ describe('PersistenceExtension.onStoreDocument — Approach-A boundary snapshot'
|
||||
// Contributors keyed by the UUID so they match the PAGE_HISTORY job (page.id).
|
||||
expect(collabHistory.addContributors.mock.calls[0][0]).toBe(PAGE_ID);
|
||||
});
|
||||
|
||||
// #370 — explicit save-version (Cmd+S / agent save tool) over the stateless
|
||||
// seam. The tier is derived from the SIGNED connection actor, the store path
|
||||
// is reused, and promote-not-dup avoids duplicating heavy content rows.
|
||||
describe('save-version (#370)', () => {
|
||||
const emitSave = (document: any, actor: 'user' | 'agent') =>
|
||||
ext.onStateless({
|
||||
connection: {
|
||||
readOnly: false,
|
||||
context: { user: { id: USER_ID, name: 'Alice' }, actor },
|
||||
} as any,
|
||||
documentName: `page.${PAGE_ID}`,
|
||||
document: document as any,
|
||||
payload: JSON.stringify({ type: 'save-version' }),
|
||||
} as any);
|
||||
|
||||
// findById returns a page whose content already equals the live doc, so the
|
||||
// store path is a no-op and we isolate the versioning decision.
|
||||
const pageMatchingDoc = (document: any) => ({
|
||||
...persistedHumanPage('IGNORED'),
|
||||
content: TiptapTransformer.fromYdoc(document, 'default'),
|
||||
});
|
||||
|
||||
it('human save with no prior snapshot → writes a manual version + broadcasts', async () => {
|
||||
const document = ydocFor(doc('VERSION ME'));
|
||||
pageRepo.findById.mockResolvedValue(pageMatchingDoc(document));
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue(null);
|
||||
|
||||
await emitSave(document, 'user');
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).toHaveBeenCalledTimes(1);
|
||||
expect(pageHistoryRepo.saveHistory.mock.calls[0][1]).toEqual(
|
||||
expect.objectContaining({ kind: 'manual' }),
|
||||
);
|
||||
// The pending idle autosnapshot is cancelled by the explicit version.
|
||||
expect(historyQueue.remove).toHaveBeenCalledWith(PAGE_ID);
|
||||
const msg = JSON.parse(
|
||||
(document as any).broadcastStateless.mock.calls[(document as any).broadcastStateless.mock.calls.length - 1][0],
|
||||
);
|
||||
expect(msg).toMatchObject({
|
||||
type: 'version.saved',
|
||||
kind: 'manual',
|
||||
alreadySaved: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('agent save derives kind=agent from the signed actor', async () => {
|
||||
const document = ydocFor(doc('AGENT VERSION'));
|
||||
pageRepo.findById.mockResolvedValue(pageMatchingDoc(document));
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue(null);
|
||||
|
||||
await emitSave(document, 'agent');
|
||||
|
||||
expect(pageHistoryRepo.saveHistory.mock.calls[pageHistoryRepo.saveHistory.mock.calls.length - 1][1]).toEqual(
|
||||
expect.objectContaining({ kind: 'agent' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('promote-not-dup: latest snapshot is an autosave with identical content → upgrades in place', async () => {
|
||||
const document = ydocFor(doc('SAME'));
|
||||
const page = pageMatchingDoc(document);
|
||||
pageRepo.findById.mockResolvedValue(page);
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue({
|
||||
id: 'auto-1',
|
||||
content: page.content,
|
||||
kind: 'idle',
|
||||
});
|
||||
|
||||
await emitSave(document, 'user');
|
||||
|
||||
// No heavy new content row — the existing autosave is promoted to manual.
|
||||
expect(pageHistoryRepo.updateHistoryKind).toHaveBeenCalledWith(
|
||||
'auto-1',
|
||||
'manual',
|
||||
expect.anything(),
|
||||
);
|
||||
expect(pageHistoryRepo.saveHistory).not.toHaveBeenCalled();
|
||||
const msg = JSON.parse(
|
||||
(document as any).broadcastStateless.mock.calls[(document as any).broadcastStateless.mock.calls.length - 1][0],
|
||||
);
|
||||
expect(msg).toMatchObject({ historyId: 'auto-1', alreadySaved: false });
|
||||
});
|
||||
|
||||
it('no-op when the latest snapshot is already a manual version of this content', async () => {
|
||||
const document = ydocFor(doc('ALREADY SAVED'));
|
||||
const page = pageMatchingDoc(document);
|
||||
pageRepo.findById.mockResolvedValue(page);
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue({
|
||||
id: 'ver-1',
|
||||
content: page.content,
|
||||
kind: 'manual',
|
||||
});
|
||||
|
||||
await emitSave(document, 'user');
|
||||
|
||||
expect(pageHistoryRepo.updateHistoryKind).not.toHaveBeenCalled();
|
||||
expect(pageHistoryRepo.saveHistory).not.toHaveBeenCalled();
|
||||
const msg = JSON.parse(
|
||||
(document as any).broadcastStateless.mock.calls[(document as any).broadcastStateless.mock.calls.length - 1][0],
|
||||
);
|
||||
expect(msg).toMatchObject({ alreadySaved: true, kind: 'manual' });
|
||||
});
|
||||
|
||||
it('a read-only connection cannot save a version', async () => {
|
||||
const document = ydocFor(doc('READER'));
|
||||
pageRepo.findById.mockResolvedValue(pageMatchingDoc(document));
|
||||
|
||||
await ext.onStateless({
|
||||
connection: {
|
||||
readOnly: true,
|
||||
context: { user: { id: USER_ID }, actor: 'user' },
|
||||
} as any,
|
||||
documentName: `page.${PAGE_ID}`,
|
||||
document: document as any,
|
||||
payload: JSON.stringify({ type: 'save-version' }),
|
||||
} as any);
|
||||
|
||||
expect(pageHistoryRepo.saveHistory).not.toHaveBeenCalled();
|
||||
expect(pageHistoryRepo.updateHistoryKind).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// #370 F8-twin — a COMMIT abort (serialization/deadlock/conn-drop) rejects
|
||||
// OUTSIDE the tx callback, AFTER the destructive popContributors (SPOP) and
|
||||
// saveHistory ran but the INSERT rolled back. onStateless has no retry, so
|
||||
// the outer catch MUST re-add (SADD) the popped set or attribution is lost
|
||||
// irrecoverably. MUTATION: drop the outer catch → addContributors is never
|
||||
// called → this reddens.
|
||||
it('restores popped contributors when the commit aborts after the callback', async () => {
|
||||
const document = ydocFor(doc('VERSION ME'));
|
||||
pageRepo.findById.mockResolvedValue(pageMatchingDoc(document));
|
||||
// No matching snapshot → fresh version branch → pops contributors.
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue(null);
|
||||
collabHistory.popContributors.mockResolvedValue(['u1', 'u2']);
|
||||
|
||||
// A db whose commit REJECTS after the callback body resolved: the SPOP and
|
||||
// saveHistory already ran, then the tx aborts. onStoreDocument's flush uses
|
||||
// the same db but its content matches (no-op branch) and its own retry loop
|
||||
// swallows the throw, so only the versioning tx exercises the restore.
|
||||
const commitFailingDb = {
|
||||
transaction: () => ({
|
||||
execute: async (fn: (trx: any) => Promise<any>) => {
|
||||
await fn(trxStub);
|
||||
throw new Error('commit aborted (serialization_failure)');
|
||||
},
|
||||
}),
|
||||
};
|
||||
const ext2 = new PersistenceExtension(
|
||||
pageRepo as any,
|
||||
pageHistoryRepo as any,
|
||||
commitFailingDb as any,
|
||||
aiQueue as any,
|
||||
historyQueue as any,
|
||||
notificationQueue as any,
|
||||
collabHistory as any,
|
||||
transclusionService as any,
|
||||
);
|
||||
jest.spyOn(ext2['logger'], 'debug').mockImplementation(() => undefined);
|
||||
jest.spyOn(ext2['logger'], 'warn').mockImplementation(() => undefined);
|
||||
jest.spyOn(ext2['logger'], 'error').mockImplementation(() => undefined);
|
||||
|
||||
await expect(
|
||||
ext2.onStateless({
|
||||
connection: {
|
||||
readOnly: false,
|
||||
context: { user: { id: USER_ID, name: 'Alice' }, actor: 'user' },
|
||||
} as any,
|
||||
documentName: `page.${PAGE_ID}`,
|
||||
document: document as any,
|
||||
payload: JSON.stringify({ type: 'save-version' }),
|
||||
} as any),
|
||||
).rejects.toThrow();
|
||||
|
||||
// Attribution preserved: the popped set is SADD-restored, keyed by the page
|
||||
// UUID it was popped under.
|
||||
expect(collabHistory.addContributors).toHaveBeenCalledWith(PAGE_ID, [
|
||||
'u1',
|
||||
'u2',
|
||||
]);
|
||||
});
|
||||
|
||||
// #370 #260 — for a `page.<slugId>` document the idle job is armed under the
|
||||
// page UUID (computeHistoryJob's jobId = page.id), so the supersede-remove
|
||||
// must target page.id, not the raw slugId doc-name id, or it silently misses.
|
||||
it('cancels the superseded idle job by the page UUID for a slugId doc', async () => {
|
||||
const SLUG = 'slug-1'; // persistedHumanPage.slugId
|
||||
const document = ydocFor(doc('VERSION ME'));
|
||||
pageRepo.findById.mockResolvedValue(pageMatchingDoc(document));
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue(null);
|
||||
|
||||
await ext.onStateless({
|
||||
connection: {
|
||||
readOnly: false,
|
||||
context: { user: { id: USER_ID, name: 'Alice' }, actor: 'user' },
|
||||
} as any,
|
||||
documentName: `page.${SLUG}`,
|
||||
document: document as any,
|
||||
payload: JSON.stringify({ type: 'save-version' }),
|
||||
} as any);
|
||||
|
||||
// remove() keyed by the UUID (the real jobId), never the slugId.
|
||||
expect(historyQueue.remove).toHaveBeenCalledWith(PAGE_ID);
|
||||
expect(historyQueue.remove).not.toHaveBeenCalledWith(SLUG);
|
||||
});
|
||||
});
|
||||
|
||||
// #370 — the in-memory idle-burst marker must be dropped on doc unload (like
|
||||
// its sibling per-document maps) or it grows unbounded for every page that was
|
||||
// edited but never manually saved. MUTATION: drop the afterUnloadDocument
|
||||
// delete → the entry survives → this reddens.
|
||||
describe('idleBurstStart housekeeping', () => {
|
||||
it('afterUnloadDocument clears the idle-burst marker armed by a store', async () => {
|
||||
const document = ydocFor(doc('EDIT'));
|
||||
pageRepo.findById.mockResolvedValue(persistedHumanPage('EDIT'));
|
||||
|
||||
await ext.onStoreDocument(buildData(document, 'user') as any);
|
||||
|
||||
const map = ext['idleBurstStart'] as Map<string, number>;
|
||||
// Keyed by documentName (buildData uses `page.${PAGE_ID}`).
|
||||
expect(map.has(`page.${PAGE_ID}`)).toBe(true);
|
||||
|
||||
await ext.afterUnloadDocument({
|
||||
documentName: `page.${PAGE_ID}`,
|
||||
} as any);
|
||||
|
||||
expect(map.has(`page.${PAGE_ID}`)).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -37,9 +37,11 @@ import { Page } from '@docmost/db/types/entity.types';
|
||||
import { CollabHistoryService } from '../services/collab-history.service';
|
||||
import {
|
||||
EMBED_DEBOUNCE_MS,
|
||||
HISTORY_FAST_INTERVAL,
|
||||
HISTORY_FAST_THRESHOLD,
|
||||
HISTORY_INTERVAL,
|
||||
IDLE_INTERVAL_AGENT,
|
||||
IDLE_INTERVAL_USER,
|
||||
IDLE_MAX_WAIT_AGENT,
|
||||
IDLE_MAX_WAIT_USER,
|
||||
PageHistoryKind,
|
||||
} from '../constants';
|
||||
import { TransclusionService } from '../../core/page/transclusion/transclusion.service';
|
||||
import {
|
||||
@@ -56,6 +58,16 @@ import { hasTransclusionFamilyNodes } from '../../core/page/transclusion/utils/t
|
||||
*/
|
||||
export const INTENTIONAL_CLEAR_MESSAGE_TYPE = 'intentional-clear';
|
||||
|
||||
/**
|
||||
* #370 — wire format of the client→server "save a version" signal. Sent by the
|
||||
* human (Cmd+S / Save button) and by the agent's explicit save tool over the
|
||||
* SAME stateless channel. The intentionality tier ('manual' vs 'agent') is
|
||||
* derived SERVER-SIDE from the signed connection actor, never from this
|
||||
* payload, so a version's type is unforgeable. The document is taken from the
|
||||
* connection (not the payload), so the signal cannot be aimed at another page.
|
||||
*/
|
||||
export const SAVE_VERSION_MESSAGE_TYPE = 'save-version';
|
||||
|
||||
/**
|
||||
* #251 — how long an intentional-clear signal stays "pending" before it is
|
||||
* ignored. The signal is set on the clearing keystroke but consumed by the
|
||||
@@ -92,35 +104,39 @@ export function resolveSource(
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the BullMQ job id + delay for a page-history snapshot job. Pure so
|
||||
* the data-loss-sensitive timing arithmetic is unit-testable; `now` is injected
|
||||
* (caller passes `Date.now()`) for determinism.
|
||||
* #370 — compute the BullMQ job id + delay for a page's trailing idle-flush
|
||||
* autosnapshot. Pure so the timing is unit-testable.
|
||||
*
|
||||
* - Agent edits: delay 0 and a source-keyed job id `${page.id}-agent`. The
|
||||
* delay MUST stay 0 — the worker re-reads the page row at run time, so any
|
||||
* delay risks reading content a later human edit has already overwritten
|
||||
* (mis-tagged snapshot). 0 minimizes that window. The `-agent` suffix keeps
|
||||
* the job from coalescing with the bare-page.id human job.
|
||||
* - Human edits: age-based debounce so rapid human edits coalesce into one
|
||||
* snapshot; job id is the bare `page.id`.
|
||||
*
|
||||
* BullMQ forbids ':' in custom job ids (Redis key separator), so '-' is used;
|
||||
* page.id is a UUID, so `${page.id}-agent` cannot collide with a human job.
|
||||
* Both humans and the agent now share ONE idle pipeline (the agent's old
|
||||
* `delay=0` fast path is gone — intentional agent points arrive via the
|
||||
* explicit save-version signal instead). The job id is the bare `page.id`, so a
|
||||
* page has at most one pending idle job; the caller removes-and-re-adds it on
|
||||
* every store to keep it debounced to the trailing edge of an edit burst. The
|
||||
* window differs by source only: the agent flushes sooner than a human.
|
||||
*/
|
||||
export function computeHistoryJob(
|
||||
page: Pick<Page, 'id' | 'createdAt'>,
|
||||
page: Pick<Page, 'id'>,
|
||||
source: string,
|
||||
now: number,
|
||||
// Epoch ms of the FIRST edit in the current burst (when the pending idle job
|
||||
// was first armed). Used to enforce the max-wait ceiling so a continuous
|
||||
// editing session cannot re-arm the trailing timer forever. `now` is injectable
|
||||
// for tests; both default to a live clock / no ceiling when omitted.
|
||||
burstStart?: number,
|
||||
now: number = Date.now(),
|
||||
): { jobId: string; delay: number } {
|
||||
const isAgent = source === 'agent';
|
||||
const pageAge = now - new Date(page.createdAt).getTime();
|
||||
const delay = isAgent
|
||||
? 0
|
||||
: pageAge < HISTORY_FAST_THRESHOLD
|
||||
? HISTORY_FAST_INTERVAL
|
||||
: HISTORY_INTERVAL;
|
||||
const jobId = isAgent ? `${page.id}-agent` : page.id;
|
||||
return { jobId, delay };
|
||||
const interval = isAgent ? IDLE_INTERVAL_AGENT : IDLE_INTERVAL_USER;
|
||||
const maxWait = isAgent ? IDLE_MAX_WAIT_AGENT : IDLE_MAX_WAIT_USER;
|
||||
|
||||
let delay = interval;
|
||||
if (burstStart !== undefined) {
|
||||
// Time already elapsed since the burst's first edit; the snapshot must fire
|
||||
// no later than `maxWait` after that, so shrink the trailing delay to the
|
||||
// remaining budget (never negative, so BullMQ fires it promptly).
|
||||
const remaining = burstStart + maxWait - now;
|
||||
delay = Math.max(0, Math.min(interval, remaining));
|
||||
}
|
||||
return { jobId: page.id, delay };
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
@@ -132,6 +148,28 @@ export class PersistenceExtension implements Extension {
|
||||
// coalescing window" per document and OR it across all edits in the window,
|
||||
// so the snapshot is marked 'agent' regardless of who wrote last.
|
||||
private agentTouched: Map<string, boolean> = new Map();
|
||||
// #370 — epoch ms of the FIRST edit in the current idle-flush burst. Keyed by
|
||||
// documentName (like its sibling per-document maps above), NOT by page.id, so
|
||||
// it can be cleaned in afterUnloadDocument alongside `contributors` /
|
||||
// `agentTouched` / `intentionalClear` when the doc unloads — otherwise any page
|
||||
// that was edited but never manually saved (the common case) would keep its
|
||||
// entry forever and the Map would grow unbounded in this long-lived process.
|
||||
// Set when the pending idle job is first armed (empty entry), read to enforce
|
||||
// the max-wait ceiling in computeHistoryJob, and cleared on doc unload or when
|
||||
// a manual save cancels the idle job so the next burst starts a fresh window.
|
||||
//
|
||||
// Single-process assumption (like `contributors` / `agentTouched` above): this
|
||||
// lives only in THIS collab process's memory. A restart, or a page's ownership
|
||||
// moving to another node, loses the burst-start marker. Consequence: a burst
|
||||
// that spans the restart looks like a fresh burst to the surviving process, so
|
||||
// its max-wait ceiling is re-anchored to the first post-restart edit — a single
|
||||
// continuous session straddling a restart can therefore wait up to ~2× the cap
|
||||
// for its idle snapshot (once for the lost pre-restart window, once for the new
|
||||
// one). Bounded and benign (it only DELAYS a safety-net autosnapshot; manual
|
||||
// saves are unaffected and the next quiet period always flushes), but the
|
||||
// assumption and its consequence are recorded here so no one mistakes the
|
||||
// in-memory marker for a durable, cross-process guarantee.
|
||||
private idleBurstStart: Map<string, number> = new Map();
|
||||
// #251 — per-document "intentional clear pending" flags. Keyed by
|
||||
// documentName, value = expiry timestamp (ms). Set by onStateless when the
|
||||
// client reports a deliberate clear; consumed once by the next
|
||||
@@ -363,20 +401,19 @@ export class PersistenceExtension implements Extension {
|
||||
//this.logger.debug('Contributors error:' + err?.['message']);
|
||||
}
|
||||
|
||||
// Approach A — boundary snapshot before the agent's first edit.
|
||||
// When this store is the agent's and the page's currently persisted
|
||||
// state was authored by a human, pin that human state as its own
|
||||
// history version BEFORE the agent overwrites it. `page` still holds
|
||||
// the OLD content/provenance here, so saveHistory(page) captures the
|
||||
// pre-agent state tagged 'user'. The agent's new content is
|
||||
// snapshotted later by the debounced PAGE_HISTORY job ('agent'). Skip
|
||||
// if the prior state is already agent-authored (boundary already
|
||||
// pinned on the user->agent transition), if the page is effectively
|
||||
// empty, or if the latest existing snapshot already equals this human
|
||||
// state (avoid duplicates).
|
||||
// #370 — boundary snapshot on ANY source transition. When the store
|
||||
// flips the page's provenance (user↔agent↔git), pin the OUTGOING
|
||||
// state as its own history version BEFORE the incoming source
|
||||
// overwrites it. `page` still holds the OLD content/provenance here,
|
||||
// so saveHistory(page) captures the pre-transition state tagged with
|
||||
// its own source, kind='boundary'. The incoming content is snapshotted
|
||||
// later by the debounced idle job. Skip if the page is effectively
|
||||
// empty or if the latest existing snapshot already equals this state
|
||||
// (the shared isDeepStrictEqual gate — avoids duplicates). Generalizing
|
||||
// beyond the old user→agent special-case also covers git-sync for free.
|
||||
if (
|
||||
lastUpdatedSource === 'agent' &&
|
||||
page.lastUpdatedSource !== 'agent'
|
||||
page.lastUpdatedSource &&
|
||||
page.lastUpdatedSource !== lastUpdatedSource
|
||||
) {
|
||||
// pageHistory.pageId is uuid-typed; use page.id (never the doc-name
|
||||
// slugId) so a `page.<slugId>` doc cannot throw 22P02 here (#260).
|
||||
@@ -384,15 +421,13 @@ export class PersistenceExtension implements Extension {
|
||||
page.id,
|
||||
{ includeContent: true, trx },
|
||||
);
|
||||
const humanBaselineMissing =
|
||||
const baselineMissing =
|
||||
!lastHistory ||
|
||||
!isDeepStrictEqual(lastHistory.content, page.content);
|
||||
if (
|
||||
!isEmptyParagraphDoc(page.content as any) &&
|
||||
humanBaselineMissing
|
||||
) {
|
||||
if (!isEmptyParagraphDoc(page.content as any) && baselineMissing) {
|
||||
await this.pageHistoryRepo.saveHistory(page, {
|
||||
contributorIds: page.contributorIds ?? undefined,
|
||||
kind: 'boundary',
|
||||
trx,
|
||||
});
|
||||
}
|
||||
@@ -522,7 +557,7 @@ export class PersistenceExtension implements Extension {
|
||||
{ jobId: `embed-${page.id}`, delay: EMBED_DEBOUNCE_MS },
|
||||
);
|
||||
|
||||
await this.enqueuePageHistory(page, lastUpdatedSource);
|
||||
await this.enqueuePageHistory(page, documentName, lastUpdatedSource);
|
||||
}
|
||||
|
||||
// #402 — report the serialized size for the store histogram's size_bucket.
|
||||
@@ -554,6 +589,14 @@ export class PersistenceExtension implements Extension {
|
||||
return; // unrelated / malformed stateless message
|
||||
}
|
||||
|
||||
// #370 — explicit "save a version" (human Cmd+S / agent save tool). Edit
|
||||
// rights are already enforced by the readOnly reject above (a reader can't
|
||||
// create a version), exactly as intentional-clear requires.
|
||||
if (message?.type === SAVE_VERSION_MESSAGE_TYPE) {
|
||||
await this.handleSaveVersion(data);
|
||||
return;
|
||||
}
|
||||
|
||||
if (message?.type !== INTENTIONAL_CLEAR_MESSAGE_TYPE) return;
|
||||
|
||||
this.intentionalClear.set(
|
||||
@@ -562,6 +605,160 @@ export class PersistenceExtension implements Extension {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* #370 — persist an intentional version from the live in-memory ydoc.
|
||||
*
|
||||
* One stateless path serves BOTH the human and the agent; the tier is derived
|
||||
* SERVER-SIDE from the signed connection actor ('agent' → 'agent', anything
|
||||
* else → 'manual'), so the version type cannot be spoofed by the client. We
|
||||
* take the fresh ydoc from the collab process memory and run it through the
|
||||
* EXISTING store path first (so pages.content/ydoc reflect the exact content
|
||||
* being versioned — a REST endpoint would race the up-to-10s-stale page row),
|
||||
* then snapshot it into page_history with the intentional kind.
|
||||
*
|
||||
* Promote-not-dup: if the latest history row already holds this exact content
|
||||
* and it is an autosave (idle/boundary/legacy-null), upgrade its kind in place
|
||||
* instead of duplicating a heavy content row; if it is already 'manual', it is
|
||||
* a no-op (the client shows an "already saved" toast). Otherwise a fresh
|
||||
* version row is written, popping the aggregated contributors from Redis.
|
||||
*/
|
||||
private async handleSaveVersion(data: onStatelessPayload): Promise<void> {
|
||||
const { connection, document, documentName } = data;
|
||||
const context = connection?.context;
|
||||
const pageId = getPageId(documentName);
|
||||
// Unforgeable: 'agent' only for a signed agent connection, else 'manual'.
|
||||
const kind: PageHistoryKind =
|
||||
context?.actor === 'agent' ? 'agent' : 'manual';
|
||||
|
||||
// Flush the live ydoc through the normal store path so the page row + ydoc
|
||||
// hold exactly what we are about to version (also fires the idle enqueue we
|
||||
// supersede below, plus any source-transition boundary). onStoreDocument
|
||||
// only needs document/documentName/context.
|
||||
await this.onStoreDocument({
|
||||
document,
|
||||
documentName,
|
||||
context,
|
||||
} as onStoreDocumentPayload);
|
||||
|
||||
let result:
|
||||
| { historyId: string; kind: PageHistoryKind; alreadySaved: boolean }
|
||||
| undefined;
|
||||
|
||||
// #370 F8-twin — the contributor set popped from Redis (destructive SPOP)
|
||||
// must be restored if the version row does not durably land. The inner
|
||||
// try/catch below only covers a throw INSIDE the callback; but executeTx
|
||||
// COMMITS after the callback, so a commit-abort (serialization/deadlock/
|
||||
// connection drop — the transient class the epic retries in the processor)
|
||||
// rejects OUTSIDE the callback, after saveHistory already ran and the SPOP
|
||||
// already happened, while the INSERT rolls back. onStateless does NOT retry,
|
||||
// so an unrestored pop is a one-shot irrecoverable attribution loss (the
|
||||
// processor got exactly this fix: poppedForRestore + an outer catch). We
|
||||
// track the popped set here (keyed by the page UUID it was popped by — never
|
||||
// the doc-name id, which may be a slugId, #260) and restore it in the outer
|
||||
// catch. addContributors is an idempotent Redis SADD, so a double-restore is
|
||||
// harmless. versionedPageId is also reused below to remove the superseded
|
||||
// idle job by its real jobId (page.id).
|
||||
let poppedForRestore: string[] = [];
|
||||
let versionedPageId: string | undefined;
|
||||
|
||||
try {
|
||||
await executeTx(this.db, async (trx) => {
|
||||
const page = await this.pageRepo.findById(pageId, {
|
||||
withLock: true,
|
||||
includeContent: true,
|
||||
trx,
|
||||
});
|
||||
if (!page) return;
|
||||
versionedPageId = page.id;
|
||||
// Never version an effectively-empty page (mirrors the processor's
|
||||
// first-history guard); there is nothing intentional to pin.
|
||||
if (isEmptyParagraphDoc(page.content as any)) return;
|
||||
|
||||
const lastHistory = await this.pageHistoryRepo.findPageLastHistory(
|
||||
page.id,
|
||||
{ includeContent: true, trx },
|
||||
);
|
||||
|
||||
if (
|
||||
lastHistory &&
|
||||
isDeepStrictEqual(lastHistory.content, page.content)
|
||||
) {
|
||||
// Content is already snapshotted. Promote-not-dup.
|
||||
if (lastHistory.kind === 'manual') {
|
||||
result = {
|
||||
historyId: lastHistory.id,
|
||||
kind: 'manual',
|
||||
alreadySaved: true,
|
||||
};
|
||||
return;
|
||||
}
|
||||
await this.pageHistoryRepo.updateHistoryKind(
|
||||
lastHistory.id,
|
||||
kind,
|
||||
trx,
|
||||
);
|
||||
result = { historyId: lastHistory.id, kind, alreadySaved: false };
|
||||
return;
|
||||
}
|
||||
|
||||
// Fresh version row. Pop the contributors aggregated since the last
|
||||
// snapshot (SPOP); restore them if the write fails so they aren't lost.
|
||||
const contributorIds = await this.collabHistory.popContributors(
|
||||
page.id,
|
||||
);
|
||||
poppedForRestore = contributorIds;
|
||||
try {
|
||||
const saved = await this.pageHistoryRepo.saveHistory(page, {
|
||||
contributorIds,
|
||||
kind,
|
||||
trx,
|
||||
});
|
||||
result = { historyId: saved.id, kind, alreadySaved: false };
|
||||
} catch (err) {
|
||||
await this.collabHistory.addContributors(page.id, contributorIds);
|
||||
poppedForRestore = [];
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
// A throw here means the tx did NOT commit (callback threw, or the commit
|
||||
// itself failed and rolled back). If we popped contributors and the inner
|
||||
// catch did not already restore them, restore now so attribution is not
|
||||
// lost — onStateless has no retry to recover it. Restore by the page UUID
|
||||
// the pop was keyed under (versionedPageId is always set before the pop).
|
||||
if (poppedForRestore.length && versionedPageId) {
|
||||
await this.collabHistory.addContributors(
|
||||
versionedPageId,
|
||||
poppedForRestore,
|
||||
);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
// Housekeeping: this explicit version supersedes the page's pending idle
|
||||
// autosnapshot, so cancel it and end the current idle burst so the next edit
|
||||
// starts a fresh max-wait window. Remove the idle job by its REAL jobId
|
||||
// (page.id UUID — computeHistoryJob arms it under page.id), not the raw
|
||||
// doc-name id which may be a slugId for a `page.<slugId>` doc (#260), or the
|
||||
// remove silently misses. The burst marker is keyed by documentName (like its
|
||||
// sibling per-document maps), and is also cleaned in afterUnloadDocument.
|
||||
if (versionedPageId) {
|
||||
await this.historyQueue.remove(versionedPageId).catch(() => undefined);
|
||||
}
|
||||
this.idleBurstStart.delete(documentName);
|
||||
|
||||
if (result) {
|
||||
document.broadcastStateless(
|
||||
JSON.stringify({
|
||||
type: 'version.saved',
|
||||
historyId: result.historyId,
|
||||
kind: result.kind,
|
||||
alreadySaved: result.alreadySaved,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async onChange(data: onChangePayload) {
|
||||
const documentName = data.documentName;
|
||||
const userId = data.context?.user?.id;
|
||||
@@ -586,6 +783,10 @@ export class PersistenceExtension implements Extension {
|
||||
this.contributors.delete(documentName);
|
||||
this.agentTouched.delete(documentName);
|
||||
this.intentionalClear.delete(documentName);
|
||||
// #370 — drop the idle-burst marker with the other per-document maps so it
|
||||
// cannot accumulate across the process lifetime for never-manually-saved
|
||||
// pages. The pending idle job (if any) is a self-expiring BullMQ delayed job.
|
||||
this.idleBurstStart.delete(documentName);
|
||||
}
|
||||
|
||||
private consumeContributors(documentName: string): string[] {
|
||||
@@ -617,19 +818,80 @@ export class PersistenceExtension implements Extension {
|
||||
|
||||
private async enqueuePageHistory(
|
||||
page: Page,
|
||||
documentName: string,
|
||||
lastUpdatedSource: string,
|
||||
): Promise<void> {
|
||||
// Job id + delay arithmetic lives in the pure `computeHistoryJob` (see its
|
||||
// doc comment for the agent-delay-0 / age-based-debounce invariants).
|
||||
// #370 — trailing idle debounce with a max-wait ceiling. One pending idle
|
||||
// job per page (jobId = page.id); on every store we remove the pending
|
||||
// delayed job and re-add it, so the snapshot lands `delay` after edits go
|
||||
// quiet rather than once per store (precedent: workspace.service.ts).
|
||||
// remove() on a delayed job simply deletes it (0 if absent, no throw); if the
|
||||
// job is already ACTIVE and the remove is a no-op, the add still de-dups and
|
||||
// the processor's isDeepStrictEqual gate collapses the duplicate content.
|
||||
//
|
||||
// The FIRST arm of a burst records `burstStart`; computeHistoryJob shrinks
|
||||
// the delay to the remaining max-wait budget from that point, so a continuous
|
||||
// session cannot re-arm the trailing timer forever and starve the snapshot.
|
||||
// A burst marker older than THIS TIER's max-wait means the previous idle job
|
||||
// has already fired — start a fresh window instead of firing immediately on
|
||||
// the next edit. Must use the SAME source-specific max-wait computeHistoryJob
|
||||
// uses (agent 5m / user 10m): a hardcoded USER ceiling would leave an agent
|
||||
// burst's marker stale for 5..10m, forcing delay=0 on every store in that
|
||||
// window and writing one idle row per store — exactly the per-store bloat the
|
||||
// debounce exists to prevent, on the continuous-agent path.
|
||||
const maxWait =
|
||||
lastUpdatedSource === 'agent' ? IDLE_MAX_WAIT_AGENT : IDLE_MAX_WAIT_USER;
|
||||
const now = Date.now();
|
||||
// Keyed by documentName (see the map declaration) so afterUnloadDocument can
|
||||
// clean it; the queue jobId stays page.id (computeHistoryJob) as required.
|
||||
let burstStart = this.idleBurstStart.get(documentName);
|
||||
if (burstStart === undefined || now - burstStart >= maxWait) {
|
||||
burstStart = now;
|
||||
this.idleBurstStart.set(documentName, burstStart);
|
||||
}
|
||||
|
||||
const { jobId, delay } = computeHistoryJob(
|
||||
page,
|
||||
lastUpdatedSource,
|
||||
Date.now(),
|
||||
burstStart,
|
||||
now,
|
||||
);
|
||||
|
||||
// remove-then-add trailing-debounce idiom, and its ONE race. We delete the
|
||||
// pending delayed job and re-add it under the same jobId so the timer resets
|
||||
// to the trailing edge of the burst. The race is the small window between
|
||||
// these two awaits: if the delayed job's `delay` elapses in that gap it goes
|
||||
// ACTIVE, and then:
|
||||
// - remove() on an active/locked job is a no-op (BullMQ won't yank a job a
|
||||
// worker holds), and our `.catch(() => undefined)` swallows that too; and
|
||||
// - add() with a jobId that already exists (the now-active job's id) is
|
||||
// DROPPED by BullMQ — a duplicate add is a no-op.
|
||||
// So this store fails to re-arm the trailing job: the just-fired snapshot
|
||||
// captured content up to the moment it went active, and THIS edit is left
|
||||
// without a pending trailing job. It is bounded and self-healing — the NEXT
|
||||
// store re-arms a fresh delayed job (the id is free again once the active job
|
||||
// completes / removeOnComplete frees it), and the processor's
|
||||
// isDeepStrictEqual gate collapses any content-identical duplicate. The only
|
||||
// uncovered case is when the racing store was the LAST in the session: the
|
||||
// tail edits made after the job went active get NO trailing snapshot until
|
||||
// the next edit re-arms one. That is an acceptable safety-net gap (a manual
|
||||
// Save, a source-transition boundary, or simply the next edit all still cover
|
||||
// it), which is why the reviewer accepts documenting it here rather than
|
||||
// adding a post-add "did the add actually arm a job?" re-check.
|
||||
//
|
||||
// NOTE — do NOT "unify" this with the neighbouring embed-debounce idiom
|
||||
// (aiQueue.add of PAGE_CONTENT_UPDATED above): that one uses a STABLE jobId
|
||||
// and NO remove(), relying purely on BullMQ coalescing a repeated add under
|
||||
// the same id, because a re-embed only needs to eventually run once on the
|
||||
// latest content and re-anchoring its delay on every keystroke is undesirable.
|
||||
// THIS idiom deliberately removes-then-adds precisely to PUSH the delay back
|
||||
// to the trailing edge on every store (a true debounce), which coalescing
|
||||
// alone cannot do. Collapsing them would silently change the history cadence.
|
||||
await this.historyQueue.remove(jobId).catch(() => undefined);
|
||||
|
||||
await this.historyQueue.add(
|
||||
QueueJob.PAGE_HISTORY,
|
||||
{ pageId: page.id } as IPageHistoryJob,
|
||||
{ pageId: page.id, kind: 'idle' } as IPageHistoryJob,
|
||||
{ jobId, delay },
|
||||
);
|
||||
}
|
||||
|
||||
@@ -66,6 +66,15 @@ describe('HistoryProcessor.process', () => {
|
||||
notificationQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
generalQueue = { add: jest.fn().mockResolvedValue(undefined) };
|
||||
|
||||
// #370 F3 — the processor now serializes its find+save under a page-row lock
|
||||
// via executeTx. A db whose transaction().execute(fn) runs fn with a trx stub
|
||||
// drives the real executeTx() helper without a database.
|
||||
const db = {
|
||||
transaction: () => ({
|
||||
execute: (fn: (trx: any) => Promise<any>) => fn({ __trx: true }),
|
||||
}),
|
||||
};
|
||||
|
||||
// WorkerHost's constructor reads `this.worker`; passing repos positionally
|
||||
// matches the constructor and avoids the Nest DI container.
|
||||
proc = new HistoryProcessor(
|
||||
@@ -73,6 +82,7 @@ describe('HistoryProcessor.process', () => {
|
||||
pageRepo as any,
|
||||
collabHistory as any,
|
||||
watcherService as any,
|
||||
db as any,
|
||||
notificationQueue as any,
|
||||
generalQueue as any,
|
||||
);
|
||||
@@ -126,15 +136,26 @@ describe('HistoryProcessor.process', () => {
|
||||
await proc.process(buildJob());
|
||||
|
||||
expect(collabHistory.popContributors).toHaveBeenCalledWith(PAGE_ID);
|
||||
// #370 F3/F9 — the snapshot decision runs under a page-row lock. Pin the lock
|
||||
// structurally so a refactor that drops withLock/trx (silently reintroducing
|
||||
// the TOCTOU double-insert) turns this red. The tx stub is { __trx: true }.
|
||||
expect(pageRepo.findById).toHaveBeenCalledWith(
|
||||
PAGE_ID,
|
||||
expect.objectContaining({ withLock: true, trx: { __trx: true } }),
|
||||
);
|
||||
// #370 F7 — addPageWatchers MUST receive the trx, or its FK-check runs on a
|
||||
// separate connection and self-deadlocks against our FOR UPDATE. Asserting
|
||||
// the trx arg here is exactly what would have caught that regression.
|
||||
expect(watcherService.addPageWatchers).toHaveBeenCalledWith(
|
||||
['u1', 'u2'],
|
||||
PAGE_ID,
|
||||
SPACE_ID,
|
||||
WORKSPACE_ID,
|
||||
{ __trx: true },
|
||||
);
|
||||
expect(pageHistoryRepo.saveHistory).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ id: PAGE_ID }),
|
||||
{ contributorIds: ['u1', 'u2'] },
|
||||
{ contributorIds: ['u1', 'u2'], kind: 'idle', trx: { __trx: true } },
|
||||
);
|
||||
expect(generalQueue.add).toHaveBeenCalledWith(
|
||||
QueueJob.PAGE_BACKLINKS,
|
||||
@@ -186,6 +207,48 @@ describe('HistoryProcessor.process', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('COMMIT failure (throw outside the tx callback) → contributors RESTORED', async () => {
|
||||
// #370 F8 — a commit-time failure throws OUTSIDE the callback, so the inner
|
||||
// try/catch does not run; the outer catch must restore the popped set (else a
|
||||
// BullMQ retry writes an unattributed version). Use a db whose execute() runs
|
||||
// the callback THEN throws, simulating a commit abort.
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue({
|
||||
content: { type: 'doc', content: [] },
|
||||
});
|
||||
const commitFail = {
|
||||
transaction: () => ({
|
||||
execute: async (fn: (trx: any) => Promise<any>) => {
|
||||
await fn({ __trx: true }); // callback succeeds (saveHistory ok)
|
||||
throw new Error('commit aborted'); // ...but the COMMIT fails
|
||||
},
|
||||
}),
|
||||
};
|
||||
const procCommitFail = new HistoryProcessor(
|
||||
pageHistoryRepo as any,
|
||||
pageRepo as any,
|
||||
collabHistory as any,
|
||||
watcherService as any,
|
||||
commitFail as any,
|
||||
notificationQueue as any,
|
||||
generalQueue as any,
|
||||
);
|
||||
jest
|
||||
.spyOn(procCommitFail['logger'], 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
|
||||
await expect(procCommitFail.process(buildJob())).rejects.toThrow(
|
||||
'commit aborted',
|
||||
);
|
||||
// The inner catch did NOT run (save succeeded), so only the outer catch can
|
||||
// restore — assert it did.
|
||||
expect(collabHistory.addContributors).toHaveBeenCalledWith(PAGE_ID, [
|
||||
'u1',
|
||||
'u2',
|
||||
]);
|
||||
// And the post-snapshot queue work must NOT have run (we rethrew).
|
||||
expect(generalQueue.add).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('backlinks + notification queue failures are swallowed (history still committed)', async () => {
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue({
|
||||
content: { type: 'doc', content: [] },
|
||||
|
||||
@@ -19,6 +19,9 @@ import { isDeepStrictEqual } from 'node:util';
|
||||
import { CollabHistoryService } from '../services/collab-history.service';
|
||||
import { WatcherService } from '../../core/watcher/watcher.service';
|
||||
import { isEmptyParagraphDoc } from '../collaboration.util';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { executeTx } from '@docmost/db/utils';
|
||||
|
||||
@Processor(QueueName.HISTORY_QUEUE)
|
||||
export class HistoryProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
@@ -29,6 +32,7 @@ export class HistoryProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
private readonly pageRepo: PageRepo,
|
||||
private readonly collabHistory: CollabHistoryService,
|
||||
private readonly watcherService: WatcherService,
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
@InjectQueue(QueueName.NOTIFICATION_QUEUE) private notificationQueue: Queue,
|
||||
@InjectQueue(QueueName.GENERAL_QUEUE) private generalQueue: Queue,
|
||||
) {
|
||||
@@ -41,6 +45,9 @@ export class HistoryProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
try {
|
||||
const { pageId } = job.data;
|
||||
|
||||
// Read the page WITHOUT a lock first, only to bail early on the two cheap
|
||||
// no-write cases (page gone / empty first snapshot) without opening a
|
||||
// transaction. The authoritative check-then-write happens locked below.
|
||||
const page = await this.pageRepo.findById(pageId, {
|
||||
includeContent: true,
|
||||
});
|
||||
@@ -51,40 +58,109 @@ export class HistoryProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
return;
|
||||
}
|
||||
|
||||
const lastHistory = await this.pageHistoryRepo.findPageLastHistory(
|
||||
pageId,
|
||||
{ includeContent: true },
|
||||
);
|
||||
// #370 F3 — the snapshot decision (findPageLastHistory → saveHistory) must
|
||||
// be serialized against manual-save/boundary writers, which run under a
|
||||
// page-row lock in onStoreDocument. Without it, this processor and a
|
||||
// concurrent manual-save each read the same lastHistory (MVCC), both see
|
||||
// content != lastHistory, and both insert — producing two page_history rows
|
||||
// with IDENTICAL content (one 'idle', one 'manual'), defeating
|
||||
// promote-not-dup and the version-vs-autosave split. Taking the same
|
||||
// page-row lock makes the second writer observe the first's committed row so
|
||||
// the isDeepStrictEqual gate collapses the duplicate. Only the read+write
|
||||
// is transacted; the post-snapshot queue work stays outside.
|
||||
let contributorIds: string[] = [];
|
||||
let snapshotWritten = false;
|
||||
let lastHistoryContent: unknown;
|
||||
// #370 F8 — the contributor set popped from Redis (destructive SPOP) must be
|
||||
// restored if the snapshot does not durably land. The inner try/catch only
|
||||
// covers a throw INSIDE the callback; a COMMIT failure (connection drop,
|
||||
// serialization/deadlock abort on commit — the transient class the epic
|
||||
// already retries) throws OUTSIDE it, rolling the snapshot back while the
|
||||
// pop is already gone. We track the popped set here and restore it in the
|
||||
// outer catch so a BullMQ retry re-attributes the version. addContributors
|
||||
// is an idempotent Redis SADD, so a double-restore is harmless.
|
||||
let poppedForRestore: string[] = [];
|
||||
|
||||
if (!lastHistory && isEmptyParagraphDoc(page.content as any)) {
|
||||
this.logger.debug(
|
||||
`Skipping first history for page ${pageId}: empty content`,
|
||||
);
|
||||
await this.collabHistory.clearContributors(pageId);
|
||||
try {
|
||||
await executeTx(this.db, async (trx) => {
|
||||
const lockedPage = await this.pageRepo.findById(pageId, {
|
||||
includeContent: true,
|
||||
withLock: true,
|
||||
trx,
|
||||
});
|
||||
if (!lockedPage) return;
|
||||
|
||||
const lastHistory = await this.pageHistoryRepo.findPageLastHistory(
|
||||
pageId,
|
||||
{ includeContent: true, trx },
|
||||
);
|
||||
lastHistoryContent = lastHistory?.content;
|
||||
|
||||
if (!lastHistory && isEmptyParagraphDoc(lockedPage.content as any)) {
|
||||
this.logger.debug(
|
||||
`Skipping first history for page ${pageId}: empty content`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
lastHistory &&
|
||||
isDeepStrictEqual(lastHistory.content, lockedPage.content)
|
||||
) {
|
||||
return; // already snapshotted at this content — nothing to write
|
||||
}
|
||||
|
||||
contributorIds = await this.collabHistory.popContributors(pageId);
|
||||
poppedForRestore = contributorIds;
|
||||
try {
|
||||
// Pass `trx` so the watcher insert's FK check (FOR KEY SHARE on
|
||||
// pages[pageId]) runs on the SAME connection that already holds the
|
||||
// FOR UPDATE lock from findById — otherwise it takes the FK lock on a
|
||||
// separate pool connection and self-deadlocks against our own tx.
|
||||
await this.watcherService.addPageWatchers(
|
||||
contributorIds,
|
||||
pageId,
|
||||
lockedPage.spaceId,
|
||||
lockedPage.workspaceId,
|
||||
trx,
|
||||
);
|
||||
|
||||
// #370 — every job on this queue is a trailing idle-flush autosnapshot.
|
||||
await this.pageHistoryRepo.saveHistory(lockedPage, {
|
||||
contributorIds,
|
||||
kind: job.data.kind ?? 'idle',
|
||||
trx,
|
||||
});
|
||||
snapshotWritten = true;
|
||||
this.logger.debug(`History created for page: ${pageId}`);
|
||||
} catch (err) {
|
||||
await this.collabHistory.addContributors(pageId, contributorIds);
|
||||
poppedForRestore = [];
|
||||
throw err;
|
||||
}
|
||||
});
|
||||
} catch (err) {
|
||||
// A throw here means the tx did NOT commit (callback threw, or the commit
|
||||
// itself failed and rolled back). If we popped contributors and the inner
|
||||
// catch did not already restore them, restore now so the retry keeps
|
||||
// attribution. snapshotWritten is irrelevant: it is set before commit, so
|
||||
// it can be true even when the commit rolled the snapshot back.
|
||||
if (poppedForRestore.length) {
|
||||
await this.collabHistory.addContributors(pageId, poppedForRestore);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
// No snapshot written (page vanished / empty-first / unchanged content) →
|
||||
// clear the contributor set for the skip cases and stop.
|
||||
if (!snapshotWritten) {
|
||||
if (!lastHistoryContent && isEmptyParagraphDoc(page.content as any)) {
|
||||
await this.collabHistory.clearContributors(pageId);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
!lastHistory ||
|
||||
!isDeepStrictEqual(lastHistory.content, page.content)
|
||||
) {
|
||||
const contributorIds = await this.collabHistory.popContributors(pageId);
|
||||
|
||||
try {
|
||||
await this.watcherService.addPageWatchers(
|
||||
contributorIds,
|
||||
pageId,
|
||||
page.spaceId,
|
||||
page.workspaceId,
|
||||
);
|
||||
|
||||
await this.pageHistoryRepo.saveHistory(page, { contributorIds });
|
||||
this.logger.debug(`History created for page: ${pageId}`);
|
||||
} catch (err) {
|
||||
await this.collabHistory.addContributors(pageId, contributorIds);
|
||||
throw err;
|
||||
}
|
||||
|
||||
{
|
||||
const mentions = extractMentions(page.content);
|
||||
const pageMentions = extractPageMentions(mentions);
|
||||
const internalLinkSlugIds = extractInternalLinkSlugIds(page.content);
|
||||
@@ -102,7 +178,7 @@ export class HistoryProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
);
|
||||
});
|
||||
|
||||
if (contributorIds.length > 0 && lastHistory?.content) {
|
||||
if (contributorIds.length > 0 && lastHistoryContent) {
|
||||
await this.notificationQueue
|
||||
.add(QueueJob.PAGE_UPDATED, {
|
||||
pageId,
|
||||
|
||||
@@ -35,6 +35,7 @@ import {
|
||||
import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { UserThrottlerGuard } from '../../integrations/throttle/user-throttler.guard';
|
||||
import { AI_CHAT_THROTTLER } from '../../integrations/throttle/throttler-names';
|
||||
@@ -43,6 +44,8 @@ import {
|
||||
AiChatRunHooks,
|
||||
AiChatService,
|
||||
AiChatStreamBody,
|
||||
rowHasInlineParts,
|
||||
hydrateAssistantParts,
|
||||
} from './ai-chat.service';
|
||||
import { AiChatRunService } from './ai-chat-run.service';
|
||||
import { AiTranscriptionService } from './ai-transcription.service';
|
||||
@@ -129,8 +132,39 @@ export class AiChatController {
|
||||
// production. Only touched on the resumable-stream (flag-on) path.
|
||||
private readonly streamRegistry?: AiChatStreamRegistryService,
|
||||
private readonly environment?: EnvironmentService,
|
||||
// #492: reconstruct a #492 mid-run record's parts from the steps table before
|
||||
// returning rows to the client / export. OPTIONAL so positional controller
|
||||
// specs compile unchanged; when absent, hydration is skipped (old-era rows
|
||||
// already carry inline parts, so nothing to reconstruct).
|
||||
private readonly aiChatRunStepRepo?: AiChatRunStepRepo,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Reconstruct parts for any assistant rows that don't carry them INLINE — a
|
||||
* #492 mid-run record whose per-step parts live in `ai_chat_run_steps` (the
|
||||
* append-persist backend). Every FINISHED row (old-era + #492) and every old-era
|
||||
* streaming snapshot already has inline `metadata.parts`, so the common path
|
||||
* fetches NOTHING and returns the rows untouched; only an actively-streaming
|
||||
* new-style row triggers the batch step fetch. Consumers (seed/poll/export) read
|
||||
* `metadata.parts` off the returned rows exactly as before — the era switch is
|
||||
* invisible to them (reconstructRunParts contract).
|
||||
*/
|
||||
private async withReconstructedParts(
|
||||
rows: AiChatMessage[],
|
||||
workspaceId: string,
|
||||
): Promise<AiChatMessage[]> {
|
||||
if (!this.aiChatRunStepRepo) return rows;
|
||||
const needy = rows.filter(
|
||||
(r) => r.role === 'assistant' && !rowHasInlineParts(r),
|
||||
);
|
||||
if (needy.length === 0) return rows;
|
||||
const stepsByMessage = await this.aiChatRunStepRepo.findByMessageIds(
|
||||
needy.map((r) => r.id),
|
||||
workspaceId,
|
||||
);
|
||||
return hydrateAssistantParts(rows, stepsByMessage);
|
||||
}
|
||||
|
||||
/** List the requesting user's chats in this workspace (paginated). */
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('chats')
|
||||
@@ -184,11 +218,17 @@ export class AiChatController {
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
) {
|
||||
await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
return this.aiChatMessageRepo.findByChat(
|
||||
const page = await this.aiChatMessageRepo.findByChat(
|
||||
dto.chatId,
|
||||
workspace.id,
|
||||
pagination,
|
||||
);
|
||||
// #492: reconstruct parts for any active new-style row so the client seed sees
|
||||
// `metadata.parts` unchanged (a no-op for the finished rows that fill a page).
|
||||
return {
|
||||
...page,
|
||||
items: await this.withReconstructedParts(page.items, workspace.id),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -225,7 +265,10 @@ export class AiChatController {
|
||||
workspace.id,
|
||||
);
|
||||
return {
|
||||
rows,
|
||||
// #492: the delta of an actively-streaming new-style row carries its parts
|
||||
// reconstructed from the steps table, so the degraded poll shows persisted
|
||||
// progress exactly as the pre-#492 full-row snapshot did.
|
||||
rows: await this.withReconstructedParts(rows, workspace.id),
|
||||
cursor,
|
||||
run: run ? { id: run.id, status: run.status } : null,
|
||||
};
|
||||
@@ -247,8 +290,10 @@ export class AiChatController {
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<{ markdown: string }> {
|
||||
const chat = await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
const rows = await this.aiChatMessageRepo.findAllByChat(
|
||||
dto.chatId,
|
||||
const rows = await this.withReconstructedParts(
|
||||
await this.aiChatMessageRepo.findAllByChat(dto.chatId, workspace.id),
|
||||
// #492: an interrupted-but-still-active turn exports its persisted steps
|
||||
// (reconstructed from the steps table) just like the pre-#492 full row did.
|
||||
workspace.id,
|
||||
);
|
||||
const markdown = buildChatMarkdown({
|
||||
@@ -288,7 +333,13 @@ export class AiChatController {
|
||||
workspace.id,
|
||||
)
|
||||
: undefined;
|
||||
return { run, message: message ?? null };
|
||||
// #492: reconnect to an IN-FLIGHT run reconstructs the projection row's parts
|
||||
// from the steps table (the row itself carries only the step marker mid-run);
|
||||
// a finished run's row already has inline parts, so this is a no-op.
|
||||
const [hydrated] = message
|
||||
? await this.withReconstructedParts([message], workspace.id)
|
||||
: [undefined];
|
||||
return { run, message: hydrated ?? null };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -22,6 +22,7 @@ import { AiSettingsService } from '../../integrations/ai/ai-settings.service';
|
||||
import { describeProviderError } from '../../integrations/ai/ai-error.util';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatPageSnapshotRepo } from '@docmost/db/repos/ai-chat/ai-chat-page-snapshot.repo';
|
||||
import { AiAgentRoleRepo } from '@docmost/db/repos/ai-agent-roles/ai-agent-roles.repo';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
@@ -189,10 +190,11 @@ export function stepBudgetWarning(stepNumber: number): string {
|
||||
//
|
||||
// `system` is the in-scope system prompt; we CONCATENATE so the original
|
||||
// persona/context is preserved — a bare `system` override would REPLACE the
|
||||
// whole system prompt for the step. `activatedTools` is PER-TURN mutable state
|
||||
// owned by the streaming loop (a closure Set grown by loadTools); it is passed
|
||||
// in (not module-global, not persisted) so this stays a pure function of its
|
||||
// arguments.
|
||||
// whole system prompt for the step. `activatedTools` is a closure Set grown by
|
||||
// loadTools and owned by the streaming loop; the caller seeds it from and
|
||||
// persists it to the chat's metadata across turns (#490), but this function only
|
||||
// READS the Set it is handed, so it stays a pure function of its arguments (not
|
||||
// module-global).
|
||||
//
|
||||
// NOTE: at AI SDK v7 the per-step `system` field is renamed to `instructions`.
|
||||
// On v6 (`^6.0.134`) `system` is the correct field — adjust when bumping.
|
||||
@@ -517,6 +519,12 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// constructions compile unchanged; Nest always injects the real singleton, so
|
||||
// reconcile sees the SAME in-memory active/zombie maps the runner mutates.
|
||||
private readonly aiChatRunService?: AiChatRunService,
|
||||
// #492 append-persist: per-step INSERT into the lightweight steps table (the
|
||||
// O(Σ steps) replacement for the O(n²) full-row `metadata.parts` rewrite).
|
||||
// OPTIONAL so existing positional constructions (int-specs) compile unchanged;
|
||||
// Nest injects the real singleton. When ABSENT the per-step path falls back to
|
||||
// the pre-#492 full-row flush (no regression, only no WAL win).
|
||||
private readonly aiChatRunStepRepo?: AiChatRunStepRepo,
|
||||
) {}
|
||||
|
||||
// #487: periodic reconcile timer (single-process phase 1). Started in
|
||||
@@ -1113,8 +1121,34 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
chatId,
|
||||
workspace.id,
|
||||
);
|
||||
// #492: HYDRATE needy assistant rows from the steps table BEFORE the replay
|
||||
// map. A #492 mid-run assistant row carries only a step marker
|
||||
// (metadata.parts:[]); its real per-step parts live in `ai_chat_run_steps`.
|
||||
// The graceful terminal callbacks (onFinish/onError/onAbort -> flushAssistant)
|
||||
// assemble the full inline parts, so a normally-ended turn already has them.
|
||||
// But a HARD crash mid-run (SIGKILL/OOM) fires NO terminal callback, so the
|
||||
// row stays parts:[]; without this, rowToUiMessage falls back to an empty
|
||||
// text part and the partial tool-calls/results/text — durable in the steps
|
||||
// table — would DROP OUT of the model's replay context (regressing #183
|
||||
// step-granular durability for the model consumer). Mirrors the controller's
|
||||
// withReconstructedParts EXACTLY (same needy predicate + hydration helper).
|
||||
// Guarded on the optional repo: absent (positional test builds) degrades to
|
||||
// the current behavior rather than crashing.
|
||||
let replayHistory = oldHistory;
|
||||
if (this.aiChatRunStepRepo) {
|
||||
const needy = oldHistory.filter(
|
||||
(r) => r.role === 'assistant' && !rowHasInlineParts(r),
|
||||
);
|
||||
if (needy.length > 0) {
|
||||
const stepsByMessage = await this.aiChatRunStepRepo.findByMessageIds(
|
||||
needy.map((r) => r.id),
|
||||
workspace.id,
|
||||
);
|
||||
replayHistory = hydrateAssistantParts(oldHistory, stepsByMessage);
|
||||
}
|
||||
}
|
||||
const uiMessages: Array<Omit<UIMessage, 'id'> & { id: string }> = [
|
||||
...oldHistory.map(rowToUiMessage),
|
||||
...replayHistory.map(rowToUiMessage),
|
||||
{
|
||||
id: 'pending-user',
|
||||
role: 'user',
|
||||
@@ -1153,7 +1187,9 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// hint — confirm it against the persisted history (the preceding assistant
|
||||
// turn must really be aborted/streaming) so a spoofed flag cannot inject the
|
||||
// interrupt note onto an ordinary turn. The partial output the model needs is
|
||||
// already in `messages` (the aborted assistant row replays via findRecent).
|
||||
// already in `messages`: a #492 mid-run row's per-step parts live only in the
|
||||
// `ai_chat_run_steps` table and were hydrated into the replay history above,
|
||||
// so the aborted assistant turn replays WITH its partial parts intact.
|
||||
// Append the new user turn (shape-only) so index -2 is the prior assistant.
|
||||
const interrupted = isInterruptResume(
|
||||
[...oldHistory, { role: 'user', status: null, metadata: null }],
|
||||
@@ -1410,10 +1446,11 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
const baseTools = { ...external.tools, ...docmostTools };
|
||||
|
||||
// Deferred tool loading state (#332), scoped to THIS streaming loop:
|
||||
// - `activatedTools` is per-TURN mutable state — a fresh closure Set created
|
||||
// per streamText call, NOT module-global and NOT persisted, so a new turn
|
||||
// starts cold. loadTools.execute adds to it; prepareAgentStep reads it to
|
||||
// widen `activeTools` on the NEXT step.
|
||||
// - `activatedTools` is a fresh closure Set per streamText call (not
|
||||
// module-global), SEEDED from the chat's persisted metadata.activatedTools
|
||||
// (#490, just below) so activation carries across turns. loadTools.execute
|
||||
// adds to it; prepareAgentStep reads it to widen `activeTools` on the NEXT
|
||||
// step; turn end persists it back.
|
||||
// - `validDeferredNames` = every tool that is NOT core (the in-app deferred
|
||||
// tools + ALL external MCP tools), computed from the ACTUAL toolset so an
|
||||
// external tool is loadable by its namespaced name. loadTools rejects any
|
||||
@@ -1557,17 +1594,57 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// connection when finalize runs, so the SQL `WHERE status='streaming'`
|
||||
// (not this flag) is what prevents it clobbering the terminal row.
|
||||
if (finalized) return null;
|
||||
// Build the flush ONCE so the returned count is EXACTLY the persisted
|
||||
// `stepsPersisted` (both derive from capturedSteps.length at this instant).
|
||||
const flushed = flushAssistant(capturedSteps, '', 'streaming', {
|
||||
pageChanged,
|
||||
partsCache,
|
||||
});
|
||||
const stepsPersisted = flushed.metadata.stepsPersisted as number;
|
||||
// The count derives from capturedSteps.length at THIS instant, so the
|
||||
// returned value is EXACTLY the persisted `stepsPersisted` the ring rotates
|
||||
// on (whether we take the append-persist path or the legacy fallback).
|
||||
const stepsPersisted = capturedSteps.length;
|
||||
try {
|
||||
await this.aiChatMessageRepo.update(assistantId, workspace.id, flushed, {
|
||||
onlyIfStreaming: true,
|
||||
});
|
||||
if (this.aiChatRunStepRepo) {
|
||||
// #492 APPEND-PERSIST: write only THIS finished step's parts to the
|
||||
// steps table (O(step) WAL), then bump the row's CHEAP step marker —
|
||||
// NO growing `metadata.parts` blob (that O(n²) full-row rewrite is
|
||||
// exactly what this removes). The full `metadata.parts` is assembled
|
||||
// once at finalize; a mid-run resume seed is reconstructed from the
|
||||
// step rows (reconstructRunParts). The INSERT is idempotent
|
||||
// (ON CONFLICT DO NOTHING), so a re-fired step never doubles the parts.
|
||||
const index = stepsPersisted - 1;
|
||||
if (index >= 0) {
|
||||
const stepParts = assistantParts(
|
||||
[capturedSteps[index]],
|
||||
'',
|
||||
partsCache,
|
||||
);
|
||||
await this.aiChatRunStepRepo.insertStep(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
index,
|
||||
stepParts,
|
||||
);
|
||||
}
|
||||
// Marker UPDATE: advance stepsPersisted + keep the toolTrace era marker
|
||||
// (bumps updatedAt so the delta poll observes the step, and carries the
|
||||
// frontier a resuming client attaches from). Scoped onlyIfStreaming so a
|
||||
// late marker never clobbers the terminal finalize.
|
||||
await this.aiChatMessageRepo.update(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
{ metadata: stepMarkerMetadata(stepsPersisted) },
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
} else {
|
||||
// Legacy fallback (no steps table wired — positional test builds): the
|
||||
// pre-#492 full-row flush, so parts still land inline on the row.
|
||||
const flushed = flushAssistant(capturedSteps, '', 'streaming', {
|
||||
pageChanged,
|
||||
partsCache,
|
||||
});
|
||||
await this.aiChatMessageRepo.update(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
flushed,
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
return stepsPersisted;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
@@ -2747,6 +2824,122 @@ export function rowToUiMessage(row: AiChatMessage): Omit<UIMessage, 'id'> & {
|
||||
return { id: row.id, role, parts: parts as UIMessage['parts'] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap step-marker metadata for the #492 per-step UPDATE. Advances
|
||||
* `stepsPersisted` (the resume attach frontier) and keeps the `toolTraceVersion`
|
||||
* era marker, WITHOUT the growing `parts` blob (those live in the steps table
|
||||
* now; the full `metadata.parts` is assembled once at finalize by flushAssistant).
|
||||
* `parts: []` is kept for shape stability — it reads as an empty inline-parts row,
|
||||
* which is exactly the discriminator that routes reconstruction to the steps table.
|
||||
*/
|
||||
export function stepMarkerMetadata(
|
||||
stepsPersisted: number,
|
||||
): Record<string, unknown> {
|
||||
return { parts: [], toolTraceVersion: 2, stepsPersisted };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an assistant row already carries its full UI parts INLINE on the row
|
||||
* (`metadata.parts`). TRUE for every FINISHED row — old-era rows AND #492 rows,
|
||||
* whose full parts are assembled once at finalize — and for old-era streaming
|
||||
* snapshots (the pre-#492 per-step full-row flush). FALSE for a #492 MID-RUN
|
||||
* record, whose per-step parts live in the `ai_chat_run_steps` table. This is the
|
||||
* era discriminator the reconstruct seam branches on — no schema flag needed.
|
||||
*/
|
||||
export function rowHasInlineParts(row: { metadata?: unknown }): boolean {
|
||||
const meta = (row.metadata ?? {}) as { parts?: unknown };
|
||||
return Array.isArray(meta.parts) && meta.parts.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenate persisted per-step parts (in `stepIndex` order) into the turn's UI
|
||||
* parts (#492). Reproduces EXACTLY what flushAssistant → assistantParts would have
|
||||
* written to `metadata.parts` for those finished steps, since each step row stored
|
||||
* `assistantParts([step])` at persist time.
|
||||
*/
|
||||
export function assembleStepParts(
|
||||
stepRows: ReadonlyArray<{ stepIndex: number; parts: unknown }>,
|
||||
): UIMessage['parts'] {
|
||||
const parts: Array<Record<string, unknown>> = [];
|
||||
for (const step of [...stepRows].sort((a, b) => a.stepIndex - b.stepIndex)) {
|
||||
if (Array.isArray(step.parts)) {
|
||||
parts.push(...(step.parts as Array<Record<string, unknown>>));
|
||||
}
|
||||
}
|
||||
return parts as UIMessage['parts'];
|
||||
}
|
||||
|
||||
/**
|
||||
* reconstructRunParts (#492) — the single backend-switch seam. Given an assistant
|
||||
* ROW and its persisted step rows, return the turn's UI `parts` + the persisted
|
||||
* step count, reading from the ROW when it already carries inline parts (old-era
|
||||
* records AND every finished record) and from the STEPS TABLE otherwise (a #492
|
||||
* mid-run record). The higher-level consumers (attach seed, delta poll, export)
|
||||
* route their row→parts through this / {@link hydrateAssistantParts}, so old and
|
||||
* new records reconstruct identically WITHOUT the consumers branching on the era.
|
||||
*/
|
||||
export function reconstructRunParts(
|
||||
row: { metadata?: unknown; content?: string | null },
|
||||
stepRows: ReadonlyArray<{ stepIndex: number; parts: unknown }>,
|
||||
): { parts: UIMessage['parts']; stepsPersisted: number } {
|
||||
if (rowHasInlineParts(row)) {
|
||||
const meta = row.metadata as {
|
||||
parts: UIMessage['parts'];
|
||||
stepsPersisted?: number;
|
||||
};
|
||||
return {
|
||||
parts: meta.parts,
|
||||
stepsPersisted:
|
||||
typeof meta.stepsPersisted === 'number'
|
||||
? meta.stepsPersisted
|
||||
: stepRows.length,
|
||||
};
|
||||
}
|
||||
if (stepRows.length > 0) {
|
||||
return {
|
||||
parts: assembleStepParts(stepRows),
|
||||
stepsPersisted: stepRows.length,
|
||||
};
|
||||
}
|
||||
// No inline parts and no step rows: an old-era seed / empty streaming row. Fall
|
||||
// back to a single text part from `content` (mirrors rowToUiMessage).
|
||||
return {
|
||||
parts: textPart(row.content ?? '') as UIMessage['parts'],
|
||||
stepsPersisted: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill each assistant row's `metadata.parts` from its step rows when the row does
|
||||
* not already carry them inline (a #492 mid-run record), so a consumer that reads
|
||||
* `metadata.parts` off the RAW row (the client seed/poll, the Markdown export)
|
||||
* sees the reconstructed parts with NO change to itself. Rows that already have
|
||||
* inline parts (old-era + finished) and non-assistant rows pass through untouched.
|
||||
* Pure: returns new row objects, never mutates the inputs.
|
||||
*/
|
||||
export function hydrateAssistantParts<
|
||||
T extends { id: string; role?: string; metadata?: unknown },
|
||||
>(
|
||||
rows: ReadonlyArray<T>,
|
||||
stepsByMessage: Map<
|
||||
string,
|
||||
ReadonlyArray<{ stepIndex: number; parts: unknown }>
|
||||
>,
|
||||
): T[] {
|
||||
return rows.map((row) => {
|
||||
if (row.role !== 'assistant' || rowHasInlineParts(row)) return row;
|
||||
const steps = stepsByMessage.get(row.id);
|
||||
if (!steps || steps.length === 0) return row;
|
||||
return {
|
||||
...row,
|
||||
metadata: {
|
||||
...((row.metadata ?? {}) as Record<string, unknown>),
|
||||
parts: assembleStepParts(steps),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The persisted-row patch shape produced by {@link flushAssistant}. It is the
|
||||
* SAME shape the assistant repo insert/update consume (content + toolCalls +
|
||||
|
||||
@@ -32,6 +32,7 @@ import { TemplateRepo } from '@docmost/db/repos/template/template.repo';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunRepo } from '@docmost/db/repos/ai-chat/ai-chat-run.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatPageSnapshotRepo } from '@docmost/db/repos/ai-chat/ai-chat-page-snapshot.repo';
|
||||
import { AiProviderCredentialsRepo } from '@docmost/db/repos/ai-chat/ai-provider-credentials.repo';
|
||||
import { AiMcpServerRepo } from '@docmost/db/repos/ai-chat/ai-mcp-server.repo';
|
||||
@@ -125,6 +126,7 @@ import { firstSqlToken } from '../integrations/metrics/metrics.constants';
|
||||
AiChatRepo,
|
||||
AiChatMessageRepo,
|
||||
AiChatRunRepo,
|
||||
AiChatRunStepRepo,
|
||||
AiChatPageSnapshotRepo,
|
||||
AiProviderCredentialsRepo,
|
||||
AiMcpServerRepo,
|
||||
@@ -161,6 +163,7 @@ import { firstSqlToken } from '../integrations/metrics/metrics.constants';
|
||||
AiChatRepo,
|
||||
AiChatMessageRepo,
|
||||
AiChatRunRepo,
|
||||
AiChatRunStepRepo,
|
||||
AiChatPageSnapshotRepo,
|
||||
AiProviderCredentialsRepo,
|
||||
AiMcpServerRepo,
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { type Kysely } from 'kysely';
|
||||
|
||||
/**
|
||||
* #370 — page-versioning intentionality tier on a history snapshot.
|
||||
*
|
||||
* Adds `page_history.kind`, the three-tier "how intentional was this snapshot"
|
||||
* marker that lets versions (intentional points) be told apart from autosaves:
|
||||
* - 'manual' — a human explicitly saved a version (Cmd+S / Save button)
|
||||
* - 'agent' — the AI agent explicitly saved a version
|
||||
* - 'idle' — trailing idle-flush autosnapshot (safety net)
|
||||
* - 'boundary' — autosnapshot pinned on a source transition (user↔agent↔git)
|
||||
*
|
||||
* Nullable with NO default (mirrors last_updated_source in the agent-provenance
|
||||
* migration): legacy rows predate the marker and read back as `null`, which the
|
||||
* client renders as a plain autosave. Stored as a short varchar to stay
|
||||
* forward-compatible without an enum migration.
|
||||
*/
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.alterTable('page_history')
|
||||
.addColumn('kind', 'varchar(20)', (col) => col)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.alterTable('page_history').dropColumn('kind').execute();
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { type Kysely, sql } from 'kysely';
|
||||
|
||||
/**
|
||||
* `ai_chat_run_steps` — append-only per-step persistence for an assistant turn
|
||||
* (#492 wave C). Each finished agent step's UI `parts` (its text part + a part
|
||||
* per tool call, WITH the tool output) is INSERTed as its own lightweight row the
|
||||
* moment the step ends, instead of REWRITING the whole assistant row's growing
|
||||
* `metadata.parts` jsonb on every `onStepFinish`.
|
||||
*
|
||||
* WHY a separate table + INSERT (not a jsonb `||` append on the message row): a
|
||||
* Postgres jsonb UPDATE rewrites the ENTIRE TOASTed row version under MVCC, so
|
||||
* re-persisting a growing `metadata.parts` on every step is O(n²) write volume
|
||||
* (a 50-step run with ~100 KB tool outputs wrote hundreds of MB of WAL / dead
|
||||
* tuples per turn, hammering autovacuum). `||` would only shave the network
|
||||
* payload — the WAL/TOAST rewrite harm remains. An INSERT into a per-step table
|
||||
* writes ONLY that step's bytes, so the per-turn write volume is O(Σ steps).
|
||||
*
|
||||
* The full `metadata.parts` on the message row is assembled ONCE at finalize (the
|
||||
* terminal completed/error/aborted write). Mid-run, a resuming client's seed is
|
||||
* reconstructed by concatenating these step rows in `step_index` order — which
|
||||
* reproduces exactly what the old per-step full-row rewrite persisted. Records
|
||||
* written the OLD way (full `metadata.parts` on the row, no step rows) still
|
||||
* reconstruct from the row unchanged; the two eras are distinguished by whether
|
||||
* the row already carries non-empty `metadata.parts` (see reconstructRunParts /
|
||||
* assembleStepParts in ai-chat.service.ts).
|
||||
*
|
||||
* ON DELETE CASCADE on `message_id`: the step rows are a derived projection of the
|
||||
* assistant message; they must vanish with it (or with its workspace).
|
||||
*/
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.createTable('ai_chat_run_steps')
|
||||
.ifNotExists()
|
||||
.addColumn('id', 'uuid', (col) =>
|
||||
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
|
||||
)
|
||||
// The assistant message row this step belongs to (the #183 projection). The
|
||||
// step rows are a derived, per-step slice of that message, so they cascade.
|
||||
.addColumn('message_id', 'uuid', (col) =>
|
||||
col.references('ai_chat_messages.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('workspace_id', 'uuid', (col) =>
|
||||
col.references('workspaces.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
// 0-based index of the finished step within the turn. Ordering key for
|
||||
// reconstruction; unique per message (idempotent step re-persist).
|
||||
.addColumn('step_index', 'integer', (col) => col.notNull())
|
||||
// The step's UI parts (text part + a `tool-*` part per call, WITH output).
|
||||
// Concatenated in step order to rebuild the turn's `metadata.parts`.
|
||||
.addColumn('parts', 'jsonb', (col) => col.notNull())
|
||||
.addColumn('created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.execute();
|
||||
|
||||
// Idempotent per-step persist: a retried INSERT of the same (message, step)
|
||||
// is a no-op (the service uses ON CONFLICT DO NOTHING). This also serves the
|
||||
// reconstruction read (WHERE message_id ORDER BY step_index).
|
||||
await db.schema
|
||||
.createIndex('ai_chat_run_steps_message_step_uidx')
|
||||
.ifNotExists()
|
||||
.on('ai_chat_run_steps')
|
||||
.columns(['message_id', 'step_index'])
|
||||
.unique()
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.dropTable('ai_chat_run_steps').ifExists().execute();
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '../../types/kysely.types';
|
||||
import { dbOrTx } from '../../utils';
|
||||
import { AiChatRunStep } from '@docmost/db/types/entity.types';
|
||||
|
||||
/**
|
||||
* Append-only per-step persistence for an assistant turn (#492). Each finished
|
||||
* agent step's UI `parts` (its text part + a `tool-*` part per call, WITH the
|
||||
* tool output) is INSERTed as its own lightweight row the moment the step ends —
|
||||
* instead of REWRITING the assistant row's growing `metadata.parts` jsonb on every
|
||||
* `onStepFinish` (a Postgres jsonb UPDATE rewrites the whole TOASTed row version
|
||||
* under MVCC, so that was O(n²) WAL/dead-tuple churn per turn).
|
||||
*
|
||||
* The full `metadata.parts` on the message row is assembled ONCE at finalize;
|
||||
* mid-run, a resuming client's seed is rebuilt from these rows in `stepIndex`
|
||||
* order (see `assembleStepParts` / the reconstruct seam in ai-chat.service.ts).
|
||||
* Every method is workspace-scoped as defense-in-depth.
|
||||
*/
|
||||
@Injectable()
|
||||
export class AiChatRunStepRepo {
|
||||
constructor(@InjectKysely() private readonly db: KyselyDB) {}
|
||||
|
||||
/**
|
||||
* Append one finished step's parts. Idempotent: a retried persist of the SAME
|
||||
* (message, stepIndex) is a no-op via ON CONFLICT DO NOTHING — the per-step
|
||||
* writes are fired fire-and-forget + serialized, and a duplicate must never
|
||||
* throw into the stream or double the parts. Returns whether a NEW row landed
|
||||
* (false = the step was already persisted).
|
||||
*/
|
||||
async insertStep(
|
||||
messageId: string,
|
||||
workspaceId: string,
|
||||
stepIndex: number,
|
||||
parts: unknown,
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<boolean> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
const inserted = await db
|
||||
.insertInto('aiChatRunSteps')
|
||||
.values({
|
||||
messageId,
|
||||
workspaceId,
|
||||
stepIndex,
|
||||
// jsonb column: cast through never (same pattern as the message repo).
|
||||
parts: parts as never,
|
||||
})
|
||||
.onConflict((oc) => oc.columns(['messageId', 'stepIndex']).doNothing())
|
||||
.returning('id')
|
||||
.executeTakeFirst();
|
||||
return inserted !== undefined;
|
||||
}
|
||||
|
||||
/** All persisted steps for ONE assistant message, in step order. */
|
||||
async findByMessage(
|
||||
messageId: string,
|
||||
workspaceId: string,
|
||||
): Promise<AiChatRunStep[]> {
|
||||
return this.db
|
||||
.selectFrom('aiChatRunSteps')
|
||||
.selectAll('aiChatRunSteps')
|
||||
.where('messageId', '=', messageId)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.orderBy('stepIndex', 'asc')
|
||||
.execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* All persisted steps for a SET of assistant messages, grouped by messageId
|
||||
* (each group in step order). One query for the batch — the hydration seam
|
||||
* (getMessages / delta / export) calls this only for the rows that actually
|
||||
* need reconstruction (an active new-style row whose `metadata.parts` is still
|
||||
* empty), which is usually none, so this is skipped on the common path.
|
||||
*/
|
||||
async findByMessageIds(
|
||||
messageIds: string[],
|
||||
workspaceId: string,
|
||||
): Promise<Map<string, AiChatRunStep[]>> {
|
||||
const byMessage = new Map<string, AiChatRunStep[]>();
|
||||
if (messageIds.length === 0) return byMessage;
|
||||
const rows = await this.db
|
||||
.selectFrom('aiChatRunSteps')
|
||||
.selectAll('aiChatRunSteps')
|
||||
.where('messageId', 'in', messageIds)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.orderBy('stepIndex', 'asc')
|
||||
.execute();
|
||||
for (const row of rows) {
|
||||
const list = byMessage.get(row.messageId);
|
||||
if (list) list.push(row);
|
||||
else byMessage.set(row.messageId, [row]);
|
||||
}
|
||||
return byMessage;
|
||||
}
|
||||
}
|
||||
@@ -13,6 +13,7 @@ import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
|
||||
import { ExpressionBuilder, sql } from 'kysely';
|
||||
import { DB } from '@docmost/db/types/db';
|
||||
import { resolveAgentProvenance } from '../agent-provenance';
|
||||
import { PageHistoryKind } from '../../../collaboration/constants';
|
||||
|
||||
/**
|
||||
* Role-resolution subquery for a page-history row's bound AI chat (#300). Joins
|
||||
@@ -46,6 +47,9 @@ export class PageHistoryRepo {
|
||||
'lastUpdatedById',
|
||||
'lastUpdatedSource',
|
||||
'lastUpdatedAiChatId',
|
||||
// #370 — intentionality tier ('manual' | 'agent' | 'idle' | 'boundary');
|
||||
// null on legacy rows (= autosave). Selected so callers can read/promote it.
|
||||
'kind',
|
||||
'contributorIds',
|
||||
'spaceId',
|
||||
'workspaceId',
|
||||
@@ -85,9 +89,15 @@ export class PageHistoryRepo {
|
||||
|
||||
async saveHistory(
|
||||
page: Page,
|
||||
opts?: { contributorIds?: string[]; trx?: KyselyTransaction },
|
||||
): Promise<void> {
|
||||
await this.insertPageHistory(
|
||||
opts?: {
|
||||
contributorIds?: string[];
|
||||
// #370 — intentionality tier for this snapshot. Omitted → null (legacy
|
||||
// autosave semantics). Callers derive it server-side, never from a client.
|
||||
kind?: PageHistoryKind;
|
||||
trx?: KyselyTransaction;
|
||||
},
|
||||
): Promise<PageHistory> {
|
||||
return await this.insertPageHistory(
|
||||
{
|
||||
pageId: page.id,
|
||||
slugId: page.slugId,
|
||||
@@ -99,6 +109,7 @@ export class PageHistoryRepo {
|
||||
// Copy the provenance marker off the page row, as for lastUpdatedById.
|
||||
lastUpdatedSource: page.lastUpdatedSource,
|
||||
lastUpdatedAiChatId: page.lastUpdatedAiChatId,
|
||||
kind: opts?.kind ?? null,
|
||||
contributorIds: opts?.contributorIds,
|
||||
spaceId: page.spaceId,
|
||||
workspaceId: page.workspaceId,
|
||||
@@ -107,6 +118,25 @@ export class PageHistoryRepo {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* #370 — promote an existing snapshot's intentionality tier in place. Used by
|
||||
* the manual-save "promote-not-dup" path: when the latest history row already
|
||||
* holds the exact content being versioned, we upgrade its `kind` instead of
|
||||
* duplicating a heavy content row.
|
||||
*/
|
||||
async updateHistoryKind(
|
||||
pageHistoryId: string,
|
||||
kind: PageHistoryKind,
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<void> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
await db
|
||||
.updateTable('pageHistory')
|
||||
.set({ kind })
|
||||
.where('id', '=', pageHistoryId)
|
||||
.execute();
|
||||
}
|
||||
|
||||
async findPageHistoryByPageId(pageId: string, pagination: PaginationOptions) {
|
||||
const query = this.db
|
||||
.selectFrom('pageHistory')
|
||||
|
||||
+18
@@ -280,6 +280,7 @@ export interface PageHistory {
|
||||
createdAt: Generated<Timestamp>;
|
||||
icon: string | null;
|
||||
id: Generated<string>;
|
||||
kind: string | null;
|
||||
lastUpdatedAiChatId: string | null;
|
||||
lastUpdatedById: string | null;
|
||||
lastUpdatedSource: string | null;
|
||||
@@ -691,6 +692,22 @@ export interface AiChatRuns {
|
||||
updatedAt: Generated<Timestamp>;
|
||||
}
|
||||
|
||||
// Append-only per-step persistence for an assistant turn (#492). Mirrors
|
||||
// migration 20260708T120000-ai-chat-run-steps.ts. Each finished agent step's UI
|
||||
// `parts` are INSERTed as their own row (instead of rewriting the message row's
|
||||
// growing `metadata.parts` jsonb every step — an O(n²) WAL/TOAST churn). The full
|
||||
// `metadata.parts` is assembled once at finalize; mid-run a resuming client's seed
|
||||
// is rebuilt by concatenating these rows in `stepIndex` order. Cascades with the
|
||||
// assistant message row it projects.
|
||||
export interface AiChatRunSteps {
|
||||
id: Generated<string>;
|
||||
messageId: string;
|
||||
workspaceId: string;
|
||||
stepIndex: number;
|
||||
parts: Json;
|
||||
createdAt: Generated<Timestamp>;
|
||||
}
|
||||
|
||||
// Per-(chat,page) snapshot of the open page's Markdown at the END of the agent's
|
||||
// previous turn (#274). Mirrors migration 20260702T120000-ai-chat-page-snapshot.ts.
|
||||
// The next turn diffs the CURRENT Markdown against `contentMd` to surface edits a
|
||||
@@ -728,6 +745,7 @@ export interface DB {
|
||||
aiChats: AiChats;
|
||||
aiChatMessages: AiChatMessages;
|
||||
aiChatRuns: AiChatRuns;
|
||||
aiChatRunSteps: AiChatRunSteps;
|
||||
aiChatPageSnapshots: AiChatPageSnapshots;
|
||||
apiKeys: ApiKeys;
|
||||
attachments: Attachments;
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
AiChats,
|
||||
AiChatMessages,
|
||||
AiChatRuns,
|
||||
AiChatRunSteps,
|
||||
AiChatPageSnapshots,
|
||||
Attachments,
|
||||
Comments,
|
||||
@@ -64,6 +65,12 @@ export type InsertableAiChatMessage = Omit<Insertable<AiChatMessages>, 'tsv'>;
|
||||
export type AiChatRun = Selectable<AiChatRuns>;
|
||||
export type InsertableAiChatRun = Insertable<AiChatRuns>;
|
||||
|
||||
// AI Chat Run Step (#492): append-only per-step parts persistence. Each finished
|
||||
// agent step's UI parts are stored as their own row; the full turn's parts are
|
||||
// assembled from these (in stepIndex order) for a mid-run resume seed.
|
||||
export type AiChatRunStep = Selectable<AiChatRunSteps>;
|
||||
export type InsertableAiChatRunStep = Insertable<AiChatRunSteps>;
|
||||
|
||||
// AI Chat Page Snapshot (#274): per-(chat,page) Markdown snapshot taken at the
|
||||
// end of the agent's previous turn, diffed against the current page next turn to
|
||||
// detect human edits made between turns.
|
||||
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
// p-limit and @sindresorhus/slugify are ESM-only and not in jest's transform
|
||||
// allowlist; both are irrelevant to createDrawioSvg (a pure fs + string method),
|
||||
// so they are mocked out to keep the module graph loadable under ts-jest.
|
||||
jest.mock('p-limit', () => ({
|
||||
__esModule: true,
|
||||
default: () => (fn: () => unknown) => fn(),
|
||||
}));
|
||||
jest.mock('@sindresorhus/slugify', () => ({
|
||||
__esModule: true,
|
||||
default: (input: string) => String(input),
|
||||
}));
|
||||
|
||||
import { promises as fs } from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { ImportAttachmentService } from './import-attachment.service';
|
||||
|
||||
/**
|
||||
* Unit test for ImportAttachmentService.createDrawioSvg (issue #507).
|
||||
*
|
||||
* The Confluence import wraps a `.drawio` file into a `.drawio.svg` attachment.
|
||||
* The `content=` payload MUST be the mxfile XML entity-escaped (draw.io's native
|
||||
* form), NOT base64 — draw.io's editor decodes a base64 content= via Latin-1
|
||||
* atob, mangling every non-ASCII char into mojibake. createDrawioSvg touches no
|
||||
* injected dependency, so the service is built with placeholder deps.
|
||||
*/
|
||||
describe('ImportAttachmentService.createDrawioSvg (#507)', () => {
|
||||
const service = new ImportAttachmentService(
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
);
|
||||
const call = (p: string): Promise<Buffer> =>
|
||||
(service as any).createDrawioSvg(p);
|
||||
|
||||
let tmpDir: string;
|
||||
beforeAll(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'drawio-507-'));
|
||||
});
|
||||
afterAll(async () => {
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
const writeDrawio = async (name: string, xml: string): Promise<string> => {
|
||||
const p = path.join(tmpDir, name);
|
||||
await fs.writeFile(p, xml, 'utf-8');
|
||||
return p;
|
||||
};
|
||||
|
||||
it('writes content= as entity-encoded XML, not base64', async () => {
|
||||
const drawio =
|
||||
'<mxfile host="Confluence"><diagram name="Схема — ёж">' +
|
||||
'<mxGraphModel><root><mxCell id="0"/>' +
|
||||
'<mxCell id="2" value="Старт-бит" vertex="1" parent="0"/>' +
|
||||
'</root></mxGraphModel></diagram></mxfile>';
|
||||
const p = await writeDrawio('cyrillic.drawio', drawio);
|
||||
|
||||
const svg = (await call(p)).toString('utf-8');
|
||||
const content = /content="([^"]*)"/.exec(svg)?.[1];
|
||||
expect(content).toBeDefined();
|
||||
|
||||
// Entity-encoded XML form, starting with <mxfile — never a base64 blob.
|
||||
expect(content).toMatch(/^<mxfile/);
|
||||
expect(content).toContain('<');
|
||||
// Non-ASCII survives as raw UTF-8, with no Latin-1 mojibake.
|
||||
expect(content).toContain('Старт-бит');
|
||||
expect(content).toContain('Схема — ёж');
|
||||
expect(content).not.toContain('Ð');
|
||||
|
||||
// Decoding the attribute (un-escaping) yields the original drawio file.
|
||||
const decoded = content!
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&/g, '&');
|
||||
expect(decoded).toBe(drawio);
|
||||
});
|
||||
|
||||
it('encodes literal tab/newline/CR as numeric char-refs, not literal control chars (#507 F1)', async () => {
|
||||
// A literal tab/newline/CR inside the mxfile XML would be collapsed to a
|
||||
// single space by XML attribute-value normalization when the draw.io editor
|
||||
// reads content=, silently flattening multi-line labels and tab-bearing
|
||||
// values. They must be emitted as numeric char-refs instead.
|
||||
const drawio =
|
||||
'<mxfile><diagram name="p">' +
|
||||
'<mxGraphModel><root><mxCell id="0"/>' +
|
||||
'<mxCell id="2" value="col1\tcol2" style="html=1;\nshadow=0" vertex="1" parent="0"/>' +
|
||||
'<mxCell id="3" value="Line1\nLine2\rLine3" vertex="1" parent="0"/>' +
|
||||
'</root></mxGraphModel></diagram></mxfile>';
|
||||
const p = await writeDrawio('ctrl.drawio', drawio);
|
||||
|
||||
const svg = (await call(p)).toString('utf-8');
|
||||
const content = /content="([^"]*)"/.exec(svg)?.[1];
|
||||
expect(content).toBeDefined();
|
||||
// No literal control chars survive in the attribute value.
|
||||
expect(content).not.toMatch(/[\t\n\r]/);
|
||||
// They round-trip as numeric char-refs.
|
||||
expect(content).toContain('	');
|
||||
expect(content).toContain('
');
|
||||
expect(content).toContain('
');
|
||||
// Decoding (char-refs back to literal, entities back) recovers the file.
|
||||
const decoded = content!
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/	/gi, '\t')
|
||||
.replace(/
/gi, '\n')
|
||||
.replace(/
/gi, '\r')
|
||||
.replace(/&/g, '&');
|
||||
expect(decoded).toBe(drawio);
|
||||
});
|
||||
|
||||
it('escapes XML metacharacters in the drawio payload', async () => {
|
||||
const drawio = '<mxfile><diagram name="a & b">"q" <x></diagram></mxfile>';
|
||||
const p = await writeDrawio('meta.drawio', drawio);
|
||||
|
||||
const svg = (await call(p)).toString('utf-8');
|
||||
const content = /content="([^"]*)"/.exec(svg)?.[1];
|
||||
expect(content).toBeDefined();
|
||||
// The attribute value must contain no bare `<`, `>` or `"` that would break
|
||||
// out of the content="..." attribute or the SVG element.
|
||||
expect(content).not.toMatch(/[<>"]/);
|
||||
const decoded = content!
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&/g, '&');
|
||||
expect(decoded).toBe(drawio);
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,7 @@ import { createReadStream } from 'node:fs';
|
||||
import { promises as fs } from 'fs';
|
||||
import { Readable } from 'stream';
|
||||
import { getMimeType, sanitizeFileName } from '../../../common/helpers';
|
||||
import { htmlEscape } from '../../../common/helpers/html-escaper';
|
||||
import { v7 } from 'uuid';
|
||||
import { FileTask } from '@docmost/db/types/entity.types';
|
||||
import { getAttachmentFolderPath } from '../../../core/attachment/attachment.utils';
|
||||
@@ -849,7 +850,12 @@ export class ImportAttachmentService {
|
||||
): Promise<Buffer> {
|
||||
try {
|
||||
const drawioContent = await fs.readFile(drawioPath, 'utf-8');
|
||||
const drawioBase64 = Buffer.from(drawioContent).toString('base64');
|
||||
// Write the mxfile XML XML-entity-escaped (draw.io's native content= form),
|
||||
// NOT base64. draw.io's editor decodes a base64 content= via Latin-1 atob
|
||||
// (no UTF-8 step), turning every non-ASCII char (Cyrillic, ё, —) into
|
||||
// mojibake; the entity-encoded form is decoded by the DOM as UTF-8 and
|
||||
// opens intact. Docmost's own decoder reads both forms.
|
||||
const drawioEscaped = this.xmlEscapeContent(drawioContent);
|
||||
|
||||
let imageElement = '';
|
||||
// If we have a PNG, include it in the SVG
|
||||
@@ -875,7 +881,7 @@ export class ImportAttachmentService {
|
||||
width="600"
|
||||
height="400"
|
||||
viewBox="0 0 600 400"
|
||||
content="${drawioBase64}">${imageElement}</svg>`;
|
||||
content="${drawioEscaped}">${imageElement}</svg>`;
|
||||
|
||||
return Buffer.from(svgContent, 'utf-8');
|
||||
} catch (error) {
|
||||
@@ -884,6 +890,24 @@ export class ImportAttachmentService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape a string so it is safe as the value of a double-quoted XML attribute
|
||||
* (the `content=` payload of a `.drawio.svg`). The shared `htmlEscape` covers
|
||||
* `& < > " '` (a strict superset of what this attribute needs; the extra `'`
|
||||
* escape is harmless in a `"`-delimited value). On top of that, the numeric
|
||||
* char-refs for tab/newline/CR are required: a literal tab/newline/CR inside
|
||||
* an attribute value is collapsed to a single space by XML attribute-value
|
||||
* normalization on DOM read (both our decoder and the real draw.io editor),
|
||||
* silently flattening multi-line labels and tab-bearing values. Char-refs
|
||||
* survive that normalization (#507).
|
||||
*/
|
||||
private xmlEscapeContent(s: string): string {
|
||||
return htmlEscape(s)
|
||||
.replace(/\t/g, '	')
|
||||
.replace(/\n/g, '
')
|
||||
.replace(/\r/g, '
');
|
||||
}
|
||||
|
||||
private async uploadWithRetry(opts: {
|
||||
abs: string;
|
||||
storageFilePath: string;
|
||||
|
||||
@@ -20,6 +20,10 @@ export interface IStripeSeatsSyncJob {
|
||||
|
||||
export interface IPageHistoryJob {
|
||||
pageId: string;
|
||||
// #370 — intentionality tier the worker stamps on the snapshot. All jobs on
|
||||
// this queue are trailing idle-flush autosnapshots, so this is 'idle' (absent
|
||||
// → treated as 'idle' by the processor).
|
||||
kind?: 'idle';
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
import * as http from 'node:http';
|
||||
import { Kysely } from 'kysely';
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { MockLanguageModelV3, convertArrayToReadableStream } from 'ai/test';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import {
|
||||
AiChatService,
|
||||
assembleStepParts,
|
||||
assistantParts,
|
||||
rowHasInlineParts,
|
||||
stepMarkerMetadata,
|
||||
} from 'src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
createMessage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — the REAL onStep WRITE path (F2) and the model-REPLAY
|
||||
* hydration path (F1), driven through `AiChatService.stream` against a LIVE
|
||||
* Postgres with a REAL `AiChatRunStepRepo` INJECTED. The existing append-persist
|
||||
* int-specs hand-roll the insert+marker cycle via the repos directly and build
|
||||
* the service with `aiChatRunStepRepo: undefined` (only the legacy-fallback branch
|
||||
* is covered), so an off-by-one on `stepsPersisted-1`, a wrong `capturedSteps`
|
||||
* slice, or a broken marker payload would pass all of them. These tests exercise
|
||||
* the actual `updateStreaming` append-persist branch end to end.
|
||||
*
|
||||
* The seam is the injected `model` (a seeded `MockLanguageModelV3` from `ai/test`)
|
||||
* plus a REAL Node `ServerResponse` as the hijacked socket — mirrors
|
||||
* ai-chat-stream.int-spec.ts.
|
||||
*/
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
async function waitFor(
|
||||
cond: () => Promise<boolean> | boolean,
|
||||
{ timeoutMs = 15_000, stepMs = 25 } = {},
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
if (await cond()) return;
|
||||
await sleep(stepMs);
|
||||
}
|
||||
throw new Error('waitFor: condition not met within timeout');
|
||||
}
|
||||
|
||||
// A real Node ServerResponse wired to a live socket (identical helper to the
|
||||
// stream int-spec) so the SDK's pipe/heartbeat writes behave as in prod.
|
||||
function makeRealResponse(): Promise<{
|
||||
res: http.ServerResponse;
|
||||
cleanup: () => Promise<void>;
|
||||
}> {
|
||||
return new Promise((resolve) => {
|
||||
const server = http.createServer((_req, res) => {
|
||||
resolve({
|
||||
res,
|
||||
cleanup: () =>
|
||||
new Promise<void>((done) => {
|
||||
try {
|
||||
if (!res.writableEnded) res.end();
|
||||
} catch {
|
||||
/* socket already gone */
|
||||
}
|
||||
server.close(() => done());
|
||||
}),
|
||||
});
|
||||
});
|
||||
server.listen(0, () => {
|
||||
const port = (server.address() as any).port;
|
||||
const creq = http.request({ port, method: 'GET' }, (cres) => {
|
||||
cres.resume();
|
||||
});
|
||||
creq.on('error', () => undefined);
|
||||
creq.end();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Stream parts for a normal, successful single-step turn.
|
||||
function successStream() {
|
||||
return convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: 't1' },
|
||||
{ type: 'text-delta', id: 't1', delta: 'Hello' },
|
||||
{ type: 'text-delta', id: 't1', delta: ' there' },
|
||||
{ type: 'text-end', id: 't1' },
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
},
|
||||
] as any);
|
||||
}
|
||||
|
||||
// A THREE-step turn: steps 0 and 1 each emit text + an `echo` tool call (the SDK
|
||||
// runs the tool and continues); step 2 answers and stops. Three steps is
|
||||
// deliberate: the LAST finished step's append-persist write races the terminal
|
||||
// finalize (which writes the full inline parts anyway, so a lost last-step row is
|
||||
// by design), but the NON-final steps 0 and 1 always drain to the steps table
|
||||
// before finalize — so those are what the test asserts on deterministically.
|
||||
function threeStepModel(): MockLanguageModelV3 {
|
||||
let step = 0;
|
||||
const toolStep = (i: number) => ({
|
||||
stream: convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: `s${i}` },
|
||||
{ type: 'text-delta', id: `s${i}`, delta: `step ${i} ` },
|
||||
{ type: 'text-end', id: `s${i}` },
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'echo',
|
||||
input: JSON.stringify({ msg: `m${i}` }),
|
||||
},
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'tool-calls',
|
||||
usage: { inputTokens: 5, outputTokens: 3, totalTokens: 8 },
|
||||
},
|
||||
] as any),
|
||||
});
|
||||
return new MockLanguageModelV3({
|
||||
doStream: async () => {
|
||||
const n = step++;
|
||||
// Realistic inter-step latency. A real model spends seconds per step, so the
|
||||
// fire-and-forget per-step write chain drains to the steps table BETWEEN
|
||||
// steps; the mock otherwise collapses all steps into microseconds and the
|
||||
// terminal finalize wins the race before any but the first step persists.
|
||||
if (n > 0) await sleep(200);
|
||||
if (n < 2) return toolStep(n);
|
||||
return {
|
||||
stream: convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: 's2' },
|
||||
{ type: 'text-delta', id: 's2', delta: 'final answer' },
|
||||
{ type: 'text-end', id: 's2' },
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 6, outputTokens: 4, totalTokens: 10 },
|
||||
},
|
||||
] as any),
|
||||
};
|
||||
},
|
||||
} as any);
|
||||
}
|
||||
|
||||
describe('#492 append-persist service paths [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let aiChatRepo: AiChatRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
|
||||
let closeCalls: number;
|
||||
const mcpClients = {
|
||||
toolsFor: async () => ({
|
||||
tools: {},
|
||||
clients: [
|
||||
{
|
||||
close: async () => {
|
||||
closeCalls += 1;
|
||||
},
|
||||
},
|
||||
],
|
||||
outcomes: [],
|
||||
instructions: [],
|
||||
}),
|
||||
};
|
||||
|
||||
// Build the service WITH a REAL AiChatRunStepRepo injected (the property under
|
||||
// test) — unlike the legacy-fallback harness that passes it as undefined.
|
||||
const echoTool = tool({
|
||||
description: 'echo the message back',
|
||||
inputSchema: z.object({ msg: z.string() }),
|
||||
execute: async ({ msg }) => ({ echoed: msg }),
|
||||
});
|
||||
|
||||
function buildService(): AiChatService {
|
||||
return new AiChatService(
|
||||
{ getChatModel: async () => null } as any,
|
||||
aiChatRepo,
|
||||
msgRepo,
|
||||
{} as any, // aiChatPageSnapshotRepo
|
||||
{ resolve: async () => null } as any, // aiSettings
|
||||
{ forUser: async () => ({ echo: echoTool }) } as any, // tools
|
||||
mcpClients as any,
|
||||
{} as any, // aiAgentRoleRepo
|
||||
{} as any, // pageRepo
|
||||
{} as any, // pageAccess
|
||||
{
|
||||
isAiChatDeferredToolsEnabled: () => false,
|
||||
isAiChatFinalStepLockdownEnabled: () => false,
|
||||
} as any, // environment (deferred OFF -> all tools active every step)
|
||||
undefined, // streamRegistry
|
||||
undefined, // aiChatRunService
|
||||
stepRepo, // #492 aiChatRunStepRepo — the append-persist backend
|
||||
);
|
||||
}
|
||||
|
||||
function userUiMessage(text: string) {
|
||||
return {
|
||||
id: `u-${Math.random()}`,
|
||||
role: 'user',
|
||||
parts: [{ type: 'text', text }],
|
||||
};
|
||||
}
|
||||
|
||||
async function runStream(opts: {
|
||||
model: MockLanguageModelV3;
|
||||
chatId: string;
|
||||
body: any;
|
||||
}): Promise<void> {
|
||||
closeCalls = 0;
|
||||
const service = buildService();
|
||||
const { res, cleanup } = await makeRealResponse();
|
||||
try {
|
||||
await service.stream({
|
||||
user: { id: userId, workspaceId } as any,
|
||||
workspace: { id: workspaceId, name: 'WS' } as any,
|
||||
sessionId: 'sess-1',
|
||||
body: opts.body,
|
||||
res: { raw: res } as any,
|
||||
signal: new AbortController().signal,
|
||||
model: opts.model as any,
|
||||
role: null,
|
||||
} as any);
|
||||
await waitFor(async () => {
|
||||
const rows = await msgRepo.findAllByChat(opts.chatId, workspaceId);
|
||||
return rows.some(
|
||||
(r) =>
|
||||
r.role === 'assistant' &&
|
||||
['completed', 'error', 'aborted'].includes(r.status as string),
|
||||
);
|
||||
});
|
||||
await waitFor(() => closeCalls > 0, { timeoutMs: 5_000 });
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
aiChatRepo = new AiChatRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
// --- F2: the real onStep append-persist WRITE branch -----------------------
|
||||
it('drives steps through the real onStep path: per-step rows + marker match a single-row flush', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
const model = threeStepModel();
|
||||
|
||||
// Capture the mid-run step-marker UPDATEs the append-persist branch writes on
|
||||
// the assistant row (a { parts: [], toolTraceVersion, stepsPersisted } patch).
|
||||
const updateSpy = jest.spyOn(msgRepo, 'update');
|
||||
try {
|
||||
await runStream({
|
||||
model,
|
||||
chatId,
|
||||
body: { chatId, messages: [userUiMessage('call the tool then answer')] },
|
||||
});
|
||||
|
||||
const rows = await msgRepo.findAllByChat(chatId, workspaceId);
|
||||
const assistant = rows.find((r) => r.role === 'assistant')!;
|
||||
expect(assistant).toBeDefined();
|
||||
expect(assistant.status).toBe('completed');
|
||||
// The turn finalizes with the FULL inline parts assembled by a single-row
|
||||
// flush (assistantParts over every step) — the baseline the per-step slices
|
||||
// must reproduce.
|
||||
expect(rowHasInlineParts(assistant)).toBe(true);
|
||||
const finalParts = (assistant.metadata as { parts: any[] }).parts;
|
||||
|
||||
// The two NON-final finished steps each landed their own row, in stepIndex
|
||||
// order. (The fire-and-forget write chain drains before the next step, so
|
||||
// poll until both are on disk; the LAST step's write may lose the finalize
|
||||
// race, which is by design — its parts are already in `finalParts`.)
|
||||
await waitFor(async () => {
|
||||
const s = await stepRepo.findByMessage(assistant.id, workspaceId);
|
||||
return s.length >= 2;
|
||||
});
|
||||
const steps = await stepRepo.findByMessage(assistant.id, workspaceId);
|
||||
expect(steps[0].stepIndex).toBe(0);
|
||||
expect(steps[1].stepIndex).toBe(1);
|
||||
|
||||
// Each per-step row carries a NON-trivial slice: this step's text part + its
|
||||
// paired tool part (guards a mutation that persists empty/whole-turn parts).
|
||||
const s0 = steps[0].parts as any[];
|
||||
expect(s0).toContainEqual({ type: 'text', text: 'step 0 ' });
|
||||
expect(s0.some((p) => p.type === 'tool-echo')).toBe(true);
|
||||
|
||||
// The per-step slices are EXACTLY the corresponding prefix of the single-row
|
||||
// flush: assembleStepParts([step0, step1]) === finalParts[0 .. len0+len1].
|
||||
// This is what an off-by-one on `stepsPersisted-1` (a wrong `capturedSteps`
|
||||
// slice) or a shifted stepIndex breaks — the prefix no longer aligns.
|
||||
const prefixLen =
|
||||
(steps[0].parts as any[]).length + (steps[1].parts as any[]).length;
|
||||
expect(assembleStepParts([steps[0], steps[1]] as any)).toEqual(
|
||||
finalParts.slice(0, prefixLen),
|
||||
);
|
||||
|
||||
// The mid-run step markers advanced 1 -> 2 -> ... (the resume frontier), each
|
||||
// a shape-stable empty-parts marker equal to a single-row flush's marker.
|
||||
const markerCounts = updateSpy.mock.calls
|
||||
.map((c) => (c[2] as any)?.metadata)
|
||||
.filter(
|
||||
(m) =>
|
||||
m &&
|
||||
Array.isArray(m.parts) &&
|
||||
m.parts.length === 0 &&
|
||||
typeof m.stepsPersisted === 'number',
|
||||
)
|
||||
.map((m) => m.stepsPersisted);
|
||||
// Monotonic from 1, covering at least the two non-final steps.
|
||||
expect(markerCounts.slice(0, 2)).toEqual([1, 2]);
|
||||
expect(
|
||||
updateSpy.mock.calls
|
||||
.map((c) => (c[2] as any)?.metadata)
|
||||
.find((m) => m && m.stepsPersisted === 2),
|
||||
).toEqual(stepMarkerMetadata(2));
|
||||
} finally {
|
||||
updateSpy.mockRestore();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
// --- F1: model-REPLAY hydrates a hard-crashed mid-run turn from the steps table
|
||||
it('replays a hard-crashed mid-run turn WITH its partial steps hydrated from the steps table', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
|
||||
// Prior turn: a genuine user question...
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
userId,
|
||||
role: 'user',
|
||||
content: 'What is in the design doc?',
|
||||
createdAt: new Date(Date.now() - 3000),
|
||||
});
|
||||
// ...and an assistant row that a HARD crash (SIGKILL/OOM) left mid-run: only a
|
||||
// step marker on the row (metadata.parts:[] , content:''), NO terminal
|
||||
// callback ever fired, so its real parts live ONLY in ai_chat_run_steps.
|
||||
const crashed = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'aborted',
|
||||
metadata: stepMarkerMetadata(1),
|
||||
createdAt: new Date(Date.now() - 2000),
|
||||
});
|
||||
// The durable partial step: some reasoning text + a completed getPage tool
|
||||
// call (input + output), exactly what #183 step-granular durability preserves.
|
||||
await stepRepo.insertStep(
|
||||
crashed.id,
|
||||
workspaceId,
|
||||
0,
|
||||
assistantParts(
|
||||
[
|
||||
{
|
||||
text: 'HYDRATED_PARTIAL_STEP the doc says',
|
||||
toolCalls: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', input: { id: 'p1' } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: 'g1',
|
||||
toolName: 'getPage',
|
||||
output: { id: 'p1', body: 'PARTIAL_TOOL_OUTPUT budget section' },
|
||||
},
|
||||
],
|
||||
} as any,
|
||||
],
|
||||
'',
|
||||
),
|
||||
);
|
||||
|
||||
// The NEXT turn: the model just answers. The service must REPLAY the crashed
|
||||
// assistant turn with its partial parts hydrated from the steps table.
|
||||
const model = new MockLanguageModelV3({
|
||||
doStream: async () => ({ stream: successStream() }),
|
||||
} as any);
|
||||
await runStream({
|
||||
model,
|
||||
chatId,
|
||||
body: { chatId, messages: [userUiMessage('Continue please')] },
|
||||
});
|
||||
|
||||
expect(model.doStreamCalls.length).toBeGreaterThan(0);
|
||||
const prompt = JSON.stringify(model.doStreamCalls[0].prompt);
|
||||
// The partial step's TEXT reached the model context (it would be an empty text
|
||||
// part without hydration — rowToUiMessage falls back to `content:''`).
|
||||
expect(prompt).toContain('HYDRATED_PARTIAL_STEP');
|
||||
// The partial TOOL RESULT survived too (durable in the steps table, replayed).
|
||||
expect(prompt).toContain('PARTIAL_TOOL_OUTPUT');
|
||||
// The genuine prior user turn is present as well (sanity: real history replay).
|
||||
expect(prompt).toContain('What is in the design doc?');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,173 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import {
|
||||
assistantParts,
|
||||
flushAssistant,
|
||||
stepMarkerMetadata,
|
||||
} from '../../src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — WRITE-VOLUME regression on a LIVE Postgres, measured via
|
||||
* the `pg_current_wal_lsn()` delta around a realistic multi-step run driven through
|
||||
* the REAL repos (not a mock — a mock cannot observe MVCC/TOAST rewrite volume, the
|
||||
* whole point). Proves the core claim:
|
||||
*
|
||||
* NEW (per-step INSERT into ai_chat_run_steps + a CHEAP step-marker UPDATE on the
|
||||
* message row) writes O(Σ steps) of WAL — each step writes only its own bytes.
|
||||
*
|
||||
* OLD (the pre-#492 full-row rewrite: re-persist the GROWING metadata.parts on
|
||||
* every onStepFinish) writes O(n²) — step k rewrites the whole TOASTed jsonb of
|
||||
* all k prior outputs.
|
||||
*
|
||||
* The OLD path here IS the reverted behavior, so this doubles as the mutation
|
||||
* check: swapping the new path back to `flushAssistant` full-row UPDATEs reddens
|
||||
* the assertion (OLD is many times larger).
|
||||
*/
|
||||
type Step = {
|
||||
text: string;
|
||||
toolCalls: Array<{ toolCallId: string; toolName: string; input: unknown }>;
|
||||
toolResults: Array<{ toolCallId: string; toolName: string; output: unknown }>;
|
||||
};
|
||||
|
||||
// ~100 KB INCOMPRESSIBLE output per step (a page read). Random base64 so TOAST
|
||||
// cannot compress it away and hide the real write volume.
|
||||
function makeStep(i: number, outputBytes = 100_000): Step {
|
||||
const body = randomBytes(Math.ceil(outputBytes * 0.75)).toString('base64');
|
||||
return {
|
||||
text: `step ${i} reasoning`,
|
||||
toolCalls: [
|
||||
{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'getPage',
|
||||
output: { id: `p${i}`, title: `Page ${i}`, body },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function walDelta(
|
||||
db: Kysely<any>,
|
||||
fn: () => Promise<void>,
|
||||
): Promise<number> {
|
||||
const before = (
|
||||
await sql<{ l: string }>`select pg_current_wal_lsn() as l`.execute(db)
|
||||
).rows[0].l;
|
||||
await fn();
|
||||
// NOTE: no pg_switch_wal() — a segment switch pads the LSN to the next 16 MB
|
||||
// boundary and would swamp the delta. The raw LSN advances by the WAL bytes.
|
||||
const after = (
|
||||
await sql<{ l: string }>`select pg_current_wal_lsn() as l`.execute(db)
|
||||
).rows[0].l;
|
||||
return Number(
|
||||
(
|
||||
await sql<{
|
||||
d: string;
|
||||
}>`select pg_wal_lsn_diff(${after}::pg_lsn, ${before}::pg_lsn) as d`.execute(
|
||||
db,
|
||||
)
|
||||
).rows[0].d,
|
||||
);
|
||||
}
|
||||
|
||||
describe('#492 append-persist write volume (pg_current_wal_lsn delta) [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
let chatId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
const seedRow = () =>
|
||||
msgRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'streaming',
|
||||
metadata: stepMarkerMetadata(0) as never,
|
||||
});
|
||||
|
||||
const STEPS = 40;
|
||||
|
||||
it('NEW per-step INSERT is O(Σ steps); OLD full-row rewrite is O(n²)', async () => {
|
||||
const steps: Step[] = [];
|
||||
for (let i = 0; i < STEPS; i++) steps.push(makeStep(i));
|
||||
|
||||
// NEW: per-step INSERT of THIS step's parts + a cheap marker UPDATE.
|
||||
const newRow = await seedRow();
|
||||
const newWal = await walDelta(db, async () => {
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
await stepRepo.insertStep(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
i,
|
||||
assistantParts([steps[i]], ''),
|
||||
);
|
||||
await msgRepo.update(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
{ metadata: stepMarkerMetadata(i + 1) },
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// OLD (the pre-#492 revert): re-persist the GROWING metadata.parts on the
|
||||
// message row on every step.
|
||||
const oldRow = await seedRow();
|
||||
const oldWal = await walDelta(db, async () => {
|
||||
const acc: Step[] = [];
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
acc.push(steps[i]);
|
||||
await msgRepo.update(
|
||||
oldRow.id,
|
||||
workspaceId,
|
||||
flushAssistant(acc as never, '', 'streaming'),
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[#492 WAL] ${STEPS} steps ×100KB: new=${(newWal / 1e6).toFixed(1)}MB ` +
|
||||
`old=${(oldWal / 1e6).toFixed(1)}MB (${(oldWal / newWal).toFixed(
|
||||
1,
|
||||
)}x smaller)`,
|
||||
);
|
||||
|
||||
// O(Σ steps): ~STEPS × (100KB output + marker) of WAL. 40 × ~100KB parts plus
|
||||
// 40 tiny markers is a few tens of MB at most — bounded, linear in step count.
|
||||
expect(newWal).toBeLessThan(30_000_000);
|
||||
// O(n²): step k rewrites ~k × 100KB. Σ over 40 steps ≈ 80+ MB — far larger.
|
||||
expect(oldWal).toBeGreaterThan(30_000_000);
|
||||
// The load-bearing claim: the new path writes a small FRACTION of the old.
|
||||
expect(newWal).toBeLessThan(oldWal * 0.35);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { AiChatController } from 'src/core/ai-chat/ai-chat.controller';
|
||||
import {
|
||||
assembleStepParts,
|
||||
assistantParts,
|
||||
stepMarkerMetadata,
|
||||
} from 'src/core/ai-chat/ai-chat.service';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import type { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
createMessage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 controller hydration (crash-before-finalize RESUME) on a LIVE Postgres.
|
||||
* `AiChatController.withReconstructedParts` is wired into getMessages/delta/export/
|
||||
* run, but `aiChatRunStepRepo` is OPTIONAL and every controller unit spec passes it
|
||||
* as `undefined`, so the hydration branch early-returns and NEVER executes in those
|
||||
* tests. This drives the real read path — a mid-run streaming row (marker only,
|
||||
* empty inline parts) PLUS its `ai_chat_run_steps` rows — through getMessages WITH
|
||||
* the repo present, exercising the `role==='assistant' && !rowHasInlineParts`
|
||||
* needy predicate, the workspace-scoped batch step fetch, and the endpoint binding.
|
||||
*/
|
||||
describe('#492 controller hydration read path [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let aiChatRepo: AiChatRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let workspaceId: string;
|
||||
let otherWorkspaceId: string;
|
||||
let userId: string;
|
||||
|
||||
// Build the controller WITH a real AiChatRunStepRepo injected (position 9), the
|
||||
// seam the unit specs leave undefined. Only the read-path deps are real.
|
||||
function buildController(): AiChatController {
|
||||
return new AiChatController(
|
||||
{} as any, // aiChatService
|
||||
{} as any, // aiChatRunService
|
||||
aiChatRepo,
|
||||
msgRepo,
|
||||
{} as any, // aiTranscription
|
||||
{} as any, // pageRepo
|
||||
undefined, // streamRegistry
|
||||
undefined, // environment
|
||||
stepRepo, // #492 aiChatRunStepRepo
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
aiChatRepo = new AiChatRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
otherWorkspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('getMessages reconstructs a mid-run row from the steps table (finished rows untouched)', async () => {
|
||||
const chatId = (
|
||||
await createChat(db, { workspaceId, creatorId: userId })
|
||||
).id;
|
||||
const user = { id: userId } as User;
|
||||
const workspace = { id: workspaceId } as Workspace;
|
||||
|
||||
// A prior FINISHED assistant row that already carries inline parts — the needy
|
||||
// predicate must SKIP it (no step fetch), returned untouched.
|
||||
const finishedParts = assistantParts(
|
||||
[{ text: 'done earlier', toolCalls: [], toolResults: [] } as any],
|
||||
'',
|
||||
);
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: 'done earlier',
|
||||
status: 'completed',
|
||||
metadata: { parts: finishedParts, toolTraceVersion: 2, stepsPersisted: 1 },
|
||||
createdAt: new Date(Date.now() - 3000),
|
||||
});
|
||||
|
||||
// The mid-run row a crash-before-finalize left behind: a step marker only
|
||||
// (parts:[] , content:''), status 'streaming'. Its real parts live ONLY in the
|
||||
// steps table.
|
||||
const midRun = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'streaming',
|
||||
metadata: stepMarkerMetadata(2),
|
||||
createdAt: new Date(Date.now() - 1000),
|
||||
});
|
||||
|
||||
const step0 = assistantParts(
|
||||
[
|
||||
{
|
||||
text: 'reasoning about the page',
|
||||
toolCalls: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', input: { id: 'p1' } },
|
||||
],
|
||||
toolResults: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', output: { id: 'p1', body: 'B' } },
|
||||
],
|
||||
} as any,
|
||||
],
|
||||
'',
|
||||
);
|
||||
const step1 = assistantParts(
|
||||
[{ text: 'partial synthesis so far', toolCalls: [], toolResults: [] } as any],
|
||||
'',
|
||||
);
|
||||
await stepRepo.insertStep(midRun.id, workspaceId, 0, step0);
|
||||
await stepRepo.insertStep(midRun.id, workspaceId, 1, step1);
|
||||
|
||||
// Workspace-scoping guard: a step row for the SAME message id under a DIFFERENT
|
||||
// workspace must NEVER leak into this workspace's reconstruction.
|
||||
await stepRepo.insertStep(midRun.id, otherWorkspaceId, 99, [
|
||||
{ type: 'text', text: 'FOREIGN_WORKSPACE_LEAK' },
|
||||
]);
|
||||
|
||||
const res = await buildController().getMessages(
|
||||
{ chatId } as any,
|
||||
{ limit: 50 } as any,
|
||||
user,
|
||||
workspace,
|
||||
);
|
||||
|
||||
const items = res.items as any[];
|
||||
const finished = items.find((r) => r.status === 'completed');
|
||||
const reconstructed = items.find((r) => r.id === midRun.id);
|
||||
|
||||
// The finished row passed through with its inline parts unchanged.
|
||||
expect(finished.metadata.parts).toEqual(finishedParts);
|
||||
|
||||
// The mid-run row's parts were reconstructed from the two step rows, in order,
|
||||
// exactly as assembleStepParts concatenates them — the client seed sees the
|
||||
// persisted progress with no change to itself.
|
||||
const expected = assembleStepParts([
|
||||
{ stepIndex: 0, parts: step0 },
|
||||
{ stepIndex: 1, parts: step1 },
|
||||
] as any);
|
||||
expect(reconstructed.metadata.parts).toEqual(expected);
|
||||
// The foreign-workspace step row did NOT leak in.
|
||||
expect(JSON.stringify(reconstructed.metadata.parts)).not.toContain(
|
||||
'FOREIGN_WORKSPACE_LEAK',
|
||||
);
|
||||
// Sanity: reconstruction produced real content (text + the paired tool part +
|
||||
// the second step's text), not an empty fallback.
|
||||
expect(reconstructed.metadata.parts).toContainEqual({
|
||||
type: 'text',
|
||||
text: 'reasoning about the page',
|
||||
});
|
||||
expect(
|
||||
(reconstructed.metadata.parts as any[]).some((p) => p.type === 'tool-getPage'),
|
||||
).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { Kysely } from 'kysely';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import {
|
||||
assistantParts,
|
||||
reconstructRunParts,
|
||||
hydrateAssistantParts,
|
||||
stepMarkerMetadata,
|
||||
rowHasInlineParts,
|
||||
} from '../../src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — the reconstruct CONTRACT on a live Postgres. Proves that a
|
||||
* turn persisted the NEW way (per-step rows in `ai_chat_run_steps`, only a step
|
||||
* marker on the message row) reconstructs to the SAME UI parts as a turn persisted
|
||||
* the OLD way (full `metadata.parts` inline on the row, no step rows) — so the
|
||||
* era-switch is invisible to attach / delta-poll / export. Real repos + real jsonb
|
||||
* roundtrip, not a mock (a mock cannot prove the parts survive the jsonb column
|
||||
* byte-identical).
|
||||
*/
|
||||
type Step = {
|
||||
text: string;
|
||||
toolCalls: Array<{ toolCallId: string; toolName: string; input: unknown }>;
|
||||
toolResults: Array<{ toolCallId: string; toolName: string; output: unknown }>;
|
||||
};
|
||||
|
||||
// A realistic step: some text + a getPage tool call whose ~100 KB body is
|
||||
// INCOMPRESSIBLE random base64 (a 'x'.repeat filler would TOAST away and hide the
|
||||
// real bytes). Under MAX_TOOL_OUTPUT_BYTES (200 KB) it is stored uncompacted.
|
||||
function makeStep(i: number, outputBytes = 4_000): Step {
|
||||
const body = randomBytes(Math.ceil(outputBytes * 0.75)).toString('base64');
|
||||
return {
|
||||
text: `step ${i} text`,
|
||||
toolCalls: [
|
||||
{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'getPage',
|
||||
output: { id: `p${i}`, title: `Page ${i}`, body },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('AiChatRunStepRepo + reconstruct contract [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
let chatId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
const seedRow = (metadata: unknown, status: string) =>
|
||||
msgRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status,
|
||||
metadata: metadata as never,
|
||||
});
|
||||
|
||||
it('insertStep is idempotent per (message, stepIndex) and reads back in order', async () => {
|
||||
const row = await seedRow(stepMarkerMetadata(0), 'streaming');
|
||||
const parts0 = assistantParts([makeStep(0)], '');
|
||||
const parts1 = assistantParts([makeStep(1)], '');
|
||||
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 0, parts0)).toBe(true);
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 1, parts1)).toBe(true);
|
||||
// A retried persist of the SAME step is a no-op (ON CONFLICT DO NOTHING).
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 0, parts0)).toBe(
|
||||
false,
|
||||
);
|
||||
|
||||
const steps = await stepRepo.findByMessage(row.id, workspaceId);
|
||||
expect(steps.map((s) => s.stepIndex)).toEqual([0, 1]);
|
||||
|
||||
// Batch fetch groups by message id in step order.
|
||||
const map = await stepRepo.findByMessageIds([row.id], workspaceId);
|
||||
expect(map.get(row.id)!.map((s) => s.stepIndex)).toEqual([0, 1]);
|
||||
});
|
||||
|
||||
it('a NEW-style (step-table) run reconstructs identically to an OLD-style (inline) run', async () => {
|
||||
const steps = [makeStep(10), makeStep(11)];
|
||||
// The inline parts the OLD full-row flush would have written.
|
||||
const fullParts = assistantParts(steps, '');
|
||||
|
||||
// OLD-style record: full parts inline on the row, NO step rows.
|
||||
const oldRow = await seedRow(
|
||||
{ parts: fullParts, toolTraceVersion: 2, stepsPersisted: 2 },
|
||||
'completed',
|
||||
);
|
||||
|
||||
// NEW-style record: only a step marker on the row + per-step rows.
|
||||
const newRow = await seedRow(stepMarkerMetadata(2), 'streaming');
|
||||
for (let i = 0; i < steps.length; i++) {
|
||||
await stepRepo.insertStep(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
i,
|
||||
assistantParts([steps[i]], ''),
|
||||
);
|
||||
}
|
||||
|
||||
// Re-read both from the DB (proves the jsonb roundtrip).
|
||||
const oldFetched = await msgRepo.findById(oldRow.id, workspaceId);
|
||||
const newFetched = await msgRepo.findById(newRow.id, workspaceId);
|
||||
const oldSteps = await stepRepo.findByMessage(oldRow.id, workspaceId);
|
||||
const newSteps = await stepRepo.findByMessage(newRow.id, workspaceId);
|
||||
|
||||
// The discriminator: the old row carries inline parts, the new one does not.
|
||||
expect(rowHasInlineParts(oldFetched!)).toBe(true);
|
||||
expect(rowHasInlineParts(newFetched!)).toBe(false);
|
||||
expect(oldSteps).toHaveLength(0);
|
||||
expect(newSteps).toHaveLength(2);
|
||||
|
||||
const oldRecon = reconstructRunParts(oldFetched!, oldSteps);
|
||||
const newRecon = reconstructRunParts(newFetched!, newSteps);
|
||||
|
||||
// Both reconstruct to the SAME parts + step count — the era is invisible.
|
||||
expect(newRecon.parts).toEqual(fullParts);
|
||||
expect(oldRecon.parts).toEqual(fullParts);
|
||||
expect(newRecon.parts).toEqual(oldRecon.parts);
|
||||
expect(newRecon.stepsPersisted).toBe(2);
|
||||
expect(oldRecon.stepsPersisted).toBe(2);
|
||||
|
||||
// hydrateAssistantParts fills the new row's metadata.parts to match the old
|
||||
// row's inline parts — so a consumer reading `metadata.parts` off the raw row
|
||||
// (the client seed/poll, export) is unchanged across the era.
|
||||
const map = await stepRepo.findByMessageIds([newRow.id], workspaceId);
|
||||
const [hydrated] = hydrateAssistantParts([newFetched!], map);
|
||||
expect((hydrated.metadata as { parts: unknown }).parts).toEqual(fullParts);
|
||||
// A row that already has inline parts passes through untouched (same ref-shape).
|
||||
const [oldPassThrough] = hydrateAssistantParts([oldFetched!], map);
|
||||
expect((oldPassThrough.metadata as { parts: unknown }).parts).toEqual(
|
||||
fullParts,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import {
|
||||
up,
|
||||
down,
|
||||
} from '../../src/database/migrations/20260708T120000-ai-chat-run-steps';
|
||||
import { getTestDb, destroyTestDb } from './db';
|
||||
|
||||
/**
|
||||
* #492 migration up/down roundtrip on a LIVE Postgres. global-setup already
|
||||
* migrated docmost_test to latest (so the table exists at start); this drives the
|
||||
* migration's own down()/up() and asserts the table presence toggles, then leaves
|
||||
* it PRESENT (up) so the shared test DB is intact for any spec that runs after.
|
||||
*/
|
||||
async function tableExists(db: Kysely<any>): Promise<boolean> {
|
||||
const row = (
|
||||
await sql<{ t: string | null }>`select to_regclass('ai_chat_run_steps') as t`.execute(
|
||||
db,
|
||||
)
|
||||
).rows[0];
|
||||
return row.t !== null;
|
||||
}
|
||||
|
||||
async function uniqueIndexExists(db: Kysely<any>): Promise<boolean> {
|
||||
const row = (
|
||||
await sql<{
|
||||
t: string | null;
|
||||
}>`select to_regclass('ai_chat_run_steps_message_step_uidx') as t`.execute(db)
|
||||
).rows[0];
|
||||
return row.t !== null;
|
||||
}
|
||||
|
||||
describe('20260708 ai_chat_run_steps migration roundtrip [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
|
||||
beforeAll(() => {
|
||||
db = getTestDb();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Belt-and-suspenders: guarantee the table is present for later specs even if
|
||||
// an assertion threw mid-roundtrip.
|
||||
if (!(await tableExists(db))) await up(db);
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('down() drops the table+index and up() recreates them (idempotent)', async () => {
|
||||
// Starts applied (global-setup migrated to latest).
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
expect(await uniqueIndexExists(db)).toBe(true);
|
||||
|
||||
await down(db);
|
||||
expect(await tableExists(db)).toBe(false);
|
||||
expect(await uniqueIndexExists(db)).toBe(false);
|
||||
|
||||
await up(db);
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
expect(await uniqueIndexExists(db)).toBe(true);
|
||||
|
||||
// up() is idempotent (ifNotExists) — a second run is a harmless no-op.
|
||||
await expect(up(db)).resolves.not.toThrow();
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -322,20 +322,21 @@ describe('AiChatService.stream [integration]', () => {
|
||||
});
|
||||
|
||||
/**
|
||||
* #332 deferred tool loading, the ON path. The riskiest property is that the
|
||||
* per-turn `activatedTools` Set is created FRESH inside each stream() call, so a
|
||||
* tool a previous turn activated via loadTools is NOT still active when the next
|
||||
* turn starts — the new turn begins "cold" (CORE + loadTools only). The unit
|
||||
* tests only exercise pure prepareAgentStep with hand-fed Sets; this pins the
|
||||
* real wiring end-to-end (loadTools.execute -> activatedTools -> prepareStep ->
|
||||
* per-step activeTools) against the real streamText loop, and proves there is no
|
||||
* cross-turn leak. We drive a MockLanguageModelV3 whose step 1 calls
|
||||
* loadTools(['createPage']) and assert, via the model's recorded per-step
|
||||
* CallOptions.tools (the AI SDK filters the provider tool list by activeTools),
|
||||
* that the deferred tool becomes active on the SAME turn's next step but NOT on a
|
||||
* fresh turn's first step.
|
||||
* #332 + #490 deferred tool loading, the ON path. Turn 1 starts COLD (CORE +
|
||||
* loadTools only) and activates a deferred tool via loadTools; that activation
|
||||
* is PERSISTED into the chat's metadata.activatedTools (#490) so the NEXT turn
|
||||
* SEEDS from it and the tool is active from the fresh turn's FIRST step — the
|
||||
* model never re-runs loadTools to re-activate the same tool. The unit tests
|
||||
* only exercise pure prepareAgentStep with hand-fed Sets; this pins the real
|
||||
* wiring end-to-end (loadTools.execute -> activatedTools -> persist -> next-turn
|
||||
* seed -> prepareStep -> per-step activeTools) against the real streamText loop.
|
||||
* We drive a MockLanguageModelV3 whose step 1 calls loadTools(['createPage'])
|
||||
* and assert, via the model's recorded per-step CallOptions.tools (the AI SDK
|
||||
* filters the provider tool list by activeTools), that the deferred tool becomes
|
||||
* active on the SAME turn's next step AND, seeded from metadata, on the next
|
||||
* turn's first step.
|
||||
*/
|
||||
describe('deferred tool loading ON — per-turn activation, no leak (#332)', () => {
|
||||
describe('deferred tool loading ON — cross-turn activation persistence (#332 + #490)', () => {
|
||||
// A stub deferred (non-core) tool the agent can activate. Its execute is never
|
||||
// called — the model only needs to SEE it become active — but it must be a
|
||||
// valid AI-SDK tool so the SDK includes it in a step's tool list once active.
|
||||
@@ -451,7 +452,7 @@ describe('AiChatService.stream [integration]', () => {
|
||||
} as any);
|
||||
}
|
||||
|
||||
it('activates a deferred tool for the SAME turn, and a NEW turn starts cold (no leak)', async () => {
|
||||
it('activates a deferred tool for the SAME turn, and a NEW turn SEEDS it from persisted chat metadata (#490)', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
|
||||
// --- Turn 1: loadTools(createPage) on step 1, then answer on step 2. ---
|
||||
@@ -474,7 +475,7 @@ describe('AiChatService.stream [integration]', () => {
|
||||
// Step 2 of the SAME turn sees the just-activated deferred tool.
|
||||
expect(step2Tools).toContain('createPage');
|
||||
|
||||
// --- Turn 2 on the SAME chat: must start cold again. ---
|
||||
// --- Turn 2 on the SAME chat: seeds the persisted activation (#490). ---
|
||||
const model2 = new MockLanguageModelV3({
|
||||
doStream: async () => ({ stream: successStream() }),
|
||||
} as any);
|
||||
@@ -485,9 +486,10 @@ describe('AiChatService.stream [integration]', () => {
|
||||
|
||||
const nextTurnFirstStep = toolNames(model2.doStreamCalls[0]);
|
||||
expect(nextTurnFirstStep).toContain('loadTools');
|
||||
// The activated set is per-turn: the prior turn's createPage did NOT leak,
|
||||
// so the fresh turn's first step sees it deferred again.
|
||||
expect(nextTurnFirstStep).not.toContain('createPage');
|
||||
// #490: activation PERSISTS across turns — turn 1 wrote createPage into the
|
||||
// chat's metadata.activatedTools, so the next turn seeds from it and the
|
||||
// deferred tool is active from the FIRST step (no need to re-run loadTools).
|
||||
expect(nextTurnFirstStep).toContain('createPage');
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { Queue, Worker } from 'bullmq';
|
||||
import { PersistenceExtension } from '../../src/collaboration/extensions/persistence.extension';
|
||||
|
||||
/**
|
||||
* #370 — integration property of the idle-snapshot pipeline against REAL BullMQ.
|
||||
*
|
||||
* This is deliberately NOT a unit test of computeHistoryJob (that lives in
|
||||
* compute-history-job.spec.ts). The point here is the OBSERVABLE end-to-end
|
||||
* behaviour of the production `enqueuePageHistory` remove-then-add debounce
|
||||
* driving a real Redis-backed delayed queue + worker (the #431→#439 class: a
|
||||
* locally-correct function whose queue/timer property was never exercised):
|
||||
*
|
||||
* - a CONTINUOUS burst of stores lasting several caps yields periodic idle
|
||||
* snapshots — at least one per max-wait cap, NOT one-per-store; and
|
||||
* - an INTERMITTENT burst (a few stores, then quiet) yields exactly ONE
|
||||
* trailing snapshot.
|
||||
*
|
||||
* We shrink the idle windows to milliseconds (jest.mock of collaboration
|
||||
* constants) so real BullMQ delayed jobs actually promote within the test —
|
||||
* fake timers cannot advance Redis's own delayed-set clock, so the intervals
|
||||
* must be real but tiny. The production method under test is called verbatim.
|
||||
*/
|
||||
|
||||
// NOTE: jest.mock is hoisted above the module's const initializers, so its
|
||||
// factory cannot close over MAX_WAIT_MS/INTERVAL_MS — the literals are inlined
|
||||
// here and MUST stay in sync with the consts below (a single source of truth is
|
||||
// impossible across the hoist boundary).
|
||||
jest.mock('../../src/collaboration/constants', () => {
|
||||
const actual = jest.requireActual('../../src/collaboration/constants');
|
||||
return {
|
||||
...actual,
|
||||
IDLE_MAX_WAIT_USER: 300,
|
||||
IDLE_MAX_WAIT_AGENT: 300,
|
||||
IDLE_INTERVAL_USER: 1000,
|
||||
IDLE_INTERVAL_AGENT: 1000,
|
||||
};
|
||||
});
|
||||
|
||||
// Mirrors the mocked IDLE_MAX_WAIT_* above (IDLE_INTERVAL_* is 1000 > this, so
|
||||
// the max-wait ceiling is what actually governs the trailing delay).
|
||||
const MAX_WAIT_MS = 300;
|
||||
|
||||
const REDIS_CONNECTION = {
|
||||
host: process.env.TEST_REDIS_HOST ?? '127.0.0.1',
|
||||
port: Number(process.env.TEST_REDIS_PORT ?? 6379),
|
||||
};
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
describe('#370 idle-snapshot pipeline (real BullMQ)', () => {
|
||||
let queue: Queue;
|
||||
let worker: Worker;
|
||||
let extension: PersistenceExtension;
|
||||
// Every processed snapshot, tagged by pageId so the two scenarios stay isolated.
|
||||
const processed: Array<{ pageId: string; kind: string; at: number }> = [];
|
||||
const queueName = `history-idle-int-${randomUUID()}`;
|
||||
|
||||
beforeAll(async () => {
|
||||
queue = new Queue(queueName, {
|
||||
connection: REDIS_CONNECTION,
|
||||
// Mirror the production default (BullModule.forRoot removeOnComplete): the
|
||||
// enqueue idiom relies on the jobId being freed once a job completes so the
|
||||
// next burst can re-arm the same id.
|
||||
defaultJobOptions: { removeOnComplete: true, removeOnFail: true },
|
||||
});
|
||||
await queue.waitUntilReady();
|
||||
|
||||
worker = new Worker(
|
||||
queueName,
|
||||
async (job) => {
|
||||
processed.push({
|
||||
pageId: job.data?.pageId,
|
||||
kind: job.data?.kind,
|
||||
at: Date.now(),
|
||||
});
|
||||
},
|
||||
{ connection: REDIS_CONNECTION },
|
||||
);
|
||||
await worker.waitUntilReady();
|
||||
|
||||
// Construct the real extension; only historyQueue (5th ctor arg) and the
|
||||
// internal idleBurstStart map are exercised by enqueuePageHistory, so the
|
||||
// other collaborators can be null — the constructor only assigns fields.
|
||||
extension = new PersistenceExtension(
|
||||
null as any, // pageRepo
|
||||
null as any, // pageHistoryRepo
|
||||
null as any, // db
|
||||
null as any, // aiQueue
|
||||
queue as any, // historyQueue
|
||||
null as any, // notificationQueue
|
||||
null as any, // collabHistory
|
||||
null as any, // transclusionService
|
||||
);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Force-close and fully drain so no BullMQ background activity (delayed-set
|
||||
// polling, blocking BRPOPLPUSH) bleeds into later suites in this single
|
||||
// shared jest worker (maxWorkers: 1).
|
||||
await worker?.close(true).catch(() => undefined);
|
||||
await queue?.obliterate({ force: true }).catch(() => undefined);
|
||||
await queue?.close();
|
||||
// Let the redis sockets settle before the next suite starts.
|
||||
await sleep(150);
|
||||
});
|
||||
|
||||
const arm = (pageId: string) =>
|
||||
(extension as any).enqueuePageHistory({ id: pageId }, 'user');
|
||||
|
||||
it('continuous burst over several caps → periodic idle snapshots (≥1 per cap, not one-per-store)', async () => {
|
||||
const pageId = randomUUID();
|
||||
const runMs = 6 * MAX_WAIT_MS; // ~6 caps of unbroken editing
|
||||
// Store cadence that does NOT evenly divide the cap: real hocuspocus stores
|
||||
// are not aligned to cap boundaries, so a boundary job promotes in the gap
|
||||
// before the next store's remove(). A cap-aligned cadence would instead land
|
||||
// a store exactly on every boundary and lose the snapshot to the documented
|
||||
// remove-vs-active race — an artefact of the test clock, not the pipeline.
|
||||
const stepMs = 70;
|
||||
const stores = Math.floor(runMs / stepMs);
|
||||
|
||||
const start = Date.now();
|
||||
let count = 0;
|
||||
while (Date.now() - start < runMs) {
|
||||
await arm(pageId);
|
||||
count++;
|
||||
await sleep(stepMs);
|
||||
}
|
||||
// Let the final armed job flush.
|
||||
await sleep(2 * MAX_WAIT_MS);
|
||||
|
||||
const snaps = processed.filter((p) => p.pageId === pageId);
|
||||
|
||||
// Every autosnapshot is an idle-kind row.
|
||||
expect(snaps.every((s) => s.kind === 'idle')).toBe(true);
|
||||
// Periodic: at least one per cap over a multi-cap burst (lower-bounded loosely
|
||||
// to stay robust; the property is "fires at least every cap", not a single
|
||||
// trailing snapshot).
|
||||
expect(snaps.length).toBeGreaterThanOrEqual(3);
|
||||
// But NOT one-per-store: ~`stores` stores were issued; the debounce must
|
||||
// collapse them to a small multiple of the cap count, nowhere near per-store.
|
||||
expect(snaps.length).toBeLessThanOrEqual(Math.ceil(stores / 2));
|
||||
});
|
||||
|
||||
it('intermittent burst (a few stores, then quiet) → exactly ONE trailing snapshot', async () => {
|
||||
const pageId = randomUUID();
|
||||
|
||||
// A short burst well within a single cap window, then silence.
|
||||
await arm(pageId);
|
||||
await sleep(40);
|
||||
await arm(pageId);
|
||||
await sleep(40);
|
||||
await arm(pageId);
|
||||
|
||||
// Wait comfortably past the cap so the single pending trailing job fires.
|
||||
await sleep(4 * MAX_WAIT_MS);
|
||||
|
||||
const snaps = processed.filter((p) => p.pageId === pageId);
|
||||
expect(snaps).toHaveLength(1);
|
||||
expect(snaps[0].kind).toBe('idle');
|
||||
});
|
||||
});
|
||||
@@ -387,7 +387,7 @@ bucketByDay(sessions, tz):
|
||||
факты ниже — ground truth, можно дозапросить файлы через gitea MCP по указанному SHA):
|
||||
|
||||
- `page_history.kind` — `varchar(20)`, NULLABLE, БЕЗ дефолта (migration
|
||||
`20260705T120000-page-history-kind.ts`). Домен: `manual`/`agent`/`idle`/`boundary`;
|
||||
`20260707T120000-page-history-kind.ts`). Домен: `manual`/`agent`/`idle`/`boundary`;
|
||||
legacy `null` = автосейв (`collaboration/constants.ts`, `PageHistoryKind`).
|
||||
- `kind` УЖЕ включён в `PageHistoryRepo.baseFields` (`page-history.repo.ts`) — читается всеми
|
||||
выборками истории. `saveHistory({kind})` и `updateHistoryKind(id, kind)` существуют.
|
||||
|
||||
@@ -178,10 +178,11 @@ function sliceModel(xml: string): string | null {
|
||||
// --- decode chain ----------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Read the `content=` attribute out of a `.drawio.svg` string. Docmost stores a
|
||||
* base64 payload there (createDrawioSvg); draw.io's own SVG export may store the
|
||||
* XML entity-encoded instead. The DOM decodes entities for us, so the caller
|
||||
* only has to distinguish "starts with '<'" (raw XML) from base64.
|
||||
* Read the `content=` attribute out of a `.drawio.svg` string. Docmost writes
|
||||
* the mxfile XML entity-encoded there (buildDrawioSvg / createDrawioSvg), which
|
||||
* is also how draw.io's own SVG export stores it; older attachments stored a
|
||||
* base64 payload instead. The DOM decodes entities for us, so the caller only
|
||||
* has to distinguish "starts with '<'" (raw XML) from base64.
|
||||
*/
|
||||
export function extractContentAttr(svg: string): string {
|
||||
const { doc, error } = parseXml(svg);
|
||||
@@ -195,12 +196,23 @@ export function extractContentAttr(svg: string): string {
|
||||
// value itself never contains a double-quote (base64 / entity-encoded XML).
|
||||
const m = /content="([^"]*)"/.exec(svg);
|
||||
if (m) {
|
||||
// Decode the handful of XML entities a raw regex would leave encoded.
|
||||
// Decode the handful of XML entities a raw regex would leave encoded. The
|
||||
// numeric char-refs for tab/newline/CR MUST be decoded here too: the DOM
|
||||
// path above turns them back into the literal control chars, so this
|
||||
// regex fallback has to agree or the two decode paths diverge (#507).
|
||||
// `&` is decoded last so an escaped `&#x9;` reads back as the
|
||||
// literal text `	`, not a tab.
|
||||
return m[1]
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/	/gi, "\t")
|
||||
.replace(/	/g, "\t")
|
||||
.replace(/
/gi, "\n")
|
||||
.replace(/ /g, "\n")
|
||||
.replace(/
/gi, "\r")
|
||||
.replace(/ /g, "\r")
|
||||
.replace(/&/g, "&");
|
||||
}
|
||||
throw new Error("drawio: SVG has no content= attribute to decode");
|
||||
@@ -307,9 +319,16 @@ export function encodeDrawioFile(modelXml: string, title = "Page-1"): string {
|
||||
/**
|
||||
* Build the `diagram.drawio.svg` attachment. Mirrors the import service's
|
||||
* createDrawioSvg contract exactly:
|
||||
* <svg xmlns=… xmlns:xlink=… content="${base64(drawioFile)}">${inner}</svg>
|
||||
* <svg xmlns=… xmlns:xlink=… content="${xmlEscape(drawioFile)}">${inner}</svg>
|
||||
* plus width/height/viewBox from the diagram bounding box and the schematic
|
||||
* preview as the visible children (`inner`).
|
||||
*
|
||||
* The `content=` value is the mxfile XML XML-entity-escaped (draw.io's own
|
||||
* native form), NOT base64. draw.io's editor decodes a base64 content= via
|
||||
* Latin-1 atob (no UTF-8 step), turning every non-ASCII char (e.g. Cyrillic,
|
||||
* ё, —) into mojibake; the entity-encoded form is decoded by the DOM as UTF-8
|
||||
* and opens intact. Our decoder (decodeDrawioSvg) reads both forms, so old
|
||||
* base64 attachments still round-trip.
|
||||
*/
|
||||
export function buildDrawioSvg(
|
||||
modelXml: string,
|
||||
@@ -318,14 +337,14 @@ export function buildDrawioSvg(
|
||||
title = "Page-1",
|
||||
): string {
|
||||
const file = encodeDrawioFile(modelXml, title);
|
||||
const base64 = Buffer.from(file, "utf-8").toString("base64");
|
||||
const content = xmlEscape(file);
|
||||
const w = Math.max(1, Math.round(bbox.width));
|
||||
const h = Math.max(1, Math.round(bbox.height));
|
||||
return (
|
||||
`<svg xmlns="http://www.w3.org/2000/svg" ` +
|
||||
`xmlns:xlink="http://www.w3.org/1999/xlink" ` +
|
||||
`width="${w}" height="${h}" viewBox="0 0 ${w} ${h}" ` +
|
||||
`content="${base64}">${inner}</svg>`
|
||||
`content="${content}">${inner}</svg>`
|
||||
);
|
||||
}
|
||||
|
||||
@@ -334,7 +353,15 @@ function xmlEscape(s: string): string {
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """);
|
||||
.replace(/"/g, """)
|
||||
// A literal tab/newline/CR inside an attribute value is collapsed to a
|
||||
// single space by XML attribute-value normalization on DOM read (both jsdom
|
||||
// here and the real draw.io editor), silently flattening multi-line labels
|
||||
// and tab-bearing values. Numeric char-refs survive that normalization, so
|
||||
// emit them the way draw.io's own native export does (#507).
|
||||
.replace(/\t/g, "	")
|
||||
.replace(/\n/g, "
")
|
||||
.replace(/\r/g, "
");
|
||||
}
|
||||
|
||||
// --- normalization + hash --------------------------------------------------
|
||||
|
||||
@@ -51,6 +51,14 @@ const hasRule = (issues, rule, cellId) =>
|
||||
(i) => i.rule === rule && (cellId === undefined || i.cellId === cellId),
|
||||
);
|
||||
|
||||
// Reverse the attribute-value XML escaping used in the `content=` payload.
|
||||
const unescapeAttr = (s) =>
|
||||
s
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, '"')
|
||||
.replace(/&/g, "&");
|
||||
|
||||
// --- style parsing ---------------------------------------------------------
|
||||
|
||||
test("parseStyle: base stylename + key=value pairs", () => {
|
||||
@@ -335,16 +343,20 @@ test("encode/build: a title with < > \" & round-trips without corrupting the SVG
|
||||
|
||||
// The outer content="..." attribute must not be broken by the title: the raw
|
||||
// title metacharacters never appear literally in the SVG markup (they are
|
||||
// base64-encoded inside content=, and escaped inside the file XML).
|
||||
// entity-escaped inside content=, doubly so where the file XML already escaped
|
||||
// them inside name="...").
|
||||
const contentMatch = /content="([^"]*)"/.exec(svg);
|
||||
assert.ok(contentMatch, "SVG has a single well-formed content= attribute");
|
||||
|
||||
// The diagram model still decodes losslessly despite the exotic title.
|
||||
assert.equal(decodeDrawioSvg(svg), model);
|
||||
|
||||
// content= is now entity-encoded XML (draw.io's native form), never base64.
|
||||
assert.match(contentMatch[1], /^<mxfile/);
|
||||
|
||||
// The file XML is well-formed: the title lives in name="..." as escaped
|
||||
// entities, so unescaping recovers the original title byte-for-byte.
|
||||
const fileXml = Buffer.from(contentMatch[1], "base64").toString("utf-8");
|
||||
const fileXml = unescapeAttr(contentMatch[1]);
|
||||
const nameMatch = /<diagram id="[^"]*" name="([^"]*)">/.exec(fileXml);
|
||||
assert.ok(nameMatch, "the diagram name attribute is intact and quote-safe");
|
||||
const decodedTitle = nameMatch[1]
|
||||
@@ -358,3 +370,112 @@ test("encode/build: a title with < > \" & round-trips without corrupting the SVG
|
||||
const file = encodeDrawioFile(model, title);
|
||||
assert.match(file, /name="A < B > C " D & E">/);
|
||||
});
|
||||
|
||||
// --- #507: content= is entity-encoded XML, never base64 --------------------
|
||||
|
||||
// A model whose cell values carry Cyrillic, ё and an em dash — exactly the
|
||||
// characters draw.io's Latin-1 atob mangles when content= is base64.
|
||||
const CYRILLIC_MODEL =
|
||||
"<mxGraphModel><root>" +
|
||||
'<mxCell id="0"/><mxCell id="1" parent="0"/>' +
|
||||
'<mxCell id="2" value="Старт-бит — ёж" style="rounded=1;html=1;" vertex="1" parent="1">' +
|
||||
'<mxGeometry x="100" y="100" width="120" height="60" as="geometry"/></mxCell>' +
|
||||
"</root></mxGraphModel>";
|
||||
|
||||
test("#507: buildDrawioSvg writes content= as entity-encoded XML (not base64)", () => {
|
||||
const model = normalizeXml(CYRILLIC_MODEL);
|
||||
const svg = buildDrawioSvg(model, "<g/>", { width: 200, height: 120 }, "Диаграмма");
|
||||
|
||||
const contentMatch = /content="([^"]*)"/.exec(svg);
|
||||
assert.ok(contentMatch, "SVG has a single well-formed content= attribute");
|
||||
const content = contentMatch[1];
|
||||
|
||||
// The content= value is the entity-encoded mxfile XML — starts with `<mxfile`.
|
||||
assert.match(content, /^<mxfile/, "content= is entity-encoded mxfile XML");
|
||||
// It must NOT be a base64 blob: base64 has no XML entities and no literal `<`.
|
||||
assert.ok(content.includes("<"), "content= carries XML entities, not base64");
|
||||
|
||||
// Cyrillic / ё / — survive verbatim in the attribute (raw UTF-8, not atob-mangled).
|
||||
assert.ok(content.includes("Старт-бит — ёж"), "non-ASCII value is raw UTF-8 in content=");
|
||||
assert.ok(content.includes("Диаграмма"), "non-ASCII title is raw UTF-8 in content=");
|
||||
// The mojibake that base64+atob would have produced must be absent.
|
||||
assert.ok(!content.includes("Ð"), "no Latin-1 mojibake in content=");
|
||||
});
|
||||
|
||||
test("#507: Cyrillic model round-trips byte-stable through buildDrawioSvg -> decodeDrawioSvg", () => {
|
||||
const model = normalizeXml(CYRILLIC_MODEL);
|
||||
const svg = buildDrawioSvg(model, "<g/>", { width: 200, height: 120 }, "Заголовок — ё");
|
||||
assert.equal(decodeDrawioSvg(svg), model);
|
||||
});
|
||||
|
||||
test("#507 back-compat: an OLD base64-form .drawio.svg still decodes losslessly", () => {
|
||||
const model = normalizeXml(CYRILLIC_MODEL);
|
||||
// Reproduce the pre-fix write path: encodeDrawioFile -> base64 in content=.
|
||||
const file = encodeDrawioFile(model, "Старая диаграмма");
|
||||
const base64 = Buffer.from(file, "utf-8").toString("base64");
|
||||
const svg =
|
||||
`<svg xmlns="http://www.w3.org/2000/svg" content="${base64}"><g/></svg>`;
|
||||
// No XML entities, purely base64 alphabet — this is the legacy form.
|
||||
assert.ok(!base64.includes("<") && !base64.includes("&"));
|
||||
assert.equal(decodeDrawioSvg(svg), model);
|
||||
});
|
||||
|
||||
test("#507 negative: non-ASCII in a cell value AND in the title round-trip clean", () => {
|
||||
const model = normalizeXml(CYRILLIC_MODEL);
|
||||
const title = "Тест — ёмкость № 5";
|
||||
const svg = buildDrawioSvg(model, "<g/>", { width: 200, height: 120 }, title);
|
||||
|
||||
// Model recovered byte-for-byte.
|
||||
assert.equal(decodeDrawioSvg(svg), model);
|
||||
|
||||
// Title lands in the <diagram name="..."> of the decoded file XML, intact.
|
||||
const content = /content="([^"]*)"/.exec(svg)[1];
|
||||
const fileXml = unescapeAttr(content);
|
||||
const nameMatch = /<diagram id="[^"]*" name="([^"]*)">/.exec(fileXml);
|
||||
assert.ok(nameMatch, "diagram name attribute present");
|
||||
assert.equal(unescapeAttr(nameMatch[1]), title);
|
||||
});
|
||||
|
||||
// --- #507 F1: literal tab/newline/CR in an attribute value survive the
|
||||
// content= round-trip. The whole mxfile XML lives in one content="..." attr;
|
||||
// XML attribute-value normalization collapses a LITERAL tab/newline/CR to a
|
||||
// single space on DOM read, so the escape must emit numeric char-refs instead.
|
||||
const CTRL_MODEL =
|
||||
"<mxGraphModel><root>" +
|
||||
'<mxCell id="0"/><mxCell id="1" parent="0"/>' +
|
||||
'<mxCell id="2" value="col1\tcol2" style="html=1;\nshadow=0" vertex="1" parent="1">' +
|
||||
'<mxGeometry x="10" y="10" width="120" height="60" as="geometry"/></mxCell>' +
|
||||
'<mxCell id="3" value="Line1\nLine2\rLine3" vertex="1" parent="1">' +
|
||||
'<mxGeometry x="10" y="100" width="120" height="60" as="geometry"/></mxCell>' +
|
||||
"</root></mxGraphModel>";
|
||||
|
||||
test("#507 F1: literal tab/newline/CR in a value round-trip byte-stable (DOM decode)", () => {
|
||||
const svg = buildDrawioSvg(CTRL_MODEL, "<g/>", { width: 200, height: 200 });
|
||||
const content = /content="([^"]*)"/.exec(svg)[1];
|
||||
// The tab/newline/CR must be emitted as numeric char-refs, never as literal
|
||||
// control chars (which DOM attribute-value normalization would eat).
|
||||
assert.ok(
|
||||
!/[\t\n\r]/.test(content),
|
||||
"no literal tab/newline/CR survive in the content= attribute",
|
||||
);
|
||||
assert.ok(
|
||||
content.includes("	") &&
|
||||
content.includes("
") &&
|
||||
content.includes("
"),
|
||||
"tab/newline/CR are emitted as numeric char-refs",
|
||||
);
|
||||
// Full DOM decode recovers the model byte-for-byte, control chars intact.
|
||||
assert.equal(decodeDrawioSvg(svg), CTRL_MODEL);
|
||||
});
|
||||
|
||||
test("#507 F1: regex fallback decodes tab/newline/CR char-refs (agrees with DOM path)", () => {
|
||||
const svg = buildDrawioSvg(CTRL_MODEL, "<g/>", { width: 200, height: 200 });
|
||||
const content = /content="([^"]*)"/.exec(svg)[1];
|
||||
// A bare `&` makes the SVG wrapper malformed, forcing extractContentAttr onto
|
||||
// its regex fallback branch. That branch must decode the tab/newline/CR
|
||||
// char-refs exactly like the DOM path, or the two decoders diverge.
|
||||
const malformedSvg = `<svg content="${content}">&</svg>`;
|
||||
assert.equal(decodeDrawioSvg(malformedSvg), CTRL_MODEL);
|
||||
// The well-formed (DOM) path yields the identical result.
|
||||
assert.equal(decodeDrawioSvg(svg), CTRL_MODEL);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user