Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 agent_coder
|
ed3a8f8174 | ||
|
agent_coder
|
3163f50c98 | ||
|
agent_coder
|
44cdb5a4c3 |
@@ -27,7 +27,6 @@ vi.mock("@/features/ai-chat/utils/markdown.ts", async () => {
|
||||
|
||||
import MessageItem from "./message-item";
|
||||
import { messageSignature } from "@/features/ai-chat/utils/message-signature.ts";
|
||||
import { splitPlainChunks } from "./streaming-plain-text";
|
||||
|
||||
// matchMedia (read by MantineProvider) is stubbed globally in vitest.setup.ts.
|
||||
|
||||
@@ -115,89 +114,3 @@ describe("MessageItem markdown memoization", () => {
|
||||
expect(queryByText("streamed answer")).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// PERF SMOKE (#492): the whole point of the incremental streaming render is that
|
||||
// the ANSWER path costs O(number of markdown blocks), NOT O(number of throttled
|
||||
// ~20Hz ticks). Pre-#492 the finalized MarkdownPart re-parsed the WHOLE growing
|
||||
// answer on every delta — a synthetic ~100 KB stream measured 394 renderChatMarkdown
|
||||
// calls (one per tick). With the incremental render each STABILIZED block is parsed
|
||||
// exactly once (memoized in MarkdownChunk) and the live tail is cheap plain text, so
|
||||
// the call count collapses to ~= the block count regardless of tick granularity.
|
||||
describe("MessageItem streaming answer render is O(blocks), not O(ticks)", () => {
|
||||
// ~100 KB answer. Each section is a heading + a paragraph — TWO blank-line
|
||||
// delimited markdown blocks — so the safe-cut block count is ~2× the section
|
||||
// count. The perf claim is about the BLOCK count (the memoization granularity),
|
||||
// measured directly with splitPlainChunks below, not the section count.
|
||||
const buildAnswer = () => {
|
||||
const SECTIONS = 100;
|
||||
const paragraphs: string[] = [];
|
||||
for (let i = 0; i < SECTIONS; i++) {
|
||||
paragraphs.push(`## Section ${i}\n\n` + "lorem ipsum dolor ".repeat(55));
|
||||
}
|
||||
const full = paragraphs.join("\n\n");
|
||||
// The number of memoized markdown blocks the incremental render splits into
|
||||
// (all but the live tail are parsed once each).
|
||||
return { full, blocks: splitPlainChunks(full).length };
|
||||
};
|
||||
|
||||
const streamMsg = (text: string, state: "streaming" | "done"): UIMessage =>
|
||||
({
|
||||
id: "m1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text, state }],
|
||||
}) as UIMessage;
|
||||
|
||||
it("parses each block ~once over a 100KB stream (≈blocks, ≪ ticks)", () => {
|
||||
renderChatMarkdownSpy.mockClear();
|
||||
const { full, blocks } = buildAnswer();
|
||||
const CHUNK = 128; // a realistic ~20Hz throttled delta size
|
||||
const ticks = Math.ceil(full.length / CHUNK);
|
||||
|
||||
let msg = streamMsg(full.slice(0, CHUNK), "streaming");
|
||||
const { rerender } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg}
|
||||
signature={messageSignature(msg)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
for (let end = 2 * CHUNK; end < full.length; end += CHUNK) {
|
||||
msg = streamMsg(full.slice(0, end), "streaming");
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg}
|
||||
signature={messageSignature(msg)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
}
|
||||
// Finalize: the streaming→done flip renders the whole answer through ONE
|
||||
// canonical pass (visual parity), so the finished DOM matches the pre-#492
|
||||
// output. This is the single extra parse on top of the per-block ones.
|
||||
const done = streamMsg(full, "done");
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem message={done} signature={messageSignature(done)} />
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
const calls = renderChatMarkdownSpy.mock.calls.length;
|
||||
// Sanity: the stream really had far more ticks than blocks (else the test is
|
||||
// vacuous — the point is that calls scale with blocks, not ticks).
|
||||
expect(ticks).toBeGreaterThan(blocks * 3);
|
||||
// O(blocks): each stabilized block parsed once + the single final whole-text
|
||||
// parse. A small constant absorbs the finalize render and the live-tail block;
|
||||
// the load-bearing claim is the bound below.
|
||||
expect(calls).toBeLessThanOrEqual(blocks + 2);
|
||||
// ≪ ticks — and, non-vacuously, the blocks WERE parsed (not skipped entirely).
|
||||
expect(calls).toBeLessThan(ticks / 3);
|
||||
expect(calls).toBeGreaterThan(blocks / 2);
|
||||
// MUTATION-VERIFY (documented, not run here): dropping the `memo()` wrapper on
|
||||
// MarkdownChunk (so every stable block re-parses each tick) drives `calls`
|
||||
// toward `ticks` (~394), reddening both upper-bound assertions above.
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { render } from "@testing-library/react";
|
||||
import { MantineProvider } from "@mantine/core";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
|
||||
// Stub react-i18next (the component reads `useTranslation`). Mirrors the other
|
||||
// message-item specs.
|
||||
vi.mock("react-i18next", () => ({
|
||||
useTranslation: () => ({ t: (key: string) => key }),
|
||||
}));
|
||||
|
||||
import MessageItem from "./message-item";
|
||||
import { messageSignature } from "@/features/ai-chat/utils/message-signature.ts";
|
||||
// The REAL canonical renderer (NOT the spy the memo test installs): this file
|
||||
// exercises the actual markdown output so the visual-regression assertions below
|
||||
// compare against genuine HTML (incl. the schema's `<li><p>` wrappers).
|
||||
import { renderChatMarkdown } from "@/features/ai-chat/utils/markdown.ts";
|
||||
import classes from "./ai-chat.module.css";
|
||||
|
||||
const msg = (
|
||||
parts: UIMessage["parts"],
|
||||
extra?: Partial<UIMessage>,
|
||||
): UIMessage =>
|
||||
({ id: "m1", role: "assistant", parts, ...extra }) as UIMessage;
|
||||
|
||||
const renderRow = (message: UIMessage, turnStreaming = false) =>
|
||||
render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={message}
|
||||
signature={messageSignature(message)}
|
||||
turnStreaming={turnStreaming}
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
// A rich multi-block answer that exercises headings, a list (the `<li><p>` case
|
||||
// the scoped CSS tightens), inline emphasis, and multiple paragraphs.
|
||||
const ANSWER = [
|
||||
"# Заголовок",
|
||||
"",
|
||||
"Первый абзац с **жирным** и `кодом`.",
|
||||
"",
|
||||
"- пункт один",
|
||||
"- пункт два",
|
||||
"",
|
||||
"Второй абзац.",
|
||||
].join("\n");
|
||||
|
||||
describe("MessageItem final render — visual parity with the canonical pipeline", () => {
|
||||
it("a finalized text part renders exactly renderChatMarkdown(text)", () => {
|
||||
const { container } = renderRow(
|
||||
msg([{ type: "text", text: ANSWER, state: "done" }]),
|
||||
);
|
||||
const block = container.querySelector(`.${classes.markdown}`);
|
||||
expect(block).not.toBeNull();
|
||||
// Byte-for-byte the canonical output (the SAME whole-text pass the pre-#492
|
||||
// MarkdownPart produced), including `<li><p>…</p></li>` wrappers.
|
||||
expect(block!.innerHTML).toBe(renderChatMarkdown(ANSWER, {}));
|
||||
// The list wrapper is really present (guards against a vacuous empty render).
|
||||
expect(container.querySelectorAll("li p").length).toBe(2);
|
||||
});
|
||||
|
||||
it("the streaming incremental view CONVERGES to the canonical render on finish", () => {
|
||||
// Mount mid-stream (live tail) — the DOM here is the incremental view.
|
||||
const { container, rerender } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg([{ type: "text", text: ANSWER, state: "streaming" }])}
|
||||
signature={messageSignature(
|
||||
msg([{ type: "text", text: ANSWER, state: "streaming" }]),
|
||||
)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
// Finish the turn: state flips to done AND the turn is no longer streaming.
|
||||
const done = msg([{ type: "text", text: ANSWER, state: "done" }]);
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem message={done} signature={messageSignature(done)} />
|
||||
</MantineProvider>,
|
||||
);
|
||||
// After finish there is exactly ONE canonical markdown container whose HTML is
|
||||
// the whole-text render — identical to the non-streaming path above.
|
||||
const blocks = container.querySelectorAll(`.${classes.markdown}`);
|
||||
expect(blocks.length).toBe(1);
|
||||
expect(blocks[0].innerHTML).toBe(renderChatMarkdown(ANSWER, {}));
|
||||
});
|
||||
|
||||
it("neutralizeInternalLinks is honored on the finalized render", () => {
|
||||
const linkAnswer = "См. [страницу](/p/abc).";
|
||||
const { container } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg([{ type: "text", text: linkAnswer, state: "done" }])}
|
||||
signature={messageSignature(
|
||||
msg([{ type: "text", text: linkAnswer, state: "done" }]),
|
||||
)}
|
||||
neutralizeInternalLinks
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
const block = container.querySelector(`.${classes.markdown}`);
|
||||
expect(block!.innerHTML).toBe(
|
||||
renderChatMarkdown(linkAnswer, { neutralizeInternalLinks: true }),
|
||||
);
|
||||
// The internal link was made inert (no href) by the neutralization flag.
|
||||
const a = container.querySelector("a");
|
||||
expect(a?.hasAttribute("href")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,6 @@ import { useTranslation } from "react-i18next";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
import ToolCallCard from "@/features/ai-chat/components/tool-call-card.tsx";
|
||||
import ReasoningBlock from "@/features/ai-chat/components/reasoning-block.tsx";
|
||||
import { StreamingMarkdownText } from "@/features/ai-chat/components/streaming-markdown-text.tsx";
|
||||
import ChatErrorAlert from "@/features/ai-chat/components/chat-error-alert.tsx";
|
||||
import ChatStoppedNotice from "@/features/ai-chat/components/chat-stopped-notice.tsx";
|
||||
import { ToolUiPart, isToolPart } from "@/features/ai-chat/utils/tool-parts.tsx";
|
||||
@@ -87,39 +86,17 @@ interface MessageItemProps {
|
||||
* One assistant text part rendered as sanitized markdown. Memoized on its inputs
|
||||
* so a finalized text part is NOT re-parsed on every streamed delta: during a
|
||||
* turn only the actively-growing tail part changes its `text`, so every earlier
|
||||
* part hits the memo and skips the expensive canonical parse + DOMPurify pass.
|
||||
* Props are primitives, so React.memo's default shallow compare is exactly right
|
||||
* (the `text` string is compared by value).
|
||||
*
|
||||
* Streaming gate (#492) — mirrors ReasoningBlock:
|
||||
* - `streaming` (this is the live, actively-growing tail part of an in-flight
|
||||
* turn): render incrementally via StreamingMarkdownText — the stabilized blocks
|
||||
* go through the canonical pipeline (each parsed ONCE, memoized) and only the
|
||||
* live tail is cheap plain text. This makes the per-tick cost O(new blocks),
|
||||
* not the pre-#492 O(ticks) whole-answer re-parse on every ~20Hz delta.
|
||||
* - finalized (the common case, and the turn-end flip): render the WHOLE text
|
||||
* through ONE canonical pass — byte-identical to the pre-#492 output (visual
|
||||
* parity). The row re-renders on the streaming→done flip because
|
||||
* `messageSignature` tracks each part's `state` (and `turnStreaming` flips at
|
||||
* turn end), so the incremental view always converges to this single render.
|
||||
* part hits the memo and skips the expensive marked + DOMPurify pass. Props are
|
||||
* primitives, so React.memo's default shallow compare is exactly right (the
|
||||
* `text` string is compared by value).
|
||||
*/
|
||||
const MarkdownPart = memo(function MarkdownPart({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
streaming,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
streaming: boolean;
|
||||
}) {
|
||||
if (streaming) {
|
||||
return (
|
||||
<StreamingMarkdownText
|
||||
text={text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
);
|
||||
}
|
||||
const html = renderChatMarkdown(text, { neutralizeInternalLinks });
|
||||
if (html) {
|
||||
return (
|
||||
@@ -202,10 +179,47 @@ function MessageItem({
|
||||
{resolveAssistantName(assistantName) ?? t("AI agent")}
|
||||
</Text>
|
||||
{message.parts.map((part, index) => {
|
||||
// Tool parts (`tool-*` / `dynamic-tool`) are template-literal kinds, so
|
||||
// they cannot be a `switch` case; the runtime guard handles them, and the
|
||||
// switch below covers every CLOSED (literal-typed) part kind with a
|
||||
// compile-time exhaustiveness check in its default.
|
||||
if (part.type === "reasoning") {
|
||||
// Reasoning ("thinking") -> a collapsible block with its own token
|
||||
// count. Empty/whitespace reasoning with no authoritative count carries
|
||||
// nothing to show, so skip it (avoids an empty 0-token block).
|
||||
const text = (part as { text?: string }).text ?? "";
|
||||
if (!text.trim() && !(reasoningTokens && reasoningTokens > 0))
|
||||
return null;
|
||||
// Absent state (persisted rows) and "done" both mean finalized.
|
||||
// `messageSignature` already includes each part's `state`, so the
|
||||
// streaming→done flip changes the row signature and re-renders this
|
||||
// row — which is what lets ReasoningBlock switch from chunked plain
|
||||
// text to its one-time markdown parse (see reasoning-block.tsx).
|
||||
// ALSO require the turn to be live: a part stranded at
|
||||
// `state:"streaming"` after the turn ended (no `reasoning-end` — see
|
||||
// the `turnStreaming` prop doc) must still finalize and parse.
|
||||
const streaming =
|
||||
turnStreaming && (part as { state?: string }).state === "streaming";
|
||||
return (
|
||||
<ReasoningBlock
|
||||
key={index}
|
||||
text={text}
|
||||
tokens={reasoningTokens}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (part.type === "text") {
|
||||
// Skip empty/whitespace-only text parts (a streaming message often
|
||||
// starts with an empty text part before the first token arrives); the
|
||||
// typing indicator covers that gap until real content streams in.
|
||||
if (!part.text.trim()) return null;
|
||||
return (
|
||||
<MarkdownPart
|
||||
key={index}
|
||||
text={part.text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (isToolPart(part.type)) {
|
||||
return (
|
||||
<ToolCallCard
|
||||
@@ -218,76 +232,7 @@ function MessageItem({
|
||||
);
|
||||
}
|
||||
|
||||
switch (part.type) {
|
||||
case "reasoning": {
|
||||
// Reasoning ("thinking") -> a collapsible block with its own token
|
||||
// count. Empty/whitespace reasoning with no authoritative count
|
||||
// carries nothing to show, so skip it (avoids an empty 0-token block).
|
||||
const text = part.text ?? "";
|
||||
if (!text.trim() && !(reasoningTokens && reasoningTokens > 0))
|
||||
return null;
|
||||
// Absent state (persisted rows) and "done" both mean finalized.
|
||||
// `messageSignature` already includes each part's `state`, so the
|
||||
// streaming→done flip changes the row signature and re-renders this
|
||||
// row — which is what lets ReasoningBlock switch from chunked plain
|
||||
// text to its one-time markdown parse (see reasoning-block.tsx).
|
||||
// ALSO require the turn to be live: a part stranded at
|
||||
// `state:"streaming"` after the turn ended (no `reasoning-end` — see
|
||||
// the `turnStreaming` prop doc) must still finalize and parse.
|
||||
const streaming = turnStreaming && part.state === "streaming";
|
||||
return (
|
||||
<ReasoningBlock
|
||||
key={index}
|
||||
text={text}
|
||||
tokens={reasoningTokens}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
case "text": {
|
||||
// Skip empty/whitespace-only text parts (a streaming message often
|
||||
// starts with an empty text part before the first token arrives); the
|
||||
// typing indicator covers that gap until real content streams in.
|
||||
if (!part.text.trim()) return null;
|
||||
// The live, actively-growing tail part of the in-flight turn renders
|
||||
// incrementally (see MarkdownPart); a finalized part (persisted, or
|
||||
// the turn-end flip) renders the whole text through one canonical
|
||||
// pass. Same liveness rule as the reasoning branch above.
|
||||
const streaming = turnStreaming && part.state === "streaming";
|
||||
return (
|
||||
<MarkdownPart
|
||||
key={index}
|
||||
text={part.text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
case "source-url":
|
||||
case "source-document":
|
||||
case "file":
|
||||
case "step-start":
|
||||
// Not surfaced in the chat bubble (v1) — same as the pre-#492 default.
|
||||
return null;
|
||||
|
||||
default: {
|
||||
// Compile-time exhaustiveness over the CLOSED union members: every
|
||||
// literal-typed part kind is handled above, so the only kinds that
|
||||
// can reach here are the OPEN template-literal ones (`tool-*` — caught
|
||||
// by the guard at runtime — and `data-*`) plus `dynamic-tool`. Adding
|
||||
// a NEW closed part kind to UIMessagePart makes this assignment fail
|
||||
// to compile, forcing it to be handled instead of silently ignored
|
||||
// (this replaces the pre-#492 fall-through `return null` + WARNING).
|
||||
const _exhaustive:
|
||||
| `tool-${string}`
|
||||
| "dynamic-tool"
|
||||
| `data-${string}` = part.type;
|
||||
void _exhaustive;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
})}
|
||||
{/* A persisted turn error (server stored it in metadata.error). Rendered
|
||||
here so it survives a thread remount and shows in reopened history. */}
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
import { memo, useMemo } from "react";
|
||||
import { splitPlainChunks } from "@/features/ai-chat/components/streaming-plain-text.tsx";
|
||||
import { renderChatMarkdown } from "@/features/ai-chat/utils/markdown.ts";
|
||||
import classes from "@/features/ai-chat/components/ai-chat.module.css";
|
||||
|
||||
/**
|
||||
* One STABILIZED markdown block, rendered through the canonical pipeline and
|
||||
* memoized on its string prop. During streaming only the TAIL chunk grows (the
|
||||
* `splitPlainChunks` append-only invariant guarantees every earlier chunk is
|
||||
* byte-identical across deltas), so React skips every stable block and each one
|
||||
* is parsed by `renderChatMarkdown` EXACTLY ONCE — turning the pre-#492
|
||||
* "re-parse the whole accumulated answer on every ~20Hz tick" (O(ticks)) into
|
||||
* O(number of blocks). The markup is DOMPurify-sanitized inside renderChatMarkdown
|
||||
* before it reaches `dangerouslySetInnerHTML`.
|
||||
*
|
||||
* NOTE (transient streaming-only artifact): a safe cut is a blank-line boundary,
|
||||
* so a construct that legitimately contains a blank line (e.g. a fenced code block
|
||||
* with an empty line) can be split across chunks and render oddly WHILE it is still
|
||||
* streaming. This is cosmetic and self-heals: the moment the part finalizes,
|
||||
* MarkdownPart renders the WHOLE text through one canonical pass (visual parity
|
||||
* with the pre-#492 output). The reasoning path makes the same trade (plain text
|
||||
* while streaming, one markdown parse at the end).
|
||||
*/
|
||||
const MarkdownChunk = memo(function MarkdownChunk({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
}) {
|
||||
const html = renderChatMarkdown(text, { neutralizeInternalLinks });
|
||||
if (html) {
|
||||
return (
|
||||
<div
|
||||
className={classes.markdown}
|
||||
// Sanitized by renderChatMarkdown (DOMPurify) before insertion.
|
||||
dangerouslySetInnerHTML={{ __html: html }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
// Malformed/unsupported markdown could not render synchronously: raw text.
|
||||
return (
|
||||
<div className={classes.markdown} style={{ whiteSpace: "pre-wrap" }}>
|
||||
{text}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* The cheap streaming-time stand-in for the finalized answer's one-time markdown
|
||||
* parse (see MarkdownPart in message-item.tsx). Mirrors StreamingPlainText's
|
||||
* chunked-memo pattern but renders the STABILIZED prefix as real markdown (each
|
||||
* block parsed once, memoized) and only the LIVE tail as flat plain text — so the
|
||||
* user sees formatted output for everything up to the last safe cut, and the not-
|
||||
* yet-stable tail (which markdown-parsing every tick would make O(ticks)) stays a
|
||||
* single cheap escaped text node until it stabilizes into a new block.
|
||||
*
|
||||
* `splitPlainChunks` yields chunks where, under append-only growth, every chunk
|
||||
* except the LAST is immutable; the last chunk is the live tail. Index keys are
|
||||
* therefore stable (a given index never changes to a different chunk's content).
|
||||
*/
|
||||
export function StreamingMarkdownText({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
}) {
|
||||
const chunks = useMemo(() => splitPlainChunks(text), [text]);
|
||||
return (
|
||||
<>
|
||||
{chunks.map((chunk, index) =>
|
||||
index < chunks.length - 1 ? (
|
||||
<MarkdownChunk
|
||||
key={index}
|
||||
text={chunk}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
) : (
|
||||
// The live tail: flat, React-escaped plain text (no markdown parse, no
|
||||
// sanitizer, no innerHTML). `pre-wrap` preserves its newlines; trailing
|
||||
// separator newlines are dropped at display time so the block gap comes
|
||||
// from the markdown margins, not a doubled empty line (mirrors
|
||||
// PlainChunk in streaming-plain-text.tsx).
|
||||
<div
|
||||
key={index}
|
||||
className={classes.markdown}
|
||||
style={{ whiteSpace: "pre-wrap" }}
|
||||
>
|
||||
{chunk.replace(/\n+$/, "")}
|
||||
</div>
|
||||
),
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -141,7 +141,57 @@ export function htmlToJson(html: string) {
|
||||
}
|
||||
}
|
||||
|
||||
export function jsonToText(tiptapJson: JSONContent) {
|
||||
/**
|
||||
* Deterministic text-serializer overrides for the `format:"text"` page read
|
||||
* (#502). Non-text nodes render to a STABLE placeholder instead of their
|
||||
* (structure-dependent) inner text, so a machine diff of two text reads is
|
||||
* driven only by the page's actual prose — output stability across package
|
||||
* versions IS the contract (pinned by a snapshot test). Returning a string from
|
||||
* a `textSerializer` also stops `generateText` descending into the node, so a
|
||||
* table renders as ONE token rather than its flattened cell text.
|
||||
*
|
||||
* Only nodes with no meaningful flat-text form are overridden; every other node
|
||||
* (paragraph/heading/list/code/blockquote/callout/…) keeps its natural text so
|
||||
* a config written as markdown reads back byte-identical.
|
||||
*/
|
||||
const TEXT_READ_SERIALIZERS: Record<string, (props: { node: any }) => string> =
|
||||
{
|
||||
// Image atom: no inner text -> a fixed placeholder.
|
||||
image: () => '[image]',
|
||||
// Table: `[table RxC]` where R = row count, C = the first row's cell count
|
||||
// (a table's columns are uniform per the schema). Computed from the PM node,
|
||||
// so it is independent of cell contents.
|
||||
table: ({ node }) => {
|
||||
const rows = node?.childCount ?? 0;
|
||||
const cols = rows > 0 ? (node.child(0)?.childCount ?? 0) : 0;
|
||||
return `[table ${rows}x${cols}]`;
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Serialize a ProseMirror/TipTap document to plain text.
|
||||
*
|
||||
* Default (no options): the long-standing search-index behavior — bare
|
||||
* concatenated node text with `generateText`'s default `\n\n` block separator.
|
||||
* This feeds the page `textContent` tsvector and MUST NOT change.
|
||||
*
|
||||
* `deterministic:true` (#502 `format:"text"` page read): a flat, machine-diffable
|
||||
* rendering — one line per block (`\n` block separator; `hardBreak` already
|
||||
* serializes to `\n`), inline marks/anchors/autoformat dropped, and non-text
|
||||
* nodes replaced by the stable placeholders above (`[image]`, `[table RxC]`).
|
||||
*/
|
||||
export function jsonToText(
|
||||
// `any` (like jsonToHtml/jsonToMarkdown) so a loosely-typed DB `page.content`
|
||||
// (JsonValue) can be passed straight through, as the controller does.
|
||||
tiptapJson: any,
|
||||
options?: { deterministic?: boolean },
|
||||
) {
|
||||
if (options?.deterministic) {
|
||||
return generateText(tiptapJson, tiptapExtensions, {
|
||||
blockSeparator: '\n',
|
||||
textSerializers: TEXT_READ_SERIALIZERS,
|
||||
});
|
||||
}
|
||||
return generateText(tiptapJson, tiptapExtensions);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import { jsonToText } from './collaboration.util';
|
||||
|
||||
// #502 READ contract: `jsonToText(json, { deterministic: true })` — the flat,
|
||||
// machine-diffable text rendering behind getPage `format:"text"`. Block-per-line
|
||||
// (`\n` separator), inline marks/anchors dropped, hardBreak -> `\n`, and non-text
|
||||
// nodes replaced by STABLE placeholders (`[image]`, `[table RxC]`). Output
|
||||
// stability across package versions IS the contract, so it is pinned by a
|
||||
// snapshot below. The DEFAULT (no options) path is the search-index serializer
|
||||
// and MUST be unchanged — asserted separately.
|
||||
|
||||
const doc = (...content: any[]) => ({ type: 'doc', content });
|
||||
const para = (...content: any[]) => ({ type: 'paragraph', content });
|
||||
const text = (t: string, marks?: any[]) =>
|
||||
marks ? { type: 'text', text: t, marks } : { type: 'text', text: t };
|
||||
|
||||
describe('jsonToText — default (search index) behavior is unchanged', () => {
|
||||
it('uses the `\\n\\n` block separator and drops non-text nodes to empty', () => {
|
||||
const d = doc(para(text('alpha')), para(text('beta')));
|
||||
expect(jsonToText(d)).toBe('alpha\n\nbeta');
|
||||
});
|
||||
|
||||
it('an image contributes no text in the default (tsvector) mode', () => {
|
||||
const d = doc(para(text('cap')), { type: 'image', attrs: { src: 's' } });
|
||||
// No `[image]` placeholder leaks into the search index.
|
||||
expect(jsonToText(d)).not.toContain('[image]');
|
||||
});
|
||||
});
|
||||
|
||||
describe('jsonToText — deterministic:true (getPage format:"text")', () => {
|
||||
it('renders one line per block with `\\n` separators, marks dropped', () => {
|
||||
const d = doc(
|
||||
{ type: 'heading', attrs: { level: 2 }, content: [text('Title')] },
|
||||
para(
|
||||
text('hello ', [{ type: 'bold' }]),
|
||||
text('world', [{ type: 'italic' }]),
|
||||
),
|
||||
);
|
||||
expect(jsonToText(d, { deterministic: true })).toBe('Title\nhello world');
|
||||
});
|
||||
|
||||
it('a hardBreak renders as a newline', () => {
|
||||
const d = doc(para(text('a'), { type: 'hardBreak' }, text('b')));
|
||||
expect(jsonToText(d, { deterministic: true })).toBe('a\nb');
|
||||
});
|
||||
|
||||
it('an image node -> the stable `[image]` placeholder', () => {
|
||||
const d = doc(para(text('before')), { type: 'image', attrs: { src: 's' } });
|
||||
expect(jsonToText(d, { deterministic: true })).toBe('before\n[image]');
|
||||
});
|
||||
|
||||
it('a table -> `[table RxC]` (rows x columns) and its cell text is NOT flattened in', () => {
|
||||
const cell = (t: string) => ({
|
||||
type: 'tableCell',
|
||||
content: [para(text(t))],
|
||||
});
|
||||
const row = (...cells: any[]) => ({ type: 'tableRow', content: cells });
|
||||
const table = {
|
||||
type: 'table',
|
||||
content: [
|
||||
row(cell('CELLONE'), cell('CELLTWO'), cell('CELLTHREE')),
|
||||
row(cell('CELLFOUR'), cell('CELLFIVE'), cell('CELLSIX')),
|
||||
],
|
||||
};
|
||||
const d = doc(para(text('grid:')), table);
|
||||
const out = jsonToText(d, { deterministic: true });
|
||||
expect(out).toBe('grid:\n[table 2x3]');
|
||||
expect(out).not.toContain('CELL'); // cell text is not flattened into the read
|
||||
});
|
||||
|
||||
it('SNAPSHOT: a mixed document renders to a stable, deterministic string', () => {
|
||||
const cell = (t: string) => ({
|
||||
type: 'tableCell',
|
||||
content: [para(text(t))],
|
||||
});
|
||||
const d = doc(
|
||||
{ type: 'heading', attrs: { level: 1 }, content: [text('Config')] },
|
||||
para(text('key = ', [{ type: 'bold' }]), text('value')),
|
||||
{
|
||||
type: 'bulletList',
|
||||
content: [
|
||||
{ type: 'listItem', content: [para(text('one'))] },
|
||||
{ type: 'listItem', content: [para(text('two'))] },
|
||||
],
|
||||
},
|
||||
{ type: 'image', attrs: { src: 's' } },
|
||||
{
|
||||
type: 'table',
|
||||
content: [{ type: 'tableRow', content: [cell('x'), cell('y')] }],
|
||||
},
|
||||
);
|
||||
expect(jsonToText(d, { deterministic: true })).toMatchInlineSnapshot(`
|
||||
"Config
|
||||
key = value
|
||||
|
||||
|
||||
one
|
||||
|
||||
two
|
||||
[image]
|
||||
[table 1x2]"
|
||||
`);
|
||||
});
|
||||
|
||||
it('SCENARIO: a config written as a code block reads back byte-identical (diff empty)', () => {
|
||||
const config =
|
||||
'export TICKET_LIFETIME=$2592000\nservers:\n - www.internal.host';
|
||||
const d = doc({
|
||||
type: 'codeBlock',
|
||||
attrs: { language: 'yaml' },
|
||||
content: [text(config)],
|
||||
});
|
||||
// The code block is one block; its text (dollars, bare domain, newlines) is
|
||||
// preserved verbatim, so a read-as-text of a stored config diffs empty.
|
||||
expect(jsonToText(d, { deterministic: true })).toBe(config);
|
||||
});
|
||||
});
|
||||
@@ -35,7 +35,6 @@ import {
|
||||
import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { UserThrottlerGuard } from '../../integrations/throttle/user-throttler.guard';
|
||||
import { AI_CHAT_THROTTLER } from '../../integrations/throttle/throttler-names';
|
||||
@@ -44,8 +43,6 @@ import {
|
||||
AiChatRunHooks,
|
||||
AiChatService,
|
||||
AiChatStreamBody,
|
||||
rowHasInlineParts,
|
||||
hydrateAssistantParts,
|
||||
} from './ai-chat.service';
|
||||
import { AiChatRunService } from './ai-chat-run.service';
|
||||
import { AiTranscriptionService } from './ai-transcription.service';
|
||||
@@ -132,39 +129,8 @@ export class AiChatController {
|
||||
// production. Only touched on the resumable-stream (flag-on) path.
|
||||
private readonly streamRegistry?: AiChatStreamRegistryService,
|
||||
private readonly environment?: EnvironmentService,
|
||||
// #492: reconstruct a #492 mid-run record's parts from the steps table before
|
||||
// returning rows to the client / export. OPTIONAL so positional controller
|
||||
// specs compile unchanged; when absent, hydration is skipped (old-era rows
|
||||
// already carry inline parts, so nothing to reconstruct).
|
||||
private readonly aiChatRunStepRepo?: AiChatRunStepRepo,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Reconstruct parts for any assistant rows that don't carry them INLINE — a
|
||||
* #492 mid-run record whose per-step parts live in `ai_chat_run_steps` (the
|
||||
* append-persist backend). Every FINISHED row (old-era + #492) and every old-era
|
||||
* streaming snapshot already has inline `metadata.parts`, so the common path
|
||||
* fetches NOTHING and returns the rows untouched; only an actively-streaming
|
||||
* new-style row triggers the batch step fetch. Consumers (seed/poll/export) read
|
||||
* `metadata.parts` off the returned rows exactly as before — the era switch is
|
||||
* invisible to them (reconstructRunParts contract).
|
||||
*/
|
||||
private async withReconstructedParts(
|
||||
rows: AiChatMessage[],
|
||||
workspaceId: string,
|
||||
): Promise<AiChatMessage[]> {
|
||||
if (!this.aiChatRunStepRepo) return rows;
|
||||
const needy = rows.filter(
|
||||
(r) => r.role === 'assistant' && !rowHasInlineParts(r),
|
||||
);
|
||||
if (needy.length === 0) return rows;
|
||||
const stepsByMessage = await this.aiChatRunStepRepo.findByMessageIds(
|
||||
needy.map((r) => r.id),
|
||||
workspaceId,
|
||||
);
|
||||
return hydrateAssistantParts(rows, stepsByMessage);
|
||||
}
|
||||
|
||||
/** List the requesting user's chats in this workspace (paginated). */
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('chats')
|
||||
@@ -218,17 +184,11 @@ export class AiChatController {
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
) {
|
||||
await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
const page = await this.aiChatMessageRepo.findByChat(
|
||||
return this.aiChatMessageRepo.findByChat(
|
||||
dto.chatId,
|
||||
workspace.id,
|
||||
pagination,
|
||||
);
|
||||
// #492: reconstruct parts for any active new-style row so the client seed sees
|
||||
// `metadata.parts` unchanged (a no-op for the finished rows that fill a page).
|
||||
return {
|
||||
...page,
|
||||
items: await this.withReconstructedParts(page.items, workspace.id),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -265,10 +225,7 @@ export class AiChatController {
|
||||
workspace.id,
|
||||
);
|
||||
return {
|
||||
// #492: the delta of an actively-streaming new-style row carries its parts
|
||||
// reconstructed from the steps table, so the degraded poll shows persisted
|
||||
// progress exactly as the pre-#492 full-row snapshot did.
|
||||
rows: await this.withReconstructedParts(rows, workspace.id),
|
||||
rows,
|
||||
cursor,
|
||||
run: run ? { id: run.id, status: run.status } : null,
|
||||
};
|
||||
@@ -290,10 +247,8 @@ export class AiChatController {
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<{ markdown: string }> {
|
||||
const chat = await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
const rows = await this.withReconstructedParts(
|
||||
await this.aiChatMessageRepo.findAllByChat(dto.chatId, workspace.id),
|
||||
// #492: an interrupted-but-still-active turn exports its persisted steps
|
||||
// (reconstructed from the steps table) just like the pre-#492 full row did.
|
||||
const rows = await this.aiChatMessageRepo.findAllByChat(
|
||||
dto.chatId,
|
||||
workspace.id,
|
||||
);
|
||||
const markdown = buildChatMarkdown({
|
||||
@@ -333,13 +288,7 @@ export class AiChatController {
|
||||
workspace.id,
|
||||
)
|
||||
: undefined;
|
||||
// #492: reconnect to an IN-FLIGHT run reconstructs the projection row's parts
|
||||
// from the steps table (the row itself carries only the step marker mid-run);
|
||||
// a finished run's row already has inline parts, so this is a no-op.
|
||||
const [hydrated] = message
|
||||
? await this.withReconstructedParts([message], workspace.id)
|
||||
: [undefined];
|
||||
return { run, message: hydrated ?? null };
|
||||
return { run, message: message ?? null };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -22,7 +22,6 @@ import { AiSettingsService } from '../../integrations/ai/ai-settings.service';
|
||||
import { describeProviderError } from '../../integrations/ai/ai-error.util';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatPageSnapshotRepo } from '@docmost/db/repos/ai-chat/ai-chat-page-snapshot.repo';
|
||||
import { AiAgentRoleRepo } from '@docmost/db/repos/ai-agent-roles/ai-agent-roles.repo';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
@@ -519,12 +518,6 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// constructions compile unchanged; Nest always injects the real singleton, so
|
||||
// reconcile sees the SAME in-memory active/zombie maps the runner mutates.
|
||||
private readonly aiChatRunService?: AiChatRunService,
|
||||
// #492 append-persist: per-step INSERT into the lightweight steps table (the
|
||||
// O(Σ steps) replacement for the O(n²) full-row `metadata.parts` rewrite).
|
||||
// OPTIONAL so existing positional constructions (int-specs) compile unchanged;
|
||||
// Nest injects the real singleton. When ABSENT the per-step path falls back to
|
||||
// the pre-#492 full-row flush (no regression, only no WAL win).
|
||||
private readonly aiChatRunStepRepo?: AiChatRunStepRepo,
|
||||
) {}
|
||||
|
||||
// #487: periodic reconcile timer (single-process phase 1). Started in
|
||||
@@ -1121,34 +1114,8 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
chatId,
|
||||
workspace.id,
|
||||
);
|
||||
// #492: HYDRATE needy assistant rows from the steps table BEFORE the replay
|
||||
// map. A #492 mid-run assistant row carries only a step marker
|
||||
// (metadata.parts:[]); its real per-step parts live in `ai_chat_run_steps`.
|
||||
// The graceful terminal callbacks (onFinish/onError/onAbort -> flushAssistant)
|
||||
// assemble the full inline parts, so a normally-ended turn already has them.
|
||||
// But a HARD crash mid-run (SIGKILL/OOM) fires NO terminal callback, so the
|
||||
// row stays parts:[]; without this, rowToUiMessage falls back to an empty
|
||||
// text part and the partial tool-calls/results/text — durable in the steps
|
||||
// table — would DROP OUT of the model's replay context (regressing #183
|
||||
// step-granular durability for the model consumer). Mirrors the controller's
|
||||
// withReconstructedParts EXACTLY (same needy predicate + hydration helper).
|
||||
// Guarded on the optional repo: absent (positional test builds) degrades to
|
||||
// the current behavior rather than crashing.
|
||||
let replayHistory = oldHistory;
|
||||
if (this.aiChatRunStepRepo) {
|
||||
const needy = oldHistory.filter(
|
||||
(r) => r.role === 'assistant' && !rowHasInlineParts(r),
|
||||
);
|
||||
if (needy.length > 0) {
|
||||
const stepsByMessage = await this.aiChatRunStepRepo.findByMessageIds(
|
||||
needy.map((r) => r.id),
|
||||
workspace.id,
|
||||
);
|
||||
replayHistory = hydrateAssistantParts(oldHistory, stepsByMessage);
|
||||
}
|
||||
}
|
||||
const uiMessages: Array<Omit<UIMessage, 'id'> & { id: string }> = [
|
||||
...replayHistory.map(rowToUiMessage),
|
||||
...oldHistory.map(rowToUiMessage),
|
||||
{
|
||||
id: 'pending-user',
|
||||
role: 'user',
|
||||
@@ -1187,9 +1154,7 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// hint — confirm it against the persisted history (the preceding assistant
|
||||
// turn must really be aborted/streaming) so a spoofed flag cannot inject the
|
||||
// interrupt note onto an ordinary turn. The partial output the model needs is
|
||||
// already in `messages`: a #492 mid-run row's per-step parts live only in the
|
||||
// `ai_chat_run_steps` table and were hydrated into the replay history above,
|
||||
// so the aborted assistant turn replays WITH its partial parts intact.
|
||||
// already in `messages` (the aborted assistant row replays via findRecent).
|
||||
// Append the new user turn (shape-only) so index -2 is the prior assistant.
|
||||
const interrupted = isInterruptResume(
|
||||
[...oldHistory, { role: 'user', status: null, metadata: null }],
|
||||
@@ -1594,57 +1559,17 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// connection when finalize runs, so the SQL `WHERE status='streaming'`
|
||||
// (not this flag) is what prevents it clobbering the terminal row.
|
||||
if (finalized) return null;
|
||||
// The count derives from capturedSteps.length at THIS instant, so the
|
||||
// returned value is EXACTLY the persisted `stepsPersisted` the ring rotates
|
||||
// on (whether we take the append-persist path or the legacy fallback).
|
||||
const stepsPersisted = capturedSteps.length;
|
||||
// Build the flush ONCE so the returned count is EXACTLY the persisted
|
||||
// `stepsPersisted` (both derive from capturedSteps.length at this instant).
|
||||
const flushed = flushAssistant(capturedSteps, '', 'streaming', {
|
||||
pageChanged,
|
||||
partsCache,
|
||||
});
|
||||
const stepsPersisted = flushed.metadata.stepsPersisted as number;
|
||||
try {
|
||||
if (this.aiChatRunStepRepo) {
|
||||
// #492 APPEND-PERSIST: write only THIS finished step's parts to the
|
||||
// steps table (O(step) WAL), then bump the row's CHEAP step marker —
|
||||
// NO growing `metadata.parts` blob (that O(n²) full-row rewrite is
|
||||
// exactly what this removes). The full `metadata.parts` is assembled
|
||||
// once at finalize; a mid-run resume seed is reconstructed from the
|
||||
// step rows (reconstructRunParts). The INSERT is idempotent
|
||||
// (ON CONFLICT DO NOTHING), so a re-fired step never doubles the parts.
|
||||
const index = stepsPersisted - 1;
|
||||
if (index >= 0) {
|
||||
const stepParts = assistantParts(
|
||||
[capturedSteps[index]],
|
||||
'',
|
||||
partsCache,
|
||||
);
|
||||
await this.aiChatRunStepRepo.insertStep(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
index,
|
||||
stepParts,
|
||||
);
|
||||
}
|
||||
// Marker UPDATE: advance stepsPersisted + keep the toolTrace era marker
|
||||
// (bumps updatedAt so the delta poll observes the step, and carries the
|
||||
// frontier a resuming client attaches from). Scoped onlyIfStreaming so a
|
||||
// late marker never clobbers the terminal finalize.
|
||||
await this.aiChatMessageRepo.update(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
{ metadata: stepMarkerMetadata(stepsPersisted) },
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
} else {
|
||||
// Legacy fallback (no steps table wired — positional test builds): the
|
||||
// pre-#492 full-row flush, so parts still land inline on the row.
|
||||
const flushed = flushAssistant(capturedSteps, '', 'streaming', {
|
||||
pageChanged,
|
||||
partsCache,
|
||||
});
|
||||
await this.aiChatMessageRepo.update(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
flushed,
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
await this.aiChatMessageRepo.update(assistantId, workspace.id, flushed, {
|
||||
onlyIfStreaming: true,
|
||||
});
|
||||
return stepsPersisted;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
@@ -2824,122 +2749,6 @@ export function rowToUiMessage(row: AiChatMessage): Omit<UIMessage, 'id'> & {
|
||||
return { id: row.id, role, parts: parts as UIMessage['parts'] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap step-marker metadata for the #492 per-step UPDATE. Advances
|
||||
* `stepsPersisted` (the resume attach frontier) and keeps the `toolTraceVersion`
|
||||
* era marker, WITHOUT the growing `parts` blob (those live in the steps table
|
||||
* now; the full `metadata.parts` is assembled once at finalize by flushAssistant).
|
||||
* `parts: []` is kept for shape stability — it reads as an empty inline-parts row,
|
||||
* which is exactly the discriminator that routes reconstruction to the steps table.
|
||||
*/
|
||||
export function stepMarkerMetadata(
|
||||
stepsPersisted: number,
|
||||
): Record<string, unknown> {
|
||||
return { parts: [], toolTraceVersion: 2, stepsPersisted };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an assistant row already carries its full UI parts INLINE on the row
|
||||
* (`metadata.parts`). TRUE for every FINISHED row — old-era rows AND #492 rows,
|
||||
* whose full parts are assembled once at finalize — and for old-era streaming
|
||||
* snapshots (the pre-#492 per-step full-row flush). FALSE for a #492 MID-RUN
|
||||
* record, whose per-step parts live in the `ai_chat_run_steps` table. This is the
|
||||
* era discriminator the reconstruct seam branches on — no schema flag needed.
|
||||
*/
|
||||
export function rowHasInlineParts(row: { metadata?: unknown }): boolean {
|
||||
const meta = (row.metadata ?? {}) as { parts?: unknown };
|
||||
return Array.isArray(meta.parts) && meta.parts.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenate persisted per-step parts (in `stepIndex` order) into the turn's UI
|
||||
* parts (#492). Reproduces EXACTLY what flushAssistant → assistantParts would have
|
||||
* written to `metadata.parts` for those finished steps, since each step row stored
|
||||
* `assistantParts([step])` at persist time.
|
||||
*/
|
||||
export function assembleStepParts(
|
||||
stepRows: ReadonlyArray<{ stepIndex: number; parts: unknown }>,
|
||||
): UIMessage['parts'] {
|
||||
const parts: Array<Record<string, unknown>> = [];
|
||||
for (const step of [...stepRows].sort((a, b) => a.stepIndex - b.stepIndex)) {
|
||||
if (Array.isArray(step.parts)) {
|
||||
parts.push(...(step.parts as Array<Record<string, unknown>>));
|
||||
}
|
||||
}
|
||||
return parts as UIMessage['parts'];
|
||||
}
|
||||
|
||||
/**
|
||||
* reconstructRunParts (#492) — the single backend-switch seam. Given an assistant
|
||||
* ROW and its persisted step rows, return the turn's UI `parts` + the persisted
|
||||
* step count, reading from the ROW when it already carries inline parts (old-era
|
||||
* records AND every finished record) and from the STEPS TABLE otherwise (a #492
|
||||
* mid-run record). The higher-level consumers (attach seed, delta poll, export)
|
||||
* route their row→parts through this / {@link hydrateAssistantParts}, so old and
|
||||
* new records reconstruct identically WITHOUT the consumers branching on the era.
|
||||
*/
|
||||
export function reconstructRunParts(
|
||||
row: { metadata?: unknown; content?: string | null },
|
||||
stepRows: ReadonlyArray<{ stepIndex: number; parts: unknown }>,
|
||||
): { parts: UIMessage['parts']; stepsPersisted: number } {
|
||||
if (rowHasInlineParts(row)) {
|
||||
const meta = row.metadata as {
|
||||
parts: UIMessage['parts'];
|
||||
stepsPersisted?: number;
|
||||
};
|
||||
return {
|
||||
parts: meta.parts,
|
||||
stepsPersisted:
|
||||
typeof meta.stepsPersisted === 'number'
|
||||
? meta.stepsPersisted
|
||||
: stepRows.length,
|
||||
};
|
||||
}
|
||||
if (stepRows.length > 0) {
|
||||
return {
|
||||
parts: assembleStepParts(stepRows),
|
||||
stepsPersisted: stepRows.length,
|
||||
};
|
||||
}
|
||||
// No inline parts and no step rows: an old-era seed / empty streaming row. Fall
|
||||
// back to a single text part from `content` (mirrors rowToUiMessage).
|
||||
return {
|
||||
parts: textPart(row.content ?? '') as UIMessage['parts'],
|
||||
stepsPersisted: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill each assistant row's `metadata.parts` from its step rows when the row does
|
||||
* not already carry them inline (a #492 mid-run record), so a consumer that reads
|
||||
* `metadata.parts` off the RAW row (the client seed/poll, the Markdown export)
|
||||
* sees the reconstructed parts with NO change to itself. Rows that already have
|
||||
* inline parts (old-era + finished) and non-assistant rows pass through untouched.
|
||||
* Pure: returns new row objects, never mutates the inputs.
|
||||
*/
|
||||
export function hydrateAssistantParts<
|
||||
T extends { id: string; role?: string; metadata?: unknown },
|
||||
>(
|
||||
rows: ReadonlyArray<T>,
|
||||
stepsByMessage: Map<
|
||||
string,
|
||||
ReadonlyArray<{ stepIndex: number; parts: unknown }>
|
||||
>,
|
||||
): T[] {
|
||||
return rows.map((row) => {
|
||||
if (row.role !== 'assistant' || rowHasInlineParts(row)) return row;
|
||||
const steps = stepsByMessage.get(row.id);
|
||||
if (!steps || steps.length === 0) return row;
|
||||
return {
|
||||
...row,
|
||||
metadata: {
|
||||
...((row.metadata ?? {}) as Record<string, unknown>),
|
||||
parts: assembleStepParts(steps),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The persisted-row patch shape produced by {@link flushAssistant}. It is the
|
||||
* SAME shape the assistant repo insert/update consume (content + toolCalls +
|
||||
|
||||
@@ -10,6 +10,12 @@ import { Transform } from 'class-transformer';
|
||||
|
||||
export type ContentFormat = 'json' | 'markdown' | 'html';
|
||||
|
||||
// READ-only rendering formats for `getPage` (#502). A superset of the writable
|
||||
// `ContentFormat` with `text` — a flat, deterministic, machine-diffable text
|
||||
// rendering — added. Kept SEPARATE from `ContentFormat` so the write path
|
||||
// (createPage/updatePage `parseProsemirrorContent`) can never be handed `text`.
|
||||
export type PageReadFormat = ContentFormat | 'text';
|
||||
|
||||
export class CreatePageDto {
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
} from 'class-validator';
|
||||
import { Transform } from 'class-transformer';
|
||||
|
||||
import { ContentFormat } from './create-page.dto';
|
||||
import { PageReadFormat } from './create-page.dto';
|
||||
import { IsPageIdOrSlugId } from './page-identity.validator';
|
||||
|
||||
export class PageIdDto {
|
||||
@@ -43,8 +43,8 @@ export class PageInfoDto extends PageIdDto {
|
||||
|
||||
@IsOptional()
|
||||
@Transform(({ value }) => value?.toLowerCase())
|
||||
@IsIn(['json', 'markdown', 'html'])
|
||||
format?: ContentFormat;
|
||||
@IsIn(['json', 'markdown', 'html', 'text'])
|
||||
format?: PageReadFormat;
|
||||
}
|
||||
|
||||
export class DeletePageDto extends PageIdDto {
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { NotFoundException } from '@nestjs/common';
|
||||
import { PageController } from './page.controller';
|
||||
import { jsonToText, jsonToMarkdown } from '../../collaboration/collaboration.util';
|
||||
|
||||
// #502 READ: getPage `format:"text"` routes through the deterministic jsonToText
|
||||
// path (placeholders for non-text nodes, block-per-line), while `format:"json"`
|
||||
// (or none) returns the raw content and `format:"markdown"` still converts. This
|
||||
// pins the CONTROLLER wiring with lightweight mocks (no DB needed — the jsonToText
|
||||
// output contract itself is pinned in json-to-text-deterministic.spec.ts).
|
||||
|
||||
const CONTENT = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{ type: 'heading', attrs: { level: 1 }, content: [{ type: 'text', text: 'Config' }] },
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{ type: 'text', text: 'key=', marks: [{ type: 'bold' }] },
|
||||
{ type: 'text', text: 'value' },
|
||||
],
|
||||
},
|
||||
{ type: 'image', attrs: { src: 's' } },
|
||||
],
|
||||
};
|
||||
|
||||
function makeController(page: any): PageController {
|
||||
const pageRepo = { findById: jest.fn().mockResolvedValue(page) } as any;
|
||||
const pageAccessService = {
|
||||
validateCanViewWithPermissions: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ canEdit: true, hasRestriction: false }),
|
||||
} as any;
|
||||
// Only pageRepo + pageAccessService are exercised by getPage; the rest are
|
||||
// never touched on this path, so undefined placeholders are fine.
|
||||
return new PageController(
|
||||
undefined as any, // pageService
|
||||
pageRepo,
|
||||
undefined as any, // pageHistoryService
|
||||
undefined as any, // spaceAbility
|
||||
pageAccessService,
|
||||
undefined as any, // backlinkService
|
||||
undefined as any, // labelService
|
||||
undefined as any, // auditService
|
||||
);
|
||||
}
|
||||
|
||||
const user = { id: 'u1' } as any;
|
||||
|
||||
describe('PageController.getPage — format:"text" (#502)', () => {
|
||||
it('returns deterministic flat text with placeholders for non-text nodes', async () => {
|
||||
const controller = makeController({ id: 'p1', content: CONTENT });
|
||||
const res: any = await controller.getPage({ pageId: 'p1', format: 'text' } as any, user);
|
||||
expect(res.content).toBe(jsonToText(CONTENT, { deterministic: true }));
|
||||
expect(res.content).toBe('Config\nkey=value\n[image]');
|
||||
expect(res.permissions).toEqual({ canEdit: true, hasRestriction: false });
|
||||
});
|
||||
|
||||
it('markdown format still converts to markdown (unchanged)', async () => {
|
||||
const controller = makeController({ id: 'p1', content: CONTENT });
|
||||
const res: any = await controller.getPage({ pageId: 'p1', format: 'markdown' } as any, user);
|
||||
expect(res.content).toBe(jsonToMarkdown(CONTENT));
|
||||
});
|
||||
|
||||
it('json / no format returns the raw ProseMirror content object', async () => {
|
||||
const controller = makeController({ id: 'p1', content: CONTENT });
|
||||
const res: any = await controller.getPage({ pageId: 'p1' } as any, user);
|
||||
expect(res.content).toEqual(CONTENT); // untouched object, not a string
|
||||
});
|
||||
|
||||
it('missing page -> NotFoundException', async () => {
|
||||
const controller = makeController(null);
|
||||
await expect(
|
||||
controller.getPage({ pageId: 'nope', format: 'text' } as any, user),
|
||||
).rejects.toBeInstanceOf(NotFoundException);
|
||||
});
|
||||
});
|
||||
@@ -49,6 +49,7 @@ import { AddLabelsDto, RemoveLabelDto } from '../label/dto/label.dto';
|
||||
import {
|
||||
jsonToHtml,
|
||||
jsonToMarkdown,
|
||||
jsonToText,
|
||||
} from '../../collaboration/collaboration.util';
|
||||
import { AuditEvent, AuditResource } from '../../common/events/audit-events';
|
||||
import {
|
||||
@@ -93,10 +94,16 @@ export class PageController {
|
||||
const permissions = { canEdit, hasRestriction };
|
||||
|
||||
if (dto.format && dto.format !== 'json' && page.content) {
|
||||
const contentOutput =
|
||||
dto.format === 'markdown'
|
||||
? jsonToMarkdown(page.content)
|
||||
: jsonToHtml(page.content);
|
||||
let contentOutput: string;
|
||||
if (dto.format === 'markdown') {
|
||||
contentOutput = jsonToMarkdown(page.content);
|
||||
} else if (dto.format === 'text') {
|
||||
// #502: flat, deterministic, machine-diffable text (block-per-line,
|
||||
// inline marks/anchors dropped, non-text nodes -> stable placeholders).
|
||||
contentOutput = jsonToText(page.content, { deterministic: true });
|
||||
} else {
|
||||
contentOutput = jsonToHtml(page.content);
|
||||
}
|
||||
return {
|
||||
...page,
|
||||
content: contentOutput,
|
||||
|
||||
@@ -32,7 +32,6 @@ import { TemplateRepo } from '@docmost/db/repos/template/template.repo';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunRepo } from '@docmost/db/repos/ai-chat/ai-chat-run.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatPageSnapshotRepo } from '@docmost/db/repos/ai-chat/ai-chat-page-snapshot.repo';
|
||||
import { AiProviderCredentialsRepo } from '@docmost/db/repos/ai-chat/ai-provider-credentials.repo';
|
||||
import { AiMcpServerRepo } from '@docmost/db/repos/ai-chat/ai-mcp-server.repo';
|
||||
@@ -126,7 +125,6 @@ import { firstSqlToken } from '../integrations/metrics/metrics.constants';
|
||||
AiChatRepo,
|
||||
AiChatMessageRepo,
|
||||
AiChatRunRepo,
|
||||
AiChatRunStepRepo,
|
||||
AiChatPageSnapshotRepo,
|
||||
AiProviderCredentialsRepo,
|
||||
AiMcpServerRepo,
|
||||
@@ -163,7 +161,6 @@ import { firstSqlToken } from '../integrations/metrics/metrics.constants';
|
||||
AiChatRepo,
|
||||
AiChatMessageRepo,
|
||||
AiChatRunRepo,
|
||||
AiChatRunStepRepo,
|
||||
AiChatPageSnapshotRepo,
|
||||
AiProviderCredentialsRepo,
|
||||
AiMcpServerRepo,
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
import { type Kysely, sql } from 'kysely';
|
||||
|
||||
/**
|
||||
* `ai_chat_run_steps` — append-only per-step persistence for an assistant turn
|
||||
* (#492 wave C). Each finished agent step's UI `parts` (its text part + a part
|
||||
* per tool call, WITH the tool output) is INSERTed as its own lightweight row the
|
||||
* moment the step ends, instead of REWRITING the whole assistant row's growing
|
||||
* `metadata.parts` jsonb on every `onStepFinish`.
|
||||
*
|
||||
* WHY a separate table + INSERT (not a jsonb `||` append on the message row): a
|
||||
* Postgres jsonb UPDATE rewrites the ENTIRE TOASTed row version under MVCC, so
|
||||
* re-persisting a growing `metadata.parts` on every step is O(n²) write volume
|
||||
* (a 50-step run with ~100 KB tool outputs wrote hundreds of MB of WAL / dead
|
||||
* tuples per turn, hammering autovacuum). `||` would only shave the network
|
||||
* payload — the WAL/TOAST rewrite harm remains. An INSERT into a per-step table
|
||||
* writes ONLY that step's bytes, so the per-turn write volume is O(Σ steps).
|
||||
*
|
||||
* The full `metadata.parts` on the message row is assembled ONCE at finalize (the
|
||||
* terminal completed/error/aborted write). Mid-run, a resuming client's seed is
|
||||
* reconstructed by concatenating these step rows in `step_index` order — which
|
||||
* reproduces exactly what the old per-step full-row rewrite persisted. Records
|
||||
* written the OLD way (full `metadata.parts` on the row, no step rows) still
|
||||
* reconstruct from the row unchanged; the two eras are distinguished by whether
|
||||
* the row already carries non-empty `metadata.parts` (see reconstructRunParts /
|
||||
* assembleStepParts in ai-chat.service.ts).
|
||||
*
|
||||
* ON DELETE CASCADE on `message_id`: the step rows are a derived projection of the
|
||||
* assistant message; they must vanish with it (or with its workspace).
|
||||
*/
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.createTable('ai_chat_run_steps')
|
||||
.ifNotExists()
|
||||
.addColumn('id', 'uuid', (col) =>
|
||||
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
|
||||
)
|
||||
// The assistant message row this step belongs to (the #183 projection). The
|
||||
// step rows are a derived, per-step slice of that message, so they cascade.
|
||||
.addColumn('message_id', 'uuid', (col) =>
|
||||
col.references('ai_chat_messages.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('workspace_id', 'uuid', (col) =>
|
||||
col.references('workspaces.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
// 0-based index of the finished step within the turn. Ordering key for
|
||||
// reconstruction; unique per message (idempotent step re-persist).
|
||||
.addColumn('step_index', 'integer', (col) => col.notNull())
|
||||
// The step's UI parts (text part + a `tool-*` part per call, WITH output).
|
||||
// Concatenated in step order to rebuild the turn's `metadata.parts`.
|
||||
.addColumn('parts', 'jsonb', (col) => col.notNull())
|
||||
.addColumn('created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.execute();
|
||||
|
||||
// Idempotent per-step persist: a retried INSERT of the same (message, step)
|
||||
// is a no-op (the service uses ON CONFLICT DO NOTHING). This also serves the
|
||||
// reconstruction read (WHERE message_id ORDER BY step_index).
|
||||
await db.schema
|
||||
.createIndex('ai_chat_run_steps_message_step_uidx')
|
||||
.ifNotExists()
|
||||
.on('ai_chat_run_steps')
|
||||
.columns(['message_id', 'step_index'])
|
||||
.unique()
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.dropTable('ai_chat_run_steps').ifExists().execute();
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '../../types/kysely.types';
|
||||
import { dbOrTx } from '../../utils';
|
||||
import { AiChatRunStep } from '@docmost/db/types/entity.types';
|
||||
|
||||
/**
|
||||
* Append-only per-step persistence for an assistant turn (#492). Each finished
|
||||
* agent step's UI `parts` (its text part + a `tool-*` part per call, WITH the
|
||||
* tool output) is INSERTed as its own lightweight row the moment the step ends —
|
||||
* instead of REWRITING the assistant row's growing `metadata.parts` jsonb on every
|
||||
* `onStepFinish` (a Postgres jsonb UPDATE rewrites the whole TOASTed row version
|
||||
* under MVCC, so that was O(n²) WAL/dead-tuple churn per turn).
|
||||
*
|
||||
* The full `metadata.parts` on the message row is assembled ONCE at finalize;
|
||||
* mid-run, a resuming client's seed is rebuilt from these rows in `stepIndex`
|
||||
* order (see `assembleStepParts` / the reconstruct seam in ai-chat.service.ts).
|
||||
* Every method is workspace-scoped as defense-in-depth.
|
||||
*/
|
||||
@Injectable()
|
||||
export class AiChatRunStepRepo {
|
||||
constructor(@InjectKysely() private readonly db: KyselyDB) {}
|
||||
|
||||
/**
|
||||
* Append one finished step's parts. Idempotent: a retried persist of the SAME
|
||||
* (message, stepIndex) is a no-op via ON CONFLICT DO NOTHING — the per-step
|
||||
* writes are fired fire-and-forget + serialized, and a duplicate must never
|
||||
* throw into the stream or double the parts. Returns whether a NEW row landed
|
||||
* (false = the step was already persisted).
|
||||
*/
|
||||
async insertStep(
|
||||
messageId: string,
|
||||
workspaceId: string,
|
||||
stepIndex: number,
|
||||
parts: unknown,
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<boolean> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
const inserted = await db
|
||||
.insertInto('aiChatRunSteps')
|
||||
.values({
|
||||
messageId,
|
||||
workspaceId,
|
||||
stepIndex,
|
||||
// jsonb column: cast through never (same pattern as the message repo).
|
||||
parts: parts as never,
|
||||
})
|
||||
.onConflict((oc) => oc.columns(['messageId', 'stepIndex']).doNothing())
|
||||
.returning('id')
|
||||
.executeTakeFirst();
|
||||
return inserted !== undefined;
|
||||
}
|
||||
|
||||
/** All persisted steps for ONE assistant message, in step order. */
|
||||
async findByMessage(
|
||||
messageId: string,
|
||||
workspaceId: string,
|
||||
): Promise<AiChatRunStep[]> {
|
||||
return this.db
|
||||
.selectFrom('aiChatRunSteps')
|
||||
.selectAll('aiChatRunSteps')
|
||||
.where('messageId', '=', messageId)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.orderBy('stepIndex', 'asc')
|
||||
.execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* All persisted steps for a SET of assistant messages, grouped by messageId
|
||||
* (each group in step order). One query for the batch — the hydration seam
|
||||
* (getMessages / delta / export) calls this only for the rows that actually
|
||||
* need reconstruction (an active new-style row whose `metadata.parts` is still
|
||||
* empty), which is usually none, so this is skipped on the common path.
|
||||
*/
|
||||
async findByMessageIds(
|
||||
messageIds: string[],
|
||||
workspaceId: string,
|
||||
): Promise<Map<string, AiChatRunStep[]>> {
|
||||
const byMessage = new Map<string, AiChatRunStep[]>();
|
||||
if (messageIds.length === 0) return byMessage;
|
||||
const rows = await this.db
|
||||
.selectFrom('aiChatRunSteps')
|
||||
.selectAll('aiChatRunSteps')
|
||||
.where('messageId', 'in', messageIds)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.orderBy('stepIndex', 'asc')
|
||||
.execute();
|
||||
for (const row of rows) {
|
||||
const list = byMessage.get(row.messageId);
|
||||
if (list) list.push(row);
|
||||
else byMessage.set(row.messageId, [row]);
|
||||
}
|
||||
return byMessage;
|
||||
}
|
||||
}
|
||||
-17
@@ -692,22 +692,6 @@ export interface AiChatRuns {
|
||||
updatedAt: Generated<Timestamp>;
|
||||
}
|
||||
|
||||
// Append-only per-step persistence for an assistant turn (#492). Mirrors
|
||||
// migration 20260708T120000-ai-chat-run-steps.ts. Each finished agent step's UI
|
||||
// `parts` are INSERTed as their own row (instead of rewriting the message row's
|
||||
// growing `metadata.parts` jsonb every step — an O(n²) WAL/TOAST churn). The full
|
||||
// `metadata.parts` is assembled once at finalize; mid-run a resuming client's seed
|
||||
// is rebuilt by concatenating these rows in `stepIndex` order. Cascades with the
|
||||
// assistant message row it projects.
|
||||
export interface AiChatRunSteps {
|
||||
id: Generated<string>;
|
||||
messageId: string;
|
||||
workspaceId: string;
|
||||
stepIndex: number;
|
||||
parts: Json;
|
||||
createdAt: Generated<Timestamp>;
|
||||
}
|
||||
|
||||
// Per-(chat,page) snapshot of the open page's Markdown at the END of the agent's
|
||||
// previous turn (#274). Mirrors migration 20260702T120000-ai-chat-page-snapshot.ts.
|
||||
// The next turn diffs the CURRENT Markdown against `contentMd` to surface edits a
|
||||
@@ -745,7 +729,6 @@ export interface DB {
|
||||
aiChats: AiChats;
|
||||
aiChatMessages: AiChatMessages;
|
||||
aiChatRuns: AiChatRuns;
|
||||
aiChatRunSteps: AiChatRunSteps;
|
||||
aiChatPageSnapshots: AiChatPageSnapshots;
|
||||
apiKeys: ApiKeys;
|
||||
attachments: Attachments;
|
||||
|
||||
@@ -4,7 +4,6 @@ import {
|
||||
AiChats,
|
||||
AiChatMessages,
|
||||
AiChatRuns,
|
||||
AiChatRunSteps,
|
||||
AiChatPageSnapshots,
|
||||
Attachments,
|
||||
Comments,
|
||||
@@ -65,12 +64,6 @@ export type InsertableAiChatMessage = Omit<Insertable<AiChatMessages>, 'tsv'>;
|
||||
export type AiChatRun = Selectable<AiChatRuns>;
|
||||
export type InsertableAiChatRun = Insertable<AiChatRuns>;
|
||||
|
||||
// AI Chat Run Step (#492): append-only per-step parts persistence. Each finished
|
||||
// agent step's UI parts are stored as their own row; the full turn's parts are
|
||||
// assembled from these (in stepIndex order) for a mid-run resume seed.
|
||||
export type AiChatRunStep = Selectable<AiChatRunSteps>;
|
||||
export type InsertableAiChatRunStep = Insertable<AiChatRunSteps>;
|
||||
|
||||
// AI Chat Page Snapshot (#274): per-(chat,page) Markdown snapshot taken at the
|
||||
// end of the agent's previous turn, diffed against the current page next turn to
|
||||
// detect human edits made between turns.
|
||||
|
||||
@@ -85,6 +85,14 @@ export class ImportController {
|
||||
throw new BadRequestException('spaceId is required');
|
||||
}
|
||||
|
||||
// #502: optional multipart field. Only the MCP agent `createPage` path sends
|
||||
// `disableMarkdownExtensions=true` (its body is agent-authored plain prose /
|
||||
// config, so a `$…$` span must stay literal and a bare `www.host` must not
|
||||
// autolink). A HUMAN file upload omits the field, so it stays false and math
|
||||
// + autolink remain ON for human imports. Settable ONLY via this API param.
|
||||
const disableMarkdownExtensions =
|
||||
file.fields?.disableMarkdownExtensions?.value === 'true';
|
||||
|
||||
const ability = await this.spaceAbility.createForUser(user, spaceId);
|
||||
if (ability.cannot(SpaceCaslAction.Edit, SpaceCaslSubject.Page)) {
|
||||
throw new ForbiddenException();
|
||||
@@ -95,6 +103,7 @@ export class ImportController {
|
||||
user.id,
|
||||
spaceId,
|
||||
workspace.id,
|
||||
disableMarkdownExtensions,
|
||||
);
|
||||
|
||||
const ext = path.extname(file.filename).toLowerCase();
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
// Importing ImportService transitively loads import-formatter.ts, which imports
|
||||
// the ESM-only @sindresorhus/slugify (not in jest's transform allowlist). It is
|
||||
// irrelevant to this path, so mock it to keep the module graph loadable (mirrors
|
||||
// the sibling import.service specs).
|
||||
jest.mock('@sindresorhus/slugify', () => ({
|
||||
__esModule: true,
|
||||
default: (input: string) => String(input),
|
||||
}));
|
||||
|
||||
import { ImportService } from './import.service';
|
||||
|
||||
// #502 BLOCKER 1: the server markdown import path (`/pages/import`) now accepts an
|
||||
// optional `disableMarkdownExtensions` flag threaded into `processMarkdown`.
|
||||
// - MCP agent `createPage` sends it TRUE -> extensions OFF (a `$…$` config span
|
||||
// stays literal, a schemeless `www.host` is not autolinked).
|
||||
// - a HUMAN file upload omits it (default FALSE) -> extensions ON, so a real
|
||||
// `$x^2$` still becomes a formula (human imports unaffected).
|
||||
// `processMarkdown` only uses the imported converter (no injected deps on this
|
||||
// path), so the service is constructed with null deps for this focused unit test.
|
||||
|
||||
function makeService(): ImportService {
|
||||
return new ImportService(null as any, null as any, null as any, null as any);
|
||||
}
|
||||
|
||||
function findAll(node: any, type: string, acc: any[] = []): any[] {
|
||||
if (!node || typeof node !== 'object') return acc;
|
||||
if (node.type === type) acc.push(node);
|
||||
if (Array.isArray(node.content)) for (const c of node.content) findAll(c, type, acc);
|
||||
return acc;
|
||||
}
|
||||
function hasLink(node: any): boolean {
|
||||
return findAll(node, 'text').some((t: any) =>
|
||||
t.marks?.some((m: any) => m.type === 'link'),
|
||||
);
|
||||
}
|
||||
|
||||
describe('ImportService.processMarkdown — #502 disableMarkdownExtensions', () => {
|
||||
it('disableMarkdownExtensions=true (MCP createPage): `$…$` stays literal, no math', async () => {
|
||||
const svc = makeService();
|
||||
const doc = await svc.processMarkdown('export A=$FOO and B=$BAR done', true);
|
||||
expect(findAll(doc, 'mathInline')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('disableMarkdownExtensions=true: a schemeless www is NOT autolinked', async () => {
|
||||
const svc = makeService();
|
||||
const doc = await svc.processMarkdown('see www.example.com here', true);
|
||||
expect(hasLink(doc)).toBe(false);
|
||||
});
|
||||
|
||||
it('disableMarkdownExtensions=true: an explicit https:// STILL links', async () => {
|
||||
const svc = makeService();
|
||||
const doc = await svc.processMarkdown('see https://example.com here', true);
|
||||
expect(hasLink(doc)).toBe(true);
|
||||
});
|
||||
|
||||
it('DEFAULT (human upload): a real `$x^2$` DOES become a math node', async () => {
|
||||
const svc = makeService();
|
||||
const doc = await svc.processMarkdown('$x^2$');
|
||||
expect(findAll(doc, 'mathInline')).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('DEFAULT (human upload): a schemeless www IS autolinked', async () => {
|
||||
const svc = makeService();
|
||||
const doc = await svc.processMarkdown('see www.example.com here');
|
||||
expect(hasLink(doc)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -52,6 +52,12 @@ export class ImportService {
|
||||
userId: string,
|
||||
spaceId: string,
|
||||
workspaceId: string,
|
||||
// #502: when true, the markdown importer runs with the two layered
|
||||
// extensions OFF (a `$…$` span stays literal text; a schemeless `www.host` is
|
||||
// NOT autolinked). ONLY the MCP agent `createPage` path sets this; a HUMAN
|
||||
// file upload never passes it, so it defaults false and math/autolink stay ON
|
||||
// for human imports (their `$x^2$` still becomes a formula).
|
||||
disableMarkdownExtensions = false,
|
||||
) {
|
||||
const file = await filePromise;
|
||||
const fileBuffer = await file.toBuffer();
|
||||
@@ -66,7 +72,10 @@ export class ImportService {
|
||||
|
||||
try {
|
||||
if (fileExtension.endsWith('.md')) {
|
||||
prosemirrorState = await this.processMarkdown(fileContent);
|
||||
prosemirrorState = await this.processMarkdown(
|
||||
fileContent,
|
||||
disableMarkdownExtensions,
|
||||
);
|
||||
} else if (fileExtension.endsWith('.html')) {
|
||||
prosemirrorState = await this.processHTML(fileContent);
|
||||
}
|
||||
@@ -138,7 +147,12 @@ export class ImportService {
|
||||
return createdPage;
|
||||
}
|
||||
|
||||
async processMarkdown(markdownInput: string): Promise<any> {
|
||||
async processMarkdown(
|
||||
markdownInput: string,
|
||||
// #502: forwarded to the importer. DEFAULT false keeps math + fuzzy autolink
|
||||
// ON (human uploads unaffected); the MCP agent `createPage` path passes true.
|
||||
disableMarkdownExtensions = false,
|
||||
): Promise<any> {
|
||||
// Canonical markdown -> ProseMirror JSON directly via
|
||||
// `@docmost/prosemirror-markdown` (issue #345) — no HTML intermediate and no
|
||||
// second editor-ext markdown layer. Foreign markdown surfaces the strict
|
||||
@@ -147,7 +161,12 @@ export class ImportService {
|
||||
// The HTML-cleanup pass (`normalizeImportHtml`) is intentionally skipped here:
|
||||
// it targets foreign *HTML* (Notion/XWiki), which only ever arrives on the
|
||||
// `.html` path (`processHTML`), never as canonical markdown.
|
||||
return markdownToProseMirror(normalizeForeignMarkdown(markdownInput));
|
||||
return markdownToProseMirror(
|
||||
normalizeForeignMarkdown(markdownInput),
|
||||
disableMarkdownExtensions
|
||||
? { parseMath: false, fuzzyLinkify: false }
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
|
||||
async processHTML(htmlInput: string): Promise<any> {
|
||||
|
||||
@@ -1,412 +0,0 @@
|
||||
import * as http from 'node:http';
|
||||
import { Kysely } from 'kysely';
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { MockLanguageModelV3, convertArrayToReadableStream } from 'ai/test';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import {
|
||||
AiChatService,
|
||||
assembleStepParts,
|
||||
assistantParts,
|
||||
rowHasInlineParts,
|
||||
stepMarkerMetadata,
|
||||
} from 'src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
createMessage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — the REAL onStep WRITE path (F2) and the model-REPLAY
|
||||
* hydration path (F1), driven through `AiChatService.stream` against a LIVE
|
||||
* Postgres with a REAL `AiChatRunStepRepo` INJECTED. The existing append-persist
|
||||
* int-specs hand-roll the insert+marker cycle via the repos directly and build
|
||||
* the service with `aiChatRunStepRepo: undefined` (only the legacy-fallback branch
|
||||
* is covered), so an off-by-one on `stepsPersisted-1`, a wrong `capturedSteps`
|
||||
* slice, or a broken marker payload would pass all of them. These tests exercise
|
||||
* the actual `updateStreaming` append-persist branch end to end.
|
||||
*
|
||||
* The seam is the injected `model` (a seeded `MockLanguageModelV3` from `ai/test`)
|
||||
* plus a REAL Node `ServerResponse` as the hijacked socket — mirrors
|
||||
* ai-chat-stream.int-spec.ts.
|
||||
*/
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
async function waitFor(
|
||||
cond: () => Promise<boolean> | boolean,
|
||||
{ timeoutMs = 15_000, stepMs = 25 } = {},
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
if (await cond()) return;
|
||||
await sleep(stepMs);
|
||||
}
|
||||
throw new Error('waitFor: condition not met within timeout');
|
||||
}
|
||||
|
||||
// A real Node ServerResponse wired to a live socket (identical helper to the
|
||||
// stream int-spec) so the SDK's pipe/heartbeat writes behave as in prod.
|
||||
function makeRealResponse(): Promise<{
|
||||
res: http.ServerResponse;
|
||||
cleanup: () => Promise<void>;
|
||||
}> {
|
||||
return new Promise((resolve) => {
|
||||
const server = http.createServer((_req, res) => {
|
||||
resolve({
|
||||
res,
|
||||
cleanup: () =>
|
||||
new Promise<void>((done) => {
|
||||
try {
|
||||
if (!res.writableEnded) res.end();
|
||||
} catch {
|
||||
/* socket already gone */
|
||||
}
|
||||
server.close(() => done());
|
||||
}),
|
||||
});
|
||||
});
|
||||
server.listen(0, () => {
|
||||
const port = (server.address() as any).port;
|
||||
const creq = http.request({ port, method: 'GET' }, (cres) => {
|
||||
cres.resume();
|
||||
});
|
||||
creq.on('error', () => undefined);
|
||||
creq.end();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Stream parts for a normal, successful single-step turn.
|
||||
function successStream() {
|
||||
return convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: 't1' },
|
||||
{ type: 'text-delta', id: 't1', delta: 'Hello' },
|
||||
{ type: 'text-delta', id: 't1', delta: ' there' },
|
||||
{ type: 'text-end', id: 't1' },
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
},
|
||||
] as any);
|
||||
}
|
||||
|
||||
// A THREE-step turn: steps 0 and 1 each emit text + an `echo` tool call (the SDK
|
||||
// runs the tool and continues); step 2 answers and stops. Three steps is
|
||||
// deliberate: the LAST finished step's append-persist write races the terminal
|
||||
// finalize (which writes the full inline parts anyway, so a lost last-step row is
|
||||
// by design), but the NON-final steps 0 and 1 always drain to the steps table
|
||||
// before finalize — so those are what the test asserts on deterministically.
|
||||
function threeStepModel(): MockLanguageModelV3 {
|
||||
let step = 0;
|
||||
const toolStep = (i: number) => ({
|
||||
stream: convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: `s${i}` },
|
||||
{ type: 'text-delta', id: `s${i}`, delta: `step ${i} ` },
|
||||
{ type: 'text-end', id: `s${i}` },
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'echo',
|
||||
input: JSON.stringify({ msg: `m${i}` }),
|
||||
},
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'tool-calls',
|
||||
usage: { inputTokens: 5, outputTokens: 3, totalTokens: 8 },
|
||||
},
|
||||
] as any),
|
||||
});
|
||||
return new MockLanguageModelV3({
|
||||
doStream: async () => {
|
||||
const n = step++;
|
||||
// Realistic inter-step latency. A real model spends seconds per step, so the
|
||||
// fire-and-forget per-step write chain drains to the steps table BETWEEN
|
||||
// steps; the mock otherwise collapses all steps into microseconds and the
|
||||
// terminal finalize wins the race before any but the first step persists.
|
||||
if (n > 0) await sleep(200);
|
||||
if (n < 2) return toolStep(n);
|
||||
return {
|
||||
stream: convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: 's2' },
|
||||
{ type: 'text-delta', id: 's2', delta: 'final answer' },
|
||||
{ type: 'text-end', id: 's2' },
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 6, outputTokens: 4, totalTokens: 10 },
|
||||
},
|
||||
] as any),
|
||||
};
|
||||
},
|
||||
} as any);
|
||||
}
|
||||
|
||||
describe('#492 append-persist service paths [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let aiChatRepo: AiChatRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
|
||||
let closeCalls: number;
|
||||
const mcpClients = {
|
||||
toolsFor: async () => ({
|
||||
tools: {},
|
||||
clients: [
|
||||
{
|
||||
close: async () => {
|
||||
closeCalls += 1;
|
||||
},
|
||||
},
|
||||
],
|
||||
outcomes: [],
|
||||
instructions: [],
|
||||
}),
|
||||
};
|
||||
|
||||
// Build the service WITH a REAL AiChatRunStepRepo injected (the property under
|
||||
// test) — unlike the legacy-fallback harness that passes it as undefined.
|
||||
const echoTool = tool({
|
||||
description: 'echo the message back',
|
||||
inputSchema: z.object({ msg: z.string() }),
|
||||
execute: async ({ msg }) => ({ echoed: msg }),
|
||||
});
|
||||
|
||||
function buildService(): AiChatService {
|
||||
return new AiChatService(
|
||||
{ getChatModel: async () => null } as any,
|
||||
aiChatRepo,
|
||||
msgRepo,
|
||||
{} as any, // aiChatPageSnapshotRepo
|
||||
{ resolve: async () => null } as any, // aiSettings
|
||||
{ forUser: async () => ({ echo: echoTool }) } as any, // tools
|
||||
mcpClients as any,
|
||||
{} as any, // aiAgentRoleRepo
|
||||
{} as any, // pageRepo
|
||||
{} as any, // pageAccess
|
||||
{
|
||||
isAiChatDeferredToolsEnabled: () => false,
|
||||
isAiChatFinalStepLockdownEnabled: () => false,
|
||||
} as any, // environment (deferred OFF -> all tools active every step)
|
||||
undefined, // streamRegistry
|
||||
undefined, // aiChatRunService
|
||||
stepRepo, // #492 aiChatRunStepRepo — the append-persist backend
|
||||
);
|
||||
}
|
||||
|
||||
function userUiMessage(text: string) {
|
||||
return {
|
||||
id: `u-${Math.random()}`,
|
||||
role: 'user',
|
||||
parts: [{ type: 'text', text }],
|
||||
};
|
||||
}
|
||||
|
||||
async function runStream(opts: {
|
||||
model: MockLanguageModelV3;
|
||||
chatId: string;
|
||||
body: any;
|
||||
}): Promise<void> {
|
||||
closeCalls = 0;
|
||||
const service = buildService();
|
||||
const { res, cleanup } = await makeRealResponse();
|
||||
try {
|
||||
await service.stream({
|
||||
user: { id: userId, workspaceId } as any,
|
||||
workspace: { id: workspaceId, name: 'WS' } as any,
|
||||
sessionId: 'sess-1',
|
||||
body: opts.body,
|
||||
res: { raw: res } as any,
|
||||
signal: new AbortController().signal,
|
||||
model: opts.model as any,
|
||||
role: null,
|
||||
} as any);
|
||||
await waitFor(async () => {
|
||||
const rows = await msgRepo.findAllByChat(opts.chatId, workspaceId);
|
||||
return rows.some(
|
||||
(r) =>
|
||||
r.role === 'assistant' &&
|
||||
['completed', 'error', 'aborted'].includes(r.status as string),
|
||||
);
|
||||
});
|
||||
await waitFor(() => closeCalls > 0, { timeoutMs: 5_000 });
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
aiChatRepo = new AiChatRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
// --- F2: the real onStep append-persist WRITE branch -----------------------
|
||||
it('drives steps through the real onStep path: per-step rows + marker match a single-row flush', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
const model = threeStepModel();
|
||||
|
||||
// Capture the mid-run step-marker UPDATEs the append-persist branch writes on
|
||||
// the assistant row (a { parts: [], toolTraceVersion, stepsPersisted } patch).
|
||||
const updateSpy = jest.spyOn(msgRepo, 'update');
|
||||
try {
|
||||
await runStream({
|
||||
model,
|
||||
chatId,
|
||||
body: { chatId, messages: [userUiMessage('call the tool then answer')] },
|
||||
});
|
||||
|
||||
const rows = await msgRepo.findAllByChat(chatId, workspaceId);
|
||||
const assistant = rows.find((r) => r.role === 'assistant')!;
|
||||
expect(assistant).toBeDefined();
|
||||
expect(assistant.status).toBe('completed');
|
||||
// The turn finalizes with the FULL inline parts assembled by a single-row
|
||||
// flush (assistantParts over every step) — the baseline the per-step slices
|
||||
// must reproduce.
|
||||
expect(rowHasInlineParts(assistant)).toBe(true);
|
||||
const finalParts = (assistant.metadata as { parts: any[] }).parts;
|
||||
|
||||
// The two NON-final finished steps each landed their own row, in stepIndex
|
||||
// order. (The fire-and-forget write chain drains before the next step, so
|
||||
// poll until both are on disk; the LAST step's write may lose the finalize
|
||||
// race, which is by design — its parts are already in `finalParts`.)
|
||||
await waitFor(async () => {
|
||||
const s = await stepRepo.findByMessage(assistant.id, workspaceId);
|
||||
return s.length >= 2;
|
||||
});
|
||||
const steps = await stepRepo.findByMessage(assistant.id, workspaceId);
|
||||
expect(steps[0].stepIndex).toBe(0);
|
||||
expect(steps[1].stepIndex).toBe(1);
|
||||
|
||||
// Each per-step row carries a NON-trivial slice: this step's text part + its
|
||||
// paired tool part (guards a mutation that persists empty/whole-turn parts).
|
||||
const s0 = steps[0].parts as any[];
|
||||
expect(s0).toContainEqual({ type: 'text', text: 'step 0 ' });
|
||||
expect(s0.some((p) => p.type === 'tool-echo')).toBe(true);
|
||||
|
||||
// The per-step slices are EXACTLY the corresponding prefix of the single-row
|
||||
// flush: assembleStepParts([step0, step1]) === finalParts[0 .. len0+len1].
|
||||
// This is what an off-by-one on `stepsPersisted-1` (a wrong `capturedSteps`
|
||||
// slice) or a shifted stepIndex breaks — the prefix no longer aligns.
|
||||
const prefixLen =
|
||||
(steps[0].parts as any[]).length + (steps[1].parts as any[]).length;
|
||||
expect(assembleStepParts([steps[0], steps[1]] as any)).toEqual(
|
||||
finalParts.slice(0, prefixLen),
|
||||
);
|
||||
|
||||
// The mid-run step markers advanced 1 -> 2 -> ... (the resume frontier), each
|
||||
// a shape-stable empty-parts marker equal to a single-row flush's marker.
|
||||
const markerCounts = updateSpy.mock.calls
|
||||
.map((c) => (c[2] as any)?.metadata)
|
||||
.filter(
|
||||
(m) =>
|
||||
m &&
|
||||
Array.isArray(m.parts) &&
|
||||
m.parts.length === 0 &&
|
||||
typeof m.stepsPersisted === 'number',
|
||||
)
|
||||
.map((m) => m.stepsPersisted);
|
||||
// Monotonic from 1, covering at least the two non-final steps.
|
||||
expect(markerCounts.slice(0, 2)).toEqual([1, 2]);
|
||||
expect(
|
||||
updateSpy.mock.calls
|
||||
.map((c) => (c[2] as any)?.metadata)
|
||||
.find((m) => m && m.stepsPersisted === 2),
|
||||
).toEqual(stepMarkerMetadata(2));
|
||||
} finally {
|
||||
updateSpy.mockRestore();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
// --- F1: model-REPLAY hydrates a hard-crashed mid-run turn from the steps table
|
||||
it('replays a hard-crashed mid-run turn WITH its partial steps hydrated from the steps table', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
|
||||
// Prior turn: a genuine user question...
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
userId,
|
||||
role: 'user',
|
||||
content: 'What is in the design doc?',
|
||||
createdAt: new Date(Date.now() - 3000),
|
||||
});
|
||||
// ...and an assistant row that a HARD crash (SIGKILL/OOM) left mid-run: only a
|
||||
// step marker on the row (metadata.parts:[] , content:''), NO terminal
|
||||
// callback ever fired, so its real parts live ONLY in ai_chat_run_steps.
|
||||
const crashed = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'aborted',
|
||||
metadata: stepMarkerMetadata(1),
|
||||
createdAt: new Date(Date.now() - 2000),
|
||||
});
|
||||
// The durable partial step: some reasoning text + a completed getPage tool
|
||||
// call (input + output), exactly what #183 step-granular durability preserves.
|
||||
await stepRepo.insertStep(
|
||||
crashed.id,
|
||||
workspaceId,
|
||||
0,
|
||||
assistantParts(
|
||||
[
|
||||
{
|
||||
text: 'HYDRATED_PARTIAL_STEP the doc says',
|
||||
toolCalls: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', input: { id: 'p1' } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: 'g1',
|
||||
toolName: 'getPage',
|
||||
output: { id: 'p1', body: 'PARTIAL_TOOL_OUTPUT budget section' },
|
||||
},
|
||||
],
|
||||
} as any,
|
||||
],
|
||||
'',
|
||||
),
|
||||
);
|
||||
|
||||
// The NEXT turn: the model just answers. The service must REPLAY the crashed
|
||||
// assistant turn with its partial parts hydrated from the steps table.
|
||||
const model = new MockLanguageModelV3({
|
||||
doStream: async () => ({ stream: successStream() }),
|
||||
} as any);
|
||||
await runStream({
|
||||
model,
|
||||
chatId,
|
||||
body: { chatId, messages: [userUiMessage('Continue please')] },
|
||||
});
|
||||
|
||||
expect(model.doStreamCalls.length).toBeGreaterThan(0);
|
||||
const prompt = JSON.stringify(model.doStreamCalls[0].prompt);
|
||||
// The partial step's TEXT reached the model context (it would be an empty text
|
||||
// part without hydration — rowToUiMessage falls back to `content:''`).
|
||||
expect(prompt).toContain('HYDRATED_PARTIAL_STEP');
|
||||
// The partial TOOL RESULT survived too (durable in the steps table, replayed).
|
||||
expect(prompt).toContain('PARTIAL_TOOL_OUTPUT');
|
||||
// The genuine prior user turn is present as well (sanity: real history replay).
|
||||
expect(prompt).toContain('What is in the design doc?');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -1,173 +0,0 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import {
|
||||
assistantParts,
|
||||
flushAssistant,
|
||||
stepMarkerMetadata,
|
||||
} from '../../src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — WRITE-VOLUME regression on a LIVE Postgres, measured via
|
||||
* the `pg_current_wal_lsn()` delta around a realistic multi-step run driven through
|
||||
* the REAL repos (not a mock — a mock cannot observe MVCC/TOAST rewrite volume, the
|
||||
* whole point). Proves the core claim:
|
||||
*
|
||||
* NEW (per-step INSERT into ai_chat_run_steps + a CHEAP step-marker UPDATE on the
|
||||
* message row) writes O(Σ steps) of WAL — each step writes only its own bytes.
|
||||
*
|
||||
* OLD (the pre-#492 full-row rewrite: re-persist the GROWING metadata.parts on
|
||||
* every onStepFinish) writes O(n²) — step k rewrites the whole TOASTed jsonb of
|
||||
* all k prior outputs.
|
||||
*
|
||||
* The OLD path here IS the reverted behavior, so this doubles as the mutation
|
||||
* check: swapping the new path back to `flushAssistant` full-row UPDATEs reddens
|
||||
* the assertion (OLD is many times larger).
|
||||
*/
|
||||
type Step = {
|
||||
text: string;
|
||||
toolCalls: Array<{ toolCallId: string; toolName: string; input: unknown }>;
|
||||
toolResults: Array<{ toolCallId: string; toolName: string; output: unknown }>;
|
||||
};
|
||||
|
||||
// ~100 KB INCOMPRESSIBLE output per step (a page read). Random base64 so TOAST
|
||||
// cannot compress it away and hide the real write volume.
|
||||
function makeStep(i: number, outputBytes = 100_000): Step {
|
||||
const body = randomBytes(Math.ceil(outputBytes * 0.75)).toString('base64');
|
||||
return {
|
||||
text: `step ${i} reasoning`,
|
||||
toolCalls: [
|
||||
{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'getPage',
|
||||
output: { id: `p${i}`, title: `Page ${i}`, body },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function walDelta(
|
||||
db: Kysely<any>,
|
||||
fn: () => Promise<void>,
|
||||
): Promise<number> {
|
||||
const before = (
|
||||
await sql<{ l: string }>`select pg_current_wal_lsn() as l`.execute(db)
|
||||
).rows[0].l;
|
||||
await fn();
|
||||
// NOTE: no pg_switch_wal() — a segment switch pads the LSN to the next 16 MB
|
||||
// boundary and would swamp the delta. The raw LSN advances by the WAL bytes.
|
||||
const after = (
|
||||
await sql<{ l: string }>`select pg_current_wal_lsn() as l`.execute(db)
|
||||
).rows[0].l;
|
||||
return Number(
|
||||
(
|
||||
await sql<{
|
||||
d: string;
|
||||
}>`select pg_wal_lsn_diff(${after}::pg_lsn, ${before}::pg_lsn) as d`.execute(
|
||||
db,
|
||||
)
|
||||
).rows[0].d,
|
||||
);
|
||||
}
|
||||
|
||||
describe('#492 append-persist write volume (pg_current_wal_lsn delta) [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
let chatId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
const seedRow = () =>
|
||||
msgRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'streaming',
|
||||
metadata: stepMarkerMetadata(0) as never,
|
||||
});
|
||||
|
||||
const STEPS = 40;
|
||||
|
||||
it('NEW per-step INSERT is O(Σ steps); OLD full-row rewrite is O(n²)', async () => {
|
||||
const steps: Step[] = [];
|
||||
for (let i = 0; i < STEPS; i++) steps.push(makeStep(i));
|
||||
|
||||
// NEW: per-step INSERT of THIS step's parts + a cheap marker UPDATE.
|
||||
const newRow = await seedRow();
|
||||
const newWal = await walDelta(db, async () => {
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
await stepRepo.insertStep(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
i,
|
||||
assistantParts([steps[i]], ''),
|
||||
);
|
||||
await msgRepo.update(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
{ metadata: stepMarkerMetadata(i + 1) },
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// OLD (the pre-#492 revert): re-persist the GROWING metadata.parts on the
|
||||
// message row on every step.
|
||||
const oldRow = await seedRow();
|
||||
const oldWal = await walDelta(db, async () => {
|
||||
const acc: Step[] = [];
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
acc.push(steps[i]);
|
||||
await msgRepo.update(
|
||||
oldRow.id,
|
||||
workspaceId,
|
||||
flushAssistant(acc as never, '', 'streaming'),
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[#492 WAL] ${STEPS} steps ×100KB: new=${(newWal / 1e6).toFixed(1)}MB ` +
|
||||
`old=${(oldWal / 1e6).toFixed(1)}MB (${(oldWal / newWal).toFixed(
|
||||
1,
|
||||
)}x smaller)`,
|
||||
);
|
||||
|
||||
// O(Σ steps): ~STEPS × (100KB output + marker) of WAL. 40 × ~100KB parts plus
|
||||
// 40 tiny markers is a few tens of MB at most — bounded, linear in step count.
|
||||
expect(newWal).toBeLessThan(30_000_000);
|
||||
// O(n²): step k rewrites ~k × 100KB. Σ over 40 steps ≈ 80+ MB — far larger.
|
||||
expect(oldWal).toBeGreaterThan(30_000_000);
|
||||
// The load-bearing claim: the new path writes a small FRACTION of the old.
|
||||
expect(newWal).toBeLessThan(oldWal * 0.35);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -1,169 +0,0 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { AiChatController } from 'src/core/ai-chat/ai-chat.controller';
|
||||
import {
|
||||
assembleStepParts,
|
||||
assistantParts,
|
||||
stepMarkerMetadata,
|
||||
} from 'src/core/ai-chat/ai-chat.service';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import type { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
createMessage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 controller hydration (crash-before-finalize RESUME) on a LIVE Postgres.
|
||||
* `AiChatController.withReconstructedParts` is wired into getMessages/delta/export/
|
||||
* run, but `aiChatRunStepRepo` is OPTIONAL and every controller unit spec passes it
|
||||
* as `undefined`, so the hydration branch early-returns and NEVER executes in those
|
||||
* tests. This drives the real read path — a mid-run streaming row (marker only,
|
||||
* empty inline parts) PLUS its `ai_chat_run_steps` rows — through getMessages WITH
|
||||
* the repo present, exercising the `role==='assistant' && !rowHasInlineParts`
|
||||
* needy predicate, the workspace-scoped batch step fetch, and the endpoint binding.
|
||||
*/
|
||||
describe('#492 controller hydration read path [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let aiChatRepo: AiChatRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let workspaceId: string;
|
||||
let otherWorkspaceId: string;
|
||||
let userId: string;
|
||||
|
||||
// Build the controller WITH a real AiChatRunStepRepo injected (position 9), the
|
||||
// seam the unit specs leave undefined. Only the read-path deps are real.
|
||||
function buildController(): AiChatController {
|
||||
return new AiChatController(
|
||||
{} as any, // aiChatService
|
||||
{} as any, // aiChatRunService
|
||||
aiChatRepo,
|
||||
msgRepo,
|
||||
{} as any, // aiTranscription
|
||||
{} as any, // pageRepo
|
||||
undefined, // streamRegistry
|
||||
undefined, // environment
|
||||
stepRepo, // #492 aiChatRunStepRepo
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
aiChatRepo = new AiChatRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
otherWorkspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('getMessages reconstructs a mid-run row from the steps table (finished rows untouched)', async () => {
|
||||
const chatId = (
|
||||
await createChat(db, { workspaceId, creatorId: userId })
|
||||
).id;
|
||||
const user = { id: userId } as User;
|
||||
const workspace = { id: workspaceId } as Workspace;
|
||||
|
||||
// A prior FINISHED assistant row that already carries inline parts — the needy
|
||||
// predicate must SKIP it (no step fetch), returned untouched.
|
||||
const finishedParts = assistantParts(
|
||||
[{ text: 'done earlier', toolCalls: [], toolResults: [] } as any],
|
||||
'',
|
||||
);
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: 'done earlier',
|
||||
status: 'completed',
|
||||
metadata: { parts: finishedParts, toolTraceVersion: 2, stepsPersisted: 1 },
|
||||
createdAt: new Date(Date.now() - 3000),
|
||||
});
|
||||
|
||||
// The mid-run row a crash-before-finalize left behind: a step marker only
|
||||
// (parts:[] , content:''), status 'streaming'. Its real parts live ONLY in the
|
||||
// steps table.
|
||||
const midRun = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'streaming',
|
||||
metadata: stepMarkerMetadata(2),
|
||||
createdAt: new Date(Date.now() - 1000),
|
||||
});
|
||||
|
||||
const step0 = assistantParts(
|
||||
[
|
||||
{
|
||||
text: 'reasoning about the page',
|
||||
toolCalls: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', input: { id: 'p1' } },
|
||||
],
|
||||
toolResults: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', output: { id: 'p1', body: 'B' } },
|
||||
],
|
||||
} as any,
|
||||
],
|
||||
'',
|
||||
);
|
||||
const step1 = assistantParts(
|
||||
[{ text: 'partial synthesis so far', toolCalls: [], toolResults: [] } as any],
|
||||
'',
|
||||
);
|
||||
await stepRepo.insertStep(midRun.id, workspaceId, 0, step0);
|
||||
await stepRepo.insertStep(midRun.id, workspaceId, 1, step1);
|
||||
|
||||
// Workspace-scoping guard: a step row for the SAME message id under a DIFFERENT
|
||||
// workspace must NEVER leak into this workspace's reconstruction.
|
||||
await stepRepo.insertStep(midRun.id, otherWorkspaceId, 99, [
|
||||
{ type: 'text', text: 'FOREIGN_WORKSPACE_LEAK' },
|
||||
]);
|
||||
|
||||
const res = await buildController().getMessages(
|
||||
{ chatId } as any,
|
||||
{ limit: 50 } as any,
|
||||
user,
|
||||
workspace,
|
||||
);
|
||||
|
||||
const items = res.items as any[];
|
||||
const finished = items.find((r) => r.status === 'completed');
|
||||
const reconstructed = items.find((r) => r.id === midRun.id);
|
||||
|
||||
// The finished row passed through with its inline parts unchanged.
|
||||
expect(finished.metadata.parts).toEqual(finishedParts);
|
||||
|
||||
// The mid-run row's parts were reconstructed from the two step rows, in order,
|
||||
// exactly as assembleStepParts concatenates them — the client seed sees the
|
||||
// persisted progress with no change to itself.
|
||||
const expected = assembleStepParts([
|
||||
{ stepIndex: 0, parts: step0 },
|
||||
{ stepIndex: 1, parts: step1 },
|
||||
] as any);
|
||||
expect(reconstructed.metadata.parts).toEqual(expected);
|
||||
// The foreign-workspace step row did NOT leak in.
|
||||
expect(JSON.stringify(reconstructed.metadata.parts)).not.toContain(
|
||||
'FOREIGN_WORKSPACE_LEAK',
|
||||
);
|
||||
// Sanity: reconstruction produced real content (text + the paired tool part +
|
||||
// the second step's text), not an empty fallback.
|
||||
expect(reconstructed.metadata.parts).toContainEqual({
|
||||
type: 'text',
|
||||
text: 'reasoning about the page',
|
||||
});
|
||||
expect(
|
||||
(reconstructed.metadata.parts as any[]).some((p) => p.type === 'tool-getPage'),
|
||||
).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -1,163 +0,0 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { Kysely } from 'kysely';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import {
|
||||
assistantParts,
|
||||
reconstructRunParts,
|
||||
hydrateAssistantParts,
|
||||
stepMarkerMetadata,
|
||||
rowHasInlineParts,
|
||||
} from '../../src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — the reconstruct CONTRACT on a live Postgres. Proves that a
|
||||
* turn persisted the NEW way (per-step rows in `ai_chat_run_steps`, only a step
|
||||
* marker on the message row) reconstructs to the SAME UI parts as a turn persisted
|
||||
* the OLD way (full `metadata.parts` inline on the row, no step rows) — so the
|
||||
* era-switch is invisible to attach / delta-poll / export. Real repos + real jsonb
|
||||
* roundtrip, not a mock (a mock cannot prove the parts survive the jsonb column
|
||||
* byte-identical).
|
||||
*/
|
||||
type Step = {
|
||||
text: string;
|
||||
toolCalls: Array<{ toolCallId: string; toolName: string; input: unknown }>;
|
||||
toolResults: Array<{ toolCallId: string; toolName: string; output: unknown }>;
|
||||
};
|
||||
|
||||
// A realistic step: some text + a getPage tool call whose ~100 KB body is
|
||||
// INCOMPRESSIBLE random base64 (a 'x'.repeat filler would TOAST away and hide the
|
||||
// real bytes). Under MAX_TOOL_OUTPUT_BYTES (200 KB) it is stored uncompacted.
|
||||
function makeStep(i: number, outputBytes = 4_000): Step {
|
||||
const body = randomBytes(Math.ceil(outputBytes * 0.75)).toString('base64');
|
||||
return {
|
||||
text: `step ${i} text`,
|
||||
toolCalls: [
|
||||
{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'getPage',
|
||||
output: { id: `p${i}`, title: `Page ${i}`, body },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('AiChatRunStepRepo + reconstruct contract [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
let chatId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
const seedRow = (metadata: unknown, status: string) =>
|
||||
msgRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status,
|
||||
metadata: metadata as never,
|
||||
});
|
||||
|
||||
it('insertStep is idempotent per (message, stepIndex) and reads back in order', async () => {
|
||||
const row = await seedRow(stepMarkerMetadata(0), 'streaming');
|
||||
const parts0 = assistantParts([makeStep(0)], '');
|
||||
const parts1 = assistantParts([makeStep(1)], '');
|
||||
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 0, parts0)).toBe(true);
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 1, parts1)).toBe(true);
|
||||
// A retried persist of the SAME step is a no-op (ON CONFLICT DO NOTHING).
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 0, parts0)).toBe(
|
||||
false,
|
||||
);
|
||||
|
||||
const steps = await stepRepo.findByMessage(row.id, workspaceId);
|
||||
expect(steps.map((s) => s.stepIndex)).toEqual([0, 1]);
|
||||
|
||||
// Batch fetch groups by message id in step order.
|
||||
const map = await stepRepo.findByMessageIds([row.id], workspaceId);
|
||||
expect(map.get(row.id)!.map((s) => s.stepIndex)).toEqual([0, 1]);
|
||||
});
|
||||
|
||||
it('a NEW-style (step-table) run reconstructs identically to an OLD-style (inline) run', async () => {
|
||||
const steps = [makeStep(10), makeStep(11)];
|
||||
// The inline parts the OLD full-row flush would have written.
|
||||
const fullParts = assistantParts(steps, '');
|
||||
|
||||
// OLD-style record: full parts inline on the row, NO step rows.
|
||||
const oldRow = await seedRow(
|
||||
{ parts: fullParts, toolTraceVersion: 2, stepsPersisted: 2 },
|
||||
'completed',
|
||||
);
|
||||
|
||||
// NEW-style record: only a step marker on the row + per-step rows.
|
||||
const newRow = await seedRow(stepMarkerMetadata(2), 'streaming');
|
||||
for (let i = 0; i < steps.length; i++) {
|
||||
await stepRepo.insertStep(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
i,
|
||||
assistantParts([steps[i]], ''),
|
||||
);
|
||||
}
|
||||
|
||||
// Re-read both from the DB (proves the jsonb roundtrip).
|
||||
const oldFetched = await msgRepo.findById(oldRow.id, workspaceId);
|
||||
const newFetched = await msgRepo.findById(newRow.id, workspaceId);
|
||||
const oldSteps = await stepRepo.findByMessage(oldRow.id, workspaceId);
|
||||
const newSteps = await stepRepo.findByMessage(newRow.id, workspaceId);
|
||||
|
||||
// The discriminator: the old row carries inline parts, the new one does not.
|
||||
expect(rowHasInlineParts(oldFetched!)).toBe(true);
|
||||
expect(rowHasInlineParts(newFetched!)).toBe(false);
|
||||
expect(oldSteps).toHaveLength(0);
|
||||
expect(newSteps).toHaveLength(2);
|
||||
|
||||
const oldRecon = reconstructRunParts(oldFetched!, oldSteps);
|
||||
const newRecon = reconstructRunParts(newFetched!, newSteps);
|
||||
|
||||
// Both reconstruct to the SAME parts + step count — the era is invisible.
|
||||
expect(newRecon.parts).toEqual(fullParts);
|
||||
expect(oldRecon.parts).toEqual(fullParts);
|
||||
expect(newRecon.parts).toEqual(oldRecon.parts);
|
||||
expect(newRecon.stepsPersisted).toBe(2);
|
||||
expect(oldRecon.stepsPersisted).toBe(2);
|
||||
|
||||
// hydrateAssistantParts fills the new row's metadata.parts to match the old
|
||||
// row's inline parts — so a consumer reading `metadata.parts` off the raw row
|
||||
// (the client seed/poll, export) is unchanged across the era.
|
||||
const map = await stepRepo.findByMessageIds([newRow.id], workspaceId);
|
||||
const [hydrated] = hydrateAssistantParts([newFetched!], map);
|
||||
expect((hydrated.metadata as { parts: unknown }).parts).toEqual(fullParts);
|
||||
// A row that already has inline parts passes through untouched (same ref-shape).
|
||||
const [oldPassThrough] = hydrateAssistantParts([oldFetched!], map);
|
||||
expect((oldPassThrough.metadata as { parts: unknown }).parts).toEqual(
|
||||
fullParts,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,63 +0,0 @@
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import {
|
||||
up,
|
||||
down,
|
||||
} from '../../src/database/migrations/20260708T120000-ai-chat-run-steps';
|
||||
import { getTestDb, destroyTestDb } from './db';
|
||||
|
||||
/**
|
||||
* #492 migration up/down roundtrip on a LIVE Postgres. global-setup already
|
||||
* migrated docmost_test to latest (so the table exists at start); this drives the
|
||||
* migration's own down()/up() and asserts the table presence toggles, then leaves
|
||||
* it PRESENT (up) so the shared test DB is intact for any spec that runs after.
|
||||
*/
|
||||
async function tableExists(db: Kysely<any>): Promise<boolean> {
|
||||
const row = (
|
||||
await sql<{ t: string | null }>`select to_regclass('ai_chat_run_steps') as t`.execute(
|
||||
db,
|
||||
)
|
||||
).rows[0];
|
||||
return row.t !== null;
|
||||
}
|
||||
|
||||
async function uniqueIndexExists(db: Kysely<any>): Promise<boolean> {
|
||||
const row = (
|
||||
await sql<{
|
||||
t: string | null;
|
||||
}>`select to_regclass('ai_chat_run_steps_message_step_uidx') as t`.execute(db)
|
||||
).rows[0];
|
||||
return row.t !== null;
|
||||
}
|
||||
|
||||
describe('20260708 ai_chat_run_steps migration roundtrip [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
|
||||
beforeAll(() => {
|
||||
db = getTestDb();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Belt-and-suspenders: guarantee the table is present for later specs even if
|
||||
// an assertion threw mid-roundtrip.
|
||||
if (!(await tableExists(db))) await up(db);
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('down() drops the table+index and up() recreates them (idempotent)', async () => {
|
||||
// Starts applied (global-setup migrated to latest).
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
expect(await uniqueIndexExists(db)).toBe(true);
|
||||
|
||||
await down(db);
|
||||
expect(await tableExists(db)).toBe(false);
|
||||
expect(await uniqueIndexExists(db)).toBe(false);
|
||||
|
||||
await up(db);
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
expect(await uniqueIndexExists(db)).toBe(true);
|
||||
|
||||
// up() is idempotent (ifNotExists) — a second run is a harmless no-op.
|
||||
await expect(up(db)).resolves.not.toThrow();
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -118,7 +118,10 @@ All 41 tools, grouped by what you'd reach for them.
|
||||
- **`getPage`** — A page's content as clean **Markdown** (canonical for text; drops only
|
||||
block ids, resolved-comment anchors, and a fixed no-Markdown-representation attr set —
|
||||
table spans/colwidth/background, indent, `callout.icon`, `orderedList.type`, and link
|
||||
`internal`/`target`/`rel`/`class`; use `getPageJson` when you need those).
|
||||
`internal`/`target`/`rel`/`class`; use `getPageJson` when you need those). Pass
|
||||
`format:"text"` for a flat, deterministic plain-text rendering (one line per block,
|
||||
marks dropped, stable `[image]`/`[table RxC]` placeholders) to machine-diff what you
|
||||
wrote against what was stored.
|
||||
- **`getPageJson`** — A page's **lossless ProseMirror/TipTap JSON**, including every
|
||||
block's `attrs.id` and the `slugId` used in URLs. This is what the per-block editing
|
||||
tools consume.
|
||||
|
||||
@@ -123,7 +123,10 @@ Docmost-MCP не сочетают:
|
||||
лишь id блоков, якоря разрешённых комментариев и фиксированный набор атрибутов без
|
||||
markdown-представления — спаны/colwidth/фон ячеек таблиц, отступы (indent),
|
||||
`callout.icon`, `orderedList.type` и `internal`/`target`/`rel`/`class` у ссылок;
|
||||
используйте `getPageJson`, когда они нужны).
|
||||
используйте `getPageJson`, когда они нужны). Передайте `format:"text"` для плоского
|
||||
детерминированного plain-text рендера (одна строка на блок, маркировка убрана,
|
||||
стабильные плейсхолдеры `[image]`/`[table RxC]`) — чтобы machine-diff'ом сверить то,
|
||||
что вы записали, с тем, что сохранилось.
|
||||
- **`getPageJson`** — **Lossless ProseMirror/TipTap JSON** страницы, включая `attrs.id`
|
||||
каждого блока и `slugId`, используемый в URL. Именно его потребляют инструменты
|
||||
поблочного редактирования.
|
||||
|
||||
@@ -701,10 +701,20 @@ export abstract class DocmostClientContext {
|
||||
}
|
||||
|
||||
|
||||
/** Raw page info including the ProseMirror JSON content and slugId. */
|
||||
async getPageRaw(pageId: string) {
|
||||
/**
|
||||
* Raw page info including the ProseMirror JSON content and slugId.
|
||||
*
|
||||
* With `format:"text"` (#502) the server instead renders `content` as a flat,
|
||||
* deterministic text string (its `jsonToText` path — the SAME serializer that
|
||||
* feeds search), so the MCP text read reuses the server's ONE serializer
|
||||
* rather than shipping a second one. Every other caller omits `format` and
|
||||
* gets the JSON content unchanged.
|
||||
*/
|
||||
async getPageRaw(pageId: string, format?: "text") {
|
||||
await this.ensureAuthenticated();
|
||||
const response = await this.client.post("/pages/info", { pageId });
|
||||
const body: Record<string, unknown> = { pageId };
|
||||
if (format) body.format = format;
|
||||
const response = await this.client.post("/pages/info", body);
|
||||
return response.data?.data ?? response.data;
|
||||
}
|
||||
|
||||
|
||||
@@ -93,6 +93,13 @@ export function PagesMixin<TBase extends GConstructor<DocmostClientContext>>(Bas
|
||||
const buildForm = () => {
|
||||
const form = new FormData();
|
||||
form.append("spaceId", spaceId);
|
||||
// #502: this is an AGENT-authored body (plain prose / config), so tell the
|
||||
// server import path to run the markdown importer with the two layered
|
||||
// extensions OFF — a `$…$` span stays literal text (real math via
|
||||
// `update_page_json`) and a schemeless `www.host`/email is not autolinked
|
||||
// (an explicit `https://…` still links). A human file upload never sends
|
||||
// this field, so human imports keep math + autolink ON.
|
||||
form.append("disableMarkdownExtensions", "true");
|
||||
form.append("file", fileContent, {
|
||||
filename: `${title || "import"}.md`,
|
||||
contentType: "text/markdown",
|
||||
|
||||
@@ -72,7 +72,7 @@ export interface IReadMixin {
|
||||
getTree(spaceId: string, rootPageId?: string, maxDepth?: number): any;
|
||||
getPageContext(pageId: string): any;
|
||||
listSidebarPages(spaceId: string, pageId?: string): any;
|
||||
getPage(pageId: string): any;
|
||||
getPage(pageId: string, format?: "markdown" | "text"): any;
|
||||
getPageJson(pageId: string): any;
|
||||
getOutline(pageId: string): any;
|
||||
getNode(pageId: string, nodeId: string, format?: "markdown" | "json"): any;
|
||||
@@ -420,8 +420,34 @@ export function ReadMixin<TBase extends GConstructor<DocmostClientContext>>(Base
|
||||
return convertProseMirrorToMarkdown(content, options);
|
||||
}
|
||||
|
||||
async getPage(pageId: string) {
|
||||
async getPage(pageId: string, format: "markdown" | "text" = "markdown") {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// #502 `format:"text"`: a flat, deterministic, machine-diffable rendering
|
||||
// (block-per-line, inline marks/comment anchors dropped, non-text nodes ->
|
||||
// stable placeholders like `[image]` / `[table RxC]`). The server produces
|
||||
// it via its `jsonToText` path (the SAME serializer that feeds search), so
|
||||
// there is no second serializer here — we just request it and pass the
|
||||
// string through. Distinct from the markdown default: no PM->markdown walk,
|
||||
// no markdown conversion cache, and no `{{SUBPAGES}}` substitution (the text
|
||||
// renderer emits no such placeholder). Use it to diff a page you wrote as a
|
||||
// config/prose against what was stored.
|
||||
if (format === "text") {
|
||||
const textData = await this.getPageRaw(pageId, "text");
|
||||
let subpages: any[] = [];
|
||||
try {
|
||||
subpages = await this.listSidebarPages(textData.spaceId, textData.id);
|
||||
} catch (e: any) {
|
||||
console.warn("Failed to fetch subpages:", e);
|
||||
}
|
||||
const textContent =
|
||||
typeof textData.content === "string" ? textData.content : "";
|
||||
return {
|
||||
data: filterPage(textData, textContent, subpages),
|
||||
success: true,
|
||||
};
|
||||
}
|
||||
|
||||
const resultData = await this.getPageRaw(pageId);
|
||||
|
||||
// Agent read: hide resolved-comment anchors so the agent sees only active
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
markdownToProseMirror,
|
||||
normalizeAgentMarkdown,
|
||||
} from "@docmost/prosemirror-markdown";
|
||||
import type { MarkdownImportOptions } from "@docmost/prosemirror-markdown";
|
||||
import { docmostExtensions, docmostSchema } from "./docmost-schema.js";
|
||||
import { withPageLock } from "./page-lock.js";
|
||||
import type { PageId } from "./page-id.js";
|
||||
@@ -111,16 +112,31 @@ global.WebSocket = WebSocket;
|
||||
* horizontalRule the serializer emitted, and stripping it would silently drop the
|
||||
* page's leading content (#493 review). The front-matter strip stays on the
|
||||
* server FILE-import boundary only (`normalizeForeignMarkdown`).
|
||||
*
|
||||
* #502 IMPORTANT — the two layered markdown extensions (`$…$` math, schemeless
|
||||
* fuzzy autolink) are NOT decided here; they are the CALLER's choice via
|
||||
* `options`, because the two callers of this wrapper need OPPOSITE behavior:
|
||||
* - AGENT-authored plain markdown (`updatePageMarkdown`) → both OFF, so a
|
||||
* `$…$` config span stays literal and a bare `www.host` is not autolinked
|
||||
* (real math is authored via `update_page_json`).
|
||||
* - FULL-FILE round-trip import (`import_page_markdown`, #328 lossless) →
|
||||
* DEFAULTS (both ON), because the exporter serializes a math node as readable
|
||||
* `$x^2$`; re-importing with math OFF would degrade it to literal text and
|
||||
* BREAK the lossless export→import pair.
|
||||
* So `options` DEFAULTS to `undefined` → the package importer's defaults (ON),
|
||||
* which is the safe round-trip behavior; the agent-write caller opts OUT
|
||||
* explicitly. (The fragment path `importMarkdownFragment` opts out on its own.)
|
||||
*/
|
||||
export async function markdownToProseMirrorCanonical(
|
||||
markdownContent: string,
|
||||
options?: MarkdownImportOptions,
|
||||
): Promise<any> {
|
||||
// #419: normalize + merge glyph-forked footnote definitions BEFORE
|
||||
// canonicalizing, so the canonicalizer re-hangs references and drops the
|
||||
// now-orphaned duplicate definitions.
|
||||
return canonicalizeFootnotes(
|
||||
normalizeAndMergeFootnotes(
|
||||
await markdownToProseMirror(normalizeAgentMarkdown(markdownContent)),
|
||||
await markdownToProseMirror(normalizeAgentMarkdown(markdownContent), options),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -358,7 +374,17 @@ export async function updatePageContentRealtime(
|
||||
): Promise<MutationResult> {
|
||||
// PAGE write: canonicalize footnotes (markdown import builds the bottom list in
|
||||
// definition order; numbering is reference-ordered).
|
||||
const tiptapJson = await markdownToProseMirrorCanonical(markdownContent);
|
||||
//
|
||||
// #502: this is the AGENT-authored `updatePageMarkdown` body — plain prose /
|
||||
// config — so the two layered markdown extensions are turned OFF: a `$…$` span
|
||||
// stays literal text (real math via `update_page_json`) and a SCHEMELESS
|
||||
// `www.host`/email is not autolinked (an explicit `https://…` still links).
|
||||
// Contrast `import_page_markdown`, which keeps DEFAULTS for the #328 lossless
|
||||
// round-trip.
|
||||
const tiptapJson = await markdownToProseMirrorCanonical(markdownContent, {
|
||||
parseMath: false,
|
||||
fuzzyLinkify: false,
|
||||
});
|
||||
return await mutatePageContent(
|
||||
pageId,
|
||||
collabToken,
|
||||
|
||||
@@ -135,7 +135,17 @@ export interface MarkdownFragment {
|
||||
export async function importMarkdownFragment(
|
||||
markdown: string,
|
||||
): Promise<MarkdownFragment> {
|
||||
const doc = await markdownToProseMirror(markdown);
|
||||
// #502: the fragment path is an MCP agent WRITE (patch_node/insert_node
|
||||
// markdown), so it uses the SAME extensions-OFF importer options as the
|
||||
// full-page write (markdownToProseMirrorCanonical): a `$…$` span stays literal
|
||||
// and a schemeless domain/email is not autolinked. This keeps a block written
|
||||
// via markdown canonically identical to the same content in a full-page write
|
||||
// (no "second canon"). Explicit `https://…` links and block structure are
|
||||
// unaffected.
|
||||
const doc = await markdownToProseMirror(markdown, {
|
||||
parseMath: false,
|
||||
fuzzyLinkify: false,
|
||||
});
|
||||
const content: any[] = Array.isArray(doc?.content) ? doc.content : [];
|
||||
|
||||
const blocks: any[] = [];
|
||||
|
||||
@@ -469,7 +469,11 @@ export const SHARED_TOOL_SPECS = {
|
||||
'(markdown). The fragment may be SEVERAL blocks (a "1 → N" splice: rewrite a ' +
|
||||
'whole section in one call) — the first block inherits this block id, the ' +
|
||||
'rest get fresh ids. `^[...]` footnotes are supported (their definitions ' +
|
||||
"merge into the page's footnote list). REJECTED when the target is a table " +
|
||||
"merge into the page's footnote list). Markdown is taken LITERALLY — " +
|
||||
'`$...$`/`$$...$$` are NOT parsed as math and schemeless `www.host` / bare ' +
|
||||
'emails are NOT auto-linked (an explicit `https://` URL still links); for a ' +
|
||||
'real formula pass a `mathInline`/`mathBlock` ProseMirror node via `node` ' +
|
||||
'(or updatePageJson). REJECTED when the target is a table ' +
|
||||
'cell with attributes markdown cannot represent (merged/colored/fixed-width) ' +
|
||||
'— use the table tools or `node`. ' +
|
||||
'`node` (for precise attr/mark work): a raw ProseMirror node, e.g. a ' +
|
||||
@@ -535,6 +539,10 @@ export const SHARED_TOOL_SPECS = {
|
||||
'Provide EXACTLY ONE of `markdown` or `node`. ' +
|
||||
'`markdown` (RECOMMENDED): a canonical markdown fragment — may be SEVERAL ' +
|
||||
'blocks, inserted in order at the anchor; `^[...]` footnotes supported. ' +
|
||||
'Markdown is taken LITERALLY — `$...$`/`$$...$$` are NOT parsed as math and ' +
|
||||
'schemeless `www.host` / bare emails are NOT auto-linked (an explicit ' +
|
||||
'`https://` URL still links); for a real formula pass a ' +
|
||||
'`mathInline`/`mathBlock` ProseMirror node via `node` (or updatePageJson). ' +
|
||||
'`node` (for precise attr/mark work OR table structure): a raw ProseMirror ' +
|
||||
'node. Table structure is JSON-only (not expressible in markdown): to add a ' +
|
||||
'tableRow, pass a tableRow node with position before/after and anchor INSIDE ' +
|
||||
@@ -913,28 +921,46 @@ export const SHARED_TOOL_SPECS = {
|
||||
inAppKey: 'getPage',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Fetch a single page as Markdown by its id. Returns the page title and ' +
|
||||
'its Markdown content. The converter is canonical (round-trips text and ' +
|
||||
'block structure), so this is sufficient for text edits; use the ' +
|
||||
'page-JSON read tool only when you need what Markdown cannot carry. The ' +
|
||||
'Markdown drops exactly: (1) block ids (not visible in Markdown); ' +
|
||||
'(2) resolved-comment anchors (hidden here; only active <span ' +
|
||||
'data-comment-id> anchors remain); (3) a fixed set of attributes with no ' +
|
||||
'Markdown representation — table-cell colspan/rowspan/colwidth/' +
|
||||
'backgroundColor/backgroundColorName, heading/paragraph indent, ' +
|
||||
'callout.icon, orderedList.type, and link internal/target/rel/class. ' +
|
||||
'Inline <span data-comment-id> tags in the markdown are comment highlight ' +
|
||||
'anchors — treat them as markup, not page text.',
|
||||
'Fetch a single page by its id. Returns the page title and its content. ' +
|
||||
'format:"markdown" (DEFAULT) returns canonical Markdown (round-trips text ' +
|
||||
'and block structure), sufficient for text edits; use the page-JSON read ' +
|
||||
'tool only when you need what Markdown cannot carry. The Markdown drops ' +
|
||||
'exactly: (1) block ids (not visible in Markdown); (2) resolved-comment ' +
|
||||
'anchors (hidden here; only active <span data-comment-id> anchors remain); ' +
|
||||
'(3) a fixed set of attributes with no Markdown representation — table-cell ' +
|
||||
'colspan/rowspan/colwidth/backgroundColor/backgroundColorName, ' +
|
||||
'heading/paragraph indent, callout.icon, orderedList.type, and link ' +
|
||||
'internal/target/rel/class. Inline <span data-comment-id> tags in the ' +
|
||||
'markdown are comment highlight anchors — treat them as markup, not page ' +
|
||||
'text. format:"text" returns a FLAT, DETERMINISTIC plain-text rendering ' +
|
||||
'for machine diffing: one line per block, ALL inline marks/formatting and ' +
|
||||
'comment anchors dropped, a hardBreak is a newline, and non-text nodes ' +
|
||||
'become STABLE placeholders — an image is "[image]" and a table is ' +
|
||||
'"[table RxC]" (R rows x C columns). This output is stable across versions ' +
|
||||
'(pinned by a snapshot test); use it to diff a config/prose you wrote ' +
|
||||
'against what was stored (an empty diff means it was stored verbatim).',
|
||||
tier: 'core',
|
||||
catalogLine: 'getPage — fetch a page as Markdown by its id.',
|
||||
catalogLine:
|
||||
'getPage — fetch a page by its id (format:"markdown" default, or "text" for a flat machine-diffable read).',
|
||||
// Reconciled: MCP's stricter .min(1) kept; in-app's more-informative
|
||||
// "(or slugId)" describe kept.
|
||||
buildShape: (z) => ({
|
||||
pageId: z.string().min(1).describe('The id (or slugId) of the page.'),
|
||||
format: z
|
||||
.enum(['markdown', 'text'])
|
||||
.optional()
|
||||
.describe(
|
||||
'Output format: "markdown" (default, canonical round-trippable) or ' +
|
||||
'"text" (flat deterministic plain text for machine diffing).',
|
||||
),
|
||||
}),
|
||||
// MCP wraps the raw `{ data, success }` as JSON. The in-app host instead
|
||||
// projects a token-efficient `{ title, markdown }` (its long-standing shape).
|
||||
execute: (client, { pageId }) => client.getPage(pageId as string),
|
||||
execute: (client, { pageId, format }) =>
|
||||
client.getPage(
|
||||
pageId as string,
|
||||
(format as 'markdown' | 'text' | undefined) ?? 'markdown',
|
||||
),
|
||||
inAppExecute: async (client, { pageId }) => {
|
||||
// getPage(pageId) -> { data: filterPage(page, markdown), success }.
|
||||
const result = (await client.getPage(pageId as string)) as {
|
||||
@@ -1077,8 +1103,12 @@ export const SHARED_TOOL_SPECS = {
|
||||
description:
|
||||
'Create a new page with a Markdown body in a space, optionally under a ' +
|
||||
'parent page (omit parentPageId to create at the space root). Returns ' +
|
||||
'the new page id and title. Reversible: a page can be moved to trash ' +
|
||||
'later.',
|
||||
'the new page id and title. Body text is taken LITERALLY — ' +
|
||||
'`$...$`/`$$...$$` are NOT parsed into a math formula and schemeless ' +
|
||||
'`www.host` / bare emails are NOT auto-linked (an explicit `https://` URL ' +
|
||||
'still links); for a real formula use updatePageJson with ' +
|
||||
'`mathInline`/`mathBlock` nodes instead. Reversible: a page can be moved ' +
|
||||
'to trash later.',
|
||||
tier: 'deferred',
|
||||
catalogLine: 'createPage — create a new page with a Markdown body in a space.',
|
||||
// Reconciled schema DRIFT: the MCP copy pinned `content` to .min(1) while
|
||||
@@ -1337,7 +1367,11 @@ export const SHARED_TOOL_SPECS = {
|
||||
'title). The whole body is re-imported from the markdown (block ids ' +
|
||||
'regenerate — for surgical or id-preserving edits use the find/replace, ' +
|
||||
'node-patch or page-JSON tools instead). Docmost-flavoured markdown is ' +
|
||||
'parsed, including `^[...]` inline footnotes. Reversible: the previous ' +
|
||||
'parsed, including `^[...]` inline footnotes. Text is taken LITERALLY — ' +
|
||||
'`$...$`/`$$...$$` are NOT parsed into a math formula and schemeless ' +
|
||||
'`www.host` / bare emails are NOT auto-linked (an explicit `https://` URL ' +
|
||||
'still links); for a real formula use updatePageJson with ' +
|
||||
'`mathInline`/`mathBlock` nodes instead. Reversible: the previous ' +
|
||||
'version is kept in page history.',
|
||||
tier: 'deferred',
|
||||
catalogLine:
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
// #502 BLOCKER 1 (client half): the MCP `createPage` tool builds its body as an
|
||||
// AGENT-authored markdown file and POSTs it to the server `/pages/import`
|
||||
// endpoint. It must send the `disableMarkdownExtensions=true` multipart field so
|
||||
// the SERVER importer runs with math + fuzzy-autolink OFF (a human file upload
|
||||
// omits the field and keeps them ON). This mock http server captures the raw
|
||||
// multipart body and asserts the field is present.
|
||||
import { test, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { DocmostClient } from "../../build/client.js";
|
||||
|
||||
function sendJson(res, status, obj, extra = {}) {
|
||||
res.writeHead(status, { "Content-Type": "application/json", ...extra });
|
||||
res.end(JSON.stringify(obj));
|
||||
}
|
||||
function readRaw(req) {
|
||||
return new Promise((resolve) => {
|
||||
const chunks = [];
|
||||
req.on("data", (c) => chunks.push(c));
|
||||
req.on("end", () => resolve(Buffer.concat(chunks).toString("utf-8")));
|
||||
});
|
||||
}
|
||||
|
||||
const openServers = [];
|
||||
after(async () => {
|
||||
await Promise.all(openServers.map((s) => new Promise((r) => s.close(r))));
|
||||
});
|
||||
|
||||
const NEW_ID = "00000000-0000-4000-8000-000000000042";
|
||||
const SPACE = "00000000-0000-4000-8000-0000000000aa";
|
||||
|
||||
function spawn(state) {
|
||||
return new Promise((resolve) => {
|
||||
const server = http.createServer(async (req, res) => {
|
||||
const raw = await readRaw(req);
|
||||
if (req.url === "/api/auth/login") {
|
||||
return sendJson(res, 200, { success: true }, { "Set-Cookie": "authToken=t; Path=/; HttpOnly" });
|
||||
}
|
||||
if (req.url === "/api/pages/import") {
|
||||
state.importBody = raw; // the raw multipart payload
|
||||
return sendJson(res, 200, { data: { id: NEW_ID } });
|
||||
}
|
||||
if (req.url === "/api/pages/update") {
|
||||
return sendJson(res, 200, { data: { id: NEW_ID } });
|
||||
}
|
||||
if (req.url === "/api/pages/info") {
|
||||
return sendJson(res, 200, {
|
||||
data: {
|
||||
id: NEW_ID,
|
||||
slugId: "slugnew1234",
|
||||
title: "T",
|
||||
spaceId: SPACE,
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
content: { type: "doc", content: [{ type: "paragraph", content: [{ type: "text", text: "x" }] }] },
|
||||
},
|
||||
});
|
||||
}
|
||||
if (req.url === "/api/pages/sidebar-pages") {
|
||||
return sendJson(res, 200, { data: { items: [], meta: { hasNextPage: false, nextCursor: null } } });
|
||||
}
|
||||
return sendJson(res, 404, { message: "not found" });
|
||||
});
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
openServers.push(server);
|
||||
resolve(`http://127.0.0.1:${server.address().port}/api`);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test("createPage sends disableMarkdownExtensions=true in the /pages/import multipart", async () => {
|
||||
const state = {};
|
||||
const baseURL = await spawn(state);
|
||||
const client = new DocmostClient(baseURL, "e@x.com", "pw");
|
||||
|
||||
await client.createPage("My Config", "ticket $x=1$ at www.host.com", SPACE);
|
||||
|
||||
assert.ok(state.importBody, "the import endpoint received a body");
|
||||
// The multipart payload carries the field name and its "true" value.
|
||||
assert.match(state.importBody, /name="disableMarkdownExtensions"/);
|
||||
assert.match(state.importBody, /name="disableMarkdownExtensions"[\s\S]*?\r?\n\r?\ntrue\r?\n/);
|
||||
// The agent body itself is still sent as the file part.
|
||||
assert.match(state.importBody, /ticket \$x=1\$ at www\.host\.com/);
|
||||
});
|
||||
@@ -0,0 +1,116 @@
|
||||
// #502 READ: the getPage tool gains a `format:"text"` mode. It requests the
|
||||
// server's flat, deterministic text rendering (the server's jsonToText path — the
|
||||
// SAME serializer that feeds search), passing it through unchanged. This mock
|
||||
// stands up a local http server (same harness style as getpage-conversion-cache)
|
||||
// and asserts end-to-end that:
|
||||
// - format:"text" sends `format:"text"` in the /pages/info body and returns the
|
||||
// server's text string verbatim (no client-side markdown conversion);
|
||||
// - the DEFAULT (no format) still returns markdown-converted content;
|
||||
// - the text read resolves page + subpages like the markdown read.
|
||||
import { test, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { DocmostClient } from "../../build/client.js";
|
||||
|
||||
function readBody(req) {
|
||||
return new Promise((resolve) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => resolve(raw));
|
||||
});
|
||||
}
|
||||
function sendJson(res, status, obj, extra = {}) {
|
||||
res.writeHead(status, { "Content-Type": "application/json", ...extra });
|
||||
res.end(JSON.stringify(obj));
|
||||
}
|
||||
|
||||
const openServers = [];
|
||||
after(async () => {
|
||||
await Promise.all(openServers.map((s) => new Promise((r) => s.close(r))));
|
||||
});
|
||||
|
||||
const PAGE_UUID = "00000000-0000-4000-8000-000000000010";
|
||||
const SPACE_UUID = "00000000-0000-4000-8000-0000000000aa";
|
||||
const CHILD_UUID = "00000000-0000-4000-8000-0000000000bb";
|
||||
|
||||
// The deterministic text the SERVER's jsonToText would produce for this page.
|
||||
const SERVER_TEXT = "Title line\nsecond block\n[image]\n[table 2x3]";
|
||||
|
||||
function makeDoc(text) {
|
||||
return { type: "doc", content: [{ type: "paragraph", content: [{ type: "text", text }] }] };
|
||||
}
|
||||
|
||||
function spawn(state) {
|
||||
return new Promise((resolve) => {
|
||||
const server = http.createServer(async (req, res) => {
|
||||
const body = await readBody(req);
|
||||
if (req.url === "/api/auth/login") {
|
||||
return sendJson(res, 200, { success: true }, { "Set-Cookie": "authToken=t; Path=/; HttpOnly" });
|
||||
}
|
||||
if (req.url === "/api/pages/info") {
|
||||
const parsed = body ? JSON.parse(body) : {};
|
||||
state.lastInfoBody = parsed;
|
||||
// Emulate the server: when format:"text" is requested, `content` is the
|
||||
// flat text string; otherwise it is the raw ProseMirror JSON.
|
||||
const content = parsed.format === "text" ? SERVER_TEXT : makeDoc("Title line");
|
||||
return sendJson(res, 200, {
|
||||
success: true,
|
||||
data: {
|
||||
id: PAGE_UUID,
|
||||
slugId: "slug123456",
|
||||
title: "Text Page",
|
||||
parentPageId: null,
|
||||
spaceId: SPACE_UUID,
|
||||
updatedAt: "2026-01-01T00:00:00Z",
|
||||
content,
|
||||
},
|
||||
});
|
||||
}
|
||||
if (req.url === "/api/pages/sidebar-pages") {
|
||||
return sendJson(res, 200, {
|
||||
success: true,
|
||||
data: {
|
||||
items: [{ id: CHILD_UUID, title: "Child", hasChildren: false }],
|
||||
meta: { hasNextPage: false, nextCursor: null },
|
||||
},
|
||||
});
|
||||
}
|
||||
return sendJson(res, 404, { message: "not found" });
|
||||
});
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
openServers.push(server);
|
||||
resolve(`http://127.0.0.1:${server.address().port}/api`);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function makeClient(baseURL) {
|
||||
return new DocmostClient({ apiUrl: baseURL, getToken: async () => "access" });
|
||||
}
|
||||
|
||||
test('getPage(format:"text") requests text and returns the server text verbatim', async () => {
|
||||
const state = {};
|
||||
const client = makeClient(await spawn(state));
|
||||
|
||||
const result = await client.getPage(PAGE_UUID, "text");
|
||||
assert.equal(state.lastInfoBody.format, "text", "the info request carried format:text");
|
||||
assert.equal(result.success, true);
|
||||
assert.equal(result.data.content, SERVER_TEXT, "returns the server's flat text unchanged");
|
||||
// Deterministic placeholders are surfaced to the agent.
|
||||
assert.ok(result.data.content.includes("[image]"));
|
||||
assert.ok(result.data.content.includes("[table 2x3]"));
|
||||
// No client-side markdown artifacts leaked in.
|
||||
assert.ok(!result.data.content.includes("{{SUBPAGES}}"));
|
||||
// Subpages still resolve for context.
|
||||
assert.deepEqual(result.data.subpages, [{ id: CHILD_UUID, title: "Child" }]);
|
||||
});
|
||||
|
||||
test('getPage default (no format) still returns markdown, not text', async () => {
|
||||
const state = {};
|
||||
const client = makeClient(await spawn(state));
|
||||
|
||||
const result = await client.getPage(PAGE_UUID);
|
||||
assert.equal(state.lastInfoBody.format, undefined, "default read sends no format");
|
||||
assert.ok(result.data.content.includes("Title line"), "markdown content converted client-side");
|
||||
assert.notEqual(result.data.content, SERVER_TEXT);
|
||||
});
|
||||
@@ -0,0 +1,181 @@
|
||||
// #502 caller-WIRING: end-to-end through a live Hocuspocus collab stack (same
|
||||
// harness style as markdown-patch-insert), driving the REAL client methods and
|
||||
// reading the persisted document back, so the test proves each write tool passes
|
||||
// the RIGHT importer options — not just that the shared wrapper can:
|
||||
// - updatePageMarkdown (client.updatePage) -> extensions OFF (`$…$` literal, www not linked)
|
||||
// - import_page_markdown (client.importPageMarkdown) -> DEFAULTS (`$x^2$` -> math node, #328)
|
||||
// Mutating either caller's option flips the matching assertion (see the coder's
|
||||
// mutation note), so this file guards the wiring, not only the wrapper.
|
||||
import { test, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { WebSocketServer } from "ws";
|
||||
import { Hocuspocus } from "@hocuspocus/server";
|
||||
import { DocmostClient } from "../../build/client.js";
|
||||
import { buildYDoc } from "../../build/lib/collaboration.js";
|
||||
import { serializeDocmostMarkdown } from "@docmost/prosemirror-markdown";
|
||||
|
||||
const PAGE = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
function findAll(node, type, acc = []) {
|
||||
if (!node || typeof node !== "object") return acc;
|
||||
if (node.type === type) acc.push(node);
|
||||
if (Array.isArray(node.content)) for (const c of node.content) findAll(c, type, acc);
|
||||
return acc;
|
||||
}
|
||||
function allText(node, acc = []) {
|
||||
if (!node || typeof node !== "object") return acc.join("");
|
||||
if (node.type === "text" && typeof node.text === "string") acc.push(node.text);
|
||||
if (Array.isArray(node.content)) for (const c of node.content) allText(c, acc);
|
||||
return acc.join("");
|
||||
}
|
||||
function hasLink(node) {
|
||||
return findAll(node, "text").some((t) => t.marks?.some((m) => m.type === "link"));
|
||||
}
|
||||
|
||||
function fragmentToJson(frag) {
|
||||
const decodeNode = (el) => {
|
||||
if (el.constructor.name === "YXmlText") {
|
||||
const delta = el.toDelta();
|
||||
return delta.map((d) => {
|
||||
const node = { type: "text", text: d.insert };
|
||||
if (d.attributes && Object.keys(d.attributes).length) {
|
||||
node.marks = Object.entries(d.attributes).map(([type, attrs]) =>
|
||||
attrs && typeof attrs === "object" && Object.keys(attrs).length
|
||||
? { type, attrs }
|
||||
: { type },
|
||||
);
|
||||
}
|
||||
return node;
|
||||
});
|
||||
}
|
||||
const node = { type: el.nodeName };
|
||||
const attrs = el.getAttributes();
|
||||
if (attrs && Object.keys(attrs).length) node.attrs = attrs;
|
||||
const children = [];
|
||||
for (const child of el.toArray()) {
|
||||
const decoded = decodeNode(child);
|
||||
if (Array.isArray(decoded)) children.push(...decoded);
|
||||
else children.push(decoded);
|
||||
}
|
||||
if (children.length) node.content = children;
|
||||
return node;
|
||||
};
|
||||
const content = [];
|
||||
for (const child of frag.toArray()) content.push(decodeNode(child));
|
||||
return { type: "doc", content };
|
||||
}
|
||||
|
||||
const openStacks = [];
|
||||
after(async () => {
|
||||
await Promise.all(
|
||||
openStacks.map(
|
||||
({ server, hocuspocus }) =>
|
||||
new Promise((resolve) => {
|
||||
server.close(() => {
|
||||
Promise.resolve(hocuspocus.destroy?.()).finally(resolve);
|
||||
});
|
||||
}),
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
async function spawnCollabStack(seedDoc) {
|
||||
const state = { lastDoc: null };
|
||||
const hocuspocus = new Hocuspocus({
|
||||
quiet: true,
|
||||
async onLoadDocument() {
|
||||
return buildYDoc(seedDoc);
|
||||
},
|
||||
async onChange(data) {
|
||||
try {
|
||||
state.lastDoc = fragmentToJson(data.document.getXmlFragment("default"));
|
||||
} catch {
|
||||
/* ignore teardown-race decode errors */
|
||||
}
|
||||
},
|
||||
});
|
||||
const wss = new WebSocketServer({ noServer: true });
|
||||
const server = http.createServer((req, res) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => {
|
||||
if (req.url === "/api/auth/login") {
|
||||
res.writeHead(200, {
|
||||
"Content-Type": "application/json",
|
||||
"Set-Cookie": "authToken=t; Path=/; HttpOnly",
|
||||
});
|
||||
res.end(JSON.stringify({ success: true }));
|
||||
return;
|
||||
}
|
||||
if (req.url === "/api/auth/collab-token") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ data: { token: "collab-jwt" } }));
|
||||
return;
|
||||
}
|
||||
res.writeHead(404, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ message: "not found" }));
|
||||
});
|
||||
});
|
||||
server.on("upgrade", (request, socket, head) => {
|
||||
if (!request.url || !request.url.startsWith("/collab")) {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
wss.handleUpgrade(request, socket, head, (ws) => {
|
||||
hocuspocus.handleConnection(ws, request);
|
||||
});
|
||||
});
|
||||
const baseURL = await new Promise((resolve) => {
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
resolve(`http://127.0.0.1:${server.address().port}/api`);
|
||||
});
|
||||
});
|
||||
openStacks.push({ server, hocuspocus });
|
||||
return { state, baseURL };
|
||||
}
|
||||
|
||||
function seed() {
|
||||
return {
|
||||
type: "doc",
|
||||
content: [
|
||||
{ type: "paragraph", attrs: { id: "p-id" }, content: [{ type: "text", text: "seed" }] },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
test("updatePageMarkdown wiring: extensions OFF — `$…$` literal, www not linked, https links", async () => {
|
||||
const { state, baseURL } = await spawnCollabStack(seed());
|
||||
const client = new DocmostClient(baseURL, "e@x.com", "pw");
|
||||
|
||||
await client.updatePage(PAGE, "cfg $x=1$ and www.host.com and https://ex.com");
|
||||
|
||||
assert.ok(state.lastDoc, "a document was persisted");
|
||||
assert.equal(findAll(state.lastDoc, "mathInline").length, 0, "no phantom math from an agent write");
|
||||
assert.ok(allText(state.lastDoc).includes("$x=1$"), "literal dollars preserved");
|
||||
assert.ok(allText(state.lastDoc).includes("www.host.com"), "bare domain preserved as text");
|
||||
// The explicit https URL still links (only the schemeless autolink is off).
|
||||
const links = findAll(state.lastDoc, "text").filter((t) =>
|
||||
t.marks?.some((m) => m.type === "link"),
|
||||
);
|
||||
assert.ok(links.some((t) => t.text?.includes("ex.com")), "explicit https still links");
|
||||
});
|
||||
|
||||
test("import_page_markdown wiring: DEFAULTS — exported `$x^2$` re-imports AS a math node (#328)", async () => {
|
||||
const { state, baseURL } = await spawnCollabStack(seed());
|
||||
const client = new DocmostClient(baseURL, "e@x.com", "pw");
|
||||
|
||||
// A self-contained docmost markdown file whose body carries a math span (as the
|
||||
// exporter emits it). import_page_markdown must import it with math ON.
|
||||
const meta = { version: 1, pageId: PAGE, slugId: "s", title: "T", spaceId: "sp", parentPageId: null };
|
||||
const fullMd = serializeDocmostMarkdown(meta, "energy is $x^2$ here", []);
|
||||
|
||||
await client.importPageMarkdown(PAGE, fullMd);
|
||||
|
||||
assert.ok(state.lastDoc, "a document was persisted");
|
||||
assert.equal(
|
||||
findAll(state.lastDoc, "mathInline").length,
|
||||
1,
|
||||
"the lossless round-trip is intact: math survives import_page_markdown",
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
// #502: the two layered markdown extensions (`$…$` math, schemeless fuzzy
|
||||
// autolink) are decided PER CALLER of the shared write importer, not hardcoded in
|
||||
// the wrapper — because the callers need OPPOSITE behavior:
|
||||
// - AGENT-authored plain markdown (updatePageMarkdown, patch_node/insert_node)
|
||||
// -> extensions OFF: a `$…$` config span stays literal, a bare `www.host` is
|
||||
// not autolinked, an explicit `https://…` still links.
|
||||
// - FULL-FILE round-trip import (import_page_markdown, #328 lossless) ->
|
||||
// DEFAULTS (extensions ON): an exported math node's `$x^2$` re-imports AS a
|
||||
// math node, so the export→import pair is NOT broken.
|
||||
// This unit file pins the importer contract at each of those semantics. The
|
||||
// caller-WIRING (that each client method passes the right options) is pinned by
|
||||
// the collab-backed test in mock/write-path-extensions-wiring.test.mjs.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { markdownToProseMirrorCanonical } from "../../build/lib/collaboration.js";
|
||||
import { importMarkdownFragment } from "../../build/lib/markdown-fragment.js";
|
||||
import {
|
||||
markdownToProseMirror,
|
||||
convertProseMirrorToMarkdown,
|
||||
} from "@docmost/prosemirror-markdown";
|
||||
|
||||
const OFF = { parseMath: false, fuzzyLinkify: false };
|
||||
|
||||
function findAll(node, type, acc = []) {
|
||||
if (!node || typeof node !== "object") return acc;
|
||||
if (node.type === type) acc.push(node);
|
||||
if (Array.isArray(node.content)) for (const c of node.content) findAll(c, type, acc);
|
||||
return acc;
|
||||
}
|
||||
function allText(node, acc = []) {
|
||||
if (!node || typeof node !== "object") return acc.join("");
|
||||
if (node.type === "text" && typeof node.text === "string") acc.push(node.text);
|
||||
if (Array.isArray(node.content)) for (const c of node.content) allText(c, acc);
|
||||
return acc.join("");
|
||||
}
|
||||
function hasLink(node) {
|
||||
return findAll(node, "text").some((t) => t.marks?.some((m) => m.type === "link"));
|
||||
}
|
||||
|
||||
// --- AGENT-write semantics (updatePageMarkdown): extensions OFF -----------------
|
||||
|
||||
test("agent-write importer (OFF): `$…$` config stays literal, no math node", async () => {
|
||||
const doc = await markdownToProseMirrorCanonical("export A=$FOO and B=$BAR done", OFF);
|
||||
assert.equal(findAll(doc, "mathInline").length, 0);
|
||||
assert.equal(allText(doc), "export A=$FOO and B=$BAR done");
|
||||
});
|
||||
|
||||
test("agent-write importer (OFF): schemeless www NOT linked; explicit https STILL linked", async () => {
|
||||
const bare = await markdownToProseMirrorCanonical("see www.example.com here", OFF);
|
||||
assert.equal(hasLink(bare), false);
|
||||
assert.equal(allText(bare), "see www.example.com here");
|
||||
|
||||
const explicit = await markdownToProseMirrorCanonical("see https://example.com here", OFF);
|
||||
assert.equal(hasLink(explicit), true);
|
||||
});
|
||||
|
||||
test("agent-write importer (OFF): heading + list structure preserved", async () => {
|
||||
const doc = await markdownToProseMirrorCanonical("## Heading\n\n- one\n- two", OFF);
|
||||
assert.equal(findAll(doc, "heading").length, 1);
|
||||
assert.equal(findAll(doc, "bulletList").length, 1);
|
||||
});
|
||||
|
||||
test("fragment importer (patch_node/insert_node) is OFF: `$x=1$` literal, https links", async () => {
|
||||
const { blocks } = await importMarkdownFragment("cfg $x=1$ and https://ex.com");
|
||||
const doc = { type: "doc", content: blocks };
|
||||
assert.equal(findAll(doc, "mathInline").length, 0);
|
||||
assert.equal(hasLink(doc), true);
|
||||
assert.ok(allText(doc).includes("$x=1$"));
|
||||
});
|
||||
|
||||
// --- FULL-FILE import semantics (import_page_markdown): DEFAULTS (math ON) ------
|
||||
|
||||
test("import_page_markdown importer (DEFAULTS): `$x^2$` DOES create a math node", async () => {
|
||||
// markdownToProseMirrorCanonical WITHOUT options == what import_page_markdown
|
||||
// passes. Math must survive (this is the REAL importer, not the package default).
|
||||
const doc = await markdownToProseMirrorCanonical("$x^2$");
|
||||
assert.equal(findAll(doc, "mathInline").length, 1);
|
||||
assert.equal(findAll(doc, "mathInline")[0].attrs.text, "x^2");
|
||||
});
|
||||
|
||||
test("import_page_markdown (DEFAULTS): #328 lossless export->import keeps math (round-trip)", async () => {
|
||||
// The exporter serializes a math node as readable `$x^2$`; re-importing through
|
||||
// the REAL import_page_markdown importer (markdownToProseMirrorCanonical, no
|
||||
// options) must yield a math node again and be byte-stable. Under the BUGGY code
|
||||
// (canonical hardcoded parseMath:false) doc2 would be literal text -> this test
|
||||
// would REDDEN.
|
||||
const source = {
|
||||
type: "doc",
|
||||
content: [{ type: "paragraph", content: [{ type: "mathInline", attrs: { text: "x^2" } }] }],
|
||||
};
|
||||
const md1 = convertProseMirrorToMarkdown(source);
|
||||
const doc2 = await markdownToProseMirrorCanonical(md1); // real import path, defaults
|
||||
assert.equal(findAll(doc2, "mathInline").length, 1, "math survives the round-trip import");
|
||||
const md2 = convertProseMirrorToMarkdown(doc2);
|
||||
assert.equal(md2, md1, "export is byte-stable across the round-trip");
|
||||
});
|
||||
|
||||
test("import_page_markdown (DEFAULTS): a schemeless www IS autolinked", async () => {
|
||||
const doc = await markdownToProseMirrorCanonical("see www.example.com here");
|
||||
assert.equal(hasLink(doc), true);
|
||||
});
|
||||
|
||||
// --- Package default importer (editor/file-import/git-sync) is UNCHANGED --------
|
||||
|
||||
test("PACKAGE default importer keeps math ON (editor/file/git-sync path)", async () => {
|
||||
const doc = await markdownToProseMirror("$x^2$");
|
||||
assert.equal(findAll(doc, "mathInline").length, 1);
|
||||
});
|
||||
@@ -25,6 +25,7 @@ export {
|
||||
markdownToProseMirror,
|
||||
markdownToProseMirrorSync,
|
||||
} from "./markdown-to-prosemirror.js";
|
||||
export type { MarkdownImportOptions } from "./markdown-to-prosemirror.js";
|
||||
|
||||
// Foreign-markdown normalizer (#493): the input-liberal pre-pass that rewrites
|
||||
// GFM `[^id]` reference footnotes to canonical inline `^[body]`. Two variants:
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* natively through the collab gateway, so no websocket/Yjs write-path lives
|
||||
* here.
|
||||
*/
|
||||
import { Marked } from "marked";
|
||||
import { Marked, Tokenizer } from "marked";
|
||||
import { parseHtmlDocument, generateJsonWith } from "./dom-parser.js";
|
||||
import type { TokenizerExtension, RendererExtension } from "marked";
|
||||
import { docmostExtensions } from "./docmost-schema.js";
|
||||
@@ -230,19 +230,88 @@ function escapeFootnoteAttr(value: string): string {
|
||||
return String(value).replace(/&/g, "&").replace(/"/g, """);
|
||||
}
|
||||
|
||||
// Dedicated marked instance: default (GFM) options plus the `==` highlight
|
||||
// inline extension, the `$…$` / `$$…$$` math extensions (#293 canon #6), and the
|
||||
// `^[…]` inline-footnote extension (#293 canon #2). Constructed once at module
|
||||
// load so the extensions are registered exactly once and never mutate the global
|
||||
// `marked` singleton.
|
||||
const markedInstance = new Marked().use({
|
||||
extensions: [
|
||||
/**
|
||||
* Options controlling which of the two *layered* markdown extensions the
|
||||
* canonical importer applies. Both default to `true`, so the human editor,
|
||||
* file-import and git-sync paths keep their existing behavior byte-for-byte;
|
||||
* ONLY the MCP markdown-write path opts OUT (see #502).
|
||||
*
|
||||
* The extensions are optional because they are the SOURCE of two silent
|
||||
* corruptions when an AGENT writes plain prose/config as markdown:
|
||||
* - `parseMath`: a `$…$` span becomes a `mathInline` node. An agent writing a
|
||||
* config like `export A=$FOO and B=$BAR` gets `$FOO and B=$` silently turned
|
||||
* into a formula. With `parseMath:false` the `$` stays literal text (real
|
||||
* formulas go through `update_page_json` with `mathInline`/`mathBlock`).
|
||||
* - `fuzzyLinkify`: marked's GFM autolinker turns a SCHEMELESS `www.foo.com`
|
||||
* (and email) into a link. With `fuzzyLinkify:false` a schemeless domain
|
||||
* stays literal text; an EXPLICIT `https://…` STILL becomes a link (only the
|
||||
* fuzzy, schemeless autolink is suppressed).
|
||||
*/
|
||||
export interface MarkdownImportOptions {
|
||||
/** Apply the `$…$` / `$$…$$` math extensions (default true). */
|
||||
parseMath?: boolean;
|
||||
/** Apply marked's GFM schemeless (fuzzy) autolinker (default true). */
|
||||
fuzzyLinkify?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* `fuzzyLinkify:false` override of marked's built-in GFM `url` inline tokenizer.
|
||||
*
|
||||
* The stock tokenizer autolinks THREE shapes: a schemeless `www.host` domain, a
|
||||
* bare email, and an EXPLICIT `scheme://…` URL. #502 wants only the last kept —
|
||||
* a schemeless domain/email an agent typed as prose must stay literal text, but
|
||||
* a deliberate `https://…` still links. We delegate to the original tokenizer
|
||||
* and, when it matched, DROP the token (returning `undefined`, so the run stays
|
||||
* literal text) unless the matched RAW text carries an explicit `scheme:` prefix.
|
||||
* `www.`/email matches have no scheme in their raw text, so they are dropped;
|
||||
* `https://…`/`ftp://…` keep their link.
|
||||
*/
|
||||
const SCHEME_PREFIX_RE = /^[a-zA-Z][a-zA-Z0-9+.-]*:/;
|
||||
const noFuzzyUrlTokenizer = {
|
||||
url(this: any, src: string) {
|
||||
const token = Tokenizer.prototype.url.call(this, src) as any;
|
||||
if (!token) return token;
|
||||
// Keep only explicit-scheme URLs; schemeless `www.`/email matches -> literal.
|
||||
return SCHEME_PREFIX_RE.test(token.raw) ? token : undefined;
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Build a dedicated `marked` instance for a given extension combination: default
|
||||
* (GFM) options plus the `==` highlight and `^[…]` footnote inline extensions
|
||||
* ALWAYS, the `$…$`/`$$…$$` math extensions only when `parseMath`, and the
|
||||
* schemeless-autolink suppressor only when `!fuzzyLinkify`. Built on a private
|
||||
* `Marked` instance so nothing leaks into the global `marked` singleton.
|
||||
*/
|
||||
function buildMarkedInstance(parseMath: boolean, fuzzyLinkify: boolean): Marked {
|
||||
const extensions: (TokenizerExtension & RendererExtension)[] = [
|
||||
highlightMarkExtension,
|
||||
mathInlineExtension,
|
||||
mathBlockExtension,
|
||||
footnoteInlineExtension,
|
||||
],
|
||||
});
|
||||
];
|
||||
if (parseMath) {
|
||||
extensions.push(mathInlineExtension, mathBlockExtension);
|
||||
}
|
||||
const instance = new Marked().use({ extensions });
|
||||
if (!fuzzyLinkify) {
|
||||
instance.use({ tokenizer: noFuzzyUrlTokenizer as any });
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
// Memoize one instance per (parseMath, fuzzyLinkify) combination so the
|
||||
// extensions are registered exactly once per combo (never on the global
|
||||
// singleton). The default `(true, true)` instance preserves the pre-#502
|
||||
// behavior exactly for the editor/file-import/git-sync paths.
|
||||
const markedInstanceCache = new Map<string, Marked>();
|
||||
function getMarkedInstance(parseMath: boolean, fuzzyLinkify: boolean): Marked {
|
||||
const key = `${parseMath}:${fuzzyLinkify}`;
|
||||
let instance = markedInstanceCache.get(key);
|
||||
if (!instance) {
|
||||
instance = buildMarkedInstance(parseMath, fuzzyLinkify);
|
||||
markedInstanceCache.set(key, instance);
|
||||
}
|
||||
return instance;
|
||||
}
|
||||
|
||||
// NOTE: this module no longer installs a module-level `global.window`/`document`
|
||||
// jsdom shim. The HTML->DOM passes below (bridgeTaskLists / applyCommentDirectives
|
||||
@@ -301,7 +370,7 @@ const CODE_FENCE_RE = /^(\s*)(`{3,}|~{3,})/;
|
||||
// preprocess is sync. Keeping it sync lets a sync converter entry
|
||||
// (`markdownToProseMirrorSync`, used by the client's chat renderer which must
|
||||
// stay synchronous) share this exact logic with the async entry.
|
||||
function preprocessCallouts(markdown: string): string {
|
||||
function preprocessCallouts(markdown: string, markedInstance: Marked): string {
|
||||
// Defensive cap: skip preprocessing for pathologically large inputs.
|
||||
if (markdown.length > MAX_CALLOUT_PREPROCESS_BYTES) {
|
||||
return markdown;
|
||||
@@ -955,7 +1024,7 @@ function applyCommentDirectives(html: string): string {
|
||||
// sups stay inert) rather than hang.
|
||||
const MAX_FOOTNOTE_ROUNDS = 10000;
|
||||
|
||||
function assembleFootnotes(html: string): string {
|
||||
function assembleFootnotes(html: string, markedInstance: Marked): string {
|
||||
// Cheap early-out: nothing carries a footnote body -> nothing to assemble.
|
||||
if (!html.includes("data-fn-text")) return html;
|
||||
const document = parseHtmlDocument(html);
|
||||
@@ -1081,8 +1150,18 @@ function stripEmptyParagraphs(node: any): any {
|
||||
* for every existing Node consumer). A sync entry is REQUIRED by the client's
|
||||
* chat renderer, which runs inside a React render/useMemo and cannot await.
|
||||
*/
|
||||
export function markdownToProseMirrorSync(markdownContent: string): any {
|
||||
const withCallouts = preprocessCallouts(markdownContent);
|
||||
export function markdownToProseMirrorSync(
|
||||
markdownContent: string,
|
||||
options?: MarkdownImportOptions,
|
||||
): any {
|
||||
// Select the marked instance for this call's extension combination. Defaults
|
||||
// (math + fuzzy autolink ON) preserve the editor/file-import/git-sync paths;
|
||||
// the MCP markdown-write path passes both false (#502).
|
||||
const markedInstance = getMarkedInstance(
|
||||
options?.parseMath ?? true,
|
||||
options?.fuzzyLinkify ?? true,
|
||||
);
|
||||
const withCallouts = preprocessCallouts(markdownContent, markedInstance);
|
||||
const html = markedInstance.parse(withCallouts) as string;
|
||||
// Materialize comment directives (#293 #9 attached textAlign; #5 standalone
|
||||
// subpages/pageBreak) while the comment nodes still exist, before generateJSON
|
||||
@@ -1091,7 +1170,7 @@ export function markdownToProseMirrorSync(markdownContent: string): any {
|
||||
// #293 canon #2: assemble the doc-level footnote list from the `<sup
|
||||
// data-fn-text>` markers (from `^[…]` or the raw-HTML column form) before
|
||||
// generateJSON, so references + definitions materialize into the schema model.
|
||||
const withFootnotes = assembleFootnotes(withAttrs);
|
||||
const withFootnotes = assembleFootnotes(withAttrs, markedInstance);
|
||||
const bridged = bridgeTaskLists(withFootnotes);
|
||||
const doc = generateJsonWith(bridged, docmostExtensions);
|
||||
return stripEmptyParagraphs(doc);
|
||||
@@ -1104,6 +1183,7 @@ export function markdownToProseMirrorSync(markdownContent: string): any {
|
||||
*/
|
||||
export async function markdownToProseMirror(
|
||||
markdownContent: string,
|
||||
options?: MarkdownImportOptions,
|
||||
): Promise<any> {
|
||||
return markdownToProseMirrorSync(markdownContent);
|
||||
return markdownToProseMirrorSync(markdownContent, options);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
// Import DIRECTLY from src (like math.test.ts) so we exercise the real
|
||||
// converter and its module-load jsdom setup.
|
||||
import { convertProseMirrorToMarkdown } from '../src/lib/markdown-converter.js';
|
||||
import { markdownToProseMirror } from '../src/lib/markdown-to-prosemirror.js';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// #502: the canonical importer is parameterized with `{ parseMath, fuzzyLinkify }`
|
||||
// (both DEFAULT true). The MCP markdown-WRITE path passes both false so an agent's
|
||||
// plain prose/config is imported LITERALLY:
|
||||
// - parseMath:false -> a `$…$` span stays literal text (no phantom mathInline)
|
||||
// - fuzzyLinkify:false -> a SCHEMELESS `www.host`/email stays literal text; an
|
||||
// EXPLICIT `https://…` STILL links (only the fuzzy autolink is suppressed).
|
||||
// The DEFAULTS (editor/file-import/git-sync) keep math + fuzzy autolink ON, so
|
||||
// this file also pins that the defaults are UNCHANGED.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const OFF = { parseMath: false, fuzzyLinkify: false } as const;
|
||||
|
||||
// Collect the first paragraph's inline children (the common assertion target).
|
||||
function firstParaInline(doc: any): any[] {
|
||||
const p = doc.content?.find((n: any) => n.type === 'paragraph');
|
||||
return p?.content ?? [];
|
||||
}
|
||||
|
||||
function findAll(node: any, type: string, acc: any[] = []): any[] {
|
||||
if (!node || typeof node !== 'object') return acc;
|
||||
if (node.type === type) acc.push(node);
|
||||
if (Array.isArray(node.content)) {
|
||||
for (const c of node.content) findAll(c, type, acc);
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
|
||||
// Flatten every text node's text (ignoring marks/structure) into one string.
|
||||
function allText(node: any, acc: string[] = []): string {
|
||||
if (!node || typeof node !== 'object') return acc.join('');
|
||||
if (node.type === 'text' && typeof node.text === 'string') acc.push(node.text);
|
||||
if (Array.isArray(node.content)) {
|
||||
for (const c of node.content) allText(c, acc);
|
||||
}
|
||||
return acc.join('');
|
||||
}
|
||||
|
||||
describe('#502 importer options — extensions OFF (MCP write path)', () => {
|
||||
it('a `$…$` config span stays literal text (no mathInline node)', async () => {
|
||||
const md = 'export A=$FOO and B=$BAR done';
|
||||
const doc = await markdownToProseMirror(md, OFF);
|
||||
expect(findAll(doc, 'mathInline')).toHaveLength(0);
|
||||
expect(allText(doc)).toBe('export A=$FOO and B=$BAR done');
|
||||
});
|
||||
|
||||
it('a real-looking `$x=1$` span stays literal text', async () => {
|
||||
const doc = await markdownToProseMirror('$x=1$', OFF);
|
||||
expect(findAll(doc, 'mathInline')).toHaveLength(0);
|
||||
expect(allText(doc)).toBe('$x=1$');
|
||||
});
|
||||
|
||||
it('the reported `($ticket_lifetime=2592000)` config stays literal', async () => {
|
||||
const doc = await markdownToProseMirror('($ticket_lifetime=2592000)', OFF);
|
||||
expect(findAll(doc, 'mathInline')).toHaveLength(0);
|
||||
expect(allText(doc)).toBe('($ticket_lifetime=2592000)');
|
||||
});
|
||||
|
||||
it('a `$$…$$` block stays literal (no mathBlock node)', async () => {
|
||||
const doc = await markdownToProseMirror('$$\nx^2\n$$', OFF);
|
||||
expect(findAll(doc, 'mathBlock')).toHaveLength(0);
|
||||
expect(allText(doc)).toContain('x^2');
|
||||
});
|
||||
|
||||
it('a SCHEMELESS `www.example.com` is NOT autolinked', async () => {
|
||||
const doc = await markdownToProseMirror('see www.example.com here', OFF);
|
||||
const inline = firstParaInline(doc);
|
||||
expect(inline.some((n: any) => n.marks?.some((m: any) => m.type === 'link'))).toBe(false);
|
||||
expect(allText(doc)).toBe('see www.example.com here');
|
||||
});
|
||||
|
||||
it('a bare dotted domain `gitea.vvzvlad.xyz` stays literal text', async () => {
|
||||
const doc = await markdownToProseMirror('see gitea.vvzvlad.xyz here', OFF);
|
||||
expect(findAll(doc, 'text').every((t: any) => !t.marks?.some((m: any) => m.type === 'link'))).toBe(true);
|
||||
});
|
||||
|
||||
it('an EXPLICIT `https://…` STILL becomes a link', async () => {
|
||||
const doc = await markdownToProseMirror('see https://example.com here', OFF);
|
||||
const linked = firstParaInline(doc).find((n: any) =>
|
||||
n.marks?.some((m: any) => m.type === 'link'),
|
||||
);
|
||||
expect(linked?.text).toBe('https://example.com');
|
||||
const link = linked.marks.find((m: any) => m.type === 'link');
|
||||
expect(link.attrs.href).toBe('https://example.com');
|
||||
});
|
||||
|
||||
it('`foo_bar_baz` is not italicized (CommonMark, unaffected by options)', async () => {
|
||||
const doc = await markdownToProseMirror('foo_bar_baz', OFF);
|
||||
expect(findAll(doc, 'text').some((t: any) => t.marks?.some((m: any) => m.type === 'italic' || m.type === 'em'))).toBe(false);
|
||||
expect(allText(doc)).toBe('foo_bar_baz');
|
||||
});
|
||||
|
||||
it('block STRUCTURE (headings, lists, code fence) is preserved with extensions off', async () => {
|
||||
const md = '## Heading\n\n- one\n- two\n\n```\ncode $x$ here\n```';
|
||||
const doc = await markdownToProseMirror(md, OFF);
|
||||
expect(findAll(doc, 'heading')).toHaveLength(1);
|
||||
expect(findAll(doc, 'bulletList')).toHaveLength(1);
|
||||
expect(findAll(doc, 'codeBlock')).toHaveLength(1);
|
||||
// The `$x$` inside the code fence never becomes math regardless.
|
||||
expect(findAll(doc, 'mathInline')).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('#502 importer options — DEFAULTS unchanged (editor/file/git-sync)', () => {
|
||||
it('DEFAULT: `$x^2$` DOES create a mathInline node (file-import unaffected)', async () => {
|
||||
const doc = await markdownToProseMirror('$x^2$');
|
||||
expect(findAll(doc, 'mathInline')).toHaveLength(1);
|
||||
expect(findAll(doc, 'mathInline')[0].attrs.text).toBe('x^2');
|
||||
});
|
||||
|
||||
it('DEFAULT: a schemeless `www.example.com` IS autolinked', async () => {
|
||||
const doc = await markdownToProseMirror('see www.example.com here');
|
||||
const linked = firstParaInline(doc).find((n: any) =>
|
||||
n.marks?.some((m: any) => m.type === 'link'),
|
||||
);
|
||||
expect(linked?.text).toBe('www.example.com');
|
||||
});
|
||||
|
||||
it('DEFAULT: explicitly passing {parseMath:true, fuzzyLinkify:true} equals no-options', async () => {
|
||||
const a = await markdownToProseMirror('$x^2$ and www.foo.com');
|
||||
const b = await markdownToProseMirror('$x^2$ and www.foo.com', {
|
||||
parseMath: true,
|
||||
fuzzyLinkify: true,
|
||||
});
|
||||
expect(JSON.stringify(b)).toBe(JSON.stringify(a));
|
||||
});
|
||||
|
||||
it('round-trip export->import at the PACKAGE-DEFAULT layer keeps math (file-import / #328)', async () => {
|
||||
// The package DEFAULT importer is what the server file-import path uses. A
|
||||
// page holding real math is exported, then re-imported with DEFAULTS (math
|
||||
// ON): the mathInline survives. (The tool-level import_page_markdown round-
|
||||
// trip, which goes through mcp's markdownToProseMirrorCanonical, is pinned
|
||||
// authoritatively in @docmost/mcp's mcp-write-extensions-off test.)
|
||||
const source = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{ type: 'paragraph', content: [{ type: 'mathInline', attrs: { text: 'x^2' } }] },
|
||||
],
|
||||
};
|
||||
const md1 = convertProseMirrorToMarkdown(source);
|
||||
const doc2 = await markdownToProseMirror(md1); // DEFAULTS -> math on
|
||||
expect(findAll(doc2, 'mathInline')).toHaveLength(1);
|
||||
const md2 = convertProseMirrorToMarkdown(doc2);
|
||||
expect(md2).toBe(md1); // byte-stable
|
||||
});
|
||||
});
|
||||
|
||||
describe('#502 mutation guard', () => {
|
||||
// If a future change silently flipped the MCP write path back to parseMath:true,
|
||||
// the OFF assertion below would go RED — this pins the discriminating behavior.
|
||||
it('with parseMath:true the same span DOES become math (proves the flag drives it)', async () => {
|
||||
const on = await markdownToProseMirror('$x=1$', { parseMath: true, fuzzyLinkify: false });
|
||||
expect(findAll(on, 'mathInline')).toHaveLength(1);
|
||||
const off = await markdownToProseMirror('$x=1$', OFF);
|
||||
expect(findAll(off, 'mathInline')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('with fuzzyLinkify:true the same www domain DOES link (proves the flag drives it)', async () => {
|
||||
const on = await markdownToProseMirror('www.example.com', { parseMath: false, fuzzyLinkify: true });
|
||||
expect(findAll(on, 'text').some((t: any) => t.marks?.some((m: any) => m.type === 'link'))).toBe(true);
|
||||
const off = await markdownToProseMirror('www.example.com', OFF);
|
||||
expect(findAll(off, 'text').some((t: any) => t.marks?.some((m: any) => m.type === 'link'))).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user