Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 agent_coder
|
cc2373f101 | ||
|
agent_coder
|
747b125912 | ||
|
agent_coder
|
01637fa86b | ||
|
agent_coder
|
1413b26287 | ||
|
agent_coder
|
f0a64829bf | ||
|
agent_coder
|
4636eaa7d0 | ||
|
agent_coder
|
e4f0d0f0ed | ||
|
agent_coder
|
a4ea22a7f5 | ||
|
agent_coder
|
87d5d7ac26 |
@@ -27,7 +27,6 @@ vi.mock("@/features/ai-chat/utils/markdown.ts", async () => {
|
||||
|
||||
import MessageItem from "./message-item";
|
||||
import { messageSignature } from "@/features/ai-chat/utils/message-signature.ts";
|
||||
import { splitPlainChunks } from "./streaming-plain-text";
|
||||
|
||||
// matchMedia (read by MantineProvider) is stubbed globally in vitest.setup.ts.
|
||||
|
||||
@@ -115,89 +114,3 @@ describe("MessageItem markdown memoization", () => {
|
||||
expect(queryByText("streamed answer")).not.toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// PERF SMOKE (#492): the whole point of the incremental streaming render is that
|
||||
// the ANSWER path costs O(number of markdown blocks), NOT O(number of throttled
|
||||
// ~20Hz ticks). Pre-#492 the finalized MarkdownPart re-parsed the WHOLE growing
|
||||
// answer on every delta — a synthetic ~100 KB stream measured 394 renderChatMarkdown
|
||||
// calls (one per tick). With the incremental render each STABILIZED block is parsed
|
||||
// exactly once (memoized in MarkdownChunk) and the live tail is cheap plain text, so
|
||||
// the call count collapses to ~= the block count regardless of tick granularity.
|
||||
describe("MessageItem streaming answer render is O(blocks), not O(ticks)", () => {
|
||||
// ~100 KB answer. Each section is a heading + a paragraph — TWO blank-line
|
||||
// delimited markdown blocks — so the safe-cut block count is ~2× the section
|
||||
// count. The perf claim is about the BLOCK count (the memoization granularity),
|
||||
// measured directly with splitPlainChunks below, not the section count.
|
||||
const buildAnswer = () => {
|
||||
const SECTIONS = 100;
|
||||
const paragraphs: string[] = [];
|
||||
for (let i = 0; i < SECTIONS; i++) {
|
||||
paragraphs.push(`## Section ${i}\n\n` + "lorem ipsum dolor ".repeat(55));
|
||||
}
|
||||
const full = paragraphs.join("\n\n");
|
||||
// The number of memoized markdown blocks the incremental render splits into
|
||||
// (all but the live tail are parsed once each).
|
||||
return { full, blocks: splitPlainChunks(full).length };
|
||||
};
|
||||
|
||||
const streamMsg = (text: string, state: "streaming" | "done"): UIMessage =>
|
||||
({
|
||||
id: "m1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text, state }],
|
||||
}) as UIMessage;
|
||||
|
||||
it("parses each block ~once over a 100KB stream (≈blocks, ≪ ticks)", () => {
|
||||
renderChatMarkdownSpy.mockClear();
|
||||
const { full, blocks } = buildAnswer();
|
||||
const CHUNK = 128; // a realistic ~20Hz throttled delta size
|
||||
const ticks = Math.ceil(full.length / CHUNK);
|
||||
|
||||
let msg = streamMsg(full.slice(0, CHUNK), "streaming");
|
||||
const { rerender } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg}
|
||||
signature={messageSignature(msg)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
for (let end = 2 * CHUNK; end < full.length; end += CHUNK) {
|
||||
msg = streamMsg(full.slice(0, end), "streaming");
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg}
|
||||
signature={messageSignature(msg)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
}
|
||||
// Finalize: the streaming→done flip renders the whole answer through ONE
|
||||
// canonical pass (visual parity), so the finished DOM matches the pre-#492
|
||||
// output. This is the single extra parse on top of the per-block ones.
|
||||
const done = streamMsg(full, "done");
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem message={done} signature={messageSignature(done)} />
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
const calls = renderChatMarkdownSpy.mock.calls.length;
|
||||
// Sanity: the stream really had far more ticks than blocks (else the test is
|
||||
// vacuous — the point is that calls scale with blocks, not ticks).
|
||||
expect(ticks).toBeGreaterThan(blocks * 3);
|
||||
// O(blocks): each stabilized block parsed once + the single final whole-text
|
||||
// parse. A small constant absorbs the finalize render and the live-tail block;
|
||||
// the load-bearing claim is the bound below.
|
||||
expect(calls).toBeLessThanOrEqual(blocks + 2);
|
||||
// ≪ ticks — and, non-vacuously, the blocks WERE parsed (not skipped entirely).
|
||||
expect(calls).toBeLessThan(ticks / 3);
|
||||
expect(calls).toBeGreaterThan(blocks / 2);
|
||||
// MUTATION-VERIFY (documented, not run here): dropping the `memo()` wrapper on
|
||||
// MarkdownChunk (so every stable block re-parses each tick) drives `calls`
|
||||
// toward `ticks` (~394), reddening both upper-bound assertions above.
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { render } from "@testing-library/react";
|
||||
import { MantineProvider } from "@mantine/core";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
|
||||
// Stub react-i18next (the component reads `useTranslation`). Mirrors the other
|
||||
// message-item specs.
|
||||
vi.mock("react-i18next", () => ({
|
||||
useTranslation: () => ({ t: (key: string) => key }),
|
||||
}));
|
||||
|
||||
import MessageItem from "./message-item";
|
||||
import { messageSignature } from "@/features/ai-chat/utils/message-signature.ts";
|
||||
// The REAL canonical renderer (NOT the spy the memo test installs): this file
|
||||
// exercises the actual markdown output so the visual-regression assertions below
|
||||
// compare against genuine HTML (incl. the schema's `<li><p>` wrappers).
|
||||
import { renderChatMarkdown } from "@/features/ai-chat/utils/markdown.ts";
|
||||
import classes from "./ai-chat.module.css";
|
||||
|
||||
const msg = (
|
||||
parts: UIMessage["parts"],
|
||||
extra?: Partial<UIMessage>,
|
||||
): UIMessage =>
|
||||
({ id: "m1", role: "assistant", parts, ...extra }) as UIMessage;
|
||||
|
||||
const renderRow = (message: UIMessage, turnStreaming = false) =>
|
||||
render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={message}
|
||||
signature={messageSignature(message)}
|
||||
turnStreaming={turnStreaming}
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
// A rich multi-block answer that exercises headings, a list (the `<li><p>` case
|
||||
// the scoped CSS tightens), inline emphasis, and multiple paragraphs.
|
||||
const ANSWER = [
|
||||
"# Заголовок",
|
||||
"",
|
||||
"Первый абзац с **жирным** и `кодом`.",
|
||||
"",
|
||||
"- пункт один",
|
||||
"- пункт два",
|
||||
"",
|
||||
"Второй абзац.",
|
||||
].join("\n");
|
||||
|
||||
describe("MessageItem final render — visual parity with the canonical pipeline", () => {
|
||||
it("a finalized text part renders exactly renderChatMarkdown(text)", () => {
|
||||
const { container } = renderRow(
|
||||
msg([{ type: "text", text: ANSWER, state: "done" }]),
|
||||
);
|
||||
const block = container.querySelector(`.${classes.markdown}`);
|
||||
expect(block).not.toBeNull();
|
||||
// Byte-for-byte the canonical output (the SAME whole-text pass the pre-#492
|
||||
// MarkdownPart produced), including `<li><p>…</p></li>` wrappers.
|
||||
expect(block!.innerHTML).toBe(renderChatMarkdown(ANSWER, {}));
|
||||
// The list wrapper is really present (guards against a vacuous empty render).
|
||||
expect(container.querySelectorAll("li p").length).toBe(2);
|
||||
});
|
||||
|
||||
it("the streaming incremental view CONVERGES to the canonical render on finish", () => {
|
||||
// Mount mid-stream (live tail) — the DOM here is the incremental view.
|
||||
const { container, rerender } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg([{ type: "text", text: ANSWER, state: "streaming" }])}
|
||||
signature={messageSignature(
|
||||
msg([{ type: "text", text: ANSWER, state: "streaming" }]),
|
||||
)}
|
||||
turnStreaming
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
// Finish the turn: state flips to done AND the turn is no longer streaming.
|
||||
const done = msg([{ type: "text", text: ANSWER, state: "done" }]);
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem message={done} signature={messageSignature(done)} />
|
||||
</MantineProvider>,
|
||||
);
|
||||
// After finish there is exactly ONE canonical markdown container whose HTML is
|
||||
// the whole-text render — identical to the non-streaming path above.
|
||||
const blocks = container.querySelectorAll(`.${classes.markdown}`);
|
||||
expect(blocks.length).toBe(1);
|
||||
expect(blocks[0].innerHTML).toBe(renderChatMarkdown(ANSWER, {}));
|
||||
});
|
||||
|
||||
it("neutralizeInternalLinks is honored on the finalized render", () => {
|
||||
const linkAnswer = "См. [страницу](/p/abc).";
|
||||
const { container } = render(
|
||||
<MantineProvider>
|
||||
<MessageItem
|
||||
message={msg([{ type: "text", text: linkAnswer, state: "done" }])}
|
||||
signature={messageSignature(
|
||||
msg([{ type: "text", text: linkAnswer, state: "done" }]),
|
||||
)}
|
||||
neutralizeInternalLinks
|
||||
/>
|
||||
</MantineProvider>,
|
||||
);
|
||||
const block = container.querySelector(`.${classes.markdown}`);
|
||||
expect(block!.innerHTML).toBe(
|
||||
renderChatMarkdown(linkAnswer, { neutralizeInternalLinks: true }),
|
||||
);
|
||||
// The internal link was made inert (no href) by the neutralization flag.
|
||||
const a = container.querySelector("a");
|
||||
expect(a?.hasAttribute("href")).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -4,7 +4,6 @@ import { useTranslation } from "react-i18next";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
import ToolCallCard from "@/features/ai-chat/components/tool-call-card.tsx";
|
||||
import ReasoningBlock from "@/features/ai-chat/components/reasoning-block.tsx";
|
||||
import { StreamingMarkdownText } from "@/features/ai-chat/components/streaming-markdown-text.tsx";
|
||||
import ChatErrorAlert from "@/features/ai-chat/components/chat-error-alert.tsx";
|
||||
import ChatStoppedNotice from "@/features/ai-chat/components/chat-stopped-notice.tsx";
|
||||
import { ToolUiPart, isToolPart } from "@/features/ai-chat/utils/tool-parts.tsx";
|
||||
@@ -87,39 +86,17 @@ interface MessageItemProps {
|
||||
* One assistant text part rendered as sanitized markdown. Memoized on its inputs
|
||||
* so a finalized text part is NOT re-parsed on every streamed delta: during a
|
||||
* turn only the actively-growing tail part changes its `text`, so every earlier
|
||||
* part hits the memo and skips the expensive canonical parse + DOMPurify pass.
|
||||
* Props are primitives, so React.memo's default shallow compare is exactly right
|
||||
* (the `text` string is compared by value).
|
||||
*
|
||||
* Streaming gate (#492) — mirrors ReasoningBlock:
|
||||
* - `streaming` (this is the live, actively-growing tail part of an in-flight
|
||||
* turn): render incrementally via StreamingMarkdownText — the stabilized blocks
|
||||
* go through the canonical pipeline (each parsed ONCE, memoized) and only the
|
||||
* live tail is cheap plain text. This makes the per-tick cost O(new blocks),
|
||||
* not the pre-#492 O(ticks) whole-answer re-parse on every ~20Hz delta.
|
||||
* - finalized (the common case, and the turn-end flip): render the WHOLE text
|
||||
* through ONE canonical pass — byte-identical to the pre-#492 output (visual
|
||||
* parity). The row re-renders on the streaming→done flip because
|
||||
* `messageSignature` tracks each part's `state` (and `turnStreaming` flips at
|
||||
* turn end), so the incremental view always converges to this single render.
|
||||
* part hits the memo and skips the expensive marked + DOMPurify pass. Props are
|
||||
* primitives, so React.memo's default shallow compare is exactly right (the
|
||||
* `text` string is compared by value).
|
||||
*/
|
||||
const MarkdownPart = memo(function MarkdownPart({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
streaming,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
streaming: boolean;
|
||||
}) {
|
||||
if (streaming) {
|
||||
return (
|
||||
<StreamingMarkdownText
|
||||
text={text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
);
|
||||
}
|
||||
const html = renderChatMarkdown(text, { neutralizeInternalLinks });
|
||||
if (html) {
|
||||
return (
|
||||
@@ -202,10 +179,47 @@ function MessageItem({
|
||||
{resolveAssistantName(assistantName) ?? t("AI agent")}
|
||||
</Text>
|
||||
{message.parts.map((part, index) => {
|
||||
// Tool parts (`tool-*` / `dynamic-tool`) are template-literal kinds, so
|
||||
// they cannot be a `switch` case; the runtime guard handles them, and the
|
||||
// switch below covers every CLOSED (literal-typed) part kind with a
|
||||
// compile-time exhaustiveness check in its default.
|
||||
if (part.type === "reasoning") {
|
||||
// Reasoning ("thinking") -> a collapsible block with its own token
|
||||
// count. Empty/whitespace reasoning with no authoritative count carries
|
||||
// nothing to show, so skip it (avoids an empty 0-token block).
|
||||
const text = (part as { text?: string }).text ?? "";
|
||||
if (!text.trim() && !(reasoningTokens && reasoningTokens > 0))
|
||||
return null;
|
||||
// Absent state (persisted rows) and "done" both mean finalized.
|
||||
// `messageSignature` already includes each part's `state`, so the
|
||||
// streaming→done flip changes the row signature and re-renders this
|
||||
// row — which is what lets ReasoningBlock switch from chunked plain
|
||||
// text to its one-time markdown parse (see reasoning-block.tsx).
|
||||
// ALSO require the turn to be live: a part stranded at
|
||||
// `state:"streaming"` after the turn ended (no `reasoning-end` — see
|
||||
// the `turnStreaming` prop doc) must still finalize and parse.
|
||||
const streaming =
|
||||
turnStreaming && (part as { state?: string }).state === "streaming";
|
||||
return (
|
||||
<ReasoningBlock
|
||||
key={index}
|
||||
text={text}
|
||||
tokens={reasoningTokens}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (part.type === "text") {
|
||||
// Skip empty/whitespace-only text parts (a streaming message often
|
||||
// starts with an empty text part before the first token arrives); the
|
||||
// typing indicator covers that gap until real content streams in.
|
||||
if (!part.text.trim()) return null;
|
||||
return (
|
||||
<MarkdownPart
|
||||
key={index}
|
||||
text={part.text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
if (isToolPart(part.type)) {
|
||||
return (
|
||||
<ToolCallCard
|
||||
@@ -218,76 +232,7 @@ function MessageItem({
|
||||
);
|
||||
}
|
||||
|
||||
switch (part.type) {
|
||||
case "reasoning": {
|
||||
// Reasoning ("thinking") -> a collapsible block with its own token
|
||||
// count. Empty/whitespace reasoning with no authoritative count
|
||||
// carries nothing to show, so skip it (avoids an empty 0-token block).
|
||||
const text = part.text ?? "";
|
||||
if (!text.trim() && !(reasoningTokens && reasoningTokens > 0))
|
||||
return null;
|
||||
// Absent state (persisted rows) and "done" both mean finalized.
|
||||
// `messageSignature` already includes each part's `state`, so the
|
||||
// streaming→done flip changes the row signature and re-renders this
|
||||
// row — which is what lets ReasoningBlock switch from chunked plain
|
||||
// text to its one-time markdown parse (see reasoning-block.tsx).
|
||||
// ALSO require the turn to be live: a part stranded at
|
||||
// `state:"streaming"` after the turn ended (no `reasoning-end` — see
|
||||
// the `turnStreaming` prop doc) must still finalize and parse.
|
||||
const streaming = turnStreaming && part.state === "streaming";
|
||||
return (
|
||||
<ReasoningBlock
|
||||
key={index}
|
||||
text={text}
|
||||
tokens={reasoningTokens}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
case "text": {
|
||||
// Skip empty/whitespace-only text parts (a streaming message often
|
||||
// starts with an empty text part before the first token arrives); the
|
||||
// typing indicator covers that gap until real content streams in.
|
||||
if (!part.text.trim()) return null;
|
||||
// The live, actively-growing tail part of the in-flight turn renders
|
||||
// incrementally (see MarkdownPart); a finalized part (persisted, or
|
||||
// the turn-end flip) renders the whole text through one canonical
|
||||
// pass. Same liveness rule as the reasoning branch above.
|
||||
const streaming = turnStreaming && part.state === "streaming";
|
||||
return (
|
||||
<MarkdownPart
|
||||
key={index}
|
||||
text={part.text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
streaming={streaming}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
case "source-url":
|
||||
case "source-document":
|
||||
case "file":
|
||||
case "step-start":
|
||||
// Not surfaced in the chat bubble (v1) — same as the pre-#492 default.
|
||||
return null;
|
||||
|
||||
default: {
|
||||
// Compile-time exhaustiveness over the CLOSED union members: every
|
||||
// literal-typed part kind is handled above, so the only kinds that
|
||||
// can reach here are the OPEN template-literal ones (`tool-*` — caught
|
||||
// by the guard at runtime — and `data-*`) plus `dynamic-tool`. Adding
|
||||
// a NEW closed part kind to UIMessagePart makes this assignment fail
|
||||
// to compile, forcing it to be handled instead of silently ignored
|
||||
// (this replaces the pre-#492 fall-through `return null` + WARNING).
|
||||
const _exhaustive:
|
||||
| `tool-${string}`
|
||||
| "dynamic-tool"
|
||||
| `data-${string}` = part.type;
|
||||
void _exhaustive;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
})}
|
||||
{/* A persisted turn error (server stored it in metadata.error). Rendered
|
||||
here so it survives a thread remount and shows in reopened history. */}
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
import { memo, useMemo } from "react";
|
||||
import { splitPlainChunks } from "@/features/ai-chat/components/streaming-plain-text.tsx";
|
||||
import { renderChatMarkdown } from "@/features/ai-chat/utils/markdown.ts";
|
||||
import classes from "@/features/ai-chat/components/ai-chat.module.css";
|
||||
|
||||
/**
|
||||
* One STABILIZED markdown block, rendered through the canonical pipeline and
|
||||
* memoized on its string prop. During streaming only the TAIL chunk grows (the
|
||||
* `splitPlainChunks` append-only invariant guarantees every earlier chunk is
|
||||
* byte-identical across deltas), so React skips every stable block and each one
|
||||
* is parsed by `renderChatMarkdown` EXACTLY ONCE — turning the pre-#492
|
||||
* "re-parse the whole accumulated answer on every ~20Hz tick" (O(ticks)) into
|
||||
* O(number of blocks). The markup is DOMPurify-sanitized inside renderChatMarkdown
|
||||
* before it reaches `dangerouslySetInnerHTML`.
|
||||
*
|
||||
* NOTE (transient streaming-only artifact): a safe cut is a blank-line boundary,
|
||||
* so a construct that legitimately contains a blank line (e.g. a fenced code block
|
||||
* with an empty line) can be split across chunks and render oddly WHILE it is still
|
||||
* streaming. This is cosmetic and self-heals: the moment the part finalizes,
|
||||
* MarkdownPart renders the WHOLE text through one canonical pass (visual parity
|
||||
* with the pre-#492 output). The reasoning path makes the same trade (plain text
|
||||
* while streaming, one markdown parse at the end).
|
||||
*/
|
||||
const MarkdownChunk = memo(function MarkdownChunk({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
}) {
|
||||
const html = renderChatMarkdown(text, { neutralizeInternalLinks });
|
||||
if (html) {
|
||||
return (
|
||||
<div
|
||||
className={classes.markdown}
|
||||
// Sanitized by renderChatMarkdown (DOMPurify) before insertion.
|
||||
dangerouslySetInnerHTML={{ __html: html }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
// Malformed/unsupported markdown could not render synchronously: raw text.
|
||||
return (
|
||||
<div className={classes.markdown} style={{ whiteSpace: "pre-wrap" }}>
|
||||
{text}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* The cheap streaming-time stand-in for the finalized answer's one-time markdown
|
||||
* parse (see MarkdownPart in message-item.tsx). Mirrors StreamingPlainText's
|
||||
* chunked-memo pattern but renders the STABILIZED prefix as real markdown (each
|
||||
* block parsed once, memoized) and only the LIVE tail as flat plain text — so the
|
||||
* user sees formatted output for everything up to the last safe cut, and the not-
|
||||
* yet-stable tail (which markdown-parsing every tick would make O(ticks)) stays a
|
||||
* single cheap escaped text node until it stabilizes into a new block.
|
||||
*
|
||||
* `splitPlainChunks` yields chunks where, under append-only growth, every chunk
|
||||
* except the LAST is immutable; the last chunk is the live tail. Index keys are
|
||||
* therefore stable (a given index never changes to a different chunk's content).
|
||||
*/
|
||||
export function StreamingMarkdownText({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
}) {
|
||||
const chunks = useMemo(() => splitPlainChunks(text), [text]);
|
||||
return (
|
||||
<>
|
||||
{chunks.map((chunk, index) =>
|
||||
index < chunks.length - 1 ? (
|
||||
<MarkdownChunk
|
||||
key={index}
|
||||
text={chunk}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
) : (
|
||||
// The live tail: flat, React-escaped plain text (no markdown parse, no
|
||||
// sanitizer, no innerHTML). `pre-wrap` preserves its newlines; trailing
|
||||
// separator newlines are dropped at display time so the block gap comes
|
||||
// from the markdown margins, not a doubled empty line (mirrors
|
||||
// PlainChunk in streaming-plain-text.tsx).
|
||||
<div
|
||||
key={index}
|
||||
className={classes.markdown}
|
||||
style={{ whiteSpace: "pre-wrap" }}
|
||||
>
|
||||
{chunk.replace(/\n+$/, "")}
|
||||
</div>
|
||||
),
|
||||
)}
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Spotlight } from "@mantine/spotlight";
|
||||
import { IconSearch } from "@tabler/icons-react";
|
||||
import { Group, VisuallyHidden } from "@mantine/core";
|
||||
import { Group, Text, VisuallyHidden } from "@mantine/core";
|
||||
import { useState, useMemo } from "react";
|
||||
import { useDebouncedValue } from "@mantine/hooks";
|
||||
import { useTranslation } from "react-i18next";
|
||||
@@ -84,6 +84,11 @@ export function SearchSpotlight({ spaceId }: SearchSpotlightProps) {
|
||||
onFiltersChange={handleFiltersChange}
|
||||
spaceId={spaceId}
|
||||
/>
|
||||
{/* #529: operator hint — matches ANY word by default; "…" for an exact
|
||||
phrase, +term to require, -term to exclude. */}
|
||||
<Text size="xs" c="dimmed" mt={4}>
|
||||
{t('Tip: "exact phrase", +required, -excluded')}
|
||||
</Text>
|
||||
</div>
|
||||
|
||||
<VisuallyHidden role="status" aria-live="polite">
|
||||
|
||||
@@ -5,6 +5,9 @@ import { IPage } from "@/features/page/types/page.types.ts";
|
||||
|
||||
export interface IPageSearch {
|
||||
id: string;
|
||||
// #529 A7 superset: `pageId` aliases `id`; `rank`/`highlight` are null for
|
||||
// substring-only hits (the UI already falls back to the title/snippet).
|
||||
pageId?: string;
|
||||
title: string;
|
||||
icon: string;
|
||||
parentPageId: string;
|
||||
@@ -12,9 +15,36 @@ export interface IPageSearch {
|
||||
creatorId: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
rank: string;
|
||||
highlight: string;
|
||||
rank: string | number | null;
|
||||
highlight: string | null;
|
||||
space: Partial<ISpace>;
|
||||
// New #529 fields (present from the native Postgres search driver).
|
||||
snippet?: string;
|
||||
score?: number;
|
||||
path?: string[];
|
||||
matchedFields?: string[];
|
||||
matchedTerms?: string[];
|
||||
}
|
||||
|
||||
// #529 A5 pagination envelope returned by POST /search (native driver). The web
|
||||
// list helpers read `items`; these travel alongside for pagination + diagnostics.
|
||||
export interface IPageSearchResponse {
|
||||
items: IPageSearch[];
|
||||
total: number;
|
||||
hasMore: boolean;
|
||||
truncatedAtCap: boolean;
|
||||
offset: number;
|
||||
query?: {
|
||||
raw: string;
|
||||
parsed: {
|
||||
positive: string[];
|
||||
required: string[];
|
||||
excluded: string[];
|
||||
reason?: string;
|
||||
};
|
||||
mode: "or" | "and";
|
||||
match: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface SearchSuggestionParams {
|
||||
@@ -37,6 +67,10 @@ export interface IPageSearchParams {
|
||||
query: string;
|
||||
spaceId?: string;
|
||||
shareId?: string;
|
||||
// #529 A9: match mode (auto default) + pagination.
|
||||
match?: "auto" | "word" | "prefix" | "substring";
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export interface IAttachmentSearch {
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"migration:reset": "tsx src/database/migrate.ts down-to NO_MIGRATIONS",
|
||||
"migration:codegen": "kysely-codegen --dialect=postgres --camel-case --env-file=../../.env --out-file=./src/database/types/db.d.ts",
|
||||
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
||||
"pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build && pnpm --filter @docmost/token-estimate build",
|
||||
"pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build && pnpm --filter @docmost/token-estimate build && pnpm --filter @docmost/mcp build",
|
||||
"test": "jest",
|
||||
"test:int": "jest --config test/jest-integration.json",
|
||||
"test:watch": "jest --watch",
|
||||
|
||||
@@ -35,7 +35,6 @@ import {
|
||||
import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { UserThrottlerGuard } from '../../integrations/throttle/user-throttler.guard';
|
||||
import { AI_CHAT_THROTTLER } from '../../integrations/throttle/throttler-names';
|
||||
@@ -44,8 +43,6 @@ import {
|
||||
AiChatRunHooks,
|
||||
AiChatService,
|
||||
AiChatStreamBody,
|
||||
rowHasInlineParts,
|
||||
hydrateAssistantParts,
|
||||
} from './ai-chat.service';
|
||||
import { AiChatRunService } from './ai-chat-run.service';
|
||||
import { AiTranscriptionService } from './ai-transcription.service';
|
||||
@@ -132,39 +129,8 @@ export class AiChatController {
|
||||
// production. Only touched on the resumable-stream (flag-on) path.
|
||||
private readonly streamRegistry?: AiChatStreamRegistryService,
|
||||
private readonly environment?: EnvironmentService,
|
||||
// #492: reconstruct a #492 mid-run record's parts from the steps table before
|
||||
// returning rows to the client / export. OPTIONAL so positional controller
|
||||
// specs compile unchanged; when absent, hydration is skipped (old-era rows
|
||||
// already carry inline parts, so nothing to reconstruct).
|
||||
private readonly aiChatRunStepRepo?: AiChatRunStepRepo,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Reconstruct parts for any assistant rows that don't carry them INLINE — a
|
||||
* #492 mid-run record whose per-step parts live in `ai_chat_run_steps` (the
|
||||
* append-persist backend). Every FINISHED row (old-era + #492) and every old-era
|
||||
* streaming snapshot already has inline `metadata.parts`, so the common path
|
||||
* fetches NOTHING and returns the rows untouched; only an actively-streaming
|
||||
* new-style row triggers the batch step fetch. Consumers (seed/poll/export) read
|
||||
* `metadata.parts` off the returned rows exactly as before — the era switch is
|
||||
* invisible to them (reconstructRunParts contract).
|
||||
*/
|
||||
private async withReconstructedParts(
|
||||
rows: AiChatMessage[],
|
||||
workspaceId: string,
|
||||
): Promise<AiChatMessage[]> {
|
||||
if (!this.aiChatRunStepRepo) return rows;
|
||||
const needy = rows.filter(
|
||||
(r) => r.role === 'assistant' && !rowHasInlineParts(r),
|
||||
);
|
||||
if (needy.length === 0) return rows;
|
||||
const stepsByMessage = await this.aiChatRunStepRepo.findByMessageIds(
|
||||
needy.map((r) => r.id),
|
||||
workspaceId,
|
||||
);
|
||||
return hydrateAssistantParts(rows, stepsByMessage);
|
||||
}
|
||||
|
||||
/** List the requesting user's chats in this workspace (paginated). */
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('chats')
|
||||
@@ -218,17 +184,11 @@ export class AiChatController {
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
) {
|
||||
await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
const page = await this.aiChatMessageRepo.findByChat(
|
||||
return this.aiChatMessageRepo.findByChat(
|
||||
dto.chatId,
|
||||
workspace.id,
|
||||
pagination,
|
||||
);
|
||||
// #492: reconstruct parts for any active new-style row so the client seed sees
|
||||
// `metadata.parts` unchanged (a no-op for the finished rows that fill a page).
|
||||
return {
|
||||
...page,
|
||||
items: await this.withReconstructedParts(page.items, workspace.id),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -265,10 +225,7 @@ export class AiChatController {
|
||||
workspace.id,
|
||||
);
|
||||
return {
|
||||
// #492: the delta of an actively-streaming new-style row carries its parts
|
||||
// reconstructed from the steps table, so the degraded poll shows persisted
|
||||
// progress exactly as the pre-#492 full-row snapshot did.
|
||||
rows: await this.withReconstructedParts(rows, workspace.id),
|
||||
rows,
|
||||
cursor,
|
||||
run: run ? { id: run.id, status: run.status } : null,
|
||||
};
|
||||
@@ -290,10 +247,8 @@ export class AiChatController {
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<{ markdown: string }> {
|
||||
const chat = await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
const rows = await this.withReconstructedParts(
|
||||
await this.aiChatMessageRepo.findAllByChat(dto.chatId, workspace.id),
|
||||
// #492: an interrupted-but-still-active turn exports its persisted steps
|
||||
// (reconstructed from the steps table) just like the pre-#492 full row did.
|
||||
const rows = await this.aiChatMessageRepo.findAllByChat(
|
||||
dto.chatId,
|
||||
workspace.id,
|
||||
);
|
||||
const markdown = buildChatMarkdown({
|
||||
@@ -333,13 +288,7 @@ export class AiChatController {
|
||||
workspace.id,
|
||||
)
|
||||
: undefined;
|
||||
// #492: reconnect to an IN-FLIGHT run reconstructs the projection row's parts
|
||||
// from the steps table (the row itself carries only the step marker mid-run);
|
||||
// a finished run's row already has inline parts, so this is a no-op.
|
||||
const [hydrated] = message
|
||||
? await this.withReconstructedParts([message], workspace.id)
|
||||
: [undefined];
|
||||
return { run, message: hydrated ?? null };
|
||||
return { run, message: message ?? null };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -22,7 +22,6 @@ import { AiSettingsService } from '../../integrations/ai/ai-settings.service';
|
||||
import { describeProviderError } from '../../integrations/ai/ai-error.util';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatPageSnapshotRepo } from '@docmost/db/repos/ai-chat/ai-chat-page-snapshot.repo';
|
||||
import { AiAgentRoleRepo } from '@docmost/db/repos/ai-agent-roles/ai-agent-roles.repo';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
@@ -519,12 +518,6 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// constructions compile unchanged; Nest always injects the real singleton, so
|
||||
// reconcile sees the SAME in-memory active/zombie maps the runner mutates.
|
||||
private readonly aiChatRunService?: AiChatRunService,
|
||||
// #492 append-persist: per-step INSERT into the lightweight steps table (the
|
||||
// O(Σ steps) replacement for the O(n²) full-row `metadata.parts` rewrite).
|
||||
// OPTIONAL so existing positional constructions (int-specs) compile unchanged;
|
||||
// Nest injects the real singleton. When ABSENT the per-step path falls back to
|
||||
// the pre-#492 full-row flush (no regression, only no WAL win).
|
||||
private readonly aiChatRunStepRepo?: AiChatRunStepRepo,
|
||||
) {}
|
||||
|
||||
// #487: periodic reconcile timer (single-process phase 1). Started in
|
||||
@@ -1121,34 +1114,8 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
chatId,
|
||||
workspace.id,
|
||||
);
|
||||
// #492: HYDRATE needy assistant rows from the steps table BEFORE the replay
|
||||
// map. A #492 mid-run assistant row carries only a step marker
|
||||
// (metadata.parts:[]); its real per-step parts live in `ai_chat_run_steps`.
|
||||
// The graceful terminal callbacks (onFinish/onError/onAbort -> flushAssistant)
|
||||
// assemble the full inline parts, so a normally-ended turn already has them.
|
||||
// But a HARD crash mid-run (SIGKILL/OOM) fires NO terminal callback, so the
|
||||
// row stays parts:[]; without this, rowToUiMessage falls back to an empty
|
||||
// text part and the partial tool-calls/results/text — durable in the steps
|
||||
// table — would DROP OUT of the model's replay context (regressing #183
|
||||
// step-granular durability for the model consumer). Mirrors the controller's
|
||||
// withReconstructedParts EXACTLY (same needy predicate + hydration helper).
|
||||
// Guarded on the optional repo: absent (positional test builds) degrades to
|
||||
// the current behavior rather than crashing.
|
||||
let replayHistory = oldHistory;
|
||||
if (this.aiChatRunStepRepo) {
|
||||
const needy = oldHistory.filter(
|
||||
(r) => r.role === 'assistant' && !rowHasInlineParts(r),
|
||||
);
|
||||
if (needy.length > 0) {
|
||||
const stepsByMessage = await this.aiChatRunStepRepo.findByMessageIds(
|
||||
needy.map((r) => r.id),
|
||||
workspace.id,
|
||||
);
|
||||
replayHistory = hydrateAssistantParts(oldHistory, stepsByMessage);
|
||||
}
|
||||
}
|
||||
const uiMessages: Array<Omit<UIMessage, 'id'> & { id: string }> = [
|
||||
...replayHistory.map(rowToUiMessage),
|
||||
...oldHistory.map(rowToUiMessage),
|
||||
{
|
||||
id: 'pending-user',
|
||||
role: 'user',
|
||||
@@ -1187,9 +1154,7 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// hint — confirm it against the persisted history (the preceding assistant
|
||||
// turn must really be aborted/streaming) so a spoofed flag cannot inject the
|
||||
// interrupt note onto an ordinary turn. The partial output the model needs is
|
||||
// already in `messages`: a #492 mid-run row's per-step parts live only in the
|
||||
// `ai_chat_run_steps` table and were hydrated into the replay history above,
|
||||
// so the aborted assistant turn replays WITH its partial parts intact.
|
||||
// already in `messages` (the aborted assistant row replays via findRecent).
|
||||
// Append the new user turn (shape-only) so index -2 is the prior assistant.
|
||||
const interrupted = isInterruptResume(
|
||||
[...oldHistory, { role: 'user', status: null, metadata: null }],
|
||||
@@ -1594,57 +1559,17 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy {
|
||||
// connection when finalize runs, so the SQL `WHERE status='streaming'`
|
||||
// (not this flag) is what prevents it clobbering the terminal row.
|
||||
if (finalized) return null;
|
||||
// The count derives from capturedSteps.length at THIS instant, so the
|
||||
// returned value is EXACTLY the persisted `stepsPersisted` the ring rotates
|
||||
// on (whether we take the append-persist path or the legacy fallback).
|
||||
const stepsPersisted = capturedSteps.length;
|
||||
// Build the flush ONCE so the returned count is EXACTLY the persisted
|
||||
// `stepsPersisted` (both derive from capturedSteps.length at this instant).
|
||||
const flushed = flushAssistant(capturedSteps, '', 'streaming', {
|
||||
pageChanged,
|
||||
partsCache,
|
||||
});
|
||||
const stepsPersisted = flushed.metadata.stepsPersisted as number;
|
||||
try {
|
||||
if (this.aiChatRunStepRepo) {
|
||||
// #492 APPEND-PERSIST: write only THIS finished step's parts to the
|
||||
// steps table (O(step) WAL), then bump the row's CHEAP step marker —
|
||||
// NO growing `metadata.parts` blob (that O(n²) full-row rewrite is
|
||||
// exactly what this removes). The full `metadata.parts` is assembled
|
||||
// once at finalize; a mid-run resume seed is reconstructed from the
|
||||
// step rows (reconstructRunParts). The INSERT is idempotent
|
||||
// (ON CONFLICT DO NOTHING), so a re-fired step never doubles the parts.
|
||||
const index = stepsPersisted - 1;
|
||||
if (index >= 0) {
|
||||
const stepParts = assistantParts(
|
||||
[capturedSteps[index]],
|
||||
'',
|
||||
partsCache,
|
||||
);
|
||||
await this.aiChatRunStepRepo.insertStep(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
index,
|
||||
stepParts,
|
||||
);
|
||||
}
|
||||
// Marker UPDATE: advance stepsPersisted + keep the toolTrace era marker
|
||||
// (bumps updatedAt so the delta poll observes the step, and carries the
|
||||
// frontier a resuming client attaches from). Scoped onlyIfStreaming so a
|
||||
// late marker never clobbers the terminal finalize.
|
||||
await this.aiChatMessageRepo.update(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
{ metadata: stepMarkerMetadata(stepsPersisted) },
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
} else {
|
||||
// Legacy fallback (no steps table wired — positional test builds): the
|
||||
// pre-#492 full-row flush, so parts still land inline on the row.
|
||||
const flushed = flushAssistant(capturedSteps, '', 'streaming', {
|
||||
pageChanged,
|
||||
partsCache,
|
||||
});
|
||||
await this.aiChatMessageRepo.update(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
flushed,
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
await this.aiChatMessageRepo.update(assistantId, workspace.id, flushed, {
|
||||
onlyIfStreaming: true,
|
||||
});
|
||||
return stepsPersisted;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
@@ -2824,122 +2749,6 @@ export function rowToUiMessage(row: AiChatMessage): Omit<UIMessage, 'id'> & {
|
||||
return { id: row.id, role, parts: parts as UIMessage['parts'] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Cheap step-marker metadata for the #492 per-step UPDATE. Advances
|
||||
* `stepsPersisted` (the resume attach frontier) and keeps the `toolTraceVersion`
|
||||
* era marker, WITHOUT the growing `parts` blob (those live in the steps table
|
||||
* now; the full `metadata.parts` is assembled once at finalize by flushAssistant).
|
||||
* `parts: []` is kept for shape stability — it reads as an empty inline-parts row,
|
||||
* which is exactly the discriminator that routes reconstruction to the steps table.
|
||||
*/
|
||||
export function stepMarkerMetadata(
|
||||
stepsPersisted: number,
|
||||
): Record<string, unknown> {
|
||||
return { parts: [], toolTraceVersion: 2, stepsPersisted };
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether an assistant row already carries its full UI parts INLINE on the row
|
||||
* (`metadata.parts`). TRUE for every FINISHED row — old-era rows AND #492 rows,
|
||||
* whose full parts are assembled once at finalize — and for old-era streaming
|
||||
* snapshots (the pre-#492 per-step full-row flush). FALSE for a #492 MID-RUN
|
||||
* record, whose per-step parts live in the `ai_chat_run_steps` table. This is the
|
||||
* era discriminator the reconstruct seam branches on — no schema flag needed.
|
||||
*/
|
||||
export function rowHasInlineParts(row: { metadata?: unknown }): boolean {
|
||||
const meta = (row.metadata ?? {}) as { parts?: unknown };
|
||||
return Array.isArray(meta.parts) && meta.parts.length > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Concatenate persisted per-step parts (in `stepIndex` order) into the turn's UI
|
||||
* parts (#492). Reproduces EXACTLY what flushAssistant → assistantParts would have
|
||||
* written to `metadata.parts` for those finished steps, since each step row stored
|
||||
* `assistantParts([step])` at persist time.
|
||||
*/
|
||||
export function assembleStepParts(
|
||||
stepRows: ReadonlyArray<{ stepIndex: number; parts: unknown }>,
|
||||
): UIMessage['parts'] {
|
||||
const parts: Array<Record<string, unknown>> = [];
|
||||
for (const step of [...stepRows].sort((a, b) => a.stepIndex - b.stepIndex)) {
|
||||
if (Array.isArray(step.parts)) {
|
||||
parts.push(...(step.parts as Array<Record<string, unknown>>));
|
||||
}
|
||||
}
|
||||
return parts as UIMessage['parts'];
|
||||
}
|
||||
|
||||
/**
|
||||
* reconstructRunParts (#492) — the single backend-switch seam. Given an assistant
|
||||
* ROW and its persisted step rows, return the turn's UI `parts` + the persisted
|
||||
* step count, reading from the ROW when it already carries inline parts (old-era
|
||||
* records AND every finished record) and from the STEPS TABLE otherwise (a #492
|
||||
* mid-run record). The higher-level consumers (attach seed, delta poll, export)
|
||||
* route their row→parts through this / {@link hydrateAssistantParts}, so old and
|
||||
* new records reconstruct identically WITHOUT the consumers branching on the era.
|
||||
*/
|
||||
export function reconstructRunParts(
|
||||
row: { metadata?: unknown; content?: string | null },
|
||||
stepRows: ReadonlyArray<{ stepIndex: number; parts: unknown }>,
|
||||
): { parts: UIMessage['parts']; stepsPersisted: number } {
|
||||
if (rowHasInlineParts(row)) {
|
||||
const meta = row.metadata as {
|
||||
parts: UIMessage['parts'];
|
||||
stepsPersisted?: number;
|
||||
};
|
||||
return {
|
||||
parts: meta.parts,
|
||||
stepsPersisted:
|
||||
typeof meta.stepsPersisted === 'number'
|
||||
? meta.stepsPersisted
|
||||
: stepRows.length,
|
||||
};
|
||||
}
|
||||
if (stepRows.length > 0) {
|
||||
return {
|
||||
parts: assembleStepParts(stepRows),
|
||||
stepsPersisted: stepRows.length,
|
||||
};
|
||||
}
|
||||
// No inline parts and no step rows: an old-era seed / empty streaming row. Fall
|
||||
// back to a single text part from `content` (mirrors rowToUiMessage).
|
||||
return {
|
||||
parts: textPart(row.content ?? '') as UIMessage['parts'],
|
||||
stepsPersisted: 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill each assistant row's `metadata.parts` from its step rows when the row does
|
||||
* not already carry them inline (a #492 mid-run record), so a consumer that reads
|
||||
* `metadata.parts` off the RAW row (the client seed/poll, the Markdown export)
|
||||
* sees the reconstructed parts with NO change to itself. Rows that already have
|
||||
* inline parts (old-era + finished) and non-assistant rows pass through untouched.
|
||||
* Pure: returns new row objects, never mutates the inputs.
|
||||
*/
|
||||
export function hydrateAssistantParts<
|
||||
T extends { id: string; role?: string; metadata?: unknown },
|
||||
>(
|
||||
rows: ReadonlyArray<T>,
|
||||
stepsByMessage: Map<
|
||||
string,
|
||||
ReadonlyArray<{ stepIndex: number; parts: unknown }>
|
||||
>,
|
||||
): T[] {
|
||||
return rows.map((row) => {
|
||||
if (row.role !== 'assistant' || rowHasInlineParts(row)) return row;
|
||||
const steps = stepsByMessage.get(row.id);
|
||||
if (!steps || steps.length === 0) return row;
|
||||
return {
|
||||
...row,
|
||||
metadata: {
|
||||
...((row.metadata ?? {}) as Record<string, unknown>),
|
||||
parts: assembleStepParts(steps),
|
||||
},
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The persisted-row patch shape produced by {@link flushAssistant}. It is the
|
||||
* SAME shape the assistant repo insert/update consume (content + toolCalls +
|
||||
|
||||
@@ -1,33 +1,56 @@
|
||||
import { Space } from '@docmost/db/types/entity.types';
|
||||
|
||||
export class SearchResponseDto {
|
||||
// #529 A7 — the single per-hit SUPERSET returned by the unified search engine.
|
||||
// The web-UI reads id/highlight/icon/space/title/…; the MCP agent maps id→pageId
|
||||
// and reads snippet/score/path. `rank`/`highlight` are null for substring-only
|
||||
// hits (the web already falls back). Nothing the legacy web response carried is
|
||||
// dropped.
|
||||
export class SearchResultDto {
|
||||
id: string;
|
||||
title: string;
|
||||
// Alias of `id` for the MCP layer (it addresses pages by pageId).
|
||||
pageId: string;
|
||||
slugId: string;
|
||||
icon: string;
|
||||
parentPageId: string;
|
||||
title: string;
|
||||
space?: Partial<Space>;
|
||||
creatorId: string;
|
||||
rank: number;
|
||||
highlight: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
space: Partial<Space>;
|
||||
// ts_rank_cd of the FTS branch; null for substring-only hits.
|
||||
rank: number | null;
|
||||
// ts_headline marked HTML; null for substring-only hits.
|
||||
highlight: string | null;
|
||||
// Plain windowed snippet around the match (empty for titleOnly).
|
||||
snippet: string;
|
||||
// Ancestor titles root → direct parent ([] for a root page).
|
||||
path: string[];
|
||||
// Per-response ordering proxy (falls back to rank).
|
||||
score: number;
|
||||
// Which fields matched: 'title' and/or 'text'.
|
||||
matchedFields: string[];
|
||||
// Which parsed positive/required terms this hit matched.
|
||||
matchedTerms: string[];
|
||||
}
|
||||
|
||||
// Response shape for the opt-in agent-lookup mode (#443, `substring: true`).
|
||||
// Additive to the FTS response: carries the location (`path`), a windowed
|
||||
// `snippet` around the first match and a per-response sort `score`. The MCP
|
||||
// layer maps `id → pageId`; `slugId` is never exposed.
|
||||
export class SearchLookupResponseDto {
|
||||
id: string;
|
||||
slugId: string;
|
||||
title: string;
|
||||
parentPageId: string | null;
|
||||
// Ancestor titles from the space root down to the direct parent; [] for a
|
||||
// root page.
|
||||
path: string[];
|
||||
// ~300–500 chars around the first match (or a leading text window / extended
|
||||
// ts_headline fallback).
|
||||
snippet: string;
|
||||
// 0..1 float, meaningful ONLY for sorting within one response.
|
||||
score: number;
|
||||
// The paginated envelope (A5). `total` is the EXACT permission-filtered count of
|
||||
// pages matching the positive lexical query (fail-closed). `hasMore` is true when
|
||||
// more results exist WITHIN the fusion window; `truncatedAtCap` signals the match
|
||||
// set exceeded CANDIDATE_CAP and the tail is unreachable by pagination.
|
||||
export class SearchResponseDto {
|
||||
items: SearchResultDto[];
|
||||
total: number;
|
||||
hasMore: boolean;
|
||||
truncatedAtCap: boolean;
|
||||
offset: number;
|
||||
query: {
|
||||
raw: string;
|
||||
parsed: {
|
||||
positive: string[];
|
||||
required: string[];
|
||||
excluded: string[];
|
||||
reason?: string;
|
||||
};
|
||||
mode: 'or' | 'and';
|
||||
match: string;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
IsBoolean,
|
||||
IsIn,
|
||||
IsNotEmpty,
|
||||
IsNumber,
|
||||
IsOptional,
|
||||
@@ -11,6 +12,13 @@ export class SearchDTO {
|
||||
@IsString()
|
||||
query: string;
|
||||
|
||||
// #529 A3 — match mode. `auto` (default) routes identifier-like terms
|
||||
// (10.31.41, esp32, WB-MGE-30D86B) to the substring/trigram branch and words
|
||||
// to full-text; `word`/`prefix`/`substring` are explicit overrides.
|
||||
@IsOptional()
|
||||
@IsIn(['auto', 'word', 'prefix', 'substring'])
|
||||
match?: 'auto' | 'word' | 'prefix' | 'substring';
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
spaceId: string;
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
import { parseSearchQuery, hasPositiveRecall } from './search-query-parser';
|
||||
|
||||
describe('parseSearchQuery — tokenization & operators (A2)', () => {
|
||||
it('splits on whitespace into positive terms (OR recall)', () => {
|
||||
const p = parseSearchQuery('стамбул роснефть');
|
||||
expect(p.positive.map((t) => t.text)).toEqual(['стамбул', 'роснефть']);
|
||||
expect(p.required).toEqual([]);
|
||||
expect(p.excluded).toEqual([]);
|
||||
expect(p.mode).toBe('or');
|
||||
});
|
||||
|
||||
it('treats +term as required and -term as excluded', () => {
|
||||
const p = parseSearchQuery('+кофейня -архив');
|
||||
expect(p.required.map((t) => t.text)).toEqual(['кофейня']);
|
||||
expect(p.excluded.map((t) => t.text)).toEqual(['архив']);
|
||||
expect(p.positive).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps a leading-operator-free hyphen/dot/colon token as ONE literal term', () => {
|
||||
// WB-MGE-30D86B, 10.0.12.5, a:b — internal -,.,: are literal, one term each.
|
||||
expect(parseSearchQuery('WB-MGE-30D86B').positive[0].text).toBe(
|
||||
'WB-MGE-30D86B',
|
||||
);
|
||||
expect(parseSearchQuery('10.0.12.5').positive[0].text).toBe('10.0.12.5');
|
||||
expect(parseSearchQuery('host:8080').positive[0].text).toBe('host:8080');
|
||||
});
|
||||
|
||||
it('only a LEADING +/- is an operator; -архив excludes архив', () => {
|
||||
const p = parseSearchQuery('-архив');
|
||||
expect(p.excluded.map((t) => t.text)).toEqual(['архив']);
|
||||
expect(p.positive).toEqual([]);
|
||||
});
|
||||
|
||||
it('drops a bare "-" / "+" and an all-operator remainder', () => {
|
||||
const p = parseSearchQuery('- + foo -- ++');
|
||||
expect(p.positive.map((t) => t.text)).toEqual(['foo']);
|
||||
expect(p.required).toEqual([]);
|
||||
expect(p.excluded).toEqual([]);
|
||||
});
|
||||
|
||||
it('parses a quoted phrase as one adjacency term', () => {
|
||||
const p = parseSearchQuery('"воздушный шар" кофе');
|
||||
expect(p.positive[0]).toMatchObject({ text: 'воздушный шар', branch: 'phrase' });
|
||||
expect(p.positive[1].text).toBe('кофе');
|
||||
});
|
||||
|
||||
it('applies +/- to a phrase', () => {
|
||||
const req = parseSearchQuery('+"воздушный шар"');
|
||||
expect(req.required[0]).toMatchObject({ text: 'воздушный шар', branch: 'phrase' });
|
||||
const exc = parseSearchQuery('-"воздушный шар"');
|
||||
expect(exc.excluded[0]).toMatchObject({ text: 'воздушный шар', branch: 'phrase' });
|
||||
});
|
||||
|
||||
it('drops an unbalanced quote token', () => {
|
||||
const p = parseSearchQuery('kafka "unclosed here');
|
||||
expect(p.positive.map((t) => t.text)).toEqual(['kafka']);
|
||||
});
|
||||
|
||||
it('strips tsquery metacharacters from a bare FTS term (no 500)', () => {
|
||||
const p = parseSearchQuery('foo|bar');
|
||||
// `|` is not an identifier signal → FTS branch; the metachar is stripped so
|
||||
// the term becomes the two words that survive.
|
||||
expect(p.positive[0].branch === 'fts' || p.positive[0].branch === 'ftsPrefix').toBe(true);
|
||||
expect(p.positive[0].text).toBe('foo bar');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSearchQuery — match=auto classification (A3)', () => {
|
||||
it('routes identifier-like terms to the substring branch', () => {
|
||||
expect(parseSearchQuery('10.31.41').positive[0].branch).toBe('substring');
|
||||
expect(parseSearchQuery('esp32').positive[0].branch).toBe('substring');
|
||||
expect(parseSearchQuery('WB-MGE-30D86B').positive[0].branch).toBe('substring');
|
||||
});
|
||||
|
||||
it('routes purely-alphabetic words to the FTS (prefix) branch', () => {
|
||||
expect(parseSearchQuery('печат').positive[0].branch).toBe('ftsPrefix');
|
||||
expect(parseSearchQuery('ресторан').positive[0].branch).toBe('ftsPrefix');
|
||||
});
|
||||
|
||||
it('explicit match overrides: word / prefix / substring', () => {
|
||||
expect(parseSearchQuery('печат', { match: 'word' }).positive[0].branch).toBe('fts');
|
||||
expect(parseSearchQuery('печат', { match: 'prefix' }).positive[0].branch).toBe(
|
||||
'ftsPrefix',
|
||||
);
|
||||
expect(parseSearchQuery('печат', { match: 'substring' }).positive[0].branch).toBe(
|
||||
'substring',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSearchQuery — reasons & recall', () => {
|
||||
it('only-negation yields reason only-negation and no positive recall', () => {
|
||||
const p = parseSearchQuery('-архив');
|
||||
expect(p.reason).toBe('only-negation');
|
||||
expect(hasPositiveRecall(p)).toBe(false);
|
||||
});
|
||||
|
||||
it('empty / whitespace / garbage yields reason empty', () => {
|
||||
expect(parseSearchQuery('').reason).toBe('empty');
|
||||
expect(parseSearchQuery(' ').reason).toBe('empty');
|
||||
// A bare operator drops to nothing → empty (no exclusion survived).
|
||||
expect(parseSearchQuery('+ -').reason).toBe('empty');
|
||||
});
|
||||
|
||||
it('a required term alone IS positive recall (no reason)', () => {
|
||||
const p = parseSearchQuery('+кофейня');
|
||||
expect(p.reason).toBeUndefined();
|
||||
expect(hasPositiveRecall(p)).toBe(true);
|
||||
});
|
||||
|
||||
it('mode flag flows through', () => {
|
||||
expect(parseSearchQuery('a b', { mode: 'and' }).mode).toBe('and');
|
||||
expect(parseSearchQuery('a b').mode).toBe('or');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,224 @@
|
||||
// #529 Phase A — server-side query parser (the SINGLE source of query semantics).
|
||||
//
|
||||
// Clients (web-UI, MCP agent, public share) send a RAW query string plus flags
|
||||
// (`match`, `mode`); ALL query interpretation happens HERE, so every consumer
|
||||
// gets identical operator/phrase/morphology behaviour. The parser is PURE (no
|
||||
// SQL, no DB) so it is exhaustively unit-testable; SearchService turns the parsed
|
||||
// AST into a parameterized tsquery/predicate tree (never string-concatenated SQL).
|
||||
//
|
||||
// Grammar (A2):
|
||||
// - Whitespace splits tokens, but a double-quoted run is ONE token ("a b" is a
|
||||
// phrase). An unbalanced quote is dropped.
|
||||
// - A token is an OPERATOR token only when it STARTS with `+` or `-`, the
|
||||
// remainder is non-empty, and the remainder is not itself only operators.
|
||||
// `+`/`-` inside a token (`WB-MGE-30D86B`, `10.0.12.5`, `a:b`) is a LITERAL —
|
||||
// the token is a single term. A bare `-`/`+` is dropped.
|
||||
// - `"phrase"` → phrase term; `+"phrase"`/`-"phrase"` apply the operator to it.
|
||||
// - Positive terms (bare + phrase, no operator) form the OR recall set.
|
||||
// `+term` is a REQUIRED predicate, `-term` an EXCLUDED predicate (A2): both
|
||||
// are applied in SQL WHERE against the whole candidate set, not folded into
|
||||
// the positive tsquery.
|
||||
// - Only-negation (no positive term) short-circuits to an empty result with
|
||||
// reason `only-negation` (never runs a costly NOT-scan).
|
||||
|
||||
export type SearchMatchMode = 'auto' | 'word' | 'prefix' | 'substring';
|
||||
export type SearchBooleanMode = 'or' | 'and';
|
||||
|
||||
// How a single term is matched against the index.
|
||||
// - 'fts' : full-text lexeme, exact (no trailing prefix).
|
||||
// - 'ftsPrefix' : full-text lexeme with a `:*` prefix match.
|
||||
// - 'phrase' : an adjacency phrase (phraseto_tsquery).
|
||||
// - 'substring' : a literal LOWER(f_unaccent(col)) LIKE '%needle%' branch
|
||||
// (identifiers the tokenizer mangles: IPs, hostnames, IDs).
|
||||
export type SearchTermBranch = 'fts' | 'ftsPrefix' | 'phrase' | 'substring';
|
||||
|
||||
export interface ParsedTerm {
|
||||
// The user-visible term text, operator stripped, quotes removed. This is what
|
||||
// `matchedTerms` echoes back per hit.
|
||||
text: string;
|
||||
branch: SearchTermBranch;
|
||||
}
|
||||
|
||||
export interface ParsedQuery {
|
||||
raw: string;
|
||||
// OR-recall set (bare + phrase terms with no operator).
|
||||
positive: ParsedTerm[];
|
||||
// AND predicates (`+term`) — the candidate MUST match each of these.
|
||||
required: ParsedTerm[];
|
||||
// NOT predicates (`-term`) — the candidate must match NONE of these.
|
||||
excluded: ParsedTerm[];
|
||||
mode: SearchBooleanMode;
|
||||
// Set only when the query yields no positive recall: 'empty' (nothing usable)
|
||||
// or 'only-negation' (there were exclusions but no positive term).
|
||||
reason?: 'empty' | 'only-negation';
|
||||
}
|
||||
|
||||
interface RawToken {
|
||||
op: '' | '+' | '-';
|
||||
kind: 'word' | 'phrase';
|
||||
text: string;
|
||||
}
|
||||
|
||||
// tsquery metacharacters that must never reach to_tsquery from a bare term — they
|
||||
// are what turned adversarial input into a 500 before (#139). Stripped for the FTS
|
||||
// branch; the substring branch keeps them (they are literal there).
|
||||
const TSQUERY_META = /[:&|!()*<>\\]+/g;
|
||||
|
||||
// A term is "identifier-like" when it carries a digit or one of . _ : / - AND is
|
||||
// not purely alphabetic (letters only). Such tokens (10.31.41, esp32,
|
||||
// WB-MGE-30D86B) are mangled by the FTS tokenizer, so `match: auto` routes them
|
||||
// to the substring branch. A purely-alphabetic word (печат, ресторан) stays FTS.
|
||||
const IDENTIFIER_SIGNAL = /[0-9._:/\\-]/;
|
||||
const PURELY_ALPHA = /^\p{L}+$/u;
|
||||
|
||||
function isIdentifierLike(text: string): boolean {
|
||||
return IDENTIFIER_SIGNAL.test(text) && !PURELY_ALPHA.test(text);
|
||||
}
|
||||
|
||||
// Clean a bare term for the FTS branch: NFC-normalize, drop tsquery metacharacters
|
||||
// and collapse whitespace. Returns '' when nothing usable remains.
|
||||
export function cleanFtsLexeme(raw: string): string {
|
||||
return (raw ?? '')
|
||||
.normalize('NFC')
|
||||
.replace(TSQUERY_META, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// Split a raw query into tokens, honouring double quotes and a single leading
|
||||
// +/- operator. Unbalanced quotes and bare operators are dropped.
|
||||
function tokenize(raw: string): RawToken[] {
|
||||
const tokens: RawToken[] = [];
|
||||
const s = raw ?? '';
|
||||
let i = 0;
|
||||
const n = s.length;
|
||||
|
||||
const isSpace = (c: string) => /\s/.test(c);
|
||||
|
||||
while (i < n) {
|
||||
// Skip leading whitespace.
|
||||
while (i < n && isSpace(s[i])) i++;
|
||||
if (i >= n) break;
|
||||
|
||||
let op: '' | '+' | '-' = '';
|
||||
// A single leading +/- is a tentative operator. Only ONE leading operator is
|
||||
// consumed; a second (`--x`) leaves `-x` as the remainder (a literal dash).
|
||||
if (s[i] === '+' || s[i] === '-') {
|
||||
op = s[i] as '+' | '-';
|
||||
i++;
|
||||
}
|
||||
|
||||
if (i < n && s[i] === '"') {
|
||||
// Quoted phrase: read until the closing quote.
|
||||
const close = s.indexOf('"', i + 1);
|
||||
if (close === -1) {
|
||||
// Unbalanced quote → drop this token and everything the open quote would
|
||||
// have consumed (the rest of the string).
|
||||
break;
|
||||
}
|
||||
const phrase = s.slice(i + 1, close);
|
||||
i = close + 1;
|
||||
if (phrase.trim().length > 0) {
|
||||
tokens.push({ op, kind: 'phrase', text: phrase.trim() });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Bare word: read until the next whitespace.
|
||||
let j = i;
|
||||
while (j < n && !isSpace(s[j])) j++;
|
||||
const word = s.slice(i, j);
|
||||
i = j;
|
||||
|
||||
// A bare operator (`-`/`+` with no remainder) or an all-operator remainder is
|
||||
// dropped.
|
||||
if (word.length === 0) continue;
|
||||
if (op && /^[+-]+$/.test(word)) continue;
|
||||
|
||||
tokens.push({ op, kind: 'word', text: word });
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
// Resolve the match branch for a single term given the global match mode.
|
||||
function branchForTerm(
|
||||
text: string,
|
||||
kind: 'word' | 'phrase',
|
||||
mode: SearchMatchMode,
|
||||
): SearchTermBranch {
|
||||
if (kind === 'phrase') return 'phrase';
|
||||
switch (mode) {
|
||||
case 'word':
|
||||
return 'fts';
|
||||
case 'prefix':
|
||||
return 'ftsPrefix';
|
||||
case 'substring':
|
||||
return 'substring';
|
||||
case 'auto':
|
||||
default:
|
||||
// Identifiers the tokenizer mangles go to substring; words get a prefix
|
||||
// FTS match (so `печат` still finds `печатать`, but `печат` no longer drags
|
||||
// in `впечатления` because the russian stemmer anchors the stem).
|
||||
return isIdentifierLike(text) ? 'substring' : 'ftsPrefix';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a raw user query + flags into a structured, SQL-agnostic AST.
|
||||
* Pure and total: never throws, always returns a ParsedQuery.
|
||||
*/
|
||||
export function parseSearchQuery(
|
||||
raw: string,
|
||||
opts: { match?: SearchMatchMode; mode?: SearchBooleanMode } = {},
|
||||
): ParsedQuery {
|
||||
const match: SearchMatchMode = opts.match ?? 'auto';
|
||||
const mode: SearchBooleanMode = opts.mode ?? 'or';
|
||||
|
||||
const positive: ParsedTerm[] = [];
|
||||
const required: ParsedTerm[] = [];
|
||||
const excluded: ParsedTerm[] = [];
|
||||
|
||||
for (const tok of tokenize(raw)) {
|
||||
// For an FTS branch, the token must survive metacharacter cleaning; for the
|
||||
// substring/phrase branch the literal text is used. A term that cleans to
|
||||
// nothing AND is not usable as a substring is dropped.
|
||||
const branch = branchForTerm(tok.text, tok.kind, match);
|
||||
|
||||
let usableText: string;
|
||||
if (branch === 'fts' || branch === 'ftsPrefix') {
|
||||
usableText = cleanFtsLexeme(tok.text);
|
||||
} else {
|
||||
// phrase / substring keep the literal (trimmed) text.
|
||||
usableText = tok.text.trim();
|
||||
}
|
||||
if (!usableText) continue;
|
||||
|
||||
const term: ParsedTerm = { text: usableText, branch };
|
||||
|
||||
if (tok.op === '+') required.push(term);
|
||||
else if (tok.op === '-') excluded.push(term);
|
||||
else positive.push(term);
|
||||
}
|
||||
|
||||
const parsed: ParsedQuery = { raw: raw ?? '', positive, required, excluded, mode };
|
||||
|
||||
if (positive.length === 0) {
|
||||
// Required terms with no positive recall still form a valid positive set (the
|
||||
// required predicates ARE the recall). Only when there is neither a positive
|
||||
// nor a required term is the query empty / only-negation.
|
||||
if (required.length === 0) {
|
||||
parsed.reason = excluded.length > 0 ? 'only-negation' : 'empty';
|
||||
}
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Does this parsed query have any positive recall to run? False means we must
|
||||
* short-circuit to an empty result (with `reason`), never a costly NOT-only scan.
|
||||
*/
|
||||
export function hasPositiveRecall(parsed: ParsedQuery): boolean {
|
||||
return parsed.positive.length > 0 || parsed.required.length > 0;
|
||||
}
|
||||
@@ -1,19 +1,14 @@
|
||||
import {
|
||||
computeLookupScore,
|
||||
escapeLikePattern,
|
||||
SearchLookupTier,
|
||||
} from './search.service';
|
||||
import { escapeLikePattern } from './search.service';
|
||||
|
||||
/**
|
||||
* Pure-function coverage for the #443 agent-lookup helpers:
|
||||
* - escapeLikePattern: LIKE-metacharacter escaping so `%`/`_`/`\` are literals
|
||||
* (the acceptance-table requirement that a query of `%` or `_` does NOT match
|
||||
* everything);
|
||||
* - computeLookupScore: the tiered 0..1 ranking score, where a stronger tier
|
||||
* always outranks a weaker one regardless of the in-tier secondary signal.
|
||||
* Pure-function coverage for `escapeLikePattern` — LIKE-metacharacter escaping so
|
||||
* `%`/`_`/`\` are matched literally (the acceptance requirement that a query of
|
||||
* `%` or `_` does NOT match everything, #529 acceptance #10). The substring
|
||||
* branch's DB behaviour is covered by the integration spec.
|
||||
*
|
||||
* The DB-touching branch (substring UNION FTS, path CTE, snippet window) is
|
||||
* covered by the integration spec against the real schema.
|
||||
* NOTE (#529): the old tiered `computeLookupScore` was replaced by RRF rank
|
||||
* fusion in the unified engine, so its unit coverage moved to the integration
|
||||
* ordering tests; only the escaping helper remains a pure unit here.
|
||||
*/
|
||||
describe('escapeLikePattern', () => {
|
||||
it('escapes the LIKE metacharacters % _ and \\', () => {
|
||||
@@ -43,53 +38,3 @@ describe('escapeLikePattern', () => {
|
||||
expect(escapeLikePattern(null as any)).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeLookupScore', () => {
|
||||
it('keeps every score within (0, 1]', () => {
|
||||
for (const tier of [
|
||||
SearchLookupTier.TITLE_EXACT,
|
||||
SearchLookupTier.TITLE_SUBSTRING,
|
||||
SearchLookupTier.TEXT,
|
||||
]) {
|
||||
for (const secondary of [0, 0.001, 1, 100, 1e6]) {
|
||||
const s = computeLookupScore({ tier, secondary });
|
||||
expect(s).toBeGreaterThan(0);
|
||||
expect(s).toBeLessThanOrEqual(1);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('a stronger tier ALWAYS outranks a weaker tier, whatever the secondary', () => {
|
||||
// Weak tier with a huge secondary must still lose to a strong tier with a
|
||||
// tiny secondary — tiers dominate.
|
||||
const strongLowSecondary = computeLookupScore({
|
||||
tier: SearchLookupTier.TITLE_EXACT,
|
||||
secondary: 0,
|
||||
});
|
||||
const weakHighSecondary = computeLookupScore({
|
||||
tier: SearchLookupTier.TEXT,
|
||||
secondary: 1e9,
|
||||
});
|
||||
expect(strongLowSecondary).toBeGreaterThan(weakHighSecondary);
|
||||
});
|
||||
|
||||
it('within a tier a larger secondary sorts higher', () => {
|
||||
const lo = computeLookupScore({
|
||||
tier: SearchLookupTier.TEXT,
|
||||
secondary: 0.1,
|
||||
});
|
||||
const hi = computeLookupScore({
|
||||
tier: SearchLookupTier.TEXT,
|
||||
secondary: 5,
|
||||
});
|
||||
expect(hi).toBeGreaterThan(lo);
|
||||
});
|
||||
|
||||
it('treats a negative/absent secondary as 0', () => {
|
||||
const zero = computeLookupScore({ tier: SearchLookupTier.TEXT, secondary: 0 });
|
||||
expect(computeLookupScore({ tier: SearchLookupTier.TEXT })).toBe(zero);
|
||||
expect(
|
||||
computeLookupScore({ tier: SearchLookupTier.TEXT, secondary: -5 }),
|
||||
).toBe(zero);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,74 +1,45 @@
|
||||
import { SearchService } from './search.service';
|
||||
|
||||
/**
|
||||
* Coverage for SearchService.searchPage query-mode selection (search.service.ts
|
||||
* @25). searchPage chooses HOW the result set is scoped — by explicit space, by
|
||||
* the authenticated user's member spaces, or by a share — and must return an
|
||||
* empty set (without leaking data) for every disallowed combination.
|
||||
* Unit coverage for SearchService.searchPage SCOPE-SECURITY early returns — the
|
||||
* branches that must yield an empty result WITHOUT ever touching the DB, so they
|
||||
* can leak nothing. The happy-path scope SQL (explicit space / member spaces /
|
||||
* share id set) is covered against the real schema in the integration spec.
|
||||
*
|
||||
* The kysely query builder is mocked with the same chainable pattern as the
|
||||
* existing search.service.spec.ts: every builder method returns the same builder
|
||||
* and `.execute()` resolves the supplied rows. Each `.where(...)` call is
|
||||
* recorded so we can assert exactly which scope clause was applied — that is the
|
||||
* mutation-resistant signal that distinguishes one query mode from another.
|
||||
*
|
||||
* These specs catch cross-space / cross-workspace search leakage and
|
||||
* share-scope bypass (data exposure).
|
||||
* Every case here returns BEFORE the raw-SQL candidate query runs, so a bare `db`
|
||||
* stub (never called) is enough — a call to it would itself be a failure signal.
|
||||
*/
|
||||
describe('SearchService.searchPage — query-mode selection', () => {
|
||||
// Build a chainable selectFrom('pages') builder that records its calls. The
|
||||
// builder is returned from `db.selectFrom` and is the single object every
|
||||
// chained call mutates/returns, mirroring the existing spec's pattern.
|
||||
function makeBuilder(rows: Array<{ id: string; highlight?: string }>) {
|
||||
const builder: any = {};
|
||||
builder.select = jest.fn(() => builder);
|
||||
builder.where = jest.fn(() => builder);
|
||||
builder.$if = jest.fn(() => builder);
|
||||
builder.orderBy = jest.fn(() => builder);
|
||||
builder.limit = jest.fn(() => builder);
|
||||
builder.offset = jest.fn(() => builder);
|
||||
builder.execute = jest.fn(async () => rows);
|
||||
return builder;
|
||||
}
|
||||
|
||||
describe('SearchService.searchPage — scope-security early returns', () => {
|
||||
function makeService(opts?: {
|
||||
rows?: Array<{ id: string; highlight?: string }>;
|
||||
share?: any;
|
||||
isRestricted?: boolean;
|
||||
descendants?: Array<{ id: string }>;
|
||||
memberSpaceIds?: string[];
|
||||
}) {
|
||||
const builder = makeBuilder(opts?.rows ?? []);
|
||||
|
||||
const db: any = {
|
||||
selectFrom: jest.fn(() => builder),
|
||||
};
|
||||
|
||||
// `getUserSpaceIdsQuery` returns a sub-query object that searchPage passes
|
||||
// straight into `.where('spaceId', 'in', <subquery>)`. A sentinel is enough
|
||||
// to assert the user-scoped branch was taken.
|
||||
const userSpaceIdsQuery = { __userSpaceIdsQuery: true };
|
||||
// A db that THROWS if touched — these branches must not reach SQL.
|
||||
const db: any = new Proxy(
|
||||
{},
|
||||
{
|
||||
get() {
|
||||
throw new Error('db must not be touched on an empty-scope branch');
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const pageRepo = {
|
||||
// `.select((eb) => this.pageRepo.withSpace(eb))` — value ignored by stub.
|
||||
withSpace: jest.fn(() => ({ __withSpace: true })),
|
||||
getPageAndDescendantsExcludingRestricted: jest
|
||||
.fn()
|
||||
.mockResolvedValue(opts?.descendants ?? []),
|
||||
getPageAndDescendantsExcludingRestricted: jest.fn(),
|
||||
getPageAndDescendants: jest.fn(),
|
||||
};
|
||||
const shareRepo = {
|
||||
findById: jest.fn().mockResolvedValue(opts?.share ?? null),
|
||||
};
|
||||
const spaceMemberRepo = {
|
||||
getUserSpaceIdsQuery: jest.fn(() => userSpaceIdsQuery),
|
||||
getUserSpaceIds: jest.fn().mockResolvedValue(opts?.memberSpaceIds ?? []),
|
||||
};
|
||||
const pagePermissionRepo = {
|
||||
hasRestrictedAncestor: jest
|
||||
.fn()
|
||||
.mockResolvedValue(opts?.isRestricted ?? false),
|
||||
// Let everything through page-level permission filtering by default.
|
||||
filterAccessiblePageIds: jest
|
||||
.fn()
|
||||
.mockImplementation(async ({ pageIds }: { pageIds: string[] }) => pageIds),
|
||||
filterAccessiblePageIds: jest.fn(),
|
||||
};
|
||||
|
||||
const service = new SearchService(
|
||||
@@ -78,145 +49,81 @@ describe('SearchService.searchPage — query-mode selection', () => {
|
||||
spaceMemberRepo as any,
|
||||
pagePermissionRepo as any,
|
||||
);
|
||||
|
||||
return {
|
||||
service,
|
||||
db,
|
||||
builder,
|
||||
pageRepo,
|
||||
shareRepo,
|
||||
spaceMemberRepo,
|
||||
pagePermissionRepo,
|
||||
userSpaceIdsQuery,
|
||||
};
|
||||
return { service, pageRepo, shareRepo, spaceMemberRepo, pagePermissionRepo };
|
||||
}
|
||||
|
||||
const whereCallFor = (builder: any, column: any) =>
|
||||
builder.where.mock.calls.find((c: any[]) => c[0] === column);
|
||||
|
||||
it('returns {items:[]} for a blank query WITHOUT touching the DB', async () => {
|
||||
const { service, db } = makeService();
|
||||
|
||||
it('returns total:0 for a blank query WITHOUT touching the DB or any repo', async () => {
|
||||
const { service, shareRepo, spaceMemberRepo } = makeService();
|
||||
const result = await service.searchPage(
|
||||
{ query: '' } as any,
|
||||
{ userId: 'user-1', workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ items: [] });
|
||||
// Blank query is rejected before any query builder is constructed.
|
||||
expect(db.selectFrom).not.toHaveBeenCalled();
|
||||
expect(result.items).toEqual([]);
|
||||
expect(result.total).toBe(0);
|
||||
expect(shareRepo.findById).not.toHaveBeenCalled();
|
||||
expect(spaceMemberRepo.getUserSpaceIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('scopes to the explicit spaceId branch', async () => {
|
||||
const { service, builder, db, spaceMemberRepo, shareRepo } = makeService({
|
||||
rows: [{ id: 'p-1' }],
|
||||
});
|
||||
|
||||
it('only-negation short-circuits with reason "only-negation", never scanning', async () => {
|
||||
const { service, spaceMemberRepo } = makeService();
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan', spaceId: 'space-42' } as any,
|
||||
{ query: '-архив' } as any,
|
||||
{ userId: 'user-1', workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(db.selectFrom).toHaveBeenCalledWith('pages');
|
||||
// The explicit-space branch adds exactly `.where('spaceId', '=', 'space-42')`.
|
||||
expect(whereCallFor(builder, 'spaceId')).toEqual([
|
||||
'spaceId',
|
||||
'=',
|
||||
'space-42',
|
||||
]);
|
||||
// It must NOT fall through to the user-member-spaces or share branch.
|
||||
expect(spaceMemberRepo.getUserSpaceIdsQuery).not.toHaveBeenCalled();
|
||||
expect(shareRepo.findById).not.toHaveBeenCalled();
|
||||
expect(result.items.map((i: any) => i.id)).toEqual(['p-1']);
|
||||
expect(result.total).toBe(0);
|
||||
expect(result.query.parsed.reason).toBe('only-negation');
|
||||
// Never resolves scope (returns before) — no expensive NOT-only scan.
|
||||
expect(spaceMemberRepo.getUserSpaceIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('scopes an authenticated user WITHOUT spaceId to their member spaces', async () => {
|
||||
const { service, builder, spaceMemberRepo, userSpaceIdsQuery, shareRepo } =
|
||||
makeService({ rows: [{ id: 'p-9' }] });
|
||||
|
||||
await service.searchPage(
|
||||
{ query: 'plan' } as any,
|
||||
{ userId: 'user-7', workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
// The user-scoped branch resolves the member-spaces sub-query for that user
|
||||
// and restricts both spaceId (to that sub-query) and workspaceId.
|
||||
expect(spaceMemberRepo.getUserSpaceIdsQuery).toHaveBeenCalledWith('user-7');
|
||||
expect(whereCallFor(builder, 'spaceId')).toEqual([
|
||||
'spaceId',
|
||||
'in',
|
||||
userSpaceIdsQuery,
|
||||
]);
|
||||
expect(whereCallFor(builder, 'workspaceId')).toEqual([
|
||||
'workspaceId',
|
||||
'=',
|
||||
'ws-1',
|
||||
]);
|
||||
// Authenticated user path must not consult shares.
|
||||
expect(shareRepo.findById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns {items:[]} when the share belongs to a DIFFERENT workspace', async () => {
|
||||
const { service, builder, shareRepo, pagePermissionRepo } = makeService({
|
||||
share: {
|
||||
id: 'share-1',
|
||||
pageId: 'page-1',
|
||||
workspaceId: 'OTHER-ws',
|
||||
includeSubPages: false,
|
||||
},
|
||||
it('returns empty when the share belongs to a DIFFERENT workspace (no leak)', async () => {
|
||||
const { service, shareRepo, pagePermissionRepo } = makeService({
|
||||
share: { id: 's1', pageId: 'p1', workspaceId: 'OTHER', includeSubPages: false },
|
||||
});
|
||||
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan', shareId: 'share-1' } as any,
|
||||
{ query: 'plan', shareId: 's1' } as any,
|
||||
{ workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(shareRepo.findById).toHaveBeenCalledWith('share-1');
|
||||
expect(result).toEqual({ items: [] });
|
||||
// Workspace mismatch short-circuits before any restricted-ancestor / id
|
||||
// scoping or DB execution: no leak across workspaces.
|
||||
expect(shareRepo.findById).toHaveBeenCalledWith('s1');
|
||||
expect(result.items).toEqual([]);
|
||||
// Workspace mismatch short-circuits before restricted-ancestor / enumeration.
|
||||
expect(pagePermissionRepo.hasRestrictedAncestor).not.toHaveBeenCalled();
|
||||
expect(builder.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns {items:[]} when the shared page has a restricted ancestor', async () => {
|
||||
const { service, builder, pagePermissionRepo, pageRepo } = makeService({
|
||||
share: {
|
||||
id: 'share-1',
|
||||
pageId: 'page-1',
|
||||
workspaceId: 'ws-1',
|
||||
includeSubPages: true,
|
||||
},
|
||||
it('returns empty when the shared page has a restricted ancestor', async () => {
|
||||
const { service, pagePermissionRepo, pageRepo } = makeService({
|
||||
share: { id: 's1', pageId: 'p1', workspaceId: 'ws-1', includeSubPages: true },
|
||||
isRestricted: true,
|
||||
});
|
||||
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan', shareId: 'share-1' } as any,
|
||||
{ query: 'plan', shareId: 's1' } as any,
|
||||
{ workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(pagePermissionRepo.hasRestrictedAncestor).toHaveBeenCalledWith(
|
||||
'page-1',
|
||||
);
|
||||
expect(result).toEqual({ items: [] });
|
||||
// Restricted ancestor must block before page enumeration and DB execution.
|
||||
expect(pagePermissionRepo.hasRestrictedAncestor).toHaveBeenCalledWith('p1');
|
||||
expect(result.items).toEqual([]);
|
||||
expect(
|
||||
pageRepo.getPageAndDescendantsExcludingRestricted,
|
||||
).not.toHaveBeenCalled();
|
||||
expect(builder.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns {items:[]} with no userId, no spaceId and no shareId', async () => {
|
||||
const { service, builder, shareRepo } = makeService();
|
||||
|
||||
it('returns empty with no userId, no spaceId and no shareId', async () => {
|
||||
const { service, shareRepo } = makeService();
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan' } as any,
|
||||
{ workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ items: [] });
|
||||
// The catch-all else returns empty without scoping/executing or hitting shares.
|
||||
expect(result.items).toEqual([]);
|
||||
expect(shareRepo.findById).not.toHaveBeenCalled();
|
||||
expect(builder.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('an authenticated user with NO member spaces gets an empty result', async () => {
|
||||
const { service, spaceMemberRepo } = makeService({ memberSpaceIds: [] });
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan' } as any,
|
||||
{ userId: 'user-1', workspaceId: 'ws-1' },
|
||||
);
|
||||
expect(spaceMemberRepo.getUserSpaceIds).toHaveBeenCalledWith('user-1');
|
||||
expect(result.items).toEqual([]);
|
||||
expect(result.total).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -32,7 +32,6 @@ import { TemplateRepo } from '@docmost/db/repos/template/template.repo';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunRepo } from '@docmost/db/repos/ai-chat/ai-chat-run.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatPageSnapshotRepo } from '@docmost/db/repos/ai-chat/ai-chat-page-snapshot.repo';
|
||||
import { AiProviderCredentialsRepo } from '@docmost/db/repos/ai-chat/ai-provider-credentials.repo';
|
||||
import { AiMcpServerRepo } from '@docmost/db/repos/ai-chat/ai-mcp-server.repo';
|
||||
@@ -126,7 +125,6 @@ import { firstSqlToken } from '../integrations/metrics/metrics.constants';
|
||||
AiChatRepo,
|
||||
AiChatMessageRepo,
|
||||
AiChatRunRepo,
|
||||
AiChatRunStepRepo,
|
||||
AiChatPageSnapshotRepo,
|
||||
AiProviderCredentialsRepo,
|
||||
AiMcpServerRepo,
|
||||
@@ -163,7 +161,6 @@ import { firstSqlToken } from '../integrations/metrics/metrics.constants';
|
||||
AiChatRepo,
|
||||
AiChatMessageRepo,
|
||||
AiChatRunRepo,
|
||||
AiChatRunStepRepo,
|
||||
AiChatPageSnapshotRepo,
|
||||
AiProviderCredentialsRepo,
|
||||
AiMcpServerRepo,
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
import { type Kysely, sql } from 'kysely';
|
||||
|
||||
/**
|
||||
* #529 Phase A1 — the `ru_en` text-search configuration + the config swap.
|
||||
*
|
||||
* WHY: the search stack was pinned to the `english` FTS config, which stems only
|
||||
* Latin words. On a Russian-language wiki that is a morphology black hole:
|
||||
* «ресторанов москвы» never matched a page titled «ресторан в москве». `ru_en`
|
||||
* layers the russian_stem over the Cyrillic token classes and english_stem over
|
||||
* the ascii ones, so BOTH languages get proper morphology from one config.
|
||||
*
|
||||
* CREATE TEXT SEARCH CONFIGURATION ru_en (COPY = simple);
|
||||
* ALTER ... asciiword/asciihword/hword_asciipart WITH english_stem;
|
||||
* ALTER ... word/hword/hword_part WITH russian_stem;
|
||||
*
|
||||
* `to_tsvector('ru_en', …)` with the LITERAL config name is IMMUTABLE, so it is
|
||||
* valid inside a trigger, a generated column and an index expression.
|
||||
*
|
||||
* THE INVARIANT (acceptance #13): the config of the STORED column and the config
|
||||
* of the QUERY must change together. This migration flips BOTH stored sides
|
||||
* (pages.tsv via its trigger + a reindex; page_embeddings.fts, the RAG lexical
|
||||
* leg, via its generated expression); the matching QUERY-side flips
|
||||
* (search.service.ts and page-embedding.repo.ts `hybridSearch`) ship in the SAME
|
||||
* commit. Trigram indexes are LOWER(f_unaccent(...)) and do NOT depend on the FTS
|
||||
* config, so they are untouched.
|
||||
*
|
||||
* REINDEX / LOCK MODEL (deploy-critical). Kysely runs EACH migration in its OWN
|
||||
* transaction (see sibling 20260706T120000 "Kysely runs each migration in a
|
||||
* transaction"; migrate.ts / migration.service.ts set no `disableTransactions`,
|
||||
* and PostgresJSDialect has transactional DDL). A single migration file therefore
|
||||
* cannot commit between batches, so the issue's "procedural batch job OUTSIDE the
|
||||
* transaction + dual-config read window + migration_complete gate" is not
|
||||
* expressible in-migration here. That machinery bridges a reindex spread over
|
||||
* MANY committed batches, during which some rows are still `english` while others
|
||||
* are already `ru_en`. Our pages.tsv reindex is a SINGLE `UPDATE pages SET tsv`,
|
||||
* ATOMIC within THIS migration's own transaction: at COMMIT every row is `ru_en`
|
||||
* at once, so no morphology-desync window exists and no dual-config read path is
|
||||
* required — the query config flips to `ru_en` in the very same release. This is
|
||||
* the deliberate, correct adaptation to this framework (see the PR notes).
|
||||
*
|
||||
* - pages.tsv: swapping the trigger is a cheap catalog change (no table lock),
|
||||
* and the reindex is a single `UPDATE pages SET tsv = <ru_en expr>` — a
|
||||
* ROW-level-lock (RowExclusiveLock) backfill, NOT an ACCESS EXCLUSIVE rewrite
|
||||
* (mirrors the existing space_id backfill in 20250725T052004). On a LARGE
|
||||
* tenant this still writes every row + its WAL and leaves dead tuples, so it
|
||||
* can take MINUTES and blocks the startup migrator for that time — but it
|
||||
* never blocks concurrent reads. It stays inline unconditionally.
|
||||
*
|
||||
* - page_embeddings.fts is a GENERATED STORED column; Postgres cannot change a
|
||||
* generated expression without DROP+ADD, which is a full-table ACCESS
|
||||
* EXCLUSIVE REWRITE of page_embeddings — it blocks ALL reads AND writes on
|
||||
* that table (including the RAG agent) for the rewrite's duration. That inline
|
||||
* rewrite is appropriate for small/typical tenants (this fork's target) and
|
||||
* is the DEFAULT.
|
||||
*
|
||||
* LARGE TENANTS have two documented escape hatches, either of which makes the
|
||||
* migration genuinely no-op the rewrite (it is NOT a blind DROP+ADD):
|
||||
* (a) Set `SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE=false`. The migration then
|
||||
* SKIPS the embeddings rewrite entirely and logs a WARNING. The operator
|
||||
* MUST perform the ru_en fts swap out-of-band; until they do, the RAG
|
||||
* lexical leg stays on `english` while the query config is `ru_en` — a
|
||||
* documented, operator-owned desync window. (pages.tsv still swaps
|
||||
* inline — the gate is ONLY the embeddings rewrite.)
|
||||
* (b) Perform the swap out-of-band BEFORE deploy — add a plain column →
|
||||
* batched backfill → brief-lock swap → CREATE INDEX CONCURRENTLY — so
|
||||
* the `fts` column's generated expression already references the TARGET
|
||||
* config when the migration runs. The migration detects this (it reads
|
||||
* the column's actual generation expression from pg_catalog) and does a
|
||||
* TRUE no-op — no DROP, no ADD, no rewrite. This is real idempotency,
|
||||
* not the old (false) "IF-EXISTS guards no-op" claim: `DROP COLUMN IF
|
||||
* EXISTS` guards against ABSENCE, not presence, so it would have dropped
|
||||
* and recreated an existing `fts` regardless. The at-target check is the
|
||||
* only honest no-op path.
|
||||
*
|
||||
* Same documented trade-off family as the #443 trgm GIN migration (20260706T120000).
|
||||
*/
|
||||
|
||||
// pages.tsv trigger body for a given FTS config — mirrors the latest form
|
||||
// (20250729T213756): f_unaccent + a 1MB text cap on text_content, weights A/B.
|
||||
function pagesTriggerSql(config: 'ru_en' | 'english') {
|
||||
return sql`
|
||||
CREATE OR REPLACE FUNCTION pages_tsvector_trigger() RETURNS trigger AS $$
|
||||
begin
|
||||
new.tsv :=
|
||||
setweight(to_tsvector('${sql.raw(config)}', f_unaccent(coalesce(new.title, ''))), 'A') ||
|
||||
setweight(to_tsvector('${sql.raw(config)}', f_unaccent(substring(coalesce(new.text_content, ''), 1, 1000000))), 'B');
|
||||
return new;
|
||||
end;
|
||||
$$ LANGUAGE plpgsql;
|
||||
`;
|
||||
}
|
||||
|
||||
async function swapPagesConfig(db: Kysely<any>, config: 'ru_en' | 'english') {
|
||||
// 1. Point the trigger at the target config (new/edited rows use it going
|
||||
// forward). CREATE OR REPLACE FUNCTION takes only a brief catalog lock.
|
||||
await pagesTriggerSql(config).execute(db);
|
||||
|
||||
// 2. Reindex existing rows: recompute tsv directly with the target config. A
|
||||
// plain UPDATE — row locks, no ACCESS EXCLUSIVE. Equivalent to firing the
|
||||
// trigger but cheaper (no self-update round trip).
|
||||
await sql`
|
||||
UPDATE pages
|
||||
SET tsv =
|
||||
setweight(to_tsvector('${sql.raw(config)}', f_unaccent(coalesce(title, ''))), 'A') ||
|
||||
setweight(to_tsvector('${sql.raw(config)}', f_unaccent(substring(coalesce(text_content, ''), 1, 1000000))), 'B')
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
// The default in-migration ACCESS EXCLUSIVE rewrite of page_embeddings.fts is
|
||||
// ON unless the operator explicitly opts out with the env flag. Parsed strictly
|
||||
// (mirrors CLIENT_TELEMETRY_ENABLED / DEBUG_MODE in common/): only a literal
|
||||
// (case-insensitive) 'false' disables it; anything else — unset included —
|
||||
// keeps the default true.
|
||||
function inlineEmbeddingsRewriteEnabled(): boolean {
|
||||
return (
|
||||
(process.env.SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE ?? 'true').toLowerCase() !==
|
||||
'false'
|
||||
);
|
||||
}
|
||||
|
||||
// Read page_embeddings.fts's ACTUAL generated-column expression from pg_catalog
|
||||
// (the generation expression is stored as a column default marked generated).
|
||||
// Returns '' when the column is absent.
|
||||
async function embeddingsFtsExpr(db: Kysely<any>): Promise<string> {
|
||||
const r = await sql<{ def: string }>`
|
||||
SELECT pg_get_expr(d.adbin, d.adrelid) AS def
|
||||
FROM pg_attrdef d
|
||||
JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum
|
||||
WHERE a.attname = 'fts'
|
||||
AND a.attrelid = 'page_embeddings'::regclass
|
||||
AND NOT a.attisdropped
|
||||
`.execute(db);
|
||||
return r.rows[0]?.def ?? '';
|
||||
}
|
||||
|
||||
async function swapEmbeddingsFtsConfig(
|
||||
db: Kysely<any>,
|
||||
config: 'ru_en' | 'english',
|
||||
) {
|
||||
// 1. TRUE no-op path (real out-of-band escape hatch): if the column's current
|
||||
// generation expression already references the TARGET config, there is
|
||||
// nothing to do. An operator who pre-swapped the column out-of-band lands
|
||||
// here and the migration does NOT rewrite the table. ('ru_en' and 'english'
|
||||
// are disjoint tokens, neither a substring of the other or of the rest of
|
||||
// the expression, so a plain contains-check is unambiguous.)
|
||||
const currentExpr = await embeddingsFtsExpr(db);
|
||||
if (currentExpr.includes(config)) return;
|
||||
|
||||
// 2. Env-gated opt-out: large tenants set SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE
|
||||
// =false to skip the ACCESS EXCLUSIVE rewrite in-migration and own the swap
|
||||
// out-of-band. Warn loudly so the desync window is not silent.
|
||||
if (!inlineEmbeddingsRewriteEnabled()) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[migration 20260707T130000] SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE=false: ` +
|
||||
`SKIPPING the page_embeddings.fts rewrite to '${config}'. The operator MUST ` +
|
||||
`perform this fts swap out-of-band. Until then the RAG lexical leg stays on ` +
|
||||
`its current config while the query config is '${config}' (documented, ` +
|
||||
`operator-owned desync window).`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// 3. Default inline path: the generated `fts` expression can only change via
|
||||
// DROP+ADD (a full-table ACCESS EXCLUSIVE rewrite; see the lock note in the
|
||||
// header). The GIN index depends on the column, so it is dropped with it and
|
||||
// recreated.
|
||||
await sql`DROP INDEX IF EXISTS idx_page_embeddings_fts`.execute(db);
|
||||
await sql`ALTER TABLE page_embeddings DROP COLUMN IF EXISTS fts`.execute(db);
|
||||
await sql`
|
||||
ALTER TABLE page_embeddings
|
||||
ADD COLUMN fts tsvector
|
||||
GENERATED ALWAYS AS (to_tsvector('${sql.raw(config)}', f_unaccent(content))) STORED
|
||||
`.execute(db);
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_page_embeddings_fts
|
||||
ON page_embeddings USING gin(fts)
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
async function ruEnConfigExists(db: Kysely<any>): Promise<boolean> {
|
||||
const r = await sql<{ n: number }>`
|
||||
SELECT count(*)::int AS n FROM pg_ts_config WHERE cfgname = 'ru_en'
|
||||
`.execute(db);
|
||||
return (r.rows[0]?.n ?? 0) > 0;
|
||||
}
|
||||
|
||||
async function ensureRuEnConfig(db: Kysely<any>): Promise<void> {
|
||||
// Idempotent by EXISTENCE, not by drop-recreate. The old `DROP ... IF EXISTS;
|
||||
// CREATE` was safe only on a first run: on a re-run the page_embeddings.fts
|
||||
// generated column already has a hard dependency on ru_en, so dropping the
|
||||
// config would fail. Create only when it is genuinely missing.
|
||||
if (await ruEnConfigExists(db)) return;
|
||||
await sql`CREATE TEXT SEARCH CONFIGURATION ru_en (COPY = simple)`.execute(db);
|
||||
// Latin token classes → english_stem.
|
||||
await sql`
|
||||
ALTER TEXT SEARCH CONFIGURATION ru_en
|
||||
ALTER MAPPING FOR asciiword, asciihword, hword_asciipart
|
||||
WITH english_stem
|
||||
`.execute(db);
|
||||
// Cyrillic / non-ascii token classes → russian_stem.
|
||||
await sql`
|
||||
ALTER TEXT SEARCH CONFIGURATION ru_en
|
||||
ALTER MAPPING FOR word, hword, hword_part
|
||||
WITH russian_stem
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await ensureRuEnConfig(db);
|
||||
|
||||
// Flip both stored sides to ru_en (query-side flips in the same commit).
|
||||
// swapEmbeddingsFtsConfig no-ops when fts already references ru_en, so a
|
||||
// re-run of up() is idempotent and does NOT re-rewrite the embeddings table.
|
||||
await swapPagesConfig(db, 'ru_en');
|
||||
await swapEmbeddingsFtsConfig(db, 'ru_en');
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
// Reverse ORDER matters: the trigger and the generated column reference the
|
||||
// `ru_en` config by name, so they must be moved back to `english` BEFORE the
|
||||
// config can be dropped (a generated column that still depends on `ru_en` would
|
||||
// block the DROP with a dependency error).
|
||||
await swapEmbeddingsFtsConfig(db, 'english');
|
||||
await swapPagesConfig(db, 'english');
|
||||
|
||||
// Drop the config ONLY if nothing still references it. When the embeddings
|
||||
// rewrite was gated off (SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE=false), the fts
|
||||
// column can still reference ru_en — dropping the config would then fail with a
|
||||
// dependency error. Skip + warn so down() stays non-fatal; the operator drops
|
||||
// ru_en after completing the out-of-band english swap.
|
||||
if ((await embeddingsFtsExpr(db)).includes('ru_en')) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[migration 20260707T130000] down(): page_embeddings.fts still references ` +
|
||||
`ru_en (inline rewrite was gated off) — leaving the ru_en text-search ` +
|
||||
`configuration in place. Drop it out-of-band once fts is back on 'english'.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
await sql`DROP TEXT SEARCH CONFIGURATION IF EXISTS ru_en`.execute(db);
|
||||
}
|
||||
@@ -1,70 +0,0 @@
|
||||
import { type Kysely, sql } from 'kysely';
|
||||
|
||||
/**
|
||||
* `ai_chat_run_steps` — append-only per-step persistence for an assistant turn
|
||||
* (#492 wave C). Each finished agent step's UI `parts` (its text part + a part
|
||||
* per tool call, WITH the tool output) is INSERTed as its own lightweight row the
|
||||
* moment the step ends, instead of REWRITING the whole assistant row's growing
|
||||
* `metadata.parts` jsonb on every `onStepFinish`.
|
||||
*
|
||||
* WHY a separate table + INSERT (not a jsonb `||` append on the message row): a
|
||||
* Postgres jsonb UPDATE rewrites the ENTIRE TOASTed row version under MVCC, so
|
||||
* re-persisting a growing `metadata.parts` on every step is O(n²) write volume
|
||||
* (a 50-step run with ~100 KB tool outputs wrote hundreds of MB of WAL / dead
|
||||
* tuples per turn, hammering autovacuum). `||` would only shave the network
|
||||
* payload — the WAL/TOAST rewrite harm remains. An INSERT into a per-step table
|
||||
* writes ONLY that step's bytes, so the per-turn write volume is O(Σ steps).
|
||||
*
|
||||
* The full `metadata.parts` on the message row is assembled ONCE at finalize (the
|
||||
* terminal completed/error/aborted write). Mid-run, a resuming client's seed is
|
||||
* reconstructed by concatenating these step rows in `step_index` order — which
|
||||
* reproduces exactly what the old per-step full-row rewrite persisted. Records
|
||||
* written the OLD way (full `metadata.parts` on the row, no step rows) still
|
||||
* reconstruct from the row unchanged; the two eras are distinguished by whether
|
||||
* the row already carries non-empty `metadata.parts` (see reconstructRunParts /
|
||||
* assembleStepParts in ai-chat.service.ts).
|
||||
*
|
||||
* ON DELETE CASCADE on `message_id`: the step rows are a derived projection of the
|
||||
* assistant message; they must vanish with it (or with its workspace).
|
||||
*/
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.createTable('ai_chat_run_steps')
|
||||
.ifNotExists()
|
||||
.addColumn('id', 'uuid', (col) =>
|
||||
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
|
||||
)
|
||||
// The assistant message row this step belongs to (the #183 projection). The
|
||||
// step rows are a derived, per-step slice of that message, so they cascade.
|
||||
.addColumn('message_id', 'uuid', (col) =>
|
||||
col.references('ai_chat_messages.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('workspace_id', 'uuid', (col) =>
|
||||
col.references('workspaces.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
// 0-based index of the finished step within the turn. Ordering key for
|
||||
// reconstruction; unique per message (idempotent step re-persist).
|
||||
.addColumn('step_index', 'integer', (col) => col.notNull())
|
||||
// The step's UI parts (text part + a `tool-*` part per call, WITH output).
|
||||
// Concatenated in step order to rebuild the turn's `metadata.parts`.
|
||||
.addColumn('parts', 'jsonb', (col) => col.notNull())
|
||||
.addColumn('created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.execute();
|
||||
|
||||
// Idempotent per-step persist: a retried INSERT of the same (message, step)
|
||||
// is a no-op (the service uses ON CONFLICT DO NOTHING). This also serves the
|
||||
// reconstruction read (WHERE message_id ORDER BY step_index).
|
||||
await db.schema
|
||||
.createIndex('ai_chat_run_steps_message_step_uidx')
|
||||
.ifNotExists()
|
||||
.on('ai_chat_run_steps')
|
||||
.columns(['message_id', 'step_index'])
|
||||
.unique()
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.dropTable('ai_chat_run_steps').ifExists().execute();
|
||||
}
|
||||
@@ -1,95 +0,0 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '../../types/kysely.types';
|
||||
import { dbOrTx } from '../../utils';
|
||||
import { AiChatRunStep } from '@docmost/db/types/entity.types';
|
||||
|
||||
/**
|
||||
* Append-only per-step persistence for an assistant turn (#492). Each finished
|
||||
* agent step's UI `parts` (its text part + a `tool-*` part per call, WITH the
|
||||
* tool output) is INSERTed as its own lightweight row the moment the step ends —
|
||||
* instead of REWRITING the assistant row's growing `metadata.parts` jsonb on every
|
||||
* `onStepFinish` (a Postgres jsonb UPDATE rewrites the whole TOASTed row version
|
||||
* under MVCC, so that was O(n²) WAL/dead-tuple churn per turn).
|
||||
*
|
||||
* The full `metadata.parts` on the message row is assembled ONCE at finalize;
|
||||
* mid-run, a resuming client's seed is rebuilt from these rows in `stepIndex`
|
||||
* order (see `assembleStepParts` / the reconstruct seam in ai-chat.service.ts).
|
||||
* Every method is workspace-scoped as defense-in-depth.
|
||||
*/
|
||||
@Injectable()
|
||||
export class AiChatRunStepRepo {
|
||||
constructor(@InjectKysely() private readonly db: KyselyDB) {}
|
||||
|
||||
/**
|
||||
* Append one finished step's parts. Idempotent: a retried persist of the SAME
|
||||
* (message, stepIndex) is a no-op via ON CONFLICT DO NOTHING — the per-step
|
||||
* writes are fired fire-and-forget + serialized, and a duplicate must never
|
||||
* throw into the stream or double the parts. Returns whether a NEW row landed
|
||||
* (false = the step was already persisted).
|
||||
*/
|
||||
async insertStep(
|
||||
messageId: string,
|
||||
workspaceId: string,
|
||||
stepIndex: number,
|
||||
parts: unknown,
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<boolean> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
const inserted = await db
|
||||
.insertInto('aiChatRunSteps')
|
||||
.values({
|
||||
messageId,
|
||||
workspaceId,
|
||||
stepIndex,
|
||||
// jsonb column: cast through never (same pattern as the message repo).
|
||||
parts: parts as never,
|
||||
})
|
||||
.onConflict((oc) => oc.columns(['messageId', 'stepIndex']).doNothing())
|
||||
.returning('id')
|
||||
.executeTakeFirst();
|
||||
return inserted !== undefined;
|
||||
}
|
||||
|
||||
/** All persisted steps for ONE assistant message, in step order. */
|
||||
async findByMessage(
|
||||
messageId: string,
|
||||
workspaceId: string,
|
||||
): Promise<AiChatRunStep[]> {
|
||||
return this.db
|
||||
.selectFrom('aiChatRunSteps')
|
||||
.selectAll('aiChatRunSteps')
|
||||
.where('messageId', '=', messageId)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.orderBy('stepIndex', 'asc')
|
||||
.execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* All persisted steps for a SET of assistant messages, grouped by messageId
|
||||
* (each group in step order). One query for the batch — the hydration seam
|
||||
* (getMessages / delta / export) calls this only for the rows that actually
|
||||
* need reconstruction (an active new-style row whose `metadata.parts` is still
|
||||
* empty), which is usually none, so this is skipped on the common path.
|
||||
*/
|
||||
async findByMessageIds(
|
||||
messageIds: string[],
|
||||
workspaceId: string,
|
||||
): Promise<Map<string, AiChatRunStep[]>> {
|
||||
const byMessage = new Map<string, AiChatRunStep[]>();
|
||||
if (messageIds.length === 0) return byMessage;
|
||||
const rows = await this.db
|
||||
.selectFrom('aiChatRunSteps')
|
||||
.selectAll('aiChatRunSteps')
|
||||
.where('messageId', 'in', messageIds)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.orderBy('stepIndex', 'asc')
|
||||
.execute();
|
||||
for (const row of rows) {
|
||||
const list = byMessage.get(row.messageId);
|
||||
if (list) list.push(row);
|
||||
else byMessage.set(row.messageId, [row]);
|
||||
}
|
||||
return byMessage;
|
||||
}
|
||||
}
|
||||
@@ -200,8 +200,11 @@ export class PageEmbeddingRepo {
|
||||
*
|
||||
* The `model_dimensions = $dim` filter applies ONLY on the semantic side
|
||||
* (cosine compares same-dimension vectors; pgvector errors otherwise). The
|
||||
* lexical side (`fts`) is dimension-independent. If `websearch_to_tsquery`
|
||||
* yields an EMPTY query (e.g. the text is all stopwords) the `@@` matches
|
||||
* lexical side (`fts`) is dimension-independent. Its query config is `ru_en`,
|
||||
* matched IN LOCKSTEP with the `page_embeddings.fts` generated column's config
|
||||
* (#529 acceptance #13): a mismatch silently breaks Cyrillic RAG retrieval. If
|
||||
* `websearch_to_tsquery` yields an EMPTY query (e.g. the text is all stopwords)
|
||||
* the `@@` matches
|
||||
* nothing and the lexical CTE is empty, so results degrade to pure-semantic —
|
||||
* which is correct behaviour, not an error.
|
||||
*
|
||||
@@ -249,7 +252,7 @@ export class PageEmbeddingRepo {
|
||||
row_number() OVER (ORDER BY ts_rank(pe.fts, q.query) DESC) AS rank_ix
|
||||
FROM page_embeddings pe
|
||||
JOIN pages p ON p.id = pe.page_id,
|
||||
websearch_to_tsquery('english', f_unaccent(${queryText})) AS q(query)
|
||||
websearch_to_tsquery('ru_en', f_unaccent(${queryText})) AS q(query)
|
||||
WHERE pe.workspace_id = ${workspaceId}
|
||||
AND pe.space_id IN (${spaceList})
|
||||
AND p.deleted_at IS NULL
|
||||
|
||||
-17
@@ -692,22 +692,6 @@ export interface AiChatRuns {
|
||||
updatedAt: Generated<Timestamp>;
|
||||
}
|
||||
|
||||
// Append-only per-step persistence for an assistant turn (#492). Mirrors
|
||||
// migration 20260708T120000-ai-chat-run-steps.ts. Each finished agent step's UI
|
||||
// `parts` are INSERTed as their own row (instead of rewriting the message row's
|
||||
// growing `metadata.parts` jsonb every step — an O(n²) WAL/TOAST churn). The full
|
||||
// `metadata.parts` is assembled once at finalize; mid-run a resuming client's seed
|
||||
// is rebuilt by concatenating these rows in `stepIndex` order. Cascades with the
|
||||
// assistant message row it projects.
|
||||
export interface AiChatRunSteps {
|
||||
id: Generated<string>;
|
||||
messageId: string;
|
||||
workspaceId: string;
|
||||
stepIndex: number;
|
||||
parts: Json;
|
||||
createdAt: Generated<Timestamp>;
|
||||
}
|
||||
|
||||
// Per-(chat,page) snapshot of the open page's Markdown at the END of the agent's
|
||||
// previous turn (#274). Mirrors migration 20260702T120000-ai-chat-page-snapshot.ts.
|
||||
// The next turn diffs the CURRENT Markdown against `contentMd` to surface edits a
|
||||
@@ -745,7 +729,6 @@ export interface DB {
|
||||
aiChats: AiChats;
|
||||
aiChatMessages: AiChatMessages;
|
||||
aiChatRuns: AiChatRuns;
|
||||
aiChatRunSteps: AiChatRunSteps;
|
||||
aiChatPageSnapshots: AiChatPageSnapshots;
|
||||
apiKeys: ApiKeys;
|
||||
attachments: Attachments;
|
||||
|
||||
@@ -4,7 +4,6 @@ import {
|
||||
AiChats,
|
||||
AiChatMessages,
|
||||
AiChatRuns,
|
||||
AiChatRunSteps,
|
||||
AiChatPageSnapshots,
|
||||
Attachments,
|
||||
Comments,
|
||||
@@ -65,12 +64,6 @@ export type InsertableAiChatMessage = Omit<Insertable<AiChatMessages>, 'tsv'>;
|
||||
export type AiChatRun = Selectable<AiChatRuns>;
|
||||
export type InsertableAiChatRun = Insertable<AiChatRuns>;
|
||||
|
||||
// AI Chat Run Step (#492): append-only per-step parts persistence. Each finished
|
||||
// agent step's UI parts are stored as their own row; the full turn's parts are
|
||||
// assembled from these (in stepIndex order) for a mid-run resume seed.
|
||||
export type AiChatRunStep = Selectable<AiChatRunSteps>;
|
||||
export type InsertableAiChatRunStep = Insertable<AiChatRunSteps>;
|
||||
|
||||
// AI Chat Page Snapshot (#274): per-(chat,page) Markdown snapshot taken at the
|
||||
// end of the agent's previous turn, diffed against the current page next turn to
|
||||
// detect human edits made between turns.
|
||||
|
||||
@@ -1,412 +0,0 @@
|
||||
import * as http from 'node:http';
|
||||
import { Kysely } from 'kysely';
|
||||
import { tool } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { MockLanguageModelV3, convertArrayToReadableStream } from 'ai/test';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import {
|
||||
AiChatService,
|
||||
assembleStepParts,
|
||||
assistantParts,
|
||||
rowHasInlineParts,
|
||||
stepMarkerMetadata,
|
||||
} from 'src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
createMessage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — the REAL onStep WRITE path (F2) and the model-REPLAY
|
||||
* hydration path (F1), driven through `AiChatService.stream` against a LIVE
|
||||
* Postgres with a REAL `AiChatRunStepRepo` INJECTED. The existing append-persist
|
||||
* int-specs hand-roll the insert+marker cycle via the repos directly and build
|
||||
* the service with `aiChatRunStepRepo: undefined` (only the legacy-fallback branch
|
||||
* is covered), so an off-by-one on `stepsPersisted-1`, a wrong `capturedSteps`
|
||||
* slice, or a broken marker payload would pass all of them. These tests exercise
|
||||
* the actual `updateStreaming` append-persist branch end to end.
|
||||
*
|
||||
* The seam is the injected `model` (a seeded `MockLanguageModelV3` from `ai/test`)
|
||||
* plus a REAL Node `ServerResponse` as the hijacked socket — mirrors
|
||||
* ai-chat-stream.int-spec.ts.
|
||||
*/
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
async function waitFor(
|
||||
cond: () => Promise<boolean> | boolean,
|
||||
{ timeoutMs = 15_000, stepMs = 25 } = {},
|
||||
): Promise<void> {
|
||||
const start = Date.now();
|
||||
while (Date.now() - start < timeoutMs) {
|
||||
if (await cond()) return;
|
||||
await sleep(stepMs);
|
||||
}
|
||||
throw new Error('waitFor: condition not met within timeout');
|
||||
}
|
||||
|
||||
// A real Node ServerResponse wired to a live socket (identical helper to the
|
||||
// stream int-spec) so the SDK's pipe/heartbeat writes behave as in prod.
|
||||
function makeRealResponse(): Promise<{
|
||||
res: http.ServerResponse;
|
||||
cleanup: () => Promise<void>;
|
||||
}> {
|
||||
return new Promise((resolve) => {
|
||||
const server = http.createServer((_req, res) => {
|
||||
resolve({
|
||||
res,
|
||||
cleanup: () =>
|
||||
new Promise<void>((done) => {
|
||||
try {
|
||||
if (!res.writableEnded) res.end();
|
||||
} catch {
|
||||
/* socket already gone */
|
||||
}
|
||||
server.close(() => done());
|
||||
}),
|
||||
});
|
||||
});
|
||||
server.listen(0, () => {
|
||||
const port = (server.address() as any).port;
|
||||
const creq = http.request({ port, method: 'GET' }, (cres) => {
|
||||
cres.resume();
|
||||
});
|
||||
creq.on('error', () => undefined);
|
||||
creq.end();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Stream parts for a normal, successful single-step turn.
|
||||
function successStream() {
|
||||
return convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: 't1' },
|
||||
{ type: 'text-delta', id: 't1', delta: 'Hello' },
|
||||
{ type: 'text-delta', id: 't1', delta: ' there' },
|
||||
{ type: 'text-end', id: 't1' },
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
},
|
||||
] as any);
|
||||
}
|
||||
|
||||
// A THREE-step turn: steps 0 and 1 each emit text + an `echo` tool call (the SDK
|
||||
// runs the tool and continues); step 2 answers and stops. Three steps is
|
||||
// deliberate: the LAST finished step's append-persist write races the terminal
|
||||
// finalize (which writes the full inline parts anyway, so a lost last-step row is
|
||||
// by design), but the NON-final steps 0 and 1 always drain to the steps table
|
||||
// before finalize — so those are what the test asserts on deterministically.
|
||||
function threeStepModel(): MockLanguageModelV3 {
|
||||
let step = 0;
|
||||
const toolStep = (i: number) => ({
|
||||
stream: convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: `s${i}` },
|
||||
{ type: 'text-delta', id: `s${i}`, delta: `step ${i} ` },
|
||||
{ type: 'text-end', id: `s${i}` },
|
||||
{
|
||||
type: 'tool-call',
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'echo',
|
||||
input: JSON.stringify({ msg: `m${i}` }),
|
||||
},
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'tool-calls',
|
||||
usage: { inputTokens: 5, outputTokens: 3, totalTokens: 8 },
|
||||
},
|
||||
] as any),
|
||||
});
|
||||
return new MockLanguageModelV3({
|
||||
doStream: async () => {
|
||||
const n = step++;
|
||||
// Realistic inter-step latency. A real model spends seconds per step, so the
|
||||
// fire-and-forget per-step write chain drains to the steps table BETWEEN
|
||||
// steps; the mock otherwise collapses all steps into microseconds and the
|
||||
// terminal finalize wins the race before any but the first step persists.
|
||||
if (n > 0) await sleep(200);
|
||||
if (n < 2) return toolStep(n);
|
||||
return {
|
||||
stream: convertArrayToReadableStream([
|
||||
{ type: 'stream-start', warnings: [] },
|
||||
{ type: 'text-start', id: 's2' },
|
||||
{ type: 'text-delta', id: 's2', delta: 'final answer' },
|
||||
{ type: 'text-end', id: 's2' },
|
||||
{
|
||||
type: 'finish',
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 6, outputTokens: 4, totalTokens: 10 },
|
||||
},
|
||||
] as any),
|
||||
};
|
||||
},
|
||||
} as any);
|
||||
}
|
||||
|
||||
describe('#492 append-persist service paths [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let aiChatRepo: AiChatRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
|
||||
let closeCalls: number;
|
||||
const mcpClients = {
|
||||
toolsFor: async () => ({
|
||||
tools: {},
|
||||
clients: [
|
||||
{
|
||||
close: async () => {
|
||||
closeCalls += 1;
|
||||
},
|
||||
},
|
||||
],
|
||||
outcomes: [],
|
||||
instructions: [],
|
||||
}),
|
||||
};
|
||||
|
||||
// Build the service WITH a REAL AiChatRunStepRepo injected (the property under
|
||||
// test) — unlike the legacy-fallback harness that passes it as undefined.
|
||||
const echoTool = tool({
|
||||
description: 'echo the message back',
|
||||
inputSchema: z.object({ msg: z.string() }),
|
||||
execute: async ({ msg }) => ({ echoed: msg }),
|
||||
});
|
||||
|
||||
function buildService(): AiChatService {
|
||||
return new AiChatService(
|
||||
{ getChatModel: async () => null } as any,
|
||||
aiChatRepo,
|
||||
msgRepo,
|
||||
{} as any, // aiChatPageSnapshotRepo
|
||||
{ resolve: async () => null } as any, // aiSettings
|
||||
{ forUser: async () => ({ echo: echoTool }) } as any, // tools
|
||||
mcpClients as any,
|
||||
{} as any, // aiAgentRoleRepo
|
||||
{} as any, // pageRepo
|
||||
{} as any, // pageAccess
|
||||
{
|
||||
isAiChatDeferredToolsEnabled: () => false,
|
||||
isAiChatFinalStepLockdownEnabled: () => false,
|
||||
} as any, // environment (deferred OFF -> all tools active every step)
|
||||
undefined, // streamRegistry
|
||||
undefined, // aiChatRunService
|
||||
stepRepo, // #492 aiChatRunStepRepo — the append-persist backend
|
||||
);
|
||||
}
|
||||
|
||||
function userUiMessage(text: string) {
|
||||
return {
|
||||
id: `u-${Math.random()}`,
|
||||
role: 'user',
|
||||
parts: [{ type: 'text', text }],
|
||||
};
|
||||
}
|
||||
|
||||
async function runStream(opts: {
|
||||
model: MockLanguageModelV3;
|
||||
chatId: string;
|
||||
body: any;
|
||||
}): Promise<void> {
|
||||
closeCalls = 0;
|
||||
const service = buildService();
|
||||
const { res, cleanup } = await makeRealResponse();
|
||||
try {
|
||||
await service.stream({
|
||||
user: { id: userId, workspaceId } as any,
|
||||
workspace: { id: workspaceId, name: 'WS' } as any,
|
||||
sessionId: 'sess-1',
|
||||
body: opts.body,
|
||||
res: { raw: res } as any,
|
||||
signal: new AbortController().signal,
|
||||
model: opts.model as any,
|
||||
role: null,
|
||||
} as any);
|
||||
await waitFor(async () => {
|
||||
const rows = await msgRepo.findAllByChat(opts.chatId, workspaceId);
|
||||
return rows.some(
|
||||
(r) =>
|
||||
r.role === 'assistant' &&
|
||||
['completed', 'error', 'aborted'].includes(r.status as string),
|
||||
);
|
||||
});
|
||||
await waitFor(() => closeCalls > 0, { timeoutMs: 5_000 });
|
||||
} finally {
|
||||
await cleanup();
|
||||
}
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
aiChatRepo = new AiChatRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
// --- F2: the real onStep append-persist WRITE branch -----------------------
|
||||
it('drives steps through the real onStep path: per-step rows + marker match a single-row flush', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
const model = threeStepModel();
|
||||
|
||||
// Capture the mid-run step-marker UPDATEs the append-persist branch writes on
|
||||
// the assistant row (a { parts: [], toolTraceVersion, stepsPersisted } patch).
|
||||
const updateSpy = jest.spyOn(msgRepo, 'update');
|
||||
try {
|
||||
await runStream({
|
||||
model,
|
||||
chatId,
|
||||
body: { chatId, messages: [userUiMessage('call the tool then answer')] },
|
||||
});
|
||||
|
||||
const rows = await msgRepo.findAllByChat(chatId, workspaceId);
|
||||
const assistant = rows.find((r) => r.role === 'assistant')!;
|
||||
expect(assistant).toBeDefined();
|
||||
expect(assistant.status).toBe('completed');
|
||||
// The turn finalizes with the FULL inline parts assembled by a single-row
|
||||
// flush (assistantParts over every step) — the baseline the per-step slices
|
||||
// must reproduce.
|
||||
expect(rowHasInlineParts(assistant)).toBe(true);
|
||||
const finalParts = (assistant.metadata as { parts: any[] }).parts;
|
||||
|
||||
// The two NON-final finished steps each landed their own row, in stepIndex
|
||||
// order. (The fire-and-forget write chain drains before the next step, so
|
||||
// poll until both are on disk; the LAST step's write may lose the finalize
|
||||
// race, which is by design — its parts are already in `finalParts`.)
|
||||
await waitFor(async () => {
|
||||
const s = await stepRepo.findByMessage(assistant.id, workspaceId);
|
||||
return s.length >= 2;
|
||||
});
|
||||
const steps = await stepRepo.findByMessage(assistant.id, workspaceId);
|
||||
expect(steps[0].stepIndex).toBe(0);
|
||||
expect(steps[1].stepIndex).toBe(1);
|
||||
|
||||
// Each per-step row carries a NON-trivial slice: this step's text part + its
|
||||
// paired tool part (guards a mutation that persists empty/whole-turn parts).
|
||||
const s0 = steps[0].parts as any[];
|
||||
expect(s0).toContainEqual({ type: 'text', text: 'step 0 ' });
|
||||
expect(s0.some((p) => p.type === 'tool-echo')).toBe(true);
|
||||
|
||||
// The per-step slices are EXACTLY the corresponding prefix of the single-row
|
||||
// flush: assembleStepParts([step0, step1]) === finalParts[0 .. len0+len1].
|
||||
// This is what an off-by-one on `stepsPersisted-1` (a wrong `capturedSteps`
|
||||
// slice) or a shifted stepIndex breaks — the prefix no longer aligns.
|
||||
const prefixLen =
|
||||
(steps[0].parts as any[]).length + (steps[1].parts as any[]).length;
|
||||
expect(assembleStepParts([steps[0], steps[1]] as any)).toEqual(
|
||||
finalParts.slice(0, prefixLen),
|
||||
);
|
||||
|
||||
// The mid-run step markers advanced 1 -> 2 -> ... (the resume frontier), each
|
||||
// a shape-stable empty-parts marker equal to a single-row flush's marker.
|
||||
const markerCounts = updateSpy.mock.calls
|
||||
.map((c) => (c[2] as any)?.metadata)
|
||||
.filter(
|
||||
(m) =>
|
||||
m &&
|
||||
Array.isArray(m.parts) &&
|
||||
m.parts.length === 0 &&
|
||||
typeof m.stepsPersisted === 'number',
|
||||
)
|
||||
.map((m) => m.stepsPersisted);
|
||||
// Monotonic from 1, covering at least the two non-final steps.
|
||||
expect(markerCounts.slice(0, 2)).toEqual([1, 2]);
|
||||
expect(
|
||||
updateSpy.mock.calls
|
||||
.map((c) => (c[2] as any)?.metadata)
|
||||
.find((m) => m && m.stepsPersisted === 2),
|
||||
).toEqual(stepMarkerMetadata(2));
|
||||
} finally {
|
||||
updateSpy.mockRestore();
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
// --- F1: model-REPLAY hydrates a hard-crashed mid-run turn from the steps table
|
||||
it('replays a hard-crashed mid-run turn WITH its partial steps hydrated from the steps table', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
|
||||
// Prior turn: a genuine user question...
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
userId,
|
||||
role: 'user',
|
||||
content: 'What is in the design doc?',
|
||||
createdAt: new Date(Date.now() - 3000),
|
||||
});
|
||||
// ...and an assistant row that a HARD crash (SIGKILL/OOM) left mid-run: only a
|
||||
// step marker on the row (metadata.parts:[] , content:''), NO terminal
|
||||
// callback ever fired, so its real parts live ONLY in ai_chat_run_steps.
|
||||
const crashed = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'aborted',
|
||||
metadata: stepMarkerMetadata(1),
|
||||
createdAt: new Date(Date.now() - 2000),
|
||||
});
|
||||
// The durable partial step: some reasoning text + a completed getPage tool
|
||||
// call (input + output), exactly what #183 step-granular durability preserves.
|
||||
await stepRepo.insertStep(
|
||||
crashed.id,
|
||||
workspaceId,
|
||||
0,
|
||||
assistantParts(
|
||||
[
|
||||
{
|
||||
text: 'HYDRATED_PARTIAL_STEP the doc says',
|
||||
toolCalls: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', input: { id: 'p1' } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: 'g1',
|
||||
toolName: 'getPage',
|
||||
output: { id: 'p1', body: 'PARTIAL_TOOL_OUTPUT budget section' },
|
||||
},
|
||||
],
|
||||
} as any,
|
||||
],
|
||||
'',
|
||||
),
|
||||
);
|
||||
|
||||
// The NEXT turn: the model just answers. The service must REPLAY the crashed
|
||||
// assistant turn with its partial parts hydrated from the steps table.
|
||||
const model = new MockLanguageModelV3({
|
||||
doStream: async () => ({ stream: successStream() }),
|
||||
} as any);
|
||||
await runStream({
|
||||
model,
|
||||
chatId,
|
||||
body: { chatId, messages: [userUiMessage('Continue please')] },
|
||||
});
|
||||
|
||||
expect(model.doStreamCalls.length).toBeGreaterThan(0);
|
||||
const prompt = JSON.stringify(model.doStreamCalls[0].prompt);
|
||||
// The partial step's TEXT reached the model context (it would be an empty text
|
||||
// part without hydration — rowToUiMessage falls back to `content:''`).
|
||||
expect(prompt).toContain('HYDRATED_PARTIAL_STEP');
|
||||
// The partial TOOL RESULT survived too (durable in the steps table, replayed).
|
||||
expect(prompt).toContain('PARTIAL_TOOL_OUTPUT');
|
||||
// The genuine prior user turn is present as well (sanity: real history replay).
|
||||
expect(prompt).toContain('What is in the design doc?');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -1,173 +0,0 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import {
|
||||
assistantParts,
|
||||
flushAssistant,
|
||||
stepMarkerMetadata,
|
||||
} from '../../src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — WRITE-VOLUME regression on a LIVE Postgres, measured via
|
||||
* the `pg_current_wal_lsn()` delta around a realistic multi-step run driven through
|
||||
* the REAL repos (not a mock — a mock cannot observe MVCC/TOAST rewrite volume, the
|
||||
* whole point). Proves the core claim:
|
||||
*
|
||||
* NEW (per-step INSERT into ai_chat_run_steps + a CHEAP step-marker UPDATE on the
|
||||
* message row) writes O(Σ steps) of WAL — each step writes only its own bytes.
|
||||
*
|
||||
* OLD (the pre-#492 full-row rewrite: re-persist the GROWING metadata.parts on
|
||||
* every onStepFinish) writes O(n²) — step k rewrites the whole TOASTed jsonb of
|
||||
* all k prior outputs.
|
||||
*
|
||||
* The OLD path here IS the reverted behavior, so this doubles as the mutation
|
||||
* check: swapping the new path back to `flushAssistant` full-row UPDATEs reddens
|
||||
* the assertion (OLD is many times larger).
|
||||
*/
|
||||
type Step = {
|
||||
text: string;
|
||||
toolCalls: Array<{ toolCallId: string; toolName: string; input: unknown }>;
|
||||
toolResults: Array<{ toolCallId: string; toolName: string; output: unknown }>;
|
||||
};
|
||||
|
||||
// ~100 KB INCOMPRESSIBLE output per step (a page read). Random base64 so TOAST
|
||||
// cannot compress it away and hide the real write volume.
|
||||
function makeStep(i: number, outputBytes = 100_000): Step {
|
||||
const body = randomBytes(Math.ceil(outputBytes * 0.75)).toString('base64');
|
||||
return {
|
||||
text: `step ${i} reasoning`,
|
||||
toolCalls: [
|
||||
{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'getPage',
|
||||
output: { id: `p${i}`, title: `Page ${i}`, body },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async function walDelta(
|
||||
db: Kysely<any>,
|
||||
fn: () => Promise<void>,
|
||||
): Promise<number> {
|
||||
const before = (
|
||||
await sql<{ l: string }>`select pg_current_wal_lsn() as l`.execute(db)
|
||||
).rows[0].l;
|
||||
await fn();
|
||||
// NOTE: no pg_switch_wal() — a segment switch pads the LSN to the next 16 MB
|
||||
// boundary and would swamp the delta. The raw LSN advances by the WAL bytes.
|
||||
const after = (
|
||||
await sql<{ l: string }>`select pg_current_wal_lsn() as l`.execute(db)
|
||||
).rows[0].l;
|
||||
return Number(
|
||||
(
|
||||
await sql<{
|
||||
d: string;
|
||||
}>`select pg_wal_lsn_diff(${after}::pg_lsn, ${before}::pg_lsn) as d`.execute(
|
||||
db,
|
||||
)
|
||||
).rows[0].d,
|
||||
);
|
||||
}
|
||||
|
||||
describe('#492 append-persist write volume (pg_current_wal_lsn delta) [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
let chatId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
const seedRow = () =>
|
||||
msgRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'streaming',
|
||||
metadata: stepMarkerMetadata(0) as never,
|
||||
});
|
||||
|
||||
const STEPS = 40;
|
||||
|
||||
it('NEW per-step INSERT is O(Σ steps); OLD full-row rewrite is O(n²)', async () => {
|
||||
const steps: Step[] = [];
|
||||
for (let i = 0; i < STEPS; i++) steps.push(makeStep(i));
|
||||
|
||||
// NEW: per-step INSERT of THIS step's parts + a cheap marker UPDATE.
|
||||
const newRow = await seedRow();
|
||||
const newWal = await walDelta(db, async () => {
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
await stepRepo.insertStep(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
i,
|
||||
assistantParts([steps[i]], ''),
|
||||
);
|
||||
await msgRepo.update(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
{ metadata: stepMarkerMetadata(i + 1) },
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// OLD (the pre-#492 revert): re-persist the GROWING metadata.parts on the
|
||||
// message row on every step.
|
||||
const oldRow = await seedRow();
|
||||
const oldWal = await walDelta(db, async () => {
|
||||
const acc: Step[] = [];
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
acc.push(steps[i]);
|
||||
await msgRepo.update(
|
||||
oldRow.id,
|
||||
workspaceId,
|
||||
flushAssistant(acc as never, '', 'streaming'),
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[#492 WAL] ${STEPS} steps ×100KB: new=${(newWal / 1e6).toFixed(1)}MB ` +
|
||||
`old=${(oldWal / 1e6).toFixed(1)}MB (${(oldWal / newWal).toFixed(
|
||||
1,
|
||||
)}x smaller)`,
|
||||
);
|
||||
|
||||
// O(Σ steps): ~STEPS × (100KB output + marker) of WAL. 40 × ~100KB parts plus
|
||||
// 40 tiny markers is a few tens of MB at most — bounded, linear in step count.
|
||||
expect(newWal).toBeLessThan(30_000_000);
|
||||
// O(n²): step k rewrites ~k × 100KB. Σ over 40 steps ≈ 80+ MB — far larger.
|
||||
expect(oldWal).toBeGreaterThan(30_000_000);
|
||||
// The load-bearing claim: the new path writes a small FRACTION of the old.
|
||||
expect(newWal).toBeLessThan(oldWal * 0.35);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -1,169 +0,0 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { AiChatController } from 'src/core/ai-chat/ai-chat.controller';
|
||||
import {
|
||||
assembleStepParts,
|
||||
assistantParts,
|
||||
stepMarkerMetadata,
|
||||
} from 'src/core/ai-chat/ai-chat.service';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import type { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
createMessage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 controller hydration (crash-before-finalize RESUME) on a LIVE Postgres.
|
||||
* `AiChatController.withReconstructedParts` is wired into getMessages/delta/export/
|
||||
* run, but `aiChatRunStepRepo` is OPTIONAL and every controller unit spec passes it
|
||||
* as `undefined`, so the hydration branch early-returns and NEVER executes in those
|
||||
* tests. This drives the real read path — a mid-run streaming row (marker only,
|
||||
* empty inline parts) PLUS its `ai_chat_run_steps` rows — through getMessages WITH
|
||||
* the repo present, exercising the `role==='assistant' && !rowHasInlineParts`
|
||||
* needy predicate, the workspace-scoped batch step fetch, and the endpoint binding.
|
||||
*/
|
||||
describe('#492 controller hydration read path [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let aiChatRepo: AiChatRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let workspaceId: string;
|
||||
let otherWorkspaceId: string;
|
||||
let userId: string;
|
||||
|
||||
// Build the controller WITH a real AiChatRunStepRepo injected (position 9), the
|
||||
// seam the unit specs leave undefined. Only the read-path deps are real.
|
||||
function buildController(): AiChatController {
|
||||
return new AiChatController(
|
||||
{} as any, // aiChatService
|
||||
{} as any, // aiChatRunService
|
||||
aiChatRepo,
|
||||
msgRepo,
|
||||
{} as any, // aiTranscription
|
||||
{} as any, // pageRepo
|
||||
undefined, // streamRegistry
|
||||
undefined, // environment
|
||||
stepRepo, // #492 aiChatRunStepRepo
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
aiChatRepo = new AiChatRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
otherWorkspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('getMessages reconstructs a mid-run row from the steps table (finished rows untouched)', async () => {
|
||||
const chatId = (
|
||||
await createChat(db, { workspaceId, creatorId: userId })
|
||||
).id;
|
||||
const user = { id: userId } as User;
|
||||
const workspace = { id: workspaceId } as Workspace;
|
||||
|
||||
// A prior FINISHED assistant row that already carries inline parts — the needy
|
||||
// predicate must SKIP it (no step fetch), returned untouched.
|
||||
const finishedParts = assistantParts(
|
||||
[{ text: 'done earlier', toolCalls: [], toolResults: [] } as any],
|
||||
'',
|
||||
);
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: 'done earlier',
|
||||
status: 'completed',
|
||||
metadata: { parts: finishedParts, toolTraceVersion: 2, stepsPersisted: 1 },
|
||||
createdAt: new Date(Date.now() - 3000),
|
||||
});
|
||||
|
||||
// The mid-run row a crash-before-finalize left behind: a step marker only
|
||||
// (parts:[] , content:''), status 'streaming'. Its real parts live ONLY in the
|
||||
// steps table.
|
||||
const midRun = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'streaming',
|
||||
metadata: stepMarkerMetadata(2),
|
||||
createdAt: new Date(Date.now() - 1000),
|
||||
});
|
||||
|
||||
const step0 = assistantParts(
|
||||
[
|
||||
{
|
||||
text: 'reasoning about the page',
|
||||
toolCalls: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', input: { id: 'p1' } },
|
||||
],
|
||||
toolResults: [
|
||||
{ toolCallId: 'g1', toolName: 'getPage', output: { id: 'p1', body: 'B' } },
|
||||
],
|
||||
} as any,
|
||||
],
|
||||
'',
|
||||
);
|
||||
const step1 = assistantParts(
|
||||
[{ text: 'partial synthesis so far', toolCalls: [], toolResults: [] } as any],
|
||||
'',
|
||||
);
|
||||
await stepRepo.insertStep(midRun.id, workspaceId, 0, step0);
|
||||
await stepRepo.insertStep(midRun.id, workspaceId, 1, step1);
|
||||
|
||||
// Workspace-scoping guard: a step row for the SAME message id under a DIFFERENT
|
||||
// workspace must NEVER leak into this workspace's reconstruction.
|
||||
await stepRepo.insertStep(midRun.id, otherWorkspaceId, 99, [
|
||||
{ type: 'text', text: 'FOREIGN_WORKSPACE_LEAK' },
|
||||
]);
|
||||
|
||||
const res = await buildController().getMessages(
|
||||
{ chatId } as any,
|
||||
{ limit: 50 } as any,
|
||||
user,
|
||||
workspace,
|
||||
);
|
||||
|
||||
const items = res.items as any[];
|
||||
const finished = items.find((r) => r.status === 'completed');
|
||||
const reconstructed = items.find((r) => r.id === midRun.id);
|
||||
|
||||
// The finished row passed through with its inline parts unchanged.
|
||||
expect(finished.metadata.parts).toEqual(finishedParts);
|
||||
|
||||
// The mid-run row's parts were reconstructed from the two step rows, in order,
|
||||
// exactly as assembleStepParts concatenates them — the client seed sees the
|
||||
// persisted progress with no change to itself.
|
||||
const expected = assembleStepParts([
|
||||
{ stepIndex: 0, parts: step0 },
|
||||
{ stepIndex: 1, parts: step1 },
|
||||
] as any);
|
||||
expect(reconstructed.metadata.parts).toEqual(expected);
|
||||
// The foreign-workspace step row did NOT leak in.
|
||||
expect(JSON.stringify(reconstructed.metadata.parts)).not.toContain(
|
||||
'FOREIGN_WORKSPACE_LEAK',
|
||||
);
|
||||
// Sanity: reconstruction produced real content (text + the paired tool part +
|
||||
// the second step's text), not an empty fallback.
|
||||
expect(reconstructed.metadata.parts).toContainEqual({
|
||||
type: 'text',
|
||||
text: 'reasoning about the page',
|
||||
});
|
||||
expect(
|
||||
(reconstructed.metadata.parts as any[]).some((p) => p.type === 'tool-getPage'),
|
||||
).toBe(true);
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -1,163 +0,0 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { Kysely } from 'kysely';
|
||||
import { AiChatRunStepRepo } from '@docmost/db/repos/ai-chat/ai-chat-run-step.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import {
|
||||
assistantParts,
|
||||
reconstructRunParts,
|
||||
hydrateAssistantParts,
|
||||
stepMarkerMetadata,
|
||||
rowHasInlineParts,
|
||||
} from '../../src/core/ai-chat/ai-chat.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #492 append-persist — the reconstruct CONTRACT on a live Postgres. Proves that a
|
||||
* turn persisted the NEW way (per-step rows in `ai_chat_run_steps`, only a step
|
||||
* marker on the message row) reconstructs to the SAME UI parts as a turn persisted
|
||||
* the OLD way (full `metadata.parts` inline on the row, no step rows) — so the
|
||||
* era-switch is invisible to attach / delta-poll / export. Real repos + real jsonb
|
||||
* roundtrip, not a mock (a mock cannot prove the parts survive the jsonb column
|
||||
* byte-identical).
|
||||
*/
|
||||
type Step = {
|
||||
text: string;
|
||||
toolCalls: Array<{ toolCallId: string; toolName: string; input: unknown }>;
|
||||
toolResults: Array<{ toolCallId: string; toolName: string; output: unknown }>;
|
||||
};
|
||||
|
||||
// A realistic step: some text + a getPage tool call whose ~100 KB body is
|
||||
// INCOMPRESSIBLE random base64 (a 'x'.repeat filler would TOAST away and hide the
|
||||
// real bytes). Under MAX_TOOL_OUTPUT_BYTES (200 KB) it is stored uncompacted.
|
||||
function makeStep(i: number, outputBytes = 4_000): Step {
|
||||
const body = randomBytes(Math.ceil(outputBytes * 0.75)).toString('base64');
|
||||
return {
|
||||
text: `step ${i} text`,
|
||||
toolCalls: [
|
||||
{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } },
|
||||
],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'getPage',
|
||||
output: { id: `p${i}`, title: `Page ${i}`, body },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('AiChatRunStepRepo + reconstruct contract [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let stepRepo: AiChatRunStepRepo;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
let chatId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
stepRepo = new AiChatRunStepRepo(db as any);
|
||||
msgRepo = new AiChatMessageRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
const seedRow = (metadata: unknown, status: string) =>
|
||||
msgRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status,
|
||||
metadata: metadata as never,
|
||||
});
|
||||
|
||||
it('insertStep is idempotent per (message, stepIndex) and reads back in order', async () => {
|
||||
const row = await seedRow(stepMarkerMetadata(0), 'streaming');
|
||||
const parts0 = assistantParts([makeStep(0)], '');
|
||||
const parts1 = assistantParts([makeStep(1)], '');
|
||||
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 0, parts0)).toBe(true);
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 1, parts1)).toBe(true);
|
||||
// A retried persist of the SAME step is a no-op (ON CONFLICT DO NOTHING).
|
||||
expect(await stepRepo.insertStep(row.id, workspaceId, 0, parts0)).toBe(
|
||||
false,
|
||||
);
|
||||
|
||||
const steps = await stepRepo.findByMessage(row.id, workspaceId);
|
||||
expect(steps.map((s) => s.stepIndex)).toEqual([0, 1]);
|
||||
|
||||
// Batch fetch groups by message id in step order.
|
||||
const map = await stepRepo.findByMessageIds([row.id], workspaceId);
|
||||
expect(map.get(row.id)!.map((s) => s.stepIndex)).toEqual([0, 1]);
|
||||
});
|
||||
|
||||
it('a NEW-style (step-table) run reconstructs identically to an OLD-style (inline) run', async () => {
|
||||
const steps = [makeStep(10), makeStep(11)];
|
||||
// The inline parts the OLD full-row flush would have written.
|
||||
const fullParts = assistantParts(steps, '');
|
||||
|
||||
// OLD-style record: full parts inline on the row, NO step rows.
|
||||
const oldRow = await seedRow(
|
||||
{ parts: fullParts, toolTraceVersion: 2, stepsPersisted: 2 },
|
||||
'completed',
|
||||
);
|
||||
|
||||
// NEW-style record: only a step marker on the row + per-step rows.
|
||||
const newRow = await seedRow(stepMarkerMetadata(2), 'streaming');
|
||||
for (let i = 0; i < steps.length; i++) {
|
||||
await stepRepo.insertStep(
|
||||
newRow.id,
|
||||
workspaceId,
|
||||
i,
|
||||
assistantParts([steps[i]], ''),
|
||||
);
|
||||
}
|
||||
|
||||
// Re-read both from the DB (proves the jsonb roundtrip).
|
||||
const oldFetched = await msgRepo.findById(oldRow.id, workspaceId);
|
||||
const newFetched = await msgRepo.findById(newRow.id, workspaceId);
|
||||
const oldSteps = await stepRepo.findByMessage(oldRow.id, workspaceId);
|
||||
const newSteps = await stepRepo.findByMessage(newRow.id, workspaceId);
|
||||
|
||||
// The discriminator: the old row carries inline parts, the new one does not.
|
||||
expect(rowHasInlineParts(oldFetched!)).toBe(true);
|
||||
expect(rowHasInlineParts(newFetched!)).toBe(false);
|
||||
expect(oldSteps).toHaveLength(0);
|
||||
expect(newSteps).toHaveLength(2);
|
||||
|
||||
const oldRecon = reconstructRunParts(oldFetched!, oldSteps);
|
||||
const newRecon = reconstructRunParts(newFetched!, newSteps);
|
||||
|
||||
// Both reconstruct to the SAME parts + step count — the era is invisible.
|
||||
expect(newRecon.parts).toEqual(fullParts);
|
||||
expect(oldRecon.parts).toEqual(fullParts);
|
||||
expect(newRecon.parts).toEqual(oldRecon.parts);
|
||||
expect(newRecon.stepsPersisted).toBe(2);
|
||||
expect(oldRecon.stepsPersisted).toBe(2);
|
||||
|
||||
// hydrateAssistantParts fills the new row's metadata.parts to match the old
|
||||
// row's inline parts — so a consumer reading `metadata.parts` off the raw row
|
||||
// (the client seed/poll, export) is unchanged across the era.
|
||||
const map = await stepRepo.findByMessageIds([newRow.id], workspaceId);
|
||||
const [hydrated] = hydrateAssistantParts([newFetched!], map);
|
||||
expect((hydrated.metadata as { parts: unknown }).parts).toEqual(fullParts);
|
||||
// A row that already has inline parts passes through untouched (same ref-shape).
|
||||
const [oldPassThrough] = hydrateAssistantParts([oldFetched!], map);
|
||||
expect((oldPassThrough.metadata as { parts: unknown }).parts).toEqual(
|
||||
fullParts,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,63 +0,0 @@
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import {
|
||||
up,
|
||||
down,
|
||||
} from '../../src/database/migrations/20260708T120000-ai-chat-run-steps';
|
||||
import { getTestDb, destroyTestDb } from './db';
|
||||
|
||||
/**
|
||||
* #492 migration up/down roundtrip on a LIVE Postgres. global-setup already
|
||||
* migrated docmost_test to latest (so the table exists at start); this drives the
|
||||
* migration's own down()/up() and asserts the table presence toggles, then leaves
|
||||
* it PRESENT (up) so the shared test DB is intact for any spec that runs after.
|
||||
*/
|
||||
async function tableExists(db: Kysely<any>): Promise<boolean> {
|
||||
const row = (
|
||||
await sql<{ t: string | null }>`select to_regclass('ai_chat_run_steps') as t`.execute(
|
||||
db,
|
||||
)
|
||||
).rows[0];
|
||||
return row.t !== null;
|
||||
}
|
||||
|
||||
async function uniqueIndexExists(db: Kysely<any>): Promise<boolean> {
|
||||
const row = (
|
||||
await sql<{
|
||||
t: string | null;
|
||||
}>`select to_regclass('ai_chat_run_steps_message_step_uidx') as t`.execute(db)
|
||||
).rows[0];
|
||||
return row.t !== null;
|
||||
}
|
||||
|
||||
describe('20260708 ai_chat_run_steps migration roundtrip [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
|
||||
beforeAll(() => {
|
||||
db = getTestDb();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Belt-and-suspenders: guarantee the table is present for later specs even if
|
||||
// an assertion threw mid-roundtrip.
|
||||
if (!(await tableExists(db))) await up(db);
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('down() drops the table+index and up() recreates them (idempotent)', async () => {
|
||||
// Starts applied (global-setup migrated to latest).
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
expect(await uniqueIndexExists(db)).toBe(true);
|
||||
|
||||
await down(db);
|
||||
expect(await tableExists(db)).toBe(false);
|
||||
expect(await uniqueIndexExists(db)).toBe(false);
|
||||
|
||||
await up(db);
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
expect(await uniqueIndexExists(db)).toBe(true);
|
||||
|
||||
// up() is idempotent (ifNotExists) — a second run is a harmless no-op.
|
||||
await expect(up(db)).resolves.not.toThrow();
|
||||
expect(await tableExists(db)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,464 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { SearchService } from 'src/core/search/search.service';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createSpace,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #529 Phase A — the lexical overhaul, on the REAL migrated schema (ru_en config).
|
||||
*
|
||||
* Covers every acceptance criterion of the issue: RU+EN morphology + OR default,
|
||||
* match=auto identifier routing, "phrase"/+/- operators, RRF ordering, exact
|
||||
* permission-filtered total (fail-closed) + pagination, only-negation / garbage
|
||||
* short-circuits, the A8 path fix, the response superset, and the RAG lockstep
|
||||
* config (acceptance #13).
|
||||
*
|
||||
* The tsv column is populated by the pages_tsvector_trigger (now ru_en), so the
|
||||
* FTS branch is exercised end to end.
|
||||
*/
|
||||
describe('SearchService #529 lexical overhaul [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let workspaceId: string;
|
||||
let spaceId: string;
|
||||
|
||||
async function insertPage(args: {
|
||||
title: string;
|
||||
textContent?: string;
|
||||
parentPageId?: string | null;
|
||||
spaceId?: string;
|
||||
deletedAt?: Date | null;
|
||||
}): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await db
|
||||
.insertInto('pages')
|
||||
.values({
|
||||
id,
|
||||
slugId: `slug-${id.slice(0, 12)}`,
|
||||
title: args.title,
|
||||
textContent: args.textContent ?? null,
|
||||
parentPageId: args.parentPageId ?? null,
|
||||
spaceId: args.spaceId ?? spaceId,
|
||||
workspaceId,
|
||||
deletedAt: args.deletedAt ?? null,
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
// Service wired to the real DB + real PageRepo (recursive descendants) with
|
||||
// stubbed space-membership + permission repos so a test controls scope and the
|
||||
// permission filter explicitly. `accessibleIds` (when set) is the KEEP list.
|
||||
function buildService(opts?: {
|
||||
userSpaceIds?: string[];
|
||||
accessibleIds?: string[] | null;
|
||||
filterThrows?: boolean;
|
||||
}): SearchService {
|
||||
const pageRepo = new PageRepo(db as any, null as any, null as any);
|
||||
const spaceMemberRepo = {
|
||||
getUserSpaceIds: async () => opts?.userSpaceIds ?? [spaceId],
|
||||
};
|
||||
const pagePermissionRepo = {
|
||||
hasRestrictedAncestor: async () => false,
|
||||
filterAccessiblePageIds: async ({ pageIds }: { pageIds: string[] }) => {
|
||||
if (opts?.filterThrows) throw new Error('permission query failed');
|
||||
return opts?.accessibleIds
|
||||
? pageIds.filter((id) => opts.accessibleIds!.includes(id))
|
||||
: pageIds;
|
||||
},
|
||||
};
|
||||
return new SearchService(
|
||||
db as any,
|
||||
pageRepo as any,
|
||||
{} as any,
|
||||
spaceMemberRepo as any,
|
||||
pagePermissionRepo as any,
|
||||
);
|
||||
}
|
||||
|
||||
const search = (service: SearchService, params: any) =>
|
||||
service.searchPage(params, { userId: 'u-1', workspaceId }) as any;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
spaceId = (await createSpace(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
// 1. RU morphology + OR: «ресторанов москвы» finds «ресторан в москве».
|
||||
it('#1 russian morphology + OR: finds «ресторан в москве» for «ресторанов москвы»', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'ресторан в москве',
|
||||
textContent: 'Лучший ресторан столицы.',
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: 'ресторанов москвы',
|
||||
spaceId,
|
||||
});
|
||||
expect(res.items.map((i: any) => i.id)).toContain(page);
|
||||
expect(res.total).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
// 2. OR non-empty: «Стамбул Роснефть».
|
||||
it('#2 OR yields a hit when only one term matches', async () => {
|
||||
const page = await insertPage({ title: 'Роснефть отчёт', textContent: 'x' });
|
||||
const res = await search(buildService(), {
|
||||
query: 'Стамбул Роснефть',
|
||||
spaceId,
|
||||
});
|
||||
expect(res.items.map((i: any) => i.id)).toContain(page);
|
||||
});
|
||||
|
||||
// 3. Multi-word OR: a page matching >=1 term is returned.
|
||||
it('#3 «3D принтер» returns pages that matched at least one term', async () => {
|
||||
const models = await insertPage({
|
||||
title: 'Модели для печати 3D',
|
||||
textContent: 'коллекция моделей',
|
||||
});
|
||||
const wish = await insertPage({
|
||||
title: 'Хотеть напечатать на принтере',
|
||||
textContent: 'очередь печати',
|
||||
});
|
||||
const res = await search(buildService(), { query: '3D принтер', spaceId });
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(models); // matched "3D"
|
||||
expect(ids).toContain(wish); // matched "принтер"
|
||||
// matchedTerms is populated per hit.
|
||||
const hit = res.items.find((i: any) => i.id === wish);
|
||||
expect(hit.matchedTerms).toContain('принтер');
|
||||
});
|
||||
|
||||
// 4. match=auto FTS stemming: «печат» must NOT drag in «впечатления».
|
||||
it('#4 «печат» (auto) matches «печать» but NOT «впечатления»', async () => {
|
||||
const good = await insertPage({
|
||||
title: 'Печать документов',
|
||||
textContent: 'настройка печати',
|
||||
});
|
||||
const bad = await insertPage({
|
||||
title: 'Впечатления от поездки',
|
||||
textContent: 'много впечатлений',
|
||||
});
|
||||
const res = await search(buildService(), { query: 'печат', spaceId });
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(good);
|
||||
expect(ids).not.toContain(bad);
|
||||
});
|
||||
|
||||
// 5. Identifier → substring branch.
|
||||
it('#5 `10.31.41` (auto→substring) finds the page with that IP', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'Сетевой узел',
|
||||
textContent: 'Адрес устройства: 10.31.41.7 в сети.',
|
||||
});
|
||||
const res = await search(buildService(), { query: '10.31.41', spaceId });
|
||||
const hit = res.items.find((i: any) => i.id === page);
|
||||
expect(hit).toBeDefined();
|
||||
expect(hit.matchedFields).toContain('text');
|
||||
});
|
||||
|
||||
// 6. +required / -excluded.
|
||||
it('#6 `+кофейня -архив`: keeps «кофейня», drops pages with «архив»', async () => {
|
||||
const keep = await insertPage({ title: 'Кофейня в центре', textContent: 'уют' });
|
||||
const drop = await insertPage({
|
||||
title: 'Кофейня старый архив',
|
||||
textContent: 'архивные записи',
|
||||
});
|
||||
const res = await search(buildService(), { query: '+кофейня -архив', spaceId });
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(keep);
|
||||
expect(ids).not.toContain(drop);
|
||||
});
|
||||
|
||||
// 7. Phrase operator: only adjacent phrase hits survive.
|
||||
it('#7 `+"воздушный шар" кофе`: every hit contains the adjacent phrase', async () => {
|
||||
const adjacent = await insertPage({
|
||||
title: 'Воздушный шар и кофе',
|
||||
textContent: 'воздушный шар над городом, чашка кофе',
|
||||
});
|
||||
const nonAdjacent = await insertPage({
|
||||
title: 'Красный воздушный большой шар',
|
||||
textContent: 'воздушный красный шар и кофе рядом',
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: '+"воздушный шар" кофе',
|
||||
spaceId,
|
||||
});
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(adjacent);
|
||||
// The non-adjacent page (words separated) must NOT match the phrase.
|
||||
expect(ids).not.toContain(nonAdjacent);
|
||||
});
|
||||
|
||||
// 8. Pagination determinism + exact total.
|
||||
it('#8 >50 matches: total>50, hasMore, and offset paginates without dupes', async () => {
|
||||
const svc = buildService();
|
||||
const created: string[] = [];
|
||||
for (let i = 0; i < 60; i++) {
|
||||
created.push(
|
||||
await insertPage({
|
||||
title: `паджинация запись ${i}`,
|
||||
textContent: 'общий паджинационный маркер',
|
||||
}),
|
||||
);
|
||||
}
|
||||
const p1 = await search(svc, {
|
||||
query: 'паджинационный',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
offset: 0,
|
||||
});
|
||||
expect(p1.total).toBeGreaterThanOrEqual(60);
|
||||
expect(p1.hasMore).toBe(true);
|
||||
const p2 = await search(svc, {
|
||||
query: 'паджинационный',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
offset: 25,
|
||||
});
|
||||
const p3 = await search(svc, {
|
||||
query: 'паджинационный',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
offset: 50,
|
||||
});
|
||||
const ids = [
|
||||
...p1.items.map((i: any) => i.id),
|
||||
...p2.items.map((i: any) => i.id),
|
||||
...p3.items.map((i: any) => i.id),
|
||||
];
|
||||
// No duplicates across the three pages.
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
// Deterministic: same query twice → identical order.
|
||||
const p1b = await search(svc, {
|
||||
query: 'паджинационный',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
offset: 0,
|
||||
});
|
||||
expect(p1b.items.map((i: any) => i.id)).toEqual(p1.items.map((i: any) => i.id));
|
||||
});
|
||||
|
||||
// 9. Only-negation.
|
||||
it('#9 `-архив` only-negation: total 0, reason only-negation, no throw', async () => {
|
||||
const res = await search(buildService(), { query: '-архив', spaceId });
|
||||
expect(res.total).toBe(0);
|
||||
expect(res.items).toEqual([]);
|
||||
expect(res.query.parsed.reason).toBe('only-negation');
|
||||
});
|
||||
|
||||
// 10. Garbage input.
|
||||
it('#10 garbage `%` / `_` / empty: total 0, does not match everything', async () => {
|
||||
await insertPage({ title: 'какая-то страница', textContent: 'текст' });
|
||||
for (const q of ['%', '_', ' ', '%%__']) {
|
||||
const res = await search(buildService(), { query: q, spaceId });
|
||||
expect(res.total).toBe(0);
|
||||
expect(res.items).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
// 11. Permission-filtered total (fail-closed) + mutation guard.
|
||||
it('#11 a permission-hidden page is absent from items AND total', async () => {
|
||||
const visible = await insertPage({
|
||||
title: 'разрешённая пермишен-страница',
|
||||
textContent: 'пермишенмаркер',
|
||||
});
|
||||
const hidden = await insertPage({
|
||||
title: 'скрытая пермишен-страница',
|
||||
textContent: 'пермишенмаркер',
|
||||
});
|
||||
// Filter keeps only the visible page.
|
||||
const filtered = buildService({ accessibleIds: [visible] });
|
||||
const res = await search(filtered, { query: 'пермишенмаркер', spaceId });
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(visible);
|
||||
expect(ids).not.toContain(hidden);
|
||||
// total is the POST-permission count — the hidden page does not leak into it.
|
||||
expect(res.total).toBe(1);
|
||||
|
||||
// MUTATION: disable the guard (passthrough) → the hidden page reappears in
|
||||
// BOTH items and total. If this did NOT change, the guard is not load-bearing.
|
||||
const open = buildService({ accessibleIds: null });
|
||||
const res2 = await search(open, { query: 'пермишенмаркер', spaceId });
|
||||
expect(res2.items.map((i: any) => i.id)).toContain(hidden);
|
||||
expect(res2.total).toBe(2);
|
||||
});
|
||||
|
||||
it('#11b a permission-query error PROPAGATES (fail-closed, never empty)', async () => {
|
||||
await insertPage({ title: 'failclosed маркер', textContent: 'failclosedmarker' });
|
||||
const svc = buildService({ filterThrows: true });
|
||||
await expect(
|
||||
search(svc, { query: 'failclosedmarker', spaceId }),
|
||||
).rejects.toThrow(/permission query failed/);
|
||||
});
|
||||
|
||||
// A8 path fix.
|
||||
it('#11c path: a soft-deleted / cross-space ancestor title does not leak', async () => {
|
||||
const otherSpace = (await createSpace(db, workspaceId)).id;
|
||||
const root = await insertPage({ title: 'Живой корень' });
|
||||
const deletedMid = await insertPage({
|
||||
title: 'УдалённыйПредок',
|
||||
parentPageId: root,
|
||||
deletedAt: new Date(),
|
||||
});
|
||||
const leaf = await insertPage({
|
||||
title: 'a8leaf уникальный',
|
||||
parentPageId: deletedMid,
|
||||
textContent: 'a8leafmarker',
|
||||
});
|
||||
const res = await search(buildService(), { query: 'a8leafmarker', spaceId });
|
||||
const hit = res.items.find((i: any) => i.id === leaf);
|
||||
expect(hit).toBeDefined();
|
||||
// The walk stops at the deleted ancestor — no deleted title in the path.
|
||||
expect(hit.path).not.toContain('УдалённыйПредок');
|
||||
expect(hit.path).not.toContain('Живой корень');
|
||||
|
||||
// Cross-space parent must also not leak.
|
||||
const foreignParent = await insertPage({
|
||||
title: 'ЧужойСпейс',
|
||||
spaceId: otherSpace,
|
||||
});
|
||||
const crossLeaf = await insertPage({
|
||||
title: 'crossleaf узел',
|
||||
parentPageId: foreignParent,
|
||||
textContent: 'crossleafmarker',
|
||||
});
|
||||
const res2 = await search(buildService(), {
|
||||
query: 'crossleafmarker',
|
||||
spaceId,
|
||||
});
|
||||
const hit2 = res2.items.find((i: any) => i.id === crossLeaf);
|
||||
expect(hit2).toBeDefined();
|
||||
expect(hit2.path).not.toContain('ЧужойСпейс');
|
||||
});
|
||||
|
||||
// 12. Web-UI superset.
|
||||
it('#12 web path (no flags) returns the OR result with the icon/space/highlight superset', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'веб суперсет страница',
|
||||
textContent: 'суперсетмаркер контент',
|
||||
});
|
||||
const res = await search(buildService(), { query: 'суперсетмаркер', spaceId });
|
||||
const hit = res.items.find((i: any) => i.id === page);
|
||||
expect(hit).toBeDefined();
|
||||
// Superset fields the web-UI relies on.
|
||||
expect('icon' in hit).toBe(true);
|
||||
expect('space' in hit).toBe(true);
|
||||
expect('highlight' in hit).toBe(true);
|
||||
expect('rank' in hit).toBe(true);
|
||||
// Plus the new fields.
|
||||
expect('path' in hit).toBe(true);
|
||||
expect('snippet' in hit).toBe(true);
|
||||
expect('score' in hit).toBe(true);
|
||||
// FTS hit carries a non-null rank + highlight.
|
||||
expect(hit.rank).not.toBeNull();
|
||||
});
|
||||
|
||||
// 13. RAG lockstep: the page_embeddings.fts generated column is ru_en.
|
||||
it('#13 page_embeddings.fts uses ru_en (cyrillic stemming) — RAG lockstep', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'rag страница',
|
||||
textContent: 'ресторанов много',
|
||||
});
|
||||
// Insert a chunk row; the generated fts column is computed by Postgres.
|
||||
await sql`
|
||||
INSERT INTO page_embeddings
|
||||
(id, page_id, workspace_id, space_id, attachment_id, chunk_index,
|
||||
chunk_start, chunk_length, content, model_name, model_dimensions, embedding)
|
||||
VALUES
|
||||
(${randomUUID()}, ${pageId}, ${workspaceId}, ${spaceId}, NULL, 0,
|
||||
0, 20, ${'ресторанов москвы много'}, 'test-model', 3, '[0.1,0.2,0.3]'::vector)
|
||||
`.execute(db);
|
||||
// A ru_en query stems «москва» → «москв», matching the stored «москвы».
|
||||
// Under the old `english` config the cyrillic word would not stem and this
|
||||
// inflected-form query would miss — so this asserts the ru_en lockstep.
|
||||
const row = await sql<{ m: boolean }>`
|
||||
SELECT fts @@ to_tsquery('ru_en', f_unaccent('москва')) AS m
|
||||
FROM page_embeddings WHERE page_id = ${pageId}
|
||||
`.execute(db);
|
||||
expect(row.rows[0].m).toBe(true);
|
||||
});
|
||||
|
||||
// W4 — substring-tier dominance under RRF: title-exact (tier 3) > title-
|
||||
// substring (tier 2) > text-only (tier 1). The engine encodes this via
|
||||
// sub_tier DESC → rn_sub → RRF; no test asserted the end-to-end ordering, so
|
||||
// this restores that guarantee. match:'substring' routes the term to the
|
||||
// substring branch (no FTS leg), so sub_tier alone drives the order.
|
||||
it('W4 tier dominance: title-exact > title-substring > text-only', async () => {
|
||||
const exact = await insertPage({ title: 'tierdomxyz' }); // tier 3
|
||||
const titleSub = await insertPage({
|
||||
title: 'prefix tierdomxyz suffix', // tier 2
|
||||
});
|
||||
const textOnly = await insertPage({
|
||||
title: 'w4 unrelated heading',
|
||||
textContent: 'body has tierdomxyz here', // tier 1
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: 'tierdomxyz',
|
||||
match: 'substring',
|
||||
spaceId,
|
||||
});
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(exact);
|
||||
expect(ids).toContain(titleSub);
|
||||
expect(ids).toContain(textOnly);
|
||||
// Strict tier order.
|
||||
expect(ids.indexOf(exact)).toBeLessThan(ids.indexOf(titleSub));
|
||||
expect(ids.indexOf(titleSub)).toBeLessThan(ids.indexOf(textOnly));
|
||||
});
|
||||
|
||||
// S3 — an exact-title hit must NOT be lost when the match set exceeds
|
||||
// CANDIDATE_CAP: it ranks first under RRF (tier 3) so it lands in the reachable
|
||||
// window even with a tiny cap. Restores a guarantee the old lookup suite gave.
|
||||
it('S3 exact-title survives the CANDIDATE_CAP window', async () => {
|
||||
const exact = await insertPage({ title: 'capmarkerxyz' }); // tier 3
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await insertPage({
|
||||
title: `cap filler ${i}`,
|
||||
textContent: 'noise capmarkerxyz noise', // tier 1
|
||||
});
|
||||
}
|
||||
process.env.SEARCH_CANDIDATE_CAP = '2';
|
||||
try {
|
||||
const res = await search(buildService(), {
|
||||
query: 'capmarkerxyz',
|
||||
match: 'substring',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
});
|
||||
// More matches than the cap → truncated, but the exact-title survives.
|
||||
expect(res.total).toBeGreaterThan(2);
|
||||
expect(res.truncatedAtCap).toBe(true);
|
||||
expect(res.items.length).toBe(2); // the reachable window
|
||||
expect(res.items.map((i: any) => i.id)).toContain(exact);
|
||||
} finally {
|
||||
delete process.env.SEARCH_CANDIDATE_CAP;
|
||||
}
|
||||
});
|
||||
|
||||
// S1 — a required-only query (`+term`, no bare positive) really matches, so its
|
||||
// hits must report matchedFields / rank / highlight, not [] / null. Before the
|
||||
// fix these detail exprs were built from parsed.positive only.
|
||||
it('S1 required-only query populates matchedFields + rank on a title match', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'Кофейня s1маркер центр',
|
||||
textContent: 'обычный текст без ключевого слова',
|
||||
});
|
||||
const res = await search(buildService(), { query: '+кофейня', spaceId });
|
||||
const hit = res.items.find((i: any) => i.id === page);
|
||||
expect(hit).toBeDefined();
|
||||
// The title matches the required term → matchedFields includes 'title'.
|
||||
expect(hit.matchedFields).toContain('title');
|
||||
// FTS rank is populated (was null before the S1 fix).
|
||||
expect(hit.rank).not.toBeNull();
|
||||
// matchedTerms already echoed the required term; still true.
|
||||
expect(hit.matchedTerms).toContain('кофейня');
|
||||
});
|
||||
});
|
||||
@@ -1,462 +0,0 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { Kysely } from 'kysely';
|
||||
import { SearchService } from 'src/core/search/search.service';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createSpace,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #443 — agent-lookup search mode, acceptance on the REAL DB schema.
|
||||
*
|
||||
* Exercises SearchService.searchPage(..., { substring: true }) against a
|
||||
* migrated Postgres: substring matching of technical tokens the FTS tokenizer
|
||||
* mangles (backup-srv.local, 10.0.12.5, WB-MGE-30D86B, "Теги: Docker"), the
|
||||
* populated path + snippet, parentPageId subtree scoping, titleOnly, the empty
|
||||
* result, LIKE-metacharacter escaping (`%`/`_` must NOT match everything), the
|
||||
* permission post-filter applied BEFORE the limit, and the web-UI path staying
|
||||
* on the legacy FTS shape when `substring` is absent.
|
||||
*
|
||||
* The tsv column is populated by the pages_tsvector_trigger on insert, so the
|
||||
* FTS branch is exercised too.
|
||||
*/
|
||||
describe('SearchService agent-lookup mode [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let service: SearchService;
|
||||
let workspaceId: string;
|
||||
let spaceId: string;
|
||||
|
||||
// Direct page insert (the shared createPage seeder omits text_content /
|
||||
// parent_page_id, both of which this mode depends on). Returns the id.
|
||||
async function insertPage(args: {
|
||||
title: string;
|
||||
textContent?: string;
|
||||
parentPageId?: string | null;
|
||||
spaceId?: string;
|
||||
}): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await db
|
||||
.insertInto('pages')
|
||||
.values({
|
||||
id,
|
||||
slugId: `slug-${id.slice(0, 12)}`,
|
||||
title: args.title,
|
||||
textContent: args.textContent ?? null,
|
||||
parentPageId: args.parentPageId ?? null,
|
||||
spaceId: args.spaceId ?? spaceId,
|
||||
workspaceId,
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
// Build a SearchService wired to the real DB + a real PageRepo (only its
|
||||
// recursive-descendants method is used by this mode, and it needs only `db`),
|
||||
// with lightweight stubs for the space-membership and permission repos so a
|
||||
// test can drive scope + the permission post-filter explicitly.
|
||||
function buildService(opts?: {
|
||||
userSpaceIds?: string[];
|
||||
// ids to KEEP after the permission post-filter; undefined = keep all.
|
||||
accessibleIds?: string[];
|
||||
}): SearchService {
|
||||
const pageRepo = new PageRepo(db as any, null as any, null as any);
|
||||
const spaceMemberRepo = {
|
||||
getUserSpaceIds: async () => opts?.userSpaceIds ?? [spaceId],
|
||||
};
|
||||
const pagePermissionRepo = {
|
||||
filterAccessiblePageIds: async ({ pageIds }: { pageIds: string[] }) =>
|
||||
opts?.accessibleIds
|
||||
? pageIds.filter((id) => opts.accessibleIds!.includes(id))
|
||||
: pageIds,
|
||||
};
|
||||
return new SearchService(
|
||||
db as any,
|
||||
pageRepo as any,
|
||||
{} as any, // shareRepo — unused by the lookup path
|
||||
spaceMemberRepo as any,
|
||||
pagePermissionRepo as any,
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
spaceId = (await createSpace(db, workspaceId)).id;
|
||||
service = buildService();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('finds `backup-srv.local` by the fragment `srv.local`', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'backup-srv.local',
|
||||
textContent: 'A backup server node.',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'srv.local', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
expect(items.map((i: any) => i.id)).toContain(pageId);
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit.title).toBe('backup-srv.local');
|
||||
// slugId must never be part of the server response shape.
|
||||
expect('slugId' in hit).toBe(true); // server carries it; MCP strips it
|
||||
});
|
||||
|
||||
it('finds a page whose TEXT contains `10.0.12.5` by the fragment `10.0.12` (empty-tsquery case)', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'Server inventory',
|
||||
textContent: 'The backup box lives at IP: 10.0.12.5. Debian 12, backups.',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: '10.0.12', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit).toBeDefined();
|
||||
// The windowed snippet must include the matched text.
|
||||
expect(hit.snippet).toContain('10.0.12.5');
|
||||
});
|
||||
|
||||
it('finds `WB-MGE-30D86B` (alphanumeric token with dashes) by title', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'WB-MGE-30D86B',
|
||||
textContent: 'Device page.',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'WB-MGE-30D86B', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit).toBeDefined();
|
||||
// Exact title match → top tier (TITLE_EXACT=3) → score in [0.75, 1].
|
||||
expect(hit.score).toBeGreaterThanOrEqual(0.75);
|
||||
// And it is the top-ranked hit of its own result set.
|
||||
expect(items[0].id).toBe(pageId);
|
||||
});
|
||||
|
||||
it('finds every page whose text literally contains `Теги: Docker`', async () => {
|
||||
const a = await insertPage({
|
||||
title: 'Container host A',
|
||||
textContent: 'Some notes.\nТеги: Docker, compose\nmore.',
|
||||
});
|
||||
const b = await insertPage({
|
||||
title: 'Container host B',
|
||||
textContent: 'Prelude.\nТеги: Docker\nepilogue.',
|
||||
});
|
||||
const noise = await insertPage({
|
||||
title: 'Unrelated',
|
||||
textContent: 'Теги: Kubernetes',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'Теги: Docker', spaceId, substring: true, limit: 50 } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
expect(ids).toContain(a);
|
||||
expect(ids).toContain(b);
|
||||
expect(ids).not.toContain(noise);
|
||||
});
|
||||
|
||||
it('populates a non-empty `path` for a nested hit and `[]` for a root hit', async () => {
|
||||
const root = await insertPage({ title: 'Infrastructure' });
|
||||
const mid = await insertPage({ title: 'Datacenter A', parentPageId: root });
|
||||
const leaf = await insertPage({
|
||||
title: 'unique-nested-host',
|
||||
parentPageId: mid,
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'unique-nested-host', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === leaf);
|
||||
expect(hit.path).toEqual(['Infrastructure', 'Datacenter A']);
|
||||
|
||||
const rootHits = (await service.searchPage(
|
||||
{ query: 'Infrastructure', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
const rootHit = rootHits.items.find((i: any) => i.id === root);
|
||||
expect(rootHit.path).toEqual([]);
|
||||
});
|
||||
|
||||
it('scopes to a subtree with parentPageId (cutting off sibling branches)', async () => {
|
||||
const branchA = await insertPage({ title: 'BranchA-root' });
|
||||
const inA = await insertPage({
|
||||
title: 'scoped-target-xyz',
|
||||
parentPageId: branchA,
|
||||
});
|
||||
const branchB = await insertPage({ title: 'BranchB-root' });
|
||||
const inB = await insertPage({
|
||||
title: 'scoped-target-xyz',
|
||||
parentPageId: branchB,
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{
|
||||
query: 'scoped-target-xyz',
|
||||
spaceId,
|
||||
substring: true,
|
||||
parentPageId: branchA,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
expect(ids).toContain(inA);
|
||||
expect(ids).not.toContain(inB);
|
||||
});
|
||||
|
||||
it('includes the parent page itself in the parentPageId subtree', async () => {
|
||||
const parent = await insertPage({ title: 'self-included-parent' });
|
||||
await insertPage({ title: 'child-of-self', parentPageId: parent });
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{
|
||||
query: 'self-included-parent',
|
||||
spaceId,
|
||||
substring: true,
|
||||
parentPageId: parent,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
expect(items.map((i: any) => i.id)).toContain(parent);
|
||||
});
|
||||
|
||||
it('titleOnly does NOT match on text_content', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'Plain title',
|
||||
textContent: 'body mentions the-secret-token here',
|
||||
});
|
||||
|
||||
const withText = (await service.searchPage(
|
||||
{ query: 'the-secret-token', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
expect(withText.items.map((i: any) => i.id)).toContain(pageId);
|
||||
|
||||
const titleOnly = (await service.searchPage(
|
||||
{
|
||||
query: 'the-secret-token',
|
||||
spaceId,
|
||||
substring: true,
|
||||
titleOnly: true,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
expect(titleOnly.items.map((i: any) => i.id)).not.toContain(pageId);
|
||||
});
|
||||
|
||||
// #443 Fix #1 regression: f_unaccent is NOT length-preserving, so an
|
||||
// expanding char (ß→ss, …→...) BEFORE the match shifted the strpos position
|
||||
// relative to the ORIGINAL text and the snippet slice ran past end → empty.
|
||||
// The position and the slice now share the LOWER(f_unaccent(...)) space, so
|
||||
// the window is aligned and always contains the matched (unaccented) token.
|
||||
it('returns a populated snippet when an unaccent-EXPANDING char precedes the match', async () => {
|
||||
// 300 × `ß` (each f_unaccent-expands to `ss`) before the needle. Under the
|
||||
// old code strpos returned a position ~593 in the expanded space but the
|
||||
// slice ran over the ORIGINAL (~360 char) text → empty snippet, match lost.
|
||||
const prefix = 'ß'.repeat(300);
|
||||
const pageId = await insertPage({
|
||||
title: 'Expanding-unaccent page',
|
||||
textContent: `${prefix} needle-token-xyz trailing.`,
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'needle-token-xyz', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit).toBeDefined();
|
||||
// Snippet must be non-empty AND contain the matched token (unaccented form).
|
||||
expect(hit.snippet.length).toBeGreaterThan(0);
|
||||
expect(hit.snippet).toContain('needle-token-xyz');
|
||||
});
|
||||
|
||||
// #443 Fix #2 regression: >200 matching pages for a broad substring, with
|
||||
// exactly ONE exact-title hit. Without an ORDER BY on the 200-cap the exact
|
||||
// hit could be among the arbitrarily-dropped rows; the ORDER BY keeps the
|
||||
// strongest candidates so it must survive the cap and rank at the top.
|
||||
it('keeps an exact-title hit through the 200-cap on a >200-row match set', async () => {
|
||||
const isoSpace = (await createSpace(db, workspaceId)).id;
|
||||
const svc = buildService({ userSpaceIds: [isoSpace] });
|
||||
|
||||
// 250 low-tier TEXT hits: the shared substring `capword` appears only in the
|
||||
// body, never the title, so each is a TEXT-tier match (weakest tier).
|
||||
for (let i = 0; i < 250; i++) {
|
||||
await insertPage({
|
||||
title: `filler-page-${i}`,
|
||||
textContent: `body contains capword here #${i}`,
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
}
|
||||
// Exactly one EXACT-title hit for the same query token.
|
||||
const exact = await insertPage({
|
||||
title: 'capword',
|
||||
textContent: 'unrelated body text',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
|
||||
const { items } = (await svc.searchPage(
|
||||
{ query: 'capword', spaceId: isoSpace, substring: true, limit: 10 } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
// The exact-title hit must survive the 200-cap and appear in the top `limit`.
|
||||
expect(ids).toContain(exact);
|
||||
// And, being TITLE_EXACT, it must be the single strongest hit.
|
||||
expect(items[0].id).toBe(exact);
|
||||
});
|
||||
|
||||
// #443 Fix #3: titleOnly matches only the title, so it must not leak the page
|
||||
// body as the snippet (the old "first 300 chars of text_content" fallback).
|
||||
it('titleOnly does NOT return a text-body snippet', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'titleonly-snippet-page',
|
||||
textContent: 'SECRET-BODY-CONTENT-NOT-IN-TITLE that must not leak.',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{
|
||||
query: 'titleonly-snippet-page',
|
||||
spaceId,
|
||||
substring: true,
|
||||
titleOnly: true,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit).toBeDefined();
|
||||
// The body text must not appear in the snippet; titleOnly → empty snippet.
|
||||
expect(hit.snippet).not.toContain('SECRET-BODY-CONTENT-NOT-IN-TITLE');
|
||||
expect(hit.snippet).toBe('');
|
||||
});
|
||||
|
||||
it('returns [] (not an error) for a query that matches nothing', async () => {
|
||||
const { items } = (await service.searchPage(
|
||||
{
|
||||
query: 'zzz-no-such-string-anywhere-42',
|
||||
spaceId,
|
||||
substring: true,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
expect(items).toEqual([]);
|
||||
});
|
||||
|
||||
it('a `%` query does NOT match everything (LIKE metacharacter escaped)', async () => {
|
||||
// Fresh space so we can assert on total counts without cross-test noise.
|
||||
const isoSpace = (await createSpace(db, workspaceId)).id;
|
||||
const svc = buildService({ userSpaceIds: [isoSpace] });
|
||||
await insertPage({ title: 'alpha', spaceId: isoSpace });
|
||||
await insertPage({ title: 'beta', spaceId: isoSpace });
|
||||
const literal = await insertPage({
|
||||
title: '100%-coverage',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
|
||||
const { items } = (await svc.searchPage(
|
||||
{ query: '%', spaceId: isoSpace, substring: true, limit: 50 } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
// `%` is a literal → matches only the page that actually contains '%'.
|
||||
expect(ids).toContain(literal);
|
||||
expect(ids).not.toContain(
|
||||
items.find((i: any) => i.title === 'alpha')?.id,
|
||||
);
|
||||
expect(items.length).toBe(1);
|
||||
});
|
||||
|
||||
it('an `_` query does NOT match everything (LIKE metacharacter escaped)', async () => {
|
||||
const isoSpace = (await createSpace(db, workspaceId)).id;
|
||||
const svc = buildService({ userSpaceIds: [isoSpace] });
|
||||
await insertPage({ title: 'gamma', spaceId: isoSpace });
|
||||
const literal = await insertPage({
|
||||
title: 'snake_case_name',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
|
||||
const { items } = (await svc.searchPage(
|
||||
{ query: '_', spaceId: isoSpace, substring: true, limit: 50 } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
expect(ids).toContain(literal);
|
||||
expect(items.length).toBe(1);
|
||||
});
|
||||
|
||||
it('applies the permission post-filter to the MERGED set BEFORE the limit', async () => {
|
||||
const isoSpace = (await createSpace(db, workspaceId)).id;
|
||||
const keep = await insertPage({
|
||||
title: 'perm-visible-target',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
const hidden = await insertPage({
|
||||
title: 'perm-hidden-target',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
|
||||
// Authenticated (userId set) so the permission filter runs; only `keep` is
|
||||
// accessible. limit 1 must NOT be able to select `hidden`.
|
||||
const svc = buildService({
|
||||
userSpaceIds: [isoSpace],
|
||||
accessibleIds: [keep],
|
||||
});
|
||||
const { items } = (await svc.searchPage(
|
||||
{
|
||||
query: 'perm-',
|
||||
spaceId: isoSpace,
|
||||
substring: true,
|
||||
limit: 1,
|
||||
} as any,
|
||||
{ userId: 'user-1', workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
expect(ids).toContain(keep);
|
||||
expect(ids).not.toContain(hidden);
|
||||
});
|
||||
|
||||
it('web-UI path (no `substring` flag) keeps the legacy FTS response shape', async () => {
|
||||
await insertPage({
|
||||
title: 'legacy shape page',
|
||||
textContent: 'searchable legacyword content',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'legacyword', spaceId } as any,
|
||||
{ userId: 'user-1', workspaceId },
|
||||
)) as any;
|
||||
|
||||
// Legacy hits carry rank + highlight + space, and NO path/snippet/score.
|
||||
const hit = items[0];
|
||||
expect(hit).toBeDefined();
|
||||
expect('rank' in hit).toBe(true);
|
||||
expect('highlight' in hit).toBe(true);
|
||||
expect('path' in hit).toBe(false);
|
||||
expect('snippet' in hit).toBe(false);
|
||||
expect('score' in hit).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { getTestDb, destroyTestDb } from './db';
|
||||
import * as migration from '../../src/database/migrations/20260707T130000-search-ru-en-config';
|
||||
|
||||
/**
|
||||
* #529 A1 — the ru_en config migration must be REVERSIBLE in the correct order:
|
||||
* down() moves pages.tsv + page_embeddings.fts back to `english` BEFORE dropping
|
||||
* the ru_en config (a generated column still depending on ru_en would block the
|
||||
* DROP). This roundtrips down()→up() on the already-migrated test DB and asserts
|
||||
* the config, the pages trigger and the fts generated expression each flip and
|
||||
* flip back — the deploy-critical property.
|
||||
*/
|
||||
describe('search ru_en config migration [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
|
||||
const configExists = async () => {
|
||||
const r = await sql<{ n: number }>`
|
||||
SELECT count(*)::int AS n FROM pg_ts_config WHERE cfgname = 'ru_en'
|
||||
`.execute(db);
|
||||
return r.rows[0].n > 0;
|
||||
};
|
||||
|
||||
const ftsDef = async () => {
|
||||
const r = await sql<{ def: string }>`
|
||||
SELECT pg_get_expr(adbin, adrelid) AS def
|
||||
FROM pg_attrdef d
|
||||
JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum
|
||||
WHERE a.attname = 'fts' AND a.attrelid = 'page_embeddings'::regclass
|
||||
`.execute(db);
|
||||
return r.rows[0]?.def ?? '';
|
||||
};
|
||||
|
||||
const triggerSrc = async () => {
|
||||
const r = await sql<{ src: string }>`
|
||||
SELECT prosrc AS src FROM pg_proc WHERE proname = 'pages_tsvector_trigger'
|
||||
`.execute(db);
|
||||
return r.rows[0]?.src ?? '';
|
||||
};
|
||||
|
||||
beforeAll(() => {
|
||||
db = getTestDb();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Restore the canonical ru_en state for any later suite regardless of where
|
||||
// a test left off. up() is now safe on an existing config (ensureRuEnConfig
|
||||
// no-ops) and re-asserts both stored sides to ru_en.
|
||||
delete process.env.SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE;
|
||||
await migration.up(db);
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('starts at ru_en (config present, trigger + fts on ru_en)', async () => {
|
||||
expect(await configExists()).toBe(true);
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('ru_en');
|
||||
});
|
||||
|
||||
it('down() reverts tsv + fts to english THEN drops the config', async () => {
|
||||
await migration.down(db);
|
||||
expect(await configExists()).toBe(false);
|
||||
expect(await ftsDef()).toContain('english');
|
||||
expect(await ftsDef()).not.toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('english');
|
||||
});
|
||||
|
||||
it('up() re-applies ru_en cleanly (idempotent config create)', async () => {
|
||||
await migration.up(db);
|
||||
expect(await configExists()).toBe(true);
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('ru_en');
|
||||
});
|
||||
|
||||
// B1.1 — running up() a SECOND time on an already-migrated DB is a true no-op:
|
||||
// it must NOT throw (the old drop-recreate-config would fail on the fts hard
|
||||
// dependency) and must NOT re-rewrite the embeddings table — the fts column
|
||||
// already references ru_en, so the at-target check short-circuits.
|
||||
it('up() is idempotent: a 2nd run does not error and leaves ru_en intact', async () => {
|
||||
await expect(migration.up(db)).resolves.toBeUndefined();
|
||||
expect(await configExists()).toBe(true);
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('ru_en');
|
||||
});
|
||||
|
||||
// B1.2 — env-gate opt-out: with SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE=false the
|
||||
// embeddings rewrite is SKIPPED (fts stays where it was) but pages.tsv still
|
||||
// swaps inline, and the ru_en config is left in place (fts still depends on it).
|
||||
// Runs down() as the vehicle: english is NOT the current fts config, so the
|
||||
// skip path — not the at-target no-op — is exercised.
|
||||
it('env-gate=false: skips the fts rewrite but still swaps pages.tsv', async () => {
|
||||
// Precondition: fts + trigger on ru_en (from the prior test).
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
process.env.SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE = 'false';
|
||||
try {
|
||||
await migration.down(db);
|
||||
// fts rewrite skipped → still ru_en (the ACCESS EXCLUSIVE rewrite avoided).
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await ftsDef()).not.toContain('english');
|
||||
// pages.tsv trigger still swapped inline to english (gate is fts-only).
|
||||
expect(await triggerSrc()).toContain('english');
|
||||
// Config left in place because fts still references it (guarded drop).
|
||||
expect(await configExists()).toBe(true);
|
||||
} finally {
|
||||
// Restore ru_en fully for the afterAll / later suites.
|
||||
delete process.env.SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE;
|
||||
await migration.up(db);
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('ru_en');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -78,7 +78,7 @@ export interface IReadMixin {
|
||||
getNode(pageId: string, nodeId: string, format?: "markdown" | "json"): any;
|
||||
searchInPage(pageId: string, query: string, opts?: SearchOptions): any;
|
||||
getTable(pageId: string, tableRef: string): any;
|
||||
search(query: string, spaceId?: string, limit?: number, opts?: { parentPageId?: string; titleOnly?: boolean }): any;
|
||||
search(query: string, spaceId?: string, limit?: number, opts?: { parentPageId?: string; titleOnly?: boolean; offset?: number; match?: "auto" | "word" | "prefix" | "substring" }): any;
|
||||
}
|
||||
|
||||
export function ReadMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & IReadMixin> & TBase {
|
||||
@@ -681,13 +681,19 @@ export function ReadMixin<TBase extends GConstructor<DocmostClientContext>>(Base
|
||||
query: string,
|
||||
spaceId?: string,
|
||||
limit?: number,
|
||||
opts: { parentPageId?: string; titleOnly?: boolean } = {},
|
||||
opts: {
|
||||
parentPageId?: string;
|
||||
titleOnly?: boolean;
|
||||
offset?: number;
|
||||
match?: "auto" | "word" | "prefix" | "substring";
|
||||
} = {},
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
// Opt into the #443 agent-lookup mode: `substring: true` turns on the hybrid
|
||||
// substring + FTS branch that returns path + snippet + score. A stock
|
||||
// upstream server strips these unknown DTO fields (whitelist:true) and
|
||||
// silently degrades to plain FTS — see the tool-registration comment.
|
||||
// #529 unified engine: the query is parsed SERVER-SIDE (operators
|
||||
// "phrase"/+/-, OR default, RU+EN morphology, match=auto). We forward the RAW
|
||||
// query plus flags. `substring: true` is kept as a back-compat hint for a
|
||||
// stock upstream server (whitelist:true strips the unknown #529 fields and it
|
||||
// degrades to plain FTS — see the tool-registration comment).
|
||||
const payload: Record<string, any> = {
|
||||
query,
|
||||
spaceId,
|
||||
@@ -695,20 +701,31 @@ export function ReadMixin<TBase extends GConstructor<DocmostClientContext>>(Base
|
||||
};
|
||||
if (opts.parentPageId) payload.parentPageId = opts.parentPageId;
|
||||
if (opts.titleOnly) payload.titleOnly = true;
|
||||
if (opts.match) payload.match = opts.match;
|
||||
// Clamp an optional caller-supplied limit into the lookup range (1..50)
|
||||
// before forwarding; omit it when not provided so the server default applies.
|
||||
if (limit !== undefined) {
|
||||
payload.limit = Math.max(1, Math.min(50, limit));
|
||||
}
|
||||
if (opts.offset !== undefined) {
|
||||
payload.offset = Math.max(0, Math.floor(opts.offset));
|
||||
}
|
||||
const response = await this.client.post("/search", payload);
|
||||
|
||||
// Normalize both response shapes: bare array and paginated { items: [...] }
|
||||
// Normalize both response shapes: bare array and paginated { items: [...] }.
|
||||
const data = response.data?.data;
|
||||
const items = Array.isArray(data) ? data : data?.items || [];
|
||||
const filteredItems = items.map((item: any) => filterSearchResult(item));
|
||||
|
||||
// Surface the #529 pagination envelope when present (a stock upstream has
|
||||
// none — the fields are simply undefined and the caller sees just `items`).
|
||||
const envelope = Array.isArray(data) ? undefined : data;
|
||||
return {
|
||||
items: filteredItems,
|
||||
total: envelope?.total,
|
||||
hasMore: envelope?.hasMore,
|
||||
truncatedAtCap: envelope?.truncatedAtCap,
|
||||
offset: envelope?.offset,
|
||||
success: response.data?.success || false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -450,15 +450,22 @@ server.registerTool(
|
||||
"search",
|
||||
{
|
||||
description:
|
||||
"Find pages by a fragment of a technical string (hostnames, IPs, IDs " +
|
||||
"like `srv.local`, `10.0.12`, `WB-MGE-30D86B`) — one call returns each " +
|
||||
"hit's location (`path`: ancestor titles root→parent) and a `snippet` " +
|
||||
"around the first match, so you rarely need a follow-up get_page. " +
|
||||
"Matches substrings literally (dots/dashes/digits are not tokenized) as " +
|
||||
"well as full-text. Returns `{ pageId, title, path, snippet, score }` " +
|
||||
"sorted by `score` (a per-response relevance float).",
|
||||
"Search pages across the wiki. OR by default with relevance ranking " +
|
||||
"(RU+EN morphology): multi-word queries match ANY term, not all. " +
|
||||
"Operators: \"exact phrase\" (adjacent words), +term (require), -term " +
|
||||
"(exclude) — e.g. `+кофейня -архив`, `+\"воздушный шар\" кофе`. A leading " +
|
||||
"-/+ is the operator; -,.,: INSIDE a token are literal (`WB-MGE-30D86B`, " +
|
||||
"`10.0.12.5` stay one term). Technical fragments (hostnames, IPs, IDs) " +
|
||||
"auto-match as substrings; words use full-text. Each hit returns its " +
|
||||
"location (`path`: ancestor titles root→parent), a `snippet`, `score`, " +
|
||||
"`matchedTerms` and `matchedFields`, so you rarely need a follow-up " +
|
||||
"getPage. Paginate with limit + offset; the response carries " +
|
||||
"`total` (exact, permission-filtered), `hasMore` and `truncatedAtCap`. " +
|
||||
"NOTE: results past the relevance cap (~500) are unreachable by " +
|
||||
"pagination — narrow the query (add terms / +required / a spaceId) " +
|
||||
"instead when `truncatedAtCap` is true.",
|
||||
inputSchema: {
|
||||
query: z.string().min(1).describe("Search query"),
|
||||
query: z.string().min(1).describe("Search query (supports \"phrase\", +require, -exclude)"),
|
||||
spaceId: z
|
||||
.string()
|
||||
.optional()
|
||||
@@ -473,6 +480,13 @@ server.registerTool(
|
||||
.boolean()
|
||||
.optional()
|
||||
.describe("Match page titles only; skip page text"),
|
||||
match: z
|
||||
.enum(["auto", "word", "prefix", "substring"])
|
||||
.optional()
|
||||
.describe(
|
||||
"Match mode (default auto: identifiers→substring, words→full-text). " +
|
||||
"Override with word/prefix/substring.",
|
||||
),
|
||||
limit: z
|
||||
.number()
|
||||
.int()
|
||||
@@ -480,12 +494,20 @@ server.registerTool(
|
||||
.max(50)
|
||||
.optional()
|
||||
.describe("Max results to return (1-50, default 10)"),
|
||||
offset: z
|
||||
.number()
|
||||
.int()
|
||||
.min(0)
|
||||
.optional()
|
||||
.describe("Pagination offset (default 0); use with total/hasMore"),
|
||||
},
|
||||
},
|
||||
async ({ query, spaceId, parentPageId, titleOnly, limit }) => {
|
||||
async ({ query, spaceId, parentPageId, titleOnly, match, limit, offset }) => {
|
||||
const result = await docmostClient.search(query, spaceId, limit, {
|
||||
parentPageId,
|
||||
titleOnly,
|
||||
match,
|
||||
offset,
|
||||
});
|
||||
return jsonContent(result);
|
||||
},
|
||||
|
||||
@@ -40,7 +40,7 @@ import { SHARED_TOOL_SPECS, SharedToolSpec } from "./tool-specs.js";
|
||||
*/
|
||||
export const ROUTING_PROSE =
|
||||
"Docmost editing guide — choose the tool by intent. The <tool_inventory> at the end lists every tool with a one-line purpose; the notes below are the routing hints for WHEN to reach for each.\n" +
|
||||
"READ: find a page by a fragment of a technical string (hostname/IP/ID like srv.local, 10.0.12, WB-MGE-30D86B) -> search — hybrid substring + full-text, returns each hit's location (path: root->parent titles) and a snippet around the match, so you rarely need a follow-up getPage; scope with spaceId or parentPageId (a subtree), titleOnly to match titles only. A space's page HIERARCHY (or one subtree) -> getTree (one request, complete, `{pageId,title,children?}`; rootPageId for a subtree, maxDepth to trim depth — a trimmed node gets hasChildren:true); prefer it over listPages tree:true (deprecated). Have a pageId, need WHERE-AM-I / what's around it (its breadcrumbs + direct children, metadata only) -> getPageContext (one call; parent = last breadcrumb, [] for a root page). list -> listPages / listSpaces. Locate blocks and their ids CHEAPLY -> getOutline (compact top-level map; start here, not getPageJson). One block, for editing -> getNode (by attrs.id, or \"#<index>\" for tables, which carry no id) — returns MARKDOWN by default (comment anchors kept for safe write-back); pass format:\"json\" for the raw ProseMirror subtree. Find every occurrence of a string/regex ON a page (and where each is) -> searchInPage, NOT block-by-block getNode — it returns each hit's node ref + block index + context for a targeted comment. Whole page -> getPage (Markdown, canonical for text; drops only block ids, resolved-comment anchors, and a fixed no-md-representation attr set: table spans/colwidth/bg, indent, callout.icon, orderedList.type, link internal/target/rel/class; inline <span data-comment-id> tags are comment anchors — markup, not text) or getPageJson (full ProseMirror with block ids, for those dropped attrs). Hand a huge page (with images) to an external consumer without pulling it through the model context -> stashPage (returns a short-lived anonymous URL).\n" +
|
||||
"READ: find pages across the wiki -> search — OR by default with relevance ranking and RU+EN morphology (multi-word matches ANY term). Operators: \"exact phrase\", +require, -exclude (e.g. `+кофейня -архив`, `+\"воздушный шар\" кофе`); a leading -/+ is the operator, but -,.,: inside a token are literal (WB-MGE-30D86B, 10.0.12.5 stay one term, auto-matched as substrings). Each hit returns its location (path: root->parent titles), a snippet, score, matchedTerms/matchedFields, so you rarely need a follow-up getPage; scope with spaceId or parentPageId (a subtree), titleOnly to match titles only, match to override auto. Paginate with limit+offset; total is exact and permission-filtered, hasMore/truncatedAtCap flag more. Results past the relevance cap (~500) are UNREACHABLE by pagination — when truncatedAtCap is true, narrow the query (add terms / +required / a spaceId) rather than page deeper. A space's page HIERARCHY (or one subtree) -> getTree (one request, complete, `{pageId,title,children?}`; rootPageId for a subtree, maxDepth to trim depth — a trimmed node gets hasChildren:true); prefer it over listPages tree:true (deprecated). Have a pageId, need WHERE-AM-I / what's around it (its breadcrumbs + direct children, metadata only) -> getPageContext (one call; parent = last breadcrumb, [] for a root page). list -> listPages / listSpaces. Locate blocks and their ids CHEAPLY -> getOutline (compact top-level map; start here, not getPageJson). One block, for editing -> getNode (by attrs.id, or \"#<index>\" for tables, which carry no id) — returns MARKDOWN by default (comment anchors kept for safe write-back); pass format:\"json\" for the raw ProseMirror subtree. Find every occurrence of a string/regex ON a page (and where each is) -> searchInPage, NOT block-by-block getNode — it returns each hit's node ref + block index + context for a targeted comment. Whole page -> getPage (Markdown, canonical for text; drops only block ids, resolved-comment anchors, and a fixed no-md-representation attr set: table spans/colwidth/bg, indent, callout.icon, orderedList.type, link internal/target/rel/class; inline <span data-comment-id> tags are comment anchors — markup, not text) or getPageJson (full ProseMirror with block ids, for those dropped attrs). Hand a huge page (with images) to an external consumer without pulling it through the model context -> stashPage (returns a short-lived anonymous URL).\n" +
|
||||
"EDIT: fix wording/typos/numbers -> editPageText (find/replace inside blocks, no node id needed). Edit a block -> getNode(markdown) -> edit the markdown -> patchNode(markdown) (by attrs.id from getOutline; the markdown fragment may be several blocks — a 1->N section rewrite in one call, the first block keeps the id). Reach for patchNode's `node`-JSON only for fine attr/mark work; a table cell with spans/colors/fixed width -> the table tools (patchNode markdown refuses it). Add a block -> insertNode (markdown, before/after a block by attrs.id or by anchor text, or append; `node` for raw JSON or bare table structure). Remove a block -> deleteNode (by attrs.id). Tables -> tableGet / tableUpdateCell / tableInsertRow / tableDeleteRow (address by \"#<index>\" from getOutline; table nodes have no attrs.id). Images -> insertImage (add from a web URL) / replaceImage (swap an existing image). Draw.io diagrams -> PREFER the high-level semantic tools that hide coordinates/styles: drawioFromGraph (architecture/cloud/network diagrams — describe nodes/groups/edges by kind+icon, the server picks layout, colors and verified icons; hints layer/sameLayerAs/pinned and layout:full|incremental|none) and drawioFromMermaid (standard flowcharts — write Mermaid, get an editable diagram). For targeted tweaks of an existing diagram use drawioEditCells (id-based add/update/delete with cascade delete + baseHash lock). Raw mxGraph XML via drawioCreate/drawioUpdate is the escape-hatch for exotic/wireframe diagrams; drawioGet reads a diagram as mxGraph XML + a hash (pass it as baseHash to drawioUpdate/drawioEditCells for optimistic locking). Before authoring raw XML, drawioShapes (look up verified stencil style-strings so a shape name never renders as an empty box) and drawioGuide (on-demand authoring reference: skeleton/layout/containers/icons-aws/icons-azure), and pass layout:\"elk\" to drawioCreate/drawioUpdate to auto-place nodes. Footnotes -> insertFootnote. Bulk/structural rewrite -> updatePageJson (full ProseMirror replace) or updatePageMarkdown (full plain-Markdown body replace, re-imported — block ids regenerate); prefer the granular tools above to avoid resending the whole ~100KB+ document. Complex/scripted rewrite (multiple coordinated edits, renumbering) -> docmostTransform: write a JS `(doc, ctx) => doc` transform, preview the diff with dryRun (default), then apply with dryRun:false; ctx.helpers includes commentsToFootnotes for turning inline comments into numbered footnotes.\n" +
|
||||
"PAGES: new -> createPage (Markdown). Rename (title only) -> renamePage. Move -> movePage. Delete -> deletePage (SOFT delete — the page goes to trash and is restorable; nothing is permanent). Copy/replace a page's whole content from another page (server-side, no document through the model) -> copyPageContent. Sharing -> sharePage / unsharePage / listShares; sharePage makes the page PUBLICLY accessible — do it only when explicitly asked.\n" +
|
||||
"COMMENTS: createComment is always inline and requires an EXACT selection — contiguous text from a single block, <=250 chars (fails rather than leaving an unanchored comment); reply to a thread via parentCommentId. Propose a concrete text fix for one-click human approval -> createComment with suggestedText (the exact plain-text replacement for the selection; the selection must then be UNIQUE in the page — extend it with context if needed); prefer this over editing directly when the change is subjective or needs the author's sign-off. Manage -> listComments, updateComment, resolveComment (resolve/reopen, reversible — prefer over delete to close), deleteComment, checkNewComments.\n" +
|
||||
@@ -72,6 +72,12 @@ export const PROSE_NON_TOOL_TERMS: ReadonlySet<string> = new Set([
|
||||
"parentCommentId",
|
||||
"suggestedText",
|
||||
"historyId",
|
||||
// search RESPONSE fields documented in the routing prose (#529) — schema
|
||||
// fields the search tool returns, not tools themselves
|
||||
"matchedTerms",
|
||||
"matchedFields",
|
||||
"hasMore",
|
||||
"truncatedAtCap",
|
||||
// helper / value fragments
|
||||
"orderedList", // "orderedList.type" (a dropped attr, not a tool)
|
||||
"mxGraph", // "mxGraph XML"
|
||||
@@ -234,7 +240,7 @@ export const INLINE_MCP_INVENTORY: ToolInventoryLine[] = [
|
||||
{
|
||||
name: "search",
|
||||
purpose:
|
||||
"find pages by a fragment of a technical string (hybrid substring + full-text); returns each hit's path and a snippet.",
|
||||
"search pages across the wiki (OR default, RU+EN morphology, \"phrase\"/+/- operators, pagination); returns each hit's path, snippet, score and matched terms.",
|
||||
},
|
||||
{
|
||||
name: "docmostTransform",
|
||||
|
||||
@@ -175,3 +175,26 @@ test("#494: PROSE_NON_TOOL_TERMS holds no actually-registered tool name", () =>
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// #529: the search routing prose must document the new engine contract — the
|
||||
// operators, OR/morphology default, pagination fields and the relevance-CAP
|
||||
// caveat — so an agent uses the operators and understands the unreachable tail.
|
||||
test("SERVER_INSTRUCTIONS documents the #529 search operators, pagination and CAP", () => {
|
||||
const read = ROUTING_PROSE.split("EDIT:")[0]; // the READ family section
|
||||
// Operators.
|
||||
assert.ok(/\+require/.test(read), "search prose missing +require operator");
|
||||
assert.ok(/-exclude/.test(read), "search prose missing -exclude operator");
|
||||
assert.ok(/phrase/i.test(read), "search prose missing phrase operator");
|
||||
// OR default + morphology.
|
||||
assert.ok(/\bOR\b/.test(read), "search prose missing OR-default note");
|
||||
assert.ok(/morpholog/i.test(read), "search prose missing morphology note");
|
||||
// Pagination + exact permission-filtered total.
|
||||
assert.ok(/offset/.test(read), "search prose missing offset/pagination");
|
||||
assert.ok(/total is exact/i.test(read), "search prose missing exact total");
|
||||
assert.ok(/hasMore|truncatedAtCap/.test(read), "search prose missing hasMore/cap flags");
|
||||
// The relevance CAP caveat (tail unreachable by pagination).
|
||||
assert.ok(
|
||||
/cap/i.test(read) && /unreachable/i.test(read),
|
||||
"search prose missing the relevance-CAP unreachable-tail caveat",
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user