feat(search) Фаза A: лексический оверхол — ru_en, OR/RRF, операторы, пагинация (#529) #538
@@ -1,6 +1,6 @@
|
||||
import { Spotlight } from "@mantine/spotlight";
|
||||
import { IconSearch } from "@tabler/icons-react";
|
||||
import { Group, VisuallyHidden } from "@mantine/core";
|
||||
import { Group, Text, VisuallyHidden } from "@mantine/core";
|
||||
import { useState, useMemo } from "react";
|
||||
import { useDebouncedValue } from "@mantine/hooks";
|
||||
import { useTranslation } from "react-i18next";
|
||||
@@ -84,6 +84,11 @@ export function SearchSpotlight({ spaceId }: SearchSpotlightProps) {
|
||||
onFiltersChange={handleFiltersChange}
|
||||
spaceId={spaceId}
|
||||
/>
|
||||
{/* #529: operator hint — matches ANY word by default; "…" for an exact
|
||||
phrase, +term to require, -term to exclude. */}
|
||||
<Text size="xs" c="dimmed" mt={4}>
|
||||
{t('Tip: "exact phrase", +required, -excluded')}
|
||||
</Text>
|
||||
</div>
|
||||
|
||||
<VisuallyHidden role="status" aria-live="polite">
|
||||
|
||||
@@ -5,6 +5,9 @@ import { IPage } from "@/features/page/types/page.types.ts";
|
||||
|
||||
export interface IPageSearch {
|
||||
id: string;
|
||||
// #529 A7 superset: `pageId` aliases `id`; `rank`/`highlight` are null for
|
||||
// substring-only hits (the UI already falls back to the title/snippet).
|
||||
pageId?: string;
|
||||
title: string;
|
||||
icon: string;
|
||||
parentPageId: string;
|
||||
@@ -12,9 +15,36 @@ export interface IPageSearch {
|
||||
creatorId: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
rank: string;
|
||||
highlight: string;
|
||||
rank: string | number | null;
|
||||
highlight: string | null;
|
||||
space: Partial<ISpace>;
|
||||
// New #529 fields (present from the native Postgres search driver).
|
||||
snippet?: string;
|
||||
score?: number;
|
||||
path?: string[];
|
||||
matchedFields?: string[];
|
||||
matchedTerms?: string[];
|
||||
}
|
||||
|
||||
// #529 A5 pagination envelope returned by POST /search (native driver). The web
|
||||
// list helpers read `items`; these travel alongside for pagination + diagnostics.
|
||||
export interface IPageSearchResponse {
|
||||
items: IPageSearch[];
|
||||
total: number;
|
||||
hasMore: boolean;
|
||||
truncatedAtCap: boolean;
|
||||
offset: number;
|
||||
query?: {
|
||||
raw: string;
|
||||
parsed: {
|
||||
positive: string[];
|
||||
required: string[];
|
||||
excluded: string[];
|
||||
reason?: string;
|
||||
};
|
||||
mode: "or" | "and";
|
||||
match: string;
|
||||
};
|
||||
}
|
||||
|
||||
export interface SearchSuggestionParams {
|
||||
@@ -37,6 +67,10 @@ export interface IPageSearchParams {
|
||||
query: string;
|
||||
spaceId?: string;
|
||||
shareId?: string;
|
||||
// #529 A9: match mode (auto default) + pagination.
|
||||
match?: "auto" | "word" | "prefix" | "substring";
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
export interface IAttachmentSearch {
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"migration:reset": "tsx src/database/migrate.ts down-to NO_MIGRATIONS",
|
||||
"migration:codegen": "kysely-codegen --dialect=postgres --camel-case --env-file=../../.env --out-file=./src/database/types/db.d.ts",
|
||||
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
||||
"pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build && pnpm --filter @docmost/token-estimate build",
|
||||
"pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build && pnpm --filter @docmost/token-estimate build && pnpm --filter @docmost/mcp build",
|
||||
"test": "jest",
|
||||
"test:int": "jest --config test/jest-integration.json",
|
||||
"test:watch": "jest --watch",
|
||||
|
||||
@@ -1,33 +1,56 @@
|
||||
import { Space } from '@docmost/db/types/entity.types';
|
||||
|
||||
export class SearchResponseDto {
|
||||
// #529 A7 — the single per-hit SUPERSET returned by the unified search engine.
|
||||
// The web-UI reads id/highlight/icon/space/title/…; the MCP agent maps id→pageId
|
||||
// and reads snippet/score/path. `rank`/`highlight` are null for substring-only
|
||||
// hits (the web already falls back). Nothing the legacy web response carried is
|
||||
// dropped.
|
||||
export class SearchResultDto {
|
||||
id: string;
|
||||
title: string;
|
||||
// Alias of `id` for the MCP layer (it addresses pages by pageId).
|
||||
pageId: string;
|
||||
slugId: string;
|
||||
icon: string;
|
||||
parentPageId: string;
|
||||
title: string;
|
||||
space?: Partial<Space>;
|
||||
creatorId: string;
|
||||
rank: number;
|
||||
highlight: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
space: Partial<Space>;
|
||||
// ts_rank_cd of the FTS branch; null for substring-only hits.
|
||||
rank: number | null;
|
||||
// ts_headline marked HTML; null for substring-only hits.
|
||||
highlight: string | null;
|
||||
// Plain windowed snippet around the match (empty for titleOnly).
|
||||
snippet: string;
|
||||
// Ancestor titles root → direct parent ([] for a root page).
|
||||
path: string[];
|
||||
// Per-response ordering proxy (falls back to rank).
|
||||
score: number;
|
||||
// Which fields matched: 'title' and/or 'text'.
|
||||
matchedFields: string[];
|
||||
// Which parsed positive/required terms this hit matched.
|
||||
matchedTerms: string[];
|
||||
}
|
||||
|
||||
// Response shape for the opt-in agent-lookup mode (#443, `substring: true`).
|
||||
// Additive to the FTS response: carries the location (`path`), a windowed
|
||||
// `snippet` around the first match and a per-response sort `score`. The MCP
|
||||
// layer maps `id → pageId`; `slugId` is never exposed.
|
||||
export class SearchLookupResponseDto {
|
||||
id: string;
|
||||
slugId: string;
|
||||
title: string;
|
||||
parentPageId: string | null;
|
||||
// Ancestor titles from the space root down to the direct parent; [] for a
|
||||
// root page.
|
||||
path: string[];
|
||||
// ~300–500 chars around the first match (or a leading text window / extended
|
||||
// ts_headline fallback).
|
||||
snippet: string;
|
||||
// 0..1 float, meaningful ONLY for sorting within one response.
|
||||
score: number;
|
||||
// The paginated envelope (A5). `total` is the EXACT permission-filtered count of
|
||||
// pages matching the positive lexical query (fail-closed). `hasMore` is true when
|
||||
// more results exist WITHIN the fusion window; `truncatedAtCap` signals the match
|
||||
// set exceeded CANDIDATE_CAP and the tail is unreachable by pagination.
|
||||
export class SearchResponseDto {
|
||||
items: SearchResultDto[];
|
||||
total: number;
|
||||
hasMore: boolean;
|
||||
truncatedAtCap: boolean;
|
||||
offset: number;
|
||||
query: {
|
||||
raw: string;
|
||||
parsed: {
|
||||
positive: string[];
|
||||
required: string[];
|
||||
excluded: string[];
|
||||
reason?: string;
|
||||
};
|
||||
mode: 'or' | 'and';
|
||||
match: string;
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,16 +1,30 @@
|
||||
import {
|
||||
IsBoolean,
|
||||
IsIn,
|
||||
IsNotEmpty,
|
||||
IsNumber,
|
||||
IsOptional,
|
||||
IsString,
|
||||
MaxLength,
|
||||
} from 'class-validator';
|
||||
|
||||
export class SearchDTO {
|
||||
// Defense-in-depth cap on the raw query length. The real stack-depth bound is
|
||||
// the parser's MAX_PARSED_TERMS term cap (see search-query-parser.ts); this
|
||||
// just rejects absurd payloads early. 10k chars still comfortably holds any
|
||||
// legitimate query.
|
||||
@IsNotEmpty()
|
||||
@IsString()
|
||||
@MaxLength(10000)
|
||||
query: string;
|
||||
|
||||
// #529 A3 — match mode. `auto` (default) routes identifier-like terms
|
||||
// (10.31.41, esp32, WB-MGE-30D86B) to the substring/trigram branch and words
|
||||
// to full-text; `word`/`prefix`/`substring` are explicit overrides.
|
||||
@IsOptional()
|
||||
@IsIn(['auto', 'word', 'prefix', 'substring'])
|
||||
match?: 'auto' | 'word' | 'prefix' | 'substring';
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
spaceId: string;
|
||||
@@ -33,15 +47,19 @@ export class SearchDTO {
|
||||
|
||||
// --- Opt-in agent-lookup mode (#443). ------------------------------------
|
||||
// These fields are ADDITIVE and default-off: a web client that sends none of
|
||||
// them gets byte-identical FTS behaviour and result shape. They are only read
|
||||
// by the substring/path/snippet code path in SearchService.searchPage.
|
||||
// them gets byte-identical FTS behaviour and result shape. In the unified #529
|
||||
// engine, `parentPageId` and `titleOnly` are read by SearchService.searchPage
|
||||
// (subtree scoping and title-only matching, respectively). `substring` is NOT
|
||||
// read by the native driver — it is accepted-but-ignored, kept only for
|
||||
// back-compat with the upstream lookup request shape.
|
||||
//
|
||||
// NOTE (standalone stdio vs stock upstream): stock upstream validates this DTO
|
||||
// with `whitelist: true`, so an older server silently strips these unknown
|
||||
// fields and the request degrades gracefully to the plain FTS behaviour.
|
||||
|
||||
// Enables the hybrid substring branch (title + text_content LIKE) merged with
|
||||
// the existing FTS branch, plus tiered ranking, path and windowed snippet.
|
||||
// Accepted-but-ignored by the #529 native driver (kept for upstream lookup
|
||||
// back-compat). The unified engine ALWAYS runs the hybrid FTS + substring/
|
||||
// trigram branches with tiered ranking, so this flag no longer toggles anything.
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
substring?: boolean;
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
import {
|
||||
parseSearchQuery,
|
||||
hasPositiveRecall,
|
||||
MAX_PARSED_TERMS,
|
||||
} from './search-query-parser';
|
||||
|
||||
describe('parseSearchQuery — tokenization & operators (A2)', () => {
|
||||
it('splits on whitespace into positive terms (OR recall)', () => {
|
||||
const p = parseSearchQuery('стамбул роснефть');
|
||||
expect(p.positive.map((t) => t.text)).toEqual(['стамбул', 'роснефть']);
|
||||
expect(p.required).toEqual([]);
|
||||
expect(p.excluded).toEqual([]);
|
||||
expect(p.mode).toBe('or');
|
||||
});
|
||||
|
||||
it('treats +term as required and -term as excluded', () => {
|
||||
const p = parseSearchQuery('+кофейня -архив');
|
||||
expect(p.required.map((t) => t.text)).toEqual(['кофейня']);
|
||||
expect(p.excluded.map((t) => t.text)).toEqual(['архив']);
|
||||
expect(p.positive).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps a leading-operator-free hyphen/dot/colon token as ONE literal term', () => {
|
||||
// WB-MGE-30D86B, 10.0.12.5, a:b — internal -,.,: are literal, one term each.
|
||||
expect(parseSearchQuery('WB-MGE-30D86B').positive[0].text).toBe(
|
||||
'WB-MGE-30D86B',
|
||||
);
|
||||
expect(parseSearchQuery('10.0.12.5').positive[0].text).toBe('10.0.12.5');
|
||||
expect(parseSearchQuery('host:8080').positive[0].text).toBe('host:8080');
|
||||
});
|
||||
|
||||
it('only a LEADING +/- is an operator; -архив excludes архив', () => {
|
||||
const p = parseSearchQuery('-архив');
|
||||
expect(p.excluded.map((t) => t.text)).toEqual(['архив']);
|
||||
expect(p.positive).toEqual([]);
|
||||
});
|
||||
|
||||
it('drops a bare "-" / "+" and an all-operator remainder', () => {
|
||||
const p = parseSearchQuery('- + foo -- ++');
|
||||
expect(p.positive.map((t) => t.text)).toEqual(['foo']);
|
||||
expect(p.required).toEqual([]);
|
||||
expect(p.excluded).toEqual([]);
|
||||
});
|
||||
|
||||
it('parses a quoted phrase as one adjacency term', () => {
|
||||
const p = parseSearchQuery('"воздушный шар" кофе');
|
||||
expect(p.positive[0]).toMatchObject({ text: 'воздушный шар', branch: 'phrase' });
|
||||
expect(p.positive[1].text).toBe('кофе');
|
||||
});
|
||||
|
||||
it('applies +/- to a phrase', () => {
|
||||
const req = parseSearchQuery('+"воздушный шар"');
|
||||
expect(req.required[0]).toMatchObject({ text: 'воздушный шар', branch: 'phrase' });
|
||||
const exc = parseSearchQuery('-"воздушный шар"');
|
||||
expect(exc.excluded[0]).toMatchObject({ text: 'воздушный шар', branch: 'phrase' });
|
||||
});
|
||||
|
||||
it('drops an unbalanced quote token', () => {
|
||||
const p = parseSearchQuery('kafka "unclosed here');
|
||||
expect(p.positive.map((t) => t.text)).toEqual(['kafka']);
|
||||
});
|
||||
|
||||
it('strips tsquery metacharacters from a bare FTS term (no 500)', () => {
|
||||
const p = parseSearchQuery('foo|bar');
|
||||
// `|` is not an identifier signal → FTS branch; the metachar is stripped so
|
||||
// the term becomes the two words that survive.
|
||||
expect(p.positive[0].branch === 'fts' || p.positive[0].branch === 'ftsPrefix').toBe(true);
|
||||
expect(p.positive[0].text).toBe('foo bar');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSearchQuery — match=auto classification (A3)', () => {
|
||||
it('routes identifier-like terms to the substring branch', () => {
|
||||
expect(parseSearchQuery('10.31.41').positive[0].branch).toBe('substring');
|
||||
expect(parseSearchQuery('esp32').positive[0].branch).toBe('substring');
|
||||
expect(parseSearchQuery('WB-MGE-30D86B').positive[0].branch).toBe('substring');
|
||||
});
|
||||
|
||||
it('routes purely-alphabetic words to the FTS (prefix) branch', () => {
|
||||
expect(parseSearchQuery('печат').positive[0].branch).toBe('ftsPrefix');
|
||||
expect(parseSearchQuery('ресторан').positive[0].branch).toBe('ftsPrefix');
|
||||
});
|
||||
|
||||
it('explicit match overrides: word / prefix / substring', () => {
|
||||
expect(parseSearchQuery('печат', { match: 'word' }).positive[0].branch).toBe('fts');
|
||||
expect(parseSearchQuery('печат', { match: 'prefix' }).positive[0].branch).toBe(
|
||||
'ftsPrefix',
|
||||
);
|
||||
expect(parseSearchQuery('печат', { match: 'substring' }).positive[0].branch).toBe(
|
||||
'substring',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSearchQuery — reasons & recall', () => {
|
||||
it('only-negation yields reason only-negation and no positive recall', () => {
|
||||
const p = parseSearchQuery('-архив');
|
||||
expect(p.reason).toBe('only-negation');
|
||||
expect(hasPositiveRecall(p)).toBe(false);
|
||||
});
|
||||
|
||||
it('empty / whitespace / garbage yields reason empty', () => {
|
||||
expect(parseSearchQuery('').reason).toBe('empty');
|
||||
expect(parseSearchQuery(' ').reason).toBe('empty');
|
||||
// A bare operator drops to nothing → empty (no exclusion survived).
|
||||
expect(parseSearchQuery('+ -').reason).toBe('empty');
|
||||
});
|
||||
|
||||
it('a required term alone IS positive recall (no reason)', () => {
|
||||
const p = parseSearchQuery('+кофейня');
|
||||
expect(p.reason).toBeUndefined();
|
||||
expect(hasPositiveRecall(p)).toBe(true);
|
||||
});
|
||||
|
||||
it('mode flag flows through', () => {
|
||||
expect(parseSearchQuery('a b', { mode: 'and' }).mode).toBe('and');
|
||||
expect(parseSearchQuery('a b').mode).toBe('or');
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseSearchQuery — term cap (stack-depth guard)', () => {
|
||||
it('caps the total parsed terms at MAX_PARSED_TERMS without throwing', () => {
|
||||
// A pasted text block: far more words than the cap. The parser must bound the
|
||||
// SQL tsquery nesting depth (else Postgres blows its stack → HTTP 500).
|
||||
const words = Array.from({ length: 5000 }, (_, i) => `w${i}`);
|
||||
let p!: ReturnType<typeof parseSearchQuery>;
|
||||
expect(() => {
|
||||
p = parseSearchQuery(words.join(' '));
|
||||
}).not.toThrow();
|
||||
const total = p.positive.length + p.required.length + p.excluded.length;
|
||||
expect(total).toBe(MAX_PARSED_TERMS);
|
||||
// Stable order: the FIRST cap terms are kept.
|
||||
expect(p.positive.slice(0, 3).map((t) => t.text)).toEqual(['w0', 'w1', 'w2']);
|
||||
expect(p.positive[MAX_PARSED_TERMS - 1].text).toBe(`w${MAX_PARSED_TERMS - 1}`);
|
||||
});
|
||||
|
||||
it('counts positive + required + excluded together toward the cap', () => {
|
||||
// Interleave operators so overflow can fall on any bucket. Total must still
|
||||
// never exceed the cap, and the first cap terms (in stable order) win.
|
||||
const tokens: string[] = [];
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const op = i % 3 === 0 ? '+' : i % 3 === 1 ? '-' : '';
|
||||
tokens.push(`${op}t${i}`);
|
||||
}
|
||||
const p = parseSearchQuery(tokens.join(' '));
|
||||
const total = p.positive.length + p.required.length + p.excluded.length;
|
||||
expect(total).toBe(MAX_PARSED_TERMS);
|
||||
// t0 (+, required) and t1 (-, excluded) and t2 (bare, positive) are all within
|
||||
// the first cap tokens → each bucket got its leading terms.
|
||||
expect(p.required[0].text).toBe('t0');
|
||||
expect(p.excluded[0].text).toBe('t1');
|
||||
expect(p.positive[0].text).toBe('t2');
|
||||
});
|
||||
|
||||
it('leaves a normal (<= cap) query completely unchanged', () => {
|
||||
const p = parseSearchQuery('+кофейня -архив "воздушный шар" ресторан 10.31.41');
|
||||
expect(p.required.map((t) => t.text)).toEqual(['кофейня']);
|
||||
expect(p.excluded.map((t) => t.text)).toEqual(['архив']);
|
||||
expect(p.positive.map((t) => t.text)).toEqual([
|
||||
'воздушный шар',
|
||||
'ресторан',
|
||||
'10.31.41',
|
||||
]);
|
||||
// Phrase / substring branch handling survives under the cap.
|
||||
expect(p.positive[0].branch).toBe('phrase');
|
||||
expect(p.positive[2].branch).toBe('substring');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,242 @@
|
||||
// #529 Phase A — server-side query parser (the SINGLE source of query semantics).
|
||||
//
|
||||
// Clients (web-UI, MCP agent, public share) send a RAW query string plus flags
|
||||
// (`match`, `mode`); ALL query interpretation happens HERE, so every consumer
|
||||
// gets identical operator/phrase/morphology behaviour. The parser is PURE (no
|
||||
// SQL, no DB) so it is exhaustively unit-testable; SearchService turns the parsed
|
||||
// AST into a parameterized tsquery/predicate tree (never string-concatenated SQL).
|
||||
//
|
||||
// Grammar (A2):
|
||||
// - Whitespace splits tokens, but a double-quoted run is ONE token ("a b" is a
|
||||
// phrase). An unbalanced quote is dropped.
|
||||
// - A token is an OPERATOR token only when it STARTS with `+` or `-`, the
|
||||
// remainder is non-empty, and the remainder is not itself only operators.
|
||||
// `+`/`-` inside a token (`WB-MGE-30D86B`, `10.0.12.5`, `a:b`) is a LITERAL —
|
||||
// the token is a single term. A bare `-`/`+` is dropped.
|
||||
// - `"phrase"` → phrase term; `+"phrase"`/`-"phrase"` apply the operator to it.
|
||||
// - Positive terms (bare + phrase, no operator) form the OR recall set.
|
||||
// `+term` is a REQUIRED predicate, `-term` an EXCLUDED predicate (A2): both
|
||||
// are applied in SQL WHERE against the whole candidate set, not folded into
|
||||
// the positive tsquery.
|
||||
// - Only-negation (no positive term) short-circuits to an empty result with
|
||||
// reason `only-negation` (never runs a costly NOT-scan).
|
||||
|
||||
export type SearchMatchMode = 'auto' | 'word' | 'prefix' | 'substring';
|
||||
export type SearchBooleanMode = 'or' | 'and';
|
||||
|
||||
// Hard cap on the total number of parsed terms (positive + required + excluded).
|
||||
// SearchService folds each FTS term into a LEFT-NESTED SQL tsquery expression
|
||||
// `(((t1)||(t2))||(t3))…`, embedded several times per query — so nesting depth
|
||||
// grows with the term count. A pasted text block (thousands of words) would nest
|
||||
// deep enough to blow Postgres' `stack depth limit` → an ERROR → HTTP 500 for the
|
||||
// caller. Capping in the parser bounds that depth for EVERY consumer (web / MCP /
|
||||
// share), since they all route through here. 64 comfortably covers any real query
|
||||
// while keeping the SQL nesting shallow. Overflow terms (beyond the first 64, in
|
||||
// stable order) are dropped rather than throwing.
|
||||
export const MAX_PARSED_TERMS = 64;
|
||||
|
||||
// How a single term is matched against the index.
|
||||
// - 'fts' : full-text lexeme, exact (no trailing prefix).
|
||||
// - 'ftsPrefix' : full-text lexeme with a `:*` prefix match.
|
||||
// - 'phrase' : an adjacency phrase (phraseto_tsquery).
|
||||
// - 'substring' : a literal LOWER(f_unaccent(col)) LIKE '%needle%' branch
|
||||
// (identifiers the tokenizer mangles: IPs, hostnames, IDs).
|
||||
export type SearchTermBranch = 'fts' | 'ftsPrefix' | 'phrase' | 'substring';
|
||||
|
||||
export interface ParsedTerm {
|
||||
// The user-visible term text, operator stripped, quotes removed. This is what
|
||||
// `matchedTerms` echoes back per hit.
|
||||
text: string;
|
||||
branch: SearchTermBranch;
|
||||
}
|
||||
|
||||
export interface ParsedQuery {
|
||||
raw: string;
|
||||
// OR-recall set (bare + phrase terms with no operator).
|
||||
positive: ParsedTerm[];
|
||||
// AND predicates (`+term`) — the candidate MUST match each of these.
|
||||
required: ParsedTerm[];
|
||||
// NOT predicates (`-term`) — the candidate must match NONE of these.
|
||||
excluded: ParsedTerm[];
|
||||
mode: SearchBooleanMode;
|
||||
// Set only when the query yields no positive recall: 'empty' (nothing usable)
|
||||
// or 'only-negation' (there were exclusions but no positive term).
|
||||
reason?: 'empty' | 'only-negation';
|
||||
}
|
||||
|
||||
interface RawToken {
|
||||
op: '' | '+' | '-';
|
||||
kind: 'word' | 'phrase';
|
||||
text: string;
|
||||
}
|
||||
|
||||
// tsquery metacharacters that must never reach to_tsquery from a bare term — they
|
||||
// are what turned adversarial input into a 500 before (#139). Stripped for the FTS
|
||||
// branch; the substring branch keeps them (they are literal there).
|
||||
const TSQUERY_META = /[:&|!()*<>\\]+/g;
|
||||
|
||||
// A term is "identifier-like" when it carries a digit or one of . _ : / - AND is
|
||||
// not purely alphabetic (letters only). Such tokens (10.31.41, esp32,
|
||||
// WB-MGE-30D86B) are mangled by the FTS tokenizer, so `match: auto` routes them
|
||||
// to the substring branch. A purely-alphabetic word (печат, ресторан) stays FTS.
|
||||
const IDENTIFIER_SIGNAL = /[0-9._:/\\-]/;
|
||||
const PURELY_ALPHA = /^\p{L}+$/u;
|
||||
|
||||
function isIdentifierLike(text: string): boolean {
|
||||
return IDENTIFIER_SIGNAL.test(text) && !PURELY_ALPHA.test(text);
|
||||
}
|
||||
|
||||
// Clean a bare term for the FTS branch: NFC-normalize, drop tsquery metacharacters
|
||||
// and collapse whitespace. Returns '' when nothing usable remains.
|
||||
export function cleanFtsLexeme(raw: string): string {
|
||||
return (raw ?? '')
|
||||
.normalize('NFC')
|
||||
.replace(TSQUERY_META, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
// Split a raw query into tokens, honouring double quotes and a single leading
|
||||
// +/- operator. Unbalanced quotes and bare operators are dropped.
|
||||
function tokenize(raw: string): RawToken[] {
|
||||
const tokens: RawToken[] = [];
|
||||
const s = raw ?? '';
|
||||
let i = 0;
|
||||
const n = s.length;
|
||||
|
||||
const isSpace = (c: string) => /\s/.test(c);
|
||||
|
||||
while (i < n) {
|
||||
// Skip leading whitespace.
|
||||
while (i < n && isSpace(s[i])) i++;
|
||||
if (i >= n) break;
|
||||
|
||||
let op: '' | '+' | '-' = '';
|
||||
// A single leading +/- is a tentative operator. Only ONE leading operator is
|
||||
// consumed; a second (`--x`) leaves `-x` as the remainder (a literal dash).
|
||||
if (s[i] === '+' || s[i] === '-') {
|
||||
op = s[i] as '+' | '-';
|
||||
i++;
|
||||
}
|
||||
|
||||
if (i < n && s[i] === '"') {
|
||||
// Quoted phrase: read until the closing quote.
|
||||
const close = s.indexOf('"', i + 1);
|
||||
if (close === -1) {
|
||||
// Unbalanced quote → drop this token and everything the open quote would
|
||||
// have consumed (the rest of the string).
|
||||
break;
|
||||
}
|
||||
const phrase = s.slice(i + 1, close);
|
||||
i = close + 1;
|
||||
if (phrase.trim().length > 0) {
|
||||
tokens.push({ op, kind: 'phrase', text: phrase.trim() });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Bare word: read until the next whitespace.
|
||||
let j = i;
|
||||
while (j < n && !isSpace(s[j])) j++;
|
||||
const word = s.slice(i, j);
|
||||
i = j;
|
||||
|
||||
// A bare operator (`-`/`+` with no remainder) or an all-operator remainder is
|
||||
// dropped.
|
||||
if (word.length === 0) continue;
|
||||
if (op && /^[+-]+$/.test(word)) continue;
|
||||
|
||||
tokens.push({ op, kind: 'word', text: word });
|
||||
}
|
||||
|
||||
return tokens;
|
||||
}
|
||||
|
||||
// Resolve the match branch for a single term given the global match mode.
|
||||
function branchForTerm(
|
||||
text: string,
|
||||
kind: 'word' | 'phrase',
|
||||
mode: SearchMatchMode,
|
||||
): SearchTermBranch {
|
||||
if (kind === 'phrase') return 'phrase';
|
||||
switch (mode) {
|
||||
case 'word':
|
||||
return 'fts';
|
||||
case 'prefix':
|
||||
return 'ftsPrefix';
|
||||
case 'substring':
|
||||
return 'substring';
|
||||
case 'auto':
|
||||
default:
|
||||
// Identifiers the tokenizer mangles go to substring; words get a prefix
|
||||
// FTS match (so `печат` still finds `печатать`, but `печат` no longer drags
|
||||
// in `впечатления` because the russian stemmer anchors the stem).
|
||||
return isIdentifierLike(text) ? 'substring' : 'ftsPrefix';
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a raw user query + flags into a structured, SQL-agnostic AST.
|
||||
* Pure and total: never throws, always returns a ParsedQuery.
|
||||
*/
|
||||
export function parseSearchQuery(
|
||||
raw: string,
|
||||
opts: { match?: SearchMatchMode; mode?: SearchBooleanMode } = {},
|
||||
): ParsedQuery {
|
||||
const match: SearchMatchMode = opts.match ?? 'auto';
|
||||
const mode: SearchBooleanMode = opts.mode ?? 'or';
|
||||
|
||||
const positive: ParsedTerm[] = [];
|
||||
const required: ParsedTerm[] = [];
|
||||
const excluded: ParsedTerm[] = [];
|
||||
|
||||
for (const tok of tokenize(raw)) {
|
||||
// For an FTS branch, the token must survive metacharacter cleaning; for the
|
||||
// substring/phrase branch the literal text is used. A term that cleans to
|
||||
// nothing AND is not usable as a substring is dropped.
|
||||
const branch = branchForTerm(tok.text, tok.kind, match);
|
||||
|
||||
let usableText: string;
|
||||
if (branch === 'fts' || branch === 'ftsPrefix') {
|
||||
usableText = cleanFtsLexeme(tok.text);
|
||||
} else {
|
||||
// phrase / substring keep the literal (trimmed) text.
|
||||
usableText = tok.text.trim();
|
||||
}
|
||||
if (!usableText) continue;
|
||||
|
||||
// Stack-depth guard: stop after MAX_PARSED_TERMS surviving terms (positive +
|
||||
// required + excluded, combined) so the SQL tsquery nesting stays shallow.
|
||||
// The first 64 terms are kept in stable order; the rest are dropped.
|
||||
if (positive.length + required.length + excluded.length >= MAX_PARSED_TERMS) {
|
||||
break;
|
||||
}
|
||||
|
||||
const term: ParsedTerm = { text: usableText, branch };
|
||||
|
||||
if (tok.op === '+') required.push(term);
|
||||
else if (tok.op === '-') excluded.push(term);
|
||||
else positive.push(term);
|
||||
}
|
||||
|
||||
const parsed: ParsedQuery = { raw: raw ?? '', positive, required, excluded, mode };
|
||||
|
||||
if (positive.length === 0) {
|
||||
// Required terms with no positive recall still form a valid positive set (the
|
||||
// required predicates ARE the recall). Only when there is neither a positive
|
||||
// nor a required term is the query empty / only-negation.
|
||||
if (required.length === 0) {
|
||||
parsed.reason = excluded.length > 0 ? 'only-negation' : 'empty';
|
||||
}
|
||||
}
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Does this parsed query have any positive recall to run? False means we must
|
||||
* short-circuit to an empty result (with `reason`), never a costly NOT-only scan.
|
||||
*/
|
||||
export function hasPositiveRecall(parsed: ParsedQuery): boolean {
|
||||
return parsed.positive.length > 0 || parsed.required.length > 0;
|
||||
}
|
||||
@@ -1,19 +1,14 @@
|
||||
import {
|
||||
computeLookupScore,
|
||||
escapeLikePattern,
|
||||
SearchLookupTier,
|
||||
} from './search.service';
|
||||
import { escapeLikePattern } from './search.service';
|
||||
|
||||
/**
|
||||
* Pure-function coverage for the #443 agent-lookup helpers:
|
||||
* - escapeLikePattern: LIKE-metacharacter escaping so `%`/`_`/`\` are literals
|
||||
* (the acceptance-table requirement that a query of `%` or `_` does NOT match
|
||||
* everything);
|
||||
* - computeLookupScore: the tiered 0..1 ranking score, where a stronger tier
|
||||
* always outranks a weaker one regardless of the in-tier secondary signal.
|
||||
* Pure-function coverage for `escapeLikePattern` — LIKE-metacharacter escaping so
|
||||
* `%`/`_`/`\` are matched literally (the acceptance requirement that a query of
|
||||
* `%` or `_` does NOT match everything, #529 acceptance #10). The substring
|
||||
* branch's DB behaviour is covered by the integration spec.
|
||||
*
|
||||
* The DB-touching branch (substring UNION FTS, path CTE, snippet window) is
|
||||
* covered by the integration spec against the real schema.
|
||||
* NOTE (#529): the old tiered `computeLookupScore` was replaced by RRF rank
|
||||
* fusion in the unified engine, so its unit coverage moved to the integration
|
||||
* ordering tests; only the escaping helper remains a pure unit here.
|
||||
*/
|
||||
describe('escapeLikePattern', () => {
|
||||
it('escapes the LIKE metacharacters % _ and \\', () => {
|
||||
@@ -43,53 +38,3 @@ describe('escapeLikePattern', () => {
|
||||
expect(escapeLikePattern(null as any)).toBe('');
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeLookupScore', () => {
|
||||
it('keeps every score within (0, 1]', () => {
|
||||
for (const tier of [
|
||||
SearchLookupTier.TITLE_EXACT,
|
||||
SearchLookupTier.TITLE_SUBSTRING,
|
||||
SearchLookupTier.TEXT,
|
||||
]) {
|
||||
for (const secondary of [0, 0.001, 1, 100, 1e6]) {
|
||||
const s = computeLookupScore({ tier, secondary });
|
||||
expect(s).toBeGreaterThan(0);
|
||||
expect(s).toBeLessThanOrEqual(1);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('a stronger tier ALWAYS outranks a weaker tier, whatever the secondary', () => {
|
||||
// Weak tier with a huge secondary must still lose to a strong tier with a
|
||||
// tiny secondary — tiers dominate.
|
||||
const strongLowSecondary = computeLookupScore({
|
||||
tier: SearchLookupTier.TITLE_EXACT,
|
||||
secondary: 0,
|
||||
});
|
||||
const weakHighSecondary = computeLookupScore({
|
||||
tier: SearchLookupTier.TEXT,
|
||||
secondary: 1e9,
|
||||
});
|
||||
expect(strongLowSecondary).toBeGreaterThan(weakHighSecondary);
|
||||
});
|
||||
|
||||
it('within a tier a larger secondary sorts higher', () => {
|
||||
const lo = computeLookupScore({
|
||||
tier: SearchLookupTier.TEXT,
|
||||
secondary: 0.1,
|
||||
});
|
||||
const hi = computeLookupScore({
|
||||
tier: SearchLookupTier.TEXT,
|
||||
secondary: 5,
|
||||
});
|
||||
expect(hi).toBeGreaterThan(lo);
|
||||
});
|
||||
|
||||
it('treats a negative/absent secondary as 0', () => {
|
||||
const zero = computeLookupScore({ tier: SearchLookupTier.TEXT, secondary: 0 });
|
||||
expect(computeLookupScore({ tier: SearchLookupTier.TEXT })).toBe(zero);
|
||||
expect(
|
||||
computeLookupScore({ tier: SearchLookupTier.TEXT, secondary: -5 }),
|
||||
).toBe(zero);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,74 +1,45 @@
|
||||
import { SearchService } from './search.service';
|
||||
|
||||
/**
|
||||
* Coverage for SearchService.searchPage query-mode selection (search.service.ts
|
||||
* @25). searchPage chooses HOW the result set is scoped — by explicit space, by
|
||||
* the authenticated user's member spaces, or by a share — and must return an
|
||||
* empty set (without leaking data) for every disallowed combination.
|
||||
* Unit coverage for SearchService.searchPage SCOPE-SECURITY early returns — the
|
||||
* branches that must yield an empty result WITHOUT ever touching the DB, so they
|
||||
* can leak nothing. The happy-path scope SQL (explicit space / member spaces /
|
||||
* share id set) is covered against the real schema in the integration spec.
|
||||
*
|
||||
* The kysely query builder is mocked with the same chainable pattern as the
|
||||
* existing search.service.spec.ts: every builder method returns the same builder
|
||||
* and `.execute()` resolves the supplied rows. Each `.where(...)` call is
|
||||
* recorded so we can assert exactly which scope clause was applied — that is the
|
||||
* mutation-resistant signal that distinguishes one query mode from another.
|
||||
*
|
||||
* These specs catch cross-space / cross-workspace search leakage and
|
||||
* share-scope bypass (data exposure).
|
||||
* Every case here returns BEFORE the raw-SQL candidate query runs, so a bare `db`
|
||||
* stub (never called) is enough — a call to it would itself be a failure signal.
|
||||
*/
|
||||
describe('SearchService.searchPage — query-mode selection', () => {
|
||||
// Build a chainable selectFrom('pages') builder that records its calls. The
|
||||
// builder is returned from `db.selectFrom` and is the single object every
|
||||
// chained call mutates/returns, mirroring the existing spec's pattern.
|
||||
function makeBuilder(rows: Array<{ id: string; highlight?: string }>) {
|
||||
const builder: any = {};
|
||||
builder.select = jest.fn(() => builder);
|
||||
builder.where = jest.fn(() => builder);
|
||||
builder.$if = jest.fn(() => builder);
|
||||
builder.orderBy = jest.fn(() => builder);
|
||||
builder.limit = jest.fn(() => builder);
|
||||
builder.offset = jest.fn(() => builder);
|
||||
builder.execute = jest.fn(async () => rows);
|
||||
return builder;
|
||||
}
|
||||
|
||||
describe('SearchService.searchPage — scope-security early returns', () => {
|
||||
function makeService(opts?: {
|
||||
rows?: Array<{ id: string; highlight?: string }>;
|
||||
share?: any;
|
||||
isRestricted?: boolean;
|
||||
descendants?: Array<{ id: string }>;
|
||||
memberSpaceIds?: string[];
|
||||
}) {
|
||||
const builder = makeBuilder(opts?.rows ?? []);
|
||||
|
||||
const db: any = {
|
||||
selectFrom: jest.fn(() => builder),
|
||||
};
|
||||
|
||||
// `getUserSpaceIdsQuery` returns a sub-query object that searchPage passes
|
||||
// straight into `.where('spaceId', 'in', <subquery>)`. A sentinel is enough
|
||||
// to assert the user-scoped branch was taken.
|
||||
const userSpaceIdsQuery = { __userSpaceIdsQuery: true };
|
||||
// A db that THROWS if touched — these branches must not reach SQL.
|
||||
const db: any = new Proxy(
|
||||
{},
|
||||
{
|
||||
get() {
|
||||
throw new Error('db must not be touched on an empty-scope branch');
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const pageRepo = {
|
||||
// `.select((eb) => this.pageRepo.withSpace(eb))` — value ignored by stub.
|
||||
withSpace: jest.fn(() => ({ __withSpace: true })),
|
||||
getPageAndDescendantsExcludingRestricted: jest
|
||||
.fn()
|
||||
.mockResolvedValue(opts?.descendants ?? []),
|
||||
getPageAndDescendantsExcludingRestricted: jest.fn(),
|
||||
getPageAndDescendants: jest.fn(),
|
||||
};
|
||||
const shareRepo = {
|
||||
findById: jest.fn().mockResolvedValue(opts?.share ?? null),
|
||||
};
|
||||
const spaceMemberRepo = {
|
||||
getUserSpaceIdsQuery: jest.fn(() => userSpaceIdsQuery),
|
||||
getUserSpaceIds: jest.fn().mockResolvedValue(opts?.memberSpaceIds ?? []),
|
||||
};
|
||||
const pagePermissionRepo = {
|
||||
hasRestrictedAncestor: jest
|
||||
.fn()
|
||||
.mockResolvedValue(opts?.isRestricted ?? false),
|
||||
// Let everything through page-level permission filtering by default.
|
||||
filterAccessiblePageIds: jest
|
||||
.fn()
|
||||
.mockImplementation(async ({ pageIds }: { pageIds: string[] }) => pageIds),
|
||||
filterAccessiblePageIds: jest.fn(),
|
||||
};
|
||||
|
||||
const service = new SearchService(
|
||||
@@ -78,145 +49,81 @@ describe('SearchService.searchPage — query-mode selection', () => {
|
||||
spaceMemberRepo as any,
|
||||
pagePermissionRepo as any,
|
||||
);
|
||||
|
||||
return {
|
||||
service,
|
||||
db,
|
||||
builder,
|
||||
pageRepo,
|
||||
shareRepo,
|
||||
spaceMemberRepo,
|
||||
pagePermissionRepo,
|
||||
userSpaceIdsQuery,
|
||||
};
|
||||
return { service, pageRepo, shareRepo, spaceMemberRepo, pagePermissionRepo };
|
||||
}
|
||||
|
||||
const whereCallFor = (builder: any, column: any) =>
|
||||
builder.where.mock.calls.find((c: any[]) => c[0] === column);
|
||||
|
||||
it('returns {items:[]} for a blank query WITHOUT touching the DB', async () => {
|
||||
const { service, db } = makeService();
|
||||
|
||||
it('returns total:0 for a blank query WITHOUT touching the DB or any repo', async () => {
|
||||
const { service, shareRepo, spaceMemberRepo } = makeService();
|
||||
const result = await service.searchPage(
|
||||
{ query: '' } as any,
|
||||
{ userId: 'user-1', workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ items: [] });
|
||||
// Blank query is rejected before any query builder is constructed.
|
||||
expect(db.selectFrom).not.toHaveBeenCalled();
|
||||
expect(result.items).toEqual([]);
|
||||
expect(result.total).toBe(0);
|
||||
expect(shareRepo.findById).not.toHaveBeenCalled();
|
||||
expect(spaceMemberRepo.getUserSpaceIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('scopes to the explicit spaceId branch', async () => {
|
||||
const { service, builder, db, spaceMemberRepo, shareRepo } = makeService({
|
||||
rows: [{ id: 'p-1' }],
|
||||
});
|
||||
|
||||
it('only-negation short-circuits with reason "only-negation", never scanning', async () => {
|
||||
const { service, spaceMemberRepo } = makeService();
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan', spaceId: 'space-42' } as any,
|
||||
{ query: '-архив' } as any,
|
||||
{ userId: 'user-1', workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(db.selectFrom).toHaveBeenCalledWith('pages');
|
||||
// The explicit-space branch adds exactly `.where('spaceId', '=', 'space-42')`.
|
||||
expect(whereCallFor(builder, 'spaceId')).toEqual([
|
||||
'spaceId',
|
||||
'=',
|
||||
'space-42',
|
||||
]);
|
||||
// It must NOT fall through to the user-member-spaces or share branch.
|
||||
expect(spaceMemberRepo.getUserSpaceIdsQuery).not.toHaveBeenCalled();
|
||||
expect(shareRepo.findById).not.toHaveBeenCalled();
|
||||
expect(result.items.map((i: any) => i.id)).toEqual(['p-1']);
|
||||
expect(result.total).toBe(0);
|
||||
expect(result.query.parsed.reason).toBe('only-negation');
|
||||
// Never resolves scope (returns before) — no expensive NOT-only scan.
|
||||
expect(spaceMemberRepo.getUserSpaceIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('scopes an authenticated user WITHOUT spaceId to their member spaces', async () => {
|
||||
const { service, builder, spaceMemberRepo, userSpaceIdsQuery, shareRepo } =
|
||||
makeService({ rows: [{ id: 'p-9' }] });
|
||||
|
||||
await service.searchPage(
|
||||
{ query: 'plan' } as any,
|
||||
{ userId: 'user-7', workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
// The user-scoped branch resolves the member-spaces sub-query for that user
|
||||
// and restricts both spaceId (to that sub-query) and workspaceId.
|
||||
expect(spaceMemberRepo.getUserSpaceIdsQuery).toHaveBeenCalledWith('user-7');
|
||||
expect(whereCallFor(builder, 'spaceId')).toEqual([
|
||||
'spaceId',
|
||||
'in',
|
||||
userSpaceIdsQuery,
|
||||
]);
|
||||
expect(whereCallFor(builder, 'workspaceId')).toEqual([
|
||||
'workspaceId',
|
||||
'=',
|
||||
'ws-1',
|
||||
]);
|
||||
// Authenticated user path must not consult shares.
|
||||
expect(shareRepo.findById).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns {items:[]} when the share belongs to a DIFFERENT workspace', async () => {
|
||||
const { service, builder, shareRepo, pagePermissionRepo } = makeService({
|
||||
share: {
|
||||
id: 'share-1',
|
||||
pageId: 'page-1',
|
||||
workspaceId: 'OTHER-ws',
|
||||
includeSubPages: false,
|
||||
},
|
||||
it('returns empty when the share belongs to a DIFFERENT workspace (no leak)', async () => {
|
||||
const { service, shareRepo, pagePermissionRepo } = makeService({
|
||||
share: { id: 's1', pageId: 'p1', workspaceId: 'OTHER', includeSubPages: false },
|
||||
});
|
||||
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan', shareId: 'share-1' } as any,
|
||||
{ query: 'plan', shareId: 's1' } as any,
|
||||
{ workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(shareRepo.findById).toHaveBeenCalledWith('share-1');
|
||||
expect(result).toEqual({ items: [] });
|
||||
// Workspace mismatch short-circuits before any restricted-ancestor / id
|
||||
// scoping or DB execution: no leak across workspaces.
|
||||
expect(shareRepo.findById).toHaveBeenCalledWith('s1');
|
||||
expect(result.items).toEqual([]);
|
||||
// Workspace mismatch short-circuits before restricted-ancestor / enumeration.
|
||||
expect(pagePermissionRepo.hasRestrictedAncestor).not.toHaveBeenCalled();
|
||||
expect(builder.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns {items:[]} when the shared page has a restricted ancestor', async () => {
|
||||
const { service, builder, pagePermissionRepo, pageRepo } = makeService({
|
||||
share: {
|
||||
id: 'share-1',
|
||||
pageId: 'page-1',
|
||||
workspaceId: 'ws-1',
|
||||
includeSubPages: true,
|
||||
},
|
||||
it('returns empty when the shared page has a restricted ancestor', async () => {
|
||||
const { service, pagePermissionRepo, pageRepo } = makeService({
|
||||
share: { id: 's1', pageId: 'p1', workspaceId: 'ws-1', includeSubPages: true },
|
||||
isRestricted: true,
|
||||
});
|
||||
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan', shareId: 'share-1' } as any,
|
||||
{ query: 'plan', shareId: 's1' } as any,
|
||||
{ workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(pagePermissionRepo.hasRestrictedAncestor).toHaveBeenCalledWith(
|
||||
'page-1',
|
||||
);
|
||||
expect(result).toEqual({ items: [] });
|
||||
// Restricted ancestor must block before page enumeration and DB execution.
|
||||
expect(pagePermissionRepo.hasRestrictedAncestor).toHaveBeenCalledWith('p1');
|
||||
expect(result.items).toEqual([]);
|
||||
expect(
|
||||
pageRepo.getPageAndDescendantsExcludingRestricted,
|
||||
).not.toHaveBeenCalled();
|
||||
expect(builder.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns {items:[]} with no userId, no spaceId and no shareId', async () => {
|
||||
const { service, builder, shareRepo } = makeService();
|
||||
|
||||
it('returns empty with no userId, no spaceId and no shareId', async () => {
|
||||
const { service, shareRepo } = makeService();
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan' } as any,
|
||||
{ workspaceId: 'ws-1' },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ items: [] });
|
||||
// The catch-all else returns empty without scoping/executing or hitting shares.
|
||||
expect(result.items).toEqual([]);
|
||||
expect(shareRepo.findById).not.toHaveBeenCalled();
|
||||
expect(builder.execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('an authenticated user with NO member spaces gets an empty result', async () => {
|
||||
const { service, spaceMemberRepo } = makeService({ memberSpaceIds: [] });
|
||||
const result = await service.searchPage(
|
||||
{ query: 'plan' } as any,
|
||||
{ userId: 'user-1', workspaceId: 'ws-1' },
|
||||
);
|
||||
expect(spaceMemberRepo.getUserSpaceIds).toHaveBeenCalledWith('user-1');
|
||||
expect(result.items).toEqual([]);
|
||||
expect(result.total).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { SearchService, buildTsQuery } from './search.service';
|
||||
import { SearchService } from './search.service';
|
||||
|
||||
describe('SearchService', () => {
|
||||
it('should be defined', () => {
|
||||
@@ -99,59 +99,3 @@ describe('SearchService.searchSuggestions — onlyTemplates filter', () => {
|
||||
expect(isTemplateWhereCall(pageBuilder)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// Unit tests for `buildTsQuery` (extracted from search.service.ts). It turns a raw
|
||||
// user query into a prefix tsquery string fed to `to_tsquery('english', ...)`.
|
||||
//
|
||||
// REAL BUG (Gitea #139, item 10): the previous inline `tsquery(query.trim() + '*')`
|
||||
// let to_tsquery operator characters through, so adversarial inputs could produce a
|
||||
// fragment that to_tsquery rejects -> 500. The extraction sanitizes the input
|
||||
// (strip everything but letters/numbers/whitespace) so these inputs degrade to a
|
||||
// safe, neutral query with NO throw, while normal queries keep working.
|
||||
describe('buildTsQuery', () => {
|
||||
it('builds a prefix query for a normal single word', () => {
|
||||
expect(buildTsQuery('hello')).toBe('hello:*');
|
||||
});
|
||||
|
||||
it('joins multiple words with AND and a trailing prefix match', () => {
|
||||
expect(buildTsQuery('foo bar')).toBe('foo&bar:*');
|
||||
});
|
||||
|
||||
it('preserves accented and non-Latin words', () => {
|
||||
expect(buildTsQuery('héllo café')).toBe('héllo&café:*');
|
||||
expect(buildTsQuery('日本語')).toBe('日本語:*');
|
||||
});
|
||||
|
||||
it('neutralizes to_tsquery operator inputs without throwing', () => {
|
||||
// Each of these previously risked an invalid to_tsquery -> 500. They must now
|
||||
// produce a safe (here empty) query and never throw.
|
||||
for (const input of ['&', '!', '*', '<->', '\\']) {
|
||||
expect(() => buildTsQuery(input)).not.toThrow();
|
||||
expect(buildTsQuery(input)).toBe('');
|
||||
}
|
||||
});
|
||||
|
||||
it('handles stopword-only input safely', () => {
|
||||
// pg-tsquery still tokenizes stopwords; to_tsquery reduces them to nothing.
|
||||
// The important contract is: no throw, and a deterministic string.
|
||||
expect(() => buildTsQuery('the a of')).not.toThrow();
|
||||
expect(buildTsQuery('the a of')).toBe('the&a&of:*');
|
||||
});
|
||||
|
||||
it('returns empty string for empty / whitespace-only / null-ish input', () => {
|
||||
expect(buildTsQuery('')).toBe('');
|
||||
expect(buildTsQuery(' ')).toBe('');
|
||||
expect(buildTsQuery(undefined as unknown as string)).toBe('');
|
||||
});
|
||||
|
||||
it('handles a very long input without throwing', () => {
|
||||
const long = 'a'.repeat(10000);
|
||||
expect(() => buildTsQuery(long)).not.toThrow();
|
||||
expect(buildTsQuery(long)).toBe(`${long}:*`);
|
||||
});
|
||||
|
||||
it('strips punctuation embedded in otherwise valid words', () => {
|
||||
expect(buildTsQuery('c++ code')).toBe('c&code:*');
|
||||
expect(buildTsQuery('a-b-c')).toBe('a&b&c:*');
|
||||
});
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,242 @@
|
||||
import { type Kysely, sql } from 'kysely';
|
||||
|
||||
/**
|
||||
* #529 Phase A1 — the `ru_en` text-search configuration + the config swap.
|
||||
*
|
||||
* WHY: the search stack was pinned to the `english` FTS config, which stems only
|
||||
* Latin words. On a Russian-language wiki that is a morphology black hole:
|
||||
* «ресторанов москвы» never matched a page titled «ресторан в москве». `ru_en`
|
||||
* layers the russian_stem over the Cyrillic token classes and english_stem over
|
||||
* the ascii ones, so BOTH languages get proper morphology from one config.
|
||||
*
|
||||
* CREATE TEXT SEARCH CONFIGURATION ru_en (COPY = simple);
|
||||
* ALTER ... asciiword/asciihword/hword_asciipart WITH english_stem;
|
||||
* ALTER ... word/hword/hword_part WITH russian_stem;
|
||||
*
|
||||
* `to_tsvector('ru_en', …)` with the LITERAL config name is IMMUTABLE, so it is
|
||||
* valid inside a trigger, a generated column and an index expression.
|
||||
*
|
||||
* THE INVARIANT (acceptance #13): the config of the STORED column and the config
|
||||
* of the QUERY must change together. This migration flips BOTH stored sides
|
||||
* (pages.tsv via its trigger + a reindex; page_embeddings.fts, the RAG lexical
|
||||
* leg, via its generated expression); the matching QUERY-side flips
|
||||
* (search.service.ts and page-embedding.repo.ts `hybridSearch`) ship in the SAME
|
||||
* commit. Trigram indexes are LOWER(f_unaccent(...)) and do NOT depend on the FTS
|
||||
* config, so they are untouched.
|
||||
*
|
||||
* REINDEX / LOCK MODEL (deploy-critical). Kysely runs EACH migration in its OWN
|
||||
* transaction (see sibling 20260706T120000 "Kysely runs each migration in a
|
||||
* transaction"; migrate.ts / migration.service.ts set no `disableTransactions`,
|
||||
* and PostgresJSDialect has transactional DDL). A single migration file therefore
|
||||
* cannot commit between batches, so the issue's "procedural batch job OUTSIDE the
|
||||
* transaction + dual-config read window + migration_complete gate" is not
|
||||
* expressible in-migration here. That machinery bridges a reindex spread over
|
||||
* MANY committed batches, during which some rows are still `english` while others
|
||||
* are already `ru_en`. Our pages.tsv reindex is a SINGLE `UPDATE pages SET tsv`,
|
||||
* ATOMIC within THIS migration's own transaction: at COMMIT every row is `ru_en`
|
||||
* at once, so no morphology-desync window exists and no dual-config read path is
|
||||
* required — the query config flips to `ru_en` in the very same release. This is
|
||||
* the deliberate, correct adaptation to this framework (see the PR notes).
|
||||
*
|
||||
* - pages.tsv: swapping the trigger is a cheap catalog change (no table lock),
|
||||
* and the reindex is a single `UPDATE pages SET tsv = <ru_en expr>` — a
|
||||
* ROW-level-lock (RowExclusiveLock) backfill, NOT an ACCESS EXCLUSIVE rewrite
|
||||
* (mirrors the existing space_id backfill in 20250725T052004). On a LARGE
|
||||
* tenant this still writes every row + its WAL and leaves dead tuples, so it
|
||||
* can take MINUTES and blocks the startup migrator for that time — but it
|
||||
* never blocks concurrent reads. It stays inline unconditionally.
|
||||
*
|
||||
* - page_embeddings.fts is a GENERATED STORED column; Postgres cannot change a
|
||||
* generated expression without DROP+ADD, which is a full-table ACCESS
|
||||
* EXCLUSIVE REWRITE of page_embeddings — it blocks ALL reads AND writes on
|
||||
* that table (including the RAG agent) for the rewrite's duration. That inline
|
||||
* rewrite is appropriate for small/typical tenants (this fork's target) and
|
||||
* is the DEFAULT.
|
||||
*
|
||||
* LARGE TENANTS have two documented escape hatches, either of which makes the
|
||||
* migration genuinely no-op the rewrite (it is NOT a blind DROP+ADD):
|
||||
* (a) Set `SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE=false`. The migration then
|
||||
* SKIPS the embeddings rewrite entirely and logs a WARNING. The operator
|
||||
* MUST perform the ru_en fts swap out-of-band; until they do, the RAG
|
||||
* lexical leg stays on `english` while the query config is `ru_en` — a
|
||||
* documented, operator-owned desync window. (pages.tsv still swaps
|
||||
* inline — the gate is ONLY the embeddings rewrite.)
|
||||
* (b) Perform the swap out-of-band BEFORE deploy — add a plain column →
|
||||
* batched backfill → brief-lock swap → CREATE INDEX CONCURRENTLY — so
|
||||
* the `fts` column's generated expression already references the TARGET
|
||||
* config when the migration runs. The migration detects this (it reads
|
||||
* the column's actual generation expression from pg_catalog) and does a
|
||||
* TRUE no-op — no DROP, no ADD, no rewrite. This is real idempotency,
|
||||
* not the old (false) "IF-EXISTS guards no-op" claim: `DROP COLUMN IF
|
||||
* EXISTS` guards against ABSENCE, not presence, so it would have dropped
|
||||
* and recreated an existing `fts` regardless. The at-target check is the
|
||||
* only honest no-op path.
|
||||
*
|
||||
* Same documented trade-off family as the #443 trgm GIN migration (20260706T120000).
|
||||
*/
|
||||
|
||||
// pages.tsv trigger body for a given FTS config — mirrors the latest form
|
||||
// (20250729T213756): f_unaccent + a 1MB text cap on text_content, weights A/B.
|
||||
function pagesTriggerSql(config: 'ru_en' | 'english') {
|
||||
return sql`
|
||||
CREATE OR REPLACE FUNCTION pages_tsvector_trigger() RETURNS trigger AS $$
|
||||
begin
|
||||
new.tsv :=
|
||||
setweight(to_tsvector('${sql.raw(config)}', f_unaccent(coalesce(new.title, ''))), 'A') ||
|
||||
setweight(to_tsvector('${sql.raw(config)}', f_unaccent(substring(coalesce(new.text_content, ''), 1, 1000000))), 'B');
|
||||
return new;
|
||||
end;
|
||||
$$ LANGUAGE plpgsql;
|
||||
`;
|
||||
}
|
||||
|
||||
async function swapPagesConfig(db: Kysely<any>, config: 'ru_en' | 'english') {
|
||||
// 1. Point the trigger at the target config (new/edited rows use it going
|
||||
// forward). CREATE OR REPLACE FUNCTION takes only a brief catalog lock.
|
||||
await pagesTriggerSql(config).execute(db);
|
||||
|
||||
// 2. Reindex existing rows: recompute tsv directly with the target config. A
|
||||
// plain UPDATE — row locks, no ACCESS EXCLUSIVE. Equivalent to firing the
|
||||
// trigger but cheaper (no self-update round trip).
|
||||
await sql`
|
||||
UPDATE pages
|
||||
SET tsv =
|
||||
setweight(to_tsvector('${sql.raw(config)}', f_unaccent(coalesce(title, ''))), 'A') ||
|
||||
setweight(to_tsvector('${sql.raw(config)}', f_unaccent(substring(coalesce(text_content, ''), 1, 1000000))), 'B')
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
// The default in-migration ACCESS EXCLUSIVE rewrite of page_embeddings.fts is
|
||||
// ON unless the operator explicitly opts out with the env flag. Parsed strictly
|
||||
// (mirrors CLIENT_TELEMETRY_ENABLED / DEBUG_MODE in common/): only a literal
|
||||
// (case-insensitive) 'false' disables it; anything else — unset included —
|
||||
// keeps the default true.
|
||||
function inlineEmbeddingsRewriteEnabled(): boolean {
|
||||
return (
|
||||
(process.env.SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE ?? 'true').toLowerCase() !==
|
||||
'false'
|
||||
);
|
||||
}
|
||||
|
||||
// Read page_embeddings.fts's ACTUAL generated-column expression from pg_catalog
|
||||
// (the generation expression is stored as a column default marked generated).
|
||||
// Returns '' when the column is absent.
|
||||
async function embeddingsFtsExpr(db: Kysely<any>): Promise<string> {
|
||||
const r = await sql<{ def: string }>`
|
||||
SELECT pg_get_expr(d.adbin, d.adrelid) AS def
|
||||
FROM pg_attrdef d
|
||||
JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum
|
||||
WHERE a.attname = 'fts'
|
||||
AND a.attrelid = 'page_embeddings'::regclass
|
||||
AND NOT a.attisdropped
|
||||
`.execute(db);
|
||||
return r.rows[0]?.def ?? '';
|
||||
}
|
||||
|
||||
async function swapEmbeddingsFtsConfig(
|
||||
db: Kysely<any>,
|
||||
config: 'ru_en' | 'english',
|
||||
) {
|
||||
// 1. TRUE no-op path (real out-of-band escape hatch): if the column's current
|
||||
// generation expression already references the TARGET config, there is
|
||||
// nothing to do. An operator who pre-swapped the column out-of-band lands
|
||||
// here and the migration does NOT rewrite the table. ('ru_en' and 'english'
|
||||
// are disjoint tokens, neither a substring of the other or of the rest of
|
||||
// the expression, so a plain contains-check is unambiguous.)
|
||||
const currentExpr = await embeddingsFtsExpr(db);
|
||||
if (currentExpr.includes(config)) return;
|
||||
|
||||
// 2. Env-gated opt-out: large tenants set SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE
|
||||
// =false to skip the ACCESS EXCLUSIVE rewrite in-migration and own the swap
|
||||
// out-of-band. Warn loudly so the desync window is not silent.
|
||||
if (!inlineEmbeddingsRewriteEnabled()) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[migration 20260707T130000] SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE=false: ` +
|
||||
`SKIPPING the page_embeddings.fts rewrite to '${config}'. The operator MUST ` +
|
||||
`perform this fts swap out-of-band. Until then the RAG lexical leg stays on ` +
|
||||
`its current config while the query config is '${config}' (documented, ` +
|
||||
`operator-owned desync window).`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// 3. Default inline path: the generated `fts` expression can only change via
|
||||
// DROP+ADD (a full-table ACCESS EXCLUSIVE rewrite; see the lock note in the
|
||||
// header). The GIN index depends on the column, so it is dropped with it and
|
||||
// recreated.
|
||||
await sql`DROP INDEX IF EXISTS idx_page_embeddings_fts`.execute(db);
|
||||
await sql`ALTER TABLE page_embeddings DROP COLUMN IF EXISTS fts`.execute(db);
|
||||
await sql`
|
||||
ALTER TABLE page_embeddings
|
||||
ADD COLUMN fts tsvector
|
||||
GENERATED ALWAYS AS (to_tsvector('${sql.raw(config)}', f_unaccent(content))) STORED
|
||||
`.execute(db);
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_page_embeddings_fts
|
||||
ON page_embeddings USING gin(fts)
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
async function ruEnConfigExists(db: Kysely<any>): Promise<boolean> {
|
||||
const r = await sql<{ n: number }>`
|
||||
SELECT count(*)::int AS n FROM pg_ts_config WHERE cfgname = 'ru_en'
|
||||
`.execute(db);
|
||||
return (r.rows[0]?.n ?? 0) > 0;
|
||||
}
|
||||
|
||||
async function ensureRuEnConfig(db: Kysely<any>): Promise<void> {
|
||||
// Idempotent by EXISTENCE, not by drop-recreate. The old `DROP ... IF EXISTS;
|
||||
// CREATE` was safe only on a first run: on a re-run the page_embeddings.fts
|
||||
// generated column already has a hard dependency on ru_en, so dropping the
|
||||
// config would fail. Create only when it is genuinely missing.
|
||||
if (await ruEnConfigExists(db)) return;
|
||||
await sql`CREATE TEXT SEARCH CONFIGURATION ru_en (COPY = simple)`.execute(db);
|
||||
// Latin token classes → english_stem.
|
||||
await sql`
|
||||
ALTER TEXT SEARCH CONFIGURATION ru_en
|
||||
ALTER MAPPING FOR asciiword, asciihword, hword_asciipart
|
||||
WITH english_stem
|
||||
`.execute(db);
|
||||
// Cyrillic / non-ascii token classes → russian_stem.
|
||||
await sql`
|
||||
ALTER TEXT SEARCH CONFIGURATION ru_en
|
||||
ALTER MAPPING FOR word, hword, hword_part
|
||||
WITH russian_stem
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await ensureRuEnConfig(db);
|
||||
|
||||
// Flip both stored sides to ru_en (query-side flips in the same commit).
|
||||
// swapEmbeddingsFtsConfig no-ops when fts already references ru_en, so a
|
||||
// re-run of up() is idempotent and does NOT re-rewrite the embeddings table.
|
||||
await swapPagesConfig(db, 'ru_en');
|
||||
await swapEmbeddingsFtsConfig(db, 'ru_en');
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
// Reverse ORDER matters: the trigger and the generated column reference the
|
||||
// `ru_en` config by name, so they must be moved back to `english` BEFORE the
|
||||
// config can be dropped (a generated column that still depends on `ru_en` would
|
||||
// block the DROP with a dependency error).
|
||||
await swapEmbeddingsFtsConfig(db, 'english');
|
||||
await swapPagesConfig(db, 'english');
|
||||
|
||||
// Drop the config ONLY if nothing still references it. When the embeddings
|
||||
// rewrite was gated off (SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE=false), the fts
|
||||
// column can still reference ru_en — dropping the config would then fail with a
|
||||
// dependency error. Skip + warn so down() stays non-fatal; the operator drops
|
||||
// ru_en after completing the out-of-band english swap.
|
||||
if ((await embeddingsFtsExpr(db)).includes('ru_en')) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.warn(
|
||||
`[migration 20260707T130000] down(): page_embeddings.fts still references ` +
|
||||
`ru_en (inline rewrite was gated off) — leaving the ru_en text-search ` +
|
||||
`configuration in place. Drop it out-of-band once fts is back on 'english'.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
await sql`DROP TEXT SEARCH CONFIGURATION IF EXISTS ru_en`.execute(db);
|
||||
}
|
||||
@@ -200,8 +200,11 @@ export class PageEmbeddingRepo {
|
||||
*
|
||||
* The `model_dimensions = $dim` filter applies ONLY on the semantic side
|
||||
* (cosine compares same-dimension vectors; pgvector errors otherwise). The
|
||||
* lexical side (`fts`) is dimension-independent. If `websearch_to_tsquery`
|
||||
* yields an EMPTY query (e.g. the text is all stopwords) the `@@` matches
|
||||
* lexical side (`fts`) is dimension-independent. Its query config is `ru_en`,
|
||||
* matched IN LOCKSTEP with the `page_embeddings.fts` generated column's config
|
||||
* (#529 acceptance #13): a mismatch silently breaks Cyrillic RAG retrieval. If
|
||||
* `websearch_to_tsquery` yields an EMPTY query (e.g. the text is all stopwords)
|
||||
* the `@@` matches
|
||||
* nothing and the lexical CTE is empty, so results degrade to pure-semantic —
|
||||
* which is correct behaviour, not an error.
|
||||
*
|
||||
@@ -249,7 +252,7 @@ export class PageEmbeddingRepo {
|
||||
row_number() OVER (ORDER BY ts_rank(pe.fts, q.query) DESC) AS rank_ix
|
||||
FROM page_embeddings pe
|
||||
JOIN pages p ON p.id = pe.page_id,
|
||||
websearch_to_tsquery('english', f_unaccent(${queryText})) AS q(query)
|
||||
websearch_to_tsquery('ru_en', f_unaccent(${queryText})) AS q(query)
|
||||
WHERE pe.workspace_id = ${workspaceId}
|
||||
AND pe.space_id IN (${spaceList})
|
||||
AND p.deleted_at IS NULL
|
||||
|
||||
@@ -0,0 +1,612 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { SearchService } from 'src/core/search/search.service';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createSpace,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #529 Phase A — the lexical overhaul, on the REAL migrated schema (ru_en config).
|
||||
*
|
||||
* Covers every acceptance criterion of the issue: RU+EN morphology + OR default,
|
||||
* match=auto identifier routing, "phrase"/+/- operators, RRF ordering, exact
|
||||
* permission-filtered total (fail-closed) + pagination, only-negation / garbage
|
||||
* short-circuits, the A8 path fix, the response superset, and the RAG lockstep
|
||||
* config (acceptance #13).
|
||||
*
|
||||
* The tsv column is populated by the pages_tsvector_trigger (now ru_en), so the
|
||||
* FTS branch is exercised end to end.
|
||||
*/
|
||||
describe('SearchService #529 lexical overhaul [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let workspaceId: string;
|
||||
let spaceId: string;
|
||||
|
||||
async function insertPage(args: {
|
||||
title: string;
|
||||
textContent?: string;
|
||||
parentPageId?: string | null;
|
||||
spaceId?: string;
|
||||
deletedAt?: Date | null;
|
||||
}): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await db
|
||||
.insertInto('pages')
|
||||
.values({
|
||||
id,
|
||||
slugId: `slug-${id.slice(0, 12)}`,
|
||||
title: args.title,
|
||||
textContent: args.textContent ?? null,
|
||||
parentPageId: args.parentPageId ?? null,
|
||||
spaceId: args.spaceId ?? spaceId,
|
||||
workspaceId,
|
||||
deletedAt: args.deletedAt ?? null,
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
// Service wired to the real DB + real PageRepo (recursive descendants) with
|
||||
// stubbed space-membership + permission repos so a test controls scope and the
|
||||
// permission filter explicitly. `accessibleIds` (when set) is the KEEP list.
|
||||
function buildService(opts?: {
|
||||
userSpaceIds?: string[];
|
||||
accessibleIds?: string[] | null;
|
||||
filterThrows?: boolean;
|
||||
}): SearchService {
|
||||
const pageRepo = new PageRepo(db as any, null as any, null as any);
|
||||
const spaceMemberRepo = {
|
||||
getUserSpaceIds: async () => opts?.userSpaceIds ?? [spaceId],
|
||||
};
|
||||
const pagePermissionRepo = {
|
||||
hasRestrictedAncestor: async () => false,
|
||||
filterAccessiblePageIds: async ({ pageIds }: { pageIds: string[] }) => {
|
||||
if (opts?.filterThrows) throw new Error('permission query failed');
|
||||
return opts?.accessibleIds
|
||||
? pageIds.filter((id) => opts.accessibleIds!.includes(id))
|
||||
: pageIds;
|
||||
},
|
||||
};
|
||||
return new SearchService(
|
||||
db as any,
|
||||
pageRepo as any,
|
||||
{} as any,
|
||||
spaceMemberRepo as any,
|
||||
pagePermissionRepo as any,
|
||||
);
|
||||
}
|
||||
|
||||
const search = (service: SearchService, params: any) =>
|
||||
service.searchPage(params, { userId: 'u-1', workspaceId }) as any;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
spaceId = (await createSpace(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
// 1. RU morphology + OR: «ресторанов москвы» finds «ресторан в москве».
|
||||
it('#1 russian morphology + OR: finds «ресторан в москве» for «ресторанов москвы»', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'ресторан в москве',
|
||||
textContent: 'Лучший ресторан столицы.',
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: 'ресторанов москвы',
|
||||
spaceId,
|
||||
});
|
||||
expect(res.items.map((i: any) => i.id)).toContain(page);
|
||||
expect(res.total).toBeGreaterThanOrEqual(1);
|
||||
});
|
||||
|
||||
// 2. OR non-empty: «Стамбул Роснефть».
|
||||
it('#2 OR yields a hit when only one term matches', async () => {
|
||||
const page = await insertPage({ title: 'Роснефть отчёт', textContent: 'x' });
|
||||
const res = await search(buildService(), {
|
||||
query: 'Стамбул Роснефть',
|
||||
spaceId,
|
||||
});
|
||||
expect(res.items.map((i: any) => i.id)).toContain(page);
|
||||
});
|
||||
|
||||
// 3. Multi-word OR: a page matching >=1 term is returned.
|
||||
it('#3 «3D принтер» returns pages that matched at least one term', async () => {
|
||||
const models = await insertPage({
|
||||
title: 'Модели для печати 3D',
|
||||
textContent: 'коллекция моделей',
|
||||
});
|
||||
const wish = await insertPage({
|
||||
title: 'Хотеть напечатать на принтере',
|
||||
textContent: 'очередь печати',
|
||||
});
|
||||
const res = await search(buildService(), { query: '3D принтер', spaceId });
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(models); // matched "3D"
|
||||
expect(ids).toContain(wish); // matched "принтер"
|
||||
// matchedTerms is populated per hit.
|
||||
const hit = res.items.find((i: any) => i.id === wish);
|
||||
expect(hit.matchedTerms).toContain('принтер');
|
||||
});
|
||||
|
||||
// 4. match=auto FTS stemming: «печат» must NOT drag in «впечатления».
|
||||
it('#4 «печат» (auto) matches «печать» but NOT «впечатления»', async () => {
|
||||
const good = await insertPage({
|
||||
title: 'Печать документов',
|
||||
textContent: 'настройка печати',
|
||||
});
|
||||
const bad = await insertPage({
|
||||
title: 'Впечатления от поездки',
|
||||
textContent: 'много впечатлений',
|
||||
});
|
||||
const res = await search(buildService(), { query: 'печат', spaceId });
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(good);
|
||||
expect(ids).not.toContain(bad);
|
||||
});
|
||||
|
||||
// 5. Identifier → substring branch.
|
||||
it('#5 `10.31.41` (auto→substring) finds the page with that IP', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'Сетевой узел',
|
||||
textContent: 'Адрес устройства: 10.31.41.7 в сети.',
|
||||
});
|
||||
const res = await search(buildService(), { query: '10.31.41', spaceId });
|
||||
const hit = res.items.find((i: any) => i.id === page);
|
||||
expect(hit).toBeDefined();
|
||||
expect(hit.matchedFields).toContain('text');
|
||||
});
|
||||
|
||||
// 6. +required / -excluded.
|
||||
it('#6 `+кофейня -архив`: keeps «кофейня», drops pages with «архив»', async () => {
|
||||
const keep = await insertPage({ title: 'Кофейня в центре', textContent: 'уют' });
|
||||
const drop = await insertPage({
|
||||
title: 'Кофейня старый архив',
|
||||
textContent: 'архивные записи',
|
||||
});
|
||||
const res = await search(buildService(), { query: '+кофейня -архив', spaceId });
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(keep);
|
||||
expect(ids).not.toContain(drop);
|
||||
});
|
||||
|
||||
// 7. Phrase operator: only adjacent phrase hits survive.
|
||||
it('#7 `+"воздушный шар" кофе`: every hit contains the adjacent phrase', async () => {
|
||||
const adjacent = await insertPage({
|
||||
title: 'Воздушный шар и кофе',
|
||||
textContent: 'воздушный шар над городом, чашка кофе',
|
||||
});
|
||||
const nonAdjacent = await insertPage({
|
||||
title: 'Красный воздушный большой шар',
|
||||
textContent: 'воздушный красный шар и кофе рядом',
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: '+"воздушный шар" кофе',
|
||||
spaceId,
|
||||
});
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(adjacent);
|
||||
// The non-adjacent page (words separated) must NOT match the phrase.
|
||||
expect(ids).not.toContain(nonAdjacent);
|
||||
});
|
||||
|
||||
// 8. Pagination determinism + exact total.
|
||||
it('#8 >50 matches: total>50, hasMore, and offset paginates without dupes', async () => {
|
||||
const svc = buildService();
|
||||
const created: string[] = [];
|
||||
for (let i = 0; i < 60; i++) {
|
||||
created.push(
|
||||
await insertPage({
|
||||
title: `паджинация запись ${i}`,
|
||||
textContent: 'общий паджинационный маркер',
|
||||
}),
|
||||
);
|
||||
}
|
||||
const p1 = await search(svc, {
|
||||
query: 'паджинационный',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
offset: 0,
|
||||
});
|
||||
expect(p1.total).toBeGreaterThanOrEqual(60);
|
||||
expect(p1.hasMore).toBe(true);
|
||||
const p2 = await search(svc, {
|
||||
query: 'паджинационный',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
offset: 25,
|
||||
});
|
||||
const p3 = await search(svc, {
|
||||
query: 'паджинационный',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
offset: 50,
|
||||
});
|
||||
const ids = [
|
||||
...p1.items.map((i: any) => i.id),
|
||||
...p2.items.map((i: any) => i.id),
|
||||
...p3.items.map((i: any) => i.id),
|
||||
];
|
||||
// No duplicates across the three pages.
|
||||
expect(new Set(ids).size).toBe(ids.length);
|
||||
// Deterministic: same query twice → identical order.
|
||||
const p1b = await search(svc, {
|
||||
query: 'паджинационный',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
offset: 0,
|
||||
});
|
||||
expect(p1b.items.map((i: any) => i.id)).toEqual(p1.items.map((i: any) => i.id));
|
||||
});
|
||||
|
||||
// 9. Only-negation.
|
||||
it('#9 `-архив` only-negation: total 0, reason only-negation, no throw', async () => {
|
||||
const res = await search(buildService(), { query: '-архив', spaceId });
|
||||
expect(res.total).toBe(0);
|
||||
expect(res.items).toEqual([]);
|
||||
expect(res.query.parsed.reason).toBe('only-negation');
|
||||
});
|
||||
|
||||
// 10. Garbage input.
|
||||
it('#10 garbage `%` / `_` / empty: total 0, does not match everything', async () => {
|
||||
await insertPage({ title: 'какая-то страница', textContent: 'текст' });
|
||||
for (const q of ['%', '_', ' ', '%%__']) {
|
||||
const res = await search(buildService(), { query: q, spaceId });
|
||||
expect(res.total).toBe(0);
|
||||
expect(res.items).toEqual([]);
|
||||
}
|
||||
});
|
||||
|
||||
// 11. Permission-filtered total (fail-closed) + mutation guard.
|
||||
it('#11 a permission-hidden page is absent from items AND total', async () => {
|
||||
const visible = await insertPage({
|
||||
title: 'разрешённая пермишен-страница',
|
||||
textContent: 'пермишенмаркер',
|
||||
});
|
||||
const hidden = await insertPage({
|
||||
title: 'скрытая пермишен-страница',
|
||||
textContent: 'пермишенмаркер',
|
||||
});
|
||||
// Filter keeps only the visible page.
|
||||
const filtered = buildService({ accessibleIds: [visible] });
|
||||
const res = await search(filtered, { query: 'пермишенмаркер', spaceId });
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(visible);
|
||||
expect(ids).not.toContain(hidden);
|
||||
// total is the POST-permission count — the hidden page does not leak into it.
|
||||
expect(res.total).toBe(1);
|
||||
|
||||
// MUTATION: disable the guard (passthrough) → the hidden page reappears in
|
||||
// BOTH items and total. If this did NOT change, the guard is not load-bearing.
|
||||
const open = buildService({ accessibleIds: null });
|
||||
const res2 = await search(open, { query: 'пермишенмаркер', spaceId });
|
||||
expect(res2.items.map((i: any) => i.id)).toContain(hidden);
|
||||
expect(res2.total).toBe(2);
|
||||
});
|
||||
|
||||
it('#11b a permission-query error PROPAGATES (fail-closed, never empty)', async () => {
|
||||
await insertPage({ title: 'failclosed маркер', textContent: 'failclosedmarker' });
|
||||
const svc = buildService({ filterThrows: true });
|
||||
await expect(
|
||||
search(svc, { query: 'failclosedmarker', spaceId }),
|
||||
).rejects.toThrow(/permission query failed/);
|
||||
});
|
||||
|
||||
// A8 path fix.
|
||||
it('#11c path: a soft-deleted / cross-space ancestor title does not leak', async () => {
|
||||
const otherSpace = (await createSpace(db, workspaceId)).id;
|
||||
const root = await insertPage({ title: 'Живой корень' });
|
||||
const deletedMid = await insertPage({
|
||||
title: 'УдалённыйПредок',
|
||||
parentPageId: root,
|
||||
deletedAt: new Date(),
|
||||
});
|
||||
const leaf = await insertPage({
|
||||
title: 'a8leaf уникальный',
|
||||
parentPageId: deletedMid,
|
||||
textContent: 'a8leafmarker',
|
||||
});
|
||||
const res = await search(buildService(), { query: 'a8leafmarker', spaceId });
|
||||
const hit = res.items.find((i: any) => i.id === leaf);
|
||||
expect(hit).toBeDefined();
|
||||
// The walk stops at the deleted ancestor — no deleted title in the path.
|
||||
expect(hit.path).not.toContain('УдалённыйПредок');
|
||||
expect(hit.path).not.toContain('Живой корень');
|
||||
|
||||
// Cross-space parent must also not leak.
|
||||
const foreignParent = await insertPage({
|
||||
title: 'ЧужойСпейс',
|
||||
spaceId: otherSpace,
|
||||
});
|
||||
const crossLeaf = await insertPage({
|
||||
title: 'crossleaf узел',
|
||||
parentPageId: foreignParent,
|
||||
textContent: 'crossleafmarker',
|
||||
});
|
||||
const res2 = await search(buildService(), {
|
||||
query: 'crossleafmarker',
|
||||
spaceId,
|
||||
});
|
||||
const hit2 = res2.items.find((i: any) => i.id === crossLeaf);
|
||||
expect(hit2).toBeDefined();
|
||||
expect(hit2.path).not.toContain('ЧужойСпейс');
|
||||
});
|
||||
|
||||
// 12. Web-UI superset.
|
||||
it('#12 web path (no flags) returns the OR result with the icon/space/highlight superset', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'веб суперсет страница',
|
||||
textContent: 'суперсетмаркер контент',
|
||||
});
|
||||
const res = await search(buildService(), { query: 'суперсетмаркер', spaceId });
|
||||
const hit = res.items.find((i: any) => i.id === page);
|
||||
expect(hit).toBeDefined();
|
||||
// Superset fields the web-UI relies on.
|
||||
expect('icon' in hit).toBe(true);
|
||||
expect('space' in hit).toBe(true);
|
||||
expect('highlight' in hit).toBe(true);
|
||||
expect('rank' in hit).toBe(true);
|
||||
// Plus the new fields.
|
||||
expect('path' in hit).toBe(true);
|
||||
expect('snippet' in hit).toBe(true);
|
||||
expect('score' in hit).toBe(true);
|
||||
// FTS hit carries a non-null rank + highlight.
|
||||
expect(hit.rank).not.toBeNull();
|
||||
});
|
||||
|
||||
// 13. RAG lockstep: the page_embeddings.fts generated column is ru_en.
|
||||
it('#13 page_embeddings.fts uses ru_en (cyrillic stemming) — RAG lockstep', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'rag страница',
|
||||
textContent: 'ресторанов много',
|
||||
});
|
||||
// Insert a chunk row; the generated fts column is computed by Postgres.
|
||||
await sql`
|
||||
INSERT INTO page_embeddings
|
||||
(id, page_id, workspace_id, space_id, attachment_id, chunk_index,
|
||||
chunk_start, chunk_length, content, model_name, model_dimensions, embedding)
|
||||
VALUES
|
||||
(${randomUUID()}, ${pageId}, ${workspaceId}, ${spaceId}, NULL, 0,
|
||||
0, 20, ${'ресторанов москвы много'}, 'test-model', 3, '[0.1,0.2,0.3]'::vector)
|
||||
`.execute(db);
|
||||
// A ru_en query stems «москва» → «москв», matching the stored «москвы».
|
||||
// Under the old `english` config the cyrillic word would not stem and this
|
||||
// inflected-form query would miss — so this asserts the ru_en lockstep.
|
||||
const row = await sql<{ m: boolean }>`
|
||||
SELECT fts @@ to_tsquery('ru_en', f_unaccent('москва')) AS m
|
||||
FROM page_embeddings WHERE page_id = ${pageId}
|
||||
`.execute(db);
|
||||
expect(row.rows[0].m).toBe(true);
|
||||
});
|
||||
|
||||
// W4 — substring-tier dominance under RRF: title-exact (tier 3) > title-
|
||||
// substring (tier 2) > text-only (tier 1). The engine encodes this via
|
||||
// sub_tier DESC → rn_sub → RRF; no test asserted the end-to-end ordering, so
|
||||
// this restores that guarantee. match:'substring' routes the term to the
|
||||
// substring branch (no FTS leg), so sub_tier alone drives the order.
|
||||
it('W4 tier dominance: title-exact > title-substring > text-only', async () => {
|
||||
const exact = await insertPage({ title: 'tierdomxyz' }); // tier 3
|
||||
const titleSub = await insertPage({
|
||||
title: 'prefix tierdomxyz suffix', // tier 2
|
||||
});
|
||||
const textOnly = await insertPage({
|
||||
title: 'w4 unrelated heading',
|
||||
textContent: 'body has tierdomxyz here', // tier 1
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: 'tierdomxyz',
|
||||
match: 'substring',
|
||||
spaceId,
|
||||
});
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(exact);
|
||||
expect(ids).toContain(titleSub);
|
||||
expect(ids).toContain(textOnly);
|
||||
// Strict tier order.
|
||||
expect(ids.indexOf(exact)).toBeLessThan(ids.indexOf(titleSub));
|
||||
expect(ids.indexOf(titleSub)).toBeLessThan(ids.indexOf(textOnly));
|
||||
});
|
||||
|
||||
// S3 — an exact-title hit must NOT be lost when the match set exceeds
|
||||
// CANDIDATE_CAP: it ranks first under RRF (tier 3) so it lands in the reachable
|
||||
// window even with a tiny cap. Restores a guarantee the old lookup suite gave.
|
||||
it('S3 exact-title survives the CANDIDATE_CAP window', async () => {
|
||||
const exact = await insertPage({ title: 'capmarkerxyz' }); // tier 3
|
||||
for (let i = 0; i < 6; i++) {
|
||||
await insertPage({
|
||||
title: `cap filler ${i}`,
|
||||
textContent: 'noise capmarkerxyz noise', // tier 1
|
||||
});
|
||||
}
|
||||
process.env.SEARCH_CANDIDATE_CAP = '2';
|
||||
try {
|
||||
const res = await search(buildService(), {
|
||||
query: 'capmarkerxyz',
|
||||
match: 'substring',
|
||||
spaceId,
|
||||
limit: 25,
|
||||
});
|
||||
// More matches than the cap → truncated, but the exact-title survives.
|
||||
expect(res.total).toBeGreaterThan(2);
|
||||
expect(res.truncatedAtCap).toBe(true);
|
||||
expect(res.items.length).toBe(2); // the reachable window
|
||||
expect(res.items.map((i: any) => i.id)).toContain(exact);
|
||||
} finally {
|
||||
delete process.env.SEARCH_CANDIDATE_CAP;
|
||||
}
|
||||
});
|
||||
|
||||
// S1 — a required-only query (`+term`, no bare positive) really matches, so its
|
||||
// hits must report matchedFields / rank / highlight, not [] / null. Before the
|
||||
// fix these detail exprs were built from parsed.positive only.
|
||||
it('S1 required-only query populates matchedFields + rank on a title match', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'Кофейня s1маркер центр',
|
||||
textContent: 'обычный текст без ключевого слова',
|
||||
});
|
||||
const res = await search(buildService(), { query: '+кофейня', spaceId });
|
||||
const hit = res.items.find((i: any) => i.id === page);
|
||||
expect(hit).toBeDefined();
|
||||
// The title matches the required term → matchedFields includes 'title'.
|
||||
expect(hit.matchedFields).toContain('title');
|
||||
// FTS rank is populated (was null before the S1 fix).
|
||||
expect(hit.rank).not.toBeNull();
|
||||
// matchedTerms already echoed the required term; still true.
|
||||
expect(hit.matchedTerms).toContain('кофейня');
|
||||
});
|
||||
|
||||
// F1 — stack-depth guard: a pasted text block (thousands of FTS terms) used to
|
||||
// nest the combined tsquery so deep that Postgres raised `stack depth limit
|
||||
// exceeded` → HTTP 500 for the caller. The parser now caps terms at
|
||||
// MAX_PARSED_TERMS, so a huge query returns 200 and the leading (in-cap) term
|
||||
// still matches. Mutation: remove the cap → this reddens (500 / stack depth).
|
||||
it('F1 a huge multi-term query returns 200, not a stack-depth 500', async () => {
|
||||
const page = await insertPage({
|
||||
title: 'qfonemarker заголовок',
|
||||
textContent: 'тело страницы',
|
||||
});
|
||||
// Purely-alphabetic filler words → FTS branch (the branch that nests in
|
||||
// combineTsq). A digit-bearing token would route to substring and not nest.
|
||||
const alphaWord = (i: number): string => {
|
||||
let s = '';
|
||||
let n = i + 1;
|
||||
while (n > 0) {
|
||||
s = String.fromCharCode(97 + (n % 26)) + s;
|
||||
n = Math.floor(n / 26);
|
||||
}
|
||||
return 'q' + s;
|
||||
};
|
||||
const words = [
|
||||
'qfonemarker',
|
||||
...Array.from({ length: 5000 }, (_, i) => alphaWord(i)),
|
||||
];
|
||||
const res = await search(buildService(), {
|
||||
query: words.join(' '),
|
||||
spaceId,
|
||||
});
|
||||
// Bounded nesting → no 500; the leading in-cap term still recalls the page.
|
||||
expect(res.items.map((i: any) => i.id)).toContain(page);
|
||||
});
|
||||
|
||||
// F2 — titleOnly leak-guard (restores coverage the deleted lookup spec gave).
|
||||
// A term present ONLY in text_content must NOT match under titleOnly, and a
|
||||
// title hit must carry NO body snippet. Uses match:'substring' because titleOnly
|
||||
// gates the substring branch (the FTS `pages.tsv` already spans title+body).
|
||||
it('F2 titleOnly does not match text_content and yields no body snippet', async () => {
|
||||
// Marker lives only in the body, never in the title.
|
||||
const bodyOnly = await insertPage({
|
||||
title: 'нейтральный заголовок f2',
|
||||
textContent: 'секрет titleonlybody конец',
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: 'titleonlybody',
|
||||
match: 'substring',
|
||||
spaceId,
|
||||
titleOnly: true,
|
||||
});
|
||||
// titleOnly must NOT leak a body-substring match.
|
||||
expect(res.items.map((i: any) => i.id)).not.toContain(bodyOnly);
|
||||
|
||||
// Sanity: WITHOUT titleOnly the same term DOES find it via the body.
|
||||
const resOpen = await search(buildService(), {
|
||||
query: 'titleonlybody',
|
||||
match: 'substring',
|
||||
spaceId,
|
||||
});
|
||||
expect(resOpen.items.map((i: any) => i.id)).toContain(bodyOnly);
|
||||
|
||||
// A page that hits on its TITLE: the snippet must be empty (no body leaks in).
|
||||
const titleHit = await insertPage({
|
||||
title: 'titleonlytitle страница',
|
||||
textContent: 'какой-то текст тела здесь для сниппета',
|
||||
});
|
||||
const res2 = await search(buildService(), {
|
||||
query: 'titleonlytitle',
|
||||
match: 'substring',
|
||||
spaceId,
|
||||
titleOnly: true,
|
||||
});
|
||||
const hit = res2.items.find((i: any) => i.id === titleHit);
|
||||
expect(hit).toBeDefined();
|
||||
expect(hit.snippet).toBe('');
|
||||
});
|
||||
|
||||
// F3 — parentPageId subtree scoping (restores coverage the deleted lookup spec
|
||||
// gave). Two sibling subtrees share one term; scoping to ONE root returns only
|
||||
// that subtree's hits INCLUDING the root/parent page itself, and NONE of the
|
||||
// sibling subtree. Mutation: drop the ANY(descendantIds) filter → this reddens.
|
||||
it('F3 parentPageId scopes to one subtree incl. the parent, excludes siblings', async () => {
|
||||
const rootA = await insertPage({
|
||||
title: 'subtreeA корень',
|
||||
textContent: 'f3marker в корне A',
|
||||
});
|
||||
const childA = await insertPage({
|
||||
title: 'subtreeA потомок',
|
||||
textContent: 'f3marker в потомке A',
|
||||
parentPageId: rootA,
|
||||
});
|
||||
const rootB = await insertPage({
|
||||
title: 'subtreeB корень',
|
||||
textContent: 'f3marker в корне B',
|
||||
});
|
||||
const childB = await insertPage({
|
||||
title: 'subtreeB потомок',
|
||||
textContent: 'f3marker в потомке B',
|
||||
parentPageId: rootB,
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: 'f3marker',
|
||||
spaceId,
|
||||
parentPageId: rootA,
|
||||
});
|
||||
const ids = res.items.map((i: any) => i.id);
|
||||
expect(ids).toContain(rootA); // the parent/root page itself
|
||||
expect(ids).toContain(childA);
|
||||
expect(ids).not.toContain(rootB); // sibling subtree cut off
|
||||
expect(ids).not.toContain(childB);
|
||||
});
|
||||
|
||||
// F4 — positive path + snippet content asserts (the #11c test only asserts what
|
||||
// must NOT be in path). (a) a nested hit's path is root→parent ordered and a
|
||||
// root-level hit's path is []; (b) a text-body hit returns a NON-empty snippet.
|
||||
it('F4 path is root→parent ordered ([] at root) and body hits carry a snippet', async () => {
|
||||
const root = await insertPage({ title: 'F4Root' });
|
||||
const parent = await insertPage({ title: 'F4Parent', parentPageId: root });
|
||||
const leaf = await insertPage({
|
||||
title: 'f4leaf лист',
|
||||
parentPageId: parent,
|
||||
textContent: 'f4leafmarker тело с содержимым для сниппета',
|
||||
});
|
||||
const res = await search(buildService(), {
|
||||
query: 'f4leafmarker',
|
||||
spaceId,
|
||||
});
|
||||
const hit = res.items.find((i: any) => i.id === leaf);
|
||||
expect(hit).toBeDefined();
|
||||
// Positive ancestry: root → direct parent, hit's own title excluded.
|
||||
expect(hit.path).toEqual(['F4Root', 'F4Parent']);
|
||||
// Snippet content: a text-body hit returns a non-empty snippet with the term.
|
||||
expect(hit.snippet.length).toBeGreaterThan(0);
|
||||
expect(hit.snippet).toContain('f4leafmarker');
|
||||
|
||||
// A root-level hit (no parent) → empty path.
|
||||
const rootHit = await insertPage({
|
||||
title: 'f4rootlevel уникальный',
|
||||
textContent: 'f4rootmarker в теле',
|
||||
});
|
||||
const res2 = await search(buildService(), {
|
||||
query: 'f4rootmarker',
|
||||
spaceId,
|
||||
});
|
||||
const hit2 = res2.items.find((i: any) => i.id === rootHit);
|
||||
expect(hit2).toBeDefined();
|
||||
expect(hit2.path).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,462 +0,0 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { Kysely } from 'kysely';
|
||||
import { SearchService } from 'src/core/search/search.service';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createSpace,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #443 — agent-lookup search mode, acceptance on the REAL DB schema.
|
||||
*
|
||||
* Exercises SearchService.searchPage(..., { substring: true }) against a
|
||||
* migrated Postgres: substring matching of technical tokens the FTS tokenizer
|
||||
* mangles (backup-srv.local, 10.0.12.5, WB-MGE-30D86B, "Теги: Docker"), the
|
||||
* populated path + snippet, parentPageId subtree scoping, titleOnly, the empty
|
||||
* result, LIKE-metacharacter escaping (`%`/`_` must NOT match everything), the
|
||||
* permission post-filter applied BEFORE the limit, and the web-UI path staying
|
||||
* on the legacy FTS shape when `substring` is absent.
|
||||
*
|
||||
* The tsv column is populated by the pages_tsvector_trigger on insert, so the
|
||||
* FTS branch is exercised too.
|
||||
*/
|
||||
describe('SearchService agent-lookup mode [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let service: SearchService;
|
||||
let workspaceId: string;
|
||||
let spaceId: string;
|
||||
|
||||
// Direct page insert (the shared createPage seeder omits text_content /
|
||||
// parent_page_id, both of which this mode depends on). Returns the id.
|
||||
async function insertPage(args: {
|
||||
title: string;
|
||||
textContent?: string;
|
||||
parentPageId?: string | null;
|
||||
spaceId?: string;
|
||||
}): Promise<string> {
|
||||
const id = randomUUID();
|
||||
await db
|
||||
.insertInto('pages')
|
||||
.values({
|
||||
id,
|
||||
slugId: `slug-${id.slice(0, 12)}`,
|
||||
title: args.title,
|
||||
textContent: args.textContent ?? null,
|
||||
parentPageId: args.parentPageId ?? null,
|
||||
spaceId: args.spaceId ?? spaceId,
|
||||
workspaceId,
|
||||
})
|
||||
.execute();
|
||||
return id;
|
||||
}
|
||||
|
||||
// Build a SearchService wired to the real DB + a real PageRepo (only its
|
||||
// recursive-descendants method is used by this mode, and it needs only `db`),
|
||||
// with lightweight stubs for the space-membership and permission repos so a
|
||||
// test can drive scope + the permission post-filter explicitly.
|
||||
function buildService(opts?: {
|
||||
userSpaceIds?: string[];
|
||||
// ids to KEEP after the permission post-filter; undefined = keep all.
|
||||
accessibleIds?: string[];
|
||||
}): SearchService {
|
||||
const pageRepo = new PageRepo(db as any, null as any, null as any);
|
||||
const spaceMemberRepo = {
|
||||
getUserSpaceIds: async () => opts?.userSpaceIds ?? [spaceId],
|
||||
};
|
||||
const pagePermissionRepo = {
|
||||
filterAccessiblePageIds: async ({ pageIds }: { pageIds: string[] }) =>
|
||||
opts?.accessibleIds
|
||||
? pageIds.filter((id) => opts.accessibleIds!.includes(id))
|
||||
: pageIds,
|
||||
};
|
||||
return new SearchService(
|
||||
db as any,
|
||||
pageRepo as any,
|
||||
{} as any, // shareRepo — unused by the lookup path
|
||||
spaceMemberRepo as any,
|
||||
pagePermissionRepo as any,
|
||||
);
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
spaceId = (await createSpace(db, workspaceId)).id;
|
||||
service = buildService();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('finds `backup-srv.local` by the fragment `srv.local`', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'backup-srv.local',
|
||||
textContent: 'A backup server node.',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'srv.local', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
expect(items.map((i: any) => i.id)).toContain(pageId);
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit.title).toBe('backup-srv.local');
|
||||
// slugId must never be part of the server response shape.
|
||||
expect('slugId' in hit).toBe(true); // server carries it; MCP strips it
|
||||
});
|
||||
|
||||
it('finds a page whose TEXT contains `10.0.12.5` by the fragment `10.0.12` (empty-tsquery case)', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'Server inventory',
|
||||
textContent: 'The backup box lives at IP: 10.0.12.5. Debian 12, backups.',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: '10.0.12', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit).toBeDefined();
|
||||
// The windowed snippet must include the matched text.
|
||||
expect(hit.snippet).toContain('10.0.12.5');
|
||||
});
|
||||
|
||||
it('finds `WB-MGE-30D86B` (alphanumeric token with dashes) by title', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'WB-MGE-30D86B',
|
||||
textContent: 'Device page.',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'WB-MGE-30D86B', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit).toBeDefined();
|
||||
// Exact title match → top tier (TITLE_EXACT=3) → score in [0.75, 1].
|
||||
expect(hit.score).toBeGreaterThanOrEqual(0.75);
|
||||
// And it is the top-ranked hit of its own result set.
|
||||
expect(items[0].id).toBe(pageId);
|
||||
});
|
||||
|
||||
it('finds every page whose text literally contains `Теги: Docker`', async () => {
|
||||
const a = await insertPage({
|
||||
title: 'Container host A',
|
||||
textContent: 'Some notes.\nТеги: Docker, compose\nmore.',
|
||||
});
|
||||
const b = await insertPage({
|
||||
title: 'Container host B',
|
||||
textContent: 'Prelude.\nТеги: Docker\nepilogue.',
|
||||
});
|
||||
const noise = await insertPage({
|
||||
title: 'Unrelated',
|
||||
textContent: 'Теги: Kubernetes',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'Теги: Docker', spaceId, substring: true, limit: 50 } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
expect(ids).toContain(a);
|
||||
expect(ids).toContain(b);
|
||||
expect(ids).not.toContain(noise);
|
||||
});
|
||||
|
||||
it('populates a non-empty `path` for a nested hit and `[]` for a root hit', async () => {
|
||||
const root = await insertPage({ title: 'Infrastructure' });
|
||||
const mid = await insertPage({ title: 'Datacenter A', parentPageId: root });
|
||||
const leaf = await insertPage({
|
||||
title: 'unique-nested-host',
|
||||
parentPageId: mid,
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'unique-nested-host', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === leaf);
|
||||
expect(hit.path).toEqual(['Infrastructure', 'Datacenter A']);
|
||||
|
||||
const rootHits = (await service.searchPage(
|
||||
{ query: 'Infrastructure', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
const rootHit = rootHits.items.find((i: any) => i.id === root);
|
||||
expect(rootHit.path).toEqual([]);
|
||||
});
|
||||
|
||||
it('scopes to a subtree with parentPageId (cutting off sibling branches)', async () => {
|
||||
const branchA = await insertPage({ title: 'BranchA-root' });
|
||||
const inA = await insertPage({
|
||||
title: 'scoped-target-xyz',
|
||||
parentPageId: branchA,
|
||||
});
|
||||
const branchB = await insertPage({ title: 'BranchB-root' });
|
||||
const inB = await insertPage({
|
||||
title: 'scoped-target-xyz',
|
||||
parentPageId: branchB,
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{
|
||||
query: 'scoped-target-xyz',
|
||||
spaceId,
|
||||
substring: true,
|
||||
parentPageId: branchA,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
expect(ids).toContain(inA);
|
||||
expect(ids).not.toContain(inB);
|
||||
});
|
||||
|
||||
it('includes the parent page itself in the parentPageId subtree', async () => {
|
||||
const parent = await insertPage({ title: 'self-included-parent' });
|
||||
await insertPage({ title: 'child-of-self', parentPageId: parent });
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{
|
||||
query: 'self-included-parent',
|
||||
spaceId,
|
||||
substring: true,
|
||||
parentPageId: parent,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
expect(items.map((i: any) => i.id)).toContain(parent);
|
||||
});
|
||||
|
||||
it('titleOnly does NOT match on text_content', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'Plain title',
|
||||
textContent: 'body mentions the-secret-token here',
|
||||
});
|
||||
|
||||
const withText = (await service.searchPage(
|
||||
{ query: 'the-secret-token', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
expect(withText.items.map((i: any) => i.id)).toContain(pageId);
|
||||
|
||||
const titleOnly = (await service.searchPage(
|
||||
{
|
||||
query: 'the-secret-token',
|
||||
spaceId,
|
||||
substring: true,
|
||||
titleOnly: true,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
expect(titleOnly.items.map((i: any) => i.id)).not.toContain(pageId);
|
||||
});
|
||||
|
||||
// #443 Fix #1 regression: f_unaccent is NOT length-preserving, so an
|
||||
// expanding char (ß→ss, …→...) BEFORE the match shifted the strpos position
|
||||
// relative to the ORIGINAL text and the snippet slice ran past end → empty.
|
||||
// The position and the slice now share the LOWER(f_unaccent(...)) space, so
|
||||
// the window is aligned and always contains the matched (unaccented) token.
|
||||
it('returns a populated snippet when an unaccent-EXPANDING char precedes the match', async () => {
|
||||
// 300 × `ß` (each f_unaccent-expands to `ss`) before the needle. Under the
|
||||
// old code strpos returned a position ~593 in the expanded space but the
|
||||
// slice ran over the ORIGINAL (~360 char) text → empty snippet, match lost.
|
||||
const prefix = 'ß'.repeat(300);
|
||||
const pageId = await insertPage({
|
||||
title: 'Expanding-unaccent page',
|
||||
textContent: `${prefix} needle-token-xyz trailing.`,
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'needle-token-xyz', spaceId, substring: true } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit).toBeDefined();
|
||||
// Snippet must be non-empty AND contain the matched token (unaccented form).
|
||||
expect(hit.snippet.length).toBeGreaterThan(0);
|
||||
expect(hit.snippet).toContain('needle-token-xyz');
|
||||
});
|
||||
|
||||
// #443 Fix #2 regression: >200 matching pages for a broad substring, with
|
||||
// exactly ONE exact-title hit. Without an ORDER BY on the 200-cap the exact
|
||||
// hit could be among the arbitrarily-dropped rows; the ORDER BY keeps the
|
||||
// strongest candidates so it must survive the cap and rank at the top.
|
||||
it('keeps an exact-title hit through the 200-cap on a >200-row match set', async () => {
|
||||
const isoSpace = (await createSpace(db, workspaceId)).id;
|
||||
const svc = buildService({ userSpaceIds: [isoSpace] });
|
||||
|
||||
// 250 low-tier TEXT hits: the shared substring `capword` appears only in the
|
||||
// body, never the title, so each is a TEXT-tier match (weakest tier).
|
||||
for (let i = 0; i < 250; i++) {
|
||||
await insertPage({
|
||||
title: `filler-page-${i}`,
|
||||
textContent: `body contains capword here #${i}`,
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
}
|
||||
// Exactly one EXACT-title hit for the same query token.
|
||||
const exact = await insertPage({
|
||||
title: 'capword',
|
||||
textContent: 'unrelated body text',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
|
||||
const { items } = (await svc.searchPage(
|
||||
{ query: 'capword', spaceId: isoSpace, substring: true, limit: 10 } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
// The exact-title hit must survive the 200-cap and appear in the top `limit`.
|
||||
expect(ids).toContain(exact);
|
||||
// And, being TITLE_EXACT, it must be the single strongest hit.
|
||||
expect(items[0].id).toBe(exact);
|
||||
});
|
||||
|
||||
// #443 Fix #3: titleOnly matches only the title, so it must not leak the page
|
||||
// body as the snippet (the old "first 300 chars of text_content" fallback).
|
||||
it('titleOnly does NOT return a text-body snippet', async () => {
|
||||
const pageId = await insertPage({
|
||||
title: 'titleonly-snippet-page',
|
||||
textContent: 'SECRET-BODY-CONTENT-NOT-IN-TITLE that must not leak.',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{
|
||||
query: 'titleonly-snippet-page',
|
||||
spaceId,
|
||||
substring: true,
|
||||
titleOnly: true,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const hit = items.find((i: any) => i.id === pageId);
|
||||
expect(hit).toBeDefined();
|
||||
// The body text must not appear in the snippet; titleOnly → empty snippet.
|
||||
expect(hit.snippet).not.toContain('SECRET-BODY-CONTENT-NOT-IN-TITLE');
|
||||
expect(hit.snippet).toBe('');
|
||||
});
|
||||
|
||||
it('returns [] (not an error) for a query that matches nothing', async () => {
|
||||
const { items } = (await service.searchPage(
|
||||
{
|
||||
query: 'zzz-no-such-string-anywhere-42',
|
||||
spaceId,
|
||||
substring: true,
|
||||
} as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
expect(items).toEqual([]);
|
||||
});
|
||||
|
||||
it('a `%` query does NOT match everything (LIKE metacharacter escaped)', async () => {
|
||||
// Fresh space so we can assert on total counts without cross-test noise.
|
||||
const isoSpace = (await createSpace(db, workspaceId)).id;
|
||||
const svc = buildService({ userSpaceIds: [isoSpace] });
|
||||
await insertPage({ title: 'alpha', spaceId: isoSpace });
|
||||
await insertPage({ title: 'beta', spaceId: isoSpace });
|
||||
const literal = await insertPage({
|
||||
title: '100%-coverage',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
|
||||
const { items } = (await svc.searchPage(
|
||||
{ query: '%', spaceId: isoSpace, substring: true, limit: 50 } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
// `%` is a literal → matches only the page that actually contains '%'.
|
||||
expect(ids).toContain(literal);
|
||||
expect(ids).not.toContain(
|
||||
items.find((i: any) => i.title === 'alpha')?.id,
|
||||
);
|
||||
expect(items.length).toBe(1);
|
||||
});
|
||||
|
||||
it('an `_` query does NOT match everything (LIKE metacharacter escaped)', async () => {
|
||||
const isoSpace = (await createSpace(db, workspaceId)).id;
|
||||
const svc = buildService({ userSpaceIds: [isoSpace] });
|
||||
await insertPage({ title: 'gamma', spaceId: isoSpace });
|
||||
const literal = await insertPage({
|
||||
title: 'snake_case_name',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
|
||||
const { items } = (await svc.searchPage(
|
||||
{ query: '_', spaceId: isoSpace, substring: true, limit: 50 } as any,
|
||||
{ workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
expect(ids).toContain(literal);
|
||||
expect(items.length).toBe(1);
|
||||
});
|
||||
|
||||
it('applies the permission post-filter to the MERGED set BEFORE the limit', async () => {
|
||||
const isoSpace = (await createSpace(db, workspaceId)).id;
|
||||
const keep = await insertPage({
|
||||
title: 'perm-visible-target',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
const hidden = await insertPage({
|
||||
title: 'perm-hidden-target',
|
||||
spaceId: isoSpace,
|
||||
});
|
||||
|
||||
// Authenticated (userId set) so the permission filter runs; only `keep` is
|
||||
// accessible. limit 1 must NOT be able to select `hidden`.
|
||||
const svc = buildService({
|
||||
userSpaceIds: [isoSpace],
|
||||
accessibleIds: [keep],
|
||||
});
|
||||
const { items } = (await svc.searchPage(
|
||||
{
|
||||
query: 'perm-',
|
||||
spaceId: isoSpace,
|
||||
substring: true,
|
||||
limit: 1,
|
||||
} as any,
|
||||
{ userId: 'user-1', workspaceId },
|
||||
)) as any;
|
||||
|
||||
const ids = items.map((i: any) => i.id);
|
||||
expect(ids).toContain(keep);
|
||||
expect(ids).not.toContain(hidden);
|
||||
});
|
||||
|
||||
it('web-UI path (no `substring` flag) keeps the legacy FTS response shape', async () => {
|
||||
await insertPage({
|
||||
title: 'legacy shape page',
|
||||
textContent: 'searchable legacyword content',
|
||||
});
|
||||
|
||||
const { items } = (await service.searchPage(
|
||||
{ query: 'legacyword', spaceId } as any,
|
||||
{ userId: 'user-1', workspaceId },
|
||||
)) as any;
|
||||
|
||||
// Legacy hits carry rank + highlight + space, and NO path/snippet/score.
|
||||
const hit = items[0];
|
||||
expect(hit).toBeDefined();
|
||||
expect('rank' in hit).toBe(true);
|
||||
expect('highlight' in hit).toBe(true);
|
||||
expect('path' in hit).toBe(false);
|
||||
expect('snippet' in hit).toBe(false);
|
||||
expect('score' in hit).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
import { Kysely, sql } from 'kysely';
|
||||
import { getTestDb, destroyTestDb } from './db';
|
||||
import * as migration from '../../src/database/migrations/20260707T130000-search-ru-en-config';
|
||||
|
||||
/**
|
||||
* #529 A1 — the ru_en config migration must be REVERSIBLE in the correct order:
|
||||
* down() moves pages.tsv + page_embeddings.fts back to `english` BEFORE dropping
|
||||
* the ru_en config (a generated column still depending on ru_en would block the
|
||||
* DROP). This roundtrips down()→up() on the already-migrated test DB and asserts
|
||||
* the config, the pages trigger and the fts generated expression each flip and
|
||||
* flip back — the deploy-critical property.
|
||||
*/
|
||||
describe('search ru_en config migration [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
|
||||
const configExists = async () => {
|
||||
const r = await sql<{ n: number }>`
|
||||
SELECT count(*)::int AS n FROM pg_ts_config WHERE cfgname = 'ru_en'
|
||||
`.execute(db);
|
||||
return r.rows[0].n > 0;
|
||||
};
|
||||
|
||||
const ftsDef = async () => {
|
||||
const r = await sql<{ def: string }>`
|
||||
SELECT pg_get_expr(adbin, adrelid) AS def
|
||||
FROM pg_attrdef d
|
||||
JOIN pg_attribute a ON a.attrelid = d.adrelid AND a.attnum = d.adnum
|
||||
WHERE a.attname = 'fts' AND a.attrelid = 'page_embeddings'::regclass
|
||||
`.execute(db);
|
||||
return r.rows[0]?.def ?? '';
|
||||
};
|
||||
|
||||
const triggerSrc = async () => {
|
||||
const r = await sql<{ src: string }>`
|
||||
SELECT prosrc AS src FROM pg_proc WHERE proname = 'pages_tsvector_trigger'
|
||||
`.execute(db);
|
||||
return r.rows[0]?.src ?? '';
|
||||
};
|
||||
|
||||
beforeAll(() => {
|
||||
db = getTestDb();
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
// Restore the canonical ru_en state for any later suite regardless of where
|
||||
// a test left off. up() is now safe on an existing config (ensureRuEnConfig
|
||||
// no-ops) and re-asserts both stored sides to ru_en.
|
||||
delete process.env.SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE;
|
||||
await migration.up(db);
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('starts at ru_en (config present, trigger + fts on ru_en)', async () => {
|
||||
expect(await configExists()).toBe(true);
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('ru_en');
|
||||
});
|
||||
|
||||
it('down() reverts tsv + fts to english THEN drops the config', async () => {
|
||||
await migration.down(db);
|
||||
expect(await configExists()).toBe(false);
|
||||
expect(await ftsDef()).toContain('english');
|
||||
expect(await ftsDef()).not.toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('english');
|
||||
});
|
||||
|
||||
it('up() re-applies ru_en cleanly (idempotent config create)', async () => {
|
||||
await migration.up(db);
|
||||
expect(await configExists()).toBe(true);
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('ru_en');
|
||||
});
|
||||
|
||||
// B1.1 — running up() a SECOND time on an already-migrated DB is a true no-op:
|
||||
// it must NOT throw (the old drop-recreate-config would fail on the fts hard
|
||||
// dependency) and must NOT re-rewrite the embeddings table — the fts column
|
||||
// already references ru_en, so the at-target check short-circuits.
|
||||
it('up() is idempotent: a 2nd run does not error and leaves ru_en intact', async () => {
|
||||
await expect(migration.up(db)).resolves.toBeUndefined();
|
||||
expect(await configExists()).toBe(true);
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('ru_en');
|
||||
});
|
||||
|
||||
// B1.2 — env-gate opt-out: with SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE=false the
|
||||
// embeddings rewrite is SKIPPED (fts stays where it was) but pages.tsv still
|
||||
// swaps inline, and the ru_en config is left in place (fts still depends on it).
|
||||
// Runs down() as the vehicle: english is NOT the current fts config, so the
|
||||
// skip path — not the at-target no-op — is exercised.
|
||||
it('env-gate=false: skips the fts rewrite but still swaps pages.tsv', async () => {
|
||||
// Precondition: fts + trigger on ru_en (from the prior test).
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
process.env.SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE = 'false';
|
||||
try {
|
||||
await migration.down(db);
|
||||
// fts rewrite skipped → still ru_en (the ACCESS EXCLUSIVE rewrite avoided).
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await ftsDef()).not.toContain('english');
|
||||
// pages.tsv trigger still swapped inline to english (gate is fts-only).
|
||||
expect(await triggerSrc()).toContain('english');
|
||||
// Config left in place because fts still references it (guarded drop).
|
||||
expect(await configExists()).toBe(true);
|
||||
} finally {
|
||||
// Restore ru_en fully for the afterAll / later suites.
|
||||
delete process.env.SEARCH_EMBEDDINGS_FTS_INLINE_REWRITE;
|
||||
await migration.up(db);
|
||||
expect(await ftsDef()).toContain('ru_en');
|
||||
expect(await triggerSrc()).toContain('ru_en');
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -78,7 +78,7 @@ export interface IReadMixin {
|
||||
getNode(pageId: string, nodeId: string, format?: "markdown" | "json"): any;
|
||||
searchInPage(pageId: string, query: string, opts?: SearchOptions): any;
|
||||
getTable(pageId: string, tableRef: string): any;
|
||||
search(query: string, spaceId?: string, limit?: number, opts?: { parentPageId?: string; titleOnly?: boolean }): any;
|
||||
search(query: string, spaceId?: string, limit?: number, opts?: { parentPageId?: string; titleOnly?: boolean; offset?: number; match?: "auto" | "word" | "prefix" | "substring" }): any;
|
||||
}
|
||||
|
||||
export function ReadMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & IReadMixin> & TBase {
|
||||
@@ -701,13 +701,19 @@ export function ReadMixin<TBase extends GConstructor<DocmostClientContext>>(Base
|
||||
query: string,
|
||||
spaceId?: string,
|
||||
limit?: number,
|
||||
opts: { parentPageId?: string; titleOnly?: boolean } = {},
|
||||
opts: {
|
||||
parentPageId?: string;
|
||||
titleOnly?: boolean;
|
||||
offset?: number;
|
||||
match?: "auto" | "word" | "prefix" | "substring";
|
||||
} = {},
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
// Opt into the #443 agent-lookup mode: `substring: true` turns on the hybrid
|
||||
// substring + FTS branch that returns path + snippet + score. A stock
|
||||
// upstream server strips these unknown DTO fields (whitelist:true) and
|
||||
// silently degrades to plain FTS — see the tool-registration comment.
|
||||
// #529 unified engine: the query is parsed SERVER-SIDE (operators
|
||||
// "phrase"/+/-, OR default, RU+EN morphology, match=auto). We forward the RAW
|
||||
// query plus flags. `substring: true` is kept as a back-compat hint for a
|
||||
// stock upstream server (whitelist:true strips the unknown #529 fields and it
|
||||
// degrades to plain FTS — see the tool-registration comment).
|
||||
const payload: Record<string, any> = {
|
||||
query,
|
||||
spaceId,
|
||||
@@ -715,22 +721,34 @@ export function ReadMixin<TBase extends GConstructor<DocmostClientContext>>(Base
|
||||
};
|
||||
if (opts.parentPageId) payload.parentPageId = opts.parentPageId;
|
||||
if (opts.titleOnly) payload.titleOnly = true;
|
||||
if (opts.match) payload.match = opts.match;
|
||||
// Clamp an optional caller-supplied limit into the lookup range (1..50)
|
||||
// before forwarding; omit it when not provided so the server default applies.
|
||||
if (limit !== undefined) {
|
||||
payload.limit = Math.max(1, Math.min(50, limit));
|
||||
}
|
||||
|
||||
if (opts.offset !== undefined) {
|
||||
payload.offset = Math.max(0, Math.floor(opts.offset));
|
||||
}
|
||||
|
||||
const runSearch = async () => {
|
||||
const response = await this.client.post("/search", payload);
|
||||
|
||||
// Normalize both response shapes: bare array and paginated { items: [...] }
|
||||
// Normalize both response shapes: bare array and paginated { items: [...] }.
|
||||
const data = response.data?.data;
|
||||
const items = Array.isArray(data) ? data : data?.items || [];
|
||||
const filteredItems = items.map((item: any) => filterSearchResult(item));
|
||||
|
||||
// Surface the #529 pagination envelope when present (a stock upstream has
|
||||
// none — the fields are simply undefined and the caller sees just `items`).
|
||||
const envelope = Array.isArray(data) ? undefined : data;
|
||||
return {
|
||||
items: filteredItems,
|
||||
total: envelope?.total,
|
||||
hasMore: envelope?.hasMore,
|
||||
truncatedAtCap: envelope?.truncatedAtCap,
|
||||
offset: envelope?.offset,
|
||||
success: response.data?.success || false,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -450,15 +450,22 @@ server.registerTool(
|
||||
"search",
|
||||
{
|
||||
description:
|
||||
"Find pages by a fragment of a technical string (hostnames, IPs, IDs " +
|
||||
"like `srv.local`, `10.0.12`, `WB-MGE-30D86B`) — one call returns each " +
|
||||
"hit's location (`path`: ancestor titles root→parent) and a `snippet` " +
|
||||
"around the first match, so you rarely need a follow-up get_page. " +
|
||||
"Matches substrings literally (dots/dashes/digits are not tokenized) as " +
|
||||
"well as full-text. Returns `{ pageId, title, path, snippet, score }` " +
|
||||
"sorted by `score` (a per-response relevance float).",
|
||||
"Search pages across the wiki. OR by default with relevance ranking " +
|
||||
"(RU+EN morphology): multi-word queries match ANY term, not all. " +
|
||||
"Operators: \"exact phrase\" (adjacent words), +term (require), -term " +
|
||||
"(exclude) — e.g. `+кофейня -архив`, `+\"воздушный шар\" кофе`. A leading " +
|
||||
"-/+ is the operator; -,.,: INSIDE a token are literal (`WB-MGE-30D86B`, " +
|
||||
"`10.0.12.5` stay one term). Technical fragments (hostnames, IPs, IDs) " +
|
||||
"auto-match as substrings; words use full-text. Each hit returns its " +
|
||||
"location (`path`: ancestor titles root→parent), a `snippet`, `score`, " +
|
||||
"`matchedTerms` and `matchedFields`, so you rarely need a follow-up " +
|
||||
"getPage. Paginate with limit + offset; the response carries " +
|
||||
"`total` (exact, permission-filtered), `hasMore` and `truncatedAtCap`. " +
|
||||
"NOTE: results past the relevance cap (~500) are unreachable by " +
|
||||
"pagination — narrow the query (add terms / +required / a spaceId) " +
|
||||
"instead when `truncatedAtCap` is true.",
|
||||
inputSchema: {
|
||||
query: z.string().min(1).describe("Search query"),
|
||||
query: z.string().min(1).describe("Search query (supports \"phrase\", +require, -exclude)"),
|
||||
spaceId: z
|
||||
.string()
|
||||
.optional()
|
||||
@@ -473,6 +480,13 @@ server.registerTool(
|
||||
.boolean()
|
||||
.optional()
|
||||
.describe("Match page titles only; skip page text"),
|
||||
match: z
|
||||
.enum(["auto", "word", "prefix", "substring"])
|
||||
.optional()
|
||||
.describe(
|
||||
"Match mode (default auto: identifiers→substring, words→full-text). " +
|
||||
"Override with word/prefix/substring.",
|
||||
),
|
||||
limit: z
|
||||
.number()
|
||||
.int()
|
||||
@@ -480,12 +494,20 @@ server.registerTool(
|
||||
.max(50)
|
||||
.optional()
|
||||
.describe("Max results to return (1-50, default 10)"),
|
||||
offset: z
|
||||
.number()
|
||||
.int()
|
||||
.min(0)
|
||||
.optional()
|
||||
.describe("Pagination offset (default 0); use with total/hasMore"),
|
||||
},
|
||||
},
|
||||
async ({ query, spaceId, parentPageId, titleOnly, limit }) => {
|
||||
async ({ query, spaceId, parentPageId, titleOnly, match, limit, offset }) => {
|
||||
const result = await docmostClient.search(query, spaceId, limit, {
|
||||
parentPageId,
|
||||
titleOnly,
|
||||
match,
|
||||
offset,
|
||||
});
|
||||
return jsonContent(result);
|
||||
},
|
||||
|
||||
@@ -40,7 +40,7 @@ import { SHARED_TOOL_SPECS, SharedToolSpec } from "./tool-specs.js";
|
||||
*/
|
||||
export const ROUTING_PROSE =
|
||||
"Docmost editing guide — choose the tool by intent. The <tool_inventory> at the end lists every tool with a one-line purpose; the notes below are the routing hints for WHEN to reach for each.\n" +
|
||||
"READ: find a page by a fragment of a technical string (hostname/IP/ID like srv.local, 10.0.12, WB-MGE-30D86B) -> search — hybrid substring + full-text, returns each hit's location (path: root->parent titles) and a snippet around the match, so you rarely need a follow-up getPage; scope with spaceId or parentPageId (a subtree), titleOnly to match titles only. A space's page HIERARCHY (or one subtree) -> getTree (one request, complete, `{pageId,title,children?}`; rootPageId for a subtree, maxDepth to trim depth — a trimmed node gets hasChildren:true); prefer it over listPages tree:true (deprecated). Have a pageId, need WHERE-AM-I / what's around it (its breadcrumbs + direct children, metadata only) -> getPageContext (one call; parent = last breadcrumb, [] for a root page). list -> listPages / listSpaces. Locate blocks and their ids CHEAPLY -> getOutline (compact top-level map; start here, not getPageJson). One block, for editing -> getNode (by attrs.id, or \"#<index>\" for tables, which carry no id) — returns MARKDOWN by default (comment anchors kept for safe write-back); pass format:\"json\" for the raw ProseMirror subtree. Find every occurrence of a string/regex ON a page (and where each is) -> searchInPage, NOT block-by-block getNode — it returns each hit's node ref + block index + context for a targeted comment. Whole page -> getPage (Markdown, canonical for text; drops only block ids, resolved-comment anchors, and a fixed no-md-representation attr set: table spans/colwidth/bg, indent, callout.icon, orderedList.type, link internal/target/rel/class; inline <span data-comment-id> tags are comment anchors — markup, not text) or getPageJson (full ProseMirror with block ids, for those dropped attrs). Hand a huge page (with images) to an external consumer without pulling it through the model context -> stashPage (returns a short-lived anonymous URL).\n" +
|
||||
"READ: find pages across the wiki -> search — OR by default with relevance ranking and RU+EN morphology (multi-word matches ANY term). Operators: \"exact phrase\", +require, -exclude (e.g. `+кофейня -архив`, `+\"воздушный шар\" кофе`); a leading -/+ is the operator, but -,.,: inside a token are literal (WB-MGE-30D86B, 10.0.12.5 stay one term, auto-matched as substrings). Each hit returns its location (path: root->parent titles), a snippet, score, matchedTerms/matchedFields, so you rarely need a follow-up getPage; scope with spaceId or parentPageId (a subtree), titleOnly to match titles only, match to override auto. Paginate with limit+offset; total is exact and permission-filtered, hasMore/truncatedAtCap flag more. Results past the relevance cap (~500) are UNREACHABLE by pagination — when truncatedAtCap is true, narrow the query (add terms / +required / a spaceId) rather than page deeper. A space's page HIERARCHY (or one subtree) -> getTree (one request, complete, `{pageId,title,children?}`; rootPageId for a subtree, maxDepth to trim depth — a trimmed node gets hasChildren:true); prefer it over listPages tree:true (deprecated). Have a pageId, need WHERE-AM-I / what's around it (its breadcrumbs + direct children, metadata only) -> getPageContext (one call; parent = last breadcrumb, [] for a root page). list -> listPages / listSpaces. Locate blocks and their ids CHEAPLY -> getOutline (compact top-level map; start here, not getPageJson). One block, for editing -> getNode (by attrs.id, or \"#<index>\" for tables, which carry no id) — returns MARKDOWN by default (comment anchors kept for safe write-back); pass format:\"json\" for the raw ProseMirror subtree. Find every occurrence of a string/regex ON a page (and where each is) -> searchInPage, NOT block-by-block getNode — it returns each hit's node ref + block index + context for a targeted comment. Whole page -> getPage (Markdown, canonical for text; drops only block ids, resolved-comment anchors, and a fixed no-md-representation attr set: table spans/colwidth/bg, indent, callout.icon, orderedList.type, link internal/target/rel/class; inline <span data-comment-id> tags are comment anchors — markup, not text) or getPageJson (full ProseMirror with block ids, for those dropped attrs). Hand a huge page (with images) to an external consumer without pulling it through the model context -> stashPage (returns a short-lived anonymous URL).\n" +
|
||||
"EDIT: fix wording/typos/numbers -> editPageText (find/replace inside blocks, no node id needed). Edit a block -> getNode(markdown) -> edit the markdown -> patchNode(markdown) (by attrs.id from getOutline; the markdown fragment may be several blocks — a 1->N section rewrite in one call, the first block keeps the id). Reach for patchNode's `node`-JSON only for fine attr/mark work; a table cell with spans/colors/fixed width -> the table tools (patchNode markdown refuses it). Add a block -> insertNode (markdown, before/after a block by attrs.id or by anchor text, or append; `node` for raw JSON or bare table structure). Remove a block -> deleteNode (by attrs.id). Tables -> tableGet / tableUpdateCell / tableInsertRow / tableDeleteRow (address by \"#<index>\" from getOutline; table nodes have no attrs.id). Images -> insertImage (add from a web URL) / replaceImage (swap an existing image). Draw.io diagrams -> PREFER the high-level semantic tools that hide coordinates/styles: drawioFromGraph (architecture/cloud/network diagrams — describe nodes/groups/edges by kind+icon, the server picks layout, colors and verified icons; hints layer/sameLayerAs/pinned and layout:full|incremental|none) and drawioFromMermaid (standard flowcharts — write Mermaid, get an editable diagram). For targeted tweaks of an existing diagram use drawioEditCells (id-based add/update/delete with cascade delete + baseHash lock). Raw mxGraph XML via drawioCreate/drawioUpdate is the escape-hatch for exotic/wireframe diagrams; drawioGet reads a diagram as mxGraph XML + a hash (pass it as baseHash to drawioUpdate/drawioEditCells for optimistic locking). Before authoring raw XML, drawioShapes (look up verified stencil style-strings so a shape name never renders as an empty box) and drawioGuide (on-demand authoring reference: skeleton/layout/containers/icons-aws/icons-azure), and pass layout:\"elk\" to drawioCreate/drawioUpdate to auto-place nodes. Footnotes -> insertFootnote. Bulk/structural rewrite -> updatePageJson (full ProseMirror replace) or updatePageMarkdown (full plain-Markdown body replace, re-imported — block ids regenerate); prefer the granular tools above to avoid resending the whole ~100KB+ document. Complex/scripted rewrite (multiple coordinated edits, renumbering) -> docmostTransform: write a JS `(doc, ctx) => doc` transform, preview the diff with dryRun (default), then apply with dryRun:false; ctx.helpers includes commentsToFootnotes for turning inline comments into numbered footnotes.\n" +
|
||||
"PAGES: new -> createPage (Markdown). Rename (title only) -> renamePage. Move -> movePage. Delete -> deletePage (SOFT delete — the page goes to trash and is restorable; nothing is permanent). Copy/replace a page's whole content from another page (server-side, no document through the model) -> copyPageContent. Sharing -> sharePage / unsharePage / listShares; sharePage makes the page PUBLICLY accessible — do it only when explicitly asked.\n" +
|
||||
"COMMENTS: createComment is always inline and requires an EXACT selection — contiguous text from a single block, <=250 chars (fails rather than leaving an unanchored comment); reply to a thread via parentCommentId. Propose a concrete text fix for one-click human approval -> createComment with suggestedText (the exact plain-text replacement for the selection; the selection must then be UNIQUE in the page — extend it with context if needed); prefer this over editing directly when the change is subjective or needs the author's sign-off. Manage -> listComments, updateComment, resolveComment (resolve/reopen, reversible — prefer over delete to close), deleteComment, checkNewComments.\n" +
|
||||
@@ -72,6 +72,12 @@ export const PROSE_NON_TOOL_TERMS: ReadonlySet<string> = new Set([
|
||||
"parentCommentId",
|
||||
"suggestedText",
|
||||
"historyId",
|
||||
// search RESPONSE fields documented in the routing prose (#529) — schema
|
||||
// fields the search tool returns, not tools themselves
|
||||
"matchedTerms",
|
||||
"matchedFields",
|
||||
"hasMore",
|
||||
"truncatedAtCap",
|
||||
// helper / value fragments
|
||||
"orderedList", // "orderedList.type" (a dropped attr, not a tool)
|
||||
"mxGraph", // "mxGraph XML"
|
||||
@@ -234,7 +240,7 @@ export const INLINE_MCP_INVENTORY: ToolInventoryLine[] = [
|
||||
{
|
||||
name: "search",
|
||||
purpose:
|
||||
"find pages by a fragment of a technical string (hybrid substring + full-text); returns each hit's path and a snippet.",
|
||||
"search pages across the wiki (OR default, RU+EN morphology, \"phrase\"/+/- operators, pagination); returns each hit's path, snippet, score and matched terms.",
|
||||
},
|
||||
{
|
||||
name: "docmostTransform",
|
||||
|
||||
@@ -175,3 +175,26 @@ test("#494: PROSE_NON_TOOL_TERMS holds no actually-registered tool name", () =>
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// #529: the search routing prose must document the new engine contract — the
|
||||
// operators, OR/morphology default, pagination fields and the relevance-CAP
|
||||
// caveat — so an agent uses the operators and understands the unreachable tail.
|
||||
test("SERVER_INSTRUCTIONS documents the #529 search operators, pagination and CAP", () => {
|
||||
const read = ROUTING_PROSE.split("EDIT:")[0]; // the READ family section
|
||||
// Operators.
|
||||
assert.ok(/\+require/.test(read), "search prose missing +require operator");
|
||||
assert.ok(/-exclude/.test(read), "search prose missing -exclude operator");
|
||||
assert.ok(/phrase/i.test(read), "search prose missing phrase operator");
|
||||
// OR default + morphology.
|
||||
assert.ok(/\bOR\b/.test(read), "search prose missing OR-default note");
|
||||
assert.ok(/morpholog/i.test(read), "search prose missing morphology note");
|
||||
// Pagination + exact permission-filtered total.
|
||||
assert.ok(/offset/.test(read), "search prose missing offset/pagination");
|
||||
assert.ok(/total is exact/i.test(read), "search prose missing exact total");
|
||||
assert.ok(/hasMore|truncatedAtCap/.test(read), "search prose missing hasMore/cap flags");
|
||||
// The relevance CAP caveat (tail unreachable by pagination).
|
||||
assert.ok(
|
||||
/cap/i.test(read) && /unreachable/i.test(read),
|
||||
"search prose missing the relevance-CAP unreachable-tail caveat",
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user