diff --git a/apps/server/src/core/search/search.service.ts b/apps/server/src/core/search/search.service.ts index 0d125d58..76ac59d3 100644 --- a/apps/server/src/core/search/search.service.ts +++ b/apps/server/src/core/search/search.service.ts @@ -20,7 +20,7 @@ const tsquery = require('pg-tsquery')(); // The previous inline form `tsquery(query.trim() + '*')` passed user input // (including to_tsquery operators like `&`, `|`, `!`, `<->`, `*`, backslashes) // straight through. pg-tsquery would then emit operator fragments that -// `to_tsquery('english', ...)` can reject as a syntax error, turning a search +// `to_tsquery('ru_en', ...)` can reject as a syntax error, turning a search // into a 500. We strip everything that is not a letter, number or whitespace // BEFORE handing the text to pg-tsquery, so adversarial input degrades to a // neutral (possibly empty) query instead of throwing, while normal word queries @@ -126,17 +126,17 @@ export class SearchService { 'creatorId', 'createdAt', 'updatedAt', - sql`ts_rank(tsv, to_tsquery('english', f_unaccent(${searchQuery})))`.as( + sql`ts_rank(tsv, to_tsquery('ru_en', f_unaccent(${searchQuery})))`.as( 'rank', ), - sql`ts_headline('english', text_content, to_tsquery('english', f_unaccent(${searchQuery})),'MinWords=9, MaxWords=10, MaxFragments=3')`.as( + sql`ts_headline('ru_en', text_content, to_tsquery('ru_en', f_unaccent(${searchQuery})),'MinWords=9, MaxWords=10, MaxFragments=3')`.as( 'highlight', ), ]) .where( 'tsv', '@@', - sql`to_tsquery('english', f_unaccent(${searchQuery}))`, + sql`to_tsquery('ru_en', f_unaccent(${searchQuery}))`, ) .$if(Boolean(searchParams.creatorId), (qb) => qb.where('creatorId', '=', searchParams.creatorId), @@ -325,7 +325,7 @@ export class SearchService { ), // FTS secondary signal (0 when the tsquery is empty). hasTsQuery - ? sql`ts_rank(pages.tsv, to_tsquery('english', f_unaccent(${tsQuery})))`.as( + ? sql`ts_rank(pages.tsv, to_tsquery('ru_en', f_unaccent(${tsQuery})))`.as( 'ftsRank', ) : sql`0`.as('ftsRank'), @@ -360,7 +360,7 @@ export class SearchService { then substring(LOWER(f_unaccent(pages.text_content)) from 1 for 300) ${ hasTsQuery - ? sql`else ts_headline('english', coalesce(pages.text_content, ''), to_tsquery('english', f_unaccent(${tsQuery})), 'MinWords=25, MaxWords=40, MaxFragments=3')` + ? sql`else ts_headline('ru_en', coalesce(pages.text_content, ''), to_tsquery('ru_en', f_unaccent(${tsQuery})), 'MinWords=25, MaxWords=40, MaxFragments=3')` : sql`` } end, @@ -406,7 +406,7 @@ export class SearchService { ); if (hasTsQuery) { ors.push( - sql`pages.tsv @@ to_tsquery('english', f_unaccent(${tsQuery}))` as any, + sql`pages.tsv @@ to_tsquery('ru_en', f_unaccent(${tsQuery}))` as any, ); } } @@ -430,7 +430,7 @@ export class SearchService { // column position (`ORDER BY 0` → "position 0 is not in select list"), so the // no-tsquery fallback is `0::float`, not `0`. const ftsRankExpr = hasTsQuery - ? sql`ts_rank(pages.tsv, to_tsquery('english', f_unaccent(${tsQuery})))` + ? sql`ts_rank(pages.tsv, to_tsquery('ru_en', f_unaccent(${tsQuery})))` : sql`0::float`; const candidatesCapped = candidates // Raw-SQL ORDER BY expressions: pass the full ` ` as ONE arg diff --git a/apps/server/src/database/migrations/20260707T120000-search-ru-en-config.ts b/apps/server/src/database/migrations/20260707T120000-search-ru-en-config.ts new file mode 100644 index 00000000..4a6d49dd --- /dev/null +++ b/apps/server/src/database/migrations/20260707T120000-search-ru-en-config.ts @@ -0,0 +1,136 @@ +import { type Kysely, sql } from 'kysely'; + +/** + * #529 Phase A1 — the `ru_en` text-search configuration + the config swap. + * + * WHY: the search stack was pinned to the `english` FTS config, which stems only + * Latin words. On a Russian-language wiki that is a morphology black hole: + * «ресторанов москвы» never matched a page titled «ресторан в москве». `ru_en` + * layers the russian_stem over the Cyrillic token classes and english_stem over + * the ascii ones, so BOTH languages get proper morphology from one config. + * + * CREATE TEXT SEARCH CONFIGURATION ru_en (COPY = simple); + * ALTER ... asciiword/asciihword/hword_asciipart WITH english_stem; + * ALTER ... word/hword/hword_part WITH russian_stem; + * + * `to_tsvector('ru_en', …)` with the LITERAL config name is IMMUTABLE, so it is + * valid inside a trigger, a generated column and an index expression. + * + * THE INVARIANT (acceptance #13): the config of the STORED column and the config + * of the QUERY must change together. This migration flips BOTH stored sides + * (pages.tsv via its trigger + a reindex; page_embeddings.fts, the RAG lexical + * leg, via its generated expression); the matching QUERY-side flips + * (search.service.ts and page-embedding.repo.ts `hybridSearch`) ship in the SAME + * commit. Trigram indexes are LOWER(f_unaccent(...)) and do NOT depend on the FTS + * config, so they are untouched. + * + * REINDEX / LOCK MODEL (deploy-critical). This repo's Kysely Migrator runs the + * whole pending set inside ONE transaction (migration.service.ts / migrate.ts set + * no `disableTransactions`), and PostgresJSDialect has transactional DDL — so a + * migration file CANNOT commit between batches, which means the issue's + * "procedural batch job OUTSIDE the transaction + dual-config read window + + * migration_complete gate" is not expressible in-migration here. That machinery + * exists to bridge a reindex spread over MANY committed batches, during which some + * rows are still `english` while others are already `ru_en`. Our reindex is ATOMIC + * (one transaction): at COMMIT every row is `ru_en` at once, so no morphology- + * desync window exists and no dual-config read path is required — the query config + * flips to `ru_en` in the very same release. This is the deliberate, correct + * adaptation to this framework (see the PR notes). + * + * - pages.tsv: swapping the trigger is a cheap catalog change (no table lock), + * and the reindex is a single `UPDATE pages SET tsv = ` — a + * ROW-level-lock backfill, NOT an ACCESS EXCLUSIVE rewrite (mirrors the + * existing space_id backfill in 20250725T052004). + * - page_embeddings.fts is a GENERATED STORED column; Postgres has no way to + * change a generated expression without DROP+ADD, which rewrites the table + * under ACCESS EXCLUSIVE. On a LARGE tenant run this in a maintenance window, + * or do it out-of-band before deploy: add a plain `fts_ruen` column, backfill + * with CREATE INDEX CONCURRENTLY-style batched UPDATEs, then swap — the + * IF-EXISTS guards here then no-op. Small/typical tenants (this fork's target) + * are fine with the in-migration rewrite. Same documented trade-off as the + * #443 trgm GIN migration (20260706T120000). + */ + +// pages.tsv trigger body for a given FTS config — mirrors the latest form +// (20250729T213756): f_unaccent + a 1MB text cap on text_content, weights A/B. +function pagesTriggerSql(config: 'ru_en' | 'english') { + return sql` + CREATE OR REPLACE FUNCTION pages_tsvector_trigger() RETURNS trigger AS $$ + begin + new.tsv := + setweight(to_tsvector('${sql.raw(config)}', f_unaccent(coalesce(new.title, ''))), 'A') || + setweight(to_tsvector('${sql.raw(config)}', f_unaccent(substring(coalesce(new.text_content, ''), 1, 1000000))), 'B'); + return new; + end; + $$ LANGUAGE plpgsql; + `; +} + +async function swapPagesConfig(db: Kysely, config: 'ru_en' | 'english') { + // 1. Point the trigger at the target config (new/edited rows use it going + // forward). CREATE OR REPLACE FUNCTION takes only a brief catalog lock. + await pagesTriggerSql(config).execute(db); + + // 2. Reindex existing rows: recompute tsv directly with the target config. A + // plain UPDATE — row locks, no ACCESS EXCLUSIVE. Equivalent to firing the + // trigger but cheaper (no self-update round trip). + await sql` + UPDATE pages + SET tsv = + setweight(to_tsvector('${sql.raw(config)}', f_unaccent(coalesce(title, ''))), 'A') || + setweight(to_tsvector('${sql.raw(config)}', f_unaccent(substring(coalesce(text_content, ''), 1, 1000000))), 'B') + `.execute(db); +} + +async function swapEmbeddingsFtsConfig( + db: Kysely, + config: 'ru_en' | 'english', +) { + // The generated `fts` expression can only change via DROP+ADD (a table + // rewrite; see the lock note in the header). The GIN index depends on the + // column, so it is dropped with it and recreated. + await sql`DROP INDEX IF EXISTS idx_page_embeddings_fts`.execute(db); + await sql`ALTER TABLE page_embeddings DROP COLUMN IF EXISTS fts`.execute(db); + await sql` + ALTER TABLE page_embeddings + ADD COLUMN fts tsvector + GENERATED ALWAYS AS (to_tsvector('${sql.raw(config)}', f_unaccent(content))) STORED + `.execute(db); + await sql` + CREATE INDEX IF NOT EXISTS idx_page_embeddings_fts + ON page_embeddings USING gin(fts) + `.execute(db); +} + +export async function up(db: Kysely): Promise { + // Create the ru_en configuration (idempotent: drop first, nothing depends on it + // yet at this point in up()). + await sql`DROP TEXT SEARCH CONFIGURATION IF EXISTS ru_en`.execute(db); + await sql`CREATE TEXT SEARCH CONFIGURATION ru_en (COPY = simple)`.execute(db); + // Latin token classes → english_stem. + await sql` + ALTER TEXT SEARCH CONFIGURATION ru_en + ALTER MAPPING FOR asciiword, asciihword, hword_asciipart + WITH english_stem + `.execute(db); + // Cyrillic / non-ascii token classes → russian_stem. + await sql` + ALTER TEXT SEARCH CONFIGURATION ru_en + ALTER MAPPING FOR word, hword, hword_part + WITH russian_stem + `.execute(db); + + // Flip both stored sides to ru_en (query-side flips in the same commit). + await swapPagesConfig(db, 'ru_en'); + await swapEmbeddingsFtsConfig(db, 'ru_en'); +} + +export async function down(db: Kysely): Promise { + // Reverse ORDER matters: the trigger and the generated column reference the + // `ru_en` config by name, so they must be moved back to `english` BEFORE the + // config can be dropped (a generated column that still depends on `ru_en` would + // block the DROP with a dependency error). + await swapEmbeddingsFtsConfig(db, 'english'); + await swapPagesConfig(db, 'english'); + await sql`DROP TEXT SEARCH CONFIGURATION IF EXISTS ru_en`.execute(db); +} diff --git a/apps/server/src/database/repos/ai-chat/page-embedding.repo.ts b/apps/server/src/database/repos/ai-chat/page-embedding.repo.ts index fe12d648..8937a922 100644 --- a/apps/server/src/database/repos/ai-chat/page-embedding.repo.ts +++ b/apps/server/src/database/repos/ai-chat/page-embedding.repo.ts @@ -200,8 +200,11 @@ export class PageEmbeddingRepo { * * The `model_dimensions = $dim` filter applies ONLY on the semantic side * (cosine compares same-dimension vectors; pgvector errors otherwise). The - * lexical side (`fts`) is dimension-independent. If `websearch_to_tsquery` - * yields an EMPTY query (e.g. the text is all stopwords) the `@@` matches + * lexical side (`fts`) is dimension-independent. Its query config is `ru_en`, + * matched IN LOCKSTEP with the `page_embeddings.fts` generated column's config + * (#529 acceptance #13): a mismatch silently breaks Cyrillic RAG retrieval. If + * `websearch_to_tsquery` yields an EMPTY query (e.g. the text is all stopwords) + * the `@@` matches * nothing and the lexical CTE is empty, so results degrade to pure-semantic — * which is correct behaviour, not an error. * @@ -249,7 +252,7 @@ export class PageEmbeddingRepo { row_number() OVER (ORDER BY ts_rank(pe.fts, q.query) DESC) AS rank_ix FROM page_embeddings pe JOIN pages p ON p.id = pe.page_id, - websearch_to_tsquery('english', f_unaccent(${queryText})) AS q(query) + websearch_to_tsquery('ru_en', f_unaccent(${queryText})) AS q(query) WHERE pe.workspace_id = ${workspaceId} AND pe.space_id IN (${spaceList}) AND p.deleted_at IS NULL