From 3cda0080127f79cf43d5423b8078a83f7380fbe8 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sun, 12 Jul 2026 19:32:20 +0300 Subject: [PATCH 1/2] =?UTF-8?q?feat(work-time):=20redesign=20=C2=ABTime=20?= =?UTF-8?q?worked=C2=BB=20modal=20with=20time-of-day=20timelines=20(#566)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port the NewDesign/TimeWorkedModal prototype onto the existing work-time feature with zero backend changes. Each daily track now shows WHEN work happened: a sticky 00/06/12/18/24 hour axis, shaded night hours (0–6, 21–24), a per-block hover tooltip "start – end · duration", and a "now" boundary on today's row. Work vs agent windows keep their existing colour semantics. The prototype's invented props (DaySummary[], pre-made labels) are replaced by a pure, unit-tested adapter (work-time-adapter.ts) over the real IPageWorkTime: windows → time-of-day blocks (epoch kept for a DST-safe tooltip), ms → labels via the shared formatters, in-place empty-run collapsing, and the today-only now-line. The agent-only fail-safe (#395/#551) is preserved — the agent estimate fills the main slot so it never renders empty. Reuses usePageWorkTime, the endpoint, tz bucketing and format-work-time; deletes the scratch prototype. New tooltip i18n key added to en-US/ru-RU. Co-Authored-By: Claude Opus 4.8 (1M context) --- NewDesign/TimeWorkedModal.tsx | 234 ------------------ .../public/locales/en-US/translation.json | 1 + .../public/locales/ru-RU/translation.json | 1 + .../work-time/work-time-adapter.test.ts | 168 +++++++++++++ .../work-time/work-time-adapter.ts | 168 +++++++++++++ .../work-time/work-time-punch-card.tsx | 207 +++++++++------- .../page-history/work-time/work-time-stat.tsx | 2 +- .../work-time/work-time.module.css | 84 +++++-- 8 files changed, 524 insertions(+), 341 deletions(-) delete mode 100644 NewDesign/TimeWorkedModal.tsx create mode 100644 apps/client/src/features/page-history/work-time/work-time-adapter.test.ts create mode 100644 apps/client/src/features/page-history/work-time/work-time-adapter.ts diff --git a/NewDesign/TimeWorkedModal.tsx b/NewDesign/TimeWorkedModal.tsx deleted file mode 100644 index 2866e2a3..00000000 --- a/NewDesign/TimeWorkedModal.tsx +++ /dev/null @@ -1,234 +0,0 @@ -/** - * TimeWorkedModal — редизайн окна «Time worked on this article». - * Mantine v7. Light/dark. - * - * ───────────────────────────────────────────────────────────────────────── - * ЧТО ЭТО - * ───────────────────────────────────────────────────────────────────────── - * Сводка трудозатрат по дням в виде суточных таймлайнов. Главное отличие от - * старой версии: на барах ВИДНО время суток. Реализовано двумя способами - * (проп `axis`): - * - 'grid' — ось часов 00–06–12–18–24 сверху + вертикальные деления, - * ночные часы (0–6, 21–24) слегка затемнены. По умолчанию. - * - 'phases' — цветные полосы фаз дня за блоками (Ночь/Утро/День/Вечер), - * период суток читается сразу, без счёта делений. - * - * Блоки позиционируются по времени: left = start/24, width = dur/24. - * Интенсивность (opacity) блока = длительность/плотность работы, чтобы - * очень короткие сессии не терялись (минимальная видимая ширина задана). - * Hover-тултип на блоке: начало–конец · длительность. - * - * ───────────────────────────────────────────────────────────────────────── - * УСТАНОВКА / ИСПОЛЬЗОВАНИЕ - * ───────────────────────────────────────────────────────────────────────── - * import { TimeWorkedModal, DaySummary } from './TimeWorkedModal'; - * - * setOpen(false)} - * totalLabel="≈ 34h" - * agentLabel="≈ 1h 20m" // undefined → строку agent не показываем - * days={days} // DaySummary[] - * axis="grid" // 'grid' | 'phases' - * tz="Europe/Moscow" - * inactivityGapMin={15} - * /> - * - * Требует MantineProvider на корне. - * - * ───────────────────────────────────────────────────────────────────────── - * ФОРМАТ ДАННЫХ - * ───────────────────────────────────────────────────────────────────────── - * interface Block { - * start: number; // час начала в сутках, 0..24 (напр. 13.7 = 13:42) - * end: number; // час конца - * kind: 'work' | 'agent'; - * } - * interface DaySummary { - * label: string; // «Mon 29 Jun» - * totalLabel: string; // «1h 24m» | «—» для пустого дня - * blocks: Block[]; // [] → пустой день (полупрозрачная дорожка, итог «—») - * isToday?: boolean; // сегодня — неполные сутки (рисуем границу «сейчас») - * nowFraction?: number;// 0..1 позиция «сейчас» для isToday - * } - * - * Данные уже есть в бэкенде трудозатрат: сессии с началом/концом + тип - * (work/agent). Часы = локальные к tz. Изменений API не требуется. - * - * ───────────────────────────────────────────────────────────────────────── - * СОСТОЯНИЯ (нарисованы/поддержаны) - * ───────────────────────────────────────────────────────────────────────── - * - День: с активностью (work+agent) / только work / только agent / пустой (—). - * - Сегодня: граница «сейчас» на дорожке (nowFraction). - * - Короткий блок: минимальная ширина, не исчезает. - * - Пустая панель целиком: нет трудозатрат → EmptyState. - * - Длинный период: вертикальный скролл списка дней, шапка/легенда липкие. - * - Тёмная тема: токены Mantine. - */ -import { Modal, Box, Group, Stack, Text, ActionIcon, ScrollArea, Tooltip, useMantineColorScheme } from '@mantine/core'; - -/* ─────────────────────────── Типы ─────────────────────────── */ - -export interface Block { start: number; end: number; kind: 'work' | 'agent'; } -export interface DaySummary { - label: string; - totalLabel: string; - blocks: Block[]; - isToday?: boolean; - nowFraction?: number; -} -export interface TimeWorkedModalProps { - opened: boolean; - onClose: () => void; - totalLabel: string; - agentLabel?: string; - days: DaySummary[]; - axis?: 'grid' | 'phases'; - tz?: string; - inactivityGapMin?: number; -} - -const WORK = '#3b82f6'; -const AGENT = '#c026d3'; - -const PHASES = [ - { name: 'Ночь', s: 0, e: 6, bg: 'rgba(99,102,241,.10)', lg: '#e8eaff', fg: '#5b60c9' }, - { name: 'Утро', s: 6, e: 12, bg: 'rgba(245,159,0,.10)', lg: '#fff2dc', fg: '#b5820e' }, - { name: 'День', s: 12, e: 18, bg: 'rgba(56,178,172,.10)', lg: '#dcf5f2', fg: '#1a857d' }, - { name: 'Вечер', s: 18, e: 24, bg: 'rgba(139,92,246,.11)', lg: '#efe7ff', fg: '#6d43c0' }, -]; - -/* ─────────────────────────── Блок активности ─────────────────────────── */ - -function fmtHour(h: number) { - const hh = Math.floor(h); - const mm = Math.round((h - hh) * 60); - return `${String(hh).padStart(2, '0')}:${String(mm).padStart(2, '0')}`; -} -function fmtDur(h: number) { - const total = Math.round(h * 60); - const hh = Math.floor(total / 60), mm = total % 60; - return hh ? `${hh}h ${mm}m` : `${mm}m`; -} - -function ActivityBlock({ b }: { b: Block }) { - const left = (b.start / 24) * 100; - const width = Math.max(((b.end - b.start) / 24) * 100, 0.6); - const dur = b.end - b.start; - const opacity = dur < 0.16 ? 0.5 : dur < 0.35 ? 0.8 : 1; - return ( - - - - ); -} - -/* ─────────────────────────── Дорожка дня ─────────────────────────── */ - -function DayTrack({ d, axis, dark }: { d: DaySummary; axis: 'grid' | 'phases'; dark: boolean }) { - const trackBg = axis === 'grid' - ? 'linear-gradient(90deg, rgba(90,100,130,.10) 0 25%, rgba(90,100,130,.02) 25% 87.5%, rgba(90,100,130,.10) 87.5% 100%)' - : (dark ? 'rgba(255,255,255,.04)' : '#f6f8fa'); - - return ( - - {d.label} - - {/* фон: полосы фаз или деления сетки */} - {axis === 'phases' - ? PHASES.map((ph) => ( - - )) - : [25, 50, 75].map((p) => ( - - ))} - {/* блоки */} - {d.blocks.map((b, i) => )} - {/* граница «сейчас» для сегодняшнего дня */} - {d.isToday && d.nowFraction != null && ( - - )} - - - {d.totalLabel} - - - ); -} - -/* ─────────────────────────── Окно ─────────────────────────── */ - -export function TimeWorkedModal(props: TimeWorkedModalProps) { - const { opened, onClose, totalLabel, agentLabel, days, axis = 'grid', tz = 'Europe/Moscow', inactivityGapMin = 15 } = props; - const { colorScheme } = useMantineColorScheme(); - const dark = colorScheme === 'dark'; - const isEmpty = days.length === 0 || days.every((d) => d.blocks.length === 0); - - return ( - - - {/* header */} - - Time worked on this article - - ✕ - - - {/* summary */} - - {totalLabel} - {agentLabel && agent: {agentLabel}} - - - {/* legend (grid only) */} - {axis === 'grid' && ( - - Work - Agent - - )} - - {isEmpty ? ( - - No time tracked yet - По этой статье ещё нет трудозатрат. - - ) : ( - <> - {/* axis header (sticky) */} - - - {axis === 'grid' ? ( - - {[['0%', '00', 'flex-start'], ['25%', '06', 'center'], ['50%', '12', 'center'], ['75%', '18', 'center'], ['100%', '24', 'flex-end']].map(([l, t, al]) => ( - {t} - ))} - - ) : ( - - {PHASES.map((ph) => ( - {ph.name} - ))} - - )} - - - - {/* day rows */} - - {days.map((d, i) => )} - - - )} - - Estimate · timezone {tz} · inactivity gap {inactivityGapMin} min - - - ); -} - -export default TimeWorkedModal; diff --git a/apps/client/public/locales/en-US/translation.json b/apps/client/public/locales/en-US/translation.json index bacdbd36..0b90b2c7 100644 --- a/apps/client/public/locales/en-US/translation.json +++ b/apps/client/public/locales/en-US/translation.json @@ -1464,6 +1464,7 @@ "agent: {{value}}": "agent: {{value}}", "Work": "Work", "Agent": "Agent", + "{{start}} – {{end}} · {{duration}}": "{{start}} – {{end}} · {{duration}}", "≈ {{hours}}h {{minutes}}m": "≈ {{hours}}h {{minutes}}m", "≈ {{hours}}h": "≈ {{hours}}h", "≈ {{minutes}}m": "≈ {{minutes}}m", diff --git a/apps/client/public/locales/ru-RU/translation.json b/apps/client/public/locales/ru-RU/translation.json index 49d6fe61..2aa13e5b 100644 --- a/apps/client/public/locales/ru-RU/translation.json +++ b/apps/client/public/locales/ru-RU/translation.json @@ -1481,6 +1481,7 @@ "agent: {{value}}": "агент: {{value}}", "Work": "Работа", "Agent": "Агент", + "{{start}} – {{end}} · {{duration}}": "{{start}} – {{end}} · {{duration}}", "≈ {{hours}}h {{minutes}}m": "≈ {{hours}} ч {{minutes}} мин", "≈ {{hours}}h": "≈ {{hours}} ч", "≈ {{minutes}}m": "≈ {{minutes}} мин", diff --git a/apps/client/src/features/page-history/work-time/work-time-adapter.test.ts b/apps/client/src/features/page-history/work-time/work-time-adapter.test.ts new file mode 100644 index 00000000..ce864444 --- /dev/null +++ b/apps/client/src/features/page-history/work-time/work-time-adapter.test.ts @@ -0,0 +1,168 @@ +import { describe, it, expect } from "vitest"; +import { + buildRows, + formatBlockTooltip, + summaryLabels, + toBlocks, + toTimelineDay, + EMPTY_RUN_COLLAPSE, +} from "./work-time-adapter"; +import { IPageWorkTime, IPerDay } from "./work-time.types"; + +const MIN = 60 * 1000; +const HOUR = 60 * MIN; +const DAY_MS = 24 * HOUR; + +// Fake translator: renders the key with {{tokens}} substituted, so the tests +// assert the mapping/branching without depending on the i18n catalogue. +const t = (key: string, opts?: Record) => + key.replace(/\{\{(\w+)\}\}/g, (_, k) => String(opts?.[k] ?? "")); + +// A day midnight anchored at a fixed UTC instant (tz handled server-side; we +// only lay out fractions of the day the server already bucketed). +const DAY0 = Date.UTC(2026, 5, 29, 0, 0, 0); // Mon 29 Jun 2026 00:00 UTC + +function day(over: Partial & { day: number }): IPerDay { + return { + dayISO: new Date(over.day).toISOString().slice(0, 10), + activeMs: 0, + agentMs: 0, + windows: [], + ...over, + }; +} + +const config: IPageWorkTime["config"] = { + tGap: 15 * MIN, + agentTGap: 15 * MIN, + pIn: 0, + pOut: 0, + pSingle: 30 * 1000, + excludeGit: false, + dedupRoundMs: 1000, +}; + +function payload(over: Partial): IPageWorkTime { + return { + workMs: 0, + agentOnlyMs: 0, + perDay: [], + config, + tz: "UTC", + ...over, + }; +} + +describe("toBlocks", () => { + it("maps work+agent windows to time-of-day segments (hour fractions)", () => { + const d = day({ + day: DAY0, + activeMs: 90 * MIN, + windows: [ + { start: DAY0 + 9 * HOUR, end: DAY0 + 10 * HOUR + 30 * MIN, class: "work" }, + { start: DAY0 + 22 * HOUR, end: DAY0 + 23 * HOUR, class: "agent_only" }, + ], + }); + const blocks = toBlocks(d); + expect(blocks).toHaveLength(2); + expect(blocks[0]).toMatchObject({ start: 9, end: 10.5, kind: "work" }); + expect(blocks[1]).toMatchObject({ start: 22, end: 23, kind: "agent" }); + // real epoch preserved for the DST-safe tooltip + expect(blocks[0].startEpoch).toBe(DAY0 + 9 * HOUR); + }); + + it("clamps out-of-day fractions to [0,24]", () => { + const d = day({ + day: DAY0, + windows: [{ start: DAY0 - HOUR, end: DAY0 + 25 * HOUR, class: "work" }], + }); + const [b] = toBlocks(d); + expect(b.start).toBe(0); + expect(b.end).toBe(24); + }); +}); + +describe("toTimelineDay", () => { + it("draws the now-line only on today's row", () => { + const today = day({ day: DAY0, activeMs: HOUR }); + const noon = DAY0 + 12 * HOUR; + const asToday = toTimelineDay(today, t, noon); + expect(asToday.isToday).toBe(true); + expect(asToday.nowFraction).toBeCloseTo(0.5, 5); + + const past = day({ day: DAY0 - DAY_MS, activeMs: HOUR }); + const asPast = toTimelineDay(past, t, noon); + expect(asPast.isToday).toBe(false); + expect(asPast.nowFraction).toBeUndefined(); + }); + + it("labels an empty day and totals it as —", () => { + const empty = day({ day: DAY0 }); + const d = toTimelineDay(empty, t, DAY0 + 12 * HOUR); + expect(d.isEmpty).toBe(true); + expect(d.totalLabel).toBe("—"); + }); +}); + +describe("buildRows (empty-run collapsing)", () => { + it("collapses a run of >= EMPTY_RUN_COLLAPSE empty days in place", () => { + const perDay: IPerDay[] = [ + day({ day: DAY0, activeMs: HOUR }), + ...Array.from({ length: EMPTY_RUN_COLLAPSE }, (_, i) => + day({ day: DAY0 + (i + 1) * DAY_MS }), + ), + day({ day: DAY0 + (EMPTY_RUN_COLLAPSE + 1) * DAY_MS, activeMs: HOUR }), + ]; + const rows = buildRows(perDay, t, 0); + expect(rows.map((r) => r.type)).toEqual(["day", "gap", "day"]); + const gap = rows[1]; + expect(gap.type === "gap" && gap.count).toBe(EMPTY_RUN_COLLAPSE); + }); + + it("keeps a short empty run as individual dimmed day rows", () => { + const perDay: IPerDay[] = [ + day({ day: DAY0, activeMs: HOUR }), + day({ day: DAY0 + DAY_MS }), + day({ day: DAY0 + 2 * DAY_MS, activeMs: HOUR }), + ]; + const rows = buildRows(perDay, t, 0); + expect(rows.map((r) => r.type)).toEqual(["day", "day", "day"]); + }); +}); + +describe("summaryLabels", () => { + it("derives totalLabel/agentLabel from ms via the shared formatter", () => { + const { total, agent } = summaryLabels( + payload({ workMs: 4 * HOUR + 27 * MIN, agentOnlyMs: 80 * MIN }), + t, + ); + expect(total).toBe("≈ 4h 25m"); + expect(agent).toBe("≈ 1h 20m"); + }); + + it("agent-only page: agent estimate fills the main slot, no secondary line", () => { + const { total, agent } = summaryLabels( + payload({ workMs: 0, agentOnlyMs: 80 * MIN }), + t, + ); + expect(total).toBe("agent: ≈ 1h 20m"); + expect(agent).toBeUndefined(); + }); + + it("human-only page: no agent line", () => { + const { agent } = summaryLabels(payload({ workMs: HOUR, agentOnlyMs: 0 }), t); + expect(agent).toBeUndefined(); + }); +}); + +describe("formatBlockTooltip", () => { + it("formats start–end from the real epoch in the data tz + duration", () => { + const d = day({ + day: DAY0, + windows: [{ start: DAY0 + 9 * HOUR, end: DAY0 + 10 * HOUR + 30 * MIN, class: "work" }], + }); + const [b] = toBlocks(d); + const label = formatBlockTooltip(b, "UTC", "en-US", t); + expect(label).toBe("09:00 – 10:30 · 1h 30m"); + }); +}); diff --git a/apps/client/src/features/page-history/work-time/work-time-adapter.ts b/apps/client/src/features/page-history/work-time/work-time-adapter.ts new file mode 100644 index 00000000..5ee9e5be --- /dev/null +++ b/apps/client/src/features/page-history/work-time/work-time-adapter.ts @@ -0,0 +1,168 @@ +// #566 — adapter: real IPageWorkTime → the render-ready shape the redesigned +// time-of-day timeline consumes. The visual prototype (NewDesign/TimeWorkedModal) +// was written against invented props (`DaySummary[]`, pre-made `totalLabel`); the +// real payload is IPageWorkTime. Everything below is pure so the mapping (windows +// → time-of-day blocks, ms → labels, the "now" boundary, empty-run collapsing) is +// unit-testable without React. No re-bucketing: the server already grouped the +// windows by the request timezone — we only lay out the windows it returned. + +import { IPageWorkTime, IPerDay, IDayWindow } from "./work-time.types"; +import { formatDayTotal, formatHeadline } from "./format-work-time"; + +type Translate = (key: string, opts?: Record) => string; + +export const DAY_MS = 24 * 60 * 60 * 1000; +// Collapse a run of this many (or more) consecutive edit-free days into a single +// "× N days" separator (§6.2 long-range) — preserved from the original punch-card. +export const EMPTY_RUN_COLLAPSE = 8; + +export interface TimelineBlock { + /** Hour fraction 0..24 within the day — drives left/width positioning. */ + start: number; + end: number; + kind: "work" | "agent"; + /** Real epoch ms, kept for a DST-safe tooltip (formatted in the data tz). */ + startEpoch: number; + endEpoch: number; +} + +export interface TimelineDay { + key: string; + label: string; + totalLabel: string; + blocks: TimelineBlock[]; + isEmpty: boolean; + /** Today lives only in the last active bucket; drives the "now" boundary. */ + isToday: boolean; + /** 0..1 position of "now" within today's track (undefined when not today). */ + nowFraction?: number; +} + +export type TimelineRow = + | { type: "day"; day: TimelineDay } + | { type: "gap"; count: number }; + +/** Weekday-day-month heading in the browser locale (matches the server tz + * bucketing, since usePageWorkTime requests buckets in the viewer tz). */ +export function dayHeading(day: number): string { + return new Date(day).toLocaleDateString(undefined, { + weekday: "short", + day: "numeric", + month: "short", + }); +} + +/** Map one day's absolute windows into time-of-day blocks. `class` work|agent_only + * → kind work|agent; positions are the in-day hour fraction (epoch kept for the + * tooltip). Fractions are clamped to [0,24] to match the original punch-card. */ +export function toBlocks(day: IPerDay): TimelineBlock[] { + return day.windows.map((w: IDayWindow) => { + const start = clampHour((w.start - day.day) / (DAY_MS / 24)); + const end = clampHour((w.end - day.day) / (DAY_MS / 24)); + return { + start, + end, + kind: w.class === "work" ? "work" : "agent", + startEpoch: w.start, + endEpoch: w.end, + }; + }); +} + +function clampHour(h: number): number { + return Math.max(0, Math.min(24, h)); +} + +/** Build a single render-ready day. `now` is injected for testability; the + * "now" boundary is drawn only when `now` falls inside this bucket's calendar + * day (so it appears on today's row and only when today has edits). */ +export function toTimelineDay( + day: IPerDay, + t: Translate, + now: number, +): TimelineDay { + const isToday = now >= day.day && now < day.day + DAY_MS; + return { + key: day.dayISO, + label: dayHeading(day.day), + totalLabel: formatDayTotal(day.activeMs, t), + blocks: toBlocks(day), + isEmpty: day.activeMs === 0 && day.agentMs === 0, + isToday, + nowFraction: isToday ? (now - day.day) / DAY_MS : undefined, + }; +} + +/** Collapse long edit-free runs (≥ EMPTY_RUN_COLLAPSE) into an in-place "gap" + * row; short runs stay as (dimmed, "—") day rows. Preserved from the original + * punch-card so a page edited over months does not render hundreds of rows. */ +export function buildRows( + perDay: IPerDay[], + t: Translate, + now: number, +): TimelineRow[] { + const rows: TimelineRow[] = []; + let emptyRun: IPerDay[] = []; + const flush = () => { + if (emptyRun.length >= EMPTY_RUN_COLLAPSE) { + rows.push({ type: "gap", count: emptyRun.length }); + } else { + for (const d of emptyRun) { + rows.push({ type: "day", day: toTimelineDay(d, t, now) }); + } + } + emptyRun = []; + }; + for (const d of perDay) { + if (d.activeMs === 0 && d.agentMs === 0) { + emptyRun.push(d); + } else { + flush(); + rows.push({ type: "day", day: toTimelineDay(d, t, now) }); + } + } + flush(); + return rows; +} + +/** The big summary slot. Fail-safe for an agent-only page (#395/#551): since + * formatHeadline(0) === "", never leave the 22px slot empty — put the agent + * estimate in the main slot, and only show the secondary `agent:` line when + * BOTH a human and an agent estimate exist. */ +export function summaryLabels( + data: IPageWorkTime, + t: Translate, +): { total: string; agent?: string } { + const total = + data.workMs > 0 + ? formatHeadline(data.workMs, t) + : t("agent: {{value}}", { value: formatHeadline(data.agentOnlyMs, t) }); + const agent = + data.workMs > 0 && data.agentOnlyMs > 0 + ? formatHeadline(data.agentOnlyMs, t) + : undefined; + return { total, agent }; +} + +/** Block hover label "start – end · duration". Times come from the REAL epoch + * rendered in the data tz (NOT the 24h fraction) so a DST-transition day does + * not skew the shown clock time. Duration reuses formatDayTotal (always > 0 + * here, so never "—"). */ +export function formatBlockTooltip( + block: TimelineBlock, + tz: string, + locale: string, + t: Translate, +): string { + const fmt = new Intl.DateTimeFormat(locale || undefined, { + hour: "2-digit", + minute: "2-digit", + hour12: false, + timeZone: tz, + }); + return t("{{start}} – {{end}} · {{duration}}", { + start: fmt.format(block.startEpoch), + end: fmt.format(block.endEpoch), + duration: formatDayTotal(block.endEpoch - block.startEpoch, t), + }); +} diff --git a/apps/client/src/features/page-history/work-time/work-time-punch-card.tsx b/apps/client/src/features/page-history/work-time/work-time-punch-card.tsx index a1285d2d..043a8a71 100644 --- a/apps/client/src/features/page-history/work-time/work-time-punch-card.tsx +++ b/apps/client/src/features/page-history/work-time/work-time-punch-card.tsx @@ -1,97 +1,86 @@ -import { Group, Stack, Text } from "@mantine/core"; +// #566 — redesigned "Time worked" body: daily time-of-day timelines. Each day is +// a 24h track showing WHEN the work happened — a sticky 00/06/12/18/24 hour axis, +// shaded night hours, per-block hover tooltip ("start – end · duration"), and a +// "now" boundary on today's row. Presentation ported from NewDesign/TimeWorkedModal; +// all data comes through the pure adapter over the real IPageWorkTime (zero backend +// change). Positioning math, empty-run collapsing and the formatters are reused. +import { Box, Group, ScrollArea, Stack, Text, Tooltip } from "@mantine/core"; import { useTranslation } from "react-i18next"; import { useMemo } from "react"; -import { IPageWorkTime, IPerDay, IDayWindow } from "./work-time.types"; +import { IPageWorkTime } from "./work-time.types"; +import { formatGapMinutes } from "./format-work-time"; import { - formatDayTotal, - formatGapMinutes, - formatHeadline, -} from "./format-work-time"; + buildRows, + formatBlockTooltip, + summaryLabels, + TimelineBlock, + TimelineDay, +} from "./work-time-adapter"; import classes from "./work-time.module.css"; -const DAY_MS = 24 * 60 * 60 * 1000; -// Collapse a run of this many (or more) consecutive edit-free days into a single -// "× N days" separator (§6.2 long-range) — the row is still always one day. -const EMPTY_RUN_COLLAPSE = 8; +// Minimum visible width so a very short session neither vanishes nor fakes dense +// work (§6.2); kept from the original punch-card. +const MIN_BLOCK_WIDTH_PCT = 0.6; -type Row = - | { type: "day"; day: IPerDay } - | { type: "gap"; count: number }; - -function collapseEmptyRuns(perDay: IPerDay[]): Row[] { - const rows: Row[] = []; - let emptyRun: IPerDay[] = []; - const flush = () => { - if (emptyRun.length >= EMPTY_RUN_COLLAPSE) { - rows.push({ type: "gap", count: emptyRun.length }); - } else { - for (const d of emptyRun) rows.push({ type: "day", day: d }); - } - emptyRun = []; - }; - for (const d of perDay) { - if (d.activeMs === 0 && d.agentMs === 0) { - emptyRun.push(d); - } else { - flush(); - rows.push({ type: "day", day: d }); - } - } - flush(); - return rows; -} - -function dayHeading(day: number): string { - return new Date(day).toLocaleDateString(undefined, { - weekday: "short", - day: "numeric", - month: "short", - }); +function ActivityBlock({ + block, + tz, + locale, +}: { + block: TimelineBlock; + tz: string; + locale: string; +}) { + const { t } = useTranslation(); + const left = (block.start / 24) * 100; + const width = Math.max(((block.end - block.start) / 24) * 100, MIN_BLOCK_WIDTH_PCT); + const cls = [ + classes.window, + block.kind === "work" ? classes.windowWork : classes.windowAgent, + ].join(" "); + return ( + +
+ + ); } function DayTrack({ day, - pSingle, + tz, + locale, }: { - day: IPerDay; - pSingle: number; + day: TimelineDay; + tz: string; + locale: string; }) { - const { t } = useTranslation(); - const ticks = [6, 12, 18]; return (
- {dayHeading(day.day)} -
- {ticks.map((h) => ( -
+ {day.label} +
+ {[25, 50, 75].map((p) => ( +
))} - {day.windows.map((w: IDayWindow, i) => { - const leftPct = ((w.start - day.day) / DAY_MS) * 100; - const widthPct = ((w.end - w.start) / DAY_MS) * 100; - const isSingle = w.end - w.start <= pSingle; - const cls = [ - classes.window, - w.class === "work" ? classes.windowWork : classes.windowAgent, - isSingle ? classes.windowSingle : "", - ].join(" "); - return ( -
- ); - })} + {day.blocks.map((b, i) => ( + + ))} + {day.isToday && day.nowFraction != null && ( +
+ )}
- - {formatDayTotal(day.activeMs, t)} + + {day.totalLabel}
); @@ -102,8 +91,16 @@ interface Props { } export default function WorkTimePunchCard({ data }: Props) { - const { t } = useTranslation(); - const rows = useMemo(() => collapseEmptyRuns(data.perDay), [data.perDay]); + const { t, i18n } = useTranslation(); + const locale = i18n.language; + const now = Date.now(); + const rows = useMemo( + () => buildRows(data.perDay, t, now), + // `now` intentionally re-read on each open; excluded so the memo tracks data. + // eslint-disable-next-line react-hooks/exhaustive-deps + [data.perDay, t], + ); + const { total, agent } = summaryLabels(data, t); const gapMin = formatGapMinutes(data.config.tGap); if (data.workMs <= 0 && data.agentOnlyMs <= 0) { @@ -116,17 +113,19 @@ export default function WorkTimePunchCard({ data }: Props) { return ( - - - {formatHeadline(data.workMs, t)} + {/* summary */} + + + {total} - {data.agentOnlyMs > 0 && ( + {agent && ( - {t("agent: {{value}}", { value: formatHeadline(data.agentOnlyMs, t) })} + {t("agent: {{value}}", { value: agent })} )} + {/* legend */} -
+ {/* sticky hour axis */} +
+ +
+ {[ + ["0%", "00", "start"], + ["25%", "06", "center"], + ["50%", "12", "center"], + ["75%", "18", "center"], + ["100%", "24", "end"], + ].map(([l, label, align]) => ( + + {label} + + ))} +
+ +
+ + {/* day rows */} + {rows.map((row, i) => row.type === "day" ? ( ) : ( -
+ {t("× {{count}} days without edits", { count: row.count })} -
+ ), )} -
+ {t("Estimate · timezone {{tz}} · inactivity gap {{gap}} min", { diff --git a/apps/client/src/features/page-history/work-time/work-time-stat.tsx b/apps/client/src/features/page-history/work-time/work-time-stat.tsx index b963456f..8e370a35 100644 --- a/apps/client/src/features/page-history/work-time/work-time-stat.tsx +++ b/apps/client/src/features/page-history/work-time/work-time-stat.tsx @@ -60,7 +60,7 @@ export default function WorkTimeStat({ pageId }: Props) { opened={opened} onClose={close} title={t("Time worked on this article")} - size="lg" + size="46rem" > diff --git a/apps/client/src/features/page-history/work-time/work-time.module.css b/apps/client/src/features/page-history/work-time/work-time.module.css index 397d45b7..6dc848e0 100644 --- a/apps/client/src/features/page-history/work-time/work-time.module.css +++ b/apps/client/src/features/page-history/work-time/work-time.module.css @@ -1,5 +1,7 @@ -/* #395 — 24h × days punch-card. Custom CSS segments on a fixed 24-hour track - (position = offset-in-day / 24h, width = duration / 24h), no chart library. */ +/* #566 — time-of-day timeline. Custom CSS segments on a fixed 24-hour track + (position = offset-in-day / 24h, width = duration / 24h), no chart library. + Night hours (0–6, 21–24) are shaded so the eye reads morning-vs-evening; a + sticky hour axis labels 00/06/12/18/24. Theme-aware via Mantine tokens. */ .row { display: grid; @@ -16,17 +18,29 @@ } .track { + --wt-night: rgba(90, 100, 130, 0.16); + --wt-day: rgba(90, 100, 130, 0.03); position: relative; - height: 16px; - border-radius: 4px; - background-color: light-dark( - var(--mantine-color-gray-1), - var(--mantine-color-dark-6) - ); + height: 20px; + border-radius: 5px; overflow: hidden; + /* base fill + night shading: 0–6h and 21–24h (87.5%) darker than daytime */ + background: + linear-gradient( + 90deg, + var(--wt-night) 0 25%, + var(--wt-day) 25% 87.5%, + var(--wt-night) 87.5% 100% + ), + light-dark(var(--mantine-color-gray-1), var(--mantine-color-dark-6)); } -/* Faint hour grid so the eye can read "morning vs evening". */ +/* Short (< EMPTY_RUN_COLLAPSE) edit-free days: dimmed, empty track. */ +.trackEmpty { + opacity: 0.5; +} + +/* Quarter-day grid divisions (06/12/18) inside the track. */ .hourTick { position: absolute; top: 0; @@ -40,10 +54,11 @@ .window { position: absolute; - top: 2px; - bottom: 2px; + top: 3px; + height: 14px; border-radius: 3px; min-width: 3px; + cursor: default; } .windowWork { @@ -54,18 +69,57 @@ background-color: var(--mantine-color-grape-5); } -/* A lone single-sample (P_single) window: minimal + dimmed, so it neither - vanishes nor fakes dense work (§6.2). */ -.windowSingle { - opacity: 0.5; +/* The "now" boundary on today's row. */ +.nowLine { + position: absolute; + top: 0; + bottom: 0; + width: 2px; + background-color: var(--mantine-color-red-6); } .daySum { font-size: var(--mantine-font-size-xs); + font-weight: 500; text-align: right; white-space: nowrap; } +.daySum[data-empty] { + color: var(--mantine-color-dimmed); + font-weight: 400; +} + +/* Sticky hour axis header aligned to the track column. */ +.axisRow { + position: sticky; + top: 0; + z-index: 2; + background-color: var(--mantine-color-body); + padding-bottom: 2px; +} + +.axis { + position: relative; + height: 14px; +} + +.axisTick { + position: absolute; + top: 0; + font-size: 10px; + font-weight: 500; + color: var(--mantine-color-dimmed); +} + +.axisTick[data-align="center"] { + transform: translateX(-50%); +} + +.axisTick[data-align="end"] { + transform: translateX(-100%); +} + .gapRow { padding: 6px 0 6px 108px; font-size: var(--mantine-font-size-xs); From 1fc9c256817b3351932079dfa7c23f520b3d8795 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sun, 12 Jul 2026 18:31:11 +0300 Subject: [PATCH 2/2] feat(search): semantic fusion layer (Phase B PR-1) over lexical RRF (#530) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fuse a vector-similarity branch into #529's lexical RRF. Query embed via a global TEI sidecar (or per-workspace provider), degrading transparently to the byte-identical Phase-A lexical path on any failure; a kill-switch and env knobs gate it. - ai.service: resolveEmbeddingProvider (workspace→global TEI fallback) with a deterministic config fingerprint; embedQuery (query prefix + short 800ms timeout); extract embedWithModel core shared with embedTexts. - page-embedding.repo: vectorCandidateArm fragment (page-level NN, dim + active-fingerprint filtered) + fingerprint on insertChunks. - search.service: 3-branch RRF over lexical ∪ vector candidates; try/catch wraps only the embed (permission filter stays outside, fail-closed); semantic degrade emits search.semantic.degraded; response gains semantic{state,available,reason}. - indexer: resolve provider, prepend doc prefix, stamp fingerprint per row. - migration 20260712T120000: add nullable page_embeddings.fingerprint + composite index. - infra: TEI embeddings sidecar in docker-compose + EMBEDDING_*/SEARCH_* in .env.example. - tests: 7 semantic int cases (vector-only hit, sidecar-down, no-provider, permission-over-union fail-closed, hung sidecar, lexical∪vector de-dup, fingerprint isolation) + fingerprint/prefix/timeout unit tests. total now = permission-filtered size of (lexical ∪ vector top-N) — a documented change from Phase A's exact lexical count (falls back to it on degrade). Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 36 ++ .../embedding-indexer.service.spec.ts | 102 ++++- .../embedding/embedding-indexer.service.ts | 37 +- .../core/search/dto/search-response.dto.ts | 28 +- apps/server/src/core/search/search.module.ts | 7 + .../search/search.service.query-mode.spec.ts | 4 + .../src/core/search/search.service.spec.ts | 4 + apps/server/src/core/search/search.service.ts | 275 ++++++++++-- ...0712T120000-page-embeddings-fingerprint.ts | 49 +++ .../repos/ai-chat/page-embedding.repo.ts | 82 +++- .../src/database/types/embeddings.types.ts | 5 + apps/server/src/database/utils.ts | 17 + .../ai/ai.service.embedding.spec.ts | 143 +++++++ apps/server/src/integrations/ai/ai.service.ts | 169 +++++++- .../integration/search-lexical.int-spec.ts | 403 ++++++++++++++++++ docker-compose.yml | 41 ++ 16 files changed, 1355 insertions(+), 47 deletions(-) create mode 100644 apps/server/src/database/migrations/20260712T120000-page-embeddings-fingerprint.ts create mode 100644 apps/server/src/integrations/ai/ai.service.embedding.spec.ts diff --git a/.env.example b/.env.example index bb189adc..68654421 100644 --- a/.env.example +++ b/.env.example @@ -198,6 +198,42 @@ MCP_DOCMOST_PASSWORD= # A slow/hung embeddings endpoint fails after this and the batch continues. # AI_EMBEDDING_TIMEOUT_MS=120000 +# ─── #530 Semantic search (Phase B) ────────────────────────────────────────── +# The GLOBAL embedding provider used by search (query embed) AND the indexer +# (document embed) when a workspace has NO embedding provider of its own. It is +# an OpenAI-compatible endpoint — the docker-compose `embeddings` (TEI) sidecar. +# A workspace that configures its own embedding provider OVERRIDES all of this. +# When neither resolves, search runs lexical-only (semantic.reason=no-provider). +EMBEDDING_ENDPOINT=http://embeddings:80/v1 +EMBEDDING_MODEL=intfloat/multilingual-e5-small +# Dummy — the self-hosted TEI sidecar is keyless. +EMBEDDING_API_KEY=unused +EMBEDDING_DIMENSIONS=384 +# PLACEHOLDER — set to a PINNED intfloat/multilingual-e5-small commit sha (used +# both as the TEI --revision and as part of the embedding fingerprint). Keep this +# in lockstep with docker-compose; a bump changes the fingerprint (PR-2 handles +# the generational swap/GC — PR-1 uses a single active fingerprint). +EMBEDDING_REVISION=REPLACE_WITH_PINNED_E5_SMALL_COMMIT_SHA +# e5 models require these input prefixes. Empty them for a non-e5 model. +EMBEDDING_QUERY_PREFIX="query: " +EMBEDDING_DOC_PREFIX="passage: " +# Per-request timeout (ms) for the interactive SEARCH query embed — much shorter +# than the batch indexer timeout: a slow/hung sidecar degrades search fast to the +# lexical-only path instead of blocking the request. Default 800. +# SEARCH_EMBED_TIMEOUT_MS=800 +# RRF weight of the vector leg relative to each lexical leg (1.0 = equal). Default 1.0. +# SEARCH_VECTOR_WEIGHT=1.0 +# Vector top-N pulled into the fused candidate union per request. Default 50. +# SEARCH_VECTOR_CANDIDATES=50 +# Per-statement timeout (ms) bounding ONLY the fused vector query (the brute-force +# KNN seq scan — there is no ANN index). If the vector scan exceeds this it is +# cancelled and search degrades to lexical-only (never hangs the request). Scoped +# per-query (SET LOCAL), so it never affects other queries. Default 2000. +# SEARCH_VECTOR_STATEMENT_TIMEOUT_MS=2000 +# Kill-switch: set to `off` to disable the semantic layer entirely (search then +# runs the byte-identical Phase-A lexical path). Default on. +# SEARCH_SEMANTIC=on + # Silence timeout (ms) for streaming chat/agent AI calls AND external-MCP traffic. # Bounds time-to-first-byte and the gap BETWEEN chunks (NOT the total turn length), # so an arbitrarily long turn that keeps streaming is never cut. Finite so a hung diff --git a/apps/server/src/core/ai-chat/embedding/embedding-indexer.service.spec.ts b/apps/server/src/core/ai-chat/embedding/embedding-indexer.service.spec.ts index 38e86d12..c7437406 100644 --- a/apps/server/src/core/ai-chat/embedding/embedding-indexer.service.spec.ts +++ b/apps/server/src/core/ai-chat/embedding/embedding-indexer.service.spec.ts @@ -32,6 +32,14 @@ describe('EmbeddingIndexerService.reindexWorkspace fail-fast', () => { const pageEmbeddingRepo = {}; const aiService = { getEmbeddingModel: jest.fn().mockResolvedValue('some-model'), + // #530: reindexWorkspace's pre-check now resolves the provider (workspace + // or global). Resolve it so the batch control flow under test proceeds. + resolveEmbeddingProvider: jest.fn().mockResolvedValue({ + model: 'some-model', + queryPrefix: '', + docPrefix: '', + fingerprint: 'fp-test', + }), }; // Progress is a best-effort cosmetic store; mock its async methods so the // batch control flow can be tested without Redis. @@ -108,6 +116,14 @@ describe('EmbeddingIndexerService.reindexWorkspace progress', () => { const pageEmbeddingRepo = {}; const aiService = { getEmbeddingModel: jest.fn().mockResolvedValue('some-model'), + // #530: reindexWorkspace's pre-check now resolves the provider (workspace + // or global). Resolve it so the batch control flow under test proceeds. + resolveEmbeddingProvider: jest.fn().mockResolvedValue({ + model: 'some-model', + queryPrefix: '', + docPrefix: '', + fingerprint: 'fp-test', + }), }; const reindexProgress = { start: jest.fn().mockResolvedValue(undefined), @@ -174,7 +190,7 @@ describe('EmbeddingIndexerService.reindexWorkspace progress', () => { const { service, aiService, reindexProgress } = makeService(); // Embeddings not configured: reindexWorkspace returns early WITHOUT starting // a fresh record, but the finally must still clear the enqueue-time seed. - aiService.getEmbeddingModel = jest + aiService.resolveEmbeddingProvider = jest .fn() .mockRejectedValue(new AiEmbeddingNotConfiguredException()); @@ -187,3 +203,87 @@ describe('EmbeddingIndexerService.reindexWorkspace progress', () => { expect(reindexProgress.clear).toHaveBeenCalledWith(WORKSPACE_ID); }); }); + +/** + * #530 PR-1: reindexPage must (a) prepend the provider's DOC prefix to each chunk + * BEFORE embedding (so stored vectors live in the same prefixed space as a + * prefixed query), and (b) stamp the ACTIVE fingerprint on every inserted row (so + * search only fuses same-generation vectors). Uses lightweight mocks; the tx is + * stubbed to run its callback inline. + */ +describe('EmbeddingIndexerService.reindexPage doc-prefix + fingerprint (#530)', () => { + const WORKSPACE_ID = 'ws-1'; + const SPACE_ID = 'space-1'; + const PAGE_ID = 'page-1'; + + function makeService(docPrefix: string) { + const pageRepo = { + findById: jest.fn().mockResolvedValue({ + id: PAGE_ID, + workspaceId: WORKSPACE_ID, + spaceId: SPACE_ID, + title: 'Заголовок', + // No ProseMirror content -> the plain-text fallback path (single chunk). + content: null, + textContent: 'простой текст страницы', + deletedAt: null, + }), + }; + const insertChunks = jest.fn().mockResolvedValue(undefined); + const pageEmbeddingRepo = { + deleteByPage: jest.fn().mockResolvedValue(undefined), + insertChunks, + }; + const embedWithModel = jest.fn().mockResolvedValue([[0.1, 0.2, 0.3]]); + const aiService = { + resolveEmbeddingProvider: jest.fn().mockResolvedValue({ + model: { modelId: 'e5-small' }, + queryPrefix: 'query: ', + docPrefix, + fingerprint: 'fp-gen-1', + }), + embedWithModel, + }; + const reindexProgress = {}; + // Stub the tx so executeTx runs its callback inline against a fake trx. + const db = { + transaction: () => ({ execute: (cb: any) => cb({}) }), + }; + const service = new EmbeddingIndexerService( + pageRepo as unknown as PageRepo, + pageEmbeddingRepo as unknown as PageEmbeddingRepo, + aiService as unknown as AiService, + reindexProgress as unknown as EmbeddingReindexProgressService, + db as unknown as KyselyDB, + ); + return { service, embedWithModel, insertChunks }; + } + + it('prepends the doc prefix to each chunk and stamps the fingerprint on rows', async () => { + const { service, embedWithModel, insertChunks } = makeService('passage: '); + await service.reindexPage(PAGE_ID); + + // Embedded values are DOC-prefixed; the model is the resolved provider model. + expect(embedWithModel).toHaveBeenCalledTimes(1); + const [modelArg, wsArg, valuesArg] = embedWithModel.mock.calls[0]; + expect(modelArg).toEqual({ modelId: 'e5-small' }); + expect(wsArg).toBe(WORKSPACE_ID); + expect(valuesArg).toEqual(['passage: простой текст страницы']); + + // Inserted rows carry the active fingerprint and the ORIGINAL (un-prefixed) + // content (the prefix is an embedding-space artifact, not stored text). + expect(insertChunks).toHaveBeenCalledTimes(1); + const rows = insertChunks.mock.calls[0][0]; + expect(rows).toHaveLength(1); + expect(rows[0].fingerprint).toBe('fp-gen-1'); + expect(rows[0].content).toBe('простой текст страницы'); + expect(rows[0].modelName).toBe('e5-small'); + }); + + it('does not prefix when the provider has an empty doc prefix', async () => { + const { service, embedWithModel } = makeService(''); + await service.reindexPage(PAGE_ID); + const [, , valuesArg] = embedWithModel.mock.calls[0]; + expect(valuesArg).toEqual(['простой текст страницы']); + }); +}); diff --git a/apps/server/src/core/ai-chat/embedding/embedding-indexer.service.ts b/apps/server/src/core/ai-chat/embedding/embedding-indexer.service.ts index 9d1a3a09..4b20813c 100644 --- a/apps/server/src/core/ai-chat/embedding/embedding-indexer.service.ts +++ b/apps/server/src/core/ai-chat/embedding/embedding-indexer.service.ts @@ -108,15 +108,24 @@ export class EmbeddingIndexerService { return; } - // Resolve embeddings config WITHOUT crashing the queue when unconfigured. + // Resolve the embeddings provider WITHOUT crashing the queue when + // unconfigured. #530: resolveEmbeddingProvider prefers the workspace provider + // and falls back to the GLOBAL env provider (TEI sidecar), and yields the + // doc-prefix + the active fingerprint stored per row so search filters by the + // active generation. let modelName = 'unknown'; + let provider: Awaited< + ReturnType + >; try { - const model = await this.aiService.getEmbeddingModel(workspaceId); + provider = await this.aiService.resolveEmbeddingProvider(workspaceId); // Record the model id per row so a future migration can detect + re-index // rows produced by a different model (see the migration header). The SDK // type is `string | EmbeddingModel{V2,V3}`; model objects carry `modelId`. modelName = - typeof model === 'string' ? model : (model.modelId ?? 'unknown'); + typeof provider.model === 'string' + ? provider.model + : (provider.model.modelId ?? 'unknown'); } catch (err) { if (err instanceof AiEmbeddingNotConfiguredException) { // No embeddings provider for this workspace: NO-OP (§6.7). The page can @@ -145,8 +154,18 @@ export class EmbeddingIndexerService { return; } - // Embed all chunks in one batch. - const vectors = await this.aiService.embedTexts(workspaceId, chunks); + // #530: prepend the provider's DOC prefix to each chunk (e5-style + // "passage: "; empty for a non-e5 provider) so stored vectors live in the + // same prefixed space as a prefixed query, then embed with the RESOLVED + // provider model (which may be the global TEI sidecar, not a workspace one). + const prefixedChunks = provider.docPrefix + ? chunks.map((c) => provider.docPrefix + c) + : chunks; + const vectors = await this.aiService.embedWithModel( + provider.model, + workspaceId, + prefixedChunks, + ); // The column is dimension-agnostic, so ANY model dimension is stored as-is. // Defensive sanity check only: all chunks of ONE page come from the SAME @@ -170,6 +189,7 @@ export class EmbeddingIndexerService { vectors, { pageId, workspaceId, spaceId }, modelName, + provider.fingerprint, ); // HARD replace in one transaction: delete then insert so search never @@ -216,7 +236,9 @@ export class EmbeddingIndexerService { // (seeded at enqueue time); the finally cleans that too. try { try { - await this.aiService.getEmbeddingModel(workspaceId); + // #530: resolve via the same path reindexPage uses (workspace provider, + // else the global TEI sidecar) so a global-only deployment is NOT skipped. + await this.aiService.resolveEmbeddingProvider(workspaceId); } catch (err) { if (err instanceof AiEmbeddingNotConfiguredException) { this.logger.log( @@ -348,6 +370,7 @@ export class EmbeddingIndexerService { vectors: number[][], ids: { pageId: string; workspaceId: string; spaceId: string }, modelName: string, + fingerprint: string | null, ): PageEmbeddingChunkRow[] { const rows: PageEmbeddingChunkRow[] = []; let cursor = 0; @@ -370,6 +393,8 @@ export class EmbeddingIndexerService { // Provenance for a future re-index sweep on model change. modelName, modelDimensions: embedding.length, + // #530: the active generation fingerprint this row belongs to. + fingerprint, embedding, }); } diff --git a/apps/server/src/core/search/dto/search-response.dto.ts b/apps/server/src/core/search/dto/search-response.dto.ts index b9460953..81bd3680 100644 --- a/apps/server/src/core/search/dto/search-response.dto.ts +++ b/apps/server/src/core/search/dto/search-response.dto.ts @@ -32,10 +32,27 @@ export class SearchResultDto { matchedTerms: string[]; } -// The paginated envelope (A5). `total` is the EXACT permission-filtered count of -// pages matching the positive lexical query (fail-closed). `hasMore` is true when -// more results exist WITHIN the fusion window; `truncatedAtCap` signals the match -// set exceeded CANDIDATE_CAP and the tail is unreachable by pagination. +// #530 Phase B — the semantic (vector) layer's per-request status. +// - `state`: 'full' when a provider resolved and the vector arm ran this +// request; 'off' otherwise (kill-switch, no provider, or a degrade). +// - `available`: whether the vector arm actually contributed this request. +// - `reason`: why the arm did NOT run — 'no-provider' (no embedding provider +// resolved) or 'degraded' (sidecar down / query embed timed out). +// - `indexed`/`total`: reserved for PR-2 coverage reporting (unused in PR-1). +export class SearchSemanticDto { + state: 'full' | 'off'; + available: boolean; + reason?: 'no-provider' | 'degraded'; + indexed?: number; + total?: number; +} + +// The paginated envelope (A5). `total` is the permission-filtered count of the +// candidate UNION (lexical ∪ vector top-N, fail-closed) — a deliberate, #530 +// documented change from Phase A's exact-lexical count (on a semantic degrade it +// falls back to exactly that lexical count). `hasMore` is true when more results +// exist WITHIN the fusion window; `truncatedAtCap` signals the match set exceeded +// CANDIDATE_CAP and the tail is unreachable by pagination. export class SearchResponseDto { items: SearchResultDto[]; total: number; @@ -53,4 +70,7 @@ export class SearchResponseDto { mode: 'or' | 'and'; match: string; }; + // Absent on the early-exit paths (empty/garbage query); present once search + // actually runs. Optional so those short-circuit responses stay valid. + semantic?: SearchSemanticDto; } diff --git a/apps/server/src/core/search/search.module.ts b/apps/server/src/core/search/search.module.ts index fa74573d..79658ab6 100644 --- a/apps/server/src/core/search/search.module.ts +++ b/apps/server/src/core/search/search.module.ts @@ -1,8 +1,15 @@ import { Module } from '@nestjs/common'; import { SearchController } from './search.controller'; import { SearchService } from './search.service'; +import { AiModule } from '../../integrations/ai/ai.module'; +/** + * #530: AiModule supplies AiService (embedQuery / resolveEmbeddingProvider). + * PageEmbeddingRepo is provided by the @Global DatabaseModule, so it is injected + * into SearchService without an explicit import here. + */ @Module({ + imports: [AiModule], controllers: [SearchController], providers: [SearchService], exports: [SearchService], diff --git a/apps/server/src/core/search/search.service.query-mode.spec.ts b/apps/server/src/core/search/search.service.query-mode.spec.ts index e2b7c0d0..1930149c 100644 --- a/apps/server/src/core/search/search.service.query-mode.spec.ts +++ b/apps/server/src/core/search/search.service.query-mode.spec.ts @@ -48,6 +48,10 @@ describe('SearchService.searchPage — scope-security early returns', () => { shareRepo as any, spaceMemberRepo as any, pagePermissionRepo as any, + // #530 semantic path off for these query-mode unit tests: a stubbed + // embedQuery that throws makes the service degrade to the lexical path. + { embedQuery: jest.fn().mockRejectedValue(new Error('no embed')) } as any, + { vectorCandidateArm: jest.fn() } as any, ); return { service, pageRepo, shareRepo, spaceMemberRepo, pagePermissionRepo }; } diff --git a/apps/server/src/core/search/search.service.spec.ts b/apps/server/src/core/search/search.service.spec.ts index efd4d2b8..01eefb42 100644 --- a/apps/server/src/core/search/search.service.spec.ts +++ b/apps/server/src/core/search/search.service.spec.ts @@ -11,6 +11,8 @@ describe('SearchService', () => { {} as any, // shareRepo {} as any, // spaceMemberRepo {} as any, // pagePermissionRepo + {} as any, // aiService + {} as any, // pageEmbeddingRepo ); expect(service).toBeDefined(); }); @@ -61,6 +63,8 @@ describe('SearchService.searchSuggestions — onlyTemplates filter', () => { shareRepo as any, spaceMemberRepo as any, pagePermissionRepo as any, + {} as any, // aiService (searchSuggestions never embeds) + {} as any, // pageEmbeddingRepo ); return { service, db, pageBuilder }; diff --git a/apps/server/src/core/search/search.service.ts b/apps/server/src/core/search/search.service.ts index ffd5d72b..f6e6faaf 100644 --- a/apps/server/src/core/search/search.service.ts +++ b/apps/server/src/core/search/search.service.ts @@ -1,8 +1,9 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { SearchDTO, SearchSuggestionDTO } from './dto/search.dto'; import { SearchResponseDto, SearchResultDto, + SearchSemanticDto, } from './dto/search-response.dto'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB } from '@docmost/db/types/kysely.types'; @@ -11,6 +12,10 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo'; import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo'; import { ShareRepo } from '@docmost/db/repos/share/share.repo'; import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission.repo'; +import { PageEmbeddingRepo } from '@docmost/db/repos/ai-chat/page-embedding.repo'; +import { AiService } from '../../integrations/ai/ai.service'; +import { AiEmbeddingNotConfiguredException } from '../../integrations/ai/ai-embedding-not-configured.exception'; +import { isStatementTimeout } from '@docmost/db/utils'; import { ParsedQuery, ParsedTerm, @@ -59,14 +64,42 @@ function defaultBooleanMode(): SearchBooleanMode { return process.env.SEARCH_MODE === 'and' ? 'and' : 'or'; } +// #530 semantic env knobs. SEARCH_SEMANTIC=off is the kill-switch (hybrid is +// otherwise transparent — no per-request mode flag). SEARCH_VECTOR_WEIGHT tunes +// the vector RRF leg's contribution (default 1.0, i.e. equal to each lexical +// leg). SEARCH_VECTOR_CANDIDATES caps the vector top-N pulled into the union. +function semanticKillSwitchOff(): boolean { + return process.env.SEARCH_SEMANTIC === 'off'; +} +function getVectorWeight(): number { + const raw = Number(process.env.SEARCH_VECTOR_WEIGHT); + return Number.isFinite(raw) && raw >= 0 ? raw : 1.0; +} +function getVectorCandidates(): number { + const raw = Number(process.env.SEARCH_VECTOR_CANDIDATES); + return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : 50; +} +// Safety net for the brute-force vector scan (no ANN index — see +// vectorCandidateArm): a per-statement timeout bounding ONLY the fused vector +// query, so a pathological seq scan is cancelled (SQLSTATE 57014) and search +// degrades to lexical instead of hanging the request. Default 2000ms. +function getVectorStatementTimeoutMs(): number { + const raw = Number(process.env.SEARCH_VECTOR_STATEMENT_TIMEOUT_MS); + return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : 2000; +} + @Injectable() export class SearchService { + private readonly logger = new Logger(SearchService.name); + constructor( @InjectKysely() private readonly db: KyselyDB, private pageRepo: PageRepo, private shareRepo: ShareRepo, private spaceMemberRepo: SpaceMemberRepo, private pagePermissionRepo: PagePermissionRepo, + private aiService: AiService, + private pageEmbeddingRepo: PageEmbeddingRepo, ) {} // === #529 SQL fragment builders (parameterized AST, never string-concat) ===== @@ -292,33 +325,102 @@ export class SearchService { const scopeSql = sql`(${sql.join(scopePreds, sql` AND `)})`; const candidateSql = sql`(${scopeSql} AND ${this.candidatePredicate(parsed, titleOnly)})`; + // --- Semantic (vector) arm: embed the query, degrade gracefully. (#530) ---- + // The try/catch wraps ONLY embedQuery + vector-arm construction. On ANY throw + // (TEI down / 800ms timeout / no provider) we omit the vector arm entirely and + // run the byte-identical Phase-A lexical path — never a 500 from the sidecar. + // The permission filter below stays OUTSIDE this try (a permission error must + // 500, never fail-open). Hybrid is transparent; SEARCH_SEMANTIC=off disables it. + let vectorArm: RawBuilder | null = null; + let semanticAvailable = false; + let semanticReason: 'no-provider' | 'degraded' | undefined; + if (!semanticKillSwitchOff()) { + try { + const { vector, fingerprint } = await this.aiService.embedQuery( + opts.workspaceId, + rawQuery, + ); + vectorArm = this.pageEmbeddingRepo.vectorCandidateArm({ + queryEmbedding: vector, + dimensions: vector.length, + fingerprint, + scope: scopeSql, + limit: getVectorCandidates(), + // Filter page_embeddings by (immutable) workspace_id directly, so the + // composite index bites on its leading column and candidates are + // workspace-scoped at the embedding level (#530 review WARNING 2). Space + // scoping stays on the pages join only — page_embeddings.space_id can be + // STALE after a cross-space move, so filtering it here would drop a moved + // page's vector hit (re-review regression fix). + workspaceId: scope.workspaceId, + }); + semanticAvailable = true; + } catch (err) { + if (err instanceof AiEmbeddingNotConfiguredException) { + // No embedding provider is the DEFAULT state of any deployment without + // the TEI sidecar — normal, not a problem. Log at DEBUG so it never + // floods WARN once per search request. + semanticReason = 'no-provider'; + this.logger.debug( + `search.semantic.no-provider workspace=${opts.workspaceId}`, + ); + } else { + // A provider IS configured but the embed call failed/timed out — a real + // sidecar problem worth a WARN. Structured, greppable event (fixed code) + // — no query text/secrets. + semanticReason = 'degraded'; + this.logger.warn( + `search.semantic.degraded reason=degraded workspace=${opts.workspaceId}`, + ); + } + } + } // --- Ranked candidate ids (ALL matches, RRF order). ----------------------- - // Two independent branches ranked SEPARATELY (FTS by ts_rank_cd, substring by - // tier) then fused with RRF: score = Σ 1/(k + rank_branch). Deterministic - // ORDER BY rrf DESC, id so pagination never dupes/skips (A4, acceptance #8). - const rankedRows = await sql<{ id: string }>` - WITH candidates AS ( - SELECT pages.id AS id, - ${this.ftsScoreExpr(parsed)} AS fts_score, - ${this.subTierExpr(parsed, titleOnly)} AS sub_tier - FROM pages - WHERE ${candidateSql} - ), - ranked AS ( - SELECT id, fts_score, sub_tier, - row_number() OVER (ORDER BY fts_score DESC NULLS LAST, id) AS rn_fts, - row_number() OVER (ORDER BY sub_tier DESC NULLS LAST, id) AS rn_sub - FROM candidates - ) - SELECT id - FROM ranked - ORDER BY - (CASE WHEN fts_score IS NOT NULL THEN 1.0/(${RRF_K} + rn_fts) ELSE 0 END) - + (CASE WHEN sub_tier IS NOT NULL THEN 1.0/(${RRF_K} + rn_sub) ELSE 0 END) DESC, - id ASC - `.execute(this.db); + // Lexical branches ranked SEPARATELY (FTS by ts_rank_cd, substring by tier); + // when a query vector is available a third VECTOR branch is fused over the + // UNION of lexical + vector candidates. RRF: score = Σ w/(k + rank_branch). + // Deterministic ORDER BY rrf DESC, id so pagination never dupes/skips (A4, + // acceptance #8). `total` is thus the union size (lexical ∪ vector top-N), + // post-permission-filter — a documented change from Phase A's exact lexical + // count (#530). On degrade the query is byte-identical to Phase A. + // + // The fused query is bounded by a per-statement timeout (see runRankedQuery): + // if the brute-force vector scan is cancelled (SQLSTATE 57014) we degrade to + // the byte-identical lexical path — search returns lexical results, never + // hangs, never 500s. Only the timeout is caught here; any OTHER SQL error + // propagates. The permission filter below stays outside, unchanged. + let orderedIds: string[]; + try { + orderedIds = await this.runRankedQuery( + candidateSql, + parsed, + titleOnly, + vectorArm, + ); + } catch (err) { + if (vectorArm && isStatementTimeout(err)) { + semanticAvailable = false; + semanticReason = 'degraded'; + this.logger.warn( + `search.semantic.degraded reason=degraded workspace=${opts.workspaceId} (vector statement timeout)`, + ); + // Re-run the exact Phase-A lexical path (no vector arm, no timeout). + orderedIds = await this.runRankedQuery( + candidateSql, + parsed, + titleOnly, + null, + ); + } else { + throw err; + } + } - let orderedIds = rankedRows.rows.map((r) => r.id); + const semantic: SearchSemanticDto = { + state: semanticAvailable ? 'full' : 'off', + available: semanticAvailable, + ...(semanticReason ? { reason: semanticReason } : {}), + }; // --- Permission filter (fail-closed, exact total). ------------------------ // filterAccessiblePageIds runs the #348 hasRestricted pre-check internally: @@ -355,14 +457,131 @@ export class SearchService { const hasMore = offset + pageIds.length < window.length; if (pageIds.length === 0) { - return { items: [], total, hasMore, truncatedAtCap, offset, query: queryMeta }; + return { + items: [], + total, + hasMore, + truncatedAtCap, + offset, + query: queryMeta, + semantic, + }; } // --- Detail fetch for the page slice only (ts_headline/snippet are costly, so // compute them ONLY for the returned rows), preserving RRF order. ------- const items = await this.fetchDetails(pageIds, parsed, titleOnly); - return { items, total, hasMore, truncatedAtCap, offset, query: queryMeta }; + return { + items, + total, + hasMore, + truncatedAtCap, + offset, + query: queryMeta, + semantic, + }; + } + + /** + * Run the ranked-candidate-ids query and return the ids in RRF order (#530). + * + * When `vectorArm` is null (degrade / semantic off / no query vector) this runs + * the BYTE-IDENTICAL Phase-A 2-branch lexical query — the semantic layer must + * be a pure superset that never changes lexical behaviour on degrade. + * + * When a vector arm is supplied it is UNION ALL'd with the lexical arm into one + * `candidates` set; `agg` collapses each page to its best per-branch signal + * (MAX fts_score / MAX sub_tier / MIN vec_distance); `ranked` assigns a + * per-branch row_number; the final ORDER BY fuses the three legs with RRF, + * adding the vector leg ONLY for rows that have a vec_distance (a lexical-only + * hit contributes 0 to the vector leg, and vice-versa). W_VEC weights the + * vector leg. + */ + private async runRankedQuery( + candidateSql: RawBuilder, + parsed: ParsedQuery, + titleOnly: boolean, + vectorArm: RawBuilder | null, + ): Promise { + if (!vectorArm) { + // Phase-A lexical path — DO NOT change (byte-identical on degrade, #529). + const rankedRows = await sql<{ id: string }>` + WITH candidates AS ( + SELECT pages.id AS id, + ${this.ftsScoreExpr(parsed)} AS fts_score, + ${this.subTierExpr(parsed, titleOnly)} AS sub_tier + FROM pages + WHERE ${candidateSql} + ), + ranked AS ( + SELECT id, fts_score, sub_tier, + row_number() OVER (ORDER BY fts_score DESC NULLS LAST, id) AS rn_fts, + row_number() OVER (ORDER BY sub_tier DESC NULLS LAST, id) AS rn_sub + FROM candidates + ) + SELECT id + FROM ranked + ORDER BY + (CASE WHEN fts_score IS NOT NULL THEN 1.0/(${RRF_K} + rn_fts) ELSE 0 END) + + (CASE WHEN sub_tier IS NOT NULL THEN 1.0/(${RRF_K} + rn_sub) ELSE 0 END) DESC, + id ASC + `.execute(this.db); + return rankedRows.rows.map((r) => r.id); + } + + // #530 fused 3-branch RRF over lexical ∪ vector candidates. The lexical arm + // adds NULL::float vec_distance to stay UNION-compatible with the vector arm. + const wVec = getVectorWeight(); + + // Bound the brute-force vector scan with a per-statement timeout, scoped to + // THIS query only. `set_config(..., is_local := true)` = SET LOCAL semantics: + // it applies within this transaction and auto-resets at COMMIT/ROLLBACK, so + // it can NEVER leak onto the next query that reuses this pooled connection. + // On a cancellation the driver raises SQLSTATE 57014, which searchPage catches + // to degrade to lexical-only. Running inside a transaction also guarantees the + // SET and the ranked query share one connection. + const timeoutMs = getVectorStatementTimeoutMs(); + return this.db.transaction().execute(async (trx) => { + await sql`SELECT set_config('statement_timeout', ${String(timeoutMs)}, true)`.execute( + trx, + ); + const rankedRows = await sql<{ id: string }>` + WITH candidates AS ( + (SELECT pages.id AS id, + ${this.ftsScoreExpr(parsed)} AS fts_score, + ${this.subTierExpr(parsed, titleOnly)} AS sub_tier, + NULL::float AS vec_distance + FROM pages + WHERE ${candidateSql}) + UNION ALL + (${vectorArm}) + ), + agg AS ( + SELECT id, + MAX(fts_score) AS fts_score, + MAX(sub_tier) AS sub_tier, + MIN(vec_distance) AS vec_distance + FROM candidates + GROUP BY id + ), + ranked AS ( + SELECT id, fts_score, sub_tier, vec_distance, + row_number() OVER (ORDER BY fts_score DESC NULLS LAST, id) AS rn_fts, + row_number() OVER (ORDER BY sub_tier DESC NULLS LAST, id) AS rn_sub, + row_number() OVER (ORDER BY vec_distance ASC NULLS LAST, id) AS rn_vec + FROM agg + ) + SELECT id + FROM ranked + ORDER BY + (CASE WHEN fts_score IS NOT NULL THEN 1.0/(${RRF_K} + rn_fts) ELSE 0 END) + + (CASE WHEN sub_tier IS NOT NULL THEN 1.0/(${RRF_K} + rn_sub) ELSE 0 END) + + (CASE WHEN vec_distance IS NOT NULL THEN ${wVec}::float/(${RRF_K} + rn_vec) ELSE 0 END) DESC, + id ASC + `.execute(trx); + return rankedRows.rows.map((r) => r.id); + }); } // Resolve the search scope: explicit space, the authed user's member spaces, or diff --git a/apps/server/src/database/migrations/20260712T120000-page-embeddings-fingerprint.ts b/apps/server/src/database/migrations/20260712T120000-page-embeddings-fingerprint.ts new file mode 100644 index 00000000..3bd17f1d --- /dev/null +++ b/apps/server/src/database/migrations/20260712T120000-page-embeddings-fingerprint.ts @@ -0,0 +1,49 @@ +import { type Kysely, sql } from 'kysely'; + +/** + * #530 Search Phase B (PR-1): add the embedding FINGERPRINT column. + * + * The fingerprint is a deterministic id of the embedding configuration that + * produced a row — model id + revision + query/doc prefix scheme + dimensions + * (see AiService.computeEmbeddingFingerprint). Search filters vector candidates + * by the workspace's ACTIVE fingerprint so a revision bump or a prefix-scheme + * change never fuses incompatible vectors into results. + * + * PR-1 scope: this migration only ADDS the column (nullable — existing rows stay + * NULL = legacy) and the composite index the vector-candidate scan uses. The full + * generational swap / GC lifecycle (target-fingerprint reindex, atomic flip, + * old-generation GC) is deliberately deferred to PR-2. + * + * Independent migration (per the #363 crash-loop net): it creates ONLY its own + * objects and never touches another migration's tables/indexes, so a partial + * failure cannot leave a shared object half-built. + */ +export async function up(db: Kysely): Promise { + // Nullable text column. Existing rows keep NULL (legacy generation) and are + // simply not matched by the active-fingerprint filter until re-indexed. + await sql` + ALTER TABLE page_embeddings + ADD COLUMN IF NOT EXISTS fingerprint text + `.execute(db); + + // Composite btree supporting the scoped, dimension- + fingerprint-filtered + // vector-candidate scan (workspace_id + space_id + fingerprint + model_dimensions). + await db.schema + .createIndex('idx_page_embeddings_ws_space_fp_dim') + .ifNotExists() + .on('page_embeddings') + .columns(['workspace_id', 'space_id', 'fingerprint', 'model_dimensions']) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .dropIndex('idx_page_embeddings_ws_space_fp_dim') + .ifExists() + .execute(); + + await sql` + ALTER TABLE page_embeddings + DROP COLUMN IF EXISTS fingerprint + `.execute(db); +} diff --git a/apps/server/src/database/repos/ai-chat/page-embedding.repo.ts b/apps/server/src/database/repos/ai-chat/page-embedding.repo.ts index 8937a922..398797bd 100644 --- a/apps/server/src/database/repos/ai-chat/page-embedding.repo.ts +++ b/apps/server/src/database/repos/ai-chat/page-embedding.repo.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; import { InjectKysely } from 'nestjs-kysely'; -import { sql } from 'kysely'; +import { RawBuilder, sql } from 'kysely'; import * as pgvector from 'pgvector'; import { KyselyDB, KyselyTransaction } from '../../types/kysely.types'; import { dbOrTx } from '../../utils'; @@ -35,6 +35,9 @@ export interface PageEmbeddingChunkRow { content: string; modelName: string; modelDimensions: number; + // #530 PR-1: the active embedding fingerprint (see computeEmbeddingFingerprint). + // null only when no provider resolves (the indexer no-ops in that case). + fingerprint: string | null; embedding: number[]; } @@ -120,6 +123,7 @@ export class PageEmbeddingRepo { content: row.content, modelName: row.modelName, modelDimensions: row.modelDimensions, + fingerprint: row.fingerprint, // pgvector.toSql -> '[1,2,3]'; cast the bound literal to vector. embedding: sql`${pgvector.toSql(row.embedding)}::vector`, })), @@ -127,6 +131,82 @@ export class PageEmbeddingRepo { .execute(); } + /** + * #530: build the VECTOR candidate arm for SearchService's fused RRF union — a + * page-level nearest-neighbour sub-select. Returns a `sql` fragment (not an + * executed query) so the caller can UNION ALL it with the lexical arm inside a + * single ranked-ids query. + * + * SCALING BOUNDARY: the column is dimension-agnostic, so it carries NO ANN + * index — this is a brute-force O(N) KNN seq scan with `<=>`. Accepted for the + * small-tenant / homelab fork target (ANN + a pinned-dimension column are + * deferred). It is NOT unbounded, though: the caller (SearchService) runs this + * fused query under a per-statement timeout (SEARCH_VECTOR_STATEMENT_TIMEOUT_MS) + * and DEGRADES to lexical-only on a 57014 cancellation, so a pathological scan + * can never hang the interactive search request. + * + * The arm collapses a page's chunks to its best (MIN) cosine distance: + * SELECT pages.id, NULL fts_score, NULL sub_tier, + * MIN(pe.embedding <=> $qvec) AS vec_distance + * FROM page_embeddings pe JOIN pages ON pages.id = pe.page_id + * WHERE AND pe.model_dimensions = $dim AND pe.fingerprint = $fp + * GROUP BY pages.id ORDER BY vec_distance LIMIT $limit + * + * `scope` is SearchService's shared scope predicate (workspace + space/id set + + * creator + descendants + deleted_at), referencing the `pages` table — it must + * be spliced verbatim so the vector candidate set mirrors the lexical scope + * EXACTLY, minus the lexical text predicate (vector candidates need not match + * text). The vector is bound via pgvector's `toSql(...)::vector`, and both the + * dimension and the ACTIVE fingerprint are filtered so `<=>` only ever compares + * compatible, same-generation vectors (pgvector errors on a dimension + * mismatch; a fingerprint mismatch would fuse incomparable vectors). + * + * `workspaceId` is ALSO filtered directly on `page_embeddings` — not only via + * the pages join. It is IMMUTABLE (there are no cross-workspace page moves), so + * an embedding row's workspace_id always matches its page's, and this drops NO + * legitimate hit; it lets the composite index idx_page_embeddings_ws_space_fp_dim + * (workspace_id, space_id, fingerprint, model_dimensions) bite from its LEADING + * column, and workspace-scopes candidates at the embedding level (defense in + * depth). + * + * SPACE is deliberately NOT filtered on page_embeddings: `page_embeddings.space_id` + * is stamped at index time and is NOT updated when a page is MOVED between spaces + * (movePageToSpace does not reindex, and PAGE_MOVED_TO_SPACE has no reindex + * consumer), so a moved page's rows carry the OLD space until the next reindex. A + * `page_embeddings.space_id = ANY(scope)` predicate would then wrongly drop a + * legitimate vector hit for a page moved INTO the searched space. Space scoping + * is therefore enforced ONLY by the join to `pages` (pages.space_id ∈ scope, + * always current) — the same way the lexical arm scopes, so no space leak. + * + * The NULL fts_score / sub_tier columns keep the arm UNION-compatible with the + * lexical arm's column list. + */ + vectorCandidateArm(params: { + queryEmbedding: number[]; + dimensions: number; + fingerprint: string; + scope: RawBuilder; + limit: number; + workspaceId: string; + }): RawBuilder { + const qvec = sql`${pgvector.toSql(params.queryEmbedding)}::vector`; + return sql` + SELECT pages.id AS id, + NULL::float AS fts_score, + NULL::int AS sub_tier, + MIN(page_embeddings.embedding <=> ${qvec}) AS vec_distance + FROM page_embeddings + JOIN pages ON pages.id = page_embeddings.page_id + WHERE ${params.scope} + AND page_embeddings.workspace_id = ${params.workspaceId} + AND page_embeddings.model_dimensions = ${params.dimensions} + AND page_embeddings.fingerprint = ${params.fingerprint} + GROUP BY pages.id + ORDER BY vec_distance ASC + LIMIT ${params.limit} + `; + } + /** * Cosine search over the embeddings, scoped to a workspace AND a set of * spaces the caller may read (see semanticSearch access-scoping). Orders by diff --git a/apps/server/src/database/types/embeddings.types.ts b/apps/server/src/database/types/embeddings.types.ts index e12f1c7a..1f650c6a 100644 --- a/apps/server/src/database/types/embeddings.types.ts +++ b/apps/server/src/database/types/embeddings.types.ts @@ -7,6 +7,11 @@ export interface PageEmbeddings { spaceId: string; modelName: string; modelDimensions: number; + // #530 PR-1: the active embedding fingerprint (model id + revision + prefix + // scheme + dimensions). NULL on legacy rows written before this column existed. + // Search filters candidates by the ACTIVE fingerprint so a revision/prefix + // change never mixes incompatible vectors. + fingerprint: string | null; workspaceId: string; // Nullable: page-body embeddings have no attachment (only attachment chunks set it). attachmentId: string | null; diff --git a/apps/server/src/database/utils.ts b/apps/server/src/database/utils.ts index c78596b9..c711c463 100644 --- a/apps/server/src/database/utils.ts +++ b/apps/server/src/database/utils.ts @@ -108,6 +108,23 @@ export function isUniqueViolation(err: unknown): boolean { return (err as { code?: unknown } | null | undefined)?.code === PG_UNIQUE_VIOLATION; } +/** Postgres `query_canceled` SQLSTATE — raised when statement_timeout fires. */ +const PG_QUERY_CANCELED = '57014'; + +/** + * Whether `err` is a Postgres statement-timeout cancellation (SQLSTATE `57014`). + * Used by #530 semantic search to degrade a slow vector scan to lexical-only + * instead of hanging/500-ing. Matches on the SQLSTATE (the driver surfaces it as + * `.code`), with a message fallback for any wrapper that drops the code. + */ +export function isStatementTimeout(err: unknown): boolean { + if ((err as { code?: unknown } | null | undefined)?.code === PG_QUERY_CANCELED) { + return true; + } + const msg = err instanceof Error ? err.message : ''; + return /canceling statement due to statement timeout/i.test(msg); +} + /** * The name of the UNIQUE index/constraint a `23505` error violated, or * undefined. The `kysely-postgres-js` / `postgres@3.x` driver surfaces it as diff --git a/apps/server/src/integrations/ai/ai.service.embedding.spec.ts b/apps/server/src/integrations/ai/ai.service.embedding.spec.ts new file mode 100644 index 00000000..0de863b6 --- /dev/null +++ b/apps/server/src/integrations/ai/ai.service.embedding.spec.ts @@ -0,0 +1,143 @@ +import { AiService, computeEmbeddingFingerprint } from './ai.service'; + +/** + * #530 Search Phase B (PR-1) unit coverage for the embedding layer: + * - computeEmbeddingFingerprint: a revision bump / prefix toggle MUST change the + * fingerprint even when the bare model name + dimension are unchanged (it is + * deliberately NOT the bare model name); + * - embedQuery: applies the QUERY prefix and embeds under the short SEARCH embed + * timeout (not the long batch timeout); + * - embedWithModel: a hung model + short timeout degrades with a clear timeout + * error (the search caller then falls back to the lexical path). + */ +describe('computeEmbeddingFingerprint (#530)', () => { + const base = { + modelId: 'intfloat/multilingual-e5-small', + revision: 'sha-aaa', + queryPrefix: 'query: ', + docPrefix: 'passage: ', + dimensions: 384, + }; + + it('is deterministic for identical inputs', () => { + expect(computeEmbeddingFingerprint(base)).toBe( + computeEmbeddingFingerprint({ ...base }), + ); + }); + + it('CHANGES on a revision bump (same model name + dimension)', () => { + const a = computeEmbeddingFingerprint(base); + const b = computeEmbeddingFingerprint({ ...base, revision: 'sha-bbb' }); + expect(b).not.toBe(a); + }); + + it('CHANGES on a query-prefix toggle (same model name + dimension + revision)', () => { + const a = computeEmbeddingFingerprint(base); + const b = computeEmbeddingFingerprint({ ...base, queryPrefix: '' }); + expect(b).not.toBe(a); + }); + + it('CHANGES on a doc-prefix toggle', () => { + const a = computeEmbeddingFingerprint(base); + const b = computeEmbeddingFingerprint({ ...base, docPrefix: '' }); + expect(b).not.toBe(a); + }); + + it('CHANGES on a dimension change', () => { + const a = computeEmbeddingFingerprint(base); + const b = computeEmbeddingFingerprint({ ...base, dimensions: 768 }); + expect(b).not.toBe(a); + }); + + it('is NOT the bare model name (guards the mutation)', () => { + // If the fingerprint were the bare model name, a revision bump would not + // change it — the tests above would red. Assert it directly too. + expect(computeEmbeddingFingerprint(base)).not.toBe(base.modelId); + }); + + it('distinguishes swapped prefixes — ("q","") != ("","q")', () => { + const a = computeEmbeddingFingerprint({ + ...base, + queryPrefix: 'q', + docPrefix: '', + }); + const b = computeEmbeddingFingerprint({ + ...base, + queryPrefix: '', + docPrefix: 'q', + }); + expect(a).not.toBe(b); + }); +}); + +describe('AiService.embedQuery (#530)', () => { + function makeService(): AiService { + // The constructor body only stores its deps; embedQuery is fully driven by + // the spied resolveEmbeddingProvider + embedWithModel below. + return new AiService({} as any, {} as any, {} as any); + } + + it('prepends the query prefix and returns the provider fingerprint', async () => { + const svc = makeService(); + jest.spyOn(svc, 'resolveEmbeddingProvider').mockResolvedValue({ + model: { modelId: 'm' } as any, + queryPrefix: 'query: ', + docPrefix: 'passage: ', + fingerprint: 'fp-active', + }); + const embedSpy = jest + .spyOn(svc, 'embedWithModel') + .mockResolvedValue([[0.1, 0.2, 0.3]]); + + const res = await svc.embedQuery('ws-1', 'кофе'); + + // The single embedded value carries the query prefix. + expect(embedSpy).toHaveBeenCalledTimes(1); + const [, wsArg, valuesArg, timeoutArg] = embedSpy.mock.calls[0]; + expect(wsArg).toBe('ws-1'); + expect(valuesArg).toEqual(['query: кофе']); + // Bound by the SHORT search-embed timeout (default 800ms), not the 120000ms + // batch timeout. + expect(timeoutArg).toBe(800); + expect(res).toEqual({ vector: [0.1, 0.2, 0.3], fingerprint: 'fp-active' }); + }); + + it('propagates a no-provider error (drives the no-provider degrade)', async () => { + const svc = makeService(); + const { AiEmbeddingNotConfiguredException } = await import( + './ai-embedding-not-configured.exception' + ); + jest + .spyOn(svc, 'resolveEmbeddingProvider') + .mockRejectedValue(new AiEmbeddingNotConfiguredException()); + await expect(svc.embedQuery('ws-1', 'x')).rejects.toBeInstanceOf( + AiEmbeddingNotConfiguredException, + ); + }); +}); + +describe('AiService.embedWithModel timeout (#530)', () => { + it('degrades a hung model with a timeout error under the short window', async () => { + const svc = new AiService({} as any, {} as any, {} as any); + // A model whose doEmbed never resolves but honours the abort signal, so the + // AbortSignal.timeout fires and embedWithModel raises a clear timeout error. + const hangingModel: any = { + specificationVersion: 'v2', + provider: 'fake', + modelId: 'fake', + maxEmbeddingsPerCall: 100, + supportsParallelCalls: true, + doEmbed: ({ abortSignal }: { abortSignal?: AbortSignal }) => + new Promise((_resolve, reject) => { + abortSignal?.addEventListener('abort', () => { + const err = new Error('The operation was aborted'); + err.name = 'AbortError'; + reject(err); + }); + }), + }; + await expect( + svc.embedWithModel(hangingModel, 'ws-1', ['x'], 30), + ).rejects.toThrow(/timed out/i); + }); +}); diff --git a/apps/server/src/integrations/ai/ai.service.ts b/apps/server/src/integrations/ai/ai.service.ts index 2f999e72..f0beec15 100644 --- a/apps/server/src/integrations/ai/ai.service.ts +++ b/apps/server/src/integrations/ai/ai.service.ts @@ -23,6 +23,50 @@ import { import { AiProviderCredentialsRepo } from '@docmost/db/repos/ai-chat/ai-provider-credentials.repo'; import { SecretBoxService } from '../crypto/secret-box'; import { AiDriver } from './ai.types'; +import { createHash } from 'node:crypto'; + +/** + * A resolved embedding provider for #530 semantic search. `model` is the AI SDK + * embedding model; `queryPrefix`/`docPrefix` are prepended to a query / a stored + * chunk respectively (e5-style `"query: "` / `"passage: "`, empty for a non-e5 + * provider); `fingerprint` is the deterministic id of the whole configuration. + */ +export interface ResolvedEmbeddingProvider { + model: EmbeddingModel; + queryPrefix: string; + docPrefix: string; + fingerprint: string; +} + +/** + * Deterministic embedding FINGERPRINT (#530). Encodes model id + revision + + * prefix scheme + dimensions so that ANY of them changing (a revision bump, a + * prefix toggle) yields a DIFFERENT fingerprint even when the bare model name and + * dimension are unchanged. Deliberately NOT the bare model name: two rows from + * the same model but a different revision/prefix must not be fused together. + * + * Exported as a pure function so it can be unit-tested in isolation and reused by + * the indexer without a service instance. + */ +export function computeEmbeddingFingerprint(parts: { + modelId: string; + revision: string; + queryPrefix: string; + docPrefix: string; + dimensions: number | null; +}): string { + // The two prefixes are SEPARATE keys (not a concatenated string): JSON.stringify + // escapes each independently, so the prefix scheme ("a","") is distinct from + // ("","a") with no separator/collision hazard even if a prefix contains spaces. + const canonical = JSON.stringify({ + m: parts.modelId, + r: parts.revision, + q: parts.queryPrefix, + d: parts.docPrefix, + dim: parts.dimensions ?? 0, + }); + return createHash('sha256').update(canonical).digest('hex').slice(0, 32); +} /** * Optional chat-model override carried by an agent role (`ai_agent_roles. @@ -362,11 +406,112 @@ export class AiService { async embedTexts(workspaceId: string, texts: string[]): Promise { if (texts.length === 0) return []; const model = await this.getEmbeddingModel(workspaceId); - // Bound the embedding call: a slow/hung embeddings endpoint must fail loudly - // (and let the caller move on to the next page) instead of blocking forever. - // The single signal caps the WHOLE call, including the SDK's internal - // retries/backoff (embedMany defaults to maxRetries: 2). - const timeoutMs = AiService.embeddingTimeoutMs(); + return this.embedWithModel(model, workspaceId, texts); + } + + /** + * #530: resolve the embedding provider for a workspace. Prefers the workspace's + * own configured embedding provider; falls back to the GLOBAL env provider (a + * TEI sidecar via the OpenAI-compatible path) when the workspace has none. + * Returns the model + the query/doc prefixes + the config fingerprint. Throws + * AiEmbeddingNotConfiguredException when NEITHER resolves, so callers can drive + * a `no-provider` degrade path. + */ + async resolveEmbeddingProvider( + workspaceId: string, + ): Promise { + // 1. Per-workspace provider (uses the workspace's own creds/endpoint). When + // it is not configured getEmbeddingModel throws the not-configured + // exception; we swallow ONLY that and fall through to the global provider. + try { + const model = await this.getEmbeddingModel(workspaceId); + const modelId = + typeof model === 'string' ? model : (model.modelId ?? 'unknown'); + // A per-workspace (typically non-e5) provider gets no e5-style prefixes. + return { + model, + queryPrefix: '', + docPrefix: '', + fingerprint: computeEmbeddingFingerprint({ + modelId, + revision: 'workspace', + queryPrefix: '', + docPrefix: '', + dimensions: null, + }), + }; + } catch (err) { + if (!(err instanceof AiEmbeddingNotConfiguredException)) throw err; + } + + // 2. Global env provider (TEI sidecar). TEI is OpenAI-compatible, so reuse + // the existing openai path — no new SDK. A dummy key is fine for a + // keyless self-hosted sidecar. + const endpoint = process.env.EMBEDDING_ENDPOINT?.trim(); + const globalModel = process.env.EMBEDDING_MODEL?.trim(); + if (endpoint && globalModel) { + const model = createOpenAI({ + baseURL: endpoint, + apiKey: process.env.EMBEDDING_API_KEY || 'unused', + }).textEmbeddingModel(globalModel); + const queryPrefix = process.env.EMBEDDING_QUERY_PREFIX ?? ''; + const docPrefix = process.env.EMBEDDING_DOC_PREFIX ?? ''; + const dimRaw = Number(process.env.EMBEDDING_DIMENSIONS); + const dimensions = Number.isFinite(dimRaw) && dimRaw > 0 ? dimRaw : null; + return { + model, + queryPrefix, + docPrefix, + fingerprint: computeEmbeddingFingerprint({ + modelId: globalModel, + revision: process.env.EMBEDDING_REVISION ?? '', + queryPrefix, + docPrefix, + dimensions, + }), + }; + } + + // Neither resolved -> drives semantic.reason=no-provider. + throw new AiEmbeddingNotConfiguredException(); + } + + /** + * #530: embed a SEARCH QUERY. Resolves the provider, prepends its query prefix, + * and embeds the single value under its OWN short timeout + * (SEARCH_EMBED_TIMEOUT_MS, default 800ms) — NOT the long batch-indexing + * timeout — so a slow/hung sidecar degrades search fast. Throws on + * timeout/error (the caller degrades to the lexical-only path). Returns the + * vector plus the active fingerprint used to filter candidate rows. + */ + async embedQuery( + workspaceId: string, + text: string, + ): Promise<{ vector: number[]; fingerprint: string }> { + const provider = await this.resolveEmbeddingProvider(workspaceId); + const [vector] = await this.embedWithModel( + provider.model, + workspaceId, + [provider.queryPrefix + text], + AiService.searchEmbedTimeoutMs(), + ); + return { vector, fingerprint: provider.fingerprint }; + } + + /** + * Embed values with an EXPLICIT model, bounded by `timeoutMs` (default: the + * batch-indexing timeout). Shared core of embedTexts / embedQuery: a slow/hung + * embeddings endpoint must fail loudly instead of blocking forever. The single + * signal caps the WHOLE call, including the SDK's internal retries/backoff + * (embedMany defaults to maxRetries: 2). + */ + async embedWithModel( + model: EmbeddingModel, + workspaceId: string, + texts: string[], + timeoutMs: number = AiService.embeddingTimeoutMs(), + ): Promise { + if (texts.length === 0) return []; const signal = AbortSignal.timeout(timeoutMs); try { const { embeddings } = await embedMany({ @@ -391,8 +536,8 @@ export class AiService { if (signal.aborted && abortLike) { throw new Error( `Embedding request timed out after ${timeoutMs}ms ` + - `(workspace ${workspaceId}, ${texts.length} chunk(s)). ` + - `Increase AI_EMBEDDING_TIMEOUT_MS or check the embeddings endpoint.`, + `(workspace ${workspaceId}, ${texts.length} value(s)). ` + + `Increase the embedding timeout or check the embeddings endpoint.`, ); } throw err; @@ -408,6 +553,16 @@ export class AiService { return Number.isFinite(raw) && raw > 0 ? raw : 120_000; } + /** + * #530: per-call timeout for the interactive SEARCH query embed. Much shorter + * than the batch-indexing timeout (a search request cannot wait 2 minutes on + * the sidecar). Configurable via SEARCH_EMBED_TIMEOUT_MS; default 800ms. + */ + private static searchEmbedTimeoutMs(): number { + const raw = Number(process.env.SEARCH_EMBED_TIMEOUT_MS); + return Number.isFinite(raw) && raw > 0 ? raw : 800; + } + // Build a tiny valid WAV (mono, 16-bit PCM, 16 kHz, ~1s of silence), used only // as a connectivity probe for the STT endpoint in testConnection. private static silentWavProbe(): Uint8Array { diff --git a/apps/server/test/integration/search-lexical.int-spec.ts b/apps/server/test/integration/search-lexical.int-spec.ts index b553c6db..38d8b0a5 100644 --- a/apps/server/test/integration/search-lexical.int-spec.ts +++ b/apps/server/test/integration/search-lexical.int-spec.ts @@ -1,7 +1,10 @@ import { randomUUID } from 'node:crypto'; import { Kysely, sql } from 'kysely'; +import { Logger } from '@nestjs/common'; import { SearchService } from 'src/core/search/search.service'; import { PageRepo } from '@docmost/db/repos/page/page.repo'; +import { PageEmbeddingRepo } from '@docmost/db/repos/ai-chat/page-embedding.repo'; +import { AiEmbeddingNotConfiguredException } from 'src/integrations/ai/ai-embedding-not-configured.exception'; import { getTestDb, destroyTestDb, @@ -53,10 +56,19 @@ describe('SearchService #529 lexical overhaul [integration]', () => { // Service wired to the real DB + real PageRepo (recursive descendants) with // stubbed space-membership + permission repos so a test controls scope and the // permission filter explicitly. `accessibleIds` (when set) is the KEEP list. + // #530: a FAKE AiService whose embedQuery is fully controlled per test. By + // DEFAULT it throws AiEmbeddingNotConfiguredException, so the semantic arm is + // omitted and every Phase-A case runs the BYTE-IDENTICAL lexical path — the + // semantic layer must never change lexical behaviour on degrade. Semantic cases + // pass `embedVector` (a deterministic 384-dim query vector) or `embedThrows`. function buildService(opts?: { userSpaceIds?: string[]; accessibleIds?: string[] | null; filterThrows?: boolean; + embedVector?: number[]; + embedFingerprint?: string; + embedThrows?: 'no-provider' | 'degraded'; + slowVectorArm?: boolean; }): SearchService { const pageRepo = new PageRepo(db as any, null as any, null as any); const spaceMemberRepo = { @@ -71,15 +83,86 @@ describe('SearchService #529 lexical overhaul [integration]', () => { : pageIds; }, }; + const aiService = { + embedQuery: async () => { + if (opts?.embedVector) { + return { + vector: opts.embedVector, + fingerprint: opts.embedFingerprint ?? ACTIVE_FP, + }; + } + if (opts?.embedThrows === 'degraded') { + const err = new Error('embedding request timed out after 1ms'); + err.name = 'TimeoutError'; + throw err; + } + // Default (and explicit 'no-provider'): no embedding provider resolved. + throw new AiEmbeddingNotConfiguredException(); + }, + }; + // Real repo on the migrated test DB — exercises the actual vector-candidate + // SQL (pgvector <=>, fingerprint + dimension filters). When slowVectorArm is + // set, wrap the arm so it sleeps well past the (tiny, test-set) statement + // timeout, forcing a 57014 cancellation to exercise the degrade-to-lexical net. + const realRepo = new PageEmbeddingRepo(db as any); + const pageEmbeddingRepo = opts?.slowVectorArm + ? { + vectorCandidateArm: (p: any) => { + const inner = realRepo.vectorCandidateArm(p); + return sql`SELECT * FROM (${inner}) AS slowarm WHERE (SELECT true FROM pg_sleep(0.5))`; + }, + } + : realRepo; return new SearchService( db as any, pageRepo as any, {} as any, spaceMemberRepo as any, pagePermissionRepo as any, + aiService as any, + pageEmbeddingRepo as any, ); } + // Active embedding fingerprint used by planted rows + the fake query embed. + const ACTIVE_FP = 'fp-active-530'; + + // A deterministic unit-ish 384-dim vector with a single 1 at `concept`. Two + // vectors of the same concept have cosine distance 0; different concepts are + // orthogonal (distance 1), so nearest-neighbour ordering is fully controlled. + function conceptVec(concept: number): number[] { + const v = new Array(384).fill(0); + v[concept % 384] = 1; + return v; + } + + // Plant one chunk embedding for a page via the REAL repo (exercises the #530 + // fingerprint insert path). Dimension is fixed at 384 to match conceptVec. + async function plantEmbedding( + pageId: string, + vector: number[], + fingerprint: string = ACTIVE_FP, + planSpaceId: string = spaceId, + ): Promise { + const repo = new PageEmbeddingRepo(db as any); + await repo.insertChunks([ + { + pageId, + workspaceId, + spaceId: planSpaceId, + attachmentId: null, + chunkIndex: 0, + chunkStart: 0, + chunkLength: 1, + content: 'planted chunk', + modelName: 'test-model', + modelDimensions: 384, + fingerprint, + embedding: vector, + }, + ]); + } + const search = (service: SearchService, params: any) => service.searchPage(params, { userId: 'u-1', workspaceId }) as any; @@ -609,4 +692,324 @@ describe('SearchService #529 lexical overhaul [integration]', () => { expect(hit2).toBeDefined(); expect(hit2.path).toEqual([]); }); + + // === #530 Phase B (PR-1): semantic fusion ================================== + // Deterministic fake embedder (no live model) + real PageEmbeddingRepo on the + // migrated DB with planted vectors + the active fingerprint, so cosine ordering + // is fully controlled. Each case runs in its OWN space so planted rows never + // leak across tests. + describe('#530 semantic fusion', () => { + // 1. Vector-only hit: nearest to a page that has NO lexical match for the + // query term — it appears via the vector arm, and a pure-lexical run does + // not return it. Mutation (a): drop the vector arm -> this reddens. + it('#530-1 a vector-only hit appears; a pure-lexical run would not return it', async () => { + const s = (await createSpace(db, workspaceId)).id; + const page = await insertPage({ + title: 'нейтральный вект заголовок', + textContent: 'содержимое без искомого термина', + spaceId: s, + }); + await plantEmbedding(page, conceptVec(1), ACTIVE_FP, s); + + const svc = buildService({ embedVector: conceptVec(1), userSpaceIds: [s] }); + const res = await search(svc, { query: 'квазонимикс', spaceId: s }); + expect(res.items.map((i: any) => i.id)).toContain(page); + expect(res.semantic.available).toBe(true); + expect(res.semantic.state).toBe('full'); + + // Pure-lexical (default degrade): the vector-only page is absent. + const lex = buildService({ userSpaceIds: [s] }); + const res2 = await search(lex, { query: 'квазонимикс', spaceId: s }); + expect(res2.items.map((i: any) => i.id)).not.toContain(page); + expect(res2.semantic.available).toBe(false); + }); + + // 2. Sidecar down: embedQuery throws -> results = the lexical set; semantic + // unavailable; the fixed 'search.semantic.degraded' event is logged. + it('#530-2 sidecar-down degrades to lexical and logs search.semantic.degraded', async () => { + const s = (await createSpace(db, workspaceId)).id; + const lexHit = await insertPage({ + title: 'семдвамаркер лексический', + textContent: 'обычный текст', + spaceId: s, + }); + const warnSpy = jest + .spyOn(Logger.prototype, 'warn') + .mockImplementation(() => undefined as any); + try { + const svc = buildService({ embedThrows: 'degraded', userSpaceIds: [s] }); + const res = await search(svc, { query: 'семдвамаркер', spaceId: s }); + expect(res.items.map((i: any) => i.id)).toContain(lexHit); + expect(res.semantic.available).toBe(false); + expect(res.semantic.reason).toBe('degraded'); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('search.semantic.degraded'), + ); + } finally { + warnSpy.mockRestore(); + } + }); + + // 3. No provider: resolveEmbeddingProvider yields none -> lexical results, + // reason no-provider, no 500. + it('#530-3 no-provider yields lexical results with reason no-provider', async () => { + const s = (await createSpace(db, workspaceId)).id; + const hit = await insertPage({ + title: 'семтримаркер тема', + textContent: 'текст', + spaceId: s, + }); + const svc = buildService({ embedThrows: 'no-provider', userSpaceIds: [s] }); + const res = await search(svc, { query: 'семтримаркер', spaceId: s }); + expect(res.items.map((i: any) => i.id)).toContain(hit); + expect(res.semantic.available).toBe(false); + expect(res.semantic.reason).toBe('no-provider'); + }); + + // 4. Permission over the union: a restricted vector-only hit is dropped from + // items AND total; with the permission filter throwing the call rejects + // (fail-closed, never fail-open). Mutation (b): move filterAccessiblePageIds + // inside the semantic try/catch -> the reject assertion reddens. + it('#530-4 permission filters a vector-only hit from items AND total (fail-closed)', async () => { + const s = (await createSpace(db, workspaceId)).id; + const visibleLex = await insertPage({ + title: 'семчетмаркер видимый', + textContent: 'доступный текст', + spaceId: s, + }); + const hiddenVec = await insertPage({ + title: 'скрытая вект страница', + textContent: 'содержимое без искомого термина', + spaceId: s, + }); + await plantEmbedding(hiddenVec, conceptVec(4), ACTIVE_FP, s); + + // Query hits visibleLex lexically; hiddenVec only via the vector arm. + const svc = buildService({ + embedVector: conceptVec(4), + accessibleIds: [visibleLex], + userSpaceIds: [s], + }); + const res = await search(svc, { query: 'семчетмаркер', spaceId: s }); + const ids = res.items.map((i: any) => i.id); + expect(ids).toContain(visibleLex); + expect(ids).not.toContain(hiddenVec); + // The hidden vector hit does not leak into total either. + expect(res.total).toBe(1); + expect(ids).toHaveLength(res.total); + + // A permission-query error PROPAGATES (never a fail-open empty result). + const boom = buildService({ + embedVector: conceptVec(4), + filterThrows: true, + userSpaceIds: [s], + }); + await expect( + search(boom, { query: 'семчетмаркер', spaceId: s }), + ).rejects.toThrow(/permission query failed/); + }); + + // 5. Hung sidecar under a short embed timeout: graceful degrade (never a 500). + it('#530-5 a hung sidecar under a short timeout degrades gracefully', async () => { + const s = (await createSpace(db, workspaceId)).id; + const hit = await insertPage({ + title: 'семпятьмаркер тема', + textContent: 'текст', + spaceId: s, + }); + process.env.SEARCH_EMBED_TIMEOUT_MS = '1'; + try { + const svc = buildService({ embedThrows: 'degraded', userSpaceIds: [s] }); + const res = await search(svc, { query: 'семпятьмаркер', spaceId: s }); + expect(res.items.map((i: any) => i.id)).toContain(hit); + expect(res.semantic.available).toBe(false); + expect(res.semantic.reason).toBe('degraded'); + } finally { + delete process.env.SEARCH_EMBED_TIMEOUT_MS; + } + }); + + // 6. A page matched BOTH lexically and by vector is fused (de-duped), not + // returned twice — the agg CTE collapses it to one row. + it('#530-6 a page matched lexically AND by vector appears once', async () => { + const s = (await createSpace(db, workspaceId)).id; + const both = await insertPage({ + title: 'семшестьмаркер общий', + textContent: 'и лексика и вектор', + spaceId: s, + }); + await plantEmbedding(both, conceptVec(6), ACTIVE_FP, s); + const svc = buildService({ embedVector: conceptVec(6), userSpaceIds: [s] }); + const res = await search(svc, { query: 'семшестьмаркер', spaceId: s }); + const ids = res.items.map((i: any) => i.id); + expect(ids.filter((id: string) => id === both)).toHaveLength(1); + expect(res.total).toBe(1); + }); + + // 7. Fingerprint isolation: a planted row under a DIFFERENT fingerprint is not + // a vector candidate for the active-fingerprint query (guards mixing + // generations). It only appears if it also matches lexically (it does not). + it('#530-7 a stale-fingerprint vector row is not fused into results', async () => { + const s = (await createSpace(db, workspaceId)).id; + const stale = await insertPage({ + title: 'нейтральный семь заголовок', + textContent: 'без искомого термина совсем', + spaceId: s, + }); + // Same concept vector as the query, but an OLD generation fingerprint. + await plantEmbedding(stale, conceptVec(7), 'fp-stale-old', s); + const svc = buildService({ embedVector: conceptVec(7), userSpaceIds: [s] }); + const res = await search(svc, { query: 'квазонимиксseven', spaceId: s }); + expect(res.items.map((i: any) => i.id)).not.toContain(stale); + }); + + // 8. Stale embedding space_id (moved page): a page currently in space B whose + // embedding rows still carry space A (movePageToSpace does NOT reindex) is + // STILL a vector hit when searching B — space scoping is enforced only by + // the pages join (pages.space_id, always current), NOT by a stale + // page_embeddings.space_id. Guards against re-adding that predicate (which + // would wrongly drop the moved page's vector-only hit). + it('#530-8 a vector hit with a STALE embedding space_id (moved page) is still returned', async () => { + const newSpace = (await createSpace(db, workspaceId)).id; + const oldSpace = (await createSpace(db, workspaceId)).id; + // The page currently lives in newSpace (as after a move into newSpace). + const page = await insertPage({ + title: 'перемещённая вект страница', + textContent: 'содержимое без искомого термина', + spaceId: newSpace, + }); + // Its embedding was stamped with the OLD space and never reindexed. + await plantEmbedding(page, conceptVec(8), ACTIVE_FP, oldSpace); + const svc = buildService({ + embedVector: conceptVec(8), + userSpaceIds: [newSpace], + }); + const res = await search(svc, { + query: 'квазонимиксeight', + spaceId: newSpace, + }); + // Survives despite page_embeddings.space_id (oldSpace) != pages.space_id. + expect(res.items.map((i: any) => i.id)).toContain(page); + expect(res.semantic.available).toBe(true); + }); + + // 9. Statement-timeout safety net: a pathologically slow vector scan is + // cancelled (SQLSTATE 57014) and search degrades to lexical-only — the + // lexical hit survives, the vector-only hit vanishes, no 500/hang, and the + // degraded event is logged. Mutation: make the fallback re-throw instead of + // degrading -> this test reds (the call rejects with 57014). + it('#530-9 a vector-query timeout degrades to lexical (57014), never 500', async () => { + const s = (await createSpace(db, workspaceId)).id; + const lexHit = await insertPage({ + title: 'семдевятьмаркер лексический', + textContent: 'обычный текст', + spaceId: s, + }); + const vecOnly = await insertPage({ + title: 'нейтральный девять заголовок', + textContent: 'без искомого термина', + spaceId: s, + }); + await plantEmbedding(vecOnly, conceptVec(9), ACTIVE_FP, s); + + const warnSpy = jest + .spyOn(Logger.prototype, 'warn') + .mockImplementation(() => undefined as any); + // Tiny per-statement timeout so the 0.5s sleeping arm is cancelled. + process.env.SEARCH_VECTOR_STATEMENT_TIMEOUT_MS = '50'; + try { + const svc = buildService({ + embedVector: conceptVec(9), + slowVectorArm: true, + userSpaceIds: [s], + }); + const res = await search(svc, { query: 'семдевятьмаркер', spaceId: s }); + // Lexical result survives; the vector-only page does NOT (arm cancelled). + const ids = res.items.map((i: any) => i.id); + expect(ids).toContain(lexHit); + expect(ids).not.toContain(vecOnly); + expect(res.semantic.available).toBe(false); + expect(res.semantic.reason).toBe('degraded'); + expect(warnSpy).toHaveBeenCalledWith( + expect.stringContaining('search.semantic.degraded'), + ); + } finally { + delete process.env.SEARCH_VECTOR_STATEMENT_TIMEOUT_MS; + warnSpy.mockRestore(); + } + }); + + // 10. SET LOCAL scoping: the search's statement timeout must NOT leak onto a + // later query on the same pooled connection. After a search that set a + // tiny timeout, a deliberately-slow standalone query still completes. + it('#530-10 the per-query statement timeout does not leak to later queries', async () => { + const s = (await createSpace(db, workspaceId)).id; + await insertPage({ + title: 'семдесятьмаркер тема', + textContent: 'текст', + spaceId: s, + }); + process.env.SEARCH_VECTOR_STATEMENT_TIMEOUT_MS = '50'; + try { + const svc = buildService({ + embedVector: conceptVec(10), + slowVectorArm: true, + userSpaceIds: [s], + }); + // This search trips + resets the local timeout (via SET LOCAL semantics). + await search(svc, { query: 'семдесятьмаркер', spaceId: s }); + // A subsequent query that sleeps 200ms must NOT be cancelled — proving the + // 50ms search timeout did not persist on the connection (it would raise + // 57014 if it had leaked). SET LOCAL auto-resets at the search tx end. + await expect(sql`SELECT pg_sleep(0.2)`.execute(db)).resolves.toBeDefined(); + } finally { + delete process.env.SEARCH_VECTOR_STATEMENT_TIMEOUT_MS; + } + }); + + // 11. COMMIT-path no-leak (locks is_local=true). #530-10 only covers the + // ROLLBACK path (a timed-out search rolls back — a plain session SET would + // ALSO be undone on rollback, so it would NOT catch is_local true->false). + // Here a NORMAL, FAST semantic search COMMITs its transaction; a leaked + // session-level statement_timeout would then persist on the pooled + // connection and cancel later queries. We prove it does NOT: after the + // committing search (bounded at a small 100ms), a 300ms query succeeds. + // Because postgres.js pools, we run several sequential slow queries so at + // least one reuses the connection the committed search ran on (which is + // where a leaked timeout would live); ALL must succeed. Mutation: flip + // is_local true->false -> the committed 100ms timeout leaks and the + // follow-up pg_sleep is cancelled (57014), reddening this test. + it('#530-11 a COMMITTED search does not leak its statement timeout (is_local)', async () => { + const s = (await createSpace(db, workspaceId)).id; + const page = await insertPage({ + title: 'семодиннадцать вект страница', + textContent: 'содержимое без искомого термина', + spaceId: s, + }); + await plantEmbedding(page, conceptVec(11), ACTIVE_FP, s); + // Small bound: the fast vector arm still completes (<100ms) so the search + // COMMITs, but small enough that a LEAKED timeout would cancel the 300ms + // follow-ups below. + process.env.SEARCH_VECTOR_STATEMENT_TIMEOUT_MS = '100'; + try { + const svc = buildService({ + embedVector: conceptVec(11), + userSpaceIds: [s], + }); + const res = await search(svc, { query: 'семодиннадцать', spaceId: s }); + // The (committing) vector path actually ran. + expect(res.semantic.available).toBe(true); + // Each 300ms query must SUCCEED — a leaked 100ms timeout would cancel the + // one that reuses the committed search's connection. 8 sweeps cover the + // whole pool (max 5 conns), so a leak on any connection is caught. + for (let i = 0; i < 8; i++) { + await expect( + sql`SELECT pg_sleep(0.3)`.execute(db), + ).resolves.toBeDefined(); + } + } finally { + delete process.env.SEARCH_VECTOR_STATEMENT_TIMEOUT_MS; + } + }); + }); }); diff --git a/docker-compose.yml b/docker-compose.yml index ded83719..fa7e2ddd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,11 +4,26 @@ services: depends_on: - db - redis + - embeddings environment: APP_URL: 'http://localhost:3000' APP_SECRET: 'REPLACE_WITH_LONG_SECRET' DATABASE_URL: 'postgresql://docmost:STRONG_DB_PASSWORD@db:5432/docmost' REDIS_URL: 'redis://redis:6379' + # #530 semantic search: the GLOBAL embedding provider (the `embeddings` TEI + # sidecar below). A workspace that configures its own embedding provider + # overrides this. TEI is OpenAI-compatible, so the endpoint is /v1. + EMBEDDING_ENDPOINT: 'http://embeddings:80/v1' + EMBEDDING_MODEL: 'intfloat/multilingual-e5-small' + EMBEDDING_API_KEY: 'unused' + EMBEDDING_DIMENSIONS: '384' + # MUST match the --revision the sidecar pins (see EMBEDDING_REVISION in + # .env). A revision bump changes the embedding fingerprint (PR-2 swaps + # generations); keep the two in lockstep. + EMBEDDING_REVISION: '${EMBEDDING_REVISION}' + # e5 models require these input prefixes; empty them for a non-e5 model. + EMBEDDING_QUERY_PREFIX: 'query: ' + EMBEDDING_DOC_PREFIX: 'passage: ' ports: - "3000:3000" restart: unless-stopped @@ -39,7 +54,33 @@ services: volumes: - redis_data:/data + # #530 Text Embeddings Inference (TEI) sidecar — the GLOBAL embedding provider + # for semantic search. OpenAI-compatible, reached only on the internal network + # (no published port). The model + revision are PINNED so the embedding + # fingerprint is stable; set EMBEDDING_REVISION to a real commit sha in .env. + embeddings: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.9 + command: + - "--model-id" + - "intfloat/multilingual-e5-small" + - "--revision" + - "${EMBEDDING_REVISION}" + restart: unless-stopped + # Cache downloaded model weights so a restart does not re-download them. + volumes: + - tei-models:/data + # GET /health is TEI's readiness probe. (Drop this block if your TEI image + # variant ships without curl.) + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://localhost:80/health || exit 1"] + interval: 30s + timeout: 5s + retries: 5 + # Model download on first boot can be slow; don't flap as unhealthy meanwhile. + start_period: 120s + volumes: docmost: db_data: redis_data: + tei-models: