Compare commits
54 Commits
dd64c2ea05
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fcf1fdec89 | ||
|
|
a7f8ee04b3 | ||
|
|
378d8b676b | ||
| 580f7bd5bb | |||
|
|
b538c729c3 | ||
| e3b23e0d26 | |||
|
|
b392219659 | ||
|
|
ba5cd02439 | ||
|
|
df50f23d58 | ||
|
|
eb5c8e6611 | ||
|
|
d32ad73158 | ||
|
|
acf2241e23 | ||
|
|
cb61274187 | ||
|
|
1d610b3a62 | ||
|
|
6bb9dfdc86 | ||
|
|
770ba70541 | ||
|
|
3d47c306fa | ||
|
|
c919d4f636 | ||
|
|
c4807022f2 | ||
|
|
00ca4ff3d6 | ||
|
|
ef7d04d1e7 | ||
|
|
5b59a70e3f | ||
|
|
eafd15f0ef | ||
|
|
fbdb8aa16c | ||
|
|
9b61024b95 | ||
|
|
63c26042ba | ||
|
|
2644fe6a83 | ||
|
|
993f884e64 | ||
|
|
2f058a6e40 | ||
|
|
3ddc329bba | ||
|
|
ed3b65c36b | ||
|
|
de115ade1e | ||
|
|
364838d0b2 | ||
|
|
aa7a115f66 | ||
|
|
30c358a2f8 | ||
|
|
ea61c96a7c | ||
|
|
f80276d41a | ||
|
|
8218c1a8ef | ||
|
|
d7e7489654 | ||
|
|
8f1af676ba | ||
|
|
34c5b557ef | ||
|
|
59f0c8b22d | ||
|
|
77ccc596ea | ||
|
|
e536c6f9a9 | ||
|
|
fdaf20ca7b | ||
|
|
47a2ae420b | ||
|
|
1cfad1f6fb | ||
|
|
a766672574 | ||
|
|
5e8cb628f0 | ||
|
|
8413185a1d | ||
|
|
8fee6a86c2 | ||
|
|
ae6faf3abc | ||
|
|
e7b719bbb8 | ||
|
|
99d0cb8773 |
@@ -187,3 +187,11 @@ MCP_DOCMOST_PASSWORD=
|
||||
# Per-request output-token ceiling for the anonymous assistant (default: 512).
|
||||
# Worst-case output per accepted call = agent steps (5) × this value.
|
||||
# SHARE_AI_MAX_OUTPUT_TOKENS=512
|
||||
#
|
||||
# Second cost backstop: a cluster-wide per-workspace rolling-DAY token budget
|
||||
# (input re-sent per step + output, summed across every accepted turn). The
|
||||
# hourly request cap above bounds how MANY calls run, not how expensive each is,
|
||||
# so this caps the owner's actual provider bill directly. Like the request cap it
|
||||
# FAILS CLOSED if Redis is unavailable (default: 1,000,000 tokens per workspace
|
||||
# per rolling day).
|
||||
# SHARE_AI_WORKSPACE_TOKEN_BUDGET_PER_DAY=1000000
|
||||
|
||||
157
.github/workflows/develop.yml
vendored
157
.github/workflows/develop.yml
vendored
@@ -56,3 +56,160 @@ jobs:
|
||||
tags: ${{ env.IMAGE }}:develop
|
||||
cache-from: type=gha,scope=develop-amd64
|
||||
cache-to: type=gha,scope=develop-amd64,mode=max,ignore-error=true
|
||||
|
||||
# e2e jobs run on every develop push but DO NOT gate the build/publish above:
|
||||
# `build` stays `needs: test` only, so the :develop image still ships even if
|
||||
# e2e fails. A failing e2e job turns the run red and triggers GitHub's email
|
||||
# to the pusher — that red run + email is the intended notification, not a
|
||||
# deploy block.
|
||||
e2e-server:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DATABASE_URL: postgresql://docmost:docmost@localhost:5432/docmost
|
||||
REDIS_URL: redis://localhost:6379
|
||||
APP_SECRET: ci-e2e-secret-change-me-min-32-characters
|
||||
APP_URL: http://localhost:3000
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg18
|
||||
env:
|
||||
POSTGRES_DB: docmost
|
||||
POSTGRES_USER: docmost
|
||||
POSTGRES_PASSWORD: docmost
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U docmost"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
redis:
|
||||
image: redis:7
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build editor-ext
|
||||
run: pnpm --filter @docmost/editor-ext build
|
||||
|
||||
- name: Run migrations
|
||||
run: pnpm --filter ./apps/server migration:latest
|
||||
|
||||
- name: Run server e2e
|
||||
run: pnpm --filter ./apps/server test:e2e
|
||||
|
||||
# Same rationale as e2e-server: this job is intentionally NOT in
|
||||
# `build.needs`. Deploy of the :develop image must not be blocked by e2e;
|
||||
# a red run plus GitHub's email to the pusher is the notification mechanism.
|
||||
e2e-mcp:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DATABASE_URL: postgresql://docmost:docmost@localhost:5432/docmost
|
||||
REDIS_URL: redis://localhost:6379
|
||||
APP_SECRET: ci-e2e-secret-change-me-min-32-characters
|
||||
APP_URL: http://localhost:3000
|
||||
NODE_ENV: production
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg18
|
||||
env:
|
||||
POSTGRES_DB: docmost
|
||||
POSTGRES_USER: docmost
|
||||
POSTGRES_PASSWORD: docmost
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U docmost"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
redis:
|
||||
image: redis:7
|
||||
ports:
|
||||
- 6379:6379
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Set up pnpm
|
||||
uses: pnpm/action-setup@v4
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build editor-ext
|
||||
run: pnpm --filter @docmost/editor-ext build
|
||||
|
||||
- name: Build server
|
||||
run: pnpm server:build
|
||||
|
||||
- name: Build mcp
|
||||
run: pnpm --filter @docmost/mcp build
|
||||
|
||||
- name: Run migrations
|
||||
run: pnpm --filter ./apps/server migration:latest
|
||||
|
||||
- name: Start server (prod)
|
||||
# Capture stdout/stderr so a start-up crash (bind error, stack trace,
|
||||
# migration mismatch) is diagnosable; without this the only signal is
|
||||
# the generic health-loop timeout below, ~120s later.
|
||||
run: pnpm --filter ./apps/server start:prod > /tmp/server.log 2>&1 &
|
||||
|
||||
- name: Wait for server health
|
||||
run: |
|
||||
for i in $(seq 1 60); do
|
||||
if curl -fsS http://localhost:3000/api/health > /dev/null; then
|
||||
echo "Server is healthy"
|
||||
exit 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "Server did not become healthy in time"
|
||||
exit 1
|
||||
|
||||
- name: Dump server log on failure
|
||||
if: failure()
|
||||
run: cat /tmp/server.log || true
|
||||
|
||||
- name: Seed admin
|
||||
run: |
|
||||
curl -fsS -X POST http://localhost:3000/api/auth/setup \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"name":"E2E","email":"e2e@example.com","password":"E2ePassword123","workspaceName":"E2E"}'
|
||||
|
||||
- name: Run mcp e2e
|
||||
env:
|
||||
DOCMOST_API_URL: http://localhost:3000/api
|
||||
DOCMOST_EMAIL: e2e@example.com
|
||||
DOCMOST_PASSWORD: E2ePassword123
|
||||
run: pnpm --filter @docmost/mcp test:e2e
|
||||
|
||||
2
.github/workflows/test.yml
vendored
2
.github/workflows/test.yml
vendored
@@ -26,7 +26,7 @@ jobs:
|
||||
# TEST_*_URL overrides are needed.
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
image: pgvector/pgvector:pg18
|
||||
env:
|
||||
POSTGRES_USER: docmost
|
||||
POSTGRES_PASSWORD: docmost_dev_pw
|
||||
|
||||
78
CHANGELOG.md
78
CHANGELOG.md
@@ -10,12 +10,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.94.0] - 2026-06-26
|
||||
|
||||
This release makes AI chat durable and fast: assistant turns are persisted to
|
||||
the database step by step and exported server-side, the desktop app no longer
|
||||
freezes at 100% CPU on long agent runs, and MCP writes are badged with
|
||||
unspoofable AI attribution. It also reworks footnotes (Pandoc-style reuse and
|
||||
per-reference back-links), hardens page moves and duplication against cycles
|
||||
and lost edits, and caps the anonymous public-share assistant with a
|
||||
per-workspace rolling-day token budget.
|
||||
|
||||
### Added
|
||||
|
||||
- **Persistent AI-chat history as the source of truth + server-side export.**
|
||||
An assistant turn is now persisted to the database step by step: the row is
|
||||
inserted upfront as `streaming` and updated as each agent step finishes, then
|
||||
finalized once to `completed`/`error`/`aborted`. A process that dies mid-turn
|
||||
keeps every finished step, and a startup sweep flips any dangling `streaming`
|
||||
row (untouched for 10 minutes) to `aborted`. Chat "Copy" now exports
|
||||
server-side from these rows (`POST /ai-chat/export`) rather than from live
|
||||
client state, so the export is identical whether a chat is freshly streaming,
|
||||
just switched to, or reloaded — and is available from the first turn of a new
|
||||
chat. (#183, #174)
|
||||
|
||||
- **AI-agent attribution for MCP writes.** Comments (and pages) created through
|
||||
the MCP endpoint by a dedicated agent account are now badged as "AI", with
|
||||
unspoofable provenance derived from a per-user `is_agent` flag (not from the
|
||||
request body). **Operator setup:** use a *dedicated* service account for the
|
||||
request body). **Operator setup:** use a _dedicated_ service account for the
|
||||
MCP fallback and set the flag with SQL —
|
||||
`UPDATE users SET is_agent = true WHERE email = '<mcp-account>'`. Never flag a
|
||||
human or shared account, or its normal edits get mis-attributed as AI. See the
|
||||
@@ -32,6 +53,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
OpenRouter, etc.; `openai` uses the official provider (real-OpenAI
|
||||
reasoning-model request shaping). Chosen explicitly rather than inferred from
|
||||
the base URL, since a custom URL can front real OpenAI too. (#175, #177)
|
||||
- **Per-MCP-server instructions in the agent prompt.** Each external MCP server
|
||||
now has an admin-authored `instructions` field ("how/when to use this server's
|
||||
tools") that is injected into the agent's system prompt next to that server's
|
||||
tool descriptions. Trusted text, rendered inside the prompt safety sandwich;
|
||||
shown only for a server that actually connected and contributed ≥1 callable
|
||||
tool. (#180)
|
||||
- **Footnote multi-backlinks.** A footnote referenced more than once now shows a
|
||||
back-link per reference (↩ a b c …), each scrolling to its own occurrence, like
|
||||
Pandoc/Wikipedia; a single-reference footnote keeps the plain ↩. (#168)
|
||||
|
||||
### Changed
|
||||
|
||||
@@ -58,6 +88,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **AI chat: the desktop app no longer freezes at 100% CPU on long agent runs.**
|
||||
`useChat` re-rendered on every streamed token and `MessageItem`/`ReasoningBlock`
|
||||
re-parsed the whole transcript markdown (marked + DOMPurify) on every delta, so
|
||||
per-turn work grew quadratically and saturated the main thread. The stream is now
|
||||
throttled (`experimental_throttle`) to ~20 Hz and each finalized message row /
|
||||
markdown part / reasoning block is memoized, so a long turn no longer re-parses
|
||||
already-finished content. (#182)
|
||||
- **Editor: caret/selection landed on the wrong line when clicking inside code
|
||||
blocks and footnotes.** The affected NodeViews rendered their non-editable
|
||||
chrome (language menu, footnotes heading, footnote number marker) before the
|
||||
@@ -67,6 +104,42 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
are nudged after a paste to refresh stale hit-testing geometry. The caret
|
||||
symptom is macOS-specific and was confirmed manually on macOS; the automated
|
||||
guard pins the DOM-order invariant, not the caret behavior itself. (#146, #147)
|
||||
- **AI chat: the live token counter now ticks between agent steps.** During a
|
||||
multi-step turn the header token badge (and the "Thinking… · N tokens" line)
|
||||
no longer froze on the previous step's authoritative usage; the current step's
|
||||
estimate is combined per-component with `max`, so the count rises smoothly and
|
||||
never jumps backwards. (#163)
|
||||
- **AI chat: "New chat" during a streaming first turn now resets the whole
|
||||
chat, not just the role badge.** Starting a new chat mid-stream cleared the
|
||||
header but left the in-flight turn's messages behind, so the fresh chat opened
|
||||
pre-populated with the previous conversation; it now fully resets. (#161)
|
||||
- **AI chat: a dropped tool argument now yields an actionable error.** When the
|
||||
model omitted a required parameter (typically `pageId`) in a parallel/batch
|
||||
tool call, the assistant forwarded zod's raw "expected string, received
|
||||
undefined" text; tool inputs now return a message naming each missing/invalid
|
||||
parameter (the JSON Schema contract is unchanged and nothing is backfilled).
|
||||
(#190)
|
||||
- **Page move: cycle checks are now atomic and depth-bounded.** Moving a page
|
||||
under one of its own descendants is rejected in the same transaction as the
|
||||
update (closing a TOCTOU window where two concurrent A→B / B→A moves could
|
||||
form a cycle), and the recursive tree-traversal CTEs carry a cycle/depth guard
|
||||
so a pre-existing cycle can no longer spin a query. (#207)
|
||||
- **Page/editor robustness batch.** Duplicating a page now copies shared
|
||||
attachments for every referencing page (not just the first); colliding block
|
||||
ids are de-duplicated on import/normalize so MCP addressed edits can't hit the
|
||||
wrong node; transient collab store failures are retried so autosave edits
|
||||
aren't lost; and an out-of-order tree move no longer drops the moved subtree.
|
||||
(#206)
|
||||
|
||||
### Security
|
||||
|
||||
- **Public share AI: per-workspace rolling-day token budget.** The anonymous
|
||||
share assistant now caps a workspace's actual token spend (input + output,
|
||||
summed across every accepted turn) over a trailing day, on top of the hourly
|
||||
request cap — so a caller who evades the per-IP throttle still cannot run up
|
||||
the owner's provider bill without bound. Cluster-wide via Redis and FAILS
|
||||
CLOSED if Redis is down; default 1,000,000 tokens/day, overridable via
|
||||
`SHARE_AI_WORKSPACE_TOKEN_BUDGET_PER_DAY`. (#159)
|
||||
|
||||
## [0.93.0] - 2026-06-21
|
||||
|
||||
@@ -150,8 +223,7 @@ embeds — plus a large batch of security hardening and test coverage.
|
||||
- Page templates: import `ThrottleModule` so collab boots, never strand an
|
||||
in-flight page-embed id, and add defense-in-depth workspace checks.
|
||||
- Pages: `movePage` cycle guard with no phantom `PAGE_MOVED` event.
|
||||
- Import: surface the real error cause from `/pages/import` instead of a generic
|
||||
400.
|
||||
- Import: surface the real error cause from `/pages/import` instead of a generic 400.
|
||||
|
||||
### Security
|
||||
|
||||
|
||||
@@ -114,7 +114,7 @@ community feature, with no enterprise license. Open it from the page header; the
|
||||
- 🔭 **Viewer comments** — let read-only viewers leave comments.
|
||||
- 🔭 **Password-protected pages** — protect individual pages / shares with a password.
|
||||
- 🔭 **Windows / Linux app** — native desktop app for Windows and Linux.
|
||||
- 🔭 **Mobile app** — mobile apps (iOS first, Android to follow), reusing the existing responsive web UI and editor via a Capacitor wrapper, with offline planned for later. See [docs/mobile-app-plan.md](docs/mobile-app-plan.md).
|
||||
- 🔭 **Mobile app** — mobile apps (iOS first, Android to follow), reusing the existing responsive web UI and editor via a Capacitor wrapper, with offline planned for later. See [issue #195](https://gitea.vvzvlad.xyz/vvzvlad/gitmost/issues/195).
|
||||
- 🔭 **Offline mode** — offline sync & PWA support.
|
||||
- 🔭 **Editor & UX improvements** — blocks inside tables (lists, to-do items), column layout, additional heading levels, highlight blocks, custom emoji in callouts, floating images, anchor links for page mentions, toggles (shared-page width, aside/sidebar, spellcheck, ligatures), sanitized space-tree export, and mentions in breadcrumbs.
|
||||
|
||||
|
||||
@@ -115,7 +115,7 @@ real-time-коллаборации Docmost, поэтому запись нико
|
||||
- 🔭 **Комментарии зрителей** — возможность комментировать для пользователей с доступом только на чтение.
|
||||
- 🔭 **Защищённые паролем страницы** — защита отдельных страниц / шар паролем.
|
||||
- 🔭 **Приложение для Windows / Linux** — нативное десктоп-приложение для Windows и Linux.
|
||||
- 🔭 **Мобильное приложение** — мобильные приложения (iOS обязательно, Android как пойдёт) на базе существующей адаптивной веб-версии и редактора через обёртку Capacitor; оффлайн запланирован на будущее. См. [docs/mobile-app-plan.md](docs/mobile-app-plan.md).
|
||||
- 🔭 **Мобильное приложение** — мобильные приложения (iOS обязательно, Android как пойдёт) на базе существующей адаптивной веб-версии и редактора через обёртку Capacitor; оффлайн запланирован на будущее. См. [issue #195](https://gitea.vvzvlad.xyz/vvzvlad/gitmost/issues/195).
|
||||
- 🔭 **Офлайн-режим** — офлайн-синхронизация и поддержка PWA.
|
||||
- 🔭 **Улучшения редактора и UX** — блоки внутри таблиц (списки, чек-листы), колоночная вёрстка, дополнительные уровни заголовков, highlight-блоки, кастомные эмодзи в callout-ах, плавающие изображения, anchor-ссылки на упоминания страниц, тоглы (ширина шары, aside/сайдбар, spellcheck, лигатуры), санитизация экспорта дерева спейса и mentions в хлебных крошках.
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "client",
|
||||
"private": true,
|
||||
"version": "0.93.0",
|
||||
"version": "0.94.0",
|
||||
"scripts": {
|
||||
"dev": "node scripts/copy-vad-assets.mjs && vite",
|
||||
"build": "node scripts/copy-vad-assets.mjs && tsc && vite build",
|
||||
|
||||
@@ -258,6 +258,7 @@
|
||||
"Copy to space": "Copy to space",
|
||||
"Copy chat": "Copy chat",
|
||||
"Copied": "Copied",
|
||||
"Failed to export chat": "Failed to export chat",
|
||||
"Duplicate": "Duplicate",
|
||||
"Select a user": "Select a user",
|
||||
"Select a group": "Select a group",
|
||||
@@ -714,6 +715,8 @@
|
||||
"Test": "Test",
|
||||
"Available tools": "Available tools",
|
||||
"No tools available": "No tools available",
|
||||
"Failed": "Failed",
|
||||
"OK · {{n}}": "OK · {{n}}",
|
||||
"Created successfully": "Created successfully",
|
||||
"Deleted successfully": "Deleted successfully",
|
||||
"Clear": "Clear",
|
||||
@@ -1078,6 +1081,8 @@
|
||||
"Undo": "Undo",
|
||||
"Redo": "Redo",
|
||||
"Backlinks": "Backlinks",
|
||||
"Back to references": "Back to references",
|
||||
"Back to reference {{label}}": "Back to reference {{label}}",
|
||||
"Last updated by": "Last updated by",
|
||||
"Last updated": "Last updated",
|
||||
"Stats": "Stats",
|
||||
@@ -1164,8 +1169,9 @@
|
||||
"Pick an agent role whose persona the public assistant adopts. The safety rules always still apply.": "Pick an agent role whose persona the public assistant adopts. The safety rules always still apply.",
|
||||
"Built-in assistant persona": "Built-in assistant persona",
|
||||
"Minimize": "Minimize",
|
||||
"Current context size": "Current context size",
|
||||
"Tokens generated this turn": "Tokens generated this turn",
|
||||
"Context size / model limit": "Context size / model limit",
|
||||
"Context window (tokens)": "Context window (tokens)",
|
||||
"Shown as used / total in the chat header. Leave empty to hide the limit.": "Shown as used / total in the chat header. Leave empty to hide the limit.",
|
||||
"AI agent": "AI agent",
|
||||
"Take a look at the current document": "Take a look at the current document",
|
||||
"AI agent is typing…": "AI agent is typing…",
|
||||
|
||||
@@ -257,6 +257,7 @@
|
||||
"Copy": "Копировать",
|
||||
"Copy to space": "Копировать в пространство",
|
||||
"Copied": "Скопировано",
|
||||
"Failed to export chat": "Не удалось экспортировать чат",
|
||||
"Duplicate": "Дублировать",
|
||||
"Select a user": "Выберите пользователя",
|
||||
"Select a group": "Выберите группу",
|
||||
@@ -703,13 +704,19 @@
|
||||
"Ask the AI agent…": "Спросите AI-агента…",
|
||||
"Copy chat": "Копировать чат",
|
||||
"Created successfully": "Успешно создано",
|
||||
"Current context size": "Текущий размер контекста",
|
||||
"Tokens generated this turn": "Токенов сгенерировано за ход",
|
||||
"Context size / model limit": "Размер контекста / лимит модели",
|
||||
"Context window (tokens)": "Окно контекста (токены)",
|
||||
"Shown as used / total in the chat header. Leave empty to hide the limit.": "Показывается в шапке чата как использовано / всего. Пусто — лимит скрыт.",
|
||||
"Delete this chat?": "Удалить этот чат?",
|
||||
"Deleted successfully": "Успешно удалено",
|
||||
"Edited by AI agent on behalf of {{name}}": "Отредактировано AI-агентом от имени {{name}}",
|
||||
"Failed to delete chat": "Не удалось удалить чат",
|
||||
"Failed to rename chat": "Не удалось переименовать чат",
|
||||
"Failed": "Ошибка",
|
||||
"OK · {{n}}": "OK · {{n}}",
|
||||
"Test": "Тест",
|
||||
"No tools available": "Инструменты недоступны",
|
||||
"Available tools": "Доступные инструменты",
|
||||
"Minimize": "Свернуть",
|
||||
"No chats yet.": "Чатов пока нет.",
|
||||
"Send": "Отправить",
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import { type UIMessage } from "@ai-sdk/react";
|
||||
import { Group, Loader, Tooltip } from "@mantine/core";
|
||||
import {
|
||||
IconArrowsDiagonal,
|
||||
@@ -40,12 +39,13 @@ import {
|
||||
} from "@/features/ai-chat/queries/ai-chat-query.ts";
|
||||
import ConversationList from "@/features/ai-chat/components/conversation-list.tsx";
|
||||
import ChatThread from "@/features/ai-chat/components/chat-thread.tsx";
|
||||
import { buildChatMarkdown } from "@/features/ai-chat/utils/chat-markdown.ts";
|
||||
import { exportAiChat } from "@/features/ai-chat/services/ai-chat-service.ts";
|
||||
import { useChatSession } from "@/features/ai-chat/hooks/use-chat-session.ts";
|
||||
import {
|
||||
shouldCollapseOnOutsidePointer,
|
||||
isHeaderClick,
|
||||
} from "@/features/ai-chat/utils/collapse-helpers.ts";
|
||||
import { selectContextBadge } from "@/features/ai-chat/utils/context-badge.ts";
|
||||
import { useClipboard } from "@/hooks/use-clipboard";
|
||||
import { notifications } from "@mantine/notifications";
|
||||
import classes from "@/features/ai-chat/components/ai-chat-window.module.css";
|
||||
@@ -121,7 +121,7 @@ function clampGeom(g: {
|
||||
* ported from the GitmostAgent.jsx design.
|
||||
*/
|
||||
export default function AiChatWindow() {
|
||||
const { t } = useTranslation();
|
||||
const { t, i18n } = useTranslation();
|
||||
const clipboard = useClipboard({ timeout: 500 });
|
||||
const queryClient = useQueryClient();
|
||||
const [windowOpen, setWindowOpen] = useAtom(aiChatWindowOpenAtom);
|
||||
@@ -162,31 +162,6 @@ export default function AiChatWindow() {
|
||||
const { data: messageRows, isLoading: messagesLoading } =
|
||||
useAiChatMessagesQuery(activeChatId ?? undefined);
|
||||
|
||||
// Live snapshot of the active thread's useChat state, kept up to date by
|
||||
// ChatThread. Lets the export include the in-progress (not-yet-persisted)
|
||||
// streaming turn. A ref avoids re-rendering this window on every token.
|
||||
const liveThreadRef = useRef<{
|
||||
messages: UIMessage[];
|
||||
isStreaming: boolean;
|
||||
banner: string | null;
|
||||
}>({
|
||||
messages: [],
|
||||
isStreaming: false,
|
||||
banner: null,
|
||||
});
|
||||
|
||||
// Live turn-token total (reasoning + output) for the in-flight turn, pushed up
|
||||
// (THROTTLED to ~8 Hz inside ChatThread) so the header badge ticks mid-stream.
|
||||
// `null` means no turn is in flight -> the badge falls back to the persisted
|
||||
// context size below.
|
||||
const [liveTurnTokens, setLiveTurnTokens] = useState<number | null>(null);
|
||||
// Whether the on-screen thread currently holds at least one message. Reported
|
||||
// reactively by ChatThread (the live snapshot lives in a non-reactive ref). This
|
||||
// lets the "Copy chat" button stay available for a brand-new, not-yet-persisted
|
||||
// chat whose first turn is in flight or was interrupted — that case has no
|
||||
// persisted rows yet, so a persisted-rows-only gate would hide the button (#174).
|
||||
const [hasLiveContent, setHasLiveContent] = useState(false);
|
||||
|
||||
// The page the user is currently viewing. AiChatWindow lives in a pathless
|
||||
// parent layout route, so useParams() can't see :pageSlug. Match the full
|
||||
// pathname against the authenticated page route instead so "the current page"
|
||||
@@ -213,7 +188,9 @@ export default function AiChatWindow() {
|
||||
const {
|
||||
threadKey,
|
||||
waitingForHistory,
|
||||
startFreshThread,
|
||||
onTurnFinished,
|
||||
onServerChatId,
|
||||
cancelPendingAdoption,
|
||||
} = useChatSession({
|
||||
activeChatId,
|
||||
@@ -234,12 +211,25 @@ export default function AiChatWindow() {
|
||||
// just-failed chat after they chose a fresh one.
|
||||
const startNewChat = useCallback((): void => {
|
||||
cancelPendingAdoption();
|
||||
// Force a fresh, empty thread UNCONDITIONALLY (#161). Pressing "New chat"
|
||||
// while a brand-new chat's first turn is still streaming leaves activeChatId
|
||||
// null (the real id is adopted only at turn end), so setActiveChatId(null)
|
||||
// alone is a no-op and the reconciler never remounts — the chat/stream/history
|
||||
// would persist and only the role badge would drop. This always remounts the
|
||||
// thread into a clean new chat.
|
||||
startFreshThread();
|
||||
setActiveChatId(null);
|
||||
setHistoryOpen(false);
|
||||
setDraft("");
|
||||
// Default the picker back to "Universal assistant" for the fresh chat.
|
||||
setSelectedRoleId(null);
|
||||
}, [cancelPendingAdoption, setActiveChatId, setDraft, setSelectedRoleId]);
|
||||
}, [
|
||||
cancelPendingAdoption,
|
||||
startFreshThread,
|
||||
setActiveChatId,
|
||||
setDraft,
|
||||
setSelectedRoleId,
|
||||
]);
|
||||
|
||||
const selectChat = useCallback(
|
||||
(chatId: string): void => {
|
||||
@@ -254,20 +244,19 @@ export default function AiChatWindow() {
|
||||
[cancelPendingAdoption, setActiveChatId, setDraft, setSelectedRoleId],
|
||||
);
|
||||
|
||||
// The active chat object (for its title) and an export gate: only enable the
|
||||
// export button when an existing chat with loaded persisted rows is active.
|
||||
// The active chat object (for its title) and an export gate. The export is now
|
||||
// SERVER-sourced (the DB is the single source of truth — #183): the assistant
|
||||
// row is persisted upfront + per step, so even a brand-new chat whose first
|
||||
// turn is streaming/interrupted has a server row to render. Enable the button
|
||||
// whenever a persisted chat is active (`activeChatId` is set). For a BRAND-NEW
|
||||
// chat that id is adopted EARLY — at the stream's `start` chunk via
|
||||
// onServerChatId (#174) — so the Copy button is available during the first
|
||||
// turn's stream, not only after it terminates.
|
||||
const activeChat = useMemo(
|
||||
() => chats?.items?.find((c) => c.id === activeChatId) ?? null,
|
||||
[chats, activeChatId],
|
||||
);
|
||||
// Export is available when there is anything to export: either persisted rows
|
||||
// for the active chat, OR a live on-screen thread with at least one message.
|
||||
// The live arm covers a brand-new chat whose first turn is streaming or was
|
||||
// interrupted before the server persisted any row (#174); the persisted arm is
|
||||
// the steady-state path for an already-saved chat (#160).
|
||||
const canExport =
|
||||
hasLiveContent ||
|
||||
(!!activeChatId && !!messageRows && messageRows.length > 0);
|
||||
const canExport = !!activeChatId;
|
||||
|
||||
// The role to display in the header and as the assistant's name. Prefer the
|
||||
// persisted role of an existing chat (chat-list JOIN); fall back to the role
|
||||
@@ -284,53 +273,21 @@ export default function AiChatWindow() {
|
||||
return picked ? { name: picked.name, emoji: picked.emoji } : null;
|
||||
}, [activeChat, enabledRoles, selectedRoleId]);
|
||||
|
||||
// Build a Markdown export from the already-loaded persisted rows (no network
|
||||
// call) and copy it to the clipboard. The "Copied" notification is the
|
||||
// feedback.
|
||||
const handleCopy = useCallback(() => {
|
||||
// Export gate. There must be SOMETHING to export — either a live on-screen
|
||||
// message or a persisted row. A brand-new chat whose first turn is streaming
|
||||
// or was interrupted has live messages but no persisted rows yet; it still
|
||||
// exports the on-screen thread WYSIWYG (#174). Only a truly empty chat (no
|
||||
// live messages and no rows) is non-exportable (the button is hidden too —
|
||||
// see `canExport`).
|
||||
const live = liveThreadRef.current;
|
||||
const hasRows = !!messageRows && messageRows.length > 0;
|
||||
if (live.messages.length === 0 && !hasRows) return;
|
||||
// WYSIWYG export: the live on-screen messages ARE the document (so a partial
|
||||
// reply from an interrupted turn — which never reached the persisted rows —
|
||||
// is exported just as it appears). The persisted rows enrich each live
|
||||
// message (token usage / error / timestamp) by id and serve as the fallback
|
||||
// when the live mirror is empty. The on-screen banner is appended too. See
|
||||
// issues #160 and #174. `chatId` may be null for a not-yet-saved chat — use a
|
||||
// placeholder so the header line still renders.
|
||||
const markdown = buildChatMarkdown({
|
||||
title: activeChat?.title ?? null,
|
||||
chatId: activeChatId ?? "unsaved",
|
||||
live: live.messages.map((m) => ({
|
||||
id: m.id,
|
||||
role: m.role,
|
||||
parts: (m.parts ?? []) as { type: string; text?: string }[],
|
||||
metadata: m.metadata as
|
||||
| {
|
||||
usage?: {
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
totalTokens?: number;
|
||||
reasoningTokens?: number;
|
||||
};
|
||||
error?: string;
|
||||
}
|
||||
| undefined,
|
||||
})),
|
||||
rows: messageRows,
|
||||
isStreaming: live.isStreaming,
|
||||
banner: live.banner,
|
||||
t,
|
||||
});
|
||||
clipboard.copy(markdown);
|
||||
notifications.show({ message: t("Copied") });
|
||||
}, [activeChatId, messageRows, activeChat, clipboard, t]);
|
||||
// Fetch the server-rendered Markdown export and copy it to the clipboard. The
|
||||
// server is the single source of truth (#183): it renders the transcript from
|
||||
// the persisted rows — including an interrupted turn's in-progress row — so the
|
||||
// export is identical whether the chat is freshly streaming, just switched to,
|
||||
// or reloaded. The `lang` of the active i18n drives the few localized labels.
|
||||
const handleCopy = useCallback(async () => {
|
||||
if (!activeChatId) return;
|
||||
try {
|
||||
const markdown = await exportAiChat(activeChatId, i18n.language);
|
||||
clipboard.copy(markdown);
|
||||
notifications.show({ message: t("Copied") });
|
||||
} catch {
|
||||
notifications.show({ message: t("Failed to export chat"), color: "red" });
|
||||
}
|
||||
}, [activeChatId, clipboard, t, i18n.language]);
|
||||
|
||||
// Current context size for the active chat: how much the conversation now
|
||||
// occupies in the model's context window — NOT the cumulative tokens spent.
|
||||
@@ -339,24 +296,19 @@ export default function AiChatWindow() {
|
||||
// shipped; older rows fall back to that turn's `usage` total. NOTE: reflects
|
||||
// PERSISTED rows (updates on chat open/switch); it does not tick live
|
||||
// mid-stream — acceptable for v1.
|
||||
const contextTokens = useMemo(() => {
|
||||
if (!activeChatId || !messageRows) return 0;
|
||||
for (let i = messageRows.length - 1; i >= 0; i--) {
|
||||
const meta = messageRows[i].metadata;
|
||||
if (!meta) continue;
|
||||
if (typeof meta.contextTokens === "number" && meta.contextTokens > 0) {
|
||||
return meta.contextTokens;
|
||||
}
|
||||
const usage = meta.usage;
|
||||
if (usage) {
|
||||
const fallback =
|
||||
usage.totalTokens ??
|
||||
(usage.inputTokens ?? 0) + (usage.outputTokens ?? 0);
|
||||
if (fallback > 0) return fallback;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}, [activeChatId, messageRows]);
|
||||
//
|
||||
// The denominator `maxContextTokens` (the model's configured max window) is
|
||||
// derived in the SAME backward scan: it is stamped alongside `contextTokens`
|
||||
// on a completed turn, but the numerator and denominator are taken from the
|
||||
// most recent row carrying EACH value independently — they may land on
|
||||
// different rows (e.g. a fresh error row can carry contextTokens but not
|
||||
// maxContextTokens), so we keep scanning for whichever is still unset. 0 when
|
||||
// no row has it (older rows, or no admin-configured limit) — the badge then
|
||||
// shows just the current size with no denominator.
|
||||
const { contextTokens, maxContextTokens } = useMemo(
|
||||
() => selectContextBadge(activeChatId ? messageRows : undefined),
|
||||
[activeChatId, messageRows],
|
||||
);
|
||||
|
||||
// On (re)open, settle the geometry before paint (useLayoutEffect → no
|
||||
// first-frame jump): compute an initial top-right placement the first time,
|
||||
@@ -547,20 +499,17 @@ export default function AiChatWindow() {
|
||||
)}
|
||||
|
||||
<div style={{ flex: 1, display: "flex", justifyContent: "center" }}>
|
||||
{/* While a turn streams, show the LIVE turn-token count (ticks ~8 Hz);
|
||||
once it finishes, fall back to the persisted context size. Require
|
||||
> 0 so the very first emit (an empty tail message, count 0) does not
|
||||
flash a "0" badge before any token streams in (#151 review). */}
|
||||
{liveTurnTokens !== null && liveTurnTokens > 0 ? (
|
||||
<Tooltip label={t("Tokens generated this turn")} withArrow>
|
||||
<span className={classes.badge}>
|
||||
{formatTokens(liveTurnTokens)}
|
||||
</span>
|
||||
</Tooltip>
|
||||
) : contextTokens > 0 ? (
|
||||
<Tooltip label={t("Current context size")} withArrow>
|
||||
{/* Always show the persisted "current / max" context. The denominator
|
||||
(the admin-configured model limit) is appended only when known;
|
||||
not clamped when current > max (shown as-is, e.g. "210k / 200k").
|
||||
Hidden entirely until a turn has recorded a context figure. */}
|
||||
{contextTokens > 0 ? (
|
||||
<Tooltip label={t("Context size / model limit")} withArrow>
|
||||
<span className={classes.badge}>
|
||||
{formatTokens(contextTokens)}
|
||||
{maxContextTokens > 0
|
||||
? ` / ${formatTokens(maxContextTokens)}`
|
||||
: ""}
|
||||
</span>
|
||||
</Tooltip>
|
||||
) : null}
|
||||
@@ -674,6 +623,7 @@ export default function AiChatWindow() {
|
||||
) : (
|
||||
<ChatThread
|
||||
key={threadKey}
|
||||
threadKey={threadKey}
|
||||
chatId={activeChatId}
|
||||
initialRows={activeChatId ? messageRows : []}
|
||||
openPage={openPage}
|
||||
@@ -685,9 +635,7 @@ export default function AiChatWindow() {
|
||||
onRolePicked={(role) => setSelectedRoleId(role.id)}
|
||||
assistantName={currentRole?.name}
|
||||
onTurnFinished={onTurnFinished}
|
||||
liveStateRef={liveThreadRef}
|
||||
onLiveTurnTokens={setLiveTurnTokens}
|
||||
onLiveContentChange={setHasLiveContent}
|
||||
onServerChatId={onServerChatId}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -55,6 +55,45 @@
|
||||
padding-inline-start: 1.4em;
|
||||
}
|
||||
|
||||
/* GFM tables in assistant markdown. The chat lives in a NARROW side panel, so a
|
||||
wide LLM table must scroll horizontally instead of collapsing its columns:
|
||||
`.markdown` sets `word-break: break-word`, which (with the default table
|
||||
layout) shrinks columns to a single glyph and wraps headers mid-word
|
||||
("Секция" -> "Секци / я"). Make the table a horizontally scrollable block,
|
||||
give cells a readable minimum width, and restore word-boundary wrapping. */
|
||||
.markdown table {
|
||||
display: block;
|
||||
/* lets the table scroll horizontally on its own */
|
||||
max-width: 100%;
|
||||
overflow-x: auto;
|
||||
border-collapse: collapse;
|
||||
margin-block-end: 0.5em;
|
||||
}
|
||||
|
||||
.markdown th,
|
||||
.markdown td {
|
||||
border: 1px solid light-dark(var(--mantine-color-gray-3), var(--mantine-color-dark-4));
|
||||
padding: 3px 8px;
|
||||
/* readable floor; the block scrolls when the row exceeds the panel */
|
||||
min-width: 6em;
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
/* cancel the inherited break-word so words don't split mid-glyph */
|
||||
word-break: normal;
|
||||
/* still wrap genuinely long words / URLs at the cell edge */
|
||||
overflow-wrap: break-word;
|
||||
}
|
||||
|
||||
.markdown th {
|
||||
background: light-dark(var(--mantine-color-gray-1), var(--mantine-color-dark-5));
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
/* GFM wraps cell text in <p>; drop its default block margin inside cells. */
|
||||
.markdown table p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
/* Animated three-dot "typing" indicator shown while the agent is thinking but
|
||||
has not yet produced any visible text/tool parts. */
|
||||
.typingDots {
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
import {
|
||||
useCallback,
|
||||
useEffect,
|
||||
useMemo,
|
||||
useRef,
|
||||
useState,
|
||||
type MutableRefObject,
|
||||
} from "react";
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||
import { generateId } from "ai";
|
||||
import { ActionIcon, Box, Group, Stack, Text } from "@mantine/core";
|
||||
import { IconClockHour4, IconX } from "@tabler/icons-react";
|
||||
@@ -27,7 +20,6 @@ import {
|
||||
} from "@/features/ai-chat/utils/role-launch.ts";
|
||||
import { describeChatError } from "@/features/ai-chat/utils/error-message.ts";
|
||||
import { extractServerChatId } from "@/features/ai-chat/utils/adopt-chat-id.ts";
|
||||
import { liveTurnTokens } from "@/features/ai-chat/utils/count-stream-tokens.ts";
|
||||
import {
|
||||
dequeue,
|
||||
enqueueMessage,
|
||||
@@ -36,6 +28,14 @@ import {
|
||||
} from "@/features/ai-chat/utils/queue-helpers.ts";
|
||||
import classes from "@/features/ai-chat/components/ai-chat.module.css";
|
||||
|
||||
// Throttle how often the streamed `messages` state triggers a re-render. Without
|
||||
// it, useChat updates state on EVERY token, so the whole transcript's markdown
|
||||
// (marked + DOMPurify) is re-parsed per token — on a long agent run that grows
|
||||
// into a quadratic CPU storm that pins the main thread and freezes the UI.
|
||||
// ~50ms (20 Hz) keeps streaming visually smooth while decoupling re-render cost
|
||||
// from the token rate.
|
||||
const STREAM_THROTTLE_MS = 50;
|
||||
|
||||
/** The page the user is currently viewing, sent as chat context. */
|
||||
export interface OpenPageContext {
|
||||
id: string;
|
||||
@@ -45,6 +45,11 @@ export interface OpenPageContext {
|
||||
interface ChatThreadProps {
|
||||
/** The open chat id, or null for a brand-new (not-yet-created) chat. */
|
||||
chatId: string | null;
|
||||
/** This thread's mount key (the same value the parent uses as React `key`).
|
||||
* Forwarded to onTurnFinished so the session can tell a turn finishing on the
|
||||
* CURRENT thread from one ABANDONED by New chat mid-stream — whose onFinish/
|
||||
* onError still fire after unmount and must not adopt the abandoned chat (#161). */
|
||||
threadKey?: string;
|
||||
/** Persisted rows to seed initial messages (existing chats only). */
|
||||
initialRows?: IAiChatMessageRow[];
|
||||
/** The page currently open in the workspace, or null on a non-page route.
|
||||
@@ -66,32 +71,16 @@ interface ChatThreadProps {
|
||||
/** Called when a turn finishes; the parent refreshes the chat list and, for a
|
||||
* new chat, adopts the freshly created chat id. `serverChatId` is the
|
||||
* authoritative id the server streamed on the assistant message metadata, or
|
||||
* undefined on a failed turn — see adopt-chat-id.ts for the full #137 design. */
|
||||
onTurnFinished: (serverChatId?: string) => void;
|
||||
/** Parent-owned ref that this thread keeps updated with its live useChat
|
||||
* snapshot (full message list + streaming flag), so the header's
|
||||
* "Copy chat" export can include the in-progress, not-yet-persisted
|
||||
* assistant message. A ref (not state) avoids re-rendering the parent on
|
||||
* every streamed delta. */
|
||||
liveStateRef?: MutableRefObject<{
|
||||
messages: UIMessage[];
|
||||
isStreaming: boolean;
|
||||
banner: string | null;
|
||||
}>;
|
||||
/** Reports the live turn-token total (reasoning + output) for the in-flight
|
||||
* turn so the parent can show a header badge that ticks mid-stream. THROTTLED
|
||||
* here (~8 Hz) so the parent re-renders a handful of times a second, not on
|
||||
* every streamed delta. Called with `null` when no turn is in flight (the
|
||||
* parent then reverts the badge to the persisted context size). */
|
||||
onLiveTurnTokens?: (tokens: number | null) => void;
|
||||
/** Reports whether the live thread currently holds at least one message, so the
|
||||
* parent can gate the "Copy chat" button on the on-screen thread rather than on
|
||||
* the persisted rows alone. This stays truthy for a brand-new, not-yet-saved
|
||||
* chat the moment its first user message appears — so an interrupted very first
|
||||
* turn (no persisted rows yet) is still exportable (#174). Called with `false`
|
||||
* on unmount so a thread torn down by `key` on chat switch can't leave the
|
||||
* button enabled for the next, possibly empty, chat. */
|
||||
onLiveContentChange?: (hasContent: boolean) => void;
|
||||
* undefined on a failed turn — see adopt-chat-id.ts for the full #137 design.
|
||||
* `finishingThreadKey` (this thread's mount key) lets the session ignore a turn
|
||||
* finishing on a thread already abandoned by New chat mid-stream (#161). */
|
||||
onTurnFinished: (serverChatId?: string, finishingThreadKey?: string) => void;
|
||||
/** Called EARLY (at the stream's `start` chunk) with the authoritative server
|
||||
* chat id streamed on the assistant message metadata, so a brand-new chat
|
||||
* adopts its real id WHILE the first turn is still streaming (#174 — makes the
|
||||
* Copy/export button available mid-stream). Distinct from onTurnFinished,
|
||||
* which fires only at the terminal outcome. */
|
||||
onServerChatId?: (serverChatId?: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -128,6 +117,7 @@ function rowToUiMessage(row: IAiChatMessageRow): UIMessage {
|
||||
*/
|
||||
export default function ChatThread({
|
||||
chatId,
|
||||
threadKey,
|
||||
initialRows,
|
||||
openPage,
|
||||
roleId,
|
||||
@@ -135,9 +125,7 @@ export default function ChatThread({
|
||||
onRolePicked,
|
||||
assistantName,
|
||||
onTurnFinished,
|
||||
liveStateRef,
|
||||
onLiveTurnTokens,
|
||||
onLiveContentChange,
|
||||
onServerChatId,
|
||||
}: ChatThreadProps) {
|
||||
const { t } = useTranslation();
|
||||
|
||||
@@ -266,6 +254,8 @@ export default function ChatThread({
|
||||
id: chatStoreId,
|
||||
messages: initialMessages,
|
||||
transport,
|
||||
// See STREAM_THROTTLE_MS — bounds re-render/markdown-reparse frequency.
|
||||
experimental_throttle: STREAM_THROTTLE_MS,
|
||||
// `onFinish` (ai@6 useChat) fires from a `finally` on EVERY terminal outcome
|
||||
// — success, user Stop/abort (`isAbort`), network drop (`isDisconnect`), and
|
||||
// stream error (`isError`). Keep calling `onTurnFinished()` on all of them
|
||||
@@ -277,8 +267,10 @@ export default function ChatThread({
|
||||
onFinish: ({ message, isAbort, isDisconnect, isError }) => {
|
||||
// Forward the authoritative server chatId (streamed on the assistant
|
||||
// message metadata) so the parent adopts the REAL created chat id for a new
|
||||
// chat — see adopt-chat-id.ts for the full #137 design.
|
||||
onTurnFinished(extractServerChatId(message));
|
||||
// chat — see adopt-chat-id.ts for the full #137 design. `threadKey` lets the
|
||||
// session ignore this finish if it belongs to a thread abandoned by New chat
|
||||
// mid-stream (#161).
|
||||
onTurnFinished(extractServerChatId(message), threadKey);
|
||||
// Show a neutral "stopped" marker for an aborted turn; the red error banner
|
||||
// (via `error`) already covers isError, and a clean finish clears any marker.
|
||||
if (isError) setStopNotice(null);
|
||||
@@ -299,13 +291,33 @@ export default function ChatThread({
|
||||
// Surface the raw failure in the browser console (devtools) for debugging;
|
||||
// the UI separately shows a friendly classified banner (see errorView).
|
||||
console.error("AI chat stream error:", streamError);
|
||||
onTurnFinished();
|
||||
onTurnFinished(undefined, threadKey);
|
||||
},
|
||||
});
|
||||
|
||||
// Keep the flush helper pointed at the latest sendMessage instance.
|
||||
sendMessageRef.current = sendMessage;
|
||||
|
||||
// EARLY chat-id adoption (#174): the server streams the authoritative chat id
|
||||
// on the assistant message metadata at the `start` chunk (message.metadata.
|
||||
// chatId — see adopt-chat-id.ts / chatStreamMetadata). Forward it to the parent
|
||||
// AS SOON AS it appears (mid-stream), so a brand-new chat adopts its real id
|
||||
// WHILE the first turn is still streaming and activeChatId-gated affordances
|
||||
// (the Copy/export button) light up immediately, instead of only at onFinish.
|
||||
// Keyed by the last-seen id so we forward each distinct id exactly once. The
|
||||
// parent's onServerChatId is idempotent and a no-op once the chat has an id.
|
||||
const lastForwardedChatIdRef = useRef<string | undefined>(undefined);
|
||||
useEffect(() => {
|
||||
if (!onServerChatId) return;
|
||||
const tail = messages[messages.length - 1];
|
||||
if (tail?.role !== "assistant") return;
|
||||
const serverChatId = extractServerChatId(tail);
|
||||
if (!serverChatId || serverChatId === lastForwardedChatIdRef.current)
|
||||
return;
|
||||
lastForwardedChatIdRef.current = serverChatId;
|
||||
onServerChatId(serverChatId);
|
||||
}, [messages, onServerChatId]);
|
||||
|
||||
// Live "turn was interrupted" marker for the CURRENT session. The red error
|
||||
// banner (driven by `error`) covers the error case; this covers an aborted
|
||||
// turn, distinguishing a manual Stop (`isAbort`) from a dropped connection
|
||||
@@ -328,91 +340,6 @@ export default function ChatThread({
|
||||
// the SAME on-screen banner text can be mirrored into the export (issue #160).
|
||||
const errorView = error ? describeChatError(error.message ?? "", t) : null;
|
||||
|
||||
// The exact banner the user sees under the message list, flattened to a single
|
||||
// string for the "Copy chat" export so the artifact records the interruption
|
||||
// WYSIWYG. Mirrors the JSX precedence below: error first, else the stop notice.
|
||||
const banner = errorView
|
||||
? errorView.detail
|
||||
? `${errorView.title} — ${errorView.detail}`
|
||||
: errorView.title
|
||||
: stopNotice === "manual"
|
||||
? t("Response stopped.")
|
||||
: stopNotice === "disconnect"
|
||||
? t("Connection lost — the answer was interrupted.")
|
||||
: null;
|
||||
|
||||
// Mirror the live useChat snapshot into the parent-owned ref so the export
|
||||
// (handled in AiChatWindow) can include the in-progress streaming turn AND the
|
||||
// on-screen banner. The cleanup clears the ref on unmount so a thread torn down
|
||||
// by `key` on chat switch can't leak its (possibly still-streaming) tail into
|
||||
// the next chat's export before the new thread's effect repopulates the ref.
|
||||
useEffect(() => {
|
||||
if (!liveStateRef) return;
|
||||
liveStateRef.current = { messages, isStreaming, banner };
|
||||
return () => {
|
||||
liveStateRef.current = { messages: [], isStreaming: false, banner: null };
|
||||
};
|
||||
}, [liveStateRef, messages, isStreaming, banner]);
|
||||
|
||||
// Reactively report "the live thread has content" to the parent. `liveStateRef`
|
||||
// above is a ref (deliberately non-reactive so streaming deltas don't re-render
|
||||
// the parent), so the export button needs a SEPARATE reactive signal to flip on
|
||||
// for a not-yet-persisted chat. Keyed on the boolean only — identical values are
|
||||
// a no-op setState in the parent, so this does not add per-delta re-renders.
|
||||
const hasLiveContent = messages.length > 0;
|
||||
useEffect(() => {
|
||||
if (!onLiveContentChange) return;
|
||||
onLiveContentChange(hasLiveContent);
|
||||
return () => onLiveContentChange(false);
|
||||
}, [onLiveContentChange, hasLiveContent]);
|
||||
|
||||
// Report the live turn-token total to the parent header badge, THROTTLED to
|
||||
// ~8 Hz so the parent re-renders a few times a second instead of on every
|
||||
// streamed delta. The tail assistant message's reasoning+output (estimate while
|
||||
// streaming, authoritative once a step reports usage) is the live figure. When
|
||||
// the turn ends we emit a final exact value, then `null` so the parent reverts
|
||||
// the badge to the persisted context size.
|
||||
const lastEmitRef = useRef(0);
|
||||
const emitTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
useEffect(() => {
|
||||
if (!onLiveTurnTokens) return;
|
||||
if (!isStreaming) {
|
||||
// Turn ended (or never started): clear any pending throttle and revert.
|
||||
if (emitTimerRef.current) {
|
||||
clearTimeout(emitTimerRef.current);
|
||||
emitTimerRef.current = null;
|
||||
}
|
||||
lastEmitRef.current = 0;
|
||||
onLiveTurnTokens(null);
|
||||
return;
|
||||
}
|
||||
const tail = messages[messages.length - 1];
|
||||
const live = tail?.role === "assistant" ? liveTurnTokens(tail) : null;
|
||||
const total = live ? live.reasoning + live.output : 0;
|
||||
const now = Date.now();
|
||||
const MIN_INTERVAL = 120; // ms (~8 Hz)
|
||||
const elapsed = now - lastEmitRef.current;
|
||||
if (elapsed >= MIN_INTERVAL) {
|
||||
lastEmitRef.current = now;
|
||||
onLiveTurnTokens(total);
|
||||
} else if (!emitTimerRef.current) {
|
||||
// Schedule a trailing emit so the FINAL value of a burst is not dropped.
|
||||
emitTimerRef.current = setTimeout(() => {
|
||||
emitTimerRef.current = null;
|
||||
lastEmitRef.current = Date.now();
|
||||
onLiveTurnTokens(total);
|
||||
}, MIN_INTERVAL - elapsed);
|
||||
}
|
||||
}, [messages, isStreaming, onLiveTurnTokens]);
|
||||
|
||||
// Clear any pending throttle timer on unmount (chat switch via `key`) so a
|
||||
// trailing emit can't fire into a torn-down thread's parent.
|
||||
useEffect(() => {
|
||||
return () => {
|
||||
if (emitTimerRef.current) clearTimeout(emitTimerRef.current);
|
||||
};
|
||||
}, []);
|
||||
|
||||
// A role was picked with autoStart=false: the role is bound but NOTHING was
|
||||
// sent, so chatId stays null and the empty state would keep showing the cards.
|
||||
// This flag hides the cards and reveals the composer (with the role indicated)
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { render } from "@testing-library/react";
|
||||
import { MantineProvider } from "@mantine/core";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
|
||||
// Stub react-i18next (the component reads `useTranslation`). Mirrors the stub in
|
||||
// reasoning-block.test.tsx.
|
||||
vi.mock("react-i18next", () => ({
|
||||
useTranslation: () => ({ t: (key: string) => key }),
|
||||
}));
|
||||
|
||||
// Spy on `renderChatMarkdown` so we can count parse calls per text. We keep every
|
||||
// OTHER named export of markdown.ts intact via `importActual`, and override only
|
||||
// `renderChatMarkdown` with a `vi.fn()` that returns simple HTML so the component
|
||||
// still renders. This is the seam that proves the MarkdownPart memo works: a
|
||||
// finalized text part must NOT be re-parsed on a later streamed delta.
|
||||
// `vi.hoisted` so the spy exists when the hoisted `vi.mock` factory runs.
|
||||
const { renderChatMarkdownSpy } = vi.hoisted(() => ({
|
||||
renderChatMarkdownSpy: vi.fn((text: string) => `<p>${text}</p>`),
|
||||
}));
|
||||
vi.mock("@/features/ai-chat/utils/markdown.ts", async () => {
|
||||
const actual = await vi.importActual<
|
||||
typeof import("@/features/ai-chat/utils/markdown.ts")
|
||||
>("@/features/ai-chat/utils/markdown.ts");
|
||||
return { ...actual, renderChatMarkdown: renderChatMarkdownSpy };
|
||||
});
|
||||
|
||||
import MessageItem from "./message-item";
|
||||
|
||||
// matchMedia (read by MantineProvider) is stubbed globally in vitest.setup.ts.
|
||||
|
||||
const msg = (parts: UIMessage["parts"]): UIMessage =>
|
||||
({ id: "m1", role: "assistant", parts }) as UIMessage;
|
||||
|
||||
const renderRow = (message: UIMessage) =>
|
||||
render(
|
||||
<MantineProvider>
|
||||
<MessageItem message={message} />
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
/** Count how many spy calls parsed exactly `text` (filtering by the first arg). */
|
||||
const callsFor = (text: string) =>
|
||||
renderChatMarkdownSpy.mock.calls.filter((c) => c[0] === text).length;
|
||||
|
||||
describe("MessageItem markdown memoization", () => {
|
||||
it("does not re-parse finalized text parts when only a tail part grows", () => {
|
||||
renderChatMarkdownSpy.mockClear();
|
||||
|
||||
// Two finalized text parts.
|
||||
const first = msg([
|
||||
{ type: "text", text: "alpha" },
|
||||
{ type: "text", text: "beta" },
|
||||
]);
|
||||
const { rerender } = renderRow(first);
|
||||
|
||||
// Both finalized parts parsed exactly once on the initial render.
|
||||
expect(callsFor("alpha")).toBe(1);
|
||||
expect(callsFor("beta")).toBe(1);
|
||||
|
||||
// A streamed delta: a NEW message object where only a third tail part grows;
|
||||
// the first two parts' text is byte-identical.
|
||||
const next = msg([
|
||||
{ type: "text", text: "alpha" },
|
||||
{ type: "text", text: "beta" },
|
||||
{ type: "text", text: "gamm" },
|
||||
]);
|
||||
rerender(
|
||||
<MantineProvider>
|
||||
<MessageItem message={next} />
|
||||
</MantineProvider>,
|
||||
);
|
||||
|
||||
// The finalized parts hit the MarkdownPart memo: still parsed at most once
|
||||
// each across BOTH renders (the resilient invariant). The only new parse is
|
||||
// for the changed/added tail part.
|
||||
expect(callsFor("alpha")).toBe(1);
|
||||
expect(callsFor("beta")).toBe(1);
|
||||
expect(callsFor("gamm")).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,73 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
|
||||
// Stub react-i18next: importing the component module pulls in `useTranslation`,
|
||||
// and we only exercise the pure `arePropsEqual` comparator (no rendering), so a
|
||||
// minimal `t` that echoes the key is enough. Mirrors the stub in
|
||||
// reasoning-block.test.tsx.
|
||||
vi.mock("react-i18next", () => ({
|
||||
useTranslation: () => ({ t: (key: string) => key }),
|
||||
}));
|
||||
|
||||
import { arePropsEqual } from "./message-item";
|
||||
|
||||
/**
|
||||
* Tests for `arePropsEqual`, the `React.memo` comparator for MessageItem. It must
|
||||
* return false on any visible prop/content change (so the row re-renders) and
|
||||
* true when nothing visible changed (so a finalized row is skipped). A FIXED
|
||||
* message id is used so a content-identical clone yields an equal signature.
|
||||
*/
|
||||
const msg = (parts: UIMessage["parts"]): UIMessage =>
|
||||
({ id: "m1", role: "assistant", parts }) as UIMessage;
|
||||
|
||||
const props = (
|
||||
message: UIMessage,
|
||||
over: Record<string, unknown> = {},
|
||||
) => ({
|
||||
message,
|
||||
showCitations: true,
|
||||
neutralizeInternalLinks: false,
|
||||
assistantName: "AI",
|
||||
...over,
|
||||
});
|
||||
|
||||
describe("arePropsEqual", () => {
|
||||
it("returns false when showCitations differs", () => {
|
||||
const m = msg([{ type: "text", text: "answer" }]);
|
||||
expect(
|
||||
arePropsEqual(props(m), props(m, { showCitations: false })),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when neutralizeInternalLinks differs", () => {
|
||||
const m = msg([{ type: "text", text: "answer" }]);
|
||||
expect(
|
||||
arePropsEqual(props(m), props(m, { neutralizeInternalLinks: true })),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("returns false when assistantName differs", () => {
|
||||
const m = msg([{ type: "text", text: "answer" }]);
|
||||
expect(
|
||||
arePropsEqual(props(m), props(m, { assistantName: "Other" })),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("returns true on the identity fast path (same message object, equal props)", () => {
|
||||
const m = msg([{ type: "text", text: "answer" }]);
|
||||
expect(arePropsEqual(props(m), props(m))).toBe(true);
|
||||
});
|
||||
|
||||
it("returns true for the same content in a different message object", () => {
|
||||
const a = msg([{ type: "text", text: "answer" }]);
|
||||
const b = msg([{ type: "text", text: "answer" }]);
|
||||
expect(a).not.toBe(b);
|
||||
expect(arePropsEqual(props(a), props(b))).toBe(true);
|
||||
});
|
||||
|
||||
it("returns false when content changed in a different message object", () => {
|
||||
const a = msg([{ type: "text", text: "answer" }]);
|
||||
const b = msg([{ type: "text", text: "answer grown" }]);
|
||||
expect(arePropsEqual(props(a), props(b))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,3 +1,4 @@
|
||||
import { memo } from "react";
|
||||
import { Box, Text } from "@mantine/core";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
@@ -10,6 +11,7 @@ import { assistantMessageHasVisibleContent } from "@/features/ai-chat/utils/mess
|
||||
import { renderChatMarkdown } from "@/features/ai-chat/utils/markdown.ts";
|
||||
import { resolveAssistantName } from "@/features/ai-chat/utils/assistant-name.ts";
|
||||
import { reasoningTokensForPart } from "@/features/ai-chat/utils/reasoning-tokens.ts";
|
||||
import { messageSignature } from "@/features/ai-chat/utils/message-signature.ts";
|
||||
import { describeChatError } from "@/features/ai-chat/utils/error-message.ts";
|
||||
import classes from "@/features/ai-chat/components/ai-chat.module.css";
|
||||
|
||||
@@ -34,6 +36,39 @@ interface MessageItemProps {
|
||||
assistantName?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* One assistant text part rendered as sanitized markdown. Memoized on its inputs
|
||||
* so a finalized text part is NOT re-parsed on every streamed delta: during a
|
||||
* turn only the actively-growing tail part changes its `text`, so every earlier
|
||||
* part hits the memo and skips the expensive marked + DOMPurify pass. Props are
|
||||
* primitives, so React.memo's default shallow compare is exactly right (the
|
||||
* `text` string is compared by value).
|
||||
*/
|
||||
const MarkdownPart = memo(function MarkdownPart({
|
||||
text,
|
||||
neutralizeInternalLinks,
|
||||
}: {
|
||||
text: string;
|
||||
neutralizeInternalLinks: boolean;
|
||||
}) {
|
||||
const html = renderChatMarkdown(text, { neutralizeInternalLinks });
|
||||
if (html) {
|
||||
return (
|
||||
<div
|
||||
className={classes.markdown}
|
||||
// Sanitized by renderChatMarkdown (DOMPurify) before insertion.
|
||||
dangerouslySetInnerHTML={{ __html: html }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
// Fallback when markdown could not render synchronously: raw text.
|
||||
return (
|
||||
<Text className={classes.markdown} style={{ whiteSpace: "pre-wrap" }}>
|
||||
{text}
|
||||
</Text>
|
||||
);
|
||||
});
|
||||
|
||||
/**
|
||||
* Render a single UIMessage by iterating its `parts`:
|
||||
* - `text` parts -> sanitized markdown.
|
||||
@@ -41,12 +76,13 @@ interface MessageItemProps {
|
||||
* Other part kinds (reasoning, sources, files, step-start) are ignored for v1.
|
||||
* User messages render their text as a right-aligned plain bubble.
|
||||
*
|
||||
* This component is intentionally NOT memoized: `useChat` replaces the streaming
|
||||
* assistant message with a freshly cloned object on every streamed delta, so the
|
||||
* `message` prop identity (and its `parts`) changes each tick. Re-rendering the
|
||||
* text parts on each delta is what makes the answer stream in progressively.
|
||||
* This component is memoized (see `arePropsEqual` at the bottom) on a cheap
|
||||
* per-message content signature: the streaming TAIL message's signature changes
|
||||
* on each delta so it still re-renders and streams in, while finalized rows are
|
||||
* skipped. Each text part's markdown is itself memoized via `MarkdownPart`, so a
|
||||
* long turn no longer re-parses the whole transcript on every token.
|
||||
*/
|
||||
export default function MessageItem({
|
||||
function MessageItem({
|
||||
message,
|
||||
showCitations = true,
|
||||
neutralizeInternalLinks = false,
|
||||
@@ -109,24 +145,12 @@ export default function MessageItem({
|
||||
// starts with an empty text part before the first token arrives); the
|
||||
// typing indicator covers that gap until real content streams in.
|
||||
if (!part.text.trim()) return null;
|
||||
const html = renderChatMarkdown(part.text, {
|
||||
neutralizeInternalLinks,
|
||||
});
|
||||
if (html) {
|
||||
return (
|
||||
<div
|
||||
key={index}
|
||||
className={classes.markdown}
|
||||
// Sanitized by renderChatMarkdown (DOMPurify) before insertion.
|
||||
dangerouslySetInnerHTML={{ __html: html }}
|
||||
/>
|
||||
);
|
||||
}
|
||||
// Fallback when markdown could not render synchronously: raw text.
|
||||
return (
|
||||
<Text key={index} className={classes.markdown} style={{ whiteSpace: "pre-wrap" }}>
|
||||
{part.text}
|
||||
</Text>
|
||||
<MarkdownPart
|
||||
key={index}
|
||||
text={part.text}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -177,3 +201,26 @@ export default function MessageItem({
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
/** Skip re-rendering a message whose visible content is unchanged. The streaming
|
||||
* TAIL message gets a fresh object whose signature changes each delta, so it
|
||||
* still re-renders and streams in; every FINALIZED message is skipped, turning a
|
||||
* per-token whole-transcript re-render into a tail-only one. */
|
||||
export function arePropsEqual(
|
||||
prev: MessageItemProps,
|
||||
next: MessageItemProps,
|
||||
): boolean {
|
||||
if (
|
||||
prev.showCitations !== next.showCitations ||
|
||||
prev.neutralizeInternalLinks !== next.neutralizeInternalLinks ||
|
||||
prev.assistantName !== next.assistantName
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
// Fast path: identical message object (finalized rows keep their identity
|
||||
// across deltas) — skip without building signatures.
|
||||
if (prev.message === next.message) return true;
|
||||
return messageSignature(prev.message) === messageSignature(next.message);
|
||||
}
|
||||
|
||||
export default memo(MessageItem, arePropsEqual);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState } from "react";
|
||||
import { memo, useMemo, useState } from "react";
|
||||
import { Box, Collapse, Group, Text, UnstyledButton } from "@mantine/core";
|
||||
import { IconChevronDown } from "@tabler/icons-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
@@ -27,19 +27,23 @@ interface ReasoningBlockProps {
|
||||
* Providers that don't stream reasoning TEXT still render this block from the
|
||||
* authoritative count alone (header only, empty body) so the cost is visible.
|
||||
*/
|
||||
export default function ReasoningBlock({ text, tokens }: ReasoningBlockProps) {
|
||||
function ReasoningBlock({ text, tokens }: ReasoningBlockProps) {
|
||||
const { t } = useTranslation();
|
||||
const [open, setOpen] = useState(false);
|
||||
|
||||
// Authoritative count wins; otherwise estimate live from the streamed text.
|
||||
const count = tokens && tokens > 0 ? tokens : estimateTokens(text);
|
||||
const trimmed = text.trim();
|
||||
// Collapse the blank-line gaps the model emits between every list item /
|
||||
// paragraph so the reasoning renders compactly (tight lists, joined
|
||||
// paragraphs) — see collapseBlankLines. ONLY here, not in the normal answer.
|
||||
const html = trimmed
|
||||
? renderChatMarkdown(collapseBlankLines(trimmed), {})
|
||||
: "";
|
||||
// Memoize the markdown render so toggling `open` (or a parent re-render caused
|
||||
// by an unrelated streamed delta) does not re-parse the reasoning text; it
|
||||
// recomputes only when the reasoning text itself changes (while it streams in).
|
||||
// collapseBlankLines collapses the blank-line gaps the model emits between every
|
||||
// list item / paragraph so the reasoning renders compactly (tight lists, joined
|
||||
// paragraphs) — ONLY here, not in the normal answer.
|
||||
const html = useMemo(
|
||||
() => (trimmed ? renderChatMarkdown(collapseBlankLines(trimmed), {}) : ""),
|
||||
[trimmed],
|
||||
);
|
||||
|
||||
return (
|
||||
<Box className={classes.reasoningBlock} mb={6}>
|
||||
@@ -87,3 +91,8 @@ export default function ReasoningBlock({ text, tokens }: ReasoningBlockProps) {
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
// Memoized: re-renders only when `text`/`tokens` change (primitive props, default
|
||||
// shallow compare), so a parent re-render during streaming of OTHER content does
|
||||
// not re-run the markdown parse for an already-finalized reasoning block.
|
||||
export default memo(ReasoningBlock);
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, it, expect, vi, beforeEach } from "vitest";
|
||||
import { renderHook } from "@testing-library/react";
|
||||
import { renderHook, act } from "@testing-library/react";
|
||||
import { useChatSession } from "./use-chat-session";
|
||||
import type { UseChatSessionOptions } from "./use-chat-session";
|
||||
|
||||
@@ -64,7 +64,10 @@ describe("useChatSession", () => {
|
||||
result.current.onTurnFinished(undefined);
|
||||
expect(setActiveChatId).not.toHaveBeenCalled();
|
||||
// The refetch lands with the new row => adopt it.
|
||||
rerender({ activeChatId: null, chats: { items: [{ id: "x" }, { id: "new" }] } });
|
||||
rerender({
|
||||
activeChatId: null,
|
||||
chats: { items: [{ id: "x" }, { id: "new" }] },
|
||||
});
|
||||
expect(setActiveChatId).toHaveBeenCalledWith("new");
|
||||
});
|
||||
|
||||
@@ -88,7 +91,10 @@ describe("useChatSession", () => {
|
||||
});
|
||||
result.current.onTurnFinished(undefined);
|
||||
// a was deleted, new was added — same length, but membership changed.
|
||||
rerender({ activeChatId: null, chats: { items: [{ id: "b" }, { id: "new" }] } });
|
||||
rerender({
|
||||
activeChatId: null,
|
||||
chats: { items: [{ id: "b" }, { id: "new" }] },
|
||||
});
|
||||
expect(setActiveChatId).toHaveBeenCalledWith("new");
|
||||
});
|
||||
|
||||
@@ -171,6 +177,40 @@ describe("useChatSession", () => {
|
||||
expect(setActiveChatId).not.toHaveBeenCalledWith("late");
|
||||
});
|
||||
|
||||
it("#174 early adopt: onServerChatId adopts the streamed id mid-stream (Copy button available during the first turn)", () => {
|
||||
// Brand-new chat: no id yet. The server streams the real chat id "A" on the
|
||||
// `start` chunk WHILE the first turn is still streaming (before onTurnFinished
|
||||
// fires at the terminal outcome). The hook must adopt it immediately so the
|
||||
// window's activeChatId-gated Copy/export button lights up during the stream.
|
||||
const { result, setActiveChatId } = setup({
|
||||
activeChatId: null,
|
||||
chats: { items: [] },
|
||||
});
|
||||
result.current.onServerChatId("A");
|
||||
expect(setActiveChatId).toHaveBeenCalledWith("A");
|
||||
});
|
||||
|
||||
it("#174 early adopt is in-place: threadKey stays stable (live stream not torn down)", () => {
|
||||
const chats = { items: [] };
|
||||
const { result, rerender } = setup({ activeChatId: null, chats });
|
||||
const keyBefore = result.current.threadKey;
|
||||
result.current.onServerChatId("A");
|
||||
// Parent reflects the adopted id back in; the SAME mount key is kept so the
|
||||
// in-flight useChat store (the streaming turn) is preserved.
|
||||
rerender({ activeChatId: "A", chats });
|
||||
expect(result.current.threadKey).toBe(keyBefore);
|
||||
});
|
||||
|
||||
it("#174 early adopt: no-op for an existing chat and for a missing id", () => {
|
||||
const { result, setActiveChatId } = setup({
|
||||
activeChatId: "chat-1",
|
||||
chats: { items: [{ id: "chat-1" }] },
|
||||
});
|
||||
result.current.onServerChatId("chat-1"); // already has an id
|
||||
result.current.onServerChatId(undefined); // no streamed id
|
||||
expect(setActiveChatId).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("in-place adopt keeps threadKey stable; an external switch remounts", () => {
|
||||
const chats = { items: [{ id: "B" }] };
|
||||
const { result, rerender } = setup({ activeChatId: null, chats });
|
||||
@@ -187,6 +227,50 @@ describe("useChatSession", () => {
|
||||
expect(result.current.threadKey).toBe("C");
|
||||
});
|
||||
|
||||
it("#161: New chat during a streaming first turn forces a fresh thread (remount), not just a no-op", () => {
|
||||
// Brand-new chat whose first turn is still streaming: the id is adopted only
|
||||
// at turn end, so activeChatId AND thread.chatId are both null. Pressing "New
|
||||
// chat" must still remount to a clean thread even though the atom is unchanged
|
||||
// — the render-phase reconciler (null === null) would otherwise do nothing,
|
||||
// leaving the old chat/stream/history in place (the bug: only the role badge
|
||||
// dropped).
|
||||
const { result } = setup({ activeChatId: null, chats: { items: [] } });
|
||||
const keyBefore = result.current.threadKey;
|
||||
act(() => result.current.startFreshThread());
|
||||
expect(result.current.threadKey).not.toBe(keyBefore);
|
||||
});
|
||||
|
||||
it("#161: an abandoned thread's late onTurnFinished does NOT adopt its chat (thread-aware guard)", () => {
|
||||
// New chat mid-stream remounts to a fresh thread, but @ai-sdk/react does not
|
||||
// abort the abandoned stream on unmount: its onFinish still fires later with
|
||||
// the real server id, tagged with the OLD (abandoned) mount key. That must not
|
||||
// adopt — it would yank the user back into the chat they just left.
|
||||
const { result, setActiveChatId, onInvalidateChatList } = setup({
|
||||
activeChatId: null,
|
||||
chats: { items: [] },
|
||||
});
|
||||
const abandonedKey = result.current.threadKey;
|
||||
act(() => result.current.startFreshThread());
|
||||
expect(result.current.threadKey).not.toBe(abandonedKey);
|
||||
// The abandoned turn finishes in the background, streaming its real id "A".
|
||||
result.current.onTurnFinished("A", abandonedKey);
|
||||
expect(setActiveChatId).not.toHaveBeenCalledWith("A");
|
||||
// It still refreshes the chat list so the left-behind chat shows in history.
|
||||
expect(onInvalidateChatList).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("#161: a turn finishing on the CURRENT thread still adopts (guard is key-scoped, not blanket)", () => {
|
||||
// The happy path must keep working: onTurnFinished tagged with the mounted
|
||||
// thread's own key adopts in place as before.
|
||||
const { result, setActiveChatId } = setup({
|
||||
activeChatId: null,
|
||||
chats: { items: [] },
|
||||
});
|
||||
const currentKey = result.current.threadKey;
|
||||
result.current.onTurnFinished("A", currentKey);
|
||||
expect(setActiveChatId).toHaveBeenCalledWith("A");
|
||||
});
|
||||
|
||||
it("waitingForHistory gates the loader only while opening an unloaded existing chat", () => {
|
||||
// Open an existing chat whose history is still loading => loader on.
|
||||
const { result, rerender } = setup({
|
||||
|
||||
@@ -31,9 +31,26 @@ export interface UseChatSessionResult {
|
||||
threadKey: string;
|
||||
/** Show the history loader instead of the live thread. */
|
||||
waitingForHistory: boolean;
|
||||
/** Force a brand-new, empty thread (new mount key, no chat id) UNCONDITIONALLY,
|
||||
* even when `activeChatId` is unchanged. The window calls this from
|
||||
* startNewChat so "New chat" pressed WHILE a brand-new chat's first turn is
|
||||
* still streaming (activeChatId still null, nothing to diverge) actually
|
||||
* resets the chat instead of only dropping the role badge (#161). */
|
||||
startFreshThread: () => void;
|
||||
/** Call when a turn finishes; `serverChatId` is the authoritative streamed id
|
||||
* (undefined on a failed turn). Handles new-chat id adoption + invalidations. */
|
||||
onTurnFinished: (serverChatId?: string) => void;
|
||||
* (undefined on a failed turn). `finishingThreadKey` is the mount key of the
|
||||
* thread that produced the turn (omit => "current thread", back-compatible):
|
||||
* a turn ABANDONED by New chat mid-stream still fires this after its thread
|
||||
* unmounted, so adoption is gated to the still-mounted thread (#161). Handles
|
||||
* new-chat id adoption + invalidations. */
|
||||
onTurnFinished: (serverChatId?: string, finishingThreadKey?: string) => void;
|
||||
/** Call EARLY (at the stream's `start` chunk) with the authoritative streamed
|
||||
* chat id so a brand-new chat adopts its real id WHILE its first turn is still
|
||||
* streaming — making `activeChatId`-gated affordances (e.g. the Copy/export
|
||||
* button, #174) available immediately. In-place adoption only (same mount key,
|
||||
* no list/messages invalidation — that is left to onTurnFinished at the end).
|
||||
* Idempotent and a no-op once the chat already has an id. */
|
||||
onServerChatId: (serverChatId?: string) => void;
|
||||
/** Disarm any pending error-path new-chat fallback. The window calls this from
|
||||
* startNewChat/selectChat so a late refetch can't yank the user back into a
|
||||
* just-failed chat after they explicitly moved on. */
|
||||
@@ -85,15 +102,21 @@ export function useChatSession(
|
||||
// `newThread`/`switchThread` to (re)mount, `adoptThread` for in-place adoption.
|
||||
// Initial: a non-null activeChatId switches to it; a null one gets a fresh
|
||||
// session key with no chat id yet.
|
||||
const [thread, dispatch] = useReducer(
|
||||
threadSessionReducer,
|
||||
undefined,
|
||||
() =>
|
||||
activeChatId === null
|
||||
? newThread(`new-${generateId()}`)
|
||||
: switchThread(activeChatId),
|
||||
const [thread, dispatch] = useReducer(threadSessionReducer, undefined, () =>
|
||||
activeChatId === null
|
||||
? newThread(`new-${generateId()}`)
|
||||
: switchThread(activeChatId),
|
||||
);
|
||||
|
||||
// Live mirror of the mounted thread's mount key, read by onTurnFinished to tell
|
||||
// the CURRENT thread from one ABANDONED by New chat mid-stream. @ai-sdk/react
|
||||
// does not abort a stream on unmount and proxies callbacks through a ref, so an
|
||||
// abandoned turn's onFinish/onError still fires AFTER its ChatThread unmounted;
|
||||
// matching its key against this ref keeps that late finish from adopting the
|
||||
// abandoned chat and yanking the user out of the fresh chat they opened (#161).
|
||||
const threadKeyRef = useRef(thread.key);
|
||||
threadKeyRef.current = thread.key;
|
||||
|
||||
// Error-path fallback for new-chat id adoption. When a brand-new chat's first
|
||||
// turn errors BEFORE the server's `start` chunk, no authoritative chatId ever
|
||||
// reaches the client, so the primary metadata adoption cannot run. We then ARM
|
||||
@@ -111,7 +134,23 @@ export function useChatSession(
|
||||
// yet) we adopt the server's AUTHORITATIVE streamed id (never the newest in the
|
||||
// list, which races a second tab — #137; see adopt-chat-id.ts).
|
||||
const onTurnFinished = useCallback(
|
||||
(serverChatId?: string) => {
|
||||
(serverChatId?: string, finishingThreadKey?: string) => {
|
||||
// Thread-aware guard (#161). A turn ABANDONED by "New chat" mid-stream still
|
||||
// fires onFinish/onError after its ChatThread unmounted (@ai-sdk/react does
|
||||
// not abort on unmount and proxies callbacks through a ref). If that late
|
||||
// finish ran the adoption path it would set activeChatId to the abandoned
|
||||
// chat's real id and yank the user out of the fresh chat they just opened.
|
||||
// So adopt / arm the fallback ONLY for the still-mounted thread; an
|
||||
// abandoned one merely refreshes the chat list (so the left-behind chat
|
||||
// surfaces in history) and does nothing else. A missing key (undefined)
|
||||
// means "current thread" — keeps old call sites/tests working.
|
||||
if (
|
||||
finishingThreadKey !== undefined &&
|
||||
finishingThreadKey !== threadKeyRef.current
|
||||
) {
|
||||
onInvalidateChatList();
|
||||
return;
|
||||
}
|
||||
// Read the live id from the ref, not the closure: on a failed turn this can
|
||||
// run twice in one turn (onFinish + onError) before any re-render, and the
|
||||
// primary branch below updates the ref so the second call sees the adopted id.
|
||||
@@ -150,6 +189,31 @@ export function useChatSession(
|
||||
[chats, setActiveChatId, onInvalidateChatList, onInvalidateChatMessages],
|
||||
);
|
||||
|
||||
// EARLY adoption (#174): adopt the authoritative streamed chat id the moment
|
||||
// the server emits it on the `start` chunk, so a brand-new chat gets its real
|
||||
// `activeChatId` WHILE its first turn streams — not only at terminal
|
||||
// onTurnFinished. This makes the activeChatId-gated Copy/export button
|
||||
// available during the first turn. Pure in-place adoption (same mount key, like
|
||||
// the primary path) with NO invalidation: the list/messages refresh stays on
|
||||
// onTurnFinished at the end of the turn. Reads the live id from the ref so a
|
||||
// repeat call after adoption is a no-op (resolveAdoptedChatId only fires for a
|
||||
// still-new chat).
|
||||
const onServerChatId = useCallback(
|
||||
(serverChatId?: string) => {
|
||||
const adopted = resolveAdoptedChatId(
|
||||
activeChatIdRef.current,
|
||||
serverChatId,
|
||||
);
|
||||
if (!adopted) return;
|
||||
activeChatIdRef.current = adopted;
|
||||
setActiveChatId(adopted);
|
||||
dispatch({ type: "adopt", chatId: adopted });
|
||||
// Early adoption beat the error-path fallback to it — disarm.
|
||||
pendingNewChatRef.current = null;
|
||||
},
|
||||
[setActiveChatId],
|
||||
);
|
||||
|
||||
// FALLBACK resolver. Armed only by onTurnFinished when a brand-new chat's first
|
||||
// turn errored before the `start` chunk (no authoritative id streamed). Once
|
||||
// the per-user list refetch lands with the just-created row, adopt the SINGLE
|
||||
@@ -229,10 +293,30 @@ export function useChatSession(
|
||||
pendingNewChatRef.current = null;
|
||||
}, []);
|
||||
|
||||
// Force a fresh, empty thread regardless of `activeChatId` (#161). The render-
|
||||
// phase reconciler only remounts when activeChatId diverges from thread.chatId,
|
||||
// so "New chat" pressed while a brand-new chat's first turn is still streaming
|
||||
// (activeChatId AND thread.chatId both null — the real id is adopted only at the
|
||||
// end of the turn) is a no-op for it and the abandoned thread/stream/history
|
||||
// would persist. Dispatching reconcile with a fresh key and chatId:null here
|
||||
// always produces a new mount key, so React remounts ChatThread (a clean useChat
|
||||
// store) and the post-dispatch state (activeChatId null === thread.chatId null)
|
||||
// keeps the reconciler from interfering. Also disarms any pending fallback.
|
||||
const startFreshThread = useCallback(() => {
|
||||
pendingNewChatRef.current = null;
|
||||
dispatch({
|
||||
type: "reconcile",
|
||||
chatId: null,
|
||||
newKey: `new-${generateId()}`,
|
||||
});
|
||||
}, []);
|
||||
|
||||
return {
|
||||
threadKey: thread.key,
|
||||
waitingForHistory,
|
||||
startFreshThread,
|
||||
onTurnFinished,
|
||||
onServerChatId,
|
||||
cancelPendingAdoption,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -50,6 +50,24 @@ export async function deleteAiChat(chatId: string): Promise<void> {
|
||||
await api.post("/ai-chat/delete", { chatId });
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a chat to Markdown (#183). The server renders the transcript from the
|
||||
* persisted rows (the DB is the single source of truth — including an
|
||||
* interrupted turn's in-progress row, persisted upfront + per step), so the
|
||||
* client just copies the returned string. `lang` localizes the few fixed
|
||||
* role/tool labels; defaults to English server-side when omitted.
|
||||
*/
|
||||
export async function exportAiChat(
|
||||
chatId: string,
|
||||
lang?: string,
|
||||
): Promise<string> {
|
||||
const req = await api.post<{ markdown: string }>("/ai-chat/export", {
|
||||
chatId,
|
||||
lang,
|
||||
});
|
||||
return req.data.markdown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Agent roles API (`/ai-chat/roles`). `list` is available to any workspace
|
||||
* member (for the chat-creation picker); create/update/delete are admin-only
|
||||
@@ -76,6 +94,8 @@ export async function updateAiRole(data: IAiRoleUpdate): Promise<IAiRole> {
|
||||
|
||||
/** Soft-delete a role (admin). */
|
||||
export async function deleteAiRole(id: string): Promise<{ success: true }> {
|
||||
const req = await api.post<{ success: true }>("/ai-chat/roles/delete", { id });
|
||||
const req = await api.post<{ success: true }>("/ai-chat/roles/delete", {
|
||||
id,
|
||||
});
|
||||
return req.data;
|
||||
}
|
||||
|
||||
@@ -116,6 +116,9 @@ export interface IAiChatMessageRow {
|
||||
// turn. Distinct from `usage` (legacy cumulative totalUsage). Shown in the
|
||||
// floating window's header badge.
|
||||
contextTokens?: number;
|
||||
// The model's max context window (denominator for the header badge); set
|
||||
// alongside contextTokens on a completed turn; absent on older rows.
|
||||
maxContextTokens?: number;
|
||||
// Set on an assistant row whose turn ended in a provider/stream error; the
|
||||
// raw provider error text (e.g. "402: ...") for inline display in the thread.
|
||||
error?: string;
|
||||
|
||||
@@ -1,747 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { buildChatMarkdown } from "@/features/ai-chat/utils/chat-markdown.ts";
|
||||
import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.ts";
|
||||
|
||||
/**
|
||||
* Tests for the client-only Markdown export builder. The output embeds a live
|
||||
* `new Date().toISOString()` export timestamp; we never assert that value, only
|
||||
* the deterministic structure (headings, numbering, fenced blocks, totals).
|
||||
*
|
||||
* A pass-through translator keeps role/tool labels predictable so the
|
||||
* structural assertions are stable without an i18n runtime.
|
||||
*/
|
||||
const t = (key: string, values?: Record<string, unknown>): string => {
|
||||
if (values && typeof values.name === "string") {
|
||||
return key.replace("{{name}}", values.name);
|
||||
}
|
||||
return key;
|
||||
};
|
||||
|
||||
function row(partial: Partial<IAiChatMessageRow>): IAiChatMessageRow {
|
||||
return {
|
||||
id: partial.id ?? "id",
|
||||
role: partial.role ?? "user",
|
||||
content: partial.content ?? null,
|
||||
metadata: partial.metadata ?? null,
|
||||
createdAt: partial.createdAt ?? "2026-06-21T00:00:00.000Z",
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildChatMarkdown — structure", () => {
|
||||
it("emits the title heading, chat id and message count", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "My chat",
|
||||
chatId: "chat-123",
|
||||
rows: [],
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("# My chat");
|
||||
expect(md).toContain("- Chat ID: `chat-123`");
|
||||
expect(md).toContain("- Messages: 0");
|
||||
expect(md).toContain("- Exported:"); // timestamp present, value not asserted
|
||||
});
|
||||
|
||||
it("falls back to the translated 'Untitled chat' for empty/blank titles", () => {
|
||||
expect(
|
||||
buildChatMarkdown({ title: null, chatId: "c", rows: [], t }),
|
||||
).toContain("# Untitled chat");
|
||||
expect(
|
||||
buildChatMarkdown({ title: " ", chatId: "c", rows: [], t }),
|
||||
).toContain("# Untitled chat");
|
||||
});
|
||||
|
||||
it("numbers rows sequentially with role headings", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({ role: "user", content: "hi" }),
|
||||
row({ role: "assistant", content: "hello" }),
|
||||
row({ role: "user", content: "again" }),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("## 1. You");
|
||||
expect(md).toContain("## 2. AI agent");
|
||||
expect(md).toContain("## 3. You");
|
||||
// Heading numbering is strictly index+1, not e.g. role-relative.
|
||||
expect(md).not.toContain("## 0.");
|
||||
});
|
||||
|
||||
it("renders the per-row text content from `content` when no metadata.parts", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [row({ role: "user", content: "plain body" })],
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("plain body");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildChatMarkdown — text parts", () => {
|
||||
it("skips empty / whitespace-only text parts", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "ignored-content",
|
||||
metadata: {
|
||||
parts: [
|
||||
{ type: "text", text: " " },
|
||||
{ type: "text", text: "" },
|
||||
{ type: "text", text: "kept line" },
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
] as any,
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("kept line");
|
||||
// Whitespace-only part contributed no block of its own.
|
||||
expect(md).not.toContain(" \n\n");
|
||||
// When metadata.parts exists, the plain `content` fallback is NOT used.
|
||||
expect(md).not.toContain("ignored-content");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildChatMarkdown — tool parts", () => {
|
||||
it("renders a tool label, name, state and fenced Input/Output blocks", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "",
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: "tool-getPage",
|
||||
state: "output-available",
|
||||
input: { pageId: "p1" },
|
||||
output: { id: "p1", title: "Home" },
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any,
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
// Known tool name maps to its label key; raw name in backticks; done state.
|
||||
expect(md).toContain("**Tool: Read page** (`getPage`) — done");
|
||||
expect(md).toContain("Input:");
|
||||
expect(md).toContain("Output:");
|
||||
// Fenced JSON blocks contain the stringified payloads.
|
||||
expect(md).toContain('"pageId": "p1"');
|
||||
expect(md).toContain('"title": "Home"');
|
||||
expect(md).toContain("```json");
|
||||
});
|
||||
|
||||
it("renders the generic label for an unknown tool and surfaces errorText", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "",
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: "tool-mysteryTool",
|
||||
state: "output-error",
|
||||
input: { a: 1 },
|
||||
errorText: "boom",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any,
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(md).toContain(
|
||||
"**Tool: Ran tool mysteryTool** (`mysteryTool`) — error",
|
||||
);
|
||||
expect(md).toContain("**Error:** boom");
|
||||
});
|
||||
|
||||
it("does not throw on a circular tool input (falls back to String)", () => {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const circular: any = {};
|
||||
circular.self = circular;
|
||||
expect(() =>
|
||||
buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "",
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: "tool-getPage",
|
||||
state: "input-available",
|
||||
input: circular,
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any,
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildChatMarkdown — fence anti-breakout", () => {
|
||||
it("lengthens the delimiter so embedded ``` cannot break out of the block", () => {
|
||||
// Tool input whose stringified string form contains a literal ``` run.
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "",
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: "tool-getPage",
|
||||
state: "output-available",
|
||||
// A bare string passes through stringify() verbatim.
|
||||
input: "before ``` after",
|
||||
output: "x",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any,
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
// The fence around the 3-backtick content must use at least 4 backticks so
|
||||
// the embedded ``` run cannot terminate the block.
|
||||
expect(md).toContain("````json\nbefore ``` after\n````");
|
||||
// Robust anti-breakout check: the opening fence delimiter is strictly
|
||||
// longer than the longest backtick run inside the wrapped content. (A naive
|
||||
// `not.toContain("```json...")` is a false negative — a 4-backtick fence
|
||||
// textually contains the 3-backtick substring.)
|
||||
const open = md.match(/(`{3,})json\nbefore/);
|
||||
expect(open).not.toBeNull();
|
||||
expect(open![1].length).toBeGreaterThan(3); // > the 3-backtick run in content
|
||||
});
|
||||
|
||||
it("uses a 5-backtick fence when the content has a 4-backtick run", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "",
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: "tool-getPage",
|
||||
state: "output-available",
|
||||
input: "a ```` b",
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
} as any,
|
||||
],
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("`````json\na ```` b\n`````");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildChatMarkdown — token totals", () => {
|
||||
it("prints the total-tokens line only when the summed usage is > 0", () => {
|
||||
const withTokens = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "x",
|
||||
metadata: { usage: { inputTokens: 10, outputTokens: 5 } },
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(withTokens).toContain("- Total tokens: 15");
|
||||
// Per-row usage footer too.
|
||||
expect(withTokens).toContain("_Tokens — in: 10, out: 5, total: 15_");
|
||||
});
|
||||
|
||||
it("omits the total-tokens line when the sum is 0 / usage absent", () => {
|
||||
const noTokens = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({ role: "user", content: "hi" }),
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "x",
|
||||
metadata: { usage: { inputTokens: 0, outputTokens: 0 } },
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(noTokens).not.toContain("- Total tokens:");
|
||||
});
|
||||
|
||||
it("uses totalTokens when present rather than summing in/out", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "x",
|
||||
metadata: {
|
||||
usage: { inputTokens: 3, outputTokens: 4, totalTokens: 99 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("- Total tokens: 99");
|
||||
});
|
||||
|
||||
it("appends the reasoning figure to the row footer when reasoningTokens > 0", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "x",
|
||||
metadata: {
|
||||
usage: { inputTokens: 10, outputTokens: 8, reasoningTokens: 3 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("_Tokens — in: 10, out: 8, reasoning: 3, total: 18_");
|
||||
});
|
||||
|
||||
it("omits the reasoning figure when reasoningTokens is 0 / absent", () => {
|
||||
const zero = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "x",
|
||||
metadata: {
|
||||
usage: { inputTokens: 10, outputTokens: 5, reasoningTokens: 0 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(zero).toContain("_Tokens — in: 10, out: 5, total: 15_");
|
||||
expect(zero).not.toContain("reasoning:");
|
||||
|
||||
const absent = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "x",
|
||||
metadata: { usage: { inputTokens: 10, outputTokens: 5 } },
|
||||
}),
|
||||
],
|
||||
t,
|
||||
});
|
||||
expect(absent).not.toContain("reasoning:");
|
||||
});
|
||||
});
|
||||
|
||||
// A minimal on-screen (live) message, matching the subset buildChatMarkdown reads.
|
||||
function live(partial: {
|
||||
id?: string;
|
||||
role?: string;
|
||||
parts?: { type: string; text?: string }[];
|
||||
metadata?: { usage?: Record<string, number>; error?: string };
|
||||
}) {
|
||||
return {
|
||||
id: partial.id ?? "live-id",
|
||||
role: partial.role ?? "assistant",
|
||||
parts: partial.parts ?? [],
|
||||
metadata: partial.metadata,
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildChatMarkdown — live (WYSIWYG) source", () => {
|
||||
it("uses the live messages as the document (what's on screen), numbered from 1", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
// Persisted rows hold only the user turn; the assistant reply is live-only.
|
||||
rows: [row({ id: "u1", role: "user", content: "persisted user" })],
|
||||
live: [
|
||||
live({
|
||||
id: "u1",
|
||||
role: "user",
|
||||
parts: [{ type: "text", text: "on-screen user" }],
|
||||
}),
|
||||
live({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "on-screen reply" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: false,
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("## 1. You");
|
||||
expect(md).toContain("## 2. AI agent");
|
||||
expect(md).toContain("on-screen user");
|
||||
expect(md).toContain("on-screen reply");
|
||||
// Message count reflects the LIVE document, not rows + live.
|
||||
expect(md).toContain("- Messages: 2");
|
||||
});
|
||||
|
||||
it("captures a partial reply from an interrupted (non-streaming) turn — no 'generating' note", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [row({ id: "u1", role: "user", content: "q" })],
|
||||
live: [
|
||||
live({ id: "u1", role: "user", parts: [{ type: "text", text: "q" }] }),
|
||||
live({
|
||||
id: "a-live",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "partial plan before the drop" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: false, // the stream dropped — not streaming anymore
|
||||
banner: "Connection lost — the answer was interrupted.",
|
||||
t,
|
||||
});
|
||||
// The partial assistant answer that was on screen IS in the export.
|
||||
expect(md).toContain("partial plan before the drop");
|
||||
// It is NOT flagged still-generating (the turn is over, just interrupted).
|
||||
expect(md).not.toContain("still being generated");
|
||||
// The on-screen banner is recorded at the end.
|
||||
expect(md).toContain("Connection lost — the answer was interrupted.");
|
||||
});
|
||||
|
||||
it("flags ONLY the tail assistant as still generating, and only while streaming", () => {
|
||||
const streaming = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [],
|
||||
live: [
|
||||
live({
|
||||
id: "a",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "done earlier" }],
|
||||
}),
|
||||
live({
|
||||
id: "u",
|
||||
role: "user",
|
||||
parts: [{ type: "text", text: "next q" }],
|
||||
}),
|
||||
live({
|
||||
id: "b",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "streaming now" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: true,
|
||||
t,
|
||||
});
|
||||
// Exactly one "still being generated" note (the tail assistant).
|
||||
expect(streaming.match(/still being generated/g)?.length).toBe(1);
|
||||
|
||||
const idle = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [],
|
||||
live: [
|
||||
live({
|
||||
id: "b",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "final" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: false,
|
||||
t,
|
||||
});
|
||||
expect(idle).not.toContain("still being generated");
|
||||
});
|
||||
|
||||
it("does NOT flag a completed assistant as generating when the streaming tail is a user message", () => {
|
||||
// The `status === "submitted"` window: the user just sent, isStreaming is
|
||||
// already true, but the new assistant turn has no message yet so the tail is
|
||||
// the USER message. The previous assistant answer is complete on screen and
|
||||
// must not be marked still-generating (WYSIWYG; regression for #160 review).
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [],
|
||||
live: [
|
||||
live({
|
||||
id: "a",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "completed answer" }],
|
||||
}),
|
||||
live({
|
||||
id: "u",
|
||||
role: "user",
|
||||
parts: [{ type: "text", text: "the new question" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: true,
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("completed answer");
|
||||
expect(md).not.toContain("still being generated");
|
||||
});
|
||||
|
||||
it("emits the heading + note for a streaming tail assistant with empty parts", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [row({ id: "u1", role: "user", content: "q" })],
|
||||
live: [
|
||||
live({ id: "u1", role: "user", parts: [{ type: "text", text: "q" }] }),
|
||||
live({ id: "a-live", role: "assistant", parts: [] }),
|
||||
],
|
||||
isStreaming: true,
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("## 2. AI agent");
|
||||
expect(md).toContain("still being generated");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildChatMarkdown — live enrichment from persisted rows", () => {
|
||||
it("pulls usage / error / timestamp from the persisted row matched by id", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
content: "x",
|
||||
createdAt: "2026-06-22T10:00:00.000Z",
|
||||
metadata: {
|
||||
usage: { inputTokens: 10, outputTokens: 5 },
|
||||
error: "rate limited",
|
||||
},
|
||||
}),
|
||||
],
|
||||
live: [
|
||||
// Same id as the persisted row, but no usage/error/timestamp on the live msg.
|
||||
live({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "reply" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: false,
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("reply");
|
||||
// Token footer + total come from the enriched row.
|
||||
expect(md).toContain("_Tokens — in: 10, out: 5, total: 15_");
|
||||
expect(md).toContain("- Total tokens: 15");
|
||||
expect(md).toContain("**⚠️ Error:** rate limited");
|
||||
// The persisted timestamp is carried into the export.
|
||||
expect(md).toContain("<!-- 2026-06-22T10:00:00.000Z -->");
|
||||
});
|
||||
|
||||
it("prefers authoritative usage already on the live message over the row's", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
content: "x",
|
||||
metadata: {
|
||||
usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
live: [
|
||||
live({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "reply" }],
|
||||
metadata: {
|
||||
usage: { inputTokens: 100, outputTokens: 50, totalTokens: 150 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
isStreaming: false,
|
||||
t,
|
||||
});
|
||||
// The live (authoritative, freshest) usage wins, not the stale row usage.
|
||||
expect(md).toContain("- Total tokens: 150");
|
||||
expect(md).not.toContain("- Total tokens: 2");
|
||||
});
|
||||
|
||||
it("a current-turn live message with no matching row renders without a footer", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [row({ id: "u1", role: "user", content: "q" })],
|
||||
live: [
|
||||
live({ id: "u1", role: "user", parts: [{ type: "text", text: "q" }] }),
|
||||
live({
|
||||
id: "a-live",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "fresh reply" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: false,
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("fresh reply");
|
||||
// No persisted row for the live assistant -> no token footer, no timestamp.
|
||||
expect(md).not.toContain("_Tokens —");
|
||||
expect(md).not.toContain("<!-- undefined -->");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildChatMarkdown — fallback + banner", () => {
|
||||
it("falls back to the persisted rows when there are no live messages", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [
|
||||
row({ role: "user", content: "from rows" }),
|
||||
row({
|
||||
role: "assistant",
|
||||
content: "answer",
|
||||
metadata: { usage: { inputTokens: 4, outputTokens: 6 } },
|
||||
}),
|
||||
],
|
||||
live: [], // empty live mirror -> fallback path
|
||||
isStreaming: false,
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("## 1. You");
|
||||
expect(md).toContain("## 2. AI agent");
|
||||
expect(md).toContain("from rows");
|
||||
expect(md).toContain("- Messages: 2");
|
||||
expect(md).toContain("- Total tokens: 10");
|
||||
});
|
||||
|
||||
it("appends the on-screen banner once, after the messages", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [row({ role: "user", content: "q" })],
|
||||
live: [
|
||||
live({ id: "u", role: "user", parts: [{ type: "text", text: "q" }] }),
|
||||
],
|
||||
isStreaming: false,
|
||||
banner: "Rate limit reached — try again shortly.",
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("_⚠️ Rate limit reached — try again shortly._");
|
||||
// Banner comes after the (only) message block.
|
||||
expect(md.indexOf("Rate limit reached")).toBeGreaterThan(
|
||||
md.indexOf("## 1."),
|
||||
);
|
||||
});
|
||||
|
||||
it("omits the banner block when there is no banner", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: "t",
|
||||
chatId: "c",
|
||||
rows: [row({ role: "user", content: "q" })],
|
||||
live: [
|
||||
live({ id: "u", role: "user", parts: [{ type: "text", text: "q" }] }),
|
||||
],
|
||||
isStreaming: false,
|
||||
banner: null,
|
||||
t,
|
||||
});
|
||||
expect(md).not.toContain("_⚠️");
|
||||
});
|
||||
});
|
||||
|
||||
// #174: a brand-new, not-yet-persisted chat whose first turn is streaming (or was
|
||||
// interrupted) has live messages but NO persisted rows yet, and its chat id is not
|
||||
// known (the caller passes a placeholder). The export must still capture the
|
||||
// on-screen thread WYSIWYG from the live messages alone.
|
||||
describe("buildChatMarkdown — first-turn export with no persisted base (#174)", () => {
|
||||
it("builds the document from live messages alone when rows are empty", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: null,
|
||||
chatId: "unsaved",
|
||||
rows: [],
|
||||
live: [
|
||||
live({
|
||||
id: "u1",
|
||||
role: "user",
|
||||
parts: [{ type: "text", text: "hello" }],
|
||||
}),
|
||||
live({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "partial reply" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: true,
|
||||
t,
|
||||
});
|
||||
// Both on-screen messages are serialized, numbered from 1.
|
||||
expect(md).toContain("## 1. You");
|
||||
expect(md).toContain("hello");
|
||||
expect(md).toContain("## 2. AI agent");
|
||||
expect(md).toContain("partial reply");
|
||||
// The streaming tail assistant is flagged as in-progress.
|
||||
expect(md).toContain("still being generated");
|
||||
// The placeholder chat id and the live message count are recorded.
|
||||
expect(md).toContain("- Chat ID: `unsaved`");
|
||||
expect(md).toContain("- Messages: 2");
|
||||
// No persisted timestamp exists for a current-turn live message.
|
||||
expect(md).not.toContain("<!--");
|
||||
});
|
||||
|
||||
it("captures an interrupted first turn (no rows, not streaming) without a generating note", () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: null,
|
||||
chatId: "unsaved",
|
||||
rows: [],
|
||||
live: [
|
||||
live({ id: "u1", role: "user", parts: [{ type: "text", text: "q" }] }),
|
||||
live({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
parts: [{ type: "text", text: "half an answer" }],
|
||||
}),
|
||||
],
|
||||
isStreaming: false,
|
||||
banner: "Connection dropped — the response was cut off.",
|
||||
t,
|
||||
});
|
||||
expect(md).toContain("half an answer");
|
||||
// An interrupted (non-streaming) partial is exported as-is, no generating note.
|
||||
expect(md).not.toContain("still being generated");
|
||||
// The on-screen banner records the interruption.
|
||||
expect(md).toContain("_⚠️ Connection dropped — the response was cut off._");
|
||||
});
|
||||
});
|
||||
@@ -1,308 +0,0 @@
|
||||
/**
|
||||
* Client-only Markdown builder for an AI agent chat. Serializes the already
|
||||
* persisted message rows (loaded via `useAiChatMessagesQuery`) into a single
|
||||
* Markdown string suitable for copying to the clipboard. NO network call is
|
||||
* made and NO server/DB code is touched — this reuses the rich "request
|
||||
* internals" (tool calls with input/output, per-message token usage,
|
||||
* finish/error info) that the chat already holds client-side.
|
||||
*
|
||||
* Only role labels and tool action labels are localized via the passed-in `t`
|
||||
* translator; the structural document words (Input/Output/Error/Tokens/...) are
|
||||
* plain English constants because the output is a technical artifact.
|
||||
*/
|
||||
|
||||
import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.ts";
|
||||
import {
|
||||
ToolUiPart,
|
||||
getToolName,
|
||||
toolRunState,
|
||||
toolLabelKey,
|
||||
} from "@/features/ai-chat/utils/tool-parts.tsx";
|
||||
|
||||
// Minimal translator signature compatible with react-i18next's `t`.
|
||||
type Translate = (key: string, values?: Record<string, unknown>) => string;
|
||||
|
||||
interface BuildChatMarkdownArgs {
|
||||
title: string | null;
|
||||
chatId: string;
|
||||
/** The live, on-screen messages — the WYSIWYG source of the export. When
|
||||
* present and non-empty these DRIVE the document (so it mirrors exactly what
|
||||
* the user sees, including a partial reply from an interrupted turn). Each is
|
||||
* matched to a persisted row by `id` to enrich it with token usage / error /
|
||||
* timestamp. When absent or empty the builder falls back to `rows`. */
|
||||
live?: LiveMessage[];
|
||||
/** Persisted message rows. Enrichment source (matched to `live` by id) AND the
|
||||
* fallback document source when `live` is empty. */
|
||||
rows: IAiChatMessageRow[];
|
||||
/** Whether the live thread is still streaming. Only then is the tail assistant
|
||||
* message flagged "still generating"; an interrupted (non-streaming) partial
|
||||
* reply is exported as-is and the `banner` explains the interruption. */
|
||||
isStreaming?: boolean;
|
||||
/** The on-screen banner text (error / dropped connection / manual stop),
|
||||
* appended at the end of the export so the artifact records the interruption
|
||||
* the user saw. */
|
||||
banner?: string | null;
|
||||
t: Translate;
|
||||
}
|
||||
|
||||
/** A single AI SDK UIMessage part (text part or other). */
|
||||
interface TextLikePart {
|
||||
type: string;
|
||||
text?: string;
|
||||
}
|
||||
|
||||
/** Authoritative per-turn usage the server attaches to a message / row. */
|
||||
interface UsageLike {
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
totalTokens?: number;
|
||||
reasoningTokens?: number;
|
||||
}
|
||||
|
||||
/** A live, on-screen message (subset of the AI SDK UIMessage we consume). */
|
||||
interface LiveMessage {
|
||||
id: string;
|
||||
role: "user" | "assistant" | string;
|
||||
parts: TextLikePart[];
|
||||
metadata?: { usage?: UsageLike; error?: string };
|
||||
}
|
||||
|
||||
/** One message normalized for rendering, regardless of live/persisted origin. */
|
||||
interface ExportItem {
|
||||
role: string;
|
||||
parts: TextLikePart[];
|
||||
usage?: UsageLike;
|
||||
error?: string;
|
||||
/** ISO timestamp from the persisted row, when one is known. */
|
||||
createdAt?: string;
|
||||
/** True only for the tail assistant message while the thread is streaming. */
|
||||
generating: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stringify an arbitrary tool input/output value for a fenced block. Strings
|
||||
* pass through as-is; everything else is pretty-printed JSON, falling back to
|
||||
* `String(value)` if serialization throws (e.g. a circular structure).
|
||||
*/
|
||||
function stringify(value: unknown): string {
|
||||
if (typeof value === "string") return value;
|
||||
try {
|
||||
return JSON.stringify(value, null, 2);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap `code` in a fenced code block whose backtick delimiter is LONGER than
|
||||
* the longest backtick run inside the content, so embedded backticks (or even
|
||||
* a literal ``` fence) never break out of the block. Minimum 3 backticks.
|
||||
*/
|
||||
function fence(code: string, lang = ""): string {
|
||||
const runs: string[] = code.match(/`+/g) ?? [];
|
||||
const longest = runs.reduce((m, s) => Math.max(m, s.length), 0);
|
||||
const delim = "`".repeat(Math.max(3, longest + 1));
|
||||
return `${delim}${lang}\n${code}\n${delim}`;
|
||||
}
|
||||
|
||||
/** Per-row token count, mirroring the header sum in ai-chat-window.tsx. */
|
||||
function rowTokens(usage: {
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
totalTokens?: number;
|
||||
reasoningTokens?: number;
|
||||
}): number {
|
||||
return (
|
||||
usage.totalTokens ?? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0)
|
||||
);
|
||||
}
|
||||
|
||||
/** Render one message's UIMessage parts into an array of Markdown blocks
|
||||
* (text blocks + tool blocks). Mirrors MessageItem's part handling. */
|
||||
function renderMessageParts(parts: TextLikePart[], t: Translate): string[] {
|
||||
const out: string[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
if (part.type === "text") {
|
||||
const text = (part.text ?? "").trim();
|
||||
// Skip empty/whitespace-only text parts (matches MessageItem).
|
||||
if (text.length > 0) out.push(text);
|
||||
continue;
|
||||
}
|
||||
|
||||
const isToolPart =
|
||||
part.type.startsWith("tool-") || part.type === "dynamic-tool";
|
||||
if (!isToolPart) continue;
|
||||
|
||||
const tp = part as unknown as ToolUiPart;
|
||||
const name = getToolName(tp);
|
||||
const { key, values } = toolLabelKey(name);
|
||||
const label = t(key, values);
|
||||
const state = toolRunState(tp.state);
|
||||
|
||||
const toolLines: string[] = [
|
||||
`**Tool: ${label}** (\`${name}\`) — ${state}`,
|
||||
];
|
||||
if (tp.input !== undefined) {
|
||||
toolLines.push("Input:");
|
||||
toolLines.push(fence(stringify(tp.input), "json"));
|
||||
}
|
||||
if (tp.output !== undefined) {
|
||||
toolLines.push("Output:");
|
||||
toolLines.push(fence(stringify(tp.output), "json"));
|
||||
}
|
||||
if (tp.errorText) {
|
||||
toolLines.push(`**Error:** ${tp.errorText}`);
|
||||
}
|
||||
out.push(toolLines.join("\n\n"));
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Resolve a persisted row's parts: prefer the rich persisted parts, else a
|
||||
* single text part built from the plain-text content (mirrors `rowToUiMessage`). */
|
||||
function rowParts(row: IAiChatMessageRow): TextLikePart[] {
|
||||
return Array.isArray(row.metadata?.parts) && row.metadata.parts.length > 0
|
||||
? (row.metadata.parts as TextLikePart[])
|
||||
: [{ type: "text", text: row.content ?? "" }];
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalize the export to one ordered list of {@link ExportItem}, WYSIWYG-first:
|
||||
*
|
||||
* - When `live` messages are present, THEY are the document (what the user sees,
|
||||
* incl. an interrupted turn's partial reply). Each is matched to a persisted
|
||||
* row by `id` to pull token usage / error / timestamp — a live message of the
|
||||
* CURRENT turn has no matching row yet, so it simply renders without a footer.
|
||||
* Authoritative `usage`/`error` already on the live message metadata win over
|
||||
* the row (the server attaches usage to the streamed message at a step
|
||||
* boundary before the row is refetched). Only the tail assistant message is
|
||||
* flagged `generating`, and only while `isStreaming`.
|
||||
* - When `live` is empty (e.g. the export runs before the live mirror is
|
||||
* populated), fall back to the persisted `rows` so the format never regresses.
|
||||
*/
|
||||
function resolveItems(
|
||||
live: LiveMessage[] | undefined,
|
||||
rows: IAiChatMessageRow[],
|
||||
isStreaming: boolean,
|
||||
): ExportItem[] {
|
||||
if (live && live.length > 0) {
|
||||
const rowsById = new Map(rows.map((r) => [r.id, r]));
|
||||
// The "still generating" note may apply ONLY to an assistant message that is
|
||||
// the actual TAIL of the list — that is where the on-screen typing indicator
|
||||
// sits. While `status === "submitted"` (isStreaming true) right after the
|
||||
// user hit send, the tail is the USER message and the new assistant turn has
|
||||
// no message yet; the previous assistant answer is shown complete on screen,
|
||||
// so it must NOT be flagged (the indicator renders as a separate bottom
|
||||
// block, not on that answer).
|
||||
const lastIndex = live.length - 1;
|
||||
const tailIsStreamingAssistant =
|
||||
isStreaming && live[lastIndex]?.role === "assistant";
|
||||
return live.map((m, i) => {
|
||||
const row = rowsById.get(m.id);
|
||||
return {
|
||||
role: m.role,
|
||||
parts: m.parts ?? [],
|
||||
// Authoritative usage/error already on the live message (the server
|
||||
// attaches usage to the streamed message at a step boundary) wins over
|
||||
// the persisted row; a current-turn live message has no matching row yet
|
||||
// and simply renders without a token footer (the accepted WYSIWYG
|
||||
// tradeoff — an interrupted turn loses only its token footer, not text).
|
||||
usage: m.metadata?.usage ?? row?.metadata?.usage,
|
||||
error: m.metadata?.error ?? row?.metadata?.error ?? undefined,
|
||||
createdAt: row?.createdAt,
|
||||
generating: tailIsStreamingAssistant && i === lastIndex,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
return rows.map((row) => ({
|
||||
role: row.role,
|
||||
parts: rowParts(row),
|
||||
usage: row.metadata?.usage,
|
||||
error: row.metadata?.error ?? undefined,
|
||||
createdAt: row.createdAt,
|
||||
generating: false,
|
||||
}));
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a chat to a Markdown string. Pure (apart from `new Date()` for the
|
||||
* export timestamp), so it is straightforward to unit-test.
|
||||
*/
|
||||
export function buildChatMarkdown(args: BuildChatMarkdownArgs): string {
|
||||
const { title, chatId, live, rows, isStreaming, banner, t } = args;
|
||||
const blocks: string[] = [];
|
||||
|
||||
const items = resolveItems(live, rows, isStreaming === true);
|
||||
|
||||
const heading = (title ?? "").trim() || t("Untitled chat");
|
||||
blocks.push(`# ${heading}`);
|
||||
|
||||
// Metadata bullet list. Total tokens is only shown when there is a sum.
|
||||
const totalTokens = items.reduce(
|
||||
(sum, item) => (item.usage ? sum + rowTokens(item.usage) : sum),
|
||||
0,
|
||||
);
|
||||
const meta = [
|
||||
`- Chat ID: \`${chatId}\``,
|
||||
`- Exported: ${new Date().toISOString()}`,
|
||||
`- Messages: ${items.length}`,
|
||||
];
|
||||
if (totalTokens > 0) meta.push(`- Total tokens: ${totalTokens}`);
|
||||
blocks.push(meta.join("\n"));
|
||||
|
||||
items.forEach((item, index) => {
|
||||
blocks.push("---");
|
||||
|
||||
const roleLabel = item.role === "assistant" ? t("AI agent") : t("You");
|
||||
blocks.push(`## ${index + 1}. ${roleLabel}`);
|
||||
|
||||
// Created-at kept in source as an HTML comment (out of the rendered prose).
|
||||
// A live message of the current turn has no persisted row yet — omit it.
|
||||
if (item.createdAt) blocks.push(`<!-- ${item.createdAt} -->`);
|
||||
|
||||
blocks.push(...renderMessageParts(item.parts, t));
|
||||
|
||||
// A generating assistant may have empty/no parts yet — the heading (above)
|
||||
// and this note still record the in-progress turn.
|
||||
if (item.generating) {
|
||||
blocks.push(
|
||||
"_⏳ This message is still being generated — the export captured a partial, in-progress response._",
|
||||
);
|
||||
}
|
||||
|
||||
// A persisted per-message error (the raw provider text) may coexist with the
|
||||
// trailing `banner` (the classified on-screen alert) when the failed turn's
|
||||
// row has already been refetched by export time. They describe the same
|
||||
// failure at different fidelity; showing both is an accepted, minor redundancy.
|
||||
if (item.error) {
|
||||
blocks.push(`**⚠️ Error:** ${item.error}`);
|
||||
}
|
||||
|
||||
const usage = item.usage;
|
||||
if (usage) {
|
||||
const total = usage.totalTokens ?? rowTokens(usage);
|
||||
// Reasoning (thinking) tokens are shown only when the provider reported a
|
||||
// positive count; old rows / non-reasoning providers omit it.
|
||||
const reasoning =
|
||||
usage.reasoningTokens && usage.reasoningTokens > 0
|
||||
? `, reasoning: ${usage.reasoningTokens}`
|
||||
: "";
|
||||
blocks.push(
|
||||
`_Tokens — in: ${usage.inputTokens ?? "?"}, out: ${usage.outputTokens ?? "?"}${reasoning}, total: ${total}_`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Record the on-screen banner (error / dropped connection / manual stop) so
|
||||
// the export reflects exactly what the user saw, including an interruption.
|
||||
if (banner && banner.trim().length > 0) {
|
||||
blocks.push("---");
|
||||
blocks.push(`_⚠️ ${banner.trim()}_`);
|
||||
}
|
||||
|
||||
// Blank line between blocks so the Markdown renders cleanly.
|
||||
return blocks.join("\n\n");
|
||||
}
|
||||
90
apps/client/src/features/ai-chat/utils/context-badge.test.ts
Normal file
90
apps/client/src/features/ai-chat/utils/context-badge.test.ts
Normal file
@@ -0,0 +1,90 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.ts";
|
||||
import { selectContextBadge } from "@/features/ai-chat/utils/context-badge.ts";
|
||||
|
||||
/**
|
||||
* Pure-helper tests for the header context badge selection. Covers the two
|
||||
* non-obvious rules: numerator and denominator are each taken from the most
|
||||
* recent row carrying THAT value (they may live on different rows), and a fresh
|
||||
* row with a zero/absent value must NOT shadow an older positive one.
|
||||
*/
|
||||
const row = (metadata: IAiChatMessageRow["metadata"]): IAiChatMessageRow => ({
|
||||
id: Math.random().toString(),
|
||||
role: "assistant",
|
||||
content: null,
|
||||
metadata,
|
||||
createdAt: "2026-01-01T00:00:00.000Z",
|
||||
});
|
||||
|
||||
describe("selectContextBadge", () => {
|
||||
it("returns zeros for empty / nullish input", () => {
|
||||
expect(selectContextBadge(undefined)).toEqual({
|
||||
contextTokens: 0,
|
||||
maxContextTokens: 0,
|
||||
});
|
||||
expect(selectContextBadge(null)).toEqual({
|
||||
contextTokens: 0,
|
||||
maxContextTokens: 0,
|
||||
});
|
||||
expect(selectContextBadge([])).toEqual({
|
||||
contextTokens: 0,
|
||||
maxContextTokens: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("reads both figures from the most recent row that carries them", () => {
|
||||
expect(
|
||||
selectContextBadge([
|
||||
row({ contextTokens: 100, maxContextTokens: 200000 }),
|
||||
row({ contextTokens: 1500, maxContextTokens: 200000 }),
|
||||
]),
|
||||
).toEqual({ contextTokens: 1500, maxContextTokens: 200000 });
|
||||
});
|
||||
|
||||
it("falls back to legacy usage total for older rows without contextTokens", () => {
|
||||
expect(
|
||||
selectContextBadge([
|
||||
row({ usage: { inputTokens: 30, outputTokens: 70 } }),
|
||||
]),
|
||||
).toEqual({ contextTokens: 100, maxContextTokens: 0 });
|
||||
|
||||
expect(
|
||||
selectContextBadge([row({ usage: { totalTokens: 250 } })]),
|
||||
).toEqual({ contextTokens: 250, maxContextTokens: 0 });
|
||||
});
|
||||
|
||||
it("takes numerator and denominator from different rows", () => {
|
||||
// Freshest row (an error turn) carries contextTokens but no max; the older
|
||||
// completed turn carries the max. Each is picked from its own latest row.
|
||||
expect(
|
||||
selectContextBadge([
|
||||
row({ contextTokens: 800, maxContextTokens: 200000 }),
|
||||
row({ contextTokens: 1200, error: "402: nope" }),
|
||||
]),
|
||||
).toEqual({ contextTokens: 1200, maxContextTokens: 200000 });
|
||||
});
|
||||
|
||||
it("does not let a fresh zero/absent max shadow an older positive max", () => {
|
||||
expect(
|
||||
selectContextBadge([
|
||||
row({ contextTokens: 100, maxContextTokens: 200000 }),
|
||||
row({ contextTokens: 1200, maxContextTokens: 0 }),
|
||||
]),
|
||||
).toEqual({ contextTokens: 1200, maxContextTokens: 200000 });
|
||||
});
|
||||
|
||||
it("skips rows with null metadata", () => {
|
||||
expect(
|
||||
selectContextBadge([
|
||||
row({ contextTokens: 500, maxContextTokens: 200000 }),
|
||||
row(null),
|
||||
]),
|
||||
).toEqual({ contextTokens: 500, maxContextTokens: 200000 });
|
||||
});
|
||||
|
||||
it("reports current > max as-is (no clamp)", () => {
|
||||
expect(
|
||||
selectContextBadge([row({ contextTokens: 250000, maxContextTokens: 200000 })]),
|
||||
).toEqual({ contextTokens: 250000, maxContextTokens: 200000 });
|
||||
});
|
||||
});
|
||||
49
apps/client/src/features/ai-chat/utils/context-badge.ts
Normal file
49
apps/client/src/features/ai-chat/utils/context-badge.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.ts";
|
||||
|
||||
/**
|
||||
* Derive the header context badge figures from the persisted message rows.
|
||||
*
|
||||
* - `contextTokens` (numerator): how much the conversation now occupies in the
|
||||
* model's context window. Read from the most recent row carrying a context
|
||||
* figure — `contextTokens` (final-step input+output) on rows recorded after
|
||||
* this shipped, else that turn's legacy `usage` total for older rows.
|
||||
* - `maxContextTokens` (denominator): the model's configured max window, stamped
|
||||
* alongside `contextTokens` on a completed turn.
|
||||
*
|
||||
* Each value is taken from the most recent row carrying THAT value
|
||||
* independently — they may land on different rows (e.g. a fresh error row can
|
||||
* carry `contextTokens` but not `maxContextTokens`), so the scan continues for
|
||||
* whichever is still unset. `0` means "no row has it" (older rows, or no
|
||||
* admin-configured limit); the badge then omits the value.
|
||||
*/
|
||||
export function selectContextBadge(
|
||||
messageRows: readonly IAiChatMessageRow[] | undefined | null,
|
||||
): { contextTokens: number; maxContextTokens: number } {
|
||||
let contextTokens = 0;
|
||||
let maxContextTokens = 0;
|
||||
if (!messageRows) return { contextTokens, maxContextTokens };
|
||||
for (let i = messageRows.length - 1; i >= 0; i--) {
|
||||
const meta = messageRows[i].metadata;
|
||||
if (!meta) continue;
|
||||
if (contextTokens === 0) {
|
||||
if (typeof meta.contextTokens === "number" && meta.contextTokens > 0) {
|
||||
contextTokens = meta.contextTokens;
|
||||
} else if (meta.usage) {
|
||||
const usage = meta.usage;
|
||||
const fallback =
|
||||
usage.totalTokens ??
|
||||
(usage.inputTokens ?? 0) + (usage.outputTokens ?? 0);
|
||||
if (fallback > 0) contextTokens = fallback;
|
||||
}
|
||||
}
|
||||
if (
|
||||
maxContextTokens === 0 &&
|
||||
typeof meta.maxContextTokens === "number" &&
|
||||
meta.maxContextTokens > 0
|
||||
) {
|
||||
maxContextTokens = meta.maxContextTokens;
|
||||
}
|
||||
if (contextTokens !== 0 && maxContextTokens !== 0) break;
|
||||
}
|
||||
return { contextTokens, maxContextTokens };
|
||||
}
|
||||
@@ -1,17 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
import {
|
||||
estimateTokens,
|
||||
liveTurnTokens,
|
||||
} from "@/features/ai-chat/utils/count-stream-tokens.ts";
|
||||
|
||||
const msg = (parts: unknown[], metadata?: unknown): UIMessage =>
|
||||
({
|
||||
id: Math.random().toString(),
|
||||
role: "assistant",
|
||||
parts,
|
||||
metadata,
|
||||
}) as UIMessage;
|
||||
import { estimateTokens } from "@/features/ai-chat/utils/count-stream-tokens.ts";
|
||||
|
||||
describe("estimateTokens", () => {
|
||||
it("returns 0 for the empty string", () => {
|
||||
@@ -25,147 +13,3 @@ describe("estimateTokens", () => {
|
||||
expect(estimateTokens("12345678")).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("liveTurnTokens — estimate path", () => {
|
||||
it("is all zeros for an undefined message", () => {
|
||||
expect(liveTurnTokens(undefined)).toEqual({
|
||||
reasoning: 0,
|
||||
output: 0,
|
||||
authoritative: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("is all zeros for a parts-less message", () => {
|
||||
expect(liveTurnTokens({ id: "x", role: "assistant" } as UIMessage)).toEqual({
|
||||
reasoning: 0,
|
||||
output: 0,
|
||||
authoritative: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("estimates output from text parts", () => {
|
||||
// 8 chars -> 2 tokens.
|
||||
const r = liveTurnTokens(msg([{ type: "text", text: "12345678" }]));
|
||||
expect(r).toEqual({ reasoning: 0, output: 2, authoritative: false });
|
||||
});
|
||||
|
||||
it("estimates reasoning from reasoning parts (kept separate from output)", () => {
|
||||
const r = liveTurnTokens(
|
||||
msg([
|
||||
{ type: "reasoning", text: "12345678" },
|
||||
{ type: "text", text: "abcd" },
|
||||
]),
|
||||
);
|
||||
expect(r).toEqual({ reasoning: 2, output: 1, authoritative: false });
|
||||
});
|
||||
|
||||
it("accumulates across multiple text + reasoning parts (multi-step)", () => {
|
||||
const r = liveTurnTokens(
|
||||
msg([
|
||||
{ type: "reasoning", text: "abcd" }, // 1
|
||||
{ type: "text", text: "abcd" }, // 1
|
||||
{ type: "tool-getPage", state: "output-available" }, // ignored
|
||||
{ type: "reasoning", text: "abcd" }, // 1
|
||||
{ type: "text", text: "abcdefgh" }, // 2
|
||||
]),
|
||||
);
|
||||
expect(r).toEqual({ reasoning: 2, output: 3, authoritative: false });
|
||||
});
|
||||
|
||||
it("ignores non text/reasoning parts (tools, step-start)", () => {
|
||||
const r = liveTurnTokens(
|
||||
msg([
|
||||
{ type: "step-start" },
|
||||
{ type: "tool-getPage", state: "input-available" },
|
||||
]),
|
||||
);
|
||||
expect(r).toEqual({ reasoning: 0, output: 0, authoritative: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe("liveTurnTokens — authoritative path", () => {
|
||||
it("returns authoritative usage verbatim, splitting reasoning out of output", () => {
|
||||
// outputTokens INCLUDES reasoning in the AI SDK shape -> answer = 100 - 30.
|
||||
const r = liveTurnTokens(
|
||||
msg([{ type: "text", text: "estimate would be tiny" }], {
|
||||
usage: { inputTokens: 500, outputTokens: 100, reasoningTokens: 30 },
|
||||
}),
|
||||
);
|
||||
expect(r).toEqual({ reasoning: 30, output: 70, authoritative: true });
|
||||
});
|
||||
|
||||
it("treats missing reasoningTokens as 0 and keeps full output", () => {
|
||||
const r = liveTurnTokens(
|
||||
msg([{ type: "text", text: "x" }], {
|
||||
usage: { inputTokens: 10, outputTokens: 42 },
|
||||
}),
|
||||
);
|
||||
expect(r).toEqual({ reasoning: 0, output: 42, authoritative: true });
|
||||
});
|
||||
|
||||
it("never returns a negative output when reasoning exceeds reported output", () => {
|
||||
const r = liveTurnTokens(
|
||||
msg([], { usage: { outputTokens: 10, reasoningTokens: 40 } }),
|
||||
);
|
||||
expect(r).toEqual({ reasoning: 40, output: 0, authoritative: true });
|
||||
});
|
||||
|
||||
it("falls back to the estimate when metadata has no usage object", () => {
|
||||
const r = liveTurnTokens(
|
||||
msg([{ type: "text", text: "abcd" }], { chatId: "c1" }),
|
||||
);
|
||||
expect(r).toEqual({ reasoning: 0, output: 1, authoritative: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe("liveTurnTokens — combined authoritative + estimate (#163)", () => {
|
||||
it("ticks the in-flight step above the completed-steps authoritative base", () => {
|
||||
// The authoritative usage is the sum over COMPLETED steps (step 1). The
|
||||
// CURRENT step is streaming and its text is NOT in `usage` yet, but it IS in
|
||||
// the parts -> the running estimate must push the live figure above the base
|
||||
// so the badge keeps growing between step boundaries.
|
||||
const longText = "x".repeat(800); // 800 chars -> 200 est output tokens
|
||||
const r = liveTurnTokens(
|
||||
msg([{ type: "text", text: longText }], {
|
||||
usage: { inputTokens: 500, outputTokens: 40 }, // step-1 base: 40 output
|
||||
}),
|
||||
);
|
||||
// max(authOutput=40, estOutput=200) = 200 -> the counter ticks, not frozen.
|
||||
expect(r.output).toBe(200);
|
||||
expect(r.authoritative).toBe(true);
|
||||
});
|
||||
|
||||
it("ticks reasoning of the in-flight step above the authoritative reasoning base", () => {
|
||||
const longReasoning = "r".repeat(400); // 400 chars -> 100 est reasoning
|
||||
const r = liveTurnTokens(
|
||||
msg([{ type: "reasoning", text: longReasoning }], {
|
||||
usage: { inputTokens: 100, outputTokens: 20, reasoningTokens: 20 },
|
||||
}),
|
||||
);
|
||||
// reasoning: max(20, 100) = 100 ; output: max(max(0,20-20)=0, 0) = 0.
|
||||
expect(r.reasoning).toBe(100);
|
||||
expect(r.output).toBe(0);
|
||||
expect(r.authoritative).toBe(true);
|
||||
});
|
||||
|
||||
it("snaps to the authoritative figure once it exceeds the rough estimate", () => {
|
||||
// Short on-screen text (estimate tiny) but a large authoritative output:
|
||||
// the exact figure wins at the boundary (the counter never under-reports).
|
||||
const r = liveTurnTokens(
|
||||
msg([{ type: "text", text: "abcd" }], {
|
||||
usage: { inputTokens: 10, outputTokens: 5000 },
|
||||
}),
|
||||
);
|
||||
expect(r.output).toBe(5000);
|
||||
});
|
||||
|
||||
it("is monotonic: max never drops below the authoritative base when the estimate is smaller", () => {
|
||||
// Mirrors the legacy 'verbatim' tests: estimate < authoritative -> unchanged.
|
||||
const r = liveTurnTokens(
|
||||
msg([{ type: "text", text: "tiny" }], {
|
||||
usage: { inputTokens: 500, outputTokens: 100, reasoningTokens: 30 },
|
||||
}),
|
||||
);
|
||||
expect(r).toEqual({ reasoning: 30, output: 70, authoritative: true });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,18 +1,11 @@
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
|
||||
/**
|
||||
* Live token counting for a streaming AI-chat turn — split into REASONING
|
||||
* (thinking) and OUTPUT (answer) tokens, mirroring how Claude Code shows
|
||||
* `Thinking… · 60 tokens` next to its thinking indicator.
|
||||
* Rough client-side token estimation for AI-chat UI affordances.
|
||||
*
|
||||
* No provider streams exact per-token usage mid-stream, so the live number is a
|
||||
* CLIENT ESTIMATE (chars/≈4 heuristic) that is reconciled to AUTHORITATIVE usage
|
||||
* once the server attaches it on a step/turn boundary (see the server's
|
||||
* `chatStreamMetadata` + the client's read of `message.metadata.usage`). When
|
||||
* authoritative usage is present we return it verbatim (the number "jumps to
|
||||
* exact"); otherwise we return the running estimate. Pure + unit-testable: it
|
||||
* never runs a real BPE tokenizer (that would be O(n²) on the hot path, bloat the
|
||||
* bundle, and be wrong for Gemini/Ollama anyway).
|
||||
* No provider streams exact per-token usage mid-stream, so any in-flight figure
|
||||
* is a CLIENT ESTIMATE (chars/≈4 heuristic). Pure + unit-testable: it never runs
|
||||
* a real BPE tokenizer (that would be O(n²) on the hot path, bloat the bundle,
|
||||
* and be wrong for Gemini/Ollama anyway). Used by the in-body reasoning counter
|
||||
* ("Thinking · N tokens").
|
||||
*/
|
||||
|
||||
/**
|
||||
@@ -24,90 +17,3 @@ export function estimateTokens(text: string): number {
|
||||
if (!text) return 0;
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
/** Authoritative per-step/turn usage the server attaches to message metadata. */
|
||||
export interface AuthoritativeUsage {
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
totalTokens?: number;
|
||||
reasoningTokens?: number;
|
||||
}
|
||||
|
||||
/** Live token split for a turn's tail (streaming) assistant message. */
|
||||
export interface LiveTurnTokens {
|
||||
/** Thinking/reasoning tokens (estimate, or authoritative when available). */
|
||||
reasoning: number;
|
||||
/** Answer/output tokens (estimate, or authoritative when available). */
|
||||
output: number;
|
||||
/** True when the numbers come from authoritative server usage, not estimate. */
|
||||
authoritative: boolean;
|
||||
}
|
||||
|
||||
/** Read the authoritative usage off a UIMessage's metadata, if the server set it. */
|
||||
function metadataUsage(message: UIMessage): AuthoritativeUsage | undefined {
|
||||
const meta = message?.metadata as
|
||||
| { usage?: AuthoritativeUsage }
|
||||
| undefined;
|
||||
const usage = meta?.usage;
|
||||
if (!usage || typeof usage !== "object") return undefined;
|
||||
return usage;
|
||||
}
|
||||
|
||||
/**
|
||||
* Token split for the given (streaming) assistant message.
|
||||
*
|
||||
* COMBINES the authoritative server usage with the running text estimate so the
|
||||
* counter ticks in real time AND lands exact. The server only attaches
|
||||
* `metadata.usage` at a step/turn boundary (`finish-step`/`finish`) and it is
|
||||
* CUMULATIVE over COMPLETED steps — it does NOT yet include the in-flight step.
|
||||
* So a multi-step turn that returned the authoritative figure verbatim would
|
||||
* FREEZE between boundaries and jump in steps (issue #163).
|
||||
*
|
||||
* Instead we always compute the running ESTIMATE (chars/≈4 over the message's
|
||||
* `reasoning`/`text` parts, which grows on every streamed delta) and take the
|
||||
* per-component MAX of the authoritative base and the estimate:
|
||||
* - between boundaries the estimate of the in-flight step ticks the number up;
|
||||
* - at a boundary the authoritative figure snaps it to exact;
|
||||
* - because the server's usage is cumulative and we only ever take the max, the
|
||||
* number is MONOTONIC — it never drops.
|
||||
*
|
||||
* Providers that don't stream reasoning text still surface a reasoning count once
|
||||
* the authoritative usage arrives (`max(reasoningTokens, 0)`); on the pure
|
||||
* estimate path (no usage yet) such a turn shows `reasoning: 0` until then.
|
||||
*/
|
||||
export function liveTurnTokens(message: UIMessage | undefined): LiveTurnTokens {
|
||||
if (!message) return { reasoning: 0, output: 0, authoritative: false };
|
||||
|
||||
// Running ESTIMATE over every reasoning/text part — grows on each delta. This
|
||||
// includes the IN-FLIGHT step, which the authoritative usage does not cover yet.
|
||||
let estReasoning = 0;
|
||||
let estOutput = 0;
|
||||
for (const part of message.parts ?? []) {
|
||||
if (part.type === "reasoning") {
|
||||
estReasoning += estimateTokens((part as { text?: string }).text ?? "");
|
||||
} else if (part.type === "text") {
|
||||
estOutput += estimateTokens((part as { text?: string }).text ?? "");
|
||||
}
|
||||
}
|
||||
|
||||
const usage = metadataUsage(message);
|
||||
if (!usage) {
|
||||
// No authoritative usage streamed yet: the estimate IS the live figure.
|
||||
return { reasoning: estReasoning, output: estOutput, authoritative: false };
|
||||
}
|
||||
|
||||
// Authoritative sum over COMPLETED steps. `outputTokens` already INCLUDES
|
||||
// reasoning in the AI SDK usage shape, so subtract it out for the "answer"
|
||||
// figure (never go negative if a provider reports them inconsistently).
|
||||
const authReasoning = usage.reasoningTokens ?? 0;
|
||||
const authOutput = Math.max(0, (usage.outputTokens ?? 0) - authReasoning);
|
||||
|
||||
// Per-component max: the in-flight step's estimate ticks above the completed-
|
||||
// steps base between boundaries, and the authoritative figure wins once it
|
||||
// exceeds the (rough) estimate at the next boundary. Monotonic by construction.
|
||||
return {
|
||||
reasoning: Math.max(authReasoning, estReasoning),
|
||||
output: Math.max(authOutput, estOutput),
|
||||
authoritative: true,
|
||||
};
|
||||
}
|
||||
|
||||
241
apps/client/src/features/ai-chat/utils/message-signature.test.ts
Normal file
241
apps/client/src/features/ai-chat/utils/message-signature.test.ts
Normal file
@@ -0,0 +1,241 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
import { messageSignature } from "@/features/ai-chat/utils/message-signature.ts";
|
||||
|
||||
/**
|
||||
* Pure-helper tests for `messageSignature`, the cheap per-message content
|
||||
* signature that drives MessageItem's memo (a streaming row's signature must
|
||||
* change on every delta so it re-renders, while a finalized row's stays stable
|
||||
* so it is skipped). Each test exercises ONE change signal and asserts it flips
|
||||
* the signature; a content-identical clone must keep an EQUAL signature.
|
||||
*
|
||||
* The signature embeds `message.id` and `message.role`, so the `msg` factory
|
||||
* uses a FIXED id/role here (not `Math.random()`): otherwise two messages with
|
||||
* identical content would get different signatures and the negative case would
|
||||
* be impossible to express.
|
||||
*/
|
||||
const msg = (
|
||||
parts: UIMessage["parts"],
|
||||
metadata?: unknown,
|
||||
): UIMessage =>
|
||||
({
|
||||
id: "m1",
|
||||
role: "assistant",
|
||||
parts,
|
||||
metadata,
|
||||
}) as UIMessage;
|
||||
|
||||
describe("messageSignature", () => {
|
||||
it("changes when a text part grows", () => {
|
||||
const before = msg([{ type: "text", text: "alpha" }]);
|
||||
const after = msg([{ type: "text", text: "alpha beta" }]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("changes when a new part is appended", () => {
|
||||
const before = msg([{ type: "text", text: "alpha" }]);
|
||||
const after = msg([
|
||||
{ type: "text", text: "alpha" },
|
||||
{ type: "text", text: "beta" },
|
||||
]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("changes when a part's state flips", () => {
|
||||
const before = msg([
|
||||
{ type: "tool-getPage", state: "input-streaming" } as never,
|
||||
]);
|
||||
const after = msg([
|
||||
{ type: "tool-getPage", state: "output-available" } as never,
|
||||
]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("changes when a tool part gains an output", () => {
|
||||
const before = msg([
|
||||
{ type: "tool-getPage", state: "output-available" } as never,
|
||||
]);
|
||||
const after = msg([
|
||||
{
|
||||
type: "tool-getPage",
|
||||
state: "output-available",
|
||||
output: { ok: true },
|
||||
} as never,
|
||||
]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("changes when a part gains an errorText", () => {
|
||||
const before = msg([
|
||||
{ type: "tool-getPage", state: "output-error" } as never,
|
||||
]);
|
||||
const after = msg([
|
||||
{
|
||||
type: "tool-getPage",
|
||||
state: "output-error",
|
||||
errorText: "boom",
|
||||
} as never,
|
||||
]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("changes when usage.reasoningTokens arrives on finish-step (text/state already frozen)", () => {
|
||||
// The specifically-commented edge case: the authoritative turn total lands on
|
||||
// the final finish-step AFTER the reasoning text length and state are frozen.
|
||||
// Only the token count appears between these two snapshots, so the signature
|
||||
// MUST still flip — otherwise the "Thinking · N tokens" header would never
|
||||
// snap from the live estimate to the exact figure.
|
||||
const before = msg([
|
||||
{ type: "reasoning", text: "thinking", state: "done" } as never,
|
||||
]);
|
||||
const after = msg(
|
||||
[{ type: "reasoning", text: "thinking", state: "done" } as never],
|
||||
{ usage: { reasoningTokens: 42 } },
|
||||
);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("changes when metadata.error appears", () => {
|
||||
const before = msg([{ type: "text", text: "answer" }]);
|
||||
const after = msg([{ type: "text", text: "answer" }], { error: "boom" });
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("changes when metadata.finishReason changes (e.g. to 'aborted')", () => {
|
||||
const before = msg([{ type: "text", text: "answer" }], {
|
||||
finishReason: "stop",
|
||||
});
|
||||
const after = msg([{ type: "text", text: "answer" }], {
|
||||
finishReason: "aborted",
|
||||
});
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("is UNCHANGED for a content-identical clone (different object, same values)", () => {
|
||||
// A finalized row that is re-created as a fresh object (different parts array
|
||||
// by reference, same parts by value) must keep an EQUAL signature, so the
|
||||
// memo skips re-rendering it.
|
||||
const a = msg([
|
||||
{ type: "text", text: "alpha" },
|
||||
{ type: "tool-getPage", state: "output-available", output: { ok: true } } as never,
|
||||
]);
|
||||
const b = msg([
|
||||
{ type: "text", text: "alpha" },
|
||||
{ type: "tool-getPage", state: "output-available", output: { ok: true } } as never,
|
||||
]);
|
||||
expect(a).not.toBe(b);
|
||||
expect(messageSignature(a)).toBe(messageSignature(b));
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Per-part-kind coupling guard for the load-bearing invariant documented at the
|
||||
* top of message-signature.ts: the signature MUST sample every VISIBLE field the
|
||||
* MessageItem render body draws, or the memo freezes a stale row. This is an
|
||||
* executable lock for the part kinds rendered TODAY — read alongside
|
||||
* `MessageItem` (message-item.tsx) and the `assistantMessageHasVisibleContent`
|
||||
* helper (message-content.ts), which "mirrors MessageItem's render decisions
|
||||
* EXACTLY". For each kind, mutating a field the render body DRAWS must flip the
|
||||
* signature. If a new visible field is rendered without being added here AND to
|
||||
* the signature, the corresponding assertion below should fail — that is the
|
||||
* guard. (This intentionally stops short of the render-descriptor refactor:
|
||||
* adding a part kind or a visible field still requires a human to extend both
|
||||
* the signature and this block.)
|
||||
*/
|
||||
describe("messageSignature ↔ render coupling (per visible part kind)", () => {
|
||||
describe("text part — render draws part.text (MarkdownPart text={part.text})", () => {
|
||||
it("flips when the visible text changes", () => {
|
||||
// Streaming is append-only, so the visible text only grows; the signature
|
||||
// samples its length, so the growth is the change signal.
|
||||
const before = msg([{ type: "text", text: "answer" }]);
|
||||
const after = msg([{ type: "text", text: "answer extended" }]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
});
|
||||
|
||||
describe("reasoning part — render draws text + tokens (ReasoningBlock)", () => {
|
||||
it("flips when the visible reasoning text changes", () => {
|
||||
const before = msg([
|
||||
{ type: "reasoning", text: "think", state: "streaming" } as never,
|
||||
]);
|
||||
const after = msg([
|
||||
{ type: "reasoning", text: "think harder", state: "streaming" } as never,
|
||||
]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("flips when the visible token count (metadata.usage.reasoningTokens) lands", () => {
|
||||
// The header's "Thinking · N tokens" reads reasoningTokensForPart, fed by
|
||||
// metadata.usage.reasoningTokens — a VISIBLE field that arrives on the final
|
||||
// finish-step after text length and state are frozen.
|
||||
const before = msg([
|
||||
{ type: "reasoning", text: "think", state: "done" } as never,
|
||||
]);
|
||||
const after = msg(
|
||||
[{ type: "reasoning", text: "think", state: "done" } as never],
|
||||
{ usage: { reasoningTokens: 99 } },
|
||||
);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
});
|
||||
|
||||
describe("tool-* part — render draws state/errorText/citations (ToolCallCard)", () => {
|
||||
it("flips when the run state changes (running ↔ done icon + label)", () => {
|
||||
// toolRunState(part.state) selects the spinner/check/error icon.
|
||||
const before = msg([
|
||||
{ type: "tool-getPage", state: "input-available" } as never,
|
||||
]);
|
||||
const after = msg([
|
||||
{ type: "tool-getPage", state: "output-available" } as never,
|
||||
]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("flips when output arrives (drives the rendered citation links)", () => {
|
||||
// toolCitations reads part.output to render the "/p/{id}" anchors.
|
||||
const before = msg([
|
||||
{ type: "tool-getPage", state: "output-available" } as never,
|
||||
]);
|
||||
const after = msg([
|
||||
{
|
||||
type: "tool-getPage",
|
||||
state: "output-available",
|
||||
output: { id: "page-1", title: "Doc" },
|
||||
} as never,
|
||||
]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("flips when errorText appears (the visible red error detail line)", () => {
|
||||
const before = msg([
|
||||
{ type: "tool-getPage", state: "output-error" } as never,
|
||||
]);
|
||||
const after = msg([
|
||||
{
|
||||
type: "tool-getPage",
|
||||
state: "output-error",
|
||||
errorText: "permission denied",
|
||||
} as never,
|
||||
]);
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
});
|
||||
|
||||
describe("metadata banners — render draws error / aborted notices", () => {
|
||||
it("flips when metadata.error appears (ChatErrorAlert banner)", () => {
|
||||
const before = msg([{ type: "text", text: "answer" }]);
|
||||
const after = msg([{ type: "text", text: "answer" }], { error: "boom" });
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
|
||||
it("flips when metadata.finishReason becomes 'aborted' (ChatStoppedNotice)", () => {
|
||||
const before = msg([{ type: "text", text: "answer" }], {
|
||||
finishReason: "stop",
|
||||
});
|
||||
const after = msg([{ type: "text", text: "answer" }], {
|
||||
finishReason: "aborted",
|
||||
});
|
||||
expect(messageSignature(before)).not.toBe(messageSignature(after));
|
||||
});
|
||||
});
|
||||
});
|
||||
44
apps/client/src/features/ai-chat/utils/message-signature.ts
Normal file
44
apps/client/src/features/ai-chat/utils/message-signature.ts
Normal file
@@ -0,0 +1,44 @@
|
||||
import type { UIMessage } from "@ai-sdk/react";
|
||||
|
||||
/** Cheap content signature for one message: changes iff something VISIBLE in the
|
||||
* row changed. Streaming is APPEND-ONLY (text parts only grow, parts are only
|
||||
* appended, a tool/text part flips state once), so a per-part [type, text
|
||||
* length, state, error/output presence] tuple + the persisted metadata
|
||||
* (error/finishReason) is a sufficient change signal without comparing full
|
||||
* strings on every delta. WARNING — load-bearing for the MessageItem memo:
|
||||
* if a future part kind's VISIBLE content can change WITHOUT changing [type,
|
||||
* text length, state, error/output presence] (e.g. a tool that streams
|
||||
* `preliminary` output, or a client-side regenerate that edits a finalized
|
||||
* row in place), extend this signature or the memo will freeze a stale row. */
|
||||
export function messageSignature(message: UIMessage): string {
|
||||
const parts = message.parts
|
||||
.map((p) => {
|
||||
const any = p as {
|
||||
type: string;
|
||||
text?: string;
|
||||
state?: string;
|
||||
errorText?: string;
|
||||
output?: unknown;
|
||||
};
|
||||
return [
|
||||
any.type,
|
||||
any.text?.length ?? 0,
|
||||
any.state ?? "",
|
||||
any.errorText ? 1 : 0,
|
||||
any.output !== undefined ? 1 : 0,
|
||||
].join(":");
|
||||
})
|
||||
.join("|");
|
||||
const meta = message.metadata as
|
||||
| { error?: string; finishReason?: string; usage?: { reasoningTokens?: number } }
|
||||
| undefined;
|
||||
// `usage.reasoningTokens` is neither append-only nor part-bound: the authoritative
|
||||
// turn total arrives on the final `finish-step` AFTER the reasoning text length and
|
||||
// state are already frozen. Without it in the signature the row's signature would be
|
||||
// unchanged at that point and the re-render skipped, so the "Thinking · N tokens"
|
||||
// header (reasoningTokensForPart) would keep the live estimate instead of snapping
|
||||
// to the exact figure.
|
||||
return `${message.id}#${message.role}#${parts}#${meta?.error ?? ""}#${
|
||||
meta?.finishReason ?? ""
|
||||
}#${meta?.usage?.reasoningTokens ?? ""}`;
|
||||
}
|
||||
@@ -7,7 +7,7 @@ import classes from "./footnote.module.css";
|
||||
* A 0-based backlink index -> its lowercase letter label (0 -> "a", 25 -> "z",
|
||||
* 26 -> "aa", ...), matching the Pandoc/Wikipedia "↩ a b c" convention.
|
||||
*/
|
||||
function backlinkLabel(index: number): string {
|
||||
export function backlinkLabel(index: number): string {
|
||||
let out = "";
|
||||
let x = index;
|
||||
while (x >= 0) {
|
||||
|
||||
@@ -75,7 +75,9 @@ vi.mock("@/features/editor/components/code-block/mermaid-view.tsx", () => ({
|
||||
}));
|
||||
|
||||
import FootnotesListView from "./footnotes-list-view";
|
||||
import FootnoteDefinitionView from "./footnote-definition-view";
|
||||
import FootnoteDefinitionView, {
|
||||
backlinkLabel,
|
||||
} from "./footnote-definition-view";
|
||||
import CodeBlockView from "../code-block/code-block-view";
|
||||
|
||||
// Minimal NodeViewProps stub: definition view only touches node.attrs.id and
|
||||
@@ -214,3 +216,16 @@ describe("#168 footnote definition multi-backlinks", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// #185 re-review pt 7: backlinkLabel is base-26 (a..z, then aa…). The component
|
||||
// tests only cover a,b,c (index 0-2); pin the >= 26 carry boundary.
|
||||
describe("backlinkLabel base-26 boundary (#168)", () => {
|
||||
it("maps 0->a, 25->z, 26->aa, 27->ab, 51->az, 52->ba", () => {
|
||||
expect(backlinkLabel(0)).toBe("a");
|
||||
expect(backlinkLabel(25)).toBe("z");
|
||||
expect(backlinkLabel(26)).toBe("aa");
|
||||
expect(backlinkLabel(27)).toBe("ab");
|
||||
expect(backlinkLabel(51)).toBe("az");
|
||||
expect(backlinkLabel(52)).toBe("ba");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -274,7 +274,10 @@ export function useRestorePageMutation() {
|
||||
queryClient.setQueryData<IPage>(["pages", restoredPage.slugId], merge);
|
||||
},
|
||||
onError: (error) => {
|
||||
notifications.show({ message: t("Failed to restore page"), color: "red" });
|
||||
notifications.show({
|
||||
message: t("Failed to restore page"),
|
||||
color: "red",
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -285,10 +288,10 @@ export function useGetSidebarPagesQuery(
|
||||
return useInfiniteQuery({
|
||||
queryKey: ["sidebar-pages", data],
|
||||
enabled: !!data?.pageId || !!data?.spaceId,
|
||||
queryFn: ({ pageParam }) => getSidebarPages({ ...data, cursor: pageParam, limit: 100 }),
|
||||
queryFn: ({ pageParam }) =>
|
||||
getSidebarPages({ ...data, cursor: pageParam, limit: 100 }),
|
||||
initialPageParam: undefined,
|
||||
getNextPageParam: (lastPage) =>
|
||||
lastPage.meta?.nextCursor ?? undefined,
|
||||
getNextPageParam: (lastPage) => lastPage.meta?.nextCursor ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -296,11 +299,14 @@ export function useGetRootSidebarPagesQuery(data: SidebarPagesParams) {
|
||||
return useInfiniteQuery({
|
||||
queryKey: ["root-sidebar-pages", data.spaceId],
|
||||
queryFn: async ({ pageParam }) => {
|
||||
return getSidebarPages({ spaceId: data.spaceId, cursor: pageParam, limit: 100 });
|
||||
return getSidebarPages({
|
||||
spaceId: data.spaceId,
|
||||
cursor: pageParam,
|
||||
limit: 100,
|
||||
});
|
||||
},
|
||||
initialPageParam: undefined,
|
||||
getNextPageParam: (lastPage) =>
|
||||
lastPage.meta?.nextCursor ?? undefined,
|
||||
getNextPageParam: (lastPage) => lastPage.meta?.nextCursor ?? undefined,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -323,12 +329,17 @@ export function usePageBreadcrumbsQuery(
|
||||
});
|
||||
}
|
||||
|
||||
export async function fetchAllAncestorChildren(params: SidebarPagesParams) {
|
||||
export async function fetchAllAncestorChildren(
|
||||
params: SidebarPagesParams,
|
||||
// `fresh: true` forces a server refetch (staleTime 0) — used by the reconnect
|
||||
// refresh (#159 #8), which must NOT receive the 30-min-cached children.
|
||||
opts?: { fresh?: boolean },
|
||||
) {
|
||||
// not using a hook here, so we can call it inside a useEffect hook
|
||||
const response = await queryClient.fetchQuery({
|
||||
queryKey: ["sidebar-pages", params],
|
||||
queryFn: () => getAllSidebarPages(params),
|
||||
staleTime: 30 * 60 * 1000,
|
||||
staleTime: opts?.fresh ? 0 : 30 * 60 * 1000,
|
||||
});
|
||||
|
||||
const allItems = response.pages.flatMap((page) => page.items);
|
||||
@@ -347,11 +358,15 @@ export function useRecentChangesQuery(spaceId?: string) {
|
||||
});
|
||||
}
|
||||
|
||||
export function useCreatedByQuery(params?: { userId?: string; spaceId?: string }) {
|
||||
export function useCreatedByQuery(params?: {
|
||||
userId?: string;
|
||||
spaceId?: string;
|
||||
}) {
|
||||
const { userId, spaceId } = params ?? {};
|
||||
return useInfiniteQuery({
|
||||
queryKey: ["pages-created-by-user", { userId, spaceId }],
|
||||
queryFn: ({ pageParam }) => getCreatedByPages({ userId, spaceId, cursor: pageParam, limit: 15 }),
|
||||
queryFn: ({ pageParam }) =>
|
||||
getCreatedByPages({ userId, spaceId, cursor: pageParam, limit: 15 }),
|
||||
initialPageParam: undefined as string | undefined,
|
||||
getNextPageParam: (lastPage) =>
|
||||
lastPage.meta.hasNextPage ? lastPage.meta.nextCursor : undefined,
|
||||
|
||||
@@ -29,9 +29,11 @@ import {
|
||||
collectBranchIds,
|
||||
openBranches,
|
||||
closeIds,
|
||||
loadedOpenBranchIds,
|
||||
} from "@/features/page/tree/utils/utils.ts";
|
||||
import { SpaceTreeNode } from "@/features/page/tree/types.ts";
|
||||
import { treeModel } from "@/features/page/tree/model/tree-model";
|
||||
import { socketAtom } from "@/features/websocket/atoms/socket-atom.ts";
|
||||
import {
|
||||
getPageBreadcrumbs,
|
||||
getSpaceTree,
|
||||
@@ -39,11 +41,7 @@ import {
|
||||
import { IPage } from "@/features/page/types/page.types.ts";
|
||||
import { extractPageSlugId } from "@/lib";
|
||||
import { isCompactPageTreeEnabled } from "@/lib/config.ts";
|
||||
import {
|
||||
DocTree,
|
||||
ROW_HEIGHT_COMPACT,
|
||||
ROW_HEIGHT_STANDARD,
|
||||
} from "./doc-tree";
|
||||
import { DocTree, ROW_HEIGHT_COMPACT, ROW_HEIGHT_STANDARD } from "./doc-tree";
|
||||
import { SpaceTreeRow } from "./space-tree-row";
|
||||
|
||||
interface SpaceTreeProps {
|
||||
@@ -193,6 +191,54 @@ const SpaceTree = forwardRef<SpaceTreeApi, SpaceTreeProps>(function SpaceTree(
|
||||
[openTreeNodes],
|
||||
);
|
||||
|
||||
// Latest tree + open-state for the reconnect handler (its closure would
|
||||
// otherwise read stale snapshots).
|
||||
const [socket] = useAtom(socketAtom);
|
||||
const dataRef = useRef(data);
|
||||
dataRef.current = data;
|
||||
const openIdsRef = useRef(openIds);
|
||||
openIdsRef.current = openIds;
|
||||
|
||||
// Reconnect refresh (#159 #8): on a socket reconnect, re-fetch and reconcile
|
||||
// the children of every currently-open, already-loaded branch of THIS space,
|
||||
// so a move/rename/delete that happened INSIDE a loaded branch while events
|
||||
// were missed (laptop sleep / wifi gap) is reflected instead of left stale.
|
||||
// The ROOT level is reconciled separately by the root-query refetch +
|
||||
// mergeRootTrees; an UNLOADED branch is skipped (lazy-load fetches it fresh on
|
||||
// expand). No first-connect guard is needed: space-tree usually mounts AFTER
|
||||
// the initial connect, so every `connect` it sees is a reconnect; the rare
|
||||
// initial-connect case has an empty tree, so the refresh is a harmless no-op.
|
||||
useEffect(() => {
|
||||
if (!socket) return;
|
||||
const onConnect = async () => {
|
||||
const effectSpaceId = spaceIdRef.current;
|
||||
const branchIds = loadedOpenBranchIds(
|
||||
dataRef.current.filter((n) => n?.spaceId === effectSpaceId),
|
||||
openIdsRef.current,
|
||||
);
|
||||
if (branchIds.length === 0) return;
|
||||
for (const id of branchIds) {
|
||||
try {
|
||||
// `fresh: true` bypasses the 30-min sidebar-pages cache so the
|
||||
// reconcile sees the server's CURRENT children (handler-order
|
||||
// independent — no reliance on the global reconnect invalidation).
|
||||
const fresh = await fetchAllAncestorChildren(
|
||||
{ pageId: id, spaceId: effectSpaceId },
|
||||
{ fresh: true },
|
||||
);
|
||||
if (spaceIdRef.current !== effectSpaceId) return; // space switched
|
||||
setData((prev) => treeModel.reconcileChildren(prev, id, fresh));
|
||||
} catch (err) {
|
||||
console.error("[tree] reconnect branch refresh failed", err);
|
||||
}
|
||||
}
|
||||
};
|
||||
socket.on("connect", onConnect);
|
||||
return () => {
|
||||
socket.off("connect", onConnect);
|
||||
};
|
||||
}, [socket, setData]);
|
||||
|
||||
const handleToggle = useCallback(
|
||||
async (id: string, isOpen: boolean) => {
|
||||
setOpenTreeNodes((prev) => ({ ...prev, [id]: isOpen }));
|
||||
@@ -245,8 +291,7 @@ const SpaceTree = forwardRef<SpaceTreeApi, SpaceTreeProps>(function SpaceTree(
|
||||
notifications.show({
|
||||
color: "red",
|
||||
message: t("Couldn't expand the tree: {{reason}}", {
|
||||
reason:
|
||||
err?.response?.data?.message ?? err?.message ?? String(err),
|
||||
reason: err?.response?.data?.message ?? err?.message ?? String(err),
|
||||
}),
|
||||
});
|
||||
} finally {
|
||||
@@ -262,11 +307,11 @@ const SpaceTree = forwardRef<SpaceTreeApi, SpaceTreeProps>(function SpaceTree(
|
||||
setOpenTreeNodes((prev) => closeIds(prev, ids));
|
||||
}, [filteredData, setOpenTreeNodes]);
|
||||
|
||||
useImperativeHandle(
|
||||
ref,
|
||||
() => ({ expandAll, collapseAll, isExpanding }),
|
||||
[expandAll, collapseAll, isExpanding],
|
||||
);
|
||||
useImperativeHandle(ref, () => ({ expandAll, collapseAll, isExpanding }), [
|
||||
expandAll,
|
||||
collapseAll,
|
||||
isExpanding,
|
||||
]);
|
||||
|
||||
// Stable callbacks for DocTree. Without these, every parent render recreates
|
||||
// the props and tears down every row's draggable/dropTarget subscription,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,4 +1,4 @@
|
||||
import type { TreeNode, SiblingsInfo } from './tree-model.types';
|
||||
import type { TreeNode, SiblingsInfo } from "./tree-model.types";
|
||||
|
||||
function findInternal<T extends object>(
|
||||
nodes: TreeNode<T>[],
|
||||
@@ -19,7 +19,10 @@ export const treeModel = {
|
||||
return findInternal(tree, id)?.node ?? null;
|
||||
},
|
||||
|
||||
path<T extends object>(tree: TreeNode<T>[], id: string): TreeNode<T>[] | null {
|
||||
path<T extends object>(
|
||||
tree: TreeNode<T>[],
|
||||
id: string,
|
||||
): TreeNode<T>[] | null {
|
||||
const found = findInternal(tree, id);
|
||||
if (!found) return null;
|
||||
return [...found.parents, found.node];
|
||||
@@ -123,6 +126,23 @@ export const treeModel = {
|
||||
return treeModel.insert(tree, null, node, index(tree));
|
||||
}
|
||||
const parent = treeModel.find(tree, parentId);
|
||||
// The parent is in the tree but its children have NOT been lazy-loaded yet
|
||||
// (`children === undefined`, distinct from a loaded-but-empty `[]`). Inserting
|
||||
// here would MATERIALIZE a misleading partial child list (`[node]`) that
|
||||
// defeats the lazy-load gate — which fetches only when children are
|
||||
// absent/empty — so the parent's OTHER real children would never load and the
|
||||
// moved/added node would be the only one shown (a silent data loss, #159 #1).
|
||||
// Instead, leave the children unloaded and just flag `hasChildren` so the
|
||||
// chevron appears; expanding fetches the FULL set (including this node).
|
||||
if (parent && parent.children === undefined) {
|
||||
return treeModel.update(
|
||||
tree,
|
||||
parentId,
|
||||
// hasChildren is not part of the generic T constraint; tree nodes carry
|
||||
// it. Cast narrowly so this stays a single, well-understood exception.
|
||||
{ hasChildren: true } as unknown as Omit<Partial<T>, "id" | "children">,
|
||||
);
|
||||
}
|
||||
const kids = (parent?.children as TreeNode<T>[] | undefined) ?? [];
|
||||
return treeModel.insert(tree, parentId, node, index(kids));
|
||||
},
|
||||
@@ -203,6 +223,48 @@ export const treeModel = {
|
||||
return touched ? out : tree;
|
||||
},
|
||||
|
||||
// Replace a parent's DIRECT children with the authoritative `fresh` set while
|
||||
// PRESERVING each surviving child's already-loaded grandchildren (deeper
|
||||
// expansion). Unlike `appendChildren` (add-only), this DROPS children that are
|
||||
// no longer present and reorders to `fresh` — so a move/delete/rename that
|
||||
// happened inside a loaded branch while events were missed (a socket reconnect
|
||||
// gap) is reflected, not left stale (#159 #8). Only used to reconcile an
|
||||
// already-loaded branch against a fresh fetch; a parent with no loaded children
|
||||
// (`children === undefined`) is left untouched (lazy-load handles it).
|
||||
reconcileChildren<T extends object>(
|
||||
tree: TreeNode<T>[],
|
||||
parentId: string,
|
||||
fresh: TreeNode<T>[],
|
||||
): TreeNode<T>[] {
|
||||
let touched = false;
|
||||
const walk = (nodes: TreeNode<T>[]): TreeNode<T>[] =>
|
||||
nodes.map((n) => {
|
||||
if (n.id === parentId) {
|
||||
// Only reconcile a branch whose children were actually loaded; an
|
||||
// unloaded parent stays unloaded (lazy-load fetches it fresh later).
|
||||
if (n.children === undefined) return n;
|
||||
const prevById = new Map(n.children.map((c) => [c.id, c]));
|
||||
const merged = fresh.map((f) => {
|
||||
const prev = prevById.get(f.id);
|
||||
// Preserve the surviving child's previously loaded grandchildren so
|
||||
// deeper expansion is not collapsed by the reconcile.
|
||||
return prev?.children !== undefined
|
||||
? { ...f, children: prev.children }
|
||||
: f;
|
||||
});
|
||||
touched = true;
|
||||
return { ...n, children: merged };
|
||||
}
|
||||
if (n.children) {
|
||||
const next = walk(n.children);
|
||||
if (next !== n.children) return { ...n, children: next };
|
||||
}
|
||||
return n;
|
||||
});
|
||||
const out = walk(tree);
|
||||
return touched ? out : tree;
|
||||
},
|
||||
|
||||
place<T extends object>(
|
||||
tree: TreeNode<T>[],
|
||||
sourceId: string,
|
||||
@@ -232,6 +294,20 @@ export const treeModel = {
|
||||
const source = treeModel.find(tree, sourceId);
|
||||
if (!source) return tree;
|
||||
if (to.parentId !== null && !treeModel.find(tree, to.parentId)) return tree;
|
||||
// Cycle guard, mirroring `move`'s `isDescendant` check (#206 ui-state-races-1).
|
||||
// If the destination parent is INSIDE the moved node's own subtree (reachable
|
||||
// when server-authoritative move events arrive out of order — e.g. X moved
|
||||
// under Y, then Y under X, but on this receiver Y is still inside X), then
|
||||
// `remove(sourceId)` would drop the future parent along with the whole subtree
|
||||
// and `insertByPosition` could not find it again — the node and ALL its
|
||||
// descendants would silently vanish. Refuse the move and return the same
|
||||
// reference so callers can detect the no-op and reconcile (refetch) instead.
|
||||
if (
|
||||
to.parentId !== null &&
|
||||
treeModel.isDescendant(tree, sourceId, to.parentId)
|
||||
) {
|
||||
return tree;
|
||||
}
|
||||
const removed = treeModel.remove(tree, sourceId);
|
||||
// Reuse the same position-ordered insertion as `insertByPosition` by
|
||||
// stamping the authoritative position onto the moved node first.
|
||||
@@ -242,9 +318,10 @@ export const treeModel = {
|
||||
move<T extends object>(
|
||||
tree: TreeNode<T>[],
|
||||
sourceId: string,
|
||||
op: import('./tree-model.types').DropOp,
|
||||
): { tree: TreeNode<T>[]; result: import('./tree-model.types').DropResult } {
|
||||
if (sourceId === op.targetId) return { tree, result: { parentId: null, index: 0 } };
|
||||
op: import("./tree-model.types").DropOp,
|
||||
): { tree: TreeNode<T>[]; result: import("./tree-model.types").DropResult } {
|
||||
if (sourceId === op.targetId)
|
||||
return { tree, result: { parentId: null, index: 0 } };
|
||||
if (!treeModel.find(tree, sourceId) || !treeModel.find(tree, op.targetId)) {
|
||||
return { tree, result: { parentId: null, index: 0 } };
|
||||
}
|
||||
@@ -255,7 +332,7 @@ export const treeModel = {
|
||||
let parentId: string | null;
|
||||
let index: number;
|
||||
|
||||
if (op.kind === 'make-child') {
|
||||
if (op.kind === "make-child") {
|
||||
parentId = op.targetId;
|
||||
const target = treeModel.find(tree, op.targetId)!;
|
||||
index = target.children?.length ?? 0;
|
||||
@@ -264,9 +341,8 @@ export const treeModel = {
|
||||
parentId = info.parentId;
|
||||
const sourceInfo = treeModel.siblingsOf(tree, sourceId)!;
|
||||
const sameParent = sourceInfo.parentId === parentId;
|
||||
const adjust =
|
||||
sameParent && sourceInfo.index < info.index ? -1 : 0;
|
||||
index = info.index + adjust + (op.kind === 'reorder-after' ? 1 : 0);
|
||||
const adjust = sameParent && sourceInfo.index < info.index ? -1 : 0;
|
||||
index = info.index + adjust + (op.kind === "reorder-after" ? 1 : 0);
|
||||
}
|
||||
|
||||
const next = treeModel.place(tree, sourceId, { parentId, index });
|
||||
|
||||
@@ -6,6 +6,8 @@ import {
|
||||
collectBranchIds,
|
||||
openBranches,
|
||||
closeIds,
|
||||
mergeRootTrees,
|
||||
loadedOpenBranchIds,
|
||||
} from "./utils";
|
||||
import type { IPage } from "@/features/page/types/page.types.ts";
|
||||
import type { SpaceTreeNode } from "@/features/page/tree/types.ts";
|
||||
@@ -44,10 +46,7 @@ function flatNode(
|
||||
}
|
||||
|
||||
// Nested SpaceTreeNode factory for collectAllIds / collectBranchIds.
|
||||
function treeNode(
|
||||
id: string,
|
||||
children: SpaceTreeNode[] = [],
|
||||
): SpaceTreeNode {
|
||||
function treeNode(id: string, children: SpaceTreeNode[] = []): SpaceTreeNode {
|
||||
return {
|
||||
id,
|
||||
slugId: `slug-${id}`,
|
||||
@@ -94,11 +93,7 @@ describe("collectBranchIds", () => {
|
||||
]),
|
||||
treeNode("root2", [treeNode("leaf3")]),
|
||||
];
|
||||
expect(collectBranchIds(tree).sort()).toEqual([
|
||||
"branch1",
|
||||
"root",
|
||||
"root2",
|
||||
]);
|
||||
expect(collectBranchIds(tree).sort()).toEqual(["branch1", "root", "root2"]);
|
||||
});
|
||||
|
||||
it("returns [] for a leaf-only tree", () => {
|
||||
@@ -273,3 +268,95 @@ describe("closeIds", () => {
|
||||
expect(twice).toEqual({ keep: true, a: false, b: false });
|
||||
});
|
||||
});
|
||||
|
||||
describe("mergeRootTrees (#159 #2 reconnect reconcile)", () => {
|
||||
// Root node with a position and optional already-loaded children.
|
||||
function root(
|
||||
id: string,
|
||||
position: string,
|
||||
children?: SpaceTreeNode[],
|
||||
): SpaceTreeNode {
|
||||
return {
|
||||
id,
|
||||
slugId: `slug-${id}`,
|
||||
name: id.toUpperCase(),
|
||||
icon: undefined,
|
||||
position,
|
||||
spaceId: "space-1",
|
||||
parentPageId: null as unknown as string,
|
||||
hasChildren: !!children?.length,
|
||||
children: children as SpaceTreeNode[],
|
||||
};
|
||||
}
|
||||
|
||||
it("DROPS a stale root that is absent from the incoming (authoritative) set", () => {
|
||||
// 'ghost' was a root before the gap; the server's current roots no longer
|
||||
// include it (deleted / moved under another page). It must not linger.
|
||||
const prev = [root("a", "a0"), root("ghost", "a2"), root("b", "a4")];
|
||||
const incoming = [root("a", "a0"), root("b", "a4")];
|
||||
const merged = mergeRootTrees(prev, incoming);
|
||||
expect(merged.map((n) => n.id)).toEqual(["a", "b"]);
|
||||
expect(merged.find((n) => n.id === "ghost")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("PRESERVES a surviving root's lazy-loaded children (subtree not lost on refetch)", () => {
|
||||
const loadedChild = root("a1", "a0");
|
||||
const prev = [root("a", "a0", [loadedChild])];
|
||||
// The root query returns only top-level roots (no children).
|
||||
const incoming = [root("a", "a0")];
|
||||
const merged = mergeRootTrees(prev, incoming);
|
||||
expect(merged[0].children?.map((c) => c.id)).toEqual(["a1"]);
|
||||
});
|
||||
|
||||
it("ADDS a new incoming root", () => {
|
||||
const prev = [root("a", "a0")];
|
||||
const incoming = [root("a", "a0"), root("new", "a2")];
|
||||
const merged = mergeRootTrees(prev, incoming);
|
||||
expect(merged.map((n) => n.id)).toEqual(["a", "new"]);
|
||||
});
|
||||
|
||||
it("REFRESHES a surviving root's own fields from the incoming copy (e.g. rename)", () => {
|
||||
const prev = [{ ...root("a", "a0"), name: "OLD" }];
|
||||
const incoming = [{ ...root("a", "a0"), name: "NEW" }];
|
||||
const merged = mergeRootTrees(prev, incoming);
|
||||
expect(merged[0].name).toBe("NEW");
|
||||
});
|
||||
});
|
||||
|
||||
describe("loadedOpenBranchIds (#159 #8 reconnect refresh targets)", () => {
|
||||
function n(id: string, children?: SpaceTreeNode[]): SpaceTreeNode {
|
||||
return {
|
||||
id,
|
||||
slugId: `slug-${id}`,
|
||||
name: id.toUpperCase(),
|
||||
icon: undefined,
|
||||
position: "a0",
|
||||
spaceId: "space-1",
|
||||
parentPageId: null as unknown as string,
|
||||
hasChildren: !!children,
|
||||
children: children as SpaceTreeNode[],
|
||||
};
|
||||
}
|
||||
|
||||
it("returns OPEN branches whose children are loaded (array)", () => {
|
||||
const tree = [n("a", [n("a1")]), n("b", [n("b1")])];
|
||||
const ids = loadedOpenBranchIds(tree, new Set(["a"]));
|
||||
expect(ids).toEqual(["a"]); // b is closed; a is open+loaded
|
||||
});
|
||||
|
||||
it("skips an open branch whose children are NOT loaded (undefined)", () => {
|
||||
const tree = [n("a")]; // children undefined
|
||||
expect(loadedOpenBranchIds(tree, new Set(["a"]))).toEqual([]);
|
||||
});
|
||||
|
||||
it("includes a loaded-but-empty open branch (a child may have been added during the gap)", () => {
|
||||
const tree = [n("a", [])];
|
||||
expect(loadedOpenBranchIds(tree, new Set(["a"]))).toEqual(["a"]);
|
||||
});
|
||||
|
||||
it("walks nested open+loaded branches (deep chain refreshes every level)", () => {
|
||||
const tree = [n("a", [n("a1", [n("a1a")])])];
|
||||
const ids = loadedOpenBranchIds(tree, new Set(["a", "a1"]));
|
||||
expect(ids.sort()).toEqual(["a", "a1"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -214,21 +214,59 @@ export function appendNodeChildren(
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge root nodes; keep existing ones intact, append new ones,
|
||||
* Reconcile the loaded root nodes to the authoritative INCOMING set (the
|
||||
* server's complete current roots for the space), preserving any lazy-loaded
|
||||
* children/subtree of a root that still exists.
|
||||
*
|
||||
* This runs only once all root pages are fetched, so `incomingRoots` is the full
|
||||
* server root set and is authoritative for WHICH roots exist:
|
||||
* - a root in BOTH: kept, with its own fields refreshed from `incoming` (so a
|
||||
* rename/move during a gap shows) while PRESERVING its previously lazy-loaded
|
||||
* `children` (expanded subtrees + open-state survive a refetch);
|
||||
* - a root only in `incoming`: a new root, added as-is;
|
||||
* - a root only in `prev`: it was DELETED or moved under another page while we
|
||||
* were not receiving events (e.g. a socket reconnect after a sleep/wifi gap).
|
||||
* It is DROPPED instead of lingering as a 404 "ghost" root (#159 #2). The old
|
||||
* append-only merge kept it forever.
|
||||
*/
|
||||
export function mergeRootTrees(
|
||||
prevRoots: SpaceTreeNode[],
|
||||
incomingRoots: SpaceTreeNode[],
|
||||
): SpaceTreeNode[] {
|
||||
const seen = new Set(prevRoots.map((r) => r.id));
|
||||
const prevById = new Map(prevRoots.map((r) => [r.id, r]));
|
||||
|
||||
// add new roots that were not present before
|
||||
const merged = [...prevRoots];
|
||||
incomingRoots.forEach((node) => {
|
||||
if (!seen.has(node.id)) merged.push(node);
|
||||
const reconciled = incomingRoots.map((incoming) => {
|
||||
const prev = prevById.get(incoming.id);
|
||||
// Preserve the previously loaded children/subtree (the root query returns
|
||||
// only top-level roots, so `incoming` carries no children); refresh the
|
||||
// node's own fields from the authoritative incoming copy.
|
||||
return prev ? { ...incoming, children: prev.children } : incoming;
|
||||
});
|
||||
|
||||
return sortPositionKeys(merged);
|
||||
return sortPositionKeys(reconciled);
|
||||
}
|
||||
|
||||
/**
|
||||
* Ids of branches a socket-reconnect refresh should re-fetch and reconcile
|
||||
* (#159 #8): a node that is currently OPEN and whose children are LOADED
|
||||
* (`children` is an array — possibly empty). An unloaded branch (`children ===
|
||||
* undefined`) is skipped because lazy-load fetches it fresh on the next expand,
|
||||
* so there is nothing stale to reconcile. Walks the whole tree (a deep open
|
||||
* chain refreshes every loaded level).
|
||||
*/
|
||||
export function loadedOpenBranchIds(
|
||||
tree: SpaceTreeNode[],
|
||||
openIds: ReadonlySet<string>,
|
||||
): string[] {
|
||||
const ids: string[] = [];
|
||||
const walk = (nodes: SpaceTreeNode[]) => {
|
||||
for (const n of nodes) {
|
||||
if (openIds.has(n.id) && Array.isArray(n.children)) ids.push(n.id);
|
||||
if (n.children) walk(n.children);
|
||||
}
|
||||
};
|
||||
walk(tree);
|
||||
return ids;
|
||||
}
|
||||
|
||||
// Collect every node id in the tree (roots, branches, leaves). Used by
|
||||
|
||||
@@ -81,6 +81,38 @@ describe("applyMoveTreeNode", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("does NOT create a partial child list when the destination is loaded-but-collapsed (children unloaded) — keeps it lazy-loadable (#159)", () => {
|
||||
// `dstCollapsed` is in the tree but its children were never lazy-loaded
|
||||
// (children === undefined). The OLD behavior inserted `src` as the ONLY
|
||||
// child ([src]), which defeated the lazy-load gate and HID the parent's
|
||||
// other real children. Now the move leaves children unloaded (so expanding
|
||||
// fetches the FULL set, including src) and just flags hasChildren.
|
||||
const tree: SpaceTreeNode[] = [
|
||||
node("dstCollapsed", {
|
||||
position: "a0",
|
||||
hasChildren: false,
|
||||
children: undefined as unknown as SpaceTreeNode[],
|
||||
}),
|
||||
node("src", { position: "a9" }),
|
||||
];
|
||||
const next = applyMoveTreeNode(tree, {
|
||||
id: "src",
|
||||
parentId: "dstCollapsed",
|
||||
oldParentId: null,
|
||||
index: 0,
|
||||
position: "a4",
|
||||
pageData: {},
|
||||
});
|
||||
const dst = treeModel.find(next, "dstCollapsed");
|
||||
// Children stay unloaded -> the lazy-load gate fetches the FULL set (incl.
|
||||
// src) on expand, rather than showing a misleading partial [src] list.
|
||||
expect(dst?.children).toBeUndefined();
|
||||
expect(dst?.hasChildren).toBe(true);
|
||||
// src moved away from its old root slot (it lives under dstCollapsed
|
||||
// server-side and reappears when the parent is expanded/loaded).
|
||||
expect(next.map((n) => n.id)).not.toContain("src");
|
||||
});
|
||||
|
||||
it("flips the OLD parent's hasChildren to false when it is left childless", () => {
|
||||
// src is the only child of `old`; moving it to `dst` empties `old`.
|
||||
const tree: SpaceTreeNode[] = [
|
||||
@@ -151,6 +183,34 @@ describe("applyMoveTreeNode", () => {
|
||||
expect(moved?.hasChildren).toBe(true);
|
||||
expect(moved?.position).toBe("a4");
|
||||
});
|
||||
|
||||
it("does NOT drop a subtree on a cyclic/out-of-order move (parent inside source) (#206 ui-state-races-1)", () => {
|
||||
// Locally `b` is still nested inside `a` (an earlier "a under b" echo hasn't
|
||||
// applied yet). An out-of-order "move a under b" event now arrives — b is a
|
||||
// descendant of a, so re-parenting would make placeByPosition remove a (and
|
||||
// its whole subtree, incl. b) and fail to re-insert. Before the fix BOTH a
|
||||
// and b silently vanished; now the reducer leaves the tree untouched.
|
||||
const tree: SpaceTreeNode[] = [
|
||||
node("a", {
|
||||
position: "a0",
|
||||
hasChildren: true,
|
||||
children: [node("b", { position: "a1", parentPageId: "a" })],
|
||||
}),
|
||||
];
|
||||
const next = applyMoveTreeNode(tree, {
|
||||
id: "a",
|
||||
parentId: "b",
|
||||
oldParentId: null,
|
||||
index: 0,
|
||||
position: "a4",
|
||||
pageData: {},
|
||||
});
|
||||
// No silent data loss: both nodes survive.
|
||||
expect(treeModel.find(next, "a")).not.toBeNull();
|
||||
expect(treeModel.find(next, "b")).not.toBeNull();
|
||||
// The cyclic move is refused as a no-op (same reference) pending reconcile.
|
||||
expect(next).toBe(tree);
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyDeleteTreeNode", () => {
|
||||
@@ -164,7 +224,9 @@ describe("applyDeleteTreeNode", () => {
|
||||
position: "a1",
|
||||
parentPageId: "p",
|
||||
hasChildren: true,
|
||||
children: [node("grandchild", { position: "a1", parentPageId: "child" })],
|
||||
children: [
|
||||
node("grandchild", { position: "a1", parentPageId: "child" }),
|
||||
],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
|
||||
@@ -76,6 +76,19 @@ export function applyMoveTreeNode(
|
||||
const oldParentId = (sourceBefore as SpaceTreeNode).parentPageId ?? null;
|
||||
const newParentId = payload.parentId as string | null;
|
||||
|
||||
// Cyclic / out-of-order move guard (#206 ui-state-races-1): if the
|
||||
// authoritative new parent is currently INSIDE the moved node's own subtree on
|
||||
// this client (e.g. server moved X under Y then Y under X and the events
|
||||
// arrived such that Y is still nested in X here), re-parenting is impossible to
|
||||
// represent locally. `placeByPosition` returns `prev` for this, but the
|
||||
// `placed === prev` fallback below would then `remove` the source — dropping
|
||||
// the node AND every descendant (incl. the would-be parent) silently. Leave the
|
||||
// tree untouched instead; a later corrective event or a reconnect refetch
|
||||
// reconciles it. Never delete a subtree we cannot safely re-place.
|
||||
if (newParentId && treeModel.isDescendant(prev, payload.id, newParentId)) {
|
||||
return prev;
|
||||
}
|
||||
|
||||
// Place the node by its fractional `position` among the new siblings — NOT by
|
||||
// the sender's absolute `index` (the sender computed that against its own
|
||||
// loaded set, which differs from this receiver's). Using the position keeps
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { mcpTestButtonView } from "./ai-mcp-server-test-view";
|
||||
|
||||
/**
|
||||
* Pure-helper tests for the inline "Test" button presentation. Covers the four
|
||||
* states (idle / loading is handled by the component's `isPending`, so here:
|
||||
* idle / ok-with-tools / ok-without-tools / failed) and the tooltip text
|
||||
* branches that are easiest to break silently.
|
||||
*/
|
||||
// Identity-ish translator that echoes the key and interpolates {{n}} so the
|
||||
// label/tooltip branches are observable without the real i18n bundle.
|
||||
const t = (key: string, options?: Record<string, unknown>): string =>
|
||||
options && "n" in options
|
||||
? key.replace("{{n}}", String((options as { n: unknown }).n))
|
||||
: key;
|
||||
|
||||
describe("mcpTestButtonView", () => {
|
||||
it("idle when there is no result", () => {
|
||||
expect(mcpTestButtonView(undefined, t)).toEqual({
|
||||
state: "idle",
|
||||
color: undefined,
|
||||
variant: "default",
|
||||
label: "Test",
|
||||
tooltip: "",
|
||||
});
|
||||
});
|
||||
|
||||
it("ok with tools lists them in the tooltip", () => {
|
||||
expect(mcpTestButtonView({ ok: true, tools: ["a", "b"] }, t)).toEqual({
|
||||
state: "ok",
|
||||
color: "green",
|
||||
variant: "light",
|
||||
label: "OK · 2",
|
||||
tooltip: "a, b",
|
||||
});
|
||||
});
|
||||
|
||||
it('ok with zero tools shows "No tools available"', () => {
|
||||
expect(mcpTestButtonView({ ok: true, tools: [] }, t)).toEqual({
|
||||
state: "ok",
|
||||
color: "green",
|
||||
variant: "light",
|
||||
label: "OK · 0",
|
||||
tooltip: "No tools available",
|
||||
});
|
||||
});
|
||||
|
||||
it("failed surfaces the error text in the tooltip", () => {
|
||||
expect(
|
||||
mcpTestButtonView({ ok: false, error: "402: nope" }, t),
|
||||
).toEqual({
|
||||
state: "failed",
|
||||
color: "red",
|
||||
variant: "light",
|
||||
label: "Failed",
|
||||
tooltip: "402: nope",
|
||||
});
|
||||
});
|
||||
|
||||
it("failed when the request itself rejects (no result payload)", () => {
|
||||
// 401/403/500/network: there is no { ok } body, only a thrown error. The
|
||||
// row must still show a red "Failed" rather than reverting to idle "Test".
|
||||
expect(
|
||||
mcpTestButtonView(undefined, t, {
|
||||
response: { data: { message: "Unauthorized" } },
|
||||
}),
|
||||
).toEqual({
|
||||
state: "failed",
|
||||
color: "red",
|
||||
variant: "light",
|
||||
label: "Failed",
|
||||
tooltip: "Unauthorized",
|
||||
});
|
||||
});
|
||||
|
||||
it("reject without a server message falls back to the generic label", () => {
|
||||
// A bare network error (no response body) still surfaces as failed, using
|
||||
// the i18n fallback for the tooltip.
|
||||
expect(mcpTestButtonView(undefined, t, new Error("network down"))).toEqual({
|
||||
state: "failed",
|
||||
color: "red",
|
||||
variant: "light",
|
||||
label: "Failed",
|
||||
tooltip: "Failed to update data",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { IAiMcpServerTestResult } from "@/features/workspace/services/ai-mcp-server-service.ts";
|
||||
|
||||
/** Minimal translator shape (i18next `t`): key + optional interpolation. */
|
||||
type Translate = (key: string, options?: Record<string, unknown>) => string;
|
||||
|
||||
/** Subset of an axios-style rejection we read for the reject tooltip. */
|
||||
type McpTestRequestError = {
|
||||
response?: { data?: { message?: string } };
|
||||
};
|
||||
|
||||
/**
|
||||
* Best-effort extraction of a server-sent message from a rejected test request
|
||||
* (axios stores it at `error.response.data.message`). Returns undefined for a
|
||||
* bare/network error so the caller can fall back to a generic label.
|
||||
*/
|
||||
function readRequestErrorMessage(error: unknown): string | undefined {
|
||||
if (error && typeof error === "object" && "response" in error) {
|
||||
return (error as McpTestRequestError).response?.data?.message;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Presentation for the inline "Test" button, derived from the current test
|
||||
* result tristate (no result yet / ok / failed). Color is never the only signal
|
||||
* — the label and icon change too (a11y / colorblind-friendly). Kept as a single
|
||||
* pure derivation (rather than two parallel if/else chains) so the button and
|
||||
* tooltip can never drift apart, and so the text branches are unit-testable
|
||||
* without rendering the row.
|
||||
*/
|
||||
export interface McpTestButtonView {
|
||||
/** Tristate; the component maps this to the leftSection icon. */
|
||||
state: "idle" | "ok" | "failed";
|
||||
/** Mantine Button color; undefined = theme default (idle). */
|
||||
color?: string;
|
||||
/** Mantine Button variant. */
|
||||
variant: string;
|
||||
/** Translated button label. */
|
||||
label: string;
|
||||
/** Translated tooltip text; "" while there is no result (tooltip disabled). */
|
||||
tooltip: string;
|
||||
}
|
||||
|
||||
export function mcpTestButtonView(
|
||||
result: IAiMcpServerTestResult | undefined,
|
||||
t: Translate,
|
||||
error?: unknown,
|
||||
): McpTestButtonView {
|
||||
if (result?.ok) {
|
||||
return {
|
||||
state: "ok",
|
||||
color: "green",
|
||||
variant: "light",
|
||||
label: t("OK · {{n}}", { n: result.tools.length }),
|
||||
tooltip:
|
||||
result.tools.length > 0
|
||||
? result.tools.join(", ")
|
||||
: t("No tools available"),
|
||||
};
|
||||
}
|
||||
if (result && result.ok === false) {
|
||||
return {
|
||||
state: "failed",
|
||||
color: "red",
|
||||
variant: "light",
|
||||
label: t("Failed"),
|
||||
tooltip: result.error,
|
||||
};
|
||||
}
|
||||
if (error) {
|
||||
// The test request itself rejected (401/403/500/network) — there is no
|
||||
// `{ ok }` payload, so without this branch the row would silently revert to
|
||||
// the idle "Test" instead of reporting the failure. Tooltip prefers the
|
||||
// server-sent message, else the generic i18n fallback.
|
||||
return {
|
||||
state: "failed",
|
||||
color: "red",
|
||||
variant: "light",
|
||||
label: t("Failed"),
|
||||
tooltip: readRequestErrorMessage(error) ?? t("Failed to update data"),
|
||||
};
|
||||
}
|
||||
return {
|
||||
state: "idle",
|
||||
color: undefined,
|
||||
variant: "default",
|
||||
label: t("Test"),
|
||||
tooltip: "",
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { useState } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import {
|
||||
ActionIcon,
|
||||
Badge,
|
||||
@@ -10,18 +10,28 @@ import {
|
||||
Stack,
|
||||
Switch,
|
||||
Text,
|
||||
Tooltip,
|
||||
} from "@mantine/core";
|
||||
import { useDisclosure } from "@mantine/hooks";
|
||||
import { modals } from "@mantine/modals";
|
||||
import { IconPencil, IconPlus, IconTrash } from "@tabler/icons-react";
|
||||
import {
|
||||
IconCheck,
|
||||
IconPencil,
|
||||
IconPlugConnected,
|
||||
IconPlus,
|
||||
IconTrash,
|
||||
IconX,
|
||||
} from "@tabler/icons-react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import useUserRole from "@/hooks/use-user-role.tsx";
|
||||
import {
|
||||
useAiMcpServersQuery,
|
||||
useDeleteAiMcpServerMutation,
|
||||
useTestAiMcpServerMutation,
|
||||
useUpdateAiMcpServerMutation,
|
||||
} from "@/features/workspace/queries/ai-mcp-server-query.ts";
|
||||
import { IAiMcpServer } from "@/features/workspace/services/ai-mcp-server-service.ts";
|
||||
import { mcpTestButtonView } from "@/features/workspace/components/settings/components/ai-mcp-server-test-view.ts";
|
||||
import AiMcpServerForm from "./ai-mcp-server-form.tsx";
|
||||
|
||||
/**
|
||||
@@ -112,55 +122,15 @@ export default function AiMcpServers() {
|
||||
|
||||
<Stack gap="xs" mt="sm">
|
||||
{servers?.map((server) => (
|
||||
<Group key={server.id} justify="space-between" wrap="nowrap">
|
||||
<Stack gap={2} style={{ minWidth: 0 }}>
|
||||
<Group gap="xs">
|
||||
<Text fw={500} truncate>
|
||||
{server.name}
|
||||
</Text>
|
||||
<Badge size="xs" variant="light">
|
||||
{server.transport.toUpperCase()}
|
||||
</Badge>
|
||||
</Group>
|
||||
<Text
|
||||
size="xs"
|
||||
c="dimmed"
|
||||
truncate
|
||||
style={{ fontFamily: "ui-monospace, Menlo, monospace" }}
|
||||
>
|
||||
{server.url}
|
||||
</Text>
|
||||
</Stack>
|
||||
|
||||
<Group gap="xs" wrap="nowrap">
|
||||
<Switch
|
||||
size="sm"
|
||||
checked={server.enabled}
|
||||
aria-label={t("Enabled")}
|
||||
onChange={(event) =>
|
||||
updateMutation.mutate({
|
||||
id: server.id,
|
||||
enabled: event.currentTarget.checked,
|
||||
})
|
||||
}
|
||||
/>
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
aria-label={t("Edit")}
|
||||
onClick={() => openEdit(server)}
|
||||
>
|
||||
<IconPencil size={16} />
|
||||
</ActionIcon>
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
color="red"
|
||||
aria-label={t("Delete")}
|
||||
onClick={() => confirmDelete(server)}
|
||||
>
|
||||
<IconTrash size={16} />
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
</Group>
|
||||
<AiMcpServerRow
|
||||
key={server.id}
|
||||
server={server}
|
||||
onEdit={openEdit}
|
||||
onDelete={confirmDelete}
|
||||
onToggleEnabled={(enabled) =>
|
||||
updateMutation.mutate({ id: server.id, enabled })
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</Stack>
|
||||
|
||||
@@ -180,3 +150,127 @@ export default function AiMcpServers() {
|
||||
</Paper>
|
||||
);
|
||||
}
|
||||
|
||||
interface AiMcpServerRowProps {
|
||||
server: IAiMcpServer;
|
||||
onEdit: (server: IAiMcpServer) => void;
|
||||
onDelete: (server: IAiMcpServer) => void;
|
||||
onToggleEnabled: (enabled: boolean) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* A single external MCP server row: name/badge/url on the left and the
|
||||
* Test / Switch / Edit / Delete controls on the right. Each row owns its own
|
||||
* `useTestAiMcpServerMutation()` so the inline Test result and loading state are
|
||||
* independent per row (a shared mutation would make `isPending` global and make
|
||||
* every row flicker).
|
||||
*/
|
||||
function AiMcpServerRow({
|
||||
server,
|
||||
onEdit,
|
||||
onDelete,
|
||||
onToggleEnabled,
|
||||
}: AiMcpServerRowProps) {
|
||||
const { t } = useTranslation();
|
||||
const testMutation = useTestAiMcpServerMutation();
|
||||
const result = testMutation.data;
|
||||
|
||||
// The row is keyed by `server.id`, so editing the connection-relevant fields
|
||||
// (url/transport/headers) does NOT remount it — an old success/failure result
|
||||
// would otherwise stick. Clear the result when those fields change.
|
||||
useEffect(() => {
|
||||
testMutation.reset();
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [server.url, server.transport, server.hasHeaders]);
|
||||
|
||||
// Single derivation of the button/tooltip presentation from the test tristate
|
||||
// (idle / ok / failed), so the two can never drift apart. Tooltip is "" while
|
||||
// there is no result; the icon is mapped from `view.state` below. When the
|
||||
// request itself rejects (401/403/500/network) there is no `data` payload, so
|
||||
// we feed the mutation error in too — otherwise the row would silently revert
|
||||
// to "Test" instead of showing a red "Failed".
|
||||
const view = mcpTestButtonView(
|
||||
result,
|
||||
t,
|
||||
testMutation.isError ? testMutation.error : undefined,
|
||||
);
|
||||
const tooltipLabel = view.tooltip;
|
||||
const buttonColor = view.color;
|
||||
const buttonVariant = view.variant;
|
||||
const buttonLabel = view.label;
|
||||
const buttonIcon =
|
||||
view.state === "ok" ? (
|
||||
<IconCheck size={16} />
|
||||
) : view.state === "failed" ? (
|
||||
<IconX size={16} />
|
||||
) : (
|
||||
<IconPlugConnected size={16} />
|
||||
);
|
||||
|
||||
return (
|
||||
<Group justify="space-between" wrap="nowrap">
|
||||
<Stack gap={2} style={{ minWidth: 0 }}>
|
||||
<Group gap="xs">
|
||||
<Text fw={500} truncate>
|
||||
{server.name}
|
||||
</Text>
|
||||
<Badge size="xs" variant="light">
|
||||
{server.transport.toUpperCase()}
|
||||
</Badge>
|
||||
</Group>
|
||||
<Text
|
||||
size="xs"
|
||||
c="dimmed"
|
||||
truncate
|
||||
style={{ fontFamily: "ui-monospace, Menlo, monospace" }}
|
||||
>
|
||||
{server.url}
|
||||
</Text>
|
||||
</Stack>
|
||||
|
||||
<Group gap="xs" wrap="nowrap">
|
||||
{/* Always clickable: testing a disabled server before enabling it is useful. */}
|
||||
<Tooltip
|
||||
label={tooltipLabel}
|
||||
disabled={view.state === "idle"}
|
||||
multiline
|
||||
maw={320}
|
||||
withinPortal
|
||||
>
|
||||
<Button
|
||||
size="xs"
|
||||
miw={88}
|
||||
color={buttonColor}
|
||||
variant={buttonVariant}
|
||||
leftSection={testMutation.isPending ? undefined : buttonIcon}
|
||||
loading={testMutation.isPending}
|
||||
onClick={() => testMutation.mutate(server.id)}
|
||||
>
|
||||
{buttonLabel}
|
||||
</Button>
|
||||
</Tooltip>
|
||||
<Switch
|
||||
size="sm"
|
||||
checked={server.enabled}
|
||||
aria-label={t("Enabled")}
|
||||
onChange={(event) => onToggleEnabled(event.currentTarget.checked)}
|
||||
/>
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
aria-label={t("Edit")}
|
||||
onClick={() => onEdit(server)}
|
||||
>
|
||||
<IconPencil size={16} />
|
||||
</ActionIcon>
|
||||
<ActionIcon
|
||||
variant="subtle"
|
||||
color="red"
|
||||
aria-label={t("Delete")}
|
||||
onClick={() => onDelete(server)}
|
||||
>
|
||||
<IconTrash size={16} />
|
||||
</ActionIcon>
|
||||
</Group>
|
||||
</Group>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
Button,
|
||||
Group,
|
||||
Modal,
|
||||
NumberInput,
|
||||
Paper,
|
||||
PasswordInput,
|
||||
Select,
|
||||
@@ -83,6 +84,9 @@ const STT_LANGUAGE_OPTIONS: { value: string; label: string }[] = [
|
||||
// (empty means "leave unchanged" unless explicitly cleared).
|
||||
const formSchema = z.object({
|
||||
chatModel: z.string(),
|
||||
// Max context window in tokens shown in the chat header badge. A number, or ""
|
||||
// when the NumberInput is empty (no limit).
|
||||
chatContextWindow: z.union([z.number(), z.literal("")]),
|
||||
// Chat provider implementation (reasoning surfacing). Default openai-compatible.
|
||||
chatApiStyle: z.enum(["openai-compatible", "openai"]),
|
||||
// Cheap model id for the anonymous public-share assistant; empty = use chatModel.
|
||||
@@ -311,6 +315,7 @@ export default function AiProviderSettings() {
|
||||
validate: zod4Resolver(formSchema),
|
||||
initialValues: {
|
||||
chatModel: "",
|
||||
chatContextWindow: "",
|
||||
chatApiStyle: "openai-compatible" as ChatApiStyle,
|
||||
publicShareChatModel: "",
|
||||
publicShareAssistantRoleId: "",
|
||||
@@ -334,6 +339,7 @@ export default function AiProviderSettings() {
|
||||
if (!settings) return;
|
||||
form.setValues({
|
||||
chatModel: settings.chatModel ?? "",
|
||||
chatContextWindow: settings.chatContextWindow ?? "",
|
||||
chatApiStyle: settings.chatApiStyle ?? "openai-compatible",
|
||||
publicShareChatModel: settings.publicShareChatModel ?? "",
|
||||
publicShareAssistantRoleId: settings.publicShareAssistantRoleId ?? "",
|
||||
@@ -364,6 +370,12 @@ export default function AiProviderSettings() {
|
||||
// Everything is OpenAI-compatible.
|
||||
driver: "openai",
|
||||
chatModel: values.chatModel,
|
||||
// Max context window for the chat header badge; empty NumberInput ("") →
|
||||
// 0, which clears the limit server-side (no denominator shown).
|
||||
chatContextWindow:
|
||||
typeof values.chatContextWindow === "number"
|
||||
? values.chatContextWindow
|
||||
: 0,
|
||||
chatApiStyle: values.chatApiStyle,
|
||||
// Cheap model id for the anonymous public-share assistant; empty falls
|
||||
// back to chatModel server-side.
|
||||
@@ -767,6 +779,18 @@ export default function AiProviderSettings() {
|
||||
{t("Resolves to {{url}}", { url: chatResolved })}
|
||||
</Text>
|
||||
|
||||
<NumberInput
|
||||
mt="sm"
|
||||
label={t("Context window (tokens)")}
|
||||
description={t(
|
||||
"Shown as used / total in the chat header. Leave empty to hide the limit.",
|
||||
)}
|
||||
min={0}
|
||||
allowDecimal={false}
|
||||
disabled={isLoading}
|
||||
{...form.getInputProps("chatContextWindow")}
|
||||
/>
|
||||
|
||||
<Select
|
||||
mt="sm"
|
||||
label={t("Protocol")}
|
||||
|
||||
@@ -22,6 +22,8 @@ export type ChatApiStyle = "openai-compatible" | "openai";
|
||||
export interface IAiSettings {
|
||||
driver?: AiDriver;
|
||||
chatModel?: string;
|
||||
// Max context window in tokens shown in the chat header badge; 0/unset = no limit.
|
||||
chatContextWindow?: number;
|
||||
chatApiStyle?: ChatApiStyle;
|
||||
// Cheap model id for the anonymous public-share assistant; empty = chatModel.
|
||||
publicShareChatModel?: string;
|
||||
@@ -56,6 +58,8 @@ export interface IAiSettings {
|
||||
export interface IAiSettingsUpdate {
|
||||
driver?: AiDriver;
|
||||
chatModel?: string;
|
||||
// Max context window in tokens for the chat header badge; 0 = clear the limit.
|
||||
chatContextWindow?: number;
|
||||
chatApiStyle?: ChatApiStyle;
|
||||
publicShareChatModel?: string;
|
||||
// Agent-role id whose persona the public-share assistant adopts; empty =
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "server",
|
||||
"version": "0.93.0",
|
||||
"version": "0.94.0",
|
||||
"description": "",
|
||||
"author": "",
|
||||
"private": true,
|
||||
|
||||
@@ -182,4 +182,46 @@ describe('PersistenceExtension.onStoreDocument — Approach-A boundary snapshot'
|
||||
expect(pageHistoryRepo.saveHistory).not.toHaveBeenCalled();
|
||||
expect(historyQueue.add).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// persist-1 — a transient DB failure during store must not silently lose the
|
||||
// edit. hocuspocus unloads (destroys) the in-memory Y.Doc right after this
|
||||
// hook resolves, so the store has to retry while it still holds the only copy.
|
||||
it('retries a transient DB failure and still persists the edit (persist-1)', async () => {
|
||||
const document = ydocFor(doc('NEW HUMAN CONTENT'));
|
||||
pageRepo.findById.mockResolvedValue(persistedHumanPage('NEW HUMAN CONTENT'));
|
||||
let attempts = 0;
|
||||
pageRepo.updatePage.mockImplementation(async () => {
|
||||
attempts += 1;
|
||||
if (attempts === 1) throw new Error('deadlock detected'); // transient
|
||||
callOrder.push('updatePage');
|
||||
});
|
||||
|
||||
await ext.onStoreDocument(buildData(document, 'user') as any);
|
||||
|
||||
// First attempt failed and rolled back; the retry persisted the edit.
|
||||
expect(pageRepo.updatePage).toHaveBeenCalledTimes(2);
|
||||
// The edit WAS saved, so the post-store success path runs as normal.
|
||||
expect((document as any).broadcastStateless).toHaveBeenCalledTimes(1);
|
||||
expect(historyQueue.add).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
// persist-1 — when every attempt fails the hook must NOT report a phantom
|
||||
// success: no "page.updated" badge broadcast and no history snapshot for
|
||||
// content that was never written.
|
||||
it('does not run post-store side effects when every store attempt fails (persist-1)', async () => {
|
||||
const document = ydocFor(doc('NEW HUMAN CONTENT'));
|
||||
pageRepo.findById.mockResolvedValue(persistedHumanPage('NEW HUMAN CONTENT'));
|
||||
pageRepo.updatePage.mockRejectedValue(new Error('connection reset'));
|
||||
|
||||
await expect(
|
||||
ext.onStoreDocument(buildData(document, 'user') as any),
|
||||
).resolves.toBeUndefined();
|
||||
|
||||
// Bounded retry exhausted (MAX_STORE_ATTEMPTS).
|
||||
expect(pageRepo.updatePage).toHaveBeenCalledTimes(3);
|
||||
// No false-success: nothing downstream fires for the unsaved content.
|
||||
expect((document as any).broadcastStateless).not.toHaveBeenCalled();
|
||||
expect(historyQueue.add).not.toHaveBeenCalled();
|
||||
expect(aiQueue.add).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -181,83 +181,113 @@ export class PersistenceExtension implements Extension {
|
||||
context?.actor,
|
||||
);
|
||||
|
||||
try {
|
||||
await executeTx(this.db, async (trx) => {
|
||||
page = await this.pageRepo.findById(pageId, {
|
||||
withLock: true,
|
||||
includeContent: true,
|
||||
trx,
|
||||
});
|
||||
// Persist with a small bounded retry. The in-memory Y.Doc is the ONLY copy
|
||||
// of the latest edit until this hook returns: hocuspocus destroys/unloads the
|
||||
// doc right after onStoreDocument resolves (see storeDocumentHooks' finally
|
||||
// -> unloadDocument). If a transient DB error (deadlock, serialization
|
||||
// failure, dropped connection) is merely logged and swallowed, the function
|
||||
// resolves "successfully", the doc is unloaded, and the edit is lost silently
|
||||
// (#206 persist-1). Retrying here re-attempts the write while we still hold
|
||||
// the doc; on total failure we clear `page` so the post-store side effects
|
||||
// (badge broadcast, history snapshot) never report a save that didn't happen.
|
||||
const MAX_STORE_ATTEMPTS = 3;
|
||||
for (let attempt = 1; attempt <= MAX_STORE_ATTEMPTS; attempt++) {
|
||||
try {
|
||||
await executeTx(this.db, async (trx) => {
|
||||
page = await this.pageRepo.findById(pageId, {
|
||||
withLock: true,
|
||||
includeContent: true,
|
||||
trx,
|
||||
});
|
||||
|
||||
if (!page) {
|
||||
this.logger.error(`Page with id ${pageId} not found`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (isDeepStrictEqual(tiptapJson, page.content)) {
|
||||
page = null;
|
||||
return;
|
||||
}
|
||||
|
||||
let contributorIds = undefined;
|
||||
try {
|
||||
const existingContributors = page.contributorIds || [];
|
||||
contributorIds = Array.from(
|
||||
new Set([
|
||||
...existingContributors,
|
||||
...editingUserIds,
|
||||
page.creatorId,
|
||||
]),
|
||||
);
|
||||
} catch (err) {
|
||||
//this.logger.debug('Contributors error:' + err?.['message']);
|
||||
}
|
||||
|
||||
// Approach A — boundary snapshot before the agent's first edit.
|
||||
// When this store is the agent's and the page's currently persisted
|
||||
// state was authored by a human, pin that human state as its own
|
||||
// history version BEFORE the agent overwrites it. `page` still holds the
|
||||
// OLD content/provenance here, so saveHistory(page) captures the
|
||||
// pre-agent state tagged 'user'. The agent's new content is snapshotted
|
||||
// later by the debounced PAGE_HISTORY job ('agent'). Skip if the prior
|
||||
// state is already agent-authored (boundary already pinned on the
|
||||
// user->agent transition), if the page is effectively empty, or if the
|
||||
// latest existing snapshot already equals this human state (avoid
|
||||
// duplicates).
|
||||
if (lastUpdatedSource === 'agent' && page.lastUpdatedSource !== 'agent') {
|
||||
const lastHistory = await this.pageHistoryRepo.findPageLastHistory(
|
||||
pageId,
|
||||
{ includeContent: true, trx },
|
||||
);
|
||||
const humanBaselineMissing =
|
||||
!lastHistory || !isDeepStrictEqual(lastHistory.content, page.content);
|
||||
if (!isEmptyParagraphDoc(page.content as any) && humanBaselineMissing) {
|
||||
await this.pageHistoryRepo.saveHistory(page, {
|
||||
contributorIds: page.contributorIds ?? undefined,
|
||||
trx,
|
||||
});
|
||||
if (!page) {
|
||||
this.logger.error(`Page with id ${pageId} not found`);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await this.pageRepo.updatePage(
|
||||
{
|
||||
content: tiptapJson,
|
||||
textContent: textContent,
|
||||
ydoc: ydocState,
|
||||
lastUpdatedById: context.user.id,
|
||||
// Human stays the responsible author; these annotate the source.
|
||||
lastUpdatedSource,
|
||||
lastUpdatedAiChatId: context?.aiChatId ?? null,
|
||||
contributorIds: contributorIds,
|
||||
},
|
||||
pageId,
|
||||
trx,
|
||||
if (isDeepStrictEqual(tiptapJson, page.content)) {
|
||||
page = null;
|
||||
return;
|
||||
}
|
||||
|
||||
let contributorIds = undefined;
|
||||
try {
|
||||
const existingContributors = page.contributorIds || [];
|
||||
contributorIds = Array.from(
|
||||
new Set([
|
||||
...existingContributors,
|
||||
...editingUserIds,
|
||||
page.creatorId,
|
||||
]),
|
||||
);
|
||||
} catch (err) {
|
||||
//this.logger.debug('Contributors error:' + err?.['message']);
|
||||
}
|
||||
|
||||
// Approach A — boundary snapshot before the agent's first edit.
|
||||
// When this store is the agent's and the page's currently persisted
|
||||
// state was authored by a human, pin that human state as its own
|
||||
// history version BEFORE the agent overwrites it. `page` still holds
|
||||
// the OLD content/provenance here, so saveHistory(page) captures the
|
||||
// pre-agent state tagged 'user'. The agent's new content is
|
||||
// snapshotted later by the debounced PAGE_HISTORY job ('agent'). Skip
|
||||
// if the prior state is already agent-authored (boundary already
|
||||
// pinned on the user->agent transition), if the page is effectively
|
||||
// empty, or if the latest existing snapshot already equals this human
|
||||
// state (avoid duplicates).
|
||||
if (
|
||||
lastUpdatedSource === 'agent' &&
|
||||
page.lastUpdatedSource !== 'agent'
|
||||
) {
|
||||
const lastHistory = await this.pageHistoryRepo.findPageLastHistory(
|
||||
pageId,
|
||||
{ includeContent: true, trx },
|
||||
);
|
||||
const humanBaselineMissing =
|
||||
!lastHistory ||
|
||||
!isDeepStrictEqual(lastHistory.content, page.content);
|
||||
if (
|
||||
!isEmptyParagraphDoc(page.content as any) &&
|
||||
humanBaselineMissing
|
||||
) {
|
||||
await this.pageHistoryRepo.saveHistory(page, {
|
||||
contributorIds: page.contributorIds ?? undefined,
|
||||
trx,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
await this.pageRepo.updatePage(
|
||||
{
|
||||
content: tiptapJson,
|
||||
textContent: textContent,
|
||||
ydoc: ydocState,
|
||||
lastUpdatedById: context.user.id,
|
||||
// Human stays the responsible author; these annotate the source.
|
||||
lastUpdatedSource,
|
||||
lastUpdatedAiChatId: context?.aiChatId ?? null,
|
||||
contributorIds: contributorIds,
|
||||
},
|
||||
pageId,
|
||||
trx,
|
||||
);
|
||||
|
||||
this.logger.debug(`Page updated: ${pageId} - SlugId: ${page.slugId}`);
|
||||
});
|
||||
break;
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to update page ${pageId} (attempt ${attempt}/${MAX_STORE_ATTEMPTS})`,
|
||||
err,
|
||||
);
|
||||
|
||||
this.logger.debug(`Page updated: ${pageId} - SlugId: ${page.slugId}`);
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.error(`Failed to update page ${pageId}`, err);
|
||||
// The write failed and rolled back; clear the partially-assigned `page`
|
||||
// so the post-store success branch below is skipped (no false "saved"
|
||||
// broadcast / history snapshot for content that was never persisted).
|
||||
page = null;
|
||||
if (attempt < MAX_STORE_ATTEMPTS) {
|
||||
await new Promise((resolve) => setTimeout(resolve, attempt * 50));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (page) {
|
||||
|
||||
159
apps/server/src/core/ai-chat/ai-chat.controller.export.spec.ts
Normal file
159
apps/server/src/core/ai-chat/ai-chat.controller.export.spec.ts
Normal file
@@ -0,0 +1,159 @@
|
||||
import { ForbiddenException } from '@nestjs/common';
|
||||
import { AiChatController } from './ai-chat.controller';
|
||||
import {
|
||||
planFinalizeAssistant,
|
||||
applyFinalize,
|
||||
flushAssistant,
|
||||
type AssistantFlush,
|
||||
} from './ai-chat.service';
|
||||
import type { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
|
||||
/**
|
||||
* Wiring spec for the #183 `POST /ai-chat/export` endpoint. It must: own-gate via
|
||||
* the chat lookup (workspace-scoped + creator-owned), load the FULL transcript
|
||||
* via findAllByChat, render server-side, and return `{ markdown }`. Exercised by
|
||||
* instantiating the controller with hand-rolled mocks — no Nest graph, no DB.
|
||||
*/
|
||||
describe('AiChatController.export', () => {
|
||||
const user = { id: 'u1' } as User;
|
||||
const workspace = { id: 'ws1' } as Workspace;
|
||||
|
||||
function makeController(
|
||||
over: {
|
||||
chat?: unknown;
|
||||
rows?: unknown[];
|
||||
} = {},
|
||||
) {
|
||||
const chat =
|
||||
'chat' in over
|
||||
? over.chat
|
||||
: { id: 'c1', creatorId: 'u1', title: 'My chat' };
|
||||
const aiChatRepo = {
|
||||
findById: jest.fn().mockResolvedValue(chat),
|
||||
};
|
||||
const aiChatMessageRepo = {
|
||||
findAllByChat: jest.fn().mockResolvedValue(
|
||||
over.rows ?? [
|
||||
{
|
||||
id: 'm1',
|
||||
role: 'user',
|
||||
content: 'hi',
|
||||
metadata: null,
|
||||
status: null,
|
||||
},
|
||||
{
|
||||
id: 'm2',
|
||||
role: 'assistant',
|
||||
content: 'hello',
|
||||
metadata: null,
|
||||
status: 'completed',
|
||||
},
|
||||
],
|
||||
),
|
||||
};
|
||||
const controller = new AiChatController(
|
||||
{} as never,
|
||||
aiChatRepo as never,
|
||||
aiChatMessageRepo as never,
|
||||
{} as never,
|
||||
);
|
||||
return { controller, aiChatRepo, aiChatMessageRepo };
|
||||
}
|
||||
|
||||
it('renders the full transcript and returns { markdown }', async () => {
|
||||
const { controller, aiChatMessageRepo } = makeController();
|
||||
const res = await controller.export({ chatId: 'c1' }, user, workspace);
|
||||
expect(aiChatMessageRepo.findAllByChat).toHaveBeenCalledWith('c1', 'ws1');
|
||||
expect(res.markdown).toContain('# My chat');
|
||||
expect(res.markdown).toContain('## 1. You');
|
||||
expect(res.markdown).toContain('## 2. AI agent');
|
||||
});
|
||||
|
||||
it('forbids a chat the user does not own', async () => {
|
||||
const { controller } = makeController({
|
||||
chat: { id: 'c1', creatorId: 'someone-else', title: 'X' },
|
||||
});
|
||||
await expect(
|
||||
controller.export({ chatId: 'c1' }, user, workspace),
|
||||
).rejects.toBeInstanceOf(ForbiddenException);
|
||||
});
|
||||
|
||||
it('forbids a missing / foreign-workspace chat', async () => {
|
||||
const { controller } = makeController({ chat: null });
|
||||
await expect(
|
||||
controller.export({ chatId: 'c1' }, user, workspace),
|
||||
).rejects.toBeInstanceOf(ForbiddenException);
|
||||
});
|
||||
|
||||
it('localizes labels when lang=ru is passed', async () => {
|
||||
const { controller } = makeController();
|
||||
const res = await controller.export(
|
||||
{ chatId: 'c1', lang: 'ru' },
|
||||
user,
|
||||
workspace,
|
||||
);
|
||||
expect(res.markdown).toContain('## 1. Вы');
|
||||
expect(res.markdown).toContain('## 2. ИИ-агент');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* The terminal-finalize dispatch (#183): the assistant row is INSERTed upfront
|
||||
* as 'streaming' and finalized once on the terminal callback. When the upfront
|
||||
* insert SUCCEEDED (we hold an id) finalize UPDATEs that row; when it FAILED
|
||||
* (assistantId is undefined) finalize falls back to INSERTing the terminal row
|
||||
* so the turn is not lost — the only safety against losing the turn entirely.
|
||||
*
|
||||
* `planFinalizeAssistant` is the pure decision; `applyFinalize` is the REAL
|
||||
* dispatch the service uses, exercised here over a mock repo (not a copy of the
|
||||
* logic) so a production drift would fail the test (#186 review).
|
||||
*/
|
||||
describe('finalizeAssistant dispatch (planFinalizeAssistant + applyFinalize)', () => {
|
||||
const workspaceId = 'ws1';
|
||||
|
||||
// Drive the SAME applyFinalize the service calls (no duplicated logic).
|
||||
async function dispatchFinalize(
|
||||
repo: { insert: jest.Mock; update: jest.Mock },
|
||||
assistantId: string | undefined,
|
||||
flushed: AssistantFlush,
|
||||
): Promise<void> {
|
||||
await applyFinalize(
|
||||
repo,
|
||||
planFinalizeAssistant(assistantId),
|
||||
{ chatId: 'c1', workspaceId, userId: 'u1' },
|
||||
flushed,
|
||||
);
|
||||
}
|
||||
|
||||
it('plan: update when the upfront insert returned an id', () => {
|
||||
expect(planFinalizeAssistant('a1')).toEqual({ kind: 'update', id: 'a1' });
|
||||
});
|
||||
|
||||
it('plan: insert (fallback) when there is no upfront id', () => {
|
||||
expect(planFinalizeAssistant(undefined)).toEqual({ kind: 'insert' });
|
||||
});
|
||||
|
||||
it('(a) upfront insert succeeded -> finalize UPDATEs the row by id', async () => {
|
||||
const repo = { insert: jest.fn(), update: jest.fn() };
|
||||
const flushed = flushAssistant([], 'final answer', 'completed', {
|
||||
finishReason: 'stop',
|
||||
});
|
||||
await dispatchFinalize(repo, 'a1', flushed);
|
||||
expect(repo.update).toHaveBeenCalledWith('a1', workspaceId, flushed);
|
||||
expect(repo.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('(b) upfront insert failed -> finalize INSERTs the terminal payload', async () => {
|
||||
const repo = { insert: jest.fn(), update: jest.fn() };
|
||||
const flushed = flushAssistant([], 'partial', 'error', { error: 'boom' });
|
||||
await dispatchFinalize(repo, undefined, flushed);
|
||||
expect(repo.update).not.toHaveBeenCalled();
|
||||
expect(repo.insert).toHaveBeenCalledTimes(1);
|
||||
const arg = repo.insert.mock.calls[0][0];
|
||||
// The fallback insert carries the terminal content/status/metadata.
|
||||
expect(arg.role).toBe('assistant');
|
||||
expect(arg.content).toBe('partial');
|
||||
expect(arg.status).toBe('error');
|
||||
expect((arg.metadata as { error?: string }).error).toBe('boom');
|
||||
});
|
||||
});
|
||||
@@ -20,7 +20,7 @@ import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
|
||||
import { AuthUser } from '../../common/decorators/auth-user.decorator';
|
||||
import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
|
||||
import { SkipTransform } from '../../common/decorators/skip-transform.decorator';
|
||||
import { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
import { AiChat, User, Workspace } from '@docmost/db/types/entity.types';
|
||||
import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
|
||||
import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
@@ -31,10 +31,12 @@ import { AiChatService, AiChatStreamBody } from './ai-chat.service';
|
||||
import { AiTranscriptionService } from './ai-transcription.service';
|
||||
import {
|
||||
ChatIdDto,
|
||||
ExportChatDto,
|
||||
GetChatMessagesDto,
|
||||
RenameChatDto,
|
||||
} from './dto/ai-chat.dto';
|
||||
import { describeProviderError } from '../../integrations/ai/ai-error.util';
|
||||
import { buildChatMarkdown } from './chat-markdown.util';
|
||||
|
||||
/**
|
||||
* Per-user AI chat API (§6.1). Routes are POST to match this codebase's
|
||||
@@ -81,6 +83,36 @@ export class AiChatController {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a chat to Markdown (#183). The DB is the single source of truth: the
|
||||
* whole transcript is loaded (oldest -> newest) and rendered server-side. Now
|
||||
* that the assistant row is persisted upfront and per step, an interrupted
|
||||
* turn is included up to its last finished step. Workspace-scoped and owner-
|
||||
* gated via assertOwnedChat (same as the other read endpoints). Returns
|
||||
* `{ markdown }`. `lang` localizes the few fixed labels (default English).
|
||||
*/
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('export')
|
||||
async export(
|
||||
@Body() dto: ExportChatDto,
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<{ markdown: string }> {
|
||||
const chat = await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
const rows = await this.aiChatMessageRepo.findAllByChat(
|
||||
dto.chatId,
|
||||
workspace.id,
|
||||
);
|
||||
const markdown = buildChatMarkdown({
|
||||
title: chat.title ?? null,
|
||||
chatId: dto.chatId,
|
||||
rows,
|
||||
// normalizeLang(undefined) already yields 'en', so no `?? 'en'` is needed.
|
||||
lang: dto.lang,
|
||||
});
|
||||
return { markdown };
|
||||
}
|
||||
|
||||
/** Rename a chat. */
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('rename')
|
||||
@@ -90,7 +122,11 @@ export class AiChatController {
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
) {
|
||||
await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
await this.aiChatRepo.update(dto.chatId, { title: dto.title }, workspace.id);
|
||||
await this.aiChatRepo.update(
|
||||
dto.chatId,
|
||||
{ title: dto.title },
|
||||
workspace.id,
|
||||
);
|
||||
return { success: true };
|
||||
}
|
||||
|
||||
@@ -145,7 +181,10 @@ export class AiChatController {
|
||||
// Resolve the agent role for this turn BEFORE hijack: existing chats read it
|
||||
// from ai_chats.role_id (authoritative), a new chat from body.roleId. The
|
||||
// role drives both the persona and the optional model override below.
|
||||
const role = await this.aiChatService.resolveRoleForRequest(workspace, body);
|
||||
const role = await this.aiChatService.resolveRoleForRequest(
|
||||
workspace,
|
||||
body,
|
||||
);
|
||||
|
||||
// Resolve the model (applying the role's optional override) BEFORE hijack so
|
||||
// an unconfigured provider — including a role pointing at an unconfigured
|
||||
@@ -232,7 +271,9 @@ export class AiChatController {
|
||||
let file = null;
|
||||
try {
|
||||
// Whisper hard-caps uploads at 25MB; allow a single file.
|
||||
file = await req.file({ limits: { fileSize: 25 * 1024 * 1024, files: 1 } });
|
||||
file = await req.file({
|
||||
limits: { fileSize: 25 * 1024 * 1024, files: 1 },
|
||||
});
|
||||
} catch (err: any) {
|
||||
if (err?.statusCode === 413) {
|
||||
throw new BadRequestException('Audio file too large (max 25MB)');
|
||||
@@ -283,11 +324,12 @@ export class AiChatController {
|
||||
chatId: string,
|
||||
user: User,
|
||||
workspace: Workspace,
|
||||
): Promise<void> {
|
||||
): Promise<AiChat> {
|
||||
const chat = await this.aiChatRepo.findById(chatId, workspace.id);
|
||||
if (!chat || chat.creatorId !== user.id) {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
return chat;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -174,47 +174,10 @@ describe('buildSystemPrompt mcp tooling guidance', () => {
|
||||
const workspace = { name: 'Acme' } as unknown as Workspace;
|
||||
const SAFETY_MARKER = 'Operating rules (always in effect)';
|
||||
|
||||
it('renders the server name, tool prefix and text when guidance is present', () => {
|
||||
const prompt = buildSystemPrompt({
|
||||
workspace,
|
||||
mcpInstructions: [
|
||||
{
|
||||
serverName: 'Tavily',
|
||||
toolPrefix: 'tavily',
|
||||
instructions: 'Use tavily_search for fresh web facts; cite sources.',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(prompt).toContain('<mcp_tooling');
|
||||
expect(prompt).toContain('Tavily');
|
||||
// The header names the namespace prefix as `<prefix>_*`.
|
||||
expect(prompt).toContain('tavily_*');
|
||||
expect(prompt).toContain(
|
||||
'Use tavily_search for fresh web facts; cite sources.',
|
||||
);
|
||||
});
|
||||
|
||||
it('renders nothing for an empty list', () => {
|
||||
const prompt = buildSystemPrompt({ workspace, mcpInstructions: [] });
|
||||
expect(prompt).not.toContain('<mcp_tooling');
|
||||
});
|
||||
|
||||
it('renders nothing for an undefined list', () => {
|
||||
const prompt = buildSystemPrompt({ workspace });
|
||||
expect(prompt).not.toContain('<mcp_tooling');
|
||||
});
|
||||
|
||||
it('renders nothing when every entry has blank text', () => {
|
||||
const prompt = buildSystemPrompt({
|
||||
workspace,
|
||||
mcpInstructions: [
|
||||
{ serverName: 'A', toolPrefix: 'a', instructions: ' ' },
|
||||
{ serverName: 'B', toolPrefix: 'b', instructions: '' },
|
||||
],
|
||||
});
|
||||
expect(prompt).not.toContain('<mcp_tooling');
|
||||
});
|
||||
|
||||
// The block's CONTENT and its empty/undefined/all-blank handling are covered by
|
||||
// the buildMcpToolingBlock unit tests below; here we only pin the INTEGRATION
|
||||
// invariants that are unique to buildSystemPrompt: sandwich placement and that
|
||||
// both safety copies survive.
|
||||
it('places the block inside the safety sandwich, after context, before the trailing SAFETY', () => {
|
||||
const prompt = buildSystemPrompt({
|
||||
workspace,
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { AiChatService } from './ai-chat.service';
|
||||
|
||||
/**
|
||||
* Lifecycle unit tests for AiChatService.onModuleInit (#183 crash-recovery
|
||||
* sweep). The sweep is BEST-EFFORT: a failure must be logged (warn) but must
|
||||
* NEVER throw out of onModuleInit and block server startup. Exercised with a
|
||||
* hand-rolled mock repo — no Nest graph, no DB. Only `aiChatMessageRepo` is
|
||||
* touched by onModuleInit, so the other constructor deps are stubbed as never.
|
||||
*/
|
||||
describe('AiChatService.onModuleInit (startup sweep)', () => {
|
||||
function makeService(sweepStreaming: jest.Mock) {
|
||||
const aiChatMessageRepo = { sweepStreaming };
|
||||
const service = new AiChatService(
|
||||
{} as never, // ai
|
||||
{} as never, // aiChatRepo
|
||||
aiChatMessageRepo as never,
|
||||
{} as never, // aiSettings
|
||||
{} as never, // tools
|
||||
{} as never, // mcpClients
|
||||
{} as never, // aiAgentRoleRepo
|
||||
{} as never, // pageRepo
|
||||
{} as never, // pageAccess
|
||||
);
|
||||
return { service, aiChatMessageRepo };
|
||||
}
|
||||
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('happy path: calls sweepStreaming and resolves', async () => {
|
||||
const sweepStreaming = jest.fn().mockResolvedValue(0);
|
||||
const { service } = makeService(sweepStreaming);
|
||||
await expect(service.onModuleInit()).resolves.toBeUndefined();
|
||||
expect(sweepStreaming).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('logs how many rows were swept when > 0', async () => {
|
||||
const sweepStreaming = jest.fn().mockResolvedValue(3);
|
||||
const logSpy = jest
|
||||
.spyOn(Logger.prototype, 'log')
|
||||
.mockImplementation(() => undefined);
|
||||
const { service } = makeService(sweepStreaming);
|
||||
await service.onModuleInit();
|
||||
expect(logSpy).toHaveBeenCalledTimes(1);
|
||||
expect(String(logSpy.mock.calls[0][0])).toContain('3');
|
||||
});
|
||||
|
||||
it('sweepStreaming throws -> onModuleInit resolves (does NOT throw) and warns', async () => {
|
||||
const sweepStreaming = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('db unavailable'));
|
||||
const warnSpy = jest
|
||||
.spyOn(Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
const { service } = makeService(sweepStreaming);
|
||||
// Must not throw — a sweep failure may never block startup.
|
||||
await expect(service.onModuleInit()).resolves.toBeUndefined();
|
||||
expect(warnSpy).toHaveBeenCalledTimes(1);
|
||||
expect(String(warnSpy.mock.calls[0][0])).toContain('db unavailable');
|
||||
});
|
||||
});
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
serializeSteps,
|
||||
rowToUiMessage,
|
||||
prepareAgentStep,
|
||||
buildPartialAssistantRecord,
|
||||
flushAssistant,
|
||||
chatStreamMetadata,
|
||||
accumulateStepUsage,
|
||||
MAX_AGENT_STEPS,
|
||||
@@ -233,101 +233,126 @@ describe('prepareAgentStep', () => {
|
||||
// The synthesis instruction is appended.
|
||||
expect(result?.system).toContain(FINAL_STEP_INSTRUCTION);
|
||||
});
|
||||
|
||||
it('pins the off-by-one boundary (MAX-2 is not final, MAX-1 is)', () => {
|
||||
// Boundary expressed via the constant, not a hardcoded 18/19, so the test
|
||||
// tracks MAX_AGENT_STEPS if the cap ever changes.
|
||||
expect(prepareAgentStep(MAX_AGENT_STEPS - 2, 'SYS')).toBeUndefined();
|
||||
const atBoundary = prepareAgentStep(MAX_AGENT_STEPS - 1, 'SYS');
|
||||
expect(atBoundary).toBeDefined();
|
||||
expect(atBoundary?.toolChoice).toBe('none');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Unit test for buildPartialAssistantRecord: the pure helper that shapes the
|
||||
* assistant-message record persisted on a partial/failed turn (the streamText
|
||||
* onError / onAbort paths). It captures the PARTIAL answer the user already saw
|
||||
* (finished steps' text + tool parts, plus the in-progress step's text) so a
|
||||
* provider error / disconnect no longer throws the streamed answer away. Pinning
|
||||
* the record shape here covers the persist-partial logic without seaming
|
||||
* streamText itself.
|
||||
* flushAssistant (#183): the PURE row builder behind the step-granular durable
|
||||
* write path. It runs identically for the upfront insert (empty steps,
|
||||
* 'streaming'), every per-step update, and the terminal finalize — so a future
|
||||
* background worker can call the same function. These tests pin the four status
|
||||
* shapes and the `metadata.parts` shape that rowToUiMessage/findRecent depend on
|
||||
* (per-step text + tool parts via assistantParts, in-progress text appended).
|
||||
*/
|
||||
describe('buildPartialAssistantRecord', () => {
|
||||
describe('flushAssistant', () => {
|
||||
type AnyPart = Record<string, unknown>;
|
||||
|
||||
it('records an empty turn with the error text (preserves old behavior)', () => {
|
||||
const rec = buildPartialAssistantRecord(
|
||||
[],
|
||||
'',
|
||||
'error',
|
||||
'401: Unauthorized',
|
||||
);
|
||||
expect(rec).toEqual({
|
||||
text: '',
|
||||
toolCalls: null,
|
||||
metadata: {
|
||||
finishReason: 'error',
|
||||
parts: [],
|
||||
error: '401: Unauthorized',
|
||||
},
|
||||
});
|
||||
const toolStep = {
|
||||
text: 'looked it up',
|
||||
toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: { id: 'p1' } }],
|
||||
toolResults: [
|
||||
{ toolCallId: 'c1', toolName: 'getPage', output: { title: 'T' } },
|
||||
],
|
||||
};
|
||||
|
||||
it('upfront seed: empty streaming row (no content, no toolCalls, empty parts)', () => {
|
||||
const f = flushAssistant([], '', 'streaming');
|
||||
expect(f.status).toBe('streaming');
|
||||
expect(f.content).toBe('');
|
||||
expect(f.toolCalls).toBeNull();
|
||||
expect(f.metadata.parts).toEqual([]);
|
||||
// No finishReason while streaming (it is not a terminal state).
|
||||
expect('finishReason' in f.metadata).toBe(false);
|
||||
});
|
||||
|
||||
it('persists in-progress text (no finished steps) as the partial answer', () => {
|
||||
const rec = buildPartialAssistantRecord(
|
||||
[],
|
||||
'partial answer',
|
||||
'error',
|
||||
'boom',
|
||||
);
|
||||
expect(rec.text).toBe('partial answer');
|
||||
expect(rec.metadata.parts).toEqual([
|
||||
it('streaming update folds in finished steps but keeps status streaming', () => {
|
||||
const f = flushAssistant([toolStep], '', 'streaming');
|
||||
expect(f.status).toBe('streaming');
|
||||
expect(f.content).toBe('looked it up');
|
||||
const parts = f.metadata.parts as AnyPart[];
|
||||
expect(parts).toContainEqual({ type: 'text', text: 'looked it up' });
|
||||
const toolPart = parts.find((p) => p.type === 'tool-getPage');
|
||||
expect(toolPart!.state).toBe('output-available');
|
||||
expect(f.toolCalls).not.toBeNull();
|
||||
});
|
||||
|
||||
it('completed: attaches finishReason + normalized usage + contextTokens + maxContextTokens', () => {
|
||||
const f = flushAssistant([toolStep], '', 'completed', {
|
||||
finishReason: 'stop',
|
||||
usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 },
|
||||
contextTokens: 15,
|
||||
maxContextTokens: 200000,
|
||||
});
|
||||
expect(f.status).toBe('completed');
|
||||
expect(f.metadata.finishReason).toBe('stop');
|
||||
expect(f.metadata.usage).toEqual({
|
||||
inputTokens: 10,
|
||||
outputTokens: 5,
|
||||
totalTokens: 15,
|
||||
reasoningTokens: undefined,
|
||||
});
|
||||
expect(f.metadata.contextTokens).toBe(15);
|
||||
expect(f.metadata.maxContextTokens).toBe(200000);
|
||||
});
|
||||
|
||||
it('completed: omits maxContextTokens when unset or 0', () => {
|
||||
// No maxContextTokens in the extra (admin set no context window).
|
||||
const f = flushAssistant([toolStep], '', 'completed', {
|
||||
finishReason: 'stop',
|
||||
contextTokens: 15,
|
||||
});
|
||||
expect('maxContextTokens' in f.metadata).toBe(false);
|
||||
// Explicit 0 is treated the same as unset (no limit -> key omitted).
|
||||
const f0 = flushAssistant([toolStep], '', 'completed', {
|
||||
finishReason: 'stop',
|
||||
contextTokens: 15,
|
||||
maxContextTokens: 0,
|
||||
});
|
||||
expect('maxContextTokens' in f0.metadata).toBe(false);
|
||||
});
|
||||
|
||||
it('error: records the error and a derived finishReason', () => {
|
||||
const f = flushAssistant([], 'partial answer', 'error', { error: 'boom' });
|
||||
expect(f.status).toBe('error');
|
||||
expect(f.content).toBe('partial answer');
|
||||
expect(f.metadata.error).toBe('boom');
|
||||
// Derives finishReason from the terminal status when none is supplied.
|
||||
expect(f.metadata.finishReason).toBe('error');
|
||||
expect(f.metadata.parts).toEqual([
|
||||
{ type: 'text', text: 'partial answer' },
|
||||
]);
|
||||
expect(rec.metadata.error).toBe('boom');
|
||||
});
|
||||
|
||||
it('combines a finished tool step with trailing in-progress text', () => {
|
||||
const steps = [
|
||||
{
|
||||
text: 'looked it up',
|
||||
toolCalls: [
|
||||
{ toolCallId: 'c1', toolName: 'getPage', input: { id: 'p1' } },
|
||||
],
|
||||
toolResults: [
|
||||
{ toolCallId: 'c1', toolName: 'getPage', output: { title: 'T' } },
|
||||
],
|
||||
},
|
||||
];
|
||||
const rec = buildPartialAssistantRecord(
|
||||
steps,
|
||||
' and then',
|
||||
'error',
|
||||
'boom',
|
||||
);
|
||||
const parts = rec.metadata.parts as AnyPart[];
|
||||
// The finished step's text part is present.
|
||||
expect(parts).toContainEqual({ type: 'text', text: 'looked it up' });
|
||||
// The paired tool call+result becomes an output-available part.
|
||||
const toolPart = parts.find((p) => p.type === 'tool-getPage');
|
||||
expect(toolPart).toBeDefined();
|
||||
expect(toolPart!.state).toBe('output-available');
|
||||
// The in-progress text is appended LAST so the parts match the stream order.
|
||||
it('aborted: in-progress text appended last, no error key', () => {
|
||||
const f = flushAssistant([toolStep], ' and then', 'aborted');
|
||||
expect(f.status).toBe('aborted');
|
||||
expect(f.metadata.finishReason).toBe('aborted');
|
||||
expect('error' in f.metadata).toBe(false);
|
||||
expect(f.content).toBe('looked it up and then');
|
||||
const parts = f.metadata.parts as AnyPart[];
|
||||
expect(parts[parts.length - 1]).toEqual({
|
||||
type: 'text',
|
||||
text: ' and then',
|
||||
});
|
||||
expect(rec.text).toBe('looked it up and then');
|
||||
expect(rec.toolCalls).not.toBeNull();
|
||||
expect(rec.metadata.error).toBe('boom');
|
||||
});
|
||||
|
||||
it('omits the error key on the abort path (no errorText)', () => {
|
||||
const rec = buildPartialAssistantRecord([], 'half', 'aborted');
|
||||
expect(rec.metadata.finishReason).toBe('aborted');
|
||||
expect('error' in rec.metadata).toBe(false);
|
||||
expect(rec.text).toBe('half');
|
||||
it('combines a finished tool step with trailing in-progress text (error path)', () => {
|
||||
// The error path captures the PARTIAL answer the user already saw: each
|
||||
// finished step's text + tool parts, then the in-progress step's text last.
|
||||
const flushed = flushAssistant([toolStep], ' and then', 'error', {
|
||||
error: 'boom',
|
||||
});
|
||||
const parts = flushed.metadata.parts as AnyPart[];
|
||||
expect(parts).toContainEqual({ type: 'text', text: 'looked it up' });
|
||||
const toolPart = parts.find((p) => p.type === 'tool-getPage');
|
||||
expect(toolPart!.state).toBe('output-available');
|
||||
// In-progress text appended LAST so the parts match the stream order.
|
||||
expect(parts[parts.length - 1]).toEqual({
|
||||
type: 'text',
|
||||
text: ' and then',
|
||||
});
|
||||
expect(flushed.content).toBe('looked it up and then');
|
||||
expect(flushed.toolCalls).not.toBeNull();
|
||||
expect(flushed.metadata.error).toBe('boom');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import { ForbiddenException, Injectable, Logger } from '@nestjs/common';
|
||||
import {
|
||||
ForbiddenException,
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleInit,
|
||||
} from '@nestjs/common';
|
||||
import { FastifyReply } from 'fastify';
|
||||
import {
|
||||
streamText,
|
||||
@@ -124,7 +129,7 @@ export interface AiChatStreamArgs {
|
||||
* can be rebuilt for `convertToModelMessages`.
|
||||
*/
|
||||
@Injectable()
|
||||
export class AiChatService {
|
||||
export class AiChatService implements OnModuleInit {
|
||||
private readonly logger = new Logger(AiChatService.name);
|
||||
|
||||
constructor(
|
||||
@@ -139,6 +144,32 @@ export class AiChatService {
|
||||
private readonly pageAccess: PageAccessService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Crash-recovery sweep on server start (#183): any assistant row left in the
|
||||
* 'streaming' state is the relic of a turn whose process died before it
|
||||
* reached a terminal status. Flip those to 'aborted' so history/export show
|
||||
* them settled (with whatever finished steps were already persisted) instead
|
||||
* of perpetually "streaming". Best-effort: a sweep failure is logged but must
|
||||
* never block server startup.
|
||||
*/
|
||||
async onModuleInit(): Promise<void> {
|
||||
try {
|
||||
const swept = await this.aiChatMessageRepo.sweepStreaming();
|
||||
if (swept > 0) {
|
||||
this.logger.log(
|
||||
`Startup sweep: marked ${swept} dangling 'streaming' assistant ` +
|
||||
`message(s) as 'aborted'.`,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Startup sweep of dangling 'streaming' messages failed: ${
|
||||
err instanceof Error ? err.message : 'unknown error'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the agent role that applies to this stream request, scoped to the
|
||||
* workspace and soft-delete aware. For an EXISTING chat the role is read from
|
||||
@@ -332,38 +363,14 @@ export class AiChatService {
|
||||
);
|
||||
}
|
||||
|
||||
const system = buildSystemPrompt({
|
||||
workspace,
|
||||
adminPrompt: resolved?.systemPrompt,
|
||||
// The role (pre-resolved by the controller) REPLACES the persona layer;
|
||||
// the safety framework is still appended by buildSystemPrompt.
|
||||
roleInstructions: role?.instructions,
|
||||
// Server-validated open page (authoritative title), not the client value.
|
||||
openedPage: openPageContext,
|
||||
// Guidance only for servers that connected and yielded ≥1 callable tool.
|
||||
mcpInstructions: external.instructions,
|
||||
});
|
||||
|
||||
// Pass the resolved chatId so the write tools can mint provenance tokens
|
||||
// (access + collab) carrying { actor:'agent', aiChatId: chatId }, making
|
||||
// agent REST/collab writes attributable and non-spoofable (§6.5/§6.6).
|
||||
const docmostTools = await this.tools.forUser(
|
||||
user,
|
||||
sessionId,
|
||||
workspace.id,
|
||||
chatId,
|
||||
// Same server-validated open page used by the system prompt above; exposed
|
||||
// to the model via getCurrentPage so page identity (and the AUTHORITATIVE
|
||||
// title) survives prompt mangling and client title spoofing (#159).
|
||||
openPageContext,
|
||||
);
|
||||
|
||||
const tools = { ...external.tools, ...docmostTools };
|
||||
|
||||
// Close every external client EXACTLY ONCE across the turn's terminal
|
||||
// callbacks (onFinish/onError/onAbort all fire at most once collectively,
|
||||
// but guard anyway). Close errors are swallowed so they never break the
|
||||
// response.
|
||||
// but guard anyway). DEFINED HERE — before the prompt/toolset are built — so
|
||||
// that if buildSystemPrompt or forUser throws AFTER the external lease was
|
||||
// taken (toolsFor above), the lease is still released. Otherwise its refCount
|
||||
// stays >= 1 forever and the external undici sockets leak until restart
|
||||
// (#180 reorder moved toolsFor ahead of these; #185 review). Close errors are
|
||||
// swallowed so they never break the response.
|
||||
let clientsClosed = false;
|
||||
const closeExternalClients = async (): Promise<void> => {
|
||||
if (clientsClosed) return;
|
||||
@@ -381,30 +388,43 @@ export class AiChatService {
|
||||
);
|
||||
};
|
||||
|
||||
// Persist the assistant message. Used by onFinish (full result) and the
|
||||
// abort/error paths (partial result). Guarded so we persist at most once.
|
||||
let persisted = false;
|
||||
const persistAssistant = async (data: {
|
||||
text: string;
|
||||
toolCalls: unknown;
|
||||
metadata: Record<string, unknown>;
|
||||
}): Promise<void> => {
|
||||
if (persisted) return;
|
||||
persisted = true;
|
||||
try {
|
||||
await this.aiChatMessageRepo.insert({
|
||||
chatId,
|
||||
workspaceId: workspace.id,
|
||||
userId: user.id,
|
||||
role: 'assistant',
|
||||
content: data.text ?? '',
|
||||
toolCalls: (data.toolCalls ?? null) as never,
|
||||
metadata: data.metadata as never,
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.error('Failed to persist assistant message', err as Error);
|
||||
}
|
||||
};
|
||||
// Build the system prompt + Docmost toolset. If either throws after the
|
||||
// external MCP lease was taken above, release the lease before rethrowing so
|
||||
// the leased transports are not leaked (#185 review).
|
||||
let system: string;
|
||||
let docmostTools: Awaited<ReturnType<AiChatToolsService['forUser']>>;
|
||||
try {
|
||||
system = buildSystemPrompt({
|
||||
workspace,
|
||||
adminPrompt: resolved?.systemPrompt,
|
||||
// The role (pre-resolved by the controller) REPLACES the persona layer;
|
||||
// the safety framework is still appended by buildSystemPrompt.
|
||||
roleInstructions: role?.instructions,
|
||||
// Server-validated open page (authoritative title), not the client value.
|
||||
openedPage: openPageContext,
|
||||
// Guidance only for servers that connected and yielded ≥1 callable tool.
|
||||
mcpInstructions: external.instructions,
|
||||
});
|
||||
|
||||
// Pass the resolved chatId so the write tools can mint provenance tokens
|
||||
// (access + collab) carrying { actor:'agent', aiChatId: chatId }, making
|
||||
// agent REST/collab writes attributable and non-spoofable (§6.5/§6.6).
|
||||
docmostTools = await this.tools.forUser(
|
||||
user,
|
||||
sessionId,
|
||||
workspace.id,
|
||||
chatId,
|
||||
// Same server-validated open page used by the system prompt above;
|
||||
// exposed to the model via getCurrentPage so page identity (and the
|
||||
// AUTHORITATIVE title) survives prompt mangling / client title spoofing.
|
||||
openPageContext,
|
||||
);
|
||||
} catch (err) {
|
||||
await closeExternalClients();
|
||||
throw err;
|
||||
}
|
||||
|
||||
const tools = { ...external.tools, ...docmostTools };
|
||||
|
||||
// Accumulate the turn's streamed output so a provider error / disconnect can
|
||||
// persist the PARTIAL answer the user already saw — the SDK's onError/onAbort
|
||||
@@ -414,6 +434,101 @@ export class AiChatService {
|
||||
const capturedSteps: StepLike[] = [];
|
||||
let inProgressText = '';
|
||||
|
||||
// Step-granular durability (#183): create the assistant row UPFRONT in the
|
||||
// 'streaming' state (before any token), then UPDATE it as each step finishes
|
||||
// and finalize it once on the terminal callback. If the process dies
|
||||
// mid-turn the row survives with every finished step already persisted; the
|
||||
// startup sweep (sweepStreaming) later flips a dangling 'streaming' row to
|
||||
// 'aborted'. The DB is now the single source of truth for the turn — the
|
||||
// socket is never required for the write path. A failed upfront insert is
|
||||
// logged and leaves assistantId undefined; the per-step/terminal updates then
|
||||
// no-op (guarded below) so the turn still streams to the user.
|
||||
let assistantId: string | undefined;
|
||||
try {
|
||||
const seed = flushAssistant([], '', 'streaming');
|
||||
const seeded = await this.aiChatMessageRepo.insert({
|
||||
chatId,
|
||||
workspaceId: workspace.id,
|
||||
userId: user.id,
|
||||
role: 'assistant',
|
||||
content: seed.content,
|
||||
// jsonb columns: cast through never (same as the user insert above).
|
||||
toolCalls: (seed.toolCalls ?? null) as never,
|
||||
metadata: seed.metadata as never,
|
||||
status: seed.status,
|
||||
});
|
||||
assistantId = seeded?.id;
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to insert upfront assistant row (chat ${chatId}, workspace ${workspace.id})`,
|
||||
err as Error,
|
||||
);
|
||||
}
|
||||
|
||||
// Per-step (non-terminal) update: persist the finished steps the moment a
|
||||
// step ends. Tolerant — a failed update is logged and swallowed so it never
|
||||
// throws into the stream. Keeps status 'streaming'.
|
||||
const updateStreaming = async (): Promise<void> => {
|
||||
if (!assistantId) return;
|
||||
// Cheap short-circuit once the turn is finalized (see `finalized` below).
|
||||
// The AUTHORITATIVE guard is `onlyIfStreaming` on the UPDATE: a late
|
||||
// fire-and-forget step update could still be in flight on another pool
|
||||
// connection when finalize runs, so the SQL `WHERE status='streaming'`
|
||||
// (not this flag) is what prevents it clobbering the terminal row.
|
||||
if (finalized) return;
|
||||
try {
|
||||
await this.aiChatMessageRepo.update(
|
||||
assistantId,
|
||||
workspace.id,
|
||||
flushAssistant(capturedSteps, '', 'streaming'),
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Failed to update streaming assistant row: ${
|
||||
err instanceof Error ? err.message : 'unknown error'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Serialize the per-step updates (#183 review): onStepFinish fires them
|
||||
// without await, so two could otherwise commit out of order on different pool
|
||||
// connections (step N landing after N+1). Chaining each onto the previous
|
||||
// keeps the persisted row monotonic with step order; each link short-circuits
|
||||
// on `finalized`, so a tail of late updates is cheap.
|
||||
let stepUpdateChain: Promise<void> = Promise.resolve();
|
||||
|
||||
// Terminal finalize: write the completed/error/aborted row exactly once
|
||||
// across the (mutually-exclusive, at-most-once) onFinish/onError/onAbort
|
||||
// callbacks — mirroring the pre-#183 persist-at-most-once guard for the
|
||||
// TERMINAL status (the row may be updated many times with 'streaming' before
|
||||
// this fires once).
|
||||
let finalized = false;
|
||||
const finalizeAssistant = async (
|
||||
flushed: AssistantFlush,
|
||||
): Promise<void> => {
|
||||
if (finalized) return;
|
||||
finalized = true;
|
||||
const plan = planFinalizeAssistant(assistantId);
|
||||
try {
|
||||
// Shared dispatch (see applyFinalize): UPDATE the upfront row, or — when
|
||||
// the upfront insert failed (kind 'insert') — INSERT the terminal row as
|
||||
// the only safety against losing the turn entirely.
|
||||
await applyFinalize(
|
||||
this.aiChatMessageRepo,
|
||||
plan,
|
||||
{ chatId, workspaceId: workspace.id, userId: user.id },
|
||||
flushed,
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to finalize assistant message (kind=${plan.kind})`,
|
||||
err as Error,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// DIAGNOSTIC (Safari stream-drop investigation) — temporary. Measure
|
||||
// first-chunk latency, the model-silent gap right before a disconnect, and
|
||||
// how many SSE heartbeats were written, so a Safari drop can be classified
|
||||
@@ -462,6 +577,12 @@ export class AiChatService {
|
||||
// the in-progress accumulator for the next step.
|
||||
capturedSteps.push(step as StepLike);
|
||||
inProgressText = '';
|
||||
// Step-granular durability (#183): persist this finished step (its text +
|
||||
// tool calls + tool RESULTS) the moment it ends, so a process death after
|
||||
// this point still recovers the step. Not awaited here (never block the
|
||||
// stream), but SERIALIZED via stepUpdateChain so the writes commit in
|
||||
// step order; updateStreaming is error-tolerant (logs + swallows).
|
||||
stepUpdateChain = stepUpdateChain.then(() => updateStreaming());
|
||||
},
|
||||
onFinish: async ({ text, finishReason, totalUsage, usage, steps }) => {
|
||||
// DIAGNOSTIC (Safari stream-drop investigation) — temporary: success
|
||||
@@ -472,30 +593,35 @@ export class AiChatService {
|
||||
`firstChunkLatency=${firstModelChunkAt ? firstModelChunkAt - streamStartedAt : 'none'}ms ` +
|
||||
`heartbeatsSent=${heartbeatsSent} steps=${steps.length}`,
|
||||
);
|
||||
await persistAssistant({
|
||||
text,
|
||||
toolCalls: serializeSteps(steps),
|
||||
metadata: {
|
||||
finishReason,
|
||||
// Persist the turn's cumulative usage WITH reasoning tokens resolved
|
||||
// from either the new `outputTokenDetails` or the deprecated top-level
|
||||
// field, so reopened history / the Markdown export show the thinking
|
||||
// token cost too.
|
||||
usage:
|
||||
normalizeStreamUsage(totalUsage as StreamUsage) ?? totalUsage,
|
||||
// Final-step usage = the context actually fed to the model on the last LLM
|
||||
// call (full history + tool results) plus the answer it just generated.
|
||||
// input+output of the FINAL step ≈ the conversation's CURRENT context size,
|
||||
// distinct from totalUsage which sums every step (cumulative tokens spent).
|
||||
// Finalize the assistant row (#183): the upfront 'streaming' row is
|
||||
// UPDATEd to 'completed' with the turn's final text, cumulative usage and
|
||||
// full UIMessage parts. We pass the SDK `steps` (which carry the final
|
||||
// step's text) as the captured steps so metadata.parts matches the
|
||||
// pre-#183 onFinish record exactly; `inProgressText` is '' here (the last
|
||||
// step already finished). Final-step usage (usage.input+output) ≈ the
|
||||
// conversation's CURRENT context size, distinct from totalUsage.
|
||||
//
|
||||
// COLUMN-SEMANTICS NOTE (#183): `content` is built by flushAssistant as
|
||||
// the CONCATENATION of every step's text (stepsText), whereas pre-#183
|
||||
// it stored only the FINAL step's text. This is a deliberate, harmless
|
||||
// change: the UI and the Markdown export render from `metadata.parts`
|
||||
// (per-step text + tool parts), not from `content`; `content` is the
|
||||
// plain-text projection (full-text search / fallback). A multi-step
|
||||
// turn's `content` therefore now holds all steps' prose, not just the
|
||||
// last block.
|
||||
await finalizeAssistant(
|
||||
flushAssistant(steps as StepLike[], '', 'completed', {
|
||||
finishReason: finishReason as string,
|
||||
usage: totalUsage as StreamUsage,
|
||||
contextTokens:
|
||||
(usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0) ||
|
||||
undefined,
|
||||
// Persist the FULL set of UIMessage parts for the turn (text +
|
||||
// tool-call/result), so the rebuilt history replays prior tool
|
||||
// context to the model on later turns.
|
||||
parts: assistantParts(steps, text),
|
||||
},
|
||||
});
|
||||
// Max context window for the chat header badge denominator;
|
||||
// resolved from the admin-configured provider settings (in
|
||||
// closure scope here). Omitted/0 = no limit.
|
||||
maxContextTokens: resolved?.chatContextWindow,
|
||||
}),
|
||||
);
|
||||
// Lifecycle: release the external MCP clients leased for this turn.
|
||||
await closeExternalClients();
|
||||
|
||||
@@ -531,16 +657,14 @@ export class AiChatService {
|
||||
`firstChunkLatency=${firstModelChunkAt ? firstModelChunkAt - streamStartedAt : 'none'}ms ` +
|
||||
`silentGapBeforeDrop=${diagNow - lastModelChunkAt}ms heartbeatsSent=${heartbeatsSent}`,
|
||||
);
|
||||
// Persist the PARTIAL answer streamed before the failure (text + any
|
||||
// Finalize the PARTIAL answer streamed before the failure (text + any
|
||||
// finished tool steps) WITH the error in metadata, so the turn shows what
|
||||
// the user already saw plus the cause — not just a bare error.
|
||||
await persistAssistant(
|
||||
buildPartialAssistantRecord(
|
||||
capturedSteps,
|
||||
inProgressText,
|
||||
'error',
|
||||
errorText,
|
||||
),
|
||||
// the user already saw plus the cause — not just a bare error. Status
|
||||
// 'error' (#183).
|
||||
await finalizeAssistant(
|
||||
flushAssistant(capturedSteps, inProgressText, 'error', {
|
||||
error: errorText,
|
||||
}),
|
||||
);
|
||||
await closeExternalClients();
|
||||
},
|
||||
@@ -564,12 +688,8 @@ export class AiChatService {
|
||||
`silentGapBeforeDrop=${diagNow - lastModelChunkAt}ms heartbeatsSent=${heartbeatsSent} ` +
|
||||
`steps=${steps.length}`,
|
||||
);
|
||||
await persistAssistant(
|
||||
buildPartialAssistantRecord(
|
||||
capturedSteps,
|
||||
inProgressText,
|
||||
'aborted',
|
||||
),
|
||||
await finalizeAssistant(
|
||||
flushAssistant(capturedSteps, inProgressText, 'aborted'),
|
||||
);
|
||||
await closeExternalClients();
|
||||
},
|
||||
@@ -1018,38 +1138,136 @@ export function rowToUiMessage(row: AiChatMessage): Omit<UIMessage, 'id'> & {
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the assistant-message record persisted on a partial/failed turn (the
|
||||
* streamText onError / onAbort paths). Captures the partial answer the user
|
||||
* already saw: each finished step's text + tool parts (via assistantParts),
|
||||
* then the in-progress step's text appended last. When `errorText` is provided
|
||||
* it is recorded in metadata.error so the cause shows in history; an aborted
|
||||
* turn passes none. Pure, so the partial-recording shape is unit-testable
|
||||
* without seaming streamText.
|
||||
* The persisted-row patch shape produced by {@link flushAssistant}. It is the
|
||||
* SAME shape the assistant repo insert/update consume (content + toolCalls +
|
||||
* metadata) plus the lifecycle `status` column added in #183.
|
||||
*/
|
||||
export function buildPartialAssistantRecord(
|
||||
steps: ReadonlyArray<StepLike> | undefined,
|
||||
export interface AssistantFlush {
|
||||
content: string;
|
||||
toolCalls: unknown;
|
||||
metadata: Record<string, unknown>;
|
||||
status: 'streaming' | 'completed' | 'error' | 'aborted';
|
||||
}
|
||||
|
||||
/**
|
||||
* Pure decision for the terminal finalize (#183): given whether the upfront
|
||||
* assistant row exists (`assistantId`), choose whether the terminal payload is
|
||||
* written by UPDATEing that row or — when the upfront insert failed and there is
|
||||
* no id — by INSERTing a fresh terminal row so the turn is not lost entirely.
|
||||
* Returns `{ kind: 'update', id }` or `{ kind: 'insert' }`. Extracted so the
|
||||
* fallback-insert branch (the only safety against losing a turn whose upfront
|
||||
* insert failed) is unit-testable without seaming streamText.
|
||||
*/
|
||||
export function planFinalizeAssistant(
|
||||
assistantId: string | undefined,
|
||||
): { kind: 'update'; id: string } | { kind: 'insert' } {
|
||||
return assistantId ? { kind: 'update', id: assistantId } : { kind: 'insert' };
|
||||
}
|
||||
|
||||
/** The repo surface the terminal finalize needs (structural — the real repo and
|
||||
* a test mock both satisfy it). */
|
||||
export interface FinalizeRepo {
|
||||
insert(insertable: Record<string, unknown>): Promise<unknown>;
|
||||
update(
|
||||
id: string,
|
||||
workspaceId: string,
|
||||
patch: AssistantFlush,
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply a finalize `plan` to the repo with the terminal `flushed` payload (#183):
|
||||
* UPDATE the upfront row, or INSERT a fresh terminal row as the fallback when the
|
||||
* upfront insert failed. The SINGLE dispatch shared by the service's
|
||||
* finalizeAssistant and its test, so the test exercises the real path instead of
|
||||
* a copy (#186 review). Pure of error handling — the caller wraps it.
|
||||
*/
|
||||
export async function applyFinalize(
|
||||
repo: FinalizeRepo,
|
||||
plan: { kind: 'update'; id: string } | { kind: 'insert' },
|
||||
base: { chatId: string; workspaceId: string; userId: string },
|
||||
flushed: AssistantFlush,
|
||||
): Promise<void> {
|
||||
if (plan.kind === 'update') {
|
||||
await repo.update(plan.id, base.workspaceId, flushed);
|
||||
return;
|
||||
}
|
||||
await repo.insert({
|
||||
chatId: base.chatId,
|
||||
workspaceId: base.workspaceId,
|
||||
userId: base.userId,
|
||||
role: 'assistant',
|
||||
content: flushed.content,
|
||||
toolCalls: flushed.toolCalls ?? null,
|
||||
metadata: flushed.metadata,
|
||||
status: flushed.status,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* PURE assistant-row builder (#183 step-granular durability). Given the turn's
|
||||
* accumulated steps + the in-progress (not-yet-finished) text + the lifecycle
|
||||
* status, it returns the row patch to persist. The SAME path runs for the
|
||||
* upfront insert (empty steps, status 'streaming'), every per-step update, and
|
||||
* the terminal finalize (completed/error/aborted) — and a future background
|
||||
* worker can call it identically, so it must stay a pure function of its inputs
|
||||
* (NO `this`, no IO).
|
||||
*
|
||||
* `metadata.parts` is built by assistantParts over the finished steps, then the
|
||||
* in-progress text appended as a trailing text part, so rowToUiMessage /
|
||||
* findRecent keep replaying the turn unchanged. `metadata.finishReason`,
|
||||
* `metadata.error`, `metadata.usage`, `metadata.contextTokens` and
|
||||
* `metadata.maxContextTokens` are attached only when provided/relevant, matching
|
||||
* the pre-#183 onFinish/onError records.
|
||||
*/
|
||||
export function flushAssistant(
|
||||
capturedSteps: ReadonlyArray<StepLike> | undefined,
|
||||
inProgressText: string,
|
||||
finishReason: 'error' | 'aborted',
|
||||
errorText?: string,
|
||||
): { text: string; toolCalls: unknown; metadata: Record<string, unknown> } {
|
||||
const finished = steps ?? [];
|
||||
status: 'streaming' | 'completed' | 'error' | 'aborted',
|
||||
extra?: {
|
||||
finishReason?: string;
|
||||
usage?: ChatStreamUsage | StreamUsage | undefined;
|
||||
contextTokens?: number;
|
||||
maxContextTokens?: number;
|
||||
error?: string;
|
||||
},
|
||||
): AssistantFlush {
|
||||
const finished = capturedSteps ?? [];
|
||||
const stepsText = finished.map((s) => s.text ?? '').join('');
|
||||
const trailing = inProgressText ?? '';
|
||||
// assistantParts emits text parts only for FINISHED steps; append the
|
||||
// in-progress step's text (the answer cut off by the error) as the last text
|
||||
// part so the persisted parts match what streamed to the client.
|
||||
// in-progress step's text (the partial answer cut off by an error/abort, or
|
||||
// simply not yet flushed mid-stream) as the last text part so the persisted
|
||||
// parts match what streamed to the client.
|
||||
const parts = assistantParts(finished, '') as unknown as Array<
|
||||
Record<string, unknown>
|
||||
>;
|
||||
if (trailing) parts.push({ type: 'text', text: trailing });
|
||||
|
||||
const metadata: Record<string, unknown> = {
|
||||
parts: parts as unknown as UIMessage['parts'],
|
||||
};
|
||||
// finishReason: prefer an explicit one; else derive a sensible value from the
|
||||
// terminal status (so onError/onAbort records keep their historical reason).
|
||||
if (extra?.finishReason) {
|
||||
metadata.finishReason = extra.finishReason;
|
||||
} else if (status === 'error' || status === 'aborted') {
|
||||
metadata.finishReason = status;
|
||||
}
|
||||
if (extra?.usage !== undefined) {
|
||||
metadata.usage =
|
||||
normalizeStreamUsage(extra.usage as StreamUsage) ?? extra.usage;
|
||||
}
|
||||
if (extra?.contextTokens) metadata.contextTokens = extra.contextTokens;
|
||||
if (extra?.maxContextTokens)
|
||||
metadata.maxContextTokens = extra.maxContextTokens;
|
||||
if (extra?.error) metadata.error = extra.error;
|
||||
|
||||
return {
|
||||
text: stepsText + trailing,
|
||||
content: stepsText + trailing,
|
||||
toolCalls: serializeSteps(finished),
|
||||
metadata: {
|
||||
finishReason,
|
||||
parts: parts as unknown as UIMessage['parts'],
|
||||
...(errorText ? { error: errorText } : {}),
|
||||
},
|
||||
metadata,
|
||||
status,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
295
apps/server/src/core/ai-chat/chat-markdown.util.spec.ts
Normal file
295
apps/server/src/core/ai-chat/chat-markdown.util.spec.ts
Normal file
@@ -0,0 +1,295 @@
|
||||
import { buildChatMarkdown, normalizeLang } from './chat-markdown.util';
|
||||
import type { AiChatMessage } from '@docmost/db/types/entity.types';
|
||||
|
||||
/**
|
||||
* normalizeLang: the client sends `i18n.language` — a FULL locale tag like
|
||||
* 'en-US' / 'ru-RU', NOT a bare 'en'/'ru'. A `@IsIn(['en','ru'])` DTO rejected
|
||||
* that with a 400 (caught in real-browser testing); the export now accepts any
|
||||
* string and normalizes here. Guards that regression.
|
||||
*/
|
||||
describe('normalizeLang', () => {
|
||||
it("maps any 'ru…' locale tag to ru", () => {
|
||||
expect(normalizeLang('ru')).toBe('ru');
|
||||
expect(normalizeLang('ru-RU')).toBe('ru');
|
||||
expect(normalizeLang('RU-ru')).toBe('ru');
|
||||
});
|
||||
|
||||
it('maps everything else (incl. region-qualified English) to en', () => {
|
||||
expect(normalizeLang('en')).toBe('en');
|
||||
expect(normalizeLang('en-US')).toBe('en');
|
||||
expect(normalizeLang('fr-FR')).toBe('en');
|
||||
expect(normalizeLang(undefined)).toBe('en');
|
||||
expect(normalizeLang('')).toBe('en');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Unit tests for the SERVER Markdown export (#183). Mirrors the coverage of the
|
||||
* (now-removed) client chat-markdown tests: heading/metadata, role labels, text
|
||||
* + tool blocks, token footers, the interrupted-turn note, and NULL-status
|
||||
* (legacy) rows. The export embeds a live `new Date().toISOString()` timestamp;
|
||||
* we never assert it, only the deterministic structure.
|
||||
*/
|
||||
|
||||
function row(partial: Partial<AiChatMessage>): AiChatMessage {
|
||||
return {
|
||||
id: partial.id ?? 'id',
|
||||
chatId: partial.chatId ?? 'chat-1',
|
||||
workspaceId: partial.workspaceId ?? 'ws-1',
|
||||
userId: partial.userId ?? null,
|
||||
role: partial.role ?? 'user',
|
||||
content: partial.content ?? null,
|
||||
toolCalls: partial.toolCalls ?? null,
|
||||
metadata: partial.metadata ?? null,
|
||||
status: partial.status ?? null,
|
||||
createdAt: partial.createdAt ?? ('2026-06-21T00:00:00.000Z' as never),
|
||||
updatedAt: partial.updatedAt ?? ('2026-06-21T00:00:00.000Z' as never),
|
||||
deletedAt: partial.deletedAt ?? null,
|
||||
} as AiChatMessage;
|
||||
}
|
||||
|
||||
describe('buildChatMarkdown (server) — structure', () => {
|
||||
it('emits the title heading, chat id and message count', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'My chat',
|
||||
chatId: 'chat-123',
|
||||
rows: [],
|
||||
});
|
||||
expect(md).toContain('# My chat');
|
||||
expect(md).toContain('- Chat ID: `chat-123`');
|
||||
expect(md).toContain('- Messages: 0');
|
||||
});
|
||||
|
||||
it('falls back to "Untitled chat" with no title (en)', () => {
|
||||
const md = buildChatMarkdown({ title: null, chatId: 'c', rows: [] });
|
||||
expect(md).toContain('# Untitled chat');
|
||||
});
|
||||
|
||||
it('localizes fixed labels with lang=ru (structure stays English)', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: null,
|
||||
chatId: 'c',
|
||||
lang: 'ru',
|
||||
rows: [row({ role: 'assistant', content: 'hi' })],
|
||||
});
|
||||
expect(md).toContain('# Без названия');
|
||||
expect(md).toContain('## 1. ИИ-агент');
|
||||
// Structural words remain English.
|
||||
expect(md).toContain('- Chat ID:');
|
||||
});
|
||||
|
||||
it('numbers messages and labels roles (You / AI agent)', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({ role: 'user', content: 'question' }),
|
||||
row({ role: 'assistant', content: 'answer' }),
|
||||
],
|
||||
});
|
||||
expect(md).toContain('## 1. You');
|
||||
expect(md).toContain('question');
|
||||
expect(md).toContain('## 2. AI agent');
|
||||
expect(md).toContain('answer');
|
||||
});
|
||||
|
||||
it('renders a tool part with fenced input/output and the friendly label', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({
|
||||
role: 'assistant',
|
||||
content: 'done',
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: 'tool-getPage',
|
||||
state: 'output-available',
|
||||
input: { id: 'p1' },
|
||||
output: { title: 'Hello' },
|
||||
},
|
||||
{ type: 'text', text: 'done' },
|
||||
],
|
||||
} as never,
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(md).toContain('**Tool: Read page** (`getPage`) — done');
|
||||
expect(md).toContain('Input:');
|
||||
expect(md).toContain('"id": "p1"');
|
||||
expect(md).toContain('Output:');
|
||||
expect(md).toContain('"title": "Hello"');
|
||||
});
|
||||
|
||||
// #186 re-review pt 1: restore the parity coverage of the removed client spec —
|
||||
// error state, unknown-tool fallback (en + ru), and the circular-stringify catch.
|
||||
it('renders a tool part in the error state with its errorText', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({
|
||||
role: 'assistant',
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: 'tool-getPage',
|
||||
state: 'output-error',
|
||||
input: { id: 'p1' },
|
||||
errorText: 'page not found',
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(md).toContain('**Tool: Read page** (`getPage`) — error');
|
||||
expect(md).toContain('**Error:** page not found');
|
||||
});
|
||||
|
||||
it('falls back to "Ran tool <name>" for an unknown tool (en) and the ru variant', () => {
|
||||
const parts = [
|
||||
{
|
||||
type: 'tool-mysteryTool',
|
||||
state: 'output-available',
|
||||
output: { ok: 1 },
|
||||
},
|
||||
];
|
||||
const en = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [row({ role: 'assistant', metadata: { parts } as never })],
|
||||
});
|
||||
expect(en).toContain('**Tool: Ran tool mysteryTool** (`mysteryTool`)');
|
||||
const ru = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
lang: 'ru',
|
||||
rows: [row({ role: 'assistant', metadata: { parts } as never })],
|
||||
});
|
||||
expect(ru).toContain('Выполнил инструмент mysteryTool');
|
||||
});
|
||||
|
||||
it('does not throw on a circular tool output (falls back to String)', () => {
|
||||
const circular: Record<string, unknown> = {};
|
||||
circular.self = circular;
|
||||
expect(() =>
|
||||
buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({
|
||||
role: 'assistant',
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: 'tool-getPage',
|
||||
state: 'output-available',
|
||||
output: circular,
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
}),
|
||||
],
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it('emits a token footer + total when usage is present', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({
|
||||
role: 'assistant',
|
||||
content: 'a',
|
||||
metadata: {
|
||||
usage: {
|
||||
inputTokens: 100,
|
||||
outputTokens: 20,
|
||||
totalTokens: 120,
|
||||
reasoningTokens: 8,
|
||||
},
|
||||
} as never,
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(md).toContain('- Total tokens: 120');
|
||||
expect(md).toContain(
|
||||
'_Tokens — in: 100, out: 20, reasoning: 8, total: 120_',
|
||||
);
|
||||
});
|
||||
|
||||
it('flags a still-streaming (interrupted) row', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({ role: 'assistant', content: 'partial', status: 'streaming' }),
|
||||
],
|
||||
});
|
||||
expect(md).toContain('still being generated');
|
||||
});
|
||||
|
||||
it('does NOT flag a completed row', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [row({ role: 'assistant', content: 'final', status: 'completed' })],
|
||||
});
|
||||
expect(md).not.toContain('still being generated');
|
||||
});
|
||||
|
||||
it('renders a legacy NULL-status row (no parts) from plain content', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({ role: 'assistant', content: 'legacy answer', status: null }),
|
||||
],
|
||||
});
|
||||
expect(md).toContain('legacy answer');
|
||||
expect(md).not.toContain('still being generated');
|
||||
});
|
||||
|
||||
it('renders a persisted error', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'error',
|
||||
metadata: { error: '401: Unauthorized' } as never,
|
||||
}),
|
||||
],
|
||||
});
|
||||
expect(md).toContain('**⚠️ Error:** 401: Unauthorized');
|
||||
});
|
||||
|
||||
it('escapes embedded triple-backtick fences with a longer delimiter', () => {
|
||||
const md = buildChatMarkdown({
|
||||
title: 'T',
|
||||
chatId: 'c',
|
||||
rows: [
|
||||
row({
|
||||
role: 'assistant',
|
||||
content: 'x',
|
||||
metadata: {
|
||||
parts: [
|
||||
{
|
||||
type: 'tool-getPage',
|
||||
state: 'output-available',
|
||||
output: '```inner```',
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
}),
|
||||
],
|
||||
});
|
||||
// A 4-backtick fence wraps content that itself contains a 3-backtick run.
|
||||
expect(md).toContain('````');
|
||||
});
|
||||
});
|
||||
299
apps/server/src/core/ai-chat/chat-markdown.util.ts
Normal file
299
apps/server/src/core/ai-chat/chat-markdown.util.ts
Normal file
@@ -0,0 +1,299 @@
|
||||
/**
|
||||
* Server-side Markdown export for an AI agent chat (#183). The DB is the single
|
||||
* source of truth: this renders a chat purely from its persisted message rows
|
||||
* (`AiChatMessage[]` — role / content / metadata.parts / toolCalls / usage).
|
||||
* Because the assistant row is now persisted UPFRONT and updated per step, an
|
||||
* interrupted turn is included up to its last finished step.
|
||||
*
|
||||
* Ported from the client `utils/chat-markdown.ts`. It is a PURE function (apart
|
||||
* from `new Date()` for the export timestamp), so it is straightforward to
|
||||
* unit-test and a future background worker can reuse it.
|
||||
*
|
||||
* Only a few fixed role/tool labels are localized via the `lang` param; the
|
||||
* structural document words (Input/Output/Error/Tokens/...) stay English because
|
||||
* the output is a technical artifact.
|
||||
*/
|
||||
|
||||
import type { AiChatMessage } from '@docmost/db/types/entity.types';
|
||||
|
||||
/** Supported export label languages. Defaults to English. */
|
||||
export type ExportLang = 'en' | 'ru';
|
||||
|
||||
/**
|
||||
* Normalize an arbitrary client locale code to a supported export language. The
|
||||
* client sends `i18n.language`, which is a FULL locale tag (e.g. `en-US`,
|
||||
* `ru-RU`), not a bare `en`/`ru` — so match on the language subtag and fall back
|
||||
* to English for anything non-Russian.
|
||||
*/
|
||||
export function normalizeLang(lang?: string): ExportLang {
|
||||
return lang?.toLowerCase().startsWith('ru') ? 'ru' : 'en';
|
||||
}
|
||||
|
||||
/** A single AI SDK UIMessage part (text part or a tool part). */
|
||||
interface ExportPart {
|
||||
type: string;
|
||||
text?: string;
|
||||
state?: string;
|
||||
toolName?: string;
|
||||
input?: unknown;
|
||||
output?: unknown;
|
||||
errorText?: string;
|
||||
}
|
||||
|
||||
/** Authoritative per-turn usage the server attaches to a message row. */
|
||||
interface UsageLike {
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
totalTokens?: number;
|
||||
reasoningTokens?: number;
|
||||
}
|
||||
|
||||
/** Localized label table. The client-side Markdown builder was removed by #183
|
||||
* (the export is now server-side only), so this no longer mirrors a second
|
||||
* exporter — instead the tool-action labels are kept in parity with the
|
||||
* on-screen action-log labels in the client's `tool-parts.tsx` (`toolLabelKey`)
|
||||
* so the export reads the same as the UI. Only role + tool-action labels are
|
||||
* localized; everything structural is an English constant in the renderer. */
|
||||
const LABELS: Record<
|
||||
ExportLang,
|
||||
{
|
||||
untitled: string;
|
||||
aiAgent: string;
|
||||
you: string;
|
||||
tools: Record<string, string>;
|
||||
ranTool: (name: string) => string;
|
||||
stillGenerating: string;
|
||||
}
|
||||
> = {
|
||||
en: {
|
||||
untitled: 'Untitled chat',
|
||||
aiAgent: 'AI agent',
|
||||
you: 'You',
|
||||
tools: {
|
||||
searchPages: 'Searched pages',
|
||||
getPage: 'Read page',
|
||||
createPage: 'Created page',
|
||||
updatePageContent: 'Updated page',
|
||||
renamePage: 'Renamed page',
|
||||
movePage: 'Moved page',
|
||||
deletePage: 'Deleted page (to trash)',
|
||||
createComment: 'Commented',
|
||||
resolveComment: 'Resolved comment',
|
||||
},
|
||||
ranTool: (name) => `Ran tool ${name}`,
|
||||
stillGenerating:
|
||||
'This message is still being generated — the export captured a partial, in-progress response.',
|
||||
},
|
||||
ru: {
|
||||
untitled: 'Без названия',
|
||||
aiAgent: 'ИИ-агент',
|
||||
you: 'Вы',
|
||||
tools: {
|
||||
searchPages: 'Искал по страницам',
|
||||
getPage: 'Прочитал страницу',
|
||||
createPage: 'Создал страницу',
|
||||
updatePageContent: 'Обновил страницу',
|
||||
renamePage: 'Переименовал страницу',
|
||||
movePage: 'Переместил страницу',
|
||||
deletePage: 'Удалил страницу (в корзину)',
|
||||
createComment: 'Прокомментировал',
|
||||
resolveComment: 'Закрыл комментарий',
|
||||
},
|
||||
ranTool: (name) => `Выполнил инструмент ${name}`,
|
||||
stillGenerating:
|
||||
'Это сообщение всё ещё генерируется — экспорт захватил частичный, незавершённый ответ.',
|
||||
},
|
||||
};
|
||||
|
||||
/** True for AI SDK tool parts (static `tool-*` or `dynamic-tool`). */
|
||||
function isToolPart(type: string): boolean {
|
||||
return type.startsWith('tool-') || type === 'dynamic-tool';
|
||||
}
|
||||
|
||||
/** Extract the tool name from a part `type` of `tool-${name}` (or dynamic). */
|
||||
function getToolName(part: ExportPart): string {
|
||||
if (part.type === 'dynamic-tool') return part.toolName ?? '';
|
||||
return part.type.startsWith('tool-')
|
||||
? part.type.slice('tool-'.length)
|
||||
: part.type;
|
||||
}
|
||||
|
||||
/** Map an AI SDK tool-part state to the 3 states the action-log renders. */
|
||||
function toolRunState(state: string | undefined): 'running' | 'done' | 'error' {
|
||||
if (state === 'output-error' || state === 'output-denied') return 'error';
|
||||
if (state === 'output-available') return 'done';
|
||||
return 'running';
|
||||
}
|
||||
|
||||
/** Resolve a tool's friendly action-log label (localized) from its name. */
|
||||
function toolLabel(name: string, lang: ExportLang): string {
|
||||
return LABELS[lang].tools[name] ?? LABELS[lang].ranTool(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stringify an arbitrary tool input/output value for a fenced block. Strings
|
||||
* pass through as-is; everything else is pretty-printed JSON, falling back to
|
||||
* `String(value)` if serialization throws (e.g. a circular structure).
|
||||
*/
|
||||
function stringify(value: unknown): string {
|
||||
if (typeof value === 'string') return value;
|
||||
try {
|
||||
return JSON.stringify(value, null, 2);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap `code` in a fenced code block whose backtick delimiter is LONGER than the
|
||||
* longest backtick run inside the content, so embedded backticks (or a literal
|
||||
* ``` fence) never break out of the block. Minimum 3 backticks.
|
||||
*/
|
||||
function fence(code: string, lang = ''): string {
|
||||
const runs: string[] = code.match(/`+/g) ?? [];
|
||||
const longest = runs.reduce((m, s) => Math.max(m, s.length), 0);
|
||||
const delim = '`'.repeat(Math.max(3, longest + 1));
|
||||
return `${delim}${lang}\n${code}\n${delim}`;
|
||||
}
|
||||
|
||||
/** Per-row token count, mirroring the header sum in the client window. */
|
||||
function rowTokens(usage: UsageLike): number {
|
||||
return (
|
||||
usage.totalTokens ?? (usage.inputTokens ?? 0) + (usage.outputTokens ?? 0)
|
||||
);
|
||||
}
|
||||
|
||||
/** Render one message's UIMessage parts into an array of Markdown blocks
|
||||
* (text blocks + tool blocks). Mirrors the client renderer / MessageItem. */
|
||||
function renderMessageParts(parts: ExportPart[], lang: ExportLang): string[] {
|
||||
const out: string[] = [];
|
||||
|
||||
for (const part of parts) {
|
||||
if (part.type === 'text') {
|
||||
const text = (part.text ?? '').trim();
|
||||
if (text.length > 0) out.push(text);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!isToolPart(part.type)) continue;
|
||||
|
||||
const name = getToolName(part);
|
||||
const label = toolLabel(name, lang);
|
||||
const state = toolRunState(part.state);
|
||||
|
||||
const toolLines: string[] = [`**Tool: ${label}** (\`${name}\`) — ${state}`];
|
||||
if (part.input !== undefined) {
|
||||
toolLines.push('Input:');
|
||||
toolLines.push(fence(stringify(part.input), 'json'));
|
||||
}
|
||||
if (part.output !== undefined) {
|
||||
toolLines.push('Output:');
|
||||
toolLines.push(fence(stringify(part.output), 'json'));
|
||||
}
|
||||
if (part.errorText) {
|
||||
toolLines.push(`**Error:** ${part.errorText}`);
|
||||
}
|
||||
out.push(toolLines.join('\n\n'));
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Resolve a persisted row's parts: prefer the rich persisted parts, else a
|
||||
* single text part built from the plain-text content (mirrors rowToUiMessage). */
|
||||
function rowParts(row: AiChatMessage): ExportPart[] {
|
||||
const meta = (row.metadata ?? {}) as { parts?: ExportPart[] };
|
||||
return Array.isArray(meta.parts) && meta.parts.length > 0
|
||||
? meta.parts
|
||||
: [{ type: 'text', text: row.content ?? '' }];
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a chat to a Markdown string from its persisted rows. Source = DB
|
||||
* ONLY (no live client state). A row whose `status` is still 'streaming' is an
|
||||
* interrupted turn that the export captured mid-flight; it is rendered up to its
|
||||
* last finished step and flagged "still generating".
|
||||
*/
|
||||
export function buildChatMarkdown(args: {
|
||||
title: string | null;
|
||||
chatId: string;
|
||||
rows: AiChatMessage[];
|
||||
// Accepts a full client locale tag (e.g. 'en-US'/'ru-RU'); normalized below.
|
||||
lang?: string;
|
||||
}): string {
|
||||
const { title, chatId, rows } = args;
|
||||
const lang: ExportLang = normalizeLang(args.lang);
|
||||
const L = LABELS[lang];
|
||||
const blocks: string[] = [];
|
||||
|
||||
const heading = (title ?? '').trim() || L.untitled;
|
||||
blocks.push(`# ${heading}`);
|
||||
|
||||
const usageOf = (row: AiChatMessage): UsageLike | undefined => {
|
||||
const meta = (row.metadata ?? {}) as { usage?: UsageLike };
|
||||
return meta.usage;
|
||||
};
|
||||
const errorOf = (row: AiChatMessage): string | undefined => {
|
||||
const meta = (row.metadata ?? {}) as { error?: string };
|
||||
return meta.error;
|
||||
};
|
||||
|
||||
// Metadata bullet list. Total tokens is only shown when there is a sum.
|
||||
const totalTokens = rows.reduce((sum, row) => {
|
||||
const usage = usageOf(row);
|
||||
return usage ? sum + rowTokens(usage) : sum;
|
||||
}, 0);
|
||||
const meta = [
|
||||
`- Chat ID: \`${chatId}\``,
|
||||
`- Exported: ${new Date().toISOString()}`,
|
||||
`- Messages: ${rows.length}`,
|
||||
];
|
||||
if (totalTokens > 0) meta.push(`- Total tokens: ${totalTokens}`);
|
||||
blocks.push(meta.join('\n'));
|
||||
|
||||
rows.forEach((row, index) => {
|
||||
blocks.push('---');
|
||||
|
||||
const roleLabel = row.role === 'assistant' ? L.aiAgent : L.you;
|
||||
blocks.push(`## ${index + 1}. ${roleLabel}`);
|
||||
|
||||
// Created-at kept in source as an HTML comment (out of the rendered prose).
|
||||
if (row.createdAt) {
|
||||
const iso =
|
||||
row.createdAt instanceof Date
|
||||
? row.createdAt.toISOString()
|
||||
: String(row.createdAt);
|
||||
blocks.push(`<!-- ${iso} -->`);
|
||||
}
|
||||
|
||||
blocks.push(...renderMessageParts(rowParts(row), lang));
|
||||
|
||||
// A still-'streaming' row is an interrupted/in-progress turn captured by the
|
||||
// export; record that so the partial answer is not mistaken for complete.
|
||||
if (row.status === 'streaming') {
|
||||
blocks.push(`_⏳ ${L.stillGenerating}_`);
|
||||
}
|
||||
|
||||
const error = errorOf(row);
|
||||
if (error) {
|
||||
blocks.push(`**⚠️ Error:** ${error}`);
|
||||
}
|
||||
|
||||
const usage = usageOf(row);
|
||||
if (usage) {
|
||||
const total = usage.totalTokens ?? rowTokens(usage);
|
||||
const reasoning =
|
||||
usage.reasoningTokens && usage.reasoningTokens > 0
|
||||
? `, reasoning: ${usage.reasoningTokens}`
|
||||
: '';
|
||||
blocks.push(
|
||||
`_Tokens — in: ${usage.inputTokens ?? '?'}, out: ${
|
||||
usage.outputTokens ?? '?'
|
||||
}${reasoning}, total: ${total}_`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Blank line between blocks so the Markdown renders cleanly.
|
||||
return blocks.join('\n\n');
|
||||
}
|
||||
@@ -26,3 +26,17 @@ export class GetChatMessagesDto {
|
||||
@IsString()
|
||||
cursor?: string;
|
||||
}
|
||||
|
||||
/** Export a chat to Markdown (#183). `lang` localizes the few fixed
|
||||
* role/tool-action labels; defaults to English server-side. */
|
||||
export class ExportChatDto {
|
||||
@IsString()
|
||||
chatId: string;
|
||||
|
||||
// A full client locale tag (e.g. 'en-US', 'ru-RU') — normalized server-side to
|
||||
// a supported export language (see normalizeLang). Accept any string so a
|
||||
// region-qualified locale is not rejected (the 400 that broke the real client).
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
lang?: string;
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ describe('resolveShareAssistantRequest (extracted controller funnel)', () => {
|
||||
resolveShareRole?: jest.Mock;
|
||||
getShareChatModel?: jest.Mock;
|
||||
tryConsumeWorkspaceQuota?: jest.Mock;
|
||||
withinShareTokenBudget?: jest.Mock;
|
||||
} = {}) {
|
||||
const aiSettings = {
|
||||
isPublicShareAssistantEnabled: jest
|
||||
@@ -65,6 +66,8 @@ describe('resolveShareAssistantRequest (extracted controller funnel)', () => {
|
||||
over.getShareChatModel ?? jest.fn().mockResolvedValue('MODEL'),
|
||||
tryConsumeWorkspaceQuota:
|
||||
over.tryConsumeWorkspaceQuota ?? jest.fn().mockResolvedValue(true),
|
||||
withinShareTokenBudget:
|
||||
over.withinShareTokenBudget ?? jest.fn().mockResolvedValue(true),
|
||||
};
|
||||
const deps: ShareAssistantDeps = {
|
||||
aiSettings: aiSettings as never,
|
||||
@@ -191,6 +194,39 @@ describe('resolveShareAssistantRequest (extracted controller funnel)', () => {
|
||||
expect(publicShareChat.tryConsumeWorkspaceQuota).toHaveBeenCalledWith('ws-1');
|
||||
});
|
||||
|
||||
it('withinShareTokenBudget false => 429 thrown BEFORE any stream (cost cap, #159 #5)', async () => {
|
||||
const { deps, publicShareChat } = makeDeps({
|
||||
withinShareTokenBudget: jest.fn().mockResolvedValue(false),
|
||||
});
|
||||
expect(await statusOf(deps, body())).toBe(429);
|
||||
expect(publicShareChat.withinShareTokenBudget).toHaveBeenCalledWith('ws-1');
|
||||
// The token budget is the COST backstop: an over-budget workspace must be
|
||||
// rejected WITHOUT consuming a request slot, so the request cap never runs.
|
||||
expect(publicShareChat.tryConsumeWorkspaceQuota).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('the token budget is checked BEFORE the request cap (over-budget wins, no slot spent)', async () => {
|
||||
// Over budget AND the request cap would also reject: the read-only budget
|
||||
// gate must win so the (mutating) request-slot consume is never reached.
|
||||
const { deps, publicShareChat } = makeDeps({
|
||||
withinShareTokenBudget: jest.fn().mockResolvedValue(false),
|
||||
tryConsumeWorkspaceQuota: jest.fn().mockResolvedValue(false),
|
||||
});
|
||||
expect(await statusOf(deps, body())).toBe(429);
|
||||
expect(publicShareChat.tryConsumeWorkspaceQuota).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('the token-budget gate is checked BEFORE the payload caps (429 wins over 413)', async () => {
|
||||
const { deps } = makeDeps({
|
||||
withinShareTokenBudget: jest.fn().mockResolvedValue(false),
|
||||
});
|
||||
const huge = {
|
||||
role: 'user',
|
||||
parts: [{ type: 'text', text: 'x'.repeat(MAX_SHARE_MESSAGE_CHARS + 1) }],
|
||||
};
|
||||
expect(await statusOf(deps, body({ messages: [huge] }))).toBe(429);
|
||||
});
|
||||
|
||||
it('messages over MAX_SHARE_MESSAGES => 413', async () => {
|
||||
const { deps } = makeDeps();
|
||||
const tooMany = Array.from({ length: MAX_SHARE_MESSAGES + 1 }, () => ({
|
||||
|
||||
@@ -151,6 +151,7 @@ export interface ShareAssistantDeps {
|
||||
| 'resolveShareRole'
|
||||
| 'getShareChatModel'
|
||||
| 'tryConsumeWorkspaceQuota'
|
||||
| 'withinShareTokenBudget'
|
||||
>;
|
||||
}
|
||||
|
||||
@@ -267,9 +268,21 @@ export async function resolveShareAssistantRequest(
|
||||
throw new NotFoundException('Not found');
|
||||
}
|
||||
|
||||
// 5. Per-WORKSPACE anti-abuse cap (IP-independent; defense in depth). Checked
|
||||
// BEFORE res.hijack(), so an over-cap workspace gets a clean 429 and spends
|
||||
// nothing.
|
||||
// 5a. Per-WORKSPACE rolling-day TOKEN budget (the COST backstop). Read-only and
|
||||
// checked FIRST so a workspace that has already burned its day's token
|
||||
// budget gets a clean 429 WITHOUT consuming a request slot, and spends
|
||||
// nothing. Counting requests alone does not bound the owner's provider
|
||||
// bill (issue #159, finding #5).
|
||||
if (!(await deps.publicShareChat.withinShareTokenBudget(workspaceId))) {
|
||||
throw new HttpException(
|
||||
'This documentation assistant has reached its usage budget. Please try again later.',
|
||||
HttpStatus.TOO_MANY_REQUESTS,
|
||||
);
|
||||
}
|
||||
|
||||
// 5b. Per-WORKSPACE anti-abuse request cap (IP-independent; defense in depth).
|
||||
// Checked BEFORE res.hijack(), so an over-cap workspace gets a clean 429
|
||||
// and spends nothing.
|
||||
if (!(await deps.publicShareChat.tryConsumeWorkspaceQuota(workspaceId))) {
|
||||
throw new HttpException(
|
||||
'This documentation assistant is temporarily busy. Please try again later.',
|
||||
|
||||
@@ -17,7 +17,9 @@ import { buildShareSystemPrompt } from './public-share-chat.prompt';
|
||||
import { roleModelOverride } from './roles/role-model-config';
|
||||
import {
|
||||
PublicShareWorkspaceLimiter,
|
||||
PublicShareWorkspaceTokenBudget,
|
||||
createPublicShareWorkspaceLimiter,
|
||||
createPublicShareWorkspaceTokenBudget,
|
||||
} from './public-share-workspace-limiter';
|
||||
import { describeProviderError } from '../../integrations/ai/ai-error.util';
|
||||
import {
|
||||
@@ -125,6 +127,16 @@ export class PublicShareChatService {
|
||||
*/
|
||||
private readonly workspaceLimiter: PublicShareWorkspaceLimiter;
|
||||
|
||||
/**
|
||||
* COST contour two: a per-workspace TOKEN budget over a rolling day. The
|
||||
* request-count limiter above bounds how many anonymous calls run; this bounds
|
||||
* how many provider TOKENS they spend (input re-sent per step + output),
|
||||
* which is what the owner is actually billed for (issue #159, finding #5).
|
||||
* Checked read-only before a turn streams; the real usage is recorded once the
|
||||
* turn finishes (`onFinish`).
|
||||
*/
|
||||
private readonly tokenBudget: PublicShareWorkspaceTokenBudget;
|
||||
|
||||
constructor(
|
||||
private readonly ai: AiService,
|
||||
private readonly aiSettings: AiSettingsService,
|
||||
@@ -133,6 +145,7 @@ export class PublicShareChatService {
|
||||
private readonly aiAgentRoleRepo: AiAgentRoleRepo,
|
||||
) {
|
||||
this.workspaceLimiter = createPublicShareWorkspaceLimiter(redisService);
|
||||
this.tokenBudget = createPublicShareWorkspaceTokenBudget(redisService);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -144,6 +157,48 @@ export class PublicShareChatService {
|
||||
return this.workspaceLimiter.tryConsume(workspaceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read-only pre-stream COST gate: true while the workspace is under its
|
||||
* rolling-day token budget, false once the trailing-day token spend has
|
||||
* reached it (the controller must then 429 BEFORE starting the stream). This
|
||||
* bounds the owner's actual provider bill, which counting requests alone does
|
||||
* not (issue #159, finding #5).
|
||||
*/
|
||||
async withinShareTokenBudget(workspaceId: string): Promise<boolean> {
|
||||
return this.tokenBudget.withinBudget(workspaceId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a finished turn's real token spend against the rolling-day budget.
|
||||
* Best-effort (the turn already ran): failures are swallowed by the budget.
|
||||
*/
|
||||
async recordShareTokens(workspaceId: string, tokens: number): Promise<void> {
|
||||
return this.tokenBudget.record(workspaceId, tokens);
|
||||
}
|
||||
|
||||
/**
|
||||
* `streamText` onFinish hook body: account a finished turn's REAL token spend
|
||||
* (input re-sent per step + output, summed across all steps) against the
|
||||
* per-workspace rolling-day budget, so a future turn over budget is rejected up
|
||||
* front (issue #159, finding #5). `totalUsage` fields are `number | undefined`;
|
||||
* fall back to the sum of input+output when the provider omits `totalTokens`.
|
||||
* Fire-and-forget: the turn already streamed, so a record failure must not
|
||||
* break it.
|
||||
*/
|
||||
recordTurnUsage(
|
||||
workspaceId: string,
|
||||
totalUsage: {
|
||||
totalTokens?: number;
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
},
|
||||
): void {
|
||||
const tokens =
|
||||
totalUsage.totalTokens ??
|
||||
(totalUsage.inputTokens ?? 0) + (totalUsage.outputTokens ?? 0);
|
||||
void this.recordShareTokens(workspaceId, tokens);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the admin-selected agent role for the anonymous public-share
|
||||
* assistant, scoped to the workspace and soft-delete aware. Returns null when
|
||||
@@ -231,6 +286,8 @@ export class PublicShareChatService {
|
||||
// bill even if the per-IP throttle is evaded; worst case = steps × this.
|
||||
maxOutputTokens: resolveShareAiMaxOutputTokens(),
|
||||
abortSignal: signal,
|
||||
onFinish: ({ totalUsage }) =>
|
||||
this.recordTurnUsage(workspaceId, totalUsage),
|
||||
onError: ({ error }) => {
|
||||
// Reuse the shared formatter so provider error formatting stays
|
||||
// unified (statusCode + body) with the authenticated path.
|
||||
|
||||
@@ -11,8 +11,11 @@ import {
|
||||
import { PublicShareChatToolsService } from './tools/public-share-chat-tools.service';
|
||||
import {
|
||||
PublicShareWorkspaceLimiter,
|
||||
PublicShareWorkspaceTokenBudget,
|
||||
resolveShareAiWorkspaceMax,
|
||||
resolveShareAiWorkspaceTokenBudget,
|
||||
SHARE_AI_WORKSPACE_MAX_PER_WINDOW,
|
||||
SHARE_AI_WORKSPACE_TOKEN_BUDGET_DEFAULT,
|
||||
} from './public-share-workspace-limiter';
|
||||
|
||||
/**
|
||||
@@ -546,6 +549,228 @@ describe('PublicShareWorkspaceLimiter (cluster-wide sliding-window per-workspace
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* In-memory fake of the ioredis slice the TOKEN budget uses. Unlike the request
|
||||
* limiter (one Lua), the budget runs TWO scripts over the same sorted set:
|
||||
* - the read-only CHECK (sums the token counts encoded as each member's leading
|
||||
* integer, admits while the sum is under budget, never mutates), and
|
||||
* - the RECORD (ZADDs a finished turn's `<tokens>:<unique>` member).
|
||||
* The fake faithfully reproduces both (branching on the script body) so the spec
|
||||
* exercises the REAL budget math, not a re-implementation.
|
||||
*/
|
||||
class FakeTokenRedis {
|
||||
private sets = new Map<string, Array<{ score: number; member: string }>>();
|
||||
|
||||
async eval(
|
||||
script: string,
|
||||
_numKeys: number,
|
||||
key: string,
|
||||
nowStr: string,
|
||||
windowMsStr: string,
|
||||
arg3: string,
|
||||
): Promise<number> {
|
||||
const now = Number(nowStr);
|
||||
const windowMs = Number(windowMsStr);
|
||||
const cutoff = now - windowMs;
|
||||
const arr = (this.sets.get(key) ?? []).filter((e) => e.score > cutoff);
|
||||
if (script.includes('ZADD')) {
|
||||
// RECORD: arg3 is the `<tokens>:<unique>` member; append at score=now.
|
||||
arr.push({ score: now, member: arg3 });
|
||||
this.sets.set(key, arr);
|
||||
return 1;
|
||||
}
|
||||
// CHECK: arg3 is the budget; sum the leading integer of each survivor.
|
||||
const budget = Number(arg3);
|
||||
this.sets.set(key, arr);
|
||||
const total = arr.reduce((sum, e) => {
|
||||
const m = /^(\d+)/.exec(e.member);
|
||||
return sum + (m ? Number(m[1]) : 0);
|
||||
}, 0);
|
||||
return total >= budget ? 0 : 1;
|
||||
}
|
||||
}
|
||||
|
||||
function makeTokenBudget(budget: number, windowMs: number, clock: () => number) {
|
||||
const redis = new FakeTokenRedis() as unknown as import('ioredis').Redis;
|
||||
return new PublicShareWorkspaceTokenBudget(redis, budget, windowMs, clock);
|
||||
}
|
||||
|
||||
describe('resolveShareAiWorkspaceTokenBudget (env-overridable per-day token budget)', () => {
|
||||
const KEY = 'SHARE_AI_WORKSPACE_TOKEN_BUDGET_PER_DAY';
|
||||
const saved = process.env[KEY];
|
||||
afterEach(() => {
|
||||
if (saved === undefined) delete process.env[KEY];
|
||||
else process.env[KEY] = saved;
|
||||
});
|
||||
|
||||
it('falls back to the default when unset', () => {
|
||||
delete process.env[KEY];
|
||||
expect(resolveShareAiWorkspaceTokenBudget()).toBe(
|
||||
SHARE_AI_WORKSPACE_TOKEN_BUDGET_DEFAULT,
|
||||
);
|
||||
});
|
||||
|
||||
it('honors a positive override', () => {
|
||||
process.env[KEY] = '250000';
|
||||
expect(resolveShareAiWorkspaceTokenBudget()).toBe(250000);
|
||||
});
|
||||
|
||||
it('ignores a non-positive / unparseable value (uses the default)', () => {
|
||||
for (const bad of ['0', '-5', 'nope', '']) {
|
||||
process.env[KEY] = bad;
|
||||
expect(resolveShareAiWorkspaceTokenBudget()).toBe(
|
||||
SHARE_AI_WORKSPACE_TOKEN_BUDGET_DEFAULT,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('PublicShareWorkspaceTokenBudget (cluster-wide rolling-day token cap)', () => {
|
||||
it('admits while under budget and rejects once the recorded spend reaches it', async () => {
|
||||
const budget = makeTokenBudget(1000, 60_000, () => 1_000);
|
||||
expect(await budget.withinBudget('ws-1')).toBe(true); // nothing spent yet
|
||||
await budget.record('ws-1', 600);
|
||||
expect(await budget.withinBudget('ws-1')).toBe(true); // 600 < 1000
|
||||
await budget.record('ws-1', 400);
|
||||
// 1000 >= 1000: the budget is exhausted, so the next turn is rejected up front.
|
||||
expect(await budget.withinBudget('ws-1')).toBe(false);
|
||||
});
|
||||
|
||||
it('counts TOKENS, not requests: one fat turn can exhaust the budget alone', async () => {
|
||||
const budget = makeTokenBudget(1000, 60_000, () => 1_000);
|
||||
// A single accepted turn re-sends the whole transcript across 5 steps; here
|
||||
// it lands as 1200 tokens — already over the day budget on its own.
|
||||
await budget.record('ws-1', 1200);
|
||||
expect(await budget.withinBudget('ws-1')).toBe(false);
|
||||
});
|
||||
|
||||
it('ages out spend older than the window so the budget recovers', async () => {
|
||||
let now = 0;
|
||||
const budget = makeTokenBudget(1000, 60_000, () => now);
|
||||
await budget.record('ws-1', 1000); // at budget
|
||||
now += 59_999; // still inside the day window
|
||||
expect(await budget.withinBudget('ws-1')).toBe(false);
|
||||
now += 2; // the spend is now strictly older than windowMs
|
||||
expect(await budget.withinBudget('ws-1')).toBe(true);
|
||||
});
|
||||
|
||||
it('ignores non-positive / non-finite usage (never records phantom spend)', async () => {
|
||||
const budget = makeTokenBudget(1000, 60_000, () => 1_000);
|
||||
await budget.record('ws-1', 0);
|
||||
await budget.record('ws-1', -50);
|
||||
await budget.record('ws-1', Number.NaN);
|
||||
await budget.record('ws-1', Infinity);
|
||||
expect(await budget.withinBudget('ws-1')).toBe(true); // nothing accumulated
|
||||
});
|
||||
|
||||
it('keeps separate budgets per workspace', async () => {
|
||||
const budget = makeTokenBudget(500, 60_000, () => 1_000);
|
||||
await budget.record('ws-a', 500); // ws-a exhausted
|
||||
expect(await budget.withinBudget('ws-a')).toBe(false);
|
||||
expect(await budget.withinBudget('ws-b')).toBe(true); // ws-b untouched
|
||||
});
|
||||
|
||||
it('FAILS CLOSED on the read-only check when Redis rejects', async () => {
|
||||
const failingRedis = {
|
||||
eval: () => Promise.reject(new Error('redis down')),
|
||||
} as unknown as import('ioredis').Redis;
|
||||
const budget = new PublicShareWorkspaceTokenBudget(
|
||||
failingRedis,
|
||||
1000,
|
||||
60_000,
|
||||
() => 1_000,
|
||||
);
|
||||
const errSpy = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
expect(await budget.withinBudget('ws-1')).toBe(false);
|
||||
expect(errSpy).toHaveBeenCalled();
|
||||
errSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('SWALLOWS a record failure (best-effort post-accounting, never throws)', async () => {
|
||||
// The turn already streamed; a record failure must not surface to the caller.
|
||||
const failingRedis = {
|
||||
eval: () => Promise.reject(new Error('redis down')),
|
||||
} as unknown as import('ioredis').Redis;
|
||||
const budget = new PublicShareWorkspaceTokenBudget(
|
||||
failingRedis,
|
||||
1000,
|
||||
60_000,
|
||||
() => 1_000,
|
||||
);
|
||||
const errSpy = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined);
|
||||
await expect(budget.record('ws-1', 100)).resolves.toBeUndefined();
|
||||
expect(errSpy).toHaveBeenCalled();
|
||||
errSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('PublicShareChatService.withinShareTokenBudget / recordShareTokens', () => {
|
||||
it('delegates the cost gate + accounting to the redis-backed token budget', async () => {
|
||||
const redis = new FakeTokenRedis();
|
||||
const redisService = { getOrThrow: () => redis } as never;
|
||||
const service = new PublicShareChatService(
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
redisService,
|
||||
{} as never,
|
||||
);
|
||||
// Default budget is large, so a fresh workspace is under budget; recording a
|
||||
// modest spend keeps it under budget (asserts the wiring the controller +
|
||||
// onFinish rely on).
|
||||
expect(await service.withinShareTokenBudget('ws-1')).toBe(true);
|
||||
await service.recordShareTokens('ws-1', 1234);
|
||||
expect(await service.withinShareTokenBudget('ws-1')).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PublicShareChatService.recordTurnUsage (streamText onFinish accounting)', () => {
|
||||
function makeService() {
|
||||
const redisService = { getOrThrow: () => new FakeTokenRedis() } as never;
|
||||
const service = new PublicShareChatService(
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
redisService,
|
||||
{} as never,
|
||||
);
|
||||
const recordSpy = jest
|
||||
.spyOn(service, 'recordShareTokens')
|
||||
.mockResolvedValue(undefined);
|
||||
return { service, recordSpy };
|
||||
}
|
||||
|
||||
it('sums input+output when the provider omits totalTokens', () => {
|
||||
const { service, recordSpy } = makeService();
|
||||
// The onFinish payload shape: a totalUsage with per-component counts but no
|
||||
// authoritative total (provider omitted it).
|
||||
service.recordTurnUsage('ws-1', { inputTokens: 1200, outputTokens: 300 });
|
||||
expect(recordSpy).toHaveBeenCalledWith('ws-1', 1500);
|
||||
});
|
||||
|
||||
it('treats missing input/output components as 0 in the fallback sum', () => {
|
||||
const { service, recordSpy } = makeService();
|
||||
service.recordTurnUsage('ws-1', { outputTokens: 42 });
|
||||
expect(recordSpy).toHaveBeenCalledWith('ws-1', 42);
|
||||
});
|
||||
|
||||
it('prefers the authoritative totalTokens when present (not the sum)', () => {
|
||||
const { service, recordSpy } = makeService();
|
||||
// totalTokens is the provider's authoritative figure and may differ from a
|
||||
// naive input+output sum (e.g. cached/ reasoning tokens); it must win.
|
||||
service.recordTurnUsage('ws-1', {
|
||||
totalTokens: 5000,
|
||||
inputTokens: 1200,
|
||||
outputTokens: 300,
|
||||
});
|
||||
expect(recordSpy).toHaveBeenCalledWith('ws-1', 5000);
|
||||
});
|
||||
});
|
||||
|
||||
describe('PublicShareChatService.tryConsumeWorkspaceQuota', () => {
|
||||
it('delegates to the redis-backed per-workspace limiter', async () => {
|
||||
const redis = new FakeRedis();
|
||||
|
||||
@@ -136,6 +136,177 @@ export class PublicShareWorkspaceLimiter {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* SECOND cost contour: a per-workspace TOKEN budget over a rolling DAY.
|
||||
*
|
||||
* The request-count cap above bounds how MANY anonymous calls a workspace
|
||||
* admits, but NOT how expensive each one is: one accepted call runs the agent
|
||||
* loop up to `stepCountIs(5)`, and every step re-sends the WHOLE client-held
|
||||
* transcript (~hundreds of KB) as input, so the provider input alone can be tens
|
||||
* of thousands of tokens PER step while `maxOutputTokens` only caps the output.
|
||||
* The request cap is also hourly with no daily ceiling, so a steady stream at
|
||||
* the hourly cap sustains ~24x its count per day. Counting requests therefore
|
||||
* does not bound the owner's actual LLM bill (issue #159, finding #5).
|
||||
*
|
||||
* This contour caps the SPEND directly: the actual tokens consumed (input +
|
||||
* output, summed across all steps of every accepted turn) over the trailing
|
||||
* `windowMs` (one rolling day) must stay under `budget`. It is checked BEFORE a
|
||||
* turn streams (read-only) and the turn's real usage is recorded AFTER it
|
||||
* finishes (`streamText` onFinish). Like the request cap it is cluster-wide
|
||||
* (shared Redis) and uses a sliding-window LOG so the day boundary cannot be
|
||||
* gamed for a 2x burst.
|
||||
*
|
||||
* Pre-check is read-only, so a turn already over budget is rejected, but the
|
||||
* tokens of an in-flight turn are not yet known and are accounted only once it
|
||||
* finishes. The worst-case overshoot past the budget is therefore one turn
|
||||
* (bounded by steps x (maxOutputTokens + transcript size)) — acceptable for a
|
||||
* cost backstop on an optional anonymous assistant.
|
||||
*/
|
||||
|
||||
/** Default per-workspace token budget over the rolling day. */
|
||||
export const SHARE_AI_WORKSPACE_TOKEN_BUDGET_DEFAULT = 1_000_000;
|
||||
/** Default token-budget window length: one rolling day. */
|
||||
export const SHARE_AI_WORKSPACE_TOKEN_WINDOW_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/** Redis key namespace for the per-workspace token-spend sliding-window log. */
|
||||
const TOKEN_KEY_PREFIX = 'share-ai:ws-tokens:';
|
||||
|
||||
/**
|
||||
* Read-only sliding-window token-budget check.
|
||||
*
|
||||
* KEYS[1] = the per-workspace token sorted-set key
|
||||
* ARGV[1] = now (epoch ms)
|
||||
* ARGV[2] = windowMs
|
||||
* ARGV[3] = budget (max tokens in the trailing window)
|
||||
*
|
||||
* Drops entries older than the window, then sums the token counts encoded as the
|
||||
* leading integer of each surviving member. Returns 1 if the running total is
|
||||
* still UNDER budget (admit), 0 once it has reached/exceeded the budget. Does NOT
|
||||
* add anything — the turn's real usage is recorded separately once it finishes.
|
||||
*/
|
||||
const TOKEN_BUDGET_CHECK_LUA = `
|
||||
local key = KEYS[1]
|
||||
local now = tonumber(ARGV[1])
|
||||
local windowMs = tonumber(ARGV[2])
|
||||
local budget = tonumber(ARGV[3])
|
||||
redis.call('ZREMRANGEBYSCORE', key, 0, now - windowMs)
|
||||
local members = redis.call('ZRANGE', key, 0, -1)
|
||||
local total = 0
|
||||
for i = 1, #members do
|
||||
local t = tonumber(string.match(members[i], '^(%d+)'))
|
||||
if t then total = total + t end
|
||||
end
|
||||
if total >= budget then
|
||||
return 0
|
||||
end
|
||||
return 1
|
||||
`;
|
||||
|
||||
/**
|
||||
* Record one finished turn's token spend in the sliding-window log.
|
||||
*
|
||||
* KEYS[1] = the per-workspace token sorted-set key
|
||||
* ARGV[1] = now (epoch ms) — the entry score
|
||||
* ARGV[2] = windowMs
|
||||
* ARGV[3] = member (`<tokens>:<unique>`; the leading integer is the token count)
|
||||
*
|
||||
* Always ZADDs (the turn already ran and spent the tokens) and refreshes the
|
||||
* key TTL so idle workspaces cost no memory. Trims expired entries first so the
|
||||
* set never grows unbounded for a busy workspace.
|
||||
*/
|
||||
const TOKEN_RECORD_LUA = `
|
||||
local key = KEYS[1]
|
||||
local now = tonumber(ARGV[1])
|
||||
local windowMs = tonumber(ARGV[2])
|
||||
local member = ARGV[3]
|
||||
redis.call('ZREMRANGEBYSCORE', key, 0, now - windowMs)
|
||||
redis.call('ZADD', key, now, member)
|
||||
redis.call('PEXPIRE', key, windowMs)
|
||||
return 1
|
||||
`;
|
||||
|
||||
/**
|
||||
* Cluster-wide, sliding-window per-workspace TOKEN budget backed by Redis.
|
||||
* `withinBudget(key)` is a read-only pre-stream gate; `record(key, tokens)`
|
||||
* accounts a finished turn's real usage. Decoupled from NestJS so it is testable
|
||||
* against a mocked/real ioredis client, mirroring the request-count limiter.
|
||||
*/
|
||||
export class PublicShareWorkspaceTokenBudget {
|
||||
private readonly logger = new Logger(PublicShareWorkspaceTokenBudget.name);
|
||||
private counter = 0;
|
||||
|
||||
constructor(
|
||||
private readonly redis: Redis,
|
||||
private readonly budget: number = SHARE_AI_WORKSPACE_TOKEN_BUDGET_DEFAULT,
|
||||
private readonly windowMs: number = SHARE_AI_WORKSPACE_TOKEN_WINDOW_MS,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Read-only pre-stream check. Returns true while the workspace is under its
|
||||
* rolling-day token budget, false once the trailing-window spend has reached
|
||||
* it (caller must then 429 BEFORE streaming any tokens).
|
||||
*
|
||||
* FAILS CLOSED (false) on a Redis error: identical reasoning to the request
|
||||
* limiter — when we cannot prove the workspace is under budget we DENY rather
|
||||
* than admit an unmetered billable call. The assistant is optional, so a
|
||||
* transient Redis blip briefly disabling it beats an unbounded provider bill.
|
||||
*/
|
||||
async withinBudget(key: string): Promise<boolean> {
|
||||
const t = this.now();
|
||||
try {
|
||||
const admitted = await this.redis.eval(
|
||||
TOKEN_BUDGET_CHECK_LUA,
|
||||
1,
|
||||
TOKEN_KEY_PREFIX + key,
|
||||
String(t),
|
||||
String(this.windowMs),
|
||||
String(this.budget),
|
||||
);
|
||||
return admitted === 1;
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`share-ai token budget Redis failure for key "${key}"; failing closed`,
|
||||
err as Error,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a finished turn's token spend. Best-effort: the turn already ran, so
|
||||
* a Redis failure here is logged but not propagated — it would only cause a
|
||||
* slight under-count of the running budget, never a wrong answer to the
|
||||
* caller. Non-positive / non-finite usage is ignored.
|
||||
*/
|
||||
async record(key: string, tokens: number): Promise<void> {
|
||||
if (!Number.isFinite(tokens) || tokens <= 0) return;
|
||||
const spend = Math.floor(tokens);
|
||||
const t = this.now();
|
||||
// Member: `<tokens>:<unique>` — the check Lua sums the leading integer, and
|
||||
// the unique suffix keeps distinct turns in the same ms from colliding on
|
||||
// the sorted-set member (which would drop one entry and under-count).
|
||||
const member = `${spend}:${t}-${this.counter++}-${Math.random()
|
||||
.toString(36)
|
||||
.slice(2)}`;
|
||||
try {
|
||||
await this.redis.eval(
|
||||
TOKEN_RECORD_LUA,
|
||||
1,
|
||||
TOKEN_KEY_PREFIX + key,
|
||||
String(t),
|
||||
String(this.windowMs),
|
||||
member,
|
||||
);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`share-ai token budget record failure for key "${key}" (${spend} tokens); ignoring`,
|
||||
err as Error,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the per-workspace cap from the environment (overridable seam), falling
|
||||
* back to the sane default. A non-positive / unparseable value uses the default.
|
||||
@@ -162,3 +333,31 @@ export function createPublicShareWorkspaceLimiter(
|
||||
SHARE_AI_WORKSPACE_WINDOW_MS,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the per-workspace rolling-day token budget from the environment
|
||||
* (overridable seam), falling back to the sane default. A non-positive /
|
||||
* unparseable value uses the default.
|
||||
*/
|
||||
export function resolveShareAiWorkspaceTokenBudget(): number {
|
||||
const raw = Number(process.env.SHARE_AI_WORKSPACE_TOKEN_BUDGET_PER_DAY);
|
||||
return Number.isFinite(raw) && raw > 0
|
||||
? Math.floor(raw)
|
||||
: SHARE_AI_WORKSPACE_TOKEN_BUDGET_DEFAULT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the per-workspace token budget from the injected RedisService (the same
|
||||
* global ioredis client used by the request-count limiter). Tiny factory so the
|
||||
* service constructor stays declarative and the budget stays unit-testable with
|
||||
* a hand-rolled fake redis.
|
||||
*/
|
||||
export function createPublicShareWorkspaceTokenBudget(
|
||||
redisService: RedisService,
|
||||
): PublicShareWorkspaceTokenBudget {
|
||||
return new PublicShareWorkspaceTokenBudget(
|
||||
redisService.getOrThrow(),
|
||||
resolveShareAiWorkspaceTokenBudget(),
|
||||
SHARE_AI_WORKSPACE_TOKEN_WINDOW_MS,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -120,18 +120,25 @@ describe('AiChatToolsService deletePage guardrail (H4)', () => {
|
||||
const tools = await buildTools();
|
||||
const deletePage = tools.deletePage;
|
||||
|
||||
// The Zod input schema only allows `pageId`; parsing strips/ignores extra
|
||||
// keys, so a permanent/force flag is never part of the validated input.
|
||||
// The wrapped input schema (modelFriendlyInput) only allows `pageId`;
|
||||
// validation strips/ignores extra keys, so a permanent/force flag is never
|
||||
// part of the validated input handed to execute.
|
||||
const schema = (deletePage as unknown as { inputSchema: unknown })
|
||||
.inputSchema as {
|
||||
parse: (v: unknown) => Record<string, unknown>;
|
||||
validate: (
|
||||
v: unknown,
|
||||
) =>
|
||||
| { success: boolean; value?: Record<string, unknown> }
|
||||
| Promise<{ success: boolean; value?: Record<string, unknown> }>;
|
||||
};
|
||||
const parsed = schema.parse({
|
||||
const result = await schema.validate({
|
||||
pageId: 'page-789',
|
||||
permanentlyDelete: true,
|
||||
forceDelete: true,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const parsed = result.value as Record<string, unknown>;
|
||||
expect(parsed).toHaveProperty('pageId', 'page-789');
|
||||
expect(parsed).not.toHaveProperty('permanentlyDelete');
|
||||
expect(parsed).not.toHaveProperty('forceDelete');
|
||||
@@ -207,19 +214,26 @@ describe('AiChatToolsService expanded toolset guardrails', () => {
|
||||
const tools = await buildTools();
|
||||
const transformPage = tools.transformPage;
|
||||
|
||||
// The Zod input schema only allows pageId/transformJs/dryRun; parsing
|
||||
// strips unknown keys, so deleteComments can never reach the client.
|
||||
// The wrapped input schema only allows pageId/transformJs/dryRun;
|
||||
// validation strips unknown keys, so deleteComments can never reach the
|
||||
// client.
|
||||
const schema = (transformPage as unknown as { inputSchema: unknown })
|
||||
.inputSchema as {
|
||||
parse: (v: unknown) => Record<string, unknown>;
|
||||
validate: (
|
||||
v: unknown,
|
||||
) =>
|
||||
| { success: boolean; value?: Record<string, unknown> }
|
||||
| Promise<{ success: boolean; value?: Record<string, unknown> }>;
|
||||
};
|
||||
const parsed = schema.parse({
|
||||
const result = await schema.validate({
|
||||
pageId: 'p',
|
||||
transformJs: '(d)=>d',
|
||||
dryRun: true,
|
||||
deleteComments: true,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
const parsed = result.value as Record<string, unknown>;
|
||||
expect(parsed).toHaveProperty('pageId', 'p');
|
||||
expect(parsed).not.toHaveProperty('deleteComments');
|
||||
});
|
||||
@@ -395,3 +409,95 @@ describe('AiChatToolsService node-arg JSON-string coercion', () => {
|
||||
expect(updatePageJsonCalls).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Model-friendly tool-call validation (#190): when the model drops a required
|
||||
* `pageId` in a parallel/batch tool call, the built-in input schema must return
|
||||
* a CLEAR, actionable message (naming the parameter, reminding it not to drop
|
||||
* ids in batches) instead of zod's raw "expected string, received undefined" —
|
||||
* while a valid call still validates. This is wired centrally via
|
||||
* modelFriendlyInput, so it applies to every in-app tool; createComment (the
|
||||
* tool from the bug report) and a sharedTool-built tool (getPage's sibling
|
||||
* getOutline) are exercised here end-to-end through forUser().
|
||||
*/
|
||||
describe('AiChatToolsService model-friendly input validation (#190)', () => {
|
||||
const fakeClient: Partial<DocmostClientLike> = {};
|
||||
const tokenServiceStub = {
|
||||
generateAccessToken: jest.fn().mockResolvedValue('access-token'),
|
||||
generateCollabToken: jest.fn().mockResolvedValue('collab-token'),
|
||||
};
|
||||
let service: AiChatToolsService;
|
||||
|
||||
beforeEach(() => {
|
||||
jest.spyOn(loader, 'loadDocmostMcp').mockResolvedValue(
|
||||
mockLoaded(function () {
|
||||
return fakeClient as DocmostClientLike;
|
||||
} as unknown as loader.DocmostClientCtor),
|
||||
);
|
||||
service = new AiChatToolsService(
|
||||
tokenServiceStub as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
function buildTools() {
|
||||
return service.forUser(
|
||||
{ id: 'user-1', email: 'u@example.com', workspaceId: 'ws-1' } as never,
|
||||
'session-1',
|
||||
'ws-1',
|
||||
'chat-1',
|
||||
);
|
||||
}
|
||||
|
||||
// The AI SDK Schema produced by modelFriendlyInput exposes `validate`.
|
||||
type ValidatableSchema = {
|
||||
validate: (
|
||||
v: unknown,
|
||||
) =>
|
||||
| { success: boolean; value?: unknown; error?: Error }
|
||||
| Promise<{ success: boolean; value?: unknown; error?: Error }>;
|
||||
};
|
||||
const inputSchemaOf = (t: unknown) =>
|
||||
(t as { inputSchema: unknown }).inputSchema as ValidatableSchema;
|
||||
|
||||
it('createComment: a dropped pageId yields a clear, model-actionable message', async () => {
|
||||
const tools = await buildTools();
|
||||
// The exact failing shape from the bug report's second parallel batch:
|
||||
// content + selection, but pageId silently dropped.
|
||||
const result = await inputSchemaOf(tools.createComment).validate({
|
||||
content: 'A remark',
|
||||
selection: 'титановый проводник',
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error?.message).toContain('parameter "pageId": missing (required)');
|
||||
expect(result.error?.message).toContain('parallel/batch tool calls');
|
||||
// Not the raw zod text the model previously received.
|
||||
expect(result.error?.message).not.toContain('received undefined');
|
||||
});
|
||||
|
||||
it('createComment: a valid call with pageId validates successfully', async () => {
|
||||
const tools = await buildTools();
|
||||
const result = await inputSchemaOf(tools.createComment).validate({
|
||||
pageId: '019efe44-0000-0000-0000-000000000000',
|
||||
content: 'A remark',
|
||||
selection: 'титановый проводник',
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.value).toMatchObject({
|
||||
pageId: '019efe44-0000-0000-0000-000000000000',
|
||||
content: 'A remark',
|
||||
});
|
||||
});
|
||||
|
||||
it('sharedTool-built tools (getOutline) also get the friendly message on a dropped pageId', async () => {
|
||||
const tools = await buildTools();
|
||||
const result = await inputSchemaOf(tools.getOutline).validate({});
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.error?.message).toContain('parameter "pageId": missing (required)');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
} from './docmost-client.loader';
|
||||
import { resolveCurrentPageResult } from './current-page.util';
|
||||
import { parseNodeArg } from './parse-node-arg';
|
||||
import { modelFriendlyInput } from './model-friendly-input';
|
||||
|
||||
/**
|
||||
* Per-user, per-request adapter that exposes Docmost READ operations to the
|
||||
@@ -102,9 +103,13 @@ export class AiChatToolsService {
|
||||
): Tool =>
|
||||
tool({
|
||||
description: spec.description,
|
||||
inputSchema: spec.buildShape
|
||||
? z.object(spec.buildShape(z) as z.ZodRawShape)
|
||||
: z.object({}),
|
||||
// Wrap via modelFriendlyInput so a dropped/invalid parameter (e.g. a
|
||||
// pageId omitted in a parallel batch, #190) yields a clear, actionable
|
||||
// tool error instead of zod's raw text. No-arg specs still get an empty
|
||||
// object schema.
|
||||
inputSchema: modelFriendlyInput(
|
||||
spec.buildShape ? (spec.buildShape(z) as z.ZodRawShape) : {},
|
||||
),
|
||||
execute,
|
||||
});
|
||||
|
||||
@@ -118,7 +123,7 @@ export class AiChatToolsService {
|
||||
'and entities), not a full sentence. If the first results look weak ' +
|
||||
'or incomplete, search again with different wording or synonyms ' +
|
||||
'before answering.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
query: z.string().describe('The search query.'),
|
||||
limit: z
|
||||
.number()
|
||||
@@ -227,7 +232,7 @@ export class AiChatToolsService {
|
||||
'"the current page", or "here" refers to. Returns the page id and title, ' +
|
||||
'or null if the user is not currently on a page. Call this first whenever ' +
|
||||
'the user refers to the current page without giving an explicit id.',
|
||||
inputSchema: z.object({}),
|
||||
inputSchema: modelFriendlyInput({}),
|
||||
execute: async () => resolveCurrentPageResult(openedPage),
|
||||
}),
|
||||
|
||||
@@ -235,7 +240,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Fetch a single page as Markdown by its page id. Returns the page ' +
|
||||
'title and its Markdown content.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id (or slugId) of the page.'),
|
||||
}),
|
||||
execute: async ({ pageId }) => {
|
||||
@@ -259,7 +264,7 @@ export class AiChatToolsService {
|
||||
'Create a new page with a Markdown body in a space, optionally under ' +
|
||||
'a parent page. Returns the new page id and title. Reversible: a page ' +
|
||||
'can be moved to trash later.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
title: z.string().describe('The title of the new page.'),
|
||||
content: z
|
||||
.string()
|
||||
@@ -294,7 +299,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
"Replace a page's body with new Markdown content (and optionally its " +
|
||||
'title). Reversible: the previous version is kept in page history.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to update.'),
|
||||
content: z.string().describe('The new page body as Markdown.'),
|
||||
title: z
|
||||
@@ -316,7 +321,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
"Rename a page (change its title only; the body is untouched). " +
|
||||
'Reversible: rename back at any time.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to rename.'),
|
||||
title: z.string().describe('The new title.'),
|
||||
}),
|
||||
@@ -331,7 +336,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Move a page under a new parent page, or to the space root when no ' +
|
||||
'parent is given. Reversible: move it back at any time.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to move.'),
|
||||
parentPageId: z
|
||||
.string()
|
||||
@@ -353,7 +358,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Move a page to the trash (SOFT delete only — fully reversible; the ' +
|
||||
'page can be restored from trash). This NEVER permanently deletes.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to move to trash.'),
|
||||
}),
|
||||
// GUARDRAIL (§14 H4): the only field ever passed to the client is
|
||||
@@ -379,7 +384,7 @@ export class AiChatToolsService {
|
||||
'"selection not found" error, retry with a corrected EXACT selection ' +
|
||||
'copied verbatim from a single paragraph/block. Reversible via the ' +
|
||||
'comment UI.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to comment on.'),
|
||||
content: z.string().describe('The comment body as Markdown.'),
|
||||
selection: z
|
||||
@@ -428,7 +433,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Resolve or reopen a top-level comment thread (reversible — toggle ' +
|
||||
'the resolved flag). Only top-level comments can be resolved.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
commentId: z
|
||||
.string()
|
||||
.describe('The id of the top-level comment to resolve/reopen.'),
|
||||
@@ -460,7 +465,7 @@ export class AiChatToolsService {
|
||||
'List the most recent pages, optionally scoped to a single space. ' +
|
||||
'Returns a bounded list (default 50, max 100). Pass tree:true (with ' +
|
||||
"spaceId) to instead get the space's full page hierarchy as a nested tree.",
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
spaceId: z
|
||||
.string()
|
||||
.optional()
|
||||
@@ -488,7 +493,7 @@ export class AiChatToolsService {
|
||||
'List sidebar pages for a space. With no pageId, returns the ' +
|
||||
"space's ROOT pages; with a pageId, returns that page's direct " +
|
||||
'CHILDREN.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
spaceId: z.string().describe('The id of the space.'),
|
||||
pageId: z
|
||||
.string()
|
||||
@@ -520,7 +525,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Read a table as a matrix of cell texts (plus a parallel cellIds ' +
|
||||
'matrix so cells can be addressed for rich edits).',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page.'),
|
||||
tableRef: z
|
||||
.string()
|
||||
@@ -536,7 +541,7 @@ export class AiChatToolsService {
|
||||
listComments: tool({
|
||||
description:
|
||||
'List all comments on a page (content as Markdown).',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page.'),
|
||||
}),
|
||||
execute: async ({ pageId }) => await client.listComments(pageId),
|
||||
@@ -544,7 +549,7 @@ export class AiChatToolsService {
|
||||
|
||||
getComment: tool({
|
||||
description: 'Fetch a single comment by id (content as Markdown).',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
commentId: z.string().describe('The id of the comment.'),
|
||||
}),
|
||||
execute: async ({ commentId }) => await client.getComment(commentId),
|
||||
@@ -554,7 +559,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Find new comments across a space (optionally scoped to a subtree) ' +
|
||||
'created after a given timestamp.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
spaceId: z.string().describe('The id of the space to scan.'),
|
||||
since: z
|
||||
.string()
|
||||
@@ -586,7 +591,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Fetch a single page-history version including its lossless ' +
|
||||
'ProseMirror content.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
historyId: z.string().describe('The id of the history version.'),
|
||||
}),
|
||||
execute: async ({ historyId }) =>
|
||||
@@ -604,7 +609,7 @@ export class AiChatToolsService {
|
||||
'Export a page to a single self-contained Docmost-flavoured ' +
|
||||
'Markdown file (meta + body + comment threads). Lossless round-trip ' +
|
||||
'with importPageMarkdown.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to export.'),
|
||||
}),
|
||||
execute: async ({ pageId }) => {
|
||||
@@ -630,7 +635,7 @@ export class AiChatToolsService {
|
||||
'{"type":"text","text":"x","marks":[{"type":"bold"}]}. The node arg ' +
|
||||
'may be a JSON object or a JSON string (both accepted). Reversible: ' +
|
||||
'the previous version is kept in page history.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page.'),
|
||||
nodeId: z
|
||||
.string()
|
||||
@@ -663,7 +668,7 @@ export class AiChatToolsService {
|
||||
'{"type":"text","text":"x","marks":[{"type":"bold"}]}. The node arg ' +
|
||||
'may be a JSON object or a JSON string (both accepted). Reversible ' +
|
||||
'via page history.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page.'),
|
||||
node: z
|
||||
.any()
|
||||
@@ -722,7 +727,7 @@ export class AiChatToolsService {
|
||||
'object or a JSON string (both accepted). Omit content for a ' +
|
||||
'title-only update. Reversible: the previous version is kept in page ' +
|
||||
'history.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to update.'),
|
||||
content: z
|
||||
.any()
|
||||
@@ -753,7 +758,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Insert a row of plain-text cells into a table. Reversible via ' +
|
||||
'page history.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page.'),
|
||||
tableRef: z
|
||||
.string()
|
||||
@@ -772,7 +777,7 @@ export class AiChatToolsService {
|
||||
tableDeleteRow: tool({
|
||||
description:
|
||||
'Delete a table row at a 0-based index. Reversible via page history.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page.'),
|
||||
tableRef: z
|
||||
.string()
|
||||
@@ -787,7 +792,7 @@ export class AiChatToolsService {
|
||||
description:
|
||||
'Set the plain-text content of a table cell at [row, col] (0-based). ' +
|
||||
'Reversible via page history.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page.'),
|
||||
tableRef: z
|
||||
.string()
|
||||
@@ -817,7 +822,7 @@ export class AiChatToolsService {
|
||||
'Make a page PUBLICLY accessible and return its public URL. ' +
|
||||
'Reversible via unsharePage. Only share when the user explicitly ' +
|
||||
'asked, since this exposes the page to anyone with the link.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to share.'),
|
||||
searchIndexing: z
|
||||
.boolean()
|
||||
@@ -844,7 +849,7 @@ export class AiChatToolsService {
|
||||
"page's ProseMirror document for complex/scripted rewrites. dryRun " +
|
||||
'(default true) previews a diff WITHOUT writing; set dryRun:false to ' +
|
||||
'apply. Reversible: applying creates a new page-history snapshot.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z.string().describe('The id of the page to transform.'),
|
||||
transformJs: z
|
||||
.string()
|
||||
|
||||
101
apps/server/src/core/ai-chat/tools/model-friendly-input.spec.ts
Normal file
101
apps/server/src/core/ai-chat/tools/model-friendly-input.spec.ts
Normal file
@@ -0,0 +1,101 @@
|
||||
import { z } from 'zod';
|
||||
import {
|
||||
modelFriendlyInput,
|
||||
buildModelFriendlyMessage,
|
||||
} from './model-friendly-input';
|
||||
|
||||
/**
|
||||
* Unit tests for the centralized in-app tool input wrapper (#190). A dropped or
|
||||
* invalid parameter must surface a clear, model-actionable message (naming the
|
||||
* parameter and reminding the model not to drop ids in parallel batches), while
|
||||
* a valid call validates cleanly and strips unknown keys — and the advertised
|
||||
* JSON Schema keeps the unchanged required/description contract.
|
||||
*/
|
||||
describe('modelFriendlyInput', () => {
|
||||
// Mirrors createComment's shape: pageId is the required id the model drops in
|
||||
// parallel batches; selection is optional with a min length.
|
||||
const shape = {
|
||||
pageId: z.string().describe('The id of the page to comment on.'),
|
||||
content: z.string().describe('The comment body as Markdown.'),
|
||||
selection: z.string().min(1).max(250).optional(),
|
||||
};
|
||||
|
||||
// Loose return type: the AI SDK ValidationResult is a discriminated union, but
|
||||
// these tests assert on both branches, so a flat optional shape is simpler.
|
||||
async function validate(
|
||||
value: unknown,
|
||||
): Promise<{ success: boolean; value?: unknown; error?: Error }> {
|
||||
const schema = modelFriendlyInput(shape);
|
||||
return await schema.validate!(value);
|
||||
}
|
||||
|
||||
it('rejects a dropped required pageId with a clear, actionable message', async () => {
|
||||
const result = await validate({
|
||||
content: 'Looks off here',
|
||||
selection: 'титановый проводник',
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
const msg = result.error?.message ?? '';
|
||||
// Names the dropped parameter...
|
||||
expect(msg).toContain('parameter "pageId": missing (required)');
|
||||
// ...and gives an explicit, non-raw instruction (not zod's raw text).
|
||||
expect(msg).toContain('parallel/batch tool calls');
|
||||
expect(msg).not.toContain('expected string, received undefined');
|
||||
});
|
||||
|
||||
it('distinguishes a present-but-invalid parameter from a missing one', async () => {
|
||||
// selection is present but too short (invalid), pageId is missing.
|
||||
const result = await validate({ content: 'x', selection: '' });
|
||||
expect(result.success).toBe(false);
|
||||
const msg = result.error?.message ?? '';
|
||||
expect(msg).toContain('parameter "pageId": missing (required)');
|
||||
expect(msg).toContain('parameter "selection": invalid');
|
||||
});
|
||||
|
||||
it('accepts a valid call and strips unknown keys from the validated value', async () => {
|
||||
const result = await validate({
|
||||
pageId: 'page-1',
|
||||
content: 'A comment',
|
||||
selection: 'anchor text',
|
||||
bogus: true,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
if (!result.success) throw new Error('expected success');
|
||||
expect(result.value).toEqual({
|
||||
pageId: 'page-1',
|
||||
content: 'A comment',
|
||||
selection: 'anchor text',
|
||||
});
|
||||
expect(result.value).not.toHaveProperty('bogus');
|
||||
});
|
||||
|
||||
it('preserves the required/description contract in the advertised JSON Schema', async () => {
|
||||
const schema = modelFriendlyInput(shape);
|
||||
const json = (await schema.jsonSchema) as {
|
||||
required?: string[];
|
||||
properties?: Record<string, { description?: string }>;
|
||||
};
|
||||
// pageId + content stay required; selection stays optional.
|
||||
expect(json.required).toEqual(expect.arrayContaining(['pageId', 'content']));
|
||||
expect(json.required).not.toContain('selection');
|
||||
expect(json.properties?.pageId.description).toBe(
|
||||
'The id of the page to comment on.',
|
||||
);
|
||||
});
|
||||
|
||||
it('handles a no-arg tool (empty shape) without error', async () => {
|
||||
const schema = modelFriendlyInput({});
|
||||
const result = await schema.validate!({});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildModelFriendlyMessage', () => {
|
||||
it('falls back to a generic message when issues carry an empty path', () => {
|
||||
// safeParse on a non-object yields a root-level issue (empty path).
|
||||
const error = z.object({ a: z.string() }).safeParse('not-an-object');
|
||||
if (error.success) throw new Error('expected failure');
|
||||
const msg = buildModelFriendlyMessage(error.error, 'not-an-object');
|
||||
expect(msg).toContain('parameter "input"');
|
||||
});
|
||||
});
|
||||
93
apps/server/src/core/ai-chat/tools/model-friendly-input.ts
Normal file
93
apps/server/src/core/ai-chat/tools/model-friendly-input.ts
Normal file
@@ -0,0 +1,93 @@
|
||||
import { jsonSchema, type Schema } from 'ai';
|
||||
import type { JSONSchema7 } from '@ai-sdk/provider';
|
||||
import { z } from 'zod';
|
||||
|
||||
/**
|
||||
* Centralized input-schema wrapper for every in-app AI-chat tool.
|
||||
*
|
||||
* THE PROBLEM (#190): when the model issues PARALLEL / batch tool calls it
|
||||
* sometimes drops an "obvious" repeated required argument (typically `pageId`)
|
||||
* from some of the calls. zod v4 correctly rejects the missing value, but the
|
||||
* AI SDK forwards zod's RAW message ("Invalid input: expected string, received
|
||||
* undefined") straight back to the model, which is not actionable — the model
|
||||
* cannot tell WHICH parameter it dropped or that it must re-send it.
|
||||
*
|
||||
* THE FIX: keep the exact same validation, but replace the raw zod text with a
|
||||
* model-friendly message that names every problematic parameter and tells the
|
||||
* model to re-issue the call with all required parameters present. We do NOT
|
||||
* guess/backfill the value (a silently-assumed "current page" could comment on
|
||||
* the wrong page — cf. #159); the model is simply told to retry correctly.
|
||||
*
|
||||
* HOW IT WORKS: we build the tool's JSON Schema from the zod shape via
|
||||
* `z.toJSONSchema(..., { target: 'draft-7' })` (so the advertised contract —
|
||||
* `required` / `description` / field constraints — is unchanged) and hand the
|
||||
* AI SDK a custom `validate` that runs `z.object(shape).safeParse(value)`. On
|
||||
* failure the AI SDK wraps our returned `Error` in `InvalidToolInputError`, so
|
||||
* our clear text is what reaches the model as the tool error.
|
||||
*/
|
||||
export function modelFriendlyInput<T extends z.ZodRawShape>(
|
||||
shape: T,
|
||||
): Schema<z.output<z.ZodObject<T>>> {
|
||||
const objectSchema = z.object(shape);
|
||||
// draft-07 keeps required/description/constraints intact, matching what the
|
||||
// model already saw — the tool contract does not change.
|
||||
const json = z.toJSONSchema(objectSchema, {
|
||||
target: 'draft-7',
|
||||
}) as JSONSchema7;
|
||||
|
||||
return jsonSchema<z.output<z.ZodObject<T>>>(json, {
|
||||
validate: (value) => {
|
||||
const result = objectSchema.safeParse(value);
|
||||
if (result.success) {
|
||||
return { success: true, value: result.data };
|
||||
}
|
||||
return {
|
||||
success: false,
|
||||
error: new Error(buildModelFriendlyMessage(result.error, value)),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a zod validation failure into a clear, model-actionable message naming
|
||||
* each problematic parameter (and whether it is missing vs. invalid), plus an
|
||||
* explicit reminder not to drop required ids in parallel/batch tool calls.
|
||||
*/
|
||||
export function buildModelFriendlyMessage(
|
||||
error: z.ZodError,
|
||||
value: unknown,
|
||||
): string {
|
||||
const seen = new Set<string>();
|
||||
const parts: string[] = [];
|
||||
for (const issue of error.issues) {
|
||||
const name = issue.path.length ? issue.path.map(String).join('.') : 'input';
|
||||
// A parameter the model omitted entirely reads as `undefined` at its path;
|
||||
// anything else is present-but-invalid (wrong type, too short, etc.).
|
||||
const missing = valueAtPath(value, issue.path) === undefined;
|
||||
const part = `parameter "${name}": ${missing ? 'missing (required)' : 'invalid'}`;
|
||||
if (seen.has(part)) continue;
|
||||
seen.add(part);
|
||||
parts.push(part);
|
||||
}
|
||||
if (parts.length === 0) {
|
||||
// Defensive: a ZodError always has issues, but never emit an empty list.
|
||||
parts.push('input: invalid');
|
||||
}
|
||||
return (
|
||||
`Invalid input for this tool — ${parts.join('; ')}. ` +
|
||||
'Re-issue the call with EVERY required parameter present and valid. ' +
|
||||
"Do not drop ids like pageId, even when making parallel/batch tool calls — " +
|
||||
'each tool call must carry its own pageId.'
|
||||
);
|
||||
}
|
||||
|
||||
/** Read the value at a zod issue path; returns undefined if any hop is absent. */
|
||||
function valueAtPath(value: unknown, path: ReadonlyArray<PropertyKey>): unknown {
|
||||
let current: unknown = value;
|
||||
for (const key of path) {
|
||||
if (current === null || typeof current !== 'object') return undefined;
|
||||
current = (current as Record<PropertyKey, unknown>)[key];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { ShareService } from '../../share/share.service';
|
||||
import { SearchService } from '../../search/search.service';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { jsonToMarkdown } from '../../../collaboration/collaboration.util';
|
||||
import { modelFriendlyInput } from './model-friendly-input';
|
||||
|
||||
/**
|
||||
* Isolated, READ-ONLY toolset for the ANONYMOUS public-share assistant.
|
||||
@@ -52,7 +53,7 @@ export class PublicShareChatToolsService {
|
||||
'(key terms and entities), not a full sentence. If the first ' +
|
||||
'results look weak, search again with different wording before ' +
|
||||
'answering. Only pages inside this share are ever returned.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
query: z.string().describe('The search query.'),
|
||||
limit: z
|
||||
.number()
|
||||
@@ -87,7 +88,7 @@ export class PublicShareChatToolsService {
|
||||
'Markdown, by its page id. Returns the page title and its Markdown ' +
|
||||
'content. Only pages inside this share can be read; reading any ' +
|
||||
'other page fails.',
|
||||
inputSchema: z.object({
|
||||
inputSchema: modelFriendlyInput({
|
||||
pageId: z
|
||||
.string()
|
||||
.describe('The id (or slugId) of a page within this share.'),
|
||||
@@ -142,7 +143,7 @@ export class PublicShareChatToolsService {
|
||||
'List the pages (titles + ids) that make up THIS published ' +
|
||||
'documentation share, so you can orient yourself before reading or ' +
|
||||
'searching. Only pages inside this share are listed.',
|
||||
inputSchema: z.object({}),
|
||||
inputSchema: modelFriendlyInput({}),
|
||||
execute: async () => {
|
||||
// Reuse the same share-tree logic the public /shares/tree route uses:
|
||||
// it validates the share + workspace, excludes restricted subtrees,
|
||||
|
||||
@@ -57,11 +57,28 @@ describe('PageService', () => {
|
||||
|
||||
const eventEmitter = { emit: jest.fn() };
|
||||
|
||||
// movePage now runs the cycle-check + UPDATE inside executeTx(this.db),
|
||||
// i.e. this.db.transaction().execute(fn => fn(trx)). A permissive chainable
|
||||
// Proxy stands in for the Kysely trx so the per-space advisory-lock
|
||||
// `sql``.execute(trx)` resolves; a thrown BadRequestException still
|
||||
// propagates out of the transaction unchanged.
|
||||
const trxStub: any = new Proxy(function () {}, {
|
||||
get: (_t, p) =>
|
||||
p === 'then'
|
||||
? undefined
|
||||
: p === 'execute' || p === 'executeTakeFirst'
|
||||
? () => Promise.resolve([])
|
||||
: () => trxStub,
|
||||
});
|
||||
const db = {
|
||||
transaction: () => ({ execute: (fn: any) => fn(trxStub) }),
|
||||
};
|
||||
|
||||
const svc = new PageService(
|
||||
pageRepo as any, // pageRepo
|
||||
{} as any, // pagePermissionRepo
|
||||
{} as any, // attachmentRepo
|
||||
{} as any, // db
|
||||
db as any, // db
|
||||
{} as any, // storageService
|
||||
{} as any, // attachmentQueue
|
||||
{} as any, // aiQueue
|
||||
@@ -268,9 +285,23 @@ describe('PageService', () => {
|
||||
}),
|
||||
updatePage: jest.fn().mockResolvedValue({ numUpdatedRows: 1n }),
|
||||
};
|
||||
// movePage now runs the cycle-check + UPDATE inside executeTx(this.db),
|
||||
// which calls this.db.transaction().execute(fn => fn(trx)). A permissive
|
||||
// chainable Proxy stands in for the Kysely trx so the per-space
|
||||
// advisory-lock `sql``.execute(trx)` resolves and updatePage receives it.
|
||||
const trxStub: any = new Proxy(function () {}, {
|
||||
get: (_t, p) =>
|
||||
p === 'then'
|
||||
? undefined
|
||||
: p === 'execute' || p === 'executeTakeFirst'
|
||||
? () => Promise.resolve([])
|
||||
: () => trxStub,
|
||||
});
|
||||
const svc = makeSvc({
|
||||
pageRepo,
|
||||
db: {} as any,
|
||||
db: {
|
||||
transaction: () => ({ execute: (fn: any) => fn(trxStub) }),
|
||||
} as any,
|
||||
});
|
||||
// Legitimate move: destination ancestors do NOT include the moved page.
|
||||
jest
|
||||
|
||||
@@ -15,13 +15,13 @@ import {
|
||||
executeWithCursorPagination,
|
||||
} from '@docmost/db/pagination/cursor-pagination';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { MovePageDto } from '../dto/move-page.dto';
|
||||
import { shapeSidebarPagesTree } from './sidebar-pages-tree.util';
|
||||
import { generateSlugId } from '../../../common/helpers';
|
||||
import { getPageTitle } from '../../../common/helpers';
|
||||
import { executeTx } from '@docmost/db/utils';
|
||||
import { dbOrTx, executeTx } from '@docmost/db/utils';
|
||||
import { AttachmentRepo } from '@docmost/db/repos/attachment/attachment.repo';
|
||||
import { v7 as uuid7 } from 'uuid';
|
||||
import {
|
||||
@@ -62,6 +62,23 @@ import {
|
||||
agentSourceFields,
|
||||
} from '../../../common/decorators/auth-provenance.decorator';
|
||||
|
||||
// Hard upper bound on how deep the recursive page-tree CTEs (ancestor /
|
||||
// descendant traversals) may walk. Real page trees are only a handful of levels
|
||||
// deep, so this cap never truncates a legitimate result; it purely defends the
|
||||
// recursive CTEs against runaway iteration if a parent/child cycle ever exists
|
||||
// in the data (e.g. one slipped in before the move guard, #207 #8). Without it a
|
||||
// cycle makes `withRecursive` loop forever (hang / statement timeout), and the
|
||||
// move guard itself calls one of these CTEs — so a cycle would disable the very
|
||||
// guard meant to prevent it. Each CTE carries a depth counter and stops here.
|
||||
const MAX_PAGE_TREE_DEPTH = 10_000;
|
||||
|
||||
// Advisory-lock namespace (the first key of pg_advisory_xact_lock) used to
|
||||
// serialize concurrent page moves within a single space so the cycle check and
|
||||
// the move UPDATE stay atomic (see movePage, #207 #7). A dedicated namespace
|
||||
// constant keeps these locks from colliding with any other advisory lock; the
|
||||
// second key is hashtext(spaceId). Fits a signed int4 ('page' in ASCII).
|
||||
const PAGE_MOVE_LOCK_NAMESPACE = 0x70616765;
|
||||
|
||||
@Injectable()
|
||||
export class PageService {
|
||||
private readonly logger = new Logger(PageService.name);
|
||||
@@ -601,7 +618,13 @@ export class PageService {
|
||||
slugIdMap.set(entry.oldSlugId, entry);
|
||||
}
|
||||
|
||||
const attachmentMap = new Map<string, ICopyPageAttachment>();
|
||||
// Keyed by old attachmentId. A single attachment can be referenced by more
|
||||
// than one page in the copied subtree (e.g. a block copy-pasted into a child
|
||||
// page keeps the same attachmentId). Each referencing page needs its own
|
||||
// fresh attachment id / row / blob copy, so the value is a LIST of copy
|
||||
// entries rather than a single one — otherwise the last page's entry would
|
||||
// clobber the others and their images would 404 in the copies (#206 attach-1).
|
||||
const attachmentMap = new Map<string, ICopyPageAttachment[]>();
|
||||
|
||||
const insertablePages: InsertablePage[] = await Promise.all(
|
||||
pages.map(async (page) => {
|
||||
@@ -617,12 +640,14 @@ export class PageService {
|
||||
attachmentIds.forEach((attachmentId: string) => {
|
||||
const newPageId = pageFromMap.newPageId;
|
||||
const newAttachmentId = uuid7();
|
||||
attachmentMap.set(attachmentId, {
|
||||
const existingEntries = attachmentMap.get(attachmentId) ?? [];
|
||||
existingEntries.push({
|
||||
newPageId: newPageId,
|
||||
oldPageId: page.id,
|
||||
oldAttachmentId: attachmentId,
|
||||
newAttachmentId: newAttachmentId,
|
||||
});
|
||||
attachmentMap.set(attachmentId, existingEntries);
|
||||
|
||||
prosemirrorDoc.descendants((node: PMNode) => {
|
||||
if (isAttachmentNode(node.type.name)) {
|
||||
@@ -819,51 +844,53 @@ export class PageService {
|
||||
.execute();
|
||||
|
||||
for (const attachment of attachments) {
|
||||
try {
|
||||
const pageAttachment = attachmentMap.get(attachment.id);
|
||||
|
||||
// make sure the copied attachment belongs to the page it was copied from
|
||||
if (attachment.pageId !== pageAttachment.oldPageId) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const newAttachmentId = pageAttachment.newAttachmentId;
|
||||
|
||||
const newPageId = pageAttachment.newPageId;
|
||||
|
||||
const newPathFile = attachment.filePath.replace(
|
||||
attachment.id,
|
||||
newAttachmentId,
|
||||
);
|
||||
|
||||
// One source attachment may need to be copied for several destination
|
||||
// pages (it is referenced by more than one page in the subtree). Copy a
|
||||
// distinct blob + row for every referencing page so each copy resolves
|
||||
// (#206 attach-1). The old per-page ownership guard is gone: when the
|
||||
// same attachmentId is shared, only one page would ever match the row's
|
||||
// pageId, silently dropping the other copies.
|
||||
const pageAttachments = attachmentMap.get(attachment.id) ?? [];
|
||||
for (const pageAttachment of pageAttachments) {
|
||||
try {
|
||||
await this.storageService.copy(attachment.filePath, newPathFile);
|
||||
const newAttachmentId = pageAttachment.newAttachmentId;
|
||||
|
||||
await this.db
|
||||
.insertInto('attachments')
|
||||
.values({
|
||||
id: newAttachmentId,
|
||||
type: attachment.type,
|
||||
filePath: newPathFile,
|
||||
fileName: attachment.fileName,
|
||||
fileSize: attachment.fileSize,
|
||||
mimeType: attachment.mimeType,
|
||||
fileExt: attachment.fileExt,
|
||||
creatorId: attachment.creatorId,
|
||||
workspaceId: attachment.workspaceId,
|
||||
pageId: newPageId,
|
||||
spaceId: spaceId,
|
||||
})
|
||||
.execute();
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Duplicate page: failed to copy attachment ${attachment.id}`,
|
||||
err,
|
||||
const newPageId = pageAttachment.newPageId;
|
||||
|
||||
const newPathFile = attachment.filePath.replace(
|
||||
attachment.id,
|
||||
newAttachmentId,
|
||||
);
|
||||
// Continue with other attachments even if one fails
|
||||
|
||||
try {
|
||||
await this.storageService.copy(attachment.filePath, newPathFile);
|
||||
|
||||
await this.db
|
||||
.insertInto('attachments')
|
||||
.values({
|
||||
id: newAttachmentId,
|
||||
type: attachment.type,
|
||||
filePath: newPathFile,
|
||||
fileName: attachment.fileName,
|
||||
fileSize: attachment.fileSize,
|
||||
mimeType: attachment.mimeType,
|
||||
fileExt: attachment.fileExt,
|
||||
creatorId: attachment.creatorId,
|
||||
workspaceId: attachment.workspaceId,
|
||||
pageId: newPageId,
|
||||
spaceId: spaceId,
|
||||
})
|
||||
.execute();
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Duplicate page: failed to copy attachment ${attachment.id}`,
|
||||
err,
|
||||
);
|
||||
// Continue with other attachments even if one fails
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(err);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -915,34 +942,61 @@ export class PageService {
|
||||
}
|
||||
}
|
||||
|
||||
// Server-side cycle guard: a page may not be moved into itself or into any
|
||||
// page within its own subtree. Without this, an MCP/REST/agent caller (or a
|
||||
// fast drag racing the client check) could persist a cycle and broadcast it.
|
||||
// Only relevant when re-parenting under a concrete parent; moving to root
|
||||
// (parentPageId null/undefined) can never create a cycle.
|
||||
if (dto.parentPageId) {
|
||||
if (dto.parentPageId === dto.pageId) {
|
||||
throw new BadRequestException('Cannot move a page into its own subtree');
|
||||
}
|
||||
// Walk the destination parent's ancestor chain (reusing the breadcrumb
|
||||
// ancestor CTE). If the page being moved appears among those ancestors,
|
||||
// the destination lives inside the moved page's subtree -> cycle.
|
||||
const destAncestors = await this.getPageBreadCrumbs(dto.parentPageId);
|
||||
if (destAncestors.some((ancestor) => ancestor.id === dto.pageId)) {
|
||||
throw new BadRequestException('Cannot move a page into its own subtree');
|
||||
}
|
||||
}
|
||||
// Server-side cycle guard + the move UPDATE run in ONE transaction. A page
|
||||
// may not be moved into itself or into any page within its own subtree;
|
||||
// without this an MCP/REST/agent caller (or a fast drag racing the client
|
||||
// check) could persist a cycle and broadcast it. Crucially, doing the guard
|
||||
// and the write as two separate, unlocked statements is a TOCTOU race: two
|
||||
// concurrent moves ("A under B" and "B under A") can each read the same
|
||||
// pre-write acyclic snapshot, both pass the guard, then persist
|
||||
// A.parentPageId=B AND B.parentPageId=A — a parent/child cycle (#207 #7). A
|
||||
// per-space advisory lock (held until COMMIT) serializes all moves within a
|
||||
// space: the second mover blocks until the first commits and then sees the
|
||||
// freshly written parent, so its guard rejects the cycle.
|
||||
const updateResult = await executeTx(this.db, async (trx) => {
|
||||
await sql`select pg_advisory_xact_lock(${sql.lit(
|
||||
PAGE_MOVE_LOCK_NAMESPACE,
|
||||
)}, hashtext(${movedPage.spaceId}))`.execute(trx);
|
||||
|
||||
const updateResult = await this.pageRepo.updatePage(
|
||||
{
|
||||
position: dto.position,
|
||||
parentPageId: parentPageId,
|
||||
// Agent-edit provenance: annotate the source on an agent move. A normal
|
||||
// user request leaves the existing source value unchanged.
|
||||
...agentSourceFields(provenance, 'lastUpdatedSource', 'lastUpdatedAiChatId'),
|
||||
},
|
||||
dto.pageId,
|
||||
);
|
||||
// Only relevant when re-parenting under a concrete parent; moving to root
|
||||
// (parentPageId null/undefined) can never create a cycle.
|
||||
if (dto.parentPageId) {
|
||||
if (dto.parentPageId === dto.pageId) {
|
||||
throw new BadRequestException(
|
||||
'Cannot move a page into its own subtree',
|
||||
);
|
||||
}
|
||||
// Walk the destination parent's ancestor chain (reusing the breadcrumb
|
||||
// ancestor CTE) inside the lock. If the page being moved appears among
|
||||
// those ancestors, the destination lives inside the moved page's
|
||||
// subtree -> cycle.
|
||||
const destAncestors = await this.getPageBreadCrumbs(
|
||||
dto.parentPageId,
|
||||
trx,
|
||||
);
|
||||
if (destAncestors.some((ancestor) => ancestor.id === dto.pageId)) {
|
||||
throw new BadRequestException(
|
||||
'Cannot move a page into its own subtree',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return this.pageRepo.updatePage(
|
||||
{
|
||||
position: dto.position,
|
||||
parentPageId: parentPageId,
|
||||
// Agent-edit provenance: annotate the source on an agent move. A
|
||||
// normal user request leaves the existing source value unchanged.
|
||||
...agentSourceFields(
|
||||
provenance,
|
||||
'lastUpdatedSource',
|
||||
'lastUpdatedAiChatId',
|
||||
),
|
||||
},
|
||||
dto.pageId,
|
||||
trx,
|
||||
);
|
||||
});
|
||||
|
||||
// Guard against a phantom broadcast: if the row was concurrently deleted or
|
||||
// otherwise not updated, skip the PAGE_MOVED event so we don't replay a move
|
||||
@@ -981,8 +1035,8 @@ export class PageService {
|
||||
});
|
||||
}
|
||||
|
||||
async getPageBreadCrumbs(childPageId: string) {
|
||||
const ancestors = await this.db
|
||||
async getPageBreadCrumbs(childPageId: string, trx?: KyselyTransaction) {
|
||||
const ancestors = await dbOrTx(this.db, trx)
|
||||
.withRecursive('page_ancestors', (db) =>
|
||||
db
|
||||
.selectFrom('pages')
|
||||
@@ -996,6 +1050,9 @@ export class PageService {
|
||||
'spaceId',
|
||||
'deletedAt',
|
||||
])
|
||||
// Depth counter: bounds the walk so a parent/child cycle in the data
|
||||
// can't make this recursive CTE loop forever (#207 #8).
|
||||
.select(sql<number>`0`.as('depth'))
|
||||
.where('id', '=', childPageId)
|
||||
.where('deletedAt', 'is', null)
|
||||
.unionAll((exp) =>
|
||||
@@ -1011,12 +1068,25 @@ export class PageService {
|
||||
'p.spaceId',
|
||||
'p.deletedAt',
|
||||
])
|
||||
.select(sql<number>`pa.depth + 1`.as('depth'))
|
||||
.innerJoin('page_ancestors as pa', 'pa.parentPageId', 'p.id')
|
||||
.where('p.deletedAt', 'is', null),
|
||||
.where('p.deletedAt', 'is', null)
|
||||
.where(sql<number>`pa.depth`, '<', MAX_PAGE_TREE_DEPTH),
|
||||
),
|
||||
)
|
||||
.selectFrom('page_ancestors')
|
||||
.selectAll('page_ancestors')
|
||||
// Explicit column list (not selectAll) so the internal `depth` counter
|
||||
// never leaks into the breadcrumb result shape.
|
||||
.select([
|
||||
'id',
|
||||
'slugId',
|
||||
'title',
|
||||
'icon',
|
||||
'position',
|
||||
'parentPageId',
|
||||
'spaceId',
|
||||
'deletedAt',
|
||||
])
|
||||
.select((eb) =>
|
||||
eb
|
||||
.exists(
|
||||
@@ -1137,16 +1207,21 @@ export class PageService {
|
||||
db
|
||||
.selectFrom('pages')
|
||||
.select(['id'])
|
||||
// Depth counter: bounds the walk so a parent/child cycle in the data
|
||||
// can't make this recursive CTE loop forever (#207 #8).
|
||||
.select(sql<number>`0`.as('depth'))
|
||||
.where('id', '=', pageId)
|
||||
.unionAll((exp) =>
|
||||
exp
|
||||
.selectFrom('pages as p')
|
||||
.select(['p.id'])
|
||||
.innerJoin('page_descendants as pd', 'pd.id', 'p.parentPageId'),
|
||||
.select(sql<number>`pd.depth + 1`.as('depth'))
|
||||
.innerJoin('page_descendants as pd', 'pd.id', 'p.parentPageId')
|
||||
.where(sql<number>`pd.depth`, '<', MAX_PAGE_TREE_DEPTH),
|
||||
),
|
||||
)
|
||||
.selectFrom('page_descendants')
|
||||
.selectAll()
|
||||
.select(['id'])
|
||||
.execute();
|
||||
|
||||
const pageIds = descendants.map((d) => d.id);
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { type Kysely } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
// Step-granular durability for the assistant turn (#183). The assistant row is
|
||||
// now created UPFRONT (status 'streaming') and UPDATEd as each step completes,
|
||||
// so a process death mid-turn no longer loses the whole answer. The column is
|
||||
// NULLABLE on purpose: rows written before this migration carry NULL, which the
|
||||
// app treats as 'completed' (a settled, pre-status message). Values written by
|
||||
// the app: 'streaming' | 'completed' | 'error' | 'aborted'.
|
||||
await db.schema
|
||||
.alterTable('ai_chat_messages')
|
||||
.addColumn('status', 'text', (col) => col)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.alterTable('ai_chat_messages').dropColumn('status').execute();
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '../../types/kysely.types';
|
||||
import { dbOrTx, jsonbBind } from '../../utils';
|
||||
import { dbOrTx, jsonbBind, parseJsonbValue } from '../../utils';
|
||||
import { AiAgentRole } from '@docmost/db/types/entity.types';
|
||||
|
||||
/** The jsonb shape persisted in `model_config` (loosely typed for the column). */
|
||||
@@ -183,17 +183,13 @@ export class AiAgentRoleRepo {
|
||||
export function parseModelConfig(
|
||||
value: unknown,
|
||||
): Record<string, unknown> | null {
|
||||
let v: unknown = value;
|
||||
if (typeof v === 'string') {
|
||||
try {
|
||||
v = JSON.parse(v); // legacy double-encoded read
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return v !== null && typeof v === 'object' && !Array.isArray(v)
|
||||
? (v as Record<string, unknown>)
|
||||
: null;
|
||||
// Shape guard only; the legacy double-encoding self-heal lives in
|
||||
// parseJsonbValue (database/utils.ts).
|
||||
return parseJsonbValue(
|
||||
value,
|
||||
(v): v is Record<string, unknown> =>
|
||||
v !== null && typeof v === 'object' && !Array.isArray(v),
|
||||
);
|
||||
}
|
||||
|
||||
/** Normalize a DB row so `modelConfig` is always an object or null. The cast
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '../../types/kysely.types';
|
||||
import { dbOrTx } from '../../utils';
|
||||
@@ -9,8 +9,24 @@ import {
|
||||
import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
|
||||
import { executeWithCursorPagination } from '@docmost/db/pagination/cursor-pagination';
|
||||
|
||||
// Crash-recovery sweep recency threshold (#183 review): a 'streaming' row is
|
||||
// only swept to 'aborted' once it has been UNTOUCHED for this long. A live turn
|
||||
// bumps `updatedAt` on every step (well under this window), so its row never
|
||||
// matches; only a turn whose process truly died (no step update for >threshold)
|
||||
// is swept. Chosen safely ABOVE the longest realistic turn so a fresh replica's
|
||||
// boot-sweep can never abort a turn another replica is actively streaming
|
||||
// (multi-instance deploy).
|
||||
const SWEEP_STREAMING_STALE_MS = 10 * 60 * 1000; // 10 minutes
|
||||
|
||||
// Hard upper bound on the rows materialized by `findAllByChat` (export path).
|
||||
// A generous cap so a pathologically huge chat cannot load an unbounded result
|
||||
// into memory; far above any realistic transcript length.
|
||||
const FIND_ALL_BY_CHAT_LIMIT = 5000;
|
||||
|
||||
@Injectable()
|
||||
export class AiChatMessageRepo {
|
||||
private readonly logger = new Logger(AiChatMessageRepo.name);
|
||||
|
||||
constructor(@InjectKysely() private readonly db: KyselyDB) {}
|
||||
|
||||
// The `tsv` column is a trigger-maintained tsvector used only for
|
||||
@@ -25,6 +41,7 @@ export class AiChatMessageRepo {
|
||||
'content',
|
||||
'toolCalls',
|
||||
'metadata',
|
||||
'status',
|
||||
'createdAt',
|
||||
'updatedAt',
|
||||
'deletedAt',
|
||||
@@ -60,6 +77,46 @@ export class AiChatMessageRepo {
|
||||
});
|
||||
}
|
||||
|
||||
// Load ALL (non-deleted) messages of a chat in ascending chronological order
|
||||
// (oldest -> newest), unpaginated. Used by the server-side Markdown export
|
||||
// (#183), where the DB is the single source of truth and the whole transcript
|
||||
// must be rendered in one pass (findByChat is cursor-paginated and would only
|
||||
// return the first page).
|
||||
//
|
||||
// Hard-capped at FIND_ALL_BY_CHAT_LIMIT rows (a generous bound, far above any
|
||||
// realistic transcript) so exporting a pathologically huge chat cannot
|
||||
// materialize an unbounded result set in memory.
|
||||
async findAllByChat(
|
||||
chatId: string,
|
||||
workspaceId: string,
|
||||
// Injectable for tests so truncation can be exercised on a modest volume.
|
||||
limit: number = FIND_ALL_BY_CHAT_LIMIT,
|
||||
): Promise<AiChatMessage[]> {
|
||||
// Fetch newest-first (+1 to DETECT truncation), so on overflow we keep the
|
||||
// NEWEST `limit` messages — the recent conversation matters most for an
|
||||
// export — rather than silently dropping the tail (#183 review). Reverse back
|
||||
// to chronological for rendering, like findRecent.
|
||||
const rows = await this.db
|
||||
.selectFrom('aiChatMessages')
|
||||
.select(this.baseFields)
|
||||
.where('chatId', '=', chatId)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.where('deletedAt', 'is', null)
|
||||
.orderBy('createdAt', 'desc')
|
||||
.orderBy('id', 'desc')
|
||||
.limit(limit + 1)
|
||||
.execute();
|
||||
|
||||
if (rows.length > limit) {
|
||||
rows.length = limit; // keep the newest `limit` (rows are newest-first here)
|
||||
this.logger.warn(
|
||||
`Chat ${chatId} export truncated to the newest ${limit} messages ` +
|
||||
`(older messages omitted).`,
|
||||
);
|
||||
}
|
||||
return rows.reverse();
|
||||
}
|
||||
|
||||
// Load the most RECENT `limit` messages for a chat and return them in
|
||||
// ascending chronological order (oldest -> newest), as the model expects.
|
||||
// `findByChat` returns the FIRST page ASC (the OLDEST messages), which loses
|
||||
@@ -96,4 +153,68 @@ export class AiChatMessageRepo {
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a single message in place by id + workspace (#183 step-granular
|
||||
* durability). The assistant row is created UPFRONT (status 'streaming') and
|
||||
* patched as each step completes, then finalized once on the terminal status.
|
||||
* `updatedAt` is always bumped. Returns the updated row (baseFields) or
|
||||
* undefined when no row matched (e.g. a foreign workspace / deleted row).
|
||||
*/
|
||||
async update(
|
||||
id: string,
|
||||
workspaceId: string,
|
||||
patch: Partial<{
|
||||
content: string | null;
|
||||
toolCalls: unknown;
|
||||
metadata: unknown;
|
||||
status: string | null;
|
||||
}>,
|
||||
opts?: { onlyIfStreaming?: boolean; trx?: KyselyTransaction },
|
||||
): Promise<AiChatMessage | undefined> {
|
||||
const db = dbOrTx(this.db, opts?.trx);
|
||||
let query = db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({ ...(patch as Record<string, unknown>), updatedAt: new Date() })
|
||||
.where('id', '=', id)
|
||||
.where('workspaceId', '=', workspaceId);
|
||||
// Concurrency guard (#183 review): a per-step 'streaming' update must NEVER
|
||||
// overwrite a row the terminal callback already finalized. onStepFinish
|
||||
// fires the streaming update fire-and-forget, so its UPDATE can land AFTER
|
||||
// finalize on a DIFFERENT pool connection (commit order is not guaranteed).
|
||||
// Scoping the streaming update to rows STILL in 'streaming' makes a late
|
||||
// update a no-op once the row is completed/error/aborted — regardless of
|
||||
// commit order. The terminal finalize runs WITHOUT this guard so it always
|
||||
// wins.
|
||||
if (opts?.onlyIfStreaming) {
|
||||
query = query.where('status', '=', 'streaming');
|
||||
}
|
||||
return query.returning(this.baseFields).executeTakeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* Crash-recovery sweep (#183): flip every assistant row still left in the
|
||||
* 'streaming' state (a turn that died mid-write before reaching a terminal
|
||||
* status) to 'aborted'. Run once on server start. Returns the number of rows
|
||||
* swept so the caller can log it. Workspace-wide on purpose — a crash can have
|
||||
* dangling streaming rows across any workspace.
|
||||
*
|
||||
* Bounded by recency (#183 review): only rows UNTOUCHED for
|
||||
* SWEEP_STREAMING_STALE_MS are swept. A live turn bumps `updatedAt` on every
|
||||
* step, so an actively-streaming row never matches; this prevents a fresh
|
||||
* replica's boot-sweep from aborting a turn another replica is still streaming
|
||||
* in a multi-instance deploy.
|
||||
*/
|
||||
async sweepStreaming(trx?: KyselyTransaction): Promise<number> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
const staleBefore = new Date(Date.now() - SWEEP_STREAMING_STALE_MS);
|
||||
const rows = await db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({ status: 'aborted', updatedAt: new Date() })
|
||||
.where('status', '=', 'streaming')
|
||||
.where('updatedAt', '<', staleBefore)
|
||||
.returning('id')
|
||||
.execute();
|
||||
return rows.length;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '../../types/kysely.types';
|
||||
import { dbOrTx, jsonbBind } from '../../utils';
|
||||
import { dbOrTx, jsonbBind, parseJsonbValue } from '../../utils';
|
||||
import { AiMcpServer } from '@docmost/db/types/entity.types';
|
||||
|
||||
const logger = new Logger('AiMcpServerRepo');
|
||||
@@ -161,17 +161,13 @@ export function blankToNull(value: string | null | undefined): string | null {
|
||||
* array with a non-string element all become null (unrestricted).
|
||||
*/
|
||||
export function parseToolAllowlist(value: unknown): string[] | null {
|
||||
let v: unknown = value;
|
||||
if (typeof v === 'string') {
|
||||
try {
|
||||
v = JSON.parse(v); // legacy double-encoded read
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return Array.isArray(v) && v.every((x) => typeof x === 'string')
|
||||
? (v as string[])
|
||||
: null;
|
||||
// Shape guard only; the legacy double-encoding self-heal lives in
|
||||
// parseJsonbValue (database/utils.ts).
|
||||
return parseJsonbValue(
|
||||
value,
|
||||
(v): v is string[] =>
|
||||
Array.isArray(v) && v.every((x) => typeof x === 'string'),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -20,6 +20,7 @@ import { DB, Workspaces } from '@docmost/db/types/db';
|
||||
export const AI_PROVIDER_SETTINGS_ALLOWED: readonly string[] = [
|
||||
'driver',
|
||||
'chatModel',
|
||||
'chatContextWindow',
|
||||
'chatApiStyle',
|
||||
'embeddingModel',
|
||||
'baseUrl',
|
||||
|
||||
4
apps/server/src/database/types/db.d.ts
vendored
4
apps/server/src/database/types/db.d.ts
vendored
@@ -620,6 +620,10 @@ export interface AiChatMessages {
|
||||
content: string | null;
|
||||
toolCalls: Json | null;
|
||||
metadata: Json | null;
|
||||
// Turn lifecycle status (#183): 'streaming' | 'completed' | 'error' |
|
||||
// 'aborted'. NULL on rows written before the status column existed; the app
|
||||
// treats NULL as 'completed' (a settled, pre-status message).
|
||||
status: string | null;
|
||||
tsv: string | null;
|
||||
createdAt: Generated<Timestamp>;
|
||||
updatedAt: Generated<Timestamp>;
|
||||
|
||||
@@ -64,3 +64,29 @@ export function jsonbBind<T>(
|
||||
}
|
||||
return sql<T>`${JSON.stringify(value)}::text::jsonb`;
|
||||
}
|
||||
|
||||
/**
|
||||
* READ-side counterpart to {@link jsonbBind}: tolerantly decode a jsonb value
|
||||
* read back from the DB and validate its shape with `guard`. THE single place
|
||||
* the legacy double-encoding self-heal lives, so repos keep only a type-guard.
|
||||
*
|
||||
* A row written by the old `::jsonb` bind round-trips as a JSON STRING (see the
|
||||
* quirk in jsonbBind), so the driver hands back e.g. `'["a"]'` / `'{"k":1}'`
|
||||
* rather than the structure. This parses such a string once, then applies the
|
||||
* caller's `guard`. Returns `null` for null / an unparseable string / a value
|
||||
* the guard rejects (so a corrupt or wrong-shaped value degrades to "unset").
|
||||
*/
|
||||
export function parseJsonbValue<T>(
|
||||
value: unknown,
|
||||
guard: (v: unknown) => v is T,
|
||||
): T | null {
|
||||
let v: unknown = value;
|
||||
if (typeof v === 'string') {
|
||||
try {
|
||||
v = JSON.parse(v); // legacy double-encoded read
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return guard(v) ? v : null;
|
||||
}
|
||||
|
||||
@@ -41,3 +41,35 @@ describe('UpdateAiSettingsDto.chatApiStyle', () => {
|
||||
expect(errs.find((e) => e.property === 'chatApiStyle')).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
/** DTO validation for the new chatContextWindow field (@IsInt @Min(0)). */
|
||||
describe('UpdateAiSettingsDto.chatContextWindow', () => {
|
||||
const errorsFor = async (chatContextWindow: unknown) =>
|
||||
validate(plainToInstance(UpdateAiSettingsDto, { chatContextWindow }));
|
||||
|
||||
it('accepts a non-negative integer (incl. 0 = clear the limit)', async () => {
|
||||
for (const v of [0, 200000]) {
|
||||
const errs = await errorsFor(v);
|
||||
expect(
|
||||
errs.find((e) => e.property === 'chatContextWindow'),
|
||||
).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects a negative value', async () => {
|
||||
const errs = await errorsFor(-1);
|
||||
expect(errs.find((e) => e.property === 'chatContextWindow')).toBeDefined();
|
||||
});
|
||||
|
||||
it('rejects a non-integer value', async () => {
|
||||
const errs = await errorsFor(1.5);
|
||||
expect(errs.find((e) => e.property === 'chatContextWindow')).toBeDefined();
|
||||
});
|
||||
|
||||
it('accepts the field being omitted (optional)', async () => {
|
||||
const errs = await validate(plainToInstance(UpdateAiSettingsDto, {}));
|
||||
expect(
|
||||
errs.find((e) => e.property === 'chatContextWindow'),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
43
apps/server/src/integrations/ai/ai-settings.service.spec.ts
Normal file
43
apps/server/src/integrations/ai/ai-settings.service.spec.ts
Normal file
@@ -0,0 +1,43 @@
|
||||
import { parsePositiveInt } from './ai-settings.service';
|
||||
|
||||
/**
|
||||
* Round-trip coercion for numeric `::text` provider settings (e.g.
|
||||
* chatContextWindow). Values are stored as text and read back as strings, so
|
||||
* this guards the read path the DTO write-validation does not cover: a silent
|
||||
* loss of `Math.floor` or a `> 0` → `>= 0` drift would otherwise go unnoticed.
|
||||
*/
|
||||
describe('parsePositiveInt', () => {
|
||||
it('keeps a valid positive integer string', () => {
|
||||
expect(parsePositiveInt('200000')).toBe(200000);
|
||||
});
|
||||
|
||||
it('floors a fractional string', () => {
|
||||
expect(parsePositiveInt('1.9')).toBe(1);
|
||||
expect(parsePositiveInt('1.0')).toBe(1);
|
||||
});
|
||||
|
||||
it('returns undefined for zero', () => {
|
||||
expect(parsePositiveInt('0')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for a negative value', () => {
|
||||
expect(parsePositiveInt('-5')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for an empty string', () => {
|
||||
expect(parsePositiveInt('')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for a non-numeric string', () => {
|
||||
expect(parsePositiveInt('abc')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for undefined / null', () => {
|
||||
expect(parsePositiveInt(undefined)).toBeUndefined();
|
||||
expect(parsePositiveInt(null)).toBeUndefined();
|
||||
});
|
||||
|
||||
it('accepts a real number too (not only ::text strings)', () => {
|
||||
expect(parsePositiveInt(42)).toBe(42);
|
||||
});
|
||||
});
|
||||
@@ -18,6 +18,18 @@ import {
|
||||
PROVIDER_SETTINGS_KEYS,
|
||||
} from './ai.types';
|
||||
|
||||
/**
|
||||
* Coerce a raw provider value (stored as `::text`, so it arrives as a string —
|
||||
* see workspace.repo.ts) into a positive integer, or `undefined` when it is not
|
||||
* a finite number greater than zero. Used for numeric `::text` settings such as
|
||||
* `chatContextWindow`. Fractions are floored: `"1.9" → 1`, `"0"`/`"-5"`/`""`/
|
||||
* `"abc"`/`undefined` → `undefined`.
|
||||
*/
|
||||
export function parsePositiveInt(raw: unknown): number | undefined {
|
||||
const n = Number(raw);
|
||||
return Number.isFinite(n) && n > 0 ? Math.floor(n) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shape of the partial update accepted by `update`. Mirrors the validated
|
||||
* controller DTO. `apiKey` / `embeddingApiKey` are write-only: undefined =
|
||||
@@ -26,6 +38,8 @@ import {
|
||||
export interface UpdateAiSettingsInput {
|
||||
driver?: AiDriver;
|
||||
chatModel?: string;
|
||||
// Max context window in tokens for the chat header badge. 0/empty = no limit.
|
||||
chatContextWindow?: number;
|
||||
chatApiStyle?: ChatApiStyle;
|
||||
embeddingModel?: string;
|
||||
baseUrl?: string;
|
||||
@@ -160,6 +174,9 @@ export class AiSettingsService {
|
||||
const config: ResolvedAiConfig = {
|
||||
driver: provider.driver,
|
||||
chatModel: provider.chatModel,
|
||||
// Max context window for the chat header badge denominator. Stored as
|
||||
// ::text; 0/unset/invalid = no limit (undefined).
|
||||
chatContextWindow: parsePositiveInt(provider.chatContextWindow),
|
||||
// Plain passthrough; getChatModel defaults unset to 'openai-compatible'.
|
||||
chatApiStyle: provider.chatApiStyle,
|
||||
// Cheap model id for the anonymous public-share assistant; reuses the chat
|
||||
@@ -219,6 +236,10 @@ export class AiSettingsService {
|
||||
async getMasked(workspaceId: string): Promise<MaskedAiSettings> {
|
||||
const provider = await this.readProvider(workspaceId);
|
||||
|
||||
// Stored as ::text; coerce to a positive integer (or undefined) so the
|
||||
// client receives a real number.
|
||||
const chatContextWindow = parsePositiveInt(provider.chatContextWindow);
|
||||
|
||||
let hasApiKey = false;
|
||||
let hasEmbeddingApiKey = false;
|
||||
let hasSttApiKey = false;
|
||||
@@ -243,6 +264,7 @@ export class AiSettingsService {
|
||||
return {
|
||||
driver: provider.driver,
|
||||
chatModel: provider.chatModel,
|
||||
chatContextWindow,
|
||||
chatApiStyle: provider.chatApiStyle,
|
||||
embeddingModel: provider.embeddingModel,
|
||||
baseUrl: provider.baseUrl,
|
||||
|
||||
@@ -32,6 +32,9 @@ export const CHAT_API_STYLES: ChatApiStyle[] = ['openai-compatible', 'openai'];
|
||||
export interface AiProviderSettings {
|
||||
driver: AiDriver;
|
||||
chatModel: string;
|
||||
// Max context window in tokens; surfaced to the chat header badge as the
|
||||
// denominator ("current / max"). 0/unset = no limit (badge shows no denominator).
|
||||
chatContextWindow?: number;
|
||||
// Chat provider implementation for the `openai` driver. Unset → defaults to
|
||||
// 'openai-compatible' (so reasoning is surfaced by default). See ChatApiStyle.
|
||||
chatApiStyle?: ChatApiStyle;
|
||||
@@ -72,6 +75,7 @@ export interface AiProviderSettings {
|
||||
export const PROVIDER_SETTINGS_KEYS = [
|
||||
'driver',
|
||||
'chatModel',
|
||||
'chatContextWindow',
|
||||
'chatApiStyle',
|
||||
'embeddingModel',
|
||||
'baseUrl',
|
||||
@@ -98,6 +102,9 @@ export const PROVIDER_SETTINGS_KEYS = [
|
||||
export interface ResolvedAiConfig extends Partial<AiProviderSettings> {
|
||||
driver?: AiDriver;
|
||||
chatModel?: string;
|
||||
// Max context window in tokens; surfaced to the chat header badge as the
|
||||
// "current / max" denominator. 0/unset = no limit.
|
||||
chatContextWindow?: number;
|
||||
// Cheap model id for the public-share assistant; reuses the chat creds.
|
||||
publicShareChatModel?: string;
|
||||
// Agent-role id whose persona the public-share assistant adopts (empty/unset
|
||||
@@ -116,6 +123,9 @@ export interface ResolvedAiConfig extends Partial<AiProviderSettings> {
|
||||
export interface MaskedAiSettings {
|
||||
driver?: AiDriver;
|
||||
chatModel?: string;
|
||||
// Max context window in tokens; the chat header badge denominator. 0/unset =
|
||||
// no limit.
|
||||
chatContextWindow?: number;
|
||||
chatApiStyle?: ChatApiStyle;
|
||||
embeddingModel?: string;
|
||||
baseUrl?: string;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { IsIn, IsOptional, IsString } from 'class-validator';
|
||||
import { IsIn, IsInt, IsOptional, IsString, Min } from 'class-validator';
|
||||
import {
|
||||
AI_DRIVERS,
|
||||
AiDriver,
|
||||
@@ -25,6 +25,13 @@ export class UpdateAiSettingsDto {
|
||||
@IsString()
|
||||
chatModel?: string;
|
||||
|
||||
// Max context window in tokens shown in the chat header badge. 0/empty =
|
||||
// clear the limit (no denominator shown).
|
||||
@IsOptional()
|
||||
@IsInt()
|
||||
@Min(0)
|
||||
chatContextWindow?: number;
|
||||
|
||||
@IsOptional()
|
||||
@IsIn(CHAT_API_STYLES)
|
||||
chatApiStyle?: ChatApiStyle;
|
||||
|
||||
270
apps/server/test/integration/ai-chat-message-status.int-spec.ts
Normal file
270
apps/server/test/integration/ai-chat-message-status.int-spec.ts
Normal file
@@ -0,0 +1,270 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
createMessage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* Integration coverage for the #183 step-granular durability primitives on
|
||||
* AiChatMessageRepo: `update` (in-place patch by id+workspace, bumps updatedAt,
|
||||
* returns the row) and `sweepStreaming` (crash recovery: flip dangling
|
||||
* 'streaming' rows to 'aborted'). Real SQL against docmost_test, not a mock.
|
||||
*/
|
||||
describe('AiChatMessageRepo.update + sweepStreaming [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let repo: AiChatMessageRepo;
|
||||
let workspaceId: string;
|
||||
let otherWorkspaceId: string;
|
||||
let userId: string;
|
||||
let chatId: string;
|
||||
let otherChatId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
repo = new AiChatMessageRepo(db as any);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
otherWorkspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
const otherUser = await createUser(db, otherWorkspaceId);
|
||||
otherChatId = (
|
||||
await createChat(db, {
|
||||
workspaceId: otherWorkspaceId,
|
||||
creatorId: otherUser.id,
|
||||
})
|
||||
).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('update patches content/status/metadata and bumps updatedAt', async () => {
|
||||
const seeded = await repo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: '',
|
||||
status: 'streaming',
|
||||
metadata: { parts: [] } as never,
|
||||
});
|
||||
const before = seeded.updatedAt;
|
||||
// Ensure a measurable timestamp delta.
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
|
||||
const updated = await repo.update(seeded.id, workspaceId, {
|
||||
content: 'final answer',
|
||||
status: 'completed',
|
||||
metadata: { parts: [{ type: 'text', text: 'final answer' }] },
|
||||
});
|
||||
|
||||
expect(updated).toBeDefined();
|
||||
expect(updated!.content).toBe('final answer');
|
||||
expect(updated!.status).toBe('completed');
|
||||
expect((updated!.metadata as any).parts).toHaveLength(1);
|
||||
// The 5ms sleep above guarantees a strictly-later timestamp.
|
||||
expect(new Date(updated!.updatedAt).getTime()).toBeGreaterThan(
|
||||
new Date(before).getTime(),
|
||||
);
|
||||
});
|
||||
|
||||
it('onlyIfStreaming update is a NO-OP once the row is finalized (race guard)', async () => {
|
||||
// Reproduce the step-update-vs-finalize race (#183 review): the row is
|
||||
// finalized to 'completed', then a LATE per-step 'streaming' update lands.
|
||||
// With `onlyIfStreaming` it must match nothing and leave the finalized row
|
||||
// untouched (no clobber back to 'streaming', no lost usage).
|
||||
const seeded = await repo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: 'partial',
|
||||
status: 'streaming',
|
||||
});
|
||||
// Terminal finalize (unguarded) wins.
|
||||
await repo.update(seeded.id, workspaceId, {
|
||||
content: 'final answer',
|
||||
status: 'completed',
|
||||
metadata: { usage: { totalTokens: 42 } } as never,
|
||||
});
|
||||
// A straggler per-step update arrives AFTER finalize.
|
||||
const late = await repo.update(
|
||||
seeded.id,
|
||||
workspaceId,
|
||||
{ content: 'partial', status: 'streaming', metadata: {} as never },
|
||||
{ onlyIfStreaming: true },
|
||||
);
|
||||
expect(late).toBeUndefined(); // matched no 'streaming' row -> no-op
|
||||
const rows = await repo.findAllByChat(chatId, workspaceId);
|
||||
const row = rows.find((r) => r.id === seeded.id)!;
|
||||
expect(row.status).toBe('completed'); // NOT clobbered back to streaming
|
||||
expect(row.content).toBe('final answer');
|
||||
expect((row.metadata as any).usage.totalTokens).toBe(42); // usage preserved
|
||||
});
|
||||
|
||||
it('update is workspace-scoped: a foreign workspace id matches nothing', async () => {
|
||||
const seeded = await repo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId,
|
||||
role: 'assistant',
|
||||
content: 'orig',
|
||||
status: 'streaming',
|
||||
});
|
||||
const res = await repo.update(seeded.id, otherWorkspaceId, {
|
||||
status: 'completed',
|
||||
});
|
||||
expect(res).toBeUndefined();
|
||||
// The row in the real workspace is untouched.
|
||||
const rows = await repo.findAllByChat(chatId, workspaceId);
|
||||
const stillThere = rows.find((r) => r.id === seeded.id);
|
||||
expect(stillThere!.status).toBe('streaming');
|
||||
// Clean up so it does not pollute the sweep test below.
|
||||
await repo.update(seeded.id, workspaceId, { status: 'completed' });
|
||||
});
|
||||
|
||||
// Backdate a row's updatedAt so it qualifies as a STALE streaming row (the
|
||||
// sweep only flips rows untouched for >10 minutes — a live turn bumps
|
||||
// updatedAt every step, so it would never match).
|
||||
async function backdateUpdatedAt(
|
||||
id: string,
|
||||
minutesAgo: number,
|
||||
): Promise<void> {
|
||||
await db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({ updatedAt: new Date(Date.now() - minutesAgo * 60 * 1000) })
|
||||
.where('id', '=', id)
|
||||
.execute();
|
||||
}
|
||||
|
||||
it('sweepStreaming flips STALE dangling streaming rows to aborted and counts them', async () => {
|
||||
// Two dangling streaming rows in our workspace + one in another workspace —
|
||||
// all backdated past the staleness threshold so the sweep picks them up.
|
||||
const a = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
});
|
||||
const b = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
});
|
||||
const other = await createMessage(db, {
|
||||
workspaceId: otherWorkspaceId,
|
||||
chatId: otherChatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
});
|
||||
await backdateUpdatedAt(a.id, 20);
|
||||
await backdateUpdatedAt(b.id, 20);
|
||||
await backdateUpdatedAt(other.id, 20);
|
||||
|
||||
// A settled row must NOT be touched.
|
||||
const done = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'completed',
|
||||
});
|
||||
// A legacy NULL-status row must NOT be touched.
|
||||
const legacy = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: null,
|
||||
});
|
||||
|
||||
const swept = await repo.sweepStreaming();
|
||||
// At least the 3 stale streaming rows we created (2 here + 1 in the other ws).
|
||||
expect(swept).toBeGreaterThanOrEqual(3);
|
||||
|
||||
const rows = await repo.findAllByChat(chatId, workspaceId);
|
||||
const byId = new Map(rows.map((r) => [r.id, r]));
|
||||
expect(byId.get(a.id)!.status).toBe('aborted');
|
||||
expect(byId.get(b.id)!.status).toBe('aborted');
|
||||
expect(byId.get(done.id)!.status).toBe('completed');
|
||||
expect(byId.get(legacy.id)!.status).toBeNull();
|
||||
|
||||
// Idempotent: a second sweep finds nothing left in our seeded set.
|
||||
const again = await repo.sweepStreaming();
|
||||
const rows2 = await repo.findAllByChat(chatId, workspaceId);
|
||||
// Our two rows stay aborted regardless of `again`'s global count.
|
||||
expect(rows2.find((r) => r.id === a.id)!.status).toBe('aborted');
|
||||
expect(again).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
it('sweepStreaming does NOT sweep a FRESH streaming row (recency bound, #183 review)', async () => {
|
||||
// A row that is actively streaming (recent updatedAt) must survive the sweep:
|
||||
// a fresh replica's boot-sweep must never abort a turn another replica is
|
||||
// still streaming in a multi-instance deploy.
|
||||
const fresh = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
});
|
||||
// A STALE streaming row created alongside it IS swept — proving the sweep
|
||||
// ran and the only difference is recency.
|
||||
const stale = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
});
|
||||
await backdateUpdatedAt(stale.id, 20);
|
||||
|
||||
await repo.sweepStreaming();
|
||||
|
||||
const rows = await repo.findAllByChat(chatId, workspaceId);
|
||||
const byId = new Map(rows.map((r) => [r.id, r]));
|
||||
// Fresh (recently-updated) streaming row is left untouched...
|
||||
expect(byId.get(fresh.id)!.status).toBe('streaming');
|
||||
// ...while the stale one alongside it was swept to 'aborted'.
|
||||
expect(byId.get(stale.id)!.status).toBe('aborted');
|
||||
});
|
||||
|
||||
it('findAllByChat caps the result, keeping the NEWEST messages in order (#183 review)', async () => {
|
||||
// A dedicated chat so the cap test is independent of the rows above.
|
||||
const cappedChat = (
|
||||
await createChat(db, { workspaceId, creatorId: userId })
|
||||
).id;
|
||||
const base = Date.now();
|
||||
// Three messages at strictly increasing timestamps.
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId: cappedChat,
|
||||
content: 'm1-oldest',
|
||||
createdAt: new Date(base),
|
||||
});
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId: cappedChat,
|
||||
content: 'm2',
|
||||
createdAt: new Date(base + 1000),
|
||||
});
|
||||
await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId: cappedChat,
|
||||
content: 'm3-newest',
|
||||
createdAt: new Date(base + 2000),
|
||||
});
|
||||
|
||||
// Cap of 2 -> the OLDEST message is dropped; the newest two stay, in
|
||||
// chronological order (oldest -> newest).
|
||||
const capped = await repo.findAllByChat(cappedChat, workspaceId, 2);
|
||||
expect(capped.map((r) => r.content)).toEqual(['m2', 'm3-newest']);
|
||||
|
||||
// Without a cap (well above the row count) all three come back in order.
|
||||
const all = await repo.findAllByChat(cappedChat, workspaceId, 100);
|
||||
expect(all.map((r) => r.content)).toEqual(['m1-oldest', 'm2', 'm3-newest']);
|
||||
});
|
||||
});
|
||||
@@ -91,6 +91,25 @@ describe('AiMcpServerRepo tool_allowlist jsonb round-trip [integration]', () =>
|
||||
const healed = enabled.find((r) => r.id === id);
|
||||
expect(healed?.toolAllowlist).toEqual(['alpha', 'beta']);
|
||||
});
|
||||
|
||||
it('FAIL-OPEN: a present-but-corrupt tool_allowlist reads back as null (no restriction)', async () => {
|
||||
// #185 re-review pt 8: normalizeRow's fail-open branch — the column is
|
||||
// PRESENT but does not parse into a string[] (here a jsonb string scalar
|
||||
// holding non-array JSON). The read must degrade to `null` ("no restriction"),
|
||||
// not crash. (A warn is logged with the server id; not asserted here.)
|
||||
const id = randomUUID();
|
||||
await sql`
|
||||
INSERT INTO ai_mcp_servers (id, workspace_id, name, transport, url, tool_allowlist)
|
||||
VALUES (
|
||||
${id}, ${ws}, ${`srv-${id}`}, 'http', 'https://example.com/mcp',
|
||||
to_jsonb(${'{"not":"an array"}'}::text)
|
||||
)
|
||||
`.execute(db);
|
||||
// Sanity: the column is present (a jsonb string scalar), not SQL NULL.
|
||||
expect(await jsonbTypeof(id)).toBe('string');
|
||||
// ...yet the read degrades to null (fail-open).
|
||||
expect((await repo.findById(id, ws))?.toolAllowlist).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
|
||||
@@ -104,7 +104,8 @@ export async function createWorkspace(
|
||||
name: overrides.name ?? `ws-${suffix}`,
|
||||
// hostname is uniquely constrained; keep it unique per workspace.
|
||||
hostname: `host-${suffix}`,
|
||||
settings: overrides.settings === undefined ? null : (overrides.settings as any),
|
||||
settings:
|
||||
overrides.settings === undefined ? null : (overrides.settings as any),
|
||||
})
|
||||
.returning(['id', 'settings'])
|
||||
.executeTakeFirstOrThrow();
|
||||
@@ -226,3 +227,37 @@ export async function createChat(
|
||||
.executeTakeFirstOrThrow();
|
||||
return { id: row.id as string };
|
||||
}
|
||||
|
||||
export async function createMessage(
|
||||
db: Kysely<any>,
|
||||
args: {
|
||||
workspaceId: string;
|
||||
chatId: string;
|
||||
userId?: string | null;
|
||||
role?: string;
|
||||
content?: string | null;
|
||||
status?: string | null;
|
||||
metadata?: unknown;
|
||||
// Explicit timestamp so a test can control message ORDER (the default DB
|
||||
// now() can tie within a millisecond, and the v4 id is not time-ordered).
|
||||
createdAt?: Date;
|
||||
},
|
||||
): Promise<{ id: string }> {
|
||||
const id = randomUUID();
|
||||
const row = await db
|
||||
.insertInto('aiChatMessages')
|
||||
.values({
|
||||
id,
|
||||
workspaceId: args.workspaceId,
|
||||
chatId: args.chatId,
|
||||
userId: args.userId ?? null,
|
||||
role: args.role ?? 'assistant',
|
||||
content: args.content ?? null,
|
||||
status: args.status ?? null,
|
||||
metadata: (args.metadata ?? null) as any,
|
||||
...(args.createdAt ? { createdAt: args.createdAt } : {}),
|
||||
})
|
||||
.returning(['id'])
|
||||
.executeTakeFirstOrThrow();
|
||||
return { id: row.id as string };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { Kysely } from 'kysely';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission.repo';
|
||||
import { PageService } from 'src/core/page/services/page.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createSpace,
|
||||
createUser,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #206 attach-1 — Duplicating a subtree where the SAME attachment is referenced
|
||||
* by more than one page must copy a working blob/row for EVERY copy, not just
|
||||
* the last page processed.
|
||||
*
|
||||
* Setup: root page A and child page B both embed the same image (attachmentId X,
|
||||
* the attachment row owned by A in the DB). Duplicating A produces copies A' and
|
||||
* B'. Before the fix the per-attachmentId map held a single entry, so B's entry
|
||||
* clobbered A's and the row-ownership guard (`attachment.pageId !== oldPageId`)
|
||||
* then skipped the only DB row entirely: zero blobs copied, zero new rows, both
|
||||
* copies' images 404. The fix keys the map to a LIST and copies once per
|
||||
* referencing page, dropping the broken guard.
|
||||
*
|
||||
* This drives the real PageService.duplicatePage against a real Postgres with a
|
||||
* recording storage stub, and asserts: storage.copy called twice and two fresh
|
||||
* attachment rows exist (one owned by A', one by B'), each matching the rewritten
|
||||
* attachmentId in its page's content.
|
||||
*/
|
||||
describe('PageService.duplicatePage shared attachment [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let pageRepo: PageRepo;
|
||||
let pagePermissionRepo: PagePermissionRepo;
|
||||
let pageService: PageService;
|
||||
let workspaceId: string;
|
||||
let spaceId: string;
|
||||
let userId: string;
|
||||
|
||||
// Records every (source, dest) blob copy the service requests.
|
||||
const copyCalls: Array<{ from: string; to: string }> = [];
|
||||
const storageService = {
|
||||
copy: async (from: string, to: string) => {
|
||||
copyCalls.push({ from, to });
|
||||
},
|
||||
} as any;
|
||||
|
||||
// Duplicate persists transclusion/reference rows in best-effort try/catch
|
||||
// blocks; a no-op stub keeps the harness focused on the attachment path.
|
||||
const transclusionService = {
|
||||
insertTransclusionsForPages: async () => {},
|
||||
insertReferencesForPages: async () => {},
|
||||
insertTemplateReferencesForPages: async () => {},
|
||||
} as any;
|
||||
|
||||
const eventEmitter = { emit: () => true } as any;
|
||||
|
||||
function imageDoc(attachmentId: string) {
|
||||
return {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'image',
|
||||
attrs: {
|
||||
attachmentId,
|
||||
src: `/api/files/${attachmentId}/image.png`,
|
||||
width: '100%',
|
||||
align: 'center',
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
pageRepo = new PageRepo(db as any, {} as any, eventEmitter);
|
||||
// filterAccessiblePageIds short-circuits to the input ids when the space has
|
||||
// no restricted pages, so groupRepo/cache (2nd/3rd ctor args) are never hit.
|
||||
pagePermissionRepo = new PagePermissionRepo(
|
||||
db as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
);
|
||||
pageService = new PageService(
|
||||
pageRepo,
|
||||
pagePermissionRepo,
|
||||
undefined as any, // attachmentRepo (unused on duplicate path)
|
||||
db as any,
|
||||
storageService,
|
||||
undefined as any, // attachmentQueue
|
||||
undefined as any, // aiQueue
|
||||
undefined as any, // generalQueue
|
||||
eventEmitter,
|
||||
undefined as any, // collaborationGateway
|
||||
undefined as any, // watcherService
|
||||
transclusionService,
|
||||
);
|
||||
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
spaceId = (await createSpace(db, workspaceId)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('copies a shared attachment for every page that references it', async () => {
|
||||
copyCalls.length = 0;
|
||||
|
||||
const attachmentId = randomUUID();
|
||||
const pageAId = randomUUID();
|
||||
const pageBId = randomUUID();
|
||||
|
||||
// Root A and child B both embed the same attachmentId.
|
||||
await db
|
||||
.insertInto('pages')
|
||||
.values({
|
||||
id: pageAId,
|
||||
slugId: `a-${pageAId.slice(0, 8)}`,
|
||||
title: 'A',
|
||||
content: imageDoc(attachmentId) as any,
|
||||
position: 'a0',
|
||||
spaceId,
|
||||
workspaceId,
|
||||
creatorId: userId,
|
||||
})
|
||||
.execute();
|
||||
await db
|
||||
.insertInto('pages')
|
||||
.values({
|
||||
id: pageBId,
|
||||
slugId: `b-${pageBId.slice(0, 8)}`,
|
||||
title: 'B',
|
||||
content: imageDoc(attachmentId) as any,
|
||||
position: 'a0',
|
||||
parentPageId: pageAId,
|
||||
spaceId,
|
||||
workspaceId,
|
||||
creatorId: userId,
|
||||
})
|
||||
.execute();
|
||||
|
||||
// Single attachment row, owned by A.
|
||||
await db
|
||||
.insertInto('attachments')
|
||||
.values({
|
||||
id: attachmentId,
|
||||
type: 'image',
|
||||
filePath: `${spaceId}/${attachmentId}/image.png`,
|
||||
fileName: 'image.png',
|
||||
fileExt: 'png',
|
||||
mimeType: 'image/png',
|
||||
creatorId: userId,
|
||||
workspaceId,
|
||||
pageId: pageAId,
|
||||
spaceId,
|
||||
})
|
||||
.execute();
|
||||
|
||||
const rootPage = await pageRepo.findById(pageAId);
|
||||
const result = await pageService.duplicatePage(
|
||||
rootPage as any,
|
||||
undefined,
|
||||
{ id: userId, workspaceId } as any,
|
||||
);
|
||||
|
||||
const newRootId = result.id;
|
||||
const newChildIds = result.childPageIds;
|
||||
expect(newChildIds).toHaveLength(1);
|
||||
const newChildId = newChildIds[0];
|
||||
|
||||
// Both pages' images were copied: one blob per referencing page.
|
||||
expect(copyCalls).toHaveLength(2);
|
||||
|
||||
// Two fresh attachment rows exist, one owned by each copied page.
|
||||
const newAttachments = await db
|
||||
.selectFrom('attachments')
|
||||
.selectAll()
|
||||
.where('pageId', 'in', [newRootId, newChildId])
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.execute();
|
||||
expect(newAttachments).toHaveLength(2);
|
||||
|
||||
const ownerIds = newAttachments.map((a) => a.pageId).sort();
|
||||
expect(ownerIds).toEqual([newRootId, newChildId].sort());
|
||||
|
||||
// Each copied page's content points at a rewritten attachmentId that now has
|
||||
// a real row (i.e. the image src resolves instead of 404ing).
|
||||
for (const pageId of [newRootId, newChildId]) {
|
||||
const page = await db
|
||||
.selectFrom('pages')
|
||||
.select(['content'])
|
||||
.where('id', '=', pageId)
|
||||
.executeTakeFirstOrThrow();
|
||||
const node = (page.content as any).content[0];
|
||||
expect(node.type).toBe('image');
|
||||
const referencedId = node.attrs.attachmentId;
|
||||
expect(referencedId).not.toBe(attachmentId); // remapped to a fresh id
|
||||
const row = newAttachments.find((a) => a.id === referencedId);
|
||||
expect(row).toBeDefined();
|
||||
expect(row!.pageId).toBe(pageId);
|
||||
}
|
||||
});
|
||||
});
|
||||
133
apps/server/test/integration/page-move-cycle.int-spec.ts
Normal file
133
apps/server/test/integration/page-move-cycle.int-spec.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { PageService } from 'src/core/page/services/page.service';
|
||||
import { Page } from '@docmost/db/types/entity.types';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createSpace,
|
||||
createPage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #207 #7 — TOCTOU in PageService.movePage: two concurrent moves
|
||||
* ("A under B" + "B under A") must NOT be able to persist a parent/child cycle.
|
||||
*
|
||||
* Before the fix the cycle check (getPageBreadCrumbs) and the UPDATE were two
|
||||
* separate, unlocked statements, so both movers could read the same pre-write
|
||||
* acyclic snapshot, both pass the guard, and persist A.parentPageId=B AND
|
||||
* B.parentPageId=A. The fix runs the guard + UPDATE in one transaction behind a
|
||||
* per-space advisory lock, so the moves serialize: whichever commits second
|
||||
* sees the first's write and its guard rejects the cycle.
|
||||
*
|
||||
* This test drives the real PageService.movePage against a real Postgres,
|
||||
* firing the two opposing moves concurrently, and asserts that no cycle ever
|
||||
* persists (walking parentPageId from both pages always reaches a root with no
|
||||
* repeated id) and that exactly one of the two opposing moves is rejected.
|
||||
*/
|
||||
describe('PageService.movePage concurrent A<->B cycle guard [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let pageRepo: PageRepo;
|
||||
let pageService: PageService;
|
||||
let workspaceId: string;
|
||||
let spaceId: string;
|
||||
|
||||
// A valid fractional-index position key; movePage validates the position.
|
||||
const position = generateJitteredKeyBetween(null, null);
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
// Event emission is a side effect movePage performs but the cycle behaviour
|
||||
// does not depend on; a no-op emitter keeps the harness minimal.
|
||||
const eventEmitter = { emit: () => true } as any;
|
||||
pageRepo = new PageRepo(db as any, {} as any, eventEmitter);
|
||||
// Only pageRepo (1), db (4) and eventEmitter (9) are touched by movePage;
|
||||
// the remaining constructor deps are unused on this path.
|
||||
pageService = new PageService(
|
||||
pageRepo,
|
||||
undefined as any,
|
||||
undefined as any,
|
||||
db as any,
|
||||
undefined as any,
|
||||
undefined as any,
|
||||
undefined as any,
|
||||
undefined as any,
|
||||
eventEmitter,
|
||||
undefined as any,
|
||||
undefined as any,
|
||||
undefined as any,
|
||||
);
|
||||
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
spaceId = (await createSpace(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
async function findPage(id: string): Promise<Page> {
|
||||
const page = await pageRepo.findById(id);
|
||||
if (!page) throw new Error(`page ${id} not found`);
|
||||
return page;
|
||||
}
|
||||
|
||||
// Walk parentPageId upward from startId. Throws if a node repeats (cycle) or
|
||||
// the walk fails to terminate; returns normally only when a root is reached.
|
||||
async function assertReachesRoot(startId: string): Promise<void> {
|
||||
const seen = new Set<string>();
|
||||
let cur: string | null = startId;
|
||||
let steps = 0;
|
||||
while (cur) {
|
||||
if (seen.has(cur)) {
|
||||
throw new Error(`cycle detected: revisited ${cur}`);
|
||||
}
|
||||
seen.add(cur);
|
||||
const row: { parentPageId: string | null } | undefined = await db
|
||||
.selectFrom('pages')
|
||||
.select('parentPageId')
|
||||
.where('id', '=', cur)
|
||||
.executeTakeFirst();
|
||||
cur = row?.parentPageId ?? null;
|
||||
if (++steps > 1000) {
|
||||
throw new Error('parent walk did not terminate');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
it('two opposing concurrent moves never persist a parent/child cycle', async () => {
|
||||
// Repeat to exercise different scheduler interleavings of the two moves.
|
||||
for (let i = 0; i < 8; i++) {
|
||||
const a = await createPage(db, { workspaceId, spaceId, title: `A-${i}` });
|
||||
const b = await createPage(db, { workspaceId, spaceId, title: `B-${i}` });
|
||||
|
||||
const movedA = await findPage(a.id);
|
||||
const movedB = await findPage(b.id);
|
||||
|
||||
const results = await Promise.allSettled([
|
||||
pageService.movePage(
|
||||
{ pageId: a.id, parentPageId: b.id, position } as any,
|
||||
movedA,
|
||||
),
|
||||
pageService.movePage(
|
||||
{ pageId: b.id, parentPageId: a.id, position } as any,
|
||||
movedB,
|
||||
),
|
||||
]);
|
||||
|
||||
// No cycle may have been persisted by either ordering.
|
||||
await assertReachesRoot(a.id);
|
||||
await assertReachesRoot(b.id);
|
||||
|
||||
// The serialization guarantees exactly one of the opposing moves wins;
|
||||
// the other must be rejected as a subtree cycle.
|
||||
const rejected = results.filter(
|
||||
(r): r is PromiseRejectedResult => r.status === 'rejected',
|
||||
);
|
||||
expect(rejected).toHaveLength(1);
|
||||
expect(rejected[0].reason?.message).toMatch(/into its own subtree/);
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
import { CamelCasePlugin, Kysely } from 'kysely';
|
||||
import { PostgresJSDialect } from 'kysely-postgres-js';
|
||||
import * as postgres from 'postgres';
|
||||
import { PageService } from 'src/core/page/services/page.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createSpace,
|
||||
createPage,
|
||||
TEST_DATABASE_URL,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #207 #8 — recursive page-tree CTEs (ancestors in getPageBreadCrumbs,
|
||||
* descendants in forceDelete) must not hang when a parent/child cycle already
|
||||
* exists in the data. Before the fix neither CTE had a CYCLE clause or a depth
|
||||
* cap, so a cycle (e.g. one persisted by the #7 TOCTOU race) made withRecursive
|
||||
* loop forever — and since the move guard itself runs the ancestor CTE, a cycle
|
||||
* would disable the very guard meant to prevent it.
|
||||
*
|
||||
* The fix adds a depth counter bounded by MAX_PAGE_TREE_DEPTH to both CTEs.
|
||||
* These tests seed an A<->B cycle directly (bypassing the guard), then run the
|
||||
* real CTE paths against Postgres with a short connection-level statement_timeout
|
||||
* so a regression (an unbounded CTE) fails fast as a query timeout instead of a
|
||||
* bounded result.
|
||||
*/
|
||||
describe('recursive page-tree CTEs cycle/depth guard [integration]', () => {
|
||||
// Upper bound on rows the depth-capped CTEs can emit for a 2-node cycle: one
|
||||
// row per depth level 0..MAX. Kept loose so the assertion does not couple to
|
||||
// the exact constant, only to "bounded".
|
||||
const BOUNDED_MAX_ROWS = 20_000;
|
||||
|
||||
let db: Kysely<any>;
|
||||
// Dedicated Kysely whose connections carry a short statement_timeout, so an
|
||||
// unbounded recursive CTE aborts quickly instead of hanging the suite.
|
||||
let timeoutDb: Kysely<any>;
|
||||
let workspaceId: string;
|
||||
let spaceId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
timeoutDb = new Kysely<any>({
|
||||
dialect: new PostgresJSDialect({
|
||||
postgres: postgres(TEST_DATABASE_URL, {
|
||||
max: 2,
|
||||
onnotice: () => {},
|
||||
// Applied to every connection on connect: cap any single statement.
|
||||
connection: { statement_timeout: 4000 },
|
||||
types: {
|
||||
bigint: {
|
||||
to: 20,
|
||||
from: [20, 1700],
|
||||
serialize: (value: number) => value.toString(),
|
||||
parse: (value: string) => Number.parseInt(value),
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
spaceId = (await createSpace(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await timeoutDb.destroy();
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
// Seed two fresh pages and wire them into a direct parent/child cycle,
|
||||
// bypassing PageService.movePage's guard the way the #7 race would.
|
||||
async function seedCycle(): Promise<{ aId: string; bId: string }> {
|
||||
const a = await createPage(db, { workspaceId, spaceId, title: 'cycle-A' });
|
||||
const b = await createPage(db, { workspaceId, spaceId, title: 'cycle-B' });
|
||||
await db
|
||||
.updateTable('pages')
|
||||
.set({ parentPageId: b.id })
|
||||
.where('id', '=', a.id)
|
||||
.execute();
|
||||
await db
|
||||
.updateTable('pages')
|
||||
.set({ parentPageId: a.id })
|
||||
.where('id', '=', b.id)
|
||||
.execute();
|
||||
return { aId: a.id, bId: b.id };
|
||||
}
|
||||
|
||||
function makeService(database: Kysely<any>): PageService {
|
||||
const eventEmitter = { emit: () => true } as any;
|
||||
const attachmentQueue = { add: async () => undefined } as any;
|
||||
return new PageService(
|
||||
undefined as any, // pageRepo (unused by these paths)
|
||||
undefined as any, // pagePermissionRepo
|
||||
undefined as any, // attachmentRepo
|
||||
database as any, // db
|
||||
undefined as any, // storageService
|
||||
attachmentQueue, // attachmentQueue
|
||||
undefined as any, // aiQueue
|
||||
undefined as any, // generalQueue
|
||||
eventEmitter, // eventEmitter
|
||||
undefined as any, // collaborationGateway
|
||||
undefined as any, // watcherService
|
||||
undefined as any, // transclusionService
|
||||
);
|
||||
}
|
||||
|
||||
it('getPageBreadCrumbs returns a bounded result (no hang) when a cycle exists', async () => {
|
||||
const { aId } = await seedCycle();
|
||||
const service = makeService(timeoutDb);
|
||||
|
||||
// Must resolve (the depth cap stops the walk) rather than time out.
|
||||
const crumbs = await service.getPageBreadCrumbs(aId);
|
||||
|
||||
expect(Array.isArray(crumbs)).toBe(true);
|
||||
expect(crumbs.length).toBeGreaterThan(1);
|
||||
expect(crumbs.length).toBeLessThanOrEqual(BOUNDED_MAX_ROWS);
|
||||
});
|
||||
|
||||
it('forceDelete descendant CTE is bounded (no hang) and removes the cyclic pages', async () => {
|
||||
const { aId, bId } = await seedCycle();
|
||||
const service = makeService(timeoutDb);
|
||||
|
||||
// Must complete instead of looping on the descendant CTE.
|
||||
await service.forceDelete(aId, workspaceId);
|
||||
|
||||
const survivors = await db
|
||||
.selectFrom('pages')
|
||||
.select('id')
|
||||
.where('id', 'in', [aId, bId])
|
||||
.execute();
|
||||
expect(survivors).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
@@ -1,33 +0,0 @@
|
||||
# Отложенные интеграционные тесты `AiChatService.stream`
|
||||
|
||||
Статус: **открыто.** Это остаток от прежнего документа
|
||||
`feature-test-coverage-deferred.md` (хвост тест-плана PR #49). Два из трёх
|
||||
его разделов уже закрыты новой интеграционной обвязкой против реального
|
||||
Postgres/Redis (`apps/server/test/integration/`, PR #115):
|
||||
|
||||
- ✅ **Раздел 1 — repo-тесты против БД.** Закрыт `ai-agent-roles-repo`,
|
||||
`ai-chat-repo-find-by-creator`, `page-template-references-cascade`,
|
||||
`workspace-repo-update-setting` (`*.int-spec.ts`).
|
||||
- ✅ **Раздел 2 — достоверность Lua-окна cost-cap против реального Redis.**
|
||||
Закрыт `public-share-workspace-limiter.int-spec.ts`.
|
||||
- ⬜ **Раздел 3 (ниже) — полная интеграция `AiChatService.stream`.** Всё ещё
|
||||
не реализован; держим запись открытой, чтобы тест-долг не потерялся при
|
||||
удалении исходного документа.
|
||||
|
||||
## Полная интеграция `AiChatService.stream` (рефактор R1-stream)
|
||||
|
||||
`apps/server/src/core/ai-chat/ai-chat.service.ts`. В PR #49 извлечён и
|
||||
покрыт только чистый `buildErrorAssistantRecord`. Полные интеграционные
|
||||
сценарии всё ещё отложены:
|
||||
|
||||
- **Запись чата, упавшего на первом ходу** (`onError`) — ассистентская
|
||||
запись об ошибке должна сохраняться, даже когда первый ход стрима падает.
|
||||
- **Жизненный цикл external-MCP клиентов** — клиенты закрываются и при
|
||||
`throw`, и при `onFinish` (нет утечки соединений).
|
||||
- **Анти-tamper: история восстанавливается из БД, а не из `body.messages`** —
|
||||
клиент не может подменить историю через тело запроса.
|
||||
|
||||
Эти сценарии требуют сидирования SDK `streamText` (инъекция/seam колбэков
|
||||
`onError` / `onFinish` / `onAbort` + `res.hijack`). Отложено, чтобы не
|
||||
дестабилизировать 287-строчный `stream()`; делать вместе с выносом testable
|
||||
turn-pipeline.
|
||||
@@ -1,127 +0,0 @@
|
||||
# Дублирование определений инструментов: in-app агент vs standalone MCP-пакет
|
||||
|
||||
Статус: **частично закрыто.** Квирк «node как объект ИЛИ JSON-строка» вынесен
|
||||
в общий хелпер `parseNodeArg` (см. «Прогресс» ниже); остальной долг (единый
|
||||
реестр спеков + унификация конвертера) всё ещё открыт. Это forward-looking
|
||||
стоимость поддержки, НЕ баг — код корректен сегодня. Держим запись открытой,
|
||||
чтобы при росте набора инструментов долг не разъезжался молча.
|
||||
|
||||
## Прогресс
|
||||
|
||||
- ✅ **Квирк node-arg вынесен в хелпер** (`refactor/ai-chat-tool-spec-registry`,
|
||||
PR #114). Шесть рукописных копий нормализации «node как объект ИЛИ
|
||||
JSON-строка» свёрнуты в `parseNodeArg`: по одному источнику на пакет —
|
||||
`packages/mcp/src/lib/parse-node-arg.ts` (standalone) и
|
||||
`apps/server/src/core/ai-chat/tools/parse-node-arg.ts` (in-app). Две копии
|
||||
намеренны (ESM/CJS-граница), поведение тождественно.
|
||||
- ⏳ **Единый реестр спеков** (схема + описание на инструмент) и **вывод
|
||||
`DocmostClientLike` из реального типа** — отложены (см. «Фикс»): требуют
|
||||
пересечения ESM/CJS-границы для данных+zod и ломают тест-стабы in-app
|
||||
инструментов при точных типах. Делать инкрементально.
|
||||
- ⏳ **Унификация конвертера ProseMirror ↔ Markdown** — открыта (см. раздел
|
||||
«Расширение …» ниже); на неё опирается план git-синка
|
||||
(`docs/git-sync-plan.md`).
|
||||
|
||||
## Суть
|
||||
|
||||
Один и тот же набор инструментов поверх одного `DocmostClient` описан
|
||||
**тремя независимыми рукописными слоями**. Каждое добавление инструмента или
|
||||
правка его model-facing описания требует синхронной правки в 2–3 местах, а
|
||||
parity-баги (расхождение копий) приходится чинить/переоткрывать дважды.
|
||||
|
||||
## Где дублируется (три слоя)
|
||||
|
||||
1. **Standalone MCP-сервер** — `packages/mcp/src/index.ts` (~38 `registerTool`).
|
||||
Для внешних MCP-клиентов (stdio/http). На каждый инструмент: zod-схема +
|
||||
длинное model-facing описание + тонкий `execute`, вызывающий `DocmostClient`.
|
||||
2. **Встроенный AI-чат** — `apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts`
|
||||
(~39 `tool({...})` через `ai`-SDK). Своя zod-схема + своё описание + свой
|
||||
`execute` поверх ТОГО ЖЕ клиента (`@docmost/mcp` грузится в
|
||||
`tools/docmost-client.loader.ts:188` через динамический `import()`).
|
||||
3. **Ручная копия сигнатур** — интерфейс `DocmostClientLike` в
|
||||
`apps/server/src/core/ai-chat/tools/docmost-client.loader.ts:9` (в комментарии
|
||||
прямо: «Signatures here mirror that file exactly»), скопирован руками из
|
||||
`packages/mcp/src/client.ts`.
|
||||
|
||||
## Что именно продублировано (с подтверждением по коду)
|
||||
|
||||
- **zod-схема + описание** каждого инструмента — в слоях 1 и 2 целиком.
|
||||
- ~~**Квирк «node как объект ИЛИ JSON-строка»** реализован дважды (НЕ в общем
|
||||
клиенте)~~ — **закрыто (PR #114):** вынесен в `parseNodeArg` (по хелперу на
|
||||
пакет), 6 inline-копий устранены:
|
||||
- in-app: `patchNode`, `insertNode`, `updatePageJson` →
|
||||
`apps/server/src/core/ai-chat/tools/parse-node-arg.ts`;
|
||||
- standalone: `patch_node`, `insert_node`, `update_page_json` →
|
||||
`packages/mcp/src/lib/parse-node-arg.ts`.
|
||||
- **Guardrail/семантика `transformPage` (dryRun)** описана в обоих:
|
||||
`ai-chat-tools.service.ts:~935` и `index.ts:~1006`.
|
||||
|
||||
## Почему разделение слоёв 1 и 2 само по себе оправдано
|
||||
|
||||
У путей разный транспорт и auth-контекст, и это правильно держать раздельно:
|
||||
in-app путь чеканит per-user JWT + provenance collab-токен (подписанная
|
||||
agent-claim, `docmost-client.loader.ts:159` — `getCollabToken`; см. план §6.5),
|
||||
а standalone обслуживает внешних клиентов по stdio/http. **Но** это оправдывает
|
||||
два тонких адаптера (`execute` + auth-обвязка), а НЕ две рукописные копии
|
||||
МЕТАДАННЫХ (схема + описание + квирки). Метаданные можно объявить один раз и
|
||||
переиспользовать обоими транспортами.
|
||||
|
||||
## Доказательство стоимости (наблюдалось при фиксе edit_page_text)
|
||||
|
||||
При исправлении ложного «успеха» `edit_page_text` (refuse форматных правок +
|
||||
`verify`-отчёт):
|
||||
- **Поведение** легло в общий `DocmostClient` → автоматически дошло до обоих
|
||||
агентов ОДНОЙ правкой. Это «хороший» случай — логика в едином источнике.
|
||||
- **Описание** инструмента пришлось править ДВАЖДЫ: в `index.ts` (кодером) и
|
||||
отдельно в `ai-chat-tools.service.ts:617`, где описание продолжало рекламировать
|
||||
«Markdown wrappers tolerated via strip-and-retry» — ровно ту формулировку, что
|
||||
ввела исходного агента в заблуждение. Копия молча разъехалась и какое-то время
|
||||
встроенный агент получал устаревшую подсказку. Это и есть материализованный
|
||||
parity-баг.
|
||||
|
||||
## Расширение: дублируется не только описания инструментов — ещё и конвертер (PM ↔ Markdown)
|
||||
|
||||
Зафиксировано при планировании встраивания git-синка (`docmost-sync` → gitmost,
|
||||
нативная in-process интеграция). Та же болезнь «несколько рукописных копий одного
|
||||
кода» теперь касается слоя конвертации ProseMirror ↔ Markdown и его lib, а не
|
||||
только метаданных инструментов.
|
||||
|
||||
- **Копия в gitmost** — `packages/mcp/src/lib/`: `markdown-converter.ts` (~885
|
||||
строк), `markdown-document.ts` (~136), `node-ops.ts`, `diff.ts`,
|
||||
`docmost-schema.ts`. Канонизатора (`canonicalize.ts`) здесь НЕТ.
|
||||
- **Копия в docmost-sync** — `packages/docmost-client/src/lib/`: тот же набор +
|
||||
`canonicalize.ts` (~11 КБ, держит идемпотентность round-trip, SPEC §11) +
|
||||
`markdown-document.ts` с режимом «тело + якоря, без тредов комментов»
|
||||
(`includeCommentThreads:false`, на ~20 строк больше).
|
||||
- **Третья копия (планируется)** — план git-синка вендорит чистую часть
|
||||
конвертера в новый `packages/git-sync` (collab-файл не нужен: запись идёт
|
||||
нативно через `openDirectConnection` + `@docmost/editor-ext`).
|
||||
|
||||
Копии уже молча разъехались (docmost-sync vs `packages/mcp`): `collaboration.ts`
|
||||
~329 изменённых строк, `node-ops.ts` ~53, `markdown-converter.ts` ~24,
|
||||
`markdown-document.ts` ~20. Отдельно: `docmost-schema.ts` в lib дублирует
|
||||
**реальную** схему сервера `@docmost/editor-ext` (её использует collab/persistence)
|
||||
— расхождение схем = риск битой конвертации нод.
|
||||
|
||||
Вывод: тот же фикс-вектор (единый источник правды), что и для инструментов, стоит
|
||||
распространить на конвертер — общий пакет конвертации, потребляемый `mcp`,
|
||||
`git-sync` и (в идеале) сервером. До конвергенции git-sync держит вендоренную
|
||||
копию валидированного конвертера с гейтом round-trip против схемы `editor-ext`
|
||||
(осознанный долг «третья копия сейчас, объединяем позже»).
|
||||
|
||||
## Фикс
|
||||
|
||||
Единый реестр спеков (полное устранение дублирования).** Вынести в
|
||||
`packages/mcp` один источник на инструмент: `name` + zod-схема + model-facing
|
||||
описание + общий хелпер нормализации node-строки (для patch/insert/update).
|
||||
И `index.ts`, и `ai-chat-tools.service.ts` импортируют спеки и добавляют только
|
||||
свой `execute`/auth. `DocmostClientLike` — выводить из типа реального клиента
|
||||
(type-only import / генерация), а не копировать руками.
|
||||
- Ограничение: `@docmost/mcp` — ESM-only, сервер грузит его через трюк
|
||||
`new Function('import(specifier)')` (`docmost-client.loader.ts:174`), потому
|
||||
что `module:commonjs` даунлевелит `import()` в `require()`. Реестр спеков
|
||||
(данные + zod) должен пересекать ту же ESM/CJS-границу — выполнимо тем же
|
||||
динамическим импортом; `ai`-SDK `tool()` и MCP `registerTool()` имеют разную
|
||||
форму, поэтому реестр экспортирует транспорт-агностичные `{name, schema,
|
||||
description}`, а каждая сторона оборачивает их сама. `zod` — общая зависимость
|
||||
обоих пакетов, типы переносятся.
|
||||
@@ -1,534 +0,0 @@
|
||||
# Git-sync: спека реализации (встраивание docmost-sync в gitmost)
|
||||
|
||||
Статус: **спецификация, код не менялся.** Детальный план реализации фичи
|
||||
«двусторонний синк страниц Docmost ↔ локальная git-папка Markdown», встроенной
|
||||
прямо в gitmost.
|
||||
|
||||
Источник движка: `https://gitea.vvzvlad.xyz/vvzvlad/docmost-sync`
|
||||
(ветка `main`, на момент спеки HEAD `b03eb35`). Все сигнатуры ниже сверены с этим
|
||||
исходником и с текущим кодом gitmost.
|
||||
|
||||
Предыстория и обоснование архитектурных развилок — в бэклоге
|
||||
[ai-chat-tool-definitions-duplicated.md](backlog/ai-chat-tool-definitions-duplicated.md)
|
||||
(раздел про дублирование конвертера) и в исходном `SPEC.md` репозитория
|
||||
docmost-sync (нумерация §-параграфов ниже ссылается на него).
|
||||
|
||||
---
|
||||
|
||||
## 0. Зафиксированные решения
|
||||
|
||||
Из обсуждения архитектуры (выбор пользователя) и трёх суб-решений:
|
||||
|
||||
1. **Нативная in-process интеграция.** Никаких REST-к-себе и сервис-юзера: чтение
|
||||
через репозитории gitmost, запись тела — через collab `openDirectConnection`,
|
||||
триггеры — через `EventEmitter2` вместо поллинга `/recent`.
|
||||
2. **Встроенный NestJS-модуль** `GitSyncModule` в `apps/server/src/integrations/git-sync`
|
||||
с `@Interval`/событиями и **leader-lock на Redis** (single-writer при нескольких
|
||||
репликах).
|
||||
3. **Настройка по спейсам в UI** — флаг в `space.settings.gitSync`, секреты
|
||||
(git-remote) — через ENV/`EnvironmentService`.
|
||||
4. **Конвертер** — вендорим *чистую* часть из docmost-sync в `packages/git-sync`,
|
||||
гейт = round-trip-идемпотентность против схемы `@docmost/editor-ext`.
|
||||
5. **Vault** — **репозиторий на спейс**; `move-to-space` = кросс-репо delete+create.
|
||||
6. **Провенанс** — отдельное значение `lastUpdatedSource = 'git-sync'`.
|
||||
|
||||
Вне scope v1 (как и в SPEC): комментарии (только якоря, без тредов), права/ACL,
|
||||
вложения как отдельный поток (едут ссылками внутри контента), realtime-подписка
|
||||
на Hocuspocus (остаётся поллинг-страховка + события).
|
||||
|
||||
---
|
||||
|
||||
## 1. Архитектура верхнего уровня
|
||||
|
||||
```
|
||||
gitmost server (NestJS, один процесс)
|
||||
┌─────────────────────────────────────────────────────────────┐
|
||||
│ GitSyncModule │
|
||||
│ │
|
||||
│ GitSyncOrchestrator ── @Interval + Redis leader-lock │
|
||||
│ │ (per enabled space: pull-cycle / push-cycle) │
|
||||
│ │ │
|
||||
│ ├── engine (vendored docmost-sync, IO инжектируется) │
|
||||
│ │ pull.ts / push.ts / reconcile / layout / stabilize │
|
||||
│ │ │
|
||||
│ ├── GitmostDataSource ── реализует подмножество │
|
||||
│ │ DocmostClient НАТИВНО: │
|
||||
│ │ reads → PageRepo / SpaceRepo (Kysely) │
|
||||
│ │ writes → CollaborationGateway.openDirectConnection│
|
||||
│ │ + PageService (create/move/delete/...) │
|
||||
│ │ │
|
||||
│ └── VaultGit ── shell-out в системный git (как есть) │
|
||||
│ │
|
||||
│ PageChangeListener ── подписка на EventName.PAGE_* → │
|
||||
│ debounce → enqueue push-cycle │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
▲ читает/пишет страницы ▼ git push/pull
|
||||
PostgreSQL (pages/spaces) data/git-sync/<spaceId>/ (vault) → remote
|
||||
```
|
||||
|
||||
Ключ интеграции: движок docmost-sync уже **полностью построен на dependency
|
||||
injection** — весь внешний IO (REST-клиент, git, файловая система) передаётся
|
||||
через узкие интерфейсы. Мы НЕ переписываем движок; мы подставляем нативные
|
||||
реализации в его DI-швы.
|
||||
|
||||
---
|
||||
|
||||
## 2. Состав вендоринга из docmost-sync
|
||||
|
||||
В новый пакет `packages/git-sync` копируем (с сохранением истории смысла —
|
||||
backport-friendly, как сделано с `packages/mcp`):
|
||||
|
||||
### 2.1. Движок (engine) — `src/engine/`
|
||||
| Файл | Что несёт | IO | Берём |
|
||||
| --- | --- | --- | --- |
|
||||
| `pull.ts` | Docmost→FS: reconcile + write + commit + merge | client+git+fs (инжектируется) | да |
|
||||
| `push.ts` | FS→Docmost: diff + classify + apply + refs | client+git+fs (инжектируется) | да |
|
||||
| `git.ts` | `VaultGit` — обёртка git shell-out | системный `git` | да, как есть |
|
||||
| `reconcile.ts` | чистый планировщик | нет | да |
|
||||
| `layout.ts` | чистый маппер дерево→пути | нет | да |
|
||||
| `sanitize.ts` | чистая санитизация имён | нет | да |
|
||||
| `stabilize.ts` | fixpoint-нормализация md (SPEC §11) | нет (lib-вызовы) | да |
|
||||
| `loop-guard.ts` | `bodyHash` (sha256) | нет | да |
|
||||
| `settings.ts` | zod-конфиг | `.env` | **адаптируем** (см. §7) |
|
||||
| `index.ts` | тонкий CLI-скаффолд | — | нет (заменяем на NestJS) |
|
||||
|
||||
### 2.2. Конвертер (чистая часть) — `src/lib/`
|
||||
Из `packages/docmost-client/src/lib/` берём **только** чистый конвертер и формат
|
||||
файла (collab/auth REST-части НЕ нужны — запись нативная):
|
||||
|
||||
| Файл | Экспорт |
|
||||
| --- | --- |
|
||||
| `markdown-converter.ts` | `convertProseMirrorToMarkdown(content): string` |
|
||||
| `collaboration.ts` (только конвертер-функция) | `markdownToProseMirror(md): Promise<doc>` ⚠️ |
|
||||
| `markdown-document.ts` | `serializeDocmostMarkdownBody`, `parseDocmostMarkdown`, `serializeDocmostMarkdown`, тип `DocmostMdMeta` |
|
||||
| `canonicalize.ts` | `canonicalizeContent(node)`, `docsCanonicallyEqual(a,b)` |
|
||||
| `docmost-schema.ts` | tiptap-схема для `markdownToProseMirror` |
|
||||
| `node-ops.ts`, `diff.ts` | трансформации/диф (нужны транзитивно) |
|
||||
|
||||
⚠️ `markdownToProseMirror` физически лежит в `collaboration.ts` docmost-client
|
||||
(строка 289) — это **чистая** функция (marked→HTML→generateJSON), не путать с
|
||||
collab/websocket write-path из того же файла, который НЕ берём.
|
||||
|
||||
> **Долг (зафиксирован в бэклоге):** это третья копия конвертера (есть в
|
||||
> docmost-sync, в `packages/mcp`, теперь в `packages/git-sync`). Конвергенция в
|
||||
> общий пакет — отдельная задача; здесь сознательно вендорим валидированную
|
||||
> копию ради сохранения идемпотентности.
|
||||
|
||||
### 2.3. НЕ берём
|
||||
`pull`/`push` CLI-обёртки, `roundtrip.ts` (харнес переносим в тесты, см. §13),
|
||||
`docmost-client` REST-клиент целиком, `lib/collaboration.ts` (websocket-write),
|
||||
`lib/auth-utils.ts`, `Makefile`, Docker-обвязку docmost-sync.
|
||||
|
||||
---
|
||||
|
||||
## 3. Главный шов: `GitmostDataSource`
|
||||
|
||||
Движок дёргает Docmost через `Pick<DocmostClient, …>`. Мы реализуем класс,
|
||||
**структурно совместимый** с этими сигнатурами, но нативный внутри. Это
|
||||
единственный нетривиальный новый код.
|
||||
|
||||
### 3.1. Точный набор методов, которых требует движок
|
||||
|
||||
Из `pull.ts` (`ApplyPullActionsDeps.client`) и обхода дерева:
|
||||
```ts
|
||||
listSpaceTree(spaceId: string, rootPageId?: string): Promise<{ pages: PageNode[]; complete: boolean }>;
|
||||
getPageJson(pageId: string): Promise<{ id; slugId; title; parentPageId; spaceId; updatedAt; content }>;
|
||||
```
|
||||
|
||||
Из `push.ts` (`ApplyPushDeps.client`):
|
||||
```ts
|
||||
importPageMarkdown(pageId: string, fullMarkdown: string): Promise<{ updatedAt?: string; /* … */ }>;
|
||||
createPage(title: string, content: string, spaceId: string, parentPageId?: string): Promise<{ data: { id: string }; updatedAt?: string }>;
|
||||
deletePage(pageId: string): Promise<unknown>;
|
||||
movePage(pageId: string, parentPageId: string | null, position?: string): Promise<unknown>;
|
||||
renamePage(pageId: string, title: string): Promise<unknown>;
|
||||
```
|
||||
|
||||
Для непрерывного режима/детекции удалений (фаза B+, SPEC §8):
|
||||
```ts
|
||||
listRecentSince(spaceId: string | undefined, sinceIso: string | null, hardPageCap?: number): Promise<any[]>;
|
||||
listTrash(spaceId: string): Promise<any[]>;
|
||||
restorePage(pageId: string): Promise<unknown>;
|
||||
```
|
||||
|
||||
### 3.2. Маппинг на нативные сервисы gitmost
|
||||
|
||||
| Метод адаптера | Нативная реализация |
|
||||
| --- | --- |
|
||||
| `listSpaceTree(spaceId)` | `SpaceRepo.findById(spaceId, wsId)` + `PageRepo.getSpaceDescendants(spaceId, { includeContent: false })` → map в `PageNode { id, title, slugId, parentPageId, hasChildren }`. **`complete: true` всегда** (читаем БД, не пагинированный REST) → суппрессия `incomplete-fetch` из SPEC §8 нативно не срабатывает. |
|
||||
| `getPageJson(pageId)` | `PageRepo.findById(pageId, { includeContent: true })` → `{ id, slugId, title, parentPageId, spaceId, updatedAt, content }`. `content` — ProseMirror JSON в схеме `editor-ext`. |
|
||||
| `importPageMarkdown(pageId, fullMd)` | `parseDocmostMarkdown(fullMd)` → body; `await markdownToProseMirror(body)` → doc; **запись через collab** (см. §3.3). Вернуть `{ updatedAt }` свежей страницы. |
|
||||
| `createPage(title, body, spaceId, parent?)` | `PageService.create(userId, wsId, { spaceId, title, parentPageId }, provenance)` → shell; затем тело через collab (§3.3). Вернуть `{ data: { id }, updatedAt }`. |
|
||||
| `deletePage(pageId)` | `PageService.removePage(pageId, userId, wsId)` (soft-delete → Trash, обратимо). |
|
||||
| `movePage(pageId, parent, pos?)` | `PageService.movePage({ pageId, parentPageId: parent, position }, movedPage, provenance)`. **`position` обязателен** для Docmost-move — вычисляем `fractional-indexing-jittered` ключ между соседями (соседей берём из `PageRepo`). |
|
||||
| `renamePage(pageId, title)` | `PageService.update(page, { title }, user, provenance)`. |
|
||||
| `listRecentSince` | `PageRepo.getRecentPagesInSpace(spaceId, { … })`, фильтр по `updatedAt > since`. |
|
||||
| `listTrash(spaceId)` | `PageRepo` запрос с `deletedAt IS NOT NULL` по спейсу. |
|
||||
| `restorePage(pageId)` | `PageService.restore(...)`. |
|
||||
|
||||
`userId`/`wsId` берём из конфигурации спейса (сервисный аккаунт воркспейса или
|
||||
владелец спейса — см. §7). `provenance` всегда несёт `source: 'git-sync'` (§8).
|
||||
|
||||
### 3.3. Нативная запись тела (linchpin)
|
||||
|
||||
Подтверждено в коде: `CollaborationGateway.openDirectConnection(documentName, context)`
|
||||
([collaboration.gateway.ts:148](../apps/server/src/collaboration/collaboration.gateway.ts#L148-L150))
|
||||
+ паттерн `withYdocConnection`
|
||||
([collaboration.handler.ts:118-133](../apps/server/src/collaboration/collaboration.handler.ts#L118-L133)).
|
||||
Имя документа — `page.<pageId>` ([getPageId](../apps/server/src/collaboration/collaboration.util.ts#L163-L165)).
|
||||
Схему берём из `tiptapExtensions` ([collaboration.util.ts](../apps/server/src/collaboration/collaboration.util.ts)).
|
||||
|
||||
```ts
|
||||
// In-process body write — no loopback websocket, no service-user token.
|
||||
// Mirrors collaboration.handler.ts 'replace' operation exactly.
|
||||
private async writeBody(pageId: string, prosemirrorJson: JSONContent): Promise<void> {
|
||||
const conn = await this.collabGateway.openDirectConnection(
|
||||
`page.${pageId}`,
|
||||
{ actor: 'git-sync' }, // provenance flows into PersistenceExtension (see §8)
|
||||
);
|
||||
try {
|
||||
await conn.transact((doc) => {
|
||||
const fragment = doc.getXmlFragment('default');
|
||||
if (fragment.length > 0) fragment.delete(0, fragment.length);
|
||||
const next = TiptapTransformer.toYdoc(prosemirrorJson, 'default', tiptapExtensions);
|
||||
Y.applyUpdate(doc, Y.encodeStateAsUpdate(next));
|
||||
});
|
||||
} finally {
|
||||
await conn.disconnect();
|
||||
}
|
||||
// PersistenceExtension.onStoreDocument persists ydoc+content+textContent
|
||||
// consistently, stamps lastUpdatedSource, broadcasts 'page.updated'.
|
||||
}
|
||||
```
|
||||
|
||||
**Схема-совместимость (критично).** `markdownToProseMirror` производит
|
||||
ProseMirror JSON в схеме docmost-client, а `TiptapTransformer.toYdoc` валидирует
|
||||
его в схеме `editor-ext`. Аналогично на чтении `convertProseMirrorToMarkdown`
|
||||
получает `content` в схеме `editor-ext`. Эти две схемы **должны совпадать по
|
||||
именам нод/марок/атрибутов**, иначе ноды потеряются. Это и есть гейт §13.1.
|
||||
|
||||
---
|
||||
|
||||
## 4. `VaultGit` и git-бинарь
|
||||
|
||||
`VaultGit` (engine/git.ts) оставляем как есть — он шеллит в системный `git` через
|
||||
`execFile` (args-массив, без инъекций), всегда `cwd=<vaultPath>`. Константы:
|
||||
`DEFAULT_BRANCH = "main"`, `BOT_AUTHOR_NAME = "Docmost Sync"`,
|
||||
`BOT_AUTHOR_EMAIL = "docmost-sync@local"`; в push.ts: `DOCMOST_BRANCH = "docmost"`,
|
||||
`LAST_PUSHED_REF = "refs/docmost/last-pushed"`, провенанс-трейлеры
|
||||
`Docmost-Sync-Source: docmost|local`.
|
||||
|
||||
**Ops-требование:** в рантайм-образ gitmost добавить пакет `git`
|
||||
([Dockerfile](../Dockerfile)) — сейчас его там может не быть. Без бинаря
|
||||
`VaultGit.assertGitAvailable()` падает на старте цикла.
|
||||
|
||||
**Модель веток (пер-репо, SPEC §5):** `main` (правит человек/файлы) ↔ `docmost`
|
||||
(зеркало Docmost, пишет только движок) ↔ `merge-base` как базлайн;
|
||||
`refs/docmost/last-pushed` — что из `main` уже отражено в Docmost.
|
||||
|
||||
---
|
||||
|
||||
## 5. Топология vault: репозиторий на спейс
|
||||
|
||||
- Корень: `<DATA_DIR>/git-sync/<spaceId>/` — отдельный git-репо на каждый
|
||||
включённый спейс. `layout.ts` уже спейс-скоупный (корень спейса → `segments: []`).
|
||||
- Remote — пер-спейс (из конфигурации спейса/ENV). Изоляция конфликтов, блокировок
|
||||
и blast-radius.
|
||||
- `move-to-space` (страница меняет спейс) → **кросс-репо**: `delete` в исходном
|
||||
репо + `create` в целевом. Ловим по событию `PAGE_MOVED_TO_SPACE`.
|
||||
- Redis-lock ключ — `git-sync:lock:<spaceId>` (§9).
|
||||
|
||||
---
|
||||
|
||||
## 6. NestJS-модуль `GitSyncModule`
|
||||
|
||||
Структура (шаблон — `McpModule`):
|
||||
```
|
||||
apps/server/src/integrations/git-sync/
|
||||
git-sync.module.ts
|
||||
git-sync.constants.ts # QueueJob/event-имена, дефолты
|
||||
services/
|
||||
gitmost-datasource.service.ts # §3 адаптер
|
||||
git-sync.orchestrator.ts # @Interval + leader-lock + цикл по спейсам
|
||||
vault-registry.service.ts # путь vault на спейс, VaultGit-инстансы
|
||||
fractional-index.util.ts # position для move (reuse server util)
|
||||
listeners/
|
||||
page-change.listener.ts # подписка на EventName.PAGE_* + debounce
|
||||
git-sync.controller.ts # (опц.) ручной trigger/status для админа
|
||||
```
|
||||
|
||||
```ts
|
||||
@Module({
|
||||
imports: [DatabaseModule, EnvironmentModule, ScheduleModule.forRoot()],
|
||||
providers: [
|
||||
GitmostDataSourceService,
|
||||
GitSyncOrchestrator,
|
||||
VaultRegistryService,
|
||||
PageChangeListener,
|
||||
],
|
||||
})
|
||||
export class GitSyncModule {}
|
||||
```
|
||||
- Регистрируем в [app.module.ts](../apps/server/src/app.module.ts) рядом с `McpModule`.
|
||||
- Зависимости: `PageRepo`/`SpaceRepo` (через `DatabaseModule`), `PageService`,
|
||||
`CollaborationGateway` (экспортировать из `CollaborationModule`),
|
||||
`EnvironmentService`, ioredis-клиент.
|
||||
- `ScheduleModule.forRoot()` уже подключается в `TelemetryModule`; повторный вызов
|
||||
безопасен, но лучше вынести в общий модуль или убедиться, что forRoot один раз.
|
||||
|
||||
---
|
||||
|
||||
## 7. Конфигурация
|
||||
|
||||
### 7.1. Per-space (UI) — `space.settings.gitSync`
|
||||
Расширяем существующий паттерн `settings.sharing` / `settings.comments`.
|
||||
|
||||
Сервер:
|
||||
- `UpdateSpaceDto` ([update-space.dto.ts](../apps/server/src/core/space/dto/update-space.dto.ts)):
|
||||
добавить `@IsOptional() @IsBoolean() gitSyncEnabled?: boolean;` (+ опц.
|
||||
`gitSyncRemote?: string`, если решим хранить remote в БД, а не только в ENV).
|
||||
- `SpaceService.updateSpace(dto, wsId)`
|
||||
([space.service.ts:120](../apps/server/src/core/space/services/space.service.ts#L120)):
|
||||
обработать как `disablePublicSharing`/`allowViewerComments`.
|
||||
- `SpaceRepo`: добавить `updateGitSyncSettings(spaceId, wsId, prefKey, prefValue, trx?)`
|
||||
по образцу `updateSharingSettings`
|
||||
([space.repo.ts:92](../apps/server/src/database/repos/space/space.repo.ts#L92)) —
|
||||
jsonb-merge в `settings.gitSync.<key>`.
|
||||
- Гард: CASL `SpaceCaslAction.Manage / SpaceCaslSubject.Settings` (как в
|
||||
[space.controller.ts:147](../apps/server/src/core/space/space.controller.ts#L147)).
|
||||
|
||||
Клиент:
|
||||
- Тоггл в форме настроек спейса
|
||||
([edit-space-form.tsx](../apps/client/src/features/space/components/edit-space-form.tsx))
|
||||
через `useUpdateSpaceMutation()` → `updateSpace({ spaceId, gitSyncEnabled })`.
|
||||
Образец — `mcp-settings.tsx`. `readOnly` при отсутствии `Manage/Settings`.
|
||||
|
||||
Форма `space.settings.gitSync`:
|
||||
```jsonc
|
||||
{ "gitSync": { "enabled": true, "remote": "git@…", "branch": "main" } }
|
||||
```
|
||||
|
||||
### 7.2. Секреты/тюнинг (ENV) — `EnvironmentService`
|
||||
Движковый `settings.ts` (zod, читает `.env`) **заменяем** на чтение из gitmost
|
||||
`EnvironmentService`: `parseSettings(env)` оставляем как чистую функцию для тестов,
|
||||
но в проде собираем `Settings` из `EnvironmentService`-геттеров.
|
||||
|
||||
Новые переменные (объявить в
|
||||
[environment.validation.ts](../apps/server/src/integrations/environment/environment.validation.ts)
|
||||
class-validator-декораторами, геттеры — в
|
||||
[environment.service.ts](../apps/server/src/integrations/environment/environment.service.ts)):
|
||||
|
||||
| ENV | Назначение | Обяз. |
|
||||
| --- | --- | --- |
|
||||
| `GIT_SYNC_ENABLED` | глобальный мастер-выключатель | нет (default false) |
|
||||
| `GIT_SYNC_DATA_DIR` | корень vault'ов (default `<DATA_DIR>/git-sync`) | нет |
|
||||
| `GIT_SYNC_REMOTE_TEMPLATE` | шаблон remote, напр. `git@host:vault-{spaceId}.git` | нет |
|
||||
| `GIT_SYNC_SSH_KEY_PATH` / креды remote | доступ к git-remote (secret) | по ситуации |
|
||||
| `GIT_SYNC_POLL_INTERVAL_MS` | страховочный поллинг (default 15000) | нет |
|
||||
| `GIT_SYNC_DEBOUNCE_MS` | окно дебаунса событий (default 2000) | нет |
|
||||
| `GIT_SYNC_SERVICE_USER_ID` | от чьего имени писать в Docmost | да (если синк включён) |
|
||||
|
||||
> git-remote = доступ ко всей вики спейса (SPEC §12): креды только в ENV/secret
|
||||
> store, никогда в БД/коммиты. В UI — только `enabled` (+ опц. имя remote из
|
||||
> заранее разрешённого списка).
|
||||
|
||||
---
|
||||
|
||||
## 8. Провенанс и loop-guard
|
||||
|
||||
### 8.1. Значение `'git-sync'`
|
||||
Сегодня `lastUpdatedSource ∈ { 'user', 'agent' }`
|
||||
([persistence.extension.ts:132-134](../apps/server/src/collaboration/extensions/persistence.extension.ts#L132-L134)).
|
||||
Добавляем `'git-sync'`:
|
||||
- `PersistenceExtension`: `context.actor === 'git-sync'` → `lastUpdatedSource = 'git-sync'`.
|
||||
- Снапшот истории для `'git-sync'` — дебаунс (как у человека), а не немедленный
|
||||
(немедленный — только для `'agent'`,
|
||||
[persistence.extension.ts:321](../apps/server/src/collaboration/extensions/persistence.extension.ts#L321)).
|
||||
- Для `create/move/rename/delete` через `PageService` передаём
|
||||
`AuthProvenanceData` c `source: 'git-sync'` (тип уже используется для агента —
|
||||
расширить допустимые значения; точную форму подтвердить на реализации).
|
||||
- Клиент: в истории
|
||||
([history-item.tsx:128](../apps/client/src/features/page-history/components/history-item.tsx#L128))
|
||||
не показывать агентский бейдж/дип-линк для `'git-sync'`; добавить значение в
|
||||
тип [page.types.ts:23-26](../apps/client/src/features/page-history/types/page.types.ts#L23-L26)
|
||||
(опц. свой бейдж «sync»).
|
||||
|
||||
### 8.2. Подавление петли (SPEC §10)
|
||||
На pull-стороне игнорируем страницу как «свою запись», если:
|
||||
`page.lastUpdatedSource === 'git-sync'` **И** `bodyHash(exportedBody)` совпадает
|
||||
с последним запушенным (`PushedPageRecord.bodyHash` из `push.ts`). После записи в
|
||||
Docmost сохраняем `updatedAt` ответа, чтобы поллинг-страховка не утянул свою же
|
||||
запись обратно.
|
||||
|
||||
---
|
||||
|
||||
## 9. Single-writer (Redis leader-lock)
|
||||
|
||||
В кодовой базе `@Interval`-задачи (`trash-cleanup`, `telemetry`, `session-cleanup`)
|
||||
**не защищены** от мультиинстанса. Для синка добавляем явный лок.
|
||||
|
||||
- ioredis уже есть (`RedisModule` из `@nestjs-labs/nestjs-ioredis`,
|
||||
[app.module.ts](../apps/server/src/app.module.ts); прямой `RedisClient`
|
||||
используется в collab-gateway).
|
||||
- Лок на спейс: `SET git-sync:lock:<spaceId> <instanceId> NX PX <ttl>`; держим
|
||||
цикл только при успехе, продлеваем по heartbeat, освобождаем в `finally`
|
||||
(Lua-CAS на удаление по `instanceId`, чтобы не снять чужой лок).
|
||||
- TTL > максимальной длительности цикла; на краше лок истекает сам.
|
||||
|
||||
```ts
|
||||
// Acquire per-space leadership; returns false if another replica holds it.
|
||||
private async acquire(spaceId: string): Promise<boolean> {
|
||||
const ok = await this.redis.set(`git-sync:lock:${spaceId}`, this.instanceId, 'PX', LOCK_TTL_MS, 'NX');
|
||||
return ok === 'OK';
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Планировщик и событийные триггеры
|
||||
|
||||
- **События (основной триггер).** `PageChangeListener` подписывается на
|
||||
`EventName.PAGE_CREATED | PAGE_UPDATED | PAGE_MOVED | PAGE_SOFT_DELETED |
|
||||
PAGE_RESTORED | PAGE_MOVED_TO_SPACE` и job `PAGE_CONTENT_UPDATED`
|
||||
([event.contants.ts](../apps/server/src/common/events/event.contants.ts)).
|
||||
Фильтр по `spaceId` (только включённые спейсы) → дебаунс (`GIT_SYNC_DEBOUNCE_MS`)
|
||||
→ ставит pull/push-цикл спейса в очередь оркестратора.
|
||||
- Loop-guard: события от собственных записей (`source==='git-sync'` + совпавший
|
||||
хэш) пропускаем (§8.2).
|
||||
- **Поллинг-страховка.** `@Interval(GIT_SYNC_POLL_INTERVAL_MS)` в оркестраторе:
|
||||
по каждому включённому спейсу (под локом) — реконсиляция (`listRecentSince` +
|
||||
`listTrash`), ловит пропущенные события и стартовую сверку после простоя
|
||||
(SPEC §12).
|
||||
- Один цикл на спейс за раз (внутри-процессный мьютекс на `spaceId` поверх
|
||||
Redis-лока).
|
||||
|
||||
---
|
||||
|
||||
## 11. Потоки данных (walkthroughs)
|
||||
|
||||
### 11.1. Первичный клон спейса (initial clone, SPEC §12)
|
||||
1. `VaultGit.ensureRepo()` + `ensureBranch('docmost','main')` + `checkout('docmost')`.
|
||||
2. `dataSource.listSpaceTree(spaceId)` → `{ pages, complete:true }`.
|
||||
3. `readExisting({ listTracked: () => git.listTrackedFiles('*.md'), readFile })`.
|
||||
4. `computePullActions({ pages, treeComplete:true, existing })` → план.
|
||||
5. `applyPullActions(deps, actions, vaultRoot)`: на каждую страницу
|
||||
`getPageJson` → `stabilizePageFile(content, meta)` (export→import→export
|
||||
fixpoint, SPEC §11) → запись файла; затем `stageAll` + `commit` (трейлер
|
||||
`docmost`) на `docmost`; `checkout('main')` + `merge('docmost')`.
|
||||
6. Зафиксировать max `updatedAt` как стартовый `T_last`; `git push` в remote.
|
||||
|
||||
### 11.2. Docmost → FS (pull-цикл)
|
||||
Триггер: событие/поллинг → (под локом) шаги §11.1 п.1–5 инкрементально. 3-way
|
||||
merge `docmost→main` делает git: непересекающиеся правки сливаются, реальное
|
||||
пересечение → conflict-маркеры в файле. **При конфликте push этой страницы в
|
||||
Docmost блокируется** до ручного резолва (SPEC §9; фаза D).
|
||||
|
||||
### 11.3. FS → Docmost (push-цикл)
|
||||
`runPush(deps, { dryRun })`:
|
||||
1. `git.ensureRepo` / `isMergeInProgress` (abort при merge) / `checkout('main')`.
|
||||
2. `stageAll` + `commit('local: working-tree changes')` (локально, в Docmost не шлёт).
|
||||
3. База диффа: `readRef(LAST_PUSHED_REF)` ?? `docmost`; `revParse('main')` → `pushedCommit`.
|
||||
4. `diffNameStatus(base, 'main')` → changes; префетч `metaAt(path, side)`.
|
||||
5. `computePushActions({ changes, metaAt })` → creates/updates/deletes/renamesMoves/skipped.
|
||||
6. `dryRun` → лог плана и выход (клиент НЕ создаётся).
|
||||
7. `--apply`: `makeClient(settings)` → наш `GitmostDataSource`;
|
||||
`applyPushActions`:
|
||||
- update → `importPageMarkdown(pageId, fullMd)` (collab-write, §3.3);
|
||||
- create → `createPage(...)` → записать присвоенный `pageId` обратно в meta;
|
||||
- delete → `deletePage(pageId)` (Trash);
|
||||
- rename/move → `classifyRenameMoves` → `movePage`/`renamePage`;
|
||||
- при пустых failures: `updateRef(LAST_PUSHED_REF, pushedCommit)` +
|
||||
`fastForwardBranch('docmost', pushedCommit)`.
|
||||
8. Записать `bodyHash` + `updatedAt` (loop-guard, §8.2); `git push`.
|
||||
|
||||
---
|
||||
|
||||
## 12. Фазирование
|
||||
|
||||
- **A. Каркас + односторонний pull (нативно).** `packages/git-sync` (вендоринг
|
||||
§2), `GitmostDataSource` (чтение через репозитории), `GitSyncModule`, конфиг из
|
||||
`EnvironmentService`, ручной/однократный pull-цикл на один спейс. **Гейт §13.1.**
|
||||
- **B. Push + непрерывность.** Нативная запись (§3.3), `runPush`, ветки/refs,
|
||||
loop-guard (§8), Redis-лок (§9), `@Interval` + `PageChangeListener` (§10).
|
||||
- **C. Per-space UI.** `space.settings.gitSync` (§7.1), DTO/сервис/репо/гард,
|
||||
тоггл на клиенте, скоуп оркестратора по включённым спейсам.
|
||||
- **D. Харднинг.** Conflict-gating (SPEC §9), удаления через Trash + git (§5),
|
||||
стартовая реконсиляция и `move-to-space` кросс-репо, провенанс на клиенте,
|
||||
Dockerfile `git`, полный набор тестов.
|
||||
|
||||
---
|
||||
|
||||
## 13. Тестирование
|
||||
|
||||
### 13.1. Гейт идемпотентности (блокирует фазу B)
|
||||
Перенести round-trip-харнес docmost-sync (`roundtrip.ts` + `test/fixtures/corpus`)
|
||||
в тесты `packages/git-sync`, но прогонять **против схемы `editor-ext`**:
|
||||
`content (editor-ext) → convertProseMirrorToMarkdown → markdownToProseMirror →
|
||||
TiptapTransformer.toYdoc(…, tiptapExtensions) → fromYdoc → canonicalizeContent`
|
||||
должно давать `docsCanonicallyEqual === true`. Любая потеря нод/атрибутов =
|
||||
расхождение схем → чинить `docmost-schema.ts` под `editor-ext`.
|
||||
|
||||
### 13.2. Юнит (чистая логика, переносится как есть)
|
||||
`reconcile` (planReconciliation / decideAbsenceDeletions / mass-delete guards),
|
||||
`layout` (коллизии/санитизация), `computePullActions`, `computePushActions`,
|
||||
`classifyRenameMoves`, `bodyHash`.
|
||||
|
||||
### 13.3. Интеграция (нативный адаптер)
|
||||
`GitmostDataSource` против тестовой БД: `listSpaceTree`/`getPageJson` корректно
|
||||
маппят; `createPage`/`movePage`/`deletePage`/`importPageMarkdown` пишут через
|
||||
collab и проставляют `lastUpdatedSource='git-sync'`; loop-guard не зацикливается
|
||||
(write → poll → no-op).
|
||||
|
||||
### 13.4. e2e (под локом)
|
||||
Полный pull→push round-trip на временном vault + временном спейсе: правка в
|
||||
Docmost доезжает в файл и наоборот; конфликт даёт маркеры и блокирует push.
|
||||
|
||||
---
|
||||
|
||||
## 14. Риски и открытые пункты
|
||||
|
||||
1. **Схема-совместимость конвертера** (§3.3, §13.1) — главный риск; гейт
|
||||
обязателен до фазы B.
|
||||
2. **`AuthProvenanceData`** — точную форму типа подтвердить; возможно, потребует
|
||||
расширения enum источника на сервере и в истории.
|
||||
3. **Согласованность Yjs** — писать строго через `openDirectConnection`/`transact`;
|
||||
не трогать `content`-колонку напрямую.
|
||||
4. **`position` для move** — обязателен в Docmost-move; нужен
|
||||
`fractional-indexing-jittered` между соседями (соседей брать сортировкой
|
||||
`position COLLATE "C"`).
|
||||
5. **`git` в рантайме** — добавить в Dockerfile.
|
||||
6. **`ScheduleModule.forRoot()`** — не задублировать `forRoot`.
|
||||
7. **Сервисный пользователь записи** (`GIT_SYNC_SERVICE_USER_ID`) — от чьего имени
|
||||
идут create/move (влияет на `creatorId`/права); согласовать политику.
|
||||
8. **Конфликты и удаления** — фаза D строго по SPEC §8/§9 (маркеры никогда не
|
||||
уезжают в Docmost).
|
||||
|
||||
---
|
||||
|
||||
## 15. Чек-лист изменений по файлам
|
||||
|
||||
**Новый пакет**
|
||||
- `packages/git-sync/**` — движок + чистый конвертер (§2), `package.json`
|
||||
(`@docmost/git-sync`, `workspace:*`), `tsconfig.json`.
|
||||
|
||||
**Сервер (`apps/server/src`)**
|
||||
- `integrations/git-sync/**` — модуль, оркестратор, адаптер, листенер (§6).
|
||||
- `app.module.ts` — импорт `GitSyncModule`.
|
||||
- `collaboration/collaboration.module.ts` — экспорт `CollaborationGateway`.
|
||||
- `collaboration/extensions/persistence.extension.ts` — источник `'git-sync'` (§8.1).
|
||||
- `core/space/dto/update-space.dto.ts` — `gitSyncEnabled?` (§7.1).
|
||||
- `core/space/services/space.service.ts` — обработка флага.
|
||||
- `database/repos/space/space.repo.ts` — `updateGitSyncSettings` (§7.1).
|
||||
- `integrations/environment/environment.validation.ts` + `environment.service.ts` —
|
||||
новые ENV (§7.2).
|
||||
- `Dockerfile` — пакет `git`.
|
||||
|
||||
**Клиент (`apps/client/src`)**
|
||||
- `features/space/components/edit-space-form.tsx` — тоггл git-sync.
|
||||
- `features/space/types` — поле `settings.gitSync`.
|
||||
- `features/page-history/types/page.types.ts` + `components/history-item.tsx` —
|
||||
значение `'git-sync'` в `lastUpdatedSource`.
|
||||
|
||||
**Корень**
|
||||
- `pnpm-workspace.yaml` уже покрывает `packages/*`; `apps/server/package.json` —
|
||||
зависимость `@docmost/git-sync: workspace:*`.
|
||||
@@ -1,359 +0,0 @@
|
||||
# Мобильное приложение gitmost — исследование и план
|
||||
|
||||
> Статус: исследовательский + проектный документ.
|
||||
> Контекст: gitmost — форк Docmost, чистое веб-приложение. Отдельного
|
||||
> мобильного (нативного/устанавливаемого) приложения **нет**.
|
||||
> Цель: определить путь к мобильным приложениям — **iOS обязательно, Android
|
||||
> как пойдёт** — с заделом на оффлайн в будущем (оффлайн сейчас не требуется).
|
||||
|
||||
Документ фиксирует, что уже есть в коде, почему путь к мобилке предопределён
|
||||
устройством продукта, сравнивает варианты и описывает рекомендуемый план с
|
||||
привязкой к файлам.
|
||||
|
||||
---
|
||||
|
||||
## 1. TL;DR
|
||||
|
||||
1. **Нативного приложения нет.** В проекте отсутствуют Capacitor, React Native,
|
||||
Cordova и т.п. Мобильного клиента ещё не начинали.
|
||||
2. **Адаптивная веб-версия — есть, и довольно проработанная.** Веб-клиент
|
||||
открывается с телефона как mobile-friendly сайт: сворачиваемый сайдбар-drawer,
|
||||
отдельные мобильные компоненты (история, поиск, хлебные крошки), responsive-
|
||||
примитивы Mantine, mobile-tuned `viewport`. Это готовый фундамент UI.
|
||||
3. **Ядро продукта — веб-редактор — нативно не воспроизвести.** TipTap 3
|
||||
(ProseMirror) + совместное редактирование на Yjs/Hocuspocus плотно сшиты с
|
||||
React. Production-порта Yjs под Swift/Kotlin нет. Любой реалистичный путь
|
||||
оставляет редактор в **WebView**.
|
||||
4. **API уже готов к нативному клиенту.** Сервер принимает JWT не только из
|
||||
cookie, но и из заголовка `Authorization: Bearer`. Есть точка входа для
|
||||
вебсокета совместного редактирования (`POST /auth/collab-token`).
|
||||
5. **Рекомендуемый путь — Capacitor:** обернуть существующий React-SPA в
|
||||
нативную оболочку (iOS + Android из одного кода), добавить нативные плагины
|
||||
(push, биометрия, share, файлы). Эволюция в гибрид (нативная навигация +
|
||||
WebView-редактор) делается потом инкрементально, без переписывания.
|
||||
6. **Оффлайн-будущее уже заложено** (Yjs + `y-indexeddb`). Детальный план —
|
||||
в [offline-sync-plan.md](offline-sync-plan.md); мобильное приложение этот
|
||||
план переиспользует, а не дублирует.
|
||||
7. **Главный блокер — не технический, а лицензионный.** AGPL форка несовместима
|
||||
с условиями App Store, если зашивать веб-клиент в бинарник: DRM/usage-rules
|
||||
Apple = «дополнительные ограничения», запрещённые AGPLv3 §10. Развязки —
|
||||
грузить клиент с сервера (не из `.ipa`), PWA или sideload. Детали и матрица —
|
||||
в §9; закрывать **до** кода обёртки.
|
||||
|
||||
---
|
||||
|
||||
## 2. Текущее состояние (как есть)
|
||||
|
||||
### 2.1. Стек
|
||||
|
||||
| Слой | Технологии |
|
||||
|---|---|
|
||||
| Бэкенд | NestJS 11 + Fastify, Kysely/Postgres, Redis/BullMQ. API в стиле RPC-POST (соглашение Docmost). Аутентификация — JWT. |
|
||||
| Фронт | React 18 + Vite + Mantine + TanStack Query + i18next. Обычный SPA. |
|
||||
| Ядро (редактор) | TipTap 3 (ProseMirror) + совместное редактирование на Yjs через Hocuspocus — см. [page-editor.tsx](../apps/client/src/features/editor/page-editor.tsx). |
|
||||
| Оффлайн-фундамент | `yjs` + `y-indexeddb` уже в зависимостях клиента (локальная CRDT-копия тела документа). |
|
||||
|
||||
### 2.2. Мобильного приложения нет
|
||||
|
||||
В `package.json` и `apps/*/package.json` нет `capacitor`, `react-native`,
|
||||
`cordova`, `expo`. Нативной оболочки в репозитории не заведено.
|
||||
|
||||
### 2.3. Адаптивная веб-версия — есть
|
||||
|
||||
| Что | Где |
|
||||
|---|---|
|
||||
| Адаптивная оболочка Mantine `AppShell` с `breakpoint: "sm"`, раздельные состояния `collapsed.mobile` / `collapsed.desktop` | [global-app-shell.tsx](../apps/client/src/components/layouts/global/global-app-shell.tsx) (L85–99) |
|
||||
| Отдельный мобильный сайдбар-drawer (`mobileSidebarAtom` отделён от `desktopSidebarAtom`), авто-закрытие при навигации по дереву | [sidebar-atom.ts](../apps/client/src/components/layouts/global/hooks/atoms/sidebar-atom.ts), [space-tree-row.tsx](../apps/client/src/features/page/tree/components/space-tree-row.tsx) (L147–148) |
|
||||
| Мобильная модалка истории + свой CSS | [history-modal.tsx](../apps/client/src/features/page-history/components/history-modal.tsx) (L17–19), `history-modal-mobile.tsx` |
|
||||
| Мобильный контрол поиска | [search-control.tsx](../apps/client/src/features/search/components/search-control.tsx) (L38–42) |
|
||||
| Мобильный рендер хлебных крошек через `useMediaQuery` | [breadcrumb.tsx](../apps/client/src/features/page/components/breadcrumbs/breadcrumb.tsx) (L41) |
|
||||
| Responsive-примитивы `hiddenFrom`/`visibleFrom` (~16 мест), медиа-запросы в CSS-модулях | по всему `apps/client/src` |
|
||||
| Mobile-tuned viewport (`width=device-width, user-scalable=no`) | [index.html](../apps/client/index.html) (L8) |
|
||||
|
||||
> Важно: адаптив проверялся в мобильном **браузере**, а не в WebView нативной
|
||||
> оболочки. Перед сборкой приложения нужно прогнать UI как PWA/в WebView и
|
||||
> отловить отличия (жесты, экранная клавиатура/IME в редакторе, safe-area).
|
||||
|
||||
### 2.4. Готовность API к нативному клиенту
|
||||
|
||||
- **Bearer-токен уже поддержан.** JWT извлекается из cookie **или** из заголовка
|
||||
`Authorization`: см. [jwt.strategy.ts](../apps/server/src/core/auth/strategies/jwt.strategy.ts) (L27–29).
|
||||
Серверная сторона нативной авторизации менять не нужно.
|
||||
- **Токен сейчас не возвращается в теле логина.** [`login`](../apps/server/src/core/auth/auth.controller.ts)
|
||||
(L55–105) кладёт JWT только в `httpOnly`-cookie ([`setAuthCookie`](../apps/server/src/core/auth/auth.controller.ts) L222–230).
|
||||
- **Точка входа вебсокета коллаборации:** [`POST /auth/collab-token`](../apps/server/src/core/auth/auth.controller.ts) (L187–193).
|
||||
- **CORS открыт без конфигурации:** [`app.enableCors()`](../apps/server/src/main.ts) (L144).
|
||||
- **OpenAPI/Swagger отсутствует** (`@nestjs/swagger` не подключён) — авто-генерации
|
||||
типизированного клиента сейчас нет.
|
||||
|
||||
---
|
||||
|
||||
## 3. Почему путь к мобилке предопределён
|
||||
|
||||
Три факта диктуют решение независимо от моды:
|
||||
|
||||
1. **Редактор практически невозможно переписать нативно.** ProseMirror + весь
|
||||
набор TipTap-расширений + Yjs-CRDT — это не «поле ввода». Нативного
|
||||
production-порта Yjs под Swift/Kotlin нет (есть Rust `yrs` с биндингами, но
|
||||
это отдельный тяжёлый проект). Переписывание ядра нативно = годы и вечное
|
||||
расхождение с веб-версией. **Вывод: редактор остаётся в WebView.**
|
||||
2. **API уже умеет нативного клиента** (Bearer, collab-token).
|
||||
3. **Оффлайн-фундамент уже заложен** на веб-уровне (Yjs + `y-indexeddb`),
|
||||
и он работает внутри WebView.
|
||||
|
||||
---
|
||||
|
||||
## 4. Три возможных пути
|
||||
|
||||
| Путь | Суть | Плюсы | Минусы | Вердикт |
|
||||
|---|---|---|---|---|
|
||||
| **A. Полностью нативно** (Swift/Kotlin) | Переписать всё, включая редактор и CRDT-синк | Максимально нативный UX | Воспроизвести ProseMirror + расширения + Yjs; несоразмерные трудозатраты; вечное отставание от веба | ❌ Не наш случай |
|
||||
| **B. WebView-обёртка SPA (Capacitor)** | Обернуть существующий React-клиент в нативную оболочку, native-возможности — плагинами | Реюз ~100% кода (редактор, коллаборация, оффлайн); один кодовый бэйз → iOS+Android; быстро | Менее «нативно»; риск отказа App Store за «просто сайт» (4.2) — лечится нативной ценностью | ✅ Рекомендуется |
|
||||
| **C. Гибрид: нативная оболочка + WebView-редактор** | Навигация/списки/поиск/логин — нативно (React Native/Swift), экран редактирования — web в WebView | Лучший UX; путь Notion/Linear | Заметно больше работы; нужен мост JS↔native | ⚖️ Цель эволюции из B |
|
||||
|
||||
---
|
||||
|
||||
## 5. Рекомендуемый путь
|
||||
|
||||
**B (Capacitor) как первый релиз, с заложенной эволюцией в C.**
|
||||
|
||||
Почему:
|
||||
- Capacitor создан под сценарий «есть веб-приложение → хочу его в App Store с
|
||||
нативными возможностями». Переиспользуется весь React-клиент и, главное,
|
||||
редактор — то, что нативно не сделать.
|
||||
- Один кодовый бэйз закрывает «iOS обязательно» и «Android как пойдёт»
|
||||
одновременно, без второй команды.
|
||||
- Адаптивная вёрстка уже есть (см. §2.3) — переверстывать под телефон с нуля
|
||||
не нужно; работа смещается в нативную обвязку.
|
||||
- Оффлайн-будущее подготовлено (Yjs + `y-indexeddb`); см.
|
||||
[offline-sync-plan.md](offline-sync-plan.md).
|
||||
- Когда упрётесь в UX отдельных экранов — их по одному выносят в нативную
|
||||
оболочку, оставив редактор в WebView. То есть B → C делается инкрементально.
|
||||
|
||||
Почему **не** чистый React Native сразу: редактор всё равно придётся держать в
|
||||
WebView (ядро web-only), но при этом теряется прямой реюз остального React-кода
|
||||
и появляется мост как обязательная сложность с первого дня — для iOS-first
|
||||
старта это лишний оверхед.
|
||||
|
||||
> Альтернатива: если критичен максимально нативный UX с первого релиза и есть
|
||||
> ресурс — сразу путь C на React Native (Expo) с WebView только под редактор.
|
||||
> Это сознательный размен «больше работы сейчас» за «более нативное ощущение».
|
||||
|
||||
⚠️ **Лицензионная оговорка к iOS.** Обычный Capacitor зашивает веб-билд
|
||||
`apps/client` в `.ipa` — для публикации в App Store это **нарушает AGPL**
|
||||
(см. §9). Выбор Capacitor для **Android** остаётся в силе, но на **iOS**
|
||||
веб-клиент нельзя бандлить в бинарник: либо грузить его с сервера
|
||||
(`server.url`), либо PWA. То есть рекомендация «B (Capacitor)» применима к
|
||||
Android как есть, а к iOS — только в конфигурации без зашитого AGPL.
|
||||
|
||||
---
|
||||
|
||||
## 6. Что доработать на бэкенде
|
||||
|
||||
Немного, но конкретно:
|
||||
|
||||
1. **Выдача токена в теле ответа для нативного хранения.** Сейчас логин кладёт
|
||||
JWT только в `httpOnly`-cookie и не возвращает его в body. На мобиле
|
||||
`httpOnly`-cookie между разными origin (`capacitor://localhost` ↔ API) — боль
|
||||
с SameSite/CORS. Чище: мобильный логин-флоу, возвращающий JWT в ответе, чтобы
|
||||
хранить его в Keychain/Keystore и слать как `Authorization: Bearer`. Сервер
|
||||
уже принимает Bearer — менять надо только **выдачу**.
|
||||
Файлы: [auth.controller.ts](../apps/server/src/core/auth/auth.controller.ts).
|
||||
2. **CORS.** Сейчас [`app.enableCors()`](../apps/server/src/main.ts) (L144) без
|
||||
конфигурации. Под мобильные origin'ы и для безопасности задать явный whitelist.
|
||||
3. **Push-уведомления.** Модуль `notification` уже есть — добавить регистрацию
|
||||
device-token и интеграцию **APNs** (iOS) / **FCM** (Android).
|
||||
4. **Опционально — OpenAPI/Swagger.** Сейчас спецификации нет; добавить
|
||||
`@nestjs/swagger` дёшево и сильно ускорит мобильную разработку
|
||||
(типизированный клиент).
|
||||
|
||||
---
|
||||
|
||||
## 7. Android-специфика
|
||||
|
||||
На пути Capacitor Android едет почти бесплатно (`npx cap add android` из того же
|
||||
веб-билда), но есть нюансы:
|
||||
|
||||
- **Движок в плюс.** Android System WebView (Chromium) обновляется через Play
|
||||
Store независимо от ОС и обычно свежее iOS WKWebView. Более рискованный движок
|
||||
по совместимости — это iOS, а не Android.
|
||||
- **Фрагментация.** Дешёвые/старые устройства с малой памятью и устаревшим
|
||||
WebView; стек тяжёлый (ProseMirror + Yjs + mermaid + katex + excalidraw) —
|
||||
тестировать на бюджетных аппаратах.
|
||||
- **Обвязка под Android:** аппаратная/жестовая кнопка «Назад» (навигация внутри
|
||||
приложения, а не выход), **FCM** для push, Android App Links (вместо iOS
|
||||
Universal Links), подписание и Play Console.
|
||||
- **Главный риск именно для Android — ввод текста в ProseMirror на Gboard/IME.**
|
||||
Историческая боль `contenteditable` на Android (прыжки курсора, дубли символов
|
||||
при композиции). Стало лучше, но **проверять в первую очередь и рано**.
|
||||
- **Магазин.** Google Play лояльнее к webview-обёрткам, чем App Store; риск
|
||||
«отклонят как просто сайт» для Play практически неактуален.
|
||||
|
||||
---
|
||||
|
||||
## 8. iOS-специфика
|
||||
|
||||
- **WKWebView** на движке WebKit жёстко привязан к версии ОС — это более
|
||||
рискованный по совместимости движок (тестировать прежде всего его).
|
||||
- **App Store guideline 4.2 (minimum functionality).** Чистая webview-обёртка
|
||||
рискует отклонением «это просто сайт». Лечится реальной нативной ценностью:
|
||||
push, share-extension, биометрический разблок, оффлайн-кэш — всё это Capacitor
|
||||
даёт плагинами.
|
||||
- **safe-area** под «чёлку»/системные панели, поведение экранной клавиатуры в
|
||||
редакторе.
|
||||
|
||||
---
|
||||
|
||||
## 9. Лицензионный блокер: AGPL ↔ App Store (iOS)
|
||||
|
||||
> Это не инженерная, а **лицензионная** задача — закрывать её надо **до** кода
|
||||
> обёртки, иначе можно сделать приложение, которое некуда легально опубликовать.
|
||||
> Ниже — инженерно-лицензионный разбор, **не** юридическая консультация; финально
|
||||
> подтверждать у того, кто разбирается в лицензиях.
|
||||
|
||||
### 9.1. Суть конфликта
|
||||
|
||||
gitmost — форк Docmost под **AGPL-3.0** (константа форка: «100% open, AGPL-only»).
|
||||
Две вещи несовместимы:
|
||||
|
||||
- **AGPLv3 §10** (последний абзац) запрещает накладывать на получателя кода
|
||||
**любые дополнительные ограничения** сверх самой лицензии.
|
||||
- **Стандартный EULA App Store** ровно их и накладывает: **FairPlay/DRM**,
|
||||
привязка установки к Apple ID с лимитом устройств (**usage rules**), запрет
|
||||
свободного перераспространения бинарника.
|
||||
|
||||
Приняв условия Apple, чтобы попасть в App Store, вы нарушаете AGPL кода, который
|
||||
раздаёте.
|
||||
|
||||
### 9.2. Почему это бьёт именно по форку
|
||||
|
||||
Запрет «дополнительных ограничений» связывает **лицензиатов, но не самого
|
||||
правообладателя**: владелец 100% копирайта может опубликовать свой код в App Store.
|
||||
Но в gitmost бóльшая часть копирайта принадлежит **upstream-Docmost** и
|
||||
контрибьюторам — вы выступаете дистрибьютором *чужого* AGPL-кода и не можете
|
||||
единолично добавить App-Store-исключение.
|
||||
|
||||
Прецеденты: **VLC** (удалён из App Store в 2011 по жалобе на конфликт GPL с
|
||||
условиями стора; вернулся только после перелицензирования и согласия
|
||||
правообладателей), **GNU Go** — снят по той же причине. Это не теоретический риск.
|
||||
|
||||
### 9.3. Ключевой принцип развязки: лицензия смотрит на `.ipa`, а не на устройство
|
||||
|
||||
Определяющее — **что раздаёт сам Apple** (`.ipa` под FairPlay) и **кто раздаёт
|
||||
AGPL-байты**, а не то, окажутся ли они в итоге на устройстве:
|
||||
|
||||
- AGPL **внутри `.ipa`** → получен под ограничениями Apple → **нарушение**.
|
||||
- AGPL **скачан с вашего сервера** → получен от вас под AGPL (исходники открыты,
|
||||
§13 выполнен) → ограничения Apple на него **не** накладываются, даже если бандл
|
||||
кэшируется в песочнице приложения.
|
||||
|
||||
Следствие: **офлайн на iOS легально достижим** — если кэшированный бандл пришёл с
|
||||
вашего сервера, а не из `.ipa`. Ограничение тут не лицензионное, а в **ревью
|
||||
Apple** (см. §9.5).
|
||||
|
||||
### 9.4. Варианты «грузить веб-клиент с сервера»
|
||||
|
||||
**A. WebView навигируется на хостед-клиент (`server.url`).** Capacitor умеет
|
||||
`server: { url: 'https://app.example.com' }` — оболочка грузит WebView с удалённого
|
||||
URL, мост и нативные плагины по-прежнему инжектятся. В `.ipa` — ноль AGPL.
|
||||
|
||||
- Плюс: лицензионно самый чистый; **origin = ваш домен**, поэтому cookie/CORS
|
||||
работают как в браузере (боль `capacitor://localhost` ↔ API из §6 исчезает —
|
||||
токен в body/Keychain может и не понадобиться).
|
||||
- Минус: холодный старт требует сети; сервер лёг → приложение кирпич; офлайна по
|
||||
умолчанию нет.
|
||||
|
||||
**B. OTA: пустой шелл скачивает и кэширует бандл.** Шелл при первом запуске тянет
|
||||
JS-бандл с вашего сервера и кэширует как веб-ассеты (механизм Cordova/CodePush).
|
||||
Open-source self-host-вариант — `@capgo/capacitor-updater` (важно для AGPL-проекта:
|
||||
без привязки к проприетарному Appflow).
|
||||
|
||||
- Плюс: **даёт офлайн** — кэш AGPL легален, т.к. распространён вами, а не Apple.
|
||||
- Минус: упирается в политику Apple по hot-update (§9.5).
|
||||
|
||||
**Не-обходы (мифы):** «никто не засудит» — это нарушение, а не обход; «LGPL-нуть
|
||||
обёртку» — не помогает (проблема в AGPL-веб-клиенте, а не в обёртке); «mere
|
||||
aggregation» — не катит: зашитый бандл это комбинированное распространяемое
|
||||
произведение, а не простая агрегация.
|
||||
|
||||
### 9.5. Гейты Apple
|
||||
|
||||
| # | Guideline | Суть | Влияние |
|
||||
|---|---|---|---|
|
||||
| 1 | **2.5.2** (исполняемый код) | Скачивать/исполнять **нативный** код нельзя, **но** есть исключение для скриптов, исполняемых встроенным WebKit/JavascriptCore, если они не меняют назначение приложения | Загрузка веб-клиента в `WKWebView` под исключение попадает: вариант A — чистый, B — терпимый, но с границами |
|
||||
| 2 | **4.2** (minimum functionality) | Чистый WebView-«просто сайт» рискует отклонением | Лечится нативной ценностью в оболочке (push/APNs, биометрия, share, файлы — ваш нативный код, не AGPL) |
|
||||
| 3 | конфликт двух гейтов | «Лицензионно чистый» вариант (пустой шелл качает всё) — самый рискованный для ревью; «безопасный для ревью» (зашить веб-билд в `.ipa`) — лицензионное нарушение | **Совместить (офлайн) + (чистая AGPL) + (низкий риск ревью) в одной конфигурации нельзя — выбираете любые два** |
|
||||
|
||||
Безопасность: раз исполняете удалённый код — только HTTPS, желательно cert-pinning
|
||||
(подмена сервера = произвольный JS в WebView пользователя).
|
||||
|
||||
### 9.6. Итоговая матрица распространения iOS
|
||||
|
||||
| Конфигурация | AGPL-чистота | Офлайн | Риск ревью Apple |
|
||||
|---|---|---|---|
|
||||
| A. `server.url` на хостед-клиент | ✅ чистая | ❌ нет | средний (4.2, лечится плагинами) |
|
||||
| B. OTA пустой шелл + кэш бандла | ✅ чистая | ✅ есть | выше (2.5.2 + 4.2) |
|
||||
| Зашить веб-билд в `.ipa` (обычный Capacitor) | ❌ нарушение | ✅ | низкий |
|
||||
| **PWA** | ✅ чистая | ✅ | App Store не нужен |
|
||||
| Sideload / EU DMA-маркетплейсы (iOS 17.4+) | ✅ чистая | ✅ | вне App Store; **только ЕС** |
|
||||
|
||||
**Вывод:** для iOS **PWA** — самое дешёвое решение, закрывающее всё сразу. Если
|
||||
присутствие именно в App Store критично — **вариант A** (`server.url` + нативные
|
||||
плагины под 4.2) легальный и реалистичный ценой «онлайн для холодного старта».
|
||||
Офлайн в App Store (вариант B) технически и лицензионно возможен, но это
|
||||
максимальный риск на ревью — закладывать только если офлайн на iOS обязателен.
|
||||
Совместить «App Store + зашитый офлайн AGPL» легально нельзя, пока копирайт не ваш.
|
||||
|
||||
---
|
||||
|
||||
## 10. Оффлайн в будущем
|
||||
|
||||
Оффлайн сейчас не требуется, но позиция хорошая:
|
||||
|
||||
- Тело документа уже редактируется через Yjs (CRDT) + `y-indexeddb` — локальная
|
||||
копия и автослияние правок работают, в том числе в WebView.
|
||||
- «Полностью онлайн» — это всё вокруг тела (навигация, заголовки, комментарии,
|
||||
CRUD, вложения, авторизация). Их оффлайн-синхронизация описана отдельным
|
||||
планом с этапами M0…M4 — см. [offline-sync-plan.md](offline-sync-plan.md).
|
||||
- Мобильное приложение **переиспользует** этот план, а не строит оффлайн заново.
|
||||
Нюанс Android: System WebView под нехваткой места может чистить хранилище →
|
||||
для оффлайна, возможно, понадобится дублировать критичные данные в нативное
|
||||
хранилище, чтобы локальные копии не вычищались.
|
||||
|
||||
---
|
||||
|
||||
## 11. Открытые вопросы (зафиксировать до старта)
|
||||
|
||||
- **Q1.** Путь: Capacitor (B) с эволюцией в гибрид, или сразу React Native (C)?
|
||||
Рекомендация — B.
|
||||
- **Q2.** Мобильная авторизация: отдельный логин-флоу с токеном в body + Keychain/
|
||||
Keystore + Bearer (рекомендуется) или попытка работать через cookie в WebView?
|
||||
- **Q3.** Push: APNs + FCM сразу или iOS-first?
|
||||
- **Q4.** Подключать ли OpenAPI/Swagger для генерации мобильного клиента?
|
||||
- **Q5.** Когда включать оффлайн (M0…M4 из offline-sync-plan.md) относительно
|
||||
первого мобильного релиза?
|
||||
- **Q6.** iOS-дистрибуция при AGPL (§9): App Store через `server.url`
|
||||
(онлайн-клиент, без зашитого AGPL), PWA или sideload/EU-маркетплейсы? Этот
|
||||
лицензионный путь нужно подтвердить **до** кода обёртки. Рекомендация — PWA для
|
||||
iOS, Capacitor для Android.
|
||||
|
||||
---
|
||||
|
||||
## 12. Чеклист первого шага (бутстрап Capacitor, iOS-first)
|
||||
|
||||
- [ ] **Закрыть лицензионный путь iOS (§9) ДО кода обёртки:** выбрать
|
||||
`server.url` / PWA / sideload и подтвердить у разбирающегося в лицензиях.
|
||||
- [ ] **Не бандлить AGPL-веб-клиент в iOS `.ipa`** (DRM/usage-rules App Store ⟂
|
||||
AGPLv3 §10) — на iOS грузить клиент с сервера или идти через PWA.
|
||||
- [ ] Прогнать существующий адаптивный UI как PWA/в WebView, отловить отличия
|
||||
(жесты, IME в редакторе, safe-area).
|
||||
- [ ] Добавить Capacitor в монорепо, нацелить на веб-билд `apps/client`
|
||||
(Android — зашитый билд; iOS — `server.url`/PWA без зашитого AGPL, см. §9).
|
||||
- [ ] `npx cap add ios` (Android — `npx cap add android`, когда будет готова обвязка).
|
||||
- [ ] Бэкенд: мобильный логин-флоу с токеном в body; хранить токен в Keychain/
|
||||
Keystore; слать `Authorization: Bearer`.
|
||||
- [ ] Бэкенд: явный CORS-whitelist под мобильные origin'ы.
|
||||
- [ ] Native-плагины под App Store 4.2: push, биометрия, share, файлы.
|
||||
- [ ] Push: APNs (iOS); FCM добавить вместе с Android.
|
||||
- [ ] Проверить вебсокет коллаборации из WebView (`/auth/collab-token` + Hocuspocus).
|
||||
- [ ] (Опционально) Подключить `@nestjs/swagger`.
|
||||
@@ -1,205 +0,0 @@
|
||||
# Множественные курсоры (multi-cursor editing) — анализ и подходы
|
||||
|
||||
> Статус: **черновик / обсуждение**. Код не пишется; цель этого документа — зафиксировать архитектурный вердикт, развилку подходов и рекомендацию.
|
||||
>
|
||||
> Важное уточнение термина: речь про **несколько собственных курсоров одного пользователя в одном документе** (как в VS Code: `Alt+Click` добавить курсор, `Ctrl/Cmd+D` — следующее вхождение, `Ctrl/Cmd+Shift+L` — все вхождения), чтобы править несколько мест одновременно. **Не** про collaborative-курсоры соавторов — те в проекте уже работают (`CollaborationCaret` + Hocuspocus awareness).
|
||||
>
|
||||
> Зафиксированные выводы (см. разделы ниже):
|
||||
> - Полноценный VS Code-style multi-cursor нельзя «включить флагом»: движок редактора (ProseMirror) хранит в состоянии **ровно одно выделение**, в отличие от Monaco/CodeMirror с массивом selections. Готового production-пакета в экосистеме Tiptap/ProseMirror нет.
|
||||
> - ~80% пользовательской ценности даёт ограниченный MVP («выделить все вхождения + одновременный ввод»), который опирается на **уже работающий** в проекте механизм `replaceAll` из расширения `SearchAndReplace`.
|
||||
> - Рекомендация: реализовать MVP (Вариант A); полноценный набор (Вариант B) — отдельный большой эпик, имеет смысл браться только если MVP окажется недостаточно.
|
||||
|
||||
## 0. О чём речь (и о чём НЕ речь)
|
||||
|
||||
**Что хочется** — несколько кареток в одном документе; набранный текст и `Backspace`/`Delete` применяются ко всем позициям одновременно; одно `Cmd/Ctrl+Z` откатывает всю мульти-правку целиком. Сценарии из VS Code:
|
||||
|
||||
| Действие | Горячая клавиша | Суть |
|
||||
| --- | --- | --- |
|
||||
| Добавить курсор | `Alt+Click` | Курсор в произвольной точке клика |
|
||||
| Добавить курсор строкой выше/ниже | `Ctrl/Cmd+Alt+↑/↓` | Копия курсора на соседней строке |
|
||||
| Выделить следующее вхождение | `Ctrl/Cmd+D` | Добавить к набору следующее вхождение слова |
|
||||
| Выделить все вхождения | `Ctrl/Cmd+Shift+L` | Все вхождения сразу |
|
||||
| Колонковое/блочное выделение | `Alt+drag` | Прямоугольник курсоров по строкам |
|
||||
|
||||
**О чём НЕ речь** — collaborative-курсоры (видеть, где сейчас находится другой соавтор). Это в Gitmost уже есть и работает отдельно: `CollaborationCaret` в [extensions.ts](apps/client/src/features/editor/extensions/extensions.ts) подключается через `collabExtensions(...)`, а сервер Hocuspocus по умолчанию форвардит awareness. Этот документ её не касается.
|
||||
|
||||
## 1. Архитектурный вердикт: почему это не «включить флаг»
|
||||
|
||||
Редактор Gitmost — **Tiptap поверх ProseMirror** (`@tiptap/core` 3.20.4, `@tiptap/pm` 3.20.4). Принципиальное отличие от VS Code: Monaco/CodeMirror хранит **массив selections**, а ProseMirror хранит в `EditorState` **ровно один** `Selection`:
|
||||
|
||||
```
|
||||
EditorState = { doc, selection: Selection /* единственное */, storedMarks, ... }
|
||||
```
|
||||
|
||||
На этой единственной selection завязано в ProseMirror почти всё:
|
||||
- команды ввода (`insertText`, `insertContent`) работают с текущей `selection`;
|
||||
- обработчики `handleTextInput`, `handleKeyDown`, `handlePaste`, `handleDrop` получают одно выделение;
|
||||
- история (undo/redo) оперирует transactions с одним выделением;
|
||||
- **критично для нас** — синхронизация через y-prosemirror тоже опирается на единственную selection (свою «awareness-selection» отдельно, но не на локальный массив).
|
||||
|
||||
Доказательства из первоисточников:
|
||||
- Tiptap issue [ueberdosis/tiptap#3370](https://github.com/ueberdosis/tiptap/issues/3370) «Multiple cursors per user» — открыт, официальной поддержки нет.
|
||||
- Ответ **marijnh** (автор ProseMirror) на [discuss.prosemirror.net](https://discuss.prosemirror.net/t/multi-cursor-editing-in-prosemirror-or-tiptap/8397): готовой реализации нет, но путь обозначен — **«кастомный подкласс `Selection`, по аналогии с `CellSelection` из `prosemirror-tables`, который умеет содержать несколько отдельных диапазонов»**.
|
||||
- Production-готового пакета multi-cursor для Tiptap/ProseMirror в npm **нет** — пилить с нуля.
|
||||
|
||||
**Вывод:** полноценный multi-cursor — это R&D-проект против устройства движка, а не настройка. Но самый ценный сценарий («поправить повторяющиеся одинаковые куски сразу в нескольких местах») реализуем дёшево, потому что массовая правка в одном transaction у нас уже написана.
|
||||
|
||||
## 2. Что уже есть в коде и переиспользуемо
|
||||
|
||||
В проекте уже есть расширение [SearchAndReplace](packages/editor-ext/src/lib/search-and-replace/search-and-replace.ts) (в `editor-ext`, подключено и в клиентском редакторе). Это почти готовый фундамент для главного сценария multi-cursor:
|
||||
|
||||
- [search-and-replace.ts:100-174](packages/editor-ext/src/lib/search-and-replace/search-and-replace.ts#L100-L174) — `processSearches` уже находит **все** вхождения терма и возвращает массив `results: Range[]` (диапазоны `from`/`to`).
|
||||
- [search-and-replace.ts:157-168](packages/editor-ext/src/lib/search-and-replace/search-and-replace.ts#L157-L168) — уже рисует `Decoration.inline` для **всех** совпадений одновременно (это переиспользуется для подсветки «активных» курсоров).
|
||||
- [search-and-replace.ts:213-246](packages/editor-ext/src/lib/search-and-replace/search-and-replace.ts#L213-L246) — `replaceAll` уже выполняет **массовую правку в одном transaction**, идя **с конца**, чтобы корректно учитывать сдвиг позиций после каждой вставки/удаления. Это ровно та механика, что нужна для одновременного ввода в несколько курсоров.
|
||||
|
||||
```ts
|
||||
// search-and-replace.ts:213-246 — готовый эталон массового transaction
|
||||
const replaceAll = (replaceTerm, results, { tr, dispatch }) => {
|
||||
// Process replacements in reverse order to avoid position shifting issues
|
||||
for (let i = resultsCopy.length - 1; i >= 0; i -= 1) {
|
||||
const { from, to } = resultsCopy[i];
|
||||
// ... собрать marks, удалить старый текст, вставить новый
|
||||
tr.delete(from, to);
|
||||
if (replaceTerm) tr.insert(from, tr.doc.type.schema.text(replaceTerm, marks));
|
||||
}
|
||||
dispatch(tr); // одна транзакция → одна запись в истории (один undo)
|
||||
};
|
||||
```
|
||||
|
||||
То есть самая хитрая часть multi-cursor — применить правку к N позициям за один `tr` с корректным маппингом — у нас **уже работает** в `replaceAll`.
|
||||
|
||||
Дополнительно в клиенте уже есть инфраструктура для горячих клавиш: в [page-editor.tsx:258-280](apps/client/src/features/editor/page-editor.tsx#L258-L280) есть блок `handleDOMEvents.keydown`, и используется утилита `platformModifierKey` (Cmd на macOS, Ctrl на других ОС — ровно то, что нужно для совместимых с VS Code шорткатов).
|
||||
|
||||
## 3. Развилка: три подхода
|
||||
|
||||
### 3.1 Вариант A — MVP: «выделить все вхождения + одновременный ввод» (рекомендация)
|
||||
|
||||
Реализует главный сценарий из VS Code:
|
||||
- `Ctrl/Cmd+Shift+L` — берём слово под курсором (или текущее выделение), находим все вхождения, превращаем их в «активные курсоры»;
|
||||
- `Ctrl/Cmd+D` — добавить следующее вхождение к набору;
|
||||
- дальнейший ввод текста и `Backspace`/`Delete` применяются ко всем позициям одновременно через один transaction (копия механики `replaceAll`);
|
||||
- `Esc` — выйти из multi-cursor (один курсор).
|
||||
|
||||
**Что переиспользуется:** массив `results` и логика массового `tr` берутся из [SearchAndReplace](packages/editor-ext/src/lib/search-and-replace/search-and-replace.ts) почти готовыми.
|
||||
|
||||
**Визуальные каретки:** через `Decoration.widget(pos, () => cursorDomElement)` — ProseMirror умеет «из коробки»; для диапазонов — `Decoration.inline`.
|
||||
|
||||
**Объём работы:** средний. Один новый Tiptap-extension в `packages/editor-ext/src/lib/multi-cursor/` + wiring в клиентском редакторе + горячие клавиши + CSS + юнит-тесты.
|
||||
|
||||
**Риски:** средние и ограниченные. Скоуп узкий (только текстовые вхождения), сценарии предсказуемые, тестируются конечным числом кейсов.
|
||||
|
||||
### 3.2 Вариант B — полноценный multi-cursor (как Monaco)
|
||||
|
||||
Полный набор из §0: `Alt+Click` (произвольная точка), `Alt+drag` (колонковое выделение), `Ctrl/Cmd+Alt+↑/↓` (курсор на соседней строке), а также произвольный набор **несвязанных** курсоров (не по вхождениям).
|
||||
|
||||
**Путь:** кастомный `MultiSelection extends Selection` (по подсказке мейнтейнера ProseMirror, по образцу `CellSelection` из `prosemirror-tables`), плюс **полная маршрутизация ввода**:
|
||||
- перехват `handleTextInput`, `handleKeyDown` (Backspace/Delete/стрелки/Enter/Home/End), `handlePaste`, `handleDrop`;
|
||||
- построение одного мульти-position transaction для каждого события;
|
||||
- визуальный рендер нескольких кареток и диапазонов;
|
||||
- undo-группировка (одно `Cmd/Ctrl+Z` откатывает все позиции разом);
|
||||
- перемапливание позиций курсоров при **любых** изменениях документа, включая remote Yjs-правки.
|
||||
|
||||
**Объём работы:** очень большой (многие недели). Готового референса в экосистеме нет — это самостоятельный R&D с отладкой на реальном контенте.
|
||||
|
||||
**Риски:** высокие — см. риск-карту в §4 (IME/composition, конфликты со сложными нодами вроде таблиц и code-блоков, взаимодействие с коллаборацией).
|
||||
|
||||
### 3.3 Вариант C — эмуляция через коллаборацию (отбрасываем)
|
||||
|
||||
Идея из Tiptap#3370: «проигрывать правки через отдельного pseudo-user через collaborative-слой». **Не берём:** ломает provenance правок (в проекте есть бейдж авторства «AI agent» в истории страницы, migration `20260616T130000-agent-provenance` — такой хак его загрязнит и запутает), портит историю undo, концептуально криво и хрупко.
|
||||
|
||||
### Сводка
|
||||
|
||||
| | Вариант A (MVP) | Вариант B (full) | Вариант C |
|
||||
| --- | --- | --- | --- |
|
||||
| Сценарии | «все вхождения», «+следующее вхождение» | полный набор VS Code | — |
|
||||
| База | готовый `replaceAll` | кастомный `Selection` с нуля | collaborative-слой |
|
||||
| Объём | средний | очень большой | — |
|
||||
| Риск | средний (ограниченный) | высокий | высокий |
|
||||
| Рекомендация | **да** | только если A мало | нет |
|
||||
|
||||
## 4. Риск-карта
|
||||
|
||||
Для обоих вариантов, но в варианте B каждый пункт — сильно жёстче.
|
||||
|
||||
| Зона | Суть | Где больнее |
|
||||
| --- | --- | --- |
|
||||
| **Undo/redo** | Мульти-правка должна быть **одной** записью истории (одно `Cmd/Ctrl+Z` откатывает все позиции). Группировка через мету истории, см. как `replaceAll` делает один `dispatch(tr)`. | B |
|
||||
| **Коллаборация (Yjs)** | Пока активны ваши курсоры, может прилететь remote-правка — позиции курсоров надо перемапливать через `tr.mapping.map(pos)`. Один локальный `tr` с правками в N местах Yjs переварит нормально (это несколько правок в одном Update). | B |
|
||||
| **IME / dead keys** | Ввод через composition (буквы с акцентами, CJK) одновременно в несколько курсоров — крайне хрупко; для MVP (Вариант A) проще: на время composition можно схлопывать к одному курсору. | B |
|
||||
| **Schema / сложные узлы** | Курсор внутри code-блока + курсор в заголовке: одна и та же вставка может нарушить schema одного узла, но не другого. Нужно gracefully skip конфликтующие курсоры (не ронять весь `tr`). | B (A — почти не касается, т.к. вхождения — текстовые) |
|
||||
| **Таблицы / callouts** | `CellSelection`-подобная логика внутри таблиц — отдельная вселенная; в MVP курсоры в таблицах можно просто не поддерживать (как и в `replaceAll`). | B |
|
||||
| **Производительность** | Очень много курсоров → большой `DecorationSet` и длинный `tr`. Практически редко > нескольких десятков, но заложить верхнюю границу. | общий |
|
||||
|
||||
## 5. Рекомендация
|
||||
|
||||
**Брать Вариант A.** Он закрывает главный use-case («быстро поправить повторяющиеся одинаковые куски сразу в нескольких местах»), опирается на **уже работающий** `replaceAll`-механизм, и риск ограничен. Вариант B имеет смысл отдельным эпиком — только если A окажется недостаточно и будет устойчивый спрос на произвольные курсоры; тогда начинать стоит с прототипа кастомного `MultiSelection`, чтобы доказать жизнеспособность на сложных узлах до полной реализации.
|
||||
|
||||
Сознательные границы MVP (Вариант A) — см. §6.7.
|
||||
|
||||
## 6. План реализации Варианта A (MVP) — по шагам
|
||||
|
||||
### 6.1. Новый extension
|
||||
|
||||
Создать `packages/editor-ext/src/lib/multi-cursor/multi-cursor.ts` — Tiptap `Extension`:
|
||||
- плагин (ProseMirror `Plugin`) со state = `{ cursors: {from: number, to: number}[] }` и `DecorationSet` (виджеты-каретки для точечных курсоров + `Decoration.inline` для диапазонов);
|
||||
- команды:
|
||||
- `selectAllOccurrences` — берёт слово под курсором (или текущее выделение), находит все вхождения (можно вынести общую с search-and-replace логику поиска в утилиту, чтобы не дублировать `processSearches`), заполняет `cursors`;
|
||||
- `addNextOccurrence` (`Ctrl/Cmd+D`) — добавляет следующее вхождение к `cursors`;
|
||||
- `exitMultiCursor` — очищает `cursors` (также вешается на `Esc`);
|
||||
- обработчики в `props`:
|
||||
- `handleTextInput(view, from, to, text)` — если `cursors` непустой, строит один `tr`, вставляя `text` в каждую позицию **с конца** (копия механики из [search-and-replace.ts:213-246](packages/editor-ext/src/lib/search-and-replace/search-and-replace.ts#L213-L246));
|
||||
- `handleKeyDown` — `Backspace`/`Delete` аналогично (удаление символа перед/после каждой позиции);
|
||||
- игнорировать/схлопнуть multi-cursor при начале composition (IME) — см. §4.
|
||||
|
||||
### 6.2. Маппинг позиций при изменениях документа
|
||||
|
||||
В `state.apply` плагина — при любом `docChanged` перемапливать все позиции через `tr.mapping.map(pos)` и удалять «схлопнувшиеся» (`from === to` после маппинга — это нормально для каретки). Это покрывает и собственные правки, и **remote Yjs-правки** (y-prosemirror применяет их как обычные transactions — маппинг работает одинаково).
|
||||
|
||||
### 6.3. Горячие клавиши
|
||||
|
||||
Добавить в существующий блок [page-editor.tsx:258-280](apps/client/src/features/editor/page-editor.tsx#L258-L280) (там уже есть `platformModifierKey`):
|
||||
- `platformModifierKey + Shift + KeyL` → `selectAllOccurrences`;
|
||||
- `platformModifierKey + KeyD` → `addNextOccurrence`;
|
||||
- `Escape` → `exitMultiCursor`.
|
||||
|
||||
⚠️ Проверить конфликт `Ctrl/Cmd+D` с браузерным «добавить в закладки» (предотвратить через `event.preventDefault()`) и с любыми существующими биндингами редактора.
|
||||
|
||||
### 6.4. Регистрация
|
||||
|
||||
- экспортировать расширение из `packages/editor-ext/src/lib/multi-cursor/index.ts` и добавить в `packages/editor-ext/src/index.ts`;
|
||||
- включить в `mainExtensions` в [extensions.ts](apps/client/src/features/editor/extensions/extensions.ts) (оно не зависит от коллаборации, поэтому идёт в основной набор, доступный и в обычном, и в коллаборативном редакторе).
|
||||
|
||||
### 6.5. CSS
|
||||
|
||||
Рядом с [collaboration.css](apps/client/src/features/editor/styles/collaboration.css) (и подключением через `styles/index.css`) — стили для классов вроде `.multi-cursor__caret` и `.multi-cursor__label`. Визуально отличать от collaborative-кареток (например, другим стилем/цветом), чтобы не путать свои мульти-курсоры с курсорами соавторов.
|
||||
|
||||
### 6.6. Тесты
|
||||
|
||||
Unit-тесты в `packages/editor-ext` (по образцу существующих там тестов) на:
|
||||
- корректность массового `tr` (ввод/удаление в N позициях, проверка результирующего документа);
|
||||
- маппинг позиций после локальной правки и после имитированной remote-правки;
|
||||
- граничные случаи: курсоры на границах узлов, схлопывание, пустой набор.
|
||||
|
||||
### 6.7. Скоуп v1 / что сознательно НЕ входит
|
||||
|
||||
Чтобы держать риск в пределах, в MVP **не делаем** (явно фиксируем как out-of-scope):
|
||||
- `Alt+Click` (произвольная точка) и `Alt+drag` (колонковое выделение) — это путь в Вариант B;
|
||||
- `Ctrl/Cmd+Alt+↑/↓` (курсор на соседней строке) — то же;
|
||||
- курсоры внутри таблиц, code-блоков и callouts — только обычный текст (как в `replaceAll`);
|
||||
- одновременный ввод через IME в несколько позиций (на время composition схлопываем к одному курсору);
|
||||
- курсоры, затрагивающие разные schema-узлы одновременно (если вставка нарушает schema в одной из позиций — пропускаем эту позицию, не роняем весь `tr`).
|
||||
|
||||
Эти границы — кандидаты на v2 / переход к Варианту B.
|
||||
|
||||
## 7. Открытые вопросы
|
||||
|
||||
1. **Выделение диапазонов vs точечные курсоры.** В VS Code `Ctrl/Cmd+Shift+L` выделяет целые слова (диапазоны). Делаем ли мы в MVP то же (диапазоны + одновременная замена всего слова), или только точечные каретки после конца слова? Рекомендация: диапазоны — это даёт «переименовать все эти слова сразу», что и есть главная ценность.
|
||||
2. **Общая утилита поиска.** Вынести `processSearches` из search-and-replace в общую утилиту, чтобы не дублировать, или оставить независимую реализацию в multi-cursor? Рекомендация: вынести общую часть (поиск всех вхождений слова по документу), оба расширения используют её.
|
||||
3. **Граница производительности.** Ввести ли хард-кап на число одновременных курсоров (например, 100) с предупреждением пользователю? Рекомендация: да, как страховка.
|
||||
|
||||
## 8. Источники
|
||||
|
||||
- [Tiptap issue #3370 — Multiple cursors per user](https://github.com/ueberdosis/tiptap/issues/3370)
|
||||
- [discuss.ProseMirror — Multi-cursor editing in ProseMirror (ответ автора ProseMirror о кастомном подклассе Selection)](https://discuss.prosemirror.net/t/multi-cursor-editing-in-prosemirror-or-tiptap/8397)
|
||||
- `prosemirror-tables` / `CellSelection` — референс реализации «выделения из нескольких диапазонов» для Варианта B.
|
||||
- Внутренний код: [SearchAndReplace](packages/editor-ext/src/lib/search-and-replace/search-and-replace.ts) (эталон массового transaction), [page-editor.tsx](apps/client/src/features/editor/page-editor.tsx) (точки подключения горячих клавиш), [extensions.ts](apps/client/src/features/editor/extensions/extensions.ts) (регистрация расширений).
|
||||
@@ -1,393 +0,0 @@
|
||||
# Offline-режим и синхронизация правок в gitmost
|
||||
|
||||
> Статус: проектный документ, готов к реализации.
|
||||
> Контекст: gitmost — форк Docmost. Сейчас приложение полностью онлайн.
|
||||
> Цель: дать возможность работать оффлайн (читать и редактировать) и
|
||||
> синхронизироваться при возврате сети.
|
||||
|
||||
Документ описывает текущее устройство, целевую архитектуру и пошаговый план
|
||||
реализации с привязкой к конкретным файлам. Его можно взять и реализовывать
|
||||
по этапам M0…M4.
|
||||
|
||||
---
|
||||
|
||||
## 1. TL;DR
|
||||
|
||||
1. **Половина оффлайна уже встроена.** Тело страницы редактируется через Yjs
|
||||
(CRDT) + Hocuspocus, а на клиенте уже подключён `y-indexeddb`. Правки тела
|
||||
*уже открытой* страницы переживают потерю сети и **сами мёржатся** при
|
||||
реконнекте — без конфликтов.
|
||||
2. **«Полностью онлайн» — это всё вокруг тела документа:** загрузка самого
|
||||
приложения, навигация (дерево/список), заголовки страниц, комментарии,
|
||||
создание/перемещение/удаление страниц, вложения, авторизация.
|
||||
3. **Оффлайн делится на два контура с разными механизмами синхронизации:**
|
||||
- **Контур A — тело документа:** CRDT (Yjs). Почти готов, нужно укрепить.
|
||||
- **Контур B — структурные данные (REST):** не CRDT. Нужен паттерн
|
||||
*локальный кэш + outbox (очередь мутаций) + правила разрешения конфликтов*.
|
||||
4. **PWA — обязательный фундамент, но это два слоя:**
|
||||
- *Installability* (manifest + meta-теги) — **уже есть** в gitmost
|
||||
(унаследовано от Docmost). Forkmost добавляет только косметику.
|
||||
- *Service worker* (кэш app-shell, запуск без сети) — **нет нигде**, это и
|
||||
есть реальная невыполненная часть. Без него установленное приложение без
|
||||
сети покажет пустой экран.
|
||||
|
||||
---
|
||||
|
||||
## 2. Текущее состояние (как есть)
|
||||
|
||||
### 2.1. Контур A: тело документа — CRDT, почти готово
|
||||
|
||||
| Где | Что делает |
|
||||
|---|---|
|
||||
| [page-editor.tsx](../apps/client/src/features/editor/page-editor.tsx) (L131–206) | На каждую страницу создаётся `Y.Doc`, к нему цепляются `IndexeddbPersistence("page.<id>")` (локальная копия) **и** `HocuspocusProvider` (WS-синк). |
|
||||
| [persistence.extension.ts](../apps/server/src/collaboration/extensions/persistence.extension.ts) | Сервер в `onStoreDocument` хранит в Postgres бинарный `ydoc` (Y state update) **плюс** отрендеренный tiptap-JSON `content` + `textContent`. В `onLoadDocument` поднимает `ydoc` обратно. |
|
||||
| [collaboration/extensions/redis-sync/](../apps/server/src/collaboration/extensions/redis-sync/) | Redis-синк для горизонтального масштабирования инстансов. |
|
||||
|
||||
Почему это и есть оффлайн-редактирование: Yjs — CRDT, апдейты коммутативны.
|
||||
Пока клиент оффлайн, изменения копятся в `Y.Doc` и в IndexedDB; при возврате
|
||||
сети `HocuspocusProvider` обменивается state-векторами и **детерминированно
|
||||
сливает** правки. Конфликтов «кто кого перезаписал» в теле документа нет.
|
||||
|
||||
### 2.2. Контур B: структурные данные — обычный REST, оффлайн недоступен
|
||||
|
||||
| Сущность | Где | Механизм |
|
||||
|---|---|---|
|
||||
| Заголовок страницы | [title-editor.tsx](../apps/client/src/features/editor/title-editor.tsx) (L48–152) | REST `/pages/update`, дебаунс 500 мс. **НЕ Yjs.** |
|
||||
| CRUD страниц, move, restore | [page-service.ts](../apps/client/src/features/page/services/page-service.ts) | REST `/pages/*` |
|
||||
| Комментарии | [comment-service.ts](../apps/client/src/features/comment/services/comment-service.ts) | REST `/comments/*` |
|
||||
| Watchers, favorites, labels, дерево, поиск | соответствующие `features/*/services` | REST |
|
||||
|
||||
Состояние клиента:
|
||||
- React Query: [main.tsx](../apps/client/src/main.tsx) (L26), `queryClient`
|
||||
экспортируется, `retry:false`, `staleTime: 5 мин`. **Персистентности на диск
|
||||
нет.** При перезагрузке без сети читать нечего.
|
||||
- HTTP: [api-client.ts](../apps/client/src/lib/api-client.ts) — axios `/api`,
|
||||
`withCredentials`. На `401` → `redirectToLogin()`. **Важно для оффлайна:**
|
||||
редирект на логин при сетевой ошибке недопустим (см. M4).
|
||||
|
||||
### 2.3. PWA: что уже есть
|
||||
|
||||
- [manifest.json](../apps/client/public/manifest.json) — присутствует
|
||||
(`display: standalone`, иконки).
|
||||
- [index.html](../apps/client/index.html) (L9–16) — PWA meta-теги
|
||||
(`apple-mobile-web-app-capable`, `mobile-web-app-capable`, `theme-color` и т.д.).
|
||||
- **Service worker отсутствует.** Нет `vite-plugin-pwa`, Workbox, precache.
|
||||
|
||||
> Вывод по Forkmost (`Vito0912/forkmost`): их «PWA-наработки» — это только
|
||||
> манифест и meta-теги (closing issue Docmost #328 про *устанавливаемость*).
|
||||
> Service worker / оффлайн-кэша там нет. В gitmost installability уже есть,
|
||||
> поэтому из Forkmost переносить нечего, кроме косметики.
|
||||
|
||||
### 2.4. Полезные примитивы, которые уже есть в проекте
|
||||
|
||||
- **Fractional indexing для позиций страниц:**
|
||||
[page.service.ts](../apps/server/src/core/page/services/page.service.ts)
|
||||
использует `generateJitteredKeyBetween` из `fractional-indexing-jittered`.
|
||||
Позиция — это строковый ключ (`position: string`), «jittered»-вариант
|
||||
специально снижает коллизии при конкурентных/оффлайн-вставках. Это готовый
|
||||
offline-friendly примитив для перемещений в дереве.
|
||||
- **Генерация ID:**
|
||||
[nanoid.utils.ts](../apps/server/src/common/helpers/nanoid.utils.ts) —
|
||||
`generateSlugId` (10 симв.) и `nanoIdGen`. ID можно генерировать на клиенте и
|
||||
принимать на сервере (нужно для оффлайн-создания, см. M3).
|
||||
|
||||
---
|
||||
|
||||
## 3. Целевая архитектура
|
||||
|
||||
```
|
||||
┌──────────────────────── Браузер (PWA) ────────────────────────┐
|
||||
│ │
|
||||
Тело документа │ TipTap ⟷ Y.Doc ⟷ IndexeddbPersistence (локальная копия) │
|
||||
(Контур A, CRDT) │ │ │
|
||||
│ └── HocuspocusProvider ──┐ │
|
||||
│ │ │
|
||||
Структурные данные │ React Query (read) ⟵ IndexedDB persister │ │
|
||||
(Контур B, REST) │ Мутации ⟶ Outbox (IndexedDB) ──────────┐ │ │
|
||||
│ │ │ │
|
||||
App shell │ Service Worker (Workbox precache) │ │ │
|
||||
└──────────────────────────────────────────┼────┼───────────────┘
|
||||
│ │
|
||||
(reconnect) ▼ ▼
|
||||
┌──────────────────────── Сервер ───────────────────────────────┐
|
||||
│ REST API (idempotent upsert по client-id) Hocuspocus (Yjs) │
|
||||
│ │ │ │
|
||||
│ └────────────── Postgres ───────────────┘ │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Два независимых канала синхронизации:
|
||||
- **Контур A** синкается сам через Hocuspocus (Yjs). Руками конфликты не решаем.
|
||||
- **Контур B** синкается через outbox: оффлайн-мутации пишутся в журнал в
|
||||
IndexedDB и проигрываются на сервер при реконнекте; конфликты решаются
|
||||
явными правилами (LWW / per-entity).
|
||||
|
||||
---
|
||||
|
||||
## 4. План реализации по этапам
|
||||
|
||||
Этапы инкрементальны: каждый даёт пользователю ощутимый результат и может быть
|
||||
смёржен отдельно. Рекомендуемый порядок — строго M0 → M4.
|
||||
|
||||
### M0 — PWA shell (фундамент: приложение запускается без сети)
|
||||
|
||||
**Зачем:** без service worker установленное приложение без сети не загрузится.
|
||||
Это разблокирует всё остальное.
|
||||
|
||||
**Что сделать:**
|
||||
1. Добавить `vite-plugin-pwa` (Workbox под капотом) в
|
||||
[vite.config.ts](../apps/client/vite.config.ts).
|
||||
- `registerType: 'autoUpdate'` или `prompt` (см. риск R3).
|
||||
- `workbox.globPatterns` — прекэш JS/CSS/wasm/шрифтов/иконок.
|
||||
- `manifest: false` или генерация из существующего
|
||||
[manifest.json](../apps/client/public/manifest.json) (не дублировать).
|
||||
- Навигационный fallback на `index.html` для SPA-роутов.
|
||||
- Runtime caching: `CacheFirst` для статики, **`NetworkOnly` для `/api/**`
|
||||
и `/collab`** на этом этапе (REST-кэш появится в M2; SW не должен молча
|
||||
отдавать устаревшие ответы API).
|
||||
2. Зарегистрировать SW в [main.tsx](../apps/client/src/main.tsx)
|
||||
(`registerSW` из `virtual:pwa-register`).
|
||||
3. Перенести косметику манифеста/метатегов из Forkmost при желании (бренд,
|
||||
`orientation`, `msapplication-*`). Опционально, на оффлайн не влияет.
|
||||
|
||||
**Файлы:** `apps/client/vite.config.ts`, `apps/client/src/main.tsx`,
|
||||
`apps/client/public/manifest.json`, `apps/client/index.html`.
|
||||
|
||||
**Критерий приёмки:** приложение устанавливается, после первой загрузки
|
||||
открывается **без сети** (виден shell/лэйаут, а не пустой экран);
|
||||
обновление версии SW не ломает открытую сессию.
|
||||
|
||||
**Риск:** низкий. Изолированный слой, кода приложения не трогает.
|
||||
|
||||
---
|
||||
|
||||
### M1 — Укрепление оффлайна тела документа (Контур A)
|
||||
|
||||
**Зачем:** убрать известные грабли Yjs и сделать поведение предсказуемым.
|
||||
|
||||
**Что сделать:**
|
||||
1. **Закрыть ловушку «rebuild ydoc из JSON».** В
|
||||
[persistence.extension.ts](../apps/server/src/collaboration/extensions/persistence.extension.ts)
|
||||
`onLoadDocument` при пустом `page.ydoc` пересобирает документ из
|
||||
`page.content` через `TiptapTransformer.toYdoc(...)`. Если это сработает,
|
||||
пока оффлайн-клиент держит свой `Y.Doc` со своими client-id, при мёрже
|
||||
возможно **дублирование контента** (классическая Yjs-ловушка).
|
||||
- Гарантировать, что `ydoc` всегда персистится (после первого сохранения он
|
||||
есть) и ветка rebuild не выполняется для страниц, у которых живут
|
||||
оффлайн-клиенты. Минимум — единожды мигрировать `content → ydoc` для всех
|
||||
страниц и далее считать `ydoc` единственным источником правды для тела.
|
||||
2. **Индикатор оффлайна/синка в UI.** Уже есть `yjsConnectionStatusAtom` и
|
||||
`isLocalSynced/isRemoteSynced` в
|
||||
[page-editor.tsx](../apps/client/src/features/editor/page-editor.tsx).
|
||||
Показать состояние («оффлайн», «есть несинхронизированные правки»,
|
||||
«синхронизировано»).
|
||||
3. **Заголовок страницы → в Yjs (рекомендуется).**
|
||||
[title-editor.tsx](../apps/client/src/features/editor/title-editor.tsx)
|
||||
сохраняет заголовок REST-ом (дебаунс 500 мс) — оффлайн это не работает и
|
||||
расходится с телом. Варианты:
|
||||
- (a) перенести заголовок в тот же `Y.Doc` (чистое CRDT-решение), либо
|
||||
- (b) тащить заголовок через outbox из M3 (LWW). Решение зафиксировать
|
||||
до старта M3 (см. открытый вопрос Q1).
|
||||
|
||||
**Файлы:** `apps/server/src/collaboration/extensions/persistence.extension.ts`,
|
||||
`apps/client/src/features/editor/page-editor.tsx`,
|
||||
`apps/client/src/features/editor/title-editor.tsx` (если вариант a).
|
||||
|
||||
**Критерий приёмки:** правки тела уже открытой страницы, сделанные оффлайн,
|
||||
после реконнекта появляются на сервере и у других клиентов без дублей и потерь;
|
||||
в UI виден статус синка.
|
||||
|
||||
**Риск:** средний (Yjs-семантика, миграция `content → ydoc`).
|
||||
|
||||
---
|
||||
|
||||
### M2 — Оффлайн-чтение и навигация (Контур B, read-path)
|
||||
|
||||
**Зачем:** оффлайн нужно видеть дерево, список и метаданные, иначе некуда
|
||||
переходить; и нужно префетчить страницы «на оффлайн».
|
||||
|
||||
**Что сделать:**
|
||||
1. **Персист React Query на диск.** Обернуть экспортируемый `queryClient` из
|
||||
[main.tsx](../apps/client/src/main.tsx) в
|
||||
`PersistQueryClientProvider` с IndexedDB-persister
|
||||
(`@tanstack/query-persist-client-core` + idb-хранилище).
|
||||
- Кэшировать: дерево пространства, список страниц, метаданные страницы,
|
||||
комментарии. Выставить разумный `maxAge`/`gcTime`.
|
||||
- Версионировать кэш (`buster`) по версии приложения, чтобы не «залипал»
|
||||
после деплоя.
|
||||
2. **«Сделать доступным оффлайн».** Действие для пространства/ветки: префетч
|
||||
метаданных **и** прогрев `IndexeddbPersistence` для тел страниц (открыть/
|
||||
подгрузить `ydoc` каждой целевой страницы заранее), т.к. сейчас локально
|
||||
лежат только *ранее открытые* страницы.
|
||||
3. **Runtime caching API в SW (read-only).** Для GET-эндпоинтов навигации —
|
||||
`StaleWhileRevalidate`/`NetworkFirst` с фолбэком на кэш. Мутации (POST) —
|
||||
по-прежнему мимо кэша (их берёт на себя M3).
|
||||
|
||||
**Файлы:** `apps/client/src/main.tsx`, новый модуль
|
||||
`apps/client/src/lib/offline/` (persister, prefetch), точечно — хуки списков/
|
||||
дерева в `features/page/tree`.
|
||||
|
||||
**Критерий приёмки:** после прогрева и ухода в оффлайн пользователь видит дерево
|
||||
и список, открывает заранее подготовленные страницы и читает их тело и
|
||||
комментарии.
|
||||
|
||||
**Риск:** средний (консистентность кэша, инвалидция после деплоя).
|
||||
|
||||
---
|
||||
|
||||
### M3 — Outbox для мутаций (Контур B, write-path) — ядро оффлайн-синка
|
||||
|
||||
**Зачем:** дать оффлайн-создание/редактирование структурных данных с
|
||||
последующим проигрыванием на сервер.
|
||||
|
||||
**Что сделать:**
|
||||
1. **Очередь мутаций (outbox) в IndexedDB.** Журнал операций
|
||||
`{ id, entity, op, payload, clientId, baseVersion, createdAt, status }`.
|
||||
Использовать **offline/paused mutations TanStack Query**
|
||||
(`onlineManager` + `queryClient.resumePausedMutations()` + персист пауз),
|
||||
либо отдельный модуль `apps/client/src/lib/offline/outbox.ts`.
|
||||
2. **Клиентская генерация ID.** Для оффлайн-создания страниц/комментариев
|
||||
генерировать `id`/`slugId` на клиенте тем же алфавитом, что и
|
||||
[nanoid.utils.ts](../apps/server/src/common/helpers/nanoid.utils.ts).
|
||||
Для позиций в дереве — `generateJitteredKeyBetween` из
|
||||
`fractional-indexing-jittered` (тот же пакет, что на сервере).
|
||||
3. **Идемпотентный upsert на сервере.** Эндпоинты `/pages/create`,
|
||||
`/comments/create` и т.д. должны принимать клиентский `id` и быть
|
||||
идемпотентными по нему (повторная отправка из очереди не должна плодить
|
||||
дубликаты). Точки входа:
|
||||
[page-service.ts](../apps/client/src/features/page/services/page-service.ts),
|
||||
[comment-service.ts](../apps/client/src/features/comment/services/comment-service.ts)
|
||||
и соответствующие контроллеры сервера.
|
||||
4. **Optimistic updates + откат.** Применять мутацию к кэшу сразу; при
|
||||
неуспешном проигрывании после реконнекта — откат/пометка конфликта.
|
||||
5. **Правила разрешения конфликтов** (см. §5).
|
||||
6. **Проигрывание при реконнекте** в порядке `createdAt`, с экспоненциальным
|
||||
backoff и идемпотентностью.
|
||||
|
||||
**Файлы:** новый `apps/client/src/lib/offline/outbox.ts`, обёртки над
|
||||
`features/*/services/*`, серверные контроллеры/сервисы соответствующих
|
||||
сущностей (idempotent upsert).
|
||||
|
||||
**Критерий приёмки:** оффлайн можно создать страницу, отредактировать заголовок,
|
||||
оставить комментарий, переместить страницу; после реконнекта всё появляется на
|
||||
сервере один раз (без дублей), конфликты разрешаются по заданным правилам.
|
||||
|
||||
**Риск:** высокий (это самостоятельный класс багов синхронизации; требует
|
||||
серверных изменений и тестов на конфликты).
|
||||
|
||||
---
|
||||
|
||||
### M4 — Вложения и оффлайн-авторизация
|
||||
|
||||
**Что сделать:**
|
||||
1. **Вложения/картинки оффлайн.** Очередь загрузок: blob кладётся в локальный
|
||||
кэш (Cache API/IndexedDB), в документ вставляется ссылка на локальный
|
||||
ресурс; при реконнекте файл доуплоадивается, ссылка переписывается на
|
||||
серверную. Точка входа — `features/attachments`.
|
||||
2. **Оффлайн-толерантная авторизация.** В
|
||||
[api-client.ts](../apps/client/src/lib/api-client.ts) `401`/сетевые ошибки
|
||||
**не должны** выкидывать на логин при отсутствии сети — отличать «нет сети»
|
||||
от «реально разлогинен». Collab-токен (JWT с TTL,
|
||||
[page-editor.tsx](../apps/client/src/features/editor/page-editor.tsx) L166–181)
|
||||
оффлайн не обновить — синк должен просто ждать реконнекта, не ломая
|
||||
локальную работу.
|
||||
|
||||
**Критерий приёмки:** оффлайн-вставка картинки доезжает после реконнекта;
|
||||
протухший токен/нет сети не выкидывают пользователя из приложения и не теряют
|
||||
локальные правки.
|
||||
|
||||
**Риск:** средний.
|
||||
|
||||
---
|
||||
|
||||
## 5. Правила разрешения конфликтов (Контур B)
|
||||
|
||||
CRDT здесь нет, правила задаём явно по типам сущностей:
|
||||
|
||||
| Сущность | Стратегия |
|
||||
|---|---|
|
||||
| **Тело документа** | Yjs (CRDT) — руками ничего не решаем. |
|
||||
| **Комментарии** | Почти append-only. LWW по полю + дедуп по `clientId`. Простейший случай. |
|
||||
| **Метаданные страницы** (заголовок, иконка) | Last-Write-Wins по `updatedAt`. |
|
||||
| **Перемещение в дереве** | Самый сложный случай. Позиции — строковые fractional-ключи (`generateJitteredKeyBetween`), что снижает коллизии вставок. Нужен серверный реконсилер для «родитель удалён, а ребёнок перемещён» и конкурентных move: правило «удаление побеждает перемещение» (или наоборот — зафиксировать), плюс перегенерация позиции при коллизии. |
|
||||
| **Удаление vs правка** | Зафиксировать политику: правка удалённой сущности → конфликт в UI либо «удаление выигрывает». |
|
||||
|
||||
---
|
||||
|
||||
## 6. Подводные камни (читать до старта)
|
||||
|
||||
1. **Yjs rebuild из JSON → дубли.** Ветка `content → toYdoc` в
|
||||
`onLoadDocument` опасна для долго-оффлайновых клиентов. Закрыть в M1.
|
||||
2. **Инвалидция кэша после деплоя.** Персист React Query и precache SW должны
|
||||
версионироваться по версии приложения (`buster`/`globPatterns` хэши), иначе
|
||||
пользователь застрянет на старом UI/данных.
|
||||
3. **Обновление service worker.** `autoUpdate` может перезагрузить вкладку с
|
||||
несохранёнными правками. Для редактора предпочтительнее `prompt`-стратегия
|
||||
(показать «доступно обновление», применить по согласию).
|
||||
4. **Идемпотентность обязательна.** Любая мутация из outbox может отправиться
|
||||
повторно (реконнект/ретрай). Без серверного upsert по `clientId` — дубли.
|
||||
5. **Рост IndexedDB.** Прогрев тел страниц «на оффлайн» и кэш блобов могут
|
||||
занять много места. Нужны лимиты/очистка (LRU).
|
||||
6. **Редирект на логин при сетевой ошибке.** Сейчас `401` → `redirectToLogin`.
|
||||
Оффлайн это выкинет пользователя и потеряет контекст — чинить в M4.
|
||||
|
||||
---
|
||||
|
||||
## 7. Зависимости (npm)
|
||||
|
||||
| Пакет | Зачем | Этап |
|
||||
|---|---|---|
|
||||
| `vite-plugin-pwa` (+ Workbox) | SW, precache app-shell, генерация манифеста | M0 |
|
||||
| `@tanstack/query-persist-client-core` | Персист React Query на диск | M2 |
|
||||
| `idb` или `idb-keyval` | Обёртка над IndexedDB (persister/outbox/blob-кэш) | M2–M4 |
|
||||
| `fractional-indexing-jittered` | Клиентская генерация позиций (уже есть на сервере) | M3 |
|
||||
|
||||
`yjs`, `y-indexeddb`, `@hocuspocus/provider` — **уже** в проекте, доустанавливать
|
||||
не нужно.
|
||||
|
||||
---
|
||||
|
||||
## 8. Объём работ vs ценность (для приоритизации)
|
||||
|
||||
| Уровень | Этапы | Что пользователь получает |
|
||||
|---|---|---|
|
||||
| **Минимальный** | M0 + M1 | Приложение грузится оффлайн; уже открытые страницы редактируются и синкаются (тело + заголовок). Навигация — только по закэшированному. |
|
||||
| **Средний** | + M2 + M3 | Оффлайн-навигация по подготовленным пространствам; оффлайн-создание страниц и комментариев с синком и LWW-конфликтами. |
|
||||
| **Полный** | + M4 (и при необходимости — переезд на синк-движок) | Вложения оффлайн, устойчивая авторизация. Полноценный local-first. |
|
||||
|
||||
Прагматичный путь: довести **M0+M1** (это ~80% «редактирую то, что открыл»),
|
||||
затем M2/M3 инкрементально. Полный синк-движок (RxDB / ElectricSQL / PowerSync /
|
||||
Replicache / TanStack DB) рассматривать только если оффлайн станет ключевым
|
||||
сценарием продукта — это существенный рефакторинг данных и бэкенда.
|
||||
|
||||
---
|
||||
|
||||
## 9. Открытые вопросы (зафиксировать до реализации)
|
||||
|
||||
- **Q1.** Заголовок страницы: переносим в Yjs (M1, вариант a) или гоним через
|
||||
outbox (M3, вариант b)? Рекомендация — (a), меньше конфликтных правил.
|
||||
- **Q2.** Политика конфликта «удаление vs правка»: «удаление выигрывает» или
|
||||
явный конфликт в UI?
|
||||
- **Q3.** Стратегия обновления SW для редактора: `autoUpdate` или `prompt`?
|
||||
Рекомендация — `prompt`.
|
||||
- **Q4.** Лимиты локального хранилища (сколько пространств/страниц/блобов
|
||||
держать оффлайн, политика вытеснения).
|
||||
- **Q5.** Целимся в инкрементальный путь (M0…M4) или сразу в синк-движок (уровень
|
||||
«полный»)? От этого зависит, переписывать ли REST-слой.
|
||||
|
||||
---
|
||||
|
||||
## 10. Чеклист реализации
|
||||
|
||||
- [ ] M0: `vite-plugin-pwa` подключён, SW регистрируется, app-shell в precache,
|
||||
`/api` и `/collab` — `NetworkOnly`.
|
||||
- [ ] M0: приложение открывается без сети (shell виден).
|
||||
- [ ] M1: ветка rebuild ydoc из JSON обезврежена; миграция `content → ydoc`.
|
||||
- [ ] M1: индикатор статуса синка в UI.
|
||||
- [ ] M1: заголовок переведён в Yjs (или решение Q1 принято).
|
||||
- [ ] M2: React Query персистится в IndexedDB, кэш версионирован.
|
||||
- [ ] M2: действие «сделать доступным оффлайн» (метаданные + прогрев `ydoc`).
|
||||
- [ ] M3: outbox в IndexedDB, клиентские ID, идемпотентный upsert на сервере.
|
||||
- [ ] M3: optimistic updates + откат; правила конфликтов реализованы.
|
||||
- [ ] M4: очередь загрузки вложений + локальный blob-кэш.
|
||||
- [ ] M4: авторизация толерантна к оффлайну (нет редиректа на логин при отсутствии сети).
|
||||
@@ -1,421 +0,0 @@
|
||||
# Потоковая диктовка (realtime STT) — дизайн
|
||||
|
||||
> Статус: **черновик / дизайн**. Реализация ещё не начата.
|
||||
> Исходный кейс: при диктовке текст должен появляться **по мере речи**, а не одним
|
||||
> куском после остановки записи.
|
||||
>
|
||||
> Принятые на старте предпосылки (требуют подтверждения, см. §3 «Развилки»):
|
||||
> - **Семантика** — настоящий realtime: аудио стримится во время речи, частичные
|
||||
> расшифровки (`delta`) дописываются в редактор немедленно (~150–300 мс до
|
||||
> первого частичного текста на проводном соединении).
|
||||
> - **Провайдер** — OpenAI Realtime API (или совместимый: Azure OpenAI). Это
|
||||
> ломает текущую провайдер-агностичность диктовки (см. §2) — realtime становится
|
||||
> **опциональной** возможностью поверх существующей пакетной диктовки, а не
|
||||
> заменой ей.
|
||||
|
||||
---
|
||||
|
||||
## 1. Что есть сейчас (пакетная диктовка)
|
||||
|
||||
Текущая диктовка — строго «запиши целиком → отправь → получи весь текст», без
|
||||
какого-либо стрима:
|
||||
|
||||
**Клиент.**
|
||||
- [use-dictation.ts](../apps/client/src/features/dictation/hooks/use-dictation.ts) —
|
||||
стейт-машина захвата на `MediaRecorder`. Чанки копятся в `chunksRef` в
|
||||
`recorder.ondataavailable`, но **никуда не уходят по ходу записи**; единый `Blob`
|
||||
собирается только в `recorder.onstop` и одним `multipart`-POST отправляется на
|
||||
транскрипцию. Кодек — сжатый `audio/webm;codecs=opus` (Safari: `audio/mp4`).
|
||||
- [dictation-service.ts](../apps/client/src/features/dictation/services/dictation-service.ts) —
|
||||
`transcribeAudio(blob, filename)` → `POST /ai-chat/transcribe`.
|
||||
- [mic-button.tsx](../apps/client/src/features/dictation/components/mic-button.tsx) —
|
||||
кнопка с состояниями `idle → recording → transcribing → idle`.
|
||||
- [dictation-group.tsx](../apps/client/src/features/editor/components/fixed-toolbar/groups/dictation-group.tsx) —
|
||||
снапшотит каретку в `onStart`, вставляет **готовый** текст в зафиксированную
|
||||
позицию, клампит её под текущий размер документа (учёт коллаб-дрейфа).
|
||||
- В чате — тот же `MicButton` в [chat-input.tsx](../apps/client/src/features/ai-chat/components/chat-input.tsx),
|
||||
текст дописывается в черновик сообщения.
|
||||
|
||||
**Сервер.**
|
||||
- Эндпоинт `POST /ai-chat/transcribe` в
|
||||
[ai-chat.controller.ts](../apps/server/src/core/ai-chat/ai-chat.controller.ts#L195-L281):
|
||||
гейт `settings.ai.dictation === true` (иначе 403), приём файла до 25 МБ,
|
||||
whitelist MIME, троттлинг 20 req/min на пользователя, маппинг MIME→`format`,
|
||||
вызов `AiTranscriptionService.transcribe()`.
|
||||
- [ai-transcription.service.ts](../apps/server/src/core/ai-chat/ai-transcription.service.ts) —
|
||||
тонкая обёртка над `AiService.transcribe()`.
|
||||
- [ai.service.ts](../apps/server/src/integrations/ai/ai.service.ts#L120-L187) —
|
||||
два пути по `sttApiStyle`: `multipart` (AI SDK `experimental_transcribe`,
|
||||
OpenAI/speaches/faster-whisper/Ollama) и `json` (base64 на
|
||||
`{baseURL}/audio/transcriptions`, OpenRouter). Оба возвращают **весь текст за
|
||||
один вызов**, без SSE/WS.
|
||||
- Конфиг STT — per-workspace в `settings.ai.provider` (`sttModel`, `sttBaseUrl`,
|
||||
`sttApiStyle`), ключ зашифрован в `ai_provider_credentials`, расшифровывается
|
||||
только в [ai-settings.service.ts](../apps/server/src/integrations/ai/ai-settings.service.ts#L113-L157)
|
||||
(`resolve`) и **никогда не логируется и не уходит клиенту** (только маска
|
||||
`hasSttApiKey`).
|
||||
|
||||
**Вывод.** «По мере речи» в текущей архитектуре невозможно в принципе: текст
|
||||
рисуется одним куском в `onstop`. Нужен принципиально другой транспорт.
|
||||
|
||||
---
|
||||
|
||||
## 2. Главное архитектурное противоречие
|
||||
|
||||
Пакетная диктовка **провайдер-агностична**: работает с любым OpenAI-совместимым
|
||||
`/audio/transcriptions` (включая self-hosted speaches/faster-whisper и Ollama)
|
||||
просто через `sttBaseUrl` + `sttApiStyle`.
|
||||
|
||||
Realtime STT — **не** часть OpenAI-совместимого REST. Это отдельный протокол
|
||||
(WebSocket/WebRTC + событийная модель), который реализуют единицы провайдеров:
|
||||
OpenAI Realtime, Azure OpenAI Realtime, и (с другим набором событий) пара сторонних
|
||||
вроде Together AI. Self-hosted whisper-серверы его, как правило, **не умеют**.
|
||||
|
||||
Поэтому realtime нельзя «просто включить» вместо пакетной диктовки. Дизайн исходит
|
||||
из того, что:
|
||||
|
||||
1. Пакетная диктовка (§1) **остаётся** как дефолт и фоллбэк.
|
||||
2. Realtime — **опциональная** возможность, доступная только когда workspace
|
||||
настроен на realtime-совместимый провайдер (новый флаг/поле конфига, см. §5).
|
||||
3. Если realtime не настроен или соединение не поднялось — UI прозрачно
|
||||
деградирует к пакетному пути.
|
||||
|
||||
---
|
||||
|
||||
## 3. Контракт провайдера (OpenAI Realtime, transcription session)
|
||||
|
||||
Сверено с актуальной документацией (ссылки в конце). Ключевые факты:
|
||||
|
||||
**Создание сессии и эфемерный токен.**
|
||||
- REST `POST /v1/realtime/transcription_sessions` (в GA-вариантах —
|
||||
`POST /v1/realtime/client_secrets` с телом-конфигом сессии) возвращает
|
||||
`client_secret.value` — **эфемерный** токен с коротким TTL для браузера.
|
||||
Постоянный ключ воркспейса при этом наружу не отдаётся.
|
||||
> На момент реализации сверить точный эндпоинт и форму тела с текущими доками —
|
||||
> API эволюционирует.
|
||||
|
||||
**Транспорт.**
|
||||
- **WebRTC** — рекомендуется для браузерного аудио (захват + воспроизведение).
|
||||
- **WebSocket** — для серверных аудио-пайплайнов:
|
||||
`wss://api.openai.com/v1/realtime?intent=transcription`, заголовки
|
||||
`Authorization: Bearer <key>` и `OpenAI-Beta: realtime=v1`.
|
||||
|
||||
**Формат входного аудио.** `pcm16` (raw 16-bit PCM, mono), частота 16 кГц или
|
||||
24 кГц; либо `g711`. **Не** webm/opus и **не** mp4 — то есть текущий
|
||||
`MediaRecorder`-путь для realtime неприменим (см. §6, AudioWorklet).
|
||||
|
||||
**События клиент→сервер.**
|
||||
- `transcription_session.update` (или `session.update`) — конфиг модели/VAD/языка.
|
||||
- `input_audio_buffer.append` — чанк аудио (base64 PCM16).
|
||||
- `input_audio_buffer.commit` — закрыть сегмент вручную (когда VAD выключен).
|
||||
|
||||
**События сервер→клиент.**
|
||||
- `conversation.item.input_audio_transcription.delta` — поле `delta` с
|
||||
инкрементальным текстом (частичная расшифровка).
|
||||
- `conversation.item.input_audio_transcription.completed` — поле `transcript` с
|
||||
финальным текстом сегмента. У обоих есть `item_id` для сопоставления сегментов.
|
||||
- `error` — ошибки сессии.
|
||||
|
||||
**Turn detection / VAD.** `turn_detection: { type: "server_vad" }` —
|
||||
сервер сам нарезает речь на сегменты и эмитит `completed` на границе паузы; для
|
||||
непрерывной диктовки это удобнее ручного commit. Модели: `gpt-4o-transcribe`,
|
||||
`gpt-4o-mini-transcribe`, потоковая `gpt-realtime-whisper` (у неё настраиваемая
|
||||
задержка `delay`: `minimal…xhigh` — баланс «латентность ↔ качество»).
|
||||
|
||||
> Важно: `delta`-события дают **черновой** текст, который последующие события
|
||||
> могут **переписать**. UI должен уметь заменять ранее показанный частичный текст
|
||||
> (см. §3 «Развилка B» про вставку в редактор).
|
||||
|
||||
---
|
||||
|
||||
## 4. Развилка A — транспорт: прямое WebRTC vs серверный WS-прокси
|
||||
|
||||
### Вариант A1 — браузер ↔ OpenAI напрямую (WebRTC, эфемерный токен)
|
||||
Наш сервер только минтит эфемерный токен (`/realtime/transcription_sessions`
|
||||
постоянным ключом воркспейса), браузер сам устанавливает WebRTC к OpenAI и
|
||||
получает `delta`/`completed`.
|
||||
|
||||
- **Плюсы:** минимальная латентность (нет лишнего хопа), аудио не идёт через наш
|
||||
сервер (нет нагрузки на bandwidth), меньше серверного кода.
|
||||
- **Минусы:**
|
||||
- Работает **только** с настоящим OpenAI/Azure (нужна поддержка эфемерных
|
||||
токенов и WebRTC) — `sttBaseUrl` на self-hosted/прокси-шлюз тут бесполезен.
|
||||
- Браузер устанавливает соединение с внешним хостом напрямую — мимо нашего
|
||||
[ssrf-guard](../apps/server/src/core/ai-chat/external-mcp/ssrf-guard.ts) и
|
||||
серверного троттлинга/гейтинга на уровне каждого сообщения (гейт можно
|
||||
проверить только в момент минтинга токена).
|
||||
- Эфемерный токен живёт в браузере (короткий TTL смягчает, но это всё же
|
||||
выдача наружу производного секрета).
|
||||
- WebRTC в браузере (`RTCPeerConnection`, SDP-оффер, обмен через REST) — больше
|
||||
клиентской машинерии и краевых случаев.
|
||||
|
||||
### Вариант A2 (рекомендуется) — браузер ↔ наш сервер (WS) ↔ OpenAI (WS)
|
||||
Браузер шлёт PCM16-чанки по WebSocket на наш новый gateway; сервер держит upstream
|
||||
WS к `wss://api.openai.com/v1/realtime?intent=transcription` с **постоянным**
|
||||
ключом воркспейса и проксирует `delta`/`completed` обратно браузеру.
|
||||
|
||||
- **Плюсы:**
|
||||
- Ключ **никогда не покидает сервер** — ровно как в текущем коде
|
||||
([ai-settings.service.ts](../apps/server/src/integrations/ai/ai-settings.service.ts#L138-L154)),
|
||||
эфемерные токены не нужны.
|
||||
- Работает с **любым** realtime-совместимым эндпоинтом через `sttBaseUrl`
|
||||
(OpenAI, Azure, будущий self-hosted), и upstream-URL проходит через
|
||||
SSRF-валидацию перед коннектом.
|
||||
- Гейт `settings.ai.dictation`, аутентификация (JWT воркспейса), троттлинг и
|
||||
лимиты длительности/объёма применяются **на сервере** на каждом соединении.
|
||||
- Совместимо с тем, что в проекте **уже есть WebSocket-инфраструктура** —
|
||||
коллаб-сервер на Hocuspocus + Socket.IO-адаптер на Redis
|
||||
([collaboration/](../apps/server/src/collaboration/)), и Fastify-приложение.
|
||||
- **Минусы:**
|
||||
- Аудио идёт через наш сервер (≈ десятки кбит/с на сессию для PCM16@24k ⇒
|
||||
~48 КБ/с; терпимо, но это нагрузка и нужно ограничивать конкуррентность).
|
||||
- Двойной хоп добавляет немного латентности (доли сотни мс).
|
||||
- Нужен новый WS-gateway и аккуратный proxy-стейт (бэкпрешер, очистка сокетов).
|
||||
|
||||
**Решение (предлагается): A2.** Он единственный согласуется с инвариантами
|
||||
кодовой базы — «ключ только на сервере», провайдер-агностичность через `baseURL`,
|
||||
SSRF-guard, серверные гейты и троттлинг. A1 оставить как возможную оптимизацию
|
||||
латентности «потом», если упрёмся в bandwidth.
|
||||
|
||||
Дальнейший дизайн исходит из **A2**.
|
||||
|
||||
---
|
||||
|
||||
## 5. Развилка B — куда писать частичный текст в редакторе
|
||||
|
||||
`delta` — черновой текст, который может быть переписан. Слепо вставлять каждую
|
||||
`delta` в документ Tiptap нельзя: (1) каждая правка документа порождает Yjs-апдейт,
|
||||
шумит в истории/коллабе и тяжела; (2) переписывание ранее показанного текста
|
||||
превращается в постоянные replace по диапазону.
|
||||
|
||||
### Вариант B1 — провизорная вставка в документ + замена диапазона
|
||||
Вставляем `delta` прямо в документ, запоминаем диапазон провизорного текста,
|
||||
на каждую новую `delta`/`completed` заменяем этот диапазон. На `completed` —
|
||||
«фиксируем» (диапазон становится обычным текстом).
|
||||
|
||||
- **Плюсы:** текст сразу «настоящий», работает для любого приёмника (редактор и
|
||||
чат единообразно), не нужен слой декораций.
|
||||
- **Минусы:** активный коллаб + история засоряются промежуточными апдейтами;
|
||||
замена диапазона воюет с коллаб-дрейфом (диапазон надо ремапить, как уже делает
|
||||
[dictation-group.tsx](../apps/client/src/features/editor/components/fixed-toolbar/groups/dictation-group.tsx#L24-L26));
|
||||
откат при отмене сложнее.
|
||||
|
||||
### Вариант B2 (рекомендуется для редактора) — ProseMirror-декорация для interim, коммит только финала
|
||||
Частичный текст показываем виджет-декорацией (inline widget) у каретки — он **не
|
||||
часть документа**, не порождает Yjs-апдейтов и не попадает в историю. В документ
|
||||
коммитим только текст из `completed`-сегмента (как сейчас — `insertContentAt` в
|
||||
снапшот каретки, с тем же клампом под коллаб-дрейф).
|
||||
|
||||
- **Плюсы:** ноль мусора в коллабе/истории до финала; отмена = просто снять
|
||||
декорацию; финальная вставка переиспользует уже существующую и проверенную
|
||||
логику `dictation-group`.
|
||||
- **Минусы:** нужна небольшая ProseMirror-плагин-декорация (новый код); «по мере
|
||||
речи» виден interim как подсветка-призрак, а в документ «оседает» по сегментам
|
||||
(на паузах VAD) — на практике это естественный UX (как у системных диктовок).
|
||||
|
||||
### Для чата
|
||||
В [chat-input.tsx](../apps/client/src/features/ai-chat/components/chat-input.tsx)
|
||||
приёмник — обычный `textarea`/draft, декораций нет. Там проще **B1-подобно**:
|
||||
показывать `interim` как «хвост» черновика (например, отдельным стейтом, который
|
||||
рендерится приглушённо), а на `completed` дописывать в основной черновик. То есть
|
||||
интерфейс хука должен отдавать и `interim`, и `final` (см. §6).
|
||||
|
||||
**Решение (предлагается):** редактор — **B2** (декорация + коммит финала), чат —
|
||||
показ interim-хвоста + коммит финала. Единый хук realtime отдаёт оба потока,
|
||||
а приёмник сам решает, как показывать interim.
|
||||
|
||||
---
|
||||
|
||||
## 6. Детальный дизайн (A2 + B2)
|
||||
|
||||
### 6.1 Клиент: захват аудио (PCM16 через Web Audio API)
|
||||
`MediaRecorder` отдаёт сжатый webm/opus — для realtime **не подходит**. Нужен
|
||||
сырой PCM16:
|
||||
|
||||
1. `getUserMedia({ audio: true })` (как сейчас).
|
||||
2. `AudioContext` + `AudioWorkletNode` (новый worklet-процессор): забирает
|
||||
Float32-фреймы, ресемплит к 24 кГц mono, конвертит в Int16, шлёт в основной
|
||||
поток.
|
||||
3. Чанки PCM16 → base64 → событие `input_audio_buffer.append` на наш WS-gateway
|
||||
(батчинг ~каждые 100–250 мс, чтобы не спамить сообщениями).
|
||||
4. На стоп — закрыть worklet, остановить треки (как в текущем `stopTracks`),
|
||||
дослать остаток.
|
||||
|
||||
Новый код, в идеале — отдельный хук `use-realtime-dictation.ts` рядом с
|
||||
[use-dictation.ts](../apps/client/src/features/dictation/hooks/use-dictation.ts),
|
||||
с тем же «фасадом» (`status/start/stop/cancel`) **плюс** колбэки `onInterim(text)`
|
||||
и `onFinal(text)`. `MicButton` выбирает реализацию (realtime vs batch) по флагу из
|
||||
конфига воркспейса; вся остальная обвязка (тултипы, состояния, обработка ошибок,
|
||||
гард двойного клика, очистка на unmount) переиспользуется один-в-один.
|
||||
|
||||
> AudioWorklet требует безопасного контекста (HTTPS/localhost) — то же ограничение,
|
||||
> что уже есть у `getUserMedia` в текущем хуке. Нужен бандл worklet-файла через
|
||||
> Vite (`?url`/`?worker`); сверить с тем, как проект собирает воркеры.
|
||||
|
||||
### 6.2 Сервер: WS-gateway + realtime-прокси
|
||||
Новый модуль внутри `core/ai-chat` (рядом с `ai-transcription.service.ts`):
|
||||
|
||||
- **WS endpoint** (например, `ws://…/ai-chat/realtime-transcribe`). Поднять либо
|
||||
как Nest WebSocketGateway, либо как Fastify-WS-роут — выбрать по тому, что уже
|
||||
используется в проекте (Socket.IO-адаптер на Redis в
|
||||
[collaboration/](../apps/server/src/collaboration/)). На коннекте:
|
||||
- аутентификация JWT воркспейса (как у остальных `/ai-chat` маршрутов);
|
||||
- гейт `settings.ai.dictation === true` (иначе закрыть с понятным кодом/причиной);
|
||||
- троттлинг/лимит одновременных realtime-сессий на пользователя и на воркспейс
|
||||
(realtime дороже пакетной диктовки — нужен явный потолок).
|
||||
- **Резолв конфига** через `AiSettingsService.resolve(workspaceId)`: нужны
|
||||
`sttModel`, `sttBaseUrl||baseUrl`, `sttApiKey`. **До** коннекта прогнать
|
||||
upstream-URL через [ssrf-guard](../apps/server/src/core/ai-chat/external-mcp/ssrf-guard.ts).
|
||||
- **Upstream WS** к `wss://<base>/realtime?intent=transcription` (npm `ws`),
|
||||
заголовки `Authorization: Bearer <sttApiKey>` + `OpenAI-Beta: realtime=v1`.
|
||||
Сразу отправить `transcription_session.update` с моделью/языком/`server_vad`.
|
||||
- **Прокси:** PCM16 от браузера → `input_audio_buffer.append` в upstream;
|
||||
`…transcription.delta` / `…completed` / `error` из upstream → клиенту
|
||||
(можно прозрачно ретранслировать, либо нормализовать в свой минимальный формат
|
||||
`{type:'interim'|'final'|'error', text, itemId}` — предпочтительно
|
||||
нормализовать, чтобы не привязывать клиент к сырой схеме OpenAI и упростить
|
||||
будущую поддержку Azure/иных).
|
||||
- **Очистка:** при закрытии любого из двух сокетов — закрыть второй, освободить
|
||||
ресурсы; таймаут простоя; лимит длительности сессии (аналог 120 с в текущем
|
||||
хуке) и лимит суммарного объёма аудио.
|
||||
|
||||
Расширить `AiService` (или новый `AiRealtimeService`) методом, инкапсулирующим
|
||||
upstream-WS, чтобы контроллер/gateway оставался тонким — симметрично текущему
|
||||
`transcribe()`.
|
||||
|
||||
### 6.3 Конфиг воркспейса
|
||||
Добавить в [ai.types.ts](../apps/server/src/integrations/ai/ai.types.ts) и в
|
||||
[ai-settings.service.ts](../apps/server/src/integrations/ai/ai-settings.service.ts):
|
||||
- `sttRealtime?: boolean` — включает realtime-путь для воркспейса.
|
||||
- `sttRealtimeModel?: string` — модель realtime (например `gpt-4o-mini-transcribe`
|
||||
/ `gpt-realtime-whisper`); если пусто — фоллбэк на `sttModel`.
|
||||
- (опц.) `sttRealtimeBaseUrl?` — если realtime-эндпоинт отличается от `sttBaseUrl`.
|
||||
|
||||
Ключ переиспользуется (`sttApiKey` → fallback `apiKey`), новых секретов не нужно.
|
||||
В `getMasked` отдавать новые **несекретные** поля; в `resolve` — как сейчас.
|
||||
UI настроек (Workspace settings → AI) — добавить тумблер «Realtime dictation» и
|
||||
поле модели рядом с существующими STT-полями; кнопка «Test endpoint» для realtime
|
||||
делает короткий тестовый коннект (открыть сессию, послать ~0.5 с тишины, дождаться
|
||||
`session.created`/`error`, закрыть) и возвращает `ok|error` через
|
||||
`describeProviderError`-подобную нормализацию.
|
||||
|
||||
### 6.4 Клиентский конфиг-гейт
|
||||
Realtime-кнопку показывать только если `workspace.settings.ai.dictation === true`
|
||||
**и** `…ai.provider.sttRealtime === true`. Иначе — текущая пакетная кнопка. Маска
|
||||
настроек должна отдавать эти флаги клиенту (несекретные).
|
||||
|
||||
---
|
||||
|
||||
## 7. Безопасность и соответствие конвенциям
|
||||
|
||||
- **Ключ только на сервере** (вариант A2): постоянный ключ не уходит клиенту,
|
||||
эфемерные токены не используются — инвариант
|
||||
[§8 ai-settings](../apps/server/src/integrations/ai/ai-settings.service.ts#L38-L45)
|
||||
сохранён. Ключ не логируется.
|
||||
- **SSRF:** upstream realtime-URL валидируется через
|
||||
[ssrf-guard.ts](../apps/server/src/core/ai-chat/external-mcp/ssrf-guard.ts)
|
||||
перед коннектом (особенно если разрешаем кастомный `sttRealtimeBaseUrl`).
|
||||
- **Гейт/авторизация/троттлинг** — на сервере, на каждом WS-коннекте; плюс жёсткий
|
||||
лимит одновременных realtime-сессий (это дорого) и лимит длительности.
|
||||
- **Обработка ошибок (конвенция проекта).** Любая ошибка (upstream `error`,
|
||||
разрыв сокета, провайдер-таймаут, не настроен realtime, отказ микрофона):
|
||||
- на сервере — лог полностью (имя/сообщение/стек/`cause`, статус upstream) и
|
||||
отдача клиенту **конкретной** причины (не «Something went wrong»), через
|
||||
нормализатор уровня `describeProviderError`;
|
||||
- на клиенте — `console.error(<context>, err)` + нотификация с реальной причиной
|
||||
(как уже сделано в
|
||||
[use-dictation.ts](../apps/client/src/features/dictation/hooks/use-dictation.ts#L187-L213)).
|
||||
- **Деградация:** realtime недоступен/упал на старте → молча используем пакетную
|
||||
диктовку (она всегда есть); realtime упал в середине → коммитим уже полученные
|
||||
`completed`-сегменты, показываем причину, предлагаем продолжить пакетно.
|
||||
|
||||
---
|
||||
|
||||
## 8. Краевые случаи
|
||||
|
||||
- **Коллаб-дрейф:** между `start` и каждым `completed` документ мог измениться —
|
||||
ремап/кламп позиции вставки (логика уже есть в `dictation-group`); для interim
|
||||
декорация привязывается к текущей каретке, не к абсолютной позиции.
|
||||
- **Отмена записи:** снять декорацию, ничего не коммитить, закрыть оба сокета.
|
||||
- **Тишина/нет речи:** VAD не эмитит сегментов — корректно завершить без вставки.
|
||||
- **Длинная диктовка:** server_vad нарезает на сегменты автоматически; следить за
|
||||
лимитом длительности и объёма.
|
||||
- **Переписывание interim:** поздние `delta` правят ранние — UI всегда показывает
|
||||
последнюю версию текущего (ещё не `completed`) сегмента.
|
||||
- **Языки/пунктуация:** прокидывать `language` в конфиг сессии (или авто);
|
||||
модель сама расставляет пунктуацию.
|
||||
- **Несколько вкладок / двойной старт:** гард как в текущем хуке + серверный лимит
|
||||
сессий.
|
||||
- **Старые браузеры без AudioWorklet:** фоллбэк на пакетную диктовку.
|
||||
|
||||
---
|
||||
|
||||
## 9. Поэтапный план реализации
|
||||
|
||||
1. **Конфиг и гейт.** `ai.types.ts` + `ai-settings.service.ts` (`sttRealtime`,
|
||||
`sttRealtimeModel`), маска, UI-тумблер и «Test endpoint». Без транспорта —
|
||||
просто читается/пишется.
|
||||
2. **Серверный realtime-прокси.** WS-gateway + `AiRealtimeService` (upstream WS к
|
||||
OpenAI, SSRF, гейт, троттлинг, нормализация событий, очистка). Покрыть
|
||||
юнит/моками парс событий и закрытие сокетов.
|
||||
3. **Клиентский захват PCM16.** AudioWorklet-процессор + `use-realtime-dictation`
|
||||
(фасад `status/start/stop/cancel` + `onInterim/onFinal`), подключение к WS.
|
||||
4. **UI interim.** B2-декорация в редакторе + коммит финала через существующую
|
||||
`dictation-group`-логику; в чате — interim-хвост + коммит. Переключение
|
||||
realtime/batch в `MicButton` по флагу конфига.
|
||||
5. **Закалка.** Лимиты, таймауты, фоллбэки, нотификации с реальными причинами,
|
||||
нагрузочная проверка одновременных сессий.
|
||||
|
||||
---
|
||||
|
||||
## 10. Открытые вопросы / риски
|
||||
|
||||
- **Подтвердить семантику** (предпосылки в шапке): нужен именно realtime «по мере
|
||||
речи» (A2/B2), а не просто «прогрессивный вывод после стопа» (`stream:true` на
|
||||
`gpt-4o-transcribe` — гораздо дешевле и проще, но текст идёт только **после**
|
||||
остановки записи).
|
||||
- **Точная форма Realtime API** (эндпоинт сессии, имена событий, формат аудио)
|
||||
меняется — сверить с актуальными доками на момент реализации.
|
||||
- **Стоимость/латентность** realtime заметно выше пакетной диктовки — нужен явный
|
||||
потолок одновременных сессий и, возможно, явное предупреждение админу.
|
||||
- **Нагрузка на наш сервер** (аудио через прокси) — измерить на реальной
|
||||
конкуррентности; при необходимости позднее добавить путь A1 (WebRTC напрямую).
|
||||
- **AudioWorklet-бандлинг** под Vite — проверить, как проект собирает воркеры.
|
||||
- Совместимость с Azure OpenAI Realtime (другой хост/версия API) — учесть в
|
||||
нормализации событий, чтобы клиент не зависел от сырой схемы.
|
||||
|
||||
---
|
||||
|
||||
## 11. Ориентир по затрагиваемым файлам
|
||||
|
||||
Новые:
|
||||
- `apps/client/src/features/dictation/hooks/use-realtime-dictation.ts`
|
||||
- `apps/client/src/features/dictation/audio/pcm16-worklet.*` (worklet + загрузчик)
|
||||
- `apps/client/src/features/editor/.../dictation-interim-decoration.*` (ProseMirror-плагин)
|
||||
- `apps/server/src/core/ai-chat/ai-realtime.service.ts` (+ WS-gateway)
|
||||
|
||||
Изменяемые:
|
||||
- [ai.types.ts](../apps/server/src/integrations/ai/ai.types.ts),
|
||||
[ai-settings.service.ts](../apps/server/src/integrations/ai/ai-settings.service.ts) —
|
||||
новые поля конфига + маска.
|
||||
- [ai.service.ts](../apps/server/src/integrations/ai/ai.service.ts) — realtime
|
||||
test-connection (если делать через AiService).
|
||||
- [mic-button.tsx](../apps/client/src/features/dictation/components/mic-button.tsx) —
|
||||
выбор realtime/batch по флагу.
|
||||
- [dictation-group.tsx](../apps/client/src/features/editor/components/fixed-toolbar/groups/dictation-group.tsx),
|
||||
[chat-input.tsx](../apps/client/src/features/ai-chat/components/chat-input.tsx) —
|
||||
обработка `onInterim/onFinal`.
|
||||
- Настройки AI в клиенте (Workspace settings → AI) — тумблер + модель + тест.
|
||||
- AI-модуль сервера ([app.module.ts](../apps/server/src/app.module.ts) /
|
||||
`ai-chat`-модуль) — регистрация gateway.
|
||||
|
||||
---
|
||||
|
||||
## Источники
|
||||
|
||||
- [Realtime transcription — OpenAI API](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
||||
- [Create transcription session — OpenAI API Reference](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create)
|
||||
- [Speech to text — OpenAI API](https://developers.openai.com/api/docs/guides/speech-to-text)
|
||||
- [Realtime and audio — OpenAI API](https://developers.openai.com/api/docs/guides/realtime)
|
||||
</content>
|
||||
</invoke>
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "docmost",
|
||||
"homepage": "https://docmost.com",
|
||||
"version": "0.93.0",
|
||||
"version": "0.94.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "nx run-many -t build",
|
||||
|
||||
@@ -162,6 +162,160 @@ describe('getFootnoteRefCount (cached, live editor)', () => {
|
||||
expect(getFootnoteRefCount(editor.state, 'nope')).toBe(0);
|
||||
editor.destroy();
|
||||
});
|
||||
|
||||
// #185 re-review pt 9: the cached count must update on a doc change (mirror of
|
||||
// the number-cache invalidation test) — add another `[^a]` reference and the
|
||||
// count goes 2 -> 3.
|
||||
it('recomputes the cached ref count when a reference is added', () => {
|
||||
const editor = makeEditor({
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{ type: FOOTNOTE_REFERENCE_NAME, attrs: { id: 'a' } },
|
||||
{ type: 'text', text: ' and ' },
|
||||
{ type: FOOTNOTE_REFERENCE_NAME, attrs: { id: 'a' } },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: FOOTNOTES_LIST_NAME,
|
||||
content: [
|
||||
{
|
||||
type: FOOTNOTE_DEFINITION_NAME,
|
||||
attrs: { id: 'a' },
|
||||
content: [{ type: 'paragraph' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(getFootnoteRefCount(editor.state, 'a')).toBe(2);
|
||||
|
||||
// Insert a THIRD reference to `a` at the start of the first paragraph.
|
||||
const refType = editor.schema.nodes[FOOTNOTE_REFERENCE_NAME];
|
||||
editor.view.dispatch(
|
||||
editor.state.tr.insert(1, refType.create({ id: 'a' })),
|
||||
);
|
||||
|
||||
expect(getFootnoteRefCount(editor.state, 'a')).toBe(3);
|
||||
editor.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
// #185 re-review pt 6: scrollToReference picks the index-th occurrence among the
|
||||
// reused references, falls back to the first for an out-of-range index, and is a
|
||||
// no-op (false) for an empty id. Runs the REAL command against the editor's DOM
|
||||
// (scrollIntoView is stubbed — jsdom does not implement it).
|
||||
describe('scrollToReference command (occurrence selection + fallback)', () => {
|
||||
it('selects the index-th occurrence, falls back to the first, false for empty id', () => {
|
||||
const scrolled: Element[] = [];
|
||||
const original = (Element.prototype as any).scrollIntoView;
|
||||
(Element.prototype as any).scrollIntoView = function () {
|
||||
scrolled.push(this as Element);
|
||||
};
|
||||
try {
|
||||
const editor = makeEditor({
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{ type: FOOTNOTE_REFERENCE_NAME, attrs: { id: 'a' } },
|
||||
{ type: 'text', text: ' x ' },
|
||||
{ type: FOOTNOTE_REFERENCE_NAME, attrs: { id: 'a' } },
|
||||
{ type: 'text', text: ' y ' },
|
||||
{ type: FOOTNOTE_REFERENCE_NAME, attrs: { id: 'a' } },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: FOOTNOTES_LIST_NAME,
|
||||
content: [
|
||||
{
|
||||
type: FOOTNOTE_DEFINITION_NAME,
|
||||
attrs: { id: 'a' },
|
||||
content: [{ type: 'paragraph' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
const sups = editor.view.dom.querySelectorAll(
|
||||
'sup[data-footnote-ref][data-id="a"]',
|
||||
);
|
||||
expect(sups.length).toBe(3);
|
||||
|
||||
// index 1 -> the SECOND occurrence.
|
||||
expect(editor.commands.scrollToReference('a', 1)).toBe(true);
|
||||
expect(scrolled[scrolled.length - 1]).toBe(sups[1]);
|
||||
|
||||
// out-of-range index -> falls back to the FIRST occurrence.
|
||||
expect(editor.commands.scrollToReference('a', 99)).toBe(true);
|
||||
expect(scrolled[scrolled.length - 1]).toBe(sups[0]);
|
||||
|
||||
// default index (0) -> first.
|
||||
expect(editor.commands.scrollToReference('a')).toBe(true);
|
||||
expect(scrolled[scrolled.length - 1]).toBe(sups[0]);
|
||||
|
||||
// empty id -> false, no scroll.
|
||||
const before = scrolled.length;
|
||||
expect(editor.commands.scrollToReference('')).toBe(false);
|
||||
expect(scrolled.length).toBe(before);
|
||||
|
||||
editor.destroy();
|
||||
} finally {
|
||||
(Element.prototype as any).scrollIntoView = original;
|
||||
}
|
||||
});
|
||||
|
||||
// #185 auto-review pt 2: a NON-empty id that renders ZERO references — the real
|
||||
// desync where the definition still exists but its inline ref was removed from
|
||||
// the DOM. querySelectorAll returns 0 matches, so `matches[index] ?? matches[0]`
|
||||
// is undefined and the command must bail with `false` (not throw, not scroll).
|
||||
it('returns false for a non-empty id with no rendered references', () => {
|
||||
const scrolled: Element[] = [];
|
||||
const original = (Element.prototype as any).scrollIntoView;
|
||||
(Element.prototype as any).scrollIntoView = function () {
|
||||
scrolled.push(this as Element);
|
||||
};
|
||||
try {
|
||||
// A lone definition for id 'ghost' and a reference for a DIFFERENT id, so
|
||||
// there is a footnotes structure but no `sup[data-id="ghost"]` in the DOM.
|
||||
const editor = makeEditor({
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'paragraph',
|
||||
content: [
|
||||
{ type: FOOTNOTE_REFERENCE_NAME, attrs: { id: 'other' } },
|
||||
],
|
||||
},
|
||||
{
|
||||
type: FOOTNOTES_LIST_NAME,
|
||||
content: [
|
||||
{
|
||||
type: FOOTNOTE_DEFINITION_NAME,
|
||||
attrs: { id: 'ghost' },
|
||||
content: [{ type: 'paragraph' }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(
|
||||
editor.view.dom.querySelectorAll(
|
||||
'sup[data-footnote-ref][data-id="ghost"]',
|
||||
).length,
|
||||
).toBe(0);
|
||||
|
||||
expect(editor.commands.scrollToReference('ghost')).toBe(false);
|
||||
expect(scrolled.length).toBe(0);
|
||||
|
||||
editor.destroy();
|
||||
} finally {
|
||||
(Element.prototype as any).scrollIntoView = original;
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe('setFootnote command', () => {
|
||||
|
||||
103
packages/editor-ext/src/lib/unique-id/unique-id.util.test.ts
Normal file
103
packages/editor-ext/src/lib/unique-id/unique-id.util.test.ts
Normal file
@@ -0,0 +1,103 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import StarterKit from "@tiptap/starter-kit";
|
||||
import { addUniqueIdsToDoc } from "./unique-id.util";
|
||||
import { UniqueID } from "./unique-id";
|
||||
import { TransclusionSource } from "../transclusion/transclusion-source";
|
||||
|
||||
// Minimal extension set: StarterKit (paragraph/heading) + the UniqueID config
|
||||
// the server uses for the addressing anchors.
|
||||
const extensions = [
|
||||
StarterKit,
|
||||
UniqueID.configure({ types: ["heading", "paragraph"] }),
|
||||
];
|
||||
|
||||
// `transclusionSource` is also an addressed type, but its id is a cross-reference
|
||||
// KEY (a transclusionReference / the page_transclusions table resolves a source
|
||||
// by it), so it lives in the NO_REASSIGN set: a missing id is filled, a colliding
|
||||
// id is NOT reassigned (rewriting it would orphan its references).
|
||||
const extensionsWithSource = [
|
||||
StarterKit,
|
||||
// Narrow the content expression to `paragraph+` so the schema builds from
|
||||
// StarterKit alone (the real allow-list references image/table/etc. nodes this
|
||||
// minimal harness doesn't register). The node name — what NO_REASSIGN keys on
|
||||
// — is unchanged.
|
||||
TransclusionSource.extend({ content: "paragraph+" }),
|
||||
UniqueID.configure({
|
||||
types: ["heading", "paragraph", "transclusionSource"],
|
||||
}),
|
||||
];
|
||||
|
||||
const para = (id: string | undefined, text: string) => ({
|
||||
type: "paragraph",
|
||||
...(id !== undefined ? { attrs: { id } } : {}),
|
||||
content: [{ type: "text", text }],
|
||||
});
|
||||
|
||||
const source = (id: string | undefined, text: string) => ({
|
||||
type: "transclusionSource",
|
||||
...(id !== undefined ? { attrs: { id } } : {}),
|
||||
// The schema requires at least one block child (content expression is `+`).
|
||||
content: [{ type: "paragraph", content: [{ type: "text", text }] }],
|
||||
});
|
||||
|
||||
const ids = (doc: any): (string | undefined)[] =>
|
||||
(doc.content ?? []).map((n: any) => n.attrs?.id);
|
||||
|
||||
describe("addUniqueIdsToDoc", () => {
|
||||
it("fills ids on nodes that are missing one", () => {
|
||||
const doc = { type: "doc", content: [para(undefined, "a"), para(undefined, "b")] };
|
||||
const out = addUniqueIdsToDoc(doc, extensions);
|
||||
const [a, b] = ids(out);
|
||||
expect(a).toBeTruthy();
|
||||
expect(b).toBeTruthy();
|
||||
expect(a).not.toBe(b);
|
||||
});
|
||||
|
||||
it("deduplicates two nodes that share the same id (#206 editor-pm-7)", () => {
|
||||
// A copy/paste or bulk-JSON duplicate keeps the original id on both nodes.
|
||||
const doc = {
|
||||
type: "doc",
|
||||
content: [para("dup", "first"), para("dup", "second")],
|
||||
};
|
||||
const out = addUniqueIdsToDoc(doc, extensions);
|
||||
const [first, second] = ids(out);
|
||||
// The first occurrence keeps the id (stable anchor); the duplicate is
|
||||
// reassigned a fresh one so MCP addressing can't hit the wrong/both nodes.
|
||||
expect(first).toBe("dup");
|
||||
expect(second).toBeTruthy();
|
||||
expect(second).not.toBe("dup");
|
||||
});
|
||||
|
||||
it("leaves already-unique ids untouched", () => {
|
||||
const doc = {
|
||||
type: "doc",
|
||||
content: [para("x1", "first"), para("x2", "second")],
|
||||
};
|
||||
const out = addUniqueIdsToDoc(doc, extensions);
|
||||
expect(ids(out)).toEqual(["x1", "x2"]);
|
||||
});
|
||||
|
||||
it("does NOT reassign a colliding transclusionSource id — BOTH keep it (NO_REASSIGN)", () => {
|
||||
// Two sync-block sources sharing an id: rewriting either would orphan the
|
||||
// transclusionReferences / page_transclusions rows that resolve a source by
|
||||
// this key, so the dedupe MUST leave both ids intact. If the NO_REASSIGN
|
||||
// guard is removed, the second source is reassigned a fresh id and this fails.
|
||||
const doc = {
|
||||
type: "doc",
|
||||
content: [source("src", "first"), source("src", "second")],
|
||||
};
|
||||
const out = addUniqueIdsToDoc(doc, extensionsWithSource);
|
||||
const [first, second] = ids(out);
|
||||
expect(first).toBe("src");
|
||||
expect(second).toBe("src");
|
||||
});
|
||||
|
||||
it("still FILLS a missing id on a transclusionSource (only reassignment is suppressed)", () => {
|
||||
// NO_REASSIGN suppresses dedupe of an EXISTING id, not filling a missing one:
|
||||
// a source with no id still needs a key its references can resolve.
|
||||
const doc = { type: "doc", content: [source(undefined, "only")] };
|
||||
const out = addUniqueIdsToDoc(doc, extensionsWithSource);
|
||||
const [id] = ids(out);
|
||||
expect(id).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -59,18 +59,44 @@ export function addUniqueIdsToDoc(
|
||||
]);
|
||||
const contentNode = Node.fromJSON(schema, doc);
|
||||
|
||||
// Find nodes that don't have a unique ID
|
||||
const nodesWithoutId = findChildren(contentNode, (node) => {
|
||||
return !node.attrs[attributeName] && types.includes(node.type.name);
|
||||
// All nodes of the configured types, in document order, so that the FIRST
|
||||
// occurrence of any given id keeps it and later duplicates get reassigned.
|
||||
const idNodes = findChildren(contentNode, (node) => {
|
||||
return types.includes(node.type.name);
|
||||
});
|
||||
|
||||
// Edit the document to add unique IDs to the nodes that don't have a unique ID
|
||||
// `transclusionSource` ids are cross-reference keys (a transclusionReference /
|
||||
// the page_transclusions table resolves a source by this id), so rewriting one
|
||||
// would orphan its references. We only fill a MISSING id for those, never
|
||||
// reassign an existing one; plain block anchors (heading/paragraph) are safe to
|
||||
// dedupe.
|
||||
const NO_REASSIGN = new Set(["transclusionSource"]);
|
||||
|
||||
// Edit the document to (a) add ids where missing and (b) dedupe collisions. A
|
||||
// duplicate id otherwise lets copy/paste/import produce two nodes sharing an
|
||||
// id, so MCP addressed edits (patch_node / delete_node "before/after id") hit
|
||||
// the wrong node or both (#206 editor-pm-7). This previously only filled
|
||||
// missing ids and never deduplicated existing ones.
|
||||
const seenIds = new Set<string>();
|
||||
let tr = EditorState.create({
|
||||
doc: contentNode,
|
||||
}).tr;
|
||||
// eslint-disable-next-line no-restricted-syntax
|
||||
for (const { node, pos } of nodesWithoutId) {
|
||||
tr = tr.setNodeAttribute(pos, attributeName, generateID({ node, pos }));
|
||||
for (const { node, pos } of idNodes) {
|
||||
const currentId = node.attrs[attributeName];
|
||||
const isDuplicate = currentId != null && seenIds.has(currentId);
|
||||
const needsNewId =
|
||||
currentId == null || (isDuplicate && !NO_REASSIGN.has(node.type.name));
|
||||
|
||||
if (needsNewId) {
|
||||
// setNodeAttribute only changes attributes (no size change), so positions
|
||||
// from the original node stay valid across the whole loop.
|
||||
const newId = generateID({ node, pos });
|
||||
tr = tr.setNodeAttribute(pos, attributeName, newId);
|
||||
seenIds.add(newId);
|
||||
} else if (currentId != null) {
|
||||
seenIds.add(currentId);
|
||||
}
|
||||
}
|
||||
|
||||
// Return the updated document
|
||||
|
||||
109
packages/git-sync/build/engine/client.types.d.ts
vendored
Normal file
109
packages/git-sync/build/engine/client.types.d.ts
vendored
Normal file
@@ -0,0 +1,109 @@
|
||||
/**
|
||||
* The client seam. `pull.ts`/`push.ts` depend on a narrow STRUCTURAL interface
|
||||
* rather than any concrete client, because the gitmost server writes NATIVELY —
|
||||
* through repositories + collab `openDirectConnection`.
|
||||
*
|
||||
* `GitSyncClient` is that interface: the native datasource (server side)
|
||||
* implements it, and the engine only ever uses `Pick<GitSyncClient, ...>`
|
||||
* subsets of it. The signatures below MIRROR exactly the methods the engine's
|
||||
* `pull.ts`/`push.ts` actually call (arg shapes + the fields the engine reads
|
||||
* off each result), so a REST-style client is still structurally assignable and
|
||||
* the native adapter has a precise contract.
|
||||
*/
|
||||
/**
|
||||
* A page node as returned by `listSpaceTree` (the sidebar/tree walk, no body).
|
||||
* The engine layout (`buildVaultLayout`) consumes `PageNode` from `./layout`,
|
||||
* which only requires `id` (+ optional `title`/`slugId`/`parentPageId`); this
|
||||
* lite shape documents the fields the tree walk surfaces. Real tree nodes also
|
||||
* carry `position`, `icon`, `hasChildren` — kept open via the index signature.
|
||||
*/
|
||||
export interface GitSyncPageNodeLite {
|
||||
id: string;
|
||||
slugId?: string;
|
||||
title?: string;
|
||||
parentPageId?: string | null;
|
||||
hasChildren?: boolean;
|
||||
/** `listSpaceTree` nodes carry extra fields (position, icon, …). */
|
||||
[key: string]: unknown;
|
||||
}
|
||||
/**
|
||||
* The structural client the engine depends on. Only `Pick<GitSyncClient, ...>`
|
||||
* subsets are ever used:
|
||||
* - pull reads: `getPageJson` (+ the tree walk's `listSpaceTree`),
|
||||
* - push writes: `importPageMarkdown` / `createPage` / `deletePage` /
|
||||
* `movePage` / `renamePage`,
|
||||
* - continuous (phase B+): `listRecentSince` / `listTrash` / `restorePage`.
|
||||
*/
|
||||
export interface GitSyncClient {
|
||||
/**
|
||||
* Full tree of page nodes for the space (or the subtree rooted at
|
||||
* `rootPageId`), each WITHOUT body content. `complete` is `false` when the
|
||||
* walk was truncated / a fetch failed — the pull side suppresses absence
|
||||
* deletions on an incomplete tree (SPEC §8). Native impl returns
|
||||
* `complete: true` always (reads the DB, not a paginated REST endpoint).
|
||||
*/
|
||||
listSpaceTree(spaceId: string, rootPageId?: string): Promise<{
|
||||
pages: GitSyncPageNodeLite[];
|
||||
complete: boolean;
|
||||
}>;
|
||||
/**
|
||||
* One page WITH its ProseMirror body content. `applyPullActions` reads
|
||||
* `id`, `slugId`, `title`, `parentPageId`, `spaceId` (for the file meta) and
|
||||
* `content` (to stabilize/serialize). `updatedAt` is carried for the
|
||||
* poll-suppression loop-guard.
|
||||
*/
|
||||
getPageJson(pageId: string): Promise<{
|
||||
id: string;
|
||||
slugId: string;
|
||||
title: string;
|
||||
parentPageId: string | null;
|
||||
spaceId: string;
|
||||
updatedAt: string;
|
||||
content: unknown;
|
||||
}>;
|
||||
/**
|
||||
* Merge a page's body from a self-contained markdown file (meta + body). The
|
||||
* collab/Yjs write path (SPEC §2/§15.6) — never a raw jsonb overwrite.
|
||||
* `applyPushActions` reads only an optional `updatedAt` off the result
|
||||
* (via `extractUpdatedAt`, tolerant of extra fields).
|
||||
*
|
||||
* `baseMarkdown` is the last-synced version of the file (`refs/docmost/
|
||||
* last-pushed`), the common ancestor for a THREE-WAY merge against the live
|
||||
* doc so concurrent human edits survive (review #5). Optional/null -> 2-way.
|
||||
*/
|
||||
importPageMarkdown(pageId: string, fullMarkdown: string, baseMarkdown?: string | null): Promise<{
|
||||
updatedAt?: string;
|
||||
[key: string]: unknown;
|
||||
}>;
|
||||
/**
|
||||
* Create a new page and return the assigned id at `data.id`
|
||||
* (`applyPushActions` reads `result.data.id`, then writes it back into the
|
||||
* file's meta). An optional top-level/`data.updatedAt` feeds the loop-guard.
|
||||
*/
|
||||
createPage(title: string, content: string, spaceId: string, parentPageId?: string): Promise<{
|
||||
data: {
|
||||
id: string;
|
||||
};
|
||||
updatedAt?: string;
|
||||
[key: string]: unknown;
|
||||
}>;
|
||||
/** Soft-delete a page to Trash (SPEC §8). Result is not inspected. */
|
||||
deletePage(pageId: string): Promise<unknown>;
|
||||
/**
|
||||
* Reparent a page (and optionally set its fractional-index `position`). The
|
||||
* engine passes `position` UNDEFINED for now; the native impl computes a
|
||||
* default between siblings. Result is not inspected.
|
||||
*/
|
||||
movePage(pageId: string, parentPageId: string | null, position?: string): Promise<unknown>;
|
||||
/** Change a page's title only (no body touch). Result is not inspected. */
|
||||
renamePage(pageId: string, title: string): Promise<unknown>;
|
||||
/**
|
||||
* Pages updated since `sinceIso` (the poll-safety reconciliation, SPEC §8).
|
||||
* `spaceId` may be undefined (all spaces); `hardPageCap` bounds the walk.
|
||||
*/
|
||||
listRecentSince(spaceId: string | undefined, sinceIso: string | null, hardPageCap?: number): Promise<unknown[]>;
|
||||
/** List soft-deleted (trashed) pages for the space (deletion detection). */
|
||||
listTrash(spaceId: string): Promise<unknown[]>;
|
||||
/** Restore a soft-deleted page from Trash. Result is not inspected. */
|
||||
restorePage(pageId: string): Promise<unknown>;
|
||||
}
|
||||
13
packages/git-sync/build/engine/client.types.js
Normal file
13
packages/git-sync/build/engine/client.types.js
Normal file
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* The client seam. `pull.ts`/`push.ts` depend on a narrow STRUCTURAL interface
|
||||
* rather than any concrete client, because the gitmost server writes NATIVELY —
|
||||
* through repositories + collab `openDirectConnection`.
|
||||
*
|
||||
* `GitSyncClient` is that interface: the native datasource (server side)
|
||||
* implements it, and the engine only ever uses `Pick<GitSyncClient, ...>`
|
||||
* subsets of it. The signatures below MIRROR exactly the methods the engine's
|
||||
* `pull.ts`/`push.ts` actually call (arg shapes + the fields the engine reads
|
||||
* off each result), so a REST-style client is still structurally assignable and
|
||||
* the native adapter has a precise contract.
|
||||
*/
|
||||
export {};
|
||||
1
packages/git-sync/build/engine/config-errors.d.ts
vendored
Normal file
1
packages/git-sync/build/engine/config-errors.d.ts
vendored
Normal file
@@ -0,0 +1 @@
|
||||
export declare function loadSettingsOrExit<T>(factory: () => T): T;
|
||||
50
packages/git-sync/build/engine/config-errors.js
Normal file
50
packages/git-sync/build/engine/config-errors.js
Normal file
@@ -0,0 +1,50 @@
|
||||
import { ZodError } from 'zod';
|
||||
// Turn a ZodError from settings validation into a clear, actionable startup
|
||||
// message that names the offending env var(s), then exit(1) — no raw stack
|
||||
// trace. Mirrors the Python new-project skeleton's load_settings_or_exit.
|
||||
// A non-ZodError is left to propagate unchanged.
|
||||
export function loadSettingsOrExit(factory) {
|
||||
try {
|
||||
return factory();
|
||||
}
|
||||
catch (err) {
|
||||
if (!(err instanceof ZodError))
|
||||
throw err;
|
||||
const missing = [];
|
||||
const invalid = [];
|
||||
for (const issue of err.issues) {
|
||||
const name = issue.path.length ? String(issue.path[0]) : '?';
|
||||
// A missing required variable surfaces as an `invalid_type` issue whose
|
||||
// received value was `undefined`. zod 3 exposed `issue.received` directly;
|
||||
// zod 4 dropped that field and instead folds it into the message
|
||||
// ("expected string, received undefined"). Detect both shapes so the
|
||||
// missing-vs-invalid split holds across zod majors. NOTE: an invalid (but
|
||||
// present) value uses a different code (invalid_format / invalid_value) or
|
||||
// an `invalid_type` message that reports a non-undefined received (e.g.
|
||||
// "received NaN" from a coerced number), so neither is misread as missing.
|
||||
const i = issue;
|
||||
const isMissing = issue.code === 'invalid_type' &&
|
||||
(i.received === 'undefined' ||
|
||||
/received undefined/i.test(i.message ?? ''));
|
||||
if (isMissing)
|
||||
missing.push(name);
|
||||
else
|
||||
invalid.push(`${name}: ${issue.message}`);
|
||||
}
|
||||
const lines = ['Configuration error in environment / .env:'];
|
||||
if (missing.length) {
|
||||
lines.push(' Missing required variable(s):');
|
||||
for (const n of [...new Set(missing)])
|
||||
lines.push(` - ${n}`);
|
||||
}
|
||||
if (invalid.length) {
|
||||
lines.push(' Invalid value(s):');
|
||||
for (const item of invalid)
|
||||
lines.push(` - ${item}`);
|
||||
}
|
||||
lines.push('');
|
||||
lines.push('Set them in .env (see .env.example) and try again.');
|
||||
process.stderr.write(lines.join('\n') + '\n');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user