diff --git a/.env.example b/.env.example index 97e8dba8..834ba7d7 100644 --- a/.env.example +++ b/.env.example @@ -149,6 +149,19 @@ MCP_DOCMOST_PASSWORD= # your egress drops idle connections faster than ~10s. Default 10000 (10 s). # AI_STREAM_KEEPALIVE_MS=10000 +# Silence timeout (ms) for EXTERNAL-MCP transport ONLY (not the chat provider). +# Tighter than AI_STREAM_TIMEOUT_MS so a byte-silent/hung MCP server is broken in +# ~5 min instead of 15. Note it also cuts a legitimately long but byte-silent +# single tool call (a slow crawl that emits nothing until done) and an SSE +# transport idling >5 min BETWEEN tool calls. Default 300000 (5 min). +# AI_MCP_STREAM_TIMEOUT_MS=300000 + +# Total wall-clock cap (ms) for ONE external MCP tool call (app-level, not +# transport). Aborts a tool that keeps the socket warm (SSE heartbeats / trickle) +# but never returns a result — which the silence timeout above never breaks. +# Default 900000 (15 min). +# AI_MCP_CALL_TIMEOUT_MS=900000 + # --- Anonymous public-share AI assistant --- # Opt-in per workspace (AI settings -> "public share assistant"; off by default). # When enabled, anonymous visitors of a published share can ask an AI about that diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 955b0ac2..3a756656 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,6 +15,38 @@ permissions: jobs: test: runs-on: ubuntu-latest + # Real Postgres + Redis so the server integration suite (`*.int-spec.ts`, + # behind `pnpm --filter server test:int`) runs in CI (red-team finding #7). + # Without it, cost-cap / FK-cascade / jsonb-round-trip / real-apply tests + # only ran locally, so regressions in those paths stayed green in CI. + # Postgres uses the pgvector image because migrations create vector columns + # and global-setup runs `CREATE EXTENSION vector`. Credentials/db match the + # defaults in apps/server/test/integration/db.ts + global-setup.ts + # (docmost / docmost_dev_pw, maintenance db `docmost`, redis on 6379), so no + # TEST_*_URL overrides are needed. + services: + postgres: + image: pgvector/pgvector:pg18 + env: + POSTGRES_USER: docmost + POSTGRES_PASSWORD: docmost_dev_pw + POSTGRES_DB: docmost + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U docmost" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + redis: + image: redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - name: Checkout uses: actions/checkout@v4 @@ -36,5 +68,12 @@ jobs: - name: Build editor-ext run: pnpm --filter @docmost/editor-ext build - - name: Run tests + - name: Run unit tests run: pnpm -r test + + # Integration suite against the real Postgres/Redis services above. Runs + # the FK-cascade, cost-cap, jsonb-round-trip and real-apply specs that the + # unit run (mocks only) cannot cover. global-setup drops/recreates the + # isolated `docmost_test` DB and migrates it to latest. + - name: Run server integration tests + run: pnpm --filter server test:int diff --git a/CHANGELOG.md b/CHANGELOG.md index 26adb3f9..6c2aa9c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Persistent AI-chat history as the source of truth + server-side export.** + An assistant turn is now persisted to the database step by step: the row is + inserted upfront as `streaming` and updated as each agent step finishes, then + finalized once to `completed`/`error`/`aborted`. A process that dies mid-turn + keeps every finished step, and a startup sweep flips any dangling `streaming` + row (untouched for 10 minutes) to `aborted`. Chat "Copy" now exports + server-side from these rows (`POST /ai-chat/export`) rather than from live + client state, so the export is identical whether a chat is freshly streaming, + just switched to, or reloaded — and is available from the first turn of a new + chat. (#183, #174) + - **AI-agent attribution for MCP writes.** Comments (and pages) created through the MCP endpoint by a dedicated agent account are now badged as "AI", with unspoofable provenance derived from a per-user `is_agent` flag (not from the - request body). **Operator setup:** use a *dedicated* service account for the + request body). **Operator setup:** use a _dedicated_ service account for the MCP fallback and set the flag with SQL — `UPDATE users SET is_agent = true WHERE email = ''`. Never flag a human or shared account, or its normal edits get mis-attributed as AI. See the @@ -32,6 +43,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 OpenRouter, etc.; `openai` uses the official provider (real-OpenAI reasoning-model request shaping). Chosen explicitly rather than inferred from the base URL, since a custom URL can front real OpenAI too. (#175, #177) +- **Per-MCP-server instructions in the agent prompt.** Each external MCP server + now has an admin-authored `instructions` field ("how/when to use this server's + tools") that is injected into the agent's system prompt next to that server's + tool descriptions. Trusted text, rendered inside the prompt safety sandwich; + shown only for a server that actually connected and contributed ≥1 callable + tool. (#180) +- **Footnote multi-backlinks.** A footnote referenced more than once now shows a + back-link per reference (↩ a b c …), each scrolling to its own occurrence, like + Pandoc/Wikipedia; a single-reference footnote keeps the plain ↩. (#168) ### Changed @@ -67,6 +87,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 are nudged after a paste to refresh stale hit-testing geometry. The caret symptom is macOS-specific and was confirmed manually on macOS; the automated guard pins the DOM-order invariant, not the caret behavior itself. (#146, #147) +- **AI chat: the live token counter now ticks between agent steps.** During a + multi-step turn the header token badge (and the "Thinking… · N tokens" line) + no longer froze on the previous step's authoritative usage; the current step's + estimate is combined per-component with `max`, so the count rises smoothly and + never jumps backwards. (#163) ## [0.93.0] - 2026-06-21 @@ -150,8 +175,7 @@ embeds — plus a large batch of security hardening and test coverage. - Page templates: import `ThrottleModule` so collab boots, never strand an in-flight page-embed id, and add defense-in-depth workspace checks. - Pages: `movePage` cycle guard with no phantom `PAGE_MOVED` event. -- Import: surface the real error cause from `/pages/import` instead of a generic - 400. +- Import: surface the real error cause from `/pages/import` instead of a generic 400. ### Security diff --git a/apps/client/public/locales/en-US/translation.json b/apps/client/public/locales/en-US/translation.json index 95fbfc0c..bd8c4ed3 100644 --- a/apps/client/public/locales/en-US/translation.json +++ b/apps/client/public/locales/en-US/translation.json @@ -258,6 +258,7 @@ "Copy to space": "Copy to space", "Copy chat": "Copy chat", "Copied": "Copied", + "Failed to export chat": "Failed to export chat", "Duplicate": "Duplicate", "Select a user": "Select a user", "Select a group": "Select a group", @@ -710,6 +711,7 @@ "Authorization header": "Authorization header", "Tool allowlist": "Tool allowlist", "Optional. Leave empty to allow all tools the server exposes.": "Optional. Leave empty to allow all tools the server exposes.", + "Optional guidance for the agent on how and when to use this server's tools. Injected into the system prompt. The server's tools are namespaced as \"_*\".": "Optional guidance for the agent on how and when to use this server's tools. Injected into the system prompt. The server's tools are namespaced as \"_*\".", "Test": "Test", "Available tools": "Available tools", "No tools available": "No tools available", @@ -1077,6 +1079,8 @@ "Undo": "Undo", "Redo": "Redo", "Backlinks": "Backlinks", + "Back to references": "Back to references", + "Back to reference {{label}}": "Back to reference {{label}}", "Last updated by": "Last updated by", "Last updated": "Last updated", "Stats": "Stats", diff --git a/apps/client/public/locales/ru-RU/translation.json b/apps/client/public/locales/ru-RU/translation.json index 0d4926cd..f8c59436 100644 --- a/apps/client/public/locales/ru-RU/translation.json +++ b/apps/client/public/locales/ru-RU/translation.json @@ -257,6 +257,7 @@ "Copy": "Копировать", "Copy to space": "Копировать в пространство", "Copied": "Скопировано", + "Failed to export chat": "Не удалось экспортировать чат", "Duplicate": "Дублировать", "Select a user": "Выберите пользователя", "Select a group": "Выберите группу", @@ -405,6 +406,8 @@ "Footnote {{number}}": "Сноска {{number}}", "Go to footnote": "Перейти к сноске", "Back to reference": "Вернуться к ссылке", + "Back to references": "Вернуться к ссылкам", + "Back to reference {{label}}": "Вернуться к ссылке {{label}}", "Empty footnote": "Пустая сноска", "Math inline": "Строчная формула", "Insert inline math equation.": "Вставить математическое выражение в строку.", @@ -749,6 +752,8 @@ "Manage API keys for all users in the workspace. View the API documentation for usage details.": "Управляйте API-ключами для всех пользователей в рабочем пространстве. Смотрите документацию по API для получения информации об использовании.", "View the API documentation for usage details.": "Смотрите документацию по API для получения информации об использовании.", "View the MCP documentation.": "Смотрите документацию по MCP.", + "Instructions": "Инструкции", + "Optional guidance for the agent on how and when to use this server's tools. Injected into the system prompt. The server's tools are namespaced as \"_*\".": "Необязательное указание агенту, как и когда использовать инструменты этого сервера. Добавляется в системный промпт. Инструменты сервера именуются с префиксом «<имя сервера>_*».", "Sources": "Источники", "AI Answers not available for attachments": "Ответы ИИ недоступны для вложений", "No answer available": "Ответ недоступен", diff --git a/apps/client/src/features/ai-chat/components/ai-chat-window.tsx b/apps/client/src/features/ai-chat/components/ai-chat-window.tsx index 5f6b1dde..de0b9923 100644 --- a/apps/client/src/features/ai-chat/components/ai-chat-window.tsx +++ b/apps/client/src/features/ai-chat/components/ai-chat-window.tsx @@ -6,7 +6,6 @@ import { useRef, useState, } from "react"; -import { type UIMessage } from "@ai-sdk/react"; import { Group, Loader, Tooltip } from "@mantine/core"; import { IconArrowsDiagonal, @@ -40,7 +39,7 @@ import { } from "@/features/ai-chat/queries/ai-chat-query.ts"; import ConversationList from "@/features/ai-chat/components/conversation-list.tsx"; import ChatThread from "@/features/ai-chat/components/chat-thread.tsx"; -import { buildChatMarkdown } from "@/features/ai-chat/utils/chat-markdown.ts"; +import { exportAiChat } from "@/features/ai-chat/services/ai-chat-service.ts"; import { useChatSession } from "@/features/ai-chat/hooks/use-chat-session.ts"; import { shouldCollapseOnOutsidePointer, @@ -80,17 +79,31 @@ function computeInitialGeom() { Math.min(DEFAULT_HEIGHT, window.innerHeight - 2 * EDGE_MARGIN), ); const left = Math.max(EDGE_MARGIN, window.innerWidth - width - 24); - const maxTop = Math.max(EDGE_MARGIN, window.innerHeight - height - EDGE_MARGIN); + const maxTop = Math.max( + EDGE_MARGIN, + window.innerHeight - height - EDGE_MARGIN, + ); const top = Math.min(60, maxTop); return { left, top, width, height }; } // Clamp a geometry so the window stays within the current viewport. -function clampGeom(g: { left: number; top: number; width: number; height: number }) { +function clampGeom(g: { + left: number; + top: number; + width: number; + height: number; +}) { const effWidth = Math.max(g.width, MIN_WIDTH); const effHeight = Math.max(g.height, MIN_HEIGHT); - const maxLeft = Math.max(EDGE_MARGIN, window.innerWidth - effWidth - EDGE_MARGIN); - const maxTop = Math.max(EDGE_MARGIN, window.innerHeight - effHeight - EDGE_MARGIN); + const maxLeft = Math.max( + EDGE_MARGIN, + window.innerWidth - effWidth - EDGE_MARGIN, + ); + const maxTop = Math.max( + EDGE_MARGIN, + window.innerHeight - effHeight - EDGE_MARGIN, + ); return { ...g, left: Math.min(Math.max(EDGE_MARGIN, g.left), maxLeft), @@ -107,7 +120,7 @@ function clampGeom(g: { left: number; top: number; width: number; height: number * ported from the GitmostAgent.jsx design. */ export default function AiChatWindow() { - const { t } = useTranslation(); + const { t, i18n } = useTranslation(); const clipboard = useClipboard({ timeout: 500 }); const queryClient = useQueryClient(); const [windowOpen, setWindowOpen] = useAtom(aiChatWindowOpenAtom); @@ -148,14 +161,6 @@ export default function AiChatWindow() { const { data: messageRows, isLoading: messagesLoading } = useAiChatMessagesQuery(activeChatId ?? undefined); - // Live snapshot of the active thread's useChat state, kept up to date by - // ChatThread. Lets the export include the in-progress (not-yet-persisted) - // streaming turn. A ref avoids re-rendering this window on every token. - const liveThreadRef = useRef<{ messages: UIMessage[]; isStreaming: boolean }>({ - messages: [], - isStreaming: false, - }); - // Live turn-token total (reasoning + output) for the in-flight turn, pushed up // (THROTTLED to ~8 Hz inside ChatThread) so the header badge ticks mid-stream. // `null` means no turn is in flight -> the badge falls back to the persisted @@ -185,17 +190,22 @@ export default function AiChatWindow() { // The invalidate closures are passed inline: `onTurnFinished` is read live by // useChat's onFinish (never in an effect dep array), so their identity does not // matter — no memoization ceremony needed. - const { threadKey, waitingForHistory, onTurnFinished, cancelPendingAdoption } = - useChatSession({ - activeChatId, - setActiveChatId, - chats, - messagesLoading, - onInvalidateChatList: () => - queryClient.invalidateQueries({ queryKey: AI_CHATS_RQ_KEY }), - onInvalidateChatMessages: (id) => - queryClient.invalidateQueries({ queryKey: AI_CHAT_MESSAGES_RQ_KEY(id) }), - }); + const { + threadKey, + waitingForHistory, + onTurnFinished, + onServerChatId, + cancelPendingAdoption, + } = useChatSession({ + activeChatId, + setActiveChatId, + chats, + messagesLoading, + onInvalidateChatList: () => + queryClient.invalidateQueries({ queryKey: AI_CHATS_RQ_KEY }), + onInvalidateChatMessages: (id) => + queryClient.invalidateQueries({ queryKey: AI_CHAT_MESSAGES_RQ_KEY(id) }), + }); // startNewChat/selectChat set the public atom; the hook's render-phase // reconciler handles the remount when activeChatId actually CHANGES. But @@ -225,19 +235,28 @@ export default function AiChatWindow() { [cancelPendingAdoption, setActiveChatId, setDraft, setSelectedRoleId], ); - // The active chat object (for its title) and an export gate: only enable the - // export button when an existing chat with loaded persisted rows is active. + // The active chat object (for its title) and an export gate. The export is now + // SERVER-sourced (the DB is the single source of truth — #183): the assistant + // row is persisted upfront + per step, so even a brand-new chat whose first + // turn is streaming/interrupted has a server row to render. Enable the button + // whenever a persisted chat is active (`activeChatId` is set). For a BRAND-NEW + // chat that id is adopted EARLY — at the stream's `start` chunk via + // onServerChatId (#174) — so the Copy button is available during the first + // turn's stream, not only after it terminates. const activeChat = useMemo( () => chats?.items?.find((c) => c.id === activeChatId) ?? null, [chats, activeChatId], ); - const canExport = !!activeChatId && !!messageRows && messageRows.length > 0; + const canExport = !!activeChatId; // The role to display in the header and as the assistant's name. Prefer the // persisted role of an existing chat (chat-list JOIN); fall back to the role // picked via a card click for a brand-new or just-adopted chat. selectChat // resets selectedRoleId, so this fallback never leaks into an unrelated chat. - const currentRole = useMemo<{ name: string; emoji: string | null } | null>(() => { + const currentRole = useMemo<{ + name: string; + emoji: string | null; + } | null>(() => { if (activeChat?.roleName) { return { name: activeChat.roleName, emoji: activeChat.roleEmoji ?? null }; } @@ -245,37 +264,21 @@ export default function AiChatWindow() { return picked ? { name: picked.name, emoji: picked.emoji } : null; }, [activeChat, enabledRoles, selectedRoleId]); - // Build a Markdown export from the already-loaded persisted rows (no network - // call) and copy it to the clipboard. The "Copied" notification is the - // feedback. - const handleCopy = useCallback(() => { - if (!activeChatId || !messageRows || messageRows.length === 0) return; - // While the active thread is streaming, the current user message and the - // in-progress assistant reply are NOT yet in messageRows (the persisted - // query is only refetched after the turn finishes). Pull the live tail — - // messages whose id is not among the persisted rows — and append them, - // flagging the streaming assistant message as still generating. - const live = liveThreadRef.current; - const rowIds = new Set(messageRows.map((r) => r.id)); - const pending = live.isStreaming - ? live.messages - .filter((m) => !rowIds.has(m.id)) - .map((m) => ({ - role: m.role, - parts: (m.parts ?? []) as { type: string; text?: string }[], - generating: m.role === "assistant", - })) - : []; - const markdown = buildChatMarkdown({ - title: activeChat?.title ?? null, - chatId: activeChatId, - rows: messageRows, - pending, - t, - }); - clipboard.copy(markdown); - notifications.show({ message: t("Copied") }); - }, [activeChatId, messageRows, activeChat, clipboard, t]); + // Fetch the server-rendered Markdown export and copy it to the clipboard. The + // server is the single source of truth (#183): it renders the transcript from + // the persisted rows — including an interrupted turn's in-progress row — so the + // export is identical whether the chat is freshly streaming, just switched to, + // or reloaded. The `lang` of the active i18n drives the few localized labels. + const handleCopy = useCallback(async () => { + if (!activeChatId) return; + try { + const markdown = await exportAiChat(activeChatId, i18n.language); + clipboard.copy(markdown); + notifications.show({ message: t("Copied") }); + } catch { + notifications.show({ message: t("Failed to export chat"), color: "red" }); + } + }, [activeChatId, clipboard, t, i18n.language]); // Current context size for the active chat: how much the conversation now // occupies in the model's context window — NOT the cumulative tokens spent. @@ -351,7 +354,8 @@ export default function AiChatWindow() { const width = el.offsetWidth; const height = el.offsetHeight; setGeom((prev) => { - if (!prev || (prev.width === width && prev.height === height)) return prev; + if (!prev || (prev.width === width && prev.height === height)) + return prev; return { ...prev, width, height }; }); }); @@ -497,11 +501,15 @@ export default function AiChatWindow() { flash a "0" badge before any token streams in (#151 review). */} {liveTurnTokens !== null && liveTurnTokens > 0 ? ( - {formatTokens(liveTurnTokens)} + + {formatTokens(liveTurnTokens)} + ) : contextTokens > 0 ? ( - {formatTokens(contextTokens)} + + {formatTokens(contextTokens)} + ) : null} @@ -515,7 +523,11 @@ export default function AiChatWindow() { aria-label={t("Copy chat")} onClick={handleCopy} > - {clipboard.copied ? : } + {clipboard.copied ? ( + + ) : ( + + )} )}