Compare commits
41 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 62482827c9 | |||
| 2a58b63e15 | |||
| 2dfc5c5144 | |||
| 878ede409b | |||
| b8185a9f0e | |||
| 108d2c6bef | |||
| 1e28151f7a | |||
| 6e42752a9c | |||
| 8430eea700 | |||
| 5563811aa8 | |||
| 55a1c279fc | |||
| d4b21957dc | |||
| e71212cdc6 | |||
| 763a1bf7bd | |||
| bfb6a52eea | |||
| 0503e8b4b1 | |||
| b97cad0ebe | |||
| 080dd82023 | |||
| 803bbd40b4 | |||
| c7a38e274e | |||
| 791a707eb6 | |||
| 60205398bb | |||
| 1685b32333 | |||
| 4b78e336a2 | |||
| 77e20ecb41 | |||
| d533daa45f | |||
| ba58493909 | |||
| 947796adef | |||
| 38a09c5ca1 | |||
| 1b05224b27 | |||
| b50b32bf64 | |||
| 1d704d4ec5 | |||
| 2cb07ef7fb | |||
| 9ba97b6d2b | |||
| 0b9de2c25e | |||
| 123cba7de1 | |||
| 001eb1c923 | |||
| 002c931c6b | |||
| 307ad49324 | |||
| 1031ed1ae8 | |||
| 3b6634c6be |
+51
-3
@@ -225,11 +225,26 @@ MCP_DOCMOST_PASSWORD=
|
||||
|
||||
# Silence timeout (ms) for EXTERNAL-MCP transport ONLY (not the chat provider).
|
||||
# Tighter than AI_STREAM_TIMEOUT_MS so a byte-silent/hung MCP server is broken in
|
||||
# ~1 min instead of 15. Note it also cuts a legitimately long but byte-silent
|
||||
# single tool call (a slow crawl that emits nothing until done) and an SSE
|
||||
# transport idling >1 min BETWEEN tool calls. Default 60000 (1 min).
|
||||
# ~1 min instead of 15. It cuts a legitimately long but byte-silent single tool
|
||||
# call (a slow crawl that emits nothing until done) on the HTTP (streamable)
|
||||
# transport, which opens a fresh request per call. The SSE transport — one
|
||||
# long-lived body across many calls — is NO LONGER governed by this timeout
|
||||
# (as of #489): its idle-BETWEEN-calls window has its own, raised bodyTimeout,
|
||||
# AI_MCP_SSE_BODY_TIMEOUT_MS below. Default 60000 (1 min).
|
||||
# AI_MCP_STREAM_TIMEOUT_MS=60000
|
||||
|
||||
# bodyTimeout (ms) for the EXTERNAL-MCP SSE transport ONLY (#489). The SSE
|
||||
# transport holds ONE response body open across many tool calls, so undici's
|
||||
# bodyTimeout (time between body bytes) counts the LEGITIMATE silence BETWEEN the
|
||||
# model's tool calls, not just a hung single call. At the tight 1-min silence
|
||||
# timeout above, a normal >1-min gap between calls would break the SSE socket and
|
||||
# the cache would serve a dead client until TTL — so the SSE transport gets its
|
||||
# OWN, RAISED bodyTimeout. A single stuck call is still bounded by the per-call
|
||||
# cap (AI_MCP_CALL_TIMEOUT_MS), and a socket that does break is healed by the
|
||||
# in-run transport-error retry. The HTTP (streamable) transport keeps the tight
|
||||
# timeout. Default 600000 (10 min).
|
||||
# AI_MCP_SSE_BODY_TIMEOUT_MS=600000
|
||||
|
||||
# Total wall-clock cap (ms) for ONE external MCP tool call (app-level, not
|
||||
# transport). Aborts a tool that keeps the socket warm (SSE heartbeats / trickle)
|
||||
# but never returns a result — which the silence timeout above never breaks.
|
||||
@@ -287,6 +302,39 @@ MCP_DOCMOST_PASSWORD=
|
||||
# enabled for a workspace, and the same single-instance constraint applies (the
|
||||
# registry is process-local).
|
||||
# AI_CHAT_RESUMABLE_STREAM=false
|
||||
#
|
||||
# Per-run replay ring cap (#491), in BYTES, for the resumable-stream registry
|
||||
# above. The registry buffers the run's recent SSE tail so a reopened tab can
|
||||
# attach and continue from the step it already persisted; the ring is bounded and
|
||||
# rotates on every confirmed step-persist. This caps the un-persisted tail between
|
||||
# rotations — an overflow evicts the oldest frames and a late attach falls back to
|
||||
# 204 -> degraded poll, so correctness never depends on the size. Default 4194304
|
||||
# (4MB); a 0/invalid value falls back to the default. The per-subscriber backpressure
|
||||
# cap is derived as 2x this value. Only meaningful with AI_CHAT_RESUMABLE_STREAM on.
|
||||
# AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES=4194304
|
||||
|
||||
# --- Run lifecycle tunables (#487) ---
|
||||
# These govern the universal run machinery (every turn is now a first-class run,
|
||||
# both modes) and rarely need changing.
|
||||
#
|
||||
# How long a server-side SUPERSEDE ("interrupt and send now") waits for the target
|
||||
# run to settle after issuing Stop before it degrades to a 409 SUPERSEDE_TIMEOUT
|
||||
# (nothing sent, the composer keeps the user's text). 10s is generous under a
|
||||
# healthy DB; do NOT raise it to paper over a slow DB — a SUPERSEDE_TIMEOUT is the
|
||||
# honest signal. Default 10000 (10s).
|
||||
# AI_CHAT_SUPERSEDE_TIMEOUT_MS=10000
|
||||
#
|
||||
# How often the periodic bidirectional reconcile job runs (heals runs/messages
|
||||
# left dangling by a crash or a lost terminal write). Default 120000 (2 min).
|
||||
# AI_CHAT_RECONCILE_INTERVAL_MS=120000
|
||||
#
|
||||
# Wall-clock cap for a SINGLE in-app tool call (a long paginated read, or a content
|
||||
# write whose collab commit hangs) — the per-call half of the composite abort
|
||||
# signal every in-app tool is wrapped with (the other half is the turn's Stop).
|
||||
# The reconcile staleness floor is derived as max(2 x this cap, 15min), so a very
|
||||
# high value delays stale-run recovery (the server boot-warns above 30min). Default
|
||||
# 120000 (2 min).
|
||||
# AI_CHAT_INAPP_TOOL_CALL_CAP_MS=120000
|
||||
|
||||
# --- Anonymous public-share AI assistant ---
|
||||
# Opt-in per workspace (AI settings -> "public share assistant"; off by default).
|
||||
|
||||
@@ -29,6 +29,10 @@ packages/mcp/build/
|
||||
# is a build artifact like build/ — never committed, always fresh.
|
||||
packages/mcp/src/registry-stamp.generated.ts
|
||||
|
||||
# token-estimate compiled output (#490; built in CI/Docker via `pnpm build` /
|
||||
# the server `pretest`, never committed, so src/ and prod can never diverge).
|
||||
packages/token-estimate/dist/
|
||||
|
||||
# Logs
|
||||
logs
|
||||
*.log
|
||||
|
||||
@@ -455,7 +455,7 @@ The API server is a Fastify app with a global `/api` prefix (`main.ts` excludes
|
||||
- `core/ai-chat/tools/` — the agent's ~40 read+write tools. Every tool runs under the **calling user's** CASL permissions via a per-user loopback access token (`docmost-client.loader.ts`), so the agent can never exceed what the user could do. Only **reversible** operations are exposed (page history + trash; no permanent delete). Agent edits get an "AI agent" provenance badge in page history (`20260616T130000-agent-provenance` migration).
|
||||
- `core/ai-chat/embedding/` — RAG indexer + a BullMQ consumer on `AI_QUEUE` that embeds pages into `page_embeddings` (vector search), complementing Postgres full-text search. Pages are (re)indexed on edit; `AI_EMBEDDING_TIMEOUT_MS` bounds a hung embeddings endpoint.
|
||||
- `core/ai-chat/external-mcp/` — admins can attach external MCP servers (e.g. Tavily) to give the agent web access. **`ssrf-guard.ts` validates outbound MCP URLs against SSRF** — keep that guard in the path when touching external-MCP connection logic.
|
||||
- `core/ai-chat/ai-chat-run.service.ts` + `ai_chat_runs` — **detached/autonomous agent runs** (`#184`), behind the per-workspace `settings.ai.autonomousRuns` flag (off by default). When on, a turn becomes a server-side RUN that survives a browser disconnect; only an explicit `POST /ai-chat/stop` ends it, and a client reconnects/live-follows via `POST /ai-chat/run`. **DEPLOY CONSTRAINT — single-instance only in phase 1:** Stop and the AbortController that backs it are process-local, so a Stop only aborts a run executing on the **same** replica that owns it (cross-instance pub/sub stop is phase 2). Do **not** enable `autonomousRuns` on a horizontally-scaled deployment (multiple replicas behind a load balancer, or Docmost cloud `CLOUD=true`) — run a single instance instead. The server logs a startup WARNING when it detects a multi-instance deployment (`CLOUD=true`) so the constraint is visible. The startup sweep settles any run left dangling by a restart.
|
||||
- `core/ai-chat/ai-chat-run.service.ts` + `ai_chat_runs` — **every agent turn is now a first-class server-side RUN** (`#184`, universalized in `#487`): its lifecycle is tracked in `ai_chat_runs` in **both** modes, and the single-active-run-per-chat concurrency gate is enforced universally (a legacy second tab now gets a clean `409 A_RUN_ALREADY_ACTIVE` instead of a second parallel stream that interleaved history). The per-workspace `settings.ai.autonomousRuns` flag (off by default) **no longer gates whether a turn is a run** — it now controls **only the browser-disconnect semantics**: when ON the run is *detached* (a disconnect leaves it executing server-side; only an explicit `POST /ai-chat/stop` ends it, and a client reconnects/live-follows via `POST /ai-chat/run`); when OFF (legacy) a disconnect ends the turn by stopping its run via the run's stop lever. `#487` also adds a server-side **supersede** CAS ("interrupt and send now") to `POST /ai-chat/stream` (`supersede: { runId }`): it atomically stops the chat's currently-active run and waits for it to settle before the new turn claims the slot, returning `SUPERSEDE_INVALID` / `SUPERSEDE_TARGET_MISMATCH` / `SUPERSEDE_TIMEOUT` on the non-proceed branches. **DEPLOY CONSTRAINT — single-instance only in phase 1:** Stop and the AbortController that backs it are process-local, so a Stop only aborts a run executing on the **same** replica that owns it (cross-instance pub/sub stop is phase 2). Do **not** enable `autonomousRuns` on a horizontally-scaled deployment (multiple replicas behind a load balancer, or Docmost cloud `CLOUD=true`) — run a single instance instead. The server logs a startup WARNING when it detects a multi-instance deployment (`CLOUD=true`) so the constraint is visible. The startup sweep settles any run left dangling by a restart.
|
||||
|
||||
### Client structure
|
||||
Vite SPA. Code is organized by feature under `apps/client/src/features/*` (mirrors the server domains: `page`, `space`, `comment`, `ai-chat`, `editor`, …). Conventions:
|
||||
|
||||
@@ -202,6 +202,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
dangling by a restart. Phase 1 is single-instance-only (cross-instance Stop is
|
||||
not yet reliable); the server warns at startup on a horizontally-scaled
|
||||
deployment. (#184)
|
||||
- **Server-side "interrupt and send now" (supersede) for AI chat.** `POST
|
||||
/ai-chat/stream` now accepts a `supersede: { runId }` field: when the user sends
|
||||
a new message while a run is active, the server atomically stops that run and
|
||||
waits for it to settle before the new turn claims the chat's single run slot,
|
||||
instead of the send being rejected as concurrent. The compare-and-set surfaces
|
||||
three codes on its non-proceed branches — `SUPERSEDE_INVALID` (the targeted run
|
||||
is malformed / belongs to another chat), `SUPERSEDE_TARGET_MISMATCH` (a
|
||||
different run is now active; carries the current `activeRunId`), and
|
||||
`SUPERSEDE_TIMEOUT` (the previous run did not stop within the settle window, so
|
||||
nothing was sent and the composer keeps the text). Tunable via
|
||||
`AI_CHAT_SUPERSEDE_TIMEOUT_MS` (default 10s). (#487)
|
||||
- **Out-of-band page transfer via an in-RAM blob sandbox (`stash_page`).** A
|
||||
new MCP tool serializes a whole page (its full ProseMirror JSON, with every
|
||||
internal image/file mirrored) into an ephemeral in-RAM blob and returns only
|
||||
@@ -282,6 +293,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Changed
|
||||
|
||||
- **Every AI-chat turn is now a first-class server-side run, and one run per chat
|
||||
is enforced in both modes.** The run machinery from `#184` was universalized: a
|
||||
turn is tracked in `ai_chat_runs` and gated by the single-active-run-per-chat
|
||||
index regardless of the `settings.ai.autonomousRuns` flag. **Behavior change:**
|
||||
a second tab (or a double-submit) that starts a turn while one is already active
|
||||
on the chat is now rejected up front with `409 A_RUN_ALREADY_ACTIVE` (carrying
|
||||
the `activeRunId`); previously, on the legacy path, it opened a second parallel
|
||||
stream on the same chat that interleaved history. The `autonomousRuns` flag no
|
||||
longer controls whether a turn is a run — it now governs **only** the
|
||||
browser-disconnect semantics (ON = detached/survives a disconnect; OFF = a
|
||||
disconnect stops the run). (#487)
|
||||
- **Client markdown paste/copy and AI-chat rendering now go through the canonical
|
||||
converter.** Pasting markdown into the editor, "Copy as markdown", the AI title
|
||||
generator, and the AI-chat markdown renderer all now use
|
||||
@@ -314,6 +336,44 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A long AI chat no longer bricks on the model's context window, and each turn
|
||||
stops re-persisting the whole tool-output history.** Tool outputs are now
|
||||
stored ONCE, in `metadata.parts`; the `tool_calls` trace keeps only per-step
|
||||
outcome flags (a v2 trace shape), ending the O(N²) write amplification that
|
||||
re-wrote every prior output on every step (measured on a live Postgres via the
|
||||
`pg_current_wal_lsn()` delta: the trace column shrank ~3200×, the full
|
||||
assistant row ~51%). The persisted record is unchanged in content — the full
|
||||
history still lives in `metadata.parts`. At REPLAY time only, the history sent
|
||||
to the provider is now bounded by a deterministic, prompt-cache-friendly token
|
||||
budget: `floor(0.7 × chatContextWindow)` when a window is configured (no cap —
|
||||
anti-brick protection, not a cost limiter), a flat 100k fallback for installs
|
||||
with no window set (exactly the ones that hit terminal overflow), or off when
|
||||
the window is explicitly `0`. Trimming truncates old tool outputs first, then
|
||||
mechanically collapses the oldest turns, always keeping the recent turns full
|
||||
and the tool-call/result pairing balanced. A provider context-overflow 400 is
|
||||
now classified and used as a reactive signal: the row is stamped so the NEXT
|
||||
turn re-trims aggressively (0.5×), which un-bricks a chat that just 400'd. The
|
||||
client token badge and the server budgeter now share one estimator (new
|
||||
`@docmost/token-estimate` package) so they can never diverge. Deferred-tool
|
||||
activation is also cached in the chat metadata to avoid re-resolving it each
|
||||
turn. (#490)
|
||||
- **A chat with one malformed message part no longer 500s on every turn, and a
|
||||
failed send no longer duplicates the user's message.** Incoming client parts
|
||||
are now whitelisted to `text` (a forged tool-result part can no longer reach
|
||||
the persisted history or the model context), and the turn is converted BEFORE
|
||||
the user row is inserted, so a mid-flight failure cannot leave a duplicate
|
||||
user row that a retry then compounds. A single part that still fails to convert
|
||||
degrades to a `[tool context omitted]` marker on that one row instead of
|
||||
bricking the whole chat. (#489)
|
||||
- **A transport drop to an external MCP server now heals within the same turn.**
|
||||
On an undici transport error, a read-only MCP tool reconnects its server and
|
||||
retries once within the run; a write is never auto-retried (it may already have
|
||||
applied). One flapping server no longer nulls the shared client cache, so other
|
||||
servers' cached clients are untouched. The SSE transport also gets a raised
|
||||
body-timeout so a legitimate >1-min idle between the model's tool calls no
|
||||
longer breaks a long-lived SSE socket (new `AI_MCP_SSE_BODY_TIMEOUT_MS`, default
|
||||
10 min; see `.env.example`). (#489)
|
||||
|
||||
- **The server no longer runs out of heap during long autonomous agent runs.** A
|
||||
new pnpm patch on `ai@6.0.134` stops the SDK from building a cumulative
|
||||
snapshot of the ENTIRE turn text on every streamed text-delta when no output
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
"@casl/react": "5.0.1",
|
||||
"@docmost/editor-ext": "workspace:*",
|
||||
"@docmost/prosemirror-markdown": "workspace:*",
|
||||
"@docmost/token-estimate": "workspace:*",
|
||||
"@excalidraw/excalidraw": "0.18.0-3a5ef40",
|
||||
"@mantine/core": "8.3.18",
|
||||
"@mantine/dates": "8.3.18",
|
||||
|
||||
@@ -58,8 +58,11 @@ import ConversationList from "@/features/ai-chat/components/conversation-list.ts
|
||||
import ChatThread from "@/features/ai-chat/components/chat-thread.tsx";
|
||||
import {
|
||||
exportAiChat,
|
||||
getAiChatMessagesDelta,
|
||||
stopRun,
|
||||
} from "@/features/ai-chat/services/ai-chat-service.ts";
|
||||
import { mergeDeltaRowsIntoPages } from "@/features/ai-chat/utils/resume-helpers.ts";
|
||||
import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.ts";
|
||||
import { useChatSession } from "@/features/ai-chat/hooks/use-chat-session.ts";
|
||||
import {
|
||||
shouldCollapseOnOutsidePointer,
|
||||
@@ -86,19 +89,11 @@ const MIN_HEIGHT = 400;
|
||||
// Margin kept between the window and the viewport edges while dragging.
|
||||
const EDGE_MARGIN = 8;
|
||||
|
||||
// #184 phase 1.5 / #430: backstop for the degraded-poll fallback. The poll is
|
||||
// armed when a resume attempt could not attach to the live run and disarmed by the
|
||||
// thread on settle / local stream; this cap is the ONLY backstop against an endless
|
||||
// tick (a stuck 'streaming' row before the boot-sweep, or a user-tail 204 with no
|
||||
// run).
|
||||
//
|
||||
// #430: measured from RUN ACTIVITY, not from arm-time. A real autonomous run takes
|
||||
// 11-25 min — longer than a fixed 10-min-from-start cap, which used to cut the poll
|
||||
// off mid-run. Instead we cap on INACTIVITY: keep polling as long as the run is
|
||||
// still making progress (its persisted rows keep changing), and only give up after
|
||||
// this long with NO new activity. A genuinely stuck run produces no row changes, so
|
||||
// the idle cap still bounds it; a long-but-progressing run polls to completion.
|
||||
const DEGRADED_POLL_IDLE_MAX_MS = 10 * 60_000;
|
||||
// #184 phase 1.5 / #430 / #488: the degraded-poll fallback. The window owns only
|
||||
// a DUMB 2.5s timer, gated by an armed flag; the THREAD's run-lifecycle FSM owns
|
||||
// arm/disarm AND the inactivity cap that turns a stuck run into a `stalled` banner
|
||||
// (#488 commit 4a — the cap moved into the thread so polling->stalled is a single
|
||||
// FSM transition; the window no longer silently stops polling at the cap).
|
||||
|
||||
/** Compact token formatter: 1.2M / 3.4k / 950. */
|
||||
function formatTokens(n: number): string {
|
||||
@@ -259,17 +254,13 @@ export default function AiChatWindow() {
|
||||
[roles],
|
||||
);
|
||||
|
||||
// #184 phase 1.5: degraded-poll fallback (replaces the F4/F5/F7 latches). When
|
||||
// ChatThread could not attach to a still-running run it arms this via
|
||||
// onResumeFallback(true); the thread disarms it on settle / local stream. The
|
||||
// window only OWNS the timer (armedAtRef stamps when it was armed for the cap).
|
||||
// #184 phase 1.5 / #488: degraded-poll fallback. ChatThread's FSM arms this via
|
||||
// onResumeFallback(true) when it enters a poll-bearing recovery (attach 204 /
|
||||
// starved finish / stop) and disarms it on settle / local stream / stalled. The
|
||||
// window owns ONLY the dumb 2.5s timer; the THREAD owns arm/disarm AND the
|
||||
// inactivity cap (a stuck run -> the thread's `stalled` banner disarms this).
|
||||
const [degradedPoll, setDegradedPoll] = useState(false);
|
||||
// #430: timestamp of the LAST run activity while the poll is armed — stamped on
|
||||
// arm and re-stamped whenever the polled rows change (see the effect below). The
|
||||
// idle cap is measured from this, so a long-but-progressing run keeps polling.
|
||||
const lastActivityAtRef = useRef(0);
|
||||
const onResumeFallback = useCallback((active: boolean): void => {
|
||||
if (active) lastActivityAtRef.current = Date.now();
|
||||
setDegradedPoll(active);
|
||||
}, []);
|
||||
// Reset the degraded poll whenever the open chat changes: it is scoped to the
|
||||
@@ -281,32 +272,63 @@ export default function AiChatWindow() {
|
||||
const { data: messageRows, isLoading: messagesLoading } =
|
||||
useAiChatMessagesQuery(
|
||||
activeChatId ?? undefined,
|
||||
// DELIBERATELY DUMB (invariant 8 / task 2.4): poll every 2.5s while armed
|
||||
// and while the run is still active (#430: under the INACTIVITY cap, not a
|
||||
// fixed-from-start cap); otherwise off. NO error checks (TanStack v5 resets
|
||||
// fetchFailureCount each fetch, so consecutive errors are not expressible —
|
||||
// and the poll must survive a server restart) and NO tail checks (the
|
||||
// settled/local-stream semantics live in ChatThread, which disarms via
|
||||
// onResumeFallback(false)). The idle cap is the only backstop.
|
||||
() =>
|
||||
degradedPoll === true &&
|
||||
Date.now() - lastActivityAtRef.current < DEGRADED_POLL_IDLE_MAX_MS
|
||||
? 2500
|
||||
: false,
|
||||
// #344: gate on windowOpen too — no message history is fetched (and no
|
||||
// degraded poll runs) while the window is closed; it loads when the window
|
||||
// opens with an active chat.
|
||||
// #491: the full infinite-query no longer POLLS. It seeds the thread ONCE; the
|
||||
// degraded fallback now runs a DELTA poller (below) that augments THIS cache
|
||||
// idempotently, instead of refetching every page (with full parts) every 2.5s.
|
||||
false,
|
||||
// #344: gate on windowOpen too — no message history is fetched while the window
|
||||
// is closed; it loads when the window opens with an active chat.
|
||||
windowOpen,
|
||||
);
|
||||
|
||||
// #430: re-stamp the activity clock whenever the polled rows change while the
|
||||
// poll is armed. TanStack keeps the same `messageRows` reference across refetches
|
||||
// that return deep-equal data (structural sharing), so a new reference means the
|
||||
// run genuinely progressed — which extends the inactivity cap above. A stuck run
|
||||
// yields no reference change, so the cap eventually fires and stops the poll.
|
||||
// #491 degraded DELTA poll. While armed (degradedPoll) and the window is open on a
|
||||
// chat, poll POST /ai-chat/messages/delta every 2.5s: it returns only the rows
|
||||
// CHANGED since the previous cursor (+ the run fact) in ONE round-trip. We merge
|
||||
// those rows into the SAME infinite-query cache the thread reads (idempotently by
|
||||
// id — the delta's overlap window re-delivers rows), so the thread's reconcile
|
||||
// effect follows the detached run to its terminal row from a fraction of the wire
|
||||
// cost. The run-fact settle stays the thread FSM's job (row-status reconcile), so
|
||||
// we do NOT double-poll /run here. Cursor resets when the chat changes / disarms.
|
||||
const deltaCursorRef = useRef<string | undefined>(undefined);
|
||||
useEffect(() => {
|
||||
if (degradedPoll) lastActivityAtRef.current = Date.now();
|
||||
}, [degradedPoll, messageRows]);
|
||||
deltaCursorRef.current = undefined;
|
||||
}, [activeChatId, degradedPoll]);
|
||||
useEffect(() => {
|
||||
if (!degradedPoll || !windowOpen || !activeChatId) return;
|
||||
const chatId = activeChatId;
|
||||
let cancelled = false;
|
||||
const tick = async (): Promise<void> => {
|
||||
try {
|
||||
const res = await getAiChatMessagesDelta(chatId, deltaCursorRef.current);
|
||||
if (cancelled) return;
|
||||
deltaCursorRef.current = res.cursor;
|
||||
if (res.rows.length > 0) {
|
||||
queryClient.setQueryData(
|
||||
AI_CHAT_MESSAGES_RQ_KEY(chatId),
|
||||
(
|
||||
old:
|
||||
| {
|
||||
pages: { items: IAiChatMessageRow[]; meta: unknown }[];
|
||||
pageParams: unknown[];
|
||||
}
|
||||
| undefined,
|
||||
) =>
|
||||
old
|
||||
? { ...old, pages: mergeDeltaRowsIntoPages(old.pages, res.rows) }
|
||||
: old,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// Transient failure (e.g. a server restart mid-run): swallow and retry on
|
||||
// the next tick — the poll must survive a bounce, like the old dumb refetch.
|
||||
}
|
||||
};
|
||||
const id = setInterval(() => void tick(), 2500);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(id);
|
||||
};
|
||||
}, [degradedPoll, windowOpen, activeChatId, queryClient]);
|
||||
|
||||
// #184 reconnect-and-live-follow. Whether detached agent runs are enabled for
|
||||
// this workspace. When the feature is off no runs are ever created, so the
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -57,6 +57,50 @@ export async function stopRun(
|
||||
return req.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delta poll (#491): the chat's message rows changed since `cursor` (a DB-clock
|
||||
* timestamp echoed from the previous poll) plus the current run fact, in ONE
|
||||
* round-trip — the degraded-poll fallback's payload, replacing the old "refetch
|
||||
* ALL infinite-query pages every 2.5s with full parts" poll. Omit `cursor` on the
|
||||
* first poll (returns just a fresh cursor, no rows, to start the chain). The
|
||||
* overlap window guarantees occasional REPEATS, so the caller MUST merge rows
|
||||
* idempotently by id (mergeById). Owner-gated server-side.
|
||||
*/
|
||||
export async function getAiChatMessagesDelta(
|
||||
chatId: string,
|
||||
cursor?: string,
|
||||
): Promise<{
|
||||
rows: IAiChatMessageRow[];
|
||||
cursor: string;
|
||||
run: { id: string; status: string } | null;
|
||||
}> {
|
||||
const req = await api.post<{
|
||||
rows: IAiChatMessageRow[];
|
||||
cursor: string;
|
||||
run: { id: string; status: string } | null;
|
||||
}>("/ai-chat/messages/delta", { chatId, cursor });
|
||||
return req.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* #488: the run-fact — "is a run active on this chat?" — first-class from the
|
||||
* server (POST /ai-chat/run). Called on mount to seed the client FSM's run-fact
|
||||
* and to VERIFY after a supersede mismatch (an observer following a superseded
|
||||
* run asks for the latest run and follows it). Returns the latest run row (with
|
||||
* its `id` and `status`) and its projected assistant message, or `run: null` when
|
||||
* the chat has never had a run. Owner-gated server-side.
|
||||
*/
|
||||
export async function getRun(chatId: string): Promise<{
|
||||
run: { id: string; status: string } | null;
|
||||
message: IAiChatMessageRow | null;
|
||||
}> {
|
||||
const req = await api.post<{
|
||||
run: { id: string; status: string } | null;
|
||||
message: IAiChatMessageRow | null;
|
||||
}>("/ai-chat/run", { chatId });
|
||||
return req.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the chat bound to a document (the current user's most-recent chat
|
||||
* created on that page), or null when there is none. Drives auto-open-on-page.
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
# AI-chat run-lifecycle FSM — design spec (#488)
|
||||
|
||||
This is the written design that `run-fsm.ts` implements. It ships in the PR (issue
|
||||
#488 commit 1: "the spec is written FIRST and enters the PR"). It has four parts:
|
||||
(1) the event × state transition table, (2) the map of every `chat-thread.tsx` ref
|
||||
to {FSM state | FSM context | stays data}, (3) the run-fact protocol, (4) the
|
||||
invariants.
|
||||
|
||||
The reducer is a **pure function** `reduce(machine, event) → machine`. The returned
|
||||
machine carries the **command effects** for that transition; a thin runtime in
|
||||
`chat-thread.tsx` dispatches events and executes effects. Because it is pure, the
|
||||
whole machine is enumerable and unit-tested directly (event × state → next state is
|
||||
the observable property) — see `run-fsm.test.ts`.
|
||||
|
||||
---
|
||||
|
||||
## 1. Event × state transition table
|
||||
|
||||
Phases: `idle | sending | streaming | attaching | reconnecting(attempt,failed) |
|
||||
polling(reason) | stalled | stopping | superseding | error(kind)`.
|
||||
Context (orthogonal): `epoch`, `ownership: local|observer`, `runFact: {runId}|null`,
|
||||
`liveFollow` (are we following a live run we locally streamed — the reconnect
|
||||
ladder — vs a one-shot mount-attach resume? both are `observer`, but a live-follow
|
||||
drop RE-ENTERS the ladder (#488 commit 3) while a mount-resume drop polls).
|
||||
|
||||
Legend: **†** = command-transition (bumps `epoch`, I1). Effects in `[…]`.
|
||||
|
||||
| Event (source) | From phase(s) | → To phase | Effects / ctx |
|
||||
|---|---|---|---|
|
||||
| `SEND_LOCAL` (user send) | idle, error, polling, stalled, reconnecting | sending **†** | `[cancelReconnect, disarmPoll]`, ownership=local |
|
||||
| `STREAM_START{runId}` (SDK `start` metadata) | sending, attaching, reconnecting, superseding | streaming | `[cancelReconnect, disarmPoll]`, runFact←runId |
|
||||
| `FINISH_CLEAN` (onFinish clean) | streaming, … | idle | `[disarmPoll, cancelReconnect]`, runFact←null |
|
||||
| `FINISH_ABORT` (onFinish isAbort) | streaming, stopping | idle | `[disarmPoll, cancelReconnect]`, runFact←null (I4 exits stopping by this DATA) |
|
||||
| `FINISH_DISCONNECT` (observer, NOT liveFollow) | streaming(observer) | polling(disconnect-visible) | `[armPoll]` (a mount-resume drop polls) |
|
||||
| `FINISH_DISCONNECT{hasVisibleContent}` (local drop OR liveFollow) | streaming | reconnecting(1) **†** *iff runFact\|liveFollow* | `[scheduleReconnect(1)]` (+`armPoll` if visible), ownership=observer, liveFollow=true (commit 3: repeatable) |
|
||||
| `FINISH_DISCONNECT` (no runFact, not liveFollow) | streaming | idle | runFact←null (plain terminal "connection lost") |
|
||||
| `STREAM_INCOMPLETE{reason}` (observer starved/torn clean finish) | streaming(observer) | polling(reason) | `[armPoll(reason)]` |
|
||||
| `FINISH_ERROR{kind}` (onFinish isError) | any | error(kind) | `[disarmPoll, cancelReconnect]`, runFact←null |
|
||||
| `STREAM_START{runId}` (first assistant frame of a local turn) | sending | streaming | runFact←runId, `[cancelReconnect, disarmPoll]` |
|
||||
| `ATTACH_START{runId}` (mount resume) | **idle only** (F2) | attaching **†** | `[resumeStream]`, ownership=observer, runFact←runId; ignored from any non-idle phase |
|
||||
| `ATTACH_LIVE` (attach GET 2xx) | attaching | streaming | — |
|
||||
| `ATTACH_NONE` (attach GET 204/err/throw) | attaching | polling(attach-none) | `[armPoll(attach-none)]` |
|
||||
| `RECONNECT_ATTEMPT{n}` (backoff timer) | reconnecting | reconnecting(n) **†** | `[resumeStream]` |
|
||||
| `RECONNECT_ATTACHED` (reconnect GET 2xx) | reconnecting | streaming | `[cancelReconnect, disarmPoll]` — **counter reset** (commit 3) |
|
||||
| `RECONNECT_NONE` (reconnect GET 204/err), attempt<MAX | reconnecting | reconnecting(n+1) **†** | `[armPoll(attach-none), scheduleReconnect(n+1)]` |
|
||||
| `RECONNECT_NONE`, attempt=MAX | reconnecting | reconnecting(MAX, failed) | `[armPoll(reconnect-exhausted)]` |
|
||||
| `RETRY` (manual, failed banner) | reconnecting(failed) | reconnecting(1) **†** | `[resumeStream]` |
|
||||
| `RETRY` (manual, stalled banner) | stalled | polling(attach-none) **†** | `[armPoll]` |
|
||||
| `POLL_TERMINAL` (settled tail merged) | polling, reconnecting, stopping | idle | `[disarmPoll, cancelReconnect]`, runFact←null (I4) |
|
||||
| `POLL_IDLE_CAP` (inactivity cap) | polling, reconnecting | stalled | `[disarmPoll, cancelReconnect]` (commit 4a — no more silent) |
|
||||
| `POLL_IDLE_CAP` (inactivity cap) | stopping | idle | `[disarmPoll, cancelReconnect]`, runFact←null (Review #4: a Stop-armed poll with no SDK/terminal backstop gets a bounded exit — NOT `stalled`, Stop was already pressed so nothing to retry) |
|
||||
| `RUN_FACT{null}` (POST /run → null/terminal, 204) | reconnecting/attaching/polling/stopping | idle | `[cancelReconnect, disarmPoll]`, runFact←null (I3 fresh-negative gate) |
|
||||
| `RUN_FACT{runId}` | any | (same) | runFact←runId (pessimism toward an attempt) |
|
||||
| `STOP_REQUESTED` (user Stop) | streaming, reconnecting, polling | stopping **†** | `[stopRun, abortAttach, cancelReconnect, armPoll]` (poll drives the terminal — I4 exit by data) |
|
||||
| `SUPERSEDE_REQUESTED{targetRunId}` (interrupt+send) | streaming, reconnecting, polling, error | superseding **†** | `[supersede(target), cancelReconnect, disarmPoll]` |
|
||||
| `SUPERSEDE_READY{runId}` (CAS ok) | superseding | streaming | ownership=local, runFact←runId |
|
||||
| `SUPERSEDE_MISMATCH{currentRunId}` (409 SUPERSEDE_TARGET_MISMATCH) | superseding | error(supersede-mismatch) | `[postRun(verify)]`, runFact←currentRunId |
|
||||
| `SUPERSEDE_TIMEOUT` (409 SUPERSEDE_TIMEOUT) | superseding | error(supersede-timeout) | — (composer keeps text; no auto-retry) |
|
||||
| `SUPERSEDE_INVALID` (409 SUPERSEDE_INVALID) | superseding | error(supersede-invalid) | — |
|
||||
| `RUN_ALREADY_ACTIVE{activeRunId}` (409 A_RUN_ALREADY_ACTIVE, plain POST) | sending | error(run-already-active) | runFact←activeRunId (composer offers supersede; NO auto-retry) |
|
||||
| `DISPOSE` (unmount) | any | idle **†** | `[abortAttach, cancelReconnect, disarmPoll]` (I1/I5 — epoch++ kills late callbacks) |
|
||||
|
||||
**`stopping` honors any finish (re-review MEDIUM):** BEFORE the epoch filter, a
|
||||
stream finish (`FINISH_*`/`STREAM_INCOMPLETE`) arriving in phase `stopping` exits
|
||||
`stopping -> idle` regardless of generation. A plain Stop has no successor stream,
|
||||
so the aborted stream's finish IS the expected end (I4 exit by data) — and it
|
||||
carries the PRE-stop generation (STOP_REQUESTED bumped the epoch), so the filter
|
||||
would otherwise strand the machine in `stopping` (no idle-cap covers it). The filter
|
||||
stays in force for `superseding` (that is the F1 supersede drop).
|
||||
|
||||
**Epoch filter (I1):** the reducer then drops any event carrying an `epoch` that
|
||||
does not equal the current `ctx.epoch`. Outcome events (`STREAM_START`, `ATTACH_*`,
|
||||
`RECONNECT_*`, `SUPERSEDE_*`, **`FINISH_*`/`STREAM_INCOMPLETE`**, `RUN_FACT`) are
|
||||
stamped with the generation the corresponding STREAM started under (the runtime
|
||||
holds a per-owned-stream `turnEpoch`); trigger events (user actions, fresh
|
||||
disconnects) carry no epoch. **F1:** this is what makes a SUPERSEDED stream's late
|
||||
`onFinish` (a dead stream A closing after the CAS started stream B) get dropped, so
|
||||
A cannot drive the live new run into a false reconnect or reset its run-fact. The
|
||||
supersede path additionally ABORTS A and starts B only from A's onFinish (a
|
||||
microtask), because ai@6 `AbstractChat.makeRequest` corrupts overlapping streams
|
||||
(A's `finally` reads then nulls the shared `activeResponse`).
|
||||
|
||||
**Removed events (scope-cut, internal review):** `RUN_SUPERSEDED` (a ghost feature —
|
||||
never dispatched; the observer-superseded case is handled by the degraded poll,
|
||||
which follows the latest rows regardless of runId), `RECONNECT_BEGIN` (reconnect is
|
||||
entered by `FINISH_DISCONNECT`), and `POLL_ACTIVITY` (the window's activity clock was
|
||||
removed when the idle-cap moved into the thread). The reducer and this table now
|
||||
share exactly the dispatched event set.
|
||||
|
||||
### 409-code → event map (the real #487 contract consumed here)
|
||||
|
||||
| Server response | Event dispatched | error kind → banner |
|
||||
|---|---|---|
|
||||
| 409 `A_RUN_ALREADY_ACTIVE` (+ body.activeRunId) | `RUN_ALREADY_ACTIVE{activeRunId}` | run-already-active → "already answering / interrupt & send" |
|
||||
| 409 `SUPERSEDE_TARGET_MISMATCH` (+ body.activeRunId) | `SUPERSEDE_MISMATCH{currentRunId}` | supersede-mismatch → verify via /run |
|
||||
| 409 `SUPERSEDE_TIMEOUT` | `SUPERSEDE_TIMEOUT` | supersede-timeout → "couldn't interrupt in time, resend" |
|
||||
| 409 `SUPERSEDE_INVALID` | `SUPERSEDE_INVALID` | supersede-invalid → "couldn't interrupt this run" |
|
||||
| 503 `A_RUN_BEGIN_FAILED` | `FINISH_ERROR{begin-failed}` | begin-failed → "could not start, temporary" |
|
||||
|
||||
---
|
||||
|
||||
## 2. Ref-map — every `chat-thread.tsx` ref → its new home (MIGRATION RESOLVED)
|
||||
|
||||
The migration is COMPLETE: the 13 run-lifecycle FLAGS below are GONE from
|
||||
`chat-thread.tsx` (collapsed into FSM phase/ctx/effects, or deleted). What remains
|
||||
are identity/data mirrors, effect-owned controllers/timers, and ONE React-liveness
|
||||
bit — none of which is a run-lifecycle flag, so the post-merge "no new flags" rule
|
||||
holds. **Pending column: empty.**
|
||||
|
||||
| # | Old ref | Resolved to | Where now |
|
||||
|---|---|---|---|
|
||||
| 1 | `reconcileTailRef` | **FSM phase** | reconcile-merge gated on `phase ∈ {polling, reconnecting, stopping}` |
|
||||
| 2 | `noStreamHandledRef` | **FSM epoch (I1)** | the attach outcome's epoch guard drops the stale/second outcome |
|
||||
| 3 | `onNoActiveStreamRef` | **FSM event** | transport → `handleAttachOutcome` dispatches `ATTACH_NONE`/`RECONNECT_NONE` |
|
||||
| 4 | `onReconnectAttachedRef` | **FSM event** | transport dispatches `ATTACH_LIVE` / `RECONNECT_ATTACHED` |
|
||||
| 5 | `resumedTurnRef` + `resumedTurn` state | **FSM ctx `ownership`** | `ownership==='observer'` ⇒ never flush; hides "Send now" |
|
||||
| 6 | `reconnectStateRef` + `reconnectState` state | **FSM phase** | `reconnecting(attempt,failed)` renders the banner |
|
||||
| 7 | `reconnectTimerRef` | **effect-owned timer** | owned by `scheduleReconnect`/`cancelReconnect` effects (not a flag) |
|
||||
| 8 | `flushOnAbortRef` | **DELETED** | the stop→flush dance is replaced by the CAS supersede (commit 5) |
|
||||
| 9 | `interruptNextSendRef` | **DELETED** | the server injects the interrupt note from the supersede itself |
|
||||
| 10 | `supersedeRetryRef` | **DELETED** (commit 5) | the client 409 retry ladder is gone; CAS supersede replaces it |
|
||||
| 11 | `stopPendingRef` | **FSM phase `stopping`** | the deferred stop fires from the chat-id adoption effect while `stopping` |
|
||||
| 12 | `mountedRef` | **retained (React liveness)** | orthogonal to run-lifecycle; gates imperative onFinish side-effects post-unmount. Epoch (I1) handles stale COMMAND-outcomes; DISPOSE bumps it |
|
||||
| 13 | `attemptResumeRef` | **FSM `ATTACH_START` + run-fact** | mount arms attach ONLY on a confirmed active run (commit 4b: streaming-tail status, or POST /run for a user tail) |
|
||||
| 14–15 | `anchorRef {id, stepsPersisted}` | **data** (attachStrategy) | #491 tail-only: replaced `stripRef`/`strippedRowRef`. The PERSISTED assistant row that pins the run (server invariant 6) + its step frontier N; feeds `?anchor=<id>&n=<stepsPersisted>`. No strip — the seed keeps every row; entering reconnecting re-seeds from persist |
|
||||
| 16 | `attachAbortRef` | **effect-owned controller** | aborted by the `abortAttach` effect in cleanup (I5) |
|
||||
| 17–25 | `chatIdRef`, `openPageRef`, `getEditorSelectionRef`, `roleIdRef`, `stableIdRef`, `queuedRef`, `sendMessageRef`, `statusRef`, `lastForwardedChatIdRef` | **data** (identity/send mirrors) | unchanged — not lifecycle flags |
|
||||
| NEW | `pendingSupersedeRef` | **data** (send-plumbing) | the runId injected into the next `POST /stream {supersede}`; the single replacement for the 3 DELETED one-shots (#8/#9/#10) — net −2 refs |
|
||||
| NEW | `idleCapTimerRef` | **effect-owned timer** | the stalled inactivity cap → `POLL_IDLE_CAP` (commit 4a); not a flag |
|
||||
|
||||
Net: the 13 lifecycle flags (#1–#13) are eliminated: **8** → FSM phase/ctx/epoch/event
|
||||
(#1–#6, #11, #13), **3** deleted (#8/#9/#10), **`reconnectTimerRef` (#7)** becomes an
|
||||
effect-owned controller, and **`mountedRef` (#12)** is retained as React liveness
|
||||
(8 + 3 + 1 + 1 = 13). (`attachAbortRef` (#16) is outside the #1–#13 set — it was
|
||||
already an effect-owned controller.) Two effect-owned timers + one send-plumbing data
|
||||
ref are added — none is a boolean lifecycle latch.
|
||||
|
||||
---
|
||||
|
||||
## 3. Run-fact protocol (`runFact: {runId} | null`) — I3
|
||||
|
||||
"A run is active" is first-class from the SERVER, not inferred from an assistant
|
||||
message. Sources, in the order they update `ctx.runFact`:
|
||||
|
||||
1. **Init (mount):** `POST /ai-chat/run { chatId }` → `{ run, message }`. A `run`
|
||||
with a non-terminal `status` seeds `runFact = { runId: run.id }`; a null/terminal
|
||||
run seeds `null`. This is what arms the resume attempt (`ATTACH_START`) — the
|
||||
attempt is armed ONLY on a positive fact (commit 4b: a user-tail with no active
|
||||
run no longer arms a pointless poll on every open).
|
||||
2. **Live update:** the `start` stream metadata carries `runId` → `STREAM_START{runId}`.
|
||||
3. **Attach outcomes:** `ATTACH_LIVE` (2xx) confirms active; a 204 on a non-stripped
|
||||
path is an authoritative NEGATIVE fact → the runtime dispatches `RUN_FACT{null}`,
|
||||
which cancels recovery (I3 fresh-negative gate).
|
||||
4. **Poll (#491, implemented):** the degraded poll now hits the delta endpoint
|
||||
(`POST /ai-chat/messages/delta`), which ALREADY carries the run fact
|
||||
(`run: {id, status} | null`) alongside the changed rows. The client does NOT yet
|
||||
consume that run field — it still drives to a terminal ROW (merged by id),
|
||||
dispatched as `POLL_TERMINAL` — so the run field rides the wire for a future
|
||||
client that settles straight off it.
|
||||
|
||||
Pessimism rule: a stale-but-positive fact PERMITS entering recovery (attach); the
|
||||
204 then cuts it. A fresh negative fact gates recovery OUT immediately.
|
||||
|
||||
---
|
||||
|
||||
## 4. Invariants
|
||||
|
||||
- **I1 — Epoch (generation counter).** Every command-emitting transition bumps
|
||||
`ctx.epoch`; every async outcome event carries its issuing epoch; the reducer
|
||||
drops stale-epoch outcomes. Replaces the one-shot-ref zoo (`noStreamHandledRef`,
|
||||
the flush/interrupt/supersede one-shots, the `mountedRef` late-callback gate).
|
||||
- **I2 — Ownership is context, not state.** `local | observer` is orthogonal to the
|
||||
transport phase. The queue flushes ONLY under local ownership; an observer
|
||||
following a detached run never flushes (was `resumedTurnRef`).
|
||||
- **I3 — Run-fact is first-class from the server.** Reconnect is entered by the
|
||||
run-fact, not by an assistant message (commit 2). A fresh negative fact cancels
|
||||
recovery.
|
||||
- **I4 — Exit `stopping` by DATA.** A terminal row / negative run-fact / terminal
|
||||
finish exits `stopping`, never the stopRun HTTP response (which returns after the
|
||||
abort but before finalization — keying off it would unlock the composer on a 409).
|
||||
- **I5 — Dispose protocol.** Command controllers (attach GET, POST /stream, POST
|
||||
/run) are effect-owned and aborted in cleanup (`abortAttach` on `DISPOSE`), not
|
||||
render-phase refs. A client abort of an already-sent POST does not cancel the
|
||||
server action, so disarming on unmount is safe.
|
||||
- **attachStrategy** is behind the `resumeStream` effect; #491 swapped it to
|
||||
tail-only (`?anchor=&n=`, `anchorRef` data) WITHOUT touching the FSM. Entering
|
||||
reconnecting always re-seeds from persist; on a getRun failure the live partial
|
||||
is dropped + replay-from-start so it is never the tail-apply base (no #137/#161
|
||||
duplication).
|
||||
- **Queue** stays a data structure; flush/interrupt decisions are transitions.
|
||||
@@ -0,0 +1,482 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
reduce,
|
||||
initialMachine,
|
||||
reconnectDelayMs,
|
||||
RECONNECT_MAX_ATTEMPTS,
|
||||
type Machine,
|
||||
type Effect,
|
||||
type Event,
|
||||
} from "./run-fsm";
|
||||
|
||||
// Drive a sequence of events through the reducer, returning the final machine.
|
||||
function run(m: Machine, ...events: Event[]): Machine {
|
||||
return events.reduce(reduce, m);
|
||||
}
|
||||
function withRunFact(runId = "run-1"): Machine {
|
||||
return {
|
||||
...initialMachine(),
|
||||
ctx: { epoch: 0, ownership: "local", runFact: { runId }, liveFollow: false },
|
||||
};
|
||||
}
|
||||
function effectTypes(m: Machine): string[] {
|
||||
return m.effects.map((e) => e.type);
|
||||
}
|
||||
function hasEffect(m: Machine, type: Effect["type"]): boolean {
|
||||
return m.effects.some((e) => e.type === type);
|
||||
}
|
||||
|
||||
describe("run-fsm — epoch invariant (I1)", () => {
|
||||
it("drops an outcome carrying a stale epoch", () => {
|
||||
// A command bumps the epoch; an outcome stamped with the OLD epoch is dropped.
|
||||
const m0 = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); // epoch 0->1, attaching
|
||||
expect(m0.ctx.epoch).toBe(1);
|
||||
expect(m0.phase.name).toBe("attaching");
|
||||
// A late ATTACH_LIVE from a SUPERSEDED attempt (epoch 0) must NOT drive us.
|
||||
const stale = reduce(m0, { type: "ATTACH_LIVE", epoch: 0 });
|
||||
expect(stale.phase.name).toBe("attaching");
|
||||
expect(stale.effects).toEqual([]);
|
||||
});
|
||||
|
||||
it("applies an outcome carrying the current epoch", () => {
|
||||
const m0 = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
const live = reduce(m0, { type: "ATTACH_LIVE", epoch: m0.ctx.epoch });
|
||||
expect(live.phase.name).toBe("streaming");
|
||||
});
|
||||
|
||||
it("an outcome with no epoch is never dropped (trigger events)", () => {
|
||||
const m0 = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
const disposed = reduce(m0, { type: "DISPOSE" });
|
||||
expect(disposed.phase.name).toBe("idle");
|
||||
expect(hasEffect(disposed, "abortAttach")).toBe(true);
|
||||
});
|
||||
|
||||
it("every command-transition increments the epoch exactly once", () => {
|
||||
let m = initialMachine();
|
||||
const before = m.ctx.epoch;
|
||||
m = reduce(m, { type: "SEND_LOCAL" });
|
||||
expect(m.ctx.epoch).toBe(before + 1);
|
||||
m = reduce(m, { type: "STOP_REQUESTED" });
|
||||
expect(m.ctx.epoch).toBe(before + 2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("run-fsm — local turn", () => {
|
||||
it("SEND_LOCAL → sending, local ownership, cancels recovery", () => {
|
||||
const m = reduce(withRunFact(), { type: "SEND_LOCAL" });
|
||||
expect(m.phase.name).toBe("sending");
|
||||
expect(m.ctx.ownership).toBe("local");
|
||||
expect(effectTypes(m)).toEqual(
|
||||
expect.arrayContaining(["cancelReconnect", "disarmPoll"]),
|
||||
);
|
||||
});
|
||||
|
||||
it("STREAM_START adopts the runId into the run-fact and goes streaming", () => {
|
||||
const m = run(initialMachine(), { type: "SEND_LOCAL" });
|
||||
const s = reduce(m, { type: "STREAM_START", runId: "run-9", epoch: m.ctx.epoch });
|
||||
expect(s.phase.name).toBe("streaming");
|
||||
expect(s.ctx.runFact).toEqual({ runId: "run-9" });
|
||||
});
|
||||
|
||||
it("FINISH_CLEAN → idle, run-fact cleared, poll/reconnect disarmed", () => {
|
||||
const streaming = run(initialMachine(), { type: "SEND_LOCAL" }, { type: "STREAM_START", runId: "r" });
|
||||
const done = reduce(streaming, { type: "FINISH_CLEAN" });
|
||||
expect(done.phase.name).toBe("idle");
|
||||
expect(done.ctx.runFact).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// #488 commit 2 — SSE break BEFORE the first assistant frame must still recover.
|
||||
describe("run-fsm — commit 2: reconnect by run-fact, not by assistant message", () => {
|
||||
it("FINISH_DISCONNECT with an active run-fact → reconnecting (even with no visible content)", () => {
|
||||
// Setup-phase break: no assistant frame yet, but a run-fact exists.
|
||||
const streaming = withRunFact("run-2");
|
||||
const m = reduce(streaming, {
|
||||
type: "FINISH_DISCONNECT",
|
||||
hasVisibleContent: false,
|
||||
epoch: streaming.ctx.epoch,
|
||||
});
|
||||
expect(m.phase.name).toBe("reconnecting");
|
||||
if (m.phase.name === "reconnecting") expect(m.phase.attempt).toBe(1);
|
||||
expect(m.ctx.ownership).toBe("observer");
|
||||
expect(hasEffect(m, "scheduleReconnect")).toBe(true);
|
||||
// No visible content -> no poll arm yet (the reconnect ladder rebuilds it).
|
||||
expect(hasEffect(m, "armPoll")).toBe(false);
|
||||
});
|
||||
|
||||
it("FINISH_DISCONNECT WITH visible content also arms the poll", () => {
|
||||
const m = reduce(withRunFact("run-2"), {
|
||||
type: "FINISH_DISCONNECT",
|
||||
hasVisibleContent: true,
|
||||
epoch: 0,
|
||||
});
|
||||
expect(m.phase.name).toBe("reconnecting");
|
||||
expect(hasEffect(m, "armPoll")).toBe(true);
|
||||
});
|
||||
|
||||
it("FINISH_DISCONNECT with NO run-fact → idle (plain connection-lost)", () => {
|
||||
const m = reduce(initialMachine(), {
|
||||
type: "FINISH_DISCONNECT",
|
||||
hasVisibleContent: true,
|
||||
epoch: 0,
|
||||
});
|
||||
expect(m.phase.name).toBe("idle");
|
||||
});
|
||||
});
|
||||
|
||||
// #488 commit 3 — a SECOND break after a successful re-attach starts a NEW ladder.
|
||||
describe("run-fsm — commit 3: repeated reconnect cycles", () => {
|
||||
it("two breaks in a row produce two reconnect cycles (counter resets on attach)", () => {
|
||||
let m = withRunFact("run-3");
|
||||
// First break -> reconnecting(1).
|
||||
m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("reconnecting");
|
||||
// Attempt fires, re-attaches live.
|
||||
m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: 1, epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "RECONNECT_ATTACHED", epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("streaming");
|
||||
// SECOND break: the counter was reset, so a fresh ladder starts at attempt 1
|
||||
// (the old one-shot !wasResumed gate would have sent this to silent poll).
|
||||
m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("reconnecting");
|
||||
if (m.phase.name === "reconnecting") expect(m.phase.attempt).toBe(1);
|
||||
expect(hasEffect(m, "scheduleReconnect")).toBe(true);
|
||||
});
|
||||
|
||||
it("a MOUNT-attach observer drop falls to POLL, not the reconnect ladder", () => {
|
||||
// Distinguishes commit 3 from a one-shot resume: an observer that never
|
||||
// live-followed (liveFollow false) polls on a drop.
|
||||
let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
m = reduce(m, { type: "ATTACH_LIVE", epoch: m.ctx.epoch });
|
||||
expect(m.ctx.ownership).toBe("observer");
|
||||
expect(m.ctx.liveFollow).toBe(false);
|
||||
m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: true, epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("polling");
|
||||
expect(hasEffect(m, "armPoll")).toBe(true);
|
||||
});
|
||||
|
||||
it("STREAM_INCOMPLETE (observer starved/torn finish) → polling", () => {
|
||||
let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
m = reduce(m, { type: "ATTACH_LIVE", epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "STREAM_INCOMPLETE", reason: "starved", epoch: m.ctx.epoch });
|
||||
expect(m.phase).toEqual({ name: "polling", reason: "starved" });
|
||||
expect(hasEffect(m, "armPoll")).toBe(true);
|
||||
});
|
||||
|
||||
it("liveFollow is set on the first local drop and kept across a re-attach", () => {
|
||||
let m = withRunFact("run-3");
|
||||
m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch });
|
||||
expect(m.ctx.liveFollow).toBe(true);
|
||||
m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: 1, epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "RECONNECT_ATTACHED", epoch: m.ctx.epoch });
|
||||
expect(m.ctx.liveFollow).toBe(true); // kept — so a second drop reconnects
|
||||
// A clean finish clears it.
|
||||
m = reduce(m, { type: "FINISH_CLEAN", epoch: m.ctx.epoch });
|
||||
expect(m.ctx.liveFollow).toBe(false);
|
||||
});
|
||||
|
||||
it("RECONNECT_NONE backs off through the ladder, then fails at the cap", () => {
|
||||
let m = withRunFact("run-3");
|
||||
m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch });
|
||||
for (let n = 1; n < RECONNECT_MAX_ATTEMPTS; n++) {
|
||||
m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: n, epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "RECONNECT_NONE", epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("reconnecting");
|
||||
if (m.phase.name === "reconnecting") {
|
||||
expect(m.phase.attempt).toBe(n + 1);
|
||||
expect(m.phase.failed).toBe(false);
|
||||
}
|
||||
// The belt-and-suspenders poll is armed each failed attempt.
|
||||
expect(hasEffect(m, "armPoll")).toBe(true);
|
||||
}
|
||||
// Final attempt fails -> failed banner (Retry), poll armed.
|
||||
m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: RECONNECT_MAX_ATTEMPTS, epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "RECONNECT_NONE", epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("reconnecting");
|
||||
if (m.phase.name === "reconnecting") expect(m.phase.failed).toBe(true);
|
||||
// RETRY restarts at attempt 1.
|
||||
m = reduce(m, { type: "RETRY" });
|
||||
expect(m.phase.name).toBe("reconnecting");
|
||||
if (m.phase.name === "reconnecting") {
|
||||
expect(m.phase.attempt).toBe(1);
|
||||
expect(m.phase.failed).toBe(false);
|
||||
}
|
||||
expect(hasEffect(m, "resumeStream")).toBe(true);
|
||||
});
|
||||
|
||||
it("reconnectDelayMs is the exponential backoff 1s,2s,4s,8s,16s", () => {
|
||||
expect([1, 2, 3, 4, 5].map(reconnectDelayMs)).toEqual([1000, 2000, 4000, 8000, 16000]);
|
||||
});
|
||||
});
|
||||
|
||||
// #488 commit 4 — polling stalled-state + user-tail gating.
|
||||
describe("run-fsm — commit 4: stalled + run-fact gating", () => {
|
||||
it("POLL_IDLE_CAP: polling → stalled with a banner (poll disarmed), not silent", () => {
|
||||
let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" });
|
||||
m = reduce(m, { type: "ATTACH_NONE", epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("polling");
|
||||
m = reduce(m, { type: "POLL_IDLE_CAP" });
|
||||
expect(m.phase.name).toBe("stalled");
|
||||
expect(hasEffect(m, "disarmPoll")).toBe(true);
|
||||
});
|
||||
|
||||
it("RETRY from stalled re-arms the poll", () => {
|
||||
let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" });
|
||||
m = reduce(m, { type: "ATTACH_NONE", epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "POLL_IDLE_CAP" });
|
||||
m = reduce(m, { type: "RETRY" });
|
||||
expect(m.phase.name).toBe("polling");
|
||||
expect(hasEffect(m, "armPoll")).toBe(true);
|
||||
});
|
||||
|
||||
it("a fresh NEGATIVE run-fact while attaching cancels recovery (user-tail, no active run)", () => {
|
||||
// The mount POST /run returns no active run: attaching → idle, no poll armed.
|
||||
let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" });
|
||||
m = reduce(m, { type: "RUN_FACT", runFact: null, epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
expect(m.ctx.runFact).toBeNull();
|
||||
expect(hasEffect(m, "disarmPoll")).toBe(true);
|
||||
});
|
||||
|
||||
it("a negative run-fact while polling stops the poll", () => {
|
||||
let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" });
|
||||
m = reduce(m, { type: "ATTACH_NONE", epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "RUN_FACT", runFact: null, epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
});
|
||||
|
||||
it("POLL_TERMINAL settles polling → idle (I4 data-driven exit)", () => {
|
||||
let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" });
|
||||
m = reduce(m, { type: "ATTACH_NONE", epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "POLL_TERMINAL" });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
expect(m.ctx.runFact).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
// #488 commit 5 — error classification + supersede CAS transitions.
|
||||
describe("run-fsm — commit 5: supersede CAS + error classification", () => {
|
||||
it("SUPERSEDE_REQUESTED → superseding, fires the CAS effect, bumps epoch", () => {
|
||||
const streaming = withRunFact("run-old");
|
||||
const m = reduce(streaming, { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" });
|
||||
expect(m.phase.name).toBe("superseding");
|
||||
expect(m.ctx.epoch).toBe(streaming.ctx.epoch + 1);
|
||||
const sup = m.effects.find((e) => e.type === "supersede");
|
||||
expect(sup).toEqual({ type: "supersede", targetRunId: "run-old" });
|
||||
});
|
||||
|
||||
it("SUPERSEDE_READY → streaming as the new local owner", () => {
|
||||
let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" });
|
||||
m = reduce(m, { type: "SUPERSEDE_READY", runId: "run-new", epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("streaming");
|
||||
expect(m.ctx.ownership).toBe("local");
|
||||
expect(m.ctx.runFact).toEqual({ runId: "run-new" });
|
||||
});
|
||||
|
||||
it("SUPERSEDE_MISMATCH → error(supersede-mismatch) + verify via /run (no blind banner)", () => {
|
||||
let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" });
|
||||
m = reduce(m, { type: "SUPERSEDE_MISMATCH", currentRunId: "run-x", epoch: m.ctx.epoch });
|
||||
expect(m.phase).toEqual({ name: "error", kind: "supersede-mismatch" });
|
||||
expect(hasEffect(m, "postRun")).toBe(true);
|
||||
expect(m.ctx.runFact).toEqual({ runId: "run-x" });
|
||||
});
|
||||
|
||||
it("SUPERSEDE_TIMEOUT → error(supersede-timeout), no auto-retry effect", () => {
|
||||
let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" });
|
||||
m = reduce(m, { type: "SUPERSEDE_TIMEOUT", epoch: m.ctx.epoch });
|
||||
expect(m.phase).toEqual({ name: "error", kind: "supersede-timeout" });
|
||||
expect(m.effects).toEqual([]);
|
||||
});
|
||||
|
||||
it("SUPERSEDE_INVALID → error(supersede-invalid)", () => {
|
||||
let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" });
|
||||
m = reduce(m, { type: "SUPERSEDE_INVALID", epoch: m.ctx.epoch });
|
||||
expect(m.phase).toEqual({ name: "error", kind: "supersede-invalid" });
|
||||
});
|
||||
|
||||
it("a stale SUPERSEDE outcome from a superseded epoch is dropped", () => {
|
||||
let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" });
|
||||
const supersedingEpoch = m.ctx.epoch;
|
||||
// The user retriggers, bumping the epoch again.
|
||||
m = reduce(m, { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" });
|
||||
// The first CAS's late TIMEOUT (old epoch) must NOT knock us out of superseding.
|
||||
const late = reduce(m, { type: "SUPERSEDE_TIMEOUT", epoch: supersedingEpoch });
|
||||
expect(late.phase.name).toBe("superseding");
|
||||
});
|
||||
|
||||
it("RUN_ALREADY_ACTIVE (plain POST gate) → error(run-already-active), no retry effect", () => {
|
||||
const m = reduce(run(initialMachine(), { type: "SEND_LOCAL" }), { type: "RUN_ALREADY_ACTIVE" });
|
||||
expect(m.phase).toEqual({ name: "error", kind: "run-already-active" });
|
||||
expect(m.effects).toEqual([]);
|
||||
});
|
||||
|
||||
it("#497/S4: RUN_ALREADY_ACTIVE{activeRunId} ADOPTS the server's active run as the run-fact", () => {
|
||||
// The server sends `activeRunId` so a later supersede can TARGET that run
|
||||
// instead of a blind promote+abort. Absorb it into runFact.
|
||||
const m = reduce(run(initialMachine(), { type: "SEND_LOCAL" }), {
|
||||
type: "RUN_ALREADY_ACTIVE",
|
||||
activeRunId: "run-foreign",
|
||||
});
|
||||
expect(m.phase).toEqual({ name: "error", kind: "run-already-active" });
|
||||
expect(m.ctx.runFact).toEqual({ runId: "run-foreign" });
|
||||
expect(m.effects).toEqual([]);
|
||||
});
|
||||
|
||||
it("#497/S4: RUN_ALREADY_ACTIVE without an activeRunId keeps the prior run-fact", () => {
|
||||
const seeded = reduce(run(initialMachine(), { type: "SEND_LOCAL" }), {
|
||||
type: "RUN_FACT",
|
||||
runFact: { runId: "run-prior" },
|
||||
});
|
||||
const m = reduce(seeded, { type: "RUN_ALREADY_ACTIVE" });
|
||||
expect(m.ctx.runFact).toEqual({ runId: "run-prior" });
|
||||
});
|
||||
});
|
||||
|
||||
// #488 F2 — a late mount `getRun → ATTACH_START` must not hijack a local turn.
|
||||
describe("run-fsm — F2: ATTACH_START only from idle", () => {
|
||||
it("ATTACH_START from a local `sending` turn is ignored (no observer hijack)", () => {
|
||||
const sending = reduce(initialMachine(), { type: "SEND_LOCAL" }); // idle -> sending, local
|
||||
const m = reduce(sending, { type: "ATTACH_START", runId: "r" });
|
||||
expect(m.phase.name).toBe("sending");
|
||||
expect(m.ctx.ownership).toBe("local"); // NOT flipped to observer
|
||||
expect(m.effects).toEqual([]); // no resumeStream
|
||||
});
|
||||
|
||||
it("ATTACH_START from idle attaches as normal", () => {
|
||||
const m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
expect(m.phase.name).toBe("attaching");
|
||||
expect(m.ctx.ownership).toBe("observer");
|
||||
expect(hasEffect(m, "resumeStream")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("run-fsm — stop (I4: exit by data)", () => {
|
||||
it("STOP_REQUESTED → stopping, fires stopRun + abortAttach, no data-independent exit", () => {
|
||||
const m = reduce(withRunFact(), { type: "STOP_REQUESTED" });
|
||||
expect(m.phase.name).toBe("stopping");
|
||||
expect(effectTypes(m)).toEqual(expect.arrayContaining(["stopRun", "abortAttach"]));
|
||||
});
|
||||
|
||||
it("stopping exits on the aborted stream's finish carrying the PRE-STOP epoch", () => {
|
||||
// MEDIUM (#488 re-review): STOP_REQUESTED is a command that BUMPS the epoch, but
|
||||
// the runtime stamps the aborted stream's onFinish with the stream's START (pre-
|
||||
// stop) generation — exactly what the component sends. `stopping` must HONOR
|
||||
// that finish regardless of generation (no idle-cap covers `stopping`).
|
||||
// MUTATION-VERIFY: remove the honor-in-`stopping` branch and this hangs in
|
||||
// `stopping` (the epoch filter drops the pre-stop finish) -> red.
|
||||
const preStopEpoch = withRunFact().ctx.epoch; // E1 (the stream's start epoch)
|
||||
let m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); // E1 -> E2, stopping
|
||||
expect(m.ctx.epoch).toBe(preStopEpoch + 1);
|
||||
m = reduce(m, { type: "FINISH_ABORT", epoch: preStopEpoch }); // NOT the current epoch
|
||||
expect(m.phase.name).toBe("idle");
|
||||
expect(m.ctx.runFact).toBeNull();
|
||||
});
|
||||
|
||||
it("stopping exits on a clean finish carrying the pre-stop epoch too", () => {
|
||||
const preStopEpoch = withRunFact().ctx.epoch;
|
||||
let m = reduce(withRunFact(), { type: "STOP_REQUESTED" });
|
||||
m = reduce(m, { type: "FINISH_CLEAN", epoch: preStopEpoch });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
});
|
||||
|
||||
it("stopping exits on a negative run-fact (data)", () => {
|
||||
let m = reduce(withRunFact(), { type: "STOP_REQUESTED" });
|
||||
m = reduce(m, { type: "RUN_FACT", runFact: null, epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
});
|
||||
|
||||
// Review #4: `stopping` arms the poll but had no inactivity backstop.
|
||||
it("review-4: POLL_IDLE_CAP in `stopping` exits to idle (bounded), NOT stalled", () => {
|
||||
let m = reduce(withRunFact(), { type: "STOP_REQUESTED" });
|
||||
expect(m.phase.name).toBe("stopping");
|
||||
expect(hasEffect(m, "armPoll")).toBe(true);
|
||||
// MUTATION-VERIFY: drop the `stopping` branch in POLL_IDLE_CAP and this hangs
|
||||
// in `stopping` (poll forever) -> red.
|
||||
m = reduce(m, { type: "POLL_IDLE_CAP" });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
expect(hasEffect(m, "disarmPoll")).toBe(true);
|
||||
expect(m.ctx.ownership).toBe("local");
|
||||
});
|
||||
});
|
||||
|
||||
// Review #1: positive attach outcomes must be guarded by the SOURCE phase — the
|
||||
// epoch filter alone is insufficient because POLL_TERMINAL uses to() (no epoch
|
||||
// bump) and does not abort the in-flight GET.
|
||||
describe("run-fsm — review-1: attach outcomes guarded by source phase", () => {
|
||||
it("a late RECONNECT_ATTACHED after POLL_TERMINAL stays idle (no phantom streaming)", () => {
|
||||
let m = withRunFact("run-1");
|
||||
m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: true, epoch: m.ctx.epoch });
|
||||
m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: 1, epoch: m.ctx.epoch }); // attach GET
|
||||
const epoch = m.ctx.epoch;
|
||||
// The armed degraded poll reaches the terminal row FIRST (epoch unchanged).
|
||||
m = reduce(m, { type: "POLL_TERMINAL" });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
expect(m.ctx.epoch).toBe(epoch); // POLL_TERMINAL did NOT bump the epoch
|
||||
// The slow GET returns live 2xx under the SAME epoch — must NOT resurrect.
|
||||
m = reduce(m, { type: "RECONNECT_ATTACHED", epoch });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
});
|
||||
|
||||
it("a late ATTACH_LIVE / ATTACH_NONE after leaving `attaching` is ignored", () => {
|
||||
let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
const epoch = m.ctx.epoch;
|
||||
m = reduce(m, { type: "ATTACH_NONE", epoch }); // attaching -> polling
|
||||
m = reduce(m, { type: "POLL_TERMINAL" }); // -> idle (epoch unchanged)
|
||||
expect(m.phase.name).toBe("idle");
|
||||
m = reduce(m, { type: "ATTACH_LIVE", epoch }); // late 2xx, same epoch
|
||||
expect(m.phase.name).toBe("idle");
|
||||
// And a late ATTACH_NONE (not `attaching`) is a no-op too.
|
||||
m = reduce(m, { type: "ATTACH_NONE", epoch });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
});
|
||||
});
|
||||
|
||||
// Review #2: every terminal transition resets ownership to local.
|
||||
describe("run-fsm — review-2: terminal transitions reset ownership to local", () => {
|
||||
const observer = (): Machine => {
|
||||
let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
m = reduce(m, { type: "ATTACH_LIVE", epoch: m.ctx.epoch });
|
||||
expect(m.ctx.ownership).toBe("observer");
|
||||
return m;
|
||||
};
|
||||
it("FINISH_CLEAN resets ownership", () => {
|
||||
const m = reduce(observer(), { type: "FINISH_CLEAN", epoch: observer().ctx.epoch });
|
||||
expect(m.ctx.ownership).toBe("local");
|
||||
});
|
||||
it("FINISH_ERROR / POLL_TERMINAL / RUN_FACT(null) reset ownership", () => {
|
||||
let o = observer();
|
||||
expect(reduce(o, { type: "FINISH_ERROR", kind: "stream", epoch: o.ctx.epoch }).ctx.ownership).toBe("local");
|
||||
// POLL_TERMINAL from an observer polling phase
|
||||
let p = reduce(observer(), { type: "STREAM_INCOMPLETE", reason: "starved", epoch: observer().ctx.epoch });
|
||||
expect(reduce(p, { type: "POLL_TERMINAL" }).ctx.ownership).toBe("local");
|
||||
// RUN_FACT(null) from an observer attaching phase
|
||||
let a = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
expect(reduce(a, { type: "RUN_FACT", runFact: null, epoch: a.ctx.epoch }).ctx.ownership).toBe("local");
|
||||
});
|
||||
});
|
||||
|
||||
describe("run-fsm — ownership (I2) is context, orthogonal to phase", () => {
|
||||
it("attach/reconnect set observer; send/supersede-ready set local", () => {
|
||||
let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" });
|
||||
expect(m.ctx.ownership).toBe("observer");
|
||||
m = reduce(m, { type: "ATTACH_LIVE", epoch: m.ctx.epoch });
|
||||
expect(m.phase.name).toBe("streaming");
|
||||
expect(m.ctx.ownership).toBe("observer"); // still observing a detached run
|
||||
// A local send flips ownership back to local.
|
||||
m = reduce(m, { type: "SEND_LOCAL" });
|
||||
expect(m.ctx.ownership).toBe("local");
|
||||
});
|
||||
});
|
||||
|
||||
describe("run-fsm — dispose (I5)", () => {
|
||||
it("DISPOSE from any phase aborts controllers and bumps epoch", () => {
|
||||
let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" });
|
||||
const before = m.ctx.epoch;
|
||||
m = reduce(m, { type: "DISPOSE" });
|
||||
expect(m.phase.name).toBe("idle");
|
||||
expect(m.ctx.epoch).toBe(before + 1);
|
||||
expect(effectTypes(m)).toEqual(
|
||||
expect.arrayContaining(["abortAttach", "cancelReconnect", "disarmPoll"]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,600 @@
|
||||
/**
|
||||
* Run-lifecycle finite state machine for a single AI-chat thread (#488).
|
||||
*
|
||||
* ============================================================================
|
||||
* WHY THIS EXISTS
|
||||
* ----------------------------------------------------------------------------
|
||||
* The resume/reconnect/poll/stop/supersede lifecycle used to be spread across
|
||||
* ~26 `useRef` one-shot flags in `chat-thread.tsx`, each disarmed "on every
|
||||
* path". Ownerless flag combinations produced silent UI freezes, and every fix
|
||||
* added another ref (the #381 -> #432 -> #456 spiral). This module replaces that
|
||||
* ref-zoo with ONE pure reducer whose transitions are enumerable and unit-
|
||||
* testable in isolation (event x state -> next state is the observable property).
|
||||
*
|
||||
* The reducer is PURE: it owns no timers, no fetches, no React state. It maps
|
||||
* `(machine, event) -> machine`, where the returned machine carries the list of
|
||||
* COMMAND EFFECTS to run for that transition. A thin runtime in `chat-thread.tsx`
|
||||
* dispatches events (from SDK callbacks / HTTP outcomes) and executes the
|
||||
* effects (attach GET, POST /stream, POST /run, POST /stop, backoff timers,
|
||||
* poll arm/disarm). The runtime lives in a THREAD, not the window, so a late SDK
|
||||
* callback dies with the owner (kills the "event from a dead view" class, #161).
|
||||
*
|
||||
* ============================================================================
|
||||
* INVARIANTS (see run-fsm.spec.md for the full spec + tables)
|
||||
* ----------------------------------------------------------------------------
|
||||
* I1 EPOCH (generation counter). Commands (`resumeStream`, `postRun`, `stop`,
|
||||
* `supersede`, `scheduleReconnect`) are async; their outcomes arrive on the
|
||||
* SAME SDK/HTTP callbacks. Every command-emitting transition increments
|
||||
* `ctx.epoch`; every OUTCOME event carries the epoch it was issued under;
|
||||
* the reducer DROPS an outcome whose epoch != the current epoch. This is
|
||||
* what the one-shot-ref zoo used to approximate by hand.
|
||||
* I2 OWNERSHIP is a CONTEXT FIELD (`'local' | 'observer'`), not a state —
|
||||
* orthogonal to the transport phase. The queue is flushed ONLY by a local
|
||||
* owner (an observer following a detached run never flushes).
|
||||
* I3 RUN-FACT ("a run is active") is first-class from the server: `runFact`
|
||||
* holds the server-confirmed active run id (POST /run on mount, the `start`
|
||||
* metadata runId, attach outcomes). Reconnect is entered by the RUN-FACT,
|
||||
* not by the presence of an assistant message (#488 commit 2). A fresh
|
||||
* negative fact (null) cancels reconnect immediately.
|
||||
* I4 Exit `stopping` by DATA (a terminal row / negative run-fact), NEVER by the
|
||||
* stopRun HTTP response (which returns after abort, before finalization).
|
||||
* I5 Command controllers are effect-owned (abort in cleanup), NOT render-phase
|
||||
* refs — expressed here as the `abortAttach` effect on disposing transitions.
|
||||
* ============================================================================
|
||||
*/
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Phases (the transport lifecycle). Ownership / runFact are CONTEXT, not here.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** Why the degraded poll is the active recovery. */
|
||||
export type PollReason =
|
||||
| "attach-none" // mount attach returned 204 / error — nothing live to attach
|
||||
| "starved" // a resumed finish carried no visible content
|
||||
| "disconnect-visible" // a live disconnect WITH on-screen content — poll to terminal
|
||||
| "reconnect-exhausted"; // the live re-attach ladder gave up
|
||||
|
||||
/** The classified error kind (drives the banner text + composer behavior). */
|
||||
export type ErrorKind =
|
||||
| "stream" // a generic provider/network stream error (useChat error)
|
||||
| "run-already-active" // 409 A_RUN_ALREADY_ACTIVE (a plain POST hit the gate)
|
||||
| "supersede-mismatch" // 409 SUPERSEDE_TARGET_MISMATCH (CAS target moved)
|
||||
| "supersede-timeout" // 409 SUPERSEDE_TIMEOUT (old run did not settle in W)
|
||||
| "supersede-invalid" // 409 SUPERSEDE_INVALID (bad supersede target)
|
||||
| "begin-failed"; // 503 A_RUN_BEGIN_FAILED (could not start the run)
|
||||
|
||||
export type Phase =
|
||||
| { name: "idle" }
|
||||
| { name: "sending" } // local POST in flight, before the first frame
|
||||
| { name: "streaming" } // receiving frames
|
||||
| { name: "attaching" } // mount-time attach GET in flight
|
||||
| { name: "reconnecting"; attempt: number; failed: boolean }
|
||||
| { name: "polling"; reason: PollReason }
|
||||
| { name: "stalled" } // poll hit the inactivity cap — banner + Retry
|
||||
| { name: "stopping" }
|
||||
| { name: "superseding" }
|
||||
| { name: "error"; kind: ErrorKind };
|
||||
|
||||
export type Ownership = "local" | "observer";
|
||||
|
||||
/** The server-confirmed active run, or null when no run is active. */
|
||||
export type RunFact = { runId: string } | null;
|
||||
|
||||
export interface Ctx {
|
||||
/** I1: generation counter — every command-transition increments it. */
|
||||
epoch: number;
|
||||
/** I2: does THIS client own the turn's writes (local streamer) or observe? */
|
||||
ownership: Ownership;
|
||||
/** I3: the server-confirmed active run. */
|
||||
runFact: RunFact;
|
||||
/**
|
||||
* Are we FOLLOWING a live run we were locally streaming (the reconnect ladder),
|
||||
* as opposed to a one-shot mount-attach resume? Both are `ownership: 'observer'`,
|
||||
* but they recover DIFFERENTLY on a drop: a live-follow drop RE-ENTERS the
|
||||
* reconnect ladder (#488 commit 3 — the second break after a successful re-attach
|
||||
* must reconnect again, not fall to silent poll), while a mount-resume drop falls
|
||||
* to the degraded poll. This is the ctx bit that separates the two WITHOUT a new
|
||||
* component ref (it is why commit 3 needs the FSM, not a surgical patch).
|
||||
*/
|
||||
liveFollow: boolean;
|
||||
}
|
||||
|
||||
export interface Machine {
|
||||
phase: Phase;
|
||||
ctx: Ctx;
|
||||
/** Command effects to run for the transition that produced THIS machine.
|
||||
* The runtime executes them and does not read them again. */
|
||||
effects: Effect[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Command effects (the reducer's only side-channel — executed by the runtime).
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type Effect =
|
||||
/** POST /run to (re)establish or verify the run-fact. `reason` is diagnostic. */
|
||||
| { type: "postRun"; reason: "mount" | "verify" }
|
||||
/** Trigger the SDK `resumeStream()` (attach GET via prepareReconnectToStream). */
|
||||
| { type: "resumeStream" }
|
||||
/** Schedule a reconnect attempt after a backoff, then dispatch RECONNECT_ATTEMPT. */
|
||||
| { type: "scheduleReconnect"; attempt: number; delayMs: number }
|
||||
/** Cancel any pending reconnect backoff timer. */
|
||||
| { type: "cancelReconnect" }
|
||||
/** Arm the degraded poll (the window's dumb timer follows the run in the DB). */
|
||||
| { type: "armPoll"; reason: PollReason }
|
||||
/** Disarm the degraded poll. */
|
||||
| { type: "disarmPoll" }
|
||||
/** POST /stop the chat's active run (authoritative detached-run stop). */
|
||||
| { type: "stopRun" }
|
||||
/** POST /stream { supersede: { runId } } — the CAS "interrupt and send now". */
|
||||
| { type: "supersede"; targetRunId: string }
|
||||
/** Abort the in-flight attach/reconnect GET controller (dispose / observer stop). */
|
||||
| { type: "abortAttach" };
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Events. An OUTCOME event MAY carry `epoch`; if it does and it does not equal
|
||||
// the current epoch, the reducer drops it (I1). Trigger events (user actions,
|
||||
// fresh disconnects) carry no epoch and are never dropped.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export type Event =
|
||||
// -- local turn --
|
||||
| { type: "SEND_LOCAL" }
|
||||
| { type: "STREAM_START"; runId?: string; epoch?: number }
|
||||
/** An OBSERVER's attached stream ended WITHOUT reaching terminal (a starved
|
||||
* clean replay, or a torn resume) — fall to the degraded poll to drive the row
|
||||
* to its real terminal state. (A live-follow drop uses FINISH_DISCONNECT.) */
|
||||
| { type: "STREAM_INCOMPLETE"; reason: PollReason; epoch?: number }
|
||||
| { type: "FINISH_CLEAN"; epoch?: number }
|
||||
| { type: "FINISH_ABORT"; epoch?: number }
|
||||
| { type: "FINISH_DISCONNECT"; hasVisibleContent: boolean; epoch?: number }
|
||||
| { type: "FINISH_ERROR"; kind: ErrorKind; epoch?: number }
|
||||
// -- mount attach (resume) --
|
||||
| { type: "ATTACH_START"; runId?: string }
|
||||
| { type: "ATTACH_LIVE"; epoch?: number }
|
||||
| { type: "ATTACH_NONE"; epoch?: number }
|
||||
// -- reconnect after a live disconnect (entered by FINISH_DISCONNECT, #488 c2) --
|
||||
| { type: "RECONNECT_ATTEMPT"; attempt: number; epoch?: number }
|
||||
| { type: "RECONNECT_ATTACHED"; epoch?: number }
|
||||
| { type: "RECONNECT_NONE"; epoch?: number }
|
||||
| { type: "RETRY" }
|
||||
// -- degraded poll --
|
||||
| { type: "POLL_TERMINAL" }
|
||||
| { type: "POLL_IDLE_CAP" }
|
||||
// -- run-fact (server-confirmed active run) --
|
||||
| { type: "RUN_FACT"; runFact: RunFact; epoch?: number }
|
||||
// -- stop --
|
||||
| { type: "STOP_REQUESTED" }
|
||||
// -- supersede (CAS) --
|
||||
| { type: "SUPERSEDE_REQUESTED"; targetRunId: string }
|
||||
| { type: "SUPERSEDE_READY"; runId?: string; epoch?: number }
|
||||
| { type: "SUPERSEDE_MISMATCH"; currentRunId?: string; epoch?: number }
|
||||
| { type: "SUPERSEDE_TIMEOUT"; epoch?: number }
|
||||
| { type: "SUPERSEDE_INVALID"; epoch?: number }
|
||||
| { type: "RUN_ALREADY_ACTIVE"; activeRunId?: string }
|
||||
// -- lifecycle --
|
||||
| { type: "DISPOSE" };
|
||||
|
||||
export const RECONNECT_MAX_ATTEMPTS = 5;
|
||||
export const RECONNECT_BASE_DELAY_MS = 1000;
|
||||
/** Backoff before attempt N (1-based): 1s, 2s, 4s, 8s, 16s. */
|
||||
export function reconnectDelayMs(attempt: number): number {
|
||||
return RECONNECT_BASE_DELAY_MS * 2 ** (attempt - 1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Constructors / helpers.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export function initialMachine(overrides?: Partial<Ctx>): Machine {
|
||||
return {
|
||||
phase: { name: "idle" },
|
||||
ctx: { epoch: 0, ownership: "local", runFact: null, liveFollow: false, ...overrides },
|
||||
effects: [],
|
||||
};
|
||||
}
|
||||
|
||||
/** Build a machine result: a phase, optional ctx patch, and effects. Empty
|
||||
* effects by default. Never mutates the input. */
|
||||
function to(
|
||||
m: Machine,
|
||||
phase: Phase,
|
||||
opts?: { ctx?: Partial<Ctx>; effects?: Effect[] },
|
||||
): Machine {
|
||||
return {
|
||||
phase,
|
||||
ctx: { ...m.ctx, ...(opts?.ctx ?? {}) },
|
||||
effects: opts?.effects ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/** No transition: keep the phase, clear effects (so a re-run does not re-fire). */
|
||||
function stay(m: Machine): Machine {
|
||||
return { phase: m.phase, ctx: m.ctx, effects: [] };
|
||||
}
|
||||
|
||||
/** A command-transition: same as `to` but bumps the epoch (I1). Any outcome
|
||||
* event issued under the old epoch is dropped once this lands. */
|
||||
function command(
|
||||
m: Machine,
|
||||
phase: Phase,
|
||||
effects: Effect[],
|
||||
ctx?: Partial<Ctx>,
|
||||
): Machine {
|
||||
return {
|
||||
phase,
|
||||
ctx: { ...m.ctx, ...(ctx ?? {}), epoch: m.ctx.epoch + 1 },
|
||||
effects,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The pure reducer.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** The terminal stream-finish events (one turn's stream ended). */
|
||||
function isFinishEvent(event: Event): boolean {
|
||||
return (
|
||||
event.type === "FINISH_ABORT" ||
|
||||
event.type === "FINISH_CLEAN" ||
|
||||
event.type === "FINISH_DISCONNECT" ||
|
||||
event.type === "FINISH_ERROR" ||
|
||||
event.type === "STREAM_INCOMPLETE"
|
||||
);
|
||||
}
|
||||
|
||||
export function reduce(m: Machine, event: Event): Machine {
|
||||
// MEDIUM (#488 re-review): honor ANY stream finish in `stopping` regardless of
|
||||
// generation. A plain user Stop has NO successor stream — the aborted stream's
|
||||
// finish IS the expected end of the stop, so exit `stopping -> idle` by that DATA
|
||||
// (I4). The epoch filter below must NOT drop it: STOP_REQUESTED bumped the epoch,
|
||||
// but the finish carries the PRE-stop generation (the runtime stamps it with the
|
||||
// stream's start epoch), so I1 would otherwise strand the machine in `stopping`
|
||||
// forever (no idle-cap covers `stopping`). The epoch filter stays in force for
|
||||
// `superseding` (a successor B owns) — that is the F1 supersede drop.
|
||||
if (m.phase.name === "stopping" && isFinishEvent(event)) {
|
||||
return to(m, { name: "idle" }, {
|
||||
// Reset ownership to local on this terminal transition (review #2): otherwise
|
||||
// an observer-stop leaves ownership 'observer' and hides "Send now" forever.
|
||||
ctx: { runFact: null, liveFollow: false, ownership: "local" },
|
||||
effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }],
|
||||
});
|
||||
}
|
||||
|
||||
// I1: drop a stale outcome (an event issued under a superseded epoch).
|
||||
if ("epoch" in event && event.epoch !== undefined && event.epoch !== m.ctx.epoch) {
|
||||
return stay(m);
|
||||
}
|
||||
|
||||
switch (event.type) {
|
||||
// ---- local turn ----------------------------------------------------
|
||||
case "SEND_LOCAL":
|
||||
// A local send owns the view: leave any recovery, become the local
|
||||
// streamer, disarm poll/reconnect. epoch++ so a late recovery outcome
|
||||
// from the previous phase is dropped.
|
||||
return command(
|
||||
m,
|
||||
{ name: "sending" },
|
||||
[{ type: "cancelReconnect" }, { type: "disarmPoll" }],
|
||||
{ ownership: "local", liveFollow: false },
|
||||
);
|
||||
|
||||
case "STREAM_INCOMPLETE":
|
||||
// An OBSERVER's attached stream ended incomplete (starved / torn) — follow
|
||||
// the run to terminal via the degraded poll.
|
||||
return to(m, { name: "polling", reason: event.reason }, {
|
||||
effects: [{ type: "armPoll", reason: event.reason }],
|
||||
});
|
||||
|
||||
case "STREAM_START": {
|
||||
// First frame arrived. Adopt the run-fact runId if present. sending ->
|
||||
// streaming; a reconnect/attach that just went live also lands here.
|
||||
const runFact = event.runId ? { runId: event.runId } : m.ctx.runFact;
|
||||
return to(m, { name: "streaming" }, {
|
||||
ctx: { runFact },
|
||||
effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }],
|
||||
});
|
||||
}
|
||||
|
||||
case "FINISH_CLEAN":
|
||||
// A clean terminal outcome. The run is done — clear the run-fact and go
|
||||
// idle. (The queue flush is a component concern gated by ownership; the
|
||||
// FSM only models the phase.) Review #2: reset ownership to local so a
|
||||
// just-finished observer-attach turn re-exposes "Send now" for the queue.
|
||||
return to(m, { name: "idle" }, {
|
||||
ctx: { runFact: null, liveFollow: false, ownership: "local" },
|
||||
effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }],
|
||||
});
|
||||
|
||||
case "FINISH_ABORT":
|
||||
// A user Stop / intentional abort finished. If we were stopping, the
|
||||
// terminal data has now arrived (I4) — go idle. The run-fact is cleared.
|
||||
return to(m, { name: "idle" }, {
|
||||
ctx: { runFact: null, liveFollow: false, ownership: "local" },
|
||||
effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }],
|
||||
});
|
||||
|
||||
case "FINISH_DISCONNECT":
|
||||
// A LIVE SSE drop. Recovery depends on WHO we are (I2 + liveFollow):
|
||||
// - a mount-attach OBSERVER (a one-shot resume, NOT live-follow) that drops
|
||||
// -> the degraded poll drives the row to terminal from the DB.
|
||||
if (m.ctx.ownership === "observer" && !m.ctx.liveFollow) {
|
||||
return to(m, { name: "polling", reason: "disconnect-visible" }, {
|
||||
effects: [{ type: "armPoll", reason: "disconnect-visible" }],
|
||||
});
|
||||
}
|
||||
// - a LOCAL live turn (first drop) OR a live-follow re-attach (a SUBSEQUENT
|
||||
// drop) -> (re-)enter the reconnect ladder. #488 commit 3: allowed
|
||||
// REPEATEDLY — `liveFollow` is kept across a successful re-attach, so the
|
||||
// second break reconnects again instead of falling to silent poll.
|
||||
// #488 commit 2: gated on the RUN-FACT (or an existing live-follow), NOT on
|
||||
// the presence of an assistant message — a setup-phase break still recovers.
|
||||
// - visible content already on screen -> keep it, ALSO poll to terminal
|
||||
// (a full replay could clobber the fuller live tail);
|
||||
// - no visible content -> the reconnect ladder rebuilds it.
|
||||
if (m.ctx.runFact || m.ctx.liveFollow) {
|
||||
const effects: Effect[] = [
|
||||
{ type: "scheduleReconnect", attempt: 1, delayMs: reconnectDelayMs(1) },
|
||||
];
|
||||
if (event.hasVisibleContent) effects.push({ type: "armPoll", reason: "disconnect-visible" });
|
||||
return command(m, { name: "reconnecting", attempt: 1, failed: false }, effects, {
|
||||
ownership: "observer",
|
||||
liveFollow: true,
|
||||
});
|
||||
}
|
||||
// No run to recover: a plain disconnect. Surface the terminal notice.
|
||||
return to(m, { name: "idle" }, {
|
||||
ctx: { runFact: null, liveFollow: false, ownership: "local" },
|
||||
});
|
||||
|
||||
case "FINISH_ERROR":
|
||||
return to(m, { name: "error", kind: event.kind }, {
|
||||
ctx: { runFact: null, liveFollow: false, ownership: "local" },
|
||||
effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }],
|
||||
});
|
||||
|
||||
// ---- mount attach (resume) ----------------------------------------
|
||||
case "ATTACH_START":
|
||||
// A reopened tab attaches to a still-running run: observer ownership.
|
||||
// #488 F2: ONLY from idle. The mount `getRun` round-trip resolves async, and
|
||||
// a local send may have started meanwhile (phase `sending`, ownership local);
|
||||
// a late ATTACH_START must NOT hijack that local turn into an observer-attach
|
||||
// (queue would stop flushing, "Send now" would hide). Guarding in the reducer
|
||||
// covers every dispatch source.
|
||||
if (m.phase.name !== "idle") return stay(m);
|
||||
return command(m, { name: "attaching" }, [{ type: "resumeStream" }], {
|
||||
ownership: "observer",
|
||||
runFact: event.runId ? { runId: event.runId } : m.ctx.runFact,
|
||||
});
|
||||
|
||||
case "ATTACH_LIVE":
|
||||
// The attach GET returned a live 2xx stream — follow it as an observer.
|
||||
// Review #1: guard by SOURCE phase. The epoch filter alone is not enough — a
|
||||
// POLL_TERMINAL uses to() (no epoch bump) and does not abort the in-flight
|
||||
// GET, so a slow 2xx landing after the machine already left `attaching` (e.g.
|
||||
// the armed poll saw the terminal row -> idle) would resurrect a settled run
|
||||
// into a phantom `streaming`. Only enter streaming FROM `attaching`.
|
||||
if (m.phase.name !== "attaching") return stay(m);
|
||||
return to(m, { name: "streaming" });
|
||||
|
||||
case "ATTACH_NONE":
|
||||
// 204 / non-2xx / throw: nothing live to attach. Arm the degraded poll to
|
||||
// follow the run to terminal from the DB. This is a soft-negative run-fact
|
||||
// (204 on a non-stripped path is authoritative-negative; the runtime may
|
||||
// pass a RUN_FACT null separately). Keep the run-fact as-is here.
|
||||
// Review #1: guard by source phase for consistency (a late outcome after the
|
||||
// machine already left `attaching` must not re-arm a poll).
|
||||
if (m.phase.name !== "attaching") return stay(m);
|
||||
return to(m, { name: "polling", reason: "attach-none" }, {
|
||||
effects: [{ type: "armPoll", reason: "attach-none" }],
|
||||
});
|
||||
|
||||
// ---- reconnect after a live disconnect ----------------------------
|
||||
case "RECONNECT_ATTEMPT":
|
||||
// A scheduled backoff fired — fire the attach GET. epoch++ so the previous
|
||||
// attempt's late outcome cannot drive this one.
|
||||
if (m.phase.name !== "reconnecting") return stay(m);
|
||||
return command(
|
||||
m,
|
||||
{ name: "reconnecting", attempt: event.attempt, failed: false },
|
||||
[{ type: "resumeStream" }],
|
||||
);
|
||||
|
||||
case "RECONNECT_ATTACHED":
|
||||
// #488 commit 3: a live re-attach succeeded. Reset to streaming — the
|
||||
// attempt counter is dropped, so a LATER disconnect can start a fresh
|
||||
// ladder from attempt 1 (the old one-shot `!wasResumed` gate forbade a
|
||||
// second cycle, sending the second break to silent poll).
|
||||
// Review #1: guard by SOURCE phase. The armed degraded poll can reach the
|
||||
// terminal row (POLL_TERMINAL -> idle, via to(), NO epoch bump, GET not
|
||||
// aborted) BEFORE a slow reconnect GET returns 2xx; without this guard that
|
||||
// late RECONNECT_ATTACHED (same epoch) would resurrect a settled run into a
|
||||
// phantom `streaming`. Only re-enter streaming FROM `reconnecting`.
|
||||
if (m.phase.name !== "reconnecting") return stay(m);
|
||||
return to(m, { name: "streaming" }, {
|
||||
effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }],
|
||||
});
|
||||
|
||||
case "RECONNECT_NONE": {
|
||||
// 204 / error during a reconnect attempt. Arm the degraded poll as the
|
||||
// belt-and-suspenders fallback, then either back off to the next attempt
|
||||
// or, at the cap, surface the manual Retry ("failed").
|
||||
if (m.phase.name !== "reconnecting") return stay(m);
|
||||
const attempt = m.phase.attempt;
|
||||
if (attempt < RECONNECT_MAX_ATTEMPTS) {
|
||||
return command(
|
||||
m,
|
||||
{ name: "reconnecting", attempt: attempt + 1, failed: false },
|
||||
[
|
||||
{ type: "armPoll", reason: "attach-none" },
|
||||
{ type: "scheduleReconnect", attempt: attempt + 1, delayMs: reconnectDelayMs(attempt + 1) },
|
||||
],
|
||||
);
|
||||
}
|
||||
return to(m, { name: "reconnecting", attempt, failed: true }, {
|
||||
effects: [{ type: "armPoll", reason: "reconnect-exhausted" }],
|
||||
});
|
||||
}
|
||||
|
||||
case "RETRY":
|
||||
// Manual Retry from the "failed" reconnect banner OR the stalled banner.
|
||||
if (m.phase.name === "reconnecting" && m.phase.failed) {
|
||||
return command(
|
||||
m,
|
||||
{ name: "reconnecting", attempt: 1, failed: false },
|
||||
[{ type: "resumeStream" }],
|
||||
);
|
||||
}
|
||||
if (m.phase.name === "stalled") {
|
||||
// Re-arm the poll to try to catch the run up again.
|
||||
return command(m, { name: "polling", reason: "attach-none" }, [
|
||||
{ type: "armPoll", reason: "attach-none" },
|
||||
]);
|
||||
}
|
||||
return stay(m);
|
||||
|
||||
// ---- degraded poll -------------------------------------------------
|
||||
case "POLL_TERMINAL":
|
||||
// The run reached a terminal row via the poll (or the reconcile merge). Go
|
||||
// idle and disarm everything (I4: this is a DATA-driven exit, incl. exit
|
||||
// from `stopping`). Review #2: reset ownership to local.
|
||||
return to(m, { name: "idle" }, {
|
||||
ctx: { runFact: null, liveFollow: false, ownership: "local" },
|
||||
effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }],
|
||||
});
|
||||
|
||||
case "POLL_IDLE_CAP":
|
||||
// Review #4: `stopping` also arms the poll (STOP_REQUESTED) but has NO other
|
||||
// backstop — an observer-stop with no SDK stream to fire onFinish, whose
|
||||
// server stop never drives the run terminal, would poll the DB forever. Give
|
||||
// it a bounded exit: cap -> idle + disarm (NOT `stalled`; Stop was already
|
||||
// pressed, so there is nothing for the user to retry).
|
||||
if (m.phase.name === "stopping") {
|
||||
return to(m, { name: "idle" }, {
|
||||
ctx: { runFact: null, liveFollow: false, ownership: "local" },
|
||||
effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }],
|
||||
});
|
||||
}
|
||||
// #488 commit 4a: the poll hit the inactivity cap. Instead of going SILENT
|
||||
// (the old "forever half-done answer"), surface a stalled banner + Retry.
|
||||
if (m.phase.name !== "polling" && m.phase.name !== "reconnecting") return stay(m);
|
||||
return to(m, { name: "stalled" }, {
|
||||
effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }],
|
||||
});
|
||||
|
||||
// ---- run-fact ------------------------------------------------------
|
||||
case "RUN_FACT": {
|
||||
const runFact = event.runFact;
|
||||
// A fresh NEGATIVE fact (no active run) cancels recovery immediately (I3):
|
||||
// there is nothing to reconnect to / poll for.
|
||||
if (!runFact) {
|
||||
if (
|
||||
m.phase.name === "reconnecting" ||
|
||||
m.phase.name === "attaching" ||
|
||||
m.phase.name === "polling" ||
|
||||
m.phase.name === "stopping"
|
||||
) {
|
||||
return to(m, { name: "idle" }, {
|
||||
// Review #2: reset ownership to local on this terminal transition.
|
||||
ctx: { runFact: null, liveFollow: false, ownership: "local" },
|
||||
effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }],
|
||||
});
|
||||
}
|
||||
return to(m, m.phase, { ctx: { runFact: null } });
|
||||
}
|
||||
// A positive fact just updates the context (pessimism toward an attempt: a
|
||||
// stale-but-positive fact permits entering recovery; a 204 will cut it).
|
||||
return to(m, m.phase, { ctx: { runFact } });
|
||||
}
|
||||
|
||||
// ---- stop ----------------------------------------------------------
|
||||
case "STOP_REQUESTED":
|
||||
// Authoritative stop of a detached run. Enter `stopping` and fire stopRun +
|
||||
// abort the local/attach reader. ALSO arm the poll so the terminal row is
|
||||
// observed — the exit is by DATA (I4: a terminal row / negative run-fact),
|
||||
// never by the stopRun HTTP response (which returns after abort, before
|
||||
// finalization). For a local turn the aborted stream's onFinish (ANY finish)
|
||||
// is HONORED in `stopping` at the top of reduce() — regardless of generation
|
||||
// — and exits to idle; the armed poll is the fallback for an observer stop
|
||||
// with no local onFinish.
|
||||
return command(
|
||||
m,
|
||||
{ name: "stopping" },
|
||||
[
|
||||
{ type: "stopRun" },
|
||||
{ type: "abortAttach" },
|
||||
{ type: "cancelReconnect" },
|
||||
{ type: "armPoll", reason: "attach-none" },
|
||||
],
|
||||
);
|
||||
|
||||
// ---- supersede (CAS) ----------------------------------------------
|
||||
case "SUPERSEDE_REQUESTED":
|
||||
// "Interrupt and send now": CAS POST /stream { supersede }. epoch++ so a
|
||||
// late outcome of the interrupted run is dropped.
|
||||
return command(
|
||||
m,
|
||||
{ name: "superseding" },
|
||||
[{ type: "supersede", targetRunId: event.targetRunId }, { type: "cancelReconnect" }, { type: "disarmPoll" }],
|
||||
);
|
||||
|
||||
case "SUPERSEDE_READY": {
|
||||
// CAS succeeded (old run stopped/settled, slot taken, new run begun). We
|
||||
// are now the local streamer of the NEW run. Adopt its runId if provided.
|
||||
const runFact = event.runId ? { runId: event.runId } : m.ctx.runFact;
|
||||
return to(m, { name: "streaming" }, {
|
||||
ctx: { ownership: "local", runFact, liveFollow: false },
|
||||
});
|
||||
}
|
||||
|
||||
case "SUPERSEDE_MISMATCH":
|
||||
// The active run moved between the click and the CAS. Per the spec: verify
|
||||
// via /run rather than blindly banner — the mismatch may be our own already-
|
||||
// superseded run. Surface a classified error AND fire a run-fact verify.
|
||||
return to(m, { name: "error", kind: "supersede-mismatch" }, {
|
||||
ctx: { runFact: event.currentRunId ? { runId: event.currentRunId } : m.ctx.runFact },
|
||||
effects: [{ type: "postRun", reason: "verify" }],
|
||||
});
|
||||
|
||||
case "SUPERSEDE_TIMEOUT":
|
||||
// The old run did not settle within W. Nothing persisted; the composer keeps
|
||||
// its text. Classified error, NO auto-retry (the old client retry ladder is
|
||||
// removed in #488 commit 5).
|
||||
return to(m, { name: "error", kind: "supersede-timeout" });
|
||||
|
||||
case "SUPERSEDE_INVALID":
|
||||
return to(m, { name: "error", kind: "supersede-invalid" });
|
||||
|
||||
case "RUN_ALREADY_ACTIVE":
|
||||
// A plain POST hit the one-active-run gate. NO auto-retry — the composer
|
||||
// offers "interrupt and send" (supersede) instead. #497/S4: adopt the
|
||||
// server's activeRunId as the run-fact so that supersede can TARGET the
|
||||
// (possibly foreign-tab) active run via the CAS, rather than a blind
|
||||
// promote+abort that just 409s again. A stale/absent id keeps the prior fact.
|
||||
return to(m, { name: "error", kind: "run-already-active" }, {
|
||||
ctx: { runFact: event.activeRunId ? { runId: event.activeRunId } : m.ctx.runFact },
|
||||
});
|
||||
|
||||
// ---- lifecycle -----------------------------------------------------
|
||||
case "DISPOSE":
|
||||
// Unmount: abort in-flight controllers, drop timers, and bump the epoch so
|
||||
// NO late callback can drive this (now dead) machine (I5).
|
||||
return command(
|
||||
m,
|
||||
{ name: "idle" },
|
||||
[
|
||||
{ type: "abortAttach" },
|
||||
{ type: "cancelReconnect" },
|
||||
{ type: "disarmPoll" },
|
||||
],
|
||||
{ liveFollow: false },
|
||||
);
|
||||
|
||||
default: {
|
||||
// Exhaustiveness guard.
|
||||
const _never: never = event;
|
||||
void _never;
|
||||
return stay(m);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -181,6 +181,12 @@ export interface IAiChatMessageRow {
|
||||
toolCalls?: unknown;
|
||||
metadata?: {
|
||||
parts?: UIMessage["parts"];
|
||||
// #491 step-alignment anchor: the count of FINISHED steps whose parts are in
|
||||
// THIS row, written atomically with `parts` server-side (flushAssistant). The
|
||||
// resume client reads it as its persisted step frontier N — the tail-only
|
||||
// attach asks the run-stream registry for the frames of step N onward (the
|
||||
// seed already carries steps 0..N-1). Absent on pre-#491 rows -> read as 0.
|
||||
stepsPersisted?: number;
|
||||
// AI SDK v6 `totalUsage` persisted on assistant rows. Legacy cumulative
|
||||
// figure (sum of every step's usage for the turn); kept for back-compat and
|
||||
// as the fallback for older rows that have no `contextTokens`.
|
||||
|
||||
@@ -3,6 +3,7 @@ import {
|
||||
resolveAdoptedChatId,
|
||||
newlyAddedChatIds,
|
||||
extractServerChatId,
|
||||
extractRunId,
|
||||
} from "./adopt-chat-id";
|
||||
|
||||
describe("resolveAdoptedChatId", () => {
|
||||
@@ -70,3 +71,17 @@ describe("extractServerChatId", () => {
|
||||
expect(extractServerChatId(undefined)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractRunId", () => {
|
||||
it("reads a string runId from the start metadata", () => {
|
||||
expect(extractRunId({ metadata: { runId: "run-1" } })).toBe("run-1");
|
||||
});
|
||||
it("returns undefined when runId is absent", () => {
|
||||
expect(extractRunId({ metadata: { chatId: "c" } })).toBeUndefined();
|
||||
expect(extractRunId({})).toBeUndefined();
|
||||
expect(extractRunId(undefined)).toBeUndefined();
|
||||
});
|
||||
it("returns undefined for a non-string runId", () => {
|
||||
expect(extractRunId({ metadata: { runId: 7 } })).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -56,6 +56,20 @@ export function extractServerChatId(
|
||||
return typeof m?.chatId === "string" ? m.chatId : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* #488: read the authoritative RUN id off a streaming assistant message. The
|
||||
* server attaches it as `message.metadata.runId` on the `start` part when a run
|
||||
* wraps the turn (see server `chatStreamMetadata`, #184/#487). This is the live
|
||||
* run-fact update the client FSM adopts (mirrors `extractServerChatId`). Returns
|
||||
* it only when it is a string; undefined otherwise.
|
||||
*/
|
||||
export function extractRunId(
|
||||
message: { metadata?: unknown } | undefined,
|
||||
): string | undefined {
|
||||
const m = message?.metadata as { runId?: string } | undefined;
|
||||
return typeof m?.runId === "string" ? m.runId : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The deduped set of ids present in `afterIds` but not in `beforeIds`. A
|
||||
* paginated/flatMapped list can repeat the same id, so dedupe: one genuinely-new
|
||||
|
||||
@@ -6,10 +6,13 @@ describe("estimateTokens", () => {
|
||||
expect(estimateTokens("")).toBe(0);
|
||||
});
|
||||
|
||||
it("ceils chars/4 so any non-empty text is at least 1 token", () => {
|
||||
// #490: migrated onto the shared @docmost/token-estimate module (chars/2.5, up
|
||||
// from the old client-only chars/4) so the client counter and the server replay
|
||||
// budgeter can never diverge.
|
||||
it("ceils chars/2.5 so any non-empty text is at least 1 token", () => {
|
||||
expect(estimateTokens("a")).toBe(1);
|
||||
expect(estimateTokens("abcd")).toBe(1);
|
||||
expect(estimateTokens("abcde")).toBe(2);
|
||||
expect(estimateTokens("12345678")).toBe(2);
|
||||
expect(estimateTokens("ab")).toBe(1);
|
||||
expect(estimateTokens("abcde")).toBe(2); // 5 / 2.5 = 2
|
||||
expect(estimateTokens("x".repeat(10))).toBe(4); // 10 / 2.5 = 4
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,18 +2,10 @@
|
||||
* Rough client-side token estimation for AI-chat UI affordances.
|
||||
*
|
||||
* No provider streams exact per-token usage mid-stream, so any in-flight figure
|
||||
* is a CLIENT ESTIMATE (chars/≈4 heuristic). Pure + unit-testable: it never runs
|
||||
* a real BPE tokenizer (that would be O(n²) on the hot path, bloat the bundle,
|
||||
* and be wrong for Gemini/Ollama anyway). Used by the in-body reasoning counter
|
||||
* ("Thinking · N tokens").
|
||||
* is a CLIENT ESTIMATE. This re-exports the SHARED estimator from
|
||||
* `@docmost/token-estimate` (chars/2.5) so the in-body counter and the server's
|
||||
* replay budgeter use the SAME heuristic — two divergent estimators would mean
|
||||
* "the badge shows 60%" while "the budgeter already trimmed" (#490). Used by the
|
||||
* in-body reasoning counter ("Thinking · N tokens").
|
||||
*/
|
||||
|
||||
/**
|
||||
* Rough token estimate for a piece of text using the standard chars/≈4 heuristic.
|
||||
* Returns 0 for empty/whitespace-free-of-content input, and ceils so any
|
||||
* non-empty text counts as at least one token.
|
||||
*/
|
||||
export function estimateTokens(text: string): number {
|
||||
if (!text) return 0;
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
export { estimateTokens } from "@docmost/token-estimate";
|
||||
|
||||
@@ -42,6 +42,53 @@ describe("describeChatError", () => {
|
||||
);
|
||||
});
|
||||
|
||||
// #488 commit 5: the #487 concurrency-gate / supersede 409s. FULL real bodies:
|
||||
// a ConflictException(object) whose response is serialized verbatim, carrying a
|
||||
// `code` and statusCode 409. Each must classify to a human text, not raw JSON.
|
||||
it("classifies A_RUN_ALREADY_ACTIVE (409) as already-answering, not raw JSON", () => {
|
||||
const body =
|
||||
'{"message":"A run is already active for this chat","code":"A_RUN_ALREADY_ACTIVE","statusCode":409}';
|
||||
expect(describeChatError(body, t).title).toBe(
|
||||
"The agent is already answering",
|
||||
);
|
||||
// Never leaks the raw code as the detail.
|
||||
expect(describeChatError(body, t).detail).not.toContain("A_RUN_ALREADY_ACTIVE");
|
||||
});
|
||||
|
||||
it("classifies SUPERSEDE_TARGET_MISMATCH (409) as run-changed", () => {
|
||||
// Real server body shape: the current run id is `activeRunId` (NOT `runId`) —
|
||||
// see ai-chat.controller.ts. describeChatError classifies off `code` only.
|
||||
const body =
|
||||
'{"message":"active run does not match the supersede target","code":"SUPERSEDE_TARGET_MISMATCH","activeRunId":"run-x","statusCode":409}';
|
||||
expect(describeChatError(body, t).title).toBe(
|
||||
"Couldn't interrupt — the run changed",
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies SUPERSEDE_TIMEOUT (409) as couldn't-interrupt-in-time", () => {
|
||||
const body =
|
||||
'{"message":"the run did not settle within the supersede window","code":"SUPERSEDE_TIMEOUT","statusCode":409}';
|
||||
expect(describeChatError(body, t).title).toBe("Couldn't interrupt in time");
|
||||
});
|
||||
|
||||
it("classifies SUPERSEDE_INVALID (409) as couldn't-interrupt-that-run", () => {
|
||||
const body =
|
||||
'{"message":"supervise requires chatId","code":"SUPERSEDE_INVALID","statusCode":409}';
|
||||
expect(describeChatError(body, t).title).toBe(
|
||||
"Couldn't interrupt that run",
|
||||
);
|
||||
});
|
||||
|
||||
it("ORDER GUARD: A_RUN_ALREADY_ACTIVE wins over any generic status branch", () => {
|
||||
// Even though the body could superficially look 4xx-ish, the code branch runs
|
||||
// first, so it is never mislabeled by a generic status heading.
|
||||
const body =
|
||||
'{"message":"conflict","code":"A_RUN_ALREADY_ACTIVE","statusCode":409}';
|
||||
const view = describeChatError(body, t);
|
||||
expect(view.title).not.toBe("Something went wrong");
|
||||
expect(view.title).not.toBe("AI provider not configured");
|
||||
});
|
||||
|
||||
it("classifies a dropped connection (ECONNRESET) as a lost-connection error", () => {
|
||||
expect(
|
||||
describeChatError("Cannot connect to API: read ECONNRESET", t).title,
|
||||
|
||||
@@ -39,6 +39,44 @@ export function describeChatError(
|
||||
};
|
||||
}
|
||||
|
||||
// #488 commit 5: the #487 concurrency-gate / supersede 409s. These arrive as a
|
||||
// ConflictException(object) body carrying a `code` (and statusCode 409). They
|
||||
// MUST be classified by `code` STRICTLY BEFORE any generic status branch, or the
|
||||
// user sees the raw JSON `{"code":"A_RUN_ALREADY_ACTIVE",…}`. The code strings
|
||||
// are the real #487 server contract (ai-chat.controller.ts) — do not invent.
|
||||
if (/"code"\s*:\s*"A_RUN_ALREADY_ACTIVE"/.test(msg)) {
|
||||
return {
|
||||
title: t("The agent is already answering"),
|
||||
detail: t(
|
||||
"This chat already has a run in progress. Wait for it to finish, or interrupt it and send now.",
|
||||
),
|
||||
};
|
||||
}
|
||||
if (/"code"\s*:\s*"SUPERSEDE_TARGET_MISMATCH"/.test(msg)) {
|
||||
return {
|
||||
title: t("Couldn't interrupt — the run changed"),
|
||||
detail: t(
|
||||
"The run you tried to interrupt is no longer the active one. Check the latest answer and try again.",
|
||||
),
|
||||
};
|
||||
}
|
||||
if (/"code"\s*:\s*"SUPERSEDE_TIMEOUT"/.test(msg)) {
|
||||
return {
|
||||
title: t("Couldn't interrupt in time"),
|
||||
detail: t(
|
||||
"The previous run didn't stop in time. Nothing was sent — try sending again.",
|
||||
),
|
||||
};
|
||||
}
|
||||
if (/"code"\s*:\s*"SUPERSEDE_INVALID"/.test(msg)) {
|
||||
return {
|
||||
title: t("Couldn't interrupt that run"),
|
||||
detail: t(
|
||||
"The run to interrupt doesn't belong to this chat. Reload and try again.",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (/"statusCode"\s*:\s*403\b/.test(msg)) {
|
||||
return {
|
||||
title: t("AI chat is disabled"),
|
||||
|
||||
@@ -4,7 +4,8 @@ import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.t
|
||||
import {
|
||||
isStreamingTail,
|
||||
isSettledAssistantTail,
|
||||
seedRows,
|
||||
stepsPersistedOf,
|
||||
mergeDeltaRowsIntoPages,
|
||||
mergeById,
|
||||
} from "./resume-helpers.ts";
|
||||
|
||||
@@ -12,8 +13,18 @@ function row(
|
||||
id: string,
|
||||
role: string,
|
||||
status?: string,
|
||||
stepsPersisted?: number,
|
||||
): IAiChatMessageRow {
|
||||
return { id, role, content: "", status, createdAt: "2026-01-01T00:00:00Z" };
|
||||
return {
|
||||
id,
|
||||
role,
|
||||
content: "",
|
||||
status,
|
||||
createdAt: "2026-01-01T00:00:00Z",
|
||||
...(stepsPersisted !== undefined
|
||||
? { metadata: { stepsPersisted } }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function makeMsg(id: string, text: string): UIMessage {
|
||||
@@ -65,23 +76,92 @@ describe("isSettledAssistantTail", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("seedRows", () => {
|
||||
const rows = [row("u1", "user"), row("a1", "assistant", "streaming")];
|
||||
|
||||
it("returns the rows unchanged when not stripping", () => {
|
||||
expect(seedRows(rows, false)).toBe(rows);
|
||||
describe("stepsPersistedOf", () => {
|
||||
it("reads metadata.stepsPersisted", () => {
|
||||
expect(stepsPersistedOf(row("a1", "assistant", "streaming", 3))).toBe(3);
|
||||
expect(stepsPersistedOf(row("a1", "assistant", "streaming", 0))).toBe(0);
|
||||
});
|
||||
|
||||
it("drops the last row when stripping", () => {
|
||||
const seeded = seedRows(rows, true);
|
||||
expect(seeded).toHaveLength(1);
|
||||
expect(seeded[0].id).toBe("u1");
|
||||
it("defaults to 0 for a pre-#491 row (absent), null/undefined, or a bad value", () => {
|
||||
expect(stepsPersistedOf(row("a1", "assistant", "streaming"))).toBe(0);
|
||||
expect(stepsPersistedOf(null)).toBe(0);
|
||||
expect(stepsPersistedOf(undefined)).toBe(0);
|
||||
expect(
|
||||
stepsPersistedOf({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
content: "",
|
||||
createdAt: "x",
|
||||
metadata: { stepsPersisted: -2 },
|
||||
}),
|
||||
).toBe(0);
|
||||
});
|
||||
|
||||
it("returns an empty list when stripping a single-row list", () => {
|
||||
expect(seedRows([row("a1", "assistant", "streaming")], true)).toHaveLength(
|
||||
0,
|
||||
);
|
||||
it("floors a non-integer count", () => {
|
||||
expect(
|
||||
stepsPersistedOf({
|
||||
id: "a1",
|
||||
role: "assistant",
|
||||
content: "",
|
||||
createdAt: "x",
|
||||
metadata: { stepsPersisted: 2.9 },
|
||||
}),
|
||||
).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("mergeDeltaRowsIntoPages", () => {
|
||||
const pages = () => [
|
||||
{ items: [row("u1", "user"), row("a1", "assistant", "streaming", 1)], meta: {} },
|
||||
];
|
||||
|
||||
it("returns the pages unchanged for an empty delta", () => {
|
||||
const p = pages();
|
||||
expect(mergeDeltaRowsIntoPages(p, [])).toBe(p);
|
||||
});
|
||||
|
||||
it("appends a genuinely new row to the last page in chronological order", () => {
|
||||
const merged = mergeDeltaRowsIntoPages(pages(), [row("a2", "assistant", "streaming", 0)]);
|
||||
expect(merged[0].items.map((i) => i.id)).toEqual(["u1", "a1", "a2"]);
|
||||
});
|
||||
|
||||
it("replaces a grown row in place (per-step growth), never appends a duplicate", () => {
|
||||
const merged = mergeDeltaRowsIntoPages(pages(), [
|
||||
row("a1", "assistant", "streaming", 2),
|
||||
]);
|
||||
expect(merged[0].items.map((i) => i.id)).toEqual(["u1", "a1"]);
|
||||
// the in-place replacement carries the grown step frontier.
|
||||
expect(stepsPersistedOf(merged[0].items[1])).toBe(2);
|
||||
});
|
||||
|
||||
it("does not mutate the input pages", () => {
|
||||
const input = pages();
|
||||
const before = input[0].items.slice();
|
||||
mergeDeltaRowsIntoPages(input, [row("a2", "assistant", "streaming", 0)]);
|
||||
expect(input[0].items).toEqual(before); // untouched
|
||||
});
|
||||
|
||||
// #491 CONTRACT: the delta overlap window re-delivers the same rows, so merging
|
||||
// MUST be idempotent — applying a delta twice equals applying it once (no growth,
|
||||
// no reorder). A regression re-introduces duplicate assistant bubbles per poll.
|
||||
it("is idempotent: applying the same delta twice equals once", () => {
|
||||
const delta = [
|
||||
row("a1", "assistant", "streaming", 2), // grown existing row
|
||||
row("a2", "assistant", "streaming", 0), // new row
|
||||
];
|
||||
const once = mergeDeltaRowsIntoPages(pages(), delta);
|
||||
const twice = mergeDeltaRowsIntoPages(once, delta);
|
||||
const thrice = mergeDeltaRowsIntoPages(twice, delta);
|
||||
expect(once[0].items.map((i) => i.id)).toEqual(["u1", "a1", "a2"]);
|
||||
expect(twice[0].items.map((i) => i.id)).toEqual(["u1", "a1", "a2"]);
|
||||
expect(twice).toEqual(once);
|
||||
expect(thrice).toEqual(once);
|
||||
});
|
||||
|
||||
it("seeds a first page when the cache is empty", () => {
|
||||
const merged = mergeDeltaRowsIntoPages([], [row("u1", "user")]);
|
||||
expect(merged).toHaveLength(1);
|
||||
expect(merged[0].items.map((i) => i.id)).toEqual(["u1"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -109,4 +189,37 @@ describe("mergeById", () => {
|
||||
expect(mergeById(prev, null)).toBe(prev);
|
||||
expect(mergeById(prev, undefined)).toBe(prev);
|
||||
});
|
||||
|
||||
// #491 CONTRACT: the delta poll's overlap window GUARANTEES the same row is
|
||||
// re-delivered across close polls, so merging must be IDEMPOTENT by id — merging
|
||||
// the same row (or an equal-length list of rows) twice must not duplicate or
|
||||
// reorder. This is the property the whole delta-poll design leans on; a
|
||||
// regression here would re-introduce duplicate assistant bubbles on every poll.
|
||||
it("is idempotent by id: re-merging the same row does not duplicate or reorder", () => {
|
||||
const seed = [makeMsg("u1", "hi"), makeMsg("a1", "step 1")];
|
||||
const repeat = makeMsg("a1", "step 1"); // the SAME row the overlap re-delivers
|
||||
const once = mergeById(seed, repeat);
|
||||
const twice = mergeById(once, repeat);
|
||||
const thrice = mergeById(twice, repeat);
|
||||
// Length is stable (no growth), order is stable (user then assistant).
|
||||
expect(once.map((m) => m.id)).toEqual(["u1", "a1"]);
|
||||
expect(twice.map((m) => m.id)).toEqual(["u1", "a1"]);
|
||||
expect(thrice.map((m) => m.id)).toEqual(["u1", "a1"]);
|
||||
// The repeated merge converges: the row is replaced in place, never appended.
|
||||
expect(twice[1]).toBe(repeat);
|
||||
});
|
||||
|
||||
it("is idempotent across a batch of repeated + grown rows (delta re-delivery)", () => {
|
||||
// A delta poll re-delivers a1 (unchanged) and a2 (grown one step). Applying the
|
||||
// batch twice must equal applying it once — the poll can re-send either.
|
||||
const start = [makeMsg("u1", "hi"), makeMsg("a1", "done")];
|
||||
const batch = [makeMsg("a1", "done"), makeMsg("a2", "grown step 2")];
|
||||
const apply = (list: typeof start) =>
|
||||
batch.reduce((acc, row) => mergeById(acc, row), list);
|
||||
const once = apply(start);
|
||||
const twice = apply(once);
|
||||
expect(once.map((m) => m.id)).toEqual(["u1", "a1", "a2"]);
|
||||
expect(twice.map((m) => m.id)).toEqual(["u1", "a1", "a2"]);
|
||||
expect(twice).toEqual(once);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -11,9 +11,10 @@ import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.t
|
||||
|
||||
/**
|
||||
* A STREAMING tail: the last persisted row is an assistant row still marked
|
||||
* `status === 'streaming'`. Such a tail is stripped from the seed and rebuilt by
|
||||
* the replay (`expect=live`), since the SDK's `text-start` always pushes a new
|
||||
* part and replaying over a seeded in-progress row would duplicate its text.
|
||||
* `status === 'streaming'`. #491 (tail-only): such a tail is seeded UNCHANGED —
|
||||
* it carries the persisted steps 0..N-1 — and the run-stream registry's tail
|
||||
* (frames for steps >= N) is APPENDED to it by the SDK's `readUIMessageStream`
|
||||
* continuation. Only the presence of this tail decides WHETHER to attach.
|
||||
*/
|
||||
export function isStreamingTail(rows: IAiChatMessageRow[]): boolean {
|
||||
const tail = rows[rows.length - 1];
|
||||
@@ -32,15 +33,61 @@ export function isSettledAssistantTail(rows: IAiChatMessageRow[]): boolean {
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed rows for `useChat`: return the rows unchanged, or without the last row when
|
||||
* `strip` is set (the streaming tail is stripped so the live replay rebuilds it
|
||||
* without duplicating parts).
|
||||
* #491 tail-only anchor: the count of FINISHED steps whose parts are persisted in
|
||||
* THIS assistant row (`metadata.stepsPersisted`), written atomically with `parts`
|
||||
* server-side. The resume client reads it as its persisted step frontier N — the
|
||||
* tail-only attach asks the run-stream registry for the frames of step N onward
|
||||
* (the seed already carries steps 0..N-1). Absent on pre-#491 rows => 0.
|
||||
*/
|
||||
export function seedRows(
|
||||
export function stepsPersistedOf(
|
||||
row: IAiChatMessageRow | null | undefined,
|
||||
): number {
|
||||
const n = row?.metadata?.stepsPersisted;
|
||||
return typeof n === "number" && n >= 0 ? Math.floor(n) : 0;
|
||||
}
|
||||
|
||||
/** One page of the messages infinite-query cache (`{ items, meta }`). */
|
||||
export interface IMessagePage {
|
||||
items: IAiChatMessageRow[];
|
||||
meta: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* #491 delta-poll merge: upsert the delta poll's `rows` into the messages
|
||||
* infinite-query page structure IDEMPOTENTLY by id. The delta endpoint's overlap
|
||||
* window GUARANTEES occasional REPEATS, so this MUST converge: a row already
|
||||
* present is REPLACED IN PLACE (per-step growth of an in-progress row), a new row
|
||||
* is APPENDED to the last page in chronological order (the server returns delta
|
||||
* rows oldest-first). Applying the same delta twice equals applying it once. Never
|
||||
* mutates the input pages (returns fresh page objects with cloned item arrays).
|
||||
*/
|
||||
export function mergeDeltaRowsIntoPages(
|
||||
pages: IMessagePage[],
|
||||
rows: IAiChatMessageRow[],
|
||||
strip: boolean,
|
||||
): IAiChatMessageRow[] {
|
||||
return strip ? rows.slice(0, -1) : rows;
|
||||
): IMessagePage[] {
|
||||
if (rows.length === 0) return pages;
|
||||
const next: IMessagePage[] = pages.map((p) => ({
|
||||
...p,
|
||||
items: p.items.slice(),
|
||||
}));
|
||||
const locate = (id: string): [number, number] | null => {
|
||||
for (let pi = 0; pi < next.length; pi++) {
|
||||
const ii = next[pi].items.findIndex((it) => it.id === id);
|
||||
if (ii !== -1) return [pi, ii];
|
||||
}
|
||||
return null;
|
||||
};
|
||||
for (const row of rows) {
|
||||
const at = locate(row.id);
|
||||
if (at) {
|
||||
next[at[0]].items[at[1]] = row; // replace in place — idempotent by id
|
||||
} else if (next.length > 0) {
|
||||
next[next.length - 1].items.push(row); // append chronologically
|
||||
} else {
|
||||
next.push({ items: [row], meta: undefined });
|
||||
}
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readUIMessageStream, type UIMessage } from "ai";
|
||||
import pkg from "../../../../package.json";
|
||||
|
||||
/**
|
||||
* PIN-SPEC TRIP-WIRE (#491). The tail-only attach continuation relies on THREE
|
||||
* behaviors of `ai@6.0.207`, verified line-by-line in the issue. Without this
|
||||
* test, an `ai` bump could silently break attach (the client would append the
|
||||
* live tail to the wrong message, or duplicate a step):
|
||||
*
|
||||
* 1. `readUIMessageStream({ message })` CONTINUES the passed message — it does
|
||||
* not start a fresh one — so the tail streamed after a re-seed is appended to
|
||||
* the seeded assistant row (the same DB id).
|
||||
* 2. A `start` frame does NOT reset the existing message's parts (so the seeded
|
||||
* steps 0..N-1 survive; the synthetic `start` the registry prepends only
|
||||
* carries the run-fact metadata).
|
||||
* 3. Text parts do NOT cross a `finish-step` boundary — a new `text-start` after
|
||||
* `finish-step` is a NEW part — so the reconstructed steps stay separated and
|
||||
* the step frontier stays meaningful.
|
||||
*
|
||||
* If an `ai` upgrade changes any of these, this test fails LOUD instead of the
|
||||
* resume path silently corrupting.
|
||||
*/
|
||||
describe("ai SDK continuation trip-wire (#491, tail-only attach)", () => {
|
||||
it("is pinned to the exact ai version the continuation was verified against", () => {
|
||||
// A caret/range bump is exactly what would silently break attach — require an
|
||||
// exact pin. Bumping ai MUST re-verify the behavior asserted below, then this.
|
||||
expect((pkg as { dependencies: Record<string, string> }).dependencies.ai).toBe(
|
||||
"6.0.207",
|
||||
);
|
||||
});
|
||||
|
||||
it("continues the seeded message: start does not reset parts, the tail appends as new parts", async () => {
|
||||
// A seeded assistant row with ONE finished step already reconstructed.
|
||||
const seeded: UIMessage = {
|
||||
id: "assistant-1",
|
||||
role: "assistant",
|
||||
parts: [
|
||||
{ type: "step-start" },
|
||||
{ type: "text", text: "STEP0", state: "done" },
|
||||
],
|
||||
} as UIMessage;
|
||||
|
||||
// The tail the registry delivers on re-attach: a synthetic start (run-fact),
|
||||
// then step 1's frames, then finish. As UI-message chunks (what the SSE frames
|
||||
// decode to).
|
||||
const chunks = [
|
||||
{ type: "start", messageMetadata: { runId: "r1", chatId: "c1" } },
|
||||
{ type: "start-step" },
|
||||
{ type: "text-start", id: "t1" },
|
||||
{ type: "text-delta", id: "t1", delta: "STEP1" },
|
||||
{ type: "text-end", id: "t1" },
|
||||
{ type: "finish-step" },
|
||||
{ type: "finish" },
|
||||
];
|
||||
const stream = new ReadableStream({
|
||||
start(c) {
|
||||
for (const ch of chunks) c.enqueue(ch);
|
||||
c.close();
|
||||
},
|
||||
});
|
||||
|
||||
let last: UIMessage | undefined;
|
||||
for await (const msg of readUIMessageStream({ message: seeded, stream })) {
|
||||
last = msg;
|
||||
}
|
||||
|
||||
expect(last).toBeDefined();
|
||||
// Same message id (continuation, not a fresh message).
|
||||
expect(last!.id).toBe("assistant-1");
|
||||
// The seeded step-0 parts SURVIVED the `start` frame, and step 1 was appended
|
||||
// as SEPARATE parts (text did not cross the finish-step boundary).
|
||||
const shape = last!.parts.map((p) => `${p.type}:${(p as { text?: string }).text ?? ""}`);
|
||||
expect(shape).toEqual([
|
||||
"step-start:",
|
||||
"text:STEP0",
|
||||
"step-start:",
|
||||
"text:STEP1",
|
||||
]);
|
||||
// The run-fact metadata from the synthetic start frame is applied.
|
||||
expect(last!.metadata).toMatchObject({ runId: "r1", chatId: "c1" });
|
||||
});
|
||||
});
|
||||
@@ -23,7 +23,7 @@
|
||||
"migration:reset": "tsx src/database/migrate.ts down-to NO_MIGRATIONS",
|
||||
"migration:codegen": "kysely-codegen --dialect=postgres --camel-case --env-file=../../.env --out-file=./src/database/types/db.d.ts",
|
||||
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
|
||||
"pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build",
|
||||
"pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build && pnpm --filter @docmost/token-estimate build",
|
||||
"test": "jest",
|
||||
"test:int": "jest --config test/jest-integration.json",
|
||||
"test:watch": "jest --watch",
|
||||
@@ -44,6 +44,7 @@
|
||||
"@docmost/mcp": "workspace:*",
|
||||
"@docmost/pdf-inspector": "1.9.6",
|
||||
"@docmost/prosemirror-markdown": "workspace:*",
|
||||
"@docmost/token-estimate": "workspace:*",
|
||||
"@fastify/compress": "^9.0.0",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/multipart": "^10.0.0",
|
||||
@@ -206,6 +207,7 @@
|
||||
"^@docmost/db/(.*)$": "<rootDir>/database/$1",
|
||||
"^@docmost/transactional/(.*)$": "<rootDir>/integrations/transactional/$1",
|
||||
"^@docmost/ee/(.*)$": "<rootDir>/ee/$1",
|
||||
"^@docmost/token-estimate$": "<rootDir>/../../../packages/token-estimate/src/index.ts",
|
||||
"^src/(.*)$": "<rootDir>/$1",
|
||||
"^@tiptap/react$": "<rootDir>/../test/stubs/tiptap-react.js"
|
||||
}
|
||||
|
||||
@@ -43,6 +43,9 @@ function makeRepo(overrides: Record<string, jest.Mock> = {}) {
|
||||
workspaceId: v.workspaceId,
|
||||
})),
|
||||
update: jest.fn(async () => ({ id: 'run-1' })),
|
||||
// #487: terminal finalize now goes through the CONDITIONAL write. Default
|
||||
// returns a truthy row (the run WAS active -> this call wrote it).
|
||||
finalizeIfActive: jest.fn(async () => ({ id: 'run-1', status: 'succeeded' })),
|
||||
markStopRequested: jest.fn(async () => ({ id: 'run-1' })),
|
||||
findActiveByChat: jest.fn(async () => undefined),
|
||||
findLatestByChat: jest.fn(async () => undefined),
|
||||
@@ -336,14 +339,12 @@ describe('AiChatRunService run lifecycle', () => {
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'error', 'provider blew up');
|
||||
|
||||
expect(svc.isLocallyActive('run-1')).toBe(false);
|
||||
expect(repo.update).toHaveBeenCalledWith(
|
||||
// #487: the terminal write is CONDITIONAL (finalizeIfActive); finishedAt is
|
||||
// stamped inside the repo method, so the service passes just status + error.
|
||||
expect(repo.finalizeIfActive).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'ws-1',
|
||||
expect.objectContaining({
|
||||
status: 'failed',
|
||||
error: 'provider blew up',
|
||||
finishedAt: expect.any(Date),
|
||||
}),
|
||||
expect.objectContaining({ status: 'failed', error: 'provider blew up' }),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -366,8 +367,8 @@ describe('AiChatRunService run lifecycle', () => {
|
||||
// A second settle (e.g. a streamText callback firing after the catch) no-ops.
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'completed', undefined);
|
||||
|
||||
expect(repo.update).toHaveBeenCalledTimes(1);
|
||||
expect(repo.update).toHaveBeenCalledWith(
|
||||
expect(repo.finalizeIfActive).toHaveBeenCalledTimes(1);
|
||||
expect(repo.finalizeIfActive).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'ws-1',
|
||||
expect.objectContaining({ status: 'failed', error: 'first' }),
|
||||
@@ -389,8 +390,8 @@ describe('AiChatRunService run lifecycle', () => {
|
||||
const updateGate = new Promise((res) => {
|
||||
resolveUpdate = res;
|
||||
});
|
||||
const update = jest.fn(() => updateGate);
|
||||
const repo = makeRepo({ update });
|
||||
const finalizeIfActive = jest.fn(() => updateGate);
|
||||
const repo = makeRepo({ finalizeIfActive });
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
await svc.beginRun({
|
||||
chatId: 'chat-1',
|
||||
@@ -399,23 +400,23 @@ describe('AiChatRunService run lifecycle', () => {
|
||||
});
|
||||
|
||||
// Fire both before the (pending) update resolves. The first synchronously
|
||||
// claims the entry (active.delete) and awaits update; the second, started in
|
||||
// the same macrotask, finds the entry already gone and returns at the claim
|
||||
// WITHOUT ever calling update.
|
||||
// claims the entry (active.delete) and awaits the write; the second, started
|
||||
// in the same macrotask, finds the entry already gone and returns at the claim
|
||||
// WITHOUT ever writing.
|
||||
const p1 = svc.finalizeRun('run-1', 'ws-1', 'completed');
|
||||
const p2 = svc.finalizeRun('run-1', 'ws-1', 'error', 'safety-net');
|
||||
|
||||
// The decisive assertion: exactly one caller reached the terminal UPDATE.
|
||||
expect(update).toHaveBeenCalledTimes(1);
|
||||
expect(finalizeIfActive).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Let the single in-flight update land; both calls resolve cleanly.
|
||||
resolveUpdate({ id: 'run-1' });
|
||||
resolveUpdate({ id: 'run-1', status: 'succeeded' });
|
||||
await Promise.all([p1, p2]);
|
||||
|
||||
expect(update).toHaveBeenCalledTimes(1);
|
||||
expect(finalizeIfActive).toHaveBeenCalledTimes(1);
|
||||
// The winner is the FIRST caller ('completed' -> 'succeeded'); the late
|
||||
// 'error' settle never wrote, so it could not clobber the real status.
|
||||
expect(update).toHaveBeenCalledWith(
|
||||
expect(finalizeIfActive).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'ws-1',
|
||||
expect.objectContaining({ status: 'succeeded' }),
|
||||
@@ -431,10 +432,10 @@ describe('AiChatRunService run lifecycle', () => {
|
||||
// 409s until a restart. The fix updates FIRST and retries.
|
||||
let calls = 0;
|
||||
const repo = makeRepo({
|
||||
update: jest.fn(async () => {
|
||||
finalizeIfActive: jest.fn(async () => {
|
||||
calls += 1;
|
||||
if (calls === 1) throw new Error('deadlock detected');
|
||||
return { id: 'run-1' };
|
||||
return { id: 'run-1', status: 'succeeded' };
|
||||
}),
|
||||
});
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
@@ -447,26 +448,29 @@ describe('AiChatRunService run lifecycle', () => {
|
||||
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'completed');
|
||||
|
||||
// The retry landed the terminal write: the entry is dropped (slot freed) and
|
||||
// the row carries the real terminal status — NOT stranded at 'running'.
|
||||
// The retry landed the terminal write: the entry is dropped (slot freed), no
|
||||
// zombie left, and the row carries the real terminal status.
|
||||
expect(svc.isLocallyActive('run-1')).toBe(false);
|
||||
expect(repo.update).toHaveBeenCalledTimes(2);
|
||||
expect(repo.update).toHaveBeenLastCalledWith(
|
||||
expect(svc.hasZombie('run-1')).toBe(false);
|
||||
expect(repo.finalizeIfActive).toHaveBeenCalledTimes(2);
|
||||
expect(repo.finalizeIfActive).toHaveBeenLastCalledWith(
|
||||
'run-1',
|
||||
'ws-1',
|
||||
expect.objectContaining({ status: 'succeeded' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('F6: if the terminal write keeps failing, the entry is RETAINED and a LATER settle completes it (chat not permanently 409d)', async () => {
|
||||
it('#487 give-up: if the terminal write keeps failing, finalizeRun leaves a ZOMBIE (does NOT restore the entry) and settleZombie re-drives it', async () => {
|
||||
// Worst case: the DB is down for the whole first finalize (all attempts fail).
|
||||
// The run must NOT be silently lost — the entry stays so a subsequent settle
|
||||
// (a streamText callback, requestStop -> onAbort, or a future sweep) can retry.
|
||||
// #487 changes the give-up behaviour: the entry is NOT restored (a restored
|
||||
// entry is indistinguishable from a live run). Instead a ZOMBIE record holds
|
||||
// the intended terminal status, and a re-drive (settleZombie — called by the
|
||||
// reconcile / supersede / opportunistic paths) applies it later.
|
||||
let healthy = false;
|
||||
const repo = makeRepo({
|
||||
update: jest.fn(async () => {
|
||||
finalizeIfActive: jest.fn(async () => {
|
||||
if (!healthy) throw new Error('pool exhausted');
|
||||
return { id: 'run-1' };
|
||||
return { id: 'run-1', status: 'succeeded' };
|
||||
}),
|
||||
});
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
@@ -480,35 +484,83 @@ describe('AiChatRunService run lifecycle', () => {
|
||||
userId: 'user-1',
|
||||
});
|
||||
|
||||
// First settle: every bounded attempt fails -> entry retained, NOT settled.
|
||||
// First settle: every bounded attempt fails -> ZOMBIE, entry NOT restored.
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'completed');
|
||||
expect(svc.isLocallyActive('run-1')).toBe(true);
|
||||
// F12: the give-up emits ONE explicit, greppable ERROR (run + chat context)
|
||||
// so an operator can tell "gave up, run held in memory" from a per-attempt
|
||||
// blip — distinct from the per-attempt warns.
|
||||
expect(svc.isLocallyActive('run-1')).toBe(false); // NOT a live entry
|
||||
expect(svc.hasZombie('run-1')).toBe(true);
|
||||
expect(svc.zombieRunIds()).toContain('run-1');
|
||||
// The give-up emits ONE explicit, greppable ERROR mentioning the zombie.
|
||||
const gaveUp = errorSpy.mock.calls.some(
|
||||
(c) =>
|
||||
/NON-TERMINAL/.test(String(c[0])) &&
|
||||
/ZOMBIE/.test(String(c[0])) &&
|
||||
/run-1/.test(String(c[0])) &&
|
||||
/chat-1/.test(String(c[0])),
|
||||
);
|
||||
expect(gaveUp).toBe(true);
|
||||
// The settle notifier resolved as terminalWriteFailed (a subscriber learns the
|
||||
// slot still needs the intended status applied).
|
||||
const outcome = await svc.peekSettled('run-1');
|
||||
expect(outcome).toEqual({
|
||||
status: 'succeeded',
|
||||
error: null,
|
||||
terminalWriteFailed: true,
|
||||
});
|
||||
|
||||
// The DB recovers; a later settle now succeeds and frees the slot.
|
||||
// The DB recovers; a re-drive settles the zombie via the conditional UPDATE.
|
||||
healthy = true;
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'completed');
|
||||
expect(svc.isLocallyActive('run-1')).toBe(false);
|
||||
expect(repo.update).toHaveBeenLastCalledWith(
|
||||
const redriven = await svc.settleZombie('run-1');
|
||||
expect(redriven).toBe(true);
|
||||
expect(svc.hasZombie('run-1')).toBe(false);
|
||||
expect(repo.finalizeIfActive).toHaveBeenLastCalledWith(
|
||||
'run-1',
|
||||
'ws-1',
|
||||
expect.objectContaining({ status: 'succeeded' }),
|
||||
);
|
||||
|
||||
// And it is now idempotent: a further settle no-ops (terminal row already
|
||||
// written), so a double-settle can never clobber the real status.
|
||||
const callsBefore = repo.update.mock.calls.length;
|
||||
// A later finalizeRun is idempotent (row already terminal): it no-ops at the
|
||||
// once-gate, never re-writing.
|
||||
const callsBefore = repo.finalizeIfActive.mock.calls.length;
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'error', 'late');
|
||||
expect(repo.update).toHaveBeenCalledTimes(callsBefore);
|
||||
expect(repo.finalizeIfActive).toHaveBeenCalledTimes(callsBefore);
|
||||
});
|
||||
|
||||
it('#487 double-settle collapses to a benign no-op (conditional write; notifier resolves once)', async () => {
|
||||
// A second concurrent settle is stopped at the synchronous active.delete
|
||||
// claim, so the terminal write runs exactly once and the notifier resolves
|
||||
// exactly once with the FIRST settler's outcome.
|
||||
const repo = makeRepo();
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
await svc.beginRun({ chatId: 'chat-1', workspaceId: 'ws-1', userId: 'u1' });
|
||||
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'aborted');
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'error', 'late'); // no-op
|
||||
|
||||
expect(repo.finalizeIfActive).toHaveBeenCalledTimes(1);
|
||||
const outcome = await svc.peekSettled('run-1');
|
||||
// peekSettled after resolve+delete falls through (notifier dropped, no zombie)
|
||||
// -> undefined; the FIRST settler already resolved any earlier subscriber.
|
||||
expect(outcome).toBeUndefined();
|
||||
});
|
||||
|
||||
it('#487 late settledPromise subscriber gets the resolved outcome', async () => {
|
||||
const repo = makeRepo();
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
await svc.beginRun({ chatId: 'chat-1', workspaceId: 'ws-1', userId: 'u1' });
|
||||
|
||||
// Subscribe BEFORE settle: hold the promise reference (as supersede does).
|
||||
const early = svc.peekSettled('run-1');
|
||||
expect(early).toBeDefined();
|
||||
|
||||
await svc.finalizeRun('run-1', 'ws-1', 'completed');
|
||||
|
||||
// The reference grabbed before settle resolves with the written outcome, even
|
||||
// though the notifier was dropped from the map on resolve (bounded).
|
||||
await expect(early).resolves.toEqual({
|
||||
status: 'succeeded',
|
||||
error: null,
|
||||
terminalWriteFailed: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('recordStep / linkAssistantMessage are best-effort: a repo failure is swallowed', async () => {
|
||||
@@ -525,3 +577,197 @@ describe('AiChatRunService run lifecycle', () => {
|
||||
).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('#487 AiChatRunService.supersede (CAS)', () => {
|
||||
const chat = 'chat-1';
|
||||
const ws = 'ws-1';
|
||||
|
||||
it('degrade: no active run on the chat -> caller sends a normal turn', async () => {
|
||||
const repo = makeRepo({
|
||||
findById: jest.fn(async () => undefined),
|
||||
findActiveByChat: jest.fn(async () => undefined),
|
||||
});
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
expect(await svc.supersede(chat, 'run-x', ws)).toEqual({ kind: 'degrade' });
|
||||
});
|
||||
|
||||
it('invalid: the target run belongs to a DIFFERENT chat -> 400', async () => {
|
||||
const repo = makeRepo({
|
||||
findById: jest.fn(async () => ({
|
||||
id: 'run-x',
|
||||
chatId: 'other-chat',
|
||||
workspaceId: ws,
|
||||
})),
|
||||
});
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
expect(await svc.supersede(chat, 'run-x', ws)).toEqual({ kind: 'invalid' });
|
||||
});
|
||||
|
||||
it('mismatch: a DIFFERENT run is active than the one targeted -> current runId', async () => {
|
||||
const repo = makeRepo({
|
||||
findById: jest.fn(async () => ({ id: 'run-x', chatId: chat, workspaceId: ws })),
|
||||
findActiveByChat: jest.fn(async () => ({
|
||||
id: 'run-live',
|
||||
chatId: chat,
|
||||
workspaceId: ws,
|
||||
status: 'running',
|
||||
})),
|
||||
});
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
expect(await svc.supersede(chat, 'run-x', ws)).toEqual({
|
||||
kind: 'mismatch',
|
||||
activeRunId: 'run-live',
|
||||
});
|
||||
});
|
||||
|
||||
it('ready: the target IS active -> stop it, await its (fast) settle, free the slot', async () => {
|
||||
// Simulate a live long TOOL (NOT a slow UPDATE): the run stays active until an
|
||||
// explicit Stop unwinds it; commit-1's race makes that settle land quickly.
|
||||
// The abort listener stands in for streamText's onAbort -> finalizeRun.
|
||||
const repo = makeRepo({
|
||||
findById: jest.fn(async () => ({
|
||||
id: 'run-1',
|
||||
chatId: chat,
|
||||
workspaceId: ws,
|
||||
status: 'aborted',
|
||||
error: null,
|
||||
})),
|
||||
findActiveByChat: jest.fn(async () => ({
|
||||
id: 'run-1',
|
||||
chatId: chat,
|
||||
workspaceId: ws,
|
||||
status: 'running',
|
||||
})),
|
||||
});
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
const handle = await svc.beginRun({ chatId: chat, workspaceId: ws, userId: 'u1' });
|
||||
handle.signal.addEventListener('abort', () => {
|
||||
void svc.finalizeRun('run-1', ws, 'aborted');
|
||||
});
|
||||
|
||||
// supersede: getRun -> getActiveByChat(==target) -> requestStop -> the abort
|
||||
// listener settles the run -> awaitSettled resolves -> ready.
|
||||
expect(await svc.supersede(chat, 'run-1', ws, 10_000)).toEqual({
|
||||
kind: 'ready',
|
||||
});
|
||||
expect(handle.signal.aborted).toBe(true); // Stop reached the run
|
||||
});
|
||||
|
||||
it('timeout: the target never settles within W -> 409 SUPERSEDE_TIMEOUT (nothing persisted)', async () => {
|
||||
const repo = makeRepo({
|
||||
findById: jest.fn(async () => ({ id: 'run-1', chatId: chat, workspaceId: ws })),
|
||||
findActiveByChat: jest.fn(async () => ({
|
||||
id: 'run-1',
|
||||
chatId: chat,
|
||||
workspaceId: ws,
|
||||
status: 'running',
|
||||
})),
|
||||
});
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
await svc.beginRun({ chatId: chat, workspaceId: ws, userId: 'u1' });
|
||||
// Do NOT settle the run: a tiny W elapses -> timeout.
|
||||
const result = await svc.supersede(chat, 'run-1', ws, 30);
|
||||
expect(result).toEqual({ kind: 'timeout' });
|
||||
});
|
||||
|
||||
it('ready then a DUPLICATE supersede POST degrades (the run is already gone)', async () => {
|
||||
let active: unknown = {
|
||||
id: 'run-1',
|
||||
chatId: chat,
|
||||
workspaceId: ws,
|
||||
status: 'running',
|
||||
};
|
||||
const repo = makeRepo({
|
||||
findById: jest.fn(async () => ({
|
||||
id: 'run-1',
|
||||
chatId: chat,
|
||||
workspaceId: ws,
|
||||
status: 'aborted',
|
||||
error: null,
|
||||
})),
|
||||
findActiveByChat: jest.fn(async () => active),
|
||||
finalizeIfActive: jest.fn(async () => {
|
||||
active = undefined; // settling frees the active slot
|
||||
return { id: 'run-1', status: 'aborted' };
|
||||
}),
|
||||
});
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
const handle = await svc.beginRun({ chatId: chat, workspaceId: ws, userId: 'u1' });
|
||||
handle.signal.addEventListener('abort', () => {
|
||||
void svc.finalizeRun('run-1', ws, 'aborted');
|
||||
});
|
||||
|
||||
expect(await svc.supersede(chat, 'run-1', ws, 10_000)).toEqual({
|
||||
kind: 'ready',
|
||||
});
|
||||
// The duplicate POST for the same target now finds no active run -> degrade.
|
||||
expect(await svc.supersede(chat, 'run-1', ws)).toEqual({ kind: 'degrade' });
|
||||
});
|
||||
|
||||
it('reconcileStaleRuns: aborts a stale run with NO entry/zombie; NEVER touches a live entry', async () => {
|
||||
const finalizeIfActive = jest.fn(async () => ({ id: 'x', status: 'aborted' }));
|
||||
const repo = makeRepo({
|
||||
insert: jest.fn(async (v: any) => ({
|
||||
id: 'live-1',
|
||||
status: 'running',
|
||||
chatId: v.chatId,
|
||||
workspaceId: v.workspaceId,
|
||||
})),
|
||||
finalizeIfActive,
|
||||
findStaleActive: jest.fn(async () => [
|
||||
{ id: 'orphan-1', workspaceId: ws, chatId: 'c-orphan' },
|
||||
{ id: 'live-1', workspaceId: ws, chatId: 'c-live' },
|
||||
]),
|
||||
});
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
// A LIVE run this replica owns (in the `active` map).
|
||||
await svc.beginRun({ chatId: 'c-live', workspaceId: ws, userId: 'u1' });
|
||||
expect(svc.isLocallyActive('live-1')).toBe(true);
|
||||
|
||||
const aborted = await svc.reconcileStaleRuns(15 * 60 * 1000);
|
||||
expect(aborted).toBe(1);
|
||||
// The orphan (no entry) was aborted; the live entry was NEVER passed to the DB.
|
||||
expect(finalizeIfActive).toHaveBeenCalledTimes(1);
|
||||
expect(finalizeIfActive).toHaveBeenCalledWith(
|
||||
'orphan-1',
|
||||
ws,
|
||||
expect.objectContaining({ status: 'aborted' }),
|
||||
);
|
||||
expect(svc.isLocallyActive('live-1')).toBe(true);
|
||||
});
|
||||
|
||||
it('gave-up zombie: supersede applies the intended status (settleZombie) then is ready', async () => {
|
||||
let healthy = false;
|
||||
let active: unknown = {
|
||||
id: 'run-1',
|
||||
chatId: chat,
|
||||
workspaceId: ws,
|
||||
status: 'running',
|
||||
};
|
||||
const repo = makeRepo({
|
||||
findById: jest.fn(async () => ({ id: 'run-1', chatId: chat, workspaceId: ws })),
|
||||
findActiveByChat: jest.fn(async () => active),
|
||||
finalizeIfActive: jest.fn(async () => {
|
||||
if (!healthy) throw new Error('db down');
|
||||
active = undefined;
|
||||
return { id: 'run-1', status: 'aborted' };
|
||||
}),
|
||||
});
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined);
|
||||
jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined);
|
||||
const svc = new AiChatRunService(repo as never, makeEnv() as never);
|
||||
await svc.beginRun({ chatId: chat, workspaceId: ws, userId: 'u1' });
|
||||
|
||||
// The run's terminal write gives up -> zombie (row still 'running').
|
||||
await svc.finalizeRun('run-1', ws, 'aborted');
|
||||
expect(svc.hasZombie('run-1')).toBe(true);
|
||||
|
||||
// The DB recovers; supersede awaits the (already-resolved, terminalWriteFailed)
|
||||
// settle, then settleZombie applies the intended status -> ready.
|
||||
healthy = true;
|
||||
expect(await svc.supersede(chat, 'run-1', ws, 10_000)).toEqual({
|
||||
kind: 'ready',
|
||||
});
|
||||
expect(svc.hasZombie('run-1')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -34,6 +34,88 @@ export class RunAlreadyActiveError extends Error {
|
||||
export type TurnTerminalStatus = 'completed' | 'error' | 'aborted';
|
||||
export type RunTerminalStatus = 'succeeded' | 'failed' | 'aborted';
|
||||
|
||||
/** The terminal run statuses — the row is done once it reads one of these. */
|
||||
export const RUN_TERMINAL_STATUSES: readonly RunTerminalStatus[] = [
|
||||
'succeeded',
|
||||
'failed',
|
||||
'aborted',
|
||||
];
|
||||
|
||||
/** Whether a persisted run status is terminal (settled). */
|
||||
export function isRunTerminal(status: string | null | undefined): boolean {
|
||||
return (
|
||||
status === 'succeeded' || status === 'failed' || status === 'aborted'
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: the outcome a run's {@link AiChatRunService.finalizeRun} settled with.
|
||||
* `terminalWriteFailed` = the terminal write GAVE UP after the bounded retry, so
|
||||
* the row is still non-terminal ('running') and a ZOMBIE record holds the
|
||||
* `intended` status for a later re-drive (reconcile / supersede / boot sweep). A
|
||||
* subscriber (supersede, #487 commit 3) uses this to decide whether the slot is
|
||||
* genuinely free or must first have the intended status applied.
|
||||
*/
|
||||
export interface RunSettleOutcome {
|
||||
status: RunTerminalStatus;
|
||||
error: string | null;
|
||||
terminalWriteFailed: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: how long a supersede waits for the target run to settle after Stop before
|
||||
* it degrades to `SUPERSEDE_TIMEOUT`. W=10s is generous under a HEALTHY DB: commit
|
||||
* 1's race-on-abort makes an in-app tool abort->settle in ms/hundreds of ms, so a
|
||||
* live run releases its slot well within the window. Under a DB brownout the
|
||||
* timeout is normal (the write cannot land); W must NOT be raised to paper
|
||||
* over a slow DB — a SUPERSEDE_TIMEOUT is the honest signal (nothing persisted,
|
||||
* the composer keeps the user's text). Env-tunable for ops, default 10s.
|
||||
*/
|
||||
export const SUPERSEDE_SETTLE_TIMEOUT_MS = (() => {
|
||||
const raw = Number(process.env.AI_CHAT_SUPERSEDE_TIMEOUT_MS);
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 10_000;
|
||||
})();
|
||||
|
||||
/**
|
||||
* #487: the result of the supersede CAS ({@link AiChatRunService.supersede}).
|
||||
* - `degrade` : no active run on the chat (it ended between click and POST) —
|
||||
* the caller sends a NORMAL turn (NOT a mismatch);
|
||||
* - `invalid` : the target runId belongs to a DIFFERENT chat (malformed CAS 400);
|
||||
* - `mismatch` : a DIFFERENT run is active than the one the client targeted —
|
||||
* 409 SUPERSEDE_TARGET_MISMATCH carrying the current `activeRunId`
|
||||
* (the client does NOT auto-retry);
|
||||
* - `timeout` : the target did not settle within W — 409 SUPERSEDE_TIMEOUT,
|
||||
* nothing persisted;
|
||||
* - `ready` : the target was stopped AND settled (or its zombie's intended was
|
||||
* applied) — the slot is free; the caller may beginRun the new run.
|
||||
*/
|
||||
export type SupersedeResult =
|
||||
| { kind: 'degrade' }
|
||||
| { kind: 'invalid' }
|
||||
| { kind: 'mismatch'; activeRunId: string }
|
||||
| { kind: 'timeout' }
|
||||
| { kind: 'ready' };
|
||||
|
||||
/** A one-shot settle notifier (#487): `resolve` is called EXACTLY ONCE. */
|
||||
interface Deferred<T> {
|
||||
promise: Promise<T>;
|
||||
resolve: (value: T) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: a run whose terminal write GAVE UP (every bounded attempt failed). The
|
||||
* row is stranded non-terminal ('running'); this record is the ONLY thing that
|
||||
* distinguishes it from a live run, and carries the `intended` terminal status so
|
||||
* a re-drive can apply it via the conditional UPDATE. Process-local (phase-1
|
||||
* single-process assumption): a restart drops it, and the boot sweep then writes
|
||||
* 'aborted' over the intended — a documented loss (see finalizeRun).
|
||||
*/
|
||||
interface ZombieRun {
|
||||
workspaceId: string;
|
||||
chatId: string;
|
||||
intended: { status: RunTerminalStatus; error: string | null };
|
||||
}
|
||||
|
||||
export function mapTurnStatusToRun(
|
||||
status: TurnTerminalStatus,
|
||||
): RunTerminalStatus {
|
||||
@@ -101,6 +183,22 @@ export class AiChatRunService implements OnModuleInit {
|
||||
// uptime — negligible in phase 1's single process.
|
||||
private readonly settled = new Set<string>();
|
||||
|
||||
// #487 runId -> one-shot settle notifier. Kept in a SEPARATE map from `active`
|
||||
// ON PURPOSE: it must OUTLIVE the `active.delete` claim inside finalizeRun (the
|
||||
// claim frees the slot the instant finalize starts), so a subscriber can still
|
||||
// await the outcome after the entry is gone. Created in beginRun, resolved
|
||||
// EXACTLY ONCE in finalizeRun, then removed (bounded). Absence => this replica
|
||||
// has no live notifier: a subscriber falls back to the zombie map, then to the
|
||||
// row (see peekSettled). Process-local (phase-1 single-process assumption).
|
||||
private readonly settledPromises = new Map<string, Deferred<RunSettleOutcome>>();
|
||||
|
||||
// #487 runId -> ZOMBIE record: a run whose terminal write gave up (row stranded
|
||||
// non-terminal). BOUNDED — an entry is added only on give-up and removed on a
|
||||
// successful re-drive (settleZombie) or when the row is found already terminal;
|
||||
// a process restart clears it (and the boot sweep settles the stranded row).
|
||||
// Process-local (phase-1 single-process assumption).
|
||||
private readonly zombies = new Map<string, ZombieRun>();
|
||||
|
||||
// Bounded retry for the terminal write (F6): a single PK UPDATE can fail
|
||||
// transiently under many fire-and-forget writes (pool exhaustion, deadlock, a
|
||||
// brief connection blip). Riding out that blip in-place matters because the
|
||||
@@ -224,6 +322,10 @@ export class AiChatRunService implements OnModuleInit {
|
||||
chatId: args.chatId,
|
||||
workspaceId: args.workspaceId,
|
||||
});
|
||||
// #487: arm the one-shot settle notifier BEFORE returning, so a subscriber
|
||||
// that races in immediately after begin always finds a promise to await. It
|
||||
// is resolved exactly once when the run settles (or gives up).
|
||||
this.settledPromises.set(run.id, this.makeDeferred<RunSettleOutcome>());
|
||||
return { runId: run.id, signal: controller.signal };
|
||||
}
|
||||
|
||||
@@ -263,47 +365,43 @@ export class AiChatRunService implements OnModuleInit {
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalize a run to its terminal status (succeeded / failed / aborted),
|
||||
* stamping finishedAt + any error. Best-effort, but ROBUST against a transient
|
||||
* terminal-write failure (F6) AND atomically safe against a concurrent settle.
|
||||
* Finalize a run to its terminal status (succeeded / failed / aborted) via a
|
||||
* CONDITIONAL UPDATE, stamping finishedAt + any error. Atomically safe against a
|
||||
* concurrent settle AND robust against a transient terminal-write failure.
|
||||
*
|
||||
* ATOMIC ONCE-CLAIM (the gate must close in ONE synchronous tick): two
|
||||
* finalizeRun calls for the SAME run can race — the documented real path is
|
||||
* AiChatService.stream's safety-net catch settling the turn to 'error' while a
|
||||
* streamText terminal callback (onFinish/onAbort/onError) ALSO settles it. The
|
||||
* `settled.has` check alone is NOT a gate: it is read BEFORE the awaited UPDATE,
|
||||
* so two callers can both see `false` and both write the row (last-write-wins
|
||||
* clobbers the real terminal status, and the bounded retry only widens that
|
||||
* window). The claim therefore happens via `active.delete`, a SYNCHRONOUS
|
||||
* check-and-clear with NO await between the gate and the entry removal: the
|
||||
* second concurrent caller finds the entry already gone and returns in the same
|
||||
* tick, before any UPDATE. The transition "nobody is finalizing" -> "I am
|
||||
* finalizing" is thus a single atomic step.
|
||||
* claim happens via `active.delete`, a SYNCHRONOUS check-and-clear with NO await
|
||||
* between the gate and the entry removal: the second concurrent caller finds the
|
||||
* entry already gone and returns in the same tick, before any UPDATE.
|
||||
*
|
||||
* ORDER MATTERS (F6): once we own the claim, the terminal UPDATE happens FIRST;
|
||||
* only once it SUCCEEDS do we record the run as settled. If the UPDATE fails on
|
||||
* every bounded attempt we RESTORE the in-memory entry, leave the run UNsettled,
|
||||
* and emit an ERROR signal that the row is left non-terminal 'running' (which
|
||||
* would 409 every future turn in the chat until recovery). An in-process retry
|
||||
* by a LATER settle is only POSSIBLE, never guaranteed: it needs (a) the entry
|
||||
* to have been restored at the give-up path AND (b) a fresh settler to arrive
|
||||
* AFTER that restore. A concurrent settler that arrives DURING the retry window
|
||||
* — while the entry is deleted for backoff and not yet restored — is consumed at
|
||||
* the synchronous `active.delete` claim (it finds nothing to delete and returns
|
||||
* a no-op), so it does NOT become an in-process retrier. The NO-streamText path
|
||||
* (the turn threw before streamText was wired, so ONLY the safety-net ever
|
||||
* settles) likewise has no second in-process settler at all. The UNCONDITIONAL
|
||||
* backstop in every case is the boot sweep on the next restart (phase 1 has no
|
||||
* periodic in-process sweep); the retained entry is bounded (cleared on restart)
|
||||
* and harmless meanwhile.
|
||||
* ALL TERMINAL WRITES ARE CONDITIONAL (#487): `finalizeIfActive` only flips a
|
||||
* row still in pending|running (mirror of the assistant message's
|
||||
* `onlyIfStreaming`). So even a settle that DID reach the UPDATE (e.g. a
|
||||
* reconcile stamp racing an owner finalize) can never clobber a terminal status
|
||||
* — the loser matches nothing and is a benign no-op. `active.delete` is the
|
||||
* fast, in-process gate; the conditional WHERE is the authoritative one.
|
||||
*
|
||||
* IDEMPOTENT on SUCCESS (#184 review): the terminal write happens AT MOST ONCE
|
||||
* per run. After a successful write the once-gate keys off {@link settled} (the
|
||||
* terminal row already written) so a settle arriving AFTER the entry was already
|
||||
* dropped-and-settled returns early; a settle racing the in-flight write is
|
||||
* stopped earlier still, by the `active.delete` claim. Either way a genuine
|
||||
* double-settle collapses to a single write and a late settle can never clobber
|
||||
* the real terminal status or double-write the row.
|
||||
* ZOMBIE ON GIVE-UP (#487): if every bounded attempt THROWS (the DB is down for
|
||||
* the whole finalize), we do NOT restore the entry. The row is stranded
|
||||
* non-terminal ('running'); we record a ZOMBIE `{ terminalWriteFailed, intended
|
||||
* }` (the ONLY thing distinguishing this dead run from a live one) and resolve
|
||||
* the settle notifier with `terminalWriteFailed: true`. A restore would make the
|
||||
* zombie indistinguishable from a live run to every reader; instead a re-drive
|
||||
* (settleZombie, called by the periodic reconcile / supersede / opportunistic
|
||||
* paths) applies the intended status later via the same conditional UPDATE.
|
||||
*
|
||||
* DOCUMENTED LOSS (#487, single-process phase 1): if the process RESTARTS before
|
||||
* a zombie is re-driven, the in-memory zombie map is gone and the boot sweep
|
||||
* (unconditional) writes 'aborted' over the ACTUAL intended status. This is
|
||||
* unavoidable while the run lifecycle is single-process — there is no durable
|
||||
* record of `intended`; a cross-process durable intent is deferred to phase 2.
|
||||
*
|
||||
* IDEMPOTENT: the settle notifier resolves EXACTLY ONCE; a second settle is
|
||||
* stopped at `settled.has` or the `active.delete` claim, so a double-settle
|
||||
* collapses to a single write and can never double-resolve or clobber the row.
|
||||
*/
|
||||
async finalizeRun(
|
||||
runId: string,
|
||||
@@ -314,13 +412,17 @@ export class AiChatRunService implements OnModuleInit {
|
||||
// ---- Atomic once-claim (synchronous; NO await before the gate closes) ----
|
||||
// Already terminally written -> idempotent no-op.
|
||||
if (this.settled.has(runId)) return;
|
||||
// Capture the entry BEFORE the delete so a total-failure path can restore it.
|
||||
// Capture the entry BEFORE the delete for the give-up log context.
|
||||
const entry = this.active.get(runId);
|
||||
// SYNCHRONOUS check-and-clear: the FIRST caller deletes (claims) the entry;
|
||||
// any concurrent SECOND caller finds nothing to delete and returns HERE, in
|
||||
// the same tick, before any await — so it can never reach the UPDATE.
|
||||
if (!this.active.delete(runId)) return;
|
||||
|
||||
const status = mapTurnStatusToRun(turnStatus);
|
||||
const err = error ?? null;
|
||||
const chatId = entry?.chatId ?? 'unknown';
|
||||
|
||||
let lastError: unknown;
|
||||
for (
|
||||
let attempt = 1;
|
||||
@@ -328,47 +430,294 @@ export class AiChatRunService implements OnModuleInit {
|
||||
attempt++
|
||||
) {
|
||||
try {
|
||||
await this.runRepo.update(runId, workspaceId, {
|
||||
status: mapTurnStatusToRun(turnStatus),
|
||||
finishedAt: new Date(),
|
||||
error: error ?? null,
|
||||
const row = await this.runRepo.finalizeIfActive(runId, workspaceId, {
|
||||
status,
|
||||
error: err,
|
||||
});
|
||||
// Terminal write landed: arm the once-gate. The entry is already gone
|
||||
// (claimed above); we do NOT restore it. The slot is now free.
|
||||
// No throw => the row is now terminal (we wrote it, or it was ALREADY
|
||||
// terminal — another writer won the conditional UPDATE, a benign no-op).
|
||||
this.settled.add(runId);
|
||||
this.zombies.delete(runId);
|
||||
// Resolve with the persisted outcome: our status when WE wrote it, else
|
||||
// the row's real terminal status (re-read on the already-terminal path so
|
||||
// a subscriber never sees a status we did not actually persist).
|
||||
const outcome: RunSettleOutcome = row
|
||||
? { status, error: err, terminalWriteFailed: false }
|
||||
: await this.readTerminalOutcome(runId, workspaceId, status, err);
|
||||
this.resolveSettled(runId, outcome);
|
||||
return;
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
} catch (err2) {
|
||||
lastError = err2;
|
||||
this.logger.warn(
|
||||
`Failed to finalize run ${runId} (attempt ${attempt}/${
|
||||
AiChatRunService.FINALIZE_MAX_ATTEMPTS
|
||||
}): ${err instanceof Error ? err.message : 'unknown error'}`,
|
||||
}): ${err2 instanceof Error ? err2.message : 'unknown error'}`,
|
||||
);
|
||||
if (attempt < AiChatRunService.FINALIZE_MAX_ATTEMPTS) {
|
||||
await this.delay(AiChatRunService.FINALIZE_RETRY_BASE_MS * attempt);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Every attempt failed: this is a give-up, materially worse than a per-attempt
|
||||
// blip — the row is left NON-TERMINAL ('running'), so emit ONE explicit,
|
||||
// greppable ERROR so an operator can tell "survived a blip" from "gave up, run
|
||||
// held in memory until recovery" (the last warn alone says only "attempt 3/3").
|
||||
// Every attempt threw: GIVE UP. The row is stranded non-terminal ('running').
|
||||
// Do NOT restore the entry (a restored entry is indistinguishable from a live
|
||||
// run); leave a ZOMBIE record instead, and resolve the notifier as
|
||||
// terminalWriteFailed so a subscriber knows the slot still needs the intended
|
||||
// status applied. One explicit, greppable ERROR so an operator can tell a
|
||||
// give-up from a per-attempt blip.
|
||||
this.logger.error(
|
||||
`Run ${runId} (chat ${entry?.chatId ?? 'unknown'}) left NON-TERMINAL ` +
|
||||
`('running'): terminal write failed after ${
|
||||
AiChatRunService.FINALIZE_MAX_ATTEMPTS
|
||||
} attempts; entry retained in memory, recovery deferred to next settle / ` +
|
||||
`boot sweep`,
|
||||
`Run ${runId} (chat ${chatId}) left NON-TERMINAL ('running'): terminal ` +
|
||||
`write failed after ${AiChatRunService.FINALIZE_MAX_ATTEMPTS} attempts; ` +
|
||||
`ZOMBIE recorded (intended '${status}'), recovery deferred to reconcile / ` +
|
||||
`supersede / boot sweep`,
|
||||
lastError,
|
||||
);
|
||||
// RESTORE the claimed entry (and leave the run UNsettled) so a LATER settle
|
||||
// that arrives AFTER this restore MAY retry the terminal write — but that
|
||||
// in-process retry is NOT guaranteed (a concurrent settler caught in the retry
|
||||
// window above is consumed at the `active.delete` claim, and the no-streamText
|
||||
// path has no second settler at all). The UNCONDITIONAL backstop in every case
|
||||
// is the boot sweep on the next restart; the restored entry is bounded and
|
||||
// cleared on restart.
|
||||
if (entry) this.active.set(runId, entry);
|
||||
this.zombies.set(runId, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
intended: { status, error: err },
|
||||
});
|
||||
this.resolveSettled(runId, { status, error: err, terminalWriteFailed: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: re-drive a zombie run's intended terminal write (the conditional
|
||||
* UPDATE). Called by the periodic reconcile (commit 4), an opportunistic
|
||||
* single-chat reconcile, and supersede (commit 3). On success — the row is now
|
||||
* terminal (written OR found already terminal) — the zombie is cleared and the
|
||||
* once-gate armed; on another failure the zombie is kept for a later retry.
|
||||
* Returns true when the row is now terminal. Best-effort; never throws.
|
||||
*/
|
||||
async settleZombie(runId: string): Promise<boolean> {
|
||||
const z = this.zombies.get(runId);
|
||||
if (!z) return false;
|
||||
try {
|
||||
await this.runRepo.finalizeIfActive(runId, z.workspaceId, {
|
||||
status: z.intended.status,
|
||||
error: z.intended.error,
|
||||
});
|
||||
this.zombies.delete(runId);
|
||||
this.settled.add(runId);
|
||||
return true;
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Re-drive of zombie run ${runId} (chat ${z.chatId}) failed; will retry ` +
|
||||
`later: ${err instanceof Error ? err.message : 'unknown error'}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* #487 reconcile clause (c): abort runs the DB still shows active (pending|
|
||||
* running) but that this replica does NOT own — NO live entry AND NO zombie —
|
||||
* and that have been UNTOUCHED past `staleMs` (from last-progress `updated_at`,
|
||||
* NOT startedAt, so a legit long marathon is never a candidate). "No entry" is
|
||||
* the PRIMARY gate: a live entry (an actively-executing run on this replica) is
|
||||
* NEVER aborted, whatever its age. Returns the number aborted. Best-effort —
|
||||
* never throws (a periodic-job failure must not crash the process).
|
||||
*/
|
||||
async reconcileStaleRuns(staleMs: number): Promise<number> {
|
||||
let candidates: Array<{ id: string; workspaceId: string; chatId: string }>;
|
||||
try {
|
||||
candidates = await this.runRepo.findStaleActive(staleMs);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Reconcile (stale runs) query failed: ${
|
||||
err instanceof Error ? err.message : 'unknown error'
|
||||
}`,
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
let aborted = 0;
|
||||
for (const c of candidates) {
|
||||
// PRIMARY gate: never touch a live entry, and never race a zombie we are
|
||||
// already re-driving (settleZombie owns those).
|
||||
if (this.active.has(c.id) || this.zombies.has(c.id)) continue;
|
||||
try {
|
||||
const row = await this.runRepo.finalizeIfActive(c.id, c.workspaceId, {
|
||||
status: 'aborted',
|
||||
error: 'Run aborted by reconcile: no live runner (stale).',
|
||||
});
|
||||
if (row) {
|
||||
aborted += 1;
|
||||
this.settled.add(c.id);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Reconcile abort of stale run ${c.id} failed: ${
|
||||
err instanceof Error ? err.message : 'unknown error'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return aborted;
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: the run's settle outcome as seen by THIS replica, or undefined when it
|
||||
* has no record (the caller then reads the row — the DB is the source of truth).
|
||||
* A LIVE deferred (still settling, or resolved-but-not-yet-consumed) wins; a
|
||||
* ZOMBIE synthesizes the give-up outcome. A subscriber (supersede) races this
|
||||
* against a timeout.
|
||||
*/
|
||||
peekSettled(runId: string): Promise<RunSettleOutcome> | undefined {
|
||||
const d = this.settledPromises.get(runId);
|
||||
if (d) return d.promise;
|
||||
const z = this.zombies.get(runId);
|
||||
if (z) {
|
||||
return Promise.resolve({
|
||||
status: z.intended.status,
|
||||
error: z.intended.error,
|
||||
terminalWriteFailed: true,
|
||||
});
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: await a run's settle outcome, bounded by `timeoutMs`. Returns the
|
||||
* outcome on settle, or undefined on TIMEOUT (or when this replica has no record
|
||||
* of the run and its row is not terminal). Uses the LIVE settle notifier / the
|
||||
* zombie synth when present; else reads the row (the DB is the source of truth
|
||||
* once the in-memory record is gone). The subscriber (supersede) grabs this
|
||||
* right after Stop; commit 1's race makes the settle land in ms on a healthy DB.
|
||||
*/
|
||||
async awaitSettled(
|
||||
runId: string,
|
||||
workspaceId: string,
|
||||
timeoutMs: number,
|
||||
): Promise<RunSettleOutcome | undefined> {
|
||||
const pending = this.peekSettled(runId);
|
||||
if (pending) {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeout = new Promise<undefined>((resolve) => {
|
||||
timer = setTimeout(() => resolve(undefined), timeoutMs);
|
||||
timer.unref?.();
|
||||
});
|
||||
try {
|
||||
return await Promise.race([pending, timeout]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
// No live notifier and no zombie: read the row (already settled-and-written,
|
||||
// or unknown here). A terminal row is an outcome; anything else -> undefined.
|
||||
const row = await this.runRepo.findById(runId, workspaceId);
|
||||
if (row && isRunTerminal(row.status)) {
|
||||
return {
|
||||
status: row.status as RunTerminalStatus,
|
||||
error: row.error ?? null,
|
||||
terminalWriteFailed: false,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: the SERVER supersede CAS for `POST /stream { supersede: { runId: X } }`.
|
||||
* Atomically transitions "X is the chat's active run" -> "X is stopped, settled,
|
||||
* slot free" so the caller can start a replacement run. See {@link
|
||||
* SupersedeResult} for the branch semantics.
|
||||
*
|
||||
* On a `ready` result the caller MUST still go through the normal beginRun gate
|
||||
* (the partial unique index) — between the slot freeing here and beginRun a
|
||||
* neighbouring tab's ordinary POST can win the slot (documented SLOT-THEFT: the
|
||||
* loser then gets a MISMATCH carrying the NEW runId). There is also NO side-
|
||||
* effect quiescence: an in-flight write of the stopped run may still land AFTER
|
||||
* the new run starts (commit 1 stops the NEXT call, not one already committing),
|
||||
* so the caller adds a prompt note to the new run.
|
||||
*/
|
||||
async supersede(
|
||||
chatId: string,
|
||||
targetRunId: string,
|
||||
workspaceId: string,
|
||||
timeoutMs: number = SUPERSEDE_SETTLE_TIMEOUT_MS,
|
||||
): Promise<SupersedeResult> {
|
||||
// Validate the target belongs to THIS chat (a CAS targeting another chat's run
|
||||
// is malformed -> 400). A missing row is NOT invalid: the run may have ended
|
||||
// and been pruned; the active-run check below decides degrade vs mismatch.
|
||||
const target = await this.getRun(targetRunId, workspaceId);
|
||||
if (target && target.chatId !== chatId) return { kind: 'invalid' };
|
||||
|
||||
const active = await this.getActiveForChat(chatId, workspaceId);
|
||||
// No active run: it ended between the client's click and this POST — this is a
|
||||
// DEGRADE to a normal send, NOT a mismatch (the user's intent still holds).
|
||||
if (!active) return { kind: 'degrade' };
|
||||
// A DIFFERENT run is active than the one the client saw -> mismatch. The
|
||||
// client does not auto-retry; it surfaces the new runId.
|
||||
if (active.id !== targetRunId) {
|
||||
return { kind: 'mismatch', activeRunId: active.id };
|
||||
}
|
||||
|
||||
// The target IS active: stop it, then await its settle within W.
|
||||
await this.requestStop(targetRunId, workspaceId);
|
||||
const outcome = await this.awaitSettled(targetRunId, workspaceId, timeoutMs);
|
||||
if (!outcome) return { kind: 'timeout' };
|
||||
// Gave up (terminal write failed): apply the intended status via the
|
||||
// conditional UPDATE so the slot actually frees. If that ALSO fails, the row
|
||||
// is still stranded -> treat as a timeout (nothing persisted for the new run).
|
||||
if (outcome.terminalWriteFailed) {
|
||||
const settled = await this.settleZombie(targetRunId);
|
||||
if (!settled) return { kind: 'timeout' };
|
||||
}
|
||||
return { kind: 'ready' };
|
||||
}
|
||||
|
||||
/** #487 test/diagnostic seam: whether a give-up zombie is held for this run. */
|
||||
hasZombie(runId: string): boolean {
|
||||
return this.zombies.has(runId);
|
||||
}
|
||||
|
||||
/** #487: every zombie runId held on this replica (reconcile clause a, commit 4). */
|
||||
zombieRunIds(): string[] {
|
||||
return [...this.zombies.keys()];
|
||||
}
|
||||
|
||||
/** #487: create a one-shot deferred (resolve captured for a later single call). */
|
||||
private makeDeferred<T>(): Deferred<T> {
|
||||
let resolve!: (value: T) => void;
|
||||
const promise = new Promise<T>((r) => {
|
||||
resolve = r;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
/** #487: resolve a run's settle notifier EXACTLY ONCE, then drop it (bounded).
|
||||
* A subscriber that already grabbed the promise still resolves; a later one
|
||||
* falls back to the zombie map / the row (see peekSettled). */
|
||||
private resolveSettled(runId: string, outcome: RunSettleOutcome): void {
|
||||
const d = this.settledPromises.get(runId);
|
||||
if (!d) return;
|
||||
this.settledPromises.delete(runId);
|
||||
d.resolve(outcome);
|
||||
}
|
||||
|
||||
/** #487: read the persisted terminal outcome when the conditional finalize was a
|
||||
* no-op (the row was already terminal). Falls back to the intended status when
|
||||
* the read fails or the row is unexpectedly missing/non-terminal. */
|
||||
private async readTerminalOutcome(
|
||||
runId: string,
|
||||
workspaceId: string,
|
||||
fallbackStatus: RunTerminalStatus,
|
||||
fallbackError: string | null,
|
||||
): Promise<RunSettleOutcome> {
|
||||
try {
|
||||
const row = await this.runRepo.findById(runId, workspaceId);
|
||||
if (row && isRunTerminal(row.status)) {
|
||||
return {
|
||||
status: row.status as RunTerminalStatus,
|
||||
error: row.error ?? null,
|
||||
terminalWriteFailed: false,
|
||||
};
|
||||
}
|
||||
} catch {
|
||||
// Fall through to the intended status — best-effort only.
|
||||
}
|
||||
return {
|
||||
status: fallbackStatus,
|
||||
error: fallbackError,
|
||||
terminalWriteFailed: false,
|
||||
};
|
||||
}
|
||||
|
||||
/** Small async backoff between terminal-write retries (F6). Isolated so it is
|
||||
|
||||
@@ -1,42 +1,122 @@
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
|
||||
/**
|
||||
* In-memory run-stream registry (#184 phase 1.5). A durable agent run tees its
|
||||
* SSE frames here (via `pipeUIMessageStreamToResponse({ consumeSseStream })`)
|
||||
* so a LATE tab — one that reloaded, or opened after the starter dropped — can
|
||||
* attach through `GET /ai-chat/runs/:chatId/stream`, replay the frames buffered
|
||||
* so far, and then follow the live tail as a normal streamer.
|
||||
* In-memory run-stream registry (#184 phase 1.5, step-aligned retention #491). A
|
||||
* durable agent run tees its SSE frames here (via
|
||||
* `pipeUIMessageStreamToResponse({ consumeSseStream })`) so a LATE tab — one that
|
||||
* reloaded, or opened after the starter dropped — can attach through
|
||||
* `GET /ai-chat/runs/:chatId/stream`, be handed the TAIL past the step it already
|
||||
* has persisted, and then follow the live tail as a normal streamer.
|
||||
*
|
||||
* This is deliberately single-process and best-effort: it holds nothing the DB
|
||||
* does not (the run + assistant row are the source of truth), so a process
|
||||
* restart simply drops in-flight entries and the client falls back to its
|
||||
* restore + degraded-poll path. The async `attach` return type is the seam for a
|
||||
* future phase-2 cross-process backend (Redis) — the interface does not change.
|
||||
*
|
||||
* ── #491 step-aligned retention (the OOM fix) ────────────────────────────────
|
||||
* The old registry buffered up to 32MB of raw SSE frames PER active run (V8 ~2×
|
||||
* in memory) and, on attach, blasted the WHOLE buffer to the socket synchronously
|
||||
* with no drain — a handful of marathon runs on a 1GB container OOM'd. #491 caps
|
||||
* the ring at a few MB (env-tunable, default 4MB) and keeps it there by ROTATING:
|
||||
*
|
||||
* - Every buffered frame is STAMPED with a step number at tee (see ingestFrame).
|
||||
* Convention: the stamp of a frame is the number of `finish-step` parts seen
|
||||
* BEFORE it (starting at 0). The finish-step frame itself carries the current
|
||||
* value, THEN the counter increments. So a frame stamped `s` is the content of
|
||||
* the (s+1)-th step — 0-based step index `s` — and the stamp aligns EXACTLY
|
||||
* with `metadata.stepsPersisted`: a client whose persisted `stepsPersisted` is
|
||||
* N has steps 0..N-1 on disk (and in its seed) and needs the tail `stamp >= N`.
|
||||
*
|
||||
* - The ring rotates ONLY on a CONFIRMED persist of step N
|
||||
* (`confirmPersistedStep`), dropping frames with `stamp < N` (those steps are
|
||||
* now on disk and a fresh client seed carries them). A NON-confirmed step is
|
||||
* never rotated away, so a persist FAILURE just makes the ring cover MORE
|
||||
* (auto-safe). This is the anti-inversion rule: a naive "rotate in .then()"
|
||||
* that rotated after an UNwritten step would drop a step nobody has → silent
|
||||
* hole. Rotation is gated on a real, successful persist.
|
||||
*
|
||||
* - If the ring still exceeds its byte cap after rotation (a single fat step, or
|
||||
* a lagging persist), the OLDEST frames are evicted to stay bounded. Evicting a
|
||||
* not-yet-persisted frame opens a GAP: an attach whose N falls at or below an
|
||||
* evicted step answers 204 and the client degrades to restore+poll. The gap is
|
||||
* NOT sticky — the coverage floor is recomputed from the ring, so a later
|
||||
* persist that rotates past the holey steps clears it.
|
||||
*
|
||||
* ── attach numbering / coverage (the wire convention) ────────────────────────
|
||||
* The step marker N comes ONLY FROM THE CLIENT (a query param). The server never
|
||||
* reads the row to derive N — a server-side N from a stale seed would open a
|
||||
* silent one-step hole. N is the client's persisted `stepsPersisted` (a COUNT):
|
||||
* - the tail it needs = frames with `stamp >= N`;
|
||||
* - coverage is OK ⟺ `coverageFloor(entry) <= N`, where coverageFloor is the
|
||||
* smallest step FULLY present in the ring (its smallest retained stamp, bumped
|
||||
* by one when that leading step was only partially evicted by overflow). If
|
||||
* `coverageFloor > N` the ring starts AFTER the client's frontier (a hole, or
|
||||
* the client's seed simply lagged behind a rotation) → 204 → the client
|
||||
* refetches (a larger N) and re-attaches.
|
||||
* The N cutoff is applied in ALL branches, INCLUDING the finished-retained replay.
|
||||
*
|
||||
* ── same-tick invariants (unchanged, still load-bearing) ─────────────────────
|
||||
* invariant 1: only the matching run may mutate/observe an entry (runId check).
|
||||
* invariant 2: retention deletes ONLY its own entry (a replacement may own the key).
|
||||
* invariant 3: open() over a live entry mirrors the done-path (subscribers released).
|
||||
* invariant 4: the tail SLICE + subscriber registration happen in ONE synchronous
|
||||
* tick inside attach() — no await between them — so a concurrently
|
||||
* ingested frame is EITHER in the snapshot (buffered before the sync
|
||||
* block, and the just-added subscriber never sees it) OR fanned out to
|
||||
* the paused subscriber's `pending` (ingested after) — never both and
|
||||
* never neither: no loss, no duplication. NOTE (#491): the controller
|
||||
* now AWAITS the drain-respecting tail write BEFORE calling start(), so
|
||||
* frames ingested during that await accumulate in `pending`; this is
|
||||
* bounded by the subscriber cap (an overflow degrades start() to an
|
||||
* end(), a 204-equivalent). It is the SYNCHRONOUS snapshot+registration
|
||||
* — not a same-tick start() — that makes this correct.
|
||||
* invariant 5: the controller wires close-cleanup BEFORE any write.
|
||||
* invariant 6: no cross-run replay — the `anchor` (the client's assistant row id)
|
||||
* must match this run's assistant id, or a foreign run's transcript
|
||||
* would be appended to the client's message.
|
||||
*/
|
||||
|
||||
/** How long a finished entry is retained for late attach (replay + immediate end). */
|
||||
export const RUN_STREAM_RETAIN_FINISHED_MS = 30_000;
|
||||
|
||||
/**
|
||||
* Per-run replay buffer cap. Past this the buffer is dropped (attach -> 204, and
|
||||
* the client falls back to its restore + degraded-poll path, #430).
|
||||
*
|
||||
* Raised from 4MB to 32MB (#430): marathon autonomous runs (11-25 min observed)
|
||||
* stream far more than 4MB of SSE frames, so a live disconnect mid-run would find
|
||||
* an already-overflowed buffer and could only degrade-poll instead of re-attaching
|
||||
* to the live tail. 32MB comfortably covers those runs while staying bounded.
|
||||
*
|
||||
* Memory cost: this is the WORST-CASE retained size PER ACTIVE run (the buffer is
|
||||
* freed on finish + retention, or dropped immediately on overflow). With the small
|
||||
* number of concurrent autonomous runs a single workspace realistically has, 32MB
|
||||
* each is an acceptable ceiling; the overflow->204->degraded-poll fallback remains
|
||||
* the backstop for anything larger, so correctness never depends on this bound.
|
||||
* DEFAULT per-run replay ring cap (#491, down from 32MB). SSE frames carry
|
||||
* UNcompacted tool outputs + framing overhead (×1.5–2 vs the persisted parts), so
|
||||
* a "2–3 large reads + reasoning" step routinely blows past 2MB; 4MB comfortably
|
||||
* holds a step or two of TAIL, which is all a resuming client needs (steps below
|
||||
* its persisted frontier come from the seed, not the ring). The ring stays bounded
|
||||
* because it rotates on every confirmed persist; this cap is only the ceiling for
|
||||
* the un-persisted tail between rotations. Env-tunable via
|
||||
* AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES (bytes); a 0/invalid value falls back to this.
|
||||
*/
|
||||
export const RUN_STREAM_MAX_BUFFER_BYTES = 32 * 1024 * 1024;
|
||||
export const AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES = 4 * 1024 * 1024;
|
||||
|
||||
// 2x the replay cap: a just-written full-replay burst alone can never trip the
|
||||
// per-subscriber cap (see controller); only a genuinely stalled socket can.
|
||||
export const SUBSCRIBER_MAX_BUFFERED_BYTES = 2 * RUN_STREAM_MAX_BUFFER_BYTES;
|
||||
// 2× the ring cap: a just-written full-tail burst alone can never trip the
|
||||
// per-subscriber cap (see controller); only a genuinely stalled socket can. This
|
||||
// derivative relationship is preserved even when the ring cap is env-overridden.
|
||||
export const SUBSCRIBER_MAX_BUFFERED_BYTES = 2 * AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES;
|
||||
|
||||
/**
|
||||
* A finish-step boundary frame is exactly `data: {"type":"finish-step"...}\n\n`
|
||||
* (verified empirically against ai@6.0.207 — each UI-message-stream part is a
|
||||
* single `data: {json}\n\n` event, never split across `data:` lines, and `type`
|
||||
* is always the first key). A prefix match is cheaper than JSON.parse-per-frame
|
||||
* and has no false positives: a literal `"type":"finish-step"` inside a text
|
||||
* delta is JSON-escaped (`\"type\":...`), and the frame would start with
|
||||
* `data: {"type":"text-delta"` anyway.
|
||||
*/
|
||||
const FINISH_STEP_FRAME_PREFIX = 'data: {"type":"finish-step"';
|
||||
|
||||
/** Resolve the ring cap from the environment, falling back to the default. */
|
||||
function resolveMaxBufferBytes(): number {
|
||||
const raw = process.env.AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES;
|
||||
if (!raw) return AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0
|
||||
? Math.floor(parsed)
|
||||
: AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES;
|
||||
}
|
||||
|
||||
export interface RunStreamCallbacks {
|
||||
onFrame: (frame: string) => void;
|
||||
@@ -44,6 +124,9 @@ export interface RunStreamCallbacks {
|
||||
}
|
||||
|
||||
export interface RunStreamAttachment {
|
||||
// The synthetic `start` frame (carrying { runId, chatId }) followed by the
|
||||
// buffered TAIL filtered to `stamp >= N`. The controller writes these to the
|
||||
// socket in chunks respecting drain, then calls start().
|
||||
replay: string[];
|
||||
finished: boolean;
|
||||
start(): void; // drain pending frames (order preserved) and go live
|
||||
@@ -53,14 +136,19 @@ export interface RunStreamAttachment {
|
||||
interface Subscriber extends RunStreamCallbacks {
|
||||
started: boolean;
|
||||
pending: string[];
|
||||
// Byte size of `pending`, capped at SUBSCRIBER_MAX_BUFFERED_BYTES. `start()` is
|
||||
// called in the SAME tick as `attach()` today (see attach), so `pending` never
|
||||
// holds more than one microtask of frames — but the async `attach` signature is
|
||||
// a phase-2 seam: an await between attach and start would let a stalled paused
|
||||
// subscriber buffer the WHOLE run here. The cap is the structural backstop.
|
||||
// Byte size of `pending`, capped at the subscriber cap. `start()` is called in
|
||||
// the SAME tick as `attach()` today, so `pending` never holds more than one
|
||||
// microtask of frames — but the controller writes the (potentially large) tail
|
||||
// respecting drain BEFORE start(), so a stalled socket can accumulate here; the
|
||||
// cap is the structural backstop (an overflow degrades start() to an end()).
|
||||
pendingBytes: number;
|
||||
overflowed: boolean;
|
||||
pendingEnd: boolean;
|
||||
// The client's step frontier N: this subscriber only receives frames with
|
||||
// `stamp >= minStamp` (the tail past what it already persisted). Live frames
|
||||
// always satisfy this (their stamp is the current, highest step), so it only
|
||||
// filters the rare out-of-order below-frontier frame.
|
||||
minStamp: number;
|
||||
}
|
||||
|
||||
interface Entry {
|
||||
@@ -68,8 +156,20 @@ interface Entry {
|
||||
// The persisted assistant row id of this run (set at bind; undefined if the
|
||||
// seed failed). Used by the attach anchor check (invariant 6).
|
||||
assistantMessageId?: string;
|
||||
// Parallel arrays: frames[i] is the SSE string, stamps[i] its step number.
|
||||
frames: string[];
|
||||
stamps: number[];
|
||||
bytes: number;
|
||||
// The running step counter used to stamp the NEXT frame (number of finish-step
|
||||
// frames seen so far).
|
||||
currentStamp: number;
|
||||
// The highest confirmed `stepsPersisted`: frames with stamp < persistedFloor are
|
||||
// on disk (safe to drop, never re-buffered). Monotonic (confirmPersistedStep).
|
||||
persistedFloor: number;
|
||||
// The highest stamp EVICTED by an overflow (unsafe) drop, -1 if none. Used to
|
||||
// detect a partially-evicted leading step when computing the coverage floor.
|
||||
overflowThroughStamp: number;
|
||||
// Sticky-for-logging only: at least one unsafe (overflow) eviction happened.
|
||||
overflowed: boolean;
|
||||
finished: boolean;
|
||||
subscribers: Set<Subscriber>;
|
||||
@@ -80,6 +180,10 @@ interface Entry {
|
||||
export class AiChatStreamRegistryService implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(AiChatStreamRegistryService.name);
|
||||
private readonly entries = new Map<string, Entry>(); // key: chatId
|
||||
// Env-resolved caps (per instance) so a deployment can tune the ceiling without
|
||||
// a code change. The subscriber cap keeps the documented 2× relationship.
|
||||
readonly maxBufferBytes = resolveMaxBufferBytes();
|
||||
readonly subscriberMaxBufferedBytes = 2 * this.maxBufferBytes;
|
||||
|
||||
/**
|
||||
* Register a fresh entry at the START of a run (before any frame), so a tab
|
||||
@@ -105,7 +209,11 @@ export class AiChatStreamRegistryService implements OnModuleDestroy {
|
||||
this.entries.set(chatId, {
|
||||
runId,
|
||||
frames: [],
|
||||
stamps: [],
|
||||
bytes: 0,
|
||||
currentStamp: 0,
|
||||
persistedFloor: 0,
|
||||
overflowThroughStamp: -1,
|
||||
overflowed: false,
|
||||
finished: false,
|
||||
subscribers: new Set<Subscriber>(),
|
||||
@@ -150,6 +258,34 @@ export class AiChatStreamRegistryService implements OnModuleDestroy {
|
||||
void pump();
|
||||
}
|
||||
|
||||
/**
|
||||
* Confirm that step `stepsPersisted` (a COUNT: steps 0..stepsPersisted-1) is on
|
||||
* disk for this run, and ROTATE the ring: drop the buffered frames of those
|
||||
* now-persisted steps (stamp < stepsPersisted). This is the ONLY thing that
|
||||
* rotates the ring, and it is called ONLY after a genuinely SUCCESSFUL per-step
|
||||
* persist (see ai-chat.service updateStreaming). A failed persist never calls
|
||||
* it, so the ring covers more (auto-safe). Identity-checked (invariant 1) and
|
||||
* monotonic (a stale lower count is ignored).
|
||||
*/
|
||||
confirmPersistedStep(
|
||||
chatId: string,
|
||||
runId: string,
|
||||
stepsPersisted: number,
|
||||
): void {
|
||||
const entry = this.entries.get(chatId);
|
||||
if (!entry || entry.runId !== runId) return;
|
||||
if (!Number.isFinite(stepsPersisted) || stepsPersisted <= entry.persistedFloor)
|
||||
return;
|
||||
entry.persistedFloor = stepsPersisted;
|
||||
// Clean rotation: drop the persisted steps from the head. These frames are on
|
||||
// disk + carried by a fresh client seed, so this NEVER opens a gap.
|
||||
while (entry.frames.length > 0 && entry.stamps[0] < stepsPersisted) {
|
||||
entry.bytes -= Buffer.byteLength(entry.frames[0]);
|
||||
entry.frames.shift();
|
||||
entry.stamps.shift();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminate a run's entry from the OUTER catch of the stream method (a failure
|
||||
* before/while wiring the pipe, so `done` will never arrive). Identity-checked
|
||||
@@ -162,36 +298,77 @@ export class AiChatStreamRegistryService implements OnModuleDestroy {
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach to a run's stream. Async only for the phase-2 Redis seam — the body
|
||||
* runs synchronously so the replay snapshot and the subscriber registration
|
||||
* happen in ONE tick with no await between them (invariant 4): a frame ingested
|
||||
* concurrently cannot slip into the gap and be lost or duplicated.
|
||||
* Attach to a run's stream from the client's step frontier `n` (its persisted
|
||||
* `stepsPersisted`). Async only for the phase-2 Redis seam — the body runs
|
||||
* synchronously so the tail SLICE and the subscriber registration happen in ONE
|
||||
* tick with no await between them (invariant 4).
|
||||
*
|
||||
* Returns null (-> the caller answers 204) when:
|
||||
* - there is no entry, or it overflowed (replay is gone);
|
||||
* - expect=live with an anchor that does not match this run's assistant id
|
||||
* (invariant 6: a stripped tab must never replay a FOREIGN run's transcript);
|
||||
* - the run finished and the caller did not expect a live tail.
|
||||
* A finished run with expect=live yields a replay-only attachment (no
|
||||
* subscriber registered). Otherwise a paused subscriber is registered and the
|
||||
* caller replays `replay`, then calls start() to drain and go live.
|
||||
* - there is no entry;
|
||||
* - the `anchor` does not match this run's assistant id (invariant 6);
|
||||
* - the ring does not cover the client's frontier (coverageFloor > n): a hole
|
||||
* from overflow, or the client's seed simply lagged behind a rotation. The
|
||||
* client then refetches (a larger n) and re-attaches.
|
||||
*
|
||||
* Otherwise the attachment's `replay` is a synthetic `start` frame (the run-fact
|
||||
* on re-attach) followed by the buffered tail filtered to `stamp >= n`. For a
|
||||
* FINISHED run this is replay-only (no subscriber) and ends after the replay —
|
||||
* with n = N_final that tail is just the run's `finish` frame, so the client
|
||||
* closes the stream. For a LIVE run a paused subscriber is registered; the
|
||||
* caller writes the replay (respecting drain) then calls start() to drain the
|
||||
* pending frames and go live.
|
||||
*/
|
||||
async attach(
|
||||
chatId: string,
|
||||
expectLive: boolean,
|
||||
anchor: string | undefined,
|
||||
// The client's persisted step frontier. `null` = a NOT-tail-aware client (no
|
||||
// `n` query param) — a legacy/parameterless tab that expects the old
|
||||
// "finished -> 204 -> poll" contract; distinct from `0` (a tail-aware client
|
||||
// with nothing persisted yet).
|
||||
n: number | null,
|
||||
cb: RunStreamCallbacks,
|
||||
): Promise<RunStreamAttachment | null> {
|
||||
const entry = this.entries.get(chatId);
|
||||
if (!entry || entry.overflowed) return null;
|
||||
if (!entry) return null;
|
||||
// Invariant 6: cross-run replay is forbidden. Before bind, assistantMessageId
|
||||
// is undefined and mismatches any anchor -> 204 -> client restore+poll path.
|
||||
if (expectLive && anchor && entry.assistantMessageId !== anchor) return null;
|
||||
if (entry.finished && !expectLive) return null;
|
||||
if (entry.finished && expectLive) {
|
||||
if (anchor && entry.assistantMessageId !== anchor) return null;
|
||||
// #491 regression guard (#137/#161 dup): a NOT-tail-aware client (no `n`)
|
||||
// resuming a FINISHED run must 204 and poll — the old `finished && !expectLive`
|
||||
// gate. Without this, a missing `n` collapsing to frontier 0 would serve the
|
||||
// WHOLE tail of a finished, NON-rotated run (coverageFloor 0), and a
|
||||
// parameterless client that never stripped its transcript would APPEND that
|
||||
// full replay onto the steps it already shows -> duplicated text. A tail-aware
|
||||
// client (n present, incl. n=0) still gets the tail past its frontier.
|
||||
if (entry.finished && n === null) return null;
|
||||
// A finished entry with NOTHING in the ring (aborted before the first frame,
|
||||
// or fully overflowed) has no tail to deliver -> 204 -> the client polls.
|
||||
if (entry.finished && entry.frames.length === 0) return null;
|
||||
// A LIVE run with no `n` (legacy parameterless) replays from step 0 (the old
|
||||
// behavior); a tail-aware client resumes from its frontier.
|
||||
const frontier = n ?? 0;
|
||||
const floor = this.coverageFloor(entry);
|
||||
if (floor > frontier) {
|
||||
this.logger.warn(
|
||||
`run-stream attach gap for run=${entry.runId}: coverageFloor=${floor} ` +
|
||||
`> client frontier=${frontier} -> 204 (client refetches + re-attaches)`,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
const startFrame = this.buildStartFrame(chatId, entry.runId);
|
||||
const sliceTail = (): string[] => {
|
||||
const out: string[] = [startFrame];
|
||||
for (let i = 0; i < entry.frames.length; i++) {
|
||||
if (entry.stamps[i] >= frontier) out.push(entry.frames[i]);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
if (entry.finished) {
|
||||
// Replay-only: the run is done, no subscriber is registered.
|
||||
return {
|
||||
replay: entry.frames.slice(),
|
||||
replay: sliceTail(),
|
||||
finished: true,
|
||||
start: () => undefined,
|
||||
unsubscribe: () => undefined,
|
||||
@@ -206,15 +383,12 @@ export class AiChatStreamRegistryService implements OnModuleDestroy {
|
||||
pendingBytes: 0,
|
||||
overflowed: false,
|
||||
pendingEnd: false,
|
||||
minStamp: frontier,
|
||||
};
|
||||
// Register + snapshot in the SAME synchronous block (invariant 4). No await
|
||||
// separates them, so a concurrently ingested frame cannot be lost/duplicated.
|
||||
entry.subscribers.add(sub);
|
||||
// Snapshot in the SAME synchronous block as the registration (invariant 4).
|
||||
const replay = entry.frames.slice();
|
||||
// CONTRACT: the caller MUST call start() in the SAME tick as this attach()
|
||||
// returns — no await between them. While a subscriber is paused, every frame
|
||||
// is buffered in sub.pending; a delayed start() lets a whole run accumulate
|
||||
// there. The pendingBytes cap (see ingestFrame) is the structural backstop if
|
||||
// that contract is ever broken (e.g. the phase-2 Redis await seam).
|
||||
const replay = sliceTail();
|
||||
return {
|
||||
replay,
|
||||
finished: false,
|
||||
@@ -263,24 +437,83 @@ export class AiChatStreamRegistryService implements OnModuleDestroy {
|
||||
this.entries.clear();
|
||||
}
|
||||
|
||||
/** Buffer + fan-out a single frame. See invariant/overflow semantics inline. */
|
||||
/** The synthetic `start` frame the tail is prefixed with — the source of the
|
||||
* run-fact (runId/chatId) on re-attach. A `start` frame does NOT reset the
|
||||
* client's message parts (ai@6.0.207 createStreamingUIMessageState), so it is
|
||||
* safe to prepend even when the sliced tail begins mid-message. */
|
||||
private buildStartFrame(chatId: string, runId: string): string {
|
||||
return `data: ${JSON.stringify({
|
||||
type: 'start',
|
||||
messageMetadata: { runId, chatId },
|
||||
})}\n\n`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The smallest step FULLY present in the ring: its smallest retained stamp, or
|
||||
* (when the leading step was only partially evicted by an overflow) one past it.
|
||||
* When the ring is empty it is the current step (only the live tail is coming).
|
||||
* An attach at frontier `n` is covered ⟺ coverageFloor <= n.
|
||||
*/
|
||||
private coverageFloor(entry: Entry): number {
|
||||
// Empty ring: only the live tail is coming. The floor is the current step,
|
||||
// but never below persistedFloor — a confirmed persist can rotate the ring
|
||||
// empty while currentStamp still lags a beat behind on another connection, so
|
||||
// max() keeps the invariant STRUCTURAL (a client with n = persistedFloor is
|
||||
// always covered) rather than timing-dependent.
|
||||
if (entry.frames.length === 0)
|
||||
return Math.max(entry.currentStamp, entry.persistedFloor);
|
||||
const min = entry.stamps[0];
|
||||
return entry.overflowThroughStamp >= min ? min + 1 : min;
|
||||
}
|
||||
|
||||
/**
|
||||
* Buffer (step-stamped) + fan-out a single frame. The stamp is the number of
|
||||
* finish-step frames seen BEFORE this one; a finish-step frame carries the
|
||||
* current value and THEN increments the counter (so its stamp equals the 0-based
|
||||
* index of the step it closes). Only frames at/above persistedFloor are buffered
|
||||
* (already-persisted steps are on disk); the ring is then trimmed to the byte
|
||||
* cap, an unsafe eviction opening a gap. Fan-out is always live (filtered per
|
||||
* subscriber by its frontier).
|
||||
*/
|
||||
private ingestFrame(entry: Entry, frame: string): void {
|
||||
entry.bytes += Buffer.byteLength(frame);
|
||||
if (!entry.overflowed) {
|
||||
const size = Buffer.byteLength(frame);
|
||||
const stamp = entry.currentStamp;
|
||||
if (frame.startsWith(FINISH_STEP_FRAME_PREFIX)) {
|
||||
entry.currentStamp = stamp + 1;
|
||||
}
|
||||
|
||||
// Buffer for replay only if this step is not already persisted+rotated away.
|
||||
if (stamp >= entry.persistedFloor) {
|
||||
entry.frames.push(frame);
|
||||
if (entry.bytes > RUN_STREAM_MAX_BUFFER_BYTES) {
|
||||
// The crossing frame was already counted AND (below) fanned out; only the
|
||||
// replay buffer is dropped. After overflow no more frames are buffered,
|
||||
// but live fan-out continues.
|
||||
entry.overflowed = true;
|
||||
entry.frames = [];
|
||||
this.logger.warn(
|
||||
`run-stream buffer overflow for run=${entry.runId}; ` +
|
||||
`late attach will 204 until the run ends`,
|
||||
);
|
||||
entry.stamps.push(stamp);
|
||||
entry.bytes += size;
|
||||
// Enforce the ring cap. Evicting a not-yet-persisted frame (stamp >=
|
||||
// persistedFloor) opens a GAP; a leftover persisted frame (< floor) is a
|
||||
// safe drop. Keep evicting until the ring is back under the cap.
|
||||
while (entry.bytes > this.maxBufferBytes && entry.frames.length > 0) {
|
||||
const evStamp = entry.stamps[0];
|
||||
entry.bytes -= Buffer.byteLength(entry.frames[0]);
|
||||
entry.frames.shift();
|
||||
entry.stamps.shift();
|
||||
if (evStamp >= entry.persistedFloor) {
|
||||
if (evStamp > entry.overflowThroughStamp)
|
||||
entry.overflowThroughStamp = evStamp;
|
||||
if (!entry.overflowed) {
|
||||
entry.overflowed = true;
|
||||
this.logger.warn(
|
||||
`run-stream ring overflow for run=${entry.runId}: an un-persisted ` +
|
||||
`step was evicted to stay under ${this.maxBufferBytes}B; a late ` +
|
||||
`attach at an evicted step will 204 until a later persist confirms`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fan out live, filtered to each subscriber's frontier (a subscriber only
|
||||
// wants the tail past the step it already persisted).
|
||||
for (const sub of entry.subscribers) {
|
||||
if (stamp < sub.minStamp) continue;
|
||||
if (sub.started) {
|
||||
try {
|
||||
sub.onFrame(frame);
|
||||
@@ -289,12 +522,12 @@ export class AiChatStreamRegistryService implements OnModuleDestroy {
|
||||
}
|
||||
} else {
|
||||
sub.pending.push(frame);
|
||||
sub.pendingBytes += Buffer.byteLength(frame);
|
||||
if (sub.pendingBytes > SUBSCRIBER_MAX_BUFFERED_BYTES) {
|
||||
sub.pendingBytes += size;
|
||||
if (sub.pendingBytes > this.subscriberMaxBufferedBytes) {
|
||||
// The paused subscriber's buffer overflowed — only possible if start()
|
||||
// was delayed past the same-tick contract (the phase-2 await seam).
|
||||
// Drop it rather than buffer the whole run; on start() it degrades to an
|
||||
// immediate end (a 204-equivalent) instead of replaying a partial.
|
||||
// was delayed (the controller's drain-respecting tail write, or the
|
||||
// phase-2 await seam). Drop it rather than buffer the whole run; on
|
||||
// start() it degrades to an immediate end (a 204-equivalent).
|
||||
sub.overflowed = true;
|
||||
sub.pending = [];
|
||||
entry.subscribers.delete(sub);
|
||||
|
||||
@@ -1,19 +1,27 @@
|
||||
import {
|
||||
AiChatStreamRegistryService,
|
||||
RUN_STREAM_MAX_BUFFER_BYTES,
|
||||
AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES,
|
||||
RUN_STREAM_RETAIN_FINISHED_MS,
|
||||
SUBSCRIBER_MAX_BUFFERED_BYTES,
|
||||
RunStreamCallbacks,
|
||||
} from './ai-chat-stream-registry.service';
|
||||
|
||||
/**
|
||||
* Unit tests for the in-memory run-stream registry (#184 phase 1.5). The registry
|
||||
* is the whole of the resumable-transport contract: replay ordering, paused ->
|
||||
* live hand-off, overflow, retention, the anchor check (invariant 6), and the
|
||||
* mirror-the-done-path replace semantics (invariant 3). Every enumerated case in
|
||||
* the issue's task 1.5 has a test here.
|
||||
* Unit tests for the in-memory run-stream registry (#184 phase 1.5, step-aligned
|
||||
* retention #491). The registry is the whole of the resumable-transport contract:
|
||||
* step-stamped retention, tail-only attach at the client's frontier N, the
|
||||
* confirmed-persist ring rotation (and the anti-inversion rule), the memory bound,
|
||||
* the overflow gap, paused -> live hand-off, retention, the anchor check
|
||||
* (invariant 6), and the mirror-the-done-path replace semantics (invariant 3).
|
||||
*/
|
||||
|
||||
// Real ai@6 UI-message-stream SSE frames are `data: {json}\n\n`, one part each.
|
||||
const sse = (part: Record<string, unknown>): string =>
|
||||
`data: ${JSON.stringify(part)}\n\n`;
|
||||
const finishStep = (): string => sse({ type: 'finish-step' });
|
||||
const textDelta = (id: string, delta: string): string =>
|
||||
sse({ type: 'text-delta', id, delta });
|
||||
const finish = (): string => sse({ type: 'finish' });
|
||||
|
||||
// A ReadableStream whose frames the test pushes explicitly, plus close/error.
|
||||
function makePushStream(): {
|
||||
stream: ReadableStream<string>;
|
||||
@@ -58,6 +66,9 @@ function collector(): {
|
||||
};
|
||||
}
|
||||
|
||||
// The tail past the synthetic start frame (replay[0] is always the start frame).
|
||||
const tail = (replay: string[]): string[] => replay.slice(1);
|
||||
|
||||
describe('AiChatStreamRegistryService', () => {
|
||||
const CHAT = 'chat-1';
|
||||
let registry: AiChatStreamRegistryService;
|
||||
@@ -71,7 +82,21 @@ describe('AiChatStreamRegistryService', () => {
|
||||
registry.onModuleDestroy();
|
||||
});
|
||||
|
||||
it('replays frames in arrival order (live attach)', async () => {
|
||||
it('prepends a synthetic start frame carrying { runId, chatId }', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push('a');
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!;
|
||||
const start = JSON.parse(att.replay[0].replace(/^data: /, '').trim());
|
||||
expect(start.type).toBe('start');
|
||||
expect(start.messageMetadata).toEqual({ runId: 'run-1', chatId: CHAT });
|
||||
});
|
||||
|
||||
it('replays the buffered tail (from frontier 0) in arrival order (live attach)', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
@@ -81,13 +106,13 @@ describe('AiChatStreamRegistryService', () => {
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
const att = await registry.attach(CHAT, false, undefined, c.cb);
|
||||
const att = await registry.attach(CHAT, 'assist-1', 0, c.cb);
|
||||
expect(att).not.toBeNull();
|
||||
expect(att!.replay).toEqual(['a', 'b', 'c']);
|
||||
expect(tail(att!.replay)).toEqual(['a', 'b', 'c']);
|
||||
expect(att!.finished).toBe(false);
|
||||
});
|
||||
|
||||
it('late attach gets the full prefix as replay plus the live tail', async () => {
|
||||
it('late attach gets the buffered prefix as tail plus the live tail', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
@@ -96,17 +121,16 @@ describe('AiChatStreamRegistryService', () => {
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
const att = (await registry.attach(CHAT, false, undefined, c.cb))!;
|
||||
expect(att.replay).toEqual(['a', 'b']);
|
||||
const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!;
|
||||
expect(tail(att.replay)).toEqual(['a', 'b']);
|
||||
att.start();
|
||||
// Live tail arrives after start().
|
||||
src.push('c');
|
||||
src.push('d');
|
||||
await flush();
|
||||
expect(c.frames).toEqual(['c', 'd']);
|
||||
});
|
||||
|
||||
it('a paused subscriber receives frames buffered during pause in order, then live (no loss/reorder)', async () => {
|
||||
it('a paused subscriber receives frames buffered during pause in order, then live', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
@@ -114,81 +138,45 @@ describe('AiChatStreamRegistryService', () => {
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
// Attach (paused). Frames that arrive BEFORE start() must queue, not drop.
|
||||
const att = (await registry.attach(CHAT, false, undefined, c.cb))!;
|
||||
expect(att.replay).toEqual(['a']);
|
||||
const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!;
|
||||
expect(tail(att.replay)).toEqual(['a']);
|
||||
src.push('b'); // arrives while paused -> pending
|
||||
src.push('c');
|
||||
await flush();
|
||||
expect(c.frames).toEqual([]); // nothing delivered yet (paused)
|
||||
att.start(); // drains pending in order
|
||||
att.start();
|
||||
expect(c.frames).toEqual(['b', 'c']);
|
||||
src.push('d'); // now live
|
||||
src.push('d');
|
||||
await flush();
|
||||
expect(c.frames).toEqual(['b', 'c', 'd']);
|
||||
});
|
||||
|
||||
it('a run that finishes while a subscriber is paused ends it on start()', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', makePushStream().stream);
|
||||
const c = collector();
|
||||
const att = (await registry.attach(CHAT, false, undefined, c.cb))!;
|
||||
// Terminate the run while the subscriber is still paused.
|
||||
const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!;
|
||||
registry.abortEntry(CHAT, 'run-1');
|
||||
expect(c.ended()).toBe(0); // paused: not ended yet
|
||||
att.start();
|
||||
expect(c.ended()).toBe(1); // start() drains + ends
|
||||
});
|
||||
|
||||
it('finished + expect=live returns a replay WITHOUT registering a subscriber', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push('a');
|
||||
src.push('b');
|
||||
src.close();
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
const att = (await registry.attach(CHAT, true, undefined, c.cb))!;
|
||||
expect(att.finished).toBe(true);
|
||||
expect(att.replay).toEqual(['a', 'b']);
|
||||
// No subscriber registered: start()/unsubscribe are no-ops and the entry has
|
||||
// zero subscribers.
|
||||
const entry = (registry as any).entries.get(CHAT);
|
||||
expect(entry.subscribers.size).toBe(0);
|
||||
att.start();
|
||||
expect(c.frames).toEqual([]);
|
||||
});
|
||||
|
||||
it('finished WITHOUT expect=live returns null', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push('a');
|
||||
src.close();
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
expect(await registry.attach(CHAT, false, undefined, c.cb)).toBeNull();
|
||||
});
|
||||
|
||||
it('anchor mismatch with expect=live returns null (and null before bind sets assistantMessageId)', async () => {
|
||||
it('anchor mismatch returns null (and null before bind sets assistantMessageId)', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const c = collector();
|
||||
// Before bind: assistantMessageId is undefined -> mismatches any anchor.
|
||||
expect(
|
||||
await registry.attach(CHAT, true, 'assist-1', c.cb),
|
||||
).toBeNull();
|
||||
expect(await registry.attach(CHAT, 'assist-1', 0, c.cb)).toBeNull();
|
||||
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push('a');
|
||||
await flush();
|
||||
// Wrong anchor -> null (cross-run replay forbidden, invariant 6).
|
||||
expect(await registry.attach(CHAT, true, 'other-id', c.cb)).toBeNull();
|
||||
expect(await registry.attach(CHAT, 'other-id', 0, c.cb)).toBeNull();
|
||||
});
|
||||
|
||||
it('matching anchor with expect=live attaches', async () => {
|
||||
it('matching anchor attaches', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
@@ -196,97 +184,60 @@ describe('AiChatStreamRegistryService', () => {
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
const att = await registry.attach(CHAT, true, 'assist-1', c.cb);
|
||||
const att = await registry.attach(CHAT, 'assist-1', 0, c.cb);
|
||||
expect(att).not.toBeNull();
|
||||
expect(att!.replay).toEqual(['a']);
|
||||
expect(tail(att!.replay)).toEqual(['a']);
|
||||
});
|
||||
|
||||
it('overflow: attach returns null, but the LIVE subscriber keeps receiving (incl. the crossing frame)', async () => {
|
||||
it('a throwing onFrame ejects only that subscriber; the ingest loop stays alive', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
|
||||
// A live (started) subscriber attached before the flood.
|
||||
const bad = collector();
|
||||
const badAtt = (await registry.attach(CHAT, 'assist-1', 0, {
|
||||
onFrame: () => {
|
||||
throw new Error('boom');
|
||||
},
|
||||
onEnd: bad.cb.onEnd,
|
||||
}))!;
|
||||
badAtt.start();
|
||||
|
||||
const good = collector();
|
||||
const goodAtt = (await registry.attach(CHAT, 'assist-1', 0, good.cb))!;
|
||||
goodAtt.start();
|
||||
|
||||
src.push('a'); // bad throws on this frame -> ejected
|
||||
src.push('b'); // good still receives both
|
||||
await flush();
|
||||
|
||||
const entry = (registry as any).entries.get(CHAT);
|
||||
expect(entry.subscribers.size).toBe(1);
|
||||
expect(good.frames).toEqual(['a', 'b']);
|
||||
});
|
||||
|
||||
it('open() over a LIVE entry ends started subscribers once; a late done never touches the new entry (invariant 3)', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push('a');
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
const att = (await registry.attach(CHAT, false, undefined, c.cb))!;
|
||||
const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!;
|
||||
att.start();
|
||||
|
||||
// Cap-relative so it survives a buffer-cap change (#430): a quarter-cap frame
|
||||
// means 5 frames comfortably exceed the replay cap; the last one crosses.
|
||||
const chunk = 'x'.repeat(Math.floor(RUN_STREAM_MAX_BUFFER_BYTES / 4));
|
||||
for (let i = 0; i < 5; i++) src.push(chunk + i);
|
||||
await flush();
|
||||
|
||||
const entry = (registry as any).entries.get(CHAT);
|
||||
expect(entry.overflowed).toBe(true);
|
||||
expect(entry.bytes).toBeGreaterThan(RUN_STREAM_MAX_BUFFER_BYTES);
|
||||
// The live subscriber received ALL 5 frames, including the crossing one.
|
||||
expect(c.frames).toHaveLength(5);
|
||||
expect(c.frames[4]).toBe(chunk + 4);
|
||||
|
||||
// A NEW attach after overflow gets null (replay buffer is gone).
|
||||
const c2 = collector();
|
||||
expect(await registry.attach(CHAT, false, undefined, c2.cb)).toBeNull();
|
||||
});
|
||||
|
||||
it('a paused subscriber whose pending buffer overflows is dropped and ends on start(); other subscribers keep receiving', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
|
||||
// A: paused (start() deliberately delayed to simulate the phase-2 await seam).
|
||||
const a = collector();
|
||||
const attA = (await registry.attach(CHAT, false, undefined, a.cb))!;
|
||||
// B: live (started) — its delivery must be unaffected by A's overflow.
|
||||
const b = collector();
|
||||
const attB = (await registry.attach(CHAT, false, undefined, b.cb))!;
|
||||
attB.start();
|
||||
|
||||
// Cap-relative so it survives a buffer-cap change (#430): a quarter-of-the-
|
||||
// per-subscriber-cap frame means 5 frames exceed A's paused-pending cap while
|
||||
// B streams every frame live.
|
||||
const chunk = 'x'.repeat(Math.floor(SUBSCRIBER_MAX_BUFFERED_BYTES / 4));
|
||||
for (let i = 0; i < 5; i++) src.push(chunk + i);
|
||||
await flush();
|
||||
|
||||
const entry = (registry as any).entries.get(CHAT);
|
||||
// A was dropped from the subscriber set on overflow; B (started) remains.
|
||||
expect(entry.subscribers.size).toBe(1);
|
||||
expect(a.frames).toEqual([]); // paused + overflowed: nothing was delivered
|
||||
// B received every frame live (delivery unaffected by A's overflow).
|
||||
expect(b.frames).toHaveLength(5);
|
||||
|
||||
// A's start() (arriving late) degrades to an immediate end, not a partial replay.
|
||||
attA.start();
|
||||
expect(a.frames).toEqual([]);
|
||||
expect(a.ended()).toBe(1);
|
||||
});
|
||||
|
||||
it('open() over a LIVE entry ends started subscribers exactly once and a late done does not touch the new entry (invariant 3)', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push('a');
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
const att = (await registry.attach(CHAT, false, undefined, c.cb))!;
|
||||
att.start(); // started subscriber on run-1
|
||||
|
||||
// run-2 starts on the same chat while run-1's tee is still reading.
|
||||
registry.open(CHAT, 'run-2');
|
||||
expect(c.ended()).toBe(1); // exactly one onEnd from the replace
|
||||
expect(c.ended()).toBe(1);
|
||||
|
||||
const newEntry = (registry as any).entries.get(CHAT);
|
||||
expect(newEntry.runId).toBe('run-2');
|
||||
expect(newEntry.finished).toBe(false);
|
||||
|
||||
// The old tee now completes: its late done must NOT double-end nor delete the
|
||||
// new entry.
|
||||
src.push('b');
|
||||
src.close();
|
||||
await flush();
|
||||
expect(c.ended()).toBe(1); // still exactly one
|
||||
expect(c.ended()).toBe(1);
|
||||
const still = (registry as any).entries.get(CHAT);
|
||||
expect(still).toBe(newEntry);
|
||||
expect(still.runId).toBe('run-2');
|
||||
@@ -299,7 +250,6 @@ describe('AiChatStreamRegistryService', () => {
|
||||
src.push('a');
|
||||
await flush();
|
||||
const entry = (registry as any).entries.get(CHAT);
|
||||
// Frames were NOT ingested (bind bailed), assistantMessageId untouched.
|
||||
expect(entry.frames).toEqual([]);
|
||||
expect(entry.assistantMessageId).toBeUndefined();
|
||||
});
|
||||
@@ -310,32 +260,276 @@ describe('AiChatStreamRegistryService', () => {
|
||||
const entry = (registry as any).entries.get(CHAT);
|
||||
expect(entry.finished).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
it('a throwing onFrame ejects only that subscriber; the ingest loop stays alive', async () => {
|
||||
/**
|
||||
* #491 step-stamped retention: the boundary detector, tail-only slicing at the
|
||||
* client's frontier N, the confirmed-persist rotation (+ anti-inversion), the
|
||||
* overflow gap, the memory bound, and the finished-retained tail. All observable
|
||||
* against the REAL registry driven through open/bind/ingest.
|
||||
*/
|
||||
describe('AiChatStreamRegistryService step-aligned retention (#491)', () => {
|
||||
const CHAT = 'chat-s';
|
||||
let registry: AiChatStreamRegistryService;
|
||||
|
||||
beforeEach(() => {
|
||||
registry = new AiChatStreamRegistryService();
|
||||
jest.spyOn((registry as any).logger, 'warn').mockImplementation(() => {});
|
||||
});
|
||||
afterEach(() => registry.onModuleDestroy());
|
||||
|
||||
const entryOf = () => (registry as any).entries.get(CHAT);
|
||||
|
||||
it('stamps frames by finish-step count, aligned with stepsPersisted', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
// step 0 content, its finish-step, step 1 content, its finish-step, finish.
|
||||
src.push(textDelta('t0', 'a')); // stamp 0
|
||||
src.push(finishStep()); // stamp 0 (the finish-step frame carries the pre value)
|
||||
src.push(textDelta('t1', 'b')); // stamp 1
|
||||
src.push(finishStep()); // stamp 1
|
||||
src.push(finish()); // stamp 2
|
||||
await flush();
|
||||
const e = entryOf();
|
||||
expect(e.stamps).toEqual([0, 0, 1, 1, 2]);
|
||||
expect(e.currentStamp).toBe(2);
|
||||
});
|
||||
|
||||
const bad = collector();
|
||||
const badAtt = (await registry.attach(CHAT, false, undefined, {
|
||||
onFrame: () => {
|
||||
throw new Error('boom');
|
||||
},
|
||||
onEnd: bad.cb.onEnd,
|
||||
}))!;
|
||||
badAtt.start();
|
||||
it('does NOT treat a text delta that merely quotes "finish-step" as a boundary', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
// A model that literally types "type":"finish-step" — JSON-escaped in the frame.
|
||||
src.push(textDelta('t0', '"type":"finish-step"'));
|
||||
await flush();
|
||||
expect(entryOf().currentStamp).toBe(0); // no false boundary
|
||||
});
|
||||
|
||||
const good = collector();
|
||||
const goodAtt = (await registry.attach(CHAT, false, undefined, good.cb))!;
|
||||
goodAtt.start();
|
||||
|
||||
src.push('a'); // bad throws on this frame -> ejected
|
||||
src.push('b'); // good still receives both
|
||||
it('tail-only: attach at N slices frames with stamp >= N', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push(textDelta('t0', 'a')); // 0
|
||||
src.push(finishStep()); // 0
|
||||
src.push(textDelta('t1', 'b')); // 1
|
||||
src.push(finishStep()); // 1
|
||||
src.push(textDelta('t2', 'c')); // 2 (in-progress)
|
||||
await flush();
|
||||
|
||||
const entry = (registry as any).entries.get(CHAT);
|
||||
expect(entry.subscribers.size).toBe(1); // bad ejected, good remains
|
||||
expect(good.frames).toEqual(['a', 'b']);
|
||||
const c = collector();
|
||||
// Client persisted 2 steps -> wants the tail from step 2.
|
||||
const att = (await registry.attach(CHAT, 'assist-1', 2, c.cb))!;
|
||||
expect(tail(att.replay)).toEqual([textDelta('t2', 'c')]);
|
||||
});
|
||||
|
||||
it('attach in the MIDDLE of a step (N between finish-steps) slices from that step', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push(textDelta('t0', 'a')); // 0
|
||||
src.push(finishStep()); // 0
|
||||
src.push(textDelta('t1', 'b1')); // 1
|
||||
src.push(textDelta('t1', 'b2')); // 1 (still step 1, no finish-step yet)
|
||||
await flush();
|
||||
|
||||
const c = collector();
|
||||
const att = (await registry.attach(CHAT, 'assist-1', 1, c.cb))!;
|
||||
// Step 0's frames are dropped from the tail; the whole in-progress step 1 is kept.
|
||||
expect(tail(att.replay)).toEqual([textDelta('t1', 'b1'), textDelta('t1', 'b2')]);
|
||||
});
|
||||
|
||||
it('rotates the ring ONLY on a confirmed persist (drops stamp < N)', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push(textDelta('t0', 'a')); // 0
|
||||
src.push(finishStep()); // 0
|
||||
src.push(textDelta('t1', 'b')); // 1
|
||||
await flush();
|
||||
expect(entryOf().stamps).toEqual([0, 0, 1]);
|
||||
|
||||
// Confirm step 0 persisted (stepsPersisted = 1) -> drop stamp < 1.
|
||||
registry.confirmPersistedStep(CHAT, 'run-1', 1);
|
||||
expect(entryOf().stamps).toEqual([1]);
|
||||
expect(entryOf().persistedFloor).toBe(1);
|
||||
});
|
||||
|
||||
it('persist FAILED but the ring still fits -> attach SUCCEEDS and the tail includes step N', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push(textDelta('t0', 'a')); // 0
|
||||
src.push(finishStep()); // 0
|
||||
src.push(textDelta('t1', 'b')); // 1 (step 1's persist FAILED -> no confirm)
|
||||
await flush();
|
||||
// No confirmPersistedStep for step 1: the ring still holds step 1.
|
||||
|
||||
const c = collector();
|
||||
// Client's last successful persist was step 0 -> stepsPersisted = 1.
|
||||
const att = await registry.attach(CHAT, 'assist-1', 1, c.cb);
|
||||
expect(att).not.toBeNull();
|
||||
expect(tail(att!.replay)).toEqual([textDelta('t1', 'b')]); // includes step 1
|
||||
});
|
||||
|
||||
it('persist failed AND the ring overflowed past N -> 204 (coverage gap)', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
// Step 0: a fat step that blows past the cap with NO persist confirmation.
|
||||
const big = 'x'.repeat(Math.floor(AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES / 2));
|
||||
src.push(textDelta('t0', big)); // 0
|
||||
src.push(textDelta('t0', big)); // 0
|
||||
src.push(textDelta('t0', big)); // 0 -> overflow evicts stamp-0 frames
|
||||
await flush();
|
||||
const e = entryOf();
|
||||
expect(e.overflowed).toBe(true);
|
||||
expect(e.bytes).toBeLessThanOrEqual(registry.maxBufferBytes);
|
||||
|
||||
// A client at frontier 0 falls at/below an evicted step -> gap -> null.
|
||||
const c = collector();
|
||||
expect(await registry.attach(CHAT, 'assist-1', 0, c.cb)).toBeNull();
|
||||
});
|
||||
|
||||
it('stale N (client seed lagged behind a rotation) -> 204; after a refetch (larger N) -> success', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push(textDelta('t0', 'a')); // 0
|
||||
src.push(finishStep()); // 0
|
||||
src.push(textDelta('t1', 'b')); // 1
|
||||
src.push(finishStep()); // 1
|
||||
src.push(textDelta('t2', 'c')); // 2
|
||||
await flush();
|
||||
// Server confirmed steps 0 and 1 -> rotate away stamp < 2.
|
||||
registry.confirmPersistedStep(CHAT, 'run-1', 2);
|
||||
expect(entryOf().stamps).toEqual([2]);
|
||||
|
||||
// A client whose seed still says stepsPersisted = 1 -> below minStamp -> 204.
|
||||
const stale = collector();
|
||||
expect(await registry.attach(CHAT, 'assist-1', 1, stale.cb)).toBeNull();
|
||||
|
||||
// It refetches (now stepsPersisted = 2) and re-attaches -> success.
|
||||
const fresh = collector();
|
||||
const att = await registry.attach(CHAT, 'assist-1', 2, fresh.cb);
|
||||
expect(att).not.toBeNull();
|
||||
expect(tail(att!.replay)).toEqual([textDelta('t2', 'c')]);
|
||||
});
|
||||
|
||||
it('overflow gap CLEARS once a later persist rotates out the holey steps', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
const big = 'x'.repeat(Math.floor(AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES / 2));
|
||||
src.push(textDelta('t0', big)); // 0
|
||||
src.push(textDelta('t0', big)); // 0
|
||||
src.push(finishStep()); // 0 (still stamp 0)
|
||||
src.push(textDelta('t1', 'small')); // 1
|
||||
src.push(finishStep()); // 1
|
||||
src.push(textDelta('t2', 'c')); // 2
|
||||
await flush();
|
||||
expect(entryOf().overflowed).toBe(true);
|
||||
|
||||
// Late persist confirms steps 0..1 -> rotates out the holey step-0 frames.
|
||||
registry.confirmPersistedStep(CHAT, 'run-1', 2);
|
||||
// A client at frontier 2 is now cleanly covered (the hole was below it).
|
||||
const c = collector();
|
||||
const att = await registry.attach(CHAT, 'assist-1', 2, c.cb);
|
||||
expect(att).not.toBeNull();
|
||||
expect(tail(att!.replay)).toEqual([textDelta('t2', 'c')]);
|
||||
});
|
||||
|
||||
it('finished-retained + N = N_final -> empty tail plus the finish frame', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push(textDelta('t0', 'a')); // 0
|
||||
src.push(finishStep()); // 0
|
||||
src.push(finish()); // 1 (N_final = 1)
|
||||
src.close();
|
||||
await flush();
|
||||
// The last step's per-step persist confirmed stepsPersisted = 1.
|
||||
registry.confirmPersistedStep(CHAT, 'run-1', 1);
|
||||
|
||||
const c = collector();
|
||||
const att = (await registry.attach(CHAT, 'assist-1', 1, c.cb))!;
|
||||
expect(att.finished).toBe(true);
|
||||
// Empty step tail; just the finish frame so the client's SDK closes the stream.
|
||||
expect(tail(att.replay)).toEqual([finish()]);
|
||||
// No subscriber registered for a finished run.
|
||||
expect(entryOf().subscribers.size).toBe(0);
|
||||
});
|
||||
|
||||
it('#491 regression (#137/#161 dup): a PARAMETERLESS attach (n=null) to a finished NON-rotated run -> 204, but n=0 still gets the tail', async () => {
|
||||
// A finished, non-rotated run: frames present, coverageFloor 0. A missing `n`
|
||||
// (null — a legacy/parameterless tab that never stripped its transcript) must
|
||||
// 204 -> poll, NOT receive the whole tail it would append (duplicate). A
|
||||
// tail-aware client (n=0 present) still resumes.
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push(textDelta('t0', 'a')); // 0
|
||||
src.push(finishStep()); // 0
|
||||
src.push(finish()); // 1
|
||||
src.close();
|
||||
await flush();
|
||||
// NOT rotated (no confirmPersistedStep) -> stamps[0]=0, coverageFloor=0.
|
||||
// MUTATION-VERIFY: revert the `finished && n === null -> null` gate (default n
|
||||
// to 0) and the parameterless attach below serves the full tail instead of 204.
|
||||
expect(await registry.attach(CHAT, 'assist-1', null, collector().cb)).toBeNull();
|
||||
// A tail-aware client at frontier 0 IS served (the distinction: null != 0).
|
||||
const tailAware = await registry.attach(CHAT, 'assist-1', 0, collector().cb);
|
||||
expect(tailAware).not.toBeNull();
|
||||
expect(tailAware!.finished).toBe(true);
|
||||
});
|
||||
|
||||
it('confirmPersistedStep is monotonic and identity-checked', async () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
const src = makePushStream();
|
||||
registry.bind(CHAT, 'run-1', 'assist-1', src.stream);
|
||||
src.push(textDelta('t0', 'a'));
|
||||
src.push(finishStep());
|
||||
src.push(textDelta('t1', 'b'));
|
||||
await flush();
|
||||
registry.confirmPersistedStep(CHAT, 'run-1', 1);
|
||||
expect(entryOf().persistedFloor).toBe(1);
|
||||
// A stale lower count is ignored.
|
||||
registry.confirmPersistedStep(CHAT, 'run-1', 0);
|
||||
expect(entryOf().persistedFloor).toBe(1);
|
||||
// A foreign runId is ignored.
|
||||
registry.confirmPersistedStep(CHAT, 'WRONG', 5);
|
||||
expect(entryOf().persistedFloor).toBe(1);
|
||||
});
|
||||
|
||||
it('MEMORY BOUND: 5 parallel marathon runs each stream well past 32MB; each ring stays <= the cap', async () => {
|
||||
const cap = registry.maxBufferBytes;
|
||||
const chats = ['m0', 'm1', 'm2', 'm3', 'm4'];
|
||||
const srcs = chats.map((chat) => {
|
||||
registry.open(chat, `run-${chat}`);
|
||||
const s = makePushStream();
|
||||
registry.bind(chat, `run-${chat}`, `assist-${chat}`, s.stream);
|
||||
return s;
|
||||
});
|
||||
// ~256KB frames; 160 per chat = 40MB streamed each, well past the old 32MB.
|
||||
// Interleave a finish-step every 8 frames so steps advance realistically. No
|
||||
// persist confirmation -> the ONLY thing keeping memory bounded is the cap.
|
||||
const frame = 'y'.repeat(256 * 1024);
|
||||
for (let batch = 0; batch < 20; batch++) {
|
||||
for (let i = 0; i < 8; i++) {
|
||||
for (const s of srcs) s.push(textDelta('t', frame));
|
||||
}
|
||||
for (const s of srcs) s.push(finishStep());
|
||||
await flush(); // drain the pump so queues never hold a whole run
|
||||
}
|
||||
let total = 0;
|
||||
for (const chat of chats) {
|
||||
const e = (registry as any).entries.get(chat);
|
||||
expect(e.bytes).toBeLessThanOrEqual(cap);
|
||||
total += e.bytes;
|
||||
}
|
||||
// Total retained across all 5 runs is bounded by 5x the per-run cap — the old
|
||||
// registry would have retained ~5x40MB = 200MB here.
|
||||
expect(total).toBeLessThanOrEqual(cap * chats.length);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -361,7 +555,7 @@ describe('AiChatStreamRegistryService retention timers', () => {
|
||||
|
||||
it('a finished entry is removed after the retention window', () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
registry.abortEntry(CHAT, 'run-1'); // finalize -> retention armed
|
||||
registry.abortEntry(CHAT, 'run-1');
|
||||
expect((registry as any).entries.get(CHAT)).toBeDefined();
|
||||
jest.advanceTimersByTime(RUN_STREAM_RETAIN_FINISHED_MS + 1);
|
||||
expect((registry as any).entries.get(CHAT)).toBeUndefined();
|
||||
@@ -369,20 +563,18 @@ describe('AiChatStreamRegistryService retention timers', () => {
|
||||
|
||||
it('retention deletes ONLY its own entry (invariant 2)', () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
registry.abortEntry(CHAT, 'run-1'); // arm retention for entry A
|
||||
// Simulate the race where the key was replaced without clearing A's timer.
|
||||
registry.abortEntry(CHAT, 'run-1');
|
||||
const sentinel = { marker: true };
|
||||
(registry as any).entries.set(CHAT, sentinel);
|
||||
jest.advanceTimersByTime(RUN_STREAM_RETAIN_FINISHED_MS + 1);
|
||||
// A's timer saw entries.get(CHAT) !== A, so it did NOT delete the successor.
|
||||
expect((registry as any).entries.get(CHAT)).toBe(sentinel);
|
||||
});
|
||||
|
||||
it('open() over a retained entry clears its timer and the successor survives', () => {
|
||||
registry.open(CHAT, 'run-1');
|
||||
registry.abortEntry(CHAT, 'run-1'); // retained, timer armed
|
||||
registry.abortEntry(CHAT, 'run-1');
|
||||
const clearSpy = jest.spyOn(global, 'clearTimeout');
|
||||
registry.open(CHAT, 'run-2'); // must clear run-1's retain timer
|
||||
registry.open(CHAT, 'run-2');
|
||||
expect(clearSpy).toHaveBeenCalled();
|
||||
jest.advanceTimersByTime(RUN_STREAM_RETAIN_FINISHED_MS + 1);
|
||||
const entry = (registry as any).entries.get(CHAT);
|
||||
|
||||
@@ -8,10 +8,12 @@ import { SUBSCRIBER_MAX_BUFFERED_BYTES } from './ai-chat-stream-registry.service
|
||||
import type { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
|
||||
/**
|
||||
* Wiring spec for the #184 phase 1.5 attach endpoint
|
||||
* Wiring spec for the #184 phase 1.5 attach endpoint (tail-only #491)
|
||||
* (`GET /ai-chat/runs/:chatId/stream`). Owner-gated via assertOwnedChat; the
|
||||
* registry is mocked so this exercises ONLY the controller's replay/live/204/
|
||||
* cleanup wiring against a fake raw socket. Constructor order is (aiChatService,
|
||||
* registry is mocked so this exercises ONLY the controller's tail-write/live/204/
|
||||
* cleanup wiring against a fake raw socket. The attach signature is now
|
||||
* `(chatId, anchor, n, cb)` — the client hands its persisted step frontier `n`
|
||||
* and its assistant row id `anchor`. Constructor order is (aiChatService,
|
||||
* aiChatRunService, aiChatRepo, aiChatMessageRepo, aiTranscription, pageRepo,
|
||||
* streamRegistry, environment).
|
||||
*/
|
||||
@@ -86,8 +88,8 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => {
|
||||
attach: jest.fn(
|
||||
(
|
||||
_chatId: string,
|
||||
_live: boolean,
|
||||
_anchor: string | undefined,
|
||||
_n: number,
|
||||
cb: RunStreamCallbacks,
|
||||
) => {
|
||||
capturedCb = cb;
|
||||
@@ -156,7 +158,7 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => {
|
||||
expect(res.hijack).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('threads expect=live and anchor through to the registry', async () => {
|
||||
it('threads anchor and the numeric frontier n through to the registry', async () => {
|
||||
const { controller, streamRegistry } = makeController({
|
||||
chat: owned,
|
||||
attachment: null,
|
||||
@@ -165,8 +167,8 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => {
|
||||
const { req } = makeReq();
|
||||
await controller.attachRunStream(
|
||||
'c1',
|
||||
'live',
|
||||
'anchor-1',
|
||||
'2',
|
||||
req,
|
||||
res,
|
||||
user,
|
||||
@@ -174,13 +176,44 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => {
|
||||
);
|
||||
expect(streamRegistry.attach).toHaveBeenCalledWith(
|
||||
'c1',
|
||||
true,
|
||||
'anchor-1',
|
||||
2, // parsed to a number
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('passes expect=false when the query is absent', async () => {
|
||||
it('#491: an ABSENT/invalid n passes null (not 0) so a finished run 204s (not-tail-aware)', async () => {
|
||||
// Distinguishing a MISSING `n` from `n=0` is the #137/#161 dup guard: a
|
||||
// parameterless/legacy tab must be handed null (-> the registry 204s a finished
|
||||
// run) rather than frontier 0 (which would serve a finished non-rotated run's
|
||||
// whole tail). MUTATION-VERIFY: revert to `Number(n) || 0` and this asserts 0.
|
||||
const { controller, streamRegistry } = makeController({
|
||||
chat: owned,
|
||||
attachment: null,
|
||||
});
|
||||
for (const bad of [undefined, '', 'abc']) {
|
||||
streamRegistry.attach.mockClear();
|
||||
const { res } = makeRawRes();
|
||||
const { req } = makeReq();
|
||||
await controller.attachRunStream(
|
||||
'c1',
|
||||
undefined,
|
||||
bad,
|
||||
req,
|
||||
res,
|
||||
user,
|
||||
workspace,
|
||||
);
|
||||
expect(streamRegistry.attach).toHaveBeenCalledWith(
|
||||
'c1',
|
||||
undefined,
|
||||
null,
|
||||
expect.anything(),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it('#491: a PRESENT n=0 passes 0 (tail-aware, distinct from absent)', async () => {
|
||||
const { controller, streamRegistry } = makeController({
|
||||
chat: owned,
|
||||
attachment: null,
|
||||
@@ -190,7 +223,7 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => {
|
||||
await controller.attachRunStream(
|
||||
'c1',
|
||||
undefined,
|
||||
undefined,
|
||||
'0',
|
||||
req,
|
||||
res,
|
||||
user,
|
||||
@@ -198,8 +231,8 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => {
|
||||
);
|
||||
expect(streamRegistry.attach).toHaveBeenCalledWith(
|
||||
'c1',
|
||||
false,
|
||||
undefined,
|
||||
0,
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
@@ -245,8 +278,8 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => {
|
||||
const { req } = makeReq();
|
||||
await controller.attachRunStream(
|
||||
'c1',
|
||||
'live',
|
||||
'a1',
|
||||
'1',
|
||||
req,
|
||||
res,
|
||||
user,
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import { ForbiddenException } from '@nestjs/common';
|
||||
import { AiChatController } from './ai-chat.controller';
|
||||
import type { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
|
||||
/**
|
||||
* Wiring spec for the #491 delta-poll endpoint (`POST /ai-chat/messages/delta`).
|
||||
* Owner-gated via assertOwnedChat (same gate as the other reads), NOT flag-gated.
|
||||
* The run fact rides IN the delta response (no separate /run poll). Hand-rolled
|
||||
* mocks — no Nest graph, no DB. Constructor order: (aiChatService,
|
||||
* aiChatRunService, aiChatRepo, aiChatMessageRepo, aiTranscription, pageRepo).
|
||||
*/
|
||||
describe('AiChatController POST /ai-chat/messages/delta (#491)', () => {
|
||||
const user = { id: 'u1' } as User;
|
||||
const workspace = { id: 'ws1' } as Workspace;
|
||||
|
||||
function makeController(opts: {
|
||||
chat?: unknown;
|
||||
delta?: { rows: unknown[]; cursor: string };
|
||||
run?: unknown;
|
||||
}) {
|
||||
const aiChatRunService = {
|
||||
getLatestForChat: jest.fn().mockResolvedValue(opts.run),
|
||||
};
|
||||
const aiChatRepo = {
|
||||
findById: jest.fn().mockResolvedValue(opts.chat),
|
||||
};
|
||||
const aiChatMessageRepo = {
|
||||
findByChatUpdatedAfter: jest
|
||||
.fn()
|
||||
.mockResolvedValue(opts.delta ?? { rows: [], cursor: 'C1' }),
|
||||
};
|
||||
const controller = new AiChatController(
|
||||
{} as never,
|
||||
aiChatRunService as never,
|
||||
aiChatRepo as never,
|
||||
aiChatMessageRepo as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
return { controller, aiChatRunService, aiChatRepo, aiChatMessageRepo };
|
||||
}
|
||||
|
||||
it('owner-gates: a chat the user does not own throws, never reaching the repo', async () => {
|
||||
const { controller, aiChatMessageRepo, aiChatRunService } = makeController({
|
||||
chat: { id: 'c1', creatorId: 'someone-else' },
|
||||
});
|
||||
await expect(
|
||||
controller.getMessagesDelta({ chatId: 'c1' }, user, workspace),
|
||||
).rejects.toBeInstanceOf(ForbiddenException);
|
||||
expect(aiChatMessageRepo.findByChatUpdatedAfter).not.toHaveBeenCalled();
|
||||
expect(aiChatRunService.getLatestForChat).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns { rows, cursor, run:{id,status} } with the run fact inlined', async () => {
|
||||
const rows = [{ id: 'm1' }];
|
||||
const { controller } = makeController({
|
||||
chat: { id: 'c1', creatorId: 'u1' },
|
||||
delta: { rows, cursor: 'C2' },
|
||||
run: { id: 'r1', status: 'running', error: 'ignored', stepCount: 3 },
|
||||
});
|
||||
const res = await controller.getMessagesDelta(
|
||||
{ chatId: 'c1', cursor: 'C1' },
|
||||
user,
|
||||
workspace,
|
||||
);
|
||||
expect(res).toEqual({
|
||||
rows,
|
||||
cursor: 'C2',
|
||||
// ONLY id + status — never the whole run row.
|
||||
run: { id: 'r1', status: 'running' },
|
||||
});
|
||||
});
|
||||
|
||||
it('run is null when the chat has never had a run', async () => {
|
||||
const { controller } = makeController({
|
||||
chat: { id: 'c1', creatorId: 'u1' },
|
||||
run: undefined,
|
||||
});
|
||||
const res = await controller.getMessagesDelta(
|
||||
{ chatId: 'c1' },
|
||||
user,
|
||||
workspace,
|
||||
);
|
||||
expect(res.run).toBeNull();
|
||||
});
|
||||
|
||||
it('passes cursor through, defaulting a missing cursor to null (first poll)', async () => {
|
||||
const { controller, aiChatMessageRepo } = makeController({
|
||||
chat: { id: 'c1', creatorId: 'u1' },
|
||||
});
|
||||
await controller.getMessagesDelta({ chatId: 'c1' }, user, workspace);
|
||||
expect(aiChatMessageRepo.findByChatUpdatedAfter).toHaveBeenCalledWith(
|
||||
'c1',
|
||||
'ws1',
|
||||
null,
|
||||
);
|
||||
await controller.getMessagesDelta(
|
||||
{ chatId: 'c1', cursor: 'CX' },
|
||||
user,
|
||||
workspace,
|
||||
);
|
||||
expect(aiChatMessageRepo.findByChatUpdatedAfter).toHaveBeenLastCalledWith(
|
||||
'c1',
|
||||
'ws1',
|
||||
'CX',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -115,7 +115,7 @@ describe('finalizeAssistant dispatch (planFinalizeAssistant + applyFinalize)', (
|
||||
|
||||
// Drive the SAME applyFinalize the service calls (no duplicated logic).
|
||||
async function dispatchFinalize(
|
||||
repo: { insert: jest.Mock; update: jest.Mock },
|
||||
repo: { insert: jest.Mock; finalizeOwner: jest.Mock },
|
||||
assistantId: string | undefined,
|
||||
flushed: AssistantFlush,
|
||||
): Promise<void> {
|
||||
@@ -135,21 +135,22 @@ describe('finalizeAssistant dispatch (planFinalizeAssistant + applyFinalize)', (
|
||||
expect(planFinalizeAssistant(undefined)).toEqual({ kind: 'insert' });
|
||||
});
|
||||
|
||||
it('(a) upfront insert succeeded -> finalize UPDATEs the row by id', async () => {
|
||||
const repo = { insert: jest.fn(), update: jest.fn() };
|
||||
it('(a) upfront insert succeeded -> finalize CONDITIONALLY updates the row by id (#487 owner-write)', async () => {
|
||||
const repo = { insert: jest.fn(), finalizeOwner: jest.fn() };
|
||||
const flushed = flushAssistant([], 'final answer', 'completed', {
|
||||
finishReason: 'stop',
|
||||
});
|
||||
await dispatchFinalize(repo, 'a1', flushed);
|
||||
expect(repo.update).toHaveBeenCalledWith('a1', workspaceId, flushed);
|
||||
// #487: the owner write is the CONDITIONAL finalizeOwner, not a raw update.
|
||||
expect(repo.finalizeOwner).toHaveBeenCalledWith('a1', workspaceId, flushed);
|
||||
expect(repo.insert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('(b) upfront insert failed -> finalize INSERTs the terminal payload', async () => {
|
||||
const repo = { insert: jest.fn(), update: jest.fn() };
|
||||
const repo = { insert: jest.fn(), finalizeOwner: jest.fn() };
|
||||
const flushed = flushAssistant([], 'partial', 'error', { error: 'boom' });
|
||||
await dispatchFinalize(repo, undefined, flushed);
|
||||
expect(repo.update).not.toHaveBeenCalled();
|
||||
expect(repo.finalizeOwner).not.toHaveBeenCalled();
|
||||
expect(repo.insert).toHaveBeenCalledTimes(1);
|
||||
const arg = repo.insert.mock.calls[0][0];
|
||||
// The fallback insert carries the terminal content/status/metadata.
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
ConflictException,
|
||||
ForbiddenException,
|
||||
HttpException,
|
||||
} from '@nestjs/common';
|
||||
import { AiChatController } from './ai-chat.controller';
|
||||
import type { User, Workspace } from '@docmost/db/types/entity.types';
|
||||
|
||||
/**
|
||||
* #487 commit 3 — the single concurrency GATE (both modes) + the server supersede
|
||||
* CAS, at the controller boundary. The gate + CAS run BEFORE res.hijack(), so a
|
||||
* rejected concurrent start / a CAS branch returns clean JSON (an HttpException
|
||||
* the controller's post-hijack catch re-serializes). These assert the OBSERVABLE
|
||||
* HTTP contract against the real controller + a stubbed run service.
|
||||
*/
|
||||
describe('#487 AiChatController.stream — gate + supersede', () => {
|
||||
const user = { id: 'u1' } as User;
|
||||
|
||||
function wsWith(autonomousRuns: boolean): Workspace {
|
||||
return {
|
||||
id: 'ws1',
|
||||
settings: { ai: { chat: true, autonomousRuns } },
|
||||
} as unknown as Workspace;
|
||||
}
|
||||
|
||||
function makeReqRes(body: Record<string, unknown>) {
|
||||
const req = {
|
||||
raw: { sessionId: 'sess', once: jest.fn(), destroyed: false },
|
||||
body,
|
||||
};
|
||||
const res = {
|
||||
raw: {
|
||||
writableEnded: false,
|
||||
headersSent: false,
|
||||
on: jest.fn(),
|
||||
once: jest.fn(),
|
||||
setHeader: jest.fn(),
|
||||
end: jest.fn(),
|
||||
statusCode: 200,
|
||||
flushHeaders: jest.fn(),
|
||||
},
|
||||
hijack: jest.fn(),
|
||||
status: jest.fn().mockReturnThis(),
|
||||
send: jest.fn(),
|
||||
};
|
||||
return { req, res };
|
||||
}
|
||||
|
||||
function makeController(
|
||||
runServiceOverrides: Record<string, jest.Mock>,
|
||||
// The chat assertOwnedChat resolves. Default: a chat OWNED by `user` (u1), so
|
||||
// the ownership gate is transparent to the gate/CAS assertions below. Pass a
|
||||
// foreign-owner (or undefined) chat to exercise the #487 owner rejection.
|
||||
chat: { creatorId: string } | undefined = { creatorId: 'u1' },
|
||||
) {
|
||||
const aiChatService = {
|
||||
resolveRoleForRequest: jest.fn().mockResolvedValue(null),
|
||||
getChatModel: jest.fn().mockResolvedValue({}),
|
||||
stream: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const aiChatRunService = {
|
||||
getActiveForChat: jest.fn().mockResolvedValue(undefined),
|
||||
supersede: jest.fn(),
|
||||
beginRun: jest.fn().mockResolvedValue({
|
||||
runId: 'run-new',
|
||||
signal: new AbortController().signal,
|
||||
}),
|
||||
linkAssistantMessage: jest.fn(),
|
||||
recordStep: jest.fn(),
|
||||
finalizeRun: jest.fn(),
|
||||
requestStop: jest.fn(),
|
||||
...runServiceOverrides,
|
||||
};
|
||||
const aiChatRepo = { findById: jest.fn().mockResolvedValue(chat) };
|
||||
const controller = new AiChatController(
|
||||
aiChatService as never,
|
||||
aiChatRunService as never,
|
||||
aiChatRepo as never, // aiChatRepo
|
||||
{} as never, // aiChatMessageRepo
|
||||
{} as never, // aiTranscription
|
||||
{} as never, // pageRepo
|
||||
);
|
||||
return { controller, aiChatService, aiChatRunService, aiChatRepo };
|
||||
}
|
||||
|
||||
const codeOf = (err: unknown) =>
|
||||
(((err as HttpException).getResponse() as Record<string, unknown>) ?? {})
|
||||
.code;
|
||||
|
||||
describe('single concurrency gate — BOTH modes reject the second tab with 409', () => {
|
||||
for (const autonomousRuns of [true, false]) {
|
||||
it(`rejects a concurrent start with 409 A_RUN_ALREADY_ACTIVE (autonomousRuns=${autonomousRuns})`, async () => {
|
||||
const { controller, aiChatRunService } = makeController({
|
||||
getActiveForChat: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ id: 'run-live', chatId: 'c1' }),
|
||||
});
|
||||
const { req, res } = makeReqRes({ chatId: 'c1' });
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await controller.stream(
|
||||
req as never,
|
||||
res as never,
|
||||
user,
|
||||
wsWith(autonomousRuns),
|
||||
);
|
||||
} catch (e) {
|
||||
thrown = e;
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(ConflictException);
|
||||
expect((thrown as HttpException).getStatus()).toBe(409);
|
||||
expect(codeOf(thrown)).toBe('A_RUN_ALREADY_ACTIVE');
|
||||
// Rejected BEFORE committing to the stream (no hijack, no service.stream).
|
||||
expect(res.hijack).not.toHaveBeenCalled();
|
||||
expect(aiChatRunService.getActiveForChat).toHaveBeenCalledWith(
|
||||
'c1',
|
||||
'ws1',
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// #487 [security, F1]: stream() MUST owner-gate an existing chat exactly like its
|
||||
// six sibling endpoints, BEFORE the supersede CAS. Otherwise a same-workspace
|
||||
// non-owner could POST a supersede against another user's chat and (a) harvest
|
||||
// that user's active runId from the 409 SUPERSEDE_TARGET_MISMATCH body, then (b)
|
||||
// requestStop the foreign run. The gate must reject FIRST — no run lookup, no
|
||||
// supersede, no stop, no runId leak.
|
||||
describe('cross-user ownership gate (F1)', () => {
|
||||
it('a non-owner streaming against someone else\'s chat is rejected (403) with NO runId leak and NO foreign requestStop', async () => {
|
||||
// A live run exists on the victim's chat. Without the gate the supersede CAS
|
||||
// would run and (faithful to the run service) return a MISMATCH carrying the
|
||||
// victim's runId — the exact leak. With the gate it must never be reached.
|
||||
const getActiveForChat = jest
|
||||
.fn()
|
||||
.mockResolvedValue({ id: 'run-victim', chatId: 'c-other' });
|
||||
const supersede = jest
|
||||
.fn()
|
||||
.mockResolvedValue({ kind: 'mismatch', activeRunId: 'run-victim' });
|
||||
const requestStop = jest.fn();
|
||||
const { controller, aiChatService } = makeController(
|
||||
{ getActiveForChat, supersede, requestStop },
|
||||
{ creatorId: 'someone-else' }, // the chat is NOT owned by u1
|
||||
);
|
||||
const { req, res } = makeReqRes({
|
||||
chatId: 'c-other',
|
||||
supersede: { runId: 'guessed-uuid' },
|
||||
});
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await controller.stream(req as never, res as never, user, wsWith(true));
|
||||
} catch (e) {
|
||||
thrown = e;
|
||||
}
|
||||
// Rejected by the ownership gate (403), the SAME shape the neighbors use.
|
||||
expect(thrown).toBeInstanceOf(ForbiddenException);
|
||||
expect((thrown as HttpException).getStatus()).toBe(403);
|
||||
// Crucially NOT a 409 that would carry activeRunId — no runId is leaked.
|
||||
const payload = JSON.stringify(
|
||||
(thrown as HttpException).getResponse() ?? {},
|
||||
);
|
||||
expect(payload).not.toContain('run-victim');
|
||||
expect(codeOf(thrown)).not.toBe('SUPERSEDE_TARGET_MISMATCH');
|
||||
// The gate short-circuits BEFORE any run machinery runs.
|
||||
expect(getActiveForChat).not.toHaveBeenCalled();
|
||||
expect(supersede).not.toHaveBeenCalled();
|
||||
expect(requestStop).not.toHaveBeenCalled();
|
||||
expect(aiChatService.stream).not.toHaveBeenCalled();
|
||||
expect(res.hijack).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
it('supersede MISMATCH -> 409 SUPERSEDE_TARGET_MISMATCH carrying the current runId', async () => {
|
||||
const { controller } = makeController({
|
||||
supersede: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ kind: 'mismatch', activeRunId: 'run-other' }),
|
||||
});
|
||||
const { req, res } = makeReqRes({
|
||||
chatId: 'c1',
|
||||
supersede: { runId: 'run-x' },
|
||||
});
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await controller.stream(req as never, res as never, user, wsWith(true));
|
||||
} catch (e) {
|
||||
thrown = e;
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(ConflictException);
|
||||
expect(codeOf(thrown)).toBe('SUPERSEDE_TARGET_MISMATCH');
|
||||
expect(
|
||||
((thrown as HttpException).getResponse() as Record<string, unknown>)
|
||||
.activeRunId,
|
||||
).toBe('run-other');
|
||||
expect(res.hijack).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('supersede TIMEOUT -> 409 SUPERSEDE_TIMEOUT, nothing streamed', async () => {
|
||||
const { controller } = makeController({
|
||||
supersede: jest.fn().mockResolvedValue({ kind: 'timeout' }),
|
||||
});
|
||||
const { req, res } = makeReqRes({
|
||||
chatId: 'c1',
|
||||
supersede: { runId: 'run-x' },
|
||||
});
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await controller.stream(req as never, res as never, user, wsWith(false));
|
||||
} catch (e) {
|
||||
thrown = e;
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(ConflictException);
|
||||
expect(codeOf(thrown)).toBe('SUPERSEDE_TIMEOUT');
|
||||
expect(res.hijack).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('supersede INVALID (target on another chat) -> 400 SUPERSEDE_INVALID', async () => {
|
||||
const { controller } = makeController({
|
||||
supersede: jest.fn().mockResolvedValue({ kind: 'invalid' }),
|
||||
});
|
||||
const { req, res } = makeReqRes({
|
||||
chatId: 'c1',
|
||||
supersede: { runId: 'run-x' },
|
||||
});
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await controller.stream(req as never, res as never, user, wsWith(true));
|
||||
} catch (e) {
|
||||
thrown = e;
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(BadRequestException);
|
||||
expect(codeOf(thrown)).toBe('SUPERSEDE_INVALID');
|
||||
});
|
||||
|
||||
it('supersede without chatId -> 400 SUPERSEDE_INVALID', async () => {
|
||||
const { controller, aiChatRunService } = makeController({});
|
||||
const { req, res } = makeReqRes({ supersede: { runId: 'run-x' } });
|
||||
let thrown: unknown;
|
||||
try {
|
||||
await controller.stream(req as never, res as never, user, wsWith(true));
|
||||
} catch (e) {
|
||||
thrown = e;
|
||||
}
|
||||
expect(thrown).toBeInstanceOf(BadRequestException);
|
||||
expect(codeOf(thrown)).toBe('SUPERSEDE_INVALID');
|
||||
expect(aiChatRunService.supersede).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('supersede READY -> proceeds to stream with superseded=true', async () => {
|
||||
const { controller, aiChatService } = makeController({
|
||||
supersede: jest.fn().mockResolvedValue({ kind: 'ready' }),
|
||||
getActiveForChat: jest.fn().mockResolvedValue(undefined), // slot free after CAS
|
||||
});
|
||||
const { req, res } = makeReqRes({
|
||||
chatId: 'c1',
|
||||
supersede: { runId: 'run-x' },
|
||||
});
|
||||
await controller.stream(req as never, res as never, user, wsWith(true));
|
||||
expect(res.hijack).toHaveBeenCalled();
|
||||
expect(aiChatService.stream).toHaveBeenCalledTimes(1);
|
||||
expect(aiChatService.stream.mock.calls[0][0].superseded).toBe(true);
|
||||
// The run hooks are always present now (both modes).
|
||||
expect(aiChatService.stream.mock.calls[0][0].runHooks).toBeDefined();
|
||||
});
|
||||
|
||||
it('supersede DEGRADE -> proceeds to a normal send (superseded=false)', async () => {
|
||||
const { controller, aiChatService } = makeController({
|
||||
supersede: jest.fn().mockResolvedValue({ kind: 'degrade' }),
|
||||
});
|
||||
const { req, res } = makeReqRes({
|
||||
chatId: 'c1',
|
||||
supersede: { runId: 'run-x' },
|
||||
});
|
||||
await controller.stream(req as never, res as never, user, wsWith(false));
|
||||
expect(aiChatService.stream).toHaveBeenCalledTimes(1);
|
||||
expect(aiChatService.stream.mock.calls[0][0].superseded).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -51,6 +51,7 @@ import {
|
||||
ChatIdDto,
|
||||
ExportChatDto,
|
||||
GeneratePageTitleDto,
|
||||
GetChatDeltaDto,
|
||||
GetChatMessagesDto,
|
||||
GetRunDto,
|
||||
RenameChatDto,
|
||||
@@ -63,6 +64,47 @@ import {
|
||||
SUBSCRIBER_MAX_BUFFERED_BYTES,
|
||||
} from './ai-chat-stream-registry.service';
|
||||
import { startSseHeartbeat } from './sse-resilience';
|
||||
|
||||
/**
|
||||
* Write the attach TAIL to the hijacked socket in chunks that RESPECT drain
|
||||
* (#491): each `write()` that returns false (the kernel buffer is full) is awaited
|
||||
* on the next 'drain' before continuing. The old code wrote the whole buffer
|
||||
* synchronously, which — with the pre-#491 32MB ring — spiked memory (half the
|
||||
* OOM). Bails immediately if the socket ended/errored mid-write. Frames that the
|
||||
* paused registry subscriber buffers while this awaits are delivered by start().
|
||||
*/
|
||||
async function writeTailRespectingDrain(
|
||||
raw: {
|
||||
write(chunk: string): boolean;
|
||||
writableEnded?: boolean;
|
||||
destroyed?: boolean;
|
||||
once(event: string, cb: () => void): unknown;
|
||||
removeListener?(event: string, cb: () => void): unknown;
|
||||
},
|
||||
frames: string[],
|
||||
): Promise<void> {
|
||||
for (const frame of frames) {
|
||||
if (raw.writableEnded || raw.destroyed) return;
|
||||
const ok = raw.write(frame);
|
||||
if (!ok) {
|
||||
// Kernel buffer full — wait for drain (or an early close/error) before the
|
||||
// next chunk, so a slow reader never forces the whole tail into memory.
|
||||
// Remove ALL three listeners once any fires, so a many-chunk tail with
|
||||
// repeated backpressure never leaks (MaxListenersExceededWarning).
|
||||
await new Promise<void>((resolve) => {
|
||||
const finish = (): void => {
|
||||
raw.removeListener?.('drain', finish);
|
||||
raw.removeListener?.('close', finish);
|
||||
raw.removeListener?.('error', finish);
|
||||
resolve();
|
||||
};
|
||||
raw.once('drain', finish);
|
||||
raw.once('close', finish);
|
||||
raw.once('error', finish);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
import { EnvironmentService } from '../../integrations/environment/environment.service';
|
||||
|
||||
/**
|
||||
@@ -149,6 +191,46 @@ export class AiChatController {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delta poll (#491) — the degraded-poll fallback's payload. Returns the chat's
|
||||
* message rows changed since `cursor` (a DB-clock timestamp from the previous
|
||||
* poll), a FRESH cursor, AND the current run fact `{ id, status } | null`. This
|
||||
* replaces the old degraded poll that refetched ALL infinite-query pages (full
|
||||
* parts) every 2.5s: the client seeds once and thereafter merges only the
|
||||
* deltas by id (the overlap window guarantees repeats — the merge is idempotent,
|
||||
* see mergeById). The run fact rides IN the delta (a separate /run poll would
|
||||
* double the poll QPS), so the client FSM gets the run's status on the same tick.
|
||||
* Owner-gated via assertOwnedChat (same gate as the other read endpoints).
|
||||
*/
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('messages/delta')
|
||||
async getMessagesDelta(
|
||||
@Body() dto: GetChatDeltaDto,
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<{
|
||||
rows: AiChatMessage[];
|
||||
cursor: string;
|
||||
run: { id: string; status: string } | null;
|
||||
}> {
|
||||
await this.assertOwnedChat(dto.chatId, user, workspace);
|
||||
const { rows, cursor } =
|
||||
await this.aiChatMessageRepo.findByChatUpdatedAfter(
|
||||
dto.chatId,
|
||||
workspace.id,
|
||||
dto.cursor ?? null,
|
||||
);
|
||||
const run = await this.aiChatRunService.getLatestForChat(
|
||||
dto.chatId,
|
||||
workspace.id,
|
||||
);
|
||||
return {
|
||||
rows,
|
||||
cursor,
|
||||
run: run ? { id: run.id, status: run.status } : null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a chat to Markdown (#183). The DB is the single source of truth: the
|
||||
* whole transcript is loaded (oldest -> newest) and rendered server-side. Now
|
||||
@@ -249,19 +331,25 @@ export class AiChatController {
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach to a chat's live run stream (#184 phase 1.5). A late/reloaded tab
|
||||
* replays the frames buffered so far and then follows the live tail as a normal
|
||||
* streamer. Owner-gated via assertOwnedChat (same gate as getRun). When there is
|
||||
* nothing to resume — no entry, a finished run without expect=live, an
|
||||
* overflowed buffer, or an anchor that pins a DIFFERENT run — the endpoint
|
||||
* answers 204, the ONLY "nothing to resume" signal the AI SDK's reconnect
|
||||
* accepts (it maps 204 to a silent no-op). With AI_CHAT_RESUMABLE_STREAM off the
|
||||
* registry is never populated, so attach always 204s.
|
||||
* Attach to a chat's live run stream from the client's step frontier (#184 phase
|
||||
* 1.5, tail-only #491). A late/reloaded tab hands the server the step count it
|
||||
* has PERSISTED (`n` = the seeded row's `metadata.stepsPersisted`) and its
|
||||
* assistant row id (`anchor`); the registry answers with the TAIL past step `n`
|
||||
* (a synthetic `start` frame + the buffered frames stamped >= n) and then the
|
||||
* live tail. Owner-gated via assertOwnedChat (same gate as getRun). When there
|
||||
* is nothing to resume — no entry, a ring that does not cover the client's
|
||||
* frontier (overflow gap, or the client's seed lagged a rotation), or an anchor
|
||||
* that pins a DIFFERENT run (invariant 6) — the endpoint answers 204, the ONLY
|
||||
* "nothing to resume" signal the AI SDK's reconnect accepts (it maps 204 to a
|
||||
* silent no-op); the client then refetches (a larger n) and re-attaches. With
|
||||
* AI_CHAT_RESUMABLE_STREAM off the registry is never populated, so attach always
|
||||
* 204s.
|
||||
*
|
||||
* `expect=live` opts into replaying a finished-but-retained run (safe only when
|
||||
* the client stripped the streaming tail); `anchor` is the client's assistant
|
||||
* row id, which must match this run's (invariant 6) or a foreign run's
|
||||
* transcript would be replayed into the store.
|
||||
* The step marker `n` comes ONLY from the client — the server never reads the
|
||||
* row to derive it, because a server-side n from a stale seed would open a
|
||||
* silent one-step hole. The tail is written to the socket in CHUNKS respecting
|
||||
* drain (writeTailRespectingDrain): the old code synchronously blasted the whole
|
||||
* buffer, which — with the old 32MB cap — was half the OOM.
|
||||
*/
|
||||
@SkipTransform()
|
||||
@UseGuards(JwtAuthGuard, UserThrottlerGuard)
|
||||
@@ -269,39 +357,49 @@ export class AiChatController {
|
||||
@Get('runs/:chatId/stream')
|
||||
async attachRunStream(
|
||||
@Param('chatId', new ParseUUIDPipe()) chatId: string,
|
||||
@Query('expect') expect: string | undefined,
|
||||
@Query('anchor') anchor: string | undefined,
|
||||
@Query('n') n: string | undefined,
|
||||
@Req() req: FastifyRequest,
|
||||
@Res() res: FastifyReply,
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<void> {
|
||||
await this.assertOwnedChat(chatId, user, workspace); // same gate as getRun
|
||||
// The client's persisted step frontier. #491: distinguish a MISSING/invalid `n`
|
||||
// (null — a NOT-tail-aware, legacy/parameterless tab expecting the old
|
||||
// "finished -> 204 -> poll" contract) from `n=0` (a tail-aware client with
|
||||
// nothing persisted yet). Passing 0 for a missing `n` would serve a finished,
|
||||
// non-rotated run's WHOLE tail and a parameterless client would append it onto
|
||||
// the steps it already shows -> #137/#161 duplicate. null makes the registry
|
||||
// 204 such a finished run (see attach); a tail-aware n=0 still resumes.
|
||||
const frontier: number | null =
|
||||
n === undefined || n === '' || !Number.isFinite(Number(n))
|
||||
? null
|
||||
: Math.max(0, Number(n));
|
||||
// The per-subscriber backpressure cap tracks the (env-tunable) ring cap.
|
||||
const subscriberCap =
|
||||
this.streamRegistry?.subscriberMaxBufferedBytes ??
|
||||
SUBSCRIBER_MAX_BUFFERED_BYTES;
|
||||
let stopHeartbeat: () => void = () => undefined;
|
||||
const attachment = await this.streamRegistry?.attach(
|
||||
chatId,
|
||||
expect === 'live',
|
||||
anchor,
|
||||
{
|
||||
onFrame: (frame) => {
|
||||
// Backpressure guard: 2x the replay cap, so the initial replay burst
|
||||
// alone can never trip it; only a genuinely stalled socket can.
|
||||
try {
|
||||
if (res.raw.writableLength > SUBSCRIBER_MAX_BUFFERED_BYTES) {
|
||||
res.raw.destroy(); // 'close' fires -> unsubscribe below
|
||||
return;
|
||||
}
|
||||
if (!res.raw.writableEnded) res.raw.write(frame);
|
||||
} catch {
|
||||
res.raw.destroy();
|
||||
const attachment = await this.streamRegistry?.attach(chatId, anchor, frontier, {
|
||||
onFrame: (frame) => {
|
||||
// Backpressure guard: 2x the ring cap, so the initial tail burst alone
|
||||
// can never trip it; only a genuinely stalled socket can.
|
||||
try {
|
||||
if (res.raw.writableLength > subscriberCap) {
|
||||
res.raw.destroy(); // 'close' fires -> unsubscribe below
|
||||
return;
|
||||
}
|
||||
},
|
||||
onEnd: () => {
|
||||
stopHeartbeat();
|
||||
if (!res.raw.writableEnded) res.raw.end();
|
||||
},
|
||||
if (!res.raw.writableEnded) res.raw.write(frame);
|
||||
} catch {
|
||||
res.raw.destroy();
|
||||
}
|
||||
},
|
||||
);
|
||||
onEnd: () => {
|
||||
stopHeartbeat();
|
||||
if (!res.raw.writableEnded) res.raw.end();
|
||||
},
|
||||
});
|
||||
if (!attachment) {
|
||||
res.status(204).send(); // the ONLY "nothing to resume" signal the SDK accepts
|
||||
return;
|
||||
@@ -330,13 +428,16 @@ export class AiChatController {
|
||||
// deliberately NO Connection/Keep-Alive (hop-by-hop; Safari/HTTP2)
|
||||
});
|
||||
res.raw.flushHeaders?.();
|
||||
for (const frame of attachment.replay) res.raw.write(frame);
|
||||
// Write the tail in chunks respecting drain (not a synchronous blast, which
|
||||
// was half the OOM). Frames the paused subscriber buffers meanwhile are
|
||||
// drained by start() below; its cap is the backstop for a stalled socket.
|
||||
await writeTailRespectingDrain(res.raw, attachment.replay);
|
||||
if (attachment.finished) {
|
||||
res.raw.end();
|
||||
if (!res.raw.writableEnded) res.raw.end();
|
||||
return;
|
||||
}
|
||||
stopHeartbeat = startSseHeartbeat(res.raw, 15_000);
|
||||
attachment.start(); // drain pending accumulated during replay, go live
|
||||
attachment.start(); // drain pending accumulated during the tail write, go live
|
||||
} catch {
|
||||
attachment.unsubscribe();
|
||||
stopHeartbeat();
|
||||
@@ -418,6 +519,19 @@ export class AiChatController {
|
||||
|
||||
const body = (req.body ?? {}) as AiChatStreamBody;
|
||||
|
||||
// #487 [security]: gate cross-user access to an EXISTING chat BEFORE anything
|
||||
// reads its runs. Every sibling endpoint (getRun/stop/history/rename/delete/
|
||||
// attachRunStream) owner-checks the chat via assertOwnedChat; stream() must too.
|
||||
// Without this a same-workspace member who is NOT the chat owner could POST a
|
||||
// supersede against another user's chat and (a) harvest that user's active runId
|
||||
// out of the 409 SUPERSEDE_TARGET_MISMATCH body, then (b) requestStop the foreign
|
||||
// run. Gate on the chatId the client sent, when present — a brand-new chat (no
|
||||
// chatId) has no prior owner to check. Mirrors /stop's owner check (403 as the
|
||||
// neighbors do), and runs pre-hijack so it returns clean JSON.
|
||||
if (body.chatId) {
|
||||
await this.assertOwnedChat(body.chatId, user, workspace);
|
||||
}
|
||||
|
||||
// Resolve the agent role for this turn BEFORE hijack: existing chats read it
|
||||
// from ai_chats.role_id (authoritative), a new chat from body.roleId. The
|
||||
// role drives both the persona and the optional model override below.
|
||||
@@ -432,12 +546,66 @@ export class AiChatController {
|
||||
// HttpException) instead of breaking mid-stream.
|
||||
const model = await this.aiChatService.getChatModel(workspace.id, role);
|
||||
|
||||
// #184: one active run per chat. For an EXISTING chat reject a concurrent
|
||||
// start with a clean 409 BEFORE hijack (the common double-submit / second-tab
|
||||
// case), so the user gets JSON, not a mid-stream error. A brand-new chat
|
||||
// (no chatId) cannot have a prior run, and the DB partial unique index is the
|
||||
// backstop against any race that slips past this check.
|
||||
if (autonomousRuns && body.chatId) {
|
||||
// #487: server-side supersede CAS ("interrupt and send now"). When the client
|
||||
// asks to replace a live run, atomically STOP it and wait for it to settle
|
||||
// before this turn claims the slot. Runs BEFORE hijack so every branch returns
|
||||
// clean JSON (the client keeps the composer text on a 409). See
|
||||
// AiChatRunService.supersede for the branch semantics.
|
||||
let superseded = false;
|
||||
const supersedeRunId = body.supersede?.runId;
|
||||
if (supersedeRunId) {
|
||||
if (!body.chatId) {
|
||||
throw new BadRequestException({
|
||||
message: 'supersede requires chatId',
|
||||
code: 'SUPERSEDE_INVALID',
|
||||
});
|
||||
}
|
||||
const result = await this.aiChatRunService.supersede(
|
||||
body.chatId,
|
||||
supersedeRunId,
|
||||
workspace.id,
|
||||
);
|
||||
switch (result.kind) {
|
||||
case 'invalid':
|
||||
throw new BadRequestException({
|
||||
message: 'The run to supersede does not belong to this chat',
|
||||
code: 'SUPERSEDE_INVALID',
|
||||
});
|
||||
case 'mismatch':
|
||||
// A DIFFERENT run is active than the one the client targeted. Surface
|
||||
// the CURRENT runId; the client does NOT auto-retry (a stale CAS).
|
||||
throw new ConflictException({
|
||||
message: 'A different agent run is now active on this chat',
|
||||
code: 'SUPERSEDE_TARGET_MISMATCH',
|
||||
activeRunId: result.activeRunId,
|
||||
});
|
||||
case 'timeout':
|
||||
// The target did not settle within W — nothing was persisted, the
|
||||
// composer keeps the text. NOT a rollback: the stop is already issued.
|
||||
throw new ConflictException({
|
||||
message:
|
||||
'The previous run did not stop in time; nothing was sent — please try again',
|
||||
code: 'SUPERSEDE_TIMEOUT',
|
||||
});
|
||||
case 'ready':
|
||||
// The target stopped and settled: the slot is free. Prompt the new run
|
||||
// that the old run's last operations may still be applying.
|
||||
superseded = true;
|
||||
break;
|
||||
case 'degrade':
|
||||
// The run already ended between click and POST — send normally.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// #487: one active run per chat — ENFORCED IN BOTH MODES now (legacy mode used
|
||||
// to have NO gate, so two tabs streamed two parallel turns on one chat, which
|
||||
// interleaved history and crashed convertToModelMessages). Reject a concurrent
|
||||
// start with a clean pre-hijack 409 (double-submit / second-tab). A brand-new
|
||||
// chat (no chatId) cannot have a prior run, and the DB partial unique index in
|
||||
// beginRun is the authoritative backstop for any race that slips past here
|
||||
// (including a slot stolen between a supersede release and beginRun).
|
||||
if (body.chatId) {
|
||||
const active = await this.aiChatRunService.getActiveForChat(
|
||||
body.chatId,
|
||||
workspace.id,
|
||||
@@ -446,107 +614,94 @@ export class AiChatController {
|
||||
throw new ConflictException({
|
||||
message: 'An agent run is already in progress for this chat',
|
||||
code: 'A_RUN_ALREADY_ACTIVE',
|
||||
activeRunId: active.id,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Run-lifecycle hooks (#184), only when the flag is on. They wrap the turn in
|
||||
// a durable run whose abort is governed by the run (explicit stop), persist
|
||||
// its progress, and settle its terminal status — see AiChatRunService.
|
||||
const runHooks: AiChatRunHooks | undefined = autonomousRuns
|
||||
? {
|
||||
begin: async (chatId) => {
|
||||
const handle = await this.aiChatRunService.beginRun({
|
||||
chatId,
|
||||
workspaceId: workspace.id,
|
||||
userId: user.id,
|
||||
trigger: 'user',
|
||||
});
|
||||
// #184 phase 1.5: register the run-stream entry at BEGIN (before any
|
||||
// frame) so a tab that attaches in the begin->seed window finds an
|
||||
// entry to wait on. Gated on AI_CHAT_RESUMABLE_STREAM: with the flag
|
||||
// off nothing is registered and attach always 204s.
|
||||
if (
|
||||
handle?.runId &&
|
||||
this.environment?.isAiChatResumableStreamEnabled?.()
|
||||
) {
|
||||
this.streamRegistry?.open(chatId, handle.runId);
|
||||
}
|
||||
return handle;
|
||||
},
|
||||
onAssistantSeeded: (runId, messageId) =>
|
||||
this.aiChatRunService.linkAssistantMessage(
|
||||
runId,
|
||||
workspace.id,
|
||||
messageId,
|
||||
),
|
||||
onStep: (runId, stepCount) =>
|
||||
void this.aiChatRunService.recordStep(
|
||||
runId,
|
||||
workspace.id,
|
||||
stepCount,
|
||||
),
|
||||
onSettled: (runId, status, error) =>
|
||||
this.aiChatRunService.finalizeRun(
|
||||
runId,
|
||||
workspace.id,
|
||||
status,
|
||||
error,
|
||||
),
|
||||
// #487: the turn is ALWAYS a first-class RUN now (both modes). The mode
|
||||
// difference is only the abort semantics on a browser disconnect (onClose
|
||||
// below). currentRunId is captured at begin so a legacy disconnect can stop
|
||||
// the run through its stop lever.
|
||||
let currentRunId: string | undefined;
|
||||
const runHooks: AiChatRunHooks = {
|
||||
begin: async (chatId) => {
|
||||
const handle = await this.aiChatRunService.beginRun({
|
||||
chatId,
|
||||
workspaceId: workspace.id,
|
||||
userId: user.id,
|
||||
trigger: 'user',
|
||||
});
|
||||
currentRunId = handle?.runId;
|
||||
// #184 phase 1.5: register the run-stream entry at BEGIN (before any
|
||||
// frame) so a tab that attaches in the begin->seed window finds an entry
|
||||
// to wait on. Gated on AI_CHAT_RESUMABLE_STREAM.
|
||||
if (
|
||||
handle?.runId &&
|
||||
this.environment?.isAiChatResumableStreamEnabled?.()
|
||||
) {
|
||||
this.streamRegistry?.open(chatId, handle.runId);
|
||||
}
|
||||
: undefined;
|
||||
return handle;
|
||||
},
|
||||
onAssistantSeeded: (runId, messageId) =>
|
||||
this.aiChatRunService.linkAssistantMessage(
|
||||
runId,
|
||||
workspace.id,
|
||||
messageId,
|
||||
),
|
||||
onStep: (runId, stepCount) =>
|
||||
void this.aiChatRunService.recordStep(runId, workspace.id, stepCount),
|
||||
onSettled: (runId, status, error) =>
|
||||
this.aiChatRunService.finalizeRun(runId, workspace.id, status, error),
|
||||
};
|
||||
|
||||
// Abort the agent loop when the client disconnects. `close` also fires on
|
||||
// normal completion, so only abort when the response has not finished
|
||||
// writing (a genuine disconnect). `once` fires at most once and self-removes;
|
||||
// we also drop it on response `finish` so it never lingers after the stream
|
||||
// completes normally (the AI SDK pipes the response fire-and-forget, so we
|
||||
// cannot simply remove it once `stream()` returns).
|
||||
// Handle a client disconnect. `close` also fires on normal completion, so only
|
||||
// act when the response has not finished writing (a genuine disconnect). `once`
|
||||
// fires at most once and self-removes; we also drop it on response `finish`.
|
||||
// DIAGNOSTIC (Safari stream-drop investigation) — temporary: wall-clock at
|
||||
// which a Safari disconnect is observed, measured from request receipt.
|
||||
const reqStartedAt = Date.now();
|
||||
const controller = new AbortController();
|
||||
const onClose = (): void => {
|
||||
// A genuine disconnect leaves the response unfinished (unlike a normal
|
||||
// completion, which also fires `close`). Such a drop — e.g. a reverse
|
||||
// proxy cutting the SSE mid-answer — is otherwise invisible server-side,
|
||||
// so log it here.
|
||||
if (!res.raw.writableEnded) {
|
||||
if (autonomousRuns) {
|
||||
// #184: the turn is a DETACHED run. A disconnect must NOT abort it —
|
||||
// the run keeps executing and persisting server-side; the client
|
||||
// reconnects via /ai-chat/run (or re-stops via /ai-chat/stop). Log only.
|
||||
// #184: a DETACHED run — a disconnect must NOT stop it. The run keeps
|
||||
// executing and persisting server-side; the client reconnects via
|
||||
// /ai-chat/run (or re-stops via /ai-chat/stop). Log only.
|
||||
this.logger.log(
|
||||
`AI chat stream: client disconnected; run continues server-side ` +
|
||||
`(elapsed=${Date.now() - reqStartedAt}ms since request received)`,
|
||||
);
|
||||
} else {
|
||||
// #487: legacy — a disconnect ENDS the turn, but the turn is now a RUN,
|
||||
// so stop it through the run's stop lever (requestStop). streamText no
|
||||
// longer consumes the socket signal (effectiveSignal is the run signal),
|
||||
// so aborting `controller` would do nothing; requestStop aborts the run.
|
||||
this.logger.warn(
|
||||
`AI chat stream: client disconnected before completion; aborting turn ` +
|
||||
`(elapsed=${Date.now() - reqStartedAt}ms since request received)`,
|
||||
`AI chat stream: client disconnected before completion; stopping the ` +
|
||||
`run (elapsed=${Date.now() - reqStartedAt}ms since request received)`,
|
||||
);
|
||||
controller.abort();
|
||||
if (currentRunId) {
|
||||
void this.aiChatRunService.requestStop(currentRunId, workspace.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
req.raw.once('close', onClose);
|
||||
res.raw.once('finish', () => req.raw.off('close', onClose));
|
||||
|
||||
// #184: in detached mode the turn is NOT aborted on disconnect, so the SDK's
|
||||
// pipe keeps writing to a socket the client may have dropped — for the rest of
|
||||
// the (continuing) run. A write to the dead socket can emit an 'error' on the
|
||||
// raw response; without a listener that surfaces as an unhandled error event.
|
||||
// Swallow it (the run continues server-side regardless). Legacy mode aborts on
|
||||
// disconnect, so it does not need this and keeps its exact prior behavior.
|
||||
if (autonomousRuns) {
|
||||
res.raw.on('error', (err) => {
|
||||
this.logger.debug(
|
||||
`AI chat detached stream: post-disconnect socket error swallowed: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
// #184/#487: the run/pipe can outlive the socket in BOTH modes now (autonomous
|
||||
// keeps going; legacy keeps going until requestStop's abort unwinds the turn).
|
||||
// The SDK's pipe may then write to a dropped socket and emit an 'error' on the
|
||||
// raw response — swallow it so it never surfaces as an unhandled error event.
|
||||
res.raw.on('error', (err) => {
|
||||
this.logger.debug(
|
||||
`AI chat stream: post-disconnect socket error swallowed: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
});
|
||||
|
||||
// Commit to streaming: hijack so Fastify stops managing the response and
|
||||
// the AI SDK can write the UI-message stream directly to the Node socket.
|
||||
@@ -562,8 +717,10 @@ export class AiChatController {
|
||||
signal: controller.signal,
|
||||
model,
|
||||
role,
|
||||
// #184: present only when the flag is on; wraps the turn in a durable run.
|
||||
// #487: the turn is always run-wrapped now (both modes).
|
||||
runHooks,
|
||||
// #487: warn the new run that a superseded run's last ops may still apply.
|
||||
superseded,
|
||||
});
|
||||
} catch (err) {
|
||||
// Any failure AFTER hijack can no longer go through Nest's exception
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
// #489 — client-parts validation + resilient history conversion.
|
||||
//
|
||||
// These unit tests exercise the two exported helpers against the REAL
|
||||
// `convertToModelMessages` from `ai` (NOT a mock): a genuinely malformed part
|
||||
// (a `null` element inside a parts array) makes the real converter throw
|
||||
// ("Cannot read properties of null"), which is the actual production
|
||||
// "bricked chat" mechanism this fix defends against. Asserting against the real
|
||||
// converter (rather than a mock-shaped error) is the whole point — a mock would
|
||||
// hide a version change in the converter's throw behaviour.
|
||||
import { convertToModelMessages, type UIMessage } from 'ai';
|
||||
import {
|
||||
sanitizeUserParts,
|
||||
convertHistoryResilient,
|
||||
TOOL_CONTEXT_OMITTED_MARKER,
|
||||
} from './ai-chat.service';
|
||||
|
||||
type Row = Omit<UIMessage, 'id'> & { id: string };
|
||||
|
||||
describe('sanitizeUserParts (#489, branch: validation on receipt)', () => {
|
||||
it('keeps whitelisted text parts unchanged', () => {
|
||||
const drops: string[] = [];
|
||||
const out = sanitizeUserParts(
|
||||
[
|
||||
{ type: 'text', text: 'a' },
|
||||
{ type: 'text', text: 'b' },
|
||||
] as UIMessage['parts'],
|
||||
(t) => drops.push(t),
|
||||
);
|
||||
expect(out).toEqual([
|
||||
{ type: 'text', text: 'a' },
|
||||
{ type: 'text', text: 'b' },
|
||||
]);
|
||||
expect(drops).toEqual([]);
|
||||
});
|
||||
|
||||
it('drops a non-text part (a tool-part in input-available) and reports its type', () => {
|
||||
const drops: string[] = [];
|
||||
const out = sanitizeUserParts(
|
||||
[
|
||||
{ type: 'text', text: 'hi' },
|
||||
{
|
||||
type: 'tool-getPage',
|
||||
toolCallId: 't1',
|
||||
state: 'input-available',
|
||||
input: { pageId: 'p' },
|
||||
},
|
||||
] as unknown as UIMessage['parts'],
|
||||
(t) => drops.push(t),
|
||||
);
|
||||
expect(out).toEqual([{ type: 'text', text: 'hi' }]);
|
||||
expect(drops).toEqual(['tool-getPage']);
|
||||
});
|
||||
|
||||
it('drops a null part (the shape that would poison convertToModelMessages)', () => {
|
||||
const drops: string[] = [];
|
||||
const out = sanitizeUserParts(
|
||||
[{ type: 'text', text: 'hi' }, null] as unknown as UIMessage['parts'],
|
||||
(t) => drops.push(t),
|
||||
);
|
||||
expect(out).toEqual([{ type: 'text', text: 'hi' }]);
|
||||
expect(drops).toEqual(['(unknown)']);
|
||||
});
|
||||
|
||||
it('returns undefined when nothing survives (so a null metadata is persisted)', () => {
|
||||
const out = sanitizeUserParts(
|
||||
[
|
||||
{ type: 'tool-x', toolCallId: 't', state: 'input-available' },
|
||||
] as unknown as UIMessage['parts'],
|
||||
() => undefined,
|
||||
);
|
||||
expect(out).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns undefined for a non-array input', () => {
|
||||
expect(
|
||||
sanitizeUserParts(undefined as unknown as UIMessage['parts'], () => undefined),
|
||||
).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('convertHistoryResilient (#489, branches: happy + per-row degradation)', () => {
|
||||
it('happy path: healthy history converts identically to convertToModelMessages, no degrade', async () => {
|
||||
const history: Row[] = [
|
||||
{ id: 'u1', role: 'user', parts: [{ type: 'text', text: 'hi' }] },
|
||||
{ id: 'a1', role: 'assistant', parts: [{ type: 'text', text: 'hello' }] },
|
||||
];
|
||||
const degrades: number[] = [];
|
||||
const out = await convertHistoryResilient(history, (i) => degrades.push(i));
|
||||
const expected = await convertToModelMessages(history as UIMessage[]);
|
||||
expect(out).toEqual(expected);
|
||||
expect(degrades).toEqual([]);
|
||||
});
|
||||
|
||||
it('REAL poison: a null part throws in the batch converter but is isolated and degraded to a marker', async () => {
|
||||
// Sanity: the real converter genuinely throws on this shape.
|
||||
const poisoned: Row = {
|
||||
id: 'a1',
|
||||
role: 'assistant',
|
||||
parts: [
|
||||
{ type: 'text', text: 'earlier answer' },
|
||||
null,
|
||||
] as unknown as UIMessage['parts'],
|
||||
};
|
||||
await expect(
|
||||
convertToModelMessages([poisoned as UIMessage]),
|
||||
).rejects.toThrow();
|
||||
|
||||
const history: Row[] = [
|
||||
{ id: 'u1', role: 'user', parts: [{ type: 'text', text: 'first' }] },
|
||||
poisoned,
|
||||
{ id: 'u2', role: 'user', parts: [{ type: 'text', text: 'second' }] },
|
||||
];
|
||||
const degrades: number[] = [];
|
||||
const out = await convertHistoryResilient(history, (i) => degrades.push(i));
|
||||
|
||||
// Only the poisoned row (index 1) is degraded.
|
||||
expect(degrades).toEqual([1]);
|
||||
// Healthy rows survive verbatim.
|
||||
const flat = JSON.stringify(out);
|
||||
expect(flat).toContain('first');
|
||||
expect(flat).toContain('second');
|
||||
// The degraded row carries its readable text AND the truncation marker so the
|
||||
// model sees that tool context was omitted (never a silent loss).
|
||||
expect(flat).toContain('earlier answer');
|
||||
expect(flat).toContain(TOOL_CONTEXT_OMITTED_MARKER);
|
||||
// The whole batch converted (3 model messages, none dropped).
|
||||
expect(out).toHaveLength(3);
|
||||
});
|
||||
|
||||
it('a fully-poisoned row (no readable text) still degrades to just the marker', async () => {
|
||||
const history: Row[] = [
|
||||
{
|
||||
id: 'a1',
|
||||
role: 'assistant',
|
||||
parts: [null] as unknown as UIMessage['parts'],
|
||||
},
|
||||
];
|
||||
const out = await convertHistoryResilient(history, () => undefined);
|
||||
expect(out).toHaveLength(1);
|
||||
expect(JSON.stringify(out)).toContain(TOOL_CONTEXT_OMITTED_MARKER);
|
||||
});
|
||||
});
|
||||
@@ -101,6 +101,22 @@ const INTERRUPT_NOTE =
|
||||
'assume your previous response was complete, and do not silently restart the ' +
|
||||
'partial work — build on it or follow the new instruction.';
|
||||
|
||||
/**
|
||||
* #487: injected on a turn started by SUPERSEDING a previous run (the user hit
|
||||
* "interrupt and send now" while a run was live). The previous run was Stopped,
|
||||
* but there is NO side-effect quiescence — a write it had already committed, or
|
||||
* one committing at the moment of Stop, may land with a small delay AFTER this new
|
||||
* run starts. So the model is told its picture of the page/state may be a beat
|
||||
* stale and to re-read before assuming an edit did or did not apply.
|
||||
*/
|
||||
const SUPERSEDE_NOTE =
|
||||
'NOTE: A previous agent run in this conversation was just interrupted so this ' +
|
||||
'new turn could start. That run was stopped, but any operation it had already ' +
|
||||
'begun (e.g. a page edit) may still be applied with a short delay. Do not ' +
|
||||
'assume the document/state is exactly as the interrupted run left it — if you ' +
|
||||
'need to rely on the current content, RE-READ it with the page tools before ' +
|
||||
'acting rather than trusting a cached view.';
|
||||
|
||||
/**
|
||||
* Injected on a turn where the open page was hand-edited by the user (or anyone
|
||||
* else) AFTER the agent's previous response ended (#274). The server takes a
|
||||
@@ -203,6 +219,14 @@ export interface BuildSystemPromptInput {
|
||||
* (partial) answer was cut off by the user's new message.
|
||||
*/
|
||||
interrupted?: boolean;
|
||||
/**
|
||||
* #487: true when THIS turn was started by superseding a still-live previous run
|
||||
* ("interrupt and send now"). Adds SUPERSEDE_NOTE so the model knows the previous
|
||||
* run's last operations may still be applying and to re-read state it depends on.
|
||||
* Distinct from `interrupted` (which is about a PARTIAL prior answer in history);
|
||||
* both can be set together. Self-clears — set only for the superseding turn.
|
||||
*/
|
||||
superseded?: boolean;
|
||||
/**
|
||||
* Set only when the open page was edited by the user AFTER the agent's previous
|
||||
* turn ended (#274), confirmed server-side by diffing the current page against
|
||||
@@ -311,6 +335,7 @@ export function buildSystemPrompt({
|
||||
openedPage,
|
||||
mcpInstructions,
|
||||
interrupted,
|
||||
superseded,
|
||||
pageChanged,
|
||||
deferredToolsEnabled,
|
||||
toolCatalog,
|
||||
@@ -360,6 +385,13 @@ export function buildSystemPrompt({
|
||||
context += `\n${INTERRUPT_NOTE}`;
|
||||
}
|
||||
|
||||
// Supersede note (#487): present only for a turn that stopped and replaced a
|
||||
// still-live previous run — warns the model the previous run's last operations
|
||||
// may still be applying (no side-effect quiescence).
|
||||
if (superseded) {
|
||||
context += `\n${SUPERSEDE_NOTE}`;
|
||||
}
|
||||
|
||||
// Per-turn page-change note (#274). Added to the context section (inside the
|
||||
// safety sandwich), present only when the server detected that the open page
|
||||
// was edited by the user since the agent's last turn ended. The diff content is
|
||||
|
||||
@@ -89,11 +89,22 @@ describe('AiChatService.stream run-lifecycle safety net (#184)', () => {
|
||||
const runRepo = {
|
||||
insert: jest.fn().mockResolvedValue({ id: 'run-1', status: 'running' }),
|
||||
update: jest.fn().mockResolvedValue({ id: 'run-1' }),
|
||||
// #487: the terminal settle now goes through the CONDITIONAL write.
|
||||
finalizeIfActive: jest
|
||||
.fn()
|
||||
.mockResolvedValue({ id: 'run-1', status: 'failed' }),
|
||||
findById: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const runService = new AiChatRunService(runRepo as never, { isCloud: () => false } as never);
|
||||
|
||||
// The user-message insert (the first bare await after beginRun) throws.
|
||||
// The user-message insert throws. #489 runs the history load + convert BEFORE
|
||||
// the insert (convert-before-insert, so a retry cannot duplicate the user row),
|
||||
// so `findAllByChat` (a real repo method) is now called first — stub it to an
|
||||
// empty history so the flow reaches the insert. Both awaits are AFTER beginRun,
|
||||
// so the "exception after beginRun -> settled to error" invariant is unchanged;
|
||||
// the throw point simply moved from insert to a later insert after a no-op load.
|
||||
const aiChatMessageRepo = {
|
||||
findAllByChat: jest.fn().mockResolvedValue([]),
|
||||
insert: jest.fn().mockRejectedValue(new Error('insert boom')),
|
||||
};
|
||||
const aiChatRepo = {
|
||||
@@ -148,9 +159,10 @@ describe('AiChatService.stream run-lifecycle safety net (#184)', () => {
|
||||
|
||||
// The run was begun...
|
||||
expect(runRepo.insert).toHaveBeenCalledTimes(1);
|
||||
// ...then settled to a terminal FAILED status by the safety net...
|
||||
expect(runRepo.update).toHaveBeenCalledTimes(1);
|
||||
expect(runRepo.update).toHaveBeenCalledWith(
|
||||
// ...then settled to a terminal FAILED status by the safety net (via the
|
||||
// #487 conditional write)...
|
||||
expect(runRepo.finalizeIfActive).toHaveBeenCalledTimes(1);
|
||||
expect(runRepo.finalizeIfActive).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'ws1',
|
||||
expect.objectContaining({ status: 'failed' }),
|
||||
|
||||
@@ -155,6 +155,8 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => {
|
||||
insert: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
findAllByChat: jest.fn(async () => []),
|
||||
update: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
finalizeOwner: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
findStreamingWithTerminalRun: jest.fn(async () => []),
|
||||
};
|
||||
const aiSettings = { resolve: jest.fn(async () => ({})) };
|
||||
const tools = { forUser: jest.fn(async () => ({})) };
|
||||
@@ -179,7 +181,7 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => {
|
||||
{} as never, // pageAccess
|
||||
{ isAiChatDeferredToolsEnabled: () => false, isAiChatFinalStepLockdownEnabled: () => false } as never, // environment
|
||||
);
|
||||
return { svc };
|
||||
return { svc, aiChatMessageRepo };
|
||||
}
|
||||
|
||||
const body = {
|
||||
@@ -285,7 +287,7 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => {
|
||||
// Drive stream() to the point streamText is called, capturing the options object
|
||||
// (which carries onStepFinish/onFinish/onError/onAbort) and the run hooks.
|
||||
async function captureStreamCallbacks() {
|
||||
const { svc } = makeService();
|
||||
const { svc, aiChatMessageRepo } = makeService();
|
||||
let capturedOpts: any;
|
||||
streamTextMock.mockImplementation((opts: any) => {
|
||||
capturedOpts = opts;
|
||||
@@ -312,7 +314,7 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => {
|
||||
runHooks: runHooks as never,
|
||||
});
|
||||
expect(capturedOpts).toBeDefined();
|
||||
return { capturedOpts, runHooks };
|
||||
return { capturedOpts, runHooks, aiChatMessageRepo };
|
||||
}
|
||||
|
||||
it('F9: onStepFinish bumps the run step count, onFinish settles the run "completed" (the dominant autonomous-run path)', async () => {
|
||||
@@ -332,7 +334,13 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => {
|
||||
usage: {},
|
||||
steps: [],
|
||||
});
|
||||
expect(runHooks.onSettled).toHaveBeenCalledWith('run-1', 'completed');
|
||||
// #487: onFinish passes the (undefined) error slot so a message-finalize
|
||||
// failure could error-mark the run; on the success path it is undefined.
|
||||
expect(runHooks.onSettled).toHaveBeenCalledWith(
|
||||
'run-1',
|
||||
'completed',
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('F9: onAbort settles the run "aborted"', async () => {
|
||||
@@ -361,6 +369,51 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => {
|
||||
expect.stringContaining('provider exploded'),
|
||||
);
|
||||
});
|
||||
|
||||
// #490 reactive branch: a provider CONTEXT-OVERFLOW 400 in onError is classified,
|
||||
// records a distinguishable cause, and stamps metadata.replayOverflow so the NEXT
|
||||
// turn's budgeter trims aggressively (the recovery that un-bricks the chat).
|
||||
it('#490: a context-overflow 400 stamps replayOverflow on the finalized row', async () => {
|
||||
jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined as never);
|
||||
jest
|
||||
.spyOn(Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined as never);
|
||||
const { capturedOpts, aiChatMessageRepo } = await captureStreamCallbacks();
|
||||
|
||||
const overflow = Object.assign(new Error('too large'), {
|
||||
statusCode: 400,
|
||||
message:
|
||||
"This model's maximum context length is 128000 tokens. However, your messages resulted in 214000 tokens. Please reduce the length.",
|
||||
});
|
||||
await capturedOpts.onError({ error: overflow });
|
||||
|
||||
// The seed row exists (finalizeOwner is the owner-write path).
|
||||
expect(aiChatMessageRepo.finalizeOwner).toHaveBeenCalled();
|
||||
const calls = aiChatMessageRepo.finalizeOwner.mock.calls as any[][];
|
||||
const patch = calls[calls.length - 1][2] as {
|
||||
status: string;
|
||||
metadata: Record<string, unknown>;
|
||||
};
|
||||
expect(patch.status).toBe('error');
|
||||
expect(patch.metadata.replayOverflow).toBe(true);
|
||||
expect(patch.metadata.error).toContain('контекстное окно');
|
||||
});
|
||||
|
||||
it('#490: a non-overflow error does NOT stamp replayOverflow', async () => {
|
||||
jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined as never);
|
||||
const { capturedOpts, aiChatMessageRepo } = await captureStreamCallbacks();
|
||||
await capturedOpts.onError({ error: new Error('network reset') });
|
||||
const calls = aiChatMessageRepo.finalizeOwner.mock.calls as any[][];
|
||||
const patch = calls[calls.length - 1][2] as {
|
||||
status: string;
|
||||
metadata: Record<string, unknown>;
|
||||
};
|
||||
expect('replayOverflow' in patch.metadata).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -415,6 +468,8 @@ describe('AiChatService.stream — begin-failure fails the turn (#184 F14 / #486
|
||||
insert: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
findAllByChat: jest.fn(async () => []),
|
||||
update: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
finalizeOwner: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
findStreamingWithTerminalRun: jest.fn(async () => []),
|
||||
};
|
||||
const aiSettings = { resolve: jest.fn(async () => ({})) };
|
||||
const tools = { forUser: jest.fn(async () => ({})) };
|
||||
|
||||
@@ -13,6 +13,7 @@ import {
|
||||
compactToolOutput,
|
||||
assistantParts,
|
||||
serializeSteps,
|
||||
type StepPartsCache,
|
||||
rowToUiMessage,
|
||||
prepareAgentStep,
|
||||
stepBudgetWarning,
|
||||
@@ -28,10 +29,14 @@ import {
|
||||
FINAL_STEP_NUDGE,
|
||||
STEP_LIMIT_NO_ANSWER_MARKER,
|
||||
OUTPUT_DEGENERATION_ERROR,
|
||||
lastAssistantContextTokens,
|
||||
lastAssistantReplayOverflow,
|
||||
seedActivatedTools,
|
||||
} from './ai-chat.service';
|
||||
import type { AiChatMessage, Workspace } from '@docmost/db/types/entity.types';
|
||||
import { buildSystemPrompt } from './ai-chat.prompt';
|
||||
import type { McpClientsService } from './external-mcp/mcp-clients.service';
|
||||
import { resolveEffectiveReplayThreshold } from './history-budget';
|
||||
|
||||
/**
|
||||
* Unit tests for compactToolOutput: the pure helper that shrinks tool outputs
|
||||
@@ -114,6 +119,54 @@ describe('compactToolOutput', () => {
|
||||
describe('assistantParts', () => {
|
||||
type AnyPart = Record<string, unknown>;
|
||||
|
||||
// #490 memoization: assistantParts builds each step's parts once and caches
|
||||
// them by the step OBJECT's identity, so a mid-stream flush does not
|
||||
// re-stringify every prior step's (large) output. Observable property: with a
|
||||
// shared cache, the second call over the SAME step object returns the cached
|
||||
// (identical) part array even if the step's underlying output was swapped —
|
||||
// proving the work was memoized, not redone.
|
||||
it('memoizes a step by identity (shared cache => one build per step)', () => {
|
||||
const cache: StepPartsCache = new WeakMap();
|
||||
const step = {
|
||||
text: 'x',
|
||||
toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: {} }],
|
||||
toolResults: [{ toolCallId: 'c1', toolName: 'getPage', output: { v: 1 } }],
|
||||
};
|
||||
const first = assistantParts([step], '', cache) as AnyPart[];
|
||||
expect((first.find((p) => p.type === 'tool-getPage')!.output as any).v).toBe(
|
||||
1,
|
||||
);
|
||||
// Swap the output for a NEW value; a re-build would pick it up, a cache hit
|
||||
// keeps the first result.
|
||||
step.toolResults[0] = {
|
||||
toolCallId: 'c1',
|
||||
toolName: 'getPage',
|
||||
output: { v: 2 },
|
||||
};
|
||||
const second = assistantParts([step], '', cache) as AnyPart[];
|
||||
expect((second.find((p) => p.type === 'tool-getPage')!.output as any).v).toBe(
|
||||
1,
|
||||
);
|
||||
// Same cached part objects are reused.
|
||||
expect(second.find((p) => p.type === 'tool-getPage')).toBe(
|
||||
first.find((p) => p.type === 'tool-getPage'),
|
||||
);
|
||||
});
|
||||
|
||||
it('without a cache, each call rebuilds (no stale memo)', () => {
|
||||
const step = {
|
||||
text: 'x',
|
||||
toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: {} }],
|
||||
toolResults: [{ toolCallId: 'c1', toolName: 'getPage', output: { v: 1 } }],
|
||||
};
|
||||
const first = assistantParts([step], '') as AnyPart[];
|
||||
step.toolResults[0].output = { v: 2 };
|
||||
const second = assistantParts([step], '') as AnyPart[];
|
||||
expect((second.find((p) => p.type === 'tool-getPage')!.output as any).v).toBe(
|
||||
2,
|
||||
);
|
||||
});
|
||||
|
||||
it('emits output-available for a tool-call WITH a paired result', () => {
|
||||
const steps = [
|
||||
{
|
||||
@@ -231,61 +284,320 @@ describe('assistantParts', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('serializeSteps', () => {
|
||||
// #490 trace format v2: per call the trace stores { input } for the call and an
|
||||
// OUTCOME element — { ok: true } on success, { error, kind: 'thrown' } on a
|
||||
// thrown tool-error, { error, kind: 'interrupted' } on a mid-step abort. The tool
|
||||
// OUTPUT is no longer duplicated here (it lives once in metadata.parts).
|
||||
describe('serializeSteps (trace v2)', () => {
|
||||
it('returns null when there are no calls or results', () => {
|
||||
expect(serializeSteps([])).toBeNull();
|
||||
});
|
||||
|
||||
it('flattens calls and results into a compact trace', () => {
|
||||
it('pairs a successful call with an { ok: true } outcome and NO output', () => {
|
||||
const trace = serializeSteps([
|
||||
{
|
||||
toolCalls: [{ toolName: 'getPage', input: { id: 'p1' } }],
|
||||
toolResults: [{ toolName: 'getPage', output: { title: 'T' } }],
|
||||
toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: { id: 'p1' } }],
|
||||
toolResults: [{ toolCallId: 'c1', toolName: 'getPage' }],
|
||||
},
|
||||
]) as Array<Record<string, unknown>>;
|
||||
expect(trace).toHaveLength(2);
|
||||
expect(trace[0]).toEqual({ toolName: 'getPage', input: { id: 'p1' } });
|
||||
expect(trace[1]).toEqual({ toolName: 'getPage', output: { title: 'T' } });
|
||||
expect(trace[1]).toEqual({ toolName: 'getPage', ok: true });
|
||||
// The output is NOT stored in the trace any more (dedup: it lives in parts).
|
||||
expect(trace.some((e) => 'output' in e)).toBe(false);
|
||||
});
|
||||
|
||||
it('records a THROWN tool failure (tool-error part) with its error message', () => {
|
||||
it('records a THROWN failure with { error, kind: "thrown" }', () => {
|
||||
const trace = serializeSteps([
|
||||
{
|
||||
toolCalls: [{ toolName: 'editPageText', input: { id: 'p1' } }],
|
||||
toolCalls: [
|
||||
{ toolCallId: 'c1', toolName: 'editPageText', input: { id: 'p1' } },
|
||||
],
|
||||
toolResults: [],
|
||||
content: [
|
||||
{
|
||||
type: 'tool-error',
|
||||
toolCallId: 'c1',
|
||||
toolName: 'editPageText',
|
||||
error: new Error('page is locked'),
|
||||
},
|
||||
],
|
||||
},
|
||||
]) as Array<Record<string, unknown>>;
|
||||
// The call element is followed by a paired error element (mirroring how a
|
||||
// successful result is appended), so the failure survives in the trace.
|
||||
expect(trace).toHaveLength(2);
|
||||
expect(trace[0]).toEqual({ toolName: 'editPageText', input: { id: 'p1' } });
|
||||
expect(trace[1]).toEqual({
|
||||
toolName: 'editPageText',
|
||||
error: 'page is locked',
|
||||
kind: 'thrown',
|
||||
});
|
||||
});
|
||||
|
||||
it('truncates a very long tool-error message to the tool-output limit', () => {
|
||||
it('marks an interrupted call (no result, no throw) with kind "interrupted"', () => {
|
||||
const trace = serializeSteps([
|
||||
{
|
||||
toolCalls: [
|
||||
{ toolCallId: 'c1', toolName: 'createComment', input: { x: 1 } },
|
||||
],
|
||||
toolResults: [],
|
||||
content: [],
|
||||
},
|
||||
]) as Array<Record<string, unknown>>;
|
||||
expect(trace).toHaveLength(2);
|
||||
expect(trace[1]).toEqual({
|
||||
toolName: 'createComment',
|
||||
error: 'Tool call did not complete.',
|
||||
kind: 'interrupted',
|
||||
});
|
||||
// Structurally distinct from a thrown hard-fail so it never inflates an
|
||||
// error-rate scan.
|
||||
expect((trace[1] as { kind: string }).kind).not.toBe('thrown');
|
||||
});
|
||||
|
||||
it('truncates a very long thrown-error message to the tool-output limit', () => {
|
||||
const long = 'x'.repeat(5000);
|
||||
const trace = serializeSteps([
|
||||
{
|
||||
toolCalls: [{ toolName: 'editPageText', input: {} }],
|
||||
toolCalls: [{ toolCallId: 'c1', toolName: 'editPageText', input: {} }],
|
||||
toolResults: [],
|
||||
content: [{ type: 'tool-error', toolName: 'editPageText', error: long }],
|
||||
content: [
|
||||
{
|
||||
type: 'tool-error',
|
||||
toolCallId: 'c1',
|
||||
toolName: 'editPageText',
|
||||
error: long,
|
||||
},
|
||||
],
|
||||
},
|
||||
]) as Array<Record<string, unknown>>;
|
||||
const errorText = trace[1].error as string;
|
||||
// Truncated (not the full 5000 chars) and carries the omission marker.
|
||||
expect(errorText.length).toBeLessThan(long.length);
|
||||
expect(errorText).toContain('chars omitted');
|
||||
});
|
||||
|
||||
it('pairs parallel calls in one step with their outcomes by id', () => {
|
||||
const trace = serializeSteps([
|
||||
{
|
||||
toolCalls: [
|
||||
{ toolCallId: 'a', toolName: 'getPage', input: {} },
|
||||
{ toolCallId: 'b', toolName: 'searchPages', input: {} },
|
||||
],
|
||||
toolResults: [{ toolCallId: 'b', toolName: 'searchPages' }],
|
||||
content: [
|
||||
{ type: 'tool-error', toolCallId: 'a', toolName: 'getPage', error: 'nope' },
|
||||
],
|
||||
},
|
||||
]) as Array<Record<string, unknown>>;
|
||||
// call a, outcome a (thrown), call b, outcome b (ok)
|
||||
expect(trace).toHaveLength(4);
|
||||
expect(trace[1]).toEqual({ toolName: 'getPage', error: 'nope', kind: 'thrown' });
|
||||
expect(trace[3]).toEqual({ toolName: 'searchPages', ok: true });
|
||||
});
|
||||
});
|
||||
|
||||
// #490: every assistant row flushAssistant writes carries the v2 era marker so a
|
||||
// dual-shape diagnostic query can branch on the trace shape without inspecting it.
|
||||
describe('toolTraceVersion era marker (#490)', () => {
|
||||
it('stamps metadata.toolTraceVersion = 2 on every flushed row', () => {
|
||||
const seed = flushAssistant([], '', 'streaming');
|
||||
expect(seed.metadata.toolTraceVersion).toBe(2);
|
||||
const done = flushAssistant(
|
||||
[
|
||||
{
|
||||
text: 'ok',
|
||||
toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: {} }],
|
||||
toolResults: [{ toolCallId: 'c1', toolName: 'getPage' }],
|
||||
},
|
||||
],
|
||||
'',
|
||||
'completed',
|
||||
{ finishReason: 'stop' },
|
||||
);
|
||||
expect(done.metadata.toolTraceVersion).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
// #490 replay-budget signal helpers over persisted history.
|
||||
describe('lastAssistantContextTokens', () => {
|
||||
const row = (
|
||||
role: string,
|
||||
metadata: Record<string, unknown> | null,
|
||||
): AiChatMessage => ({ role, metadata }) as unknown as AiChatMessage;
|
||||
|
||||
it('reads the most recent assistant turn contextTokens (provider fact)', () => {
|
||||
const hist = [
|
||||
row('user', null),
|
||||
row('assistant', { contextTokens: 12000 }),
|
||||
row('user', null),
|
||||
row('assistant', { contextTokens: 41000 }),
|
||||
];
|
||||
expect(lastAssistantContextTokens(hist)).toBe(41000);
|
||||
});
|
||||
|
||||
it('returns undefined when the last assistant turn recorded no usage', () => {
|
||||
const hist = [row('assistant', { error: 'boom' }), row('user', null)];
|
||||
expect(lastAssistantContextTokens(hist)).toBeUndefined();
|
||||
expect(lastAssistantContextTokens([])).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// #490 snapshotOpenPage fast-path: skip the full Markdown export + upsert when a
|
||||
// snapshot already exists at the page's CURRENT version (same updated_at instant).
|
||||
describe('snapshotOpenPage fast-path (#490)', () => {
|
||||
function makeSvc(existingSnapshot: unknown, pageUpdatedAt: Date) {
|
||||
const exportPageMarkdown = jest.fn(async () => '# md');
|
||||
const upsert = jest.fn(async () => undefined);
|
||||
const findByChatPage = jest.fn(async () => existingSnapshot);
|
||||
const pageRepo = {
|
||||
findById: jest.fn(async () => ({
|
||||
id: 'p1',
|
||||
workspaceId: 'ws1',
|
||||
updatedAt: pageUpdatedAt,
|
||||
})),
|
||||
};
|
||||
const svc = new AiChatService(
|
||||
{} as never, // ai
|
||||
{} as never, // aiChatRepo
|
||||
{} as never, // aiChatMessageRepo
|
||||
{ findByChatPage, upsert } as never, // aiChatPageSnapshotRepo
|
||||
{} as never, // aiSettings
|
||||
{ exportPageMarkdown } as never, // tools
|
||||
{} as never, // mcpClients
|
||||
{} as never, // aiAgentRoleRepo
|
||||
pageRepo as never, // pageRepo
|
||||
{} as never, // pageAccess
|
||||
{} as never, // environment
|
||||
);
|
||||
return { svc, exportPageMarkdown, upsert, findByChatPage };
|
||||
}
|
||||
|
||||
const args = () =>
|
||||
[
|
||||
'chat1',
|
||||
'p1',
|
||||
{ id: 'ws1' } as never,
|
||||
{ id: 'u1' } as never,
|
||||
'sess',
|
||||
] as const;
|
||||
|
||||
it('skips export + upsert when the snapshot is already at this page version', async () => {
|
||||
const t = new Date('2026-07-07T10:00:00Z');
|
||||
const { svc, exportPageMarkdown, upsert } = makeSvc(
|
||||
{ pageUpdatedAt: t, contentMd: '# md' },
|
||||
t,
|
||||
);
|
||||
await (svc as unknown as { snapshotOpenPage: (...a: unknown[]) => Promise<void> })
|
||||
.snapshotOpenPage(...args());
|
||||
expect(exportPageMarkdown).not.toHaveBeenCalled();
|
||||
expect(upsert).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('exports + upserts when the page advanced since the snapshot', async () => {
|
||||
const { svc, exportPageMarkdown, upsert } = makeSvc(
|
||||
{ pageUpdatedAt: new Date('2026-07-07T10:00:00Z'), contentMd: 'old' },
|
||||
new Date('2026-07-07T11:00:00Z'),
|
||||
);
|
||||
await (svc as unknown as { snapshotOpenPage: (...a: unknown[]) => Promise<void> })
|
||||
.snapshotOpenPage(...args());
|
||||
expect(exportPageMarkdown).toHaveBeenCalledTimes(1);
|
||||
expect(upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('seeds (exports + upserts) on the first turn (no snapshot yet)', async () => {
|
||||
const { svc, exportPageMarkdown, upsert } = makeSvc(
|
||||
undefined,
|
||||
new Date('2026-07-07T10:00:00Z'),
|
||||
);
|
||||
await (svc as unknown as { snapshotOpenPage: (...a: unknown[]) => Promise<void> })
|
||||
.snapshotOpenPage(...args());
|
||||
expect(exportPageMarkdown).toHaveBeenCalledTimes(1);
|
||||
expect(upsert).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// #490 deferred-tool activation persisted across turns.
|
||||
describe('seedActivatedTools', () => {
|
||||
const valid = new Set(['Search_web', 'getPageJson', 'diffPageVersions']);
|
||||
|
||||
it('seeds from persisted metadata, intersected with current valid names', () => {
|
||||
expect(
|
||||
seedActivatedTools(
|
||||
{ activatedTools: ['Search_web', 'getPageJson'] },
|
||||
valid,
|
||||
),
|
||||
).toEqual(['Search_web', 'getPageJson']);
|
||||
});
|
||||
|
||||
it('drops a stored tool that is no longer valid (allowlist/role changed)', () => {
|
||||
// 'Habr_publish' was activated before but is not in the current allowlist.
|
||||
expect(
|
||||
seedActivatedTools({ activatedTools: ['Search_web', 'Habr_publish'] }, valid),
|
||||
).toEqual(['Search_web']);
|
||||
});
|
||||
|
||||
it('is empty/robust for missing, non-array, or unknown-shaped metadata', () => {
|
||||
expect(seedActivatedTools(undefined, valid)).toEqual([]);
|
||||
expect(seedActivatedTools({}, valid)).toEqual([]);
|
||||
expect(seedActivatedTools({ activatedTools: 'nope' }, valid)).toEqual([]);
|
||||
expect(
|
||||
seedActivatedTools({ activatedTools: [1, 'getPageJson', null] }, valid),
|
||||
).toEqual(['getPageJson']);
|
||||
});
|
||||
|
||||
it('de-duplicates stored names', () => {
|
||||
expect(
|
||||
seedActivatedTools(
|
||||
{ activatedTools: ['getPageJson', 'getPageJson'] },
|
||||
valid,
|
||||
),
|
||||
).toEqual(['getPageJson']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('lastAssistantReplayOverflow', () => {
|
||||
const row = (
|
||||
role: string,
|
||||
metadata: Record<string, unknown> | null,
|
||||
): AiChatMessage => ({ role, metadata }) as unknown as AiChatMessage;
|
||||
|
||||
it('is true only when the LAST assistant turn overflowed', () => {
|
||||
expect(
|
||||
lastAssistantReplayOverflow([
|
||||
row('assistant', { replayOverflow: true }),
|
||||
row('user', null),
|
||||
]),
|
||||
).toBe(true);
|
||||
// A recovered (later, non-overflow) assistant turn clears it.
|
||||
expect(
|
||||
lastAssistantReplayOverflow([
|
||||
row('assistant', { replayOverflow: true }),
|
||||
row('user', null),
|
||||
row('assistant', { contextTokens: 5 }),
|
||||
]),
|
||||
).toBe(false);
|
||||
expect(lastAssistantReplayOverflow([])).toBe(false);
|
||||
});
|
||||
|
||||
// #490 reactive recovery: a prior turn stamped `replayOverflow` must make the
|
||||
// NEXT turn's effective budget the AGGRESSIVE 0.5x cut — that harder trim is
|
||||
// what un-bricks a chat that just 400'd on the context window. This exercises
|
||||
// the exact wiring the service uses: read the stamp, then scale the threshold.
|
||||
it('#490: a prior replayOverflow drives the next turn to the 0.5x aggressive budget', () => {
|
||||
const history = [
|
||||
row('assistant', { replayOverflow: true }),
|
||||
row('user', null),
|
||||
];
|
||||
const priorOverflowed = lastAssistantReplayOverflow(history);
|
||||
expect(priorOverflowed).toBe(true);
|
||||
// Base budget 100k -> aggressive recovery halves it to 50k this turn.
|
||||
expect(resolveEffectiveReplayThreshold(100_000, priorOverflowed)).toBe(50_000);
|
||||
// Odd base floors, not rounds.
|
||||
expect(resolveEffectiveReplayThreshold(99_999, true)).toBe(49_999);
|
||||
// No prior overflow -> the base budget is used verbatim (no aggressive cut).
|
||||
expect(resolveEffectiveReplayThreshold(100_000, false)).toBe(100_000);
|
||||
// An explicit off-switch (null) is never overridden, even on recovery.
|
||||
expect(resolveEffectiveReplayThreshold(null, true)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe('rowToUiMessage', () => {
|
||||
@@ -618,6 +930,23 @@ describe('flushAssistant', () => {
|
||||
expect(flushed.metadata.error).toBe('boom');
|
||||
});
|
||||
|
||||
// #490 observability: the replay budgeter's decision is stamped on the turn.
|
||||
it('records replayTrimmedToTokens + replayOverflow when provided', () => {
|
||||
const f = flushAssistant([], '', 'error', {
|
||||
error: 'ctx',
|
||||
replayTrimmedToTokens: 42_000,
|
||||
replayOverflow: true,
|
||||
});
|
||||
expect(f.metadata.replayTrimmedToTokens).toBe(42_000);
|
||||
expect(f.metadata.replayOverflow).toBe(true);
|
||||
});
|
||||
|
||||
it('omits the replay metadata when not provided', () => {
|
||||
const f = flushAssistant([], '', 'completed', { finishReason: 'stop' });
|
||||
expect('replayTrimmedToTokens' in f.metadata).toBe(false);
|
||||
expect('replayOverflow' in f.metadata).toBe(false);
|
||||
});
|
||||
|
||||
// #274 observability: the page-change diff the agent saw this turn is persisted
|
||||
// to metadata.pageChanged when a non-empty diff was injected, and omitted when
|
||||
// the diff is empty/whitespace or the arg is not supplied.
|
||||
@@ -1315,8 +1644,12 @@ describe('AiChatService page-change lifecycle (#274)', () => {
|
||||
describe('isInterruptResume', () => {
|
||||
// history tail is the just-inserted user row; [len-2] is the previous turn.
|
||||
const withPrev = (
|
||||
prev: { role: string; status?: string | null } | null,
|
||||
): Array<{ role: string; status?: string | null }> =>
|
||||
prev: {
|
||||
role: string;
|
||||
status?: string | null;
|
||||
metadata?: unknown;
|
||||
} | null,
|
||||
): Array<{ role: string; status?: string | null; metadata?: unknown }> =>
|
||||
prev
|
||||
? [prev, { role: 'user', status: null }]
|
||||
: [{ role: 'user', status: null }];
|
||||
@@ -1357,6 +1690,33 @@ describe('isInterruptResume', () => {
|
||||
it('false when there is no preceding turn (only the new user row)', () => {
|
||||
expect(isInterruptResume(withPrev(null), true)).toBe(false);
|
||||
});
|
||||
|
||||
it('#487 EXCLUDES a reconcile stamp (finalizeFailed) — not a genuine interruption', () => {
|
||||
// A row a reconcile settled to 'aborted' carries metadata.finalizeFailed. It
|
||||
// must NOT be treated as an interrupt-resume (that would inject a false
|
||||
// "you were interrupted" note), even though its status is 'aborted'.
|
||||
expect(
|
||||
isInterruptResume(
|
||||
withPrev({
|
||||
role: 'assistant',
|
||||
status: 'aborted',
|
||||
metadata: { finalizeFailed: true },
|
||||
}),
|
||||
true,
|
||||
),
|
||||
).toBe(false);
|
||||
// A genuine abort (no finalizeFailed) still counts.
|
||||
expect(
|
||||
isInterruptResume(
|
||||
withPrev({
|
||||
role: 'assistant',
|
||||
status: 'aborted',
|
||||
metadata: { parts: [] },
|
||||
}),
|
||||
true,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -1409,7 +1769,7 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', ()
|
||||
}
|
||||
|
||||
// Wire only the deps reached on the way to the pipe call, plus a spy registry.
|
||||
function makeService(opts: { resumable: boolean }) {
|
||||
function makeService(opts: { resumable: boolean; history?: unknown[] }) {
|
||||
const aiChatRepo = {
|
||||
findById: jest.fn(async () => ({ id: 'chat-1', workspaceId: 'ws-1' })),
|
||||
insert: jest.fn(),
|
||||
@@ -1417,8 +1777,11 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', ()
|
||||
const aiChatMessageRepo = {
|
||||
// Both the user insert and the assistant seed return the same row id.
|
||||
insert: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
findAllByChat: jest.fn(async () => []),
|
||||
findAllByChat: jest.fn(async () => opts.history ?? []),
|
||||
update: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
// #487: the terminal owner-write + the opportunistic reconcile query.
|
||||
finalizeOwner: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
findStreamingWithTerminalRun: jest.fn(async () => []),
|
||||
};
|
||||
const aiSettings = { resolve: jest.fn(async () => ({})) };
|
||||
const tools = { forUser: jest.fn(async () => ({})) };
|
||||
@@ -1453,7 +1816,7 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', ()
|
||||
} as never,
|
||||
streamRegistry as never,
|
||||
);
|
||||
return { svc, streamRegistry };
|
||||
return { svc, streamRegistry, aiChatMessageRepo };
|
||||
}
|
||||
|
||||
const body = {
|
||||
@@ -1536,6 +1899,86 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', ()
|
||||
await expect(drive(svc, makeRunHooks())).rejects.toThrow('boom');
|
||||
expect(streamRegistry.abortEntry).toHaveBeenCalledWith('chat-1', 'run-1');
|
||||
});
|
||||
|
||||
// #489 REGRESSION (against the REAL convertToModelMessages — not mocked here):
|
||||
// a persisted history row whose parts contain a `null` element makes the real
|
||||
// convertToModelMessages THROW ("Cannot read properties of null"). Pre-fix that
|
||||
// 500-ed every turn forever and each retry appended a duplicate user row. The
|
||||
// fix converts BEFORE the insert and isolates the poisoned row per-row, degrading
|
||||
// it to text with a "[tool context omitted]" marker. Assert the turn still runs,
|
||||
// the marker reaches the model, and exactly ONE user row is inserted.
|
||||
it('#489: a poisoned OLD-history row keeps the chat working; the marker reaches the model; one user insert', async () => {
|
||||
const { svc, aiChatMessageRepo } = makeService({
|
||||
resumable: false,
|
||||
history: [
|
||||
{
|
||||
id: 'old-1',
|
||||
role: 'assistant',
|
||||
content: 'earlier answer',
|
||||
// A null part is the poison: rowToUiMessage keeps it (the array is
|
||||
// non-empty) and the real convertToModelMessages throws on it.
|
||||
metadata: { parts: [{ type: 'text', text: 'earlier answer' }, null] },
|
||||
status: 'completed',
|
||||
},
|
||||
],
|
||||
});
|
||||
// Must NOT throw — the poisoned row is degraded, not fatal.
|
||||
await drive(svc, makeRunHooks());
|
||||
expect(streamTextMock).toHaveBeenCalledTimes(1);
|
||||
const passedMessages = streamTextMock.mock.calls[0][0].messages;
|
||||
const serialized = JSON.stringify(passedMessages);
|
||||
// The model sees the truncation marker (silent tool-context loss is not ok)
|
||||
// AND the row's readable text is preserved alongside it.
|
||||
expect(serialized).toContain('[tool context omitted]');
|
||||
expect(serialized).toContain('earlier answer');
|
||||
// Exactly ONE user row inserted (no duplicate), inserted AFTER conversion.
|
||||
const userInserts = aiChatMessageRepo.insert.mock.calls
|
||||
.map((c: unknown[]) => c[0] as { role?: string })
|
||||
.filter((r) => r.role === 'user');
|
||||
expect(userInserts).toHaveLength(1);
|
||||
});
|
||||
|
||||
// #489: client-supplied non-text parts (a tool-part in `input-available`, the
|
||||
// exact "bricking" payload) are dropped ON RECEIPT — never persisted — so they
|
||||
// can never poison future turns. Only the text survives into metadata.parts.
|
||||
it('#489: a non-text client part is stripped before persist (only text survives)', async () => {
|
||||
const { svc, aiChatMessageRepo } = makeService({ resumable: false });
|
||||
await svc.stream({
|
||||
user: { id: 'u1' } as never,
|
||||
workspace: { id: 'ws-1' } as never,
|
||||
sessionId: 's1',
|
||||
body: {
|
||||
chatId: 'chat-1',
|
||||
messages: [
|
||||
{
|
||||
id: 'm1',
|
||||
role: 'user',
|
||||
parts: [
|
||||
{ type: 'text', text: 'hello' },
|
||||
// untrusted tool-part — must be dropped, never persisted
|
||||
{
|
||||
type: 'tool-getPage',
|
||||
toolCallId: 't1',
|
||||
state: 'input-available',
|
||||
input: { pageId: 'p' },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
} as never,
|
||||
res: makeRes() as never,
|
||||
signal: new AbortController().signal,
|
||||
model: {} as never,
|
||||
role: null,
|
||||
runHooks: makeRunHooks() as never,
|
||||
});
|
||||
const userInsert = aiChatMessageRepo.insert.mock.calls
|
||||
.map((c: unknown[]) => c[0] as { role?: string; metadata?: unknown })
|
||||
.find((r) => r.role === 'user');
|
||||
const parts = (userInsert?.metadata as { parts?: Array<{ type: string }> })
|
||||
?.parts;
|
||||
expect(parts).toEqual([{ type: 'text', text: 'hello' }]);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
@@ -1623,6 +2066,19 @@ describe('AiChatService.stream — token-degeneration reaction (#444)', () => {
|
||||
return { id };
|
||||
},
|
||||
),
|
||||
// #487: the terminal owner-write records into the SAME `updated` recorder so
|
||||
// assertions on the terminal 'completed'/'error'/'aborted' write still hold.
|
||||
finalizeOwner: jest.fn(
|
||||
async (
|
||||
id: string,
|
||||
workspaceId: string,
|
||||
patch: Record<string, unknown>,
|
||||
) => {
|
||||
updated.push({ id, workspaceId, patch });
|
||||
return { id };
|
||||
},
|
||||
),
|
||||
findStreamingWithTerminalRun: jest.fn(async () => []),
|
||||
};
|
||||
const aiSettings = { resolve: jest.fn(async () => ({})) };
|
||||
const tools = { forUser: jest.fn(async () => ({})) };
|
||||
@@ -1882,3 +2338,148 @@ describe('AiChatService.stream — token-degeneration reaction (#444)', () => {
|
||||
expect(patch.content).not.toContain(STEP_LIMIT_NO_ANSWER_MARKER);
|
||||
});
|
||||
});
|
||||
|
||||
// #487 F3 — the reconcile() / reconcileChat() ORCHESTRATORS. The individual
|
||||
// clauses are exercised elsewhere; these pin the production orchestration the
|
||||
// per-clause specs do not: the clause ORDER, the per-clause try/catch ISOLATION
|
||||
// (one clause throwing must NOT abort the others), and reconcileChat() (which runs
|
||||
// at the start of every turn and was entirely uncovered).
|
||||
describe('AiChatService.reconcile / reconcileChat orchestrators (#487 F3)', () => {
|
||||
let warnSpy: jest.SpyInstance;
|
||||
beforeEach(() => {
|
||||
// Silence the intentional clause-failure warnings (kept out of test output).
|
||||
warnSpy = jest
|
||||
.spyOn(Logger.prototype, 'warn')
|
||||
.mockImplementation(() => undefined);
|
||||
});
|
||||
afterEach(() => {
|
||||
warnSpy.mockRestore();
|
||||
});
|
||||
|
||||
function makeService(opts: {
|
||||
messageRepo?: Record<string, jest.Mock>;
|
||||
runService?: Record<string, jest.Mock>;
|
||||
}) {
|
||||
const aiChatMessageRepo = {
|
||||
findStreamingWithTerminalRun: jest.fn(async () => []),
|
||||
stampTerminalIfStreaming: jest.fn(async () => undefined),
|
||||
sweepStreamingWithoutActiveRun: jest.fn(async () => 0),
|
||||
...(opts.messageRepo ?? {}),
|
||||
};
|
||||
const aiChatRunService = opts.runService
|
||||
? {
|
||||
zombieRunIds: jest.fn(() => []),
|
||||
settleZombie: jest.fn(async () => true),
|
||||
reconcileStaleRuns: jest.fn(async () => 0),
|
||||
...opts.runService,
|
||||
}
|
||||
: undefined;
|
||||
const svc = new AiChatService(
|
||||
{} as never, // ai
|
||||
{} as never, // aiChatRepo
|
||||
aiChatMessageRepo as never,
|
||||
{} as never, // aiChatPageSnapshotRepo
|
||||
{} as never, // aiSettings
|
||||
{} as never, // tools
|
||||
{} as never, // mcpClients
|
||||
{} as never, // aiAgentRoleRepo
|
||||
{} as never, // pageRepo
|
||||
{} as never, // pageAccess
|
||||
{} as never, // environment
|
||||
{} as never, // streamRegistry
|
||||
aiChatRunService as never, // aiChatRunService (#487)
|
||||
);
|
||||
return { svc, aiChatMessageRepo, aiChatRunService };
|
||||
}
|
||||
|
||||
it('reconcile() fires all four clauses IN ORDER (a -> b -> c -> d)', async () => {
|
||||
const order: string[] = [];
|
||||
const { svc } = makeService({
|
||||
messageRepo: {
|
||||
findStreamingWithTerminalRun: jest.fn(async () => {
|
||||
order.push('b:find');
|
||||
return [
|
||||
{ messageId: 'm1', workspaceId: 'ws1', runStatus: 'succeeded' },
|
||||
];
|
||||
}),
|
||||
stampTerminalIfStreaming: jest.fn(async () => {
|
||||
order.push('b:stamp');
|
||||
}),
|
||||
sweepStreamingWithoutActiveRun: jest.fn(async () => {
|
||||
order.push('d');
|
||||
return 0;
|
||||
}),
|
||||
},
|
||||
runService: {
|
||||
zombieRunIds: jest.fn(() => ['z1']),
|
||||
settleZombie: jest.fn(async () => {
|
||||
order.push('a');
|
||||
return true;
|
||||
}),
|
||||
reconcileStaleRuns: jest.fn(async () => {
|
||||
order.push('c');
|
||||
return 0;
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
await svc.reconcile();
|
||||
|
||||
expect(order).toEqual(['a', 'b:find', 'b:stamp', 'c', 'd']);
|
||||
});
|
||||
|
||||
it('a clause that THROWS does not abort the remaining clauses (per-clause try/catch isolation)', async () => {
|
||||
const { svc, aiChatMessageRepo, aiChatRunService } = makeService({
|
||||
messageRepo: {
|
||||
// Clause (b) blows up mid-reconcile.
|
||||
findStreamingWithTerminalRun: jest.fn(async () => {
|
||||
throw new Error('clause b DB blip');
|
||||
}),
|
||||
},
|
||||
runService: {
|
||||
zombieRunIds: jest.fn(() => ['z1']),
|
||||
},
|
||||
});
|
||||
|
||||
// reconcile() must SETTLE (the clause-b failure is swallowed), not reject.
|
||||
await expect(svc.reconcile()).resolves.toBeUndefined();
|
||||
|
||||
// (a) ran before (b); crucially (c) and (d) STILL ran despite (b) throwing —
|
||||
// the property a missing try/catch would break. MUTATION-VERIFY: drop clause
|
||||
// (b)'s try/catch and this reddens (the throw propagates, skipping c + d).
|
||||
expect(aiChatRunService!.settleZombie).toHaveBeenCalled(); // (a)
|
||||
expect(aiChatRunService!.reconcileStaleRuns).toHaveBeenCalled(); // (c)
|
||||
expect(
|
||||
aiChatMessageRepo.sweepStreamingWithoutActiveRun,
|
||||
).toHaveBeenCalled(); // (d)
|
||||
});
|
||||
|
||||
it('reconcileChat() settles THIS chat\'s stuck streaming rows by their run status', async () => {
|
||||
const { svc, aiChatMessageRepo } = makeService({
|
||||
messageRepo: {
|
||||
findStreamingWithTerminalRun: jest.fn(async () => [
|
||||
{ messageId: 'm1', workspaceId: 'ws1', runStatus: 'failed' },
|
||||
{ messageId: 'm2', workspaceId: 'ws1', runStatus: 'succeeded' },
|
||||
]),
|
||||
},
|
||||
});
|
||||
|
||||
await svc.reconcileChat('chat-1', 'ws1');
|
||||
|
||||
// Scoped to THIS chat and bounded at 50 (the user-facing opportunistic path).
|
||||
expect(
|
||||
aiChatMessageRepo.findStreamingWithTerminalRun,
|
||||
).toHaveBeenCalledWith(50, { chatId: 'chat-1', workspaceId: 'ws1' });
|
||||
// failed-run -> 'error'; every other terminal status -> 'aborted'.
|
||||
expect(aiChatMessageRepo.stampTerminalIfStreaming).toHaveBeenCalledWith(
|
||||
'm1',
|
||||
'ws1',
|
||||
'error',
|
||||
);
|
||||
expect(aiChatMessageRepo.stampTerminalIfStreaming).toHaveBeenCalledWith(
|
||||
'm2',
|
||||
'ws1',
|
||||
'aborted',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,65 @@
|
||||
import { flushAssistant } from './ai-chat.service';
|
||||
|
||||
/**
|
||||
* #491 STEP MARKER — `metadata.stepsPersisted` is written by the SAME flush that
|
||||
* builds `metadata.parts`, so the marker can never disagree with the persisted
|
||||
* parts (the step-alignment anchor the resume stack builds on). These are
|
||||
* PROPERTY tests: they assert the marker tracks the number of FINISHED steps for
|
||||
* every flush shape.
|
||||
*/
|
||||
|
||||
// A finished step carrying one line of text and one tool call/result.
|
||||
function step(i: number) {
|
||||
return {
|
||||
text: `step ${i}`,
|
||||
toolCalls: [
|
||||
{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } },
|
||||
],
|
||||
toolResults: [
|
||||
{ toolCallId: `c${i}`, toolName: 'getPage', output: { title: `T${i}` } },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe('flushAssistant step marker (#491)', () => {
|
||||
it('seed (no steps) → stepsPersisted 0', () => {
|
||||
const f = flushAssistant([], '', 'streaming');
|
||||
expect(f.metadata.stepsPersisted).toBe(0);
|
||||
});
|
||||
|
||||
it('PROPERTY: stepsPersisted equals the number of FINISHED steps, for any N', () => {
|
||||
for (let n = 0; n <= 6; n++) {
|
||||
const steps = Array.from({ length: n }, (_, i) => step(i));
|
||||
const f = flushAssistant(steps, '', 'streaming');
|
||||
expect(f.metadata.stepsPersisted).toBe(n);
|
||||
// ...and the parts actually contain those N steps' text (marker agrees with
|
||||
// the persisted parts — the atomicity the whole design relies on).
|
||||
const parts = f.metadata.parts as Array<Record<string, unknown>>;
|
||||
const textParts = parts.filter((p) => p.type === 'text');
|
||||
expect(textParts).toHaveLength(n);
|
||||
}
|
||||
});
|
||||
|
||||
it('an in-progress trailing partial does NOT increment the marker', () => {
|
||||
// 2 finished steps + a partial (not-yet-finished) trailing text: the marker
|
||||
// counts only the CONFIRMED step boundaries, not the partial.
|
||||
const f = flushAssistant([step(0), step(1)], 'partial third step', 'error', {
|
||||
error: 'boom',
|
||||
});
|
||||
expect(f.metadata.stepsPersisted).toBe(2);
|
||||
// The partial text IS persisted in parts (so the user sees it), but it is not a
|
||||
// counted step.
|
||||
const parts = f.metadata.parts as Array<Record<string, unknown>>;
|
||||
expect(parts[parts.length - 1]).toEqual({
|
||||
type: 'text',
|
||||
text: 'partial third step',
|
||||
});
|
||||
});
|
||||
|
||||
it('terminal completed flush counts all finished steps', () => {
|
||||
const f = flushAssistant([step(0), step(1), step(2)], '', 'completed', {
|
||||
finishReason: 'stop',
|
||||
});
|
||||
expect(f.metadata.stepsPersisted).toBe(3);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,209 @@
|
||||
import { randomBytes } from 'crypto';
|
||||
import { Client } from 'pg';
|
||||
import { flushAssistant, serializeSteps } from './ai-chat.service';
|
||||
|
||||
/**
|
||||
* #490 write-volume regression — an OBSERVABLE-PROPERTY test on a LIVE Postgres,
|
||||
* not "bytes through a mock repo" (a mock measures exactly the thing that does not
|
||||
* hurt). It drives a realistic 50-step run where each step returns a ~100 KB tool
|
||||
* output and, at every `onStepFinish`, UPDATEs the assistant row the way the
|
||||
* service does — then reads the REAL write volume via the `pg_current_wal_lsn()`
|
||||
* delta around the run.
|
||||
*
|
||||
* The property proven: v2 stores each tool OUTPUT only in `metadata.parts`, no
|
||||
* longer ALSO in the `tool_calls` trace. So:
|
||||
* 1. the trace (`tool_calls`) column's write volume is now O(Σ steps) — tiny,
|
||||
* linear outcome flags — vs the pre-#490 O(N²) that re-persisted every prior
|
||||
* output on every step; and
|
||||
* 2. the FULL-row write volume drops sharply (the duplicated output copy is gone).
|
||||
*
|
||||
* Connects to the local gitmost test Postgres (docker `gitmost-test-pg` on :5432);
|
||||
* SKIPS cleanly when that DB is not reachable so it never breaks a DB-less CI.
|
||||
*/
|
||||
const CONN =
|
||||
process.env.WAL_TEST_DATABASE_URL ??
|
||||
'postgresql://docmost:docmost_dev_pw@localhost:5432/docmost';
|
||||
|
||||
// A step whose tool output is ~100 KB (a page read), in the SDK StepLike shape.
|
||||
// The body is INCOMPRESSIBLE random text — a `'x'.repeat()` filler would TOAST-
|
||||
// compress to nothing and hide the real write volume (a page body does not).
|
||||
function makeStep(i: number, outputBytes = 100_000) {
|
||||
const body = randomBytes(Math.ceil(outputBytes * 0.75)).toString('base64');
|
||||
return {
|
||||
text: `step ${i} reasoning`,
|
||||
toolCalls: [{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } }],
|
||||
toolResults: [
|
||||
{
|
||||
toolCallId: `c${i}`,
|
||||
toolName: 'getPage',
|
||||
output: { id: `p${i}`, title: `Page ${i}`, body },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// The pre-#490 (v1) trace: outputs stored a SECOND time in `tool_calls`
|
||||
// (the duplication #490 removed). Mirrors the OLD serializeSteps shape.
|
||||
function v1Trace(steps: ReturnType<typeof makeStep>[]): unknown {
|
||||
const calls: unknown[] = [];
|
||||
for (const s of steps) {
|
||||
for (const c of s.toolCalls) calls.push({ toolName: c.toolName, input: c.input });
|
||||
for (const r of s.toolResults)
|
||||
calls.push({ toolName: r.toolName, output: r.output });
|
||||
}
|
||||
return calls;
|
||||
}
|
||||
|
||||
async function walDelta(
|
||||
client: Client,
|
||||
fn: () => Promise<void>,
|
||||
): Promise<number> {
|
||||
const before = (await client.query('SELECT pg_current_wal_lsn() AS l')).rows[0]
|
||||
.l as string;
|
||||
await fn();
|
||||
// NOTE: do NOT pg_switch_wal() here — a segment switch pads the LSN to the next
|
||||
// 16 MB boundary and would swamp the actual write delta. The raw LSN advances by
|
||||
// the bytes of WAL emitted, which is exactly what we want to measure.
|
||||
const after = (await client.query('SELECT pg_current_wal_lsn() AS l')).rows[0]
|
||||
.l as string;
|
||||
return Number(
|
||||
(await client.query('SELECT pg_wal_lsn_diff($1,$2) AS d', [after, before]))
|
||||
.rows[0].d,
|
||||
);
|
||||
}
|
||||
|
||||
describe('#490 write-volume on a live Postgres (pg_current_wal_lsn delta)', () => {
|
||||
let client: Client | undefined;
|
||||
let available = false;
|
||||
|
||||
beforeAll(async () => {
|
||||
try {
|
||||
client = new Client(CONN);
|
||||
await client.connect();
|
||||
await client.query('SELECT pg_current_wal_lsn()');
|
||||
available = true;
|
||||
} catch {
|
||||
available = false;
|
||||
client = undefined;
|
||||
}
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await client?.end().catch(() => undefined);
|
||||
});
|
||||
|
||||
const STEPS = 50;
|
||||
|
||||
it('v2 trace write volume is O(Σ steps) — a tiny fraction of the v1 duplicate', async () => {
|
||||
if (!available || !client) {
|
||||
console.warn('SKIP: gitmost-test-pg not reachable; skipping WAL test.');
|
||||
return;
|
||||
}
|
||||
const c = client;
|
||||
// Isolated table so we measure only the tool_calls (trace) column's writes.
|
||||
await c.query('DROP TABLE IF EXISTS _wal_trace');
|
||||
await c.query('CREATE TABLE _wal_trace(id int primary key, tool_calls jsonb)');
|
||||
await c.query("INSERT INTO _wal_trace VALUES (1, '[]'::jsonb)");
|
||||
|
||||
const steps: ReturnType<typeof makeStep>[] = [];
|
||||
|
||||
// v1: each step re-persists ALL prior outputs into the trace (the O(N²) churn).
|
||||
const v1 = await walDelta(c, async () => {
|
||||
const acc: ReturnType<typeof makeStep>[] = [];
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
acc.push(makeStep(i));
|
||||
await c.query('UPDATE _wal_trace SET tool_calls=$1 WHERE id=1', [
|
||||
JSON.stringify(v1Trace(acc)),
|
||||
]);
|
||||
}
|
||||
steps.push(...acc);
|
||||
});
|
||||
|
||||
await c.query("UPDATE _wal_trace SET tool_calls='[]'::jsonb WHERE id=1");
|
||||
|
||||
// v2: the REAL serializeSteps — outcome flags only, NO outputs.
|
||||
const v2 = await walDelta(c, async () => {
|
||||
const acc: ReturnType<typeof makeStep>[] = [];
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
acc.push(makeStep(i));
|
||||
await c.query('UPDATE _wal_trace SET tool_calls=$1 WHERE id=1', [
|
||||
JSON.stringify(serializeSteps(acc)),
|
||||
]);
|
||||
}
|
||||
});
|
||||
|
||||
await c.query('DROP TABLE IF EXISTS _wal_trace');
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[#490 WAL] trace column over ${STEPS} steps: v1=${(v1 / 1e6).toFixed(1)}MB ` +
|
||||
`v2=${(v2 / 1e6).toFixed(2)}MB (${(v1 / v2).toFixed(0)}x smaller)`,
|
||||
);
|
||||
|
||||
// The trace no longer carries outputs: v2 is a tiny fraction of v1's WAL.
|
||||
expect(v2).toBeLessThan(v1 * 0.1);
|
||||
// And v2's trace WAL is small in absolute terms — O(Σ steps) of flags, not
|
||||
// O(N² × output). 50 steps of ~40-byte flags is well under a few MB of WAL.
|
||||
expect(v2).toBeLessThan(5_000_000);
|
||||
// v1's duplicate alone is huge (≈ the 100 KB output re-written N² times).
|
||||
expect(v1).toBeGreaterThan(50_000_000);
|
||||
}, 120_000);
|
||||
|
||||
it('the full assistant row write drops sharply once the duplicate is gone', async () => {
|
||||
if (!available || !client) return;
|
||||
const c = client;
|
||||
await c.query('DROP TABLE IF EXISTS _wal_full');
|
||||
await c.query(
|
||||
'CREATE TABLE _wal_full(id int primary key, content text, tool_calls jsonb, metadata jsonb, status text)',
|
||||
);
|
||||
await c.query("INSERT INTO _wal_full VALUES (1, '', '[]'::jsonb, '{}'::jsonb, 'streaming')");
|
||||
|
||||
const writeRow = async (patch: {
|
||||
content: string;
|
||||
toolCalls: unknown;
|
||||
metadata: unknown;
|
||||
status: string;
|
||||
}) =>
|
||||
c.query(
|
||||
'UPDATE _wal_full SET content=$1, tool_calls=$2, metadata=$3, status=$4 WHERE id=1',
|
||||
[
|
||||
patch.content,
|
||||
JSON.stringify(patch.toolCalls ?? null),
|
||||
JSON.stringify(patch.metadata),
|
||||
patch.status,
|
||||
],
|
||||
);
|
||||
|
||||
// v2 (real flushAssistant): outputs live once, in metadata.parts.
|
||||
const v2 = await walDelta(c, async () => {
|
||||
const acc: ReturnType<typeof makeStep>[] = [];
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
acc.push(makeStep(i));
|
||||
await writeRow(flushAssistant(acc as never, '', 'streaming'));
|
||||
}
|
||||
});
|
||||
|
||||
await c.query("UPDATE _wal_full SET content='', tool_calls='[]'::jsonb, metadata='{}'::jsonb WHERE id=1");
|
||||
|
||||
// v1: same row PLUS the duplicated outputs in the trace column.
|
||||
const v1 = await walDelta(c, async () => {
|
||||
const acc: ReturnType<typeof makeStep>[] = [];
|
||||
for (let i = 0; i < STEPS; i++) {
|
||||
acc.push(makeStep(i));
|
||||
const f = flushAssistant(acc as never, '', 'streaming');
|
||||
await writeRow({ ...f, toolCalls: v1Trace(acc) });
|
||||
}
|
||||
});
|
||||
|
||||
await c.query('DROP TABLE IF EXISTS _wal_full');
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(
|
||||
`[#490 WAL] full row over ${STEPS} steps: v1=${(v1 / 1e6).toFixed(1)}MB ` +
|
||||
`v2=${(v2 / 1e6).toFixed(1)}MB (saved ${((1 - v2 / v1) * 100).toFixed(0)}%)`,
|
||||
);
|
||||
|
||||
// Removing the duplicated trace copy is a large, real write-volume reduction.
|
||||
expect(v2).toBeLessThan(v1 * 0.75);
|
||||
}, 120_000);
|
||||
});
|
||||
@@ -1,4 +1,10 @@
|
||||
import { IsOptional, IsString, MaxLength, MinLength } from 'class-validator';
|
||||
import {
|
||||
IsISO8601,
|
||||
IsOptional,
|
||||
IsString,
|
||||
MaxLength,
|
||||
MinLength,
|
||||
} from 'class-validator';
|
||||
|
||||
/** Identify a chat by id (workspace-scoped on the server). */
|
||||
export class ChatIdDto {
|
||||
@@ -37,6 +43,24 @@ export class GetChatMessagesDto {
|
||||
cursor?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Delta poll (#491): pull the chat's rows changed since `cursor` (a DB-clock
|
||||
* timestamp from the previous poll) plus the current run fact — the degraded-poll
|
||||
* fallback's payload, replacing the full infinite-query refetch. Omit `cursor` on
|
||||
* the first poll (returns just a fresh cursor to start the chain).
|
||||
*/
|
||||
export class GetChatDeltaDto {
|
||||
@IsString()
|
||||
chatId: string;
|
||||
|
||||
// ISO-8601 timestamp echoed from the previous poll's response. Validated as
|
||||
// ISO-8601 (not a bare string): a malformed cursor would otherwise reach the
|
||||
// `::timestamptz` cast in findByChatUpdatedAfter and 500 instead of a clean 400.
|
||||
@IsOptional()
|
||||
@IsISO8601()
|
||||
cursor?: string;
|
||||
}
|
||||
|
||||
/** Resolve the chat bound to a document (the page's most-recent owned chat). */
|
||||
export class BoundChatDto {
|
||||
@IsString()
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
import { errors } from 'undici';
|
||||
import {
|
||||
McpClientsService,
|
||||
isRetryableConnectError,
|
||||
} from './mcp-clients.service';
|
||||
|
||||
/**
|
||||
* #489 — external-MCP in-run transport recovery.
|
||||
*
|
||||
* The transport-error classification + retry gate are exercised against the REAL
|
||||
* undici error CLASSES prod throws (`errors.SocketError` / `errors.BodyTimeoutError`,
|
||||
* carrying the true `UND_ERR_*` codes and class names), wrapped EXACTLY as undici's
|
||||
* `fetch` wraps them — a `TypeError('fetch failed'|'terminated')` whose `.cause` is
|
||||
* the undici error. These are the real classes, not hand-rolled `{code:'...'}`
|
||||
* mocks: constructing the genuine class is what makes this a faithful test of the
|
||||
* prod predicate (epic root-cause #4 — a mock-shaped predicate would leave the
|
||||
* evict/retry path silently dead in production while CI stays green). We construct
|
||||
* rather than drive a live fetch because Jest's environment degrades the live-fetch
|
||||
* error to a generic `Error` cause (no undici code), which would NOT be the prod
|
||||
* shape.
|
||||
*/
|
||||
|
||||
/** A REAL undici socket reset, wrapped as fetch wraps it. */
|
||||
function realSocketResetError(): unknown {
|
||||
const err = new TypeError('fetch failed');
|
||||
(err as { cause?: unknown }).cause = new errors.SocketError('other side closed');
|
||||
return err;
|
||||
}
|
||||
|
||||
/** A REAL undici body timeout, wrapped as fetch wraps it. */
|
||||
function realBodyTimeoutError(): unknown {
|
||||
const err = new TypeError('terminated');
|
||||
(err as { cause?: unknown }).cause = new errors.BodyTimeoutError();
|
||||
return err;
|
||||
}
|
||||
|
||||
type FakeServer = {
|
||||
id: string;
|
||||
name: string;
|
||||
transport: 'http' | 'sse';
|
||||
url: string;
|
||||
headersEnc: string | null;
|
||||
toolAllowlist: string[] | null;
|
||||
instructions: string | null;
|
||||
};
|
||||
|
||||
const server = (over: Partial<FakeServer> = {}): FakeServer => ({
|
||||
id: 's1',
|
||||
name: 'srv',
|
||||
transport: 'http',
|
||||
url: 'http://example.test/mcp',
|
||||
headersEnc: null,
|
||||
toolAllowlist: null,
|
||||
instructions: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
function buildService(servers: FakeServer[], trusted = false) {
|
||||
const repo = { listEnabled: jest.fn().mockResolvedValue(servers) };
|
||||
const service = new McpClientsService(repo as never, {} as never);
|
||||
// Seed a DETERMINISTIC write-class map so the retry gate is controlled here
|
||||
// (the production map loads from @docmost/mcp via a dynamic ESM import). getPage
|
||||
// is a read, patchNode is a write — the real classifications.
|
||||
(
|
||||
service as unknown as { writeClassMapPromise: Promise<unknown> }
|
||||
).writeClassMapPromise = Promise.resolve({
|
||||
getPage: 'readOnly',
|
||||
patchNode: 'write',
|
||||
});
|
||||
// The service only APPLIES that map to a TRUSTED internal Docmost server
|
||||
// (isInternalDocmostServer, really false for every third-party row). A retry
|
||||
// test needs a trusted server to exercise the readOnly-retry path at all, so it
|
||||
// passes trusted=true to model a Docmost-origin server; the third-party
|
||||
// double-apply test leaves it at the real value (false).
|
||||
if (trusted) {
|
||||
jest
|
||||
.spyOn(
|
||||
service as unknown as {
|
||||
isInternalDocmostServer: (s: FakeServer) => boolean;
|
||||
},
|
||||
'isInternalDocmostServer',
|
||||
)
|
||||
.mockReturnValue(true);
|
||||
}
|
||||
return { service, repo };
|
||||
}
|
||||
|
||||
/** Spy the private `connect` so each call yields a controlled fake client whose
|
||||
* single tool's execute is the supplied function. Returns the connect spy. */
|
||||
function stubConnect(
|
||||
service: McpClientsService,
|
||||
toolName: string,
|
||||
execs: Array<(...a: unknown[]) => Promise<unknown>>,
|
||||
) {
|
||||
let n = 0;
|
||||
return jest
|
||||
.spyOn(
|
||||
service as unknown as { connect: (s: FakeServer) => Promise<unknown> },
|
||||
'connect',
|
||||
)
|
||||
.mockImplementation(async () => {
|
||||
const exec = execs[Math.min(n, execs.length - 1)];
|
||||
n += 1;
|
||||
return {
|
||||
tools: async () => ({ [toolName]: { description: 'x', execute: exec } }),
|
||||
close: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
const opts = (abortSignal?: AbortSignal) =>
|
||||
({ toolCallId: 't', messages: [], abortSignal }) as never;
|
||||
|
||||
describe('isRetryableConnectError (#489, REAL error shapes)', () => {
|
||||
it('classifies a real undici socket reset and body timeout as retryable', async () => {
|
||||
const socketErr = await realSocketResetError();
|
||||
const bodyErr = await realBodyTimeoutError();
|
||||
expect(isRetryableConnectError(socketErr)).toBe(true);
|
||||
expect(isRetryableConnectError(bodyErr)).toBe(true);
|
||||
// Unwraps a wrapped cause chain (e.g. an MCPClientError around the socket err).
|
||||
const wrapped = new Error('mcp call failed');
|
||||
(wrapped as { cause?: unknown }).cause = socketErr;
|
||||
expect(isRetryableConnectError(wrapped)).toBe(true);
|
||||
});
|
||||
|
||||
it('does NOT classify an application-level error as a transport break', () => {
|
||||
expect(isRetryableConnectError(new Error('validation failed'))).toBe(false);
|
||||
expect(isRetryableConnectError({ name: 'HttpError', status: 400 })).toBe(false);
|
||||
expect(isRetryableConnectError(undefined)).toBe(false);
|
||||
expect(isRetryableConnectError('boom')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('McpClientsService in-run transport recovery (#489)', () => {
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('a readOnly tool whose transport breaks reconnects and retries WITHIN the same run', async () => {
|
||||
const realErr = await realSocketResetError();
|
||||
const { service } = buildService([server()], true);
|
||||
const first = jest.fn().mockRejectedValue(realErr);
|
||||
const second = jest.fn().mockResolvedValue({ ok: true });
|
||||
const connectSpy = stubConnect(service, 'getPage', [first, second]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-1');
|
||||
const tool = toolset.tools['srv_getPage'];
|
||||
const result = await (tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(),
|
||||
);
|
||||
|
||||
// The repeat call within the run got a LIVE client and succeeded.
|
||||
expect(result).toEqual({ ok: true });
|
||||
expect(first).toHaveBeenCalledTimes(1);
|
||||
expect(second).toHaveBeenCalledTimes(1);
|
||||
// Exactly one reconnect was minted (initial build connect + one recovery).
|
||||
expect(connectSpy).toHaveBeenCalledTimes(2);
|
||||
// The run accumulated BOTH leases (old + reconnected) — released together at end.
|
||||
expect(toolset.clients).toHaveLength(2);
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
|
||||
it('a WRITE tool does NOT auto-retry on a transport error (indeterminate)', async () => {
|
||||
const realErr = await realSocketResetError();
|
||||
const { service } = buildService([server()], true);
|
||||
const exec = jest.fn().mockRejectedValue(realErr);
|
||||
const connectSpy = stubConnect(service, 'patchNode', [exec]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-2');
|
||||
const tool = toolset.tools['srv_patchNode'];
|
||||
await expect(
|
||||
(tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(),
|
||||
),
|
||||
).rejects.toThrow(/MAY have already applied/);
|
||||
|
||||
// Called exactly once — NO blind retry (avoids double-apply, the #435 class).
|
||||
expect(exec).toHaveBeenCalledTimes(1);
|
||||
// No fresh connection was minted for a write.
|
||||
expect(connectSpy).toHaveBeenCalledTimes(1);
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
|
||||
it('does NOT retry (or reconnect) after the run is aborted (Stop)', async () => {
|
||||
const realErr = await realSocketResetError();
|
||||
const { service } = buildService([server()], true);
|
||||
const controller = new AbortController();
|
||||
// The transport error arrives, but the run was Stopped in the same tick.
|
||||
const first = jest.fn().mockImplementation(async () => {
|
||||
controller.abort();
|
||||
throw realErr;
|
||||
});
|
||||
const second = jest.fn().mockResolvedValue({ ok: true });
|
||||
const connectSpy = stubConnect(service, 'getPage', [first, second]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-3');
|
||||
const tool = toolset.tools['srv_getPage'];
|
||||
await expect(
|
||||
(tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(controller.signal),
|
||||
),
|
||||
).rejects.toBeDefined();
|
||||
|
||||
// getPage IS readOnly, but the Stop blocks the retry — no second call, no mint.
|
||||
expect(second).not.toHaveBeenCalled();
|
||||
expect(connectSpy).toHaveBeenCalledTimes(1);
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
|
||||
it('an app-level (non-transport) tool error is surfaced verbatim, never retried', async () => {
|
||||
const { service } = buildService([server()], true);
|
||||
const appErr = new Error('tool says: bad input');
|
||||
const exec = jest.fn().mockRejectedValue(appErr);
|
||||
const connectSpy = stubConnect(service, 'getPage', [exec]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-4');
|
||||
const tool = toolset.tools['srv_getPage'];
|
||||
await expect(
|
||||
(tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(),
|
||||
),
|
||||
).rejects.toThrow('tool says: bad input');
|
||||
expect(exec).toHaveBeenCalledTimes(1);
|
||||
expect(connectSpy).toHaveBeenCalledTimes(1); // no reconnect for an app error
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
|
||||
// #489 (review, MEDIUM) — the Docmost write-class map keys by DOCMOST tool
|
||||
// names; a THIRD-PARTY server may name a WRITE tool `getPage` (a Docmost read
|
||||
// name). It must NOT inherit readOnly and must NOT auto-retry on a transport
|
||||
// error — a blind retry of that write is a double-apply (the #435 class). Here
|
||||
// the server is UNTRUSTED (buildService default, isInternalDocmostServer=false),
|
||||
// so the map is not applied and `getPage` classifies as a write.
|
||||
//
|
||||
// MUTATION-VERIFY: forcing the server "trusted" (buildService(..., true)) makes
|
||||
// `getPage` inherit readOnly -> it WOULD reconnect+retry (connect twice) and the
|
||||
// assertions below fail — i.e. removing the trust scope re-opens the bug.
|
||||
it('a THIRD-PARTY WRITE tool named like a Docmost read does NOT auto-retry (no double-apply)', async () => {
|
||||
const realErr = await realSocketResetError();
|
||||
// Untrusted: default trusted=false — a real third-party server.
|
||||
const { service } = buildService([server()]);
|
||||
const exec = jest.fn().mockRejectedValue(realErr);
|
||||
const connectSpy = stubConnect(service, 'getPage', [exec, exec]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-5');
|
||||
const tool = toolset.tools['srv_getPage'];
|
||||
await expect(
|
||||
(tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(),
|
||||
),
|
||||
).rejects.toThrow(/MAY have already applied/);
|
||||
|
||||
// Exactly one call, NO reconnect — the name collision granted no readOnly-retry.
|
||||
expect(exec).toHaveBeenCalledTimes(1);
|
||||
expect(connectSpy).toHaveBeenCalledTimes(1);
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
});
|
||||
@@ -106,8 +106,11 @@ describe('McpClientsService.decryptHeaders', () => {
|
||||
|
||||
describe('McpClientsService.guardedFetch (SSRF per-request guard)', () => {
|
||||
// The bound guardedFetch closure lives on the instance as a private field.
|
||||
// #489 split it into per-transport HTTP/SSE bindings (they differ only in the
|
||||
// dispatcher's bodyTimeout); the SSRF guard is identical, so testing the HTTP
|
||||
// one is sufficient.
|
||||
const guardedFetchOf = (service: McpClientsService) =>
|
||||
(service as unknown as { guardedFetch: typeof fetch }).guardedFetch;
|
||||
(service as unknown as { guardedFetchHttp: typeof fetch }).guardedFetchHttp;
|
||||
|
||||
let fetchSpy: jest.SpiedFunction<typeof fetch>;
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { isIP } from 'node:net';
|
||||
import { lookup as dnsLookup, type LookupAddress } from 'node:dns';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { type Tool, type ToolCallOptions } from 'ai';
|
||||
import { createMCPClient } from '@ai-sdk/mcp';
|
||||
@@ -10,9 +11,29 @@ import {
|
||||
streamingDispatcherOptions,
|
||||
mcpStreamTimeoutMs,
|
||||
mcpCallTimeoutMs,
|
||||
mcpSseBodyTimeoutMs,
|
||||
} from '../../../integrations/ai/ai-streaming-fetch';
|
||||
import { SecretBoxService } from '../../../integrations/crypto/secret-box';
|
||||
import { isUrlAllowed, isIpAllowed } from './ssrf-guard';
|
||||
// TYPE-ONLY (erased at compile): @docmost/mcp is ESM-only and cannot be a runtime
|
||||
// `require()` from this commonjs module (same constraint as docmost-client.loader).
|
||||
// The write-class MAP is loaded lazily via the dynamic-import trick below.
|
||||
import type { ToolWriteClass } from '@docmost/mcp';
|
||||
|
||||
// TS(commonjs) downlevels a literal `import()` to `require()`, which cannot load
|
||||
// the ESM-only @docmost/mcp. Indirect through Function so the real dynamic
|
||||
// `import()` survives compilation (same trick as docmost-client.loader.ts).
|
||||
const esmImport = new Function(
|
||||
'specifier',
|
||||
'return import(specifier)',
|
||||
) as (specifier: string) => Promise<unknown>;
|
||||
|
||||
/** Local read-only predicate — avoids a value import of the ESM-only package.
|
||||
* Only a pure read is retry-safe after a transport break (a write is
|
||||
* indeterminate). Kept in lockstep with @docmost/mcp's isRetryableWriteClass. */
|
||||
function isReadOnlyWriteClass(writeClass: ToolWriteClass | undefined): boolean {
|
||||
return writeClass === 'readOnly';
|
||||
}
|
||||
|
||||
/** A closable external MCP client handle. */
|
||||
export interface Closable {
|
||||
@@ -81,12 +102,52 @@ const MAX_TOOL_NAME_LENGTH = 64;
|
||||
* close until the turn releases it, so a TTL expiry mid-turn never closes a
|
||||
* client a stream is still executing against.
|
||||
*/
|
||||
/**
|
||||
* Where a merged (namespaced) tool came from, so the per-run recovery wrapper
|
||||
* (#489) can, on a transport error, reconnect THAT server and re-resolve the SAME
|
||||
* underlying tool by its raw name. `writeClass` gates the single auto-retry (a
|
||||
* read is retry-safe; a write is indeterminate). `serverIndex` indexes the
|
||||
* entry's `servers` array (which server config to reconnect).
|
||||
*/
|
||||
interface ToolProvenance {
|
||||
serverIndex: number;
|
||||
rawName: string;
|
||||
writeClass: ToolWriteClass | undefined;
|
||||
}
|
||||
|
||||
/** A live reconnected server (its fresh client + raw call-timeout-wrapped tools). */
|
||||
interface RecoveredServerState {
|
||||
client: McpClient;
|
||||
tools: Record<string, Tool>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-run, per-server recovery binding (#489). `current` is the server's LIVE
|
||||
* target for this run: `null` means "use the ORIGINAL cached client/template";
|
||||
* a non-null value is a reconnected throwaway client all this server's tools now
|
||||
* call. `reconnecting` dedupes concurrent reconnects so only ONE fresh client is
|
||||
* minted per death (a losing concurrent call awaits it and retries on the SAME
|
||||
* new client — the CAS-by-identity rule).
|
||||
*/
|
||||
interface ServerBinding {
|
||||
current: RecoveredServerState | null;
|
||||
reconnecting?: Promise<RecoveredServerState>;
|
||||
}
|
||||
|
||||
interface CacheEntry {
|
||||
tools: Record<string, Tool>;
|
||||
clients: McpClient[];
|
||||
outcomes: ServerOutcome[];
|
||||
/** Prompt guidance for qualifying servers (see McpServerInstruction). */
|
||||
instructions: McpServerInstruction[];
|
||||
/**
|
||||
* The enabled server configs used to build this entry (#489), so the per-run
|
||||
* recovery wrapper can reconnect a specific server by index. Parallel to the
|
||||
* indices referenced by {@link toolMeta}.
|
||||
*/
|
||||
servers: AiMcpServer[];
|
||||
/** merged-tool-key -> provenance (#489), for the per-run recovery wrapper. */
|
||||
toolMeta: Record<string, ToolProvenance>;
|
||||
expiresAt: number;
|
||||
/** Active leases (turns currently using these clients). */
|
||||
refCount: number;
|
||||
@@ -120,20 +181,82 @@ export class McpClientsService {
|
||||
*/
|
||||
private readonly cache = new Map<string, Promise<CacheEntry>>();
|
||||
/**
|
||||
* A single shared SSRF-pinned dispatcher for ALL outbound external-MCP fetches.
|
||||
* Its custom connect.lookup runs per connection, so one instance safely guards
|
||||
* every server's connections (we never connect to an unvalidated IP).
|
||||
* SSRF-pinned dispatchers for outbound external-MCP fetches. Both use the SAME
|
||||
* custom connect.lookup (so every connection is IP-validated), but carry a
|
||||
* DIFFERENT `bodyTimeout` (#489): the HTTP (streamable) transport opens a fresh
|
||||
* request per call, so it keeps the tight silence timeout; the SSE transport
|
||||
* holds ONE long-lived body open across many calls, so a >1-min idle BETWEEN
|
||||
* calls is LEGITIMATE and must not break the socket — it gets a much larger
|
||||
* bodyTimeout. (headersTimeout stays tight on both.)
|
||||
*/
|
||||
private readonly dispatcher: Dispatcher = buildPinnedDispatcher();
|
||||
/** guardedFetch bound to the pinned dispatcher; reused by every transport. */
|
||||
private readonly guardedFetch: typeof fetch = (input, init) =>
|
||||
guardedFetch(this.dispatcher, input, init);
|
||||
private readonly dispatcherHttp: Dispatcher = buildPinnedDispatcher(
|
||||
mcpStreamTimeoutMs(),
|
||||
);
|
||||
private readonly dispatcherSse: Dispatcher = buildPinnedDispatcher(
|
||||
mcpSseBodyTimeoutMs(),
|
||||
);
|
||||
/** guardedFetch bound to each dispatcher; picked by transport type in connect(). */
|
||||
private readonly guardedFetchHttp: typeof fetch = (input, init) =>
|
||||
guardedFetch(this.dispatcherHttp, input, init);
|
||||
private readonly guardedFetchSse: typeof fetch = (input, init) =>
|
||||
guardedFetch(this.dispatcherSse, input, init);
|
||||
|
||||
/**
|
||||
* Memoized write-class map (#489), loaded lazily from @docmost/mcp via the
|
||||
* dynamic-import trick. Keyed by tool name (=== mcpName). A tool NOT in the map
|
||||
* (any third-party external MCP tool) classifies as `undefined` -> treated as a
|
||||
* write by the retry gate (the safe default: never blind-retry an unknown tool).
|
||||
* On any load failure the map is `{}` (every tool -> no auto-retry), so a
|
||||
* missing/older @docmost/mcp build only DISABLES retries, never mis-retries.
|
||||
*/
|
||||
private writeClassMapPromise: Promise<Record<string, ToolWriteClass>> | null =
|
||||
null;
|
||||
|
||||
constructor(
|
||||
private readonly repo: AiMcpServerRepo,
|
||||
private readonly secretBox: SecretBoxService,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Whether an external MCP server is the TRUSTED internal Docmost MCP server —
|
||||
* the only server whose tools may be classified by the Docmost write-class map
|
||||
* (#489 review). Today this is ALWAYS false: every `ai_mcp_servers` row is an
|
||||
* admin-configured THIRD-PARTY endpoint (there is no builtin/self flag, sentinel
|
||||
* URL, or synthetic server in this path — Docmost's OWN tools are exposed via the
|
||||
* separate in-app tools path, never through this external-MCP client). So no
|
||||
* third-party tool can inherit `readOnly` by a name collision with a Docmost read
|
||||
* tool, and none is ever auto-retried on a transport error (which would risk a
|
||||
* double-apply — the #435 class). Flip this (an explicit `kind`/`isBuiltin`
|
||||
* column, or a configured self-MCP URL) if a trusted internal server is ever
|
||||
* introduced. A method (not a free function) so it is a single, mockable seam.
|
||||
*/
|
||||
private isInternalDocmostServer(_server: AiMcpServer): boolean {
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Lazily load + memoize the shared write-class map (see the field doc). */
|
||||
private getWriteClassMap(): Promise<Record<string, ToolWriteClass>> {
|
||||
if (!this.writeClassMapPromise) {
|
||||
this.writeClassMapPromise = (async () => {
|
||||
try {
|
||||
const entry = require.resolve('@docmost/mcp');
|
||||
const mod = (await esmImport(pathToFileURL(entry).href)) as {
|
||||
SHARED_TOOL_WRITE_CLASS?: Record<string, ToolWriteClass>;
|
||||
};
|
||||
return mod.SHARED_TOOL_WRITE_CLASS ?? {};
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Could not load MCP write-class map (auto-retry disabled): ${shortError(
|
||||
err,
|
||||
)}`,
|
||||
);
|
||||
return {};
|
||||
}
|
||||
})();
|
||||
}
|
||||
return this.writeClassMapPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build (or reuse a cached) external toolset for a workspace. Returns the
|
||||
* merged tools, the open client handles to release, and per-server outcomes.
|
||||
@@ -162,11 +285,37 @@ export class McpClientsService {
|
||||
}
|
||||
},
|
||||
};
|
||||
// One release handle drives the whole leased entry; closing it releases all
|
||||
// underlying clients together (they share the same lease lifecycle).
|
||||
|
||||
// #489: the run accumulates a SET of leases — the primary cache lease PLUS any
|
||||
// throwaway client minted by an in-run transport-recovery reconnect. They are
|
||||
// NEVER released mid-run (releasing a swapped-out client while a concurrent
|
||||
// in-flight call still holds it would INDUCE a second failure); the caller
|
||||
// releases the WHOLE set together at turn-end. A recovery reconnect pushes its
|
||||
// lease onto this live array, which the consumer closes over.
|
||||
const leaseSet: Closable[] = [release];
|
||||
|
||||
// #489: per-RUN transport-recovery binding, one per server, SHARED by all of
|
||||
// that server's tools so a swap by one call is seen by the next (CAS by
|
||||
// identity). Kept per-run (here, not in the cached entry) because the binding
|
||||
// + lease-set state is per-run.
|
||||
const bindings = new Map<number, ServerBinding>();
|
||||
const capMs = mcpCallTimeoutMs();
|
||||
|
||||
// Wrap each cached tool with the recovery layer. On a transport error a
|
||||
// declared readOnly tool reconnects its server and retries ONCE; a write is
|
||||
// never blind-retried (indeterminate — may have applied before the reset). A
|
||||
// tool without provenance (a minimal stub entry in a test) passes through raw.
|
||||
const tools: Record<string, Tool> = {};
|
||||
for (const [key, tool] of Object.entries(entry.tools)) {
|
||||
const meta = entry.toolMeta?.[key];
|
||||
tools[key] = meta
|
||||
? this.wrapWithTransportRecovery(entry, meta, tool, leaseSet, bindings, capMs)
|
||||
: tool;
|
||||
}
|
||||
|
||||
return {
|
||||
tools: entry.tools,
|
||||
clients: [release],
|
||||
tools,
|
||||
clients: leaseSet,
|
||||
outcomes: entry.outcomes,
|
||||
instructions: entry.instructions,
|
||||
};
|
||||
@@ -254,6 +403,16 @@ export class McpClientsService {
|
||||
// Per-call total wall-clock cap, read once for this build (env-overridable).
|
||||
const callTimeoutMs = mcpCallTimeoutMs();
|
||||
const instructions: McpServerInstruction[] = [];
|
||||
// merged-key -> provenance for the per-run recovery wrapper (#489).
|
||||
const toolMeta: Record<string, ToolProvenance> = {};
|
||||
// Shared Docmost write-class map (#489) — classifies a tool by its raw name.
|
||||
// Loaded ONLY when at least one server is a TRUSTED internal Docmost server
|
||||
// (see isInternalDocmostServer): for third-party servers the map is never
|
||||
// applied (a name collision must not grant readOnly-retry), so we skip the
|
||||
// dynamic ESM load entirely in that (currently universal) case.
|
||||
const writeClassMap = servers.some((s) => this.isInternalDocmostServer(s))
|
||||
? await this.getWriteClassMap()
|
||||
: null;
|
||||
|
||||
// Per-server connect+tools result, still tagged with its server so the merge
|
||||
// below can be applied in the SAME order as `servers` (see the parallel note).
|
||||
@@ -327,11 +486,23 @@ export class McpClientsService {
|
||||
// against names already merged from earlier servers, so no external
|
||||
// tool is silently overwritten on collision. The returned count drives
|
||||
// whether this server's prompt guidance is included (≥1 tool merged).
|
||||
// #489 (review): the Docmost write-class map keys by DOCMOST tool names and
|
||||
// may ONLY be trusted for a server KNOWN to be the internal Docmost MCP
|
||||
// server. Every row here is an admin-configured THIRD-PARTY endpoint, so a
|
||||
// third-party WRITE tool that happens to be named like a Docmost read
|
||||
// (getPage, listPages, ...) must NOT inherit readOnly — that would auto-retry
|
||||
// a mutation on a transport error (double-apply, the #435 class). Gate the
|
||||
// map on the trust check; untrusted servers get writeClass=undefined -> the
|
||||
// recovery wrapper treats them as writes and never auto-retries.
|
||||
const trustWriteClass = this.isInternalDocmostServer(server);
|
||||
const merged = this.mergeNamespaced(
|
||||
tools,
|
||||
result.guarded,
|
||||
server.name,
|
||||
server.id,
|
||||
toolMeta,
|
||||
i,
|
||||
trustWriteClass ? writeClassMap : null,
|
||||
);
|
||||
outcomes.push({ name: server.name, ok: true });
|
||||
// Include this server's guidance ONLY when it actually contributed at
|
||||
@@ -353,6 +524,8 @@ export class McpClientsService {
|
||||
clients,
|
||||
outcomes,
|
||||
instructions,
|
||||
servers,
|
||||
toolMeta,
|
||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||
refCount: 0,
|
||||
evicted: false,
|
||||
@@ -379,18 +552,33 @@ export class McpClientsService {
|
||||
picked: Record<string, Tool>,
|
||||
serverName: string,
|
||||
serverId: string,
|
||||
toolMeta: Record<string, ToolProvenance>,
|
||||
serverIndex: number,
|
||||
// The Docmost write-class map, or `null` for an UNTRUSTED (third-party)
|
||||
// server whose tools must all default to write (never auto-retried).
|
||||
writeClassMap: Record<string, ToolWriteClass> | null,
|
||||
): { count: number; prefix: string } {
|
||||
let count = 0;
|
||||
for (const [name, tool] of Object.entries(namespace(picked, serverName))) {
|
||||
let key = name;
|
||||
for (const { full, raw, tool } of namespace(picked, serverName)) {
|
||||
let key = full;
|
||||
if (key in target) {
|
||||
const original = key;
|
||||
key = disambiguate(name, serverId, (candidate) => candidate in target);
|
||||
key = disambiguate(full, serverId, (candidate) => candidate in target);
|
||||
this.logger.debug(
|
||||
`External MCP tool name "${original}" collided; renamed to "${key}"`,
|
||||
);
|
||||
}
|
||||
target[key] = tool;
|
||||
// Record provenance so the per-run recovery wrapper (#489) can reconnect
|
||||
// this tool's server and re-resolve it by its raw name. writeClass is set
|
||||
// ONLY from a TRUSTED (internal-Docmost) map; for a third-party server the
|
||||
// map is null -> writeClass stays undefined -> the wrapper treats the tool
|
||||
// as a write and never auto-retries it (no double-apply on name collision).
|
||||
toolMeta[key] = {
|
||||
serverIndex,
|
||||
rawName: raw,
|
||||
writeClass: writeClassMap ? writeClassMap[raw] : undefined,
|
||||
};
|
||||
count += 1;
|
||||
}
|
||||
return { count, prefix: namespacePrefix(serverName) };
|
||||
@@ -424,7 +612,10 @@ export class McpClientsService {
|
||||
// Defense in depth: re-validate the actual request host on EVERY fetch
|
||||
// AND pin the socket to a validated IP via the dispatcher's connect
|
||||
// lookup, closing the DNS-rebinding TOCTOU between check and connect.
|
||||
fetch: this.guardedFetch,
|
||||
// #489: the SSE transport uses the raised-bodyTimeout dispatcher (idle
|
||||
// between calls is legit); HTTP uses the tight one.
|
||||
fetch:
|
||||
transportType === 'sse' ? this.guardedFetchSse : this.guardedFetchHttp,
|
||||
},
|
||||
})) as unknown as McpClient;
|
||||
return client;
|
||||
@@ -505,6 +696,176 @@ export class McpClientsService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap one merged external tool with the per-run transport-recovery layer (#489).
|
||||
*
|
||||
* attempt 1 runs on the server's CURRENT binding (the cached client, or a client
|
||||
* a sibling tool already reconnected this run). On a REAL transport error
|
||||
* (undici/@ai-sdk socket/body-timeout shapes — {@link isRetryableConnectError},
|
||||
* NOT a mock) and ONLY for a declared readOnly tool, it reconnects the server
|
||||
* and retries EXACTLY ONCE on the fresh client; a write is surfaced as an
|
||||
* indeterminate error (it may have applied before the reset — never
|
||||
* blind-retried). A single per-call cap bounds BOTH attempts + the reconnect,
|
||||
* and the run's abort signal is checked before the retry AND before minting a
|
||||
* fresh connection (no connection is opened for a stopped run).
|
||||
*/
|
||||
private wrapWithTransportRecovery(
|
||||
entry: CacheEntry,
|
||||
meta: ToolProvenance,
|
||||
template: Tool,
|
||||
leaseSet: Closable[],
|
||||
bindings: Map<number, ServerBinding>,
|
||||
capMs: number,
|
||||
): Tool {
|
||||
const original = template.execute;
|
||||
if (typeof original !== 'function') return template;
|
||||
const service = this;
|
||||
const { serverIndex, rawName, writeClass } = meta;
|
||||
|
||||
let binding = bindings.get(serverIndex);
|
||||
if (!binding) {
|
||||
binding = { current: null };
|
||||
bindings.set(serverIndex, binding);
|
||||
}
|
||||
const boundBinding = binding;
|
||||
|
||||
const execute = async (args: unknown, options: ToolCallOptions) => {
|
||||
// The per-call cap governs the WHOLE sequence (attempt1 + reconnect +
|
||||
// attempt2). Compose it with the run's abort signal so a Stop or the cap
|
||||
// ends any awaited call — @ai-sdk/mcp does not settle on abort, so we RACE.
|
||||
const capController = new AbortController();
|
||||
const capTimer = setTimeout(() => {
|
||||
capController.abort(new Error(`MCP tool call timed out after ${capMs}ms`));
|
||||
}, capMs);
|
||||
capTimer.unref?.();
|
||||
const runSignal = options?.abortSignal;
|
||||
const composed = runSignal
|
||||
? AbortSignal.any([runSignal, capController.signal])
|
||||
: capController.signal;
|
||||
const stopped = () => runSignal?.aborted === true || capController.signal.aborted;
|
||||
|
||||
const callOn = async (
|
||||
exec: NonNullable<Tool['execute']>,
|
||||
): Promise<unknown> => {
|
||||
const aborted = new Promise<never>((_, reject) => {
|
||||
const fail = () => reject(abortReason(composed));
|
||||
if (composed.aborted) fail();
|
||||
else composed.addEventListener('abort', fail, { once: true });
|
||||
});
|
||||
return Promise.race([exec(args, { ...options, abortSignal: composed }), aborted]);
|
||||
};
|
||||
|
||||
const execFor = (
|
||||
state: RecoveredServerState | null,
|
||||
): NonNullable<Tool['execute']> | undefined =>
|
||||
state ? (state.tools[rawName]?.execute as NonNullable<Tool['execute']>) : original;
|
||||
|
||||
try {
|
||||
// Snapshot the target BEFORE the call so a swap by a concurrent call is
|
||||
// detected by identity in the catch.
|
||||
const attemptState = boundBinding.current;
|
||||
const attemptExec = execFor(attemptState);
|
||||
if (typeof attemptExec !== 'function') {
|
||||
throw new Error(`external MCP tool "${rawName}" is not callable`);
|
||||
}
|
||||
try {
|
||||
return await callOn(attemptExec);
|
||||
} catch (err) {
|
||||
// Never retry on a Stop or an exhausted cap.
|
||||
if (stopped()) throw err;
|
||||
// Only a genuine transport break is a recovery candidate.
|
||||
if (!isRetryableConnectError(err)) throw err;
|
||||
// A write tool is INDETERMINATE on a transport error (may have applied
|
||||
// before the reset) — surface that; do NOT auto-retry (double-apply is
|
||||
// the #435 incident class).
|
||||
if (!isReadOnlyWriteClass(writeClass)) {
|
||||
throw new Error(
|
||||
`external MCP tool "${rawName}" hit a transport error and MAY have already ` +
|
||||
`applied on the server — not retried automatically; verify state before ` +
|
||||
`retrying. (${shortError(err)})`,
|
||||
);
|
||||
}
|
||||
// Abort check BEFORE minting a fresh connection (no socket for a
|
||||
// stopped run). LIMITATION (#489, LOW): the reconnect's own connect is
|
||||
// bounded by CONNECT_TIMEOUT_MS but does NOT itself observe `composed`,
|
||||
// so a Stop that lands DURING the handshake is only honored at the next
|
||||
// `stopped()` gate (before the retry) — a bounded ≤5s late-abort window;
|
||||
// the throwaway client is closed at turn-end regardless. Threading
|
||||
// `composed` into the SHARED (CAS-deduped) reconnect is deliberately
|
||||
// avoided: it would let the first caller's abort tear down a reconnect a
|
||||
// concurrent still-live caller depends on.
|
||||
if (stopped()) throw err;
|
||||
// CAS-swap by IDENTITY: mint+swap only if nobody swapped since this
|
||||
// call's snapshot; a losing concurrent call awaits the same reconnect
|
||||
// and retries on the SAME fresh client.
|
||||
let target: RecoveredServerState;
|
||||
if (boundBinding.current === attemptState) {
|
||||
if (!boundBinding.reconnecting) {
|
||||
boundBinding.reconnecting = (async () => {
|
||||
const server = entry.servers[serverIndex];
|
||||
const fresh = await service.reconnectServer(server, capMs);
|
||||
leaseSet.push(fresh.lease); // accumulate; released at turn-end
|
||||
boundBinding.current = fresh.state;
|
||||
return fresh.state;
|
||||
})();
|
||||
// Clear the in-flight marker once it settles (success or failure) so
|
||||
// a LATER death of the new client can reconnect again.
|
||||
void boundBinding.reconnecting.then(
|
||||
() => (boundBinding.reconnecting = undefined),
|
||||
() => (boundBinding.reconnecting = undefined),
|
||||
);
|
||||
}
|
||||
target = await boundBinding.reconnecting;
|
||||
} else {
|
||||
target = boundBinding.current as RecoveredServerState;
|
||||
}
|
||||
// Abort check BEFORE the retry.
|
||||
if (stopped()) throw err;
|
||||
const retryExec = execFor(target);
|
||||
if (typeof retryExec !== 'function') throw err;
|
||||
return await callOn(retryExec);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(capTimer);
|
||||
}
|
||||
};
|
||||
return { ...template, execute } as unknown as Tool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconnect ONE server for an in-run recovery (#489): open a fresh client and
|
||||
* list+wrap its tools. The throwaway client is NOT cached — it is owned by the
|
||||
* RUN via the returned lease (closed at turn-end), independent of the shared
|
||||
* cache entry (whose TTL rebuild heals future turns). On a failure the fresh
|
||||
* client is closed so its socket never leaks.
|
||||
*/
|
||||
private async reconnectServer(
|
||||
server: AiMcpServer,
|
||||
capMs: number,
|
||||
): Promise<{ state: RecoveredServerState; lease: Closable }> {
|
||||
const client = await this.connectWithTimeout(server, CONNECT_TIMEOUT_MS);
|
||||
let tools: Record<string, Tool>;
|
||||
try {
|
||||
const raw = await withTimeout(client.tools(), CONNECT_TIMEOUT_MS);
|
||||
const allow = server.toolAllowlist;
|
||||
const picked =
|
||||
Array.isArray(allow) && allow.length > 0 ? pick(raw, allow) : raw;
|
||||
tools = wrapToolsWithCallTimeout(picked, capMs);
|
||||
} catch (err) {
|
||||
void client.close().catch(() => undefined);
|
||||
throw err;
|
||||
}
|
||||
let released = false;
|
||||
const lease: Closable = {
|
||||
close: async () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
await client.close().catch(() => undefined);
|
||||
},
|
||||
};
|
||||
return { state: { client, tools }, lease };
|
||||
}
|
||||
|
||||
/** Mark an entry evicted; close its clients now if nothing is leasing them. */
|
||||
private evict(entry: CacheEntry): void {
|
||||
clearTimeout(entry.timer);
|
||||
@@ -554,22 +915,21 @@ export function validateResolvedAddresses(addrs: readonly LookupAddress[]): {
|
||||
* certificate validation still uses the real hostname (we never rewrite the URL
|
||||
* to an IP literal).
|
||||
*/
|
||||
function buildPinnedDispatcher(): Agent {
|
||||
// External-MCP traffic uses a DEDICATED, shorter silence timeout
|
||||
function buildPinnedDispatcher(bodyTimeoutMs: number): Agent {
|
||||
// External-MCP traffic uses a DEDICATED, shorter HEADERS silence timeout
|
||||
// (`AI_MCP_STREAM_TIMEOUT_MS`, default 1 min) — deliberately tighter than the
|
||||
// chat provider's 15-min `streamTimeoutMs()` — so a byte-silent/hung MCP
|
||||
// upstream is broken in ~1 min instead of 15. We keep the keep-alive options
|
||||
// from `streamingDispatcherOptions()` but OVERRIDE headers/body timeouts.
|
||||
// Accepted trade-off: a legitimately long but byte-silent single tool call,
|
||||
// and an SSE transport idling >1 min BETWEEN tool calls, are also cut here; the
|
||||
// per-call total cap (wrapToolsWithCallTimeout, `AI_MCP_CALL_TIMEOUT_MS`) is the
|
||||
// complementary guard for chatty-but-stuck calls that keep the socket warm yet
|
||||
// never return.
|
||||
const mcpSilenceMs = mcpStreamTimeoutMs();
|
||||
// from `streamingDispatcherOptions()` but OVERRIDE the timeouts. `bodyTimeout`
|
||||
// is passed in per-transport (#489): tight for HTTP (fresh request per call),
|
||||
// raised for SSE (one long-lived body across calls — idle BETWEEN calls is
|
||||
// legit). The per-call total cap (`AI_MCP_CALL_TIMEOUT_MS`) is the complementary
|
||||
// guard for chatty-but-stuck calls that keep the socket warm yet never return.
|
||||
const headersMs = mcpStreamTimeoutMs();
|
||||
return new Agent({
|
||||
...streamingDispatcherOptions(),
|
||||
headersTimeout: mcpSilenceMs,
|
||||
bodyTimeout: mcpSilenceMs,
|
||||
headersTimeout: headersMs,
|
||||
bodyTimeout: bodyTimeoutMs,
|
||||
connect: {
|
||||
lookup: (hostname, _options, callback) => {
|
||||
// Always resolve ALL addresses ourselves; do not trust the caller's
|
||||
@@ -669,18 +1029,22 @@ function pick(
|
||||
function namespace(
|
||||
tools: Record<string, Tool>,
|
||||
serverName: string,
|
||||
): Record<string, Tool> {
|
||||
): Array<{ full: string; raw: string; tool: Tool }> {
|
||||
const prefix = namespacePrefix(serverName);
|
||||
const out: Record<string, Tool> = {};
|
||||
const out: Array<{ full: string; raw: string; tool: Tool }> = [];
|
||||
const taken: Record<string, true> = {};
|
||||
for (const [name, t] of Object.entries(tools)) {
|
||||
const safe = sanitizeName(name);
|
||||
let full = capName(`${prefix}_${safe}`);
|
||||
// Duplicate names within ONE server can still collide after sanitize/
|
||||
// truncate — suffix-disambiguate so the second tool is not overwritten.
|
||||
if (full in out) {
|
||||
full = disambiguate(full, '', (candidate) => candidate in out);
|
||||
if (full in taken) {
|
||||
full = disambiguate(full, '', (candidate) => candidate in taken);
|
||||
}
|
||||
out[full] = t;
|
||||
taken[full] = true;
|
||||
// Keep the RAW (un-namespaced) name alongside the merged key so the per-run
|
||||
// recovery wrapper (#489) can re-resolve the same tool on a fresh client.
|
||||
out.push({ full, raw: name, tool: t });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -804,6 +1168,69 @@ export function wrapToolWithCallTimeout(tool: Tool, ms: number): Tool {
|
||||
return { ...tool, execute } as unknown as Tool;
|
||||
}
|
||||
|
||||
/**
|
||||
* undici / Node network error CODES that mean the connection broke (not an
|
||||
* application-level error) — a transient transport failure a readOnly call may
|
||||
* safely retry after reconnecting. Matched against the REAL error shapes (#489):
|
||||
* a socket reset surfaces as `TypeError: fetch failed` whose `.cause` is an
|
||||
* undici `SocketError { code:'UND_ERR_SOCKET' }`; a body-timeout as
|
||||
* `TypeError: terminated` whose `.cause` is `BodyTimeoutError`. Classifying by
|
||||
* these real codes/names (not by mock errors) is essential — a mock-shaped
|
||||
* predicate would leave eviction silently dead in production while CI is green.
|
||||
*/
|
||||
const RETRYABLE_TRANSPORT_ERROR_CODES: ReadonlySet<string> = new Set([
|
||||
'ECONNRESET',
|
||||
'ECONNREFUSED',
|
||||
'ECONNABORTED',
|
||||
'EPIPE',
|
||||
'ETIMEDOUT',
|
||||
'EAI_AGAIN',
|
||||
'ENETUNREACH',
|
||||
'EHOSTUNREACH',
|
||||
'UND_ERR_SOCKET',
|
||||
'UND_ERR_CONNECT_TIMEOUT',
|
||||
'UND_ERR_HEADERS_TIMEOUT',
|
||||
'UND_ERR_BODY_TIMEOUT',
|
||||
'UND_ERR_CLOSED',
|
||||
'UND_ERR_DESTROYED',
|
||||
]);
|
||||
|
||||
/** undici error CLASS names for the same transport-break conditions. */
|
||||
const RETRYABLE_TRANSPORT_ERROR_NAMES: ReadonlySet<string> = new Set([
|
||||
'SocketError',
|
||||
'BodyTimeoutError',
|
||||
'HeadersTimeoutError',
|
||||
'ConnectTimeoutError',
|
||||
'ClientClosedError',
|
||||
'ClientDestroyedError',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Whether `err` is a retryable TRANSPORT break (a broken socket / body timeout),
|
||||
* classified by the REAL undici/@ai-sdk error shapes (#489). undici surfaces a
|
||||
* reset as `TypeError('fetch failed'|'terminated')` with the real error in
|
||||
* `.cause`, and @ai-sdk/mcp may wrap it again in an `MCPClientError` (cause
|
||||
* chain), so we walk `.cause` (bounded depth) checking `.code` and `.name`. An
|
||||
* app-level tool error (a 4xx, a validation failure) is NOT retryable and returns
|
||||
* false — only a connection-level failure heals with a reconnect.
|
||||
*/
|
||||
export function isRetryableConnectError(err: unknown, depth = 0): boolean {
|
||||
if (!err || typeof err !== 'object' || depth > 6) return false;
|
||||
const e = err as {
|
||||
code?: unknown;
|
||||
name?: unknown;
|
||||
cause?: unknown;
|
||||
};
|
||||
if (typeof e.code === 'string' && RETRYABLE_TRANSPORT_ERROR_CODES.has(e.code)) {
|
||||
return true;
|
||||
}
|
||||
if (typeof e.name === 'string' && RETRYABLE_TRANSPORT_ERROR_NAMES.has(e.name)) {
|
||||
return true;
|
||||
}
|
||||
if (e.cause != null) return isRetryableConnectError(e.cause, depth + 1);
|
||||
return false;
|
||||
}
|
||||
|
||||
/** The signal's reason as an Error (informative thrown value on abort/timeout). */
|
||||
function abortReason(signal: AbortSignal): Error {
|
||||
const r = signal.reason;
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
import type { ModelMessage } from 'ai';
|
||||
import {
|
||||
resolveReplayBudget,
|
||||
isContextOverflowError,
|
||||
estimateMessagesTokens,
|
||||
trimHistoryForReplay,
|
||||
REPLAY_BUDGET_DEFAULT_TOKENS,
|
||||
REPLAY_TRUNCATION_MARKER,
|
||||
REPLAY_TURN_COLLAPSED_MARKER,
|
||||
} from './history-budget';
|
||||
|
||||
describe('resolveReplayBudget', () => {
|
||||
it('uses floor(0.7 x window) for a configured window (no cap)', () => {
|
||||
// 0.7 x 60k = 42k
|
||||
expect(resolveReplayBudget(60_000)).toEqual({
|
||||
thresholdTokens: 42_000,
|
||||
usedDefault: false,
|
||||
});
|
||||
// 0.7 x 1M = 700k — NOT capped (anti-brick vs the window, not a cost limiter).
|
||||
expect(resolveReplayBudget(1_000_000)).toEqual({
|
||||
thresholdTokens: 700_000,
|
||||
usedDefault: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('accepts the raw ::text stored form', () => {
|
||||
expect(resolveReplayBudget('60000').thresholdTokens).toBe(42_000);
|
||||
});
|
||||
|
||||
// The crux (#490): a chat with NO context window configured must STILL be
|
||||
// budgeted — those are exactly the installs that hit terminal overflow.
|
||||
it('applies the flat default when the window is unset/empty', () => {
|
||||
expect(resolveReplayBudget(undefined)).toEqual({
|
||||
thresholdTokens: REPLAY_BUDGET_DEFAULT_TOKENS,
|
||||
usedDefault: true,
|
||||
});
|
||||
expect(resolveReplayBudget('')).toEqual({
|
||||
thresholdTokens: REPLAY_BUDGET_DEFAULT_TOKENS,
|
||||
usedDefault: true,
|
||||
});
|
||||
expect(resolveReplayBudget(' ')).toEqual({
|
||||
thresholdTokens: REPLAY_BUDGET_DEFAULT_TOKENS,
|
||||
usedDefault: true,
|
||||
});
|
||||
});
|
||||
|
||||
it('treats an explicit 0 as the off-switch (distinct from unset)', () => {
|
||||
expect(resolveReplayBudget(0)).toEqual({
|
||||
thresholdTokens: null,
|
||||
usedDefault: false,
|
||||
});
|
||||
expect(resolveReplayBudget('0')).toEqual({
|
||||
thresholdTokens: null,
|
||||
usedDefault: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('falls back to the default on a negative/garbage value', () => {
|
||||
expect(resolveReplayBudget(-5).usedDefault).toBe(true);
|
||||
expect(resolveReplayBudget('abc').usedDefault).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('isContextOverflowError', () => {
|
||||
it('classifies a real provider 400 context-overflow shape', () => {
|
||||
// OpenAI-compatible shape.
|
||||
expect(
|
||||
isContextOverflowError({
|
||||
statusCode: 400,
|
||||
message:
|
||||
"This model's maximum context length is 128000 tokens. However, your messages resulted in 214000 tokens. Please reduce the length of the messages.",
|
||||
}),
|
||||
).toBe(true);
|
||||
// Anthropic-style wording.
|
||||
expect(
|
||||
isContextOverflowError({
|
||||
status: 400,
|
||||
message: 'prompt is too long: 250000 tokens > 200000 maximum',
|
||||
}),
|
||||
).toBe(true);
|
||||
// Nested body + string status.
|
||||
expect(
|
||||
isContextOverflowError({
|
||||
response: { status: '400' },
|
||||
message: 'input is too long for the requested model',
|
||||
}),
|
||||
).toBe(true);
|
||||
// Error instance with the cause carrying the body.
|
||||
const e = new Error('Bad request');
|
||||
(e as any).statusCode = 400;
|
||||
(e as any).cause = new Error('maximum context window exceeded');
|
||||
expect(isContextOverflowError(e)).toBe(true);
|
||||
});
|
||||
|
||||
it('does NOT classify unrelated 400s or auth/rate-limit errors', () => {
|
||||
expect(
|
||||
isContextOverflowError({ statusCode: 400, message: 'invalid tool schema' }),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isContextOverflowError({
|
||||
statusCode: 429,
|
||||
message: 'context length exceeded but rate limited',
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(isContextOverflowError({ statusCode: 500, message: 'server error' })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(isContextOverflowError(undefined)).toBe(false);
|
||||
expect(isContextOverflowError('some random string')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// Helpers to build ModelMessage fixtures in the ai@6 shape.
|
||||
const userMsg = (text: string): ModelMessage =>
|
||||
({ role: 'user', content: [{ type: 'text', text }] }) as ModelMessage;
|
||||
const assistantMsg = (
|
||||
text: string,
|
||||
toolCallId?: string,
|
||||
toolName?: string,
|
||||
): ModelMessage =>
|
||||
({
|
||||
role: 'assistant',
|
||||
content: [
|
||||
{ type: 'text', text },
|
||||
...(toolCallId
|
||||
? [{ type: 'tool-call', toolCallId, toolName, input: {} }]
|
||||
: []),
|
||||
],
|
||||
}) as ModelMessage;
|
||||
const toolMsg = (
|
||||
toolCallId: string,
|
||||
toolName: string,
|
||||
value: unknown,
|
||||
): ModelMessage =>
|
||||
({
|
||||
role: 'tool',
|
||||
content: [
|
||||
{ type: 'tool-result', toolCallId, toolName, output: { type: 'json', value } },
|
||||
],
|
||||
}) as ModelMessage;
|
||||
|
||||
describe('trimHistoryForReplay', () => {
|
||||
it('null budget disables trimming (returns the same reference)', () => {
|
||||
const msgs = [userMsg('hi'), assistantMsg('yo')];
|
||||
const r = trimHistoryForReplay(msgs, null);
|
||||
expect(r.trimmed).toBe(false);
|
||||
expect(r.messages).toBe(msgs);
|
||||
});
|
||||
|
||||
it('leaves history under budget untouched (same reference)', () => {
|
||||
const msgs = [userMsg('hi'), assistantMsg('a short answer')];
|
||||
const r = trimHistoryForReplay(msgs, 100_000);
|
||||
expect(r.trimmed).toBe(false);
|
||||
expect(r.messages).toBe(msgs);
|
||||
});
|
||||
|
||||
it('truncates OLD tool outputs but keeps recent turns full', () => {
|
||||
const big = 'X'.repeat(40_000); // ~16k tokens on its own
|
||||
const msgs: ModelMessage[] = [];
|
||||
// 6 OLD turns (indices 0..5), each with a huge tool output.
|
||||
for (let i = 0; i < 6; i++) {
|
||||
msgs.push(userMsg(`old q${i}`));
|
||||
msgs.push(assistantMsg('looking', `c${i}`, 'getPage'));
|
||||
msgs.push(toolMsg(`c${i}`, 'getPage', { body: big }));
|
||||
msgs.push(assistantMsg(`old a${i}`));
|
||||
}
|
||||
// 3 small recent turns, then the CURRENT turn with its own huge tool output.
|
||||
// With REPLAY_KEEP_RECENT_TURNS=4 the last 4 user-turns stay full, so only
|
||||
// these small recent turns + the current big one are kept full; the 6 old
|
||||
// turns above fall in the trim region.
|
||||
for (let i = 0; i < 3; i++) {
|
||||
msgs.push(userMsg(`recent q${i}`));
|
||||
msgs.push(assistantMsg(`recent a${i}`));
|
||||
}
|
||||
msgs.push(userMsg('current q'));
|
||||
msgs.push(assistantMsg('looking', 'cR', 'getPage'));
|
||||
msgs.push(toolMsg('cR', 'getPage', { body: big }));
|
||||
msgs.push(assistantMsg('current a'));
|
||||
|
||||
// Budget large enough that phase-1 tool truncation alone brings it under.
|
||||
const r = trimHistoryForReplay(msgs, 30_000);
|
||||
expect(r.trimmed).toBe(true);
|
||||
const flat = JSON.stringify(r.messages);
|
||||
// The CURRENT turn's tool output survives in full.
|
||||
expect(flat).toContain(big);
|
||||
// Old outputs were truncated with the marker.
|
||||
expect(flat).toContain(REPLAY_TRUNCATION_MARKER);
|
||||
// Phase 1 sufficed: the oldest turns were NOT collapsed.
|
||||
expect(flat).not.toContain(REPLAY_TURN_COLLAPSED_MARKER);
|
||||
expect(estimateMessagesTokens(r.messages)).toBeLessThan(
|
||||
estimateMessagesTokens(msgs),
|
||||
);
|
||||
});
|
||||
|
||||
it('collapses the oldest turns when tool truncation is not enough', () => {
|
||||
// Many turns with LARGE assistant TEXT (not tool output) so phase 1 can't help.
|
||||
const bigText = 'слово '.repeat(8_000); // large Cyrillic text per turn
|
||||
const msgs: ModelMessage[] = [];
|
||||
for (let i = 0; i < 12; i++) {
|
||||
msgs.push(userMsg(`q${i}`));
|
||||
msgs.push(assistantMsg(bigText));
|
||||
}
|
||||
const r = trimHistoryForReplay(msgs, 30_000);
|
||||
expect(r.trimmed).toBe(true);
|
||||
// Oldest turns collapsed; result fits (best-effort) and is much smaller.
|
||||
expect(estimateMessagesTokens(r.messages)).toBeLessThan(
|
||||
estimateMessagesTokens(msgs),
|
||||
);
|
||||
// The LAST turn's text is preserved in full (recent turns stay full).
|
||||
expect(JSON.stringify(r.messages[r.messages.length - 1])).toContain(bigText);
|
||||
});
|
||||
|
||||
it('is deterministic / byte-stable for identical inputs', () => {
|
||||
const big = 'Y'.repeat(30_000);
|
||||
const build = (): ModelMessage[] => {
|
||||
const m: ModelMessage[] = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
m.push(userMsg(`q${i}`));
|
||||
m.push(assistantMsg('t', `c${i}`, 'getPage'));
|
||||
m.push(toolMsg(`c${i}`, 'getPage', { body: big }));
|
||||
}
|
||||
return m;
|
||||
};
|
||||
const a = trimHistoryForReplay(build(), 15_000);
|
||||
const b = trimHistoryForReplay(build(), 15_000);
|
||||
expect(JSON.stringify(a.messages)).toBe(JSON.stringify(b.messages));
|
||||
});
|
||||
|
||||
it('never leaves an unpaired tool-call after collapsing (balanced history)', () => {
|
||||
const big = 'Z'.repeat(40_000);
|
||||
const msgs: ModelMessage[] = [];
|
||||
for (let i = 0; i < 10; i++) {
|
||||
msgs.push(userMsg(`q${i}`));
|
||||
msgs.push(assistantMsg('t', `c${i}`, 'getPage'));
|
||||
msgs.push(toolMsg(`c${i}`, 'getPage', { body: big }));
|
||||
}
|
||||
const r = trimHistoryForReplay(msgs, 8_000);
|
||||
// Count tool-call vs tool-result parts in the trimmed output.
|
||||
let calls = 0;
|
||||
let results = 0;
|
||||
for (const m of r.messages) {
|
||||
if (!Array.isArray(m.content)) continue;
|
||||
for (const p of m.content as Array<{ type?: string }>) {
|
||||
if (p.type === 'tool-call') calls++;
|
||||
if (p.type === 'tool-result' || p.type === 'tool-error') results++;
|
||||
}
|
||||
}
|
||||
// Every surviving tool-call has a surviving result (collapsing drops BOTH).
|
||||
expect(calls).toBe(results);
|
||||
// Collapsed turns carry the marker.
|
||||
expect(JSON.stringify(r.messages)).toContain(REPLAY_TURN_COLLAPSED_MARKER);
|
||||
});
|
||||
|
||||
it('respects the provider fact: under-budget contextTokens skips trimming', () => {
|
||||
const big = 'W'.repeat(60_000);
|
||||
const msgs = [
|
||||
userMsg('q'),
|
||||
assistantMsg('t', 'c1', 'getPage'),
|
||||
toolMsg('c1', 'getPage', { body: big }),
|
||||
];
|
||||
// char-estimate is high, but the provider says we are well under budget.
|
||||
const r = trimHistoryForReplay(msgs, 100_000, 5_000);
|
||||
expect(r.trimmed).toBe(false);
|
||||
expect(r.messages).toBe(msgs);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,375 @@
|
||||
/**
|
||||
* History-replay token budget (#490).
|
||||
*
|
||||
* The whole persisted conversation is replayed to the provider on EVERY turn, so
|
||||
* a long chat eventually exceeds the model's context window and the provider 400s
|
||||
* on every turn — terminally (the chat "bricks"). This module bounds the replayed
|
||||
* history at REPLAY TIME only: it never mutates what is persisted (the DB stays
|
||||
* the full record), and its output is a deterministic, byte-stable function of its
|
||||
* input so the trimmed prefix is identical turn to turn (provider prompt-cache
|
||||
* friendliness — real money on long chats).
|
||||
*
|
||||
* The PRIMARY signal is the provider's own fact: `metadata.contextTokens` from the
|
||||
* last turn. The chars-based {@link estimateTokens} (shared with the client) is
|
||||
* used only for the DELTA of not-yet-sent messages, to decide WHAT to trim, and as
|
||||
* the fallback for chats with no usage yet.
|
||||
*/
|
||||
import type { ModelMessage } from 'ai';
|
||||
import { estimateTokens } from '@docmost/token-estimate';
|
||||
|
||||
/** Flat default budget when no context window is configured (tokens). */
|
||||
export const REPLAY_BUDGET_DEFAULT_TOKENS = 100_000;
|
||||
/** Fraction of a configured context window used as the budget. */
|
||||
export const REPLAY_BUDGET_WINDOW_FRACTION = 0.7;
|
||||
/**
|
||||
* Fraction of the normal budget used for the REACTIVE re-trim after a provider
|
||||
* context-overflow 400 — the preventive estimate under-counted, so cut harder.
|
||||
*/
|
||||
export const REPLAY_AGGRESSIVE_FRACTION = 0.5;
|
||||
/**
|
||||
* Turns (a user message + its assistant/tool replies) kept FULL at the tail,
|
||||
* including the current one — never trimmed. Older turns are compacted first.
|
||||
*/
|
||||
export const REPLAY_KEEP_RECENT_TURNS = 4;
|
||||
/** Leading chars kept from a truncated old tool output. */
|
||||
export const REPLAY_TOOL_OUTPUT_HEAD = 800;
|
||||
/** Trailing chars kept from a truncated old tool output. */
|
||||
export const REPLAY_TOOL_OUTPUT_TAIL = 300;
|
||||
/** Marker inserted where an old tool output was truncated for replay. */
|
||||
export const REPLAY_TRUNCATION_MARKER =
|
||||
'[…truncated for replay; call the tool again to read the full output]';
|
||||
/** Marker for a whole old turn collapsed to its text. */
|
||||
export const REPLAY_TURN_COLLAPSED_MARKER =
|
||||
'[earlier tool activity omitted for replay]';
|
||||
|
||||
export interface ReplayBudget {
|
||||
/** Token threshold above which replay history is trimmed; `null` = OFF. */
|
||||
thresholdTokens: number | null;
|
||||
/** True when the flat default was used (no context window configured). */
|
||||
usedDefault: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the replay budget from the RAW stored `chatContextWindow` (text/number).
|
||||
* - a positive value -> `floor(fraction × window)` (NO cap — the budgeter is
|
||||
* anti-brick protection against the window itself, not a cost/economy limiter,
|
||||
* exactly as the codebase already treats maxOutputTokens; the reactive branch
|
||||
* still guarantees anti-brick regardless of how high this budget is)
|
||||
* - explicit `0` -> OFF (admin opt-out; `null` threshold)
|
||||
* - unset/empty/invalid-> the flat default (still protects — the installations
|
||||
* that hit terminal overflow are exactly the ones that never set a window)
|
||||
*
|
||||
* Note the raw value is needed because the parsed `chatContextWindow` collapses
|
||||
* both `0` and unset to `undefined`, which would erase the explicit off-switch.
|
||||
*/
|
||||
export function resolveReplayBudget(rawContextWindow: unknown): ReplayBudget {
|
||||
let n: number | undefined;
|
||||
if (typeof rawContextWindow === 'number') {
|
||||
n = rawContextWindow;
|
||||
} else if (typeof rawContextWindow === 'string') {
|
||||
const t = rawContextWindow.trim();
|
||||
n = t === '' ? undefined : Number(t);
|
||||
}
|
||||
// Unset / empty / non-numeric / negative -> flat default (the protective case).
|
||||
if (n === undefined || !Number.isFinite(n) || n < 0) {
|
||||
return { thresholdTokens: REPLAY_BUDGET_DEFAULT_TOKENS, usedDefault: true };
|
||||
}
|
||||
// Explicit 0 -> off-switch.
|
||||
if (n === 0) {
|
||||
return { thresholdTokens: null, usedDefault: false };
|
||||
}
|
||||
return {
|
||||
thresholdTokens: Math.floor(REPLAY_BUDGET_WINDOW_FRACTION * n),
|
||||
usedDefault: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The effective replay threshold for THIS turn, given the base budget and whether
|
||||
* the PREVIOUS turn hit a context-overflow 400 (the reactive-recovery signal,
|
||||
* `metadata.replayOverflow`). On recovery the base budget is scaled down by
|
||||
* {@link REPLAY_AGGRESSIVE_FRACTION}: the overflowing turn produced no usage
|
||||
* signal, so the preventive estimate under-counted and a normal-threshold trim may
|
||||
* not shrink enough to fit — this harder cut is what un-bricks the chat.
|
||||
*
|
||||
* A `null` base budget (trimming OFF) is passed through unchanged: an explicit
|
||||
* off-switch is never overridden by the recovery path.
|
||||
*/
|
||||
export function resolveEffectiveReplayThreshold(
|
||||
thresholdTokens: number | null,
|
||||
priorOverflowed: boolean,
|
||||
): number | null {
|
||||
if (!priorOverflowed || thresholdTokens == null) return thresholdTokens;
|
||||
return Math.floor(thresholdTokens * REPLAY_AGGRESSIVE_FRACTION);
|
||||
}
|
||||
|
||||
/**
|
||||
* True when a provider error is a CONTEXT-OVERFLOW rejection (the prompt exceeds
|
||||
* the model's window). Providers surface this as an HTTP 400 with a recognizable
|
||||
* message; match both the status and the message patterns robustly across
|
||||
* OpenAI-compatible / Anthropic / Gemini wordings, since the exact shape varies.
|
||||
*/
|
||||
export function isContextOverflowError(error: unknown): boolean {
|
||||
const status = extractStatus(error);
|
||||
const msg = extractMessage(error).toLowerCase();
|
||||
// Message patterns seen across providers for "prompt too long".
|
||||
const overflowPattern =
|
||||
/context (?:length|window)|maximum context|too many tokens|too large for|reduce the length|prompt is too long|input (?:is )?too long|exceeds? the (?:maximum )?(?:context|token)|maximum.*tokens|string too long/;
|
||||
if (!overflowPattern.test(msg)) return false;
|
||||
// A 400/413 with an overflow-shaped message is an overflow. Some providers
|
||||
// omit/rewrite the status, so accept the message match when the status is
|
||||
// unknown, but reject it for auth/rate-limit statuses that never mean overflow.
|
||||
if (status === 400 || status === 413) return true;
|
||||
if (status === 401 || status === 403 || status === 429) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function extractStatus(error: unknown): number | undefined {
|
||||
if (!error || typeof error !== 'object') return undefined;
|
||||
const e = error as Record<string, unknown>;
|
||||
for (const k of ['statusCode', 'status']) {
|
||||
const v = e[k];
|
||||
if (typeof v === 'number') return v;
|
||||
if (typeof v === 'string' && /^\d+$/.test(v)) return Number(v);
|
||||
}
|
||||
// Nested (e.g. { response: { status } } / { cause: { statusCode } }).
|
||||
for (const k of ['response', 'cause', 'data']) {
|
||||
const nested = e[k];
|
||||
if (nested && typeof nested === 'object') {
|
||||
const s = extractStatus(nested);
|
||||
if (s !== undefined) return s;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function extractMessage(error: unknown): string {
|
||||
if (error == null) return '';
|
||||
if (typeof error === 'string') return error;
|
||||
if (error instanceof Error) {
|
||||
// Include nested causes (provider libs wrap the real body in `cause`).
|
||||
const cause = (error as { cause?: unknown }).cause;
|
||||
return `${error.message} ${cause ? extractMessage(cause) : ''}`;
|
||||
}
|
||||
if (typeof error === 'object') {
|
||||
const e = error as Record<string, unknown>;
|
||||
const parts: string[] = [];
|
||||
for (const k of ['message', 'error', 'body', 'responseBody', 'data']) {
|
||||
const v = e[k];
|
||||
if (typeof v === 'string') parts.push(v);
|
||||
else if (v && typeof v === 'object') parts.push(extractMessage(v));
|
||||
}
|
||||
return parts.join(' ');
|
||||
}
|
||||
return String(error);
|
||||
}
|
||||
|
||||
/** Rough token size of a ModelMessage array via the shared chars estimator. */
|
||||
export function estimateMessagesTokens(
|
||||
messages: ReadonlyArray<ModelMessage>,
|
||||
): number {
|
||||
let total = 0;
|
||||
for (const m of messages) {
|
||||
total += estimateTokens(serializeContent(m.content));
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function serializeContent(content: unknown): string {
|
||||
if (typeof content === 'string') return content;
|
||||
try {
|
||||
return JSON.stringify(content) ?? '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
/** Deep JSON string of an arbitrary value, bounded so estimation never throws. */
|
||||
function stringifyValue(value: unknown): string {
|
||||
if (typeof value === 'string') return value;
|
||||
try {
|
||||
return JSON.stringify(value) ?? String(value);
|
||||
} catch {
|
||||
return String(value);
|
||||
}
|
||||
}
|
||||
|
||||
export interface TrimResult {
|
||||
messages: ModelMessage[];
|
||||
/** Whether any trimming was applied. */
|
||||
trimmed: boolean;
|
||||
/** Estimated tokens of the returned messages (chars-based). */
|
||||
estimatedTokens: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bound the replayed history to `budgetTokens`, deterministically. Returns the
|
||||
* SAME array reference (no copy) when nothing needs trimming, so the common case
|
||||
* is free and byte-identical. Trimming order (spec #490):
|
||||
* 1. truncate OLD turns' tool outputs (head+tail + marker) — the bulk of the size
|
||||
* 2. mechanically collapse the OLDEST turns to their text (concatenation, no LLM)
|
||||
* 3. the current + last {@link REPLAY_KEEP_RECENT_TURNS} turns stay FULL
|
||||
*
|
||||
* `budgetTokens === null` disables trimming. `priorContextTokens` (the provider's
|
||||
* fact from last turn) short-circuits the decision: when it is known and already
|
||||
* under budget we skip trimming even if the char-estimate is higher (the provider
|
||||
* count is authoritative). The char-estimate drives WHAT to cut.
|
||||
*/
|
||||
export function trimHistoryForReplay(
|
||||
messages: ModelMessage[],
|
||||
budgetTokens: number | null,
|
||||
priorContextTokens?: number,
|
||||
): TrimResult {
|
||||
if (budgetTokens == null) {
|
||||
return { messages, trimmed: false, estimatedTokens: 0 };
|
||||
}
|
||||
const estimated = estimateMessagesTokens(messages);
|
||||
// Decision signal: prefer the provider's fact (last turn's contextTokens) plus
|
||||
// the estimated delta of the messages appended since; fall back to the pure
|
||||
// char-estimate for a chat with no usage yet.
|
||||
const projected =
|
||||
priorContextTokens != null
|
||||
? Math.max(priorContextTokens, estimated)
|
||||
: estimated;
|
||||
if (projected <= budgetTokens) {
|
||||
return { messages, trimmed: false, estimatedTokens: estimated };
|
||||
}
|
||||
|
||||
// The tail we always keep full: from the Nth-from-last user message onward.
|
||||
const boundary = recentBoundaryIndex(messages, REPLAY_KEEP_RECENT_TURNS);
|
||||
const tail = messages.slice(boundary);
|
||||
let head = messages.slice(0, boundary).map(cloneMessage);
|
||||
|
||||
// Phase 1: truncate old tool outputs.
|
||||
for (const m of head) {
|
||||
if (m.role === 'tool') truncateToolMessage(m);
|
||||
}
|
||||
let out = [...head, ...tail];
|
||||
let est = estimateMessagesTokens(out);
|
||||
if (est <= budgetTokens) {
|
||||
return { messages: out, trimmed: true, estimatedTokens: est };
|
||||
}
|
||||
|
||||
// Phase 2: collapse the oldest turns (in `head`) to their text, one at a time,
|
||||
// from the oldest, until we fit or the whole head is collapsed.
|
||||
const turns = splitTurns(head);
|
||||
const collapsed: ModelMessage[] = [];
|
||||
let i = 0;
|
||||
for (; i < turns.length; i++) {
|
||||
if (est <= budgetTokens) break;
|
||||
collapsed.push(...collapseTurn(turns[i]));
|
||||
// Re-estimate the whole prospective output.
|
||||
const remaining = turns.slice(i + 1).flat();
|
||||
out = [...collapsed, ...remaining, ...tail];
|
||||
est = estimateMessagesTokens(out);
|
||||
}
|
||||
// Include any turns we didn't need to collapse.
|
||||
const remaining = turns.slice(i).flat();
|
||||
out = [...collapsed, ...remaining, ...tail];
|
||||
est = estimateMessagesTokens(out);
|
||||
return { messages: out, trimmed: true, estimatedTokens: est };
|
||||
}
|
||||
|
||||
/** Index of the first message of the Nth-from-last user turn (0 if fewer). */
|
||||
function recentBoundaryIndex(
|
||||
messages: ReadonlyArray<ModelMessage>,
|
||||
keepTurns: number,
|
||||
): number {
|
||||
const userIdx: number[] = [];
|
||||
for (let i = 0; i < messages.length; i++) {
|
||||
if (messages[i].role === 'user') userIdx.push(i);
|
||||
}
|
||||
if (userIdx.length <= keepTurns) return 0;
|
||||
return userIdx[userIdx.length - keepTurns];
|
||||
}
|
||||
|
||||
/** Split a message list into turns; each turn starts at a `user` message. */
|
||||
function splitTurns(messages: ModelMessage[]): ModelMessage[][] {
|
||||
const turns: ModelMessage[][] = [];
|
||||
for (const m of messages) {
|
||||
if (m.role === 'user' || turns.length === 0) turns.push([m]);
|
||||
else turns[turns.length - 1].push(m);
|
||||
}
|
||||
return turns;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse a whole turn to its plain text (mechanical concatenation, not an LLM
|
||||
* summary). Keeps the user message; replaces the assistant/tool messages with a
|
||||
* single assistant text message = the assistant's concatenated text + a marker
|
||||
* when tool activity was dropped. Dropping BOTH the tool-call and tool-result
|
||||
* parts together keeps the rebuilt history balanced (no unpaired calls).
|
||||
*/
|
||||
function collapseTurn(turn: ModelMessage[]): ModelMessage[] {
|
||||
const out: ModelMessage[] = [];
|
||||
let assistantText = '';
|
||||
let hadTools = false;
|
||||
for (const m of turn) {
|
||||
if (m.role === 'user') {
|
||||
out.push(m);
|
||||
} else if (m.role === 'assistant') {
|
||||
const { text, tools } = extractAssistantText(m.content);
|
||||
assistantText += text;
|
||||
hadTools = hadTools || tools;
|
||||
} else if (m.role === 'tool') {
|
||||
hadTools = true;
|
||||
} else {
|
||||
out.push(m);
|
||||
}
|
||||
}
|
||||
const note =
|
||||
(assistantText ? assistantText.trimEnd() : '') +
|
||||
(hadTools
|
||||
? `${assistantText ? '\n\n' : ''}${REPLAY_TURN_COLLAPSED_MARKER}`
|
||||
: '');
|
||||
if (note) out.push({ role: 'assistant', content: note } as ModelMessage);
|
||||
return out;
|
||||
}
|
||||
|
||||
function extractAssistantText(content: unknown): {
|
||||
text: string;
|
||||
tools: boolean;
|
||||
} {
|
||||
if (typeof content === 'string') return { text: content, tools: false };
|
||||
if (!Array.isArray(content)) return { text: '', tools: false };
|
||||
let text = '';
|
||||
let tools = false;
|
||||
for (const part of content) {
|
||||
const type = (part as { type?: string })?.type;
|
||||
if (type === 'text') text += (part as { text?: string }).text ?? '';
|
||||
else if (type === 'tool-call') tools = true;
|
||||
}
|
||||
return { text, tools };
|
||||
}
|
||||
|
||||
/** Truncate every tool-result output in a `tool` message to head+tail+marker. */
|
||||
function truncateToolMessage(message: ModelMessage): void {
|
||||
const content = message.content;
|
||||
if (!Array.isArray(content)) return;
|
||||
for (const part of content) {
|
||||
const p = part as { type?: string; output?: { type?: string; value?: unknown } };
|
||||
if (p.type !== 'tool-result' && p.type !== 'tool-error') continue;
|
||||
if (!p.output) continue;
|
||||
const raw = stringifyValue(p.output.value);
|
||||
const budget = REPLAY_TOOL_OUTPUT_HEAD + REPLAY_TOOL_OUTPUT_TAIL;
|
||||
if (raw.length <= budget + REPLAY_TRUNCATION_MARKER.length) continue;
|
||||
const truncated =
|
||||
raw.slice(0, REPLAY_TOOL_OUTPUT_HEAD) +
|
||||
`\n${REPLAY_TRUNCATION_MARKER}\n` +
|
||||
raw.slice(raw.length - REPLAY_TOOL_OUTPUT_TAIL);
|
||||
// Represent the shrunk output as a text output (a valid tool-result output).
|
||||
p.output = { type: 'text', value: truncated };
|
||||
}
|
||||
}
|
||||
|
||||
/** Shallow-ish clone so trimming never mutates the caller's (persisted-derived)
|
||||
* message objects — only the OLD region is cloned before it is edited. */
|
||||
function cloneMessage(m: ModelMessage): ModelMessage {
|
||||
if (typeof m.content === 'string') return { ...m };
|
||||
return {
|
||||
...m,
|
||||
content: (m.content as unknown[]).map((p) =>
|
||||
p && typeof p === 'object' ? { ...(p as object) } : p,
|
||||
),
|
||||
} as ModelMessage;
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
import {
|
||||
wrapInAppToolWithCap,
|
||||
inAppToolCallCapMs,
|
||||
type ToolAbortSignalSink,
|
||||
} from './ai-chat-tools.service';
|
||||
import type { Tool, ToolCallOptions } from 'ai';
|
||||
|
||||
/**
|
||||
* #487 commit 1 — in-app tool race-on-abort + safe-points + per-call cap.
|
||||
*
|
||||
* Tests assert the HONEST observable property the spec names — "after Stop, NO
|
||||
* new HTTP/WS call STARTS; an already-started single call may take either
|
||||
* outcome" — against the REAL wrapper mechanism (the composite abort signal it
|
||||
* publishes on the client + the RACE it runs), NOT a timing-dependent proxy like
|
||||
* "the write didn't land".
|
||||
*/
|
||||
|
||||
// A minimal stand-in for the client's `toolAbortSignal` field. In production the
|
||||
// wrapper publishes the composite here and the client's paginateAll /
|
||||
// mutatePageContent safe-points read it; the fake "tool" below reads it the same
|
||||
// way, so this exercises the real contract without a live DB / collab socket.
|
||||
class FakeClient implements ToolAbortSignalSink {
|
||||
private signal: AbortSignal | null = null;
|
||||
setToolAbortSignal(signal: AbortSignal | null): void {
|
||||
this.signal = signal;
|
||||
}
|
||||
getToolAbortSignal(): AbortSignal | null {
|
||||
return this.signal;
|
||||
}
|
||||
}
|
||||
|
||||
// A ToolCallOptions with just the field the wrapper reads (abortSignal). The AI
|
||||
// SDK passes a fuller object; the wrapper only spreads it and reads abortSignal.
|
||||
const opts = (abortSignal?: AbortSignal): ToolCallOptions =>
|
||||
({ toolCallId: 't1', messages: [], abortSignal }) as unknown as ToolCallOptions;
|
||||
|
||||
const tick = (ms = 5) => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
describe('#487 wrapInAppToolWithCap — race-on-abort + safe-points', () => {
|
||||
it('after Stop, no NEW simulated call starts (multi-call tool)', async () => {
|
||||
const client = new FakeClient();
|
||||
const started: number[] = [];
|
||||
// A multi-call tool that mirrors paginateAll: it consults the client signal
|
||||
// at a safe-point BEFORE starting each simulated network call.
|
||||
const multiCall: Tool = {
|
||||
execute: (async (_args: unknown) => {
|
||||
for (let i = 0; i < 6; i++) {
|
||||
// Safe-point: exactly what paginateAll / mutatePageContent do.
|
||||
client.getToolAbortSignal()?.throwIfAborted();
|
||||
started.push(i);
|
||||
await tick(10);
|
||||
}
|
||||
return 'done';
|
||||
}) as unknown as Tool['execute'],
|
||||
} as Tool;
|
||||
|
||||
const wrapped = wrapInAppToolWithCap(multiCall, client, 10_000);
|
||||
const ac = new AbortController();
|
||||
const call = (
|
||||
wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise<unknown>
|
||||
)({}, opts(ac.signal));
|
||||
|
||||
// Let one or two calls start, then Stop.
|
||||
await tick(12);
|
||||
ac.abort(new Error('user stop'));
|
||||
|
||||
await expect(call).rejects.toThrow(); // wrapper rejects promptly
|
||||
const startedAtStop = started.length;
|
||||
|
||||
// Give the abandoned loser ample time; its next safe-point must throw because
|
||||
// the (aborted) composite is still published on the client.
|
||||
await tick(60);
|
||||
expect(started.length).toBe(startedAtStop);
|
||||
// It must NOT have run the whole sequence (that would mean Stop did nothing).
|
||||
expect(started.length).toBeLessThan(6);
|
||||
});
|
||||
|
||||
it('rejects immediately on Stop even if the call never settles (discard loser)', async () => {
|
||||
const client = new FakeClient();
|
||||
let settled = false;
|
||||
const hang: Tool = {
|
||||
execute: (async () => {
|
||||
await new Promise(() => undefined); // never resolves
|
||||
settled = true;
|
||||
}) as unknown as Tool['execute'],
|
||||
} as Tool;
|
||||
const wrapped = wrapInAppToolWithCap(hang, client, 10_000);
|
||||
const ac = new AbortController();
|
||||
const call = (
|
||||
wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise<unknown>
|
||||
)({}, opts(ac.signal));
|
||||
await tick(5);
|
||||
ac.abort();
|
||||
await expect(call).rejects.toThrow();
|
||||
expect(settled).toBe(false);
|
||||
});
|
||||
|
||||
it('per-call cap rejects a hung call with no Stop signal', async () => {
|
||||
const client = new FakeClient();
|
||||
const hang: Tool = {
|
||||
execute: (async () => {
|
||||
await new Promise(() => undefined);
|
||||
}) as unknown as Tool['execute'],
|
||||
} as Tool;
|
||||
// Tiny cap; no options.abortSignal at all (composite = cap only).
|
||||
const wrapped = wrapInAppToolWithCap(hang, client, 20);
|
||||
const start = Date.now();
|
||||
await expect(
|
||||
(wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise<unknown>)(
|
||||
{},
|
||||
opts(undefined),
|
||||
),
|
||||
).rejects.toThrow(/per-call cap/);
|
||||
expect(Date.now() - start).toBeLessThan(2000);
|
||||
});
|
||||
|
||||
it('publishes a composite signal on the client for the duration of the call', async () => {
|
||||
const client = new FakeClient();
|
||||
let seenDuringCall: AbortSignal | null = null;
|
||||
const probe: Tool = {
|
||||
execute: (async () => {
|
||||
seenDuringCall = client.getToolAbortSignal();
|
||||
return 'ok';
|
||||
}) as unknown as Tool['execute'],
|
||||
} as Tool;
|
||||
const wrapped = wrapInAppToolWithCap(probe, client, 10_000);
|
||||
const ac = new AbortController();
|
||||
await (
|
||||
wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise<unknown>
|
||||
)({}, opts(ac.signal));
|
||||
expect(seenDuringCall).not.toBeNull();
|
||||
// The published composite must reflect the turn's Stop signal.
|
||||
ac.abort();
|
||||
expect((seenDuringCall as unknown as AbortSignal).aborted).toBe(true);
|
||||
});
|
||||
|
||||
it('a completed call returns its raw result unchanged', async () => {
|
||||
const client = new FakeClient();
|
||||
const ok: Tool = {
|
||||
execute: (async () => ({ items: [1, 2, 3] })) as unknown as Tool['execute'],
|
||||
} as Tool;
|
||||
const wrapped = wrapInAppToolWithCap(ok, client, 10_000);
|
||||
const res = await (
|
||||
wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise<unknown>
|
||||
)({}, opts(new AbortController().signal));
|
||||
expect(res).toEqual({ items: [1, 2, 3] });
|
||||
});
|
||||
|
||||
it('cap is env-tunable with a 2-minute default', () => {
|
||||
const prev = process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS;
|
||||
delete process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS;
|
||||
expect(inAppToolCallCapMs()).toBe(120_000);
|
||||
process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS = '5000';
|
||||
expect(inAppToolCallCapMs()).toBe(5000);
|
||||
process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS = 'not-a-number';
|
||||
expect(inAppToolCallCapMs()).toBe(120_000);
|
||||
if (prev === undefined) delete process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS;
|
||||
else process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS = prev;
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { tool, type Tool } from 'ai';
|
||||
import { tool, type Tool, type ToolCallOptions } from 'ai';
|
||||
import { z } from 'zod';
|
||||
import { User } from '@docmost/db/types/entity.types';
|
||||
import { TokenService } from '../../auth/services/token.service';
|
||||
@@ -159,6 +159,129 @@ function __assertClientCallContract(client: DocmostClientLike): void {
|
||||
* existing service-account `/mcp` path already calls loopback successfully, so
|
||||
* this works for single-workspace self-host.
|
||||
*/
|
||||
/**
|
||||
* #487: wall-clock cap for a SINGLE in-app tool call, env-tunable via
|
||||
* `AI_CHAT_INAPP_TOOL_CALL_CAP_MS`. Bounds a read tool that would otherwise
|
||||
* paginate for minutes and a content write whose collab commit hangs, and is the
|
||||
* per-call CAP half of the composite abort signal every in-app tool is wrapped
|
||||
* with (the other half is the turn's Stop signal). Default 2 minutes: generous
|
||||
* for a legitimate long read/write, tight enough that a stuck call cannot pin the
|
||||
* turn. The reconcile staleness floor (#487 commit 4) is derived as
|
||||
* max(2 x this cap, 15 min), so keep this well under that.
|
||||
*/
|
||||
export function inAppToolCallCapMs(): number {
|
||||
const raw = Number(process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS);
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 120_000;
|
||||
}
|
||||
|
||||
/** #487: the composite signal's reason as an Error (informative thrown value). */
|
||||
function inAppAbortReason(signal: AbortSignal): Error {
|
||||
const r = signal.reason;
|
||||
return r instanceof Error
|
||||
? r
|
||||
: new Error(typeof r === 'string' ? r : 'In-app tool call aborted');
|
||||
}
|
||||
|
||||
/**
|
||||
* The client surface {@link wrapInAppToolWithCap} drives (#487). Both methods are
|
||||
* OPTIONAL: the real loopback DocmostClient implements them (so a Stop/cap reaches
|
||||
* its pagination / pre-commit safe-points), but a client that omits them still
|
||||
* gets the OUTER guarantee — the race rejects on abort regardless. This keeps the
|
||||
* wrapper decoupled from the exact client shape (unit-test doubles need not stub
|
||||
* the plumbing).
|
||||
*/
|
||||
export interface ToolAbortSignalSink {
|
||||
setToolAbortSignal?(signal: AbortSignal | null): void;
|
||||
getToolAbortSignal?(): AbortSignal | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: wrap an in-app tool so a Stop (the turn's `options.abortSignal`) OR the
|
||||
* per-call wall-clock cap REJECTS the call immediately, and so that SAME
|
||||
* composite signal reaches the client's pagination / pre-commit safe-points (via
|
||||
* `client.setToolAbortSignal`) — making a Stop stop the NEXT HTTP/WS call from
|
||||
* starting.
|
||||
*
|
||||
* Reuses the RACE pattern of `wrapToolWithCallTimeout` (mcp-clients.service.ts):
|
||||
* the call is raced against the composite signal, so on abort we reject in the
|
||||
* SAME tick and DISCARD the loser promise. Its network / collab teardown latency
|
||||
* therefore never blocks the turn — the supersede timeout W=10s (#487 commit 3)
|
||||
* relies on this abort->settle latency being milliseconds, not a socket teardown.
|
||||
* Awaiting the client's own signal-into-write path alone would NOT satisfy this
|
||||
* (the loser could still be tearing down a collab socket).
|
||||
*
|
||||
* The composite is SET on the client at entry and deliberately NOT restored on
|
||||
* unwind: after this wrapper rejects on abort, the ABANDONED loser promise keeps
|
||||
* running, and its safe-points read the client field — leaving the (aborted)
|
||||
* composite there is exactly what makes the loser's NEXT call throw and stop. The
|
||||
* next in-app tool call overwrites the field with its own fresh composite before
|
||||
* any of its safe-points run, so a stale settled signal never leaks forward.
|
||||
* SINGLE-WRITER by phase-1 assumption — see DocmostClientContext.toolAbortSignal
|
||||
* for the parallel-call caveat (#487).
|
||||
*
|
||||
* KNOWN LIMITATION (#487): a write tool that issues SEVERAL sequential collab
|
||||
* commits can be aborted BETWEEN commits, leaving a partially-applied operation.
|
||||
* Cancel guarantees "no NEW call starts", NOT "the write didn't land".
|
||||
*/
|
||||
export function wrapInAppToolWithCap(
|
||||
toolDef: Tool,
|
||||
client: ToolAbortSignalSink,
|
||||
capMs: number,
|
||||
): Tool {
|
||||
const original = toolDef.execute;
|
||||
if (typeof original !== 'function') return toolDef;
|
||||
const execute = async (args: unknown, options: ToolCallOptions) => {
|
||||
const capController = new AbortController();
|
||||
const timer = setTimeout(() => {
|
||||
capController.abort(
|
||||
new Error(`In-app tool call exceeded the ${capMs}ms per-call cap`),
|
||||
);
|
||||
}, capMs);
|
||||
timer.unref?.();
|
||||
const composite = options?.abortSignal
|
||||
? AbortSignal.any([options.abortSignal, capController.signal])
|
||||
: capController.signal;
|
||||
// Reject the MOMENT the composite fires, independent of whether `original`
|
||||
// ever settles (a hung collab write / read would otherwise pin the turn). The
|
||||
// losing `original` is left pending; Promise.race attaches a rejection
|
||||
// handler to both inputs so a late rejection is never unhandled.
|
||||
const aborted = new Promise<never>((_, reject) => {
|
||||
const fail = () => reject(inAppAbortReason(composite));
|
||||
if (composite.aborted) fail();
|
||||
else composite.addEventListener('abort', fail, { once: true });
|
||||
});
|
||||
// Publish the composite so the client's pagination / pre-commit safe-points
|
||||
// observe it (see the "not restored on unwind" rationale above). Guarded: a
|
||||
// client without the plumbing still gets the OUTER race guarantee below.
|
||||
client.setToolAbortSignal?.(composite);
|
||||
try {
|
||||
return await Promise.race([
|
||||
(original as (a: unknown, o: ToolCallOptions) => Promise<unknown>)(
|
||||
args,
|
||||
{ ...options, abortSignal: composite },
|
||||
),
|
||||
aborted,
|
||||
]);
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
};
|
||||
return { ...toolDef, execute } as unknown as Tool;
|
||||
}
|
||||
|
||||
/** #487: apply {@link wrapInAppToolWithCap} to every tool in a set. */
|
||||
export function wrapInAppToolsWithCap(
|
||||
tools: Record<string, Tool>,
|
||||
client: ToolAbortSignalSink,
|
||||
capMs: number,
|
||||
): Record<string, Tool> {
|
||||
const out: Record<string, Tool> = {};
|
||||
for (const [name, t] of Object.entries(tools)) {
|
||||
out[name] = wrapInAppToolWithCap(t, client, capMs);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class AiChatToolsService {
|
||||
private readonly logger = new Logger(AiChatToolsService.name);
|
||||
@@ -186,7 +309,12 @@ export class AiChatToolsService {
|
||||
sessionId: string,
|
||||
workspaceId: string,
|
||||
aiChatId: string,
|
||||
): Promise<DocmostClientLike> {
|
||||
// #487: the returned client also carries the tool-cancellation plumbing
|
||||
// (setToolAbortSignal/getToolAbortSignal). These are host plumbing, NOT part
|
||||
// of the tool-execute surface (DocmostClientMethod), so they are surfaced here
|
||||
// as an intersection rather than by widening that Pick — keeping the
|
||||
// positional-call drift-guard (#446) scoped to the actual tool methods.
|
||||
): Promise<DocmostClientLike & ToolAbortSignalSink> {
|
||||
const apiUrl =
|
||||
process.env.MCP_DOCMOST_API_URL ||
|
||||
`http://127.0.0.1:${process.env.PORT || 3000}/api`;
|
||||
@@ -630,7 +758,15 @@ export class AiChatToolsService {
|
||||
// dependency and reuses the CASL enforcement already on `client`. When the
|
||||
// loaded package predates #417 (factory undefined) or the loader is mocked in
|
||||
// a unit test, signalling is a pure no-op and results are byte-identical.
|
||||
if (!createCommentSignalTracker) return tools;
|
||||
// #487: wrap every in-app tool with the race-on-abort + per-call cap guard so
|
||||
// a Stop / cap rejects immediately AND reaches the client's write/pagination
|
||||
// safe-points. Applied as the OUTERMOST wrapper (over the comment-signal
|
||||
// wrapper below) so the race governs the whole call. The client carries the
|
||||
// per-call composite signal via setToolAbortSignal.
|
||||
const capMs = inAppToolCallCapMs();
|
||||
if (!createCommentSignalTracker) {
|
||||
return wrapInAppToolsWithCap(tools, client, capMs);
|
||||
}
|
||||
|
||||
const tracker = createCommentSignalTracker({
|
||||
probe: async (pageId: string, sinceMs: number) => {
|
||||
@@ -659,7 +795,11 @@ export class AiChatToolsService {
|
||||
},
|
||||
});
|
||||
|
||||
return wrapToolsWithCommentSignal(tools, tracker);
|
||||
return wrapInAppToolsWithCap(
|
||||
wrapToolsWithCommentSignal(tools, tracker),
|
||||
client,
|
||||
capMs,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { type Kysely, sql } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
// Chat-level metadata bag (#490). First use: the deferred-tool ACTIVATION set
|
||||
// (`activatedTools`) is persisted here so it survives across turns — previously
|
||||
// the set was reset every turn, forcing the model to re-run loadTools and pay a
|
||||
// fresh round-trip to re-activate the same tools each turn. On load the stored
|
||||
// set is intersected with the current valid deferred names, so an allowlist /
|
||||
// role change can never inject a now-nonexistent tool.
|
||||
//
|
||||
// jsonb, defaulted to '{}' so every row (incl. pre-migration ones, backfilled
|
||||
// by the default) is a readable object — the app never has to null-guard the
|
||||
// bag itself, only individual keys.
|
||||
await db.schema
|
||||
.alterTable('ai_chats')
|
||||
.addColumn('metadata', 'jsonb', (col) =>
|
||||
col.notNull().defaultTo(sql`'{}'::jsonb`),
|
||||
)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.alterTable('ai_chats').dropColumn('metadata').execute();
|
||||
}
|
||||
@@ -0,0 +1,280 @@
|
||||
import { randomUUID } from 'crypto';
|
||||
import { CamelCasePlugin, Kysely, sql } from 'kysely';
|
||||
import { PostgresJSDialect } from 'kysely-postgres-js';
|
||||
// NOT a default import: the project tsconfig is `module: commonjs` with NO
|
||||
// esModuleInterop, so `import postgres from 'postgres'` compiles to
|
||||
// `postgres_1.default(...)` and the CJS `postgres` export has no `.default` —
|
||||
// it threw in beforeAll, was swallowed as "DB unreachable", and SILENTLY voided
|
||||
// all six tests. Mirror the working integration harness (test/integration/db.ts).
|
||||
import * as postgres from 'postgres';
|
||||
import { AiChatMessageRepo } from './ai-chat-message.repo';
|
||||
import { AiChatRunRepo } from './ai-chat-run.repo';
|
||||
|
||||
/**
|
||||
* #491 delta-poll — OBSERVABLE-PROPERTY tests against a LIVE Postgres (the local
|
||||
* gitmost test DB, docker `gitmost-test-pg` on :5432), not "rows through a mock"
|
||||
* (a mock cannot observe the DB clock nor the overlap-window race — the very
|
||||
* things that matter here). Drives the REAL repo methods (`findByChatUpdatedAfter`,
|
||||
* the now()-stamped `update`) and asserts:
|
||||
* 1. delta-relevant writes stamp `updatedAt` from the DB clock, not the app clock
|
||||
* (proven by faking the process clock far into the future and observing the
|
||||
* stamp stays on real DB time);
|
||||
* 2. the poll returns only rows changed after the cursor, ordered, with a fresh
|
||||
* DB-clock cursor;
|
||||
* 3. the "committed late but stamped earlier than the cursor" RACE is caught by
|
||||
* the overlap window (a naive `updatedAt > cursor` would MISS it);
|
||||
* 4. the overlap GUARANTEES repeats across close polls — the contract behind the
|
||||
* client's idempotent merge (mergeById).
|
||||
*
|
||||
* INTEGRATION lane (`*.int-spec.ts`): runs under `test:int`, whose global-setup
|
||||
* DROPS + RE-CREATES + MIGRATES `docmost_test`, so the real `ai_chat_messages` /
|
||||
* `ai_chat_runs` tables EXIST here. (It was previously a `.spec.ts` defaulting to
|
||||
* the UNmigrated dev `docmost`; in the CI unit lane — where `WAL_TEST_DATABASE_URL`
|
||||
* is unset and only `test:int` migrates — that meant 5/6 ERROR
|
||||
* `relation "ai_chat_messages" does not exist`, silently voiding coverage of the
|
||||
* risky cursor/overlap logic. Renaming to `.int-spec.ts` + defaulting the DSN to
|
||||
* `docmost_test` fixes the CI fidelity.)
|
||||
*
|
||||
* FK triggers are bypassed (`session_replication_role = replica`) so synthetic
|
||||
* chat/workspace ids need no parent fixtures; a single pooled connection (max 1)
|
||||
* keeps that session setting for every query. SKIPS cleanly when the DB is
|
||||
* unreachable so a DB-less CI never breaks.
|
||||
*/
|
||||
const CONN =
|
||||
process.env.WAL_TEST_DATABASE_URL ??
|
||||
process.env.TEST_DATABASE_URL ??
|
||||
'postgresql://docmost:docmost_dev_pw@localhost:5432/docmost_test';
|
||||
|
||||
let db: Kysely<any>;
|
||||
let sqlClient: ReturnType<typeof postgres>;
|
||||
let msgRepo: AiChatMessageRepo;
|
||||
let runRepo: AiChatRunRepo;
|
||||
let reachable = false;
|
||||
|
||||
const workspaceId = randomUUID();
|
||||
const chatId = randomUUID();
|
||||
|
||||
beforeAll(async () => {
|
||||
try {
|
||||
sqlClient = postgres(CONN, { max: 1, onnotice: () => {} });
|
||||
db = new Kysely<any>({
|
||||
dialect: new PostgresJSDialect({ postgres: sqlClient }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
// Single connection keeps this session-scoped bypass for the whole suite.
|
||||
await sql`set session_replication_role = replica`.execute(db);
|
||||
await sql`select 1`.execute(db);
|
||||
reachable = true;
|
||||
} catch (err) {
|
||||
reachable = false;
|
||||
// A genuine connection failure (ECONNREFUSED etc.) is a legitimate skip on a
|
||||
// DB-less CI. A PROGRAMMING error (bad import, typo, driver misuse) must NOT
|
||||
// masquerade as "DB unreachable" and silently void the whole suite (that is
|
||||
// exactly the bug that hid this spec's zero coverage) — rethrow it so the
|
||||
// suite fails LOUDLY.
|
||||
const msg = String((err as Error)?.message ?? err);
|
||||
if (
|
||||
!/ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EHOSTUNREACH|connect|terminating|password|authentication|role .* does not exist|database .* does not exist/i.test(
|
||||
msg,
|
||||
)
|
||||
) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
msgRepo = new AiChatMessageRepo(db as never);
|
||||
runRepo = new AiChatRunRepo(db as never);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (db) {
|
||||
try {
|
||||
await db
|
||||
.deleteFrom('aiChatMessages')
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.execute();
|
||||
await db
|
||||
.deleteFrom('aiChatRuns')
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.execute();
|
||||
} catch {
|
||||
/* best-effort cleanup */
|
||||
}
|
||||
await db.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
async function seedMessage(overrides: Record<string, unknown> = {}) {
|
||||
return msgRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
userId: null as never,
|
||||
role: 'assistant',
|
||||
content: 'x',
|
||||
status: 'streaming',
|
||||
...overrides,
|
||||
} as never);
|
||||
}
|
||||
|
||||
async function dbNow(): Promise<string> {
|
||||
const r = await sql<{ now: Date }>`select now() as now`.execute(db);
|
||||
return r.rows[0].now.toISOString();
|
||||
}
|
||||
|
||||
// Fake ONLY the Date object (so in-process `new Date()`/`Date.now()` jump), while
|
||||
// leaving every TIMER function real. Faking timers wholesale freezes postgres.js's
|
||||
// internal connection/query timers, so the awaited DB round-trip would hang the
|
||||
// test (and the afterAll cleanup) at the jest 5s cap. With Date-only faking the
|
||||
// query resolves normally, and we still prove the stamp is the DB clock (not the
|
||||
// skewed process clock).
|
||||
function fakeDateOnly(iso: string): void {
|
||||
jest.useFakeTimers({
|
||||
doNotFake: [
|
||||
'hrtime',
|
||||
'nextTick',
|
||||
'performance',
|
||||
'queueMicrotask',
|
||||
'requestAnimationFrame',
|
||||
'cancelAnimationFrame',
|
||||
'requestIdleCallback',
|
||||
'cancelIdleCallback',
|
||||
'setImmediate',
|
||||
'clearImmediate',
|
||||
'setInterval',
|
||||
'clearInterval',
|
||||
'setTimeout',
|
||||
'clearTimeout',
|
||||
],
|
||||
now: new Date(iso),
|
||||
});
|
||||
}
|
||||
|
||||
const maybe = (name: string, fn: () => Promise<void>) =>
|
||||
it(name, async () => {
|
||||
if (!reachable) {
|
||||
console.warn(`SKIP (${name}): test DB unreachable at ${CONN}`);
|
||||
return;
|
||||
}
|
||||
await fn();
|
||||
});
|
||||
|
||||
describe('AiChatMessageRepo.findByChatUpdatedAfter (#491 delta poll)', () => {
|
||||
maybe('null cursor returns no rows and a fresh DB-clock cursor', async () => {
|
||||
const before = await dbNow();
|
||||
const { rows, cursor } = await msgRepo.findByChatUpdatedAfter(
|
||||
chatId,
|
||||
workspaceId,
|
||||
null,
|
||||
);
|
||||
expect(rows).toEqual([]);
|
||||
expect(new Date(cursor).getTime()).toBeGreaterThanOrEqual(
|
||||
new Date(before).getTime(),
|
||||
);
|
||||
});
|
||||
|
||||
maybe('returns only rows changed after the cursor', async () => {
|
||||
const { cursor: c0 } = await msgRepo.findByChatUpdatedAfter(
|
||||
chatId,
|
||||
workspaceId,
|
||||
null,
|
||||
);
|
||||
const m = await seedMessage();
|
||||
const { rows, cursor: c1 } = await msgRepo.findByChatUpdatedAfter(
|
||||
chatId,
|
||||
workspaceId,
|
||||
c0,
|
||||
);
|
||||
expect(rows.map((r) => r.id)).toContain(m.id);
|
||||
// Cursor is monotonic (advances).
|
||||
expect(new Date(c1).getTime()).toBeGreaterThanOrEqual(
|
||||
new Date(c0).getTime(),
|
||||
);
|
||||
});
|
||||
|
||||
maybe(
|
||||
'RACE: a row stamped BEFORE the cursor but seen after is caught by the overlap',
|
||||
async () => {
|
||||
// Cursor taken now; then a row appears whose updatedAt is 2s in the PAST
|
||||
// (committed late on another connection but stamped earlier). A naive
|
||||
// `updatedAt > cursor` would MISS it; the 5s overlap window catches it.
|
||||
const cursor = await dbNow();
|
||||
const m = await seedMessage();
|
||||
await sql`update ai_chat_messages set updated_at = now() - interval '2 seconds' where id = ${m.id}`.execute(
|
||||
db,
|
||||
);
|
||||
const { rows } = await msgRepo.findByChatUpdatedAfter(
|
||||
chatId,
|
||||
workspaceId,
|
||||
cursor,
|
||||
);
|
||||
expect(rows.map((r) => r.id)).toContain(m.id);
|
||||
},
|
||||
);
|
||||
|
||||
maybe(
|
||||
'overlap GUARANTEES repeats across close polls (idempotent-merge contract)',
|
||||
async () => {
|
||||
const { cursor: c0 } = await msgRepo.findByChatUpdatedAfter(
|
||||
chatId,
|
||||
workspaceId,
|
||||
null,
|
||||
);
|
||||
const m = await seedMessage();
|
||||
const first = await msgRepo.findByChatUpdatedAfter(
|
||||
chatId,
|
||||
workspaceId,
|
||||
c0,
|
||||
);
|
||||
expect(first.rows.map((r) => r.id)).toContain(m.id);
|
||||
// Immediately re-poll with the JUST-returned cursor: the row is still within
|
||||
// the overlap window, so it is returned AGAIN — the client MUST dedupe by id.
|
||||
const second = await msgRepo.findByChatUpdatedAfter(
|
||||
chatId,
|
||||
workspaceId,
|
||||
first.cursor,
|
||||
);
|
||||
expect(second.rows.map((r) => r.id)).toContain(m.id);
|
||||
},
|
||||
);
|
||||
|
||||
maybe(
|
||||
'update() stamps updatedAt from the DB clock, not the app clock',
|
||||
async () => {
|
||||
const m = await seedMessage();
|
||||
// Skew the PROCESS clock ~73 years into the future (Date only). If the stamp
|
||||
// came from `new Date()` the row would read year 2099; sql now() keeps it on
|
||||
// DB time.
|
||||
fakeDateOnly('2099-01-01T00:00:00Z');
|
||||
const updated = await msgRepo.update(m.id, workspaceId, {
|
||||
content: 'y',
|
||||
});
|
||||
jest.useRealTimers();
|
||||
expect(updated).toBeDefined();
|
||||
expect(new Date(updated!.updatedAt).getFullYear()).toBeLessThan(2099);
|
||||
},
|
||||
);
|
||||
|
||||
maybe(
|
||||
'run update() also stamps updatedAt from the DB clock',
|
||||
async () => {
|
||||
const run = await runRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
createdBy: null as never,
|
||||
trigger: 'user',
|
||||
status: 'running',
|
||||
stepCount: 0,
|
||||
} as never);
|
||||
fakeDateOnly('2099-01-01T00:00:00Z');
|
||||
const updated = await runRepo.update(run.id, workspaceId, {
|
||||
stepCount: 1,
|
||||
});
|
||||
jest.useRealTimers();
|
||||
expect(updated).toBeDefined();
|
||||
expect(new Date(updated!.updatedAt).getFullYear()).toBeLessThan(2099);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { sql } from 'kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '../../types/kysely.types';
|
||||
import { dbOrTx } from '../../utils';
|
||||
import {
|
||||
@@ -24,6 +25,20 @@ const SWEEP_STREAMING_STALE_MS = 10 * 60 * 1000; // 10 minutes
|
||||
// into memory; far above any realistic transcript length.
|
||||
const FIND_ALL_BY_CHAT_LIMIT = 5000;
|
||||
|
||||
// Delta-poll overlap (#491): the poll query reaches this far BEHIND the client's
|
||||
// echoed cursor, so a row that committed with an `updatedAt` marginally before the
|
||||
// previous cursor was taken (on another autocommit connection) is still caught.
|
||||
// Sized well above realistic single-row commit skew; the client merge is
|
||||
// idempotent by id (mergeById), so the guaranteed repeats the overlap produces are
|
||||
// harmless.
|
||||
export const DELTA_POLL_OVERLAP_SECONDS = 5;
|
||||
|
||||
// Hard cap on rows one delta poll returns — a safety bound (a poll should carry a
|
||||
// handful of just-changed rows, never a whole transcript). Ordered by (updatedAt,
|
||||
// id) asc, so on the pathological overflow the OLDEST changes win and the newest
|
||||
// are picked up by the next poll (its cursor did not advance past them).
|
||||
export const DELTA_POLL_MAX_ROWS = 500;
|
||||
|
||||
@Injectable()
|
||||
export class AiChatMessageRepo {
|
||||
private readonly logger = new Logger(AiChatMessageRepo.name);
|
||||
@@ -138,6 +153,72 @@ export class AiChatMessageRepo {
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* Delta read (#491) for the degraded poll: the chat's messages whose row
|
||||
* changed AFTER the client's `cursor`, plus a FRESH cursor taken from the DB
|
||||
* clock. Replaces the old "refetch ALL infinite-query pages every 2.5s with
|
||||
* full parts" poll — the client seeds once (findByChat) and thereafter pulls
|
||||
* only the deltas and merges them by id (mergeById).
|
||||
*
|
||||
* Cursor: a DB-clock timestamp (now()) the client echoes back each poll. All
|
||||
* delta-relevant writes stamp `updatedAt` with now() (see `update` /
|
||||
* `finalizeOwner`), so this is a SINGLE monotonic axis. The query overlaps the
|
||||
* cursor by DELTA_POLL_OVERLAP_SECONDS to catch a row committed with an
|
||||
* `updatedAt` marginally BEFORE the previous cursor was taken on another
|
||||
* connection (single-row autocommit UPDATEs; no long transactions). The overlap
|
||||
* GUARANTEES occasional REPEATS, so the client merge MUST be idempotent by id.
|
||||
*
|
||||
* `cursor === null` (first poll after the full seed) returns NO rows — there is
|
||||
* nothing "new" relative to a just-loaded seed — only the fresh cursor to start
|
||||
* the delta chain. The fresh cursor is read AFTER the rows, so it is >= every
|
||||
* returned row's `updatedAt` (they were read strictly earlier) — a row that
|
||||
* commits between the rows-read and the cursor-read is at most
|
||||
* DELTA_POLL_OVERLAP_SECONDS behind the returned cursor, so the next poll's
|
||||
* overlap window always re-includes it (no miss).
|
||||
*/
|
||||
async findByChatUpdatedAfter(
|
||||
chatId: string,
|
||||
workspaceId: string,
|
||||
cursor: string | null,
|
||||
): Promise<{ rows: AiChatMessage[]; cursor: string }> {
|
||||
if (cursor === null) {
|
||||
const nowRow = await sql<{ now: Date }>`select now() as now`.execute(
|
||||
this.db,
|
||||
);
|
||||
return { rows: [], cursor: nowRow.rows[0].now.toISOString() };
|
||||
}
|
||||
// Overlap the client cursor by DELTA_POLL_OVERLAP_SECONDS, computed in SQL off
|
||||
// the echoed cursor so the whole comparison stays on the DB clock.
|
||||
const rows = await this.db
|
||||
.selectFrom('aiChatMessages')
|
||||
.select(this.baseFields)
|
||||
.where('chatId', '=', chatId)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.where('deletedAt', 'is', null)
|
||||
.where(
|
||||
'updatedAt',
|
||||
'>',
|
||||
sql<Date>`${cursor}::timestamptz - make_interval(secs => ${DELTA_POLL_OVERLAP_SECONDS})`,
|
||||
)
|
||||
.orderBy('updatedAt', 'asc')
|
||||
.orderBy('id', 'asc')
|
||||
.limit(DELTA_POLL_MAX_ROWS)
|
||||
.execute();
|
||||
// When the page filled (pathological overflow), DO NOT advance the cursor to
|
||||
// now(): that would skip the changed rows past the cap that this poll did not
|
||||
// return. Resume from the last returned row's updatedAt instead (the next
|
||||
// poll's overlap re-includes ties by id). In the normal case the fresh DB-clock
|
||||
// now() is the cursor.
|
||||
if (rows.length === DELTA_POLL_MAX_ROWS) {
|
||||
return {
|
||||
rows,
|
||||
cursor: rows[rows.length - 1].updatedAt.toISOString(),
|
||||
};
|
||||
}
|
||||
const nowRow = await sql<{ now: Date }>`select now() as now`.execute(this.db);
|
||||
return { rows, cursor: nowRow.rows[0].now.toISOString() };
|
||||
}
|
||||
|
||||
async insert(
|
||||
insertable: InsertableAiChatMessage,
|
||||
trx?: KyselyTransaction,
|
||||
@@ -171,7 +252,13 @@ export class AiChatMessageRepo {
|
||||
const db = dbOrTx(this.db, opts?.trx);
|
||||
let query = db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({ ...(patch as Record<string, unknown>), updatedAt: new Date() })
|
||||
// #491: stamp `updatedAt` from the DB clock (sql now()), NOT the app clock
|
||||
// (new Date()). The delta-poll cursor (findByChatUpdatedAfter) is a single
|
||||
// DB-clock axis; a per-step 'streaming' UPDATE stamped with the app clock
|
||||
// would be a SECOND, skewed clock source and could leave a row's updatedAt
|
||||
// just under a cursor taken from now() on another connection — an
|
||||
// independent source of delta MISSES. All delta-relevant writes use now().
|
||||
.set({ ...(patch as Record<string, unknown>), updatedAt: sql`now()` })
|
||||
.where('id', '=', id)
|
||||
.where('workspaceId', '=', workspaceId);
|
||||
// Concurrency guard (#183 review): a per-step 'streaming' update must NEVER
|
||||
@@ -188,6 +275,150 @@ export class AiChatMessageRepo {
|
||||
return query.returning(this.baseFields).executeTakeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* #487 OWNER terminal write — the streamText terminal callback's finalize. Like
|
||||
* `update` but CONDITIONAL on `status='streaming' OR metadata.finalizeFailed`:
|
||||
* the owner writes its real content EITHER when the row is still streaming (the
|
||||
* normal case) OR when a reconcile stamp already flipped it to a terminal status
|
||||
* but marked `finalizeFailed:true` — the owner's real content OVERWRITES that
|
||||
* placeholder stamp (owner-write priority, #487). A row that is properly terminal
|
||||
* (no finalizeFailed) is left untouched (undefined) — idempotent. The `patch`
|
||||
* carries the real metadata WITHOUT finalizeFailed, so a successful write CLEARS
|
||||
* the flag. Returns the updated row, or undefined when nothing matched.
|
||||
*/
|
||||
async finalizeOwner(
|
||||
id: string,
|
||||
workspaceId: string,
|
||||
patch: Partial<{
|
||||
content: string | null;
|
||||
toolCalls: unknown;
|
||||
metadata: unknown;
|
||||
status: string | null;
|
||||
}>,
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<AiChatMessage | undefined> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.updateTable('aiChatMessages')
|
||||
// #491: DB-clock stamp (see `update`) — this terminal write flips the row's
|
||||
// status, which the delta poll must observe on the shared now() cursor axis.
|
||||
.set({ ...(patch as Record<string, unknown>), updatedAt: sql`now()` })
|
||||
.where('id', '=', id)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.where((eb) =>
|
||||
eb.or([
|
||||
eb('status', '=', 'streaming'),
|
||||
eb(sql<string>`(metadata->>'finalizeFailed')`, '=', 'true'),
|
||||
]),
|
||||
)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* #487 RECONCILE status-only stamp — settle a stuck 'streaming' row to a
|
||||
* terminal status WITHOUT the owner's real content (which lived only in the
|
||||
* dead process's memory — a documented loss). CONDITIONAL on `status='streaming'`
|
||||
* (never touches an already-terminal row) AND it MERGES `finalizeFailed:true`
|
||||
* into metadata (preserving the partial `parts` already persisted) so a LATER
|
||||
* owner-write (finalizeOwner) can still OVERWRITE this placeholder with real
|
||||
* content, and so `isInterruptResume` can EXCLUDE this row (a reconcile stamp is
|
||||
* not a genuine user interruption). Returns the updated row, or undefined.
|
||||
*/
|
||||
async stampTerminalIfStreaming(
|
||||
id: string,
|
||||
workspaceId: string,
|
||||
status: 'aborted' | 'error' | 'completed',
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<AiChatMessage | undefined> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({
|
||||
status,
|
||||
metadata: sql`coalesce(metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`,
|
||||
// #491: DB-clock stamp (see `update`) so a reconcile status flip lands on
|
||||
// the same now() cursor axis the delta poll reads.
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where('id', '=', id)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.where('status', '=', 'streaming')
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* #487 reconcile clause (b): streaming assistant rows whose linked RUN has
|
||||
* already reached a terminal status — an asymmetry ("run settled / message
|
||||
* streaming forever") the periodic reconcile heals by stamping the message.
|
||||
* Returns the message id + its run's terminal status, bounded.
|
||||
*/
|
||||
async findStreamingWithTerminalRun(
|
||||
limit = 200,
|
||||
// #487: scope to ONE chat for the opportunistic per-turn reconcile (removes
|
||||
// reconcile latency from the user-visible path); omit for the periodic sweep.
|
||||
chat?: { chatId: string; workspaceId: string },
|
||||
): Promise<
|
||||
Array<{ messageId: string; workspaceId: string; runStatus: string }>
|
||||
> {
|
||||
let query = this.db
|
||||
.selectFrom('aiChatMessages as m')
|
||||
.innerJoin('aiChatRuns as r', 'r.assistantMessageId', 'm.id')
|
||||
.select([
|
||||
'm.id as messageId',
|
||||
'm.workspaceId as workspaceId',
|
||||
'r.status as runStatus',
|
||||
])
|
||||
.where('m.status', '=', 'streaming')
|
||||
.where('r.status', 'in', ['succeeded', 'failed', 'aborted']);
|
||||
if (chat) {
|
||||
query = query
|
||||
.where('m.chatId', '=', chat.chatId)
|
||||
.where('m.workspaceId', '=', chat.workspaceId);
|
||||
}
|
||||
return query.limit(limit).execute();
|
||||
}
|
||||
|
||||
/**
|
||||
* #487 reconcile clause (d) — historical-row safety: streaming rows older than
|
||||
* `staleMs` whose chat has NO active run row (double-gated). Settle them to
|
||||
* 'aborted' + finalizeFailed (so a late owner-write could still overwrite).
|
||||
* Returns the count. Used ONLY by the periodic reconcile, never at boot.
|
||||
*/
|
||||
async sweepStreamingWithoutActiveRun(
|
||||
staleMs: number,
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<number> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
const staleBefore = new Date(Date.now() - staleMs);
|
||||
const rows = await db
|
||||
.updateTable('aiChatMessages as m')
|
||||
.set({
|
||||
status: 'aborted',
|
||||
metadata: sql`coalesce(m.metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`,
|
||||
// #491: DB-clock stamp (see `update`). The staleness WHERE below stays on
|
||||
// the app clock — a >minutes window makes the ms-scale skew irrelevant.
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where('m.status', '=', 'streaming')
|
||||
.where('m.updatedAt', '<', staleBefore)
|
||||
.where((eb) =>
|
||||
eb.not(
|
||||
eb.exists(
|
||||
eb
|
||||
.selectFrom('aiChatRuns as r')
|
||||
.select('r.id')
|
||||
.whereRef('r.chatId', '=', 'm.chatId')
|
||||
.where('r.status', 'in', ['pending', 'running']),
|
||||
),
|
||||
),
|
||||
)
|
||||
.returning('m.id')
|
||||
.execute();
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Crash-recovery sweep (#183): flip every assistant row still left in the
|
||||
* 'streaming' state (a turn that died mid-write before reaching a terminal
|
||||
@@ -200,13 +431,21 @@ export class AiChatMessageRepo {
|
||||
* step, so an actively-streaming row never matches; this prevents a fresh
|
||||
* replica's boot-sweep from aborting a turn another replica is still streaming
|
||||
* in a multi-instance deploy.
|
||||
*
|
||||
* #487: the sweep now ALSO marks `finalizeFailed:true` so a late owner-write can
|
||||
* overwrite this placeholder with real content (owner-write priority).
|
||||
*/
|
||||
async sweepStreaming(trx?: KyselyTransaction): Promise<number> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
const staleBefore = new Date(Date.now() - SWEEP_STREAMING_STALE_MS);
|
||||
const rows = await db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({ status: 'aborted', updatedAt: new Date() })
|
||||
.set({
|
||||
status: 'aborted',
|
||||
metadata: sql`coalesce(metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`,
|
||||
// #491: DB-clock stamp (see `update`). Staleness WHERE stays app-clock.
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where('status', '=', 'streaming')
|
||||
.where('updatedAt', '<', staleBefore)
|
||||
.returning('id')
|
||||
|
||||
@@ -62,10 +62,17 @@ describe('AiChatRunRepo.sweepRunning', () => {
|
||||
// ...but a fresh 'running' run (updatedAt = now) must NOT be skipped: no
|
||||
// updatedAt predicate at all on the boot path.
|
||||
expect(rec.wheres.some(([col]) => col === 'updatedAt')).toBe(false);
|
||||
// It flips to 'aborted' and stamps finishedAt.
|
||||
expect(rec.set).toEqual(
|
||||
expect.objectContaining({ status: 'aborted', finishedAt: expect.any(Date) }),
|
||||
);
|
||||
// It flips to 'aborted' and stamps finishedAt + updatedAt. #491: the stamps
|
||||
// are now DB-clock `sql now()` expressions (raw builders), NOT app-clock
|
||||
// `new Date()`, so the run row shares the delta poll's single now() cursor axis
|
||||
// — assert they are present and are the sql raw-builder objects (not a Date,
|
||||
// not undefined).
|
||||
expect(rec.set?.status).toBe('aborted');
|
||||
for (const stamp of ['finishedAt', 'updatedAt'] as const) {
|
||||
expect(rec.set?.[stamp]).toBeDefined();
|
||||
expect(rec.set?.[stamp]).not.toBeInstanceOf(Date);
|
||||
expect(typeof rec.set?.[stamp]).toBe('object');
|
||||
}
|
||||
});
|
||||
|
||||
it('phase-2 path: an explicit staleMs reintroduces the updatedAt window', async () => {
|
||||
|
||||
@@ -136,13 +136,53 @@ export class AiChatRunRepo {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.updateTable('aiChatRuns')
|
||||
.set({ ...(patch as Record<string, unknown>), updatedAt: new Date() })
|
||||
// #491: DB-clock stamp (sql now()) so the run row shares the delta poll's
|
||||
// single now() cursor axis with the assistant message rows — a run-status
|
||||
// change (the run fact the delta carries) must never sit on a skewed app
|
||||
// clock relative to the message updatedAt cursor.
|
||||
.set({ ...(patch as Record<string, unknown>), updatedAt: sql`now()` })
|
||||
.where('id', '=', id)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* #487: CONDITIONAL terminal finalize — flip a run to a terminal status and
|
||||
* stamp `finished_at` ONLY while it is still active (pending|running), mirroring
|
||||
* the assistant message's `onlyIfStreaming` guard. A double-settle (a late or
|
||||
* second writer, a supersede applying a zombie's intended, a reconcile stamp)
|
||||
* matches NOTHING once the row is terminal and is a benign no-op — so a terminal
|
||||
* status can never be clobbered by a later writer (last-writer-wins is gone).
|
||||
*
|
||||
* Returns the updated row when it WAS active (this call wrote it), else
|
||||
* undefined (the row was already terminal — another writer won). The caller
|
||||
* distinguishes the two to resolve the correct settle outcome.
|
||||
*/
|
||||
async finalizeIfActive(
|
||||
id: string,
|
||||
workspaceId: string,
|
||||
patch: { status: string; error: string | null },
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<AiChatRun | undefined> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.updateTable('aiChatRuns')
|
||||
.set({
|
||||
status: patch.status,
|
||||
error: patch.error,
|
||||
// #491: DB-clock stamps (finished_at + updated_at) so the terminal run
|
||||
// fact lands on the delta poll's now() cursor axis.
|
||||
finishedAt: sql`now()`,
|
||||
updatedAt: sql`now()`,
|
||||
})
|
||||
.where('id', '=', id)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.where('status', 'in', ACTIVE_RUN_STATUSES as unknown as string[])
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark an EXPLICIT stop request on an active run (distinct from a browser
|
||||
* disconnect, which never stops a run). Stamps `stop_requested_at` ONLY while
|
||||
@@ -157,7 +197,8 @@ export class AiChatRunRepo {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.updateTable('aiChatRuns')
|
||||
.set({ stopRequestedAt: new Date(), updatedAt: new Date() })
|
||||
// #491: DB-clock stamps (see `update`).
|
||||
.set({ stopRequestedAt: sql`now()`, updatedAt: sql`now()` })
|
||||
.where('id', '=', id)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.where('status', 'in', ACTIVE_RUN_STATUSES as unknown as string[])
|
||||
@@ -184,18 +225,44 @@ export class AiChatRunRepo {
|
||||
* sweeps only runs UNTOUCHED past the window. Phase 1 is single-process, so the
|
||||
* boot path supplies no window.
|
||||
*/
|
||||
/**
|
||||
* #487 reconcile clause (c): active (pending|running) runs UNTOUCHED past
|
||||
* `staleMs` — candidates for "no live runner" abort. Staleness is measured from
|
||||
* `updated_at` (the LAST-PROGRESS timestamp — recordStep bumps it), NOT
|
||||
* `started_at`, so a legitimate long-running marathon (11–25 min of steady
|
||||
* progress) is never a candidate. The caller filters these against its in-memory
|
||||
* `active` / zombie maps ("no entry" is the PRIMARY gate — a live entry is never
|
||||
* aborted) before settling any of them. Bounded.
|
||||
*/
|
||||
async findStaleActive(
|
||||
staleMs: number,
|
||||
limit = 200,
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<Array<{ id: string; workspaceId: string; chatId: string }>> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
const staleBefore = new Date(Date.now() - staleMs);
|
||||
return db
|
||||
.selectFrom('aiChatRuns')
|
||||
.select(['id', 'workspaceId', 'chatId'])
|
||||
.where('status', 'in', ACTIVE_RUN_STATUSES as unknown as string[])
|
||||
.where('updatedAt', '<', staleBefore)
|
||||
.limit(limit)
|
||||
.execute();
|
||||
}
|
||||
|
||||
async sweepRunning(
|
||||
opts: { staleMs?: number } = {},
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<number> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
const now = new Date();
|
||||
let query = db
|
||||
.updateTable('aiChatRuns')
|
||||
.set({
|
||||
status: 'aborted',
|
||||
finishedAt: now,
|
||||
updatedAt: now,
|
||||
// #491: DB-clock stamps (see `update`). The staleness WHERE below stays on
|
||||
// the app clock — a >minutes window makes the ms-scale skew irrelevant.
|
||||
finishedAt: sql`now()`,
|
||||
updatedAt: sql`now()`,
|
||||
error: sql`coalesce(error, ${'Run interrupted by a server restart.'})`,
|
||||
})
|
||||
.where('status', 'in', ACTIVE_RUN_STATUSES as unknown as string[]);
|
||||
@@ -203,7 +270,7 @@ export class AiChatRunRepo {
|
||||
// sibling replica's live run is never aborted. Omitted on the phase-1 boot
|
||||
// sweep -> unconditional.
|
||||
if (typeof opts.staleMs === 'number') {
|
||||
const staleBefore = new Date(now.getTime() - opts.staleMs);
|
||||
const staleBefore = new Date(Date.now() - opts.staleMs);
|
||||
query = query.where('updatedAt', '<', staleBefore);
|
||||
}
|
||||
const rows = await query.returning('id').execute();
|
||||
|
||||
+3
@@ -606,6 +606,9 @@ export interface AiChats {
|
||||
// The document the chat was created in (open page at first message). NULL =>
|
||||
// started outside any document. ON DELETE SET NULL on the page FK.
|
||||
pageId: string | null;
|
||||
// Chat-level metadata bag (#490). jsonb, defaulted to '{}'. First key:
|
||||
// `activatedTools` — the deferred-tool activation set persisted across turns.
|
||||
metadata: Generated<Json>;
|
||||
createdAt: Generated<Timestamp>;
|
||||
updatedAt: Generated<Timestamp>;
|
||||
deletedAt: Timestamp | null;
|
||||
|
||||
@@ -245,6 +245,9 @@ export class AiSettingsService {
|
||||
// Max context window for the chat header badge denominator. Stored as
|
||||
// ::text; 0/unset/invalid = no limit (undefined).
|
||||
chatContextWindow: parsePositiveInt(provider.chatContextWindow),
|
||||
// RAW stored value (#490): the replay budgeter reads this to distinguish an
|
||||
// explicit `0` (off-switch) from unset, which parsePositiveInt cannot.
|
||||
chatContextWindowRaw: provider.chatContextWindow,
|
||||
// Plain passthrough; getChatModel defaults unset to 'openai-compatible'.
|
||||
chatApiStyle: provider.chatApiStyle,
|
||||
// Cheap model id for the anonymous public-share assistant; reuses the chat
|
||||
|
||||
@@ -129,6 +129,12 @@ const DEFAULT_MCP_STREAM_TIMEOUT_MS = 60_000;
|
||||
/** Default total wall-clock cap for ONE external MCP tool call (2 min). */
|
||||
const DEFAULT_MCP_CALL_TIMEOUT_MS = 120_000;
|
||||
|
||||
/**
|
||||
* Default `bodyTimeout` for the EXTERNAL-MCP SSE transport (10 min) — #489.
|
||||
* Deliberately much LARGER than {@link DEFAULT_MCP_STREAM_TIMEOUT_MS}.
|
||||
*/
|
||||
const DEFAULT_MCP_SSE_BODY_TIMEOUT_MS = 600_000;
|
||||
|
||||
/**
|
||||
* SILENCE timeout (ms) for EXTERNAL-MCP transport ONLY. Override with
|
||||
* `AI_MCP_STREAM_TIMEOUT_MS`; a missing/invalid/non-positive value falls back to
|
||||
@@ -164,6 +170,26 @@ export function mcpCallTimeoutMs(): number {
|
||||
return positiveEnv('AI_MCP_CALL_TIMEOUT_MS', DEFAULT_MCP_CALL_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* `bodyTimeout` (ms) for the EXTERNAL-MCP **SSE** transport ONLY — #489. Override
|
||||
* with `AI_MCP_SSE_BODY_TIMEOUT_MS`; a missing/invalid/non-positive value falls
|
||||
* back to {@link DEFAULT_MCP_SSE_BODY_TIMEOUT_MS} (10 min).
|
||||
*
|
||||
* The SSE transport holds ONE long-lived response body open across many tool
|
||||
* calls, so undici's `bodyTimeout` (time between body bytes) counts the LEGITIMATE
|
||||
* silence BETWEEN calls, not just a hung single call. At the tight HTTP silence
|
||||
* timeout ({@link mcpStreamTimeoutMs}, 1 min) a normal >1-min gap between the
|
||||
* model's tool calls would break the SSE socket, and the cache would then serve a
|
||||
* dead client until TTL. So the SSE transport gets its OWN, RAISED bodyTimeout;
|
||||
* the per-call total cap ({@link mcpCallTimeoutMs}) still bounds a single stuck
|
||||
* call, and the app-level transport-error retry heals a socket that does break.
|
||||
* The HTTP (streamable) transport keeps the tight timeout — it opens a fresh
|
||||
* request per call, so idle-between-calls does not apply there.
|
||||
*/
|
||||
export function mcpSseBodyTimeoutMs(): number {
|
||||
return positiveEnv('AI_MCP_SSE_BODY_TIMEOUT_MS', DEFAULT_MCP_SSE_BODY_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* undici `Agent` options for streaming AI traffic — the (generous, finite)
|
||||
* silence timeouts plus the keep-alive recycle window. Shared by the chat
|
||||
|
||||
@@ -105,6 +105,10 @@ export interface ResolvedAiConfig extends Partial<AiProviderSettings> {
|
||||
// Max context window in tokens; surfaced to the chat header badge as the
|
||||
// "current / max" denominator. 0/unset = no limit.
|
||||
chatContextWindow?: number;
|
||||
// RAW stored context window (::text), BEFORE parsePositiveInt collapses `0` and
|
||||
// unset to `undefined`. The #490 replay budgeter needs the raw value to honor an
|
||||
// explicit `0` off-switch distinctly from "unset -> flat default".
|
||||
chatContextWindowRaw?: string | number;
|
||||
// Cheap model id for the public-share assistant; reuses the chat creds.
|
||||
publicShareChatModel?: string;
|
||||
// Agent-role id whose persona the public-share assistant adopts (empty/unset
|
||||
|
||||
@@ -30,11 +30,13 @@ import {
|
||||
* tees the SSE frames into it via `consumeSseStream` while stamping the DB row id
|
||||
* via `generateMessageId` (both gated on runId + the resumable flag).
|
||||
*
|
||||
* Proven here: a finished run's replay is the full frame sequence incl `[DONE]`
|
||||
* with `start.messageId` == the seeded DB row id; the anchor check (invariant 6);
|
||||
* an attach opened BEFORE the first frame follows the live stream from frame 0; an
|
||||
* explicit stop surfaces `{"type":"abort"}` + `[DONE]` + end to the subscriber;
|
||||
* and the legacy (non-run) path tees nothing.
|
||||
* Proven here (tail-only #491): a finished run attached at its persisted frontier
|
||||
* N_final delivers only the TAIL past N (a synthetic `start` carrying the run-fact
|
||||
* + the terminal `finish`/`[DONE]`) — the step content below N lives in the seeded
|
||||
* DB row, NOT the ring; the anchor check (invariant 6); an attach opened BEFORE the
|
||||
* first frame follows the live stream from frame 0; an explicit stop surfaces
|
||||
* `{"type":"abort"}` + `[DONE]` + end to the subscriber; and the legacy (non-run)
|
||||
* path tees nothing.
|
||||
*/
|
||||
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
|
||||
@@ -133,14 +135,16 @@ function liveSink(): {
|
||||
};
|
||||
}
|
||||
|
||||
// The SSE `start` frame carries the message id; pull it out of a `data: {...}`.
|
||||
function parseStartMessageId(frames: string[]): string | undefined {
|
||||
// Parse the first `start` frame's JSON out of a `data: {...}` sequence.
|
||||
function parseStartFrame(
|
||||
frames: string[],
|
||||
): { messageId?: string; messageMetadata?: any } | undefined {
|
||||
for (const f of frames) {
|
||||
const m = /^data: (\{.*\})\s*$/m.exec(f.trim());
|
||||
if (!m) continue;
|
||||
try {
|
||||
const json = JSON.parse(m[1]);
|
||||
if (json.type === 'start') return json.messageId;
|
||||
if (json.type === 'start') return json;
|
||||
} catch {
|
||||
/* not this frame */
|
||||
}
|
||||
@@ -270,7 +274,7 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('run-wrapped: replay is the full frame sequence incl [DONE], start.messageId == the seeded DB row id', async () => {
|
||||
it('run-wrapped, tail-only: a finished run at N_final delivers the run-fact start + finish/[DONE]; the step content lives in the seeded row', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
const registry = new AiChatStreamRegistryService();
|
||||
const runService = new AiChatRunService(runRepo, {
|
||||
@@ -300,27 +304,37 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
);
|
||||
});
|
||||
const rowId = await assistantRowId(chatId);
|
||||
// The client reads its persisted step frontier N from the seeded row.
|
||||
const row: any = await msgRepo.findById(rowId, workspaceId);
|
||||
const nFinal = row.metadata.stepsPersisted as number;
|
||||
expect(nFinal).toBe(1); // a single finished step
|
||||
// The step content is in the SEEDED row (parts/content), not the ring.
|
||||
expect(JSON.stringify(row.metadata.parts)).toContain('Hello');
|
||||
|
||||
// Finished-run replay with expect=live + the correct anchor.
|
||||
// Attach at N_final with the correct anchor: the tail past step 1 is just
|
||||
// the terminal frames; step 0's 'Hello' is BELOW the frontier (seeded).
|
||||
const sink = liveSink();
|
||||
const att = await registry.attach(chatId, true, rowId, sink.cb);
|
||||
const att = await registry.attach(chatId, rowId, nFinal, sink.cb);
|
||||
expect(att).not.toBeNull();
|
||||
expect(att!.finished).toBe(true);
|
||||
// The tee captured frames (consumeSseStream was wired).
|
||||
expect(att!.replay.length).toBeGreaterThan(0);
|
||||
// generateMessageId stamped the DB row id onto the streamed start frame.
|
||||
expect(parseStartMessageId(att!.replay)).toBe(rowId);
|
||||
// The full sequence includes the streamed text and the terminal marker.
|
||||
const joined = att!.replay.join('');
|
||||
expect(joined).toContain('Hello');
|
||||
// The synthetic start frame carries the run-fact (runId/chatId), the source
|
||||
// of the run-fact on re-attach.
|
||||
const start = parseStartFrame(att!.replay);
|
||||
expect(start?.messageMetadata).toMatchObject({
|
||||
runId: box.runId,
|
||||
chatId,
|
||||
});
|
||||
// The terminal marker is delivered so the client's SDK closes the stream.
|
||||
expect(att!.replay.some((f) => f.includes('[DONE]'))).toBe(true);
|
||||
// 'Hello' (step 0, below the frontier) is NOT re-streamed — it is seeded.
|
||||
expect(att!.replay.some((f) => f.includes('Hello'))).toBe(false);
|
||||
} finally {
|
||||
registry.onModuleDestroy();
|
||||
await cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it('anchor mismatch with expect=live returns null (invariant 6)', async () => {
|
||||
it('anchor mismatch returns null (invariant 6)', async () => {
|
||||
const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
const registry = new AiChatStreamRegistryService();
|
||||
const runService = new AiChatRunService(runRepo, {
|
||||
@@ -347,7 +361,7 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
const sink = liveSink();
|
||||
// A foreign anchor must NOT replay this run's transcript.
|
||||
expect(
|
||||
await registry.attach(chatId, true, 'a-different-run-row', sink.cb),
|
||||
await registry.attach(chatId, 'a-different-run-row', 1, sink.cb),
|
||||
).toBeNull();
|
||||
} finally {
|
||||
registry.onModuleDestroy();
|
||||
@@ -376,8 +390,11 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
try {
|
||||
// Attach while the entry exists (opened at begin) but before any frame.
|
||||
const sink = liveSink();
|
||||
const att = (await registry.attach(chatId, false, undefined, sink.cb))!;
|
||||
expect(att.replay).toEqual([]); // nothing streamed yet -> replay from 0
|
||||
const att = (await registry.attach(chatId, undefined, 0, sink.cb))!;
|
||||
// Nothing streamed yet -> the tail is just the synthetic start frame; the
|
||||
// whole live stream (start..DONE) follows via onFrame after start().
|
||||
expect(att.replay).toHaveLength(1);
|
||||
expect(att.replay[0]).toContain('"type":"start"');
|
||||
att.start(); // go live (drains nothing, then follows)
|
||||
|
||||
// Now emit the whole turn.
|
||||
@@ -448,7 +465,7 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
});
|
||||
try {
|
||||
const sink = liveSink();
|
||||
const att = (await registry.attach(chatId, false, undefined, sink.cb))!;
|
||||
const att = (await registry.attach(chatId, undefined, 0, sink.cb))!;
|
||||
att.start();
|
||||
|
||||
// Give streamText a beat to begin consuming the partial output.
|
||||
@@ -526,7 +543,9 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
expect(entry).toBeDefined();
|
||||
expect(entry.finished).toBe(true);
|
||||
const sink = liveSink();
|
||||
expect(await registry.attach(chatId, false, undefined, sink.cb)).toBeNull();
|
||||
// Finished with an EMPTY ring (aborted before any frame) -> null -> the
|
||||
// client degrades to poll instead of hanging on an empty stream.
|
||||
expect(await registry.attach(chatId, undefined, 0, sink.cb)).toBeNull();
|
||||
} finally {
|
||||
registry.onModuleDestroy();
|
||||
await cleanup();
|
||||
@@ -556,8 +575,8 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
});
|
||||
const sink = liveSink();
|
||||
// No entry was ever opened; attach always yields null.
|
||||
expect(await registry.attach(chatId, false, undefined, sink.cb)).toBeNull();
|
||||
expect(await registry.attach(chatId, true, 'anything', sink.cb)).toBeNull();
|
||||
expect(await registry.attach(chatId, undefined, 0, sink.cb)).toBeNull();
|
||||
expect(await registry.attach(chatId, 'anything', 1, sink.cb)).toBeNull();
|
||||
} finally {
|
||||
registry.onModuleDestroy();
|
||||
await cleanup();
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { AiChatRunRepo } from '@docmost/db/repos/ai-chat/ai-chat-run.repo';
|
||||
import { AiChatRunService } from '../../src/core/ai-chat/ai-chat-run.service';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createUser,
|
||||
createChat,
|
||||
createMessage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #487 commit 4 — bidirectional reconcile + owner-write priority, real SQL.
|
||||
*
|
||||
* Proves the OBSERVABLE recovery properties against docmost_test:
|
||||
* - the CONDITIONAL owner-write beats a reconcile stamp, and a stamp never
|
||||
* clobbers a proper terminal row;
|
||||
* - a LATE owner-finalize with real content OVERWRITES a reconcile 'aborted'
|
||||
* stamp (finalizeFailed);
|
||||
* - each reconcile clause (b message<-run, c stale-run, d historical row) settles
|
||||
* the stuck row/run, and a LIVE run entry is never touched;
|
||||
* - the "kill DB on finish" recovery: after the DB comes back, neither the
|
||||
* message row nor the run row stays stuck.
|
||||
*/
|
||||
describe('#487 reconcile + owner-write priority [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
let messageRepo: AiChatMessageRepo;
|
||||
let runRepo: AiChatRunRepo;
|
||||
let runService: AiChatRunService;
|
||||
let workspaceId: string;
|
||||
let userId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
messageRepo = new AiChatMessageRepo(db as any);
|
||||
runRepo = new AiChatRunRepo(db as any);
|
||||
runService = new AiChatRunService(runRepo, { isCloud: () => false } as never);
|
||||
workspaceId = (await createWorkspace(db)).id;
|
||||
userId = (await createUser(db, workspaceId)).id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
const newChat = async () =>
|
||||
(await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
|
||||
const metaOf = async (id: string): Promise<Record<string, unknown> | null> => {
|
||||
const row = await messageRepo.findById(id, workspaceId);
|
||||
return (row?.metadata as Record<string, unknown> | null) ?? null;
|
||||
};
|
||||
|
||||
it('owner finalizeOwner writes a streaming row and CLEARS finalizeFailed', async () => {
|
||||
const chatId = await newChat();
|
||||
const m = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
metadata: { parts: [] },
|
||||
});
|
||||
const wrote = await messageRepo.finalizeOwner(m.id, workspaceId, {
|
||||
content: 'final answer',
|
||||
status: 'completed',
|
||||
metadata: { parts: [{ type: 'text', text: 'final answer' }] },
|
||||
} as never);
|
||||
expect(wrote!.status).toBe('completed');
|
||||
expect((await metaOf(m.id))?.finalizeFailed).toBeUndefined();
|
||||
});
|
||||
|
||||
it('a reconcile stamp NEVER clobbers a proper terminal row (finalizeOwner is a no-op there)', async () => {
|
||||
const chatId = await newChat();
|
||||
const m = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'completed',
|
||||
content: 'real',
|
||||
metadata: { parts: [] },
|
||||
});
|
||||
// The reconcile stamp is onlyIfStreaming -> no-op on a completed row.
|
||||
const stamped = await messageRepo.stampTerminalIfStreaming(
|
||||
m.id,
|
||||
workspaceId,
|
||||
'aborted',
|
||||
);
|
||||
expect(stamped).toBeUndefined();
|
||||
expect((await messageRepo.findById(m.id, workspaceId))!.status).toBe(
|
||||
'completed',
|
||||
);
|
||||
});
|
||||
|
||||
it('LATE owner-finalize with real content OVERWRITES a reconcile aborted stamp', async () => {
|
||||
const chatId = await newChat();
|
||||
const m = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
metadata: { parts: [{ type: 'text', text: 'partial' }] },
|
||||
});
|
||||
// Reconcile stamps it aborted + finalizeFailed (final text lived only in mem).
|
||||
const stamped = await messageRepo.stampTerminalIfStreaming(
|
||||
m.id,
|
||||
workspaceId,
|
||||
'aborted',
|
||||
);
|
||||
expect(stamped!.status).toBe('aborted');
|
||||
expect((await metaOf(m.id))?.finalizeFailed).toBe(true);
|
||||
|
||||
// A LATE owner-write (finalizeFailed=true satisfies the OR) overwrites it with
|
||||
// real content, clearing the flag — owner-write priority.
|
||||
const wrote = await messageRepo.finalizeOwner(m.id, workspaceId, {
|
||||
content: 'the real final answer',
|
||||
status: 'completed',
|
||||
metadata: { parts: [{ type: 'text', text: 'the real final answer' }] },
|
||||
} as never);
|
||||
expect(wrote!.status).toBe('completed');
|
||||
expect(wrote!.content).toBe('the real final answer');
|
||||
expect((await metaOf(m.id))?.finalizeFailed).toBeUndefined();
|
||||
});
|
||||
|
||||
it('clause (c): a stale active run with NO live entry -> aborted; a LIVE entry is untouched', async () => {
|
||||
// Stale run, NOT owned by this replica (no entry) -> reconcile aborts it.
|
||||
const staleChat = await newChat();
|
||||
const stale = await runRepo.insert({
|
||||
chatId: staleChat,
|
||||
workspaceId,
|
||||
createdBy: userId,
|
||||
status: 'running',
|
||||
});
|
||||
await db
|
||||
.updateTable('aiChatRuns')
|
||||
.set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) })
|
||||
.where('id', '=', stale.id)
|
||||
.execute();
|
||||
|
||||
// A live run OWNED by this replica (beginRun registers an in-memory entry),
|
||||
// ALSO backdated stale — the "no entry" primary gate must protect it.
|
||||
const liveChat = await newChat();
|
||||
const live = await runService.beginRun({
|
||||
chatId: liveChat,
|
||||
workspaceId,
|
||||
userId,
|
||||
});
|
||||
await db
|
||||
.updateTable('aiChatRuns')
|
||||
.set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) })
|
||||
.where('id', '=', live.runId)
|
||||
.execute();
|
||||
|
||||
const aborted = await runService.reconcileStaleRuns(15 * 60 * 1000);
|
||||
expect(aborted).toBeGreaterThanOrEqual(1);
|
||||
expect((await runRepo.findById(stale.id, workspaceId))!.status).toBe(
|
||||
'aborted',
|
||||
);
|
||||
// The live entry is NEVER aborted, however stale its row looks.
|
||||
expect((await runRepo.findById(live.runId, workspaceId))!.status).toBe(
|
||||
'running',
|
||||
);
|
||||
expect(runService.isLocallyActive(live.runId)).toBe(true);
|
||||
|
||||
// cleanup the live run
|
||||
await runService.finalizeRun(live.runId, workspaceId, 'aborted');
|
||||
});
|
||||
|
||||
it('clause (b): a streaming message whose RUN is terminal is stamped by run status (succeeded -> aborted, NOT completed-empty)', async () => {
|
||||
const chatId = await newChat();
|
||||
const msg = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
metadata: { parts: [] },
|
||||
});
|
||||
// A SUCCEEDED run linked to the still-streaming message (the asymmetry).
|
||||
const run = await runRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
createdBy: userId,
|
||||
status: 'running',
|
||||
assistantMessageId: msg.id,
|
||||
});
|
||||
await runRepo.finalizeIfActive(run.id, workspaceId, {
|
||||
status: 'succeeded',
|
||||
error: null,
|
||||
});
|
||||
|
||||
const stuck = await messageRepo.findStreamingWithTerminalRun();
|
||||
const mine = stuck.find((s) => s.messageId === msg.id);
|
||||
expect(mine?.runStatus).toBe('succeeded');
|
||||
// Reconcile clause (b): succeeded run -> message 'aborted' (NOT 'completed'),
|
||||
// the final text lived only in memory (documented loss), +finalizeFailed.
|
||||
const status = mine!.runStatus === 'failed' ? 'error' : 'aborted';
|
||||
await messageRepo.stampTerminalIfStreaming(msg.id, workspaceId, status);
|
||||
const row = await messageRepo.findById(msg.id, workspaceId);
|
||||
expect(row!.status).toBe('aborted');
|
||||
expect((row!.metadata as Record<string, unknown>).finalizeFailed).toBe(true);
|
||||
});
|
||||
|
||||
it('clause (d): a stale streaming row with NO active run on the chat -> aborted+finalizeFailed', async () => {
|
||||
const chatId = await newChat();
|
||||
const msg = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
metadata: { parts: [] },
|
||||
});
|
||||
await db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) })
|
||||
.where('id', '=', msg.id)
|
||||
.execute();
|
||||
|
||||
const swept = await messageRepo.sweepStreamingWithoutActiveRun(
|
||||
15 * 60 * 1000,
|
||||
);
|
||||
expect(swept).toBeGreaterThanOrEqual(1);
|
||||
const row = await messageRepo.findById(msg.id, workspaceId);
|
||||
expect(row!.status).toBe('aborted');
|
||||
expect((row!.metadata as Record<string, unknown>).finalizeFailed).toBe(true);
|
||||
});
|
||||
|
||||
it('clause (d) is DOUBLE-GATED: a stale streaming row WITH an active run on the chat is left alone', async () => {
|
||||
const chatId = await newChat();
|
||||
const msg = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
metadata: { parts: [] },
|
||||
});
|
||||
await db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) })
|
||||
.where('id', '=', msg.id)
|
||||
.execute();
|
||||
// An ACTIVE run on the same chat -> clause (d) must NOT touch the message.
|
||||
const run = await runRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
createdBy: userId,
|
||||
status: 'running',
|
||||
});
|
||||
|
||||
await messageRepo.sweepStreamingWithoutActiveRun(15 * 60 * 1000);
|
||||
expect((await messageRepo.findById(msg.id, workspaceId))!.status).toBe(
|
||||
'streaming',
|
||||
);
|
||||
await runRepo.finalizeIfActive(run.id, workspaceId, {
|
||||
status: 'aborted',
|
||||
error: null,
|
||||
});
|
||||
});
|
||||
|
||||
it('"kill DB on finish" recovery: after the DB is back, reconcile leaves NEITHER the row nor the run stuck', async () => {
|
||||
// Simulate a process that seeded the assistant row + run, then died before
|
||||
// finalizing EITHER (a mid-turn crash): a streaming message + a running run,
|
||||
// both stale, with no in-memory entry (fresh service = fresh maps).
|
||||
const chatId = await newChat();
|
||||
const msg = await createMessage(db, {
|
||||
workspaceId,
|
||||
chatId,
|
||||
role: 'assistant',
|
||||
status: 'streaming',
|
||||
metadata: { parts: [{ type: 'text', text: 'partial' }] },
|
||||
});
|
||||
const run = await runRepo.insert({
|
||||
chatId,
|
||||
workspaceId,
|
||||
createdBy: userId,
|
||||
status: 'running',
|
||||
assistantMessageId: msg.id,
|
||||
});
|
||||
await db
|
||||
.updateTable('aiChatRuns')
|
||||
.set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) })
|
||||
.where('id', '=', run.id)
|
||||
.execute();
|
||||
await db
|
||||
.updateTable('aiChatMessages')
|
||||
.set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) })
|
||||
.where('id', '=', msg.id)
|
||||
.execute();
|
||||
|
||||
// Reconcile (as the periodic job would): (c) aborts the orphan run, then
|
||||
// (b) settles the message from the now-terminal run.
|
||||
await runService.reconcileStaleRuns(15 * 60 * 1000);
|
||||
const stuck = await messageRepo.findStreamingWithTerminalRun();
|
||||
for (const s of stuck) {
|
||||
const status = s.runStatus === 'failed' ? 'error' : 'aborted';
|
||||
await messageRepo.stampTerminalIfStreaming(s.messageId, s.workspaceId, status);
|
||||
}
|
||||
|
||||
// Neither is stuck: the run is terminal AND the message is terminal.
|
||||
expect((await runRepo.findById(run.id, workspaceId))!.status).toBe('aborted');
|
||||
const row = await messageRepo.findById(msg.id, workspaceId);
|
||||
expect(row!.status).toBe('aborted');
|
||||
expect((row!.metadata as Record<string, unknown>).finalizeFailed).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -281,6 +281,52 @@ describe('AiChatRun durable lifecycle [integration]', () => {
|
||||
});
|
||||
});
|
||||
|
||||
it('#487 finalizeIfActive is CONDITIONAL: a late terminal write cannot clobber the settled status (real SQL)', async () => {
|
||||
const c = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
const run = await runRepo.insert({
|
||||
chatId: c,
|
||||
workspaceId,
|
||||
createdBy: userId,
|
||||
status: 'running',
|
||||
});
|
||||
|
||||
// First terminal write: the run IS active, so it flips + returns the row.
|
||||
const first = await runRepo.finalizeIfActive(run.id, workspaceId, {
|
||||
status: 'succeeded',
|
||||
error: null,
|
||||
});
|
||||
expect(first!.status).toBe('succeeded');
|
||||
expect(first!.finishedAt).toBeTruthy();
|
||||
|
||||
// A late/second writer tries to flip it to 'aborted' — the WHERE status IN
|
||||
// ('pending','running') guard matches NOTHING now, so it is a benign no-op.
|
||||
const second = await runRepo.finalizeIfActive(run.id, workspaceId, {
|
||||
status: 'aborted',
|
||||
error: 'late clobber attempt',
|
||||
});
|
||||
expect(second).toBeUndefined();
|
||||
|
||||
// The persisted terminal status is UNCHANGED — last-writer-wins is gone.
|
||||
const row = await runRepo.findById(run.id, workspaceId);
|
||||
expect(row!.status).toBe('succeeded');
|
||||
expect(row!.error).toBeNull();
|
||||
});
|
||||
|
||||
it('#487 double-settle through the service collapses to one write at the SQL gate', async () => {
|
||||
const c = (await createChat(db, { workspaceId, creatorId: userId })).id;
|
||||
const handle = await service.beginRun({ chatId: c, workspaceId, userId });
|
||||
|
||||
// First settle writes 'aborted' via the conditional write.
|
||||
await service.finalizeRun(handle.runId, workspaceId, 'aborted');
|
||||
// A late safety-net settle to 'error' is a no-op (row already terminal).
|
||||
await service.finalizeRun(handle.runId, workspaceId, 'error', 'late');
|
||||
|
||||
const row = await runRepo.findById(handle.runId, workspaceId);
|
||||
expect(row!.status).toBe('aborted');
|
||||
expect(service.isLocallyActive(handle.runId)).toBe(false);
|
||||
expect(service.hasZombie(handle.runId)).toBe(false);
|
||||
});
|
||||
|
||||
it('sweepRunning() with NO args (boot sweep / variant C) aborts even a FRESH running run', async () => {
|
||||
// F1/DECISION C at the SQL level: the unconditional boot sweep has NO
|
||||
// staleness window, so a run updated just now (a fast restart) is settled too
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
"^@docmost/db/(.*)$": "<rootDir>/src/database/$1",
|
||||
"^@docmost/transactional/(.*)$": "<rootDir>/src/integrations/transactional/$1",
|
||||
"^@docmost/ee/(.*)$": "<rootDir>/src/ee/$1",
|
||||
"^@docmost/token-estimate$": "<rootDir>/../../packages/token-estimate/src/index.ts",
|
||||
"^src/(.*)$": "<rootDir>/src/$1"
|
||||
}
|
||||
}
|
||||
|
||||
+174
-78
@@ -8,19 +8,35 @@ real pain (a "which tools fail most?" analysis that confidently answered
|
||||
|
||||
Read the **Gotchas** section before you trust any error count.
|
||||
|
||||
> **TWO ERAS — check the marker first.** The `tool_calls` shape changed in **#490
|
||||
> (trace v2)**. A row written by v2 carries `metadata.toolTraceVersion = 2`; older
|
||||
> rows have no such key. The two shapes store DIFFERENT things (v2 dropped the tool
|
||||
> OUTPUT from the trace), so **every query below is dual-shape** — branch on the
|
||||
> marker. **Never compare an aggregate or trend across the era boundary**: a metric
|
||||
> jump on the cut-over week is an artifact of the shape change, not a behavior
|
||||
> change.
|
||||
|
||||
## TL;DR
|
||||
|
||||
- Agent chats live in Postgres, DB `docmost`, tables `ai_chat_*`.
|
||||
- Each tool invocation is stored as **two** array elements (a `tool-call` part and
|
||||
a `tool-result` part), so naive counting double-counts.
|
||||
- **A tool that *throws* writes no result part.** Since the #407 fix its error is
|
||||
persisted as a dedicated `{toolName, error}` element in `tool_calls` (queryable +
|
||||
replayed to the model). **Rows written before #407 still drop it** — the error is
|
||||
nowhere in the DB and shows only in the live UI. So `isError` / `success=false`
|
||||
scans under-report by design, and pre-#407 thrown errors are invisible.
|
||||
- To find where agents fail: (1) soft-failure markers in `tool_calls`, (2) the new
|
||||
`error` field for thrown errors (new rows) / the orphan-gap proxy (old rows),
|
||||
(3) server logs / the live UI for full stack traces beyond the truncated message.
|
||||
- **Era marker:** `metadata.toolTraceVersion = 2` ⇒ v2 (#490) row; absent ⇒ legacy row.
|
||||
- Each tool invocation is stored as **two** consecutive array elements — a
|
||||
`tool-call` part then an OUTCOME part — so naive counting double-counts.
|
||||
- **v2 (#490):** outcome is `{toolName, ok: true}` on success, or
|
||||
`{toolName, error, kind: 'thrown'|'interrupted'}` on failure. The tool **OUTPUT
|
||||
is NOT in `tool_calls`** any more — it lives once in `metadata.parts` (this
|
||||
removed a hundreds-of-MB-per-run write duplication). Soft-failure analysis
|
||||
therefore reads `metadata.parts`, not `tool_calls`.
|
||||
- **legacy:** outcome is `{toolName, output}` on success; a **thrown** failure is
|
||||
a `{toolName, error}` element **only on rows after #407**, and is dropped
|
||||
entirely (silent orphan) on pre-#407 rows.
|
||||
- **A tool that *throws* writes no result part.** In v2 it is a
|
||||
`{error, kind:'thrown'}` element; an interrupted/aborted call is a distinct
|
||||
`{error, kind:'interrupted'}`. `isError`/`success=false` scans read the *output*
|
||||
and so under-report thrown failures in every era.
|
||||
- To find where agents fail: (1) soft-failure markers in `metadata.parts` outputs
|
||||
(v2) / `tool_calls` outputs (legacy), (2) the `error`/`kind` fields for thrown
|
||||
failures (v2 + post-#407), (3) server logs / the live UI for full stack traces.
|
||||
|
||||
## Where the data lives
|
||||
|
||||
@@ -53,33 +69,67 @@ are rows in `workspaces`, not separate deployments.
|
||||
separate `tool` role), `content` (text), `tool_calls` (jsonb array), `metadata`
|
||||
(jsonb, holds run `error` + rendered `parts`), `status`, `tsv` (full-text index).
|
||||
|
||||
## Era marker — check this before every query
|
||||
|
||||
```sql
|
||||
-- how many rows are in each era?
|
||||
SELECT COALESCE((metadata->>'toolTraceVersion'), 'legacy') AS era, count(*)
|
||||
FROM ai_chat_messages
|
||||
WHERE role = 'assistant' AND jsonb_typeof(tool_calls) = 'array'
|
||||
GROUP BY 1 ORDER BY 2 DESC;
|
||||
```
|
||||
|
||||
- `toolTraceVersion = '2'` → **v2** (#490): outcome flags, **no output in the trace**.
|
||||
- `NULL` (`'legacy'`) → pre-#490: outcome carries the tool `output` inline.
|
||||
|
||||
**Do not trend a metric across the cut-over.** The shape change alone shifts counts
|
||||
(e.g. "elements with `output`" collapses to zero for v2), so a week that straddles
|
||||
the boundary shows an artifact, not a behavior change. Segment by era, or restrict to
|
||||
one era, before comparing.
|
||||
|
||||
## How tool calls are stored — READ THIS
|
||||
|
||||
Tool calls are **not** one-object-per-call. Each logical invocation is split into
|
||||
two consecutive elements of the `tool_calls` array:
|
||||
two consecutive elements of the `tool_calls` array — a **call** then an **outcome**.
|
||||
The outcome shape is era-dependent:
|
||||
|
||||
```text
|
||||
index 0: { "toolName": "getPage", "input": { "pageId": "…" } } ← tool-call (has input, NO output)
|
||||
index 1: { "toolName": "getPage", "output": { … } } ← tool-result (has output, NO input)
|
||||
# v2 (#490) — metadata.toolTraceVersion = 2
|
||||
index 0: { "toolName":"getPage", "input":{...} } ← call (has input)
|
||||
index 1: { "toolName":"getPage", "ok":true } ← success (NO output here)
|
||||
or : { "toolName":"getPage", "error":"…", "kind":"thrown" } ← threw
|
||||
or : { "toolName":"getPage", "error":"…", "kind":"interrupted" } ← aborted mid-step
|
||||
|
||||
# legacy — no toolTraceVersion
|
||||
index 0: { "toolName":"getPage", "input":{...} } ← call (has input, NO output)
|
||||
index 1: { "toolName":"getPage", "output":{...} } ← success (has output)
|
||||
or : { "toolName":"getPage", "error":"…" } ← threw (post-#407 only)
|
||||
```
|
||||
|
||||
The keys that appear on an element are `toolName`, `input`, `output`, and — for a
|
||||
**thrown** failure on rows written after the #407 fix — `error` (the tool's error
|
||||
message; see the "Hard failures" section below). There is no `state`, no `errorText`,
|
||||
no `type`. On pre-#407 rows a thrown failure has NO paired result element at all
|
||||
(silent orphan). Consequences:
|
||||
The keys that can appear: `toolName`, `input` (call), and on the outcome — **v2:**
|
||||
`ok` **or** `error`+`kind`; **legacy:** `output` **or** (post-#407) `error`. There is
|
||||
no `state`, no `errorText`, no `type` in `tool_calls` (those live on `metadata.parts`).
|
||||
Consequences:
|
||||
|
||||
1. **Real invocation count = elements that have `output` or `error`.** Counting every
|
||||
element double-counts (you get ~2× and a spurious "~50% of every tool has no output").
|
||||
2. **Pairing:** a call = a `tool-call` part followed by its result part. A success
|
||||
carries `output`; a thrown failure (post-#407) carries `error` instead. Both carry
|
||||
`toolName`, so you can group by tool on either.
|
||||
1. **Real invocation count** — count the OUTCOME elements, not every element (else you
|
||||
double-count): **v2** = elements with `ok` or `error`; **legacy** = elements with
|
||||
`output` or `error`.
|
||||
2. **Pairing:** a call (`input`) is followed by its outcome. `toolName` is on both, so
|
||||
you can group by tool on either. In v2 the `kind` field separates a real hard-fail
|
||||
(`thrown`) from an aborted call (`interrupted`) — a distinction legacy rows cannot
|
||||
make (both are orphans; see below).
|
||||
3. **The tool OUTPUT is only in `metadata.parts` on v2 rows.** To inspect what a tool
|
||||
returned (soft-error markers, page bodies) on a v2 row, read the parts
|
||||
(`part->>'type' LIKE 'tool-%'`, `part->>'state' = 'output-available'`, `part->'output'`),
|
||||
not `tool_calls`.
|
||||
|
||||
## The two classes of failure (and which the DB can see)
|
||||
|
||||
### 1. Soft failures — tool RAN and returned an error-shaped result → PERSISTED ✅
|
||||
|
||||
These are visible in the `tool-result` `output`. The marker differs per tool:
|
||||
These are visible in the tool `output` — **on v2 rows in `metadata.parts`** (the
|
||||
`output-available` part's `output`), on **legacy rows in the `tool_calls` outcome
|
||||
element's `output`**. The marker differs per tool:
|
||||
|
||||
| Tool(s) | Error marker in `output` |
|
||||
| --- | --- |
|
||||
@@ -91,37 +141,32 @@ These are visible in the `tool-result` `output`. The marker differs per tool:
|
||||
Note `editPageText` returns `failed: []` on success — filtering on the *presence*
|
||||
of the key gives false positives; filter on **non-empty**.
|
||||
|
||||
### 2. Hard failures — tool THREW → NOW PERSISTED ✅ (since the #407 fix)
|
||||
### 2. Hard failures — tool THREW → PERSISTED ✅
|
||||
|
||||
When a tool throws (the classic one is `patchNode` / `insertNode` / `tableUpdateCell`
|
||||
→ `Failed to encode document to Yjs (fromJSON): Unknown node type: undefined`), the
|
||||
runtime still writes **no `tool-result` part** — the failure is an ai@6 `tool-error`
|
||||
content part instead. **Since the #407 fix, that error is persisted**: `serializeSteps`
|
||||
appends a dedicated element `{toolName, error: "<message>"}` right after the failed
|
||||
call, mirroring how a successful `{toolName, output}` element is appended. So a thrown
|
||||
error now leaves a queryable `error` field carrying its (truncated) reason, and the
|
||||
same real text is replayed to the model on the next turn (an `output-error` part with
|
||||
the real `errorText`, no longer the `'Tool call did not complete.'` placeholder).
|
||||
runtime writes **no `tool-result` part** — the failure is an ai@6 `tool-error` content
|
||||
part. How that lands in `tool_calls` depends on the era:
|
||||
|
||||
**Cutover caveat — old rows keep the old blind shape.** Rows written **before** this
|
||||
change have the two-part shape (`call` + `output` only) and simply **drop** thrown
|
||||
errors, leaving a silent **orphan** (a `call` with no `output` *and* no `error`). Rows
|
||||
written **after** the fix additionally carry the `error` element. So:
|
||||
- **v2 (#490):** a `{toolName, error, kind:'thrown'}` outcome element. An interrupted /
|
||||
aborted mid-step call is a **distinct** `{toolName, error:'Tool call did not
|
||||
complete.', kind:'interrupted'}` element — so you can tell a real hard-fail from an
|
||||
abort **directly, without the orphan heuristic**. Query `kind = 'thrown'`.
|
||||
- **post-#407 legacy:** a `{toolName, error}` element (no `kind`) right after the call.
|
||||
- **pre-#407 legacy:** the error is **dropped** — a silent **orphan** (a `call` with no
|
||||
`output` *and* no `error`).
|
||||
|
||||
- **New rows:** query the `error` field directly (see the hard-error query below) — no
|
||||
orphan heuristic needed for thrown failures.
|
||||
- **Old rows (pre-#407):** the only DB-side proxy is still an **orphan**: a `tool-call`
|
||||
part with no matching `tool-result` *and* no `error`. Orphans also appear when a run
|
||||
is **aborted** mid-flight (server restart), so a high-volume tool (`createComment`,
|
||||
`searchInPage`, `Search_web_search`) shows orphans from aborts, not real errors on
|
||||
old rows. Treat the orphan gap as an *upper bound*, and cross-check the tool: a gap on
|
||||
a structural editor (`patchNode`, `insertNode`, `updatePageJson`, `transformPage`) is
|
||||
almost certainly a thrown Yjs-encode error; a gap on `createComment` is mostly aborts.
|
||||
The same real error text is replayed to the model on the next turn (an `output-error`
|
||||
part with the real `errorText`, from `metadata.parts`), in every era.
|
||||
|
||||
A note on the aborted-call fallback: a call with **neither** a result **nor** a
|
||||
`tool-error` (genuinely interrupted mid-step) still replays with the
|
||||
`'Tool call did not complete.'` placeholder and persists as an orphan — that path is
|
||||
unchanged, and is distinct from a real thrown error, which now carries `error`.
|
||||
**Cutover caveat.** Only pre-#407 legacy rows need the orphan proxy: an orphan is a
|
||||
`tool-call` with no matching outcome. Orphans there also appear when a run is **aborted**
|
||||
mid-flight (server restart), so a high-volume tool (`createComment`, `searchInPage`,
|
||||
`Search_web_search`) shows orphans from aborts, not real errors. Treat the orphan gap as
|
||||
an *upper bound* and cross-check the tool: a gap on a structural editor (`patchNode`,
|
||||
`insertNode`, `updatePageJson`, `transformPage`) is almost certainly a thrown Yjs-encode
|
||||
error; a gap on `createComment` is mostly aborts. **On v2 rows this ambiguity is gone**
|
||||
— `kind` labels each outcome.
|
||||
|
||||
### 3. Run-level failures → `ai_chat_runs`
|
||||
|
||||
@@ -134,22 +179,34 @@ the wild: `Run interrupted by a server restart.` (aborts) and
|
||||
|
||||
Run all of these via `docker exec gitmost-postgresql psql -U docmost -d docmost -P pager=off -c "…"`.
|
||||
|
||||
**Real invocation count per tool** (result parts only — the correct denominator):
|
||||
**Real invocation count per tool** (outcome parts only — the correct denominator).
|
||||
Dual-shape: a v2 outcome has `ok` or `error`; a legacy outcome has `output` or `error`:
|
||||
|
||||
```sql
|
||||
SELECT elem->>'toolName' AS tool, count(*) AS calls
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||
WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'output'
|
||||
WHERE jsonb_typeof(m.tool_calls) = 'array'
|
||||
AND (elem ? 'ok' OR elem ? 'output' OR elem ? 'error')
|
||||
GROUP BY 1 ORDER BY 2 DESC;
|
||||
```
|
||||
|
||||
**Soft errors per tool** (everything the DB can honestly see):
|
||||
**Soft errors per tool.** The soft-error marker lives in the tool OUTPUT — which on
|
||||
**v2 rows is in `metadata.parts`**, on **legacy rows is in the `tool_calls` outcome
|
||||
element**. This query UNIONs both eras, projecting each output as `o`:
|
||||
|
||||
```sql
|
||||
WITH res AS (
|
||||
-- v2 (#490): output is in metadata.parts (output-available tool parts)
|
||||
SELECT part->>'type' AS tool, part->'output' AS o
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.metadata->'parts') part
|
||||
WHERE (m.metadata->>'toolTraceVersion') = '2'
|
||||
AND part->>'type' LIKE 'tool-%' AND part->>'state' = 'output-available'
|
||||
UNION ALL
|
||||
-- legacy: output is inline in the tool_calls outcome element
|
||||
SELECT elem->>'toolName' AS tool, elem->'output' AS o
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||
WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'output'
|
||||
WHERE (m.metadata->>'toolTraceVersion') IS NULL
|
||||
AND jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'output'
|
||||
)
|
||||
SELECT tool, count(*) AS calls,
|
||||
sum(COALESCE(
|
||||
@@ -167,13 +224,23 @@ FROM res GROUP BY tool HAVING sum(COALESCE(
|
||||
ORDER BY soft_errors DESC;
|
||||
```
|
||||
|
||||
**`editPageText` failure reasons** (the most common real agent mistake — bad `find`):
|
||||
Note the v2 `tool` label is the part type (`tool-editPageText`); strip the `tool-`
|
||||
prefix if you join it against the legacy `toolName`.
|
||||
|
||||
**`editPageText` failure reasons** (the most common real agent mistake — bad `find`).
|
||||
Same dual-shape output source:
|
||||
|
||||
```sql
|
||||
WITH res AS (
|
||||
SELECT part->'output' AS o
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.metadata->'parts') part
|
||||
WHERE (m.metadata->>'toolTraceVersion') = '2'
|
||||
AND part->>'type' = 'tool-editPageText' AND part->>'state' = 'output-available'
|
||||
UNION ALL
|
||||
SELECT elem->'output' AS o
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||
WHERE jsonb_typeof(m.tool_calls) = 'array'
|
||||
WHERE (m.metadata->>'toolTraceVersion') IS NULL
|
||||
AND jsonb_typeof(m.tool_calls) = 'array'
|
||||
AND elem->>'toolName' = 'editPageText' AND elem ? 'output'
|
||||
)
|
||||
SELECT f->>'reason' AS reason, count(*)
|
||||
@@ -182,30 +249,43 @@ WHERE jsonb_typeof(o->'failed') = 'array'
|
||||
GROUP BY 1 ORDER BY 2 DESC;
|
||||
```
|
||||
|
||||
**Hard errors — persisted `error` field per tool (NEW rows, since #407)** — thrown
|
||||
tool failures now carry their real reason, so query them directly:
|
||||
**Hard errors — persisted `error` field per tool (v2 + post-#407 rows)** — thrown tool
|
||||
failures carry their real reason, so query them directly. On **v2** rows exclude the
|
||||
`interrupted` kind so an aborted call is not counted as a hard-fail:
|
||||
|
||||
```sql
|
||||
SELECT elem->>'toolName' AS tool, count(*) AS thrown_errors,
|
||||
min(elem->>'error') AS sample_error
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||
WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'error'
|
||||
-- v2 rows label the kind; a legacy error element has no kind (count it).
|
||||
AND COALESCE(elem->>'kind', 'thrown') = 'thrown'
|
||||
GROUP BY 1 ORDER BY 2 DESC;
|
||||
```
|
||||
|
||||
Aborted mid-step calls on v2 rows are a distinct, directly countable population:
|
||||
|
||||
```sql
|
||||
SELECT elem->>'toolName' AS tool, count(*) AS interrupted
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||
WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem->>'kind' = 'interrupted'
|
||||
GROUP BY 1 ORDER BY 2 DESC;
|
||||
```
|
||||
|
||||
**Hard-error proxy for OLD rows (pre-#407) — orphan gap per tool, WITH a spread column**
|
||||
(call parts minus result parts, plus how many distinct chats the gap is spread across).
|
||||
This covers rows written before thrown errors were persisted; on new rows a thrown
|
||||
failure now has its own `error` element (use the query above) and an orphan means only
|
||||
a genuinely aborted mid-step call:
|
||||
(call parts minus outcome parts, plus how many distinct chats the gap is spread across).
|
||||
This is needed ONLY for pre-#407 legacy rows (v2 and post-#407 rows carry the error /
|
||||
`kind` directly — use the queries above). The `WHERE` restricts to the legacy era so v2
|
||||
rows (where an `ok` outcome is not an `output`) never produce phantom orphans:
|
||||
|
||||
```sql
|
||||
WITH parts AS (
|
||||
SELECT m.chat_id, elem->>'toolName' AS tool,
|
||||
(elem ? 'input' AND NOT (elem ? 'output')) AS is_call,
|
||||
(elem ? 'output' OR elem ? 'error') AS is_result
|
||||
(elem ? 'input' AND NOT (elem ? 'output') AND NOT (elem ? 'ok')) AS is_call,
|
||||
(elem ? 'output' OR elem ? 'error' OR elem ? 'ok') AS is_result
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||
WHERE jsonb_typeof(m.tool_calls) = 'array' AND m.role = 'assistant'
|
||||
AND (m.metadata->>'toolTraceVersion') IS NULL
|
||||
),
|
||||
per_chat AS (
|
||||
SELECT tool, chat_id, sum(is_call::int) - sum(is_result::int) AS gap
|
||||
@@ -261,11 +341,21 @@ WHERE tsv @@ websearch_to_tsquery('english', 'some phrase') LIMIT 20;
|
||||
|
||||
## Don't blow up your context
|
||||
|
||||
A single `tool_calls` row can be **300–400 KB** (results embed full page content and
|
||||
search payloads). Never `SELECT tool_calls` (or `jsonb_pretty(tool_calls)`) raw.
|
||||
Always project just the keys you need and truncate:
|
||||
Tool outputs embed full page content and search payloads (hundreds of KB per row).
|
||||
On **legacy** rows they are in `tool_calls`; on **v2** rows they moved to
|
||||
`metadata->'parts'` (the `tool_calls` trace itself is now small). Never `SELECT
|
||||
tool_calls` / `metadata` (or `jsonb_pretty(...)`) raw — project just the keys you need
|
||||
and truncate:
|
||||
|
||||
```sql
|
||||
-- v2: outputs live in metadata.parts
|
||||
SELECT part->>'type',
|
||||
left(regexp_replace((part->'output')::text, '\s+', ' ', 'g'), 200)
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.metadata->'parts') part
|
||||
WHERE (m.metadata->>'toolTraceVersion') = '2'
|
||||
AND part->>'state' = 'output-available' LIMIT 5;
|
||||
|
||||
-- legacy: outputs live in tool_calls
|
||||
SELECT elem->>'toolName',
|
||||
left(regexp_replace((elem->'output')::text, '\s+', ' ', 'g'), 200)
|
||||
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||
@@ -280,26 +370,32 @@ docker compose -p gitmost logs -f --tail=100 # whole stack
|
||||
```
|
||||
|
||||
Logging is `json-file`, `max-size=10m max-file=5` → ~50 MB retained, then rotated,
|
||||
and **wiped on container recreate**. Since the #407 fix, thrown-tool error text is
|
||||
**persisted in the `error` field** of `tool_calls` (see the hard-error query above), so
|
||||
you no longer depend on live logs for it. Logs/live UI remain useful for **pre-#407
|
||||
rows** (whose thrown errors were dropped) and for full stack traces beyond the
|
||||
truncated stored message. A per-tool `tool_calls_total{tool,status}` metric to
|
||||
VictoriaMetrics is still a possible future add for aggregate dashboards.
|
||||
and **wiped on container recreate**. Thrown-tool error text is **persisted** — in the
|
||||
`error` field of `tool_calls` (v2 `kind:'thrown'` / post-#407 legacy) — so you no longer
|
||||
depend on live logs for it. Logs/live UI remain useful for **pre-#407 rows** (whose
|
||||
thrown errors were dropped) and for full stack traces beyond the truncated stored
|
||||
message. A per-tool `tool_calls_total{tool,status}` metric to VictoriaMetrics is still a
|
||||
possible future add for aggregate dashboards.
|
||||
|
||||
## Gotchas checklist
|
||||
|
||||
- [ ] Counting every `tool_calls` element → **overcount**. Count `output` elements; add `error` elements for thrown failures (new rows), but don't count both as invocations.
|
||||
- [ ] `isError` / `success=false` ≈ 0 does **not** mean "no errors" — thrown errors are a separate `error` element (new rows) or dropped entirely (pre-#407 rows).
|
||||
- [ ] Thrown errors persist only on rows written **after the #407 fix** — pre-#407 rows still drop them (orphan only). Mind the cutover when trending over time.
|
||||
- [ ] **Check `metadata.toolTraceVersion` first.** v2 (`= 2`) has no output in `tool_calls`; legacy has it inline. Never trend a metric across the era boundary.
|
||||
- [ ] Counting every `tool_calls` element → **overcount**. Count OUTCOME elements — v2: `ok` or `error`; legacy: `output` or `error` — never both call+outcome as invocations.
|
||||
- [ ] `isError` / `success=false` ≈ 0 does **not** mean "no errors" — thrown errors are an `error` element (v2 `kind:'thrown'` / post-#407), not in the output.
|
||||
- [ ] **v2:** soft-error markers (the tool output) are in `metadata.parts`, NOT `tool_calls`. Legacy: they are in the `tool_calls` outcome `output`.
|
||||
- [ ] **v2:** `kind` splits a real hard-fail (`thrown`) from an aborted call (`interrupted`) directly — no orphan heuristic needed. The orphan gap is a pre-#407-legacy-only proxy.
|
||||
- [ ] `editPageText.failed` is `[]` on success — test for **non-empty**, not presence.
|
||||
- [ ] Orphan gap on OLD rows mixes thrown errors **and** aborted runs — split by tool. On NEW rows a thrown error is its own `error` element, so a gap ≈ aborted call.
|
||||
- [ ] `aborted` runs = server restarts, `failed` runs = provider overload — not agent mistakes.
|
||||
- [ ] Never dump a raw `tool_calls` cell — it can be hundreds of KB.
|
||||
- [ ] Logs are ephemeral (≤50 MB, wiped on recreate) — grab hard-error text live.
|
||||
- [ ] Never dump a raw `tool_calls` **or** `metadata.parts` cell — outputs are hundreds of KB.
|
||||
- [ ] Logs are ephemeral (≤50 MB, wiped on recreate) — grab pre-#407 hard-error text live.
|
||||
|
||||
## Snapshot (2026-07-07, illustrative — rerun the queries for current numbers)
|
||||
|
||||
> All rows in this snapshot predate #490, so they are **legacy-era** (outputs inline in
|
||||
> `tool_calls`, orphan proxy for thrown errors). Do not trend these numbers against v2
|
||||
> rows — segment by `toolTraceVersion` first.
|
||||
|
||||
|
||||
- 226 chats, 732 messages, 46 runs; ~4 400 real tool invocations.
|
||||
- Soft errors (persisted): `editPageText` 4/79 (bad/non-unique `find`) + 9 markdown-in-`find` warnings; `semanticSearch` 3/4 (`unavailable`); `Habr_update_draft_from_docmost` 1/2 (`doc` sent as object, not string).
|
||||
- Missing-result proxy, read WITH the spread column:
|
||||
|
||||
@@ -59,6 +59,39 @@ import {
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
|
||||
// Max concurrent per-page comment fetches in checkNewComments (#490). The scan is
|
||||
// O(N) independent REST reads over the working set; running them one-at-a-time made
|
||||
// a large space linear in round-trips. A small cap parallelizes without hammering
|
||||
// the server (or exhausting sockets). 6 is a conservative middle of the 5–8 band.
|
||||
const COMMENT_SCAN_CONCURRENCY = 6;
|
||||
|
||||
/**
|
||||
* Map `items` through `fn` with at most `limit` in flight, preserving INPUT ORDER
|
||||
* in the returned array. A tiny bounded pool (no p-limit dependency): `limit`
|
||||
* workers pull the next index off a shared cursor until the list is drained.
|
||||
*/
|
||||
async function mapWithConcurrency<T, R>(
|
||||
items: readonly T[],
|
||||
limit: number,
|
||||
fn: (item: T, index: number) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const results = new Array<R>(items.length);
|
||||
let cursor = 0;
|
||||
const worker = async (): Promise<void> => {
|
||||
for (;;) {
|
||||
const i = cursor++;
|
||||
if (i >= items.length) return;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
};
|
||||
const workers = Array.from(
|
||||
{ length: Math.max(1, Math.min(limit, items.length)) },
|
||||
() => worker(),
|
||||
);
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
// Public method surface of CommentsMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
@@ -655,27 +688,32 @@ export function CommentsMixin<TBase extends GConstructor<DocmostClientContext>>(
|
||||
parentPageId,
|
||||
);
|
||||
|
||||
// 2. Fetch comments for each page, keep ones created after since
|
||||
const results: any[] = [];
|
||||
for (const page of pagesInScope) {
|
||||
try {
|
||||
// Full feed (incl. resolved): a "new comments since" scan reports all
|
||||
// recent activity; the active-only filter is scoped to listComments.
|
||||
const comments = (await this.listComments(page.id, true)).items;
|
||||
const newComments = comments.filter(
|
||||
(c: any) => new Date(c.createdAt) > sinceDate,
|
||||
);
|
||||
if (newComments.length > 0) {
|
||||
results.push({
|
||||
pageId: page.id,
|
||||
pageTitle: page.title,
|
||||
comments: newComments,
|
||||
});
|
||||
// 2. Fetch comments for each page, keep ones created after since. Runs with
|
||||
// bounded concurrency (#490) instead of one-at-a-time — the per-page reads are
|
||||
// independent, so a large working set no longer costs O(N) serial round-trips.
|
||||
// Order is preserved (mapWithConcurrency keeps input order), so the output is
|
||||
// deterministic regardless of which fetch finishes first.
|
||||
const perPage = await mapWithConcurrency(
|
||||
pagesInScope,
|
||||
COMMENT_SCAN_CONCURRENCY,
|
||||
async (page: any) => {
|
||||
try {
|
||||
// Full feed (incl. resolved): a "new comments since" scan reports all
|
||||
// recent activity; the active-only filter is scoped to listComments.
|
||||
const comments = (await this.listComments(page.id, true)).items;
|
||||
const newComments = comments.filter(
|
||||
(c: any) => new Date(c.createdAt) > sinceDate,
|
||||
);
|
||||
return newComments.length > 0
|
||||
? { pageId: page.id, pageTitle: page.title, comments: newComments }
|
||||
: null;
|
||||
} catch (e: any) {
|
||||
// Skip pages with errors (e.g. deleted between calls)
|
||||
return null;
|
||||
}
|
||||
} catch (e: any) {
|
||||
// Skip pages with errors (e.g. deleted between calls)
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
const results: any[] = perPage.filter((r): r is any => r !== null);
|
||||
|
||||
const totalNewComments = results.reduce(
|
||||
(sum, r) => sum + r.comments.length,
|
||||
|
||||
@@ -170,6 +170,43 @@ export abstract class DocmostClientContext {
|
||||
// cached conversion can never leak across identities. See getpage-cache.ts.
|
||||
protected getPageCache = new GetPageConversionCache();
|
||||
|
||||
// #487: an OPTIONAL abort signal the in-app tool host sets before each tool
|
||||
// call (a composite of the turn's Stop signal + a per-call wall-clock cap). It
|
||||
// is checked at safe-points BETWEEN the sequential HTTP calls of a paginated
|
||||
// read (paginateAll) and just before the atomic collab commit of a write (the
|
||||
// mutatePage/replacePage/mutateLiveContentUnlocked seams), so a Stop / cap
|
||||
// stops the NEXT network call from STARTING. An already-started single call may
|
||||
// still land — a documented limitation (#487).
|
||||
//
|
||||
// SINGLE-WRITER by phase-1 assumption: exactly one DocmostClient is built per
|
||||
// turn and shared by every tool call; the host sets this per call and does NOT
|
||||
// restore the prior value on unwind (set-and-leave) — a fresh client per turn
|
||||
// plus overwrite-by-the-next-call keeps it correct, and leaving a settled
|
||||
// call's signal in place is what makes a discarded race-loser throw on its
|
||||
// next safe-point. If the model emits PARALLEL in-app
|
||||
// tool calls they share this one field, so the per-call CAP of one call is not
|
||||
// guaranteed to bound another's in-flight pagination — but every composite the
|
||||
// host sets carries the SAME turn Stop signal, so a Stop still aborts whichever
|
||||
// signal is current. #487.
|
||||
protected toolAbortSignal: AbortSignal | null = null;
|
||||
|
||||
/**
|
||||
* #487: set (or clear with null) the in-app tool abort signal governing the
|
||||
* NEXT client call's safe-points. The host wraps each in-app tool call: it sets
|
||||
* the composite (Stop + per-call cap) here before invoking the tool and leaves
|
||||
* it in place afterwards (set-and-leave, NOT restored) — the next call
|
||||
* overwrites it, and a fresh client is built per turn. Public so the
|
||||
* server-side tool wrapper can reach it; harmless (a no-op) when never set.
|
||||
*/
|
||||
public setToolAbortSignal(signal: AbortSignal | null): void {
|
||||
this.toolAbortSignal = signal;
|
||||
}
|
||||
|
||||
/** #487: the abort signal currently governing this client's safe-points. */
|
||||
public getToolAbortSignal(): AbortSignal | null {
|
||||
return this.toolAbortSignal;
|
||||
}
|
||||
|
||||
// Two construction forms:
|
||||
// - new DocmostClient(config) // discriminated union (current)
|
||||
// - new DocmostClient(baseURL, email, password) // legacy positional creds
|
||||
@@ -571,6 +608,10 @@ export abstract class DocmostClientContext {
|
||||
this.onMetricFn?.("collab_connect_timeouts_total", 1),
|
||||
});
|
||||
try {
|
||||
// #487 PRE-COMMIT safe-point (reentrant twin of mutatePageContent): a
|
||||
// Stop/cap after acquiring the session but before the atomic write skips
|
||||
// this commit. Same limitation applies (stops the NEXT commit only).
|
||||
this.toolAbortSignal?.throwIfAborted();
|
||||
return await session.mutate(transform);
|
||||
} catch (e) {
|
||||
// Drop the session on any failure so the next call reconnects fresh.
|
||||
@@ -602,6 +643,11 @@ export abstract class DocmostClientContext {
|
||||
let truncated = false;
|
||||
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
// #487 safe-point: a Stop (or the in-app tool per-call cap) that fires
|
||||
// BETWEEN sequential page fetches must stop the NEXT request from starting
|
||||
// — a read tool that would otherwise paginate for minutes is interrupted
|
||||
// here. throwIfAborted() rejects with the signal's reason.
|
||||
this.toolAbortSignal?.throwIfAborted();
|
||||
const payload: Record<string, any> = {
|
||||
...basePayload,
|
||||
limit: clampedLimit,
|
||||
@@ -709,7 +755,15 @@ export abstract class DocmostClientContext {
|
||||
// #486: on a rejected collab-WS handshake, invalidate + refresh the token and
|
||||
// retry the write once (symmetric to the HTTP-401 reauth path).
|
||||
return this.writeWithCollabAuthRetry(collabToken, (token) =>
|
||||
mutatePageContent(pageUuid, token, apiUrl, transform),
|
||||
// #487: thread the in-app tool signal to mutatePageContent's pre-commit
|
||||
// safe-point so a Stop/cap during the connect/lock window skips the write.
|
||||
mutatePageContent(
|
||||
pageUuid,
|
||||
token,
|
||||
apiUrl,
|
||||
transform,
|
||||
this.toolAbortSignal ?? undefined,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -733,7 +787,14 @@ export abstract class DocmostClientContext {
|
||||
// #486: on a rejected collab-WS handshake, invalidate + refresh the token and
|
||||
// retry the write once (symmetric to the HTTP-401 reauth path).
|
||||
return this.writeWithCollabAuthRetry(collabToken, (token) =>
|
||||
replacePageContent(pageUuid, doc, token, apiUrl),
|
||||
// #487: same pre-commit safe-point as mutatePage, for full-document writes.
|
||||
replacePageContent(
|
||||
pageUuid,
|
||||
doc,
|
||||
token,
|
||||
apiUrl,
|
||||
this.toolAbortSignal ?? undefined,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -30,6 +30,13 @@ export { destroyAllSessions } from "./lib/collab-session.js";
|
||||
// internals directly; it goes through loadDocmostMcp()).
|
||||
export { SHARED_TOOL_SPECS } from "./tool-specs.js";
|
||||
export type { SharedToolSpec } from "./tool-specs.js";
|
||||
// #489 — write-class registry consumed by the in-app external-MCP retry gate.
|
||||
export {
|
||||
SHARED_TOOL_WRITE_CLASS,
|
||||
isRetryableWriteClass,
|
||||
assertEverySpecDeclaresWriteClass,
|
||||
} from "./tool-specs.js";
|
||||
export type { ToolWriteClass } from "./tool-specs.js";
|
||||
|
||||
// Re-export the build-time REGISTRY_STAMP (issue #447): a deterministic hash of
|
||||
// the tool-specs registry content, generated into src/registry-stamp.generated.ts
|
||||
|
||||
@@ -254,6 +254,12 @@ export async function mutatePageContent(
|
||||
collabToken: string,
|
||||
baseUrl: string,
|
||||
transform: (liveDoc: any) => any | null,
|
||||
// #487: optional abort signal carrying the turn's Stop + the in-app tool
|
||||
// per-call cap. Checked as the PRE-COMMIT safe-point below (after the session
|
||||
// is acquired, immediately before the atomic read->write), so a Stop that
|
||||
// arrives during the connect/lock window stops THIS write from landing. See the
|
||||
// limitation note at the check.
|
||||
signal?: AbortSignal,
|
||||
): Promise<MutationResult> {
|
||||
return withPageLock(pageId, async () => {
|
||||
if (process.env.DEBUG) {
|
||||
@@ -266,6 +272,13 @@ export async function mutatePageContent(
|
||||
|
||||
const session = await acquireCollabSession(pageId, collabToken, baseUrl);
|
||||
try {
|
||||
// #487 PRE-COMMIT safe-point: if the turn was Stopped (or the in-app tool
|
||||
// per-call cap fired) after we acquired the collab session but before the
|
||||
// atomic write, throw NOW so this commit never runs. KNOWN LIMITATION
|
||||
// (#487): this only stops THIS commit — a write tool that already committed
|
||||
// an EARLIER call this turn leaves that op applied. Cancel guarantees "no
|
||||
// NEW commit starts", NOT "the write didn't land".
|
||||
signal?.throwIfAborted();
|
||||
return await session.mutate(transform);
|
||||
} catch (e) {
|
||||
// Drop the session on any failure so the next call reconnects fresh (this
|
||||
@@ -291,6 +304,8 @@ export async function replacePageContent(
|
||||
prosemirrorDoc: any,
|
||||
collabToken: string,
|
||||
baseUrl: string,
|
||||
// #487: threaded straight to mutatePageContent's pre-commit safe-point.
|
||||
signal?: AbortSignal,
|
||||
): Promise<MutationResult> {
|
||||
// Fail fast on a bad document instead of deferring the failure into the
|
||||
// collaboration write (where TiptapTransformer.toYdoc(undefined) used to
|
||||
@@ -307,6 +322,7 @@ export async function replacePageContent(
|
||||
collabToken,
|
||||
baseUrl,
|
||||
() => prosemirrorDoc,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -127,6 +127,18 @@ export interface SharedToolSpec {
|
||||
mcpName: string;
|
||||
/** camelCase key in the ai-SDK tools object (the in-app layer). */
|
||||
inAppKey: string;
|
||||
/**
|
||||
* Write-class of the tool (#489), declared on EVERY spec (a registration-time
|
||||
* assert enforces completeness; `satisfies Record<string, SharedToolSpec>`
|
||||
* makes it a compile error to omit). 'readOnly' = a pure read that mutates
|
||||
* NOTHING durable, so it is safe to auto-retry once after a transport break.
|
||||
* 'write' = anything that mutates a page/comment/share/diagram/etc — a
|
||||
* transport error is INDETERMINATE (the server may have applied it before the
|
||||
* connection reset), so it is NEVER blind-retried (a retry would double-apply,
|
||||
* the #435 incident class). Consumed by the external-MCP retry path
|
||||
* (mcp-clients.service.ts) to gate its single auto-retry.
|
||||
*/
|
||||
writeClass: 'readOnly' | 'write';
|
||||
/** Single canonical model-facing description used by both layers. */
|
||||
description: string;
|
||||
/**
|
||||
@@ -240,6 +252,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
getWorkspace: {
|
||||
mcpName: 'getWorkspace',
|
||||
inAppKey: 'getWorkspace',
|
||||
writeClass: 'readOnly',
|
||||
description: 'Fetch metadata about the current workspace (name, settings).',
|
||||
tier: 'core',
|
||||
catalogLine: 'getWorkspace — fetch current workspace metadata (name, settings).',
|
||||
@@ -249,6 +262,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
listSpaces: {
|
||||
mcpName: 'listSpaces',
|
||||
inAppKey: 'listSpaces',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'List the spaces the current user can access. Returns the array of ' +
|
||||
'spaces (id, name, slug, ...).',
|
||||
@@ -260,6 +274,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
listShares: {
|
||||
mcpName: 'listShares',
|
||||
inAppKey: 'listShares',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'List all public shares in the workspace with page titles and public URLs.',
|
||||
tier: 'deferred',
|
||||
@@ -272,6 +287,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
getPageJson: {
|
||||
mcpName: 'getPageJson',
|
||||
inAppKey: 'getPageJson',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Get page details with the raw ProseMirror JSON content (lossless: ' +
|
||||
'includes block ids, callouts, tables, link/image attributes) plus the ' +
|
||||
@@ -289,6 +305,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
getOutline: {
|
||||
mcpName: 'getOutline',
|
||||
inAppKey: 'getOutline',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
"Return a COMPACT outline of a page's top-level blocks ({index, type, " +
|
||||
'id, level, firstText}; tables add rows/cols/header; lists add item ' +
|
||||
@@ -309,6 +326,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
getNode: {
|
||||
mcpName: 'getNode',
|
||||
inAppKey: 'getNode',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
"Fetch a single block for editing. `nodeId` is a block id from the page " +
|
||||
'outline or page-JSON view (works for headings/paragraphs/callouts/images), OR ' +
|
||||
@@ -350,6 +368,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
searchInPage: {
|
||||
mcpName: 'searchInPage',
|
||||
inAppKey: 'searchInPage',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Find every occurrence of a string (or regex) INSIDE one page and get ' +
|
||||
'WHERE each is — instead of pulling blocks one-by-one with getNode. ' +
|
||||
@@ -413,6 +432,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
deleteNode: {
|
||||
mcpName: 'deleteNode',
|
||||
inAppKey: 'deleteNode',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Remove a single block by its attrs.id (from the page outline or ' +
|
||||
'page-JSON view) WITHOUT resending the whole document.',
|
||||
@@ -438,6 +458,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
patchNode: {
|
||||
mcpName: 'patchNode',
|
||||
inAppKey: 'patchNode',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Replace a single content block identified by its attrs.id, WITHOUT ' +
|
||||
'resending the whole document; the replacement keeps the same block id. ' +
|
||||
@@ -505,6 +526,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
insertNode: {
|
||||
mcpName: 'insertNode',
|
||||
inAppKey: 'insertNode',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Insert content before/after another block (by attrs.id or anchor text) ' +
|
||||
'or append it at the end (top level). For before/after you MUST provide ' +
|
||||
@@ -597,6 +619,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
sharePage: {
|
||||
mcpName: 'sharePage',
|
||||
inAppKey: 'sharePage',
|
||||
writeClass: 'write',
|
||||
// CANONICAL: merges the MCP copy's URL-format + idempotency detail with the
|
||||
// in-app copy's reversibility note; keeps the security framing both had.
|
||||
description:
|
||||
@@ -626,6 +649,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
unsharePage: {
|
||||
mcpName: 'unsharePage',
|
||||
inAppKey: 'unsharePage',
|
||||
writeClass: 'write',
|
||||
description: 'Remove the public share of a page (revokes the public URL).',
|
||||
tier: 'deferred',
|
||||
catalogLine: "unsharePage — revoke a page's public share (removes the public URL).",
|
||||
@@ -640,6 +664,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
diffPageVersions: {
|
||||
mcpName: 'diffPageVersions',
|
||||
inAppKey: 'diffPageVersions',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Diff two versions of a page and return a Docmost-equivalent change set ' +
|
||||
'(inserted/deleted text, integrity counts for images/links/tables/' +
|
||||
@@ -672,6 +697,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
listPageHistory: {
|
||||
mcpName: 'listPageHistory',
|
||||
inAppKey: 'listPageHistory',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
"List a page's saved versions (Docmost auto-snapshots on every save), " +
|
||||
'newest first, cursor-paginated. Returns { items, nextCursor }; each ' +
|
||||
@@ -693,6 +719,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
restorePageVersion: {
|
||||
mcpName: 'restorePageVersion',
|
||||
inAppKey: 'restorePageVersion',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Restore a page to a saved version: writes that version\'s content back ' +
|
||||
'as the page\'s current content (Docmost has no restore endpoint, so ' +
|
||||
@@ -713,6 +740,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
importPageMarkdown: {
|
||||
mcpName: 'importPageMarkdown',
|
||||
inAppKey: 'importPageMarkdown',
|
||||
writeClass: 'write',
|
||||
// IN-APP ONLY (issue #411): the external /mcp surface no longer exposes
|
||||
// importPageMarkdown — the registry loop in index.ts skips inAppOnly specs,
|
||||
// so this stays available to the in-app agent (round-tripping an EXPORTED
|
||||
@@ -742,6 +770,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
copyPageContent: {
|
||||
mcpName: 'copyPageContent',
|
||||
inAppKey: 'copyPageContent',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
"Replace targetPageId's content with a copy of sourcePageId's content, " +
|
||||
'entirely server-side — the document is NOT sent through the model. The ' +
|
||||
@@ -770,6 +799,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
editPageText: {
|
||||
mcpName: 'editPageText',
|
||||
inAppKey: 'editPageText',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
"Surgical find/replace inside a page's text, preserving all block " +
|
||||
'ids and marks. A find MAY cross bold/italic/link boundaries; the ' +
|
||||
@@ -819,6 +849,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
stashPage: {
|
||||
mcpName: 'stashPage',
|
||||
inAppKey: 'stashPage',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Serialize a whole page (the full ProseMirror JSON, as getPageJson ' +
|
||||
'returns) into an ephemeral in-memory blob and return ONLY a short ' +
|
||||
@@ -880,6 +911,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
getPage: {
|
||||
mcpName: 'getPage',
|
||||
inAppKey: 'getPage',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Fetch a single page as Markdown by its id. Returns the page title and ' +
|
||||
'its Markdown content. The converter is canonical (round-trips text and ' +
|
||||
@@ -919,6 +951,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
listPages: {
|
||||
mcpName: 'listPages',
|
||||
inAppKey: 'listPages',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'List the most recent pages (ordered by updatedAt, descending), ' +
|
||||
'optionally scoped to a single space. Returns a bounded list (default ' +
|
||||
@@ -965,6 +998,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
getTree: {
|
||||
mcpName: 'getTree',
|
||||
inAppKey: 'getTree',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
"Get a space's page hierarchy (or one subtree) as a nested tree in a " +
|
||||
'SINGLE request — completely and without loss. Each node is ' +
|
||||
@@ -1009,6 +1043,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
getPageContext: {
|
||||
mcpName: 'getPageContext',
|
||||
inAppKey: 'getPageContext',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Given a pageId, get its LOCATION and immediate surroundings (metadata ' +
|
||||
'only, no page content) in one call — answers "where am I / what is ' +
|
||||
@@ -1038,6 +1073,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
createPage: {
|
||||
mcpName: 'createPage',
|
||||
inAppKey: 'createPage',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Create a new page with a Markdown body in a space, optionally under a ' +
|
||||
'parent page (omit parentPageId to create at the space root). Returns ' +
|
||||
@@ -1088,6 +1124,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
movePage: {
|
||||
mcpName: 'movePage',
|
||||
inAppKey: 'movePage',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Move a page under a new parent page, or to the space root when no ' +
|
||||
'parent is given. Reversible: move it back at any time.',
|
||||
@@ -1181,6 +1218,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
renamePage: {
|
||||
mcpName: 'renamePage',
|
||||
inAppKey: 'renamePage',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Rename a page (change its title only; the body is untouched, never ' +
|
||||
'resent). Reversible: rename back at any time.',
|
||||
@@ -1202,6 +1240,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
deletePage: {
|
||||
mcpName: 'deletePage',
|
||||
inAppKey: 'deletePage',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Move a page to the trash — SOFT delete only: the page can be restored ' +
|
||||
'from trash and nothing is ever permanently deleted.',
|
||||
@@ -1235,6 +1274,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
updatePageJson: {
|
||||
mcpName: 'updatePageJson',
|
||||
inAppKey: 'updatePageJson',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
"Replace a page's content with a raw ProseMirror JSON document (lossless " +
|
||||
'write: preserves the block ids, callouts, tables and attributes you pass ' +
|
||||
@@ -1291,6 +1331,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
updatePageMarkdown: {
|
||||
mcpName: 'updatePageMarkdown',
|
||||
inAppKey: 'updatePageMarkdown',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
"Replace a page's body with new Markdown content (and optionally its " +
|
||||
'title). The whole body is re-imported from the markdown (block ids ' +
|
||||
@@ -1325,6 +1366,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
exportPageMarkdown: {
|
||||
mcpName: 'exportPageMarkdown',
|
||||
inAppKey: 'exportPageMarkdown',
|
||||
writeClass: 'readOnly',
|
||||
// CANONICAL: the MCP copy (a strict superset of the terse in-app wording).
|
||||
description:
|
||||
'Export a page to a single self-contained Docmost-flavoured Markdown ' +
|
||||
@@ -1373,6 +1415,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
createComment: {
|
||||
mcpName: 'createComment',
|
||||
inAppKey: 'createComment',
|
||||
writeClass: 'write',
|
||||
// CANONICAL: the in-app copy (the more-maintained one). It keeps the same
|
||||
// rules as the MCP copy — inline-only, top-level requires a `selection`, no
|
||||
// page-level comments, replies inherit the anchor, suggestedText must be
|
||||
@@ -1505,6 +1548,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
listComments: {
|
||||
mcpName: 'listComments',
|
||||
inAppKey: 'listComments',
|
||||
writeClass: 'readOnly',
|
||||
// CANONICAL: the two copies are near-identical; the MCP copy is the
|
||||
// superset (it keeps the "(pagination is handled internally)" note the
|
||||
// in-app copy dropped), so it is used verbatim.
|
||||
@@ -1532,6 +1576,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
resolveComment: {
|
||||
mcpName: 'resolveComment',
|
||||
inAppKey: 'resolveComment',
|
||||
writeClass: 'write',
|
||||
// CANONICAL: the MCP copy's richer wording, minus its reference
|
||||
// to `deleteComment` (a sibling tool that does NOT exist in the in-app
|
||||
// layer) — rephrased transport-neutrally per the registry convention.
|
||||
@@ -1572,6 +1617,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
checkNewComments: {
|
||||
mcpName: 'checkNewComments',
|
||||
inAppKey: 'checkNewComments',
|
||||
writeClass: 'readOnly',
|
||||
// CANONICAL: the MCP copy (the more detailed of the two). The MCP layer's
|
||||
// execute-side guard that rejects an unparseable `since` timestamp stays in
|
||||
// its execute body (per-layer logic), not in the shared schema.
|
||||
@@ -1651,6 +1697,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
tableInsertRow: {
|
||||
mcpName: 'tableInsertRow',
|
||||
inAppKey: 'tableInsertRow',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Insert a row of plain-text cells into a table. `table` is `#<index>` ' +
|
||||
'from the page outline, or a block id inside it. `cells` is the text per ' +
|
||||
@@ -1684,6 +1731,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
tableDeleteRow: {
|
||||
mcpName: 'tableDeleteRow',
|
||||
inAppKey: 'tableDeleteRow',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Delete the row at 0-based `index` from a table (`table` is `#<index>` ' +
|
||||
'from the page outline, or a block id inside it). Refuses to delete the ' +
|
||||
@@ -1707,6 +1755,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
tableUpdateCell: {
|
||||
mcpName: 'tableUpdateCell',
|
||||
inAppKey: 'tableUpdateCell',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Set the plain-text content of cell [row, col] (0-based) in a table ' +
|
||||
'(`table` is `#<index>` from the page outline, or a block id inside it). ' +
|
||||
@@ -1747,6 +1796,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
insertFootnote: {
|
||||
mcpName: 'insertFootnote',
|
||||
inAppKey: 'insertFootnote',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Insert an AUTHOR-INLINE footnote: you specify only WHERE (anchorText) ' +
|
||||
'and WHAT (text). The footnote marker is placed right after anchorText in ' +
|
||||
@@ -1784,6 +1834,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
insertImage: {
|
||||
mcpName: 'insertImage',
|
||||
inAppKey: 'insertImage',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Download an image from a web (http/https) URL and insert it into ' +
|
||||
'a page in one step. By default ' +
|
||||
@@ -1828,6 +1879,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
replaceImage: {
|
||||
mcpName: 'replaceImage',
|
||||
inAppKey: 'replaceImage',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Replace an existing image on a page with a new image fetched from a web ' +
|
||||
'(http/https) URL: uploads the new file as a NEW ' +
|
||||
@@ -1866,6 +1918,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
drawioGet: {
|
||||
mcpName: 'drawioGet',
|
||||
inAppKey: 'drawioGet',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Read a draw.io diagram on a page as mxGraph XML (default) or as its raw ' +
|
||||
'`.drawio.svg`. `node` is the drawio node\'s attrs.id (from getOutline / ' +
|
||||
@@ -1899,6 +1952,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
drawioCreate: {
|
||||
mcpName: 'drawioCreate',
|
||||
inAppKey: 'drawioCreate',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Create a draw.io diagram from mxGraph XML and insert it as a diagram ' +
|
||||
'block. `xml` is a bare `<mxGraphModel>` OR a list of `<mxCell>` elements ' +
|
||||
@@ -1969,6 +2023,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
drawioUpdate: {
|
||||
mcpName: 'drawioUpdate',
|
||||
inAppKey: 'drawioUpdate',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Replace a draw.io diagram\'s content with new mxGraph XML (same lint ' +
|
||||
'pipeline as drawioCreate). `baseHash` is MANDATORY: pass the hash from ' +
|
||||
@@ -2020,6 +2075,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
drawioEditCells: {
|
||||
mcpName: 'drawioEditCells',
|
||||
inAppKey: 'drawioEditCells',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Make TARGETED, id-based edits to an existing draw.io diagram instead of ' +
|
||||
'resending the whole XML (a full-XML diff is fragile — draw.io reorders ' +
|
||||
@@ -2078,6 +2134,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
drawioFromGraph: {
|
||||
mcpName: 'drawioFromGraph',
|
||||
inAppKey: 'drawioFromGraph',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Build a draw.io diagram from a SEMANTIC graph — you describe nodes, groups ' +
|
||||
'and edges by MEANING and the server picks every coordinate, color and icon ' +
|
||||
@@ -2202,6 +2259,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
drawioFromMermaid: {
|
||||
mcpName: 'drawioFromMermaid',
|
||||
inAppKey: 'drawioFromMermaid',
|
||||
writeClass: 'write',
|
||||
description:
|
||||
'Convert Mermaid `flowchart` text into an EDITABLE draw.io diagram (LLMs ' +
|
||||
'write Mermaid reliably). Best for STANDARD flowcharts/decision trees: ' +
|
||||
@@ -2248,6 +2306,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
drawioShapes: {
|
||||
mcpName: 'drawioShapes',
|
||||
inAppKey: 'drawioShapes',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Look up VERIFIED draw.io stencil style-strings so you never guess a ' +
|
||||
'`shape=mxgraph.*` name (a wrong name renders as an EMPTY BOX). Searches a ' +
|
||||
@@ -2294,6 +2353,7 @@ export const SHARED_TOOL_SPECS = {
|
||||
drawioGuide: {
|
||||
mcpName: 'drawioGuide',
|
||||
inAppKey: 'drawioGuide',
|
||||
writeClass: 'readOnly',
|
||||
description:
|
||||
'Progressive-disclosure draw.io authoring reference. Call with a `section` ' +
|
||||
'to pull one focused, <=4KB chapter instead of bloating context: ' +
|
||||
@@ -2323,3 +2383,53 @@ export const SHARED_TOOL_SPECS = {
|
||||
inlineBothHosts: true,
|
||||
},
|
||||
} satisfies Record<string, SharedToolSpec>;
|
||||
|
||||
// --- write-class registry (#489) ------------------------------------------
|
||||
|
||||
/** A tool's retry-safety class. 'readOnly' may be auto-retried once after a
|
||||
* transport break; 'write' is indeterminate and must never be blind-retried. */
|
||||
export type ToolWriteClass = 'readOnly' | 'write';
|
||||
|
||||
/**
|
||||
* Name → write-class map for the shared registry, keyed by mcpName (=== inAppKey).
|
||||
* The external-MCP retry path (mcp-clients.service.ts) looks a tool up here by its
|
||||
* RAW (un-namespaced) name to decide whether a transport failure may be retried.
|
||||
* A tool NOT in this map (a third-party external MCP tool) is treated as 'write'
|
||||
* by the consumer — the safe default (never blind-retry an unknown tool).
|
||||
*/
|
||||
export const SHARED_TOOL_WRITE_CLASS: Record<string, ToolWriteClass> =
|
||||
Object.fromEntries(
|
||||
Object.values(SHARED_TOOL_SPECS).map((spec) => [spec.mcpName, spec.writeClass]),
|
||||
);
|
||||
|
||||
/** Whether a write-class permits a single automatic retry after a transport
|
||||
* break. Only a pure read is retry-safe; everything mutating is indeterminate. */
|
||||
export function isRetryableWriteClass(
|
||||
writeClass: ToolWriteClass | undefined,
|
||||
): boolean {
|
||||
return writeClass === 'readOnly';
|
||||
}
|
||||
|
||||
/**
|
||||
* Registration-time assert (#489): EVERY spec must declare a valid write-class.
|
||||
* `satisfies Record<string, SharedToolSpec>` already makes an omission a compile
|
||||
* error, but this guards a raw/cast construction path and documents the invariant
|
||||
* at the point of use. Runs once on import — both hosts import this module, so
|
||||
* both get the check. Throws (fails startup) rather than silently mis-gating a
|
||||
* retry in production.
|
||||
*/
|
||||
export function assertEverySpecDeclaresWriteClass(): void {
|
||||
for (const [key, spec] of Object.entries(SHARED_TOOL_SPECS)) {
|
||||
const wc = (spec as SharedToolSpec).writeClass;
|
||||
if (wc !== 'readOnly' && wc !== 'write') {
|
||||
throw new Error(
|
||||
`tool-specs: spec "${key}" must declare writeClass ('readOnly' | 'write'), got ${JSON.stringify(
|
||||
wc,
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Enforce at module load (registration time) on both hosts.
|
||||
assertEverySpecDeclaresWriteClass();
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
// #487 commit 1 — the in-app tool cancellation safe-point inside paginateAll.
|
||||
//
|
||||
// The in-app tool host sets a composite abort signal on the client
|
||||
// (setToolAbortSignal) before each tool call; paginateAll checks it at a
|
||||
// safe-point BEFORE every sequential page fetch, so a Stop that lands mid-read
|
||||
// stops the NEXT HTTP request from STARTING (a read tool can no longer paginate
|
||||
// for minutes past a Stop). This pins the HONEST observable property against the
|
||||
// REAL client + a real HTTP server: "after Stop, no NEW request starts".
|
||||
import { test, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { DocmostClient } from "../../build/client.js";
|
||||
|
||||
function readBody(req) {
|
||||
return new Promise((resolve) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => resolve(raw));
|
||||
});
|
||||
}
|
||||
function sendJson(res, status, obj, extra = {}) {
|
||||
res.writeHead(status, { "Content-Type": "application/json", ...extra });
|
||||
res.end(JSON.stringify(obj));
|
||||
}
|
||||
const openServers = [];
|
||||
async function spawn(handler) {
|
||||
const server = await new Promise((resolve) => {
|
||||
const s = http.createServer(handler);
|
||||
s.listen(0, "127.0.0.1", () => resolve(s));
|
||||
});
|
||||
openServers.push(server);
|
||||
const { port } = server.address();
|
||||
return { baseURL: `http://127.0.0.1:${port}/api` };
|
||||
}
|
||||
after(async () => {
|
||||
await Promise.all(openServers.map((s) => new Promise((r) => s.close(r))));
|
||||
});
|
||||
function handleLogin(req, res) {
|
||||
if (req.url === "/api/auth/login") {
|
||||
sendJson(res, 200, { success: true }, {
|
||||
"Set-Cookie": "authToken=t; Path=/; HttpOnly",
|
||||
});
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// A Stop that lands DURING pagination: the server aborts the client signal as it
|
||||
// serves page 1 (more pages remain). The loop's next safe-point must throw before
|
||||
// the page-2 request is sent.
|
||||
test("paginateAll stops the NEXT request when the signal aborts mid-pagination", async () => {
|
||||
let requests = 0;
|
||||
const ac = new AbortController();
|
||||
const { baseURL } = await spawn(async (req, res) => {
|
||||
await readBody(req);
|
||||
if (handleLogin(req, res)) return;
|
||||
if (req.url === "/api/spaces") {
|
||||
requests++;
|
||||
// Simulate a user Stop that lands while page 1 is in flight.
|
||||
if (requests === 1) ac.abort(new Error("user stop"));
|
||||
sendJson(res, 200, {
|
||||
success: true,
|
||||
data: {
|
||||
items: [{ id: `p${requests}` }],
|
||||
meta: { hasNextPage: true, nextCursor: `c${requests}` },
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
sendJson(res, 404, {});
|
||||
});
|
||||
|
||||
const client = new DocmostClient(baseURL, "user@example.com", "pw");
|
||||
client.setToolAbortSignal(ac.signal);
|
||||
|
||||
await assert.rejects(
|
||||
() => client.paginateAll("/spaces", {}),
|
||||
/user stop/,
|
||||
"the aborted safe-point rejects with the signal's reason",
|
||||
);
|
||||
assert.equal(requests, 1, "page 2 never started after the Stop");
|
||||
});
|
||||
|
||||
// A Stop that is already in effect before the read starts: zero requests fire.
|
||||
test("paginateAll starts no request when the signal is already aborted", async () => {
|
||||
let requests = 0;
|
||||
const { baseURL } = await spawn(async (req, res) => {
|
||||
await readBody(req);
|
||||
if (handleLogin(req, res)) return;
|
||||
if (req.url === "/api/spaces") {
|
||||
requests++;
|
||||
sendJson(res, 200, {
|
||||
success: true,
|
||||
data: { items: [], meta: { hasNextPage: false, nextCursor: null } },
|
||||
});
|
||||
return;
|
||||
}
|
||||
sendJson(res, 404, {});
|
||||
});
|
||||
|
||||
const client = new DocmostClient(baseURL, "user@example.com", "pw");
|
||||
// Warm the auth so ensureAuthenticated does not itself POST after the abort.
|
||||
await client.ensureAuthenticated();
|
||||
const ac = new AbortController();
|
||||
ac.abort(new Error("already stopped"));
|
||||
client.setToolAbortSignal(ac.signal);
|
||||
|
||||
await assert.rejects(() => client.paginateAll("/spaces", {}), /already stopped/);
|
||||
assert.equal(requests, 0, "no /spaces request started once already aborted");
|
||||
});
|
||||
|
||||
// Without a tool signal (default), pagination is unaffected — the safe-point is a
|
||||
// pure no-op, so pre-#487 behaviour is byte-identical.
|
||||
test("paginateAll is unaffected when no tool signal is set", async () => {
|
||||
let requests = 0;
|
||||
const PAGES = {
|
||||
"": { items: [{ id: "a" }], nextCursor: "c1" },
|
||||
c1: { items: [{ id: "b" }], nextCursor: null },
|
||||
};
|
||||
const { baseURL } = await spawn(async (req, res) => {
|
||||
const raw = await readBody(req);
|
||||
if (handleLogin(req, res)) return;
|
||||
if (req.url === "/api/spaces") {
|
||||
requests++;
|
||||
const body = JSON.parse(raw || "{}");
|
||||
const page = PAGES[body.cursor ?? ""] ?? { items: [], nextCursor: null };
|
||||
sendJson(res, 200, {
|
||||
success: true,
|
||||
data: {
|
||||
items: page.items,
|
||||
meta: { hasNextPage: page.nextCursor != null, nextCursor: page.nextCursor },
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
sendJson(res, 404, {});
|
||||
});
|
||||
|
||||
const client = new DocmostClient(baseURL, "user@example.com", "pw");
|
||||
const all = await client.paginateAll("/spaces", {});
|
||||
assert.equal(requests, 2, "both pages fetched with no signal set");
|
||||
assert.deepEqual(all.map((p) => p.id), ["a", "b"]);
|
||||
});
|
||||
@@ -442,3 +442,139 @@ test("checkNewComments subtree includes the root without a separate getPageRaw",
|
||||
assert.equal(result.checkedPages, 2, "root + one descendant scanned");
|
||||
assert.equal(result.totalNewComments, 1, "the root's fresh comment found");
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// 6) checkNewComments parallelism (#490): the per-page comment fetches run with
|
||||
// bounded concurrency (not one-at-a-time), and the results still preserve the
|
||||
// page order deterministically regardless of which fetch finishes first.
|
||||
// -----------------------------------------------------------------------------
|
||||
test("checkNewComments fetches pages concurrently (bounded) and preserves order", async () => {
|
||||
// A subtree with 12 descendants so the scan has plenty to parallelize.
|
||||
const NODES = [{ id: "parent", title: "Parent", parentPageId: null, hasChildren: true }];
|
||||
for (let i = 0; i < 12; i++) {
|
||||
NODES.push({ id: `k${i}`, title: `Kid ${i}`, parentPageId: "parent", hasChildren: false });
|
||||
}
|
||||
|
||||
let inFlight = 0;
|
||||
let maxInFlight = 0;
|
||||
|
||||
const { baseURL } = await spawn(async (req, res) => {
|
||||
const raw = await readBody(req);
|
||||
if (handleLogin(req, res)) return;
|
||||
if (req.url === "/api/pages/tree") {
|
||||
sendJson(res, 200, { success: true, data: { items: NODES } });
|
||||
return;
|
||||
}
|
||||
if (req.url === "/api/comments") {
|
||||
const body = JSON.parse(raw || "{}");
|
||||
inFlight++;
|
||||
maxInFlight = Math.max(maxInFlight, inFlight);
|
||||
// Hold the response briefly so concurrent fetches actually overlap.
|
||||
setTimeout(() => {
|
||||
inFlight--;
|
||||
// Every page carries one fresh comment so ordering is observable.
|
||||
sendJson(res, 200, {
|
||||
success: true,
|
||||
data: {
|
||||
items: [
|
||||
{ id: `c-${body.pageId}`, createdAt: "2030-01-01T00:00:00.000Z", content: null },
|
||||
],
|
||||
meta: { nextCursor: null },
|
||||
},
|
||||
});
|
||||
}, 25);
|
||||
return;
|
||||
}
|
||||
sendJson(res, 404, {});
|
||||
});
|
||||
|
||||
const client = new DocmostClient(baseURL, "user@example.com", "pw");
|
||||
const result = await client.checkNewComments(
|
||||
"space-1",
|
||||
"2020-01-01T00:00:00.000Z",
|
||||
"parent",
|
||||
);
|
||||
|
||||
// 13 pages (parent + 12 kids) were scanned; each had a fresh comment.
|
||||
assert.equal(result.checkedPages, 13, "all pages scanned");
|
||||
assert.equal(result.totalNewComments, 13, "one fresh comment per page");
|
||||
// Parallelism: more than one request was in flight at once, but never above the
|
||||
// cap (6). A serial implementation would show maxInFlight === 1.
|
||||
assert.ok(maxInFlight > 1, `expected concurrent fetches, saw max ${maxInFlight}`);
|
||||
assert.ok(maxInFlight <= 6, `concurrency must be bounded, saw ${maxInFlight}`);
|
||||
// Deterministic order: results follow the page-enumeration order (parent first).
|
||||
assert.equal(result.comments[0].pageId, "parent", "results preserve page order");
|
||||
assert.deepEqual(
|
||||
result.comments.map((r) => r.pageId),
|
||||
["parent", ...Array.from({ length: 12 }, (_, i) => `k${i}`)],
|
||||
"result order matches the enumeration order regardless of finish order",
|
||||
);
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// 7) checkNewComments partial failure (#490): the concurrent scan is resilient —
|
||||
// if ONE page's /comments fetch rejects (deleted mid-scan, a transient 500),
|
||||
// that page is skipped and the WHOLE scan still resolves with every other
|
||||
// page's fresh comments. A single failing fetch must never reject the batch
|
||||
// (Promise.all in mapWithConcurrency would otherwise abort all of it) nor
|
||||
// corrupt the deterministic order of the pages that DID succeed.
|
||||
// -----------------------------------------------------------------------------
|
||||
test("checkNewComments skips a page whose fetch fails and still reports the rest", async () => {
|
||||
const NODES = [{ id: "parent", title: "Parent", parentPageId: null, hasChildren: true }];
|
||||
for (let i = 0; i < 5; i++) {
|
||||
NODES.push({ id: `k${i}`, title: `Kid ${i}`, parentPageId: "parent", hasChildren: false });
|
||||
}
|
||||
// The one page whose comment fetch blows up (500 -> listComments rejects).
|
||||
const FAILING = "k2";
|
||||
|
||||
const { baseURL } = await spawn(async (req, res) => {
|
||||
const raw = await readBody(req);
|
||||
if (handleLogin(req, res)) return;
|
||||
if (req.url === "/api/pages/tree") {
|
||||
sendJson(res, 200, { success: true, data: { items: NODES } });
|
||||
return;
|
||||
}
|
||||
if (req.url === "/api/comments") {
|
||||
const body = JSON.parse(raw || "{}");
|
||||
if (body.pageId === FAILING) {
|
||||
// A transient server error on exactly one page's fetch.
|
||||
sendJson(res, 500, { success: false, message: "boom" });
|
||||
return;
|
||||
}
|
||||
sendJson(res, 200, {
|
||||
success: true,
|
||||
data: {
|
||||
items: [
|
||||
{ id: `c-${body.pageId}`, createdAt: "2030-01-01T00:00:00.000Z", content: null },
|
||||
],
|
||||
meta: { nextCursor: null },
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
sendJson(res, 404, {});
|
||||
});
|
||||
|
||||
const client = new DocmostClient(baseURL, "user@example.com", "pw");
|
||||
// Must RESOLVE (not reject) despite one page's fetch failing.
|
||||
const result = await client.checkNewComments(
|
||||
"space-1",
|
||||
"2020-01-01T00:00:00.000Z",
|
||||
"parent",
|
||||
);
|
||||
|
||||
// Every page in scope was still scanned (the failing one counts as checked).
|
||||
assert.equal(result.checkedPages, 6, "all pages scanned incl. the failing one");
|
||||
// The failing page contributes nothing; the other 5 each report one comment.
|
||||
assert.equal(result.pagesWithNewComments, 5, "the failing page is dropped");
|
||||
assert.equal(result.totalNewComments, 5, "only the succeeding pages' comments");
|
||||
const reportedIds = result.comments.map((r) => r.pageId);
|
||||
assert.ok(!reportedIds.includes(FAILING), "the failing page is absent from results");
|
||||
// Order of the survivors is still the deterministic enumeration order (the hole
|
||||
// left by the failing page is closed without reordering the rest).
|
||||
assert.deepEqual(
|
||||
reportedIds,
|
||||
["parent", "k0", "k1", "k3", "k4"],
|
||||
"survivors keep enumeration order with the failing page removed",
|
||||
);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
// #487 F4 — the WRITE-side cancellation safe-point.
|
||||
//
|
||||
// Every content-mutating collab write (collaboration.mutatePageContent and the
|
||||
// reentrant twin client.mutateLiveContentUnlocked used by replaceImage) checks
|
||||
// the in-app tool abort signal at a PRE-COMMIT safe-point — after the collab
|
||||
// session is acquired but immediately BEFORE the atomic read->write
|
||||
// (session.mutate). So a Stop (or the per-call cap) that lands during the
|
||||
// connect/lock window stops THIS write from landing: no new commit starts once
|
||||
// aborted. paginate-abort-safepoint.test.mjs pins the READ half; this pins the
|
||||
// integrity-critical WRITE half — remove the `throwIfAborted()` and the transform
|
||||
// would run and the doc would be mutated past a Stop.
|
||||
//
|
||||
// There is no collab server in the unit env, so we swap the provider factory
|
||||
// (__setCollabProviderFactory) for a fake that reports an immediate successful
|
||||
// sync. That makes acquireCollabSession SUCCEED and hand back a live, ready
|
||||
// session, so the ONLY thing standing between the call and session.mutate is the
|
||||
// safe-point under test. The transform is instrumented to prove it never runs.
|
||||
import { test, afterEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { mutatePageContent } from "../../build/lib/collaboration.js";
|
||||
import {
|
||||
__setCollabProviderFactory,
|
||||
destroyAllSessions,
|
||||
} from "../../build/lib/collab-session.js";
|
||||
import { DocmostClient } from "../../build/client.js";
|
||||
|
||||
const BASE_URL = "http://127.0.0.1:1/api";
|
||||
// mutatePageContent locks via withPageLock, which demands a canonical page UUID
|
||||
// (resolve-then-lock invariant, #260/#449); the unlocked twin does not.
|
||||
const PAGE_UUID = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
// A fake HocuspocusProvider that immediately reports a successful initial sync so
|
||||
// CollabSession.open() resolves to a ready session, and stays "synced" with zero
|
||||
// unsynced changes so a reached session.mutate() would resolve at once. It speaks
|
||||
// only the tiny CollabProviderLike surface the session depends on.
|
||||
function syncedProviderFactory(config) {
|
||||
// Fire the initial-sync callback so open() settles as ready.
|
||||
config.onSynced();
|
||||
return {
|
||||
synced: true,
|
||||
unsyncedChanges: 0,
|
||||
destroy() {},
|
||||
on() {},
|
||||
off() {},
|
||||
};
|
||||
}
|
||||
|
||||
// Disable the session cache so every acquire opens (and the failure path destroys)
|
||||
// its own ephemeral session — no cross-test session reuse.
|
||||
process.env.MCP_COLLAB_SESSION_IDLE_MS = "0";
|
||||
|
||||
afterEach(() => {
|
||||
__setCollabProviderFactory(null); // restore the real factory
|
||||
destroyAllSessions();
|
||||
});
|
||||
|
||||
// --- collaboration.mutatePageContent (the page-locked write path) -------------
|
||||
|
||||
test("mutatePageContent rejects at the pre-commit safe-point BEFORE session.mutate when the signal is already aborted", async () => {
|
||||
__setCollabProviderFactory(syncedProviderFactory);
|
||||
let transformCalls = 0;
|
||||
const ac = new AbortController();
|
||||
ac.abort(new Error("user stop"));
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
mutatePageContent(
|
||||
PAGE_UUID,
|
||||
"collab-jwt",
|
||||
BASE_URL,
|
||||
(liveDoc) => {
|
||||
transformCalls++;
|
||||
return liveDoc;
|
||||
},
|
||||
ac.signal,
|
||||
),
|
||||
/user stop/,
|
||||
"the aborted safe-point rejects with the signal's reason before committing",
|
||||
);
|
||||
assert.equal(
|
||||
transformCalls,
|
||||
0,
|
||||
"the transform (and therefore session.mutate) must NEVER run once aborted",
|
||||
);
|
||||
});
|
||||
|
||||
test("mutatePageContent (control) DOES reach session.mutate and invoke the transform when the signal is live", async () => {
|
||||
__setCollabProviderFactory(syncedProviderFactory);
|
||||
let transformCalls = 0;
|
||||
const ac = new AbortController(); // never aborted
|
||||
|
||||
const result = await mutatePageContent(
|
||||
PAGE_UUID,
|
||||
"collab-jwt",
|
||||
BASE_URL,
|
||||
(liveDoc) => {
|
||||
transformCalls++;
|
||||
return null; // null -> no-op write; still proves the transform was invoked
|
||||
},
|
||||
ac.signal,
|
||||
);
|
||||
assert.equal(
|
||||
transformCalls,
|
||||
1,
|
||||
"with a live signal the safe-point is a no-op and session.mutate runs the transform",
|
||||
);
|
||||
assert.ok(result && result.verify, "a MutationResult is returned");
|
||||
});
|
||||
|
||||
// --- client.mutateLiveContentUnlocked (the reentrant twin, replaceImage) ------
|
||||
|
||||
test("mutateLiveContentUnlocked rejects at the pre-commit safe-point BEFORE session.mutate when the tool signal is already aborted", async () => {
|
||||
__setCollabProviderFactory(syncedProviderFactory);
|
||||
const client = new DocmostClient({
|
||||
apiUrl: BASE_URL,
|
||||
getToken: async () => "access",
|
||||
getCollabToken: async () => "collab-jwt",
|
||||
});
|
||||
let transformCalls = 0;
|
||||
const ac = new AbortController();
|
||||
ac.abort(new Error("cap fired"));
|
||||
client.setToolAbortSignal(ac.signal);
|
||||
|
||||
await assert.rejects(
|
||||
() =>
|
||||
client.mutateLiveContentUnlocked("page-1", "collab-jwt", (liveDoc) => {
|
||||
transformCalls++;
|
||||
return liveDoc;
|
||||
}),
|
||||
/cap fired/,
|
||||
"the aborted safe-point rejects with the signal's reason before committing",
|
||||
);
|
||||
assert.equal(
|
||||
transformCalls,
|
||||
0,
|
||||
"the transform (and therefore session.mutate) must NEVER run once aborted",
|
||||
);
|
||||
});
|
||||
|
||||
test("mutateLiveContentUnlocked (control) DOES reach session.mutate and invoke the transform when the tool signal is live", async () => {
|
||||
__setCollabProviderFactory(syncedProviderFactory);
|
||||
const client = new DocmostClient({
|
||||
apiUrl: BASE_URL,
|
||||
getToken: async () => "access",
|
||||
getCollabToken: async () => "collab-jwt",
|
||||
});
|
||||
let transformCalls = 0;
|
||||
client.setToolAbortSignal(new AbortController().signal); // live
|
||||
|
||||
const result = await client.mutateLiveContentUnlocked(
|
||||
"page-1",
|
||||
"collab-jwt",
|
||||
(liveDoc) => {
|
||||
transformCalls++;
|
||||
return null;
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
transformCalls,
|
||||
1,
|
||||
"with a live signal the safe-point is a no-op and session.mutate runs the transform",
|
||||
);
|
||||
assert.ok(result && result.verify, "a MutationResult is returned");
|
||||
});
|
||||
@@ -2,7 +2,12 @@ import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { z } from "zod";
|
||||
|
||||
import { SHARED_TOOL_SPECS } from "../../build/tool-specs.js";
|
||||
import {
|
||||
SHARED_TOOL_SPECS,
|
||||
SHARED_TOOL_WRITE_CLASS,
|
||||
isRetryableWriteClass,
|
||||
assertEverySpecDeclaresWriteClass,
|
||||
} from "../../build/tool-specs.js";
|
||||
|
||||
// The shared registry is consumed by BOTH the zod-v3 MCP server and the zod-v4
|
||||
// in-app AI-SDK service, so every spec must carry the cross-layer wiring
|
||||
@@ -43,6 +48,41 @@ test("mcpName and inAppKey are each unique across the registry", () => {
|
||||
}
|
||||
});
|
||||
|
||||
// #489 — every spec must declare its write-class so the external-MCP retry path
|
||||
// can gate a single auto-retry ONLY on a pure read (a blind retry of a write =
|
||||
// double-apply). The declaration is enforced at registration time.
|
||||
test("#489: every spec declares a valid writeClass ('readOnly' | 'write')", () => {
|
||||
for (const [key, spec] of Object.entries(SHARED_TOOL_SPECS)) {
|
||||
assert.ok(
|
||||
spec.writeClass === "readOnly" || spec.writeClass === "write",
|
||||
`${key}: missing/invalid writeClass: ${JSON.stringify(spec.writeClass)}`,
|
||||
);
|
||||
}
|
||||
// The registration-time assert must not throw for the shipped registry.
|
||||
assert.doesNotThrow(() => assertEverySpecDeclaresWriteClass());
|
||||
});
|
||||
|
||||
test("#489: SHARED_TOOL_WRITE_CLASS maps every mcpName to its class; helper gates on readOnly", () => {
|
||||
const specs = Object.values(SHARED_TOOL_SPECS);
|
||||
assert.equal(Object.keys(SHARED_TOOL_WRITE_CLASS).length, specs.length);
|
||||
for (const spec of specs) {
|
||||
assert.equal(SHARED_TOOL_WRITE_CLASS[spec.mcpName], spec.writeClass);
|
||||
}
|
||||
// Only a readOnly tool is retry-eligible; a write tool and an unknown tool are not.
|
||||
assert.equal(isRetryableWriteClass("readOnly"), true);
|
||||
assert.equal(isRetryableWriteClass("write"), false);
|
||||
assert.equal(isRetryableWriteClass(undefined), false);
|
||||
});
|
||||
|
||||
test("#489: representative reads are readOnly and representative writes are write", () => {
|
||||
for (const name of ["getPage", "getTree", "searchInPage", "listComments"]) {
|
||||
assert.equal(SHARED_TOOL_SPECS[name].writeClass, "readOnly", `${name} should be readOnly`);
|
||||
}
|
||||
for (const name of ["patchNode", "createPage", "deletePage", "createComment", "drawioCreate"]) {
|
||||
assert.equal(SHARED_TOOL_SPECS[name].writeClass, "write", `${name} should be write`);
|
||||
}
|
||||
});
|
||||
|
||||
test("buildShape (when present) returns a usable ZodRawShape with a real zod", () => {
|
||||
for (const [key, spec] of Object.entries(SHARED_TOOL_SPECS)) {
|
||||
if (!spec.buildShape) continue;
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "@docmost/token-estimate",
|
||||
"version": "0.1.0",
|
||||
"description": "Shared, provider-agnostic token estimator (chars/2.5) used by the AI-chat client counter and the server history-replay budgeter, so the two never diverge.",
|
||||
"private": true,
|
||||
"main": "./dist/index.js",
|
||||
"types": "./dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"watch": "tsc --watch",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
},
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"typescript": "^5.0.0",
|
||||
"vitest": "4.1.6"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { estimateTokens, CHARS_PER_TOKEN } from './index';
|
||||
|
||||
describe('estimateTokens (shared chars/2.5)', () => {
|
||||
it('returns 0 for empty / nullish input', () => {
|
||||
expect(estimateTokens('')).toBe(0);
|
||||
expect(estimateTokens(null)).toBe(0);
|
||||
expect(estimateTokens(undefined)).toBe(0);
|
||||
});
|
||||
|
||||
it('uses the chars/2.5 ratio, ceiled', () => {
|
||||
expect(CHARS_PER_TOKEN).toBe(2.5);
|
||||
// 5 chars / 2.5 = 2
|
||||
expect(estimateTokens('abcde')).toBe(2);
|
||||
// any non-empty string is at least 1 token (ceil)
|
||||
expect(estimateTokens('a')).toBe(1);
|
||||
// 100 chars / 2.5 = 40
|
||||
expect(estimateTokens('x'.repeat(100))).toBe(40);
|
||||
});
|
||||
|
||||
it('counts Cyrillic ~2x higher than the old chars/4 rule (no undercount)', () => {
|
||||
const cyr = 'привет мир как дела'; // 19 chars
|
||||
expect(estimateTokens(cyr)).toBe(Math.ceil(19 / 2.5)); // 8
|
||||
expect(estimateTokens(cyr)).toBeGreaterThan(Math.ceil(19 / 4)); // > 5
|
||||
});
|
||||
|
||||
it('is deterministic / byte-stable (same input => same output)', () => {
|
||||
const s = 'the quick brown fox';
|
||||
expect(estimateTokens(s)).toBe(estimateTokens(s));
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Shared, provider-agnostic token estimator (#490).
|
||||
*
|
||||
* No provider exposes an exact tokenizer we can afford to run on the hot path (a
|
||||
* real BPE pass is O(n²)-ish, bloats the client bundle, and is wrong for
|
||||
* Gemini/Ollama anyway), so both the client's in-body counter AND the server's
|
||||
* history-replay budgeter use this ONE cheap chars-based heuristic. Keeping it in
|
||||
* a single shared module is deliberate: two independent estimators drift, and then
|
||||
* "the badge shows 60%" while "the budgeter already trimmed" — the exact confusion
|
||||
* this package prevents.
|
||||
*
|
||||
* Ratio: **chars / 2.5**. Most content here is Cyrillic, where a token is ~2.5
|
||||
* characters; the common English `chars/4` rule of thumb UNDER-counts Cyrillic by
|
||||
* ~2×, which for a budget check is the dangerous direction (it lets the context
|
||||
* overflow). 2.5 slightly over-estimates pure English/code, which is the SAFE
|
||||
* direction for a budget. This is an estimate, never an exact count — the
|
||||
* authoritative figure is always the provider's reported usage; the estimate is
|
||||
* for UI affordances, the delta of not-yet-sent messages, and deciding what to
|
||||
* trim.
|
||||
*/
|
||||
|
||||
/** Characters per token for the shared estimate. See the module comment. */
|
||||
export const CHARS_PER_TOKEN = 2.5;
|
||||
|
||||
/**
|
||||
* Rough token estimate for a piece of text (chars / {@link CHARS_PER_TOKEN}).
|
||||
* Returns 0 for empty/nullish input, and ceils so any non-empty text counts as at
|
||||
* least one token. Pure and deterministic (byte-stable), so the same text always
|
||||
* yields the same estimate — which the server budgeter relies on to keep replay
|
||||
* trimming stable turn to turn (provider prompt-cache friendliness).
|
||||
*/
|
||||
export function estimateTokens(text: string | null | undefined): number {
|
||||
if (!text) return 0;
|
||||
return Math.ceil(text.length / CHARS_PER_TOKEN);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "CommonJS",
|
||||
"moduleResolution": "Node",
|
||||
"outDir": "./dist",
|
||||
"rootDir": "./src",
|
||||
"strict": true,
|
||||
"esModuleInterop": true,
|
||||
"skipLibCheck": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"declaration": true
|
||||
},
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["src/**/*.test.ts"]
|
||||
}
|
||||
Generated
+15
@@ -287,6 +287,9 @@ importers:
|
||||
'@docmost/prosemirror-markdown':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/prosemirror-markdown
|
||||
'@docmost/token-estimate':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/token-estimate
|
||||
'@excalidraw/excalidraw':
|
||||
specifier: 0.18.0-3a5ef40
|
||||
version: 0.18.0-3a5ef40(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
|
||||
@@ -561,6 +564,9 @@ importers:
|
||||
'@docmost/prosemirror-markdown':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/prosemirror-markdown
|
||||
'@docmost/token-estimate':
|
||||
specifier: workspace:*
|
||||
version: link:../../packages/token-estimate
|
||||
'@fastify/compress':
|
||||
specifier: ^9.0.0
|
||||
version: 9.0.0
|
||||
@@ -1148,6 +1154,15 @@ importers:
|
||||
specifier: 4.1.6
|
||||
version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@20.19.43)(@vitest/coverage-v8@4.1.6)(happy-dom@20.8.9)(jsdom@25.0.0)(vite@8.0.5(@types/node@20.19.43)(esbuild@0.28.0)(jiti@2.4.2)(less@4.2.0)(sugarss@5.0.1(postcss@8.5.14))(terser@5.39.0)(tsx@4.21.0)(yaml@2.8.3))
|
||||
|
||||
packages/token-estimate:
|
||||
devDependencies:
|
||||
typescript:
|
||||
specifier: ^5.0.0
|
||||
version: 5.9.3
|
||||
vitest:
|
||||
specifier: 4.1.6
|
||||
version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.6)(happy-dom@20.8.9)(jsdom@27.4.0(@noble/hashes@2.0.1))(vite@8.0.5(@types/node@25.5.0)(esbuild@0.28.0)(jiti@2.4.2)(less@4.2.0)(sugarss@5.0.1(postcss@8.5.14))(terser@5.39.0)(tsx@4.21.0)(yaml@2.8.3))
|
||||
|
||||
packages:
|
||||
|
||||
'@aashutoshrathi/word-wrap@1.2.6':
|
||||
|
||||
Reference in New Issue
Block a user