From 09ab92eccff46e031bc574f14a3d8e3eed9d7d9a Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 02:25:28 +0300 Subject: [PATCH 01/41] =?UTF-8?q?fix(mcp):=20ELK-=D0=BB=D0=B5=D0=B9=D0=B0?= =?UTF-8?q?=D1=83=D1=82=20=D0=B2=20worker=5Fthread=20=E2=80=94=20=D1=82?= =?UTF-8?q?=D0=B0=D0=B9=D0=BC=D0=B0=D1=83=D1=82=20=D1=87=D0=B5=D1=80=D0=B5?= =?UTF-8?q?=D0=B7=20terminate()=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit elkjs.layout() возвращает Promise, но саму раскладку крутит СИНХРОННО и блокирует поток целиком. На in-app хосте это был главный event loop: патологический граф у капа 500 узлов вешал ВСЕ HTTP/SSE/loopback. Прежняя защита (Promise.race с setTimeout(5s)) была иллюзией — таймер физически не мог сработать, пока тот же поток заблокирован внутри elkjs (комментарий в коде это сам признавал). Теперь elk.layout() исполняется в worker_thread, а таймаут форсится worker.terminate() — единственный способ прервать синхронный JS. Главный поток остаётся свободным; на таймауте/ошибке — best-effort откат к исходной модели, как и раньше. Лживый комментарий «can never wedge the server» убран. Тесты: unit на terminate-по-таймауту (крошечный ceiling → hard-kill → исходная модель нетронута) и бенчмарк-гард на worst-case графе у капа (500 узлов/~1000 рёбер раскладывается, а главный event loop продолжает тикать во время раскладки). --- packages/mcp/src/lib/drawio-layout.ts | 105 ++++++++++++------ packages/mcp/src/lib/drawio-layout.worker.ts | 36 ++++++ packages/mcp/test/unit/drawio-layout.test.mjs | 82 ++++++++++++++ 3 files changed, 192 insertions(+), 31 deletions(-) create mode 100644 packages/mcp/src/lib/drawio-layout.worker.ts diff --git a/packages/mcp/src/lib/drawio-layout.ts b/packages/mcp/src/lib/drawio-layout.ts index 7adb166f..647bdca8 100644 --- a/packages/mcp/src/lib/drawio-layout.ts +++ b/packages/mcp/src/lib/drawio-layout.ts @@ -10,7 +10,7 @@ // exactly mxGraph's convention for a child of a container, so they map across // directly. Container sizes are computed by ELK; leaf sizes are preserved. -import ELK from "elkjs/lib/elk.bundled.js"; +import { Worker } from "node:worker_threads"; import { JSDOM } from "jsdom"; import { normalizeInput, parseCells, type DrawioCell } from "./drawio-xml.js"; @@ -18,22 +18,33 @@ import { normalizeInput, parseCells, type DrawioCell } from "./drawio-xml.js"; const DEFAULT_W = 140; const DEFAULT_H = 60; -// DoS bounds for the in-process ELK layout. The mxGraph XML is LLM-supplied -// (layout:"elk" in drawioCreate/drawioUpdate) and elkjs runs synchronously on -// the MCP server's event loop, so an unbounded graph would block it for -// seconds-to-minutes. A ~1MB XML (well under the stage-1 16MB cap) can carry -// thousands of nodes. We cap the graph size and race the layout against a -// wall-clock timeout; on either bound we fall back to the ORIGINAL model, the -// same best-effort contract the catch already honours. +// DoS bounds for the ELK layout. The mxGraph XML is LLM-supplied (layout:"elk" +// in drawioCreate/drawioUpdate). elkjs' layout() returns a Promise but runs the +// crossing-minimisation SYNCHRONOUSLY — it blocks whatever thread it runs on for +// the whole pass. A ~1MB XML (well under the stage-1 16MB cap) can carry +// thousands of nodes. We (a) cap the graph size before ever calling ELK and +// (b) run the layout in a WORKER THREAD so the main event loop stays free, with +// the wall-clock timeout enforced by terminating that worker. On either bound we +// fall back to the ORIGINAL model, the same best-effort contract the catch honours. // - 500 nodes lays out in well under a second; beyond that ELK cost climbs // steeply, so refuse and leave the (already-valid) model untouched. // - Edges dominate the layered-crossing cost, so allow a bit more headroom // (1000) than nodes but still bound them. -// - 5s is generous for any graph within the caps yet short enough that a -// pathological input can never wedge the server. +// - The timeout is a HARD kill of the worker thread — the only way to interrupt +// synchronous JS. The in-process setTimeout race we used before was an +// illusion: the timer could never fire while the SAME thread was blocked +// inside elkjs, so it "protected" nothing. Now the timer runs on the main +// thread while ELK runs on the worker, so it can actually fire and terminate. const ELK_MAX_NODES = 500; const ELK_MAX_EDGES = 1000; -const ELK_TIMEOUT_MS = 5000; +// Wall-clock ceiling for a single layout pass. Overridable for tests (a tiny +// value forces the terminate-on-timeout path deterministically); a non-positive +// or unparseable override falls back to the default. +const ELK_TIMEOUT_DEFAULT_MS = 5000; +function resolveElkTimeoutMs(): number { + const raw = Number(process.env.DRAWIO_ELK_TIMEOUT_MS); + return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : ELK_TIMEOUT_DEFAULT_MS; +} // Spacing is set >=150px on purpose so an ELK layout never trips the linter's // "gap between adjacent shapes < 150px" quality warning (acceptance #3). @@ -78,13 +89,57 @@ interface ElkGraph extends ElkNode { edges?: ElkEdge[]; } +/** + * Run one ELK layered layout on a worker thread and resolve with the laid-out + * graph. The timeout is enforced by `worker.terminate()` — a HARD kill, which is + * the only way to interrupt elkjs' synchronous crossing-minimisation once it has + * started. Rejects on timeout, worker error, or an early exit; the caller treats + * any rejection as "keep the original model" (best-effort layout). + */ +function layoutInWorker(graph: ElkGraph, timeoutMs: number): Promise { + return new Promise((resolve, reject) => { + const worker = new Worker( + new URL("./drawio-layout.worker.js", import.meta.url), + { workerData: { graph } }, + ); + let settled = false; + const finish = (fn: () => void) => { + if (settled) return; + settled = true; + clearTimeout(timer); + // Always tear the worker down: on the happy path so it does not linger, + // on timeout so the blocked synchronous ELK run is actually interrupted. + void worker.terminate(); + fn(); + }; + const timer = setTimeout( + () => finish(() => reject(new Error("ELK layout timed out"))), + timeoutMs, + ); + worker.once("message", (msg: { ok?: boolean; laid?: ElkGraph; error?: string }) => { + finish(() => + msg?.ok + ? resolve(msg.laid as ElkGraph) + : reject(new Error(msg?.error ?? "ELK layout failed")), + ); + }); + worker.once("error", (err) => finish(() => reject(err))); + worker.once("exit", (code) => { + // A clean exit after we already settled is normal (terminate()); only an + // unexpected early exit while still pending is a failure. + if (settled) return; + finish(() => reject(new Error(`ELK worker exited early (code ${code})`))); + }); + }); +} + /** * Apply an ELK layered layout to a drawio input and return a full mxGraphModel * string with rewritten geometry. Accepts the same three input forms as * drawioCreate (a bare model, an , or a list). Async because - * elkjs' layout() is promise-based. On any layout failure the ORIGINAL - * (normalized) model is returned unchanged — layout is best-effort polish, never - * a reason to fail the write. + * the layout runs on a worker thread. On any layout failure (including a + * terminate-on-timeout) the ORIGINAL (normalized) model is returned unchanged — + * layout is best-effort polish, never a reason to fail the write. */ export async function applyElkLayout(inputXml: string): Promise { const modelXml = normalizeInput(inputXml); @@ -150,26 +205,14 @@ export async function applyElkLayout(inputXml: string): Promise { }; let laid: ElkGraph; - let timer: ReturnType | undefined; try { - // elkjs ships a CJS default export whose interop shape varies across - // module systems; resolve the real constructor at runtime, then cast (the - // runtime call is verified — see the layout unit test). - const Ctor: any = (ELK as any).default ?? ELK; - const elk = new Ctor(); - // Race the layout against a wall-clock timeout so a graph that is under the - // node/edge caps but still pathologically slow can never wedge the server. - const timeout = new Promise((_, reject) => { - timer = setTimeout( - () => reject(new Error("ELK layout timed out")), - ELK_TIMEOUT_MS, - ); - }); - laid = (await Promise.race([elk.layout(graph as any), timeout])) as ElkGraph; + // Run the (synchronous-under-the-hood) ELK pass on a worker thread so the + // main event loop is never blocked, and enforce the wall-clock ceiling by + // terminating that worker on timeout. A graph under the node/edge caps but + // still pathologically slow is hard-killed instead of wedging anything. + laid = await layoutInWorker(graph, resolveElkTimeoutMs()); } catch { return modelXml; // best-effort: keep the model as-is on timeout or ELK failure - } finally { - if (timer) clearTimeout(timer); } // Collect computed geometry per node id (coords are parent-relative already). diff --git a/packages/mcp/src/lib/drawio-layout.worker.ts b/packages/mcp/src/lib/drawio-layout.worker.ts new file mode 100644 index 00000000..0313be93 --- /dev/null +++ b/packages/mcp/src/lib/drawio-layout.worker.ts @@ -0,0 +1,36 @@ +// Worker-thread entry for the ELK layered layout (issue #486, commit 1). +// +// elkjs' layout() returns a Promise but runs the actual crossing-minimisation +// SYNCHRONOUSLY — it blocks whatever thread it runs on for the whole pass. On +// the in-app MCP host that thread used to be the main NestJS event loop, so a +// pathological graph at the node/edge cap could wedge ALL HTTP/SSE/loopback +// traffic while it churned. Running it HERE, on a dedicated worker thread, keeps +// the main loop free; the parent enforces the wall-clock timeout by calling +// `worker.terminate()` — the only way to interrupt synchronous JS — since the +// in-process `setTimeout` race the parent used before could never fire while the +// same thread was blocked inside elkjs. +import { parentPort, workerData } from "node:worker_threads"; +import ELK from "elkjs/lib/elk.bundled.js"; + +interface WorkerInput { + graph: unknown; +} + +const { graph } = (workerData ?? {}) as WorkerInput; + +// elkjs ships a CJS default export whose interop shape varies across module +// systems; resolve the real constructor at runtime (same as the parent did). +const Ctor: any = (ELK as any).default ?? ELK; +const elk = new Ctor(); + +elk + .layout(graph as any) + .then((laid: unknown) => { + parentPort?.postMessage({ ok: true, laid }); + }) + .catch((err: unknown) => { + parentPort?.postMessage({ + ok: false, + error: err instanceof Error ? err.message : String(err), + }); + }); diff --git a/packages/mcp/test/unit/drawio-layout.test.mjs b/packages/mcp/test/unit/drawio-layout.test.mjs index a6998d17..ccf5eda3 100644 --- a/packages/mcp/test/unit/drawio-layout.test.mjs +++ b/packages/mcp/test/unit/drawio-layout.test.mjs @@ -101,6 +101,88 @@ test("DoS guard: a graph over the node cap is returned unchanged, quickly", asyn assert.ok(dt < 2000, `cap path should be fast, took ${dt}ms`); }); +/** Build a layered DAG near the caps: `n` vertices, up to ~2 edges each into the + * next layer of `layerSize`. Used as a real worst-case graph for the benchmark. */ +function layeredGraph(n, layerSize) { + let cells = ""; + for (let i = 2; i < 2 + n; i++) { + cells += + `` + + ``; + } + let ei = 0; + for (let i = 2; i < 2 + n; i++) { + for (const off of [layerSize, layerSize + 1]) { + const t = i + off; + if (t < 2 + n) cells += ``; + } + } + return ( + '' + + cells + + "" + ); +} + +test("terminate-on-timeout: a layout that exceeds the wall-clock ceiling is hard-killed and the original model is returned (#486)", async () => { + // A 1ms ceiling fires before the worker can even finish loading elkjs, so the + // parent must terminate() the worker and fall back to the ORIGINAL model. On + // the OLD in-process race this timer could never fire while the SAME thread was + // blocked inside elkjs — the fallback path was unreachable; here it works. + const prev = process.env.DRAWIO_ELK_TIMEOUT_MS; + process.env.DRAWIO_ELK_TIMEOUT_MS = "1"; + try { + const model = layeredGraph(400, 20); + const t0 = Date.now(); + const laid = await applyElkLayout(model); + const dt = Date.now() - t0; + // Original geometry is preserved verbatim: every vertex is still stacked at + // (10,10), proving NO ELK coordinates were applied (the pass was killed). + const verts = parseCells(laid).filter((c) => c.vertex); + assert.equal(verts.length, 400, "all vertices survived the fallback"); + for (const v of verts) { + assert.equal(v.geometry.x, 10, "x untouched -> layout was terminated"); + assert.equal(v.geometry.y, 10, "y untouched -> layout was terminated"); + } + // The kill is prompt: terminate() returns the call well under the natural + // layout time for a 400-node graph. + assert.ok(dt < 2000, `terminate path should be prompt, took ${dt}ms`); + } finally { + if (prev === undefined) delete process.env.DRAWIO_ELK_TIMEOUT_MS; + else process.env.DRAWIO_ELK_TIMEOUT_MS = prev; + } +}); + +test("benchmark guard: a worst-case graph AT the cap lays out without wedging the main event loop (#486)", async () => { + // ~500 nodes / ~1000 edges — a real worst case at the node/edge caps. The + // layout runs on a WORKER thread, so the MAIN event loop must stay responsive + // throughout: a timer scheduled on the main thread keeps firing while ELK + // churns. On the OLD synchronous-on-main-thread code this counter would be + // pinned at 0 for the whole layout (event loop wedged) — exactly the prod fire. + const model = layeredGraph(500, 20); + let mainLoopTicks = 0; + const iv = setInterval(() => { + mainLoopTicks++; + }, 2); + const t0 = Date.now(); + const laid = await applyElkLayout(model); + const dt = Date.now() - t0; + clearInterval(iv); + + assert.ok( + mainLoopTicks > 0, + "main event loop must stay responsive while ELK runs on the worker", + ); + // Benchmark guard: the worst-case graph actually LAYS OUT within the default + // ceiling (it did not fall back). At least one vertex moved off the stack. + const verts = parseCells(laid).filter((c) => c.vertex); + assert.equal(verts.length, 500, "all vertices survived"); + const moved = verts.some((v) => v.geometry.x !== 10 || v.geometry.y !== 10); + assert.ok(moved, "layout was applied (did not time out / fall back)"); + // Sanity ceiling well under the 5s wall-clock timeout. + assert.ok(dt < 5000, `worst-case layout should be under the ceiling, took ${dt}ms`); +}); + test("layout is best-effort: an empty/degenerate model is returned intact", async () => { const model = ''; From ee15278c8fbd4d95330862a6f0f958e9a65d69dc Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 02:29:41 +0300 Subject: [PATCH 02/41] =?UTF-8?q?fix(health):=20=D1=83=D1=81=D1=82=D1=80?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D1=82=D1=8C=20=D1=83=D1=82=D0=B5=D1=87=D0=BA?= =?UTF-8?q?=D1=83=20ioredis-=D0=BA=D0=BB=D0=B8=D0=B5=D0=BD=D1=82=D0=B0=20?= =?UTF-8?q?=D0=B2=20/health-=D0=BF=D1=80=D0=BE=D0=B1=D0=B5=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pingCheck строил new Redis(...) на КАЖДЫЙ вызов и делал disconnect() только на success-пути. Пока Redis лежит, каждый тик health-пробы добавлял свежий, вечно реконнектящийся клиент — неограниченный рост хэндлов/клиентов на всё время недоступности Redis. Теперь один долгоживущий probe-клиент, переиспользуемый между тиками: lazyConnect (конструктор не бросает и не коннектится жадно), maxRetriesPerRequest: 1 и enableOfflineQueue: false (проба фейлится быстро, команды не буферизуются), плюс listener на 'error' (иначе unhandled error роняет процесс). onModuleDestroy закрывает клиент при shutdown. Тест: интеграционный — N проб при лежащем Redis (реальный refused-порт, не мок поведения) создают РОВНО ОДИН клиент (на баге было бы N); onModuleDestroy освобождает клиент, следующая проба лениво строит новый. --- .../integrations/health/redis.health.spec.ts | 95 +++++++++++++++++++ .../src/integrations/health/redis.health.ts | 59 ++++++++++-- 2 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 apps/server/src/integrations/health/redis.health.spec.ts diff --git a/apps/server/src/integrations/health/redis.health.spec.ts b/apps/server/src/integrations/health/redis.health.spec.ts new file mode 100644 index 00000000..1fd59c26 --- /dev/null +++ b/apps/server/src/integrations/health/redis.health.spec.ts @@ -0,0 +1,95 @@ +import type { HealthIndicatorService } from '@nestjs/terminus'; +import type { EnvironmentService } from '../environment/environment.service'; + +/** + * Integration guard for the /health Redis-probe handle leak (#486, commit 2). + * + * The bug: `pingCheck` built `new Redis(...)` per call and only disconnected on + * the SUCCESS path, so when Redis is DOWN every probe tick added ANOTHER + * forever-reconnecting client — an unbounded handle/client leak for the duration + * of the outage. The fix reuses ONE long-lived probe client. + * + * This is an OBSERVABLE-property test, not an assertion on a mocked return value: + * we point the indicator at a REAL, refused TCP endpoint (a dead port) so ioredis + * genuinely fails to connect, run many probes, and assert the number of live + * Redis CLIENTS created stays at exactly ONE. `ioredis` is delegated to its real + * implementation (requireActual) — only the constructor is wrapped to COUNT the + * real clients it creates, which is precisely the leaking resource. + */ +const mockLiveClients: Array<{ status: string; disconnect: () => void }> = []; + +jest.mock('ioredis', () => { + const actual = jest.requireActual('ioredis'); + const RealRedis = actual.Redis ?? actual.default ?? actual; + class CountingRedis extends RealRedis { + constructor(...args: unknown[]) { + super(...(args as [])); + mockLiveClients.push(this as never); + } + } + return { ...actual, Redis: CountingRedis, default: CountingRedis }; +}); + +// Import AFTER the mock is registered so the class picks up the counting client. +import { RedisHealthIndicator } from './redis.health'; + +describe('RedisHealthIndicator handle leak (#486)', () => { + const indicatorService = { + check: (key: string) => ({ + up: () => ({ [key]: { status: 'up' } }), + down: (message: string) => ({ [key]: { status: 'down', message } }), + }), + } as unknown as HealthIndicatorService; + + // A port with (almost certainly) nothing listening -> connection refused fast. + const environmentService = { + getRedisUrl: () => 'redis://127.0.0.1:6399/0', + } as unknown as EnvironmentService; + + let indicator: RedisHealthIndicator; + + beforeEach(() => { + mockLiveClients.length = 0; + indicator = new RedisHealthIndicator(indicatorService, environmentService); + }); + + afterEach(() => { + indicator.onModuleDestroy(); + // Belt-and-braces: tear down anything the test created so ioredis reconnect + // timers do not keep the jest worker alive. + for (const c of mockLiveClients) { + try { + c.disconnect(); + } catch { + /* already gone */ + } + } + }); + + it('creates exactly ONE Redis client across many probes while Redis is DOWN', async () => { + const N = 8; + for (let i = 0; i < N; i++) { + const result = await indicator.pingCheck('redis'); + // Down endpoint -> every probe reports "down" (not an unhandled crash). + expect(result.redis.status).toBe('down'); + } + + // THE OBSERVABLE LEAK: on the buggy code this is N (a fresh, never-cleaned + // reconnecting client per probe). The fix reuses one shared client. + expect(mockLiveClients).toHaveLength(1); + }); + + it('onModuleDestroy releases the probe client (a later probe builds a fresh one)', async () => { + await indicator.pingCheck('redis'); + expect(mockLiveClients).toHaveLength(1); + + indicator.onModuleDestroy(); + // A second destroy is a safe no-op (probeClient was nulled). + indicator.onModuleDestroy(); + + // After shutdown the indicator lazily builds a NEW client on the next probe, + // proving the old one was truly released rather than reused. + await indicator.pingCheck('redis'); + expect(mockLiveClients).toHaveLength(2); + }); +}); diff --git a/apps/server/src/integrations/health/redis.health.ts b/apps/server/src/integrations/health/redis.health.ts index 96ed53b6..c9160125 100644 --- a/apps/server/src/integrations/health/redis.health.ts +++ b/apps/server/src/integrations/health/redis.health.ts @@ -2,33 +2,78 @@ import { HealthIndicatorResult, HealthIndicatorService, } from '@nestjs/terminus'; -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; import { EnvironmentService } from '../environment/environment.service'; import { Redis } from 'ioredis'; @Injectable() -export class RedisHealthIndicator { +export class RedisHealthIndicator implements OnModuleDestroy { private readonly logger = new Logger(RedisHealthIndicator.name); + /** + * ONE long-lived probe connection, reused across every /health tick. The old + * code built `new Redis(...)` per call and only `disconnect()`d on the SUCCESS + * path, so while Redis was DOWN every probe added a fresh, forever-reconnecting + * client — a handle leak that grew without bound for as long as the outage (and + * the health checker keeps polling) lasted. A single shared client keeps at most + * ONE background reconnect loop regardless of how many probes run. + */ + private probeClient: Redis | null = null; + constructor( private readonly healthIndicatorService: HealthIndicatorService, private environmentService: EnvironmentService, ) {} + private getProbeClient(): Redis { + if (!this.probeClient) { + this.probeClient = new Redis(this.environmentService.getRedisUrl(), { + // Constructing must never throw or eagerly connect; the first ping opens + // the socket. This lets us build the client once and reuse it. + lazyConnect: true, + // A health probe must fail FAST, not queue behind a stuck reconnect: one + // retry per request, and no offline queue so a ping while disconnected + // rejects immediately instead of buffering commands that pile up in RAM. + maxRetriesPerRequest: 1, + enableOfflineQueue: false, + }); + // ioredis emits 'error' on every failed (re)connect; with no listener that + // surfaces as an unhandled 'error' event and can crash the process. Swallow + // it here — pingCheck already reports health — and log at debug so a Redis + // outage does not flood the logs. + this.probeClient.on('error', (err) => { + this.logger.debug( + `Redis probe connection error: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); + } + return this.probeClient; + } + async pingCheck(key: string): Promise { const indicator = this.healthIndicatorService.check(key); try { - const redis = new Redis(this.environmentService.getRedisUrl(), { - maxRetriesPerRequest: 15, - }); - + const redis = this.getProbeClient(); await redis.ping(); - redis.disconnect(); return indicator.up(); } catch (e) { this.logger.error(e); return indicator.down(`${key} is not available`); } } + + onModuleDestroy(): void { + if (this.probeClient) { + // disconnect() (not quit()) tears the socket + reconnect loop down + // immediately without waiting on a round-trip to a possibly-down server. + // Do NOT removeAllListeners() with no event name — that would also strip + // ioredis' OWN internal listeners and break its teardown; our 'error' + // listener is harmless and dies with the dropped client reference. + this.probeClient.disconnect(); + this.probeClient = null; + } + } } From fc83ea28ab219b7fb4230c01d2f876862e3fc460 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 02:32:21 +0300 Subject: [PATCH 03/41] =?UTF-8?q?fix(metrics):=20bind=20=D0=BD=D0=B0=20127?= =?UTF-8?q?.0.0.1=20=D0=BF=D0=BE=20=D1=83=D0=BC=D0=BE=D0=BB=D1=87=D0=B0?= =?UTF-8?q?=D0=BD=D0=B8=D1=8E=20+=20METRICS=5FBIND/METRICS=5FTOKEN=20(#486?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /metrics слушал на 0.0.0.0 без какой-либо аутентификации — auth-less эндпоинт на всех интерфейсах. Теперь дефолтный bind — loopback 127.0.0.1; env METRICS_BIND переопределяет интерфейс (0.0.0.0 для скрейпера в отдельном контейнере, docmost:9464); опциональный METRICS_TOKEN включает Bearer-аутентификацию (запросы без точного токена получают 401). Доки скрейпа в .env.example обновлены. Тест: unit на дефолтный bind + env-переопределение + резолв токена; интеграционный по РЕАЛЬНОМУ сокету — listener забинден на 127.0.0.1, без токена /metrics отдаётся, с токеном без/с неверным Bearer → 401, с верным → 200. --- .env.example | 14 ++ .../metrics/metrics.server.spec.ts | 140 ++++++++++++++++++ .../integrations/metrics/metrics.server.ts | 50 ++++++- 3 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 apps/server/src/integrations/metrics/metrics.server.spec.ts diff --git a/.env.example b/.env.example index 0a032c26..e490f7c9 100644 --- a/.env.example +++ b/.env.example @@ -335,6 +335,20 @@ MCP_DOCMOST_PASSWORD= # VictoriaMetrics/Prometheus reaching it as :/metrics. # METRICS_PORT=9464 # +# METRICS_BIND — interface the /metrics listener binds to. DEFAULT 127.0.0.1 +# (loopback only), so the unauthenticated endpoint is NOT exposed on all +# interfaces. If the scraper runs in a SEPARATE container and reaches this as +# docmost:9464, set METRICS_BIND=0.0.0.0 — but then also set METRICS_TOKEN +# and/or keep the port on a private network, since /metrics is otherwise open. +# METRICS_BIND=127.0.0.1 +# +# METRICS_TOKEN — optional Bearer token guarding /metrics. When set, every +# scrape MUST send `Authorization: Bearer ` (others get 401). Configure +# the scraper with the same bearer token (e.g. VictoriaMetrics/vmagent +# `bearer_token`, Prometheus `authorization.credentials`). Leave unset only +# when the endpoint is bound to loopback or an otherwise-trusted network. +# METRICS_TOKEN= +# # 2) CLIENT_TELEMETRY_ENABLED — the public client perf-telemetry sink. # OFF by default. When true, the unauthenticated POST /api/telemetry/vitals # endpoint is registered and browsers collect + send web-vitals / editor diff --git a/apps/server/src/integrations/metrics/metrics.server.spec.ts b/apps/server/src/integrations/metrics/metrics.server.spec.ts new file mode 100644 index 00000000..1589db31 --- /dev/null +++ b/apps/server/src/integrations/metrics/metrics.server.spec.ts @@ -0,0 +1,140 @@ +import { get as httpGet } from 'node:http'; +import { AddressInfo } from 'node:net'; +import { createServer } from 'node:http'; + +// Drive the metrics HTTP server without the load-time METRICS_PORT gate: mock the +// registry so isMetricsEnabled()/getMetricsRegistry() are always satisfied. What +// we assert is observed over a REAL socket (bind address, status codes), not on +// the mock. +jest.mock('./metrics.registry', () => ({ + isMetricsEnabled: () => true, + getMetricsRegistry: () => ({ + metrics: async () => '# HELP up test\nup 1\n', + contentType: 'text/plain; version=0.0.4', + }), +})); + +import { + startMetricsServer, + closeMetricsServer, + resolveMetricsBind, + resolveMetricsToken, +} from './metrics.server'; + +/** Find a free TCP port (the metrics server requires METRICS_PORT > 0). */ +function freePort(): Promise { + return new Promise((resolve, reject) => { + const s = createServer(); + s.once('error', reject); + s.listen(0, '127.0.0.1', () => { + const p = (s.address() as AddressInfo).port; + s.close(() => resolve(p)); + }); + }); +} + +/** Minimal GET against 127.0.0.1:port with optional Authorization header. */ +function req( + port: number, + headers: Record = {}, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const r = httpGet( + { host: '127.0.0.1', port, path: '/metrics', headers }, + (res) => { + let body = ''; + res.on('data', (c) => (body += c)); + res.on('end', () => + resolve({ status: res.statusCode ?? 0, body }), + ); + }, + ); + r.on('error', reject); + }); +} + +describe('metrics server bind + auth (#486)', () => { + const saved = { + bind: process.env.METRICS_BIND, + token: process.env.METRICS_TOKEN, + port: process.env.METRICS_PORT, + }; + + afterEach(async () => { + await closeMetricsServer(); + process.env.METRICS_BIND = saved.bind; + process.env.METRICS_TOKEN = saved.token; + process.env.METRICS_PORT = saved.port; + delete process.env.METRICS_BIND; + delete process.env.METRICS_TOKEN; + }); + + describe('resolveMetricsBind', () => { + it('defaults to loopback 127.0.0.1', () => { + delete process.env.METRICS_BIND; + expect(resolveMetricsBind()).toBe('127.0.0.1'); + }); + it('honours the METRICS_BIND override', () => { + process.env.METRICS_BIND = '0.0.0.0'; + expect(resolveMetricsBind()).toBe('0.0.0.0'); + }); + it('treats a blank override as unset (loopback)', () => { + process.env.METRICS_BIND = ' '; + expect(resolveMetricsBind()).toBe('127.0.0.1'); + }); + }); + + describe('resolveMetricsToken', () => { + it('is null when unset', () => { + delete process.env.METRICS_TOKEN; + expect(resolveMetricsToken()).toBeNull(); + }); + it('returns the trimmed token when set', () => { + process.env.METRICS_TOKEN = ' s3cret '; + expect(resolveMetricsToken()).toBe('s3cret'); + }); + }); + + it('binds to loopback by default and serves /metrics without auth when no token', async () => { + delete process.env.METRICS_BIND; + delete process.env.METRICS_TOKEN; + const port = await freePort(); + process.env.METRICS_PORT = String(port); + + const server = startMetricsServer(); + expect(server).not.toBeNull(); + await new Promise((resolve) => { + if (server!.listening) resolve(); + else server!.once('listening', () => resolve()); + }); + // OBSERVABLE: the listener bound to loopback, not 0.0.0.0. + expect((server!.address() as AddressInfo).address).toBe('127.0.0.1'); + + const res = await req(port); + expect(res.status).toBe(200); + expect(res.body).toContain('up 1'); + }); + + it('rejects unauthenticated scrapes with 401 and accepts the exact Bearer token', async () => { + delete process.env.METRICS_BIND; + process.env.METRICS_TOKEN = 'topsecret'; + const port = await freePort(); + process.env.METRICS_PORT = String(port); + + const server = startMetricsServer(); + expect(server).not.toBeNull(); + + // No auth -> 401. + const noAuth = await req(port); + expect(noAuth.status).toBe(401); + + // Wrong token -> 401. + const wrong = await req(port, { authorization: 'Bearer nope' }); + expect(wrong.status).toBe(401); + + // Correct token -> 200 with the metrics body. + const ok = await req(port, { authorization: 'Bearer topsecret' }); + expect(ok.status).toBe(200); + expect(ok.body).toContain('up 1'); + }); +}); diff --git a/apps/server/src/integrations/metrics/metrics.server.ts b/apps/server/src/integrations/metrics/metrics.server.ts index a5c4d3e8..577cbecd 100644 --- a/apps/server/src/integrations/metrics/metrics.server.ts +++ b/apps/server/src/integrations/metrics/metrics.server.ts @@ -16,6 +16,30 @@ import { getMetricsRegistry, isMetricsEnabled } from './metrics.registry'; */ let metricsServer: Server | null = null; +/** + * Interface the metrics endpoint binds to. Defaults to LOOPBACK (127.0.0.1) so + * the unauthenticated `/metrics` surface is NOT exposed on all interfaces by + * default — the old `0.0.0.0` bind put an auth-less endpoint on every interface. + * Deployments where the scraper runs in a SEPARATE container (and reaches this as + * `docmost:9464`) set `METRICS_BIND=0.0.0.0`, ideally together with METRICS_TOKEN + * and/or a private network so the port is not world-readable. + */ +export function resolveMetricsBind(): string { + const raw = (process.env.METRICS_BIND ?? '').trim(); + return raw.length > 0 ? raw : '127.0.0.1'; +} + +/** + * Optional Bearer token guarding `/metrics`. When `METRICS_TOKEN` is set, every + * scrape must present `Authorization: Bearer `; unset (default) leaves the + * endpoint open (safe when bound to loopback / a trusted network). Returns the + * trimmed token or null when unset/blank. + */ +export function resolveMetricsToken(): string | null { + const raw = (process.env.METRICS_TOKEN ?? '').trim(); + return raw.length > 0 ? raw : null; +} + export function startMetricsServer(): Server | null { if (!isMetricsEnabled()) return null; @@ -31,8 +55,22 @@ export function startMetricsServer(): Server | null { return null; } + const bind = resolveMetricsBind(); + const token = resolveMetricsToken(); + const server = createServer(async (req, res) => { if (req.method === 'GET' && req.url === '/metrics') { + // Optional Bearer auth: reject scrapes without the exact token when one is + // configured. This is the auth layer the old all-interfaces bind lacked. + if (token) { + const auth = req.headers['authorization']; + if (auth !== `Bearer ${token}`) { + res.statusCode = 401; + res.setHeader('WWW-Authenticate', 'Bearer'); + res.end(); + return; + } + } try { const body = await register.metrics(); res.setHeader('Content-Type', register.contentType); @@ -48,10 +86,14 @@ export function startMetricsServer(): Server | null { res.end(); }); - // Bind on all interfaces: the scraper (VictoriaMetrics) reaches this from - // another container as docmost:9464. The port is not published to the host. - server.listen(port, '0.0.0.0', () => { - logger.log(`Metrics endpoint listening on :${port}/metrics`); + // Bind to loopback by default so the auth-less endpoint is not exposed on all + // interfaces. Set METRICS_BIND=0.0.0.0 (ideally with METRICS_TOKEN) when the + // scraper runs in a separate container and reaches this as docmost:9464. + server.listen(port, bind, () => { + logger.log( + `Metrics endpoint listening on ${bind}:${port}/metrics` + + (token ? ' (Bearer auth required)' : ''), + ); }); server.on('error', (err) => { From e1ebd79f3030a0bf5b69855790d05a571fec9107 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 02:36:43 +0300 Subject: [PATCH 04/41] =?UTF-8?q?fix(ai-chat):=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D0=BB=20beginRun=20=D1=84=D0=B5=D0=B9=D0=BB=D0=B8=D1=82?= =?UTF-8?q?=20=D1=85=D0=BE=D0=B4=20=D1=87=D0=B5=D1=81=D1=82=D0=BD=D1=8B?= =?UTF-8?q?=D0=BC=20503=20(A=5FRUN=5FBEGIN=5FFAILED)=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Раньше провал beginRun (кроме unique-violation), напр. blip пула БД, логировался и ход ПРОДОЛЖАЛСЯ без run-строки. В autonomous такой ран никто не абортит: /stop его не видит, дисконнект не абортит, one-run-гейт пропускает ВТОРОЙ ран — невидимый неостанавливаемый ран до рестарта. Теперь провал beginRun (кроме RunAlreadyActiveError → прежний 409) бросает ServiceUnavailableException с кодом A_RUN_BEGIN_FAILED ДО первого байта и до вставки user-строки (post-hijack catch контроллера отдаёт честный 503 на raw- сокет). Без ветвления по режимам — #487 наследует ту же политику. В тело кладём statusCode: 503 (object-arg исключение его не добавляет), чтобы клиент видел статус. Клиентский классификатор: ветка A_RUN_BEGIN_FAILED добавлена СТРОГО ДО generic- 503-матча — иначе показал бы «provider is not configured» вместо «временно, повторите». Тесты: unit fail-fast (stream() бросает 503 A_RUN_BEGIN_FAILED, ни байта в сокет, user-строка не вставлена; RunAlreadyActiveError по-прежнему 409); unit клиентского классификатора из ПОЛНОГО реального тела ответа с гвардом порядка. --- .../ai-chat/utils/error-message.test.ts | 19 +++ .../features/ai-chat/utils/error-message.ts | 15 +++ .../ai-chat/ai-chat.begin-run-failure.spec.ts | 109 ++++++++++++++++++ .../src/core/ai-chat/ai-chat.service.ts | 27 ++++- 4 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 apps/server/src/core/ai-chat/ai-chat.begin-run-failure.spec.ts diff --git a/apps/client/src/features/ai-chat/utils/error-message.test.ts b/apps/client/src/features/ai-chat/utils/error-message.test.ts index f60f8cb4..153455ac 100644 --- a/apps/client/src/features/ai-chat/utils/error-message.test.ts +++ b/apps/client/src/features/ai-chat/utils/error-message.test.ts @@ -23,6 +23,25 @@ describe("describeChatError", () => { }); }); + it("classifies an A_RUN_BEGIN_FAILED 503 as a temporary run-start failure, NOT provider-not-configured (#486)", () => { + // The FULL real body the server writes for a beginRun failure: a + // ServiceUnavailableException(object) whose response is serialized verbatim + // onto the raw socket, self-describing statusCode 503 + the run-start code. + const body = + '{"message":"Could not start the agent run. This is usually temporary — please try again.","code":"A_RUN_BEGIN_FAILED","statusCode":503}'; + expect(describeChatError(body, t)).toEqual({ + title: "Could not start the run", + detail: + "The agent run could not be started. This is usually temporary — please try again.", + }); + // ORDER GUARD: even though the body ALSO carries statusCode 503 (which the + // generic branch matches), the A_RUN_BEGIN_FAILED branch runs first, so it is + // never mislabeled "AI provider not configured". + expect(describeChatError(body, t).title).not.toBe( + "AI provider not configured", + ); + }); + it("classifies a dropped connection (ECONNRESET) as a lost-connection error", () => { expect( describeChatError("Cannot connect to API: read ECONNRESET", t).title, diff --git a/apps/client/src/features/ai-chat/utils/error-message.ts b/apps/client/src/features/ai-chat/utils/error-message.ts index cd6606b5..2d59c177 100644 --- a/apps/client/src/features/ai-chat/utils/error-message.ts +++ b/apps/client/src/features/ai-chat/utils/error-message.ts @@ -24,6 +24,21 @@ export function describeChatError( ): ChatErrorView { const msg = message ?? ""; + // Our own "could not start the run" gate (A_RUN_BEGIN_FAILED, #486): a 503 + // whose body carries this code is a TEMPORARY server-side failure while + // starting the run (e.g. a DB-pool blip), NOT an unconfigured provider. It MUST + // be matched STRICTLY BEFORE the generic 503 branch below, which would + // otherwise mislabel it "The AI provider is not configured" and tell the user + // to call an admin instead of just retrying. + if (/"code"\s*:\s*"A_RUN_BEGIN_FAILED"/.test(msg)) { + return { + title: t("Could not start the run"), + detail: t( + "The agent run could not be started. This is usually temporary — please try again.", + ), + }; + } + if (/"statusCode"\s*:\s*403\b/.test(msg)) { return { title: t("AI chat is disabled"), diff --git a/apps/server/src/core/ai-chat/ai-chat.begin-run-failure.spec.ts b/apps/server/src/core/ai-chat/ai-chat.begin-run-failure.spec.ts new file mode 100644 index 00000000..bd9e256b --- /dev/null +++ b/apps/server/src/core/ai-chat/ai-chat.begin-run-failure.spec.ts @@ -0,0 +1,109 @@ +import { + ConflictException, + Logger, + ServiceUnavailableException, +} from '@nestjs/common'; +import { AiChatService } from './ai-chat.service'; +import { RunAlreadyActiveError } from './ai-chat-run.service'; + +/** + * Fail-fast guard for beginRun failures (#486, commit 4). + * + * When runHooks.begin() rejects for a reason OTHER than RunAlreadyActiveError + * (e.g. a DB-pool blip), the turn must NOT continue untracked. The old code + * logged and streamed anyway, leaving a run with NO run-row: in autonomous mode + * nobody could abort it (/stop can't see it, disconnect doesn't abort it, and the + * one-run gate would admit a SECOND run) — an unstoppable invisible run until + * restart. The fix throws A_RUN_BEGIN_FAILED (503) BEFORE the first byte and + * before the user row is persisted. + * + * We drive `stream()` directly on a prototype instance wired with only the + * collaborators it touches before the throw, so the assertion is on the REAL + * control flow, not a mock of it. + */ +describe('AiChatService beginRun failure (#486)', () => { + function makeService(insertSpy: jest.Mock): AiChatService { + // Bypass the (heavy) DI constructor: exercise the real stream() method on a + // bare prototype instance with just the fields reached before the throw. + // `any` because the private `logger` field makes a typed intersection collapse. + const svc = Object.create(AiChatService.prototype); + svc.aiChatRepo = { + // Existing chat -> no insert path; chatId is kept as-is. + findById: jest.fn().mockResolvedValue({ id: 'chat1' }), + }; + svc.aiChatMessageRepo = { insert: insertSpy }; + svc.logger = new Logger('test'); + return svc as AiChatService; + } + + const baseArgs = () => { + const write = jest.fn(); + const res = { + raw: { write, writableEnded: false, headersSent: false }, + }; + return { + user: { id: 'u1' } as never, + workspace: { id: 'w1' } as never, + sessionId: 's1', + // openPage undefined -> resolveOpenPageContext returns null without any DB + // call; chatId present -> the existing-chat path. + body: { chatId: 'chat1', messages: [] } as never, + res: res as never, + signal: new AbortController().signal, + model: {} as never, + role: null, + write, + }; + }; + + it('throws A_RUN_BEGIN_FAILED (503) before the first byte and before persisting the user turn', async () => { + const insertSpy = jest.fn(); + const svc = makeService(insertSpy); + const { write, ...args } = baseArgs(); + + const runHooks = { + begin: jest.fn().mockRejectedValue(new Error('DB pool exhausted')), + } as never; + + let caught: unknown; + try { + await svc.stream({ ...args, runHooks }); + } catch (e) { + caught = e; + } + + expect(caught).toBeInstanceOf(ServiceUnavailableException); + const http = caught as ServiceUnavailableException; + expect(http.getStatus()).toBe(503); + expect(http.getResponse()).toMatchObject({ code: 'A_RUN_BEGIN_FAILED' }); + + // Fail-fast: nothing was written to the socket and NO user message row was + // persisted, so the turn left no orphan state to clean up. + expect(write).not.toHaveBeenCalled(); + expect(insertSpy).not.toHaveBeenCalled(); + }); + + it('still maps a lost-the-race RunAlreadyActiveError to a 409, not A_RUN_BEGIN_FAILED', async () => { + const insertSpy = jest.fn(); + const svc = makeService(insertSpy); + const { write, ...args } = baseArgs(); + + const runHooks = { + begin: jest.fn().mockRejectedValue(new RunAlreadyActiveError('chat1')), + } as never; + + let caught: unknown; + try { + await svc.stream({ ...args, runHooks }); + } catch (e) { + caught = e; + } + + expect(caught).toBeInstanceOf(ConflictException); + expect((caught as ConflictException).getResponse()).toMatchObject({ + code: 'A_RUN_ALREADY_ACTIVE', + }); + expect(write).not.toHaveBeenCalled(); + expect(insertSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/server/src/core/ai-chat/ai-chat.service.ts b/apps/server/src/core/ai-chat/ai-chat.service.ts index f4dbb31d..908c9200 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.ts @@ -4,6 +4,7 @@ import { Injectable, Logger, OnModuleInit, + ServiceUnavailableException, } from '@nestjs/common'; import { FastifyReply } from 'fastify'; import { @@ -797,12 +798,32 @@ export class AiChatService implements OnModuleInit { code: 'A_RUN_ALREADY_ACTIVE', }); } - // Any OTHER run-start failure must not break the turn — fall back to the - // socket signal (legacy behavior) and stream anyway. + // Any OTHER run-start failure (e.g. a DB-pool blip) must FAIL THE TURN, + // not silently stream without a run-row. The old fallback let the turn + // continue untracked: in autonomous mode nobody could then abort it — + // /stop can't see a run that doesn't exist, a client disconnect doesn't + // abort it, and the one-run-per-chat gate would let a SECOND run in. That + // is an unstoppable, invisible run until process restart. Reject NOW, + // BEFORE the first byte (nothing is written yet, no user row inserted, no + // MCP lease taken), so the controller's post-hijack catch turns this + // HttpException into an honest 503 on the raw socket. Same policy for BOTH + // modes — #487 inherits it (no mode-branching here). this.logger.error( - `Failed to begin agent run (chat ${chatId}); streaming without run tracking`, + `Failed to begin agent run (chat ${chatId}); failing the turn`, err as Error, ); + throw new ServiceUnavailableException({ + message: + 'Could not start the agent run. This is usually temporary — please try again.', + code: 'A_RUN_BEGIN_FAILED', + // Self-describe the status in the body: the controller's post-hijack + // catch writes getResponse() verbatim onto the raw socket, and an + // object-arg HttpException does NOT inject statusCode. Without it the + // client's 503 classifier (which reads the body JSON) could not see the + // status. With it present, the client's A_RUN_BEGIN_FAILED branch (which + // runs strictly before the generic-503 branch) shows "temporary, retry". + statusCode: 503, + }); } } From 26d9a70a1c3564f7fbbb45c87464b049f6c93d44 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 02:50:19 +0300 Subject: [PATCH 05/41] =?UTF-8?q?fix(share):=20=D0=BD=D0=B5=20=D1=83=D1=82?= =?UTF-8?q?=D0=B5=D0=BA=D0=B0=D1=82=D1=8C=20errorText=20=D1=82=D1=83=D0=BB?= =?UTF-8?q?=D0=BE=D0=B2=20=D0=B8=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B0=D0=B9?= =?UTF-8?q?=D0=B4=D0=B5=D1=80=D0=B0=20=D0=B0=D0=BD=D0=BE=D0=BD=D0=B8=D0=BC?= =?UTF-8?q?=D1=83=20=D0=B2=20=D0=BF=D1=83=D0=B1=D0=BB=D0=B8=D1=87=D0=BD?= =?UTF-8?q?=D0=BE=D0=BC=20=D1=88=D1=8D=D1=80=D0=B5=20(closes=20#394)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SECURITY. В публичном share-чате сырой текст ошибки тула или провайдера утекал анонимному читателю. Три слоя, все обязательны: (1) Рендер-гейт: prop showErrors в ToolCallCard (протянут через MessageList/ MessageItem), share-виджет передаёт false — сырой errorText не рисуется. Но рендер-гейт маскирует только DOM, не байты. (2) Санитизация на уровне share-тулсета (авторитетно): forShare оборачивает execute каждого тула catch'ем. Своя ShareToolError (безопасные строки: «page not available in this share») пробрасывается, ЛЮБАЯ другая ошибка → generic «tool could not complete», полный текст только в серверный лог. Одно место закрывает байты (атомарный tool-output-error фрейм v6), рендер и контекст модели; self- correction сохранена. (3) Анонимный onError пайпа: ShareToolError → её безопасное сообщение; иначе describeProviderError (statusCode + тело: внутренний baseUrl/модель) только в лог, читателю — фиксированная классифицированная строка (rate-limited/unavailable/ provider error). Тест: интеграционный с РЕАЛЬНЫМ падением тула и провайдера — assert по СЫРЫМ SSE- БАЙТАМ (не по DOM): секрет/baseUrl/стек отсутствуют, видна безопасная строка, полный текст провайдера ушёл в серверный лог. --- .../ai-chat/components/message-item.tsx | 10 + .../ai-chat/components/message-list.tsx | 8 + .../ai-chat/components/tool-call-card.tsx | 13 +- .../share/components/share-ai-widget.tsx | 4 + .../public-share-chat.error-leak.spec.ts | 241 ++++++++++++++++++ .../core/ai-chat/public-share-chat.service.ts | 56 +++- .../tools/public-share-chat-tools.service.ts | 75 +++++- 7 files changed, 396 insertions(+), 11 deletions(-) create mode 100644 apps/server/src/core/ai-chat/public-share-chat.error-leak.spec.ts diff --git a/apps/client/src/features/ai-chat/components/message-item.tsx b/apps/client/src/features/ai-chat/components/message-item.tsx index 130b15dc..878477b4 100644 --- a/apps/client/src/features/ai-chat/components/message-item.tsx +++ b/apps/client/src/features/ai-chat/components/message-item.tsx @@ -47,6 +47,13 @@ interface MessageItemProps { * agent's raw query/argument text. */ showInput?: boolean; + /** + * Forwarded to ToolCallCard: whether a failed tool card renders its raw + * errorText. Defaults to true (internal chat). The public share passes false so + * internal detail in a tool error is never painted (belt to the server-side + * byte sanitization). + */ + showErrors?: boolean; /** * Neutralize internal/relative markdown links in the rendered answer (drop * their href so they become inert text). Defaults to false (internal chat, @@ -125,6 +132,7 @@ function MessageItem({ message, showCitations = true, showInput = true, + showErrors = true, neutralizeInternalLinks = false, assistantName, turnStreaming = false, @@ -219,6 +227,7 @@ function MessageItem({ part={part as unknown as ToolUiPart} showCitations={showCitations} showInput={showInput} + showErrors={showErrors} /> ); } @@ -284,6 +293,7 @@ export function arePropsEqual( prev.signature === next.signature && prev.showCitations === next.showCitations && prev.showInput === next.showInput && + prev.showErrors === next.showErrors && prev.neutralizeInternalLinks === next.neutralizeInternalLinks && prev.assistantName === next.assistantName && // The turn-end flip re-renders every row once (cheap, terminal event) — diff --git a/apps/client/src/features/ai-chat/components/message-list.tsx b/apps/client/src/features/ai-chat/components/message-list.tsx index d5c9c377..a208361f 100644 --- a/apps/client/src/features/ai-chat/components/message-list.tsx +++ b/apps/client/src/features/ai-chat/components/message-list.tsx @@ -32,6 +32,12 @@ interface MessageListProps { * doesn't see the agent's raw query/argument text. */ showInput?: boolean; + /** + * Forwarded to MessageItem -> ToolCallCard: whether a failed tool card renders + * its raw errorText. Defaults to true (internal chat). The public share passes + * false so internal detail in a tool error is never painted. + */ + showErrors?: boolean; /** * Forwarded to MessageItem: neutralize internal/relative markdown links in * the rendered answers (drop their href so they render as inert text). @@ -127,6 +133,7 @@ export default function MessageList({ emptyState, showCitations = true, showInput = true, + showErrors = true, neutralizeInternalLinks = false, assistantName, }: MessageListProps) { @@ -217,6 +224,7 @@ export default function MessageList({ signature={messageSignature(message)} showCitations={showCitations} showInput={showInput} + showErrors={showErrors} neutralizeInternalLinks={neutralizeInternalLinks} assistantName={assistantName} // Turn-level liveness, gated to the TAIL row: only the tail message diff --git a/apps/client/src/features/ai-chat/components/tool-call-card.tsx b/apps/client/src/features/ai-chat/components/tool-call-card.tsx index 30efb19f..faaea631 100644 --- a/apps/client/src/features/ai-chat/components/tool-call-card.tsx +++ b/apps/client/src/features/ai-chat/components/tool-call-card.tsx @@ -30,6 +30,16 @@ interface ToolCallCardProps { * the extra summary line, leaving the card (the action log) intact. */ showInput?: boolean; + /** + * Whether to render the tool's raw errorText on a failed call. Defaults to true + * (the internal chat, where the operator may debug). The public share passes + * false: a tool error string can carry internal detail (an internal page title, + * a stack fragment, a provider message). This is the RENDER gate only — the + * authoritative fix also sanitizes the bytes server-side (see + * PublicShareChatToolsService.forShare), so a share reader never receives raw + * error text over the wire, not just never sees it painted (#394). + */ + showErrors?: boolean; } /** @@ -41,6 +51,7 @@ export default function ToolCallCard({ part, showCitations = true, showInput = true, + showErrors = true, }: ToolCallCardProps) { const { t } = useTranslation(); const toolName = getToolName(part); @@ -74,7 +85,7 @@ export default function ToolCallCard({ )} - {state === "error" && part.errorText && ( + {state === "error" && showErrors && part.errorText && ( {part.errorText} diff --git a/apps/client/src/features/share/components/share-ai-widget.tsx b/apps/client/src/features/share/components/share-ai-widget.tsx index 59b2efb5..6a1930c8 100644 --- a/apps/client/src/features/share/components/share-ai-widget.tsx +++ b/apps/client/src/features/share/components/share-ai-widget.tsx @@ -168,6 +168,10 @@ export default function ShareAiWidget({ // Anonymous reader: suppress the tool-argument summary line so the // agent's raw query/argument text isn't shown on the public share. showInput={false} + // Anonymous reader: never paint a tool's raw errorText (it can carry + // internal detail). This is the render gate; the bytes are also + // sanitized server-side in PublicShareChatToolsService.forShare (#394). + showErrors={false} // Anonymous reader: neutralize internal/relative links in the // assistant's markdown so internal UUIDs/auth-gated routes don't // leak as clickable links (external http(s) links are kept). diff --git a/apps/server/src/core/ai-chat/public-share-chat.error-leak.spec.ts b/apps/server/src/core/ai-chat/public-share-chat.error-leak.spec.ts new file mode 100644 index 00000000..4d6151de --- /dev/null +++ b/apps/server/src/core/ai-chat/public-share-chat.error-leak.spec.ts @@ -0,0 +1,241 @@ +// Break the editor-ext import chain (share.service -> collaboration.util -> +// @docmost/editor-ext -> @tiptap/core) that is unresolvable in this jest env and +// pre-existingly breaks these specs. jsonToMarkdown is never reached in these +// tests (the tools fail before rendering markdown). +jest.mock('../../collaboration/collaboration.util', () => ({ + jsonToMarkdown: () => '', +})); + +import { Logger } from '@nestjs/common'; +import { MockLanguageModelV3, simulateReadableStream } from 'ai/test'; +import { PublicShareChatService } from './public-share-chat.service'; +import { PublicShareChatToolsService } from './tools/public-share-chat-tools.service'; + +/** + * SECURITY integration guard for #394 (commit 5): a tool's or the provider's raw + * error text must NOT leak to an anonymous public-share reader. + * + * The render gate (ToolCallCard showErrors=false) hides the text in the DOM but + * NOT on the wire, so this test asserts on the RAW SSE BYTES the server writes — + * exactly the channel the render gate masks. We drive the real + * PublicShareChatService.stream() with a real share toolset (its underlying + * services mocked to fail) and a mock model, then inspect every byte piped to the + * fake socket. + */ + +// A minimal ServerResponse stand-in that records every written chunk. +class FakeSocket { + chunks: string[] = []; + statusCode = 200; + writableEnded = false; + destroyed = false; + headersSent = false; + writeHead(): this { + this.headersSent = true; + return this; + } + setHeader(): void {} + removeHeader(): void {} + getHeader(): undefined { + return undefined; + } + flushHeaders(): void {} + write(chunk: unknown): boolean { + this.chunks.push( + typeof chunk === 'string' ? chunk : Buffer.from(chunk as never).toString('utf8'), + ); + return true; + } + end(chunk?: unknown): void { + if (chunk) this.write(chunk); + this.writableEnded = true; + } + on(): this { + return this; + } + once(): this { + return this; + } + get body(): string { + return this.chunks.join(''); + } +} + +/** Mock model that issues one getSharePage tool call, then finishes with text. */ +function toolCallingModel(): MockLanguageModelV3 { + let call = 0; + return new MockLanguageModelV3({ + doStream: async () => { + call++; + if (call === 1) { + return { + stream: simulateReadableStream({ + chunks: [ + { type: 'stream-start' as const, warnings: [] }, + { type: 'tool-input-start' as const, id: 't1', toolName: 'getSharePage' }, + { type: 'tool-input-end' as const, id: 't1' }, + { + type: 'tool-call' as const, + toolCallId: 't1', + toolName: 'getSharePage', + input: '{"pageId":"secret-page"}', + }, + { + type: 'finish' as const, + finishReason: { unified: 'tool-calls' as const, raw: 'tool_calls' }, + usage: { + inputTokens: { total: 1, noCache: undefined, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 1, text: 1, reasoning: undefined }, + }, + }, + ], + }), + }; + } + return { + stream: simulateReadableStream({ + chunks: [ + { type: 'stream-start' as const, warnings: [] }, + { type: 'text-start' as const, id: '1' }, + { type: 'text-delta' as const, id: '1', delta: 'Sorry.' }, + { type: 'text-end' as const, id: '1' }, + { + type: 'finish' as const, + finishReason: { unified: 'stop' as const, raw: 'stop' }, + usage: { + inputTokens: { total: 1, noCache: undefined, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 1, text: 1, reasoning: undefined }, + }, + }, + ], + }), + }; + }, + }); +} + +/** Mock model whose stream emits a provider error carrying an internal secret. */ +function providerErrorModel(secret: string): MockLanguageModelV3 { + return new MockLanguageModelV3({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: [ + { type: 'stream-start' as const, warnings: [] }, + { + type: 'error' as const, + error: { + statusCode: 503, + message: 'Service Unavailable', + responseBody: `upstream ${secret} model=internal-gpt`, + }, + }, + ], + }), + }), + }); +} + +function makeService(toolsService: PublicShareChatToolsService): { + svc: PublicShareChatService; + logSpy: jest.SpyInstance; +} { + const svc = Object.create(PublicShareChatService.prototype); + const logger = new Logger('test'); + const logSpy = jest.spyOn(logger, 'error').mockImplementation(() => undefined); + jest.spyOn(logger, 'warn').mockImplementation(() => undefined); + svc.tools = toolsService; + svc.logger = logger; + svc.tokenBudget = { record: jest.fn().mockResolvedValue(undefined) }; + return { svc, logSpy }; +} + +async function runStream( + svc: PublicShareChatService, + model: MockLanguageModelV3, +): Promise { + const socket = new FakeSocket(); + await svc.stream({ + workspaceId: 'ws1', + shareId: 'share1', + share: { id: 'share1', pageId: 'p1', sharedPage: { id: 'p1', title: 'Docs' } }, + openedPage: null, + messages: [ + { id: 'm1', role: 'user', parts: [{ type: 'text', text: 'read the page' }] } as never, + ], + res: { raw: socket } as never, + signal: new AbortController().signal, + model: model as never, + role: null, + }); + // Let the piped stream drain fully. + await new Promise((r) => setTimeout(r, 300)); + return socket; +} + +describe('public share chat error leak (#394)', () => { + afterEach(() => jest.restoreAllMocks()); + + it('does NOT leak a tool\'s raw internal error to the SSE bytes (generic classified string instead)', async () => { + const SECRET = 'INTERNAL_baseUrl_http://provider.internal:8080/v1'; + const shareService = { + // The canonical boundary throws a RAW internal error (with a secret). + resolveReadableSharePage: jest + .fn() + .mockRejectedValue(new Error(`db failed at ${SECRET} stack@line42`)), + }; + const tools = new PublicShareChatToolsService( + shareService as never, + {} as never, + {} as never, + ); + const { svc } = makeService(tools); + + const socket = await runStream(svc, toolCallingModel()); + + // The tool-output-error frame is present on the wire... + expect(socket.body).toContain('tool-output-error'); + // ...but it carries ONLY the generic classified string — never the secret, + // the raw driver message, or a stack fragment. + expect(socket.body).toContain('The tool could not complete the request.'); + expect(socket.body).not.toContain(SECRET); + expect(socket.body).not.toContain('stack@line42'); + expect(socket.body).not.toContain('db failed'); + }); + + it('passes a SAFE ShareToolError message (page not available) through to the bytes', async () => { + const shareService = { + // Not found in this share -> the tool throws the classified SAFE message. + resolveReadableSharePage: jest.fn().mockResolvedValue(null), + }; + const tools = new PublicShareChatToolsService( + shareService as never, + {} as never, + {} as never, + ); + const { svc } = makeService(tools); + + const socket = await runStream(svc, toolCallingModel()); + expect(socket.body).toContain('tool-output-error'); + expect(socket.body).toContain('not available in this share'); + }); + + it('does NOT leak a provider error (statusCode + response body) to the SSE bytes', async () => { + const SECRET = 'http://provider.internal:8080'; + const tools = new PublicShareChatToolsService( + {} as never, + {} as never, + {} as never, + ); + const { svc, logSpy } = makeService(tools); + + const socket = await runStream(svc, providerErrorModel(SECRET)); + + // The anon sees a fixed classified string, not the provider body/baseUrl/model. + expect(socket.body).toContain('temporarily unavailable'); + expect(socket.body).not.toContain(SECRET); + expect(socket.body).not.toContain('internal-gpt'); + // The FULL provider detail is logged server-side only. + const logged = logSpy.mock.calls.map((c) => String(c[0])).join('\n'); + expect(logged).toContain(SECRET); + }); +}); diff --git a/apps/server/src/core/ai-chat/public-share-chat.service.ts b/apps/server/src/core/ai-chat/public-share-chat.service.ts index a98e738f..236067eb 100644 --- a/apps/server/src/core/ai-chat/public-share-chat.service.ts +++ b/apps/server/src/core/ai-chat/public-share-chat.service.ts @@ -12,7 +12,10 @@ import { AiAgentRoleRepo } from '@docmost/db/repos/ai-agent-roles/ai-agent-roles import { AiAgentRole } from '@docmost/db/types/entity.types'; import { AiService } from '../../integrations/ai/ai.service'; import { AiSettingsService } from '../../integrations/ai/ai-settings.service'; -import { PublicShareChatToolsService } from './tools/public-share-chat-tools.service'; +import { + PublicShareChatToolsService, + ShareToolError, +} from './tools/public-share-chat-tools.service'; import { buildShareSystemPrompt } from './public-share-chat.prompt'; import { roleModelOverride } from './roles/role-model-config'; import { @@ -102,6 +105,30 @@ export function filterShareTranscript(messages: UIMessage[]): UIMessage[] { ); } +/** + * Fixed, classified strings an ANONYMOUS share reader may see when the assistant + * stream fails (#394). These reveal NOTHING about the internal provider, its + * baseUrl, the model name, or the raw response body — unlike describeProviderError + * (which is for the server log / the authenticated operator only). We classify by + * HTTP status where available so the reader still gets a useful hint (retry vs. + * give up) without any internal detail. + */ +export function classifyAnonStreamError(error: unknown): string { + const status = + typeof error === 'object' && error !== null + ? (error as { statusCode?: number }).statusCode + : undefined; + if (status === 429) { + return 'The assistant is receiving too many requests right now. Please try again shortly.'; + } + if (typeof status === 'number' && status >= 500) { + return 'The assistant is temporarily unavailable. Please try again.'; + } + // Any other failure (including a bare connection error with no status): a + // single neutral line. No provider identity, no config, no response body. + return 'The assistant could not complete your request. Please try again.'; +} + /** * Anonymous, read-only AI assistant for a single PUBLIC share tree. * @@ -318,11 +345,28 @@ export class PublicShareChatService { result.pipeUIMessageStreamToResponse(res.raw, { headers: { 'X-Accel-Buffering': 'no' }, onError: (error: unknown) => { - // Reuse the shared formatter so provider error formatting stays - // unified between the log line and the streamed error message — a - // share reader sees 402/429/503 causes consistently with the - // authenticated path. - return describeProviderError(error, 'AI stream error'); + // SECURITY (#394): the string this returns is written verbatim into the + // SSE error frame delivered to an ANONYMOUS reader (for a tool failure + // it becomes the atomic `tool-output-error` frame's errorText; for a + // stream/provider failure, the terminal error frame). + // + // A ShareToolError is already a classified, safe tool message (see + // PublicShareChatToolsService.wrapToolErrors) — pass it through so the + // reader still gets the useful "page not available in this share" hint. + if (error instanceof ShareToolError) { + return error.message; + } + // Anything else is a provider/stream error. describeProviderError + // bundles the provider statusCode AND response body, which can carry the + // internal baseUrl or model name — NEVER expose that to the public. Log + // the full detail server-side only and return a fixed classified string. + this.logger.error( + `Public share chat pipe error: ${describeProviderError( + error, + 'AI stream error', + )}`, + ); + return classifyAnonStreamError(error); }, }); diff --git a/apps/server/src/core/ai-chat/tools/public-share-chat-tools.service.ts b/apps/server/src/core/ai-chat/tools/public-share-chat-tools.service.ts index 2d2da79d..5467234f 100644 --- a/apps/server/src/core/ai-chat/tools/public-share-chat-tools.service.ts +++ b/apps/server/src/core/ai-chat/tools/public-share-chat-tools.service.ts @@ -7,6 +7,22 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo'; import { jsonToMarkdown } from '../../../collaboration/collaboration.util'; import { modelFriendlyInput } from './model-friendly-input'; +/** + * A tool error whose message is DELIBERATELY safe to expose to an anonymous + * share reader (and to the model, for self-correction). Every OTHER thrown error + * is treated as internal and replaced with a generic string by `wrapToolErrors`, + * so a raw exception message — an internal page title, a DB/stack fragment, a + * driver detail — never rides the public UI stream (#394). + */ +export class ShareToolError extends Error {} + +// The only two classified strings an anonymous reader may ever see from a tool +// failure. The specific one keeps the model's self-correction useful ("try a +// different page"); the generic one reveals nothing about the internal fault. +const SHARE_TOOL_ERROR_NOT_AVAILABLE = + 'The requested page is not available in this share.'; +const SHARE_TOOL_ERROR_GENERIC = 'The tool could not complete the request.'; + /** * Isolated, READ-ONLY toolset for the ANONYMOUS public-share assistant. * @@ -44,7 +60,7 @@ export class PublicShareChatToolsService { * are NO write tools, NO comments/history, NO cross-space or external tools. */ forShare(shareId: string, workspaceId: string): Record { - return { + return this.wrapToolErrors({ searchSharePages: tool({ description: 'Search the pages of THIS published documentation share for a ' + @@ -96,7 +112,7 @@ export class PublicShareChatToolsService { execute: async ({ pageId }) => { const id = (pageId ?? '').trim(); if (!id) { - throw new Error('A pageId is required.'); + throw new ShareToolError('A pageId is required.'); } // Resolve via the SINGLE canonical share-access boundary: confirms the // page resolves to THIS share (recursive CTE up the tree, honouring @@ -112,7 +128,7 @@ export class PublicShareChatToolsService { workspaceId, ); if (!resolved) { - throw new Error('That page is not part of this published share.'); + throw new ShareToolError(SHARE_TOOL_ERROR_NOT_AVAILABLE); } const { page } = resolved; @@ -193,6 +209,57 @@ export class PublicShareChatToolsService { } }, }), - }; + }); + } + + /** + * Wrap every tool's `execute` so a THROWN error is sanitized in ONE place — + * closing the byte leak, the render, and the model context at once (#394). + * + * The AI SDK surfaces a tool-execution throw as an atomic `tool-output-error` + * frame on the v6 UI stream whose `errorText` is the thrown message; on the + * public share that frame goes straight to an anonymous reader. Unwrapped, a + * raw exception (an internal page title, a DB/stack fragment, a driver detail) + * would ride that frame verbatim. Here we catch it, LOG the full detail + * server-side only, and re-throw a CLASSIFIED, safe error: the tool's own + * intentional ShareToolError messages pass through (they keep the model's + * self-correction useful), everything else collapses to a generic string. + */ + private wrapToolErrors( + tools: Record, + ): Record { + const wrapped: Record = {}; + for (const [name, t] of Object.entries(tools)) { + const original = t.execute; + if (typeof original !== 'function') { + wrapped[name] = t; + continue; + } + wrapped[name] = { + ...t, + execute: async (args: unknown, options: unknown) => { + try { + return await ( + original as (a: unknown, o: unknown) => Promise + )(args, options); + } catch (err) { + const safe = + err instanceof ShareToolError + ? err.message + : SHARE_TOOL_ERROR_GENERIC; + // Full detail to the server log ONLY — never to the anon. + this.logger.warn( + `Public share tool "${name}" failed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + // This safe string is ALL that rides the tool-output-error frame, + // becomes model context, and could be rendered — one choke point. + throw new ShareToolError(safe); + } + }, + } as Tool; + } + return wrapped; } } From b2e5b51420ab13aa14b3e530af8aca874bdd8ac0 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 02:54:03 +0300 Subject: [PATCH 06/41] =?UTF-8?q?fix(ai):=20drain-hang=20=D0=B2=20writeToS?= =?UTF-8?q?erverResponse=20=E2=80=94=20=D1=80=D0=B0=D1=81=D1=88=D0=B8?= =?UTF-8?q?=D1=80=D0=B8=D1=82=D1=8C=20patches/ai@6.0.134.patch=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Серверная ai@6.0.134 в writeToServerResponse при backpressure (write()===false) ждала ТОЛЬКО once('drain'). Если клиент отвалился мид-запись, сокет не дренится, await не резолвится: read-цикл паркуется НАВСЕГДА, finally{response.end()} недостижим, reader и буферы висят до рестарта. В autonomous ран продолжает лить вывод после дисконнекта → КАЖДЫЙ дисконнект мид-ран оставляет висящий пайп. Плюс read() — fire-and-forget с throw → unhandledRejection. Патч расширен (index.js и index.mjs): Promise.race drain/close/error с гигиеной once-слушателей (все три снимаются на первом settle — не копятся по одному на stall); при close/error — reader.cancel() и выход (безопасно для detached-ранов: независимый дренаж делает consumeStream); rejection read() поглощается с логом. pnpm-lock patch_hash перегенерён (patch-commit). Выравнивание версии ai — отдельный коммит (#495), не здесь. Тест: трипвайр-спека в apps/server (только там резолвится патченная копия) по образцу ai-sdk-partial-output.patch.spec — дисконнект мид-запись без drain завершает ответ (end() вызван, reader не висит), read()-throw не даёт unhandledRejection, оба dist-билда несут маркер PATCH(docmost #486). --- .../ai/ai-sdk-drain-hang.patch.spec.ts | 133 ++++++++++++++++++ patches/ai@6.0.134.patch | 120 +++++++++++++++- pnpm-lock.yaml | 12 +- 3 files changed, 253 insertions(+), 12 deletions(-) create mode 100644 apps/server/src/integrations/ai/ai-sdk-drain-hang.patch.spec.ts diff --git a/apps/server/src/integrations/ai/ai-sdk-drain-hang.patch.spec.ts b/apps/server/src/integrations/ai/ai-sdk-drain-hang.patch.spec.ts new file mode 100644 index 00000000..39582ee5 --- /dev/null +++ b/apps/server/src/integrations/ai/ai-sdk-drain-hang.patch.spec.ts @@ -0,0 +1,133 @@ +import { readFileSync } from 'fs'; +import { EventEmitter } from 'node:events'; +import { streamText } from 'ai'; +import { MockLanguageModelV3, simulateReadableStream } from 'ai/test'; + +/** + * Regression tests for the writeToServerResponse drain-hang fix in + * patches/ai@6.0.134.patch (#486, commit 6). + * + * Unpatched ai@6.0.134's writeToServerResponse awaits ONLY `once("drain")` when + * response.write() returns false (backpressure). If the client disconnects + * mid-write the socket never drains, so that await never resolves: the read loop + * parks FOREVER, its `finally { response.end() }` is unreachable, and the stream + * reader + buffered chunks are pinned until process restart. In autonomous mode + * the run keeps producing output after the disconnect, so EVERY mid-run + * disconnect leaks a hung pipe. The patch races drain against close/error, and on + * a terminal socket event cancels the reader and breaks so `finally` always runs. + * + * This drives the REAL patched writeToServerResponse through the public + * pipeUIMessageStreamToResponse API with a response that never drains and closes + * mid-write — exactly the leak scenario. + */ + +/** A ServerResponse-like emitter whose first write() stalls (returns false) and + * then "closes" like a disconnecting client — never firing 'drain'. */ +class DisconnectingResponse extends EventEmitter { + ended = false; + writeCount = 0; + statusCode = 200; + writableEnded = false; + destroyed = false; + writeHead(): this { + return this; + } + setHeader(): void {} + flushHeaders(): void {} + write(): boolean { + this.writeCount++; + if (this.writeCount === 1) { + // Simulate the client vanishing mid-write: backpressure (false) and then a + // 'close' on the next tick, and CRUCIALLY never a 'drain'. Unpatched, the + // loop would await drain forever here. + setImmediate(() => this.emit('close')); + return false; + } + return true; + } + end(): void { + this.ended = true; + this.writableEnded = true; + this.emit('finish'); + } +} + +function makeModel() { + return new MockLanguageModelV3({ + doStream: async () => ({ + stream: simulateReadableStream({ + chunks: [ + { type: 'stream-start' as const, warnings: [] }, + { type: 'text-start' as const, id: '1' }, + { type: 'text-delta' as const, id: '1', delta: 'hello ' }, + { type: 'text-delta' as const, id: '1', delta: 'world' }, + { type: 'text-end' as const, id: '1' }, + { + type: 'finish' as const, + finishReason: { unified: 'stop' as const, raw: 'stop' }, + usage: { + inputTokens: { total: 1, noCache: undefined, cacheRead: undefined, cacheWrite: undefined }, + outputTokens: { total: 1, text: 1, reasoning: undefined }, + }, + }, + ], + }), + }), + }); +} + +describe('ai@6.0.134 pnpm patch: writeToServerResponse drain-hang (#486)', () => { + it('ends the response (does NOT hang) when the socket closes mid-write without draining', async () => { + const result = streamText({ model: makeModel(), prompt: 'hi' }); + const res = new DisconnectingResponse(); + // Drain the SDK stream independently, like the production detached path. + void result.consumeStream({ onError: () => undefined }); + result.pipeUIMessageStreamToResponse(res as never); + + // TRIPWIRE: the patched loop exits on 'close' and runs finally -> end(). + // Unpatched, it awaits 'drain' forever and this never becomes true. + await new Promise((resolve, reject) => { + const started = Date.now(); + const poll = setInterval(() => { + if (res.ended) { + clearInterval(poll); + resolve(); + } else if (Date.now() - started > 3000) { + clearInterval(poll); + reject(new Error('writeToServerResponse hung: response never ended')); + } + }, 20); + }); + + expect(res.ended).toBe(true); + }); + + it('does not emit an unhandledRejection when the fire-and-forget read() throws', async () => { + // The patch swallows read()'s rejection (fire-and-forget) with a log instead + // of letting it surface as a process-killing unhandledRejection. + const rejections: unknown[] = []; + const onUnhandled = (e: unknown) => rejections.push(e); + process.on('unhandledRejection', onUnhandled); + // Silence the patch's diagnostic console.error for the throwing read(). + const errSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined); + try { + const result = streamText({ model: makeModel(), prompt: 'hi' }); + const res = new DisconnectingResponse(); + void result.consumeStream({ onError: () => undefined }); + result.pipeUIMessageStreamToResponse(res as never); + await new Promise((r) => setTimeout(r, 300)); + } finally { + process.off('unhandledRejection', onUnhandled); + errSpy.mockRestore(); + } + expect(rejections).toEqual([]); + }); + + it('both installed dist builds (CJS and ESM) carry the #486 patch marker', () => { + const cjsPath = require.resolve('ai'); + const mjsPath = cjsPath.replace(/index\.js$/, 'index.mjs'); + expect(cjsPath).toMatch(/index\.js$/); + expect(readFileSync(cjsPath, 'utf8')).toContain('PATCH(docmost #486)'); + expect(readFileSync(mjsPath, 'utf8')).toContain('PATCH(docmost #486)'); + }); +}); diff --git a/patches/ai@6.0.134.patch b/patches/ai@6.0.134.patch index 4371975e..9c7b81ee 100644 --- a/patches/ai@6.0.134.patch +++ b/patches/ai@6.0.134.patch @@ -1,8 +1,62 @@ diff --git a/dist/index.js b/dist/index.js -index ae447a12f7823ec0a00837ee9f0eb809a610d5f8..a3402b2c2d021ef432cfa76e35d370073d525135 100644 +index ae447a12f7823ec0a00837ee9f0eb809a610d5f8..210b5a9009e5cf1537cb2007c524007c1a01253c 100644 --- a/dist/index.js +++ b/dist/index.js -@@ -6578,9 +6578,19 @@ function createOutputTransformStream(output) { +@@ -5036,9 +5036,40 @@ function writeToServerResponse({ + break; + const canContinue = response.write(value); + if (!canContinue) { +- await new Promise((resolve3) => { +- response.once("drain", resolve3); ++ // PATCH(docmost #486): race "drain" against "close"/"error". The ++ // original awaited ONLY "drain", so a client that disconnected mid-write ++ // (the socket never drains) parked this loop FOREVER: the finally never ++ // ran, response.end() was unreachable, and the reader + buffered chunks ++ // were held until process restart. On close/error we cancel the reader ++ // and break so the finally always runs (safe for detached runs: ++ // consumeStream drains the SDK stream independently). Listener hygiene: ++ // all three once-listeners are removed on the first settle, so they ++ // cannot pile up one-per-stall. ++ const closed = await new Promise((resolve3) => { ++ function finish(isClosed) { ++ response.removeListener("drain", onDrain); ++ response.removeListener("close", onClose); ++ response.removeListener("error", onError); ++ resolve3(isClosed); ++ } ++ function onDrain() { ++ finish(false); ++ } ++ function onClose() { ++ finish(true); ++ } ++ function onError() { ++ finish(true); ++ } ++ response.once("drain", onDrain); ++ response.once("close", onClose); ++ response.once("error", onError); + }); ++ if (closed) { ++ await reader.cancel().catch(() => { ++ }); ++ break; ++ } + } + } + } catch (error) { +@@ -5047,7 +5078,9 @@ function writeToServerResponse({ + response.end(); + } + }; +- read(); ++ read().catch((error) => { ++ console.error("ai writeToServerResponse read() failed:", error); ++ }); + } + + // src/text-stream/pipe-text-stream-to-response.ts +@@ -6578,9 +6611,19 @@ function createOutputTransformStream(output) { controller.enqueue({ part: chunk, partialOutput: void 0 }); return; } @@ -23,7 +77,7 @@ index ae447a12f7823ec0a00837ee9f0eb809a610d5f8..a3402b2c2d021ef432cfa76e35d37007 const result = await output.parsePartialOutput({ text: text2 }); if (result !== void 0) { const currentJson = JSON.stringify(result.partial); -@@ -6959,7 +6969,7 @@ var DefaultStreamTextResult = class { +@@ -6959,7 +7002,7 @@ var DefaultStreamTextResult = class { }) ); } @@ -33,10 +87,64 @@ index ae447a12f7823ec0a00837ee9f0eb809a610d5f8..a3402b2c2d021ef432cfa76e35d37007 maxRetries: maxRetriesArg, abortSignal diff --git a/dist/index.mjs b/dist/index.mjs -index 663875332e3f9a9bd167c25583c515876f42951b..b840b0502c9894df983e0154805abb80e70e6331 100644 +index 663875332e3f9a9bd167c25583c515876f42951b..a51514a390d22811a407edd8b703e21793586cc8 100644 --- a/dist/index.mjs +++ b/dist/index.mjs -@@ -6501,9 +6501,19 @@ function createOutputTransformStream(output) { +@@ -4957,9 +4957,40 @@ function writeToServerResponse({ + break; + const canContinue = response.write(value); + if (!canContinue) { +- await new Promise((resolve3) => { +- response.once("drain", resolve3); ++ // PATCH(docmost #486): race "drain" against "close"/"error". The ++ // original awaited ONLY "drain", so a client that disconnected mid-write ++ // (the socket never drains) parked this loop FOREVER: the finally never ++ // ran, response.end() was unreachable, and the reader + buffered chunks ++ // were held until process restart. On close/error we cancel the reader ++ // and break so the finally always runs (safe for detached runs: ++ // consumeStream drains the SDK stream independently). Listener hygiene: ++ // all three once-listeners are removed on the first settle, so they ++ // cannot pile up one-per-stall. ++ const closed = await new Promise((resolve3) => { ++ function finish(isClosed) { ++ response.removeListener("drain", onDrain); ++ response.removeListener("close", onClose); ++ response.removeListener("error", onError); ++ resolve3(isClosed); ++ } ++ function onDrain() { ++ finish(false); ++ } ++ function onClose() { ++ finish(true); ++ } ++ function onError() { ++ finish(true); ++ } ++ response.once("drain", onDrain); ++ response.once("close", onClose); ++ response.once("error", onError); + }); ++ if (closed) { ++ await reader.cancel().catch(() => { ++ }); ++ break; ++ } + } + } + } catch (error) { +@@ -4968,7 +4999,9 @@ function writeToServerResponse({ + response.end(); + } + }; +- read(); ++ read().catch((error) => { ++ console.error("ai writeToServerResponse read() failed:", error); ++ }); + } + + // src/text-stream/pipe-text-stream-to-response.ts +@@ -6501,9 +6534,19 @@ function createOutputTransformStream(output) { controller.enqueue({ part: chunk, partialOutput: void 0 }); return; } @@ -57,7 +165,7 @@ index 663875332e3f9a9bd167c25583c515876f42951b..b840b0502c9894df983e0154805abb80 const result = await output.parsePartialOutput({ text: text2 }); if (result !== void 0) { const currentJson = JSON.stringify(result.partial); -@@ -6882,7 +6892,7 @@ var DefaultStreamTextResult = class { +@@ -6882,7 +6925,7 @@ var DefaultStreamTextResult = class { }) ); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 05a4aa01..bdf58b2b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -48,7 +48,7 @@ patchedDependencies: hash: d8dc66e5ec3b9d23a876b979f493b6aa901fd2d965be54729495da5136296a42 path: patches/@hocuspocus__server@3.4.4.patch ai@6.0.134: - hash: f60bfc3357e01e1f3978c6c40fdd65aeb33fefaad7179cde8676465b6c5ff4d9 + hash: e8c599b3963eb01b9ed1481683b7b795ce94137aa1a0c951917c20c8b870299b path: patches/ai@6.0.134.patch scimmy@1.3.5: hash: 775d80f86830b2c5dd1a250c9802c10f8fc3da3c7898373de5aa0c23993d1673 @@ -644,10 +644,10 @@ importers: version: 8.3.0(socket.io-adapter@2.5.4) ai: specifier: ^6.0.134 - version: 6.0.134(patch_hash=f60bfc3357e01e1f3978c6c40fdd65aeb33fefaad7179cde8676465b6c5ff4d9)(zod@4.3.6) + version: 6.0.134(patch_hash=e8c599b3963eb01b9ed1481683b7b795ce94137aa1a0c951917c20c8b870299b)(zod@4.3.6) ai-sdk-ollama: specifier: ^3.8.1 - version: 3.8.1(ai@6.0.134(patch_hash=f60bfc3357e01e1f3978c6c40fdd65aeb33fefaad7179cde8676465b6c5ff4d9)(zod@4.3.6))(zod@4.3.6) + version: 3.8.1(ai@6.0.134(patch_hash=e8c599b3963eb01b9ed1481683b7b795ce94137aa1a0c951917c20c8b870299b)(zod@4.3.6))(zod@4.3.6) bcrypt: specifier: ^6.0.0 version: 6.0.0 @@ -16455,17 +16455,17 @@ snapshots: agent-base@7.1.4: {} - ai-sdk-ollama@3.8.1(ai@6.0.134(patch_hash=f60bfc3357e01e1f3978c6c40fdd65aeb33fefaad7179cde8676465b6c5ff4d9)(zod@4.3.6))(zod@4.3.6): + ai-sdk-ollama@3.8.1(ai@6.0.134(patch_hash=e8c599b3963eb01b9ed1481683b7b795ce94137aa1a0c951917c20c8b870299b)(zod@4.3.6))(zod@4.3.6): dependencies: '@ai-sdk/provider': 3.0.8 '@ai-sdk/provider-utils': 4.0.21(zod@4.3.6) - ai: 6.0.134(patch_hash=f60bfc3357e01e1f3978c6c40fdd65aeb33fefaad7179cde8676465b6c5ff4d9)(zod@4.3.6) + ai: 6.0.134(patch_hash=e8c599b3963eb01b9ed1481683b7b795ce94137aa1a0c951917c20c8b870299b)(zod@4.3.6) jsonrepair: 3.13.3 ollama: 0.6.3 transitivePeerDependencies: - zod - ai@6.0.134(patch_hash=f60bfc3357e01e1f3978c6c40fdd65aeb33fefaad7179cde8676465b6c5ff4d9)(zod@4.3.6): + ai@6.0.134(patch_hash=e8c599b3963eb01b9ed1481683b7b795ce94137aa1a0c951917c20c8b870299b)(zod@4.3.6): dependencies: '@ai-sdk/gateway': 3.0.77(zod@4.3.6) '@ai-sdk/provider': 3.0.8 From 5f0c49d47c2c3f848e13c4fe1e084221314bd3ba Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 03:16:02 +0300 Subject: [PATCH 07/41] =?UTF-8?q?fix(mcp):=20REGISTRY=5FSTAMP=20=D1=85?= =?UTF-8?q?=D1=8D=D1=88=D0=B8=D1=80=D1=83=D0=B5=D1=82=20=D0=B2=D1=81=D1=91?= =?UTF-8?q?=20src/**,=20=D0=B0=20=D0=BD=D0=B5=20=D1=82=D0=BE=D0=BB=D1=8C?= =?UTF-8?q?=D0=BA=D0=BE=20tool-specs.ts=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Стамп детектил build/src-skew только по tool-specs.ts — правка client.ts, client/*-модуля, comment-signal или drawio-* без пересборки проходила молча, и build/ отдавал старый код. Теперь codegen и рантайм-лоадер хэшируют весь src/**/*.ts (кроме *.generated.ts — иначе цикл через собственный выход), симметрично: одинаковый обход, POSIX-сортировка, нормализация и sha256. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tools/docmost-client.loader.spec.ts | 144 ++++++++-- .../ai-chat/tools/docmost-client.loader.ts | 73 +++-- packages/mcp/scripts/gen-registry-stamp.mjs | 114 +++++--- .../mcp/test/unit/registry-stamp.test.mjs | 259 +++++++++++++----- 4 files changed, 439 insertions(+), 151 deletions(-) diff --git a/apps/server/src/core/ai-chat/tools/docmost-client.loader.spec.ts b/apps/server/src/core/ai-chat/tools/docmost-client.loader.spec.ts index 5ca7bbfa..973dec3d 100644 --- a/apps/server/src/core/ai-chat/tools/docmost-client.loader.spec.ts +++ b/apps/server/src/core/ai-chat/tools/docmost-client.loader.spec.ts @@ -1,7 +1,15 @@ import { createHash } from 'node:crypto'; -import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { + mkdtempSync, + mkdirSync, + writeFileSync, + rmSync, + readdirSync, + statSync, + readFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join, relative, sep } from 'node:path'; import { computeSrcRegistryStamp } from './docmost-client.loader'; @@ -30,10 +38,14 @@ function assertStaleGuard( } } -// Build a throwaway `/build/index.js` + optional `/src/tool-specs.ts` -// layout so `computeSrcRegistryStamp(/build/index.js)` resolves src the same -// way the loader does (dirname(dirname(entry))/src/tool-specs.ts). -function makeFakePackage(toolSpecsSource: string | null): { +// Build a throwaway `/build/index.js` + optional `/src/` tree so +// `computeSrcRegistryStamp(/build/index.js)` resolves src the same way the +// loader does (dirname(dirname(entry))/src). Since #486 the stamp hashes the WHOLE +// src tree, so a fixture is a { relPath: content } map. A bare string is sugar for +// a single `tool-specs.ts`; `null` means "no src tree" (the prod no-op path). +function makeFakePackage( + src: string | Record | null, +): { entry: string; cleanup: () => void; } { @@ -42,10 +54,15 @@ function makeFakePackage(toolSpecsSource: string | null): { mkdirSync(buildDir, { recursive: true }); const entry = join(buildDir, 'index.js'); writeFileSync(entry, '// fake @docmost/mcp build entry\n', 'utf8'); - if (toolSpecsSource !== null) { + if (src !== null) { + const files = + typeof src === 'string' ? { 'tool-specs.ts': src } : src; const srcDir = join(root, 'src'); - mkdirSync(srcDir, { recursive: true }); - writeFileSync(join(srcDir, 'tool-specs.ts'), toolSpecsSource, 'utf8'); + for (const [rel, content] of Object.entries(files)) { + const full = join(srcDir, rel); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, content, 'utf8'); + } } return { entry, cleanup: () => rmSync(root, { recursive: true, force: true }) }; } @@ -93,34 +110,109 @@ describe('computeSrcRegistryStamp (#447 stale-build guard)', () => { } }); - // CROSS-IMPL EQUALITY (covers reviewer suggestion 2). The SAME fixed input and + // #486 CORE (negative): an edit to a NON-tool-specs src file (client.ts) with a + // rebuild NOT run must move the src stamp away from the built REGISTRY_STAMP, so + // the loader's stale-check refuses. Under the old tool-specs.ts-only hash this + // edit was invisible and a stale build/ served the old client.ts silently. + it('a client.ts edit (no rebuild) moves the src stamp -> loader refuses (#486)', () => { + // "Built" state: the package as it was compiled. + const built = makeFakePackage({ + 'tool-specs.ts': 'export const SPECS = 1;\n', + 'client.ts': "export const impl = 'v1';\n", + }); + // "Dev edited src, forgot to rebuild": client.ts changed, tool-specs.ts not. + const edited = makeFakePackage({ + 'tool-specs.ts': 'export const SPECS = 1;\n', + 'client.ts': "export const impl = 'v2';\n", + }); + try { + const builtStamp = computeSrcRegistryStamp(built.entry); + const editedStamp = computeSrcRegistryStamp(edited.entry); + expect(builtStamp).not.toBeNull(); + expect(editedStamp).not.toBe(builtStamp); + // build/ still carries builtStamp; src now hashes to editedStamp -> refuse. + expect(() => assertStaleGuard(editedStamp, builtStamp as string)).toThrow( + STALE_BUILD_MESSAGE, + ); + } finally { + built.cleanup(); + edited.cleanup(); + } + }); + + // *.generated.ts is excluded (the codegen's own output — a fixed-point cycle + // otherwise): its presence/content must not move the stamp. + it('excludes *.generated.ts from the stamp', () => { + const without = makeFakePackage({ 'tool-specs.ts': 'x\n' }); + const withGen = makeFakePackage({ + 'tool-specs.ts': 'x\n', + 'registry-stamp.generated.ts': 'export const REGISTRY_STAMP = "abc";\n', + }); + try { + expect(computeSrcRegistryStamp(withGen.entry)).toBe( + computeSrcRegistryStamp(without.entry), + ); + } finally { + without.cleanup(); + withGen.cleanup(); + } + }); + + // CROSS-IMPL EQUALITY (covers reviewer suggestion 2). The SAME fixed tree and // EXPECTED hash are asserted in the mcp-side node test // (packages/mcp/test/unit/registry-stamp.test.mjs) against the codegen's // `computeRegistryStamp`. Asserting the SAME pair here against the loader's - // `computeSrcRegistryStamp` proves both implementations normalize+hash + // `computeSrcRegistryStamp` proves both implementations enumerate+normalize+hash // identically; a divergence in EITHER side reddens one of the two tests. - it('matches the documented cross-impl hash for a fixed input', () => { - const FIXED_INPUT = 'line1\r\nline2\n'; - const EXPECTED = - '683376e290829b482c2655745caffa7a1dccfa10afaa62dac2b42dd6c68d0f83'; - const { entry, cleanup } = makeFakePackage(FIXED_INPUT); + const CROSS_IMPL_TREE = { + 'tool-specs.ts': 'line1\r\nline2\n', + 'client/read.ts': 'export const R = 1;\n', + 'registry-stamp.generated.ts': 'export const REGISTRY_STAMP="ignored";\n', + }; + const CROSS_IMPL_EXPECTED = + '131c1b9e4e2f5a7d6cef91ca8df619822b442f52bc45ebd09474a4c1d6728616'; + + it('matches the documented cross-impl hash for a fixed tree', () => { + const { entry, cleanup } = makeFakePackage(CROSS_IMPL_TREE); try { - expect(computeSrcRegistryStamp(entry)).toBe(EXPECTED); + expect(computeSrcRegistryStamp(entry)).toBe(CROSS_IMPL_EXPECTED); } finally { cleanup(); } }); - it('the documented EXPECTED is the normalize+sha256 of the fixed input', () => { - // Proves EXPECTED is not a magic constant but the documented computation. - const FIXED_INPUT = 'line1\r\nline2\n'; - const normalized = FIXED_INPUT.replace(/\r\n/g, '\n').replace(/\n$/, ''); - const expected = createHash('sha256') - .update(normalized, 'utf8') - .digest('hex'); - const { entry, cleanup } = makeFakePackage(FIXED_INPUT); + it('the documented EXPECTED is the enumerate+normalize+sha256 of the tree', () => { + // Proves EXPECTED is not a magic constant but the documented computation — a + // local re-implementation of the loader's tree walk. + const { entry, cleanup } = makeFakePackage(CROSS_IMPL_TREE); try { - expect(computeSrcRegistryStamp(entry)).toBe(expected); + const srcDir = join(dirname(dirname(entry)), 'src'); + const collect = (dir: string): string[] => { + const out: string[] = []; + for (const e of readdirSync(dir)) { + const f = join(dir, e); + if (statSync(f).isDirectory()) out.push(...collect(f)); + else if (e.endsWith('.ts') && !e.endsWith('.generated.ts')) + out.push(f); + } + return out; + }; + const files = collect(srcDir) + .map((abs) => ({ rel: relative(srcDir, abs).split(sep).join('/'), abs })) + .sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0)); + const h = createHash('sha256'); + for (const { rel, abs } of files) { + const n = readFileSync(abs, 'utf8') + .replace(/\r\n/g, '\n') + .replace(/\n$/, ''); + h.update(rel, 'utf8'); + h.update('\0', 'utf8'); + h.update(n, 'utf8'); + h.update('\0', 'utf8'); + } + const localHash = h.digest('hex'); + expect(computeSrcRegistryStamp(entry)).toBe(localHash); + expect(localHash).toBe(CROSS_IMPL_EXPECTED); } finally { cleanup(); } diff --git a/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts b/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts index 24f16f6a..8029a239 100644 --- a/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts +++ b/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto'; -import { existsSync, readFileSync } from 'node:fs'; -import { dirname, join } from 'node:path'; +import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs'; +import { dirname, join, relative, sep } from 'node:path'; import { pathToFileURL } from 'node:url'; import type { DocmostClient, SharedToolSpec } from '@docmost/mcp'; @@ -191,33 +191,52 @@ interface DocmostMcpModule { * present. Returns the stamp string, or `null` when the source is absent (a prod * image ships only build/, no src/). MUST stay byte-for-byte identical to * packages/mcp/scripts/gen-registry-stamp.mjs's `computeRegistryStamp` so the - * build-time and src-time hashes agree: same input file (src/tool-specs.ts), same - * normalization (CRLF -> LF, strip a single trailing newline), same sha256. + * build-time and src-time hashes agree: same file set (every src/**\/*.ts except + * *.generated.ts), same POSIX-relative sort, same per-file normalization (CRLF -> + * LF, strip a single trailing newline) with the same path+content framing, same + * sha256. Hashing the WHOLE src tree (not just tool-specs.ts) is #486: an edit to + * client.ts / a client/* module / comment-signal / drawio-* without a rebuild + * must also be caught, otherwise build/ silently serves the old code. * * DEV vs PROD detection is by FILE EXISTENCE, not NODE_ENV: we resolve the * package's own directory from `require.resolve('@docmost/mcp')` (which points at - * build/index.js) and look for ../src/tool-specs.ts next to it. In a dev/test - * worktree that file exists; in a prod image (build/ only, src/ stripped) it does - * not, so this returns null and the caller skips the check. Any error (ENOENT, a - * bad resolve) is swallowed to null — the stale-check must NEVER break startup. + * build/index.js) and look for ../src next to it. In a dev/test worktree that + * directory exists; in a prod image (build/ only, src/ stripped) it does not, so + * this returns null and the caller skips the check. Any error (ENOENT, a bad + * resolve) is swallowed to null — the stale-check must NEVER break startup. * * Exported for unit testing (docmost-client.loader.spec.ts): the export keyword * is behaviourally a no-op — the module-internal caller `loadDocmostMcp` is * unaffected. The test drives the null (no-src) path and asserts this - * normalize+sha256 stays identical to the codegen's `computeRegistryStamp`. + * enumerate+normalize+sha256 stays identical to the codegen's + * `computeRegistryStamp`. */ export function computeSrcRegistryStamp(packageEntry: string): string | null { try { // packageEntry is /build/index.js; the source lives at /src/. - const toolSpecsPath = join( - dirname(dirname(packageEntry)), - 'src', - 'tool-specs.ts', - ); - if (!existsSync(toolSpecsPath)) return null; // prod: no src tree -> skip. - const source = readFileSync(toolSpecsPath, 'utf8'); - const normalized = source.replace(/\r\n/g, '\n').replace(/\n$/, ''); - return createHash('sha256').update(normalized, 'utf8').digest('hex'); + const srcDir = join(dirname(dirname(packageEntry)), 'src'); + if (!existsSync(srcDir)) return null; // prod: no src tree -> skip. + // Enumerate every src/**\/*.ts except the codegen's own *.generated.ts + // output (including it would be a fixed-point cycle). Sort by POSIX-relative + // path so ordering is platform-independent, then fold each file's relative + // path + normalized content into one hash — identical to the codegen. + const files = collectStampFiles(srcDir) + .map((abs) => ({ + rel: relative(srcDir, abs).split(sep).join('/'), + abs, + })) + .sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0)); + const hash = createHash('sha256'); + for (const { rel, abs } of files) { + const normalized = readFileSync(abs, 'utf8') + .replace(/\r\n/g, '\n') + .replace(/\n$/, ''); + hash.update(rel, 'utf8'); + hash.update('\0', 'utf8'); + hash.update(normalized, 'utf8'); + hash.update('\0', 'utf8'); + } + return hash.digest('hex'); } catch { // Never let a resolution/read hiccup break server startup — treat as "no // src available" and skip the check (identical to the prod no-op path). @@ -225,6 +244,24 @@ export function computeSrcRegistryStamp(packageEntry: string): string | null { } } +/** + * Recursively enumerate every `*.ts` under `dir`, EXCLUDING `*.generated.ts`. + * Mirror of the codegen's `collectStampFiles` (packages/mcp/scripts/ + * gen-registry-stamp.mjs) — keep the two walk/filter rules identical. + */ +function collectStampFiles(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) { + out.push(...collectStampFiles(full)); + } else if (entry.endsWith('.ts') && !entry.endsWith('.generated.ts')) { + out.push(full); + } + } + return out; +} + // TS with module:commonjs downlevels a literal `import()` to `require()`, which // cannot load the ESM-only `@docmost/mcp` package. Indirect through Function so // the real dynamic `import()` survives compilation and can load ESM from diff --git a/packages/mcp/scripts/gen-registry-stamp.mjs b/packages/mcp/scripts/gen-registry-stamp.mjs index ecf865a5..82a559ab 100644 --- a/packages/mcp/scripts/gen-registry-stamp.mjs +++ b/packages/mcp/scripts/gen-registry-stamp.mjs @@ -1,60 +1,100 @@ // Codegen: emit src/registry-stamp.generated.ts with a REGISTRY_STAMP hash of -// the tool-specs REGISTRY CONTENT, so a build/ vs src/ skew (issue #447) is -// detectable at runtime. +// the ENTIRE src/ tree, so a build/ vs src/ skew (issue #447) is detectable at +// runtime for ANY source file — not just tool-specs.ts. // -// WHY hash the raw source text (not extracted structured data): -// SHARED_TOOL_SPECS carries `buildShape` functions (the input SCHEMAS) which are -// NOT serializable. The input schema is exactly one of the things that MUST stay -// in sync between build/ and src/, so we cannot drop it from the hash. Rather -// than probe zod with a fragile shim to reconstruct the schema shape, we hash the -// STABLE, deterministic source TEXT of tool-specs.ts. That text fully captures -// every field that must stay in sync — mcpName, inAppKey, description, tier, -// catalogLine AND the buildShape bodies (input schemas) — with zero probing -// fragility. Any edit to a spec (a renamed tool, a reworded description, a -// changed schema field) changes the text and therefore the stamp. +// WHY hash the whole src tree (not just tool-specs.ts): the runtime tools are +// assembled from far more than the spec registry — client.ts, the client/* +// domain modules, comment-signal.ts and the drawio-* helpers all ship in build/ +// and are loaded by the in-app server. Hashing ONLY tool-specs.ts meant an edit +// to any of those (e.g. a behavioural fix in client.ts) left the stamp unchanged, +// so a stale build/ served the OLD code silently (issue #486). Hashing every +// src/**/*.ts closes that gap: any source edit changes the stamp. // -// DETERMINISM: the hash is computed over the file bytes with line endings -// normalized to LF and a single trailing newline stripped, so a CRLF checkout or -// an editor's trailing-newline habit cannot make build/ and src/ disagree. No -// Date.now / randomness. The loader's dev-only stale-check (docmost-client.loader.ts) -// re-runs THIS SAME normalization + sha256 over src/tool-specs.ts and compares to -// the built REGISTRY_STAMP; the two must compute identically. +// WHY hash the raw source text (not extracted structured data): the tool input +// SCHEMAS live as `buildShape` functions which are NOT serializable, so we cannot +// reduce them to structured data without a fragile zod shim. Hashing the STABLE, +// deterministic source TEXT captures every field that must stay in sync with zero +// probing fragility. Any edit to any source file changes the text → the stamp. +// +// DETERMINISM: files are enumerated recursively, filtered to *.ts EXCLUDING +// *.generated.ts (the codegen's OWN output — including it would create a +// fixed-point cycle), and sorted by their POSIX-normalized path relative to src/ +// so the order is platform-independent. Each file contributes its relative path +// AND its content with line endings normalized to LF and a single trailing +// newline stripped, so a CRLF checkout or an editor's trailing-newline habit +// cannot make build/ and src/ disagree. No Date.now / randomness. The loader's +// dev-only stale-check (docmost-client.loader.ts) re-runs THIS SAME enumeration + +// normalization + sha256 and compares to the built REGISTRY_STAMP; the two must +// compute identically. // // This script runs from the `build` and `pretest` npm scripts BEFORE tsc, so -// build/ always carries a stamp derived from the tool-specs.ts that was compiled. +// build/ always carries a stamp derived from the src/ tree that was compiled. import { createHash } from 'node:crypto'; -import { readFileSync, writeFileSync } from 'node:fs'; +import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; -import { dirname, join } from 'node:path'; +import { dirname, join, relative, sep } from 'node:path'; const __dirname = dirname(fileURLToPath(import.meta.url)); const SRC_DIR = join(__dirname, '..', 'src'); -const TOOL_SPECS_PATH = join(SRC_DIR, 'tool-specs.ts'); const OUT_PATH = join(SRC_DIR, 'registry-stamp.generated.ts'); /** - * Deterministic stamp of the tool-specs registry content. Kept as a plain - * function (exported) so the algorithm has a single home; the loader duplicates - * only the tiny normalize+sha256 steps because it lives in the CJS server build - * and cannot import this ESM script. If you change the normalization here, mirror - * it in apps/server/src/core/ai-chat/tools/docmost-client.loader.ts. + * Recursively enumerate every `*.ts` file under `dir`, EXCLUDING the codegen's + * own `*.generated.ts` output (a self-referential cycle otherwise). Returns + * absolute paths, unsorted (the caller sorts by relative path for determinism). + * Kept as a plain exported function so the algorithm has a single home; the + * loader duplicates it because it lives in the CJS server build and cannot import + * this ESM script. If you change the walk/filter here, mirror it in + * apps/server/src/core/ai-chat/tools/docmost-client.loader.ts. */ -export function computeRegistryStamp(toolSpecsSource) { - const normalized = toolSpecsSource.replace(/\r\n/g, '\n').replace(/\n$/, ''); - return createHash('sha256').update(normalized, 'utf8').digest('hex'); +export function collectStampFiles(dir) { + const out = []; + for (const entry of readdirSync(dir)) { + const full = join(dir, entry); + if (statSync(full).isDirectory()) { + out.push(...collectStampFiles(full)); + } else if (entry.endsWith('.ts') && !entry.endsWith('.generated.ts')) { + out.push(full); + } + } + return out; +} + +/** + * Deterministic stamp of the whole src/ tree. Enumerate + sort by POSIX-relative + * path, then fold each file's relative path AND normalized content into one + * sha256. MUST stay byte-for-byte identical to the loader's recompute. + */ +export function computeRegistryStamp(srcDir) { + const files = collectStampFiles(srcDir) + .map((abs) => ({ + rel: relative(srcDir, abs).split(sep).join('/'), + abs, + })) + .sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0)); + const hash = createHash('sha256'); + for (const { rel, abs } of files) { + const normalized = readFileSync(abs, 'utf8') + .replace(/\r\n/g, '\n') + .replace(/\n$/, ''); + hash.update(rel, 'utf8'); + hash.update('\0', 'utf8'); + hash.update(normalized, 'utf8'); + hash.update('\0', 'utf8'); + } + return hash.digest('hex'); } function main() { - const source = readFileSync(TOOL_SPECS_PATH, 'utf8'); - const stamp = computeRegistryStamp(source); + const stamp = computeRegistryStamp(SRC_DIR); const out = '// AUTO-GENERATED by scripts/gen-registry-stamp.mjs — DO NOT EDIT BY HAND.\n' + - '// A deterministic hash of src/tool-specs.ts content (tool names, descriptions,\n' + - '// tiers, catalog lines and input schemas). Regenerated on every build/pretest\n' + - '// so build/ always matches the compiled src. The in-app loader recomputes this\n' + - '// from src and refuses to run on a mismatch (issue #447). This file is\n' + - '// gitignored and produced by the build — see .gitignore.\n' + + '// A deterministic hash of the whole src/ tree (every src/**/*.ts except\n' + + '// *.generated.ts). Regenerated on every build/pretest so build/ always\n' + + '// matches the compiled src. The in-app loader recomputes this from src and\n' + + '// refuses to run on a mismatch (issue #447/#486). This file is gitignored\n' + + '// and produced by the build — see .gitignore.\n' + `export const REGISTRY_STAMP = ${JSON.stringify(stamp)};\n`; writeFileSync(OUT_PATH, out, 'utf8'); // eslint-disable-next-line no-console diff --git a/packages/mcp/test/unit/registry-stamp.test.mjs b/packages/mcp/test/unit/registry-stamp.test.mjs index b9d992c4..32006688 100644 --- a/packages/mcp/test/unit/registry-stamp.test.mjs +++ b/packages/mcp/test/unit/registry-stamp.test.mjs @@ -1,101 +1,220 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { createHash } from "node:crypto"; -import { readFileSync } from "node:fs"; +import { + mkdtempSync, + mkdirSync, + writeFileSync, + rmSync, + readdirSync, + statSync, + readFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; -import { dirname, join } from "node:path"; +import { dirname, join, relative, sep } from "node:path"; +import { createHash } from "node:crypto"; import { computeRegistryStamp } from "../../scripts/gen-registry-stamp.mjs"; import { REGISTRY_STAMP } from "../../build/index.js"; -// Guard tests for the build/src-skew stamp (issue #447). The codegen script -// exports `computeRegistryStamp(sourceText)` — a sha256 over normalized source -// text (CRLF->LF, single trailing newline stripped). The in-app loader -// (apps/server/.../docmost-client.loader.ts) DUPLICATES that normalize+sha256 to -// recompute the stamp from src and refuse a stale build. These tests pin the -// algorithm's behaviour AND assert the built stamp matches the current src, so a -// stale generated file OR a normalize divergence reddens. +// Guard tests for the build/src-skew stamp (issues #447/#486). The codegen script +// exports `computeRegistryStamp(srcDir)` — a sha256 over the WHOLE src/ tree +// (every src/**/*.ts EXCEPT *.generated.ts), each file folded in as its +// POSIX-relative path + its normalized content (CRLF->LF, single trailing newline +// stripped). Hashing the whole tree (not just tool-specs.ts) is #486: an edit to +// client.ts / a client/* module without a rebuild must ALSO redden. The in-app +// loader (apps/server/.../docmost-client.loader.ts) DUPLICATES this enumerate+ +// normalize+sha256 to refuse a stale build. These tests pin the algorithm and +// assert the built stamp matches the current src. const __dirname = dirname(fileURLToPath(import.meta.url)); -const TOOL_SPECS_PATH = join(__dirname, "..", "..", "src", "tool-specs.ts"); +const SRC_DIR = join(__dirname, "..", "..", "src"); -test("computeRegistryStamp is deterministic: same input -> same hash", () => { - const input = "export const X = 1;\nexport const Y = 2;\n"; - assert.equal(computeRegistryStamp(input), computeRegistryStamp(input)); +// Build a throwaway src/ tree from a { relPath: content } map and return its dir. +function makeSrcTree(files) { + const root = mkdtempSync(join(tmpdir(), "mcp-stamp-tree-")); + const src = join(root, "src"); + for (const [rel, content] of Object.entries(files)) { + const full = join(src, rel); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, content, "utf8"); + } + return { src, cleanup: () => rmSync(root, { recursive: true, force: true }) }; +} + +test("computeRegistryStamp is deterministic: same tree -> same hash", () => { + const a = makeSrcTree({ "tool-specs.ts": "export const X = 1;\n" }); + const b = makeSrcTree({ "tool-specs.ts": "export const X = 1;\n" }); + try { + assert.equal(computeRegistryStamp(a.src), computeRegistryStamp(b.src)); + } finally { + a.cleanup(); + b.cleanup(); + } }); test("computeRegistryStamp returns a 64-char lowercase hex sha256", () => { - const stamp = computeRegistryStamp("anything"); - assert.match(stamp, /^[0-9a-f]{64}$/); + const t = makeSrcTree({ "tool-specs.ts": "anything\n" }); + try { + assert.match(computeRegistryStamp(t.src), /^[0-9a-f]{64}$/); + } finally { + t.cleanup(); + } }); -test("normalizes CRLF vs LF: the same content hashes equal", () => { - const lf = "line1\nline2\nline3"; - const crlf = "line1\r\nline2\r\nline3"; - assert.equal(computeRegistryStamp(crlf), computeRegistryStamp(lf)); +// #486 CORE: an edit to a NON-tool-specs source file (client.ts) must change the +// stamp. Under the old single-file (tool-specs.ts only) hash this edit was +// invisible and a stale build/ served the old client.ts silently. +test("editing client.ts (not tool-specs.ts) changes the stamp (#486)", () => { + const before = makeSrcTree({ + "tool-specs.ts": "export const SPECS = 1;\n", + "client.ts": "export const impl = 'v1';\n", + }); + const after = makeSrcTree({ + "tool-specs.ts": "export const SPECS = 1;\n", + "client.ts": "export const impl = 'v2';\n", + }); + try { + assert.notEqual( + computeRegistryStamp(before.src), + computeRegistryStamp(after.src), + "a client.ts edit with an unchanged tool-specs.ts must move the stamp", + ); + } finally { + before.cleanup(); + after.cleanup(); + } }); -test("normalizes a trailing newline: with/without a final \\n hashes equal", () => { - const noTrailing = "alpha\nbeta"; - const trailing = "alpha\nbeta\n"; - assert.equal(computeRegistryStamp(trailing), computeRegistryStamp(noTrailing)); +test("editing a nested client/* module changes the stamp", () => { + const before = makeSrcTree({ + "tool-specs.ts": "x\n", + "client/read.ts": "export const READ = 1;\n", + }); + const after = makeSrcTree({ + "tool-specs.ts": "x\n", + "client/read.ts": "export const READ = 2;\n", + }); + try { + assert.notEqual( + computeRegistryStamp(before.src), + computeRegistryStamp(after.src), + ); + } finally { + before.cleanup(); + after.cleanup(); + } }); -test("a CRLF checkout WITH a trailing CRLF still hashes equal to bare LF", () => { - // A worst-case Windows checkout: CRLF line endings + a trailing CRLF. Both the - // \r\n->\n replace and the trailing-newline strip must apply for parity. - const bare = "alpha\nbeta"; - const crlfTrailing = "alpha\r\nbeta\r\n"; - assert.equal( - computeRegistryStamp(crlfTrailing), - computeRegistryStamp(bare), - ); +// *.generated.ts is EXCLUDED (else the codegen's own output is a fixed-point +// cycle): adding/removing/changing it must not move the stamp. +test("*.generated.ts is excluded from the stamp", () => { + const without = makeSrcTree({ "tool-specs.ts": "x\n" }); + const withGen = makeSrcTree({ + "tool-specs.ts": "x\n", + "registry-stamp.generated.ts": 'export const REGISTRY_STAMP = "abc";\n', + }); + try { + assert.equal( + computeRegistryStamp(without.src), + computeRegistryStamp(withGen.src), + "a *.generated.ts file must not affect the stamp", + ); + } finally { + without.cleanup(); + withGen.cleanup(); + } }); -test("a real content change hashes differently", () => { - const before = "export const description = 'search a page';\n"; - const after = "export const description = 'search a PAGE';\n"; - assert.notEqual(computeRegistryStamp(before), computeRegistryStamp(after)); +test("a CRLF checkout WITH trailing CRLF hashes equal to bare LF", () => { + const bare = makeSrcTree({ "tool-specs.ts": "alpha\nbeta" }); + const crlfTrailing = makeSrcTree({ "tool-specs.ts": "alpha\r\nbeta\r\n" }); + try { + assert.equal( + computeRegistryStamp(crlfTrailing.src), + computeRegistryStamp(bare.src), + ); + } finally { + bare.cleanup(); + crlfTrailing.cleanup(); + } }); -// Only a SINGLE trailing newline is stripped — a second blank line is content and -// must change the hash. This pins the exact `/\n$/` semantics the loader mirrors. +// Only a SINGLE trailing newline is stripped — a second blank line is content. test("only ONE trailing newline is stripped (two differ from one)", () => { - assert.notEqual( - computeRegistryStamp("x\n"), - computeRegistryStamp("x\n\n"), - ); + const one = makeSrcTree({ "tool-specs.ts": "x\n" }); + const two = makeSrcTree({ "tool-specs.ts": "x\n\n" }); + try { + assert.notEqual( + computeRegistryStamp(one.src), + computeRegistryStamp(two.src), + ); + } finally { + one.cleanup(); + two.cleanup(); + } }); -// Cross-impl equality against a fixed, documented input. The SAME literal input -// and expected hash are asserted in the server-side jest test -// (docmost-client.loader.spec.ts). If either side's normalize+sha256 ever -// diverges, one of the two tests reddens. Input exercises BOTH normalize steps. -test("fixed-input hash matches the documented cross-impl value", () => { - const FIXED_INPUT = "line1\r\nline2\n"; - const EXPECTED = - "683376e290829b482c2655745caffa7a1dccfa10afaa62dac2b42dd6c68d0f83"; - assert.equal(computeRegistryStamp(FIXED_INPUT), EXPECTED); +// Cross-impl equality against a fixed, documented tree. The SAME literal tree and +// expected hash are asserted in the server-side jest test +// (docmost-client.loader.spec.ts). If either side's enumerate+normalize+sha256 +// ever diverges, one of the two tests reddens. The tree exercises: a nested file, +// BOTH normalize steps (tool-specs.ts uses CRLF + trailing \n) and the +// *.generated.ts exclusion. +const CROSS_IMPL_TREE = { + "tool-specs.ts": "line1\r\nline2\n", + "client/read.ts": "export const R = 1;\n", + "registry-stamp.generated.ts": 'export const REGISTRY_STAMP="ignored";\n', +}; +const CROSS_IMPL_EXPECTED = + "131c1b9e4e2f5a7d6cef91ca8df619822b442f52bc45ebd09474a4c1d6728616"; + +test("fixed-tree hash matches the documented cross-impl value", () => { + const t = makeSrcTree(CROSS_IMPL_TREE); + try { + assert.equal(computeRegistryStamp(t.src), CROSS_IMPL_EXPECTED); + } finally { + t.cleanup(); + } }); -// DESYNC GUARD (covers reviewer suggestion 2). Recompute the stamp from the -// actual src/tool-specs.ts and assert it equals the REGISTRY_STAMP baked into the -// freshly-built build/index.js. This reddens if the generated file is stale OR if -// the codegen normalize ever diverges from what produced the built stamp. -test("built REGISTRY_STAMP equals the stamp recomputed from src/tool-specs.ts", () => { - const source = readFileSync(TOOL_SPECS_PATH, "utf8"); - assert.equal(computeRegistryStamp(source), REGISTRY_STAMP); +// Sanity: the EXPECTED constant is not a magic value but the documented +// enumerate+normalize+sha256 of CROSS_IMPL_TREE (a local re-implementation). +test("the documented EXPECTED is the enumerate+normalize+sha256 of the tree", () => { + const t = makeSrcTree(CROSS_IMPL_TREE); + try { + const collect = (dir) => { + const out = []; + for (const e of readdirSync(dir)) { + const f = join(dir, e); + if (statSync(f).isDirectory()) out.push(...collect(f)); + else if (e.endsWith(".ts") && !e.endsWith(".generated.ts")) out.push(f); + } + return out; + }; + const files = collect(t.src) + .map((abs) => ({ rel: relative(t.src, abs).split(sep).join("/"), abs })) + .sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0)); + const h = createHash("sha256"); + for (const { rel, abs } of files) { + const n = readFileSync(abs, "utf8") + .replace(/\r\n/g, "\n") + .replace(/\n$/, ""); + h.update(rel, "utf8"); + h.update("\0", "utf8"); + h.update(n, "utf8"); + h.update("\0", "utf8"); + } + assert.equal(h.digest("hex"), CROSS_IMPL_EXPECTED); + } finally { + t.cleanup(); + } }); -// Sanity: the fixed-input helper computes the SAME way the codegen does, proving -// the EXPECTED constant above is not an arbitrary magic value but the documented -// normalize+sha256 of FIXED_INPUT. Belt-and-braces so a bad EXPECTED can't hide a -// real regression. -test("the documented EXPECTED constant is the normalize+sha256 of FIXED_INPUT", () => { - const FIXED_INPUT = "line1\r\nline2\n"; - const normalized = FIXED_INPUT.replace(/\r\n/g, "\n").replace(/\n$/, ""); - const expected = createHash("sha256") - .update(normalized, "utf8") - .digest("hex"); - assert.equal(computeRegistryStamp(FIXED_INPUT), expected); +// DESYNC GUARD. Recompute the stamp from the REAL src/ tree and assert it equals +// the REGISTRY_STAMP baked into the freshly-built build/index.js. This reddens if +// the generated file is stale OR if the codegen ever diverges from what produced +// the built stamp. +test("built REGISTRY_STAMP equals the stamp recomputed from src/", () => { + assert.equal(computeRegistryStamp(SRC_DIR), REGISTRY_STAMP); }); From 31176d5f93edb2c979081380a63a678f15be1758 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 03:19:15 +0300 Subject: [PATCH 08/41] =?UTF-8?q?fix(mcp):=20=D1=81=D0=B1=D1=80=D0=BE?= =?UTF-8?q?=D1=81=20collab-=D1=82=D0=BE=D0=BA=D0=B5=D0=BD=D0=B0=20=D0=BD?= =?UTF-8?q?=D0=B0=20WS-auth-failure=20=D1=81=20=D1=80=D0=B5=D1=82=D1=80?= =?UTF-8?q?=D0=B0=D0=B5=D0=BC=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Кэш collab-токена (#435) инвалидировался только на HTTP-401/403 (REST- интерцептор и login()); отклонённый Hocuspocus-handshake оставлял протухший токен в кэше — каждая последующая мутация переотправляла тот же битый токен до истечения TTL (минуты) без self-heal. collab-session помечает ошибку onAuthenticationFailed маркером; клиентские write-швы (mutatePage/replacePage/ mutateLiveContentUnlocked) обёрнуты в writeWithCollabAuthRetry: на помеченной ошибке кэш сбрасывается и запись ретраится ровно один раз со свежим токеном — симметрично HTTP-пути. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp/src/client/context.ts | 77 ++++++++-- packages/mcp/src/lib/collab-session.ts | 30 +++- .../mock/collab-auth-failure-reset.test.mjs | 137 ++++++++++++++++++ 3 files changed, 225 insertions(+), 19 deletions(-) create mode 100644 packages/mcp/test/mock/collab-auth-failure-reset.test.mjs diff --git a/packages/mcp/src/client/context.ts b/packages/mcp/src/client/context.ts index b8818863..12a8b1a0 100644 --- a/packages/mcp/src/client/context.ts +++ b/packages/mcp/src/client/context.ts @@ -19,7 +19,10 @@ import { assertYjsEncodable, MutationResult, } from "../lib/collaboration.js"; -import { acquireCollabSession } from "../lib/collab-session.js"; +import { + acquireCollabSession, + isCollabAuthFailedError, +} from "../lib/collab-session.js"; import { withPageLock, isUuid } from "../lib/page-lock.js"; import { getCollabToken, performLogin } from "../lib/auth-utils.js"; import { formatDocmostAxiosError } from "./errors.js"; @@ -492,6 +495,37 @@ export abstract class DocmostClientContext { ); } + /** + * Run a collab write and, on a Hocuspocus HANDSHAKE auth failure, self-heal + * once (#486). Symmetric to the HTTP-401 path in getCollabTokenWithReauth: the + * REST interceptor and login() already drop the cached collab token on a 401/ + * 403, but a rejected WEBSOCKET handshake left the stale token in the cache, so + * every subsequent mutation kept re-presenting the same bad token for up to the + * collab-token TTL (minutes) with no self-heal. Here, when the write rejects + * with the tagged collab-auth error, we invalidate the cached token and retry + * the write EXACTLY once with a force-refreshed token. Not a loop: a second + * failure (or any non-auth error) propagates unchanged. + * + * `write` receives the token to use, so the retry can hand it a genuinely fresh + * one rather than re-running with the same stale string. + */ + protected async writeWithCollabAuthRetry( + collabToken: string, + write: (token: string) => Promise, + ): Promise { + try { + return await write(collabToken); + } catch (e) { + if (!isCollabAuthFailedError(e)) throw e; + // The WS handshake rejected our token: drop it from the cache so it can't + // be reused for the rest of the TTL, mint a fresh one (forceRefresh bypasses + // the cache and re-invokes the provider/login), and retry the write once. + this.collabTokenCache = null; + const fresh = await this.getCollabTokenWithReauth(true); + return await write(fresh); + } + } + /** * Connect to the collaboration websocket, read the live doc, apply * `transform`, write the result, and wait for the server to persist it — @@ -526,19 +560,24 @@ export abstract class DocmostClientContext { // unsyncedChanges/connectionLost ack logic live in CollabSession.mutate, // preserved verbatim from the old inline machine (incl. the #152 structural // diff that keeps a live editor's cursor anchored). - const session = await acquireCollabSession(pageId, collabToken, this.apiUrl, { - // Only the actual 25s collab connect timeout emits this — the connect-vs- - // unload signal; the other failure paths must NOT emit it. - onConnectTimeout: () => - this.onMetricFn?.("collab_connect_timeouts_total", 1), + // Wrap in the collab-auth self-heal (#486): a rejected WS handshake drops the + // cached collab token and retries once with a fresh one (the retry passes the + // refreshed token down to acquireCollabSession via `token`). + return this.writeWithCollabAuthRetry(collabToken, async (token) => { + const session = await acquireCollabSession(pageId, token, this.apiUrl, { + // Only the actual 25s collab connect timeout emits this — the connect-vs- + // unload signal; the other failure paths must NOT emit it. + onConnectTimeout: () => + this.onMetricFn?.("collab_connect_timeouts_total", 1), + }); + try { + return await session.mutate(transform); + } catch (e) { + // Drop the session on any failure so the next call reconnects fresh. + session.destroy("mutate failed"); + throw e; + } }); - try { - return await session.mutate(transform); - } catch (e) { - // Drop the session on any failure so the next call reconnects fresh. - session.destroy("mutate failed"); - throw e; - } } /** @@ -667,7 +706,11 @@ export abstract class DocmostClientContext { transform: (doc: any) => any, ): Promise<{ doc?: any; verify?: any }> { const pageUuid = await this.resolvePageId(pageId); - return mutatePageContent(pageUuid, collabToken, apiUrl, transform); + // #486: on a rejected collab-WS handshake, invalidate + refresh the token and + // retry the write once (symmetric to the HTTP-401 reauth path). + return this.writeWithCollabAuthRetry(collabToken, (token) => + mutatePageContent(pageUuid, token, apiUrl, transform), + ); } /** @@ -687,7 +730,11 @@ export abstract class DocmostClientContext { apiUrl: string, ): Promise<{ doc?: any; verify?: any }> { const pageUuid = await this.resolvePageId(pageId); - return replacePageContent(pageUuid, doc, collabToken, apiUrl); + // #486: on a rejected collab-WS handshake, invalidate + refresh the token and + // retry the write once (symmetric to the HTTP-401 reauth path). + return this.writeWithCollabAuthRetry(collabToken, (token) => + replacePageContent(pageUuid, doc, token, apiUrl), + ); } /** diff --git a/packages/mcp/src/lib/collab-session.ts b/packages/mcp/src/lib/collab-session.ts index 3dc16161..5e1e40cc 100644 --- a/packages/mcp/src/lib/collab-session.ts +++ b/packages/mcp/src/lib/collab-session.ts @@ -37,6 +37,25 @@ const CONNECT_TIMEOUT_MS = 25000; /** Time we wait for the server to acknowledge our write before giving up. */ const PERSIST_TIMEOUT_MS = 20000; +/** + * Marker property set on the Error thrown when the Hocuspocus handshake REJECTS + * our collab token (onAuthenticationFailed). The client wraps content writes so + * that on this specific failure it invalidates its cached collab token and + * retries once with a fresh one — symmetric to the HTTP-401 reauth path (#486). + * A plain message-match would be brittle; a tagged property is unambiguous and + * survives teardown (which rejects pending ops with this SAME error object). + */ +const COLLAB_AUTH_FAILED_MARKER = "collabAuthFailed"; + +/** True when `e` is the tagged collab-WS auth-failure error (see marker above). */ +export function isCollabAuthFailedError(e: unknown): boolean { + return !!( + e && + typeof e === "object" && + (e as Record)[COLLAB_AUTH_FAILED_MARKER] === true + ); +} + /** * Tunables, read fresh from the environment on every acquire so tests (and a * live rollback) can change them without reloading the module. Mirrors how @@ -302,10 +321,13 @@ export class CollabSession { this.openResolve?.(); }, onAuthenticationFailed: () => { - this.teardown( - new Error("Authentication failed for collaboration connection"), - true, - ); + // Tag the error so the client can tell a REJECTED collab token apart + // from a generic disconnect and invalidate + refresh it (#486). + const err = new Error( + "Authentication failed for collaboration connection", + ) as Error & { [COLLAB_AUTH_FAILED_MARKER]?: boolean }; + err[COLLAB_AUTH_FAILED_MARKER] = true; + this.teardown(err, true); }, }); }); diff --git a/packages/mcp/test/mock/collab-auth-failure-reset.test.mjs b/packages/mcp/test/mock/collab-auth-failure-reset.test.mjs new file mode 100644 index 00000000..573098ce --- /dev/null +++ b/packages/mcp/test/mock/collab-auth-failure-reset.test.mjs @@ -0,0 +1,137 @@ +// Unit tests for the collab-token reset on a Hocuspocus WS auth failure (#486). +// +// Before this fix the cached collab token (#435) was dropped ONLY on an HTTP +// 401/403 (the REST interceptor + login()); a rejected collab-WEBSOCKET handshake +// left the stale token in the cache, so every subsequent mutation re-presented +// the SAME bad token for up to the collab-token TTL (minutes) with no self-heal. +// +// The fix wraps collab writes in `writeWithCollabAuthRetry`: when the write +// rejects with the tagged collab-auth error (collab-session.ts's +// onAuthenticationFailed), it invalidates the cached token and retries the write +// ONCE with a force-refreshed token — symmetric to the HTTP-401 path. +// +// writeWithCollabAuthRetry / getCollabTokenWithReauth are protected in TS but +// plain methods on the compiled build, so the tests call them directly (same +// convention as collab-token-cache.test.mjs). +import { test, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { DocmostClient } from "../../build/client.js"; + +const ENV_KEY = "MCP_COLLAB_TOKEN_TTL_MS"; +afterEach(() => { + delete process.env[ENV_KEY]; +}); + +// A counting provider that returns a distinct token each call so a cached +// (reused) token is visibly the SAME string while a fresh mint is different. +function countingProvider() { + let n = 0; + const fn = async () => { + n++; + return `provider-token-${n}`; + }; + return { + fn, + get calls() { + return n; + }, + }; +} + +// The tagged error collab-session.ts throws on a rejected WS handshake. +function collabAuthError() { + const err = new Error("Authentication failed for collaboration connection"); + err.collabAuthFailed = true; + return err; +} + +test("a WS auth failure clears the cached token and retries the write with a FRESH one (#486)", async () => { + process.env[ENV_KEY] = "300000"; // 5 min: the cache is warm across the burst. + const p = countingProvider(); + const client = new DocmostClient({ + apiUrl: "http://127.0.0.1:1/api", + getToken: async () => "access", + getCollabToken: p.fn, + }); + + // Warm the cache the way a real write would (mints provider-token-1). + const initial = await client.getCollabTokenWithReauth(); + assert.equal(initial, "provider-token-1"); + assert.equal(p.calls, 1); + + const tokensSeen = []; + const write = async (token) => { + tokensSeen.push(token); + // The FIRST attempt (with the stale cached token) fails the WS handshake; + // the retry (with a fresh token) succeeds. + if (tokensSeen.length === 1) throw collabAuthError(); + return `written-with:${token}`; + }; + + const result = await client.writeWithCollabAuthRetry(initial, write); + + assert.equal(tokensSeen.length, 2, "write attempted exactly twice (one retry)"); + assert.equal(tokensSeen[0], "provider-token-1", "first attempt used the stale token"); + assert.equal( + tokensSeen[1], + "provider-token-2", + "retry used a FRESH force-refreshed token, not the stale cached one", + ); + assert.equal(result, "written-with:provider-token-2", "the retry's result wins"); + assert.equal(p.calls, 2, "exactly one extra mint for the retry — no loop"); + + // The cache now holds the fresh token, so a subsequent op reuses it (proving + // the stale token was evicted and the fresh one cached, not re-minted). + const next = await client.getCollabTokenWithReauth(); + assert.equal(next, "provider-token-2", "the fresh token replaced the stale cache"); + assert.equal(p.calls, 2, "served from cache — provider not re-invoked"); +}); + +test("a successful write is NOT retried and mints nothing extra", async () => { + process.env[ENV_KEY] = "300000"; + const p = countingProvider(); + const client = new DocmostClient({ + apiUrl: "http://127.0.0.1:1/api", + getToken: async () => "access", + getCollabToken: p.fn, + }); + + const initial = await client.getCollabTokenWithReauth(); // provider-token-1 + let attempts = 0; + const result = await client.writeWithCollabAuthRetry(initial, async (token) => { + attempts++; + return `ok:${token}`; + }); + + assert.equal(attempts, 1, "no retry on success"); + assert.equal(result, "ok:provider-token-1"); + assert.equal(p.calls, 1, "no extra mint"); +}); + +test("a NON-auth write error propagates unchanged (no reset, no retry)", async () => { + process.env[ENV_KEY] = "300000"; + const p = countingProvider(); + const client = new DocmostClient({ + apiUrl: "http://127.0.0.1:1/api", + getToken: async () => "access", + getCollabToken: p.fn, + }); + + const initial = await client.getCollabTokenWithReauth(); // provider-token-1 + let attempts = 0; + await assert.rejects( + client.writeWithCollabAuthRetry(initial, async () => { + attempts++; + throw new Error("collab connection closed before persist"); // NOT tagged. + }), + /closed before persist/, + ); + + assert.equal(attempts, 1, "a non-auth error is not retried"); + assert.equal(p.calls, 1, "the cache is untouched -> no fresh mint"); + + // Cache still holds the original token (was never invalidated). + const still = await client.getCollabTokenWithReauth(); + assert.equal(still, "provider-token-1"); + assert.equal(p.calls, 1); +}); From 8ebdfe156f26914cc061d5cfb4c11b58bdf4aa6c Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 03:22:14 +0300 Subject: [PATCH 09/41] =?UTF-8?q?fix(ai-chat):=20=D1=81=D0=B1=D1=80=D0=BE?= =?UTF-8?q?=D1=81=20lastDegenerationCheckLen=20=D0=B2=20onStepFinish=20(#4?= =?UTF-8?q?86)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit onStepFinish обнулял inProgressText, но НЕ lastDegenerationCheckLen — а это смещение В аккумулятор. После первого длинного шага протухшая (большая) отметка делала throttle-условие отрицательным, и детектор токен-лупов молчал весь следующий шаг, пока текст не перерастал старую отметку. Обнуляем отметку в onStepFinish. Throttle-предикат вынесен в output-degeneration (shouldCheckDegeneration + DEGENERATION_CHECK_STEP), чтобы юнит гонял ту же логику, что и стрим. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/core/ai-chat/ai-chat.service.ts | 15 +++++-- .../core/ai-chat/output-degeneration.spec.ts | 41 +++++++++++++++++++ .../src/core/ai-chat/output-degeneration.ts | 26 ++++++++++++ 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/apps/server/src/core/ai-chat/ai-chat.service.ts b/apps/server/src/core/ai-chat/ai-chat.service.ts index 908c9200..dfa1bb75 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.ts @@ -56,6 +56,7 @@ import { import { isDegenerateOutput, truncateDegeneratedTail, + shouldCheckDegeneration, } from './output-degeneration'; // Max agent steps per turn. One step = one model generation; a step that calls @@ -1101,7 +1102,6 @@ export class AiChatService implements OnModuleInit { const degenerationController = new AbortController(); let degenerationDetected = false; let lastDegenerationCheckLen = 0; - const DEGENERATION_CHECK_STEP = 2000; // Step-granular durability (#183): create the assistant row UPFRONT in the // 'streaming' state (before any token), then UPDATE it as each step finishes @@ -1275,8 +1275,10 @@ export class AiChatService implements OnModuleInit { // trigger, abort the run ONCE with a distinguishable reason. if ( !degenerationDetected && - inProgressText.length - lastDegenerationCheckLen >= - DEGENERATION_CHECK_STEP + shouldCheckDegeneration( + inProgressText.length, + lastDegenerationCheckLen, + ) ) { lastDegenerationCheckLen = inProgressText.length; if (isDegenerateOutput(inProgressText)) { @@ -1296,6 +1298,13 @@ export class AiChatService implements OnModuleInit { // the in-progress accumulator for the next step. capturedSteps.push(step as StepLike); inProgressText = ''; + // Reset the degeneration-check watermark too (#486): it tracks a byte + // offset INTO inProgressText, so once that resets to '' a stale (large) + // mark makes `inProgressText.length - lastDegenerationCheckLen` go + // negative and the throttled detector stays silent until a later step's + // text re-grows past the old offset — a whole degenerate step could slip + // through undetected. Zeroing it re-arms the check from the next byte. + lastDegenerationCheckLen = 0; // Step-granular durability (#183): persist this finished step (its text + // tool calls + tool RESULTS) the moment it ends, so a process death after // this point still recovers the step. Not awaited here (never block the diff --git a/apps/server/src/core/ai-chat/output-degeneration.spec.ts b/apps/server/src/core/ai-chat/output-degeneration.spec.ts index a4f6c36e..426f4e40 100644 --- a/apps/server/src/core/ai-chat/output-degeneration.spec.ts +++ b/apps/server/src/core/ai-chat/output-degeneration.spec.ts @@ -3,6 +3,8 @@ import { hasPeriodicTail, isDegenerateOutput, truncateDegeneratedTail, + shouldCheckDegeneration, + DEGENERATION_CHECK_STEP, REPEATED_LINES_THRESHOLD, MIN_PERIOD_REPEATS, } from './output-degeneration'; @@ -180,3 +182,42 @@ describe('truncateDegeneratedTail', () => { expect(truncateDegeneratedTail(text)).toBe(text); }); }); + +/** + * Throttle + step-boundary reset (#486). The stream keeps a watermark + * (`lastDegenerationCheckLen`) that is an OFFSET into the accumulated step text. + * On a step boundary the accumulator resets to '', so the watermark MUST reset to + * 0 too — otherwise the throttle goes silent for the whole next step. These tests + * pin the pure decision AND the reset property that ai-chat.service.onStepFinish + * now enforces. + */ +describe('shouldCheckDegeneration (throttle) + step-boundary reset (#486)', () => { + it('fires once the text grows a full DEGENERATION_CHECK_STEP past the mark', () => { + expect(shouldCheckDegeneration(DEGENERATION_CHECK_STEP, 0)).toBe(true); + expect(shouldCheckDegeneration(DEGENERATION_CHECK_STEP - 1, 0)).toBe(false); + expect(shouldCheckDegeneration(5000, 3000)).toBe(true); // grew 2000 since mark + expect(shouldCheckDegeneration(4000, 3000)).toBe(false); // grew only 1000 + }); + + it('BUG (no reset): a stale large watermark silences the next step', () => { + // End of a long step: the watermark sits at 5000. The step ends and the + // accumulator resets to '' — but if the watermark is NOT reset, a fresh short + // degenerate burst (length 2000) never triggers a check: 2000 - 5000 < STEP. + const staleWatermark = 5000; + const nextStepLen = DEGENERATION_CHECK_STEP; // a fresh 2KB burst + expect(shouldCheckDegeneration(nextStepLen, staleWatermark)).toBe(false); + }); + + it('FIX (reset to 0): the same short degenerate burst IS checked and detected', () => { + // onStepFinish now zeroes the watermark, so the fresh burst re-arms the check. + const resetWatermark = 0; + const degenerateBurst = 'loadTools.\n'.repeat(300); // real degeneration + expect(degenerateBurst.length).toBeGreaterThanOrEqual(DEGENERATION_CHECK_STEP); + // The throttle now fires... + expect( + shouldCheckDegeneration(degenerateBurst.length, resetWatermark), + ).toBe(true); + // ...and the detector catches the loop that would otherwise stream unchecked. + expect(isDegenerateOutput(degenerateBurst)).toBe(true); + }); +}); diff --git a/apps/server/src/core/ai-chat/output-degeneration.ts b/apps/server/src/core/ai-chat/output-degeneration.ts index 2834ce7e..eff4c080 100644 --- a/apps/server/src/core/ai-chat/output-degeneration.ts +++ b/apps/server/src/core/ai-chat/output-degeneration.ts @@ -131,6 +131,32 @@ export function isDegenerateOutput(text: string): boolean { return hasRepeatedLineRun(text) || hasPeriodicTail(text); } +/** + * How many bytes the in-progress text must grow before the (amortized) tail + * heuristics are re-run. Shared with ai-chat.service so the throttle the stream + * applies is the SAME one the unit test drives. + */ +export const DEGENERATION_CHECK_STEP = 2000; + +/** + * Throttle decision for the degeneration guard (#444/#486). Returns true when + * the accumulated text has grown at least DEGENERATION_CHECK_STEP bytes past the + * last-checked offset, so the pure rules only fire every ~2KB. Pure; the caller + * updates its watermark to `textLen` when this returns true. + * + * The watermark is an offset INTO the accumulator, so when the accumulator is + * reset to '' on a step boundary the caller MUST reset the watermark to 0 too + * (#486). Otherwise `textLen - lastCheckLen` goes negative after the reset and + * this returns false until a later step re-grows past the stale offset — a whole + * degenerate step could stream unchecked. + */ +export function shouldCheckDegeneration( + textLen: number, + lastCheckLen: number, +): boolean { + return textLen - lastCheckLen >= DEGENERATION_CHECK_STEP; +} + /** * Truncate a degenerated tail before persist so hundreds of KB of garbage never * reach the DB / replay (#444). Keeps everything up to and including the FIRST From de8f9c804ca4fa1a3e948ed67d749b4306311da3 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 03:24:19 +0300 Subject: [PATCH 10/41] =?UTF-8?q?fix(client):=20flushNext=20=D0=B2=20onFin?= =?UTF-8?q?ish=20=D0=B3=D0=B5=D0=B9=D1=82=D0=B8=D1=82=D1=81=D1=8F=20mounte?= =?UTF-8?q?dRef=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Финальный onFinish->flushNext() не проверял live-mount флаг. Чистый onFinish может прийти ПОСЛЕ анмаунта треда (New-chat / переключение чата мид-стрим — асинхронные attach/resume оседают поздно): flush дергал очередь и re-POST'ил сообщение из брошенного треда — «призрачные» отправки/чаты-призраки. Остальные обращения к очереди уже гейтятся mountedRef; закрываем последнюю дыру. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/chat-thread.test.tsx | 46 +++++++++++++++++++ .../ai-chat/components/chat-thread.tsx | 8 +++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index 3d500495..0cebf8e6 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -203,6 +203,52 @@ describe("ChatThread — send now (#198)", () => { }); }); +// #486: the final onFinish -> flushNext() must be gated on the live-mount flag. +// A clean onFinish can land AFTER the thread unmounts (New-chat / chat-switch +// mid-stream — the async attach/resume settles late); flushing then dequeues and +// re-POSTs a queued message from an abandoned thread (a "ghost" send). +describe("ChatThread — onFinish flush gated on mount (#486)", () => { + beforeEach(resetState); + afterEach(cleanup); + + it("a clean onFinish WHILE MOUNTED flushes the queued message (control)", () => { + renderThread(); + fireEvent.click(screen.getByTestId("queue-btn")); // enqueue "queued text" + expect(h.state.sendMessage).not.toHaveBeenCalled(); + + act(() => { + h.state.onFinish?.({ + message: { id: "a", role: "assistant", parts: [] }, + isAbort: false, + isDisconnect: false, + isError: false, + }); + }); + // Mounted: the queue flushes normally. + expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); + }); + + it("a clean onFinish AFTER unmount does NOT flush (no ghost send)", () => { + const { unmount } = renderThread(); + fireEvent.click(screen.getByTestId("queue-btn")); // enqueue "queued text" + h.state.sendMessage.mockClear(); + + // Chat switched away mid-stream: the streamer unmounts... + unmount(); + // ...and a late, clean onFinish lands on the abandoned thread. + act(() => { + h.state.onFinish?.({ + message: { id: "a", role: "assistant", parts: [] }, + isAbort: false, + isDisconnect: false, + isError: false, + }); + }); + // Gated on mountedRef: NOTHING is sent from the dead thread. + expect(h.state.sendMessage).not.toHaveBeenCalled(); + }); +}); + // #396: in autonomous mode a live sendNow must additionally request the // AUTHORITATIVE server stop of the detached run (a local abort is only a client // disconnect the server ignores) and arm a bounded 409 retry so the re-POST diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index 29f88dba..4c3662f7 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -659,7 +659,13 @@ export default function ChatThread({ return; } if (isAbort || isDisconnect || isError) return; - flushNext(); + // Gate the final flush on the live-mount flag (#486): a clean onFinish can + // land AFTER this thread unmounted (a New-chat / chat-switch mid-stream — + // the async attach/resume settles late). Flushing then dequeues and POSTs a + // queued message from an abandoned thread — a "ghost" send / ghost chat. + // Every other queue side effect already guards on mountedRef; this last one + // was the gap. + if (mountedRef.current) flushNext(); }, // `onError` runs in addition to `onFinish` (which ai@6 also calls on error). // Log the raw failure here for devtools; the UI shows a friendly classified From 14da2951852386fb448412128aa2e5fa162ad3b2 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 03:25:56 +0300 Subject: [PATCH 11/41] =?UTF-8?q?fix(auth):=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D0=BD=D0=B0=D0=BD=D1=81=20=D0=B4=D0=BB=D1=8F=20API-key?= =?UTF-8?q?=20=D0=BF=D1=83=D1=82=D0=B8=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit validateApiKey возвращал результат до резолва провенанса — REST-записи по is_agent API-ключу не получали маркер 'agent'. Перенести «выше» нельзя: payload API-ключа не несёт подписанный actor-клейм, а user (с isAgent) неизвестен до валидации ключа. Резолвим провенанс от возвращённого user: isAgent -> 'agent', иначе 'user'; aiChatId у API-ключа всегда null (нет ai_chats-строки). Загрузка EE ApiKeyService вынесена в переопределяемый шов resolveApiKeyService для юнит- теста без EE-бандла. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../core/auth/strategies/jwt.strategy.spec.ts | 99 +++++++++++++++++++ .../src/core/auth/strategies/jwt.strategy.ts | 51 +++++++--- 2 files changed, 135 insertions(+), 15 deletions(-) diff --git a/apps/server/src/core/auth/strategies/jwt.strategy.spec.ts b/apps/server/src/core/auth/strategies/jwt.strategy.spec.ts index 11544df7..20a669ca 100644 --- a/apps/server/src/core/auth/strategies/jwt.strategy.spec.ts +++ b/apps/server/src/core/auth/strategies/jwt.strategy.spec.ts @@ -120,3 +120,102 @@ describe('JwtStrategy — provenance derivation', () => { expect(req.raw.actor).toBeUndefined(); }); }); + +/** + * Provenance derivation on the API-KEY path (jwt.strategy.validateApiKey, #486). + * + * The access-token path stamped provenance; the API-key path returned early + * WITHOUT it, so an is_agent API key's REST writes recorded no 'agent' marker. + * The API-key payload carries no signed claim, so provenance is resolved from the + * SERVER-SIDE user returned by ApiKeyService.validateApiKey: isAgent -> 'agent', + * otherwise 'user'; aiChatId is always null (an API key has no ai_chats row). + * + * The enterprise ApiKeyService is not bundled in the OSS build, so the strategy + * loads it through an overridable `resolveApiKeyService` seam that we stub here. + */ +describe('JwtStrategy — API-key provenance derivation (#486)', () => { + function makeApiKeyStrategy(validateApiKeyImpl: (p: any) => Promise) { + const userRepo: any = { findById: jest.fn() }; + const workspaceRepo: any = { findById: jest.fn() }; + const userSessionRepo: any = { findActiveById: jest.fn() }; + const sessionActivityService: any = { trackActivity: jest.fn() }; + const environmentService: any = { getAppSecret: () => 'test-secret' }; + const moduleRef: any = {}; + + const strategy = new JwtStrategy( + userRepo, + workspaceRepo, + userSessionRepo, + sessionActivityService, + environmentService, + moduleRef, + ); + // Stub the EE ApiKeyService seam (the real module is not in the OSS build). + const validateApiKey = jest.fn(validateApiKeyImpl); + jest + .spyOn(strategy as any, 'resolveApiKeyService') + .mockReturnValue({ validateApiKey }); + return { strategy, validateApiKey }; + } + + const makeReq = () => ({ raw: {} as Record }); + const apiKeyPayload = () => ({ + sub: 'svc-1', + workspaceId: 'ws-1', + apiKeyId: 'key-1', + type: JwtType.API_KEY, + }); + + it("stamps actor='agent' for an is_agent API key (from the validated user)", async () => { + const validated = { + user: { id: 'svc-1', isAgent: true }, + workspace: { id: 'ws-1' }, + }; + const { strategy, validateApiKey } = makeApiKeyStrategy( + async () => validated, + ); + const req = makeReq(); + + const result = await strategy.validate(req, apiKeyPayload() as any); + + expect(validateApiKey).toHaveBeenCalledTimes(1); + expect(req.raw.actor).toBe('agent'); + // API keys carry no internal ai_chats row -> null. + expect(req.raw.aiChatId).toBeNull(); + // The validated auth object is returned unchanged (req.user shape preserved). + expect(result).toBe(validated); + }); + + it("stamps actor='user' for an ordinary (non-agent) API key", async () => { + const { strategy } = makeApiKeyStrategy(async () => ({ + user: { id: 'u-1', isAgent: false }, + workspace: { id: 'ws-1' }, + })); + const req = makeReq(); + + await strategy.validate(req, apiKeyPayload() as any); + + expect(req.raw.actor).toBe('user'); + expect(req.raw.aiChatId).toBeNull(); + }); + + it('throws Unauthorized (and stamps nothing) when the EE module is missing', async () => { + const userRepo: any = { findById: jest.fn() }; + const strategy = new JwtStrategy( + userRepo, + { findById: jest.fn() } as any, + { findActiveById: jest.fn() } as any, + { trackActivity: jest.fn() } as any, + { getAppSecret: () => 'test-secret' } as any, + {} as any, + ); + // EE not bundled: the seam returns null. + jest.spyOn(strategy as any, 'resolveApiKeyService').mockReturnValue(null); + const req = makeReq(); + + await expect( + strategy.validate(req, apiKeyPayload() as any), + ).rejects.toThrow(UnauthorizedException); + expect(req.raw.actor).toBeUndefined(); + }); +}); diff --git a/apps/server/src/core/auth/strategies/jwt.strategy.ts b/apps/server/src/core/auth/strategies/jwt.strategy.ts index 9e5604a9..2abd65c2 100644 --- a/apps/server/src/core/auth/strategies/jwt.strategy.ts +++ b/apps/server/src/core/auth/strategies/jwt.strategy.ts @@ -102,28 +102,49 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') { } private async validateApiKey(req: any, payload: JwtApiKeyPayload) { - let ApiKeyModule: any; - let isApiKeyModuleReady = false; + const apiKeyService = this.resolveApiKeyService(); + if (!apiKeyService) { + throw new UnauthorizedException('Enterprise API Key module missing'); + } + const result = await apiKeyService.validateApiKey(payload); + + // Stamp the agent-edit provenance for the API-KEY path too (#486). Unlike the + // access-token path above, it CANNOT be resolved before this point: the + // API-key payload carries no signed actor/aiChatId claim, and the user (with + // its isAgent flag) is unknown until the key is validated. Claim semantics for + // API keys: an is_agent API key (an agent service account) stamps 'agent' on + // every REST write; an ordinary API key resolves to 'user'. An API key has no + // internal ai_chats row, so aiChatId is always null. Derived from the + // SERVER-SIDE user (never a client field), so an 'agent' badge is unspoofable + // — mirroring the access-token path. Passing `null` for the claim means the + // actor is decided solely by user.isAgent. + const provenance = resolveProvenance((result as any)?.user, null); + req.raw.actor = provenance.actor; + req.raw.aiChatId = provenance.aiChatId; + + return result; + } + + /** + * Resolve the enterprise ApiKeyService, or `null` when the EE module is not + * bundled in this build (community build). Extracted as an overridable seam so + * the API-key provenance stamping can be unit-tested without the EE package + * present (docmost is OSS + a separate EE bundle; `require` of the EE path + * throws here). Any load/resolve error is treated as "module missing". + */ + protected resolveApiKeyService(): { + validateApiKey: (payload: JwtApiKeyPayload) => Promise; + } | null { try { // eslint-disable-next-line @typescript-eslint/no-require-imports - ApiKeyModule = require('./../../../ee/api-key/api-key.service'); - isApiKeyModuleReady = true; + const ApiKeyModule = require('./../../../ee/api-key/api-key.service'); + return this.moduleRef.get(ApiKeyModule.ApiKeyService, { strict: false }); } catch (err) { this.logger.debug( 'API Key module requested but enterprise module not bundled in this build', ); - isApiKeyModuleReady = false; + return null; } - - if (isApiKeyModuleReady) { - const ApiKeyService = this.moduleRef.get(ApiKeyModule.ApiKeyService, { - strict: false, - }); - - return ApiKeyService.validateApiKey(payload); - } - - throw new UnauthorizedException('Enterprise API Key module missing'); } } From b934fb2292adf782f76f395880c9ff790eb5db4e Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 03:28:08 +0300 Subject: [PATCH 12/41] =?UTF-8?q?fix(mcp):=20=D0=BF=D1=80=D0=BE=D0=B1?= =?UTF-8?q?=D1=80=D0=BE=D1=81=D0=B8=D1=82=D1=8C=20truncated=20=D0=B2=20tre?= =?UTF-8?q?e-mode=20listPages=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit listPages(tree:true) деструктурировал только pages из enumerateSpacePages и возвращал голое дерево, теряя truncated — на неполном дереве (stdio-fallback BFS упёрся в node-cap) вызывающий не знал, что страницы потеряны. Возвращаем { tree, truncated } (по образцу check_new_comments); основной /pages/tree путь беспредельный, там false. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp/src/client/read.ts | 9 ++- .../mock/list-pages-tree-truncated.test.mjs | 55 +++++++++++++++++++ .../mcp/test/mock/pagination-cursor.test.mjs | 10 +++- 3 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 packages/mcp/test/mock/list-pages-tree-truncated.test.mjs diff --git a/packages/mcp/src/client/read.ts b/packages/mcp/src/client/read.ts index 479a8c62..0587ab47 100644 --- a/packages/mcp/src/client/read.ts +++ b/packages/mcp/src/client/read.ts @@ -127,8 +127,13 @@ export function ReadMixin>(Base "listPages: tree mode requires a spaceId (a page tree is scoped to one space). Pass spaceId, or omit tree to get the recent-pages list.", ); } - const { pages } = await this.enumerateSpacePages(spaceId); - return buildPageTree(pages); + // #486: propagate `truncated` (same pattern as check_new_comments). The old + // code dropped it, so a caller handed an INCOMPLETE tree (the stdio-fallback + // BFS hit its node cap) had no way to know pages were missing. Return the + // tree alongside the flag; the primary /pages/tree path is uncapped so this + // is false there. + const { pages, truncated } = await this.enumerateSpacePages(spaceId); + return { tree: buildPageTree(pages), truncated }; } const clampedLimit = Math.max(1, Math.min(100, limit)); diff --git a/packages/mcp/test/mock/list-pages-tree-truncated.test.mjs b/packages/mcp/test/mock/list-pages-tree-truncated.test.mjs new file mode 100644 index 00000000..0d39ab23 --- /dev/null +++ b/packages/mcp/test/mock/list-pages-tree-truncated.test.mjs @@ -0,0 +1,55 @@ +// Unit test: listPages tree mode must propagate the `truncated` flag (#486). +// +// enumerateSpacePages returns { pages, truncated } — truncated is true ONLY when +// the stdio-fallback BFS hit its node cap (the primary /pages/tree path is +// uncapped). The old tree-mode listPages destructured only `pages` and returned a +// bare tree, dropping `truncated`, so a caller handed an INCOMPLETE tree had no +// way to know pages were missing. The fix returns { tree, truncated } (same +// pattern check_new_comments uses). +// +// Reaching the real cap (MAX_NODES = 10000) in a mock is impractical, so we stub +// enumerateSpacePages directly to assert the flag is threaded through verbatim. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { DocmostClient } from "../../build/client.js"; + +function stubClient() { + const client = new DocmostClient({ + apiUrl: "http://127.0.0.1:1/api", + getToken: async () => "access", + }); + // No network: the tree path only calls ensureAuthenticated + enumerateSpacePages. + client.ensureAuthenticated = async () => {}; + return client; +} + +const onePage = [{ id: "r1", title: "Root", parentPageId: null }]; + +test("tree mode carries truncated:true when the enumeration truncated (#486)", async () => { + const client = stubClient(); + client.enumerateSpacePages = async () => ({ pages: onePage, truncated: true }); + + const res = await client.listPages("space-1", 50, true); + + assert.equal(res.truncated, true, "the truncated flag is threaded through"); + assert.ok(Array.isArray(res.tree), "the built tree rides alongside the flag"); + assert.equal(res.tree[0].id, "r1"); +}); + +test("tree mode carries truncated:false for a complete enumeration", async () => { + const client = stubClient(); + client.enumerateSpacePages = async () => ({ pages: onePage, truncated: false }); + + const res = await client.listPages("space-1", 50, true); + + assert.equal(res.truncated, false); + assert.equal(res.tree[0].id, "r1"); +}); + +test("tree mode still requires a spaceId", async () => { + const client = stubClient(); + await assert.rejects( + client.listPages(undefined, 50, true), + /tree mode requires a spaceId/, + ); +}); diff --git a/packages/mcp/test/mock/pagination-cursor.test.mjs b/packages/mcp/test/mock/pagination-cursor.test.mjs index a81a158e..fd6ab47f 100644 --- a/packages/mcp/test/mock/pagination-cursor.test.mjs +++ b/packages/mcp/test/mock/pagination-cursor.test.mjs @@ -184,12 +184,14 @@ test("enumerateSpacePages (via listPages tree) uses one /pages/tree request", as }); const client = new DocmostClient(baseURL, "user@example.com", "pw"); - // listPages tree:true -> enumerateSpacePages(spaceId) -> buildPageTree. - const tree = await client.listPages("space-1", 50, true); + // listPages tree:true -> enumerateSpacePages(spaceId) -> { tree, truncated }. + const { tree, truncated } = await client.listPages("space-1", 50, true); assert.equal(treeRequests, 1, "exactly one /pages/tree request for the space"); assert.equal(sidebarRequests, 0, "no per-node sidebar BFS requests"); assert.deepEqual(treeBody, { spaceId: "space-1" }, "space scope posts spaceId only"); + // The uncapped /pages/tree path is never truncated (#486). + assert.equal(truncated, false, "primary /pages/tree path is not truncated"); // buildPageTree nests c1 under r1; two roots at the top level. assert.equal(tree.length, 2, "two root nodes"); const r1 = tree.find((n) => n.id === "r1"); @@ -249,7 +251,7 @@ test("enumerateSpacePages falls back to the cursor BFS on /pages/tree 404", asyn }); const client = new DocmostClient(baseURL, "user@example.com", "pw"); - const tree = await client.listPages("space-1", 50, true); + const { tree, truncated } = await client.listPages("space-1", 50, true); assert.ok(treeRequests >= 1, "the tree endpoint was attempted first"); assert.deepEqual( @@ -257,6 +259,8 @@ test("enumerateSpacePages falls back to the cursor BFS on /pages/tree 404", asyn ["", "r1"], "fell back to the sidebar BFS: roots then the root's children", ); + // Small fallback walk well under the node cap -> not truncated (#486). + assert.equal(truncated, false, "fallback BFS below the cap is not truncated"); assert.equal(tree.length, 1, "one root in the built tree"); assert.equal(tree[0].children[0].id, "c1", "leaf nested via the BFS"); }); From 255024fbe2179de649b813feb8d654a4e21e320f Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 03:31:24 +0300 Subject: [PATCH 13/41] =?UTF-8?q?chore(mcp):=20McpService.onModuleDestroy?= =?UTF-8?q?=20=E2=86=92=20destroyAllSessions=20+=20=D1=83=D0=B4=D0=B0?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BC=D1=91=D1=80=D1=82=D0=B2?= =?UTF-8?q?=D0=BE=D0=B3=D0=BE=20=D0=BA=D0=BE=D0=B4=D0=B0=20(#486)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit onModuleDestroy чистил только sweep-таймер; loopback-collab-сессии держали доки открытыми на collab-сервере до идла — рестарт мог гонять с доком, запиненным умирающим воркером. Теперь дергает destroyAllSessions() через переопределяемый шов (для юнит-теста без ESM-пакета), best-effort. Плюс удалён мёртвый код: неиспользуемый импорт parseNodeArg в mcp/index.ts и мёртвые enum-члены SEARCH_REMOVE_* в queue.constants (подтверждено grep'ом — ни диспетчера, ни процессора). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/integrations/mcp/mcp.service.spec.ts | 44 +++++++++++++++++++ .../src/integrations/mcp/mcp.service.ts | 34 +++++++++++++- .../queue/constants/queue.constants.ts | 3 -- packages/mcp/src/index.ts | 1 - 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/apps/server/src/integrations/mcp/mcp.service.spec.ts b/apps/server/src/integrations/mcp/mcp.service.spec.ts index a6f93d5d..6894b97e 100644 --- a/apps/server/src/integrations/mcp/mcp.service.spec.ts +++ b/apps/server/src/integrations/mcp/mcp.service.spec.ts @@ -16,6 +16,7 @@ import { } from './mcp-auth.helpers'; import { JwtType } from '../../core/auth/dto/jwt-payload'; import { CREDENTIALS_MISMATCH_MESSAGE } from '../../core/auth/auth.constants'; +import { McpService } from './mcp.service'; // The /mcp per-user auth decision logic is tested through the framework-free // `resolveMcpSessionConfig` helper that McpService delegates to. McpService @@ -1179,3 +1180,46 @@ describe('mapAuthResultToResponse (handle status/body mapping, refactor R2)', () }); }); }); + +// #486: onModuleDestroy must ALSO tear down the live loopback CollabSessions, not +// just clear the sweep timer — otherwise the embedded MCP's collab sockets keep +// docs pinned open on the collab server past process exit. The teardown goes +// through an overridable seam (destroyAllMcpSessions) so it can be spied without +// loading the ESM-only @docmost/mcp package. +describe('McpService.onModuleDestroy — CollabSession teardown (#486)', () => { + function makeService(): McpService { + // The constructor only stores its deps and starts the (unref'd) sweep timer, + // so bare stubs suffice. onModuleDestroy clears that timer, so no leak. + return new McpService( + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + {} as any, + ); + } + + it('destroys all sessions AND clears the sweep timer on shutdown', async () => { + const svc = makeService(); + const destroy = jest.fn().mockResolvedValue(undefined); + (svc as any).destroyAllMcpSessions = destroy; + const clearSpy = jest.spyOn(global, 'clearInterval'); + + await svc.onModuleDestroy(); + + expect(destroy).toHaveBeenCalledTimes(1); + expect(clearSpy).toHaveBeenCalledWith((svc as any).sweepTimer); + clearSpy.mockRestore(); + }); + + it('swallows a teardown failure so shutdown never throws', async () => { + const svc = makeService(); + (svc as any).destroyAllMcpSessions = jest + .fn() + .mockRejectedValue(new Error('collab teardown boom')); + + await expect(svc.onModuleDestroy()).resolves.toBeUndefined(); + }); +}); diff --git a/apps/server/src/integrations/mcp/mcp.service.ts b/apps/server/src/integrations/mcp/mcp.service.ts index e59c442c..e72ee8ac 100644 --- a/apps/server/src/integrations/mcp/mcp.service.ts +++ b/apps/server/src/integrations/mcp/mcp.service.ts @@ -119,10 +119,42 @@ export class McpService implements OnModuleDestroy { this.sweepTimer.unref?.(); } - onModuleDestroy(): void { + async onModuleDestroy(): Promise { clearInterval(this.sweepTimer); + // Tear down any live loopback CollabSession providers at shutdown (#486). The + // embedded MCP (and the in-app AI agent) open Hocuspocus collab sockets against + // THIS process; without an explicit teardown those sessions keep their docs + // "open" on the collab server and hold providers/buffers until they idle out, + // so a restart can race a doc still pinned by the dying worker. Best-effort: + // any failure is logged, never allowed to break shutdown. + try { + await this.destroyAllMcpSessions(); + } catch (err) { + this.logger.error( + 'MCP CollabSession teardown on shutdown failed', + err as Error, + ); + } } + /** + * Resolve @docmost/mcp's `destroyAllSessions` and invoke it (#486). The live + * CollabSession registry is a module-level singleton in the ESM package, shared + * by every entry (`.`/`./http`), so this tears down ALL sessions regardless of + * which surface opened them. The module is already loaded whenever MCP was used; + * if it was never loaded (or is absent) the import + no-op is harmless. + * + * Held as an overridable field so a unit test can spy the teardown without + * loading the ESM-only package or standing up the DI graph. + */ + private destroyAllMcpSessions: () => Promise = async () => { + const entry = require.resolve('@docmost/mcp'); + const mod = (await esmImport(pathToFileURL(entry).href)) as { + destroyAllSessions?: () => void; + }; + mod.destroyAllSessions?.(); + }; + // Service account the embedded MCP uses to talk back to this Docmost // instance over loopback REST + the collaboration WebSocket. Now OPTIONAL: // it is only a fallback when no per-user Basic/Bearer credentials are sent. diff --git a/apps/server/src/integrations/queue/constants/queue.constants.ts b/apps/server/src/integrations/queue/constants/queue.constants.ts index d1ad9b7d..6bc95ffc 100644 --- a/apps/server/src/integrations/queue/constants/queue.constants.ts +++ b/apps/server/src/integrations/queue/constants/queue.constants.ts @@ -31,9 +31,6 @@ export enum QueueJob { IMPORT_TASK = 'import-task', EXPORT_TASK = 'export-task', - SEARCH_REMOVE_PAGE = 'search-remove-page', - SEARCH_REMOVE_ASSET = 'search-remove-attachment', - SEARCH_REMOVE_FACE = 'search-remove-comment', TYPESENSE_FLUSH = 'typesense-flush', PAGE_CREATED = 'page-created', diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 88c5b305..15fed6ac 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -4,7 +4,6 @@ import { readFileSync } from "fs"; import { fileURLToPath } from "url"; import { dirname, join } from "path"; import { DocmostClient, DocmostMcpConfig } from "./client.js"; -import { parseNodeArg } from "@docmost/prosemirror-markdown"; import { searchShapes } from "./lib/drawio-shapes.js"; import { getGuideSection } from "./lib/drawio-guide.js"; import { SHARED_TOOL_SPECS, SharedToolSpec } from "./tool-specs.js"; From ea7c4d7cd2423e575239a25ff92fe4d9407900cc Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 03:54:50 +0300 Subject: [PATCH 14/41] =?UTF-8?q?fix(#486):=20=D1=80=D0=B5=D0=B2=D1=8C?= =?UTF-8?q?=D1=8E=20=E2=80=94=20run-race=20=D0=BA=D0=BE=D0=BD=D1=82=D1=80?= =?UTF-8?q?=D0=B0=D0=BA=D1=82=20=D0=BF=D0=BE=D0=B4=20=D0=BD=D0=BE=D0=B2?= =?UTF-8?q?=D1=83=D1=8E=20=D0=BF=D0=BE=D0=BB=D0=B8=D1=82=D0=B8=D0=BA=D1=83?= =?UTF-8?q?=20+=20timing-safe=20metrics-=D1=82=D0=BE=D0=BA=D0=B5=D0=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ревью полной ветки #486 нашло два пункта в моих коммитах (3 и 4): - BLOCKER (коммит 4): ai-chat.service.run-race.spec.ts (#184 F14) пинил СТАРУЮ политику «plain begin() failure → swallow + стрим UNTRACKED» (resolves toBeUndefined). Коммит 4 её развернул → тест падал (1 failed/6). Кейс ИНВЕРТИРОВАН под новую политику: plain begin-failure теперь REJECTS с 503 A_RUN_BEGIN_FAILED, до первого байта, user-строка не вставлена, streamText не вызван — путь остаётся явно запинён, а не удалён. - NIT (коммит 3): metrics-токен сравнивался наивным !== (единственный слой auth эндпоинта) → тайминг-утечка токена. Заменено на crypto.timingSafeEqual с length-guard (разная длина → reject), семантика 401/200 без изменений. Внесено отдельным fixup-коммитом (rebase -i недоступен в окружении; ветка не запушена). Тесты: run-race 7/7 + metrics.server 7/7 зелёные. --- .../ai-chat/ai-chat.service.run-race.spec.ts | 60 ++++++++++--------- .../integrations/metrics/metrics.server.ts | 22 ++++++- 2 files changed, 52 insertions(+), 30 deletions(-) diff --git a/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts index a5db3b3f..d167abbe 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts @@ -1,4 +1,8 @@ -import { ConflictException, Logger } from '@nestjs/common'; +import { + ConflictException, + Logger, + ServiceUnavailableException, +} from '@nestjs/common'; // Mock the AI SDK so we can PROVE no provider call is made for the turn we are // about to reject. The race rejection happens at runHooks.begin(), long before @@ -360,22 +364,22 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => { }); /** - * F14 — the begin-failure RESILIENCE branch (the `else` of the run-race guard). + * F14 — the begin-failure branch (the `else` of the run-race guard). * * stream() wraps runHooks.begin in try/catch with TWO branches: * - RunAlreadyActiveError -> 409 ConflictException (pinned above). - * - ANY OTHER begin failure -> SWALLOW + continue UNTRACKED on the socket signal - * (legacy fallback): it logs "...streaming without run tracking", leaves - * `effectiveSignal = signal` (runId undefined) and serves the turn anyway. + * - ANY OTHER begin failure -> throw ServiceUnavailableException(A_RUN_BEGIN_FAILED) + * BEFORE the first byte (#486, commit 4). * - * The contract: a transient beginRun failure (e.g. a non-unique DB error inserting - * the run row) must STILL serve the user's turn — it must NOT re-throw and must NOT - * be misclassified as a 409. A regression that re-threw here would break EVERY turn - * on a begin failure with nothing to catch it. This branch is otherwise undriven by - * any spec, so it is pinned here SEPARATELY from the 409 path: a plain begin error - * proceeds to streamText with the SOCKET signal and still persists the user turn. + * POLICY CHANGE (#486): the OLD contract here was "SWALLOW + stream the turn + * UNTRACKED on the socket signal". That was reversed: an untracked run is + * invisible to /stop, is not aborted on disconnect, and slips past the one-run + * gate — an unstoppable ghost run in autonomous mode. Now a plain begin failure + * FAILS the turn fast with a 503 A_RUN_BEGIN_FAILED, before any user row is + * persisted and before streamText runs. This case is INVERTED (not deleted) so + * the "plain begin failure" path stays explicitly pinned under the new policy. */ -describe('AiChatService.stream — begin-failure resilience / legacy fallback (#184 F14)', () => { +describe('AiChatService.stream — begin-failure fails the turn (#184 F14 / #486)', () => { const streamTextMock = streamText as unknown as jest.Mock; function makeStreamResult() { @@ -455,7 +459,7 @@ describe('AiChatService.stream — begin-failure resilience / legacy fallback (# afterEach(() => jest.restoreAllMocks()); - it('a PLAIN begin() failure (NOT RunAlreadyActiveError) does NOT 409 — it swallows, logs, and streams the turn UNTRACKED on the socket signal', async () => { + it('a PLAIN begin() failure (NOT RunAlreadyActiveError) FAILS the turn with a 503 A_RUN_BEGIN_FAILED before the first byte — NO untracked stream (#486)', async () => { const errorSpy = jest .spyOn(Logger.prototype, 'error') .mockImplementation(() => undefined as never); @@ -487,28 +491,26 @@ describe('AiChatService.stream — begin-failure resilience / legacy fallback (# } as never, }); - // The turn proceeds: NO throw at all (in particular NOT a 409). - await expect(promise).resolves.toBeUndefined(); + // NEW POLICY: the turn is REJECTED with a 503 A_RUN_BEGIN_FAILED (not a 409, + // and NOT swallowed into an untracked stream). + await expect(promise).rejects.toBeInstanceOf(ServiceUnavailableException); + const err = (await promise.catch( + (e) => e, + )) as ServiceUnavailableException; + expect(err.getStatus()).toBe(503); + expect(err.getResponse()).toMatchObject({ code: 'A_RUN_BEGIN_FAILED' }); expect(begin).toHaveBeenCalledTimes(1); - // The resilience branch logged the legacy-fallback warning. + // It logged the fail-the-turn line. expect(errorSpy).toHaveBeenCalledWith( - expect.stringContaining('streaming without run tracking'), + expect.stringContaining('failing the turn'), expect.anything(), ); - // The turn really streamed: the user message was persisted and streamText ran. - expect(aiChatMessageRepo.insert).toHaveBeenCalled(); - expect(streamTextMock).toHaveBeenCalledTimes(1); - - // The decisive wiring: with no run handle, the fallback uses the SOCKET signal - // (effectiveSignal = signal, runId undefined) — not a run-bound signal. #444: - // the signal is unioned with the degeneration controller via AbortSignal.any, - // so assert the socket abort still reaches the turn rather than identity. - const passed = streamTextMock.mock.calls[0][0].abortSignal as AbortSignal; - expect(passed.aborted).toBe(false); - socketController.abort(); - expect(passed.aborted).toBe(true); + // Fail-fast: the turn NEVER streamed — no user row persisted, no streamText + // call, so no orphan/untracked run was left behind. + expect(aiChatMessageRepo.insert).not.toHaveBeenCalled(); + expect(streamTextMock).not.toHaveBeenCalled(); }); }); diff --git a/apps/server/src/integrations/metrics/metrics.server.ts b/apps/server/src/integrations/metrics/metrics.server.ts index 577cbecd..bd579f84 100644 --- a/apps/server/src/integrations/metrics/metrics.server.ts +++ b/apps/server/src/integrations/metrics/metrics.server.ts @@ -1,7 +1,27 @@ import { createServer, Server } from 'node:http'; +import { timingSafeEqual } from 'node:crypto'; import { Logger } from '@nestjs/common'; import { getMetricsRegistry, isMetricsEnabled } from './metrics.registry'; +/** + * Constant-time compare of the presented Authorization header against the + * expected `Bearer `. This is the ONLY auth layer for the metrics + * endpoint, so a naive `!==` would leak the token byte-by-byte via timing. + * timingSafeEqual requires equal-length buffers, so a length mismatch short- + * circuits to "not equal" (its own length is not itself a useful oracle: the + * expected string length is fixed by config, not secret-derived). + */ +function bearerMatches( + presented: string | undefined, + expected: string, +): boolean { + if (typeof presented !== 'string') return false; + const a = Buffer.from(presented); + const b = Buffer.from(expected); + if (a.length !== b.length) return false; + return timingSafeEqual(a, b); +} + /** * Start the Prometheus scrape endpoint on a SEPARATE port, taken from * `METRICS_PORT`. There is NO default port: when `METRICS_PORT` is unset the @@ -64,7 +84,7 @@ export function startMetricsServer(): Server | null { // configured. This is the auth layer the old all-interfaces bind lacked. if (token) { const auth = req.headers['authorization']; - if (auth !== `Bearer ${token}`) { + if (!bearerMatches(auth, `Bearer ${token}`)) { res.statusCode = 401; res.setHeader('WWW-Authenticate', 'Bearer'); res.end(); From 4a750c1e7f96829a59110e751eede372da4403e4 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 04:38:16 +0300 Subject: [PATCH 15/41] =?UTF-8?q?fix(#486):=20=D1=80=D0=B5=D0=B2=D1=8C?= =?UTF-8?q?=D1=8E=20=E2=80=94=20CHANGELOG=20+=20AGENTS.md=20+=20=D0=B4?= =?UTF-8?q?=D0=B2=D0=B0=20=D1=82=D0=B5=D1=81=D1=82=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ре-ревью PR #500 (changes-requested, 4 мелких, все doc/test): F1 [CHANGELOG] CHANGELOG.md [Unreleased]: - Breaking Changes: metrics-листенер 0.0.0.0→127.0.0.1 — кросс-контейнерный скрейп (docmost:9464) молча умрёт без METRICS_BIND=0.0.0.0 + METRICS_TOKEN (миграция в .env.example). - Security: утечка errorText тулов/провайдера анониму (closes #394); /metrics под Bearer (METRICS_TOKEN). - Fixed: ioredis-утечка в /health; ELK вешал event loop; beginRun-призрак → честный 503 A_RUN_BEGIN_FAILED; ai drain-hang. F2 [AGENTS.md] строка про ai-патч: теперь он несёт ДВА фикса (#184 O(n²) partialOutput И #486 drain-hang), оба тривайра названы. F3 [test] metrics.server.spec: добавлен кейс токена ТОЙ ЖЕ длины (Bearer topsecreX) → 401 — пиннит constant-time сравнение (прежние кейсы коротили на length-guard, до timingSafeEqual не доходили). F4 [test] output-degeneration.spec: behavior-тест, гоняющий РЕАЛЬНЫЕ onChunk/onStepFinish из stream() — длинный чистый шаг → граница → свежий дегенеративный бёрст → ассерт abortSignal.aborted (было хардкодом resetWatermark=0, ревёрт правки не краснил). Мутационные пруфы (non-vacuous): F3 — форс compare→true роняет same-length кейс (401→200); F4 — ревёрт lastDegenerationCheckLen=0 роняет behavior-тест (aborted true→false). Оба восстановлены, специи зелёные (34/34). --- AGENTS.md | 2 +- CHANGELOG.md | 63 +++++++ .../core/ai-chat/output-degeneration.spec.ts | 157 ++++++++++++++++++ .../metrics/metrics.server.spec.ts | 10 +- 4 files changed, 230 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 642b927f..08756773 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -470,7 +470,7 @@ Vite SPA. Code is organized by feature under `apps/client/src/features/*` (mirro - **Errors must never be swallowed or shown as generic messages.** Every caught error MUST (1) be logged in full to the console/logger — error name, message, stack, `cause`, and (for HTTP/provider failures) the status code and response body — and (2) be surfaced to the user with a *specific, human-readable explanation of what actually went wrong*, never a bare generic string like "Something went wrong" / "Could not start recording" / "Transcription failed". Include the real reason (the underlying error/provider message) in the user-facing text. On the server, wrap third-party/provider failures with `describeProviderError` (or equivalent) and rethrow as a meaningful HTTP status + message — never let them collapse into an opaque 500. On the client, `console.error(, err)` the raw error AND show the extracted reason (e.g. `err.response?.data?.message`, or the error `name: message`) in the notification. - The version string shown in the UI comes from `APP_VERSION` (CI/Docker) or `git describe --tags --always` (local), resolved in `vite.config.ts` — not from `package.json`. - Server TS config is permissive (`noImplicitAny: false`, `strictNullChecks: false`, `no-explicit-any` lint disabled). Follow the existing relaxed style rather than tightening types broadly. -- Dependency versions are heavily pinned via `pnpm.overrides` and `pnpm.patchedDependencies` (`scimmy`, `yjs`, `ai`) in the root `package.json`. Don't bump pinned/patched deps casually; the patches and overrides exist for compatibility/security reasons. The `ai@6.0.134` patch disables the SDK's O(n²) cumulative `partialOutput` accumulation when no output strategy is requested (server heap OOM on long agent runs, #184; tripwire test: `apps/server/src/integrations/ai/ai-sdk-partial-output.patch.spec.ts`) — it MUST be re-created via `pnpm patch` when bumping `ai`. +- Dependency versions are heavily pinned via `pnpm.overrides` and `pnpm.patchedDependencies` (`scimmy`, `yjs`, `ai`) in the root `package.json`. Don't bump pinned/patched deps casually; the patches and overrides exist for compatibility/security reasons. The `ai@6.0.134` patch carries TWO independent server fixes, each with its own tripwire test: (1) it disables the SDK's O(n²) cumulative `partialOutput` accumulation when no output strategy is requested (server heap OOM on long agent runs, #184; tripwire: `apps/server/src/integrations/ai/ai-sdk-partial-output.patch.spec.ts`); (2) it fixes `writeToServerResponse`'s drain-hang — the loop awaited only `"drain"` under backpressure, so a mid-write client disconnect parked the pipe forever and leaked the reader/buffers until restart; it now races `"drain"` against `"close"`/`"error"`, cancels the reader on disconnect, and swallows the fire-and-forget read rejection (#486; tripwire: `apps/server/src/integrations/ai/ai-sdk-drain-hang.patch.spec.ts`). Both tripwires assert BOTH installed dist builds carry their patch marker. The patch MUST be re-created via `pnpm patch` when bumping `ai`. - **The MCP tool inventory in `SERVER_INSTRUCTIONS` is GENERATED from the registry** (`packages/mcp/src/server-instructions.ts`: `buildToolInventory()` over `SHARED_TOOL_SPECS`) and spliced into the hand-written routing prose (`ROUTING_PROSE`). So adding/renaming/removing a **shared** spec in `packages/mcp/src/tool-specs.ts` auto-updates the `` — no manual `SERVER_INSTRUCTIONS` edit needed. Only an **inline** MCP-only tool (those registered via `server.registerTool(...)` in `index.ts`, not through the registry) needs a one-line entry in `INLINE_MCP_INVENTORY`. Enforced by `packages/mcp/test/unit/tool-inventory.test.mjs`, which fails when a registered tool is missing from the generated inventory (there is no `EXCEPTIONS` opt-out anymore — every tool must appear). Update `ROUTING_PROSE` when a tool's *intent guidance* (when-to-use) changes. `packages/mcp/build/` is gitignored and rebuilt in CI/Docker via `pnpm build` (same convention as `git-sync`/`prosemirror-markdown`) — never commit it; rebuild locally after editing to run the tests. ## CI / release diff --git a/CHANGELOG.md b/CHANGELOG.md index 795c7502..ed4d5a03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -115,6 +115,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the old ProseMirror-JSON output. Released together with the `#411`/`#412` breaking window so external configs break exactly once. (#413) +- **The Prometheus `/metrics` listener now binds to `127.0.0.1` (loopback) by + default instead of `0.0.0.0` (all interfaces).** This closes an unauthenticated + endpoint that was previously reachable on every interface. **DEPLOY MIGRATION — + cross-container scraping breaks silently otherwise:** if your scraper runs in a + SEPARATE container and reaches the app as `docmost:9464` (the exact topology the + old `0.0.0.0` hardcode served), you MUST now set `METRICS_BIND=0.0.0.0` — and, + because that re-exposes the endpoint, also set `METRICS_TOKEN=` and + configure the scraper with a matching Bearer token. Without `METRICS_BIND`, the + scraper can no longer connect and metrics go dark with no error. See the + `METRICS_BIND` / `METRICS_TOKEN` block in `.env.example` for the migration. + Same-host (loopback) scrapers need no change. (#486) + ### Added - **Place several images side by side in a row.** A new "Inline (side by @@ -310,6 +322,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `tee()` branch of the stream result — a ~20-step, ~28k-chunk agent run retained ~1.7 GB and OOM'd the 2 GB JS heap. Streaming granularity is unchanged; the patch must be re-created if `ai` is ever bumped. (#184) + +- **The server no longer leaks a hung stream pipe on every mid-run client + disconnect.** The same `ai@6.0.134` pnpm patch now also fixes the SDK's + `writeToServerResponse`, which awaited only a `"drain"` event under + backpressure: when a client disconnected mid-write the socket never drained, so + the write loop parked forever, `response.end()` was unreachable, and the stream + reader plus buffered chunks were pinned until process restart (every mid-run + disconnect in autonomous mode leaked one). The patch races `"drain"` against + `"close"`/`"error"`, cancels the reader and ends the response on disconnect, and + swallows the fire-and-forget read rejection instead of crashing on an + unhandledRejection. (#486) + +- **A failed autonomous agent-run start no longer becomes an unstoppable ghost + run.** When `beginRun` failed for a transient reason (e.g. a DB-pool blip), + the turn previously continued with NO run row — invisible to `/stop`, not + aborted on disconnect, and able to slip a second run past the one-run-per-chat + gate, leaving an unstoppable run until restart. The turn now fails fast with an + honest `503 A_RUN_BEGIN_FAILED` before the first byte (no orphan state), and the + client shows a "temporary — please try again" message instead of a misleading + "provider not configured". (#486) + +- **A pathological draw.io graph can no longer wedge the whole server.** The ELK + auto-layout (`layout:"elk"`) ran elkjs synchronously on the main event loop, so + a graph at the node/edge cap blocked ALL HTTP/SSE/loopback traffic while it + churned — and the old `setTimeout` "timeout" could never fire because the same + thread was blocked. Layout now runs in a worker thread with the timeout enforced + by `worker.terminate()`; the main loop stays responsive. (#486) + +- **The `/health` Redis probe no longer leaks a client on every tick while Redis + is down.** It built a new `ioredis` client per probe and disconnected it only on + success, so during an outage each health tick added another forever-reconnecting + client (an unbounded handle leak). A single long-lived probe client is now + reused and closed on shutdown. (#486) - **Internal links in exported Markdown no longer lose their visible text.** A link whose target page name had no file extension (e.g. a bare title) was collapsed to empty text during export, producing an unclickable, label-less @@ -386,6 +431,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 share); any other value now returns the generic "not found" instead of serving the page. (#218) +- **Tool and provider error text no longer leaks to anonymous readers in the + public-share AI chat.** A failing tool's raw error (which could carry an + internal page title or a stack fragment) and a provider error (which bundles the + provider `statusCode` and response body — potentially the internal baseUrl or + model name) were streamed verbatim to the anonymous reader over SSE. Errors are + now sanitized at the source: the share toolset collapses any unclassified tool + error to a safe generic string (safe, classified tool messages still pass + through for the model's self-correction), and the anonymous stream `onError` + maps provider failures to a fixed set of neutral strings — the full detail goes + only to the server log. A UI render gate is layered on top. (closes #394) + +- **The Prometheus `/metrics` endpoint can now require Bearer authentication and + is loopback-bound by default.** Previously it listened on all interfaces with no + auth. Setting `METRICS_TOKEN` requires every scrape to present + `Authorization: Bearer ` (compared in constant time), and the listener + defaults to `127.0.0.1` (see the Breaking Changes entry for the cross-container + migration). (#486) + ## [0.94.0] - 2026-06-26 This release makes AI chat durable and fast: assistant turns are persisted to diff --git a/apps/server/src/core/ai-chat/output-degeneration.spec.ts b/apps/server/src/core/ai-chat/output-degeneration.spec.ts index 426f4e40..3595465d 100644 --- a/apps/server/src/core/ai-chat/output-degeneration.spec.ts +++ b/apps/server/src/core/ai-chat/output-degeneration.spec.ts @@ -1,3 +1,5 @@ +import { Logger } from '@nestjs/common'; +import { streamText } from 'ai'; import { hasRepeatedLineRun, hasPeriodicTail, @@ -8,6 +10,15 @@ import { REPEATED_LINES_THRESHOLD, MIN_PERIOD_REPEATS, } from './output-degeneration'; +import { AiChatService } from './ai-chat.service'; + +// Mock ONLY streamText so we can capture the onChunk/onStepFinish callbacks the +// service registers and drive them by hand; every other `ai` export the service +// uses (convertToModelMessages, stepCountIs, …) stays real. +jest.mock('ai', () => { + const actual = jest.requireActual('ai'); + return { ...actual, streamText: jest.fn() }; +}); /** * Unit tests for the token-degeneration detector (#444) — the sole anti-babble @@ -221,3 +232,149 @@ describe('shouldCheckDegeneration (throttle) + step-boundary reset (#486)', () = expect(isDegenerateOutput(degenerateBurst)).toBe(true); }); }); + +/** + * BEHAVIOR guard for the ACTUAL fix (#486, ai-chat.service.onStepFinish resets + * lastDegenerationCheckLen to 0). The pure tests above use a hard-coded + * resetWatermark, so a REVERT of the real `lastDegenerationCheckLen = 0` line + * would not redden any of them. This drives the REAL onChunk/onStepFinish + * closures from stream() end to end and asserts the run is aborted when a fresh + * degenerate burst arrives in the step AFTER a long clean step — which only + * happens if the watermark was actually zeroed on the step boundary. + */ +describe('AiChatService: onStepFinish re-arms the degeneration watermark (#486)', () => { + const streamTextMock = streamText as unknown as jest.Mock; + + function makeRes() { + return { + raw: { + writeHead: jest.fn(), + write: jest.fn(), + once: jest.fn(), + on: jest.fn(), + flushHeaders: jest.fn(), + writableEnded: false, + destroyed: false, + }, + }; + } + + function makeService() { + const aiChatRepo = { + findById: jest.fn(async () => ({ id: 'chat-1', workspaceId: 'ws-1' })), + insert: jest.fn(), + }; + const aiChatMessageRepo = { + insert: jest.fn(async () => ({ id: 'msg-1' })), + findAllByChat: jest.fn(async () => []), + update: jest.fn(async () => ({ id: 'msg-1' })), + }; + const aiSettings = { resolve: jest.fn(async () => ({})) }; + const tools = { forUser: jest.fn(async () => ({})) }; + const mcpClients = { + toolsFor: jest.fn(async () => ({ + tools: {}, + clients: [], + outcomes: [], + instructions: [], + })), + }; + return new AiChatService( + {} as never, // ai + aiChatRepo as never, + aiChatMessageRepo as never, + {} as never, // aiChatPageSnapshotRepo + aiSettings as never, + tools as never, + mcpClients as never, + {} as never, // aiAgentRoleRepo + {} as never, // pageRepo + {} as never, // pageAccess + { + isAiChatDeferredToolsEnabled: () => false, + // Lockdown OFF -> the degeneration guard is the active anti-babble path. + isAiChatFinalStepLockdownEnabled: () => false, + } as never, // environment + ); + } + + beforeEach(() => { + streamTextMock.mockReset(); + jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined as never); + jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined as never); + }); + + afterEach(() => jest.restoreAllMocks()); + + it('aborts on a fresh degenerate burst in the NEXT step (reverting the reset line reddens this)', async () => { + let captured: + | { + onChunk?: (e: { chunk: { type: string; text: string } }) => void; + onStepFinish?: (step: unknown) => void; + abortSignal?: AbortSignal; + } + | undefined; + streamTextMock.mockImplementation((opts: never) => { + captured = opts; + return { + consumeStream: jest.fn(), + pipeUIMessageStreamToResponse: jest.fn(), + }; + }); + + const svc = makeService(); + await svc.stream({ + user: { id: 'user-1' } as never, + workspace: { id: 'ws-1' } as never, + sessionId: 'sess-1', + body: { + chatId: 'chat-1', + messages: [ + { id: 'm1', role: 'user', parts: [{ type: 'text', text: 'hi' }] }, + ], + } as never, + res: makeRes() as never, + signal: new AbortController().signal, + model: {} as never, + role: null, + // No runHooks -> legacy path (socket signal), degeneration guard active. + }); + + expect(streamTextMock).toHaveBeenCalledTimes(1); + const onChunk = captured!.onChunk!; + const onStepFinish = captured!.onStepFinish!; + const abortSignal = captured!.abortSignal!; + expect(abortSignal.aborted).toBe(false); + + // STEP 1: a LONG, non-degenerate first step. Distinct lines never trip the + // detector, but they advance the throttle watermark far past the burst size + // that follows (to ~5x the step). This is the stale watermark that, WITHOUT + // the reset, would silence step 2. + let counter = 0; + let accumulated = 0; + while (accumulated < DEGENERATION_CHECK_STEP * 5) { + const line = `unique clean line number ${counter++} with distinct words\n`; + accumulated += line.length; + onChunk({ chunk: { type: 'text-delta', text: line } }); + } + expect(abortSignal.aborted).toBe(false); // clean step must not abort + + // STEP BOUNDARY: the real onStepFinish resets inProgressText AND (the fix) + // zeroes lastDegenerationCheckLen. + onStepFinish({ text: 'a clean first step', toolCalls: [], toolResults: [] }); + + // STEP 2: a FRESH, short degenerate burst (~3.3KB). Its length is far below + // the step-1 stale watermark (~10KB), so WITHOUT the reset the throttle stays + // silent and this streams unchecked. WITH the reset (watermark 0) it re-arms, + // the detector fires, and the run aborts. + const burst = 'loadTools.\n'.repeat(300); + expect(burst.length).toBeGreaterThanOrEqual(DEGENERATION_CHECK_STEP); + expect(burst.length).toBeLessThan(DEGENERATION_CHECK_STEP * 5); + onChunk({ chunk: { type: 'text-delta', text: burst } }); + + // The decisive assertion: the composed abortSignal (unioned with the + // degeneration controller) is now aborted. Reverting `lastDegenerationCheckLen + // = 0` in onStepFinish makes this stay false. + expect(abortSignal.aborted).toBe(true); + }); +}); diff --git a/apps/server/src/integrations/metrics/metrics.server.spec.ts b/apps/server/src/integrations/metrics/metrics.server.spec.ts index 1589db31..116961ea 100644 --- a/apps/server/src/integrations/metrics/metrics.server.spec.ts +++ b/apps/server/src/integrations/metrics/metrics.server.spec.ts @@ -128,10 +128,18 @@ describe('metrics server bind + auth (#486)', () => { const noAuth = await req(port); expect(noAuth.status).toBe(401); - // Wrong token -> 401. + // Wrong token, DIFFERENT length -> 401 (short-circuits on the length guard). const wrong = await req(port, { authorization: 'Bearer nope' }); expect(wrong.status).toBe(401); + // Wrong token, SAME length -> 401. This drives the timingSafeEqual compare + // itself (the length guard passes: 'Bearer topsecreX' has the same length as + // 'Bearer topsecret'). Pins the constant-time compare: a regression that made + // it return true would let this equal-length wrong token through — the + // different-length case above would NOT catch that. + const sameLen = await req(port, { authorization: 'Bearer topsecreX' }); + expect(sameLen.status).toBe(401); + // Correct token -> 200 with the metrics body. const ok = await req(port, { authorization: 'Bearer topsecret' }); expect(ok.status).toBe(200); From fdc37de3e8c0103d23f5a684794e29a2e4c98a46 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:18:43 +0300 Subject: [PATCH 16/41] =?UTF-8?q?fix(ai-chat):=20=D1=80=D0=B5=D0=B2=D1=8C?= =?UTF-8?q?=D1=8E-=D1=80=D0=B0=D1=83=D0=BD=D0=B4=20#500=20=E2=80=94=20?= =?UTF-8?q?=D0=BA=D1=80=D0=B0=D1=81=D0=BD=D1=8B=D0=B9=20=D1=81=D0=B5=D1=80?= =?UTF-8?q?=D0=B2=D0=B5=D1=80=D0=BD=D1=8B=D0=B9=20=D1=81=D1=8C=D1=8E=D1=82?= =?UTF-8?q?=20+=20=D1=80=D0=B5=D0=B3=D1=80=D0=B5=D1=81=D1=81=D0=B8=D1=8F?= =?UTF-8?q?=20Redis-health?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit B1: переименование строки ошибки (#394) не обновило 4 предсуществующих share-ассерта → полный серверный сьют был красный. Обновлены ассерты в public-share-chat.spec.ts и public-share-chat-tools.service.spec.ts под новую классифицированную строку. B2: /health первая проба после старта врала DOWN при живом Redis (lazyConnect + enableOfflineQueue:false + maxRetriesPerRequest:1 → первый ping до открытия сокета). Добавлен ensureConnected() с bounded-таймаутом перед первым ping; покрывает и путь пересоздания после onModuleDestroy. Тест UP-с-первой-пробы против реального ioredis (mutation-verified). Устранён open-handle leak в redis.health.spec (drain ioredis force-destroy-таймера в teardown; без forceExit) и окно ложного DOWN при конкурентных пробах (мемоизированный connectingPromise). B3: комментарий про orphan-чат при провале beginRun (insert до begin). B4: описание listPages упоминает поле truncated в tree-режиме. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/core/ai-chat/ai-chat.service.ts | 7 + .../core/ai-chat/public-share-chat.spec.ts | 6 +- .../public-share-chat-tools.service.spec.ts | 2 +- .../integrations/health/redis.health.spec.ts | 138 ++++++++++++++++-- .../src/integrations/health/redis.health.ts | 95 ++++++++++++ packages/mcp/src/tool-specs.ts | 7 +- 6 files changed, 237 insertions(+), 18 deletions(-) diff --git a/apps/server/src/core/ai-chat/ai-chat.service.ts b/apps/server/src/core/ai-chat/ai-chat.service.ts index dfa1bb75..14826bd6 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.ts @@ -758,6 +758,13 @@ export class AiChatService implements OnModuleInit { // or violate the page_id FK on insert (this runs after res.hijack(), so a // DB error would break the stream). const originPageId: string | null = openPageContext?.id ?? null; + // ORPHAN-ON-BEGIN-FAILURE tradeoff (#486, B3): the chat row is inserted + // HERE, before runHooks.begin below. If begin fails (e.g. a 503 / run-slot + // rejection) the turn aborts before the client is told this new chatId, so + // an empty chat is left behind and a retry mints ANOTHER one. We accept this + // over reordering: begin needs a chatId to bind the run to, and inserting + // the chat first keeps the id stable + the FK/history-join invariants above + // intact. Orphan empty chats are cheap and swept by normal chat cleanup. const chat = await this.aiChatRepo.insert({ creatorId: user.id, workspaceId: workspace.id, diff --git a/apps/server/src/core/ai-chat/public-share-chat.spec.ts b/apps/server/src/core/ai-chat/public-share-chat.spec.ts index f65058d9..40e4a7e6 100644 --- a/apps/server/src/core/ai-chat/public-share-chat.spec.ts +++ b/apps/server/src/core/ai-chat/public-share-chat.spec.ts @@ -808,7 +808,7 @@ describe('PublicShareChatToolsService share scoping', () => { }; await expect(getSharePage.execute({ pageId: 'p-outside' })).rejects.toThrow( - /not part of this published share/i, + /not available in this share/i, ); // The tool delegated the resolve to the canonical boundary with the // forShare-scoped shareId, and returned NO content for a non-resolving page. @@ -841,7 +841,7 @@ describe('PublicShareChatToolsService share scoping', () => { await expect( getSharePage.execute({ pageId: 'p-restricted' }), - ).rejects.toThrow(/not part of this published share/i); + ).rejects.toThrow(/not available in this share/i); // No content was ever sanitized/returned for the blocked page. expect(shareService.updatePublicAttachments).not.toHaveBeenCalled(); }); @@ -1003,7 +1003,7 @@ describe('public-share assistant boundary locks (red-team regression guards)', ( }; await expect( getSharePage.execute({ pageId: 'p-elsewhere' }), - ).rejects.toThrow(/not part of this published share/i); + ).rejects.toThrow(/not available in this share/i); // The forged share id is the scope the boundary re-derivation rejects against. expect(shareService.resolveReadableSharePage).toHaveBeenCalledWith( 'FORGED-SHARE', diff --git a/apps/server/src/core/ai-chat/tools/public-share-chat-tools.service.spec.ts b/apps/server/src/core/ai-chat/tools/public-share-chat-tools.service.spec.ts index bc80acd6..eadcd3a6 100644 --- a/apps/server/src/core/ai-chat/tools/public-share-chat-tools.service.spec.ts +++ b/apps/server/src/core/ai-chat/tools/public-share-chat-tools.service.spec.ts @@ -224,7 +224,7 @@ describe('PublicShareChatToolsService.forShare', () => { (tools.getSharePage as unknown as ToolExec).execute({ pageId: 'page-1', }), - ).rejects.toThrow('That page is not part of this published share.'); + ).rejects.toThrow('The requested page is not available in this share.'); // No content is ever fetched/returned for a non-resolving page. expect(shareService.updatePublicAttachments).not.toHaveBeenCalled(); diff --git a/apps/server/src/integrations/health/redis.health.spec.ts b/apps/server/src/integrations/health/redis.health.spec.ts index 1fd59c26..daed278c 100644 --- a/apps/server/src/integrations/health/redis.health.spec.ts +++ b/apps/server/src/integrations/health/redis.health.spec.ts @@ -16,7 +16,72 @@ import type { EnvironmentService } from '../environment/environment.service'; * implementation (requireActual) — only the constructor is wrapped to COUNT the * real clients it creates, which is precisely the leaking resource. */ -const mockLiveClients: Array<{ status: string; disconnect: () => void }> = []; +import type { Redis } from 'ioredis'; + +const mockLiveClients: Redis[] = []; + +/** + * Fully tear a REAL ioredis client down so NO timer survives jest's 1s exit + * window (this suite must exit cleanly WITHOUT forceExit; see #382). + * + * `connector.disconnect()` arms a ~12s "force-destroy the stream" `setTimeout` + * that is cleared ONLY by the stream's 'close' event — but only when the + * connector still holds a stream. Two problem cases: + * - a LIVE/connecting socket: disconnect arms the timer and 'close' may lag + * past jest's window, so we destroy the socket to make 'close' fire NOW; + * - a client BETWEEN reconnect attempts to a dead port: the held socket is + * ALREADY destroyed (its 'close' fired long ago), so disconnect would arm a + * timer whose clearing 'close' can never come again. We drop that dead stream + * reference BEFORE disconnect so the doomed timer is never armed. + * `disconnect()` itself also clears ioredis' own reconnect backoff timer. + */ +type DrainableStream = { destroyed?: boolean; destroy?: () => void } | null; +type DrainableClient = { + removeAllListeners: (event: string) => void; + disconnect: () => void; + stream?: DrainableStream; + connector?: { stream?: DrainableStream }; +}; + +async function drainClient(client: Redis): Promise { + if (!client || client.status === 'end') return; + const c = client as unknown as DrainableClient; + c.removeAllListeners('error'); + + // Drop an already-dead held socket so disconnect() can't arm a timer whose + // clearing 'close' will never fire again. + if (c.connector?.stream && c.connector.stream.destroyed) { + c.connector.stream = null; + } + if (c.stream && c.stream.destroyed) { + c.stream = null; + } + + await new Promise((resolve) => { + let done = false; + const finish = () => { + if (done) return; + done = true; + resolve(); + }; + client.once('end', finish); + // reconnect=false (the default): stop the retry loop and close the socket. + client.disconnect(); + // Force any still-live socket closed NOW so the connector's stream-destroy + // timer clears inside jest's window instead of lagging behind a real 'close'. + if (c.stream && !c.stream.destroyed) { + c.stream.destroy?.(); + } + // Fallback for a client with no live stream to emit 'end' (unref'd so it + // can never itself hold the loop open). + const fallback = setTimeout(finish, 500); + (fallback as { unref?: () => void }).unref?.(); + }); +} + +async function drainAll(): Promise { + await Promise.all(mockLiveClients.map((c) => drainClient(c))); +} jest.mock('ioredis', () => { const actual = jest.requireActual('ioredis'); @@ -53,17 +118,12 @@ describe('RedisHealthIndicator handle leak (#486)', () => { indicator = new RedisHealthIndicator(indicatorService, environmentService); }); - afterEach(() => { + afterEach(async () => { + // Drain (destroy socket + AWAIT 'end') every client the test created FIRST, + // so each is fully 'end' before onModuleDestroy's disconnect runs — that way + // no ioredis reconnect / stream-destroy timer outlives jest's exit window. + await drainAll(); indicator.onModuleDestroy(); - // Belt-and-braces: tear down anything the test created so ioredis reconnect - // timers do not keep the jest worker alive. - for (const c of mockLiveClients) { - try { - c.disconnect(); - } catch { - /* already gone */ - } - } }); it('creates exactly ONE Redis client across many probes while Redis is DOWN', async () => { @@ -93,3 +153,59 @@ describe('RedisHealthIndicator handle leak (#486)', () => { expect(mockLiveClients).toHaveLength(2); }); }); + +/** + * Happy-path regression guard (#486, B2): the FIRST probe against a LIVE Redis + * must report UP. + * + * With `lazyConnect: true` + `enableOfflineQueue: false`, a freshly-built client + * is in the `wait` state and the socket opens lazily. If the very first `ping()` + * is issued before an explicit `connect()`, ioredis rejects it instantly with + * "Stream isn't writeable and enableOfflineQueue options is false" — a FALSE + * DOWN even though Redis is alive. The fix opens the socket before the first + * ping. This exercises a REAL ioredis client against a REAL TCP redis server + * (not a mock), so a regression genuinely reddens it. + */ +describe('RedisHealthIndicator live Redis first-probe (#486, B2)', () => { + const indicatorService = { + check: (key: string) => ({ + up: () => ({ [key]: { status: 'up' } }), + down: (message: string) => ({ [key]: { status: 'down', message } }), + }), + } as unknown as HealthIndicatorService; + + // A REAL running redis (see the neighboring harness / CI env). + const environmentService = { + getRedisUrl: () => 'redis://127.0.0.1:6379/0', + } as unknown as EnvironmentService; + + let indicator: RedisHealthIndicator; + + beforeEach(() => { + mockLiveClients.length = 0; + indicator = new RedisHealthIndicator(indicatorService, environmentService); + }); + + afterEach(async () => { + // Await full socket close of every live client (see drainClient) BEFORE + // onModuleDestroy: a real, connected ioredis client MUST be drained to 'end' + // or its stream-destroy timer keeps the jest worker alive past the 1s window. + await drainAll(); + indicator.onModuleDestroy(); + }); + + it('reports UP on the FIRST probe against a live Redis', async () => { + // The VERY FIRST probe — no warm-up ping — must be UP. + const result = await indicator.pingCheck('redis'); + expect(result.redis.status).toBe('up'); + }); + + it('stays UP on a probe AFTER onModuleDestroy re-creates the client', async () => { + await indicator.pingCheck('redis'); + indicator.onModuleDestroy(); + // The re-created client is again in `wait`; the first ping on it must still + // open the socket (the false-DOWN also recurs on the post-destroy path). + const result = await indicator.pingCheck('redis'); + expect(result.redis.status).toBe('up'); + }); +}); diff --git a/apps/server/src/integrations/health/redis.health.ts b/apps/server/src/integrations/health/redis.health.ts index c9160125..324d5a64 100644 --- a/apps/server/src/integrations/health/redis.health.ts +++ b/apps/server/src/integrations/health/redis.health.ts @@ -20,6 +20,26 @@ export class RedisHealthIndicator implements OnModuleDestroy { */ private probeClient: Redis | null = null; + /** + * How long the first-ping `connect()` may take before a probe gives up and + * reports DOWN. A `connect()` against a truly-down Redis never settles on its + * own (ioredis retries the socket indefinitely per its retryStrategy), so the + * probe MUST bound it or the /health handler would hang. Kept short so a real + * outage is reported fast; localhost/live Redis connects well within it. + */ + private static readonly CONNECT_TIMEOUT_MS = 2000; + + /** + * The single in-flight first-`connect()`, memoized so CONCURRENT probes share + * it. k8s liveness+readiness hit /health in parallel on startup: without this, + * probe A drives `connect()` (the client leaves the `wait` state) and probe B, + * seeing a not-`wait`/not-`ready` client, would skip connect and fire `ping()` + * at a still-opening socket → an instant FALSE DOWN. With the memo, B awaits + * the SAME connect. Cleared once it settles so a later disconnect / re-create + * starts a fresh connect. + */ + private connectingPromise: Promise | null = null; + constructor( private readonly healthIndicatorService: HealthIndicatorService, private environmentService: EnvironmentService, @@ -52,11 +72,82 @@ export class RedisHealthIndicator implements OnModuleDestroy { return this.probeClient; } + /** + * Open the probe socket BEFORE the first ping. `lazyConnect: true` leaves a + * freshly-built (or post-destroy re-built) client in the `wait` state: the + * socket is NOT open yet, so with `enableOfflineQueue: false` the very first + * `ping()` rejects instantly with "Stream isn't writeable and + * enableOfflineQueue options is false" even when Redis is perfectly alive — a + * false DOWN on the happy path. We drive `connect()` ONLY from `wait`; once + * the client is connected, ioredis owns its own (re)connect loop and a ping + * issued while it reconnects still fast-fails to a correct DOWN (offline queue + * stays off). A failed/timed-out connect rejects → reported DOWN, which is the + * right signal for a truly-down Redis. + */ + private ensureConnected(client: Redis): Promise { + // Already open — steady state, nothing to do. + if (client.status === 'ready') return Promise.resolve(); + // A first-connect is already in flight (possibly started by a CONCURRENT + // probe): await the SAME one instead of racing a second connect() (ioredis + // throws "already connecting") or firing ping() at a not-yet-open socket. + if (this.connectingPromise) return this.connectingPromise; + // Only DRIVE connect() from the initial `wait` state (fresh / post-destroy + // re-created client). In any other non-ready state ioredis already owns its + // (re)connect loop; a ping there fast-fails to a correct DOWN, so we must not + // start a competing connect. + if (client.status !== 'wait') return Promise.resolve(); + + const promise = this.connectWithTimeout(client).finally(() => { + // Clear only if still ours, so a later disconnect / re-create can connect + // again. Whether it resolved or rejected, the memo has served its window. + if (this.connectingPromise === promise) { + this.connectingPromise = null; + } + }); + this.connectingPromise = promise; + return promise; + } + + private connectWithTimeout(client: Redis): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const timer = setTimeout(() => { + if (settled) return; + settled = true; + reject(new Error('Redis probe connect timed out')); + }, RedisHealthIndicator.CONNECT_TIMEOUT_MS); + // Never let THIS timer alone keep the event loop (or a jest worker) alive; + // it is cleared on settle anyway, this is belt-and-braces. + timer.unref?.(); + // `.catch` is always attached, so a connect() that rejects AFTER we have + // already timed out is handled here (guarded by `settled`) and never + // surfaces as an unhandled rejection. + client + .connect() + .then(() => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(); + }) + .catch((err) => { + if (settled) return; + settled = true; + clearTimeout(timer); + reject(err); + }); + }); + } + async pingCheck(key: string): Promise { const indicator = this.healthIndicatorService.check(key); try { const redis = this.getProbeClient(); + // Open the socket before the first ping (see ensureConnected); without + // this the first probe after (re)creation falsely reports DOWN on a live + // Redis because lazyConnect defers the connect past the first ping. + await this.ensureConnected(redis); await redis.ping(); return indicator.up(); } catch (e) { @@ -75,5 +166,9 @@ export class RedisHealthIndicator implements OnModuleDestroy { this.probeClient.disconnect(); this.probeClient = null; } + // Drop any in-flight first-connect memo so the NEXT client (lazily rebuilt on + // the next probe) starts a fresh connect rather than awaiting a promise tied + // to the client we just tore down. + this.connectingPromise = null; } } diff --git a/packages/mcp/src/tool-specs.ts b/packages/mcp/src/tool-specs.ts index 44534ed1..941eead9 100644 --- a/packages/mcp/src/tool-specs.ts +++ b/packages/mcp/src/tool-specs.ts @@ -923,9 +923,10 @@ export const SHARED_TOOL_SPECS = { 'List the most recent pages (ordered by updatedAt, descending), ' + 'optionally scoped to a single space. Returns a bounded list (default ' + '50, max 100) — use search for lookups in large spaces. tree:true (with ' + - "spaceId) returns the space's full page hierarchy as a nested tree, but " + - 'is DEPRECATED — use getTree instead (leaner nodes, plus rootPageId / ' + - 'maxDepth).', + "spaceId) returns { tree, truncated } — the space's full page hierarchy " + + 'as a nested tree, plus a `truncated` flag that is true when the tree was ' + + 'capped and is INCOMPLETE — but is DEPRECATED, use getTree instead ' + + '(leaner nodes, plus rootPageId / maxDepth).', tier: 'core', catalogLine: "listPages — list recent pages (tree:true is deprecated; use getTree for the hierarchy).", From 3b6634c6bead401a7852ac3cf36e3eeb1414f58b Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 04:23:31 +0300 Subject: [PATCH 17/41] =?UTF-8?q?fix(ai-chat):=20in-app=20=D1=82=D1=83?= =?UTF-8?q?=D0=BB=D1=8B=20=E2=80=94=20race-on-abort=20+=20safe-points=20+?= =?UTF-8?q?=20per-call=20cap=20(#487)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In-app тул-обёртки отбрасывали второй аргумент options с abortSignal — это был единственный класс тулов без отмены и без wall-clock cap. Контент-мутации идут через collab-WS (mutatePageContent), не через axios, поэтому «прокинуть signal в axios» главный write-путь не покрывал. - Переиспользован race-паттерн wrapToolWithCallTimeout: каждый in-app тул гонится против композитного сигнала (Stop + per-call cap); на abort — реджект немедленно, проигравший промис отбрасывается (латентность мс, не сетевой teardown — от этого зависит таймаут supersede в коммите 3). - Safe-point проверки сигнала между последовательными вызовами paginateAll и пре-коммитная проверка в mutatePageContent (+ реентрантный близнец) через DocmostClient.setToolAbortSignal, который обёртка публикует перед каждым вызовом. - Per-call cap покрывает весь вызов, env-tunable (AI_CHAT_INAPP_TOOL_CALL_CAP_MS, дефолт 2 мин). - Задокументировано ограничение (#487): abort между вызовами многовызовного write-тула оставляет частично применённую операцию — отмена гарантирует «новый вызов не стартует», не «запись не доехала». Тесты (честное свойство «после Stop не стартует новый HTTP/WS-вызов»): юнит wrapInAppToolWithCap (реджект-на-abort, отсутствие новых вызовов, per-call cap, публикация сигнала) + mock-HTTP тест реального paginateAll safe-point. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/tools/ai-chat-tools.cap.spec.ts | 160 ++++++++++++++++++ .../ai-chat/tools/ai-chat-tools.service.ts | 148 +++++++++++++++- packages/mcp/src/client/context.ts | 62 ++++++- packages/mcp/src/lib/collaboration.ts | 16 ++ .../mock/paginate-abort-safepoint.test.mjs | 143 ++++++++++++++++ 5 files changed, 523 insertions(+), 6 deletions(-) create mode 100644 apps/server/src/core/ai-chat/tools/ai-chat-tools.cap.spec.ts create mode 100644 packages/mcp/test/mock/paginate-abort-safepoint.test.mjs diff --git a/apps/server/src/core/ai-chat/tools/ai-chat-tools.cap.spec.ts b/apps/server/src/core/ai-chat/tools/ai-chat-tools.cap.spec.ts new file mode 100644 index 00000000..7f504f48 --- /dev/null +++ b/apps/server/src/core/ai-chat/tools/ai-chat-tools.cap.spec.ts @@ -0,0 +1,160 @@ +import { + wrapInAppToolWithCap, + inAppToolCallCapMs, + type ToolAbortSignalSink, +} from './ai-chat-tools.service'; +import type { Tool, ToolCallOptions } from 'ai'; + +/** + * #487 commit 1 — in-app tool race-on-abort + safe-points + per-call cap. + * + * Tests assert the HONEST observable property the spec names — "after Stop, NO + * new HTTP/WS call STARTS; an already-started single call may take either + * outcome" — against the REAL wrapper mechanism (the composite abort signal it + * publishes on the client + the RACE it runs), NOT a timing-dependent proxy like + * "the write didn't land". + */ + +// A minimal stand-in for the client's `toolAbortSignal` field. In production the +// wrapper publishes the composite here and the client's paginateAll / +// mutatePageContent safe-points read it; the fake "tool" below reads it the same +// way, so this exercises the real contract without a live DB / collab socket. +class FakeClient implements ToolAbortSignalSink { + private signal: AbortSignal | null = null; + setToolAbortSignal(signal: AbortSignal | null): void { + this.signal = signal; + } + getToolAbortSignal(): AbortSignal | null { + return this.signal; + } +} + +// A ToolCallOptions with just the field the wrapper reads (abortSignal). The AI +// SDK passes a fuller object; the wrapper only spreads it and reads abortSignal. +const opts = (abortSignal?: AbortSignal): ToolCallOptions => + ({ toolCallId: 't1', messages: [], abortSignal }) as unknown as ToolCallOptions; + +const tick = (ms = 5) => new Promise((r) => setTimeout(r, ms)); + +describe('#487 wrapInAppToolWithCap — race-on-abort + safe-points', () => { + it('after Stop, no NEW simulated call starts (multi-call tool)', async () => { + const client = new FakeClient(); + const started: number[] = []; + // A multi-call tool that mirrors paginateAll: it consults the client signal + // at a safe-point BEFORE starting each simulated network call. + const multiCall: Tool = { + execute: (async (_args: unknown) => { + for (let i = 0; i < 6; i++) { + // Safe-point: exactly what paginateAll / mutatePageContent do. + client.getToolAbortSignal()?.throwIfAborted(); + started.push(i); + await tick(10); + } + return 'done'; + }) as unknown as Tool['execute'], + } as Tool; + + const wrapped = wrapInAppToolWithCap(multiCall, client, 10_000); + const ac = new AbortController(); + const call = ( + wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise + )({}, opts(ac.signal)); + + // Let one or two calls start, then Stop. + await tick(12); + ac.abort(new Error('user stop')); + + await expect(call).rejects.toThrow(); // wrapper rejects promptly + const startedAtStop = started.length; + + // Give the abandoned loser ample time; its next safe-point must throw because + // the (aborted) composite is still published on the client. + await tick(60); + expect(started.length).toBe(startedAtStop); + // It must NOT have run the whole sequence (that would mean Stop did nothing). + expect(started.length).toBeLessThan(6); + }); + + it('rejects immediately on Stop even if the call never settles (discard loser)', async () => { + const client = new FakeClient(); + let settled = false; + const hang: Tool = { + execute: (async () => { + await new Promise(() => undefined); // never resolves + settled = true; + }) as unknown as Tool['execute'], + } as Tool; + const wrapped = wrapInAppToolWithCap(hang, client, 10_000); + const ac = new AbortController(); + const call = ( + wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise + )({}, opts(ac.signal)); + await tick(5); + ac.abort(); + await expect(call).rejects.toThrow(); + expect(settled).toBe(false); + }); + + it('per-call cap rejects a hung call with no Stop signal', async () => { + const client = new FakeClient(); + const hang: Tool = { + execute: (async () => { + await new Promise(() => undefined); + }) as unknown as Tool['execute'], + } as Tool; + // Tiny cap; no options.abortSignal at all (composite = cap only). + const wrapped = wrapInAppToolWithCap(hang, client, 20); + const start = Date.now(); + await expect( + (wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise)( + {}, + opts(undefined), + ), + ).rejects.toThrow(/per-call cap/); + expect(Date.now() - start).toBeLessThan(2000); + }); + + it('publishes a composite signal on the client for the duration of the call', async () => { + const client = new FakeClient(); + let seenDuringCall: AbortSignal | null = null; + const probe: Tool = { + execute: (async () => { + seenDuringCall = client.getToolAbortSignal(); + return 'ok'; + }) as unknown as Tool['execute'], + } as Tool; + const wrapped = wrapInAppToolWithCap(probe, client, 10_000); + const ac = new AbortController(); + await ( + wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise + )({}, opts(ac.signal)); + expect(seenDuringCall).not.toBeNull(); + // The published composite must reflect the turn's Stop signal. + ac.abort(); + expect((seenDuringCall as unknown as AbortSignal).aborted).toBe(true); + }); + + it('a completed call returns its raw result unchanged', async () => { + const client = new FakeClient(); + const ok: Tool = { + execute: (async () => ({ items: [1, 2, 3] })) as unknown as Tool['execute'], + } as Tool; + const wrapped = wrapInAppToolWithCap(ok, client, 10_000); + const res = await ( + wrapped.execute as (a: unknown, o: ToolCallOptions) => Promise + )({}, opts(new AbortController().signal)); + expect(res).toEqual({ items: [1, 2, 3] }); + }); + + it('cap is env-tunable with a 2-minute default', () => { + const prev = process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS; + delete process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS; + expect(inAppToolCallCapMs()).toBe(120_000); + process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS = '5000'; + expect(inAppToolCallCapMs()).toBe(5000); + process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS = 'not-a-number'; + expect(inAppToolCallCapMs()).toBe(120_000); + if (prev === undefined) delete process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS; + else process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS = prev; + }); +}); diff --git a/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts b/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts index dc2270fd..58561ea2 100644 --- a/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts +++ b/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts @@ -1,5 +1,5 @@ import { Injectable, Logger } from '@nestjs/common'; -import { tool, type Tool } from 'ai'; +import { tool, type Tool, type ToolCallOptions } from 'ai'; import { z } from 'zod'; import { User } from '@docmost/db/types/entity.types'; import { TokenService } from '../../auth/services/token.service'; @@ -159,6 +159,129 @@ function __assertClientCallContract(client: DocmostClientLike): void { * existing service-account `/mcp` path already calls loopback successfully, so * this works for single-workspace self-host. */ +/** + * #487: wall-clock cap for a SINGLE in-app tool call, env-tunable via + * `AI_CHAT_INAPP_TOOL_CALL_CAP_MS`. Bounds a read tool that would otherwise + * paginate for minutes and a content write whose collab commit hangs, and is the + * per-call CAP half of the composite abort signal every in-app tool is wrapped + * with (the other half is the turn's Stop signal). Default 2 minutes: generous + * for a legitimate long read/write, tight enough that a stuck call cannot pin the + * turn. The reconcile staleness floor (#487 commit 4) is derived as + * max(2 x this cap, 15 min), so keep this well under that. + */ +export function inAppToolCallCapMs(): number { + const raw = Number(process.env.AI_CHAT_INAPP_TOOL_CALL_CAP_MS); + return Number.isFinite(raw) && raw > 0 ? raw : 120_000; +} + +/** #487: the composite signal's reason as an Error (informative thrown value). */ +function inAppAbortReason(signal: AbortSignal): Error { + const r = signal.reason; + return r instanceof Error + ? r + : new Error(typeof r === 'string' ? r : 'In-app tool call aborted'); +} + +/** + * The client surface {@link wrapInAppToolWithCap} drives (#487). Both methods are + * OPTIONAL: the real loopback DocmostClient implements them (so a Stop/cap reaches + * its pagination / pre-commit safe-points), but a client that omits them still + * gets the OUTER guarantee — the race rejects on abort regardless. This keeps the + * wrapper decoupled from the exact client shape (unit-test doubles need not stub + * the plumbing). + */ +export interface ToolAbortSignalSink { + setToolAbortSignal?(signal: AbortSignal | null): void; + getToolAbortSignal?(): AbortSignal | null; +} + +/** + * #487: wrap an in-app tool so a Stop (the turn's `options.abortSignal`) OR the + * per-call wall-clock cap REJECTS the call immediately, and so that SAME + * composite signal reaches the client's pagination / pre-commit safe-points (via + * `client.setToolAbortSignal`) — making a Stop stop the NEXT HTTP/WS call from + * starting. + * + * Reuses the RACE pattern of `wrapToolWithCallTimeout` (mcp-clients.service.ts): + * the call is raced against the composite signal, so on abort we reject in the + * SAME tick and DISCARD the loser promise. Its network / collab teardown latency + * therefore never blocks the turn — the supersede timeout W=10s (#487 commit 3) + * relies on this abort->settle latency being milliseconds, not a socket teardown. + * Awaiting the client's own signal-into-write path alone would NOT satisfy this + * (the loser could still be tearing down a collab socket). + * + * The composite is SET on the client at entry and deliberately NOT restored on + * unwind: after this wrapper rejects on abort, the ABANDONED loser promise keeps + * running, and its safe-points read the client field — leaving the (aborted) + * composite there is exactly what makes the loser's NEXT call throw and stop. The + * next in-app tool call overwrites the field with its own fresh composite before + * any of its safe-points run, so a stale settled signal never leaks forward. + * SINGLE-WRITER by phase-1 assumption — see DocmostClientContext.toolAbortSignal + * for the parallel-call caveat (#487). + * + * KNOWN LIMITATION (#487): a write tool that issues SEVERAL sequential collab + * commits can be aborted BETWEEN commits, leaving a partially-applied operation. + * Cancel guarantees "no NEW call starts", NOT "the write didn't land". + */ +export function wrapInAppToolWithCap( + toolDef: Tool, + client: ToolAbortSignalSink, + capMs: number, +): Tool { + const original = toolDef.execute; + if (typeof original !== 'function') return toolDef; + const execute = async (args: unknown, options: ToolCallOptions) => { + const capController = new AbortController(); + const timer = setTimeout(() => { + capController.abort( + new Error(`In-app tool call exceeded the ${capMs}ms per-call cap`), + ); + }, capMs); + timer.unref?.(); + const composite = options?.abortSignal + ? AbortSignal.any([options.abortSignal, capController.signal]) + : capController.signal; + // Reject the MOMENT the composite fires, independent of whether `original` + // ever settles (a hung collab write / read would otherwise pin the turn). The + // losing `original` is left pending; Promise.race attaches a rejection + // handler to both inputs so a late rejection is never unhandled. + const aborted = new Promise((_, reject) => { + const fail = () => reject(inAppAbortReason(composite)); + if (composite.aborted) fail(); + else composite.addEventListener('abort', fail, { once: true }); + }); + // Publish the composite so the client's pagination / pre-commit safe-points + // observe it (see the "not restored on unwind" rationale above). Guarded: a + // client without the plumbing still gets the OUTER race guarantee below. + client.setToolAbortSignal?.(composite); + try { + return await Promise.race([ + (original as (a: unknown, o: ToolCallOptions) => Promise)( + args, + { ...options, abortSignal: composite }, + ), + aborted, + ]); + } finally { + clearTimeout(timer); + } + }; + return { ...toolDef, execute } as unknown as Tool; +} + +/** #487: apply {@link wrapInAppToolWithCap} to every tool in a set. */ +export function wrapInAppToolsWithCap( + tools: Record, + client: ToolAbortSignalSink, + capMs: number, +): Record { + const out: Record = {}; + for (const [name, t] of Object.entries(tools)) { + out[name] = wrapInAppToolWithCap(t, client, capMs); + } + return out; +} + @Injectable() export class AiChatToolsService { private readonly logger = new Logger(AiChatToolsService.name); @@ -186,7 +309,12 @@ export class AiChatToolsService { sessionId: string, workspaceId: string, aiChatId: string, - ): Promise { + // #487: the returned client also carries the tool-cancellation plumbing + // (setToolAbortSignal/getToolAbortSignal). These are host plumbing, NOT part + // of the tool-execute surface (DocmostClientMethod), so they are surfaced here + // as an intersection rather than by widening that Pick — keeping the + // positional-call drift-guard (#446) scoped to the actual tool methods. + ): Promise { const apiUrl = process.env.MCP_DOCMOST_API_URL || `http://127.0.0.1:${process.env.PORT || 3000}/api`; @@ -630,7 +758,15 @@ export class AiChatToolsService { // dependency and reuses the CASL enforcement already on `client`. When the // loaded package predates #417 (factory undefined) or the loader is mocked in // a unit test, signalling is a pure no-op and results are byte-identical. - if (!createCommentSignalTracker) return tools; + // #487: wrap every in-app tool with the race-on-abort + per-call cap guard so + // a Stop / cap rejects immediately AND reaches the client's write/pagination + // safe-points. Applied as the OUTERMOST wrapper (over the comment-signal + // wrapper below) so the race governs the whole call. The client carries the + // per-call composite signal via setToolAbortSignal. + const capMs = inAppToolCallCapMs(); + if (!createCommentSignalTracker) { + return wrapInAppToolsWithCap(tools, client, capMs); + } const tracker = createCommentSignalTracker({ probe: async (pageId: string, sinceMs: number) => { @@ -659,7 +795,11 @@ export class AiChatToolsService { }, }); - return wrapToolsWithCommentSignal(tools, tracker); + return wrapInAppToolsWithCap( + wrapToolsWithCommentSignal(tools, tracker), + client, + capMs, + ); } } diff --git a/packages/mcp/src/client/context.ts b/packages/mcp/src/client/context.ts index 12a8b1a0..4cf996a1 100644 --- a/packages/mcp/src/client/context.ts +++ b/packages/mcp/src/client/context.ts @@ -170,6 +170,40 @@ export abstract class DocmostClientContext { // cached conversion can never leak across identities. See getpage-cache.ts. protected getPageCache = new GetPageConversionCache(); + // #487: an OPTIONAL abort signal the in-app tool host sets before each tool + // call (a composite of the turn's Stop signal + a per-call wall-clock cap). It + // is checked at safe-points BETWEEN the sequential HTTP calls of a paginated + // read (paginateAll) and just before the atomic collab commit of a write (the + // mutatePage/replacePage/mutateLiveContentUnlocked seams), so a Stop / cap + // stops the NEXT network call from STARTING. An already-started single call may + // still land — a documented limitation (#487). + // + // SINGLE-WRITER by phase-1 assumption: exactly one DocmostClient is built per + // turn and shared by every tool call; the host sets this per call and restores + // the prior value when the call unwinds. If the model emits PARALLEL in-app + // tool calls they share this one field, so the per-call CAP of one call is not + // guaranteed to bound another's in-flight pagination — but every composite the + // host sets carries the SAME turn Stop signal, so a Stop still aborts whichever + // signal is current. #487. + protected toolAbortSignal: AbortSignal | null = null; + + /** + * #487: set (or clear with null) the in-app tool abort signal governing the + * NEXT client call's safe-points. The host wraps each in-app tool call: it sets + * the composite (Stop + per-call cap) here before invoking the tool and + * restores the prior value afterwards. Public so the server-side tool wrapper + * can reach it; harmless (a no-op) when never set. + */ + public setToolAbortSignal(signal: AbortSignal | null): void { + this.toolAbortSignal = signal; + } + + /** #487: the abort signal currently governing this client's safe-points. */ + public getToolAbortSignal(): AbortSignal | null { + return this.toolAbortSignal; + } +>>>>>>> 917c4064 (fix(ai-chat): in-app тулы — race-on-abort + safe-points + per-call cap (#487)) + // Two construction forms: // - new DocmostClient(config) // discriminated union (current) // - new DocmostClient(baseURL, email, password) // legacy positional creds @@ -571,6 +605,10 @@ export abstract class DocmostClientContext { this.onMetricFn?.("collab_connect_timeouts_total", 1), }); try { + // #487 PRE-COMMIT safe-point (reentrant twin of mutatePageContent): a + // Stop/cap after acquiring the session but before the atomic write skips + // this commit. Same limitation applies (stops the NEXT commit only). + this.toolAbortSignal?.throwIfAborted(); return await session.mutate(transform); } catch (e) { // Drop the session on any failure so the next call reconnects fresh. @@ -602,6 +640,11 @@ export abstract class DocmostClientContext { let truncated = false; for (let page = 0; page < MAX_PAGES; page++) { + // #487 safe-point: a Stop (or the in-app tool per-call cap) that fires + // BETWEEN sequential page fetches must stop the NEXT request from starting + // — a read tool that would otherwise paginate for minutes is interrupted + // here. throwIfAborted() rejects with the signal's reason. + this.toolAbortSignal?.throwIfAborted(); const payload: Record = { ...basePayload, limit: clampedLimit, @@ -709,7 +752,15 @@ export abstract class DocmostClientContext { // #486: on a rejected collab-WS handshake, invalidate + refresh the token and // retry the write once (symmetric to the HTTP-401 reauth path). return this.writeWithCollabAuthRetry(collabToken, (token) => - mutatePageContent(pageUuid, token, apiUrl, transform), + // #487: thread the in-app tool signal to mutatePageContent's pre-commit + // safe-point so a Stop/cap during the connect/lock window skips the write. + mutatePageContent( + pageUuid, + token, + apiUrl, + transform, + this.toolAbortSignal ?? undefined, + ), ); } @@ -733,7 +784,14 @@ export abstract class DocmostClientContext { // #486: on a rejected collab-WS handshake, invalidate + refresh the token and // retry the write once (symmetric to the HTTP-401 reauth path). return this.writeWithCollabAuthRetry(collabToken, (token) => - replacePageContent(pageUuid, doc, token, apiUrl), + // #487: same pre-commit safe-point as mutatePage, for full-document writes. + replacePageContent( + pageUuid, + doc, + token, + apiUrl, + this.toolAbortSignal ?? undefined, + ), ); } diff --git a/packages/mcp/src/lib/collaboration.ts b/packages/mcp/src/lib/collaboration.ts index c7a77c5d..66d1d9ce 100644 --- a/packages/mcp/src/lib/collaboration.ts +++ b/packages/mcp/src/lib/collaboration.ts @@ -254,6 +254,12 @@ export async function mutatePageContent( collabToken: string, baseUrl: string, transform: (liveDoc: any) => any | null, + // #487: optional abort signal carrying the turn's Stop + the in-app tool + // per-call cap. Checked as the PRE-COMMIT safe-point below (after the session + // is acquired, immediately before the atomic read->write), so a Stop that + // arrives during the connect/lock window stops THIS write from landing. See the + // limitation note at the check. + signal?: AbortSignal, ): Promise { return withPageLock(pageId, async () => { if (process.env.DEBUG) { @@ -266,6 +272,13 @@ export async function mutatePageContent( const session = await acquireCollabSession(pageId, collabToken, baseUrl); try { + // #487 PRE-COMMIT safe-point: if the turn was Stopped (or the in-app tool + // per-call cap fired) after we acquired the collab session but before the + // atomic write, throw NOW so this commit never runs. KNOWN LIMITATION + // (#487): this only stops THIS commit — a write tool that already committed + // an EARLIER call this turn leaves that op applied. Cancel guarantees "no + // NEW commit starts", NOT "the write didn't land". + signal?.throwIfAborted(); return await session.mutate(transform); } catch (e) { // Drop the session on any failure so the next call reconnects fresh (this @@ -291,6 +304,8 @@ export async function replacePageContent( prosemirrorDoc: any, collabToken: string, baseUrl: string, + // #487: threaded straight to mutatePageContent's pre-commit safe-point. + signal?: AbortSignal, ): Promise { // Fail fast on a bad document instead of deferring the failure into the // collaboration write (where TiptapTransformer.toYdoc(undefined) used to @@ -307,6 +322,7 @@ export async function replacePageContent( collabToken, baseUrl, () => prosemirrorDoc, + signal, ); } diff --git a/packages/mcp/test/mock/paginate-abort-safepoint.test.mjs b/packages/mcp/test/mock/paginate-abort-safepoint.test.mjs new file mode 100644 index 00000000..af4c4e16 --- /dev/null +++ b/packages/mcp/test/mock/paginate-abort-safepoint.test.mjs @@ -0,0 +1,143 @@ +// #487 commit 1 — the in-app tool cancellation safe-point inside paginateAll. +// +// The in-app tool host sets a composite abort signal on the client +// (setToolAbortSignal) before each tool call; paginateAll checks it at a +// safe-point BEFORE every sequential page fetch, so a Stop that lands mid-read +// stops the NEXT HTTP request from STARTING (a read tool can no longer paginate +// for minutes past a Stop). This pins the HONEST observable property against the +// REAL client + a real HTTP server: "after Stop, no NEW request starts". +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { DocmostClient } from "../../build/client.js"; + +function readBody(req) { + return new Promise((resolve) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => resolve(raw)); + }); +} +function sendJson(res, status, obj, extra = {}) { + res.writeHead(status, { "Content-Type": "application/json", ...extra }); + res.end(JSON.stringify(obj)); +} +const openServers = []; +async function spawn(handler) { + const server = await new Promise((resolve) => { + const s = http.createServer(handler); + s.listen(0, "127.0.0.1", () => resolve(s)); + }); + openServers.push(server); + const { port } = server.address(); + return { baseURL: `http://127.0.0.1:${port}/api` }; +} +after(async () => { + await Promise.all(openServers.map((s) => new Promise((r) => s.close(r)))); +}); +function handleLogin(req, res) { + if (req.url === "/api/auth/login") { + sendJson(res, 200, { success: true }, { + "Set-Cookie": "authToken=t; Path=/; HttpOnly", + }); + return true; + } + return false; +} + +// A Stop that lands DURING pagination: the server aborts the client signal as it +// serves page 1 (more pages remain). The loop's next safe-point must throw before +// the page-2 request is sent. +test("paginateAll stops the NEXT request when the signal aborts mid-pagination", async () => { + let requests = 0; + const ac = new AbortController(); + const { baseURL } = await spawn(async (req, res) => { + await readBody(req); + if (handleLogin(req, res)) return; + if (req.url === "/api/spaces") { + requests++; + // Simulate a user Stop that lands while page 1 is in flight. + if (requests === 1) ac.abort(new Error("user stop")); + sendJson(res, 200, { + success: true, + data: { + items: [{ id: `p${requests}` }], + meta: { hasNextPage: true, nextCursor: `c${requests}` }, + }, + }); + return; + } + sendJson(res, 404, {}); + }); + + const client = new DocmostClient(baseURL, "user@example.com", "pw"); + client.setToolAbortSignal(ac.signal); + + await assert.rejects( + () => client.paginateAll("/spaces", {}), + /user stop/, + "the aborted safe-point rejects with the signal's reason", + ); + assert.equal(requests, 1, "page 2 never started after the Stop"); +}); + +// A Stop that is already in effect before the read starts: zero requests fire. +test("paginateAll starts no request when the signal is already aborted", async () => { + let requests = 0; + const { baseURL } = await spawn(async (req, res) => { + await readBody(req); + if (handleLogin(req, res)) return; + if (req.url === "/api/spaces") { + requests++; + sendJson(res, 200, { + success: true, + data: { items: [], meta: { hasNextPage: false, nextCursor: null } }, + }); + return; + } + sendJson(res, 404, {}); + }); + + const client = new DocmostClient(baseURL, "user@example.com", "pw"); + // Warm the auth so ensureAuthenticated does not itself POST after the abort. + await client.ensureAuthenticated(); + const ac = new AbortController(); + ac.abort(new Error("already stopped")); + client.setToolAbortSignal(ac.signal); + + await assert.rejects(() => client.paginateAll("/spaces", {}), /already stopped/); + assert.equal(requests, 0, "no /spaces request started once already aborted"); +}); + +// Without a tool signal (default), pagination is unaffected — the safe-point is a +// pure no-op, so pre-#487 behaviour is byte-identical. +test("paginateAll is unaffected when no tool signal is set", async () => { + let requests = 0; + const PAGES = { + "": { items: [{ id: "a" }], nextCursor: "c1" }, + c1: { items: [{ id: "b" }], nextCursor: null }, + }; + const { baseURL } = await spawn(async (req, res) => { + const raw = await readBody(req); + if (handleLogin(req, res)) return; + if (req.url === "/api/spaces") { + requests++; + const body = JSON.parse(raw || "{}"); + const page = PAGES[body.cursor ?? ""] ?? { items: [], nextCursor: null }; + sendJson(res, 200, { + success: true, + data: { + items: page.items, + meta: { hasNextPage: page.nextCursor != null, nextCursor: page.nextCursor }, + }, + }); + return; + } + sendJson(res, 404, {}); + }); + + const client = new DocmostClient(baseURL, "user@example.com", "pw"); + const all = await client.paginateAll("/spaces", {}); + assert.equal(requests, 2, "both pages fetched with no signal set"); + assert.deepEqual(all.map((p) => p.id), ["a", "b"]); +}); From 1031ed1ae829c80718965a01e052c4010e1e80e6 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 04:33:39 +0300 Subject: [PATCH 18/41] =?UTF-8?q?feat(ai-chat):=20=D0=B7=D0=BE=D0=BC=D0=B1?= =?UTF-8?q?=D0=B8-=D0=BC=D0=B5=D1=85=D0=B0=D0=BD=D0=B8=D0=BA=D0=B0=20final?= =?UTF-8?q?izeRun=20+=20settledPromise=20+=20=D1=83=D1=81=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=D0=BD=D1=8B=D0=B5=20=D1=82=D0=B5=D1=80=D0=BC=D0=B8=D0=BD?= =?UTF-8?q?=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D0=B5=20UPDATE=20(#487)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Раньше give-up-путь finalizeRun ВОССТАНАВЛИВАЛ entry (зомби неотличим от живого рана), а терминальный runRepo.update был БЕЗУСЛОВНЫМ (последний писатель перетирал терминальный статус). - Все терминальные UPDATE ранов теперь УСЛОВНЫЕ: новый repo.finalizeIfActive пишет только пока строка pending|running (зеркально onlyIfStreaming у сообщений) → двойной settle схлопывается в benign no-op, терминальный статус не перетереть. - При исчерпании ретраев finalizeRun НЕ восстанавливает entry, а оставляет ЗОМБИ-запись { terminalWriteFailed, intended:{status,error} } в отдельной map; settleZombie пере-драйвит intended условным UPDATE (зовётся reconcile/supersede/ boot sweep). Зомби удаляется после успешного UPDATE или обнаружения строки уже терминальной. - Per-run settledPromise в ОТДЕЛЬНОЙ map runId→deferred (переживает active.delete), создаётся в beginRun, резолвится ровно один раз исходом (записано/сдался); поздний подписчик через ранее взятую ссылку получает резолвленный; peekSettled: live deferred → зомби-синтез → undefined (читать строку). Обе map bounded. - Задокументирована потеря (single-process): рестарт до пере-драйва → boot sweep пишет 'aborted' поверх фактического intended — неустранимо в phase 1. Тесты: юниты give-up-пути (зомби вместо restore + notifier terminalWriteFailed + settleZombie), двойного settle, позднего подписчика; интеграционные против реальной Б, что условный finalizeIfActive не перетирает терминал и двойной settle схлопывается. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../core/ai-chat/ai-chat-run.service.spec.ts | 134 +++++--- .../src/core/ai-chat/ai-chat-run.service.ts | 302 ++++++++++++++---- .../ai-chat/ai-chat.service.lifecycle.spec.ts | 12 +- .../repos/ai-chat/ai-chat-run.repo.ts | 35 ++ .../test/integration/ai-chat-run.int-spec.ts | 46 +++ 5 files changed, 424 insertions(+), 105 deletions(-) diff --git a/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts b/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts index 4843a979..4a24909b 100644 --- a/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts @@ -43,6 +43,9 @@ function makeRepo(overrides: Record = {}) { workspaceId: v.workspaceId, })), update: jest.fn(async () => ({ id: 'run-1' })), + // #487: terminal finalize now goes through the CONDITIONAL write. Default + // returns a truthy row (the run WAS active -> this call wrote it). + finalizeIfActive: jest.fn(async () => ({ id: 'run-1', status: 'succeeded' })), markStopRequested: jest.fn(async () => ({ id: 'run-1' })), findActiveByChat: jest.fn(async () => undefined), findLatestByChat: jest.fn(async () => undefined), @@ -336,14 +339,12 @@ describe('AiChatRunService run lifecycle', () => { await svc.finalizeRun('run-1', 'ws-1', 'error', 'provider blew up'); expect(svc.isLocallyActive('run-1')).toBe(false); - expect(repo.update).toHaveBeenCalledWith( + // #487: the terminal write is CONDITIONAL (finalizeIfActive); finishedAt is + // stamped inside the repo method, so the service passes just status + error. + expect(repo.finalizeIfActive).toHaveBeenCalledWith( 'run-1', 'ws-1', - expect.objectContaining({ - status: 'failed', - error: 'provider blew up', - finishedAt: expect.any(Date), - }), + expect.objectContaining({ status: 'failed', error: 'provider blew up' }), ); }); @@ -366,8 +367,8 @@ describe('AiChatRunService run lifecycle', () => { // A second settle (e.g. a streamText callback firing after the catch) no-ops. await svc.finalizeRun('run-1', 'ws-1', 'completed', undefined); - expect(repo.update).toHaveBeenCalledTimes(1); - expect(repo.update).toHaveBeenCalledWith( + expect(repo.finalizeIfActive).toHaveBeenCalledTimes(1); + expect(repo.finalizeIfActive).toHaveBeenCalledWith( 'run-1', 'ws-1', expect.objectContaining({ status: 'failed', error: 'first' }), @@ -389,8 +390,8 @@ describe('AiChatRunService run lifecycle', () => { const updateGate = new Promise((res) => { resolveUpdate = res; }); - const update = jest.fn(() => updateGate); - const repo = makeRepo({ update }); + const finalizeIfActive = jest.fn(() => updateGate); + const repo = makeRepo({ finalizeIfActive }); const svc = new AiChatRunService(repo as never, makeEnv() as never); await svc.beginRun({ chatId: 'chat-1', @@ -399,23 +400,23 @@ describe('AiChatRunService run lifecycle', () => { }); // Fire both before the (pending) update resolves. The first synchronously - // claims the entry (active.delete) and awaits update; the second, started in - // the same macrotask, finds the entry already gone and returns at the claim - // WITHOUT ever calling update. + // claims the entry (active.delete) and awaits the write; the second, started + // in the same macrotask, finds the entry already gone and returns at the claim + // WITHOUT ever writing. const p1 = svc.finalizeRun('run-1', 'ws-1', 'completed'); const p2 = svc.finalizeRun('run-1', 'ws-1', 'error', 'safety-net'); // The decisive assertion: exactly one caller reached the terminal UPDATE. - expect(update).toHaveBeenCalledTimes(1); + expect(finalizeIfActive).toHaveBeenCalledTimes(1); // Let the single in-flight update land; both calls resolve cleanly. - resolveUpdate({ id: 'run-1' }); + resolveUpdate({ id: 'run-1', status: 'succeeded' }); await Promise.all([p1, p2]); - expect(update).toHaveBeenCalledTimes(1); + expect(finalizeIfActive).toHaveBeenCalledTimes(1); // The winner is the FIRST caller ('completed' -> 'succeeded'); the late // 'error' settle never wrote, so it could not clobber the real status. - expect(update).toHaveBeenCalledWith( + expect(finalizeIfActive).toHaveBeenCalledWith( 'run-1', 'ws-1', expect.objectContaining({ status: 'succeeded' }), @@ -431,10 +432,10 @@ describe('AiChatRunService run lifecycle', () => { // 409s until a restart. The fix updates FIRST and retries. let calls = 0; const repo = makeRepo({ - update: jest.fn(async () => { + finalizeIfActive: jest.fn(async () => { calls += 1; if (calls === 1) throw new Error('deadlock detected'); - return { id: 'run-1' }; + return { id: 'run-1', status: 'succeeded' }; }), }); jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined); @@ -447,26 +448,29 @@ describe('AiChatRunService run lifecycle', () => { await svc.finalizeRun('run-1', 'ws-1', 'completed'); - // The retry landed the terminal write: the entry is dropped (slot freed) and - // the row carries the real terminal status — NOT stranded at 'running'. + // The retry landed the terminal write: the entry is dropped (slot freed), no + // zombie left, and the row carries the real terminal status. expect(svc.isLocallyActive('run-1')).toBe(false); - expect(repo.update).toHaveBeenCalledTimes(2); - expect(repo.update).toHaveBeenLastCalledWith( + expect(svc.hasZombie('run-1')).toBe(false); + expect(repo.finalizeIfActive).toHaveBeenCalledTimes(2); + expect(repo.finalizeIfActive).toHaveBeenLastCalledWith( 'run-1', 'ws-1', expect.objectContaining({ status: 'succeeded' }), ); }); - it('F6: if the terminal write keeps failing, the entry is RETAINED and a LATER settle completes it (chat not permanently 409d)', async () => { + it('#487 give-up: if the terminal write keeps failing, finalizeRun leaves a ZOMBIE (does NOT restore the entry) and settleZombie re-drives it', async () => { // Worst case: the DB is down for the whole first finalize (all attempts fail). - // The run must NOT be silently lost — the entry stays so a subsequent settle - // (a streamText callback, requestStop -> onAbort, or a future sweep) can retry. + // #487 changes the give-up behaviour: the entry is NOT restored (a restored + // entry is indistinguishable from a live run). Instead a ZOMBIE record holds + // the intended terminal status, and a re-drive (settleZombie — called by the + // reconcile / supersede / opportunistic paths) applies it later. let healthy = false; const repo = makeRepo({ - update: jest.fn(async () => { + finalizeIfActive: jest.fn(async () => { if (!healthy) throw new Error('pool exhausted'); - return { id: 'run-1' }; + return { id: 'run-1', status: 'succeeded' }; }), }); jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined); @@ -480,35 +484,83 @@ describe('AiChatRunService run lifecycle', () => { userId: 'user-1', }); - // First settle: every bounded attempt fails -> entry retained, NOT settled. + // First settle: every bounded attempt fails -> ZOMBIE, entry NOT restored. await svc.finalizeRun('run-1', 'ws-1', 'completed'); - expect(svc.isLocallyActive('run-1')).toBe(true); - // F12: the give-up emits ONE explicit, greppable ERROR (run + chat context) - // so an operator can tell "gave up, run held in memory" from a per-attempt - // blip — distinct from the per-attempt warns. + expect(svc.isLocallyActive('run-1')).toBe(false); // NOT a live entry + expect(svc.hasZombie('run-1')).toBe(true); + expect(svc.zombieRunIds()).toContain('run-1'); + // The give-up emits ONE explicit, greppable ERROR mentioning the zombie. const gaveUp = errorSpy.mock.calls.some( (c) => /NON-TERMINAL/.test(String(c[0])) && + /ZOMBIE/.test(String(c[0])) && /run-1/.test(String(c[0])) && /chat-1/.test(String(c[0])), ); expect(gaveUp).toBe(true); + // The settle notifier resolved as terminalWriteFailed (a subscriber learns the + // slot still needs the intended status applied). + const outcome = await svc.peekSettled('run-1'); + expect(outcome).toEqual({ + status: 'succeeded', + error: null, + terminalWriteFailed: true, + }); - // The DB recovers; a later settle now succeeds and frees the slot. + // The DB recovers; a re-drive settles the zombie via the conditional UPDATE. healthy = true; - await svc.finalizeRun('run-1', 'ws-1', 'completed'); - expect(svc.isLocallyActive('run-1')).toBe(false); - expect(repo.update).toHaveBeenLastCalledWith( + const redriven = await svc.settleZombie('run-1'); + expect(redriven).toBe(true); + expect(svc.hasZombie('run-1')).toBe(false); + expect(repo.finalizeIfActive).toHaveBeenLastCalledWith( 'run-1', 'ws-1', expect.objectContaining({ status: 'succeeded' }), ); - // And it is now idempotent: a further settle no-ops (terminal row already - // written), so a double-settle can never clobber the real status. - const callsBefore = repo.update.mock.calls.length; + // A later finalizeRun is idempotent (row already terminal): it no-ops at the + // once-gate, never re-writing. + const callsBefore = repo.finalizeIfActive.mock.calls.length; await svc.finalizeRun('run-1', 'ws-1', 'error', 'late'); - expect(repo.update).toHaveBeenCalledTimes(callsBefore); + expect(repo.finalizeIfActive).toHaveBeenCalledTimes(callsBefore); + }); + + it('#487 double-settle collapses to a benign no-op (conditional write; notifier resolves once)', async () => { + // A second concurrent settle is stopped at the synchronous active.delete + // claim, so the terminal write runs exactly once and the notifier resolves + // exactly once with the FIRST settler's outcome. + const repo = makeRepo(); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + await svc.beginRun({ chatId: 'chat-1', workspaceId: 'ws-1', userId: 'u1' }); + + await svc.finalizeRun('run-1', 'ws-1', 'aborted'); + await svc.finalizeRun('run-1', 'ws-1', 'error', 'late'); // no-op + + expect(repo.finalizeIfActive).toHaveBeenCalledTimes(1); + const outcome = await svc.peekSettled('run-1'); + // peekSettled after resolve+delete falls through (notifier dropped, no zombie) + // -> undefined; the FIRST settler already resolved any earlier subscriber. + expect(outcome).toBeUndefined(); + }); + + it('#487 late settledPromise subscriber gets the resolved outcome', async () => { + const repo = makeRepo(); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + await svc.beginRun({ chatId: 'chat-1', workspaceId: 'ws-1', userId: 'u1' }); + + // Subscribe BEFORE settle: hold the promise reference (as supersede does). + const early = svc.peekSettled('run-1'); + expect(early).toBeDefined(); + + await svc.finalizeRun('run-1', 'ws-1', 'completed'); + + // The reference grabbed before settle resolves with the written outcome, even + // though the notifier was dropped from the map on resolve (bounded). + await expect(early).resolves.toEqual({ + status: 'succeeded', + error: null, + terminalWriteFailed: false, + }); }); it('recordStep / linkAssistantMessage are best-effort: a repo failure is swallowed', async () => { diff --git a/apps/server/src/core/ai-chat/ai-chat-run.service.ts b/apps/server/src/core/ai-chat/ai-chat-run.service.ts index b2e0d393..2c2c379d 100644 --- a/apps/server/src/core/ai-chat/ai-chat-run.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat-run.service.ts @@ -34,6 +34,54 @@ export class RunAlreadyActiveError extends Error { export type TurnTerminalStatus = 'completed' | 'error' | 'aborted'; export type RunTerminalStatus = 'succeeded' | 'failed' | 'aborted'; +/** The terminal run statuses — the row is done once it reads one of these. */ +export const RUN_TERMINAL_STATUSES: readonly RunTerminalStatus[] = [ + 'succeeded', + 'failed', + 'aborted', +]; + +/** Whether a persisted run status is terminal (settled). */ +export function isRunTerminal(status: string | null | undefined): boolean { + return ( + status === 'succeeded' || status === 'failed' || status === 'aborted' + ); +} + +/** + * #487: the outcome a run's {@link AiChatRunService.finalizeRun} settled with. + * `terminalWriteFailed` = the terminal write GAVE UP after the bounded retry, so + * the row is still non-terminal ('running') and a ZOMBIE record holds the + * `intended` status for a later re-drive (reconcile / supersede / boot sweep). A + * subscriber (supersede, #487 commit 3) uses this to decide whether the slot is + * genuinely free or must first have the intended status applied. + */ +export interface RunSettleOutcome { + status: RunTerminalStatus; + error: string | null; + terminalWriteFailed: boolean; +} + +/** A one-shot settle notifier (#487): `resolve` is called EXACTLY ONCE. */ +interface Deferred { + promise: Promise; + resolve: (value: T) => void; +} + +/** + * #487: a run whose terminal write GAVE UP (every bounded attempt failed). The + * row is stranded non-terminal ('running'); this record is the ONLY thing that + * distinguishes it from a live run, and carries the `intended` terminal status so + * a re-drive can apply it via the conditional UPDATE. Process-local (phase-1 + * single-process assumption): a restart drops it, and the boot sweep then writes + * 'aborted' over the intended — a documented loss (see finalizeRun). + */ +interface ZombieRun { + workspaceId: string; + chatId: string; + intended: { status: RunTerminalStatus; error: string | null }; +} + export function mapTurnStatusToRun( status: TurnTerminalStatus, ): RunTerminalStatus { @@ -101,6 +149,22 @@ export class AiChatRunService implements OnModuleInit { // uptime — negligible in phase 1's single process. private readonly settled = new Set(); + // #487 runId -> one-shot settle notifier. Kept in a SEPARATE map from `active` + // ON PURPOSE: it must OUTLIVE the `active.delete` claim inside finalizeRun (the + // claim frees the slot the instant finalize starts), so a subscriber can still + // await the outcome after the entry is gone. Created in beginRun, resolved + // EXACTLY ONCE in finalizeRun, then removed (bounded). Absence => this replica + // has no live notifier: a subscriber falls back to the zombie map, then to the + // row (see peekSettled). Process-local (phase-1 single-process assumption). + private readonly settledPromises = new Map>(); + + // #487 runId -> ZOMBIE record: a run whose terminal write gave up (row stranded + // non-terminal). BOUNDED — an entry is added only on give-up and removed on a + // successful re-drive (settleZombie) or when the row is found already terminal; + // a process restart clears it (and the boot sweep settles the stranded row). + // Process-local (phase-1 single-process assumption). + private readonly zombies = new Map(); + // Bounded retry for the terminal write (F6): a single PK UPDATE can fail // transiently under many fire-and-forget writes (pool exhaustion, deadlock, a // brief connection blip). Riding out that blip in-place matters because the @@ -224,6 +288,10 @@ export class AiChatRunService implements OnModuleInit { chatId: args.chatId, workspaceId: args.workspaceId, }); + // #487: arm the one-shot settle notifier BEFORE returning, so a subscriber + // that races in immediately after begin always finds a promise to await. It + // is resolved exactly once when the run settles (or gives up). + this.settledPromises.set(run.id, this.makeDeferred()); return { runId: run.id, signal: controller.signal }; } @@ -263,47 +331,43 @@ export class AiChatRunService implements OnModuleInit { } /** - * Finalize a run to its terminal status (succeeded / failed / aborted), - * stamping finishedAt + any error. Best-effort, but ROBUST against a transient - * terminal-write failure (F6) AND atomically safe against a concurrent settle. + * Finalize a run to its terminal status (succeeded / failed / aborted) via a + * CONDITIONAL UPDATE, stamping finishedAt + any error. Atomically safe against a + * concurrent settle AND robust against a transient terminal-write failure. * * ATOMIC ONCE-CLAIM (the gate must close in ONE synchronous tick): two * finalizeRun calls for the SAME run can race — the documented real path is * AiChatService.stream's safety-net catch settling the turn to 'error' while a * streamText terminal callback (onFinish/onAbort/onError) ALSO settles it. The - * `settled.has` check alone is NOT a gate: it is read BEFORE the awaited UPDATE, - * so two callers can both see `false` and both write the row (last-write-wins - * clobbers the real terminal status, and the bounded retry only widens that - * window). The claim therefore happens via `active.delete`, a SYNCHRONOUS - * check-and-clear with NO await between the gate and the entry removal: the - * second concurrent caller finds the entry already gone and returns in the same - * tick, before any UPDATE. The transition "nobody is finalizing" -> "I am - * finalizing" is thus a single atomic step. + * claim happens via `active.delete`, a SYNCHRONOUS check-and-clear with NO await + * between the gate and the entry removal: the second concurrent caller finds the + * entry already gone and returns in the same tick, before any UPDATE. * - * ORDER MATTERS (F6): once we own the claim, the terminal UPDATE happens FIRST; - * only once it SUCCEEDS do we record the run as settled. If the UPDATE fails on - * every bounded attempt we RESTORE the in-memory entry, leave the run UNsettled, - * and emit an ERROR signal that the row is left non-terminal 'running' (which - * would 409 every future turn in the chat until recovery). An in-process retry - * by a LATER settle is only POSSIBLE, never guaranteed: it needs (a) the entry - * to have been restored at the give-up path AND (b) a fresh settler to arrive - * AFTER that restore. A concurrent settler that arrives DURING the retry window - * — while the entry is deleted for backoff and not yet restored — is consumed at - * the synchronous `active.delete` claim (it finds nothing to delete and returns - * a no-op), so it does NOT become an in-process retrier. The NO-streamText path - * (the turn threw before streamText was wired, so ONLY the safety-net ever - * settles) likewise has no second in-process settler at all. The UNCONDITIONAL - * backstop in every case is the boot sweep on the next restart (phase 1 has no - * periodic in-process sweep); the retained entry is bounded (cleared on restart) - * and harmless meanwhile. + * ALL TERMINAL WRITES ARE CONDITIONAL (#487): `finalizeIfActive` only flips a + * row still in pending|running (mirror of the assistant message's + * `onlyIfStreaming`). So even a settle that DID reach the UPDATE (e.g. a + * reconcile stamp racing an owner finalize) can never clobber a terminal status + * — the loser matches nothing and is a benign no-op. `active.delete` is the + * fast, in-process gate; the conditional WHERE is the authoritative one. * - * IDEMPOTENT on SUCCESS (#184 review): the terminal write happens AT MOST ONCE - * per run. After a successful write the once-gate keys off {@link settled} (the - * terminal row already written) so a settle arriving AFTER the entry was already - * dropped-and-settled returns early; a settle racing the in-flight write is - * stopped earlier still, by the `active.delete` claim. Either way a genuine - * double-settle collapses to a single write and a late settle can never clobber - * the real terminal status or double-write the row. + * ZOMBIE ON GIVE-UP (#487): if every bounded attempt THROWS (the DB is down for + * the whole finalize), we do NOT restore the entry. The row is stranded + * non-terminal ('running'); we record a ZOMBIE `{ terminalWriteFailed, intended + * }` (the ONLY thing distinguishing this dead run from a live one) and resolve + * the settle notifier with `terminalWriteFailed: true`. A restore would make the + * zombie indistinguishable from a live run to every reader; instead a re-drive + * (settleZombie, called by the periodic reconcile / supersede / opportunistic + * paths) applies the intended status later via the same conditional UPDATE. + * + * DOCUMENTED LOSS (#487, single-process phase 1): if the process RESTARTS before + * a zombie is re-driven, the in-memory zombie map is gone and the boot sweep + * (unconditional) writes 'aborted' over the ACTUAL intended status. This is + * unavoidable while the run lifecycle is single-process — there is no durable + * record of `intended`; a cross-process durable intent is deferred to phase 2. + * + * IDEMPOTENT: the settle notifier resolves EXACTLY ONCE; a second settle is + * stopped at `settled.has` or the `active.delete` claim, so a double-settle + * collapses to a single write and can never double-resolve or clobber the row. */ async finalizeRun( runId: string, @@ -314,13 +378,17 @@ export class AiChatRunService implements OnModuleInit { // ---- Atomic once-claim (synchronous; NO await before the gate closes) ---- // Already terminally written -> idempotent no-op. if (this.settled.has(runId)) return; - // Capture the entry BEFORE the delete so a total-failure path can restore it. + // Capture the entry BEFORE the delete for the give-up log context. const entry = this.active.get(runId); // SYNCHRONOUS check-and-clear: the FIRST caller deletes (claims) the entry; // any concurrent SECOND caller finds nothing to delete and returns HERE, in // the same tick, before any await — so it can never reach the UPDATE. if (!this.active.delete(runId)) return; + const status = mapTurnStatusToRun(turnStatus); + const err = error ?? null; + const chatId = entry?.chatId ?? 'unknown'; + let lastError: unknown; for ( let attempt = 1; @@ -328,47 +396,159 @@ export class AiChatRunService implements OnModuleInit { attempt++ ) { try { - await this.runRepo.update(runId, workspaceId, { - status: mapTurnStatusToRun(turnStatus), - finishedAt: new Date(), - error: error ?? null, + const row = await this.runRepo.finalizeIfActive(runId, workspaceId, { + status, + error: err, }); - // Terminal write landed: arm the once-gate. The entry is already gone - // (claimed above); we do NOT restore it. The slot is now free. + // No throw => the row is now terminal (we wrote it, or it was ALREADY + // terminal — another writer won the conditional UPDATE, a benign no-op). this.settled.add(runId); + this.zombies.delete(runId); + // Resolve with the persisted outcome: our status when WE wrote it, else + // the row's real terminal status (re-read on the already-terminal path so + // a subscriber never sees a status we did not actually persist). + const outcome: RunSettleOutcome = row + ? { status, error: err, terminalWriteFailed: false } + : await this.readTerminalOutcome(runId, workspaceId, status, err); + this.resolveSettled(runId, outcome); return; - } catch (err) { - lastError = err; + } catch (err2) { + lastError = err2; this.logger.warn( `Failed to finalize run ${runId} (attempt ${attempt}/${ AiChatRunService.FINALIZE_MAX_ATTEMPTS - }): ${err instanceof Error ? err.message : 'unknown error'}`, + }): ${err2 instanceof Error ? err2.message : 'unknown error'}`, ); if (attempt < AiChatRunService.FINALIZE_MAX_ATTEMPTS) { await this.delay(AiChatRunService.FINALIZE_RETRY_BASE_MS * attempt); } } } - // Every attempt failed: this is a give-up, materially worse than a per-attempt - // blip — the row is left NON-TERMINAL ('running'), so emit ONE explicit, - // greppable ERROR so an operator can tell "survived a blip" from "gave up, run - // held in memory until recovery" (the last warn alone says only "attempt 3/3"). + // Every attempt threw: GIVE UP. The row is stranded non-terminal ('running'). + // Do NOT restore the entry (a restored entry is indistinguishable from a live + // run); leave a ZOMBIE record instead, and resolve the notifier as + // terminalWriteFailed so a subscriber knows the slot still needs the intended + // status applied. One explicit, greppable ERROR so an operator can tell a + // give-up from a per-attempt blip. this.logger.error( - `Run ${runId} (chat ${entry?.chatId ?? 'unknown'}) left NON-TERMINAL ` + - `('running'): terminal write failed after ${ - AiChatRunService.FINALIZE_MAX_ATTEMPTS - } attempts; entry retained in memory, recovery deferred to next settle / ` + - `boot sweep`, + `Run ${runId} (chat ${chatId}) left NON-TERMINAL ('running'): terminal ` + + `write failed after ${AiChatRunService.FINALIZE_MAX_ATTEMPTS} attempts; ` + + `ZOMBIE recorded (intended '${status}'), recovery deferred to reconcile / ` + + `supersede / boot sweep`, lastError, ); - // RESTORE the claimed entry (and leave the run UNsettled) so a LATER settle - // that arrives AFTER this restore MAY retry the terminal write — but that - // in-process retry is NOT guaranteed (a concurrent settler caught in the retry - // window above is consumed at the `active.delete` claim, and the no-streamText - // path has no second settler at all). The UNCONDITIONAL backstop in every case - // is the boot sweep on the next restart; the restored entry is bounded and - // cleared on restart. - if (entry) this.active.set(runId, entry); + this.zombies.set(runId, { + workspaceId, + chatId, + intended: { status, error: err }, + }); + this.resolveSettled(runId, { status, error: err, terminalWriteFailed: true }); + } + + /** + * #487: re-drive a zombie run's intended terminal write (the conditional + * UPDATE). Called by the periodic reconcile (commit 4), an opportunistic + * single-chat reconcile, and supersede (commit 3). On success — the row is now + * terminal (written OR found already terminal) — the zombie is cleared and the + * once-gate armed; on another failure the zombie is kept for a later retry. + * Returns true when the row is now terminal. Best-effort; never throws. + */ + async settleZombie(runId: string): Promise { + const z = this.zombies.get(runId); + if (!z) return false; + try { + await this.runRepo.finalizeIfActive(runId, z.workspaceId, { + status: z.intended.status, + error: z.intended.error, + }); + this.zombies.delete(runId); + this.settled.add(runId); + return true; + } catch (err) { + this.logger.warn( + `Re-drive of zombie run ${runId} (chat ${z.chatId}) failed; will retry ` + + `later: ${err instanceof Error ? err.message : 'unknown error'}`, + ); + return false; + } + } + + /** + * #487: the run's settle outcome as seen by THIS replica, or undefined when it + * has no record (the caller then reads the row — the DB is the source of truth). + * A LIVE deferred (still settling, or resolved-but-not-yet-consumed) wins; a + * ZOMBIE synthesizes the give-up outcome. A subscriber (supersede) races this + * against a timeout. + */ + peekSettled(runId: string): Promise | undefined { + const d = this.settledPromises.get(runId); + if (d) return d.promise; + const z = this.zombies.get(runId); + if (z) { + return Promise.resolve({ + status: z.intended.status, + error: z.intended.error, + terminalWriteFailed: true, + }); + } + return undefined; + } + + /** #487 test/diagnostic seam: whether a give-up zombie is held for this run. */ + hasZombie(runId: string): boolean { + return this.zombies.has(runId); + } + + /** #487: every zombie runId held on this replica (reconcile clause a, commit 4). */ + zombieRunIds(): string[] { + return [...this.zombies.keys()]; + } + + /** #487: create a one-shot deferred (resolve captured for a later single call). */ + private makeDeferred(): Deferred { + let resolve!: (value: T) => void; + const promise = new Promise((r) => { + resolve = r; + }); + return { promise, resolve }; + } + + /** #487: resolve a run's settle notifier EXACTLY ONCE, then drop it (bounded). + * A subscriber that already grabbed the promise still resolves; a later one + * falls back to the zombie map / the row (see peekSettled). */ + private resolveSettled(runId: string, outcome: RunSettleOutcome): void { + const d = this.settledPromises.get(runId); + if (!d) return; + this.settledPromises.delete(runId); + d.resolve(outcome); + } + + /** #487: read the persisted terminal outcome when the conditional finalize was a + * no-op (the row was already terminal). Falls back to the intended status when + * the read fails or the row is unexpectedly missing/non-terminal. */ + private async readTerminalOutcome( + runId: string, + workspaceId: string, + fallbackStatus: RunTerminalStatus, + fallbackError: string | null, + ): Promise { + try { + const row = await this.runRepo.findById(runId, workspaceId); + if (row && isRunTerminal(row.status)) { + return { + status: row.status as RunTerminalStatus, + error: row.error ?? null, + terminalWriteFailed: false, + }; + } + } catch { + // Fall through to the intended status — best-effort only. + } + return { + status: fallbackStatus, + error: fallbackError, + terminalWriteFailed: false, + }; } /** Small async backoff between terminal-write retries (F6). Isolated so it is diff --git a/apps/server/src/core/ai-chat/ai-chat.service.lifecycle.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.lifecycle.spec.ts index 66eb5b42..a6f3e75c 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.lifecycle.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.lifecycle.spec.ts @@ -89,6 +89,11 @@ describe('AiChatService.stream run-lifecycle safety net (#184)', () => { const runRepo = { insert: jest.fn().mockResolvedValue({ id: 'run-1', status: 'running' }), update: jest.fn().mockResolvedValue({ id: 'run-1' }), + // #487: the terminal settle now goes through the CONDITIONAL write. + finalizeIfActive: jest + .fn() + .mockResolvedValue({ id: 'run-1', status: 'failed' }), + findById: jest.fn().mockResolvedValue(undefined), }; const runService = new AiChatRunService(runRepo as never, { isCloud: () => false } as never); @@ -148,9 +153,10 @@ describe('AiChatService.stream run-lifecycle safety net (#184)', () => { // The run was begun... expect(runRepo.insert).toHaveBeenCalledTimes(1); - // ...then settled to a terminal FAILED status by the safety net... - expect(runRepo.update).toHaveBeenCalledTimes(1); - expect(runRepo.update).toHaveBeenCalledWith( + // ...then settled to a terminal FAILED status by the safety net (via the + // #487 conditional write)... + expect(runRepo.finalizeIfActive).toHaveBeenCalledTimes(1); + expect(runRepo.finalizeIfActive).toHaveBeenCalledWith( 'run-1', 'ws1', expect.objectContaining({ status: 'failed' }), diff --git a/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts b/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts index 7bb6bcdb..e7976ebe 100644 --- a/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts +++ b/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts @@ -143,6 +143,41 @@ export class AiChatRunRepo { .executeTakeFirst(); } + /** + * #487: CONDITIONAL terminal finalize — flip a run to a terminal status and + * stamp `finished_at` ONLY while it is still active (pending|running), mirroring + * the assistant message's `onlyIfStreaming` guard. A double-settle (a late or + * second writer, a supersede applying a zombie's intended, a reconcile stamp) + * matches NOTHING once the row is terminal and is a benign no-op — so a terminal + * status can never be clobbered by a later writer (last-writer-wins is gone). + * + * Returns the updated row when it WAS active (this call wrote it), else + * undefined (the row was already terminal — another writer won). The caller + * distinguishes the two to resolve the correct settle outcome. + */ + async finalizeIfActive( + id: string, + workspaceId: string, + patch: { status: string; error: string | null }, + trx?: KyselyTransaction, + ): Promise { + const db = dbOrTx(this.db, trx); + const now = new Date(); + return db + .updateTable('aiChatRuns') + .set({ + status: patch.status, + error: patch.error, + finishedAt: now, + updatedAt: now, + }) + .where('id', '=', id) + .where('workspaceId', '=', workspaceId) + .where('status', 'in', ACTIVE_RUN_STATUSES as unknown as string[]) + .returning(this.baseFields) + .executeTakeFirst(); + } + /** * Mark an EXPLICIT stop request on an active run (distinct from a browser * disconnect, which never stops a run). Stamps `stop_requested_at` ONLY while diff --git a/apps/server/test/integration/ai-chat-run.int-spec.ts b/apps/server/test/integration/ai-chat-run.int-spec.ts index f6a753a1..aca31c03 100644 --- a/apps/server/test/integration/ai-chat-run.int-spec.ts +++ b/apps/server/test/integration/ai-chat-run.int-spec.ts @@ -281,6 +281,52 @@ describe('AiChatRun durable lifecycle [integration]', () => { }); }); + it('#487 finalizeIfActive is CONDITIONAL: a late terminal write cannot clobber the settled status (real SQL)', async () => { + const c = (await createChat(db, { workspaceId, creatorId: userId })).id; + const run = await runRepo.insert({ + chatId: c, + workspaceId, + createdBy: userId, + status: 'running', + }); + + // First terminal write: the run IS active, so it flips + returns the row. + const first = await runRepo.finalizeIfActive(run.id, workspaceId, { + status: 'succeeded', + error: null, + }); + expect(first!.status).toBe('succeeded'); + expect(first!.finishedAt).toBeTruthy(); + + // A late/second writer tries to flip it to 'aborted' — the WHERE status IN + // ('pending','running') guard matches NOTHING now, so it is a benign no-op. + const second = await runRepo.finalizeIfActive(run.id, workspaceId, { + status: 'aborted', + error: 'late clobber attempt', + }); + expect(second).toBeUndefined(); + + // The persisted terminal status is UNCHANGED — last-writer-wins is gone. + const row = await runRepo.findById(run.id, workspaceId); + expect(row!.status).toBe('succeeded'); + expect(row!.error).toBeNull(); + }); + + it('#487 double-settle through the service collapses to one write at the SQL gate', async () => { + const c = (await createChat(db, { workspaceId, creatorId: userId })).id; + const handle = await service.beginRun({ chatId: c, workspaceId, userId }); + + // First settle writes 'aborted' via the conditional write. + await service.finalizeRun(handle.runId, workspaceId, 'aborted'); + // A late safety-net settle to 'error' is a no-op (row already terminal). + await service.finalizeRun(handle.runId, workspaceId, 'error', 'late'); + + const row = await runRepo.findById(handle.runId, workspaceId); + expect(row!.status).toBe('aborted'); + expect(service.isLocallyActive(handle.runId)).toBe(false); + expect(service.hasZombie(handle.runId)).toBe(false); + }); + it('sweepRunning() with NO args (boot sweep / variant C) aborts even a FRESH running run', async () => { // F1/DECISION C at the SQL level: the unconditional boot sweep has NO // staleness window, so a run updated just now (a fast restart) is settled too From 307ad493245569305ff6c626a9007dfabcabe1b6 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 04:51:44 +0300 Subject: [PATCH 19/41] =?UTF-8?q?feat(ai-chat):=20=D0=B5=D0=B4=D0=B8=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20=D0=B3=D0=B5=D0=B9=D1=82=20=D0=BA=D0=BE=D0=BD?= =?UTF-8?q?=D0=BA=D1=83=D1=80=D0=B5=D0=BD=D1=82=D0=BD=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B8=20+=20=D1=81=D0=B5=D1=80=D0=B2=D0=B5=D1=80=D0=BD=D1=8B?= =?UTF-8?q?=D0=B9=20supersede=20(CAS)=20(#487)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Раньше в legacy-режиме (дефолт!) гейта конкурентности НЕ было вообще — проверка 409 сидела внутри if (autonomousRuns && chatId): два таба = два параллельных стрима на один чат (интерливинг истории, падения convertToModelMessages). Серверная часть (клиентская лестница ретраев — отдельной клиентской итерацией FSM, см. ниже): - Run-строка ОБЯЗАТЕЛЬНА в ОБОИХ режимах: legacy тоже проходит beginRun/ finalizeRun (гейт партиального уникального индекса + runId в start-метаданных). Различие режимов сведено к семантике abort: legacy onClose при дисконнекте зовёт requestStop(runId) вместо аборта сокет-сигнала (которого streamText уже не потребляет). Второй таб в ЛЮБОМ режиме → 409 A_RUN_ALREADY_ACTIVE. - Supersede CAS POST /stream { supersede: { runId: X } } (AiChatRunService .supersede): валидация X.chatId===body.chatId (иначе 400 SUPERSEDE_INVALID); нет активного рана → degrade в обычный send; активный ≠ X → 409 SUPERSEDE_TARGET_MISMATCH + текущий runId; активный = X → requestStop → awaitSettled (таймаут W=10c) → «записано/сдался» (сдался → settleZombie применяет intended условным UPDATE) → ready; таймаут → 409 SUPERSEDE_TIMEOUT, ничего не персистится. W обоснован race-on-abort'ом коммита 1; DB-brownout → TIMEOUT штатен, W не увеличивать (env-tunable). - Задокументированы ограничения: нет квиесценции сайд-эффектов (в промпт нового рана добавлена строка SUPERSEDE_NOTE «предыдущий ран прерван, его последние операции могли примениться с задержкой»); кража слота между освобождением и beginRun (→ MISMATCH с новым runId, бэкстоп — уникальный индекс). Тесты: два параллельных старта (оба режима) → строго 409 второму; supersede при живом долгом ТУЛЕ (не UPDATE-задержке) settle'ится быстро; все CAS-ветки; дубль supersede-POST → degrade; зомби-путь через settleZombie; HTTP-маппинг ветвей. Кейс «beginRun ok → insert user-msg упал → ран settled, слот свободен» покрыт lifecycle-спекой против реального safety-net catch. КЛИЕНТ (deferred, точно flagged): удаление лестницы SUPERSEDE_RETRY_DELAYS_MS/ isRunAlreadyActive/supersedeRetryRef и переход на supersede-в-body требуют адопции runId из start-метаданных (сейчас клиент runId не трекает вовсе) — это и есть та «итерация клиентского FSM», которую данный серверный контракт разблокирует. Не трогаю фрагильный клиентский FSM без возможности E2E-валидации в браузере, чтобы не внести непроверяемую регрессию. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../core/ai-chat/ai-chat-run.service.spec.ts | 162 +++++++++++++ .../src/core/ai-chat/ai-chat-run.service.ts | 123 ++++++++++ .../ai-chat.controller.supersede.spec.ts | 221 ++++++++++++++++++ .../src/core/ai-chat/ai-chat.controller.ts | 207 +++++++++------- .../server/src/core/ai-chat/ai-chat.prompt.ts | 32 +++ .../src/core/ai-chat/ai-chat.service.ts | 17 ++ 6 files changed, 680 insertions(+), 82 deletions(-) create mode 100644 apps/server/src/core/ai-chat/ai-chat.controller.supersede.spec.ts diff --git a/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts b/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts index 4a24909b..1518dc73 100644 --- a/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts @@ -577,3 +577,165 @@ describe('AiChatRunService run lifecycle', () => { ).resolves.toBeUndefined(); }); }); + +describe('#487 AiChatRunService.supersede (CAS)', () => { + const chat = 'chat-1'; + const ws = 'ws-1'; + + it('degrade: no active run on the chat -> caller sends a normal turn', async () => { + const repo = makeRepo({ + findById: jest.fn(async () => undefined), + findActiveByChat: jest.fn(async () => undefined), + }); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + expect(await svc.supersede(chat, 'run-x', ws)).toEqual({ kind: 'degrade' }); + }); + + it('invalid: the target run belongs to a DIFFERENT chat -> 400', async () => { + const repo = makeRepo({ + findById: jest.fn(async () => ({ + id: 'run-x', + chatId: 'other-chat', + workspaceId: ws, + })), + }); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + expect(await svc.supersede(chat, 'run-x', ws)).toEqual({ kind: 'invalid' }); + }); + + it('mismatch: a DIFFERENT run is active than the one targeted -> current runId', async () => { + const repo = makeRepo({ + findById: jest.fn(async () => ({ id: 'run-x', chatId: chat, workspaceId: ws })), + findActiveByChat: jest.fn(async () => ({ + id: 'run-live', + chatId: chat, + workspaceId: ws, + status: 'running', + })), + }); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + expect(await svc.supersede(chat, 'run-x', ws)).toEqual({ + kind: 'mismatch', + activeRunId: 'run-live', + }); + }); + + it('ready: the target IS active -> stop it, await its (fast) settle, free the slot', async () => { + // Simulate a live long TOOL (NOT a slow UPDATE): the run stays active until an + // explicit Stop unwinds it; commit-1's race makes that settle land quickly. + // The abort listener stands in for streamText's onAbort -> finalizeRun. + const repo = makeRepo({ + findById: jest.fn(async () => ({ + id: 'run-1', + chatId: chat, + workspaceId: ws, + status: 'aborted', + error: null, + })), + findActiveByChat: jest.fn(async () => ({ + id: 'run-1', + chatId: chat, + workspaceId: ws, + status: 'running', + })), + }); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + const handle = await svc.beginRun({ chatId: chat, workspaceId: ws, userId: 'u1' }); + handle.signal.addEventListener('abort', () => { + void svc.finalizeRun('run-1', ws, 'aborted'); + }); + + // supersede: getRun -> getActiveByChat(==target) -> requestStop -> the abort + // listener settles the run -> awaitSettled resolves -> ready. + expect(await svc.supersede(chat, 'run-1', ws, 10_000)).toEqual({ + kind: 'ready', + }); + expect(handle.signal.aborted).toBe(true); // Stop reached the run + }); + + it('timeout: the target never settles within W -> 409 SUPERSEDE_TIMEOUT (nothing persisted)', async () => { + const repo = makeRepo({ + findById: jest.fn(async () => ({ id: 'run-1', chatId: chat, workspaceId: ws })), + findActiveByChat: jest.fn(async () => ({ + id: 'run-1', + chatId: chat, + workspaceId: ws, + status: 'running', + })), + }); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + await svc.beginRun({ chatId: chat, workspaceId: ws, userId: 'u1' }); + // Do NOT settle the run: a tiny W elapses -> timeout. + const result = await svc.supersede(chat, 'run-1', ws, 30); + expect(result).toEqual({ kind: 'timeout' }); + }); + + it('ready then a DUPLICATE supersede POST degrades (the run is already gone)', async () => { + let active: unknown = { + id: 'run-1', + chatId: chat, + workspaceId: ws, + status: 'running', + }; + const repo = makeRepo({ + findById: jest.fn(async () => ({ + id: 'run-1', + chatId: chat, + workspaceId: ws, + status: 'aborted', + error: null, + })), + findActiveByChat: jest.fn(async () => active), + finalizeIfActive: jest.fn(async () => { + active = undefined; // settling frees the active slot + return { id: 'run-1', status: 'aborted' }; + }), + }); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + const handle = await svc.beginRun({ chatId: chat, workspaceId: ws, userId: 'u1' }); + handle.signal.addEventListener('abort', () => { + void svc.finalizeRun('run-1', ws, 'aborted'); + }); + + expect(await svc.supersede(chat, 'run-1', ws, 10_000)).toEqual({ + kind: 'ready', + }); + // The duplicate POST for the same target now finds no active run -> degrade. + expect(await svc.supersede(chat, 'run-1', ws)).toEqual({ kind: 'degrade' }); + }); + + it('gave-up zombie: supersede applies the intended status (settleZombie) then is ready', async () => { + let healthy = false; + let active: unknown = { + id: 'run-1', + chatId: chat, + workspaceId: ws, + status: 'running', + }; + const repo = makeRepo({ + findById: jest.fn(async () => ({ id: 'run-1', chatId: chat, workspaceId: ws })), + findActiveByChat: jest.fn(async () => active), + finalizeIfActive: jest.fn(async () => { + if (!healthy) throw new Error('db down'); + active = undefined; + return { id: 'run-1', status: 'aborted' }; + }), + }); + jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined); + jest.spyOn(Logger.prototype, 'error').mockImplementation(() => undefined); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + await svc.beginRun({ chatId: chat, workspaceId: ws, userId: 'u1' }); + + // The run's terminal write gives up -> zombie (row still 'running'). + await svc.finalizeRun('run-1', ws, 'aborted'); + expect(svc.hasZombie('run-1')).toBe(true); + + // The DB recovers; supersede awaits the (already-resolved, terminalWriteFailed) + // settle, then settleZombie applies the intended status -> ready. + healthy = true; + expect(await svc.supersede(chat, 'run-1', ws, 10_000)).toEqual({ + kind: 'ready', + }); + expect(svc.hasZombie('run-1')).toBe(false); + }); +}); diff --git a/apps/server/src/core/ai-chat/ai-chat-run.service.ts b/apps/server/src/core/ai-chat/ai-chat-run.service.ts index 2c2c379d..8fd577c8 100644 --- a/apps/server/src/core/ai-chat/ai-chat-run.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat-run.service.ts @@ -62,6 +62,40 @@ export interface RunSettleOutcome { terminalWriteFailed: boolean; } +/** + * #487: how long a supersede waits for the target run to settle after Stop before + * it degrades to `SUPERSEDE_TIMEOUT`. W=10s is generous under a HEALTHY DB: commit + * 1's race-on-abort makes an in-app tool abort->settle in ms/hundreds of ms, so a + * live run releases its slot well within the window. Under a DB brownout the + * timeout is normal (the write cannot land); W must NOT be raised to paper + * over a slow DB — a SUPERSEDE_TIMEOUT is the honest signal (nothing persisted, + * the composer keeps the user's text). Env-tunable for ops, default 10s. + */ +export const SUPERSEDE_SETTLE_TIMEOUT_MS = (() => { + const raw = Number(process.env.AI_CHAT_SUPERSEDE_TIMEOUT_MS); + return Number.isFinite(raw) && raw > 0 ? raw : 10_000; +})(); + +/** + * #487: the result of the supersede CAS ({@link AiChatRunService.supersede}). + * - `degrade` : no active run on the chat (it ended between click and POST) — + * the caller sends a NORMAL turn (NOT a mismatch); + * - `invalid` : the target runId belongs to a DIFFERENT chat (malformed CAS 400); + * - `mismatch` : a DIFFERENT run is active than the one the client targeted — + * 409 SUPERSEDE_TARGET_MISMATCH carrying the current `activeRunId` + * (the client does NOT auto-retry); + * - `timeout` : the target did not settle within W — 409 SUPERSEDE_TIMEOUT, + * nothing persisted; + * - `ready` : the target was stopped AND settled (or its zombie's intended was + * applied) — the slot is free; the caller may beginRun the new run. + */ +export type SupersedeResult = + | { kind: 'degrade' } + | { kind: 'invalid' } + | { kind: 'mismatch'; activeRunId: string } + | { kind: 'timeout' } + | { kind: 'ready' }; + /** A one-shot settle notifier (#487): `resolve` is called EXACTLY ONCE. */ interface Deferred { promise: Promise; @@ -494,6 +528,95 @@ export class AiChatRunService implements OnModuleInit { return undefined; } + /** + * #487: await a run's settle outcome, bounded by `timeoutMs`. Returns the + * outcome on settle, or undefined on TIMEOUT (or when this replica has no record + * of the run and its row is not terminal). Uses the LIVE settle notifier / the + * zombie synth when present; else reads the row (the DB is the source of truth + * once the in-memory record is gone). The subscriber (supersede) grabs this + * right after Stop; commit 1's race makes the settle land in ms on a healthy DB. + */ + async awaitSettled( + runId: string, + workspaceId: string, + timeoutMs: number, + ): Promise { + const pending = this.peekSettled(runId); + if (pending) { + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(undefined), timeoutMs); + timer.unref?.(); + }); + try { + return await Promise.race([pending, timeout]); + } finally { + if (timer) clearTimeout(timer); + } + } + // No live notifier and no zombie: read the row (already settled-and-written, + // or unknown here). A terminal row is an outcome; anything else -> undefined. + const row = await this.runRepo.findById(runId, workspaceId); + if (row && isRunTerminal(row.status)) { + return { + status: row.status as RunTerminalStatus, + error: row.error ?? null, + terminalWriteFailed: false, + }; + } + return undefined; + } + + /** + * #487: the SERVER supersede CAS for `POST /stream { supersede: { runId: X } }`. + * Atomically transitions "X is the chat's active run" -> "X is stopped, settled, + * slot free" so the caller can start a replacement run. See {@link + * SupersedeResult} for the branch semantics. + * + * On a `ready` result the caller MUST still go through the normal beginRun gate + * (the partial unique index) — between the slot freeing here and beginRun a + * neighbouring tab's ordinary POST can win the slot (documented SLOT-THEFT: the + * loser then gets a MISMATCH carrying the NEW runId). There is also NO side- + * effect quiescence: an in-flight write of the stopped run may still land AFTER + * the new run starts (commit 1 stops the NEXT call, not one already committing), + * so the caller adds a prompt note to the new run. + */ + async supersede( + chatId: string, + targetRunId: string, + workspaceId: string, + timeoutMs: number = SUPERSEDE_SETTLE_TIMEOUT_MS, + ): Promise { + // Validate the target belongs to THIS chat (a CAS targeting another chat's run + // is malformed -> 400). A missing row is NOT invalid: the run may have ended + // and been pruned; the active-run check below decides degrade vs mismatch. + const target = await this.getRun(targetRunId, workspaceId); + if (target && target.chatId !== chatId) return { kind: 'invalid' }; + + const active = await this.getActiveForChat(chatId, workspaceId); + // No active run: it ended between the client's click and this POST — this is a + // DEGRADE to a normal send, NOT a mismatch (the user's intent still holds). + if (!active) return { kind: 'degrade' }; + // A DIFFERENT run is active than the one the client saw -> mismatch. The + // client does not auto-retry; it surfaces the new runId. + if (active.id !== targetRunId) { + return { kind: 'mismatch', activeRunId: active.id }; + } + + // The target IS active: stop it, then await its settle within W. + await this.requestStop(targetRunId, workspaceId); + const outcome = await this.awaitSettled(targetRunId, workspaceId, timeoutMs); + if (!outcome) return { kind: 'timeout' }; + // Gave up (terminal write failed): apply the intended status via the + // conditional UPDATE so the slot actually frees. If that ALSO fails, the row + // is still stranded -> treat as a timeout (nothing persisted for the new run). + if (outcome.terminalWriteFailed) { + const settled = await this.settleZombie(targetRunId); + if (!settled) return { kind: 'timeout' }; + } + return { kind: 'ready' }; + } + /** #487 test/diagnostic seam: whether a give-up zombie is held for this run. */ hasZombie(runId: string): boolean { return this.zombies.has(runId); diff --git a/apps/server/src/core/ai-chat/ai-chat.controller.supersede.spec.ts b/apps/server/src/core/ai-chat/ai-chat.controller.supersede.spec.ts new file mode 100644 index 00000000..69aa84b5 --- /dev/null +++ b/apps/server/src/core/ai-chat/ai-chat.controller.supersede.spec.ts @@ -0,0 +1,221 @@ +import { + BadRequestException, + ConflictException, + HttpException, +} from '@nestjs/common'; +import { AiChatController } from './ai-chat.controller'; +import type { User, Workspace } from '@docmost/db/types/entity.types'; + +/** + * #487 commit 3 — the single concurrency GATE (both modes) + the server supersede + * CAS, at the controller boundary. The gate + CAS run BEFORE res.hijack(), so a + * rejected concurrent start / a CAS branch returns clean JSON (an HttpException + * the controller's post-hijack catch re-serializes). These assert the OBSERVABLE + * HTTP contract against the real controller + a stubbed run service. + */ +describe('#487 AiChatController.stream — gate + supersede', () => { + const user = { id: 'u1' } as User; + + function wsWith(autonomousRuns: boolean): Workspace { + return { + id: 'ws1', + settings: { ai: { chat: true, autonomousRuns } }, + } as unknown as Workspace; + } + + function makeReqRes(body: Record) { + const req = { + raw: { sessionId: 'sess', once: jest.fn(), destroyed: false }, + body, + }; + const res = { + raw: { + writableEnded: false, + headersSent: false, + on: jest.fn(), + once: jest.fn(), + setHeader: jest.fn(), + end: jest.fn(), + statusCode: 200, + flushHeaders: jest.fn(), + }, + hijack: jest.fn(), + status: jest.fn().mockReturnThis(), + send: jest.fn(), + }; + return { req, res }; + } + + function makeController(runServiceOverrides: Record) { + const aiChatService = { + resolveRoleForRequest: jest.fn().mockResolvedValue(null), + getChatModel: jest.fn().mockResolvedValue({}), + stream: jest.fn().mockResolvedValue(undefined), + }; + const aiChatRunService = { + getActiveForChat: jest.fn().mockResolvedValue(undefined), + supersede: jest.fn(), + beginRun: jest.fn().mockResolvedValue({ + runId: 'run-new', + signal: new AbortController().signal, + }), + linkAssistantMessage: jest.fn(), + recordStep: jest.fn(), + finalizeRun: jest.fn(), + requestStop: jest.fn(), + ...runServiceOverrides, + }; + const controller = new AiChatController( + aiChatService as never, + aiChatRunService as never, + {} as never, // aiChatRepo + {} as never, // aiChatMessageRepo + {} as never, // aiTranscription + {} as never, // pageRepo + ); + return { controller, aiChatService, aiChatRunService }; + } + + const codeOf = (err: unknown) => + (((err as HttpException).getResponse() as Record) ?? {}) + .code; + + describe('single concurrency gate — BOTH modes reject the second tab with 409', () => { + for (const autonomousRuns of [true, false]) { + it(`rejects a concurrent start with 409 A_RUN_ALREADY_ACTIVE (autonomousRuns=${autonomousRuns})`, async () => { + const { controller, aiChatRunService } = makeController({ + getActiveForChat: jest + .fn() + .mockResolvedValue({ id: 'run-live', chatId: 'c1' }), + }); + const { req, res } = makeReqRes({ chatId: 'c1' }); + let thrown: unknown; + try { + await controller.stream( + req as never, + res as never, + user, + wsWith(autonomousRuns), + ); + } catch (e) { + thrown = e; + } + expect(thrown).toBeInstanceOf(ConflictException); + expect((thrown as HttpException).getStatus()).toBe(409); + expect(codeOf(thrown)).toBe('A_RUN_ALREADY_ACTIVE'); + // Rejected BEFORE committing to the stream (no hijack, no service.stream). + expect(res.hijack).not.toHaveBeenCalled(); + expect(aiChatRunService.getActiveForChat).toHaveBeenCalledWith( + 'c1', + 'ws1', + ); + }); + } + }); + + it('supersede MISMATCH -> 409 SUPERSEDE_TARGET_MISMATCH carrying the current runId', async () => { + const { controller } = makeController({ + supersede: jest + .fn() + .mockResolvedValue({ kind: 'mismatch', activeRunId: 'run-other' }), + }); + const { req, res } = makeReqRes({ + chatId: 'c1', + supersede: { runId: 'run-x' }, + }); + let thrown: unknown; + try { + await controller.stream(req as never, res as never, user, wsWith(true)); + } catch (e) { + thrown = e; + } + expect(thrown).toBeInstanceOf(ConflictException); + expect(codeOf(thrown)).toBe('SUPERSEDE_TARGET_MISMATCH'); + expect( + ((thrown as HttpException).getResponse() as Record) + .activeRunId, + ).toBe('run-other'); + expect(res.hijack).not.toHaveBeenCalled(); + }); + + it('supersede TIMEOUT -> 409 SUPERSEDE_TIMEOUT, nothing streamed', async () => { + const { controller } = makeController({ + supersede: jest.fn().mockResolvedValue({ kind: 'timeout' }), + }); + const { req, res } = makeReqRes({ + chatId: 'c1', + supersede: { runId: 'run-x' }, + }); + let thrown: unknown; + try { + await controller.stream(req as never, res as never, user, wsWith(false)); + } catch (e) { + thrown = e; + } + expect(thrown).toBeInstanceOf(ConflictException); + expect(codeOf(thrown)).toBe('SUPERSEDE_TIMEOUT'); + expect(res.hijack).not.toHaveBeenCalled(); + }); + + it('supersede INVALID (target on another chat) -> 400 SUPERSEDE_INVALID', async () => { + const { controller } = makeController({ + supersede: jest.fn().mockResolvedValue({ kind: 'invalid' }), + }); + const { req, res } = makeReqRes({ + chatId: 'c1', + supersede: { runId: 'run-x' }, + }); + let thrown: unknown; + try { + await controller.stream(req as never, res as never, user, wsWith(true)); + } catch (e) { + thrown = e; + } + expect(thrown).toBeInstanceOf(BadRequestException); + expect(codeOf(thrown)).toBe('SUPERSEDE_INVALID'); + }); + + it('supersede without chatId -> 400 SUPERSEDE_INVALID', async () => { + const { controller, aiChatRunService } = makeController({}); + const { req, res } = makeReqRes({ supersede: { runId: 'run-x' } }); + let thrown: unknown; + try { + await controller.stream(req as never, res as never, user, wsWith(true)); + } catch (e) { + thrown = e; + } + expect(thrown).toBeInstanceOf(BadRequestException); + expect(codeOf(thrown)).toBe('SUPERSEDE_INVALID'); + expect(aiChatRunService.supersede).not.toHaveBeenCalled(); + }); + + it('supersede READY -> proceeds to stream with superseded=true', async () => { + const { controller, aiChatService } = makeController({ + supersede: jest.fn().mockResolvedValue({ kind: 'ready' }), + getActiveForChat: jest.fn().mockResolvedValue(undefined), // slot free after CAS + }); + const { req, res } = makeReqRes({ + chatId: 'c1', + supersede: { runId: 'run-x' }, + }); + await controller.stream(req as never, res as never, user, wsWith(true)); + expect(res.hijack).toHaveBeenCalled(); + expect(aiChatService.stream).toHaveBeenCalledTimes(1); + expect(aiChatService.stream.mock.calls[0][0].superseded).toBe(true); + // The run hooks are always present now (both modes). + expect(aiChatService.stream.mock.calls[0][0].runHooks).toBeDefined(); + }); + + it('supersede DEGRADE -> proceeds to a normal send (superseded=false)', async () => { + const { controller, aiChatService } = makeController({ + supersede: jest.fn().mockResolvedValue({ kind: 'degrade' }), + }); + const { req, res } = makeReqRes({ + chatId: 'c1', + supersede: { runId: 'run-x' }, + }); + await controller.stream(req as never, res as never, user, wsWith(false)); + expect(aiChatService.stream).toHaveBeenCalledTimes(1); + expect(aiChatService.stream.mock.calls[0][0].superseded).toBe(false); + }); +}); diff --git a/apps/server/src/core/ai-chat/ai-chat.controller.ts b/apps/server/src/core/ai-chat/ai-chat.controller.ts index 8cc7b03d..6dbe8420 100644 --- a/apps/server/src/core/ai-chat/ai-chat.controller.ts +++ b/apps/server/src/core/ai-chat/ai-chat.controller.ts @@ -432,12 +432,66 @@ export class AiChatController { // HttpException) instead of breaking mid-stream. const model = await this.aiChatService.getChatModel(workspace.id, role); - // #184: one active run per chat. For an EXISTING chat reject a concurrent - // start with a clean 409 BEFORE hijack (the common double-submit / second-tab - // case), so the user gets JSON, not a mid-stream error. A brand-new chat - // (no chatId) cannot have a prior run, and the DB partial unique index is the - // backstop against any race that slips past this check. - if (autonomousRuns && body.chatId) { + // #487: server-side supersede CAS ("interrupt and send now"). When the client + // asks to replace a live run, atomically STOP it and wait for it to settle + // before this turn claims the slot. Runs BEFORE hijack so every branch returns + // clean JSON (the client keeps the composer text on a 409). See + // AiChatRunService.supersede for the branch semantics. + let superseded = false; + const supersedeRunId = body.supersede?.runId; + if (supersedeRunId) { + if (!body.chatId) { + throw new BadRequestException({ + message: 'supersede requires chatId', + code: 'SUPERSEDE_INVALID', + }); + } + const result = await this.aiChatRunService.supersede( + body.chatId, + supersedeRunId, + workspace.id, + ); + switch (result.kind) { + case 'invalid': + throw new BadRequestException({ + message: 'The run to supersede does not belong to this chat', + code: 'SUPERSEDE_INVALID', + }); + case 'mismatch': + // A DIFFERENT run is active than the one the client targeted. Surface + // the CURRENT runId; the client does NOT auto-retry (a stale CAS). + throw new ConflictException({ + message: 'A different agent run is now active on this chat', + code: 'SUPERSEDE_TARGET_MISMATCH', + activeRunId: result.activeRunId, + }); + case 'timeout': + // The target did not settle within W — nothing was persisted, the + // composer keeps the text. NOT a rollback: the stop is already issued. + throw new ConflictException({ + message: + 'The previous run did not stop in time; nothing was sent — please try again', + code: 'SUPERSEDE_TIMEOUT', + }); + case 'ready': + // The target stopped and settled: the slot is free. Prompt the new run + // that the old run's last operations may still be applying. + superseded = true; + break; + case 'degrade': + // The run already ended between click and POST — send normally. + break; + } + } + + // #487: one active run per chat — ENFORCED IN BOTH MODES now (legacy mode used + // to have NO gate, so two tabs streamed two parallel turns on one chat, which + // interleaved history and crashed convertToModelMessages). Reject a concurrent + // start with a clean pre-hijack 409 (double-submit / second-tab). A brand-new + // chat (no chatId) cannot have a prior run, and the DB partial unique index in + // beginRun is the authoritative backstop for any race that slips past here + // (including a slot stolen between a supersede release and beginRun). + if (body.chatId) { const active = await this.aiChatRunService.getActiveForChat( body.chatId, workspace.id, @@ -446,107 +500,94 @@ export class AiChatController { throw new ConflictException({ message: 'An agent run is already in progress for this chat', code: 'A_RUN_ALREADY_ACTIVE', + activeRunId: active.id, }); } } - // Run-lifecycle hooks (#184), only when the flag is on. They wrap the turn in - // a durable run whose abort is governed by the run (explicit stop), persist - // its progress, and settle its terminal status — see AiChatRunService. - const runHooks: AiChatRunHooks | undefined = autonomousRuns - ? { - begin: async (chatId) => { - const handle = await this.aiChatRunService.beginRun({ - chatId, - workspaceId: workspace.id, - userId: user.id, - trigger: 'user', - }); - // #184 phase 1.5: register the run-stream entry at BEGIN (before any - // frame) so a tab that attaches in the begin->seed window finds an - // entry to wait on. Gated on AI_CHAT_RESUMABLE_STREAM: with the flag - // off nothing is registered and attach always 204s. - if ( - handle?.runId && - this.environment?.isAiChatResumableStreamEnabled?.() - ) { - this.streamRegistry?.open(chatId, handle.runId); - } - return handle; - }, - onAssistantSeeded: (runId, messageId) => - this.aiChatRunService.linkAssistantMessage( - runId, - workspace.id, - messageId, - ), - onStep: (runId, stepCount) => - void this.aiChatRunService.recordStep( - runId, - workspace.id, - stepCount, - ), - onSettled: (runId, status, error) => - this.aiChatRunService.finalizeRun( - runId, - workspace.id, - status, - error, - ), + // #487: the turn is ALWAYS a first-class RUN now (both modes). The mode + // difference is only the abort semantics on a browser disconnect (onClose + // below). currentRunId is captured at begin so a legacy disconnect can stop + // the run through its stop lever. + let currentRunId: string | undefined; + const runHooks: AiChatRunHooks = { + begin: async (chatId) => { + const handle = await this.aiChatRunService.beginRun({ + chatId, + workspaceId: workspace.id, + userId: user.id, + trigger: 'user', + }); + currentRunId = handle?.runId; + // #184 phase 1.5: register the run-stream entry at BEGIN (before any + // frame) so a tab that attaches in the begin->seed window finds an entry + // to wait on. Gated on AI_CHAT_RESUMABLE_STREAM. + if ( + handle?.runId && + this.environment?.isAiChatResumableStreamEnabled?.() + ) { + this.streamRegistry?.open(chatId, handle.runId); } - : undefined; + return handle; + }, + onAssistantSeeded: (runId, messageId) => + this.aiChatRunService.linkAssistantMessage( + runId, + workspace.id, + messageId, + ), + onStep: (runId, stepCount) => + void this.aiChatRunService.recordStep(runId, workspace.id, stepCount), + onSettled: (runId, status, error) => + this.aiChatRunService.finalizeRun(runId, workspace.id, status, error), + }; - // Abort the agent loop when the client disconnects. `close` also fires on - // normal completion, so only abort when the response has not finished - // writing (a genuine disconnect). `once` fires at most once and self-removes; - // we also drop it on response `finish` so it never lingers after the stream - // completes normally (the AI SDK pipes the response fire-and-forget, so we - // cannot simply remove it once `stream()` returns). + // Handle a client disconnect. `close` also fires on normal completion, so only + // act when the response has not finished writing (a genuine disconnect). `once` + // fires at most once and self-removes; we also drop it on response `finish`. // DIAGNOSTIC (Safari stream-drop investigation) — temporary: wall-clock at // which a Safari disconnect is observed, measured from request receipt. const reqStartedAt = Date.now(); const controller = new AbortController(); const onClose = (): void => { - // A genuine disconnect leaves the response unfinished (unlike a normal - // completion, which also fires `close`). Such a drop — e.g. a reverse - // proxy cutting the SSE mid-answer — is otherwise invisible server-side, - // so log it here. if (!res.raw.writableEnded) { if (autonomousRuns) { - // #184: the turn is a DETACHED run. A disconnect must NOT abort it — - // the run keeps executing and persisting server-side; the client - // reconnects via /ai-chat/run (or re-stops via /ai-chat/stop). Log only. + // #184: a DETACHED run — a disconnect must NOT stop it. The run keeps + // executing and persisting server-side; the client reconnects via + // /ai-chat/run (or re-stops via /ai-chat/stop). Log only. this.logger.log( `AI chat stream: client disconnected; run continues server-side ` + `(elapsed=${Date.now() - reqStartedAt}ms since request received)`, ); } else { + // #487: legacy — a disconnect ENDS the turn, but the turn is now a RUN, + // so stop it through the run's stop lever (requestStop). streamText no + // longer consumes the socket signal (effectiveSignal is the run signal), + // so aborting `controller` would do nothing; requestStop aborts the run. this.logger.warn( - `AI chat stream: client disconnected before completion; aborting turn ` + - `(elapsed=${Date.now() - reqStartedAt}ms since request received)`, + `AI chat stream: client disconnected before completion; stopping the ` + + `run (elapsed=${Date.now() - reqStartedAt}ms since request received)`, ); - controller.abort(); + if (currentRunId) { + void this.aiChatRunService.requestStop(currentRunId, workspace.id); + } } } }; req.raw.once('close', onClose); res.raw.once('finish', () => req.raw.off('close', onClose)); - // #184: in detached mode the turn is NOT aborted on disconnect, so the SDK's - // pipe keeps writing to a socket the client may have dropped — for the rest of - // the (continuing) run. A write to the dead socket can emit an 'error' on the - // raw response; without a listener that surfaces as an unhandled error event. - // Swallow it (the run continues server-side regardless). Legacy mode aborts on - // disconnect, so it does not need this and keeps its exact prior behavior. - if (autonomousRuns) { - res.raw.on('error', (err) => { - this.logger.debug( - `AI chat detached stream: post-disconnect socket error swallowed: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - }); - } + // #184/#487: the run/pipe can outlive the socket in BOTH modes now (autonomous + // keeps going; legacy keeps going until requestStop's abort unwinds the turn). + // The SDK's pipe may then write to a dropped socket and emit an 'error' on the + // raw response — swallow it so it never surfaces as an unhandled error event. + res.raw.on('error', (err) => { + this.logger.debug( + `AI chat stream: post-disconnect socket error swallowed: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + }); // Commit to streaming: hijack so Fastify stops managing the response and // the AI SDK can write the UI-message stream directly to the Node socket. @@ -562,8 +603,10 @@ export class AiChatController { signal: controller.signal, model, role, - // #184: present only when the flag is on; wraps the turn in a durable run. + // #487: the turn is always run-wrapped now (both modes). runHooks, + // #487: warn the new run that a superseded run's last ops may still apply. + superseded, }); } catch (err) { // Any failure AFTER hijack can no longer go through Nest's exception diff --git a/apps/server/src/core/ai-chat/ai-chat.prompt.ts b/apps/server/src/core/ai-chat/ai-chat.prompt.ts index 5107a77d..33c222fd 100644 --- a/apps/server/src/core/ai-chat/ai-chat.prompt.ts +++ b/apps/server/src/core/ai-chat/ai-chat.prompt.ts @@ -101,6 +101,22 @@ const INTERRUPT_NOTE = 'assume your previous response was complete, and do not silently restart the ' + 'partial work — build on it or follow the new instruction.'; +/** + * #487: injected on a turn started by SUPERSEDING a previous run (the user hit + * "interrupt and send now" while a run was live). The previous run was Stopped, + * but there is NO side-effect quiescence — a write it had already committed, or + * one committing at the moment of Stop, may land with a small delay AFTER this new + * run starts. So the model is told its picture of the page/state may be a beat + * stale and to re-read before assuming an edit did or did not apply. + */ +const SUPERSEDE_NOTE = + 'NOTE: A previous agent run in this conversation was just interrupted so this ' + + 'new turn could start. That run was stopped, but any operation it had already ' + + 'begun (e.g. a page edit) may still be applied with a short delay. Do not ' + + 'assume the document/state is exactly as the interrupted run left it — if you ' + + 'need to rely on the current content, RE-READ it with the page tools before ' + + 'acting rather than trusting a cached view.'; + /** * Injected on a turn where the open page was hand-edited by the user (or anyone * else) AFTER the agent's previous response ended (#274). The server takes a @@ -203,6 +219,14 @@ export interface BuildSystemPromptInput { * (partial) answer was cut off by the user's new message. */ interrupted?: boolean; + /** + * #487: true when THIS turn was started by superseding a still-live previous run + * ("interrupt and send now"). Adds SUPERSEDE_NOTE so the model knows the previous + * run's last operations may still be applying and to re-read state it depends on. + * Distinct from `interrupted` (which is about a PARTIAL prior answer in history); + * both can be set together. Self-clears — set only for the superseding turn. + */ + superseded?: boolean; /** * Set only when the open page was edited by the user AFTER the agent's previous * turn ended (#274), confirmed server-side by diffing the current page against @@ -311,6 +335,7 @@ export function buildSystemPrompt({ openedPage, mcpInstructions, interrupted, + superseded, pageChanged, deferredToolsEnabled, toolCatalog, @@ -360,6 +385,13 @@ export function buildSystemPrompt({ context += `\n${INTERRUPT_NOTE}`; } + // Supersede note (#487): present only for a turn that stopped and replaced a + // still-live previous run — warns the model the previous run's last operations + // may still be applying (no side-effect quiescence). + if (superseded) { + context += `\n${SUPERSEDE_NOTE}`; + } + // Per-turn page-change note (#274). Added to the context section (inside the // safety sandwich), present only when the server detected that the open page // was edited by the user since the agent's last turn ended. The diff content is diff --git a/apps/server/src/core/ai-chat/ai-chat.service.ts b/apps/server/src/core/ai-chat/ai-chat.service.ts index 14826bd6..466b308b 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.ts @@ -379,6 +379,14 @@ export interface AiChatStreamBody { // it against persisted history (`isInterruptResume`) before injecting the // interrupt note, so a spoofed/stale flag on an ordinary turn is ignored. interrupted?: boolean; + // #487: server-side supersede CAS. When present, this POST asks the server to + // STOP the run `supersede.runId` (which the client saw as the chat's active run) + // and, once it has settled, start THIS turn in its place. The server validates + // the target against the chat and answers 400 (wrong chat) / 409 + // SUPERSEDE_TARGET_MISMATCH / 409 SUPERSEDE_TIMEOUT, or proceeds normally + // (degrade / ready). Absent => an ordinary send (rejected with 409 + // A_RUN_ALREADY_ACTIVE if a run is already active on the chat). + supersede?: { runId?: string } | null; // useChat sends the full UIMessage list; the last one is the new user turn. messages?: UIMessage[]; } @@ -428,6 +436,11 @@ export interface AiChatStreamArgs { // chat row (existing chat) or the request body (new chat). null => universal // assistant. Carried here so the turn never re-loads it. role: AiAgentRole | null; + // #487: true when this turn was started by SUPERSEDING a still-live previous run + // (the controller ran the supersede CAS to a `ready` result). Adds the + // SUPERSEDE_NOTE to the system prompt (the previous run's last ops may still be + // applying — no side-effect quiescence). Absent on an ordinary send. + superseded?: boolean; } /** @@ -727,6 +740,7 @@ export class AiChatService implements OnModuleInit { model, role, runHooks, + superseded, }: AiChatStreamArgs): Promise { // Resolve / create the chat. A new chat is created when no valid chatId is // supplied or the supplied one does not belong to this workspace. @@ -1040,6 +1054,9 @@ export class AiChatService implements OnModuleInit { // History-confirmed interrupt-resume flag (#198): adds the interrupt note // so the model treats the partial answer above as cut off, not finished. interrupted, + // #487: this turn superseded a still-live run — warn the model the + // previous run's last ops may still be applying (no quiescence). + superseded, // Detected between-turns human edit to the open page (#274): adds the // page_changed note + unified diff so the agent doesn't overwrite it. pageChanged, From 002c931c6b9ab3bfd14c2f2d356ec04b86391617 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 05:13:35 +0300 Subject: [PATCH 20/41] =?UTF-8?q?feat(ai-chat):=20=D1=80=D0=B5=D1=82=D1=80?= =?UTF-8?q?=D0=B0=D0=B8=20finalizeAssistant=20+=20owner-write=20=D0=BF?= =?UTF-8?q?=D1=80=D0=B8=D0=BE=D1=80=D0=B8=D1=82=D0=B5=D1=82=20+=20=D0=B4?= =?UTF-8?q?=D0=B2=D1=83=D1=81=D1=82=D0=BE=D1=80=D0=BE=D0=BD=D0=BD=D0=B8?= =?UTF-8?q?=D0=B9=20reconcile=20(#487)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Раньше finalizeAssistant ставил finalized=true ДО записи и не ретраил → один неудачный UPDATE = строка вечно 'streaming'; свип был boot-only; run-свип безусловный — асимметрия «run succeeded / message streaming навсегда». - finalizeAssistant: bounded-ретраи; once-гейт закрывается ТОЛЬКО после успешной записи; возвращает ok. Правило owner-write: терминальная запись owner'а условна на status='streaming' OR metadata.finalizeFailed (repo.finalizeOwner) — перетирает reconcile-штамп, но не проставленный терминал. ВСЕ status-only штампы reconcile (stampTerminalIfStreaming, sweepStreaming) пишут строго onlyIfStreaming И мёржат metadata.finalizeFailed:true (иначе поздний owner-write не перетрёт). - Порядок: попытка message-finalize → ран финализируется ВСЕГДА; при провале message onFinish помечает ран 'error' (не 'completed'). Ран не гейтится на message. - Периодический reconcile-джоб (setInterval, env-tunable) клаузами по порядку: (a) пере-драйв зомби; (b) message streaming + ран терминален → штамп по статусу рана (succeeded-ран + зависшая строка → 'aborted'+finalizeFailed, НЕ 'completed'-empty); (c) run running + НЕТ entry И НЕТ zombie + staleness → aborted (гейт «нет entry» первичный, staleness от last-progress updatedAt, X=max(2×per-call cap,15мин)+boot-warn); (d) message streaming + возраст>X + нет активной run-строки → aborted (двойной гейт). isInterruptResume исключает finalizeFailed-строки. Оппортунистический одно-чатовый reconcile при старте хода (best-effort, не фейлит ход). sweepStreaming boot-only → периодический. Тесты (реальная БД): owner finalizeOwner чистит finalizeFailed; штамп не перетирает терминал; поздний owner-write перетирает aborted-штамп; клаузы b/c/d (+живой entry не трогать, двойной гейт d); «убить БД на finish → после восстановления ни строка, ни ран не застряли». Юниты: finalizeFailed исключает interrupt-resume; reconcileStaleRuns «нет entry». Co-Authored-By: Claude Opus 4.8 (1M context) --- .../core/ai-chat/ai-chat-run.service.spec.ts | 32 ++ .../src/core/ai-chat/ai-chat-run.service.ts | 46 +++ .../ai-chat/ai-chat.controller.export.spec.ts | 13 +- .../ai-chat/ai-chat.service.run-race.spec.ts | 12 +- .../src/core/ai-chat/ai-chat.service.spec.ts | 51 ++- .../src/core/ai-chat/ai-chat.service.ts | 302 +++++++++++++++-- .../repos/ai-chat/ai-chat-message.repo.ts | 148 ++++++++- .../repos/ai-chat/ai-chat-run.repo.ts | 25 ++ .../integration/ai-chat-reconcile.int-spec.ts | 305 ++++++++++++++++++ 9 files changed, 889 insertions(+), 45 deletions(-) create mode 100644 apps/server/test/integration/ai-chat-reconcile.int-spec.ts diff --git a/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts b/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts index 1518dc73..ade4563c 100644 --- a/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat-run.service.spec.ts @@ -704,6 +704,38 @@ describe('#487 AiChatRunService.supersede (CAS)', () => { expect(await svc.supersede(chat, 'run-1', ws)).toEqual({ kind: 'degrade' }); }); + it('reconcileStaleRuns: aborts a stale run with NO entry/zombie; NEVER touches a live entry', async () => { + const finalizeIfActive = jest.fn(async () => ({ id: 'x', status: 'aborted' })); + const repo = makeRepo({ + insert: jest.fn(async (v: any) => ({ + id: 'live-1', + status: 'running', + chatId: v.chatId, + workspaceId: v.workspaceId, + })), + finalizeIfActive, + findStaleActive: jest.fn(async () => [ + { id: 'orphan-1', workspaceId: ws, chatId: 'c-orphan' }, + { id: 'live-1', workspaceId: ws, chatId: 'c-live' }, + ]), + }); + const svc = new AiChatRunService(repo as never, makeEnv() as never); + // A LIVE run this replica owns (in the `active` map). + await svc.beginRun({ chatId: 'c-live', workspaceId: ws, userId: 'u1' }); + expect(svc.isLocallyActive('live-1')).toBe(true); + + const aborted = await svc.reconcileStaleRuns(15 * 60 * 1000); + expect(aborted).toBe(1); + // The orphan (no entry) was aborted; the live entry was NEVER passed to the DB. + expect(finalizeIfActive).toHaveBeenCalledTimes(1); + expect(finalizeIfActive).toHaveBeenCalledWith( + 'orphan-1', + ws, + expect.objectContaining({ status: 'aborted' }), + ); + expect(svc.isLocallyActive('live-1')).toBe(true); + }); + it('gave-up zombie: supersede applies the intended status (settleZombie) then is ready', async () => { let healthy = false; let active: unknown = { diff --git a/apps/server/src/core/ai-chat/ai-chat-run.service.ts b/apps/server/src/core/ai-chat/ai-chat-run.service.ts index 8fd577c8..1530c3b5 100644 --- a/apps/server/src/core/ai-chat/ai-chat-run.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat-run.service.ts @@ -507,6 +507,52 @@ export class AiChatRunService implements OnModuleInit { } } + /** + * #487 reconcile clause (c): abort runs the DB still shows active (pending| + * running) but that this replica does NOT own — NO live entry AND NO zombie — + * and that have been UNTOUCHED past `staleMs` (from last-progress `updated_at`, + * NOT startedAt, so a legit long marathon is never a candidate). "No entry" is + * the PRIMARY gate: a live entry (an actively-executing run on this replica) is + * NEVER aborted, whatever its age. Returns the number aborted. Best-effort — + * never throws (a periodic-job failure must not crash the process). + */ + async reconcileStaleRuns(staleMs: number): Promise { + let candidates: Array<{ id: string; workspaceId: string; chatId: string }>; + try { + candidates = await this.runRepo.findStaleActive(staleMs); + } catch (err) { + this.logger.warn( + `Reconcile (stale runs) query failed: ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + return 0; + } + let aborted = 0; + for (const c of candidates) { + // PRIMARY gate: never touch a live entry, and never race a zombie we are + // already re-driving (settleZombie owns those). + if (this.active.has(c.id) || this.zombies.has(c.id)) continue; + try { + const row = await this.runRepo.finalizeIfActive(c.id, c.workspaceId, { + status: 'aborted', + error: 'Run aborted by reconcile: no live runner (stale).', + }); + if (row) { + aborted += 1; + this.settled.add(c.id); + } + } catch (err) { + this.logger.warn( + `Reconcile abort of stale run ${c.id} failed: ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + } + } + return aborted; + } + /** * #487: the run's settle outcome as seen by THIS replica, or undefined when it * has no record (the caller then reads the row — the DB is the source of truth). diff --git a/apps/server/src/core/ai-chat/ai-chat.controller.export.spec.ts b/apps/server/src/core/ai-chat/ai-chat.controller.export.spec.ts index bfbc64c0..6afefc61 100644 --- a/apps/server/src/core/ai-chat/ai-chat.controller.export.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.controller.export.spec.ts @@ -115,7 +115,7 @@ describe('finalizeAssistant dispatch (planFinalizeAssistant + applyFinalize)', ( // Drive the SAME applyFinalize the service calls (no duplicated logic). async function dispatchFinalize( - repo: { insert: jest.Mock; update: jest.Mock }, + repo: { insert: jest.Mock; finalizeOwner: jest.Mock }, assistantId: string | undefined, flushed: AssistantFlush, ): Promise { @@ -135,21 +135,22 @@ describe('finalizeAssistant dispatch (planFinalizeAssistant + applyFinalize)', ( expect(planFinalizeAssistant(undefined)).toEqual({ kind: 'insert' }); }); - it('(a) upfront insert succeeded -> finalize UPDATEs the row by id', async () => { - const repo = { insert: jest.fn(), update: jest.fn() }; + it('(a) upfront insert succeeded -> finalize CONDITIONALLY updates the row by id (#487 owner-write)', async () => { + const repo = { insert: jest.fn(), finalizeOwner: jest.fn() }; const flushed = flushAssistant([], 'final answer', 'completed', { finishReason: 'stop', }); await dispatchFinalize(repo, 'a1', flushed); - expect(repo.update).toHaveBeenCalledWith('a1', workspaceId, flushed); + // #487: the owner write is the CONDITIONAL finalizeOwner, not a raw update. + expect(repo.finalizeOwner).toHaveBeenCalledWith('a1', workspaceId, flushed); expect(repo.insert).not.toHaveBeenCalled(); }); it('(b) upfront insert failed -> finalize INSERTs the terminal payload', async () => { - const repo = { insert: jest.fn(), update: jest.fn() }; + const repo = { insert: jest.fn(), finalizeOwner: jest.fn() }; const flushed = flushAssistant([], 'partial', 'error', { error: 'boom' }); await dispatchFinalize(repo, undefined, flushed); - expect(repo.update).not.toHaveBeenCalled(); + expect(repo.finalizeOwner).not.toHaveBeenCalled(); expect(repo.insert).toHaveBeenCalledTimes(1); const arg = repo.insert.mock.calls[0][0]; // The fallback insert carries the terminal content/status/metadata. diff --git a/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts index d167abbe..8b170c11 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts @@ -155,6 +155,8 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => { insert: jest.fn(async () => ({ id: 'msg-1' })), findAllByChat: jest.fn(async () => []), update: jest.fn(async () => ({ id: 'msg-1' })), + finalizeOwner: jest.fn(async () => ({ id: 'msg-1' })), + findStreamingWithTerminalRun: jest.fn(async () => []), }; const aiSettings = { resolve: jest.fn(async () => ({})) }; const tools = { forUser: jest.fn(async () => ({})) }; @@ -332,7 +334,13 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => { usage: {}, steps: [], }); - expect(runHooks.onSettled).toHaveBeenCalledWith('run-1', 'completed'); + // #487: onFinish passes the (undefined) error slot so a message-finalize + // failure could error-mark the run; on the success path it is undefined. + expect(runHooks.onSettled).toHaveBeenCalledWith( + 'run-1', + 'completed', + undefined, + ); }); it('F9: onAbort settles the run "aborted"', async () => { @@ -415,6 +423,8 @@ describe('AiChatService.stream — begin-failure fails the turn (#184 F14 / #486 insert: jest.fn(async () => ({ id: 'msg-1' })), findAllByChat: jest.fn(async () => []), update: jest.fn(async () => ({ id: 'msg-1' })), + finalizeOwner: jest.fn(async () => ({ id: 'msg-1' })), + findStreamingWithTerminalRun: jest.fn(async () => []), }; const aiSettings = { resolve: jest.fn(async () => ({})) }; const tools = { forUser: jest.fn(async () => ({})) }; diff --git a/apps/server/src/core/ai-chat/ai-chat.service.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.spec.ts index 46e37f61..cdfa80e9 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.spec.ts @@ -1315,8 +1315,12 @@ describe('AiChatService page-change lifecycle (#274)', () => { describe('isInterruptResume', () => { // history tail is the just-inserted user row; [len-2] is the previous turn. const withPrev = ( - prev: { role: string; status?: string | null } | null, - ): Array<{ role: string; status?: string | null }> => + prev: { + role: string; + status?: string | null; + metadata?: unknown; + } | null, + ): Array<{ role: string; status?: string | null; metadata?: unknown }> => prev ? [prev, { role: 'user', status: null }] : [{ role: 'user', status: null }]; @@ -1357,6 +1361,33 @@ describe('isInterruptResume', () => { it('false when there is no preceding turn (only the new user row)', () => { expect(isInterruptResume(withPrev(null), true)).toBe(false); }); + + it('#487 EXCLUDES a reconcile stamp (finalizeFailed) — not a genuine interruption', () => { + // A row a reconcile settled to 'aborted' carries metadata.finalizeFailed. It + // must NOT be treated as an interrupt-resume (that would inject a false + // "you were interrupted" note), even though its status is 'aborted'. + expect( + isInterruptResume( + withPrev({ + role: 'assistant', + status: 'aborted', + metadata: { finalizeFailed: true }, + }), + true, + ), + ).toBe(false); + // A genuine abort (no finalizeFailed) still counts. + expect( + isInterruptResume( + withPrev({ + role: 'assistant', + status: 'aborted', + metadata: { parts: [] }, + }), + true, + ), + ).toBe(true); + }); }); /** @@ -1419,6 +1450,9 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', () insert: jest.fn(async () => ({ id: 'msg-1' })), findAllByChat: jest.fn(async () => []), update: jest.fn(async () => ({ id: 'msg-1' })), + // #487: the terminal owner-write + the opportunistic reconcile query. + finalizeOwner: jest.fn(async () => ({ id: 'msg-1' })), + findStreamingWithTerminalRun: jest.fn(async () => []), }; const aiSettings = { resolve: jest.fn(async () => ({})) }; const tools = { forUser: jest.fn(async () => ({})) }; @@ -1623,6 +1657,19 @@ describe('AiChatService.stream — token-degeneration reaction (#444)', () => { return { id }; }, ), + // #487: the terminal owner-write records into the SAME `updated` recorder so + // assertions on the terminal 'completed'/'error'/'aborted' write still hold. + finalizeOwner: jest.fn( + async ( + id: string, + workspaceId: string, + patch: Record, + ) => { + updated.push({ id, workspaceId, patch }); + return { id }; + }, + ), + findStreamingWithTerminalRun: jest.fn(async () => []), }; const aiSettings = { resolve: jest.fn(async () => ({})) }; const tools = { forUser: jest.fn(async () => ({})) }; diff --git a/apps/server/src/core/ai-chat/ai-chat.service.ts b/apps/server/src/core/ai-chat/ai-chat.service.ts index 466b308b..959d2d44 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.ts @@ -3,6 +3,7 @@ import { ForbiddenException, Injectable, Logger, + OnModuleDestroy, OnModuleInit, ServiceUnavailableException, } from '@nestjs/common'; @@ -42,7 +43,11 @@ import { makeLoadToolsTool, buildExternalToolCatalog, } from './tools/tool-tiers'; -import { RunAlreadyActiveError } from './ai-chat-run.service'; +import { + RunAlreadyActiveError, + AiChatRunService, +} from './ai-chat-run.service'; +import { inAppToolCallCapMs } from './tools/ai-chat-tools.service'; import { computePageChange } from './page-change/page-change.util'; import { sanitizeSelection, @@ -250,15 +255,23 @@ export function cleanGeneratedTitle(text: string): string { * partial output is already in history thanks to the step-granular write path). */ export function isInterruptResume( - history: Array<{ role: string; status?: string | null }>, + history: Array<{ + role: string; + status?: string | null; + metadata?: unknown; + }>, clientInterrupted: boolean | undefined, ): boolean { if (clientInterrupted !== true) return false; const prev = history[history.length - 2]; - return ( - prev?.role === 'assistant' && - (prev.status === 'aborted' || prev.status === 'streaming') - ); + if (prev?.role !== 'assistant') return false; + // #487: a reconcile STAMP (metadata.finalizeFailed) is NOT a genuine user + // interruption — the previous turn's process died and a reconcile settled the + // row as 'aborted'. Treating it as an interrupt-resume would inject a false + // "you were interrupted" note. Exclude any finalizeFailed row. + const meta = prev.metadata as { finalizeFailed?: unknown } | null | undefined; + if (meta && meta.finalizeFailed === true) return false; + return prev.status === 'aborted' || prev.status === 'streaming'; } /** @@ -457,7 +470,7 @@ export interface AiChatStreamArgs { * can be rebuilt for `convertToModelMessages`. */ @Injectable() -export class AiChatService implements OnModuleInit { +export class AiChatService implements OnModuleInit, OnModuleDestroy { private readonly logger = new Logger(AiChatService.name); constructor( @@ -478,8 +491,17 @@ export class AiChatService implements OnModuleInit { // constructions (int-specs) compile unchanged; Nest always injects the real // provider in production. Only ever touched on the run-wrapped + flag-on path. private readonly streamRegistry?: AiChatStreamRegistryService, + // #487: the run lifecycle service, for the periodic + opportunistic reconcile + // (zombie re-drive + stale-run abort). OPTIONAL so positional test + // constructions compile unchanged; Nest always injects the real singleton, so + // reconcile sees the SAME in-memory active/zombie maps the runner mutates. + private readonly aiChatRunService?: AiChatRunService, ) {} + // #487: periodic reconcile timer (single-process phase 1). Started in + // onModuleInit, cleared in onModuleDestroy. + private reconcileTimer?: ReturnType; + /** * Crash-recovery sweep on server start (#183): any assistant row left in the * 'streaming' state is the relic of a turn whose process died before it @@ -504,6 +526,158 @@ export class AiChatService implements OnModuleInit { }`, ); } + + // #487: start the PERIODIC reconcile (was boot-only). It heals both directions + // of the run<->message lifecycle asymmetry that a boot sweep alone left to the + // NEXT restart. Single-process phase 1: the in-memory active/zombie maps are + // authoritative, so "no live entry" is a safe primary gate. + const staleMs = this.reconcileStalenessMs(); + // boot-warn if the per-call cap is configured so high the derived staleness is + // unusually long (a stale run then lingers longer before reconcile aborts it). + if (staleMs > 30 * 60 * 1000) { + this.logger.warn( + `#487 reconcile staleness is ${Math.round(staleMs / 60000)}min ` + + `(derived from max(2 x per-call cap, 15min)); a per-call cap this high ` + + `delays stale-run recovery. Review AI_CHAT_INAPP_TOOL_CALL_CAP_MS.`, + ); + } + const intervalMs = this.reconcileIntervalMs(); + this.reconcileTimer = setInterval(() => { + void this.reconcile().catch((err) => { + this.logger.warn( + `Periodic reconcile failed: ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + }); + }, intervalMs); + this.reconcileTimer.unref?.(); + } + + /** #487: stop the periodic reconcile timer on shutdown. */ + onModuleDestroy(): void { + if (this.reconcileTimer) { + clearInterval(this.reconcileTimer); + this.reconcileTimer = undefined; + } + } + + /** + * #487: reconcile staleness threshold X — a run/message is only a "no live + * runner" abort candidate once UNTOUCHED past this. Derived as + * max(2 x per-call cap, 15min): 2x the longest legitimate single tool call plus + * a floor, so a marathon turn making steady progress (updatedAt bumped each + * step) is never swept. + */ + private reconcileStalenessMs(): number { + return Math.max(2 * inAppToolCallCapMs(), 15 * 60 * 1000); + } + + /** #487: how often the periodic reconcile runs (env-tunable, default 2min). */ + private reconcileIntervalMs(): number { + const raw = Number(process.env.AI_CHAT_RECONCILE_INTERVAL_MS); + return Number.isFinite(raw) && raw > 0 ? raw : 2 * 60 * 1000; + } + + /** + * #487: the periodic BIDIRECTIONAL reconcile. Runs the clauses IN ORDER; each is + * best-effort (a failure of one never blocks the others). Single-process phase 1 + * — the run service's in-memory maps are authoritative for "live entry". + * + * (a) re-drive ZOMBIE runs (a terminal write that gave up) — apply the intended + * status via the conditional UPDATE; + * (b) message 'streaming' + its RUN terminal -> stamp the message by the run's + * status (succeeded-run + stuck row -> 'aborted'+finalizeFailed, NOT + * 'completed' with empty parts — the final text lived only in the dead + * process's memory, a documented loss); + * (c) run active + NO live entry + NO zombie + stale -> aborted (the run + * service applies the "no entry" primary gate + last-progress staleness); + * (d) message 'streaming' + age>X + NO active run on the chat -> aborted + * (historical-row safety, double-gated). + */ + async reconcile(): Promise { + const staleMs = this.reconcileStalenessMs(); + + // (a) zombie re-drive. + if (this.aiChatRunService) { + for (const runId of this.aiChatRunService.zombieRunIds()) { + try { + await this.aiChatRunService.settleZombie(runId); + } catch (err) { + this.logger.warn( + `Reconcile (a) zombie ${runId} re-drive failed: ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + } + } + } + + // (b) message streaming + run terminal -> stamp message by run status. + try { + const stuck = await this.aiChatMessageRepo.findStreamingWithTerminalRun(); + for (const s of stuck) { + // succeeded-run -> 'aborted' (NOT 'completed'-empty); failed -> 'error'; + // aborted -> 'aborted'. All via the finalizeFailed stamp. + const status = s.runStatus === 'failed' ? 'error' : 'aborted'; + await this.aiChatMessageRepo.stampTerminalIfStreaming( + s.messageId, + s.workspaceId, + status, + ); + } + } catch (err) { + this.logger.warn( + `Reconcile (b) message<-run failed: ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + } + + // (c) stale active run with no live runner -> aborted. + if (this.aiChatRunService) { + try { + await this.aiChatRunService.reconcileStaleRuns(staleMs); + } catch (err) { + this.logger.warn( + `Reconcile (c) stale-run abort failed: ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + } + } + + // (d) historical streaming row, no active run on the chat, stale -> aborted. + try { + await this.aiChatMessageRepo.sweepStreamingWithoutActiveRun(staleMs); + } catch (err) { + this.logger.warn( + `Reconcile (d) historical-row sweep failed: ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + } + } + + /** + * #487: OPPORTUNISTIC single-chat reconcile at the start of a turn (beginRun / + * supersede path), so a user who returns to a chat with a stuck streaming row + * (its run already terminal) sees it settled WITHOUT waiting for the periodic + * job. Best-effort — a failure NEVER fails the turn (swallowed by the caller). + */ + async reconcileChat(chatId: string, workspaceId: string): Promise { + const stuck = await this.aiChatMessageRepo.findStreamingWithTerminalRun(50, { + chatId, + workspaceId, + }); + for (const s of stuck) { + const status = s.runStatus === 'failed' ? 'error' : 'aborted'; + await this.aiChatMessageRepo.stampTerminalIfStreaming( + s.messageId, + s.workspaceId, + status, + ); + } } /** @@ -849,6 +1023,20 @@ export class AiChatService implements OnModuleInit { } } + // #487: opportunistic single-chat reconcile — settle any streaming row on this + // chat whose run is already terminal BEFORE this turn's history load, so the + // user never waits on the periodic job and the new turn's model history is not + // polluted by a stuck 'streaming' row. Best-effort: it must NEVER fail the turn. + try { + await this.reconcileChat(chatId, workspace.id); + } catch (err) { + this.logger.debug( + `Opportunistic reconcile for chat ${chatId} failed (ignored): ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + } + try { // Extract the incoming user turn (the last user message from useChat). const incoming = lastUserMessage(body.messages); @@ -1211,29 +1399,59 @@ export class AiChatService implements OnModuleInit { // callbacks — mirroring the pre-#183 persist-at-most-once guard for the // TERMINAL status (the row may be updated many times with 'streaming' before // this fires once). + // #487: the once-gate closes ONLY AFTER a successful write, and the write is + // BOUNDED-RETRIED. Previously `finalized` was set BEFORE the write and never + // retried, so a single failed UPDATE stranded the row 'streaming' forever + // (the boot-only sweep was the only recovery). Now a transient blip is ridden + // out in place; a total give-up leaves the gate OPEN and logs, and the + // periodic reconcile (clauses b/d) later settles the row. Returns whether the + // terminal write LANDED, so the caller can error-mark the RUN on a message + // failure (the run is finalized regardless — never gated on the message). let finalized = false; + const FINALIZE_MSG_MAX_ATTEMPTS = 3; const finalizeAssistant = async ( flushed: AssistantFlush, - ): Promise => { - if (finalized) return; - finalized = true; + ): Promise => { + if (finalized) return true; const plan = planFinalizeAssistant(assistantId); - try { - // Shared dispatch (see applyFinalize): UPDATE the upfront row, or — when - // the upfront insert failed (kind 'insert') — INSERT the terminal row as - // the only safety against losing the turn entirely. - await applyFinalize( - this.aiChatMessageRepo, - plan, - { chatId, workspaceId: workspace.id, userId: user.id }, - flushed, - ); - } catch (err) { - this.logger.error( - `Failed to finalize assistant message (kind=${plan.kind})`, - err as Error, - ); + let lastError: unknown; + for (let attempt = 1; attempt <= FINALIZE_MSG_MAX_ATTEMPTS; attempt++) { + try { + // Shared dispatch (see applyFinalize): conditionally UPDATE the upfront + // row (owner-write priority), or — when the upfront insert failed (kind + // 'insert') — INSERT the terminal row as the only safety against losing + // the turn entirely. + await applyFinalize( + this.aiChatMessageRepo, + plan, + { chatId, workspaceId: workspace.id, userId: user.id }, + flushed, + ); + finalized = true; // gate closes ONLY after a successful write + return true; + } catch (err) { + lastError = err; + this.logger.warn( + `Assistant message finalize attempt ${attempt}/${FINALIZE_MSG_MAX_ATTEMPTS} ` + + `failed (kind=${plan.kind}): ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + if (attempt < FINALIZE_MSG_MAX_ATTEMPTS) { + await new Promise((r) => setTimeout(r, 50 * attempt)); + } + } } + // Gave up: leave the gate OPEN (no in-process second settler exists — the + // terminal callbacks are mutually exclusive) and log. The periodic reconcile + // settles the stranded row; a late owner-write is impossible for this turn, + // so the reconcile stamp (aborted+finalizeFailed) is the final state. + this.logger.error( + `Assistant message finalize GAVE UP after ${FINALIZE_MSG_MAX_ATTEMPTS} ` + + `attempts (row left 'streaming', chat ${chatId}); reconcile will settle it`, + lastError as Error, + ); + return false; }; // DIAGNOSTIC (Safari stream-drop investigation) — temporary. Measure @@ -1378,7 +1596,7 @@ export class AiChatService implements OnModuleInit { const stepExhausted = steps.length >= MAX_AGENT_STEPS; const emptyTurnMarker = !producedText && stepExhausted ? STEP_LIMIT_NO_ANSWER_MARKER : ''; - await finalizeAssistant( + const msgOk = await finalizeAssistant( flushAssistant(steps as StepLike[], emptyTurnMarker, 'completed', { finishReason: finishReason as string, usage: totalUsage as StreamUsage, @@ -1392,9 +1610,19 @@ export class AiChatService implements OnModuleInit { pageChanged, }), ); - // #184: settle the RUN as succeeded (best-effort, after the projection - // is finalized above). - if (runId) await runHooks?.onSettled?.(runId, 'completed'); + // #184/#487: the RUN is finalized ALWAYS (never gated on the message). + // If the message finalize GAVE UP, error-mark the run so the asymmetry + // "run succeeded / message streaming forever" cannot arise; the + // periodic reconcile then settles the stuck message from this run. + if (runId) { + await runHooks?.onSettled?.( + runId, + msgOk ? 'completed' : 'error', + msgOk + ? undefined + : 'Assistant message could not be persisted (finalize failed).', + ); + } // Lifecycle: release the external MCP clients leased for this turn. await closeExternalClients(); @@ -2135,7 +2363,10 @@ export function planFinalizeAssistant( * a test mock both satisfy it). */ export interface FinalizeRepo { insert(insertable: Record): Promise; - update( + // #487: the OWNER terminal write is CONDITIONAL (status='streaming' OR + // metadata.finalizeFailed) so the owner overwrites a reconcile stamp but never + // an already-proper terminal row (owner-write priority). + finalizeOwner( id: string, workspaceId: string, patch: AssistantFlush, @@ -2144,10 +2375,11 @@ export interface FinalizeRepo { /** * Apply a finalize `plan` to the repo with the terminal `flushed` payload (#183): - * UPDATE the upfront row, or INSERT a fresh terminal row as the fallback when the - * upfront insert failed. The SINGLE dispatch shared by the service's - * finalizeAssistant and its test, so the test exercises the real path instead of - * a copy (#186 review). Pure of error handling — the caller wraps it. + * conditionally UPDATE the upfront row (owner-write priority, #487), or INSERT a + * fresh terminal row as the fallback when the upfront insert failed. The SINGLE + * dispatch shared by the service's finalizeAssistant and its test, so the test + * exercises the real path instead of a copy (#186 review). Pure of error + * handling — the caller wraps it (and RETRIES it, #487). */ export async function applyFinalize( repo: FinalizeRepo, @@ -2156,7 +2388,7 @@ export async function applyFinalize( flushed: AssistantFlush, ): Promise { if (plan.kind === 'update') { - await repo.update(plan.id, base.workspaceId, flushed); + await repo.finalizeOwner(plan.id, base.workspaceId, flushed); return; } await repo.insert({ diff --git a/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.ts b/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.ts index 78cc7064..96070e77 100644 --- a/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.ts +++ b/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.ts @@ -1,5 +1,6 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectKysely } from 'nestjs-kysely'; +import { sql } from 'kysely'; import { KyselyDB, KyselyTransaction } from '../../types/kysely.types'; import { dbOrTx } from '../../utils'; import { @@ -188,6 +189,144 @@ export class AiChatMessageRepo { return query.returning(this.baseFields).executeTakeFirst(); } + /** + * #487 OWNER terminal write — the streamText terminal callback's finalize. Like + * `update` but CONDITIONAL on `status='streaming' OR metadata.finalizeFailed`: + * the owner writes its real content EITHER when the row is still streaming (the + * normal case) OR when a reconcile stamp already flipped it to a terminal status + * but marked `finalizeFailed:true` — the owner's real content OVERWRITES that + * placeholder stamp (owner-write priority, #487). A row that is properly terminal + * (no finalizeFailed) is left untouched (undefined) — idempotent. The `patch` + * carries the real metadata WITHOUT finalizeFailed, so a successful write CLEARS + * the flag. Returns the updated row, or undefined when nothing matched. + */ + async finalizeOwner( + id: string, + workspaceId: string, + patch: Partial<{ + content: string | null; + toolCalls: unknown; + metadata: unknown; + status: string | null; + }>, + trx?: KyselyTransaction, + ): Promise { + const db = dbOrTx(this.db, trx); + return db + .updateTable('aiChatMessages') + .set({ ...(patch as Record), updatedAt: new Date() }) + .where('id', '=', id) + .where('workspaceId', '=', workspaceId) + .where((eb) => + eb.or([ + eb('status', '=', 'streaming'), + eb(sql`(metadata->>'finalizeFailed')`, '=', 'true'), + ]), + ) + .returning(this.baseFields) + .executeTakeFirst(); + } + + /** + * #487 RECONCILE status-only stamp — settle a stuck 'streaming' row to a + * terminal status WITHOUT the owner's real content (which lived only in the + * dead process's memory — a documented loss). CONDITIONAL on `status='streaming'` + * (never touches an already-terminal row) AND it MERGES `finalizeFailed:true` + * into metadata (preserving the partial `parts` already persisted) so a LATER + * owner-write (finalizeOwner) can still OVERWRITE this placeholder with real + * content, and so `isInterruptResume` can EXCLUDE this row (a reconcile stamp is + * not a genuine user interruption). Returns the updated row, or undefined. + */ + async stampTerminalIfStreaming( + id: string, + workspaceId: string, + status: 'aborted' | 'error' | 'completed', + trx?: KyselyTransaction, + ): Promise { + const db = dbOrTx(this.db, trx); + return db + .updateTable('aiChatMessages') + .set({ + status, + metadata: sql`coalesce(metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`, + updatedAt: new Date(), + }) + .where('id', '=', id) + .where('workspaceId', '=', workspaceId) + .where('status', '=', 'streaming') + .returning(this.baseFields) + .executeTakeFirst(); + } + + /** + * #487 reconcile clause (b): streaming assistant rows whose linked RUN has + * already reached a terminal status — an asymmetry ("run settled / message + * streaming forever") the periodic reconcile heals by stamping the message. + * Returns the message id + its run's terminal status, bounded. + */ + async findStreamingWithTerminalRun( + limit = 200, + // #487: scope to ONE chat for the opportunistic per-turn reconcile (removes + // reconcile latency from the user-visible path); omit for the periodic sweep. + chat?: { chatId: string; workspaceId: string }, + ): Promise< + Array<{ messageId: string; workspaceId: string; runStatus: string }> + > { + let query = this.db + .selectFrom('aiChatMessages as m') + .innerJoin('aiChatRuns as r', 'r.assistantMessageId', 'm.id') + .select([ + 'm.id as messageId', + 'm.workspaceId as workspaceId', + 'r.status as runStatus', + ]) + .where('m.status', '=', 'streaming') + .where('r.status', 'in', ['succeeded', 'failed', 'aborted']); + if (chat) { + query = query + .where('m.chatId', '=', chat.chatId) + .where('m.workspaceId', '=', chat.workspaceId); + } + return query.limit(limit).execute(); + } + + /** + * #487 reconcile clause (d) — historical-row safety: streaming rows older than + * `staleMs` whose chat has NO active run row (double-gated). Settle them to + * 'aborted' + finalizeFailed (so a late owner-write could still overwrite). + * Returns the count. Used ONLY by the periodic reconcile, never at boot. + */ + async sweepStreamingWithoutActiveRun( + staleMs: number, + trx?: KyselyTransaction, + ): Promise { + const db = dbOrTx(this.db, trx); + const staleBefore = new Date(Date.now() - staleMs); + const rows = await db + .updateTable('aiChatMessages as m') + .set({ + status: 'aborted', + metadata: sql`coalesce(m.metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`, + updatedAt: new Date(), + }) + .where('m.status', '=', 'streaming') + .where('m.updatedAt', '<', staleBefore) + .where((eb) => + eb.not( + eb.exists( + eb + .selectFrom('aiChatRuns as r') + .select('r.id') + .whereRef('r.chatId', '=', 'm.chatId') + .where('r.status', 'in', ['pending', 'running']), + ), + ), + ) + .returning('m.id') + .execute(); + return rows.length; + } + /** * Crash-recovery sweep (#183): flip every assistant row still left in the * 'streaming' state (a turn that died mid-write before reaching a terminal @@ -200,13 +339,20 @@ export class AiChatMessageRepo { * step, so an actively-streaming row never matches; this prevents a fresh * replica's boot-sweep from aborting a turn another replica is still streaming * in a multi-instance deploy. + * + * #487: the sweep now ALSO marks `finalizeFailed:true` so a late owner-write can + * overwrite this placeholder with real content (owner-write priority). */ async sweepStreaming(trx?: KyselyTransaction): Promise { const db = dbOrTx(this.db, trx); const staleBefore = new Date(Date.now() - SWEEP_STREAMING_STALE_MS); const rows = await db .updateTable('aiChatMessages') - .set({ status: 'aborted', updatedAt: new Date() }) + .set({ + status: 'aborted', + metadata: sql`coalesce(metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`, + updatedAt: new Date(), + }) .where('status', '=', 'streaming') .where('updatedAt', '<', staleBefore) .returning('id') diff --git a/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts b/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts index e7976ebe..c27480bf 100644 --- a/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts +++ b/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts @@ -219,6 +219,31 @@ export class AiChatRunRepo { * sweeps only runs UNTOUCHED past the window. Phase 1 is single-process, so the * boot path supplies no window. */ + /** + * #487 reconcile clause (c): active (pending|running) runs UNTOUCHED past + * `staleMs` — candidates for "no live runner" abort. Staleness is measured from + * `updated_at` (the LAST-PROGRESS timestamp — recordStep bumps it), NOT + * `started_at`, so a legitimate long-running marathon (11–25 min of steady + * progress) is never a candidate. The caller filters these against its in-memory + * `active` / zombie maps ("no entry" is the PRIMARY gate — a live entry is never + * aborted) before settling any of them. Bounded. + */ + async findStaleActive( + staleMs: number, + limit = 200, + trx?: KyselyTransaction, + ): Promise> { + const db = dbOrTx(this.db, trx); + const staleBefore = new Date(Date.now() - staleMs); + return db + .selectFrom('aiChatRuns') + .select(['id', 'workspaceId', 'chatId']) + .where('status', 'in', ACTIVE_RUN_STATUSES as unknown as string[]) + .where('updatedAt', '<', staleBefore) + .limit(limit) + .execute(); + } + async sweepRunning( opts: { staleMs?: number } = {}, trx?: KyselyTransaction, diff --git a/apps/server/test/integration/ai-chat-reconcile.int-spec.ts b/apps/server/test/integration/ai-chat-reconcile.int-spec.ts new file mode 100644 index 00000000..4a38f1c2 --- /dev/null +++ b/apps/server/test/integration/ai-chat-reconcile.int-spec.ts @@ -0,0 +1,305 @@ +import { Kysely } from 'kysely'; +import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo'; +import { AiChatRunRepo } from '@docmost/db/repos/ai-chat/ai-chat-run.repo'; +import { AiChatRunService } from '../../src/core/ai-chat/ai-chat-run.service'; +import { + getTestDb, + destroyTestDb, + createWorkspace, + createUser, + createChat, + createMessage, +} from './db'; + +/** + * #487 commit 4 — bidirectional reconcile + owner-write priority, real SQL. + * + * Proves the OBSERVABLE recovery properties against docmost_test: + * - the CONDITIONAL owner-write beats a reconcile stamp, and a stamp never + * clobbers a proper terminal row; + * - a LATE owner-finalize with real content OVERWRITES a reconcile 'aborted' + * stamp (finalizeFailed); + * - each reconcile clause (b message<-run, c stale-run, d historical row) settles + * the stuck row/run, and a LIVE run entry is never touched; + * - the "kill DB on finish" recovery: after the DB comes back, neither the + * message row nor the run row stays stuck. + */ +describe('#487 reconcile + owner-write priority [integration]', () => { + let db: Kysely; + let messageRepo: AiChatMessageRepo; + let runRepo: AiChatRunRepo; + let runService: AiChatRunService; + let workspaceId: string; + let userId: string; + + beforeAll(async () => { + db = getTestDb(); + messageRepo = new AiChatMessageRepo(db as any); + runRepo = new AiChatRunRepo(db as any); + runService = new AiChatRunService(runRepo, { isCloud: () => false } as never); + workspaceId = (await createWorkspace(db)).id; + userId = (await createUser(db, workspaceId)).id; + }); + + afterAll(async () => { + await destroyTestDb(); + }); + + const newChat = async () => + (await createChat(db, { workspaceId, creatorId: userId })).id; + + const metaOf = async (id: string): Promise | null> => { + const row = await messageRepo.findById(id, workspaceId); + return (row?.metadata as Record | null) ?? null; + }; + + it('owner finalizeOwner writes a streaming row and CLEARS finalizeFailed', async () => { + const chatId = await newChat(); + const m = await createMessage(db, { + workspaceId, + chatId, + role: 'assistant', + status: 'streaming', + metadata: { parts: [] }, + }); + const wrote = await messageRepo.finalizeOwner(m.id, workspaceId, { + content: 'final answer', + status: 'completed', + metadata: { parts: [{ type: 'text', text: 'final answer' }] }, + } as never); + expect(wrote!.status).toBe('completed'); + expect((await metaOf(m.id))?.finalizeFailed).toBeUndefined(); + }); + + it('a reconcile stamp NEVER clobbers a proper terminal row (finalizeOwner is a no-op there)', async () => { + const chatId = await newChat(); + const m = await createMessage(db, { + workspaceId, + chatId, + role: 'assistant', + status: 'completed', + content: 'real', + metadata: { parts: [] }, + }); + // The reconcile stamp is onlyIfStreaming -> no-op on a completed row. + const stamped = await messageRepo.stampTerminalIfStreaming( + m.id, + workspaceId, + 'aborted', + ); + expect(stamped).toBeUndefined(); + expect((await messageRepo.findById(m.id, workspaceId))!.status).toBe( + 'completed', + ); + }); + + it('LATE owner-finalize with real content OVERWRITES a reconcile aborted stamp', async () => { + const chatId = await newChat(); + const m = await createMessage(db, { + workspaceId, + chatId, + role: 'assistant', + status: 'streaming', + metadata: { parts: [{ type: 'text', text: 'partial' }] }, + }); + // Reconcile stamps it aborted + finalizeFailed (final text lived only in mem). + const stamped = await messageRepo.stampTerminalIfStreaming( + m.id, + workspaceId, + 'aborted', + ); + expect(stamped!.status).toBe('aborted'); + expect((await metaOf(m.id))?.finalizeFailed).toBe(true); + + // A LATE owner-write (finalizeFailed=true satisfies the OR) overwrites it with + // real content, clearing the flag — owner-write priority. + const wrote = await messageRepo.finalizeOwner(m.id, workspaceId, { + content: 'the real final answer', + status: 'completed', + metadata: { parts: [{ type: 'text', text: 'the real final answer' }] }, + } as never); + expect(wrote!.status).toBe('completed'); + expect(wrote!.content).toBe('the real final answer'); + expect((await metaOf(m.id))?.finalizeFailed).toBeUndefined(); + }); + + it('clause (c): a stale active run with NO live entry -> aborted; a LIVE entry is untouched', async () => { + // Stale run, NOT owned by this replica (no entry) -> reconcile aborts it. + const staleChat = await newChat(); + const stale = await runRepo.insert({ + chatId: staleChat, + workspaceId, + createdBy: userId, + status: 'running', + }); + await db + .updateTable('aiChatRuns') + .set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) }) + .where('id', '=', stale.id) + .execute(); + + // A live run OWNED by this replica (beginRun registers an in-memory entry), + // ALSO backdated stale — the "no entry" primary gate must protect it. + const liveChat = await newChat(); + const live = await runService.beginRun({ + chatId: liveChat, + workspaceId, + userId, + }); + await db + .updateTable('aiChatRuns') + .set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) }) + .where('id', '=', live.runId) + .execute(); + + const aborted = await runService.reconcileStaleRuns(15 * 60 * 1000); + expect(aborted).toBeGreaterThanOrEqual(1); + expect((await runRepo.findById(stale.id, workspaceId))!.status).toBe( + 'aborted', + ); + // The live entry is NEVER aborted, however stale its row looks. + expect((await runRepo.findById(live.runId, workspaceId))!.status).toBe( + 'running', + ); + expect(runService.isLocallyActive(live.runId)).toBe(true); + + // cleanup the live run + await runService.finalizeRun(live.runId, workspaceId, 'aborted'); + }); + + it('clause (b): a streaming message whose RUN is terminal is stamped by run status (succeeded -> aborted, NOT completed-empty)', async () => { + const chatId = await newChat(); + const msg = await createMessage(db, { + workspaceId, + chatId, + role: 'assistant', + status: 'streaming', + metadata: { parts: [] }, + }); + // A SUCCEEDED run linked to the still-streaming message (the asymmetry). + const run = await runRepo.insert({ + chatId, + workspaceId, + createdBy: userId, + status: 'running', + assistantMessageId: msg.id, + }); + await runRepo.finalizeIfActive(run.id, workspaceId, { + status: 'succeeded', + error: null, + }); + + const stuck = await messageRepo.findStreamingWithTerminalRun(); + const mine = stuck.find((s) => s.messageId === msg.id); + expect(mine?.runStatus).toBe('succeeded'); + // Reconcile clause (b): succeeded run -> message 'aborted' (NOT 'completed'), + // the final text lived only in memory (documented loss), +finalizeFailed. + const status = mine!.runStatus === 'failed' ? 'error' : 'aborted'; + await messageRepo.stampTerminalIfStreaming(msg.id, workspaceId, status); + const row = await messageRepo.findById(msg.id, workspaceId); + expect(row!.status).toBe('aborted'); + expect((row!.metadata as Record).finalizeFailed).toBe(true); + }); + + it('clause (d): a stale streaming row with NO active run on the chat -> aborted+finalizeFailed', async () => { + const chatId = await newChat(); + const msg = await createMessage(db, { + workspaceId, + chatId, + role: 'assistant', + status: 'streaming', + metadata: { parts: [] }, + }); + await db + .updateTable('aiChatMessages') + .set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) }) + .where('id', '=', msg.id) + .execute(); + + const swept = await messageRepo.sweepStreamingWithoutActiveRun( + 15 * 60 * 1000, + ); + expect(swept).toBeGreaterThanOrEqual(1); + const row = await messageRepo.findById(msg.id, workspaceId); + expect(row!.status).toBe('aborted'); + expect((row!.metadata as Record).finalizeFailed).toBe(true); + }); + + it('clause (d) is DOUBLE-GATED: a stale streaming row WITH an active run on the chat is left alone', async () => { + const chatId = await newChat(); + const msg = await createMessage(db, { + workspaceId, + chatId, + role: 'assistant', + status: 'streaming', + metadata: { parts: [] }, + }); + await db + .updateTable('aiChatMessages') + .set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) }) + .where('id', '=', msg.id) + .execute(); + // An ACTIVE run on the same chat -> clause (d) must NOT touch the message. + const run = await runRepo.insert({ + chatId, + workspaceId, + createdBy: userId, + status: 'running', + }); + + await messageRepo.sweepStreamingWithoutActiveRun(15 * 60 * 1000); + expect((await messageRepo.findById(msg.id, workspaceId))!.status).toBe( + 'streaming', + ); + await runRepo.finalizeIfActive(run.id, workspaceId, { + status: 'aborted', + error: null, + }); + }); + + it('"kill DB on finish" recovery: after the DB is back, reconcile leaves NEITHER the row nor the run stuck', async () => { + // Simulate a process that seeded the assistant row + run, then died before + // finalizing EITHER (a mid-turn crash): a streaming message + a running run, + // both stale, with no in-memory entry (fresh service = fresh maps). + const chatId = await newChat(); + const msg = await createMessage(db, { + workspaceId, + chatId, + role: 'assistant', + status: 'streaming', + metadata: { parts: [{ type: 'text', text: 'partial' }] }, + }); + const run = await runRepo.insert({ + chatId, + workspaceId, + createdBy: userId, + status: 'running', + assistantMessageId: msg.id, + }); + await db + .updateTable('aiChatRuns') + .set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) }) + .where('id', '=', run.id) + .execute(); + await db + .updateTable('aiChatMessages') + .set({ updatedAt: new Date(Date.now() - 60 * 60 * 1000) }) + .where('id', '=', msg.id) + .execute(); + + // Reconcile (as the periodic job would): (c) aborts the orphan run, then + // (b) settles the message from the now-terminal run. + await runService.reconcileStaleRuns(15 * 60 * 1000); + const stuck = await messageRepo.findStreamingWithTerminalRun(); + for (const s of stuck) { + const status = s.runStatus === 'failed' ? 'error' : 'aborted'; + await messageRepo.stampTerminalIfStreaming(s.messageId, s.workspaceId, status); + } + + // Neither is stuck: the run is terminal AND the message is terminal. + expect((await runRepo.findById(run.id, workspaceId))!.status).toBe('aborted'); + const row = await messageRepo.findById(msg.id, workspaceId); + expect(row!.status).toBe('aborted'); + expect((row!.metadata as Record).finalizeFailed).toBe(true); + }); +}); From 001eb1c923f551f72c2ee8feb67ce9b2d786d758 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 05:34:46 +0300 Subject: [PATCH 21/41] =?UTF-8?q?docs(mcp):=20=D1=82=D0=BE=D1=87=D0=BD?= =?UTF-8?q?=D1=8B=D0=B9=20=D0=BA=D0=BE=D0=BC=D0=BC=D0=B5=D0=BD=D1=82=20too?= =?UTF-8?q?lAbortSignal=20=E2=80=94=20set-and-leave,=20=D0=BD=D0=B5=20rest?= =?UTF-8?q?ore=20(#487,=20=D1=80=D0=B5=D0=B2=D1=8C=D1=8E=20nit)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Внутреннее ревью: docstring поля/метода toolAbortSignal утверждал «restores the prior value on unwind», но wrapInAppToolWithCap намеренно НЕ восстанавливает сигнал (set-and-leave) — именно это заставляет брошенного проигравшего race бросить на следующем safe-point; корректность держится на свежем клиенте на ход + перезаписи следующим вызовом. Комментарии приведены в соответствие с механизмом. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp/src/client/context.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/mcp/src/client/context.ts b/packages/mcp/src/client/context.ts index 4cf996a1..1fb0b354 100644 --- a/packages/mcp/src/client/context.ts +++ b/packages/mcp/src/client/context.ts @@ -179,8 +179,11 @@ export abstract class DocmostClientContext { // still land — a documented limitation (#487). // // SINGLE-WRITER by phase-1 assumption: exactly one DocmostClient is built per - // turn and shared by every tool call; the host sets this per call and restores - // the prior value when the call unwinds. If the model emits PARALLEL in-app + // turn and shared by every tool call; the host sets this per call and does NOT + // restore the prior value on unwind (set-and-leave) — a fresh client per turn + // plus overwrite-by-the-next-call keeps it correct, and leaving a settled + // call's signal in place is what makes a discarded race-loser throw on its + // next safe-point. If the model emits PARALLEL in-app // tool calls they share this one field, so the per-call CAP of one call is not // guaranteed to bound another's in-flight pagination — but every composite the // host sets carries the SAME turn Stop signal, so a Stop still aborts whichever @@ -190,9 +193,10 @@ export abstract class DocmostClientContext { /** * #487: set (or clear with null) the in-app tool abort signal governing the * NEXT client call's safe-points. The host wraps each in-app tool call: it sets - * the composite (Stop + per-call cap) here before invoking the tool and - * restores the prior value afterwards. Public so the server-side tool wrapper - * can reach it; harmless (a no-op) when never set. + * the composite (Stop + per-call cap) here before invoking the tool and leaves + * it in place afterwards (set-and-leave, NOT restored) — the next call + * overwrites it, and a fresh client is built per turn. Public so the + * server-side tool wrapper can reach it; harmless (a no-op) when never set. */ public setToolAbortSignal(signal: AbortSignal | null): void { this.toolAbortSignal = signal; From 123cba7de1755eee3a043b2a6b413b29bad58085 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:48:52 +0300 Subject: [PATCH 22/41] =?UTF-8?q?fix(mcp):=20=D1=83=D0=B1=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20=D0=B7=D0=B0=D0=BB=D0=B8=D0=BF=D1=88=D0=B8=D0=B9?= =?UTF-8?q?=20=D0=BC=D0=B0=D1=80=D0=BA=D0=B5=D1=80=20=D0=BA=D0=BE=D0=BD?= =?UTF-8?q?=D1=84=D0=BB=D0=B8=D0=BA=D1=82=D0=B0=20=D0=B2=20context.ts=20(?= =?UTF-8?q?=D0=B0=D1=80=D1=82=D0=B5=D1=84=D0=B0=D0=BA=D1=82=20=D1=80=D0=B5?= =?UTF-8?q?=D0=B1=D0=B5=D0=B9=D0=B7=D0=B0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit После ребейза в src/client/context.ts:209 остался одиночный маркер `>>>>>>> 917c4064` без парного открытия — он попал в коммит и ломал сборку всего пакета mcp (TS1185 Merge conflict marker), из-за чего не запускались никакие node --test. Код выше маркера цел; удаляю только строку маркера. Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/mcp/src/client/context.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/mcp/src/client/context.ts b/packages/mcp/src/client/context.ts index 1fb0b354..8c784f53 100644 --- a/packages/mcp/src/client/context.ts +++ b/packages/mcp/src/client/context.ts @@ -206,7 +206,6 @@ export abstract class DocmostClientContext { public getToolAbortSignal(): AbortSignal | null { return this.toolAbortSignal; } ->>>>>>> 917c4064 (fix(ai-chat): in-app тулы — race-on-abort + safe-points + per-call cap (#487)) // Two construction forms: // - new DocmostClient(config) // discriminated union (current) From 0b9de2c25e2f834209826a8ba874204dc3700891 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:49:02 +0300 Subject: [PATCH 23/41] =?UTF-8?q?fix(ai-chat):=20owner-gate=20stream()=20?= =?UTF-8?q?=D0=B4=D0=BE=20supersede=20=E2=80=94=20=D0=B7=D0=B0=D0=BA=D1=80?= =?UTF-8?q?=D1=8B=D1=82=D1=8C=20=D1=83=D1=82=D0=B5=D1=87=D0=BA=D1=83=20?= =?UTF-8?q?=D1=87=D1=83=D0=B6=D0=BE=D0=B3=D0=BE=20runId=20(#487,=20F1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit stream() был единственным эндпоинтом ai-chat без assertOwnedChat: участник того же воркспейса (НЕ владелец чата) мог послать POST /stream {chatId:<чужой>, supersede:{runId:<любой>}} и (а) выудить активный runId жертвы из ответа 409 SUPERSEDE_TARGET_MISMATCH, затем (б) requestStop чужого рана. Добавляю owner-check в начале stream() (когда есть body.chatId), ровно как в /stop и соседях — pre-hijack, чистый 403. Это заодно закрывает и негейченную кросс-юзерную запись через тот же stream() (база #500). Тест: не-владелец POST /stream с чужим chatId → ForbiddenException, runId не утёк, supersede/requestStop не вызваны. MUTATION-VERIFY: снятие assertOwnedChat делает тест красным (возвращается путь утечки). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat.controller.supersede.spec.ts | 64 ++++++++++++++++++- .../src/core/ai-chat/ai-chat.controller.ts | 13 ++++ 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/apps/server/src/core/ai-chat/ai-chat.controller.supersede.spec.ts b/apps/server/src/core/ai-chat/ai-chat.controller.supersede.spec.ts index 69aa84b5..ce06d53f 100644 --- a/apps/server/src/core/ai-chat/ai-chat.controller.supersede.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.controller.supersede.spec.ts @@ -1,6 +1,7 @@ import { BadRequestException, ConflictException, + ForbiddenException, HttpException, } from '@nestjs/common'; import { AiChatController } from './ai-chat.controller'; @@ -46,7 +47,13 @@ describe('#487 AiChatController.stream — gate + supersede', () => { return { req, res }; } - function makeController(runServiceOverrides: Record) { + function makeController( + runServiceOverrides: Record, + // The chat assertOwnedChat resolves. Default: a chat OWNED by `user` (u1), so + // the ownership gate is transparent to the gate/CAS assertions below. Pass a + // foreign-owner (or undefined) chat to exercise the #487 owner rejection. + chat: { creatorId: string } | undefined = { creatorId: 'u1' }, + ) { const aiChatService = { resolveRoleForRequest: jest.fn().mockResolvedValue(null), getChatModel: jest.fn().mockResolvedValue({}), @@ -65,15 +72,16 @@ describe('#487 AiChatController.stream — gate + supersede', () => { requestStop: jest.fn(), ...runServiceOverrides, }; + const aiChatRepo = { findById: jest.fn().mockResolvedValue(chat) }; const controller = new AiChatController( aiChatService as never, aiChatRunService as never, - {} as never, // aiChatRepo + aiChatRepo as never, // aiChatRepo {} as never, // aiChatMessageRepo {} as never, // aiTranscription {} as never, // pageRepo ); - return { controller, aiChatService, aiChatRunService }; + return { controller, aiChatService, aiChatRunService, aiChatRepo }; } const codeOf = (err: unknown) => @@ -113,6 +121,56 @@ describe('#487 AiChatController.stream — gate + supersede', () => { } }); + // #487 [security, F1]: stream() MUST owner-gate an existing chat exactly like its + // six sibling endpoints, BEFORE the supersede CAS. Otherwise a same-workspace + // non-owner could POST a supersede against another user's chat and (a) harvest + // that user's active runId from the 409 SUPERSEDE_TARGET_MISMATCH body, then (b) + // requestStop the foreign run. The gate must reject FIRST — no run lookup, no + // supersede, no stop, no runId leak. + describe('cross-user ownership gate (F1)', () => { + it('a non-owner streaming against someone else\'s chat is rejected (403) with NO runId leak and NO foreign requestStop', async () => { + // A live run exists on the victim's chat. Without the gate the supersede CAS + // would run and (faithful to the run service) return a MISMATCH carrying the + // victim's runId — the exact leak. With the gate it must never be reached. + const getActiveForChat = jest + .fn() + .mockResolvedValue({ id: 'run-victim', chatId: 'c-other' }); + const supersede = jest + .fn() + .mockResolvedValue({ kind: 'mismatch', activeRunId: 'run-victim' }); + const requestStop = jest.fn(); + const { controller, aiChatService } = makeController( + { getActiveForChat, supersede, requestStop }, + { creatorId: 'someone-else' }, // the chat is NOT owned by u1 + ); + const { req, res } = makeReqRes({ + chatId: 'c-other', + supersede: { runId: 'guessed-uuid' }, + }); + let thrown: unknown; + try { + await controller.stream(req as never, res as never, user, wsWith(true)); + } catch (e) { + thrown = e; + } + // Rejected by the ownership gate (403), the SAME shape the neighbors use. + expect(thrown).toBeInstanceOf(ForbiddenException); + expect((thrown as HttpException).getStatus()).toBe(403); + // Crucially NOT a 409 that would carry activeRunId — no runId is leaked. + const payload = JSON.stringify( + (thrown as HttpException).getResponse() ?? {}, + ); + expect(payload).not.toContain('run-victim'); + expect(codeOf(thrown)).not.toBe('SUPERSEDE_TARGET_MISMATCH'); + // The gate short-circuits BEFORE any run machinery runs. + expect(getActiveForChat).not.toHaveBeenCalled(); + expect(supersede).not.toHaveBeenCalled(); + expect(requestStop).not.toHaveBeenCalled(); + expect(aiChatService.stream).not.toHaveBeenCalled(); + expect(res.hijack).not.toHaveBeenCalled(); + }); + }); + it('supersede MISMATCH -> 409 SUPERSEDE_TARGET_MISMATCH carrying the current runId', async () => { const { controller } = makeController({ supersede: jest diff --git a/apps/server/src/core/ai-chat/ai-chat.controller.ts b/apps/server/src/core/ai-chat/ai-chat.controller.ts index 6dbe8420..f9b58b98 100644 --- a/apps/server/src/core/ai-chat/ai-chat.controller.ts +++ b/apps/server/src/core/ai-chat/ai-chat.controller.ts @@ -418,6 +418,19 @@ export class AiChatController { const body = (req.body ?? {}) as AiChatStreamBody; + // #487 [security]: gate cross-user access to an EXISTING chat BEFORE anything + // reads its runs. Every sibling endpoint (getRun/stop/history/rename/delete/ + // attachRunStream) owner-checks the chat via assertOwnedChat; stream() must too. + // Without this a same-workspace member who is NOT the chat owner could POST a + // supersede against another user's chat and (a) harvest that user's active runId + // out of the 409 SUPERSEDE_TARGET_MISMATCH body, then (b) requestStop the foreign + // run. Gate on the chatId the client sent, when present — a brand-new chat (no + // chatId) has no prior owner to check. Mirrors /stop's owner check (403 as the + // neighbors do), and runs pre-hijack so it returns clean JSON. + if (body.chatId) { + await this.assertOwnedChat(body.chatId, user, workspace); + } + // Resolve the agent role for this turn BEFORE hijack: existing chats read it // from ai_chats.role_id (authoritative), a new chat from body.roleId. The // role drives both the persona and the optional model override below. From 9ba97b6d2be95f3010ae2d8fa28a33341d58bda8 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:49:14 +0300 Subject: [PATCH 24/41] =?UTF-8?q?docs(ai-chat):=20=D0=B7=D0=B0=D0=B4=D0=BE?= =?UTF-8?q?=D0=BA=D1=83=D0=BC=D0=B5=D0=BD=D1=82=D0=B8=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D0=BD=D0=B0=D0=B1=D0=BB=D1=8E=D0=B4=D0=B0?= =?UTF-8?q?=D0=B5=D0=BC=D1=83=D1=8E=20=D0=BF=D0=BE=D0=B2=D0=B5=D1=80=D1=85?= =?UTF-8?q?=D0=BD=D0=BE=D1=81=D1=82=D1=8C=20#487=20(F2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - CHANGELOG [Unreleased]: серверный supersede + три кода (SUPERSEDE_INVALID / SUPERSEDE_TARGET_MISMATCH / SUPERSEDE_TIMEOUT) в Added; смена поведения (легаси вторая вкладка теперь → 409 A_RUN_ALREADY_ACTIVE вместо второго параллельного стрима; каждый ход — ран в обоих режимах) в Changed. - .env.example: три новых AI_CHAT_* (SUPERSEDE_TIMEOUT_MS / RECONCILE_INTERVAL_MS / INAPP_TOOL_CALL_CAP_MS, дефолты 10000/120000/120000, связь cap↔staleness). - AGENTS.md:458: ран теперь универсален (оба режима), флаг autonomousRuns = только disconnect-семантика; плюс упоминание supersede. - ai-chat.service.ts:1136: устаревший коммент про «legacy → socket signal» → оба режима на run-signal, guard runId защищает лишь no-handle fallback. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 23 +++++++++++++++++++ AGENTS.md | 2 +- CHANGELOG.md | 22 ++++++++++++++++++ .../src/core/ai-chat/ai-chat.service.ts | 15 ++++++------ 4 files changed, 53 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index e490f7c9..f3acb7ba 100644 --- a/.env.example +++ b/.env.example @@ -288,6 +288,29 @@ MCP_DOCMOST_PASSWORD= # registry is process-local). # AI_CHAT_RESUMABLE_STREAM=false +# --- Run lifecycle tunables (#487) --- +# These govern the universal run machinery (every turn is now a first-class run, +# both modes) and rarely need changing. +# +# How long a server-side SUPERSEDE ("interrupt and send now") waits for the target +# run to settle after issuing Stop before it degrades to a 409 SUPERSEDE_TIMEOUT +# (nothing sent, the composer keeps the user's text). 10s is generous under a +# healthy DB; do NOT raise it to paper over a slow DB — a SUPERSEDE_TIMEOUT is the +# honest signal. Default 10000 (10s). +# AI_CHAT_SUPERSEDE_TIMEOUT_MS=10000 +# +# How often the periodic bidirectional reconcile job runs (heals runs/messages +# left dangling by a crash or a lost terminal write). Default 120000 (2 min). +# AI_CHAT_RECONCILE_INTERVAL_MS=120000 +# +# Wall-clock cap for a SINGLE in-app tool call (a long paginated read, or a content +# write whose collab commit hangs) — the per-call half of the composite abort +# signal every in-app tool is wrapped with (the other half is the turn's Stop). +# The reconcile staleness floor is derived as max(2 x this cap, 15min), so a very +# high value delays stale-run recovery (the server boot-warns above 30min). Default +# 120000 (2 min). +# AI_CHAT_INAPP_TOOL_CALL_CAP_MS=120000 + # --- Anonymous public-share AI assistant --- # Opt-in per workspace (AI settings -> "public share assistant"; off by default). # When enabled, anonymous visitors of a published share can ask an AI about that diff --git a/AGENTS.md b/AGENTS.md index 08756773..dd548743 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -455,7 +455,7 @@ The API server is a Fastify app with a global `/api` prefix (`main.ts` excludes - `core/ai-chat/tools/` — the agent's ~40 read+write tools. Every tool runs under the **calling user's** CASL permissions via a per-user loopback access token (`docmost-client.loader.ts`), so the agent can never exceed what the user could do. Only **reversible** operations are exposed (page history + trash; no permanent delete). Agent edits get an "AI agent" provenance badge in page history (`20260616T130000-agent-provenance` migration). - `core/ai-chat/embedding/` — RAG indexer + a BullMQ consumer on `AI_QUEUE` that embeds pages into `page_embeddings` (vector search), complementing Postgres full-text search. Pages are (re)indexed on edit; `AI_EMBEDDING_TIMEOUT_MS` bounds a hung embeddings endpoint. - `core/ai-chat/external-mcp/` — admins can attach external MCP servers (e.g. Tavily) to give the agent web access. **`ssrf-guard.ts` validates outbound MCP URLs against SSRF** — keep that guard in the path when touching external-MCP connection logic. - - `core/ai-chat/ai-chat-run.service.ts` + `ai_chat_runs` — **detached/autonomous agent runs** (`#184`), behind the per-workspace `settings.ai.autonomousRuns` flag (off by default). When on, a turn becomes a server-side RUN that survives a browser disconnect; only an explicit `POST /ai-chat/stop` ends it, and a client reconnects/live-follows via `POST /ai-chat/run`. **DEPLOY CONSTRAINT — single-instance only in phase 1:** Stop and the AbortController that backs it are process-local, so a Stop only aborts a run executing on the **same** replica that owns it (cross-instance pub/sub stop is phase 2). Do **not** enable `autonomousRuns` on a horizontally-scaled deployment (multiple replicas behind a load balancer, or Docmost cloud `CLOUD=true`) — run a single instance instead. The server logs a startup WARNING when it detects a multi-instance deployment (`CLOUD=true`) so the constraint is visible. The startup sweep settles any run left dangling by a restart. + - `core/ai-chat/ai-chat-run.service.ts` + `ai_chat_runs` — **every agent turn is now a first-class server-side RUN** (`#184`, universalized in `#487`): its lifecycle is tracked in `ai_chat_runs` in **both** modes, and the single-active-run-per-chat concurrency gate is enforced universally (a legacy second tab now gets a clean `409 A_RUN_ALREADY_ACTIVE` instead of a second parallel stream that interleaved history). The per-workspace `settings.ai.autonomousRuns` flag (off by default) **no longer gates whether a turn is a run** — it now controls **only the browser-disconnect semantics**: when ON the run is *detached* (a disconnect leaves it executing server-side; only an explicit `POST /ai-chat/stop` ends it, and a client reconnects/live-follows via `POST /ai-chat/run`); when OFF (legacy) a disconnect ends the turn by stopping its run via the run's stop lever. `#487` also adds a server-side **supersede** CAS ("interrupt and send now") to `POST /ai-chat/stream` (`supersede: { runId }`): it atomically stops the chat's currently-active run and waits for it to settle before the new turn claims the slot, returning `SUPERSEDE_INVALID` / `SUPERSEDE_TARGET_MISMATCH` / `SUPERSEDE_TIMEOUT` on the non-proceed branches. **DEPLOY CONSTRAINT — single-instance only in phase 1:** Stop and the AbortController that backs it are process-local, so a Stop only aborts a run executing on the **same** replica that owns it (cross-instance pub/sub stop is phase 2). Do **not** enable `autonomousRuns` on a horizontally-scaled deployment (multiple replicas behind a load balancer, or Docmost cloud `CLOUD=true`) — run a single instance instead. The server logs a startup WARNING when it detects a multi-instance deployment (`CLOUD=true`) so the constraint is visible. The startup sweep settles any run left dangling by a restart. ### Client structure Vite SPA. Code is organized by feature under `apps/client/src/features/*` (mirrors the server domains: `page`, `space`, `comment`, `ai-chat`, `editor`, …). Conventions: diff --git a/CHANGELOG.md b/CHANGELOG.md index ed4d5a03..30d27413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -202,6 +202,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 dangling by a restart. Phase 1 is single-instance-only (cross-instance Stop is not yet reliable); the server warns at startup on a horizontally-scaled deployment. (#184) +- **Server-side "interrupt and send now" (supersede) for AI chat.** `POST + /ai-chat/stream` now accepts a `supersede: { runId }` field: when the user sends + a new message while a run is active, the server atomically stops that run and + waits for it to settle before the new turn claims the chat's single run slot, + instead of the send being rejected as concurrent. The compare-and-set surfaces + three codes on its non-proceed branches — `SUPERSEDE_INVALID` (the targeted run + is malformed / belongs to another chat), `SUPERSEDE_TARGET_MISMATCH` (a + different run is now active; carries the current `activeRunId`), and + `SUPERSEDE_TIMEOUT` (the previous run did not stop within the settle window, so + nothing was sent and the composer keeps the text). Tunable via + `AI_CHAT_SUPERSEDE_TIMEOUT_MS` (default 10s). (#487) - **Out-of-band page transfer via an in-RAM blob sandbox (`stash_page`).** A new MCP tool serializes a whole page (its full ProseMirror JSON, with every internal image/file mirrored) into an ephemeral in-RAM blob and returns only @@ -282,6 +293,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **Every AI-chat turn is now a first-class server-side run, and one run per chat + is enforced in both modes.** The run machinery from `#184` was universalized: a + turn is tracked in `ai_chat_runs` and gated by the single-active-run-per-chat + index regardless of the `settings.ai.autonomousRuns` flag. **Behavior change:** + a second tab (or a double-submit) that starts a turn while one is already active + on the chat is now rejected up front with `409 A_RUN_ALREADY_ACTIVE` (carrying + the `activeRunId`); previously, on the legacy path, it opened a second parallel + stream on the same chat that interleaved history. The `autonomousRuns` flag no + longer controls whether a turn is a run — it now governs **only** the + browser-disconnect semantics (ON = detached/survives a disconnect; OFF = a + disconnect stops the run). (#487) - **Client markdown paste/copy and AI-chat rendering now go through the canonical converter.** Pasting markdown into the editor, "Copy as markdown", the AI title generator, and the AI-chat markdown renderer all now use diff --git a/apps/server/src/core/ai-chat/ai-chat.service.ts b/apps/server/src/core/ai-chat/ai-chat.service.ts index 959d2d44..02ecc9d1 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.ts @@ -1131,14 +1131,13 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { ); } catch (err) { // An explicit Stop reached the RUN's signal DURING setup: re-throw so the - // outer catch finalizes the run as aborted — never swallow a Stop. Gated on - // `runId`: the re-throw exists ONLY to finalize the run, which exists only - // in autonomous mode. On the legacy path (no runId) `effectiveSignal` is the - // SOCKET signal (it aborts on a client disconnect); re-throwing there would - // change prior behavior and make the controller write JSON to an already- - // closed socket (it only attaches res.raw.on('error') in autonomous mode). - // So legacy keeps its prior behavior — warn + proceed, and streamText then - // observes the aborted socket signal. + // outer catch finalizes the run as aborted — never swallow a Stop. #487: the + // turn is ALWAYS run-wrapped now (both modes), so `effectiveSignal` is the + // RUN signal and `runId` is set in BOTH — a Stop (from /ai-chat/stop or a + // legacy disconnect's requestStop) aborts it identically. The `runId` guard + // now only defends the theoretical no-handle fallback (`begin` returned + // nothing, leaving `effectiveSignal` as the bare socket signal): there we + // keep the old warn-and-proceed rather than re-throw. if (runId && effectiveSignal.aborted) { throw err; } From 2cb07ef7fb0171507b3eaafec5d23f3c79bc2ea6 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:49:30 +0300 Subject: [PATCH 25/41] =?UTF-8?q?test(ai-chat):=20=D0=BF=D0=BE=D0=BA=D1=80?= =?UTF-8?q?=D1=8B=D1=82=D1=8C=20=D0=BE=D1=80=D0=BA=D0=B5=D1=81=D1=82=D1=80?= =?UTF-8?q?=D0=B0=D1=82=D0=BE=D1=80=D1=8B=20reconcile()/reconcileChat()=20?= =?UTF-8?q?(#487,=20F3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Клаузы (a-d) тестировались по отдельности, но сам оркестратор reconcile() (ПОРЯДОК клауз + пер-клаузная try/catch-изоляция «одна упавшая не блокирует остальные») не проверялся, а reconcileChat() (старт каждого хода) не был покрыт вовсе. Добавляю: тест порядка a→b→c→d; тест изоляции (клауза b кидает — c и d всё равно отрабатывают, reconcile не реджектит); тест reconcileChat (скоуп по чату, bound 50, failed→error / прочее→aborted). MUTATION-VERIFY: снятие try/catch у клаузы (b) делает тест изоляции красным (исключение пробрасывается, c+d пропускаются). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/core/ai-chat/ai-chat.service.spec.ts | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/apps/server/src/core/ai-chat/ai-chat.service.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.spec.ts index cdfa80e9..ca9511ef 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.spec.ts @@ -1929,3 +1929,148 @@ describe('AiChatService.stream — token-degeneration reaction (#444)', () => { expect(patch.content).not.toContain(STEP_LIMIT_NO_ANSWER_MARKER); }); }); + +// #487 F3 — the reconcile() / reconcileChat() ORCHESTRATORS. The individual +// clauses are exercised elsewhere; these pin the production orchestration the +// per-clause specs do not: the clause ORDER, the per-clause try/catch ISOLATION +// (one clause throwing must NOT abort the others), and reconcileChat() (which runs +// at the start of every turn and was entirely uncovered). +describe('AiChatService.reconcile / reconcileChat orchestrators (#487 F3)', () => { + let warnSpy: jest.SpyInstance; + beforeEach(() => { + // Silence the intentional clause-failure warnings (kept out of test output). + warnSpy = jest + .spyOn(Logger.prototype, 'warn') + .mockImplementation(() => undefined); + }); + afterEach(() => { + warnSpy.mockRestore(); + }); + + function makeService(opts: { + messageRepo?: Record; + runService?: Record; + }) { + const aiChatMessageRepo = { + findStreamingWithTerminalRun: jest.fn(async () => []), + stampTerminalIfStreaming: jest.fn(async () => undefined), + sweepStreamingWithoutActiveRun: jest.fn(async () => 0), + ...(opts.messageRepo ?? {}), + }; + const aiChatRunService = opts.runService + ? { + zombieRunIds: jest.fn(() => []), + settleZombie: jest.fn(async () => true), + reconcileStaleRuns: jest.fn(async () => 0), + ...opts.runService, + } + : undefined; + const svc = new AiChatService( + {} as never, // ai + {} as never, // aiChatRepo + aiChatMessageRepo as never, + {} as never, // aiChatPageSnapshotRepo + {} as never, // aiSettings + {} as never, // tools + {} as never, // mcpClients + {} as never, // aiAgentRoleRepo + {} as never, // pageRepo + {} as never, // pageAccess + {} as never, // environment + {} as never, // streamRegistry + aiChatRunService as never, // aiChatRunService (#487) + ); + return { svc, aiChatMessageRepo, aiChatRunService }; + } + + it('reconcile() fires all four clauses IN ORDER (a -> b -> c -> d)', async () => { + const order: string[] = []; + const { svc } = makeService({ + messageRepo: { + findStreamingWithTerminalRun: jest.fn(async () => { + order.push('b:find'); + return [ + { messageId: 'm1', workspaceId: 'ws1', runStatus: 'succeeded' }, + ]; + }), + stampTerminalIfStreaming: jest.fn(async () => { + order.push('b:stamp'); + }), + sweepStreamingWithoutActiveRun: jest.fn(async () => { + order.push('d'); + return 0; + }), + }, + runService: { + zombieRunIds: jest.fn(() => ['z1']), + settleZombie: jest.fn(async () => { + order.push('a'); + return true; + }), + reconcileStaleRuns: jest.fn(async () => { + order.push('c'); + return 0; + }), + }, + }); + + await svc.reconcile(); + + expect(order).toEqual(['a', 'b:find', 'b:stamp', 'c', 'd']); + }); + + it('a clause that THROWS does not abort the remaining clauses (per-clause try/catch isolation)', async () => { + const { svc, aiChatMessageRepo, aiChatRunService } = makeService({ + messageRepo: { + // Clause (b) blows up mid-reconcile. + findStreamingWithTerminalRun: jest.fn(async () => { + throw new Error('clause b DB blip'); + }), + }, + runService: { + zombieRunIds: jest.fn(() => ['z1']), + }, + }); + + // reconcile() must SETTLE (the clause-b failure is swallowed), not reject. + await expect(svc.reconcile()).resolves.toBeUndefined(); + + // (a) ran before (b); crucially (c) and (d) STILL ran despite (b) throwing — + // the property a missing try/catch would break. MUTATION-VERIFY: drop clause + // (b)'s try/catch and this reddens (the throw propagates, skipping c + d). + expect(aiChatRunService!.settleZombie).toHaveBeenCalled(); // (a) + expect(aiChatRunService!.reconcileStaleRuns).toHaveBeenCalled(); // (c) + expect( + aiChatMessageRepo.sweepStreamingWithoutActiveRun, + ).toHaveBeenCalled(); // (d) + }); + + it('reconcileChat() settles THIS chat\'s stuck streaming rows by their run status', async () => { + const { svc, aiChatMessageRepo } = makeService({ + messageRepo: { + findStreamingWithTerminalRun: jest.fn(async () => [ + { messageId: 'm1', workspaceId: 'ws1', runStatus: 'failed' }, + { messageId: 'm2', workspaceId: 'ws1', runStatus: 'succeeded' }, + ]), + }, + }); + + await svc.reconcileChat('chat-1', 'ws1'); + + // Scoped to THIS chat and bounded at 50 (the user-facing opportunistic path). + expect( + aiChatMessageRepo.findStreamingWithTerminalRun, + ).toHaveBeenCalledWith(50, { chatId: 'chat-1', workspaceId: 'ws1' }); + // failed-run -> 'error'; every other terminal status -> 'aborted'. + expect(aiChatMessageRepo.stampTerminalIfStreaming).toHaveBeenCalledWith( + 'm1', + 'ws1', + 'error', + ); + expect(aiChatMessageRepo.stampTerminalIfStreaming).toHaveBeenCalledWith( + 'm2', + 'ws1', + 'aborted', + ); + }); +}); From 1d704d4ec512afe2105c54223c2dee246d58af29 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:49:30 +0300 Subject: [PATCH 26/41] =?UTF-8?q?test(mcp):=20=D0=BF=D0=BE=D0=BA=D1=80?= =?UTF-8?q?=D1=8B=D1=82=D1=8C=20write-safe-point=20=C2=AB=D0=BF=D0=BE?= =?UTF-8?q?=D1=81=D0=BB=D0=B5=20Stop=20=D0=BD=D0=BE=D0=B2=D0=B0=D1=8F=20?= =?UTF-8?q?=D0=B7=D0=B0=D0=BF=D0=B8=D1=81=D1=8C=20=D0=BD=D0=B5=20=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D1=80=D1=82=D1=83=D0=B5=D1=82=C2=BB=20(#487,=20F4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit paginate-abort-safepoint покрывал только READ-safe-point; WRITE-шов (collaboration.mutatePageContent / context.mutateLiveContentUnlocked — signal?.throwIfAborted() перед session.mutate) был без теста. Добавляю мок-collab юнит через тест-сим __setCollabProviderFactory (фейковый провайдер с мгновенным onSynced): предварительно abort-нутый сигнал → оба шва реджектят ДО session.mutate, трансформ не вызывается; контрольные кейсы (живой сигнал) подтверждают, что session.mutate достижим. MUTATION-VERIFY: снятие throwIfAborted в обоих швах делает красными ровно два abort-кейса, контрольные остаются зелёными. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test/mock/write-abort-safepoint.test.mjs | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 packages/mcp/test/mock/write-abort-safepoint.test.mjs diff --git a/packages/mcp/test/mock/write-abort-safepoint.test.mjs b/packages/mcp/test/mock/write-abort-safepoint.test.mjs new file mode 100644 index 00000000..1002b34b --- /dev/null +++ b/packages/mcp/test/mock/write-abort-safepoint.test.mjs @@ -0,0 +1,164 @@ +// #487 F4 — the WRITE-side cancellation safe-point. +// +// Every content-mutating collab write (collaboration.mutatePageContent and the +// reentrant twin client.mutateLiveContentUnlocked used by replaceImage) checks +// the in-app tool abort signal at a PRE-COMMIT safe-point — after the collab +// session is acquired but immediately BEFORE the atomic read->write +// (session.mutate). So a Stop (or the per-call cap) that lands during the +// connect/lock window stops THIS write from landing: no new commit starts once +// aborted. paginate-abort-safepoint.test.mjs pins the READ half; this pins the +// integrity-critical WRITE half — remove the `throwIfAborted()` and the transform +// would run and the doc would be mutated past a Stop. +// +// There is no collab server in the unit env, so we swap the provider factory +// (__setCollabProviderFactory) for a fake that reports an immediate successful +// sync. That makes acquireCollabSession SUCCEED and hand back a live, ready +// session, so the ONLY thing standing between the call and session.mutate is the +// safe-point under test. The transform is instrumented to prove it never runs. +import { test, afterEach } from "node:test"; +import assert from "node:assert/strict"; +import { mutatePageContent } from "../../build/lib/collaboration.js"; +import { + __setCollabProviderFactory, + destroyAllSessions, +} from "../../build/lib/collab-session.js"; +import { DocmostClient } from "../../build/client.js"; + +const BASE_URL = "http://127.0.0.1:1/api"; +// mutatePageContent locks via withPageLock, which demands a canonical page UUID +// (resolve-then-lock invariant, #260/#449); the unlocked twin does not. +const PAGE_UUID = "11111111-1111-4111-8111-111111111111"; + +// A fake HocuspocusProvider that immediately reports a successful initial sync so +// CollabSession.open() resolves to a ready session, and stays "synced" with zero +// unsynced changes so a reached session.mutate() would resolve at once. It speaks +// only the tiny CollabProviderLike surface the session depends on. +function syncedProviderFactory(config) { + // Fire the initial-sync callback so open() settles as ready. + config.onSynced(); + return { + synced: true, + unsyncedChanges: 0, + destroy() {}, + on() {}, + off() {}, + }; +} + +// Disable the session cache so every acquire opens (and the failure path destroys) +// its own ephemeral session — no cross-test session reuse. +process.env.MCP_COLLAB_SESSION_IDLE_MS = "0"; + +afterEach(() => { + __setCollabProviderFactory(null); // restore the real factory + destroyAllSessions(); +}); + +// --- collaboration.mutatePageContent (the page-locked write path) ------------- + +test("mutatePageContent rejects at the pre-commit safe-point BEFORE session.mutate when the signal is already aborted", async () => { + __setCollabProviderFactory(syncedProviderFactory); + let transformCalls = 0; + const ac = new AbortController(); + ac.abort(new Error("user stop")); + + await assert.rejects( + () => + mutatePageContent( + PAGE_UUID, + "collab-jwt", + BASE_URL, + (liveDoc) => { + transformCalls++; + return liveDoc; + }, + ac.signal, + ), + /user stop/, + "the aborted safe-point rejects with the signal's reason before committing", + ); + assert.equal( + transformCalls, + 0, + "the transform (and therefore session.mutate) must NEVER run once aborted", + ); +}); + +test("mutatePageContent (control) DOES reach session.mutate and invoke the transform when the signal is live", async () => { + __setCollabProviderFactory(syncedProviderFactory); + let transformCalls = 0; + const ac = new AbortController(); // never aborted + + const result = await mutatePageContent( + PAGE_UUID, + "collab-jwt", + BASE_URL, + (liveDoc) => { + transformCalls++; + return null; // null -> no-op write; still proves the transform was invoked + }, + ac.signal, + ); + assert.equal( + transformCalls, + 1, + "with a live signal the safe-point is a no-op and session.mutate runs the transform", + ); + assert.ok(result && result.verify, "a MutationResult is returned"); +}); + +// --- client.mutateLiveContentUnlocked (the reentrant twin, replaceImage) ------ + +test("mutateLiveContentUnlocked rejects at the pre-commit safe-point BEFORE session.mutate when the tool signal is already aborted", async () => { + __setCollabProviderFactory(syncedProviderFactory); + const client = new DocmostClient({ + apiUrl: BASE_URL, + getToken: async () => "access", + getCollabToken: async () => "collab-jwt", + }); + let transformCalls = 0; + const ac = new AbortController(); + ac.abort(new Error("cap fired")); + client.setToolAbortSignal(ac.signal); + + await assert.rejects( + () => + client.mutateLiveContentUnlocked("page-1", "collab-jwt", (liveDoc) => { + transformCalls++; + return liveDoc; + }), + /cap fired/, + "the aborted safe-point rejects with the signal's reason before committing", + ); + assert.equal( + transformCalls, + 0, + "the transform (and therefore session.mutate) must NEVER run once aborted", + ); +}); + +test("mutateLiveContentUnlocked (control) DOES reach session.mutate and invoke the transform when the tool signal is live", async () => { + __setCollabProviderFactory(syncedProviderFactory); + const client = new DocmostClient({ + apiUrl: BASE_URL, + getToken: async () => "access", + getCollabToken: async () => "collab-jwt", + }); + let transformCalls = 0; + client.setToolAbortSignal(new AbortController().signal); // live + + const result = await client.mutateLiveContentUnlocked( + "page-1", + "collab-jwt", + (liveDoc) => { + transformCalls++; + return null; + }, + ); + assert.equal( + transformCalls, + 1, + "with a live signal the safe-point is a no-op and session.mutate runs the transform", + ); + assert.ok(result && result.verify, "a MutationResult is returned"); +}); From b50b32bf64394514d521a8a8394db16a25a3db63 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 06:27:55 +0300 Subject: [PATCH 27/41] =?UTF-8?q?fix(ai-chat):=20=D0=B2=D0=B0=D0=BB=D0=B8?= =?UTF-8?q?=D0=B4=D0=B0=D1=86=D0=B8=D1=8F=20=D0=BA=D0=BB=D0=B8=D0=B5=D0=BD?= =?UTF-8?q?=D1=82=D1=81=D0=BA=D0=B8=D1=85=20parts=20+=20insert=20=D0=BF?= =?UTF-8?q?=D0=BE=D1=81=D0=BB=D0=B5=20=D0=BA=D0=BE=D0=BD=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D1=82=D0=B0=D1=86=D0=B8=D0=B8=20(#489)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Юзер-сообщение персистилось в metadata БЕЗ валидации и реплеилось через convertToModelMessages на каждом ходе; insert user-строки шёл ДО конвертации. Битая структура parts (например, null-элемент в массиве) → конвертация кидает → 500 на КАЖДОМ ходе навсегда, а каждый ретрай добавлял дубль user-строки. - Санитизация parts при приёме: whitelist { text }, прочее (в т.ч. tool-part в input-available) отбрасывается с warn — не попадает в metadata. - convertToModelMessages прогоняется ДО insert'а user-строки (ретрай не плодит дубли); при падении на СТАРОЙ истории — per-row конвертация изолирует битую строку и деградирует её до plain-text с маркером «[tool context omitted]» (молчаливая потеря tool-контекста недопустима). Тесты против РЕАЛЬНОГО convertToModelMessages (null-part реально кидает): unit трёх веток + сервис-регресс «чат с битым сообщением в истории работает, маркер доходит до модели, одна user-строка». Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/ai-chat.parts-resilience.spec.ts | 142 ++++++++++++++++ .../src/core/ai-chat/ai-chat.service.spec.ts | 86 +++++++++- .../src/core/ai-chat/ai-chat.service.ts | 156 +++++++++++++++--- 3 files changed, 362 insertions(+), 22 deletions(-) create mode 100644 apps/server/src/core/ai-chat/ai-chat.parts-resilience.spec.ts diff --git a/apps/server/src/core/ai-chat/ai-chat.parts-resilience.spec.ts b/apps/server/src/core/ai-chat/ai-chat.parts-resilience.spec.ts new file mode 100644 index 00000000..d0b66432 --- /dev/null +++ b/apps/server/src/core/ai-chat/ai-chat.parts-resilience.spec.ts @@ -0,0 +1,142 @@ +// #489 — client-parts validation + resilient history conversion. +// +// These unit tests exercise the two exported helpers against the REAL +// `convertToModelMessages` from `ai` (NOT a mock): a genuinely malformed part +// (a `null` element inside a parts array) makes the real converter throw +// ("Cannot read properties of null"), which is the actual production +// "bricked chat" mechanism this fix defends against. Asserting against the real +// converter (rather than a mock-shaped error) is the whole point — a mock would +// hide a version change in the converter's throw behaviour. +import { convertToModelMessages, type UIMessage } from 'ai'; +import { + sanitizeUserParts, + convertHistoryResilient, + TOOL_CONTEXT_OMITTED_MARKER, +} from './ai-chat.service'; + +type Row = Omit & { id: string }; + +describe('sanitizeUserParts (#489, branch: validation on receipt)', () => { + it('keeps whitelisted text parts unchanged', () => { + const drops: string[] = []; + const out = sanitizeUserParts( + [ + { type: 'text', text: 'a' }, + { type: 'text', text: 'b' }, + ] as UIMessage['parts'], + (t) => drops.push(t), + ); + expect(out).toEqual([ + { type: 'text', text: 'a' }, + { type: 'text', text: 'b' }, + ]); + expect(drops).toEqual([]); + }); + + it('drops a non-text part (a tool-part in input-available) and reports its type', () => { + const drops: string[] = []; + const out = sanitizeUserParts( + [ + { type: 'text', text: 'hi' }, + { + type: 'tool-getPage', + toolCallId: 't1', + state: 'input-available', + input: { pageId: 'p' }, + }, + ] as unknown as UIMessage['parts'], + (t) => drops.push(t), + ); + expect(out).toEqual([{ type: 'text', text: 'hi' }]); + expect(drops).toEqual(['tool-getPage']); + }); + + it('drops a null part (the shape that would poison convertToModelMessages)', () => { + const drops: string[] = []; + const out = sanitizeUserParts( + [{ type: 'text', text: 'hi' }, null] as unknown as UIMessage['parts'], + (t) => drops.push(t), + ); + expect(out).toEqual([{ type: 'text', text: 'hi' }]); + expect(drops).toEqual(['(unknown)']); + }); + + it('returns undefined when nothing survives (so a null metadata is persisted)', () => { + const out = sanitizeUserParts( + [ + { type: 'tool-x', toolCallId: 't', state: 'input-available' }, + ] as unknown as UIMessage['parts'], + () => undefined, + ); + expect(out).toBeUndefined(); + }); + + it('returns undefined for a non-array input', () => { + expect( + sanitizeUserParts(undefined as unknown as UIMessage['parts'], () => undefined), + ).toBeUndefined(); + }); +}); + +describe('convertHistoryResilient (#489, branches: happy + per-row degradation)', () => { + it('happy path: healthy history converts identically to convertToModelMessages, no degrade', async () => { + const history: Row[] = [ + { id: 'u1', role: 'user', parts: [{ type: 'text', text: 'hi' }] }, + { id: 'a1', role: 'assistant', parts: [{ type: 'text', text: 'hello' }] }, + ]; + const degrades: number[] = []; + const out = await convertHistoryResilient(history, (i) => degrades.push(i)); + const expected = await convertToModelMessages(history as UIMessage[]); + expect(out).toEqual(expected); + expect(degrades).toEqual([]); + }); + + it('REAL poison: a null part throws in the batch converter but is isolated and degraded to a marker', async () => { + // Sanity: the real converter genuinely throws on this shape. + const poisoned: Row = { + id: 'a1', + role: 'assistant', + parts: [ + { type: 'text', text: 'earlier answer' }, + null, + ] as unknown as UIMessage['parts'], + }; + await expect( + convertToModelMessages([poisoned as UIMessage]), + ).rejects.toThrow(); + + const history: Row[] = [ + { id: 'u1', role: 'user', parts: [{ type: 'text', text: 'first' }] }, + poisoned, + { id: 'u2', role: 'user', parts: [{ type: 'text', text: 'second' }] }, + ]; + const degrades: number[] = []; + const out = await convertHistoryResilient(history, (i) => degrades.push(i)); + + // Only the poisoned row (index 1) is degraded. + expect(degrades).toEqual([1]); + // Healthy rows survive verbatim. + const flat = JSON.stringify(out); + expect(flat).toContain('first'); + expect(flat).toContain('second'); + // The degraded row carries its readable text AND the truncation marker so the + // model sees that tool context was omitted (never a silent loss). + expect(flat).toContain('earlier answer'); + expect(flat).toContain(TOOL_CONTEXT_OMITTED_MARKER); + // The whole batch converted (3 model messages, none dropped). + expect(out).toHaveLength(3); + }); + + it('a fully-poisoned row (no readable text) still degrades to just the marker', async () => { + const history: Row[] = [ + { + id: 'a1', + role: 'assistant', + parts: [null] as unknown as UIMessage['parts'], + }, + ]; + const out = await convertHistoryResilient(history, () => undefined); + expect(out).toHaveLength(1); + expect(JSON.stringify(out)).toContain(TOOL_CONTEXT_OMITTED_MARKER); + }); +}); diff --git a/apps/server/src/core/ai-chat/ai-chat.service.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.spec.ts index ca9511ef..8dd32123 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.spec.ts @@ -1440,7 +1440,7 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', () } // Wire only the deps reached on the way to the pipe call, plus a spy registry. - function makeService(opts: { resumable: boolean }) { + function makeService(opts: { resumable: boolean; history?: unknown[] }) { const aiChatRepo = { findById: jest.fn(async () => ({ id: 'chat-1', workspaceId: 'ws-1' })), insert: jest.fn(), @@ -1448,7 +1448,7 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', () const aiChatMessageRepo = { // Both the user insert and the assistant seed return the same row id. insert: jest.fn(async () => ({ id: 'msg-1' })), - findAllByChat: jest.fn(async () => []), + findAllByChat: jest.fn(async () => opts.history ?? []), update: jest.fn(async () => ({ id: 'msg-1' })), // #487: the terminal owner-write + the opportunistic reconcile query. finalizeOwner: jest.fn(async () => ({ id: 'msg-1' })), @@ -1487,7 +1487,7 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', () } as never, streamRegistry as never, ); - return { svc, streamRegistry }; + return { svc, streamRegistry, aiChatMessageRepo }; } const body = { @@ -1570,6 +1570,86 @@ describe('AiChatService.stream — resumable pipe options (#184 phase 1.5)', () await expect(drive(svc, makeRunHooks())).rejects.toThrow('boom'); expect(streamRegistry.abortEntry).toHaveBeenCalledWith('chat-1', 'run-1'); }); + + // #489 REGRESSION (against the REAL convertToModelMessages — not mocked here): + // a persisted history row whose parts contain a `null` element makes the real + // convertToModelMessages THROW ("Cannot read properties of null"). Pre-fix that + // 500-ed every turn forever and each retry appended a duplicate user row. The + // fix converts BEFORE the insert and isolates the poisoned row per-row, degrading + // it to text with a "[tool context omitted]" marker. Assert the turn still runs, + // the marker reaches the model, and exactly ONE user row is inserted. + it('#489: a poisoned OLD-history row keeps the chat working; the marker reaches the model; one user insert', async () => { + const { svc, aiChatMessageRepo } = makeService({ + resumable: false, + history: [ + { + id: 'old-1', + role: 'assistant', + content: 'earlier answer', + // A null part is the poison: rowToUiMessage keeps it (the array is + // non-empty) and the real convertToModelMessages throws on it. + metadata: { parts: [{ type: 'text', text: 'earlier answer' }, null] }, + status: 'completed', + }, + ], + }); + // Must NOT throw — the poisoned row is degraded, not fatal. + await drive(svc, makeRunHooks()); + expect(streamTextMock).toHaveBeenCalledTimes(1); + const passedMessages = streamTextMock.mock.calls[0][0].messages; + const serialized = JSON.stringify(passedMessages); + // The model sees the truncation marker (silent tool-context loss is not ok) + // AND the row's readable text is preserved alongside it. + expect(serialized).toContain('[tool context omitted]'); + expect(serialized).toContain('earlier answer'); + // Exactly ONE user row inserted (no duplicate), inserted AFTER conversion. + const userInserts = aiChatMessageRepo.insert.mock.calls + .map((c: unknown[]) => c[0] as { role?: string }) + .filter((r) => r.role === 'user'); + expect(userInserts).toHaveLength(1); + }); + + // #489: client-supplied non-text parts (a tool-part in `input-available`, the + // exact "bricking" payload) are dropped ON RECEIPT — never persisted — so they + // can never poison future turns. Only the text survives into metadata.parts. + it('#489: a non-text client part is stripped before persist (only text survives)', async () => { + const { svc, aiChatMessageRepo } = makeService({ resumable: false }); + await svc.stream({ + user: { id: 'u1' } as never, + workspace: { id: 'ws-1' } as never, + sessionId: 's1', + body: { + chatId: 'chat-1', + messages: [ + { + id: 'm1', + role: 'user', + parts: [ + { type: 'text', text: 'hello' }, + // untrusted tool-part — must be dropped, never persisted + { + type: 'tool-getPage', + toolCallId: 't1', + state: 'input-available', + input: { pageId: 'p' }, + }, + ], + }, + ], + } as never, + res: makeRes() as never, + signal: new AbortController().signal, + model: {} as never, + role: null, + runHooks: makeRunHooks() as never, + }); + const userInsert = aiChatMessageRepo.insert.mock.calls + .map((c: unknown[]) => c[0] as { role?: string; metadata?: unknown }) + .find((r) => r.role === 'user'); + const parts = (userInsert?.metadata as { parts?: Array<{ type: string }> }) + ?.parts; + expect(parts).toEqual([{ type: 'text', text: 'hello' }]); + }); }); /** diff --git a/apps/server/src/core/ai-chat/ai-chat.service.ts b/apps/server/src/core/ai-chat/ai-chat.service.ts index 02ecc9d1..66d21473 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.ts @@ -14,6 +14,7 @@ import { convertToModelMessages, stepCountIs, type UIMessage, + type ModelMessage, type LanguageModel, } from 'ai'; import { AiService } from '../../integrations/ai/ai.service'; @@ -1042,7 +1043,58 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { const incoming = lastUserMessage(body.messages); const incomingText = uiMessageText(incoming); - // Persist the user message before contacting the model. + // #489: sanitize client-supplied parts ON RECEIPT. The client only ever + // sends `sendMessage({ text })` (a single text part); there is no + // file/attachment path. Any other part — most dangerously a tool-part in + // `input-available` state — is untrusted data that, once persisted to + // `metadata.parts` verbatim, is REPLAYED through convertToModelMessages on + // every later turn. A malformed tool-part makes that conversion throw, + // 500-ing every future turn of the chat forever ("bricked"). Drop any + // non-whitelisted part with a warn. + const sanitizedParts = sanitizeUserParts(incoming?.parts, (type) => + this.logger.warn( + `Dropping unsupported user message part '${type}' on chat ${chatId}`, + ), + ); + + // #489: rebuild the conversation from persisted history (not the client + // payload) and CONVERT it to model messages BEFORE persisting the user row. + // Load the OLD history (WITHOUT the new row) and append the incoming turn in + // memory for the conversion. This makes the insert happen only after a + // successful conversion, so a conversion failure cannot leave a DUPLICATE + // user row behind on the client's retry (the "bricked chat" that accreted a + // dup on every 500). `findAllByChat` returns chronological order (oldest -> + // newest) and keeps a 5000-row memory-safety backstop (on overflow it keeps + // the NEWEST rows and logs a warning); that is a safety net far above any + // realistic chat, not a conversational limit. + const oldHistory = await this.aiChatMessageRepo.findAllByChat( + chatId, + workspace.id, + ); + const uiMessages: Array & { id: string }> = [ + ...oldHistory.map(rowToUiMessage), + { + id: 'pending-user', + role: 'user', + parts: (sanitizedParts && sanitizedParts.length > 0 + ? sanitizedParts + : textPart(incomingText)) as UIMessage['parts'], + }, + ]; + // convertToModelMessages is async in ai@6.0.134 (returns Promise). + // Resilient (#489): a single poisoned row in the OLD history is isolated via + // per-row conversion and degraded to plain text with a "[tool context + // omitted]" marker rather than 500-ing the whole turn (silent loss of tool + // context is not acceptable — the model must see the truncation). + const messages = await convertHistoryResilient(uiMessages, (index, err) => + this.logger.warn( + `Degraded unconvertible history row ${index} on chat ${chatId} to text: ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ), + ); + + // Persist the user message only AFTER a successful conversion (#489). await this.aiChatMessageRepo.insert({ chatId, workspaceId: workspace.id, @@ -1050,31 +1102,21 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { role: 'user', content: incomingText, // jsonb column: UIMessage parts are JSON-serializable at runtime but not - // structurally `JsonValue`, so cast through unknown. - metadata: (incoming?.parts ? { parts: incoming.parts } : null) as never, + // structurally `JsonValue`, so cast through unknown. Persist the SANITIZED + // parts (never the raw client parts) so the row is always convertible. + metadata: (sanitizedParts ? { parts: sanitizedParts } : null) as never, }); - // Rebuild the conversation from persisted history (not the client payload), - // so the model always sees the authoritative server-side transcript. Load - // the FULL history in chronological order (oldest -> newest, incl. the user - // message just inserted above) so NO turns are dropped — there is no - // recent-tail window anymore. `findAllByChat` keeps a 5000-row memory-safety - // backstop (on overflow it keeps the NEWEST rows and logs a warning); that - // is a safety net far above any realistic chat, not a conversational limit. - const history = await this.aiChatMessageRepo.findAllByChat( - chatId, - workspace.id, - ); - const uiMessages = history.map(rowToUiMessage); - // convertToModelMessages is async in ai@6.0.134 (returns Promise). - const messages = await convertToModelMessages(uiMessages); - // Interrupt-resume detection (#198): the client "send now" flag is only a // hint — confirm it against the persisted history (the preceding assistant // turn must really be aborted/streaming) so a spoofed flag cannot inject the // interrupt note onto an ordinary turn. The partial output the model needs is // already in `messages` (the aborted assistant row replays via findRecent). - const interrupted = isInterruptResume(history, body.interrupted); + // Append the new user turn (shape-only) so index -2 is the prior assistant. + const interrupted = isInterruptResume( + [...oldHistory, { role: 'user', status: null, metadata: null }], + body.interrupted, + ); // Per-turn page-change detection (#274): if the open page was hand-edited by // the user since the agent's last turn ended, compute the unified diff so the @@ -2074,6 +2116,82 @@ function textPart(text: string): Array<{ type: 'text'; text: string }> { return text ? [{ type: 'text', text }] : []; } +/** + * Part types accepted on an INCOMING user turn (#489). The client only ever + * sends `sendMessage({ text })` (a single text part); there is no file/attachment + * path. Everything else on a client-supplied user message — most dangerously a + * tool-part in `input-available` state — is untrusted data that would be + * persisted to `metadata.parts` verbatim and replayed through + * `convertToModelMessages` on every later turn, potentially bricking the chat. + */ +const ALLOWED_USER_PART_TYPES: ReadonlySet = new Set(['text']); + +/** + * Keep only whitelisted parts on a client-supplied user message; report each + * dropped part's type via `onDrop` (the caller warns). Returns `undefined` when + * nothing survives (no parts / none whitelisted), so the caller persists a null + * metadata rather than an empty-parts object. Never throws. + */ +export function sanitizeUserParts( + parts: UIMessage['parts'] | undefined, + onDrop: (type: string) => void, +): UIMessage['parts'] | undefined { + if (!Array.isArray(parts)) return undefined; + const kept = parts.filter((p) => { + const type = + typeof (p as { type?: unknown })?.type === 'string' + ? (p as { type: string }).type + : ''; + if (ALLOWED_USER_PART_TYPES.has(type)) return true; + onDrop(type || '(unknown)'); + return false; + }); + return kept.length > 0 ? (kept as UIMessage['parts']) : undefined; +} + +/** Marker for a history row whose tool parts could not be replayed (#489). */ +export const TOOL_CONTEXT_OMITTED_MARKER = '[tool context omitted]'; + +/** + * Convert persisted UI history to model messages, tolerating a single poisoned + * row (#489). `convertToModelMessages` over the WHOLE array throws if ANY row is + * malformed (e.g. a tool-part left unbalanced / in `input-available` state), + * which would otherwise 500 every turn of the chat forever. On a batch failure we + * fall back to per-row conversion so the bad row is isolated: it is degraded to + * plain text carrying its readable text plus a `[tool context omitted]` marker + * (the model MUST see that its tool context was truncated — silent loss is not + * acceptable), while every healthy row converts normally. Because AI SDK v6 + * carries a tool call and its result inside the SAME assistant UIMessage's parts, + * per-row conversion preserves call/result pairing. + */ +export async function convertHistoryResilient( + uiMessages: Array & { id: string }>, + onDegrade: (index: number, err: unknown) => void, +): Promise { + try { + return await convertToModelMessages(uiMessages as UIMessage[]); + } catch { + const out: ModelMessage[] = []; + for (let i = 0; i < uiMessages.length; i++) { + const m = uiMessages[i]; + try { + out.push(...(await convertToModelMessages([m as UIMessage]))); + } catch (err) { + onDegrade(i, err); + const text = uiMessageText(m as UIMessage); + const degraded = text + ? `${text}\n\n${TOOL_CONTEXT_OMITTED_MARKER}` + : TOOL_CONTEXT_OMITTED_MARKER; + out.push({ + role: m.role === 'assistant' ? 'assistant' : 'user', + content: degraded, + } as ModelMessage); + } + } + return out; + } +} + /** * Minimal shapes of the AI SDK v6 step objects we read to rebuild UIMessage * parts (see ai@6.0.134 `StepResult`: `text`, `toolCalls` -> TypedToolCall, From 1b05224b27a9c03bcd6db6a36af5cc6f11ff019d Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:00:38 +0300 Subject: [PATCH 28/41] =?UTF-8?q?fix(ai-chat):=20MCP-=D0=BA=D1=8D=D1=88=20?= =?UTF-8?q?=E2=80=94=20in-run=20=D0=B2=D0=BE=D1=81=D1=81=D1=82=D0=B0=D0=BD?= =?UTF-8?q?=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5,=20=D1=80=D0=B5?= =?UTF-8?q?=D1=82=D1=80=D0=B0=D0=B8=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE?= =?UTF-8?q?=20=D0=B4=D0=BB=D1=8F=20readOnly=20(#489)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Кэш внешних MCP-клиентов не делал health-check/reconnect-on-error: после разрыва SSE-транспорта (bodyTimeout режет при тишине МЕЖДУ вызовами) отдавал труп до конца TTL, а модель жгла шаги на ретраях ВНУТРИ текущего рана. - Новое поле writeClass ('readOnly'|'write') на КАЖДОМ SHARED_TOOL_SPEC + registration-time assert (и compile-time через satisfies). Все 48 спеков расклассифицированы: чтения → readOnly, любая мутация страницы/коммента/шэра/ диаграммы → write. Экспортированы SHARED_TOOL_WRITE_CLASS + isRetryableWriteClass. - Per-run обёртка восстановления транспорта: при транспортной ошибке readOnly-тул реконнектит свой сервер и ретраит РОВНО 1 раз ВНУТРИ рана; write-тул НЕ авторетраится (indeterminate — «могло примениться, проверь», класс инцидента #435). CAS-своп байндинга по identity (проигравший конкурентный вызов ретраит на текущем клиенте, не минтит второй). Лизы не освобождаются mid-run — ран копит set (старая+новая) и релизит на turn-end. - Проверка abortSignal ПЕРЕД ретраем И ПЕРЕД чеканкой свежего клиента; per-call cap покрывает оба attempt'а + connect. - Классификация транспортной ошибки по РЕАЛЬНЫМ шейпам undici (SocketError/ BodyTimeoutError, cause-цепочка), не по мок-ошибкам. - Отдельный, поднятый bodyTimeout для SSE-транспорта MCP (тишина между вызовами легальна) — DEFAULT 10 мин, AI_MCP_SSE_BODY_TIMEOUT_MS. write-class map грузится в mcp-clients лениво через dynamic import (пакет ESM), type-only импорт — без static require ESM из commonjs. Тесты на РЕАЛЬНЫХ error-шейпах: «повтор после обрыва ВНУТРИ рана получает живой клиент», «write-тул не авторетраится», «ретрай после Stop не происходит», + writeClass-контракт в mcp node --test. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../external-mcp/mcp-clients.recovery.spec.ts | 214 +++++++++ .../external-mcp/mcp-clients.service.spec.ts | 5 +- .../external-mcp/mcp-clients.service.ts | 447 ++++++++++++++++-- .../src/integrations/ai/ai-streaming-fetch.ts | 26 + packages/mcp/src/index.ts | 7 + packages/mcp/src/tool-specs.ts | 110 +++++ packages/mcp/test/unit/tool-specs.test.mjs | 42 +- 7 files changed, 818 insertions(+), 33 deletions(-) create mode 100644 apps/server/src/core/ai-chat/external-mcp/mcp-clients.recovery.spec.ts diff --git a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.recovery.spec.ts b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.recovery.spec.ts new file mode 100644 index 00000000..5b129a58 --- /dev/null +++ b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.recovery.spec.ts @@ -0,0 +1,214 @@ +import { errors } from 'undici'; +import { + McpClientsService, + isRetryableConnectError, +} from './mcp-clients.service'; + +/** + * #489 — external-MCP in-run transport recovery. + * + * The transport-error classification + retry gate are exercised against the REAL + * undici error CLASSES prod throws (`errors.SocketError` / `errors.BodyTimeoutError`, + * carrying the true `UND_ERR_*` codes and class names), wrapped EXACTLY as undici's + * `fetch` wraps them — a `TypeError('fetch failed'|'terminated')` whose `.cause` is + * the undici error. These are the real classes, not hand-rolled `{code:'...'}` + * mocks: constructing the genuine class is what makes this a faithful test of the + * prod predicate (epic root-cause #4 — a mock-shaped predicate would leave the + * evict/retry path silently dead in production while CI stays green). We construct + * rather than drive a live fetch because Jest's environment degrades the live-fetch + * error to a generic `Error` cause (no undici code), which would NOT be the prod + * shape. + */ + +/** A REAL undici socket reset, wrapped as fetch wraps it. */ +function realSocketResetError(): unknown { + const err = new TypeError('fetch failed'); + (err as { cause?: unknown }).cause = new errors.SocketError('other side closed'); + return err; +} + +/** A REAL undici body timeout, wrapped as fetch wraps it. */ +function realBodyTimeoutError(): unknown { + const err = new TypeError('terminated'); + (err as { cause?: unknown }).cause = new errors.BodyTimeoutError(); + return err; +} + +type FakeServer = { + id: string; + name: string; + transport: 'http' | 'sse'; + url: string; + headersEnc: string | null; + toolAllowlist: string[] | null; + instructions: string | null; +}; + +const server = (over: Partial = {}): FakeServer => ({ + id: 's1', + name: 'srv', + transport: 'http', + url: 'http://example.test/mcp', + headersEnc: null, + toolAllowlist: null, + instructions: null, + ...over, +}); + +function buildService(servers: FakeServer[]) { + const repo = { listEnabled: jest.fn().mockResolvedValue(servers) }; + const service = new McpClientsService(repo as never, {} as never); + // Seed a DETERMINISTIC write-class map so the retry gate is controlled here + // (the production map loads from @docmost/mcp via a dynamic ESM import). getPage + // is a read, patchNode is a write — the real classifications. + ( + service as unknown as { writeClassMapPromise: Promise } + ).writeClassMapPromise = Promise.resolve({ + getPage: 'readOnly', + patchNode: 'write', + }); + return { service, repo }; +} + +/** Spy the private `connect` so each call yields a controlled fake client whose + * single tool's execute is the supplied function. Returns the connect spy. */ +function stubConnect( + service: McpClientsService, + toolName: string, + execs: Array<(...a: unknown[]) => Promise>, +) { + let n = 0; + return jest + .spyOn( + service as unknown as { connect: (s: FakeServer) => Promise }, + 'connect', + ) + .mockImplementation(async () => { + const exec = execs[Math.min(n, execs.length - 1)]; + n += 1; + return { + tools: async () => ({ [toolName]: { description: 'x', execute: exec } }), + close: jest.fn().mockResolvedValue(undefined), + }; + }); +} + +const opts = (abortSignal?: AbortSignal) => + ({ toolCallId: 't', messages: [], abortSignal }) as never; + +describe('isRetryableConnectError (#489, REAL error shapes)', () => { + it('classifies a real undici socket reset and body timeout as retryable', async () => { + const socketErr = await realSocketResetError(); + const bodyErr = await realBodyTimeoutError(); + expect(isRetryableConnectError(socketErr)).toBe(true); + expect(isRetryableConnectError(bodyErr)).toBe(true); + // Unwraps a wrapped cause chain (e.g. an MCPClientError around the socket err). + const wrapped = new Error('mcp call failed'); + (wrapped as { cause?: unknown }).cause = socketErr; + expect(isRetryableConnectError(wrapped)).toBe(true); + }); + + it('does NOT classify an application-level error as a transport break', () => { + expect(isRetryableConnectError(new Error('validation failed'))).toBe(false); + expect(isRetryableConnectError({ name: 'HttpError', status: 400 })).toBe(false); + expect(isRetryableConnectError(undefined)).toBe(false); + expect(isRetryableConnectError('boom')).toBe(false); + }); +}); + +describe('McpClientsService in-run transport recovery (#489)', () => { + afterEach(() => jest.restoreAllMocks()); + + it('a readOnly tool whose transport breaks reconnects and retries WITHIN the same run', async () => { + const realErr = await realSocketResetError(); + const { service } = buildService([server()]); + const first = jest.fn().mockRejectedValue(realErr); + const second = jest.fn().mockResolvedValue({ ok: true }); + const connectSpy = stubConnect(service, 'getPage', [first, second]); + + const toolset = await service.toolsFor('ws-1'); + const tool = toolset.tools['srv_getPage']; + const result = await (tool.execute as (a: unknown, o: unknown) => Promise)( + { pageId: 'p' }, + opts(), + ); + + // The repeat call within the run got a LIVE client and succeeded. + expect(result).toEqual({ ok: true }); + expect(first).toHaveBeenCalledTimes(1); + expect(second).toHaveBeenCalledTimes(1); + // Exactly one reconnect was minted (initial build connect + one recovery). + expect(connectSpy).toHaveBeenCalledTimes(2); + // The run accumulated BOTH leases (old + reconnected) — released together at end. + expect(toolset.clients).toHaveLength(2); + await Promise.all(toolset.clients.map((c) => c.close())); + }); + + it('a WRITE tool does NOT auto-retry on a transport error (indeterminate)', async () => { + const realErr = await realSocketResetError(); + const { service } = buildService([server()]); + const exec = jest.fn().mockRejectedValue(realErr); + const connectSpy = stubConnect(service, 'patchNode', [exec]); + + const toolset = await service.toolsFor('ws-2'); + const tool = toolset.tools['srv_patchNode']; + await expect( + (tool.execute as (a: unknown, o: unknown) => Promise)( + { pageId: 'p' }, + opts(), + ), + ).rejects.toThrow(/MAY have already applied/); + + // Called exactly once — NO blind retry (avoids double-apply, the #435 class). + expect(exec).toHaveBeenCalledTimes(1); + // No fresh connection was minted for a write. + expect(connectSpy).toHaveBeenCalledTimes(1); + await Promise.all(toolset.clients.map((c) => c.close())); + }); + + it('does NOT retry (or reconnect) after the run is aborted (Stop)', async () => { + const realErr = await realSocketResetError(); + const { service } = buildService([server()]); + const controller = new AbortController(); + // The transport error arrives, but the run was Stopped in the same tick. + const first = jest.fn().mockImplementation(async () => { + controller.abort(); + throw realErr; + }); + const second = jest.fn().mockResolvedValue({ ok: true }); + const connectSpy = stubConnect(service, 'getPage', [first, second]); + + const toolset = await service.toolsFor('ws-3'); + const tool = toolset.tools['srv_getPage']; + await expect( + (tool.execute as (a: unknown, o: unknown) => Promise)( + { pageId: 'p' }, + opts(controller.signal), + ), + ).rejects.toBeDefined(); + + // getPage IS readOnly, but the Stop blocks the retry — no second call, no mint. + expect(second).not.toHaveBeenCalled(); + expect(connectSpy).toHaveBeenCalledTimes(1); + await Promise.all(toolset.clients.map((c) => c.close())); + }); + + it('an app-level (non-transport) tool error is surfaced verbatim, never retried', async () => { + const { service } = buildService([server()]); + const appErr = new Error('tool says: bad input'); + const exec = jest.fn().mockRejectedValue(appErr); + const connectSpy = stubConnect(service, 'getPage', [exec]); + + const toolset = await service.toolsFor('ws-4'); + const tool = toolset.tools['srv_getPage']; + await expect( + (tool.execute as (a: unknown, o: unknown) => Promise)( + { pageId: 'p' }, + opts(), + ), + ).rejects.toThrow('tool says: bad input'); + expect(exec).toHaveBeenCalledTimes(1); + expect(connectSpy).toHaveBeenCalledTimes(1); // no reconnect for an app error + await Promise.all(toolset.clients.map((c) => c.close())); + }); +}); diff --git a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.spec.ts b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.spec.ts index 53ad6191..270b2df1 100644 --- a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.spec.ts +++ b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.spec.ts @@ -106,8 +106,11 @@ describe('McpClientsService.decryptHeaders', () => { describe('McpClientsService.guardedFetch (SSRF per-request guard)', () => { // The bound guardedFetch closure lives on the instance as a private field. + // #489 split it into per-transport HTTP/SSE bindings (they differ only in the + // dispatcher's bodyTimeout); the SSRF guard is identical, so testing the HTTP + // one is sufficient. const guardedFetchOf = (service: McpClientsService) => - (service as unknown as { guardedFetch: typeof fetch }).guardedFetch; + (service as unknown as { guardedFetchHttp: typeof fetch }).guardedFetchHttp; let fetchSpy: jest.SpiedFunction; diff --git a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts index 1e688974..4c874ccf 100644 --- a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts +++ b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts @@ -1,5 +1,6 @@ import { isIP } from 'node:net'; import { lookup as dnsLookup, type LookupAddress } from 'node:dns'; +import { pathToFileURL } from 'node:url'; import { Injectable, Logger } from '@nestjs/common'; import { type Tool, type ToolCallOptions } from 'ai'; import { createMCPClient } from '@ai-sdk/mcp'; @@ -10,9 +11,29 @@ import { streamingDispatcherOptions, mcpStreamTimeoutMs, mcpCallTimeoutMs, + mcpSseBodyTimeoutMs, } from '../../../integrations/ai/ai-streaming-fetch'; import { SecretBoxService } from '../../../integrations/crypto/secret-box'; import { isUrlAllowed, isIpAllowed } from './ssrf-guard'; +// TYPE-ONLY (erased at compile): @docmost/mcp is ESM-only and cannot be a runtime +// `require()` from this commonjs module (same constraint as docmost-client.loader). +// The write-class MAP is loaded lazily via the dynamic-import trick below. +import type { ToolWriteClass } from '@docmost/mcp'; + +// TS(commonjs) downlevels a literal `import()` to `require()`, which cannot load +// the ESM-only @docmost/mcp. Indirect through Function so the real dynamic +// `import()` survives compilation (same trick as docmost-client.loader.ts). +const esmImport = new Function( + 'specifier', + 'return import(specifier)', +) as (specifier: string) => Promise; + +/** Local read-only predicate — avoids a value import of the ESM-only package. + * Only a pure read is retry-safe after a transport break (a write is + * indeterminate). Kept in lockstep with @docmost/mcp's isRetryableWriteClass. */ +function isReadOnlyWriteClass(writeClass: ToolWriteClass | undefined): boolean { + return writeClass === 'readOnly'; +} /** A closable external MCP client handle. */ export interface Closable { @@ -81,12 +102,52 @@ const MAX_TOOL_NAME_LENGTH = 64; * close until the turn releases it, so a TTL expiry mid-turn never closes a * client a stream is still executing against. */ +/** + * Where a merged (namespaced) tool came from, so the per-run recovery wrapper + * (#489) can, on a transport error, reconnect THAT server and re-resolve the SAME + * underlying tool by its raw name. `writeClass` gates the single auto-retry (a + * read is retry-safe; a write is indeterminate). `serverIndex` indexes the + * entry's `servers` array (which server config to reconnect). + */ +interface ToolProvenance { + serverIndex: number; + rawName: string; + writeClass: ToolWriteClass | undefined; +} + +/** A live reconnected server (its fresh client + raw call-timeout-wrapped tools). */ +interface RecoveredServerState { + client: McpClient; + tools: Record; +} + +/** + * Per-run, per-server recovery binding (#489). `current` is the server's LIVE + * target for this run: `null` means "use the ORIGINAL cached client/template"; + * a non-null value is a reconnected throwaway client all this server's tools now + * call. `reconnecting` dedupes concurrent reconnects so only ONE fresh client is + * minted per death (a losing concurrent call awaits it and retries on the SAME + * new client — the CAS-by-identity rule). + */ +interface ServerBinding { + current: RecoveredServerState | null; + reconnecting?: Promise; +} + interface CacheEntry { tools: Record; clients: McpClient[]; outcomes: ServerOutcome[]; /** Prompt guidance for qualifying servers (see McpServerInstruction). */ instructions: McpServerInstruction[]; + /** + * The enabled server configs used to build this entry (#489), so the per-run + * recovery wrapper can reconnect a specific server by index. Parallel to the + * indices referenced by {@link toolMeta}. + */ + servers: AiMcpServer[]; + /** merged-tool-key -> provenance (#489), for the per-run recovery wrapper. */ + toolMeta: Record; expiresAt: number; /** Active leases (turns currently using these clients). */ refCount: number; @@ -120,20 +181,65 @@ export class McpClientsService { */ private readonly cache = new Map>(); /** - * A single shared SSRF-pinned dispatcher for ALL outbound external-MCP fetches. - * Its custom connect.lookup runs per connection, so one instance safely guards - * every server's connections (we never connect to an unvalidated IP). + * SSRF-pinned dispatchers for outbound external-MCP fetches. Both use the SAME + * custom connect.lookup (so every connection is IP-validated), but carry a + * DIFFERENT `bodyTimeout` (#489): the HTTP (streamable) transport opens a fresh + * request per call, so it keeps the tight silence timeout; the SSE transport + * holds ONE long-lived body open across many calls, so a >1-min idle BETWEEN + * calls is LEGITIMATE and must not break the socket — it gets a much larger + * bodyTimeout. (headersTimeout stays tight on both.) */ - private readonly dispatcher: Dispatcher = buildPinnedDispatcher(); - /** guardedFetch bound to the pinned dispatcher; reused by every transport. */ - private readonly guardedFetch: typeof fetch = (input, init) => - guardedFetch(this.dispatcher, input, init); + private readonly dispatcherHttp: Dispatcher = buildPinnedDispatcher( + mcpStreamTimeoutMs(), + ); + private readonly dispatcherSse: Dispatcher = buildPinnedDispatcher( + mcpSseBodyTimeoutMs(), + ); + /** guardedFetch bound to each dispatcher; picked by transport type in connect(). */ + private readonly guardedFetchHttp: typeof fetch = (input, init) => + guardedFetch(this.dispatcherHttp, input, init); + private readonly guardedFetchSse: typeof fetch = (input, init) => + guardedFetch(this.dispatcherSse, input, init); + + /** + * Memoized write-class map (#489), loaded lazily from @docmost/mcp via the + * dynamic-import trick. Keyed by tool name (=== mcpName). A tool NOT in the map + * (any third-party external MCP tool) classifies as `undefined` -> treated as a + * write by the retry gate (the safe default: never blind-retry an unknown tool). + * On any load failure the map is `{}` (every tool -> no auto-retry), so a + * missing/older @docmost/mcp build only DISABLES retries, never mis-retries. + */ + private writeClassMapPromise: Promise> | null = + null; constructor( private readonly repo: AiMcpServerRepo, private readonly secretBox: SecretBoxService, ) {} + /** Lazily load + memoize the shared write-class map (see the field doc). */ + private getWriteClassMap(): Promise> { + if (!this.writeClassMapPromise) { + this.writeClassMapPromise = (async () => { + try { + const entry = require.resolve('@docmost/mcp'); + const mod = (await esmImport(pathToFileURL(entry).href)) as { + SHARED_TOOL_WRITE_CLASS?: Record; + }; + return mod.SHARED_TOOL_WRITE_CLASS ?? {}; + } catch (err) { + this.logger.warn( + `Could not load MCP write-class map (auto-retry disabled): ${shortError( + err, + )}`, + ); + return {}; + } + })(); + } + return this.writeClassMapPromise; + } + /** * Build (or reuse a cached) external toolset for a workspace. Returns the * merged tools, the open client handles to release, and per-server outcomes. @@ -162,11 +268,37 @@ export class McpClientsService { } }, }; - // One release handle drives the whole leased entry; closing it releases all - // underlying clients together (they share the same lease lifecycle). + + // #489: the run accumulates a SET of leases — the primary cache lease PLUS any + // throwaway client minted by an in-run transport-recovery reconnect. They are + // NEVER released mid-run (releasing a swapped-out client while a concurrent + // in-flight call still holds it would INDUCE a second failure); the caller + // releases the WHOLE set together at turn-end. A recovery reconnect pushes its + // lease onto this live array, which the consumer closes over. + const leaseSet: Closable[] = [release]; + + // #489: per-RUN transport-recovery binding, one per server, SHARED by all of + // that server's tools so a swap by one call is seen by the next (CAS by + // identity). Kept per-run (here, not in the cached entry) because the binding + // + lease-set state is per-run. + const bindings = new Map(); + const capMs = mcpCallTimeoutMs(); + + // Wrap each cached tool with the recovery layer. On a transport error a + // declared readOnly tool reconnects its server and retries ONCE; a write is + // never blind-retried (indeterminate — may have applied before the reset). A + // tool without provenance (a minimal stub entry in a test) passes through raw. + const tools: Record = {}; + for (const [key, tool] of Object.entries(entry.tools)) { + const meta = entry.toolMeta?.[key]; + tools[key] = meta + ? this.wrapWithTransportRecovery(entry, meta, tool, leaseSet, bindings, capMs) + : tool; + } + return { - tools: entry.tools, - clients: [release], + tools, + clients: leaseSet, outcomes: entry.outcomes, instructions: entry.instructions, }; @@ -254,6 +386,11 @@ export class McpClientsService { // Per-call total wall-clock cap, read once for this build (env-overridable). const callTimeoutMs = mcpCallTimeoutMs(); const instructions: McpServerInstruction[] = []; + // merged-key -> provenance for the per-run recovery wrapper (#489). + const toolMeta: Record = {}; + // Shared write-class map (#489): classifies each merged tool by its raw name + // so the recovery wrapper knows which tools are retry-safe reads. + const writeClassMap = await this.getWriteClassMap(); // Per-server connect+tools result, still tagged with its server so the merge // below can be applied in the SAME order as `servers` (see the parallel note). @@ -332,6 +469,9 @@ export class McpClientsService { result.guarded, server.name, server.id, + toolMeta, + i, + writeClassMap, ); outcomes.push({ name: server.name, ok: true }); // Include this server's guidance ONLY when it actually contributed at @@ -353,6 +493,8 @@ export class McpClientsService { clients, outcomes, instructions, + servers, + toolMeta, expiresAt: Date.now() + CACHE_TTL_MS, refCount: 0, evicted: false, @@ -379,18 +521,30 @@ export class McpClientsService { picked: Record, serverName: string, serverId: string, + toolMeta: Record, + serverIndex: number, + writeClassMap: Record, ): { count: number; prefix: string } { let count = 0; - for (const [name, tool] of Object.entries(namespace(picked, serverName))) { - let key = name; + for (const { full, raw, tool } of namespace(picked, serverName)) { + let key = full; if (key in target) { const original = key; - key = disambiguate(name, serverId, (candidate) => candidate in target); + key = disambiguate(full, serverId, (candidate) => candidate in target); this.logger.debug( `External MCP tool name "${original}" collided; renamed to "${key}"`, ); } target[key] = tool; + // Record provenance so the per-run recovery wrapper (#489) can reconnect + // this tool's server and re-resolve it by its raw name, and gate the retry + // on its write-class (unknown third-party tool -> undefined -> treated as a + // write, i.e. never auto-retried). + toolMeta[key] = { + serverIndex, + rawName: raw, + writeClass: writeClassMap[raw], + }; count += 1; } return { count, prefix: namespacePrefix(serverName) }; @@ -424,7 +578,10 @@ export class McpClientsService { // Defense in depth: re-validate the actual request host on EVERY fetch // AND pin the socket to a validated IP via the dispatcher's connect // lookup, closing the DNS-rebinding TOCTOU between check and connect. - fetch: this.guardedFetch, + // #489: the SSE transport uses the raised-bodyTimeout dispatcher (idle + // between calls is legit); HTTP uses the tight one. + fetch: + transportType === 'sse' ? this.guardedFetchSse : this.guardedFetchHttp, }, })) as unknown as McpClient; return client; @@ -505,6 +662,168 @@ export class McpClientsService { } } + /** + * Wrap one merged external tool with the per-run transport-recovery layer (#489). + * + * attempt 1 runs on the server's CURRENT binding (the cached client, or a client + * a sibling tool already reconnected this run). On a REAL transport error + * (undici/@ai-sdk socket/body-timeout shapes — {@link isRetryableConnectError}, + * NOT a mock) and ONLY for a declared readOnly tool, it reconnects the server + * and retries EXACTLY ONCE on the fresh client; a write is surfaced as an + * indeterminate error (it may have applied before the reset — never + * blind-retried). A single per-call cap bounds BOTH attempts + the reconnect, + * and the run's abort signal is checked before the retry AND before minting a + * fresh connection (no connection is opened for a stopped run). + */ + private wrapWithTransportRecovery( + entry: CacheEntry, + meta: ToolProvenance, + template: Tool, + leaseSet: Closable[], + bindings: Map, + capMs: number, + ): Tool { + const original = template.execute; + if (typeof original !== 'function') return template; + const service = this; + const { serverIndex, rawName, writeClass } = meta; + + let binding = bindings.get(serverIndex); + if (!binding) { + binding = { current: null }; + bindings.set(serverIndex, binding); + } + const boundBinding = binding; + + const execute = async (args: unknown, options: ToolCallOptions) => { + // The per-call cap governs the WHOLE sequence (attempt1 + reconnect + + // attempt2). Compose it with the run's abort signal so a Stop or the cap + // ends any awaited call — @ai-sdk/mcp does not settle on abort, so we RACE. + const capController = new AbortController(); + const capTimer = setTimeout(() => { + capController.abort(new Error(`MCP tool call timed out after ${capMs}ms`)); + }, capMs); + capTimer.unref?.(); + const runSignal = options?.abortSignal; + const composed = runSignal + ? AbortSignal.any([runSignal, capController.signal]) + : capController.signal; + const stopped = () => runSignal?.aborted === true || capController.signal.aborted; + + const callOn = async ( + exec: NonNullable, + ): Promise => { + const aborted = new Promise((_, reject) => { + const fail = () => reject(abortReason(composed)); + if (composed.aborted) fail(); + else composed.addEventListener('abort', fail, { once: true }); + }); + return Promise.race([exec(args, { ...options, abortSignal: composed }), aborted]); + }; + + const execFor = ( + state: RecoveredServerState | null, + ): NonNullable | undefined => + state ? (state.tools[rawName]?.execute as NonNullable) : original; + + try { + // Snapshot the target BEFORE the call so a swap by a concurrent call is + // detected by identity in the catch. + const attemptState = boundBinding.current; + const attemptExec = execFor(attemptState); + if (typeof attemptExec !== 'function') { + throw new Error(`external MCP tool "${rawName}" is not callable`); + } + try { + return await callOn(attemptExec); + } catch (err) { + // Never retry on a Stop or an exhausted cap. + if (stopped()) throw err; + // Only a genuine transport break is a recovery candidate. + if (!isRetryableConnectError(err)) throw err; + // A write tool is INDETERMINATE on a transport error (may have applied + // before the reset) — surface that; do NOT auto-retry (double-apply is + // the #435 incident class). + if (!isReadOnlyWriteClass(writeClass)) { + throw new Error( + `external MCP tool "${rawName}" hit a transport error and MAY have already ` + + `applied on the server — not retried automatically; verify state before ` + + `retrying. (${shortError(err)})`, + ); + } + // Abort check BEFORE minting a fresh connection. + if (stopped()) throw err; + // CAS-swap by IDENTITY: mint+swap only if nobody swapped since this + // call's snapshot; a losing concurrent call awaits the same reconnect + // and retries on the SAME fresh client. + let target: RecoveredServerState; + if (boundBinding.current === attemptState) { + if (!boundBinding.reconnecting) { + boundBinding.reconnecting = (async () => { + const server = entry.servers[serverIndex]; + const fresh = await service.reconnectServer(server, capMs); + leaseSet.push(fresh.lease); // accumulate; released at turn-end + boundBinding.current = fresh.state; + return fresh.state; + })(); + // Clear the in-flight marker once it settles (success or failure) so + // a LATER death of the new client can reconnect again. + void boundBinding.reconnecting.then( + () => (boundBinding.reconnecting = undefined), + () => (boundBinding.reconnecting = undefined), + ); + } + target = await boundBinding.reconnecting; + } else { + target = boundBinding.current as RecoveredServerState; + } + // Abort check BEFORE the retry. + if (stopped()) throw err; + const retryExec = execFor(target); + if (typeof retryExec !== 'function') throw err; + return await callOn(retryExec); + } + } finally { + clearTimeout(capTimer); + } + }; + return { ...template, execute } as unknown as Tool; + } + + /** + * Reconnect ONE server for an in-run recovery (#489): open a fresh client and + * list+wrap its tools. The throwaway client is NOT cached — it is owned by the + * RUN via the returned lease (closed at turn-end), independent of the shared + * cache entry (whose TTL rebuild heals future turns). On a failure the fresh + * client is closed so its socket never leaks. + */ + private async reconnectServer( + server: AiMcpServer, + capMs: number, + ): Promise<{ state: RecoveredServerState; lease: Closable }> { + const client = await this.connectWithTimeout(server, CONNECT_TIMEOUT_MS); + let tools: Record; + try { + const raw = await withTimeout(client.tools(), CONNECT_TIMEOUT_MS); + const allow = server.toolAllowlist; + const picked = + Array.isArray(allow) && allow.length > 0 ? pick(raw, allow) : raw; + tools = wrapToolsWithCallTimeout(picked, capMs); + } catch (err) { + void client.close().catch(() => undefined); + throw err; + } + let released = false; + const lease: Closable = { + close: async () => { + if (released) return; + released = true; + await client.close().catch(() => undefined); + }, + }; + return { state: { client, tools }, lease }; + } + /** Mark an entry evicted; close its clients now if nothing is leasing them. */ private evict(entry: CacheEntry): void { clearTimeout(entry.timer); @@ -554,22 +873,21 @@ export function validateResolvedAddresses(addrs: readonly LookupAddress[]): { * certificate validation still uses the real hostname (we never rewrite the URL * to an IP literal). */ -function buildPinnedDispatcher(): Agent { - // External-MCP traffic uses a DEDICATED, shorter silence timeout +function buildPinnedDispatcher(bodyTimeoutMs: number): Agent { + // External-MCP traffic uses a DEDICATED, shorter HEADERS silence timeout // (`AI_MCP_STREAM_TIMEOUT_MS`, default 1 min) — deliberately tighter than the // chat provider's 15-min `streamTimeoutMs()` — so a byte-silent/hung MCP // upstream is broken in ~1 min instead of 15. We keep the keep-alive options - // from `streamingDispatcherOptions()` but OVERRIDE headers/body timeouts. - // Accepted trade-off: a legitimately long but byte-silent single tool call, - // and an SSE transport idling >1 min BETWEEN tool calls, are also cut here; the - // per-call total cap (wrapToolsWithCallTimeout, `AI_MCP_CALL_TIMEOUT_MS`) is the - // complementary guard for chatty-but-stuck calls that keep the socket warm yet - // never return. - const mcpSilenceMs = mcpStreamTimeoutMs(); + // from `streamingDispatcherOptions()` but OVERRIDE the timeouts. `bodyTimeout` + // is passed in per-transport (#489): tight for HTTP (fresh request per call), + // raised for SSE (one long-lived body across calls — idle BETWEEN calls is + // legit). The per-call total cap (`AI_MCP_CALL_TIMEOUT_MS`) is the complementary + // guard for chatty-but-stuck calls that keep the socket warm yet never return. + const headersMs = mcpStreamTimeoutMs(); return new Agent({ ...streamingDispatcherOptions(), - headersTimeout: mcpSilenceMs, - bodyTimeout: mcpSilenceMs, + headersTimeout: headersMs, + bodyTimeout: bodyTimeoutMs, connect: { lookup: (hostname, _options, callback) => { // Always resolve ALL addresses ourselves; do not trust the caller's @@ -669,18 +987,22 @@ function pick( function namespace( tools: Record, serverName: string, -): Record { +): Array<{ full: string; raw: string; tool: Tool }> { const prefix = namespacePrefix(serverName); - const out: Record = {}; + const out: Array<{ full: string; raw: string; tool: Tool }> = []; + const taken: Record = {}; for (const [name, t] of Object.entries(tools)) { const safe = sanitizeName(name); let full = capName(`${prefix}_${safe}`); // Duplicate names within ONE server can still collide after sanitize/ // truncate — suffix-disambiguate so the second tool is not overwritten. - if (full in out) { - full = disambiguate(full, '', (candidate) => candidate in out); + if (full in taken) { + full = disambiguate(full, '', (candidate) => candidate in taken); } - out[full] = t; + taken[full] = true; + // Keep the RAW (un-namespaced) name alongside the merged key so the per-run + // recovery wrapper (#489) can re-resolve the same tool on a fresh client. + out.push({ full, raw: name, tool: t }); } return out; } @@ -804,6 +1126,69 @@ export function wrapToolWithCallTimeout(tool: Tool, ms: number): Tool { return { ...tool, execute } as unknown as Tool; } +/** + * undici / Node network error CODES that mean the connection broke (not an + * application-level error) — a transient transport failure a readOnly call may + * safely retry after reconnecting. Matched against the REAL error shapes (#489): + * a socket reset surfaces as `TypeError: fetch failed` whose `.cause` is an + * undici `SocketError { code:'UND_ERR_SOCKET' }`; a body-timeout as + * `TypeError: terminated` whose `.cause` is `BodyTimeoutError`. Classifying by + * these real codes/names (not by mock errors) is essential — a mock-shaped + * predicate would leave eviction silently dead in production while CI is green. + */ +const RETRYABLE_TRANSPORT_ERROR_CODES: ReadonlySet = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'EPIPE', + 'ETIMEDOUT', + 'EAI_AGAIN', + 'ENETUNREACH', + 'EHOSTUNREACH', + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', + 'UND_ERR_CLOSED', + 'UND_ERR_DESTROYED', +]); + +/** undici error CLASS names for the same transport-break conditions. */ +const RETRYABLE_TRANSPORT_ERROR_NAMES: ReadonlySet = new Set([ + 'SocketError', + 'BodyTimeoutError', + 'HeadersTimeoutError', + 'ConnectTimeoutError', + 'ClientClosedError', + 'ClientDestroyedError', +]); + +/** + * Whether `err` is a retryable TRANSPORT break (a broken socket / body timeout), + * classified by the REAL undici/@ai-sdk error shapes (#489). undici surfaces a + * reset as `TypeError('fetch failed'|'terminated')` with the real error in + * `.cause`, and @ai-sdk/mcp may wrap it again in an `MCPClientError` (cause + * chain), so we walk `.cause` (bounded depth) checking `.code` and `.name`. An + * app-level tool error (a 4xx, a validation failure) is NOT retryable and returns + * false — only a connection-level failure heals with a reconnect. + */ +export function isRetryableConnectError(err: unknown, depth = 0): boolean { + if (!err || typeof err !== 'object' || depth > 6) return false; + const e = err as { + code?: unknown; + name?: unknown; + cause?: unknown; + }; + if (typeof e.code === 'string' && RETRYABLE_TRANSPORT_ERROR_CODES.has(e.code)) { + return true; + } + if (typeof e.name === 'string' && RETRYABLE_TRANSPORT_ERROR_NAMES.has(e.name)) { + return true; + } + if (e.cause != null) return isRetryableConnectError(e.cause, depth + 1); + return false; +} + /** The signal's reason as an Error (informative thrown value on abort/timeout). */ function abortReason(signal: AbortSignal): Error { const r = signal.reason; diff --git a/apps/server/src/integrations/ai/ai-streaming-fetch.ts b/apps/server/src/integrations/ai/ai-streaming-fetch.ts index b4a31723..9f76112c 100644 --- a/apps/server/src/integrations/ai/ai-streaming-fetch.ts +++ b/apps/server/src/integrations/ai/ai-streaming-fetch.ts @@ -129,6 +129,12 @@ const DEFAULT_MCP_STREAM_TIMEOUT_MS = 60_000; /** Default total wall-clock cap for ONE external MCP tool call (2 min). */ const DEFAULT_MCP_CALL_TIMEOUT_MS = 120_000; +/** + * Default `bodyTimeout` for the EXTERNAL-MCP SSE transport (10 min) — #489. + * Deliberately much LARGER than {@link DEFAULT_MCP_STREAM_TIMEOUT_MS}. + */ +const DEFAULT_MCP_SSE_BODY_TIMEOUT_MS = 600_000; + /** * SILENCE timeout (ms) for EXTERNAL-MCP transport ONLY. Override with * `AI_MCP_STREAM_TIMEOUT_MS`; a missing/invalid/non-positive value falls back to @@ -164,6 +170,26 @@ export function mcpCallTimeoutMs(): number { return positiveEnv('AI_MCP_CALL_TIMEOUT_MS', DEFAULT_MCP_CALL_TIMEOUT_MS); } +/** + * `bodyTimeout` (ms) for the EXTERNAL-MCP **SSE** transport ONLY — #489. Override + * with `AI_MCP_SSE_BODY_TIMEOUT_MS`; a missing/invalid/non-positive value falls + * back to {@link DEFAULT_MCP_SSE_BODY_TIMEOUT_MS} (10 min). + * + * The SSE transport holds ONE long-lived response body open across many tool + * calls, so undici's `bodyTimeout` (time between body bytes) counts the LEGITIMATE + * silence BETWEEN calls, not just a hung single call. At the tight HTTP silence + * timeout ({@link mcpStreamTimeoutMs}, 1 min) a normal >1-min gap between the + * model's tool calls would break the SSE socket, and the cache would then serve a + * dead client until TTL. So the SSE transport gets its OWN, RAISED bodyTimeout; + * the per-call total cap ({@link mcpCallTimeoutMs}) still bounds a single stuck + * call, and the app-level transport-error retry heals a socket that does break. + * The HTTP (streamable) transport keeps the tight timeout — it opens a fresh + * request per call, so idle-between-calls does not apply there. + */ +export function mcpSseBodyTimeoutMs(): number { + return positiveEnv('AI_MCP_SSE_BODY_TIMEOUT_MS', DEFAULT_MCP_SSE_BODY_TIMEOUT_MS); +} + /** * undici `Agent` options for streaming AI traffic — the (generous, finite) * silence timeouts plus the keep-alive recycle window. Shared by the chat diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 15fed6ac..298c88d5 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -30,6 +30,13 @@ export { destroyAllSessions } from "./lib/collab-session.js"; // internals directly; it goes through loadDocmostMcp()). export { SHARED_TOOL_SPECS } from "./tool-specs.js"; export type { SharedToolSpec } from "./tool-specs.js"; +// #489 — write-class registry consumed by the in-app external-MCP retry gate. +export { + SHARED_TOOL_WRITE_CLASS, + isRetryableWriteClass, + assertEverySpecDeclaresWriteClass, +} from "./tool-specs.js"; +export type { ToolWriteClass } from "./tool-specs.js"; // Re-export the build-time REGISTRY_STAMP (issue #447): a deterministic hash of // the tool-specs registry content, generated into src/registry-stamp.generated.ts diff --git a/packages/mcp/src/tool-specs.ts b/packages/mcp/src/tool-specs.ts index 941eead9..0bc9fd24 100644 --- a/packages/mcp/src/tool-specs.ts +++ b/packages/mcp/src/tool-specs.ts @@ -127,6 +127,18 @@ export interface SharedToolSpec { mcpName: string; /** camelCase key in the ai-SDK tools object (the in-app layer). */ inAppKey: string; + /** + * Write-class of the tool (#489), declared on EVERY spec (a registration-time + * assert enforces completeness; `satisfies Record` + * makes it a compile error to omit). 'readOnly' = a pure read that mutates + * NOTHING durable, so it is safe to auto-retry once after a transport break. + * 'write' = anything that mutates a page/comment/share/diagram/etc — a + * transport error is INDETERMINATE (the server may have applied it before the + * connection reset), so it is NEVER blind-retried (a retry would double-apply, + * the #435 incident class). Consumed by the external-MCP retry path + * (mcp-clients.service.ts) to gate its single auto-retry. + */ + writeClass: 'readOnly' | 'write'; /** Single canonical model-facing description used by both layers. */ description: string; /** @@ -240,6 +252,7 @@ export const SHARED_TOOL_SPECS = { getWorkspace: { mcpName: 'getWorkspace', inAppKey: 'getWorkspace', + writeClass: 'readOnly', description: 'Fetch metadata about the current workspace (name, settings).', tier: 'core', catalogLine: 'getWorkspace — fetch current workspace metadata (name, settings).', @@ -249,6 +262,7 @@ export const SHARED_TOOL_SPECS = { listSpaces: { mcpName: 'listSpaces', inAppKey: 'listSpaces', + writeClass: 'readOnly', description: 'List the spaces the current user can access. Returns the array of ' + 'spaces (id, name, slug, ...).', @@ -260,6 +274,7 @@ export const SHARED_TOOL_SPECS = { listShares: { mcpName: 'listShares', inAppKey: 'listShares', + writeClass: 'readOnly', description: 'List all public shares in the workspace with page titles and public URLs.', tier: 'deferred', @@ -272,6 +287,7 @@ export const SHARED_TOOL_SPECS = { getPageJson: { mcpName: 'getPageJson', inAppKey: 'getPageJson', + writeClass: 'readOnly', description: 'Get page details with the raw ProseMirror JSON content (lossless: ' + 'includes block ids, callouts, tables, link/image attributes) plus the ' + @@ -289,6 +305,7 @@ export const SHARED_TOOL_SPECS = { getOutline: { mcpName: 'getOutline', inAppKey: 'getOutline', + writeClass: 'readOnly', description: "Return a COMPACT outline of a page's top-level blocks ({index, type, " + 'id, level, firstText}; tables add rows/cols/header; lists add item ' + @@ -309,6 +326,7 @@ export const SHARED_TOOL_SPECS = { getNode: { mcpName: 'getNode', inAppKey: 'getNode', + writeClass: 'readOnly', description: "Fetch a single block for editing. `nodeId` is a block id from the page " + 'outline or page-JSON view (works for headings/paragraphs/callouts/images), OR ' + @@ -350,6 +368,7 @@ export const SHARED_TOOL_SPECS = { searchInPage: { mcpName: 'searchInPage', inAppKey: 'searchInPage', + writeClass: 'readOnly', description: 'Find every occurrence of a string (or regex) INSIDE one page and get ' + 'WHERE each is — instead of pulling blocks one-by-one with getNode. ' + @@ -413,6 +432,7 @@ export const SHARED_TOOL_SPECS = { deleteNode: { mcpName: 'deleteNode', inAppKey: 'deleteNode', + writeClass: 'write', description: 'Remove a single block by its attrs.id (from the page outline or ' + 'page-JSON view) WITHOUT resending the whole document.', @@ -438,6 +458,7 @@ export const SHARED_TOOL_SPECS = { patchNode: { mcpName: 'patchNode', inAppKey: 'patchNode', + writeClass: 'write', description: 'Replace a single content block identified by its attrs.id, WITHOUT ' + 'resending the whole document; the replacement keeps the same block id. ' + @@ -505,6 +526,7 @@ export const SHARED_TOOL_SPECS = { insertNode: { mcpName: 'insertNode', inAppKey: 'insertNode', + writeClass: 'write', description: 'Insert content before/after another block (by attrs.id or anchor text) ' + 'or append it at the end (top level). For before/after you MUST provide ' + @@ -597,6 +619,7 @@ export const SHARED_TOOL_SPECS = { sharePage: { mcpName: 'sharePage', inAppKey: 'sharePage', + writeClass: 'write', // CANONICAL: merges the MCP copy's URL-format + idempotency detail with the // in-app copy's reversibility note; keeps the security framing both had. description: @@ -626,6 +649,7 @@ export const SHARED_TOOL_SPECS = { unsharePage: { mcpName: 'unsharePage', inAppKey: 'unsharePage', + writeClass: 'write', description: 'Remove the public share of a page (revokes the public URL).', tier: 'deferred', catalogLine: "unsharePage — revoke a page's public share (removes the public URL).", @@ -640,6 +664,7 @@ export const SHARED_TOOL_SPECS = { diffPageVersions: { mcpName: 'diffPageVersions', inAppKey: 'diffPageVersions', + writeClass: 'readOnly', description: 'Diff two versions of a page and return a Docmost-equivalent change set ' + '(inserted/deleted text, integrity counts for images/links/tables/' + @@ -672,6 +697,7 @@ export const SHARED_TOOL_SPECS = { listPageHistory: { mcpName: 'listPageHistory', inAppKey: 'listPageHistory', + writeClass: 'readOnly', description: "List a page's saved versions (Docmost auto-snapshots on every save), " + 'newest first, cursor-paginated. Returns { items, nextCursor }; each ' + @@ -693,6 +719,7 @@ export const SHARED_TOOL_SPECS = { restorePageVersion: { mcpName: 'restorePageVersion', inAppKey: 'restorePageVersion', + writeClass: 'write', description: 'Restore a page to a saved version: writes that version\'s content back ' + 'as the page\'s current content (Docmost has no restore endpoint, so ' + @@ -713,6 +740,7 @@ export const SHARED_TOOL_SPECS = { importPageMarkdown: { mcpName: 'importPageMarkdown', inAppKey: 'importPageMarkdown', + writeClass: 'write', // IN-APP ONLY (issue #411): the external /mcp surface no longer exposes // importPageMarkdown — the registry loop in index.ts skips inAppOnly specs, // so this stays available to the in-app agent (round-tripping an EXPORTED @@ -742,6 +770,7 @@ export const SHARED_TOOL_SPECS = { copyPageContent: { mcpName: 'copyPageContent', inAppKey: 'copyPageContent', + writeClass: 'write', description: "Replace targetPageId's content with a copy of sourcePageId's content, " + 'entirely server-side — the document is NOT sent through the model. The ' + @@ -770,6 +799,7 @@ export const SHARED_TOOL_SPECS = { editPageText: { mcpName: 'editPageText', inAppKey: 'editPageText', + writeClass: 'write', description: "Surgical find/replace inside a page's text, preserving all block " + 'ids and marks. A find MAY cross bold/italic/link boundaries; the ' + @@ -819,6 +849,7 @@ export const SHARED_TOOL_SPECS = { stashPage: { mcpName: 'stashPage', inAppKey: 'stashPage', + writeClass: 'readOnly', description: 'Serialize a whole page (the full ProseMirror JSON, as getPageJson ' + 'returns) into an ephemeral in-memory blob and return ONLY a short ' + @@ -880,6 +911,7 @@ export const SHARED_TOOL_SPECS = { getPage: { mcpName: 'getPage', inAppKey: 'getPage', + writeClass: 'readOnly', description: 'Fetch a single page as Markdown by its id. Returns the page title and ' + 'its Markdown content. The converter is canonical (round-trips text and ' + @@ -919,6 +951,7 @@ export const SHARED_TOOL_SPECS = { listPages: { mcpName: 'listPages', inAppKey: 'listPages', + writeClass: 'readOnly', description: 'List the most recent pages (ordered by updatedAt, descending), ' + 'optionally scoped to a single space. Returns a bounded list (default ' + @@ -965,6 +998,7 @@ export const SHARED_TOOL_SPECS = { getTree: { mcpName: 'getTree', inAppKey: 'getTree', + writeClass: 'readOnly', description: "Get a space's page hierarchy (or one subtree) as a nested tree in a " + 'SINGLE request — completely and without loss. Each node is ' + @@ -1009,6 +1043,7 @@ export const SHARED_TOOL_SPECS = { getPageContext: { mcpName: 'getPageContext', inAppKey: 'getPageContext', + writeClass: 'readOnly', description: 'Given a pageId, get its LOCATION and immediate surroundings (metadata ' + 'only, no page content) in one call — answers "where am I / what is ' + @@ -1038,6 +1073,7 @@ export const SHARED_TOOL_SPECS = { createPage: { mcpName: 'createPage', inAppKey: 'createPage', + writeClass: 'write', description: 'Create a new page with a Markdown body in a space, optionally under a ' + 'parent page (omit parentPageId to create at the space root). Returns ' + @@ -1088,6 +1124,7 @@ export const SHARED_TOOL_SPECS = { movePage: { mcpName: 'movePage', inAppKey: 'movePage', + writeClass: 'write', description: 'Move a page under a new parent page, or to the space root when no ' + 'parent is given. Reversible: move it back at any time.', @@ -1181,6 +1218,7 @@ export const SHARED_TOOL_SPECS = { renamePage: { mcpName: 'renamePage', inAppKey: 'renamePage', + writeClass: 'write', description: 'Rename a page (change its title only; the body is untouched, never ' + 'resent). Reversible: rename back at any time.', @@ -1202,6 +1240,7 @@ export const SHARED_TOOL_SPECS = { deletePage: { mcpName: 'deletePage', inAppKey: 'deletePage', + writeClass: 'write', description: 'Move a page to the trash — SOFT delete only: the page can be restored ' + 'from trash and nothing is ever permanently deleted.', @@ -1235,6 +1274,7 @@ export const SHARED_TOOL_SPECS = { updatePageJson: { mcpName: 'updatePageJson', inAppKey: 'updatePageJson', + writeClass: 'write', description: "Replace a page's content with a raw ProseMirror JSON document (lossless " + 'write: preserves the block ids, callouts, tables and attributes you pass ' + @@ -1291,6 +1331,7 @@ export const SHARED_TOOL_SPECS = { updatePageMarkdown: { mcpName: 'updatePageMarkdown', inAppKey: 'updatePageMarkdown', + writeClass: 'write', description: "Replace a page's body with new Markdown content (and optionally its " + 'title). The whole body is re-imported from the markdown (block ids ' + @@ -1325,6 +1366,7 @@ export const SHARED_TOOL_SPECS = { exportPageMarkdown: { mcpName: 'exportPageMarkdown', inAppKey: 'exportPageMarkdown', + writeClass: 'readOnly', // CANONICAL: the MCP copy (a strict superset of the terse in-app wording). description: 'Export a page to a single self-contained Docmost-flavoured Markdown ' + @@ -1373,6 +1415,7 @@ export const SHARED_TOOL_SPECS = { createComment: { mcpName: 'createComment', inAppKey: 'createComment', + writeClass: 'write', // CANONICAL: the in-app copy (the more-maintained one). It keeps the same // rules as the MCP copy — inline-only, top-level requires a `selection`, no // page-level comments, replies inherit the anchor, suggestedText must be @@ -1505,6 +1548,7 @@ export const SHARED_TOOL_SPECS = { listComments: { mcpName: 'listComments', inAppKey: 'listComments', + writeClass: 'readOnly', // CANONICAL: the two copies are near-identical; the MCP copy is the // superset (it keeps the "(pagination is handled internally)" note the // in-app copy dropped), so it is used verbatim. @@ -1532,6 +1576,7 @@ export const SHARED_TOOL_SPECS = { resolveComment: { mcpName: 'resolveComment', inAppKey: 'resolveComment', + writeClass: 'write', // CANONICAL: the MCP copy's richer wording, minus its reference // to `deleteComment` (a sibling tool that does NOT exist in the in-app // layer) — rephrased transport-neutrally per the registry convention. @@ -1572,6 +1617,7 @@ export const SHARED_TOOL_SPECS = { checkNewComments: { mcpName: 'checkNewComments', inAppKey: 'checkNewComments', + writeClass: 'readOnly', // CANONICAL: the MCP copy (the more detailed of the two). The MCP layer's // execute-side guard that rejects an unparseable `since` timestamp stays in // its execute body (per-layer logic), not in the shared schema. @@ -1651,6 +1697,7 @@ export const SHARED_TOOL_SPECS = { tableInsertRow: { mcpName: 'tableInsertRow', inAppKey: 'tableInsertRow', + writeClass: 'write', description: 'Insert a row of plain-text cells into a table. `table` is `#` ' + 'from the page outline, or a block id inside it. `cells` is the text per ' + @@ -1684,6 +1731,7 @@ export const SHARED_TOOL_SPECS = { tableDeleteRow: { mcpName: 'tableDeleteRow', inAppKey: 'tableDeleteRow', + writeClass: 'write', description: 'Delete the row at 0-based `index` from a table (`table` is `#` ' + 'from the page outline, or a block id inside it). Refuses to delete the ' + @@ -1707,6 +1755,7 @@ export const SHARED_TOOL_SPECS = { tableUpdateCell: { mcpName: 'tableUpdateCell', inAppKey: 'tableUpdateCell', + writeClass: 'write', description: 'Set the plain-text content of cell [row, col] (0-based) in a table ' + '(`table` is `#` from the page outline, or a block id inside it). ' + @@ -1747,6 +1796,7 @@ export const SHARED_TOOL_SPECS = { insertFootnote: { mcpName: 'insertFootnote', inAppKey: 'insertFootnote', + writeClass: 'write', description: 'Insert an AUTHOR-INLINE footnote: you specify only WHERE (anchorText) ' + 'and WHAT (text). The footnote marker is placed right after anchorText in ' + @@ -1784,6 +1834,7 @@ export const SHARED_TOOL_SPECS = { insertImage: { mcpName: 'insertImage', inAppKey: 'insertImage', + writeClass: 'write', description: 'Download an image from a web (http/https) URL and insert it into ' + 'a page in one step. By default ' + @@ -1828,6 +1879,7 @@ export const SHARED_TOOL_SPECS = { replaceImage: { mcpName: 'replaceImage', inAppKey: 'replaceImage', + writeClass: 'write', description: 'Replace an existing image on a page with a new image fetched from a web ' + '(http/https) URL: uploads the new file as a NEW ' + @@ -1866,6 +1918,7 @@ export const SHARED_TOOL_SPECS = { drawioGet: { mcpName: 'drawioGet', inAppKey: 'drawioGet', + writeClass: 'readOnly', description: 'Read a draw.io diagram on a page as mxGraph XML (default) or as its raw ' + '`.drawio.svg`. `node` is the drawio node\'s attrs.id (from getOutline / ' + @@ -1899,6 +1952,7 @@ export const SHARED_TOOL_SPECS = { drawioCreate: { mcpName: 'drawioCreate', inAppKey: 'drawioCreate', + writeClass: 'write', description: 'Create a draw.io diagram from mxGraph XML and insert it as a diagram ' + 'block. `xml` is a bare `` OR a list of `` elements ' + @@ -1969,6 +2023,7 @@ export const SHARED_TOOL_SPECS = { drawioUpdate: { mcpName: 'drawioUpdate', inAppKey: 'drawioUpdate', + writeClass: 'write', description: 'Replace a draw.io diagram\'s content with new mxGraph XML (same lint ' + 'pipeline as drawioCreate). `baseHash` is MANDATORY: pass the hash from ' + @@ -2020,6 +2075,7 @@ export const SHARED_TOOL_SPECS = { drawioEditCells: { mcpName: 'drawioEditCells', inAppKey: 'drawioEditCells', + writeClass: 'write', description: 'Make TARGETED, id-based edits to an existing draw.io diagram instead of ' + 'resending the whole XML (a full-XML diff is fragile — draw.io reorders ' + @@ -2078,6 +2134,7 @@ export const SHARED_TOOL_SPECS = { drawioFromGraph: { mcpName: 'drawioFromGraph', inAppKey: 'drawioFromGraph', + writeClass: 'write', description: 'Build a draw.io diagram from a SEMANTIC graph — you describe nodes, groups ' + 'and edges by MEANING and the server picks every coordinate, color and icon ' + @@ -2202,6 +2259,7 @@ export const SHARED_TOOL_SPECS = { drawioFromMermaid: { mcpName: 'drawioFromMermaid', inAppKey: 'drawioFromMermaid', + writeClass: 'write', description: 'Convert Mermaid `flowchart` text into an EDITABLE draw.io diagram (LLMs ' + 'write Mermaid reliably). Best for STANDARD flowcharts/decision trees: ' + @@ -2248,6 +2306,7 @@ export const SHARED_TOOL_SPECS = { drawioShapes: { mcpName: 'drawioShapes', inAppKey: 'drawioShapes', + writeClass: 'readOnly', description: 'Look up VERIFIED draw.io stencil style-strings so you never guess a ' + '`shape=mxgraph.*` name (a wrong name renders as an EMPTY BOX). Searches a ' + @@ -2294,6 +2353,7 @@ export const SHARED_TOOL_SPECS = { drawioGuide: { mcpName: 'drawioGuide', inAppKey: 'drawioGuide', + writeClass: 'readOnly', description: 'Progressive-disclosure draw.io authoring reference. Call with a `section` ' + 'to pull one focused, <=4KB chapter instead of bloating context: ' + @@ -2323,3 +2383,53 @@ export const SHARED_TOOL_SPECS = { inlineBothHosts: true, }, } satisfies Record; + +// --- write-class registry (#489) ------------------------------------------ + +/** A tool's retry-safety class. 'readOnly' may be auto-retried once after a + * transport break; 'write' is indeterminate and must never be blind-retried. */ +export type ToolWriteClass = 'readOnly' | 'write'; + +/** + * Name → write-class map for the shared registry, keyed by mcpName (=== inAppKey). + * The external-MCP retry path (mcp-clients.service.ts) looks a tool up here by its + * RAW (un-namespaced) name to decide whether a transport failure may be retried. + * A tool NOT in this map (a third-party external MCP tool) is treated as 'write' + * by the consumer — the safe default (never blind-retry an unknown tool). + */ +export const SHARED_TOOL_WRITE_CLASS: Record = + Object.fromEntries( + Object.values(SHARED_TOOL_SPECS).map((spec) => [spec.mcpName, spec.writeClass]), + ); + +/** Whether a write-class permits a single automatic retry after a transport + * break. Only a pure read is retry-safe; everything mutating is indeterminate. */ +export function isRetryableWriteClass( + writeClass: ToolWriteClass | undefined, +): boolean { + return writeClass === 'readOnly'; +} + +/** + * Registration-time assert (#489): EVERY spec must declare a valid write-class. + * `satisfies Record` already makes an omission a compile + * error, but this guards a raw/cast construction path and documents the invariant + * at the point of use. Runs once on import — both hosts import this module, so + * both get the check. Throws (fails startup) rather than silently mis-gating a + * retry in production. + */ +export function assertEverySpecDeclaresWriteClass(): void { + for (const [key, spec] of Object.entries(SHARED_TOOL_SPECS)) { + const wc = (spec as SharedToolSpec).writeClass; + if (wc !== 'readOnly' && wc !== 'write') { + throw new Error( + `tool-specs: spec "${key}" must declare writeClass ('readOnly' | 'write'), got ${JSON.stringify( + wc, + )}`, + ); + } + } +} + +// Enforce at module load (registration time) on both hosts. +assertEverySpecDeclaresWriteClass(); diff --git a/packages/mcp/test/unit/tool-specs.test.mjs b/packages/mcp/test/unit/tool-specs.test.mjs index 41f16606..aad5354a 100644 --- a/packages/mcp/test/unit/tool-specs.test.mjs +++ b/packages/mcp/test/unit/tool-specs.test.mjs @@ -2,7 +2,12 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { z } from "zod"; -import { SHARED_TOOL_SPECS } from "../../build/tool-specs.js"; +import { + SHARED_TOOL_SPECS, + SHARED_TOOL_WRITE_CLASS, + isRetryableWriteClass, + assertEverySpecDeclaresWriteClass, +} from "../../build/tool-specs.js"; // The shared registry is consumed by BOTH the zod-v3 MCP server and the zod-v4 // in-app AI-SDK service, so every spec must carry the cross-layer wiring @@ -43,6 +48,41 @@ test("mcpName and inAppKey are each unique across the registry", () => { } }); +// #489 — every spec must declare its write-class so the external-MCP retry path +// can gate a single auto-retry ONLY on a pure read (a blind retry of a write = +// double-apply). The declaration is enforced at registration time. +test("#489: every spec declares a valid writeClass ('readOnly' | 'write')", () => { + for (const [key, spec] of Object.entries(SHARED_TOOL_SPECS)) { + assert.ok( + spec.writeClass === "readOnly" || spec.writeClass === "write", + `${key}: missing/invalid writeClass: ${JSON.stringify(spec.writeClass)}`, + ); + } + // The registration-time assert must not throw for the shipped registry. + assert.doesNotThrow(() => assertEverySpecDeclaresWriteClass()); +}); + +test("#489: SHARED_TOOL_WRITE_CLASS maps every mcpName to its class; helper gates on readOnly", () => { + const specs = Object.values(SHARED_TOOL_SPECS); + assert.equal(Object.keys(SHARED_TOOL_WRITE_CLASS).length, specs.length); + for (const spec of specs) { + assert.equal(SHARED_TOOL_WRITE_CLASS[spec.mcpName], spec.writeClass); + } + // Only a readOnly tool is retry-eligible; a write tool and an unknown tool are not. + assert.equal(isRetryableWriteClass("readOnly"), true); + assert.equal(isRetryableWriteClass("write"), false); + assert.equal(isRetryableWriteClass(undefined), false); +}); + +test("#489: representative reads are readOnly and representative writes are write", () => { + for (const name of ["getPage", "getTree", "searchInPage", "listComments"]) { + assert.equal(SHARED_TOOL_SPECS[name].writeClass, "readOnly", `${name} should be readOnly`); + } + for (const name of ["patchNode", "createPage", "deletePage", "createComment", "drawioCreate"]) { + assert.equal(SHARED_TOOL_SPECS[name].writeClass, "write", `${name} should be write`); + } +}); + test("buildShape (when present) returns a usable ZodRawShape with a real zod", () => { for (const [key, spec] of Object.entries(SHARED_TOOL_SPECS)) { if (!spec.buildShape) continue; From 38a09c5ca1989c59aef89992c7f011a274ff2d33 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:24:03 +0300 Subject: [PATCH 29/41] =?UTF-8?q?fix(ai-chat):=20write-class-=D1=80=D0=B5?= =?UTF-8?q?=D1=82=D1=80=D0=B0=D0=B9=20=D1=82=D0=BE=D0=BB=D1=8C=D0=BA=D0=BE?= =?UTF-8?q?=20=D0=B4=D0=BB=D1=8F=20=D0=B4=D0=BE=D0=B2=D0=B5=D1=80=D0=B5?= =?UTF-8?q?=D0=BD=D0=BD=D0=BE=D0=B3=D0=BE=20=D0=B2=D0=BD=D1=83=D1=82=D1=80?= =?UTF-8?q?=D0=B5=D0=BD=D0=BD=D0=B5=D0=B3=D0=BE=20Docmost-=D1=81=D0=B5?= =?UTF-8?q?=D1=80=D0=B2=D0=B5=D1=80=D0=B0=20(#489=20=D1=80=D0=B5=D0=B2?= =?UTF-8?q?=D1=8C=D1=8E)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEDIUM (реальная брешь): SHARED_TOOL_WRITE_CLASS — имена ТУЛОВ Docmost, но mergeNamespaced применял мапу к тулам ЛЮБОГО стороннего MCP-сервера по совпадению rawName. Сторонний WRITE-тул, названный как Docmost-read (getPage/listPages/…), наследовал readOnly → авто-ретрай при транспортной ошибке → double-apply (класс #435). Гарантия «unknown third-party → write → не ретраится» была ЛОЖНОЙ при коллизии имён. Фикс: write-class мапа применяется ТОЛЬКО к серверу, про который известно, что это внутренний Docmost-MCP (isInternalDocmostServer). Сейчас это ВСЕГДА false — в этом пути нет встроенного/доверенного Docmost-сервера: все ai_mcp_servers суть сторонние admin-конфиги, а собственные тулы Docmost идут отдельным in-app путём (docmostTools), не через mcp-clients. Значит НИ ОДИН сторонний тул не получает readOnly по коллизии и не авто-ретраится (undefined → трактуется как write). Мапа грузится лениво только если есть доверенный сервер (иначе ESM-импорт пропускается). isInternalDocmostServer — метод-сим, флипается при появлении доверенного сервера (kind/isBuiltin-колонка или сконфигурённый self-MCP URL). LOW: reconnect не наблюдает composed abort во время 5-сек handshake — задокумен- тировано (окно поздней отмены ≤5s, сокет закрывается на turn-end); проброс composed в общий CAS-дедуплицированный reconnect намеренно НЕ сделан (отменил бы реконнект, нужный конкурентному живому вызову). Тест: сторонний WRITE-тул с именем getPage при транспортной ошибке НЕ ретраится; mutation-verify — форсирование trusted делает тест красным (чужой getPage ретраится). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../external-mcp/mcp-clients.recovery.spec.ts | 57 +++++++++++++++-- .../external-mcp/mcp-clients.service.ts | 62 ++++++++++++++++--- 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.recovery.spec.ts b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.recovery.spec.ts index 5b129a58..5cf83721 100644 --- a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.recovery.spec.ts +++ b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.recovery.spec.ts @@ -55,7 +55,7 @@ const server = (over: Partial = {}): FakeServer => ({ ...over, }); -function buildService(servers: FakeServer[]) { +function buildService(servers: FakeServer[], trusted = false) { const repo = { listEnabled: jest.fn().mockResolvedValue(servers) }; const service = new McpClientsService(repo as never, {} as never); // Seed a DETERMINISTIC write-class map so the retry gate is controlled here @@ -67,6 +67,21 @@ function buildService(servers: FakeServer[]) { getPage: 'readOnly', patchNode: 'write', }); + // The service only APPLIES that map to a TRUSTED internal Docmost server + // (isInternalDocmostServer, really false for every third-party row). A retry + // test needs a trusted server to exercise the readOnly-retry path at all, so it + // passes trusted=true to model a Docmost-origin server; the third-party + // double-apply test leaves it at the real value (false). + if (trusted) { + jest + .spyOn( + service as unknown as { + isInternalDocmostServer: (s: FakeServer) => boolean; + }, + 'isInternalDocmostServer', + ) + .mockReturnValue(true); + } return { service, repo }; } @@ -121,7 +136,7 @@ describe('McpClientsService in-run transport recovery (#489)', () => { it('a readOnly tool whose transport breaks reconnects and retries WITHIN the same run', async () => { const realErr = await realSocketResetError(); - const { service } = buildService([server()]); + const { service } = buildService([server()], true); const first = jest.fn().mockRejectedValue(realErr); const second = jest.fn().mockResolvedValue({ ok: true }); const connectSpy = stubConnect(service, 'getPage', [first, second]); @@ -146,7 +161,7 @@ describe('McpClientsService in-run transport recovery (#489)', () => { it('a WRITE tool does NOT auto-retry on a transport error (indeterminate)', async () => { const realErr = await realSocketResetError(); - const { service } = buildService([server()]); + const { service } = buildService([server()], true); const exec = jest.fn().mockRejectedValue(realErr); const connectSpy = stubConnect(service, 'patchNode', [exec]); @@ -168,7 +183,7 @@ describe('McpClientsService in-run transport recovery (#489)', () => { it('does NOT retry (or reconnect) after the run is aborted (Stop)', async () => { const realErr = await realSocketResetError(); - const { service } = buildService([server()]); + const { service } = buildService([server()], true); const controller = new AbortController(); // The transport error arrives, but the run was Stopped in the same tick. const first = jest.fn().mockImplementation(async () => { @@ -194,7 +209,7 @@ describe('McpClientsService in-run transport recovery (#489)', () => { }); it('an app-level (non-transport) tool error is surfaced verbatim, never retried', async () => { - const { service } = buildService([server()]); + const { service } = buildService([server()], true); const appErr = new Error('tool says: bad input'); const exec = jest.fn().mockRejectedValue(appErr); const connectSpy = stubConnect(service, 'getPage', [exec]); @@ -211,4 +226,36 @@ describe('McpClientsService in-run transport recovery (#489)', () => { expect(connectSpy).toHaveBeenCalledTimes(1); // no reconnect for an app error await Promise.all(toolset.clients.map((c) => c.close())); }); + + // #489 (review, MEDIUM) — the Docmost write-class map keys by DOCMOST tool + // names; a THIRD-PARTY server may name a WRITE tool `getPage` (a Docmost read + // name). It must NOT inherit readOnly and must NOT auto-retry on a transport + // error — a blind retry of that write is a double-apply (the #435 class). Here + // the server is UNTRUSTED (buildService default, isInternalDocmostServer=false), + // so the map is not applied and `getPage` classifies as a write. + // + // MUTATION-VERIFY: forcing the server "trusted" (buildService(..., true)) makes + // `getPage` inherit readOnly -> it WOULD reconnect+retry (connect twice) and the + // assertions below fail — i.e. removing the trust scope re-opens the bug. + it('a THIRD-PARTY WRITE tool named like a Docmost read does NOT auto-retry (no double-apply)', async () => { + const realErr = await realSocketResetError(); + // Untrusted: default trusted=false — a real third-party server. + const { service } = buildService([server()]); + const exec = jest.fn().mockRejectedValue(realErr); + const connectSpy = stubConnect(service, 'getPage', [exec, exec]); + + const toolset = await service.toolsFor('ws-5'); + const tool = toolset.tools['srv_getPage']; + await expect( + (tool.execute as (a: unknown, o: unknown) => Promise)( + { pageId: 'p' }, + opts(), + ), + ).rejects.toThrow(/MAY have already applied/); + + // Exactly one call, NO reconnect — the name collision granted no readOnly-retry. + expect(exec).toHaveBeenCalledTimes(1); + expect(connectSpy).toHaveBeenCalledTimes(1); + await Promise.all(toolset.clients.map((c) => c.close())); + }); }); diff --git a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts index 4c874ccf..20397acc 100644 --- a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts +++ b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts @@ -217,6 +217,23 @@ export class McpClientsService { private readonly secretBox: SecretBoxService, ) {} + /** + * Whether an external MCP server is the TRUSTED internal Docmost MCP server — + * the only server whose tools may be classified by the Docmost write-class map + * (#489 review). Today this is ALWAYS false: every `ai_mcp_servers` row is an + * admin-configured THIRD-PARTY endpoint (there is no builtin/self flag, sentinel + * URL, or synthetic server in this path — Docmost's OWN tools are exposed via the + * separate in-app tools path, never through this external-MCP client). So no + * third-party tool can inherit `readOnly` by a name collision with a Docmost read + * tool, and none is ever auto-retried on a transport error (which would risk a + * double-apply — the #435 class). Flip this (an explicit `kind`/`isBuiltin` + * column, or a configured self-MCP URL) if a trusted internal server is ever + * introduced. A method (not a free function) so it is a single, mockable seam. + */ + private isInternalDocmostServer(_server: AiMcpServer): boolean { + return false; + } + /** Lazily load + memoize the shared write-class map (see the field doc). */ private getWriteClassMap(): Promise> { if (!this.writeClassMapPromise) { @@ -388,9 +405,14 @@ export class McpClientsService { const instructions: McpServerInstruction[] = []; // merged-key -> provenance for the per-run recovery wrapper (#489). const toolMeta: Record = {}; - // Shared write-class map (#489): classifies each merged tool by its raw name - // so the recovery wrapper knows which tools are retry-safe reads. - const writeClassMap = await this.getWriteClassMap(); + // Shared Docmost write-class map (#489) — classifies a tool by its raw name. + // Loaded ONLY when at least one server is a TRUSTED internal Docmost server + // (see isInternalDocmostServer): for third-party servers the map is never + // applied (a name collision must not grant readOnly-retry), so we skip the + // dynamic ESM load entirely in that (currently universal) case. + const writeClassMap = servers.some((s) => this.isInternalDocmostServer(s)) + ? await this.getWriteClassMap() + : null; // Per-server connect+tools result, still tagged with its server so the merge // below can be applied in the SAME order as `servers` (see the parallel note). @@ -464,6 +486,15 @@ export class McpClientsService { // against names already merged from earlier servers, so no external // tool is silently overwritten on collision. The returned count drives // whether this server's prompt guidance is included (≥1 tool merged). + // #489 (review): the Docmost write-class map keys by DOCMOST tool names and + // may ONLY be trusted for a server KNOWN to be the internal Docmost MCP + // server. Every row here is an admin-configured THIRD-PARTY endpoint, so a + // third-party WRITE tool that happens to be named like a Docmost read + // (getPage, listPages, ...) must NOT inherit readOnly — that would auto-retry + // a mutation on a transport error (double-apply, the #435 class). Gate the + // map on the trust check; untrusted servers get writeClass=undefined -> the + // recovery wrapper treats them as writes and never auto-retries. + const trustWriteClass = this.isInternalDocmostServer(server); const merged = this.mergeNamespaced( tools, result.guarded, @@ -471,7 +502,7 @@ export class McpClientsService { server.id, toolMeta, i, - writeClassMap, + trustWriteClass ? writeClassMap : null, ); outcomes.push({ name: server.name, ok: true }); // Include this server's guidance ONLY when it actually contributed at @@ -523,7 +554,9 @@ export class McpClientsService { serverId: string, toolMeta: Record, serverIndex: number, - writeClassMap: Record, + // The Docmost write-class map, or `null` for an UNTRUSTED (third-party) + // server whose tools must all default to write (never auto-retried). + writeClassMap: Record | null, ): { count: number; prefix: string } { let count = 0; for (const { full, raw, tool } of namespace(picked, serverName)) { @@ -537,13 +570,14 @@ export class McpClientsService { } target[key] = tool; // Record provenance so the per-run recovery wrapper (#489) can reconnect - // this tool's server and re-resolve it by its raw name, and gate the retry - // on its write-class (unknown third-party tool -> undefined -> treated as a - // write, i.e. never auto-retried). + // this tool's server and re-resolve it by its raw name. writeClass is set + // ONLY from a TRUSTED (internal-Docmost) map; for a third-party server the + // map is null -> writeClass stays undefined -> the wrapper treats the tool + // as a write and never auto-retries it (no double-apply on name collision). toolMeta[key] = { serverIndex, rawName: raw, - writeClass: writeClassMap[raw], + writeClass: writeClassMap ? writeClassMap[raw] : undefined, }; count += 1; } @@ -751,7 +785,15 @@ export class McpClientsService { `retrying. (${shortError(err)})`, ); } - // Abort check BEFORE minting a fresh connection. + // Abort check BEFORE minting a fresh connection (no socket for a + // stopped run). LIMITATION (#489, LOW): the reconnect's own connect is + // bounded by CONNECT_TIMEOUT_MS but does NOT itself observe `composed`, + // so a Stop that lands DURING the handshake is only honored at the next + // `stopped()` gate (before the retry) — a bounded ≤5s late-abort window; + // the throwaway client is closed at turn-end regardless. Threading + // `composed` into the SHARED (CAS-deduped) reconnect is deliberately + // avoided: it would let the first caller's abort tear down a reconnect a + // concurrent still-live caller depends on. if (stopped()) throw err; // CAS-swap by IDENTITY: mint+swap only if nobody swapped since this // call's snapshot; a losing concurrent call awaits the same reconnect From 947796adef9c0770e15817bfdf9e22d3434eba72 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 05:53:20 +0300 Subject: [PATCH 30/41] =?UTF-8?q?refactor(client):=20FSM=20skeleton=20+=20?= =?UTF-8?q?spec=20=D0=B4=D0=BB=D1=8F=20run-lifecycle=20(#488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Заменяет зоопарк из ~26 useRef-флагов в chat-thread.tsx на один чистый редьюсер с перечислимыми переходами (event × state → next state — впервые юнит-тестируемо напрямую). Коммит 1 из 5. Содержит СПЕКУ (пишется первой, входит в PR) и каркас: - run-fsm.spec.md: таблица «событие × состояние», карта всех ref'ов → {состояние | контекст | данные}, протокол run-факта, список инвариантов; - run-fsm.ts: чистый reduce(machine, event) → machine с epoch-инвариантом (I1), состояниями idle|sending|streaming|attaching|reconnecting|polling|stalled| stopping|superseding|error, ownership как ПОЛЕ контекста (I2), run-фактом как first-class (I3), выходом из stopping по данным (I4), dispose-протоколом (I5) и слоем command-эффектов (attach/postStream/postRun/stop/supersede); - run-fsm.test.ts: 31 тест переходов, включая поведение коммитов 2–5 как переходы автомата (reconnect по run-факту; повторные циклы reconnect; polling→stalled; supersede CAS-исходы; фильтрация позднего колбэка по epoch). 8 зафиксированных решений реализованы; epoch-инвариант неотключаем. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../features/ai-chat/state/run-fsm.spec.md | 161 ++++++ .../features/ai-chat/state/run-fsm.test.ts | 336 ++++++++++++ .../src/features/ai-chat/state/run-fsm.ts | 513 ++++++++++++++++++ 3 files changed, 1010 insertions(+) create mode 100644 apps/client/src/features/ai-chat/state/run-fsm.spec.md create mode 100644 apps/client/src/features/ai-chat/state/run-fsm.test.ts create mode 100644 apps/client/src/features/ai-chat/state/run-fsm.ts diff --git a/apps/client/src/features/ai-chat/state/run-fsm.spec.md b/apps/client/src/features/ai-chat/state/run-fsm.spec.md new file mode 100644 index 00000000..d7c7557c --- /dev/null +++ b/apps/client/src/features/ai-chat/state/run-fsm.spec.md @@ -0,0 +1,161 @@ +# AI-chat run-lifecycle FSM — design spec (#488) + +This is the written design that `run-fsm.ts` implements. It ships in the PR (issue +#488 commit 1: "the spec is written FIRST and enters the PR"). It has four parts: +(1) the event × state transition table, (2) the map of every `chat-thread.tsx` ref +to {FSM state | FSM context | stays data}, (3) the run-fact protocol, (4) the +invariants. + +The reducer is a **pure function** `reduce(machine, event) → machine`. The returned +machine carries the **command effects** for that transition; a thin runtime in +`chat-thread.tsx` dispatches events and executes effects. Because it is pure, the +whole machine is enumerable and unit-tested directly (event × state → next state is +the observable property) — see `run-fsm.test.ts`. + +--- + +## 1. Event × state transition table + +Phases: `idle | sending | streaming | attaching | reconnecting(attempt,failed) | +polling(reason) | stalled | stopping | superseding | error(kind)`. +Context (orthogonal): `epoch`, `ownership: local|observer`, `runFact: {runId}|null`. + +Legend: **†** = command-transition (bumps `epoch`, I1). Effects in `[…]`. + +| Event (source) | From phase(s) | → To phase | Effects / ctx | +|---|---|---|---| +| `SEND_LOCAL` (user send) | idle, error, polling, stalled, reconnecting | sending **†** | `[cancelReconnect, disarmPoll]`, ownership=local | +| `STREAM_START{runId}` (SDK `start` metadata) | sending, attaching, reconnecting, superseding | streaming | `[cancelReconnect, disarmPoll]`, runFact←runId | +| `FINISH_CLEAN` (onFinish clean) | streaming, … | idle | `[disarmPoll, cancelReconnect]`, runFact←null | +| `FINISH_ABORT` (onFinish isAbort) | streaming, stopping | idle | `[disarmPoll, cancelReconnect]`, runFact←null (I4 exits stopping by this DATA) | +| `FINISH_DISCONNECT{hasVisibleContent}` (onFinish isDisconnect) | streaming | reconnecting(1) **†** *iff runFact* | `[scheduleReconnect(1)]` (+`armPoll(disconnect-visible)` if visible), ownership=observer | +| `FINISH_DISCONNECT` (no runFact) | streaming | idle | runFact←null (plain terminal "connection lost") | +| `FINISH_ERROR{kind}` (onFinish isError) | any | error(kind) | `[disarmPoll, cancelReconnect]`, runFact←null | +| `ATTACH_START{runId}` (mount resume) | idle | attaching **†** | `[resumeStream]`, ownership=observer, runFact←runId | +| `ATTACH_LIVE` (attach GET 2xx) | attaching | streaming | — | +| `ATTACH_NONE` (attach GET 204/err/throw) | attaching | polling(attach-none) | `[armPoll(attach-none)]` | +| `RECONNECT_BEGIN` | streaming, idle | reconnecting(1) **†** *iff runFact* | `[scheduleReconnect(1)]`, ownership=observer | +| `RECONNECT_ATTEMPT{n}` (backoff timer) | reconnecting | reconnecting(n) **†** | `[resumeStream]` | +| `RECONNECT_ATTACHED` (reconnect GET 2xx) | reconnecting | streaming | `[cancelReconnect, disarmPoll]` — **counter reset** (commit 3) | +| `RECONNECT_NONE` (reconnect GET 204/err), attempt e.type); +} +function hasEffect(m: Machine, type: Effect["type"]): boolean { + return m.effects.some((e) => e.type === type); +} + +describe("run-fsm — epoch invariant (I1)", () => { + it("drops an outcome carrying a stale epoch", () => { + // A command bumps the epoch; an outcome stamped with the OLD epoch is dropped. + const m0 = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); // epoch 0->1, attaching + expect(m0.ctx.epoch).toBe(1); + expect(m0.phase.name).toBe("attaching"); + // A late ATTACH_LIVE from a SUPERSEDED attempt (epoch 0) must NOT drive us. + const stale = reduce(m0, { type: "ATTACH_LIVE", epoch: 0 }); + expect(stale.phase.name).toBe("attaching"); + expect(stale.effects).toEqual([]); + }); + + it("applies an outcome carrying the current epoch", () => { + const m0 = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); + const live = reduce(m0, { type: "ATTACH_LIVE", epoch: m0.ctx.epoch }); + expect(live.phase.name).toBe("streaming"); + }); + + it("an outcome with no epoch is never dropped (trigger events)", () => { + const m0 = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); + const disposed = reduce(m0, { type: "DISPOSE" }); + expect(disposed.phase.name).toBe("idle"); + expect(hasEffect(disposed, "abortAttach")).toBe(true); + }); + + it("every command-transition increments the epoch exactly once", () => { + let m = initialMachine(); + const before = m.ctx.epoch; + m = reduce(m, { type: "SEND_LOCAL" }); + expect(m.ctx.epoch).toBe(before + 1); + m = reduce(m, { type: "STOP_REQUESTED" }); + expect(m.ctx.epoch).toBe(before + 2); + }); +}); + +describe("run-fsm — local turn", () => { + it("SEND_LOCAL → sending, local ownership, cancels recovery", () => { + const m = reduce(withRunFact(), { type: "SEND_LOCAL" }); + expect(m.phase.name).toBe("sending"); + expect(m.ctx.ownership).toBe("local"); + expect(effectTypes(m)).toEqual( + expect.arrayContaining(["cancelReconnect", "disarmPoll"]), + ); + }); + + it("STREAM_START adopts the runId into the run-fact and goes streaming", () => { + const m = run(initialMachine(), { type: "SEND_LOCAL" }); + const s = reduce(m, { type: "STREAM_START", runId: "run-9", epoch: m.ctx.epoch }); + expect(s.phase.name).toBe("streaming"); + expect(s.ctx.runFact).toEqual({ runId: "run-9" }); + }); + + it("FINISH_CLEAN → idle, run-fact cleared, poll/reconnect disarmed", () => { + const streaming = run(initialMachine(), { type: "SEND_LOCAL" }, { type: "STREAM_START", runId: "r" }); + const done = reduce(streaming, { type: "FINISH_CLEAN" }); + expect(done.phase.name).toBe("idle"); + expect(done.ctx.runFact).toBeNull(); + }); +}); + +// #488 commit 2 — SSE break BEFORE the first assistant frame must still recover. +describe("run-fsm — commit 2: reconnect by run-fact, not by assistant message", () => { + it("FINISH_DISCONNECT with an active run-fact → reconnecting (even with no visible content)", () => { + // Setup-phase break: no assistant frame yet, but a run-fact exists. + const streaming = withRunFact("run-2"); + const m = reduce(streaming, { + type: "FINISH_DISCONNECT", + hasVisibleContent: false, + epoch: streaming.ctx.epoch, + }); + expect(m.phase.name).toBe("reconnecting"); + if (m.phase.name === "reconnecting") expect(m.phase.attempt).toBe(1); + expect(m.ctx.ownership).toBe("observer"); + expect(hasEffect(m, "scheduleReconnect")).toBe(true); + // No visible content -> no poll arm yet (the reconnect ladder rebuilds it). + expect(hasEffect(m, "armPoll")).toBe(false); + }); + + it("FINISH_DISCONNECT WITH visible content also arms the poll", () => { + const m = reduce(withRunFact("run-2"), { + type: "FINISH_DISCONNECT", + hasVisibleContent: true, + epoch: 0, + }); + expect(m.phase.name).toBe("reconnecting"); + expect(hasEffect(m, "armPoll")).toBe(true); + }); + + it("FINISH_DISCONNECT with NO run-fact → idle (plain connection-lost)", () => { + const m = reduce(initialMachine(), { + type: "FINISH_DISCONNECT", + hasVisibleContent: true, + epoch: 0, + }); + expect(m.phase.name).toBe("idle"); + }); +}); + +// #488 commit 3 — a SECOND break after a successful re-attach starts a NEW ladder. +describe("run-fsm — commit 3: repeated reconnect cycles", () => { + it("two breaks in a row produce two reconnect cycles (counter resets on attach)", () => { + let m = withRunFact("run-3"); + // First break -> reconnecting(1). + m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("reconnecting"); + // Attempt fires, re-attaches live. + m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: 1, epoch: m.ctx.epoch }); + m = reduce(m, { type: "RECONNECT_ATTACHED", epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("streaming"); + // SECOND break: the counter was reset, so a fresh ladder starts at attempt 1 + // (the old one-shot !wasResumed gate would have sent this to silent poll). + m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("reconnecting"); + if (m.phase.name === "reconnecting") expect(m.phase.attempt).toBe(1); + expect(hasEffect(m, "scheduleReconnect")).toBe(true); + }); + + it("RECONNECT_NONE backs off through the ladder, then fails at the cap", () => { + let m = withRunFact("run-3"); + m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch }); + for (let n = 1; n < RECONNECT_MAX_ATTEMPTS; n++) { + m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: n, epoch: m.ctx.epoch }); + m = reduce(m, { type: "RECONNECT_NONE", epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("reconnecting"); + if (m.phase.name === "reconnecting") { + expect(m.phase.attempt).toBe(n + 1); + expect(m.phase.failed).toBe(false); + } + // The belt-and-suspenders poll is armed each failed attempt. + expect(hasEffect(m, "armPoll")).toBe(true); + } + // Final attempt fails -> failed banner (Retry), poll armed. + m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: RECONNECT_MAX_ATTEMPTS, epoch: m.ctx.epoch }); + m = reduce(m, { type: "RECONNECT_NONE", epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("reconnecting"); + if (m.phase.name === "reconnecting") expect(m.phase.failed).toBe(true); + // RETRY restarts at attempt 1. + m = reduce(m, { type: "RETRY" }); + expect(m.phase.name).toBe("reconnecting"); + if (m.phase.name === "reconnecting") { + expect(m.phase.attempt).toBe(1); + expect(m.phase.failed).toBe(false); + } + expect(hasEffect(m, "resumeStream")).toBe(true); + }); + + it("reconnectDelayMs is the exponential backoff 1s,2s,4s,8s,16s", () => { + expect([1, 2, 3, 4, 5].map(reconnectDelayMs)).toEqual([1000, 2000, 4000, 8000, 16000]); + }); +}); + +// #488 commit 4 — polling stalled-state + user-tail gating. +describe("run-fsm — commit 4: stalled + run-fact gating", () => { + it("POLL_IDLE_CAP: polling → stalled with a banner (poll disarmed), not silent", () => { + let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" }); + m = reduce(m, { type: "ATTACH_NONE", epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("polling"); + m = reduce(m, { type: "POLL_IDLE_CAP" }); + expect(m.phase.name).toBe("stalled"); + expect(hasEffect(m, "disarmPoll")).toBe(true); + }); + + it("RETRY from stalled re-arms the poll", () => { + let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" }); + m = reduce(m, { type: "ATTACH_NONE", epoch: m.ctx.epoch }); + m = reduce(m, { type: "POLL_IDLE_CAP" }); + m = reduce(m, { type: "RETRY" }); + expect(m.phase.name).toBe("polling"); + expect(hasEffect(m, "armPoll")).toBe(true); + }); + + it("a fresh NEGATIVE run-fact while attaching cancels recovery (user-tail, no active run)", () => { + // The mount POST /run returns no active run: attaching → idle, no poll armed. + let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" }); + m = reduce(m, { type: "RUN_FACT", runFact: null, epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("idle"); + expect(m.ctx.runFact).toBeNull(); + expect(hasEffect(m, "disarmPoll")).toBe(true); + }); + + it("a negative run-fact while polling stops the poll", () => { + let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" }); + m = reduce(m, { type: "ATTACH_NONE", epoch: m.ctx.epoch }); + m = reduce(m, { type: "RUN_FACT", runFact: null, epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("idle"); + }); + + it("POLL_TERMINAL settles polling → idle (I4 data-driven exit)", () => { + let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" }); + m = reduce(m, { type: "ATTACH_NONE", epoch: m.ctx.epoch }); + m = reduce(m, { type: "POLL_TERMINAL" }); + expect(m.phase.name).toBe("idle"); + expect(m.ctx.runFact).toBeNull(); + }); +}); + +// #488 commit 5 — error classification + supersede CAS transitions. +describe("run-fsm — commit 5: supersede CAS + error classification", () => { + it("SUPERSEDE_REQUESTED → superseding, fires the CAS effect, bumps epoch", () => { + const streaming = withRunFact("run-old"); + const m = reduce(streaming, { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" }); + expect(m.phase.name).toBe("superseding"); + expect(m.ctx.epoch).toBe(streaming.ctx.epoch + 1); + const sup = m.effects.find((e) => e.type === "supersede"); + expect(sup).toEqual({ type: "supersede", targetRunId: "run-old" }); + }); + + it("SUPERSEDE_READY → streaming as the new local owner", () => { + let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" }); + m = reduce(m, { type: "SUPERSEDE_READY", runId: "run-new", epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("streaming"); + expect(m.ctx.ownership).toBe("local"); + expect(m.ctx.runFact).toEqual({ runId: "run-new" }); + }); + + it("SUPERSEDE_MISMATCH → error(supersede-mismatch) + verify via /run (no blind banner)", () => { + let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" }); + m = reduce(m, { type: "SUPERSEDE_MISMATCH", currentRunId: "run-x", epoch: m.ctx.epoch }); + expect(m.phase).toEqual({ name: "error", kind: "supersede-mismatch" }); + expect(hasEffect(m, "postRun")).toBe(true); + expect(m.ctx.runFact).toEqual({ runId: "run-x" }); + }); + + it("SUPERSEDE_TIMEOUT → error(supersede-timeout), no auto-retry effect", () => { + let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" }); + m = reduce(m, { type: "SUPERSEDE_TIMEOUT", epoch: m.ctx.epoch }); + expect(m.phase).toEqual({ name: "error", kind: "supersede-timeout" }); + expect(m.effects).toEqual([]); + }); + + it("SUPERSEDE_INVALID → error(supersede-invalid)", () => { + let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" }); + m = reduce(m, { type: "SUPERSEDE_INVALID", epoch: m.ctx.epoch }); + expect(m.phase).toEqual({ name: "error", kind: "supersede-invalid" }); + }); + + it("a stale SUPERSEDE outcome from a superseded epoch is dropped", () => { + let m = reduce(withRunFact("run-old"), { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" }); + const supersedingEpoch = m.ctx.epoch; + // The user retriggers, bumping the epoch again. + m = reduce(m, { type: "SUPERSEDE_REQUESTED", targetRunId: "run-old" }); + // The first CAS's late TIMEOUT (old epoch) must NOT knock us out of superseding. + const late = reduce(m, { type: "SUPERSEDE_TIMEOUT", epoch: supersedingEpoch }); + expect(late.phase.name).toBe("superseding"); + }); + + it("RUN_ALREADY_ACTIVE (plain POST gate) → error(run-already-active), no retry effect", () => { + const m = reduce(run(initialMachine(), { type: "SEND_LOCAL" }), { type: "RUN_ALREADY_ACTIVE" }); + expect(m.phase).toEqual({ name: "error", kind: "run-already-active" }); + expect(m.effects).toEqual([]); + }); + + it("RUN_SUPERSEDED (observer's run killed) → attaching + postRun(observer-follow)", () => { + const observer = { ...withRunFact("run-dead"), ctx: { epoch: 0, ownership: "observer" as const, runFact: { runId: "run-dead" } } }; + const streaming: Machine = { phase: { name: "streaming" }, ctx: observer.ctx, effects: [] }; + const m = reduce(streaming, { type: "RUN_SUPERSEDED" }); + expect(m.phase.name).toBe("attaching"); + expect(m.effects.find((e) => e.type === "postRun")).toEqual({ + type: "postRun", + reason: "observer-follow", + }); + }); +}); + +describe("run-fsm — stop (I4: exit by data)", () => { + it("STOP_REQUESTED → stopping, fires stopRun + abortAttach, no data-independent exit", () => { + const m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); + expect(m.phase.name).toBe("stopping"); + expect(effectTypes(m)).toEqual(expect.arrayContaining(["stopRun", "abortAttach"])); + }); + + it("stopping exits on a terminal FINISH_ABORT (data), not on the stopRun response", () => { + let m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); + // No transition keys off the HTTP result — only the terminal finish moves us. + m = reduce(m, { type: "FINISH_ABORT", epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("idle"); + expect(m.ctx.runFact).toBeNull(); + }); + + it("stopping exits on a negative run-fact (data)", () => { + let m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); + m = reduce(m, { type: "RUN_FACT", runFact: null, epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("idle"); + }); +}); + +describe("run-fsm — ownership (I2) is context, orthogonal to phase", () => { + it("attach/reconnect set observer; send/supersede-ready set local", () => { + let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); + expect(m.ctx.ownership).toBe("observer"); + m = reduce(m, { type: "ATTACH_LIVE", epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("streaming"); + expect(m.ctx.ownership).toBe("observer"); // still observing a detached run + // A local send flips ownership back to local. + m = reduce(m, { type: "SEND_LOCAL" }); + expect(m.ctx.ownership).toBe("local"); + }); +}); + +describe("run-fsm — dispose (I5)", () => { + it("DISPOSE from any phase aborts controllers and bumps epoch", () => { + let m = reduce(withRunFact(), { type: "ATTACH_START", runId: "r" }); + const before = m.ctx.epoch; + m = reduce(m, { type: "DISPOSE" }); + expect(m.phase.name).toBe("idle"); + expect(m.ctx.epoch).toBe(before + 1); + expect(effectTypes(m)).toEqual( + expect.arrayContaining(["abortAttach", "cancelReconnect", "disarmPoll"]), + ); + }); +}); diff --git a/apps/client/src/features/ai-chat/state/run-fsm.ts b/apps/client/src/features/ai-chat/state/run-fsm.ts new file mode 100644 index 00000000..89348384 --- /dev/null +++ b/apps/client/src/features/ai-chat/state/run-fsm.ts @@ -0,0 +1,513 @@ +/** + * Run-lifecycle finite state machine for a single AI-chat thread (#488). + * + * ============================================================================ + * WHY THIS EXISTS + * ---------------------------------------------------------------------------- + * The resume/reconnect/poll/stop/supersede lifecycle used to be spread across + * ~26 `useRef` one-shot flags in `chat-thread.tsx`, each disarmed "on every + * path". Ownerless flag combinations produced silent UI freezes, and every fix + * added another ref (the #381 -> #432 -> #456 spiral). This module replaces that + * ref-zoo with ONE pure reducer whose transitions are enumerable and unit- + * testable in isolation (event x state -> next state is the observable property). + * + * The reducer is PURE: it owns no timers, no fetches, no React state. It maps + * `(machine, event) -> machine`, where the returned machine carries the list of + * COMMAND EFFECTS to run for that transition. A thin runtime in `chat-thread.tsx` + * dispatches events (from SDK callbacks / HTTP outcomes) and executes the + * effects (attach GET, POST /stream, POST /run, POST /stop, backoff timers, + * poll arm/disarm). The runtime lives in a THREAD, not the window, so a late SDK + * callback dies with the owner (kills the "event from a dead view" class, #161). + * + * ============================================================================ + * INVARIANTS (see run-fsm.spec.md for the full spec + tables) + * ---------------------------------------------------------------------------- + * I1 EPOCH (generation counter). Commands (`resumeStream`, `postRun`, `stop`, + * `supersede`, `scheduleReconnect`) are async; their outcomes arrive on the + * SAME SDK/HTTP callbacks. Every command-emitting transition increments + * `ctx.epoch`; every OUTCOME event carries the epoch it was issued under; + * the reducer DROPS an outcome whose epoch != the current epoch. This is + * what the one-shot-ref zoo used to approximate by hand. + * I2 OWNERSHIP is a CONTEXT FIELD (`'local' | 'observer'`), not a state — + * orthogonal to the transport phase. The queue is flushed ONLY by a local + * owner (an observer following a detached run never flushes). + * I3 RUN-FACT ("a run is active") is first-class from the server: `runFact` + * holds the server-confirmed active run id (POST /run on mount, the `start` + * metadata runId, attach outcomes). Reconnect is entered by the RUN-FACT, + * not by the presence of an assistant message (#488 commit 2). A fresh + * negative fact (null) cancels reconnect immediately. + * I4 Exit `stopping` by DATA (a terminal row / negative run-fact), NEVER by the + * stopRun HTTP response (which returns after abort, before finalization). + * I5 Command controllers are effect-owned (abort in cleanup), NOT render-phase + * refs — expressed here as the `abortAttach` effect on disposing transitions. + * ============================================================================ + */ + +// --------------------------------------------------------------------------- +// Phases (the transport lifecycle). Ownership / runFact are CONTEXT, not here. +// --------------------------------------------------------------------------- + +/** Why the degraded poll is the active recovery. */ +export type PollReason = + | "attach-none" // mount attach returned 204 / error — nothing live to attach + | "starved" // a resumed finish carried no visible content + | "disconnect-visible" // a live disconnect WITH on-screen content — poll to terminal + | "reconnect-exhausted"; // the live re-attach ladder gave up + +/** The classified error kind (drives the banner text + composer behavior). */ +export type ErrorKind = + | "stream" // a generic provider/network stream error (useChat error) + | "run-already-active" // 409 A_RUN_ALREADY_ACTIVE (a plain POST hit the gate) + | "supersede-mismatch" // 409 SUPERSEDE_TARGET_MISMATCH (CAS target moved) + | "supersede-timeout" // 409 SUPERSEDE_TIMEOUT (old run did not settle in W) + | "supersede-invalid" // 409 SUPERSEDE_INVALID (bad supersede target) + | "begin-failed"; // 503 A_RUN_BEGIN_FAILED (could not start the run) + +export type Phase = + | { name: "idle" } + | { name: "sending" } // local POST in flight, before the first frame + | { name: "streaming" } // receiving frames + | { name: "attaching" } // mount-time attach GET in flight + | { name: "reconnecting"; attempt: number; failed: boolean } + | { name: "polling"; reason: PollReason } + | { name: "stalled" } // poll hit the inactivity cap — banner + Retry + | { name: "stopping" } + | { name: "superseding" } + | { name: "error"; kind: ErrorKind }; + +export type Ownership = "local" | "observer"; + +/** The server-confirmed active run, or null when no run is active. */ +export type RunFact = { runId: string } | null; + +export interface Ctx { + /** I1: generation counter — every command-transition increments it. */ + epoch: number; + /** I2: does THIS client own the turn's writes (local streamer) or observe? */ + ownership: Ownership; + /** I3: the server-confirmed active run. */ + runFact: RunFact; +} + +export interface Machine { + phase: Phase; + ctx: Ctx; + /** Command effects to run for the transition that produced THIS machine. + * The runtime executes them and does not read them again. */ + effects: Effect[]; +} + +// --------------------------------------------------------------------------- +// Command effects (the reducer's only side-channel — executed by the runtime). +// --------------------------------------------------------------------------- + +export type Effect = + /** POST /run to (re)establish or verify the run-fact. `reason` is diagnostic. */ + | { type: "postRun"; reason: "mount" | "verify" | "observer-follow" } + /** Trigger the SDK `resumeStream()` (attach GET via prepareReconnectToStream). */ + | { type: "resumeStream" } + /** Schedule a reconnect attempt after a backoff, then dispatch RECONNECT_ATTEMPT. */ + | { type: "scheduleReconnect"; attempt: number; delayMs: number } + /** Cancel any pending reconnect backoff timer. */ + | { type: "cancelReconnect" } + /** Arm the degraded poll (the window's dumb timer follows the run in the DB). */ + | { type: "armPoll"; reason: PollReason } + /** Disarm the degraded poll. */ + | { type: "disarmPoll" } + /** POST /stop the chat's active run (authoritative detached-run stop). */ + | { type: "stopRun" } + /** POST /stream { supersede: { runId } } — the CAS "interrupt and send now". */ + | { type: "supersede"; targetRunId: string } + /** Abort the in-flight attach/reconnect GET controller (dispose / observer stop). */ + | { type: "abortAttach" }; + +// --------------------------------------------------------------------------- +// Events. An OUTCOME event MAY carry `epoch`; if it does and it does not equal +// the current epoch, the reducer drops it (I1). Trigger events (user actions, +// fresh disconnects) carry no epoch and are never dropped. +// --------------------------------------------------------------------------- + +export type Event = + // -- local turn -- + | { type: "SEND_LOCAL" } + | { type: "STREAM_START"; runId?: string; epoch?: number } + | { type: "FINISH_CLEAN"; epoch?: number } + | { type: "FINISH_ABORT"; epoch?: number } + | { type: "FINISH_DISCONNECT"; hasVisibleContent: boolean; epoch?: number } + | { type: "FINISH_ERROR"; kind: ErrorKind; epoch?: number } + // -- mount attach (resume) -- + | { type: "ATTACH_START"; runId?: string } + | { type: "ATTACH_LIVE"; epoch?: number } + | { type: "ATTACH_NONE"; epoch?: number } + // -- reconnect after a live disconnect -- + /** #488 commit 2: entered by the RUN-FACT, not by an assistant message. */ + | { type: "RECONNECT_BEGIN" } + | { type: "RECONNECT_ATTEMPT"; attempt: number; epoch?: number } + | { type: "RECONNECT_ATTACHED"; epoch?: number } + | { type: "RECONNECT_NONE"; epoch?: number } + | { type: "RETRY" } + // -- degraded poll -- + | { type: "POLL_ACTIVITY" } + | { type: "POLL_TERMINAL" } + | { type: "POLL_IDLE_CAP" } + // -- run-fact (server-confirmed active run) -- + | { type: "RUN_FACT"; runFact: RunFact; epoch?: number } + // -- stop -- + | { type: "STOP_REQUESTED" } + // -- supersede (CAS) -- + | { type: "SUPERSEDE_REQUESTED"; targetRunId: string } + | { type: "SUPERSEDE_READY"; runId?: string; epoch?: number } + | { type: "SUPERSEDE_MISMATCH"; currentRunId?: string; epoch?: number } + | { type: "SUPERSEDE_TIMEOUT"; epoch?: number } + | { type: "SUPERSEDE_INVALID"; epoch?: number } + | { type: "RUN_ALREADY_ACTIVE" } + /** An observer's attached run was aborted because it was superseded server-side: + * ask for the latest run and follow it (else an observer tab freezes on the + * killed run). */ + | { type: "RUN_SUPERSEDED" } + // -- lifecycle -- + | { type: "DISPOSE" }; + +export const RECONNECT_MAX_ATTEMPTS = 5; +export const RECONNECT_BASE_DELAY_MS = 1000; +/** Backoff before attempt N (1-based): 1s, 2s, 4s, 8s, 16s. */ +export function reconnectDelayMs(attempt: number): number { + return RECONNECT_BASE_DELAY_MS * 2 ** (attempt - 1); +} + +// --------------------------------------------------------------------------- +// Constructors / helpers. +// --------------------------------------------------------------------------- + +export function initialMachine(overrides?: Partial): Machine { + return { + phase: { name: "idle" }, + ctx: { epoch: 0, ownership: "local", runFact: null, ...overrides }, + effects: [], + }; +} + +/** Build a machine result: a phase, optional ctx patch, and effects. Empty + * effects by default. Never mutates the input. */ +function to( + m: Machine, + phase: Phase, + opts?: { ctx?: Partial; effects?: Effect[] }, +): Machine { + return { + phase, + ctx: { ...m.ctx, ...(opts?.ctx ?? {}) }, + effects: opts?.effects ?? [], + }; +} + +/** No transition: keep the phase, clear effects (so a re-run does not re-fire). */ +function stay(m: Machine): Machine { + return { phase: m.phase, ctx: m.ctx, effects: [] }; +} + +/** A command-transition: same as `to` but bumps the epoch (I1). Any outcome + * event issued under the old epoch is dropped once this lands. */ +function command( + m: Machine, + phase: Phase, + effects: Effect[], + ctx?: Partial, +): Machine { + return { + phase, + ctx: { ...m.ctx, ...(ctx ?? {}), epoch: m.ctx.epoch + 1 }, + effects, + }; +} + +// --------------------------------------------------------------------------- +// The pure reducer. +// --------------------------------------------------------------------------- + +export function reduce(m: Machine, event: Event): Machine { + // I1: drop a stale outcome (an event issued under a superseded epoch). + if ("epoch" in event && event.epoch !== undefined && event.epoch !== m.ctx.epoch) { + return stay(m); + } + + switch (event.type) { + // ---- local turn ---------------------------------------------------- + case "SEND_LOCAL": + // A local send owns the view: leave any recovery, become the local + // streamer, disarm poll/reconnect. epoch++ so a late recovery outcome + // from the previous phase is dropped. + return command( + m, + { name: "sending" }, + [{ type: "cancelReconnect" }, { type: "disarmPoll" }], + { ownership: "local" }, + ); + + case "STREAM_START": { + // First frame arrived. Adopt the run-fact runId if present. sending -> + // streaming; a reconnect/attach that just went live also lands here. + const runFact = event.runId ? { runId: event.runId } : m.ctx.runFact; + return to(m, { name: "streaming" }, { + ctx: { runFact }, + effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }], + }); + } + + case "FINISH_CLEAN": + // A clean terminal outcome. The run is done — clear the run-fact and go + // idle. (The queue flush is a component concern gated by ownership; the + // FSM only models the phase.) + return to(m, { name: "idle" }, { + ctx: { runFact: null }, + effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], + }); + + case "FINISH_ABORT": + // A user Stop / intentional abort finished. If we were stopping, the + // terminal data has now arrived (I4) — go idle. The run-fact is cleared. + return to(m, { name: "idle" }, { + ctx: { runFact: null }, + effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], + }); + + case "FINISH_DISCONNECT": + // A LIVE SSE drop. If a run is (or may be) active, recover: + // - visible content already on screen -> keep it, poll to terminal + // (a full replay could clobber the fuller live tail), AND begin the + // live re-attach ladder; + // - no visible content -> the reconnect ladder rebuilds it. + // #488 commit 2: recovery is gated on the RUN-FACT, not on the presence + // of an assistant message — a break during the setup phase (before the + // first assistant frame) still has an active detached run to pick up. + if (m.ctx.runFact) { + const effects: Effect[] = [ + { type: "scheduleReconnect", attempt: 1, delayMs: reconnectDelayMs(1) }, + ]; + if (event.hasVisibleContent) effects.push({ type: "armPoll", reason: "disconnect-visible" }); + return command(m, { name: "reconnecting", attempt: 1, failed: false }, effects, { + ownership: "observer", + }); + } + // No run to recover: a plain disconnect. Surface the terminal notice. + return to(m, { name: "idle" }, { ctx: { runFact: null } }); + + case "FINISH_ERROR": + return to(m, { name: "error", kind: event.kind }, { + ctx: { runFact: null }, + effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], + }); + + // ---- mount attach (resume) ---------------------------------------- + case "ATTACH_START": + // A reopened tab attaches to a still-running run: observer ownership. + return command(m, { name: "attaching" }, [{ type: "resumeStream" }], { + ownership: "observer", + runFact: event.runId ? { runId: event.runId } : m.ctx.runFact, + }); + + case "ATTACH_LIVE": + // The attach GET returned a live 2xx stream — follow it as an observer. + return to(m, { name: "streaming" }); + + case "ATTACH_NONE": + // 204 / non-2xx / throw: nothing live to attach. Arm the degraded poll to + // follow the run to terminal from the DB. This is a soft-negative run-fact + // (204 on a non-stripped path is authoritative-negative; the runtime may + // pass a RUN_FACT null separately). Keep the run-fact as-is here. + return to(m, { name: "polling", reason: "attach-none" }, { + effects: [{ type: "armPoll", reason: "attach-none" }], + }); + + // ---- reconnect after a live disconnect ---------------------------- + case "RECONNECT_BEGIN": + // Explicit begin (used when the runtime decides to reconnect from a signal + // other than FINISH_DISCONNECT). Requires a run-fact (I3). + if (!m.ctx.runFact) return stay(m); + return command( + m, + { name: "reconnecting", attempt: 1, failed: false }, + [{ type: "scheduleReconnect", attempt: 1, delayMs: reconnectDelayMs(1) }], + { ownership: "observer" }, + ); + + case "RECONNECT_ATTEMPT": + // A scheduled backoff fired — fire the attach GET. epoch++ so the previous + // attempt's late outcome cannot drive this one. + if (m.phase.name !== "reconnecting") return stay(m); + return command( + m, + { name: "reconnecting", attempt: event.attempt, failed: false }, + [{ type: "resumeStream" }], + ); + + case "RECONNECT_ATTACHED": + // #488 commit 3: a live re-attach succeeded. Reset to streaming — the + // attempt counter is dropped, so a LATER disconnect can start a fresh + // ladder from attempt 1 (the old one-shot `!wasResumed` gate forbade a + // second cycle, sending the second break to silent poll). + return to(m, { name: "streaming" }, { + effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }], + }); + + case "RECONNECT_NONE": { + // 204 / error during a reconnect attempt. Arm the degraded poll as the + // belt-and-suspenders fallback, then either back off to the next attempt + // or, at the cap, surface the manual Retry ("failed"). + if (m.phase.name !== "reconnecting") return stay(m); + const attempt = m.phase.attempt; + if (attempt < RECONNECT_MAX_ATTEMPTS) { + return command( + m, + { name: "reconnecting", attempt: attempt + 1, failed: false }, + [ + { type: "armPoll", reason: "attach-none" }, + { type: "scheduleReconnect", attempt: attempt + 1, delayMs: reconnectDelayMs(attempt + 1) }, + ], + ); + } + return to(m, { name: "reconnecting", attempt, failed: true }, { + effects: [{ type: "armPoll", reason: "reconnect-exhausted" }], + }); + } + + case "RETRY": + // Manual Retry from the "failed" reconnect banner OR the stalled banner. + if (m.phase.name === "reconnecting" && m.phase.failed) { + return command( + m, + { name: "reconnecting", attempt: 1, failed: false }, + [{ type: "resumeStream" }], + ); + } + if (m.phase.name === "stalled") { + // Re-arm the poll to try to catch the run up again. + return command(m, { name: "polling", reason: "attach-none" }, [ + { type: "armPoll", reason: "attach-none" }, + ]); + } + return stay(m); + + // ---- degraded poll ------------------------------------------------- + case "POLL_ACTIVITY": + // The polled rows changed (progress). No phase change — the runtime resets + // its inactivity clock. Only meaningful while a poll-bearing phase is active. + return stay(m); + + case "POLL_TERMINAL": + // The run reached a terminal row via the poll (or the reconcile merge). Go + // idle and disarm everything (I4: this is a DATA-driven exit, incl. exit + // from `stopping`). + return to(m, { name: "idle" }, { + ctx: { runFact: null }, + effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], + }); + + case "POLL_IDLE_CAP": + // #488 commit 4a: the poll hit the inactivity cap. Instead of going SILENT + // (the old "forever half-done answer"), surface a stalled banner + Retry. + if (m.phase.name !== "polling" && m.phase.name !== "reconnecting") return stay(m); + return to(m, { name: "stalled" }, { + effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], + }); + + // ---- run-fact ------------------------------------------------------ + case "RUN_FACT": { + const runFact = event.runFact; + // A fresh NEGATIVE fact (no active run) cancels recovery immediately (I3): + // there is nothing to reconnect to / poll for. + if (!runFact) { + if ( + m.phase.name === "reconnecting" || + m.phase.name === "attaching" || + m.phase.name === "polling" || + m.phase.name === "stopping" + ) { + return to(m, { name: "idle" }, { + ctx: { runFact: null }, + effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }], + }); + } + return to(m, m.phase, { ctx: { runFact: null } }); + } + // A positive fact just updates the context (pessimism toward an attempt: a + // stale-but-positive fact permits entering recovery; a 204 will cut it). + return to(m, m.phase, { ctx: { runFact } }); + } + + // ---- stop ---------------------------------------------------------- + case "STOP_REQUESTED": + // Authoritative stop of a detached run. Enter `stopping` and fire stopRun + + // abort the local/attach reader. Exit is by DATA (I4), never by the HTTP + // response — so no transition keys off stopRun's return. + return command( + m, + { name: "stopping" }, + [{ type: "stopRun" }, { type: "abortAttach" }, { type: "cancelReconnect" }], + ); + + // ---- supersede (CAS) ---------------------------------------------- + case "SUPERSEDE_REQUESTED": + // "Interrupt and send now": CAS POST /stream { supersede }. epoch++ so a + // late outcome of the interrupted run is dropped. + return command( + m, + { name: "superseding" }, + [{ type: "supersede", targetRunId: event.targetRunId }, { type: "cancelReconnect" }, { type: "disarmPoll" }], + ); + + case "SUPERSEDE_READY": { + // CAS succeeded (old run stopped/settled, slot taken, new run begun). We + // are now the local streamer of the NEW run. Adopt its runId if provided. + const runFact = event.runId ? { runId: event.runId } : m.ctx.runFact; + return to(m, { name: "streaming" }, { ctx: { ownership: "local", runFact } }); + } + + case "SUPERSEDE_MISMATCH": + // The active run moved between the click and the CAS. Per the spec: verify + // via /run rather than blindly banner — the mismatch may be our own already- + // superseded run. Surface a classified error AND fire a run-fact verify. + return to(m, { name: "error", kind: "supersede-mismatch" }, { + ctx: { runFact: event.currentRunId ? { runId: event.currentRunId } : m.ctx.runFact }, + effects: [{ type: "postRun", reason: "verify" }], + }); + + case "SUPERSEDE_TIMEOUT": + // The old run did not settle within W. Nothing persisted; the composer keeps + // its text. Classified error, NO auto-retry (the old client retry ladder is + // removed in #488 commit 5). + return to(m, { name: "error", kind: "supersede-timeout" }); + + case "SUPERSEDE_INVALID": + return to(m, { name: "error", kind: "supersede-invalid" }); + + case "RUN_ALREADY_ACTIVE": + // A plain POST hit the one-active-run gate. NO auto-retry — the composer + // offers "interrupt and send" (supersede) instead. + return to(m, { name: "error", kind: "run-already-active" }); + + case "RUN_SUPERSEDED": + // An observer's attached run was killed by a server-side supersede. Ask for + // the latest run and follow it, instead of freezing on the dead run. + return command(m, { name: "attaching" }, [{ type: "postRun", reason: "observer-follow" }], { + ownership: "observer", + }); + + // ---- lifecycle ----------------------------------------------------- + case "DISPOSE": + // Unmount: abort in-flight controllers, drop timers, and bump the epoch so + // NO late callback can drive this (now dead) machine (I5). + return command(m, { name: "idle" }, [ + { type: "abortAttach" }, + { type: "cancelReconnect" }, + { type: "disarmPoll" }, + ]); + + default: { + // Exhaustiveness guard. + const _never: never = event; + void _never; + return stay(m); + } + } +} From ba584939093ef66d4eee9465bac828c2e746b9f7 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 05:58:09 +0300 Subject: [PATCH 31/41] =?UTF-8?q?fix(client):=20=D0=BA=D0=BB=D0=B0=D1=81?= =?UTF-8?q?=D1=81=D0=B8=D1=84=D0=B8=D0=BA=D0=B0=D1=86=D0=B8=D1=8F=20409-?= =?UTF-8?q?=D0=BA=D0=BE=D0=B4=D0=BE=D0=B2=20+=20run-=D1=84=D0=B0=D0=BA?= =?UTF-8?q?=D1=82=20=D0=BF=D0=BB=D1=83=D0=BC=D0=B1=D0=B8=D0=BD=D0=B3=20(#4?= =?UTF-8?q?88)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Потребляет реальный контракт #487 на клиенте (без выдумывания кодов): - error-message.ts: ветки для 409 A_RUN_ALREADY_ACTIVE / SUPERSEDE_TARGET_MISMATCH / SUPERSEDE_TIMEOUT / SUPERSEDE_INVALID — человеческие тексты СТРОГО ДО generic- веток по статусу (иначе юзер видит сырой JSON {"code":"A_RUN_ALREADY_ACTIVE"}); - extractRunId(message): чтение runId из start-метаданных (зеркало extractServerChatId) — live-обновление run-факта для FSM; - getRun(chatId): POST /ai-chat/run — first-class run-факт с сервера (init на маунте + verify после supersede-mismatch). Плумбинг под FSM-обвязку коммитов 2–5. Тесты: классификатор (все 4 кода + order- guard), extractRunId. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/services/ai-chat-service.ts | 19 ++++++++ .../ai-chat/utils/adopt-chat-id.test.ts | 15 +++++++ .../features/ai-chat/utils/adopt-chat-id.ts | 14 ++++++ .../ai-chat/utils/error-message.test.ts | 45 +++++++++++++++++++ .../features/ai-chat/utils/error-message.ts | 38 ++++++++++++++++ 5 files changed, 131 insertions(+) diff --git a/apps/client/src/features/ai-chat/services/ai-chat-service.ts b/apps/client/src/features/ai-chat/services/ai-chat-service.ts index 9946248a..34e6aa71 100644 --- a/apps/client/src/features/ai-chat/services/ai-chat-service.ts +++ b/apps/client/src/features/ai-chat/services/ai-chat-service.ts @@ -57,6 +57,25 @@ export async function stopRun( return req.data; } +/** + * #488: the run-fact — "is a run active on this chat?" — first-class from the + * server (POST /ai-chat/run). Called on mount to seed the client FSM's run-fact + * and to VERIFY after a supersede mismatch (an observer following a superseded + * run asks for the latest run and follows it). Returns the latest run row (with + * its `id` and `status`) and its projected assistant message, or `run: null` when + * the chat has never had a run. Owner-gated server-side. + */ +export async function getRun(chatId: string): Promise<{ + run: { id: string; status: string } | null; + message: IAiChatMessageRow | null; +}> { + const req = await api.post<{ + run: { id: string; status: string } | null; + message: IAiChatMessageRow | null; + }>("/ai-chat/run", { chatId }); + return req.data; +} + /** * Resolve the chat bound to a document (the current user's most-recent chat * created on that page), or null when there is none. Drives auto-open-on-page. diff --git a/apps/client/src/features/ai-chat/utils/adopt-chat-id.test.ts b/apps/client/src/features/ai-chat/utils/adopt-chat-id.test.ts index c9ff117a..27ec39b5 100644 --- a/apps/client/src/features/ai-chat/utils/adopt-chat-id.test.ts +++ b/apps/client/src/features/ai-chat/utils/adopt-chat-id.test.ts @@ -3,6 +3,7 @@ import { resolveAdoptedChatId, newlyAddedChatIds, extractServerChatId, + extractRunId, } from "./adopt-chat-id"; describe("resolveAdoptedChatId", () => { @@ -70,3 +71,17 @@ describe("extractServerChatId", () => { expect(extractServerChatId(undefined)).toBeUndefined(); }); }); + +describe("extractRunId", () => { + it("reads a string runId from the start metadata", () => { + expect(extractRunId({ metadata: { runId: "run-1" } })).toBe("run-1"); + }); + it("returns undefined when runId is absent", () => { + expect(extractRunId({ metadata: { chatId: "c" } })).toBeUndefined(); + expect(extractRunId({})).toBeUndefined(); + expect(extractRunId(undefined)).toBeUndefined(); + }); + it("returns undefined for a non-string runId", () => { + expect(extractRunId({ metadata: { runId: 7 } })).toBeUndefined(); + }); +}); diff --git a/apps/client/src/features/ai-chat/utils/adopt-chat-id.ts b/apps/client/src/features/ai-chat/utils/adopt-chat-id.ts index 0c01dd91..b7dad698 100644 --- a/apps/client/src/features/ai-chat/utils/adopt-chat-id.ts +++ b/apps/client/src/features/ai-chat/utils/adopt-chat-id.ts @@ -56,6 +56,20 @@ export function extractServerChatId( return typeof m?.chatId === "string" ? m.chatId : undefined; } +/** + * #488: read the authoritative RUN id off a streaming assistant message. The + * server attaches it as `message.metadata.runId` on the `start` part when a run + * wraps the turn (see server `chatStreamMetadata`, #184/#487). This is the live + * run-fact update the client FSM adopts (mirrors `extractServerChatId`). Returns + * it only when it is a string; undefined otherwise. + */ +export function extractRunId( + message: { metadata?: unknown } | undefined, +): string | undefined { + const m = message?.metadata as { runId?: string } | undefined; + return typeof m?.runId === "string" ? m.runId : undefined; +} + /** * The deduped set of ids present in `afterIds` but not in `beforeIds`. A * paginated/flatMapped list can repeat the same id, so dedupe: one genuinely-new diff --git a/apps/client/src/features/ai-chat/utils/error-message.test.ts b/apps/client/src/features/ai-chat/utils/error-message.test.ts index 153455ac..594503f5 100644 --- a/apps/client/src/features/ai-chat/utils/error-message.test.ts +++ b/apps/client/src/features/ai-chat/utils/error-message.test.ts @@ -42,6 +42,51 @@ describe("describeChatError", () => { ); }); + // #488 commit 5: the #487 concurrency-gate / supersede 409s. FULL real bodies: + // a ConflictException(object) whose response is serialized verbatim, carrying a + // `code` and statusCode 409. Each must classify to a human text, not raw JSON. + it("classifies A_RUN_ALREADY_ACTIVE (409) as already-answering, not raw JSON", () => { + const body = + '{"message":"A run is already active for this chat","code":"A_RUN_ALREADY_ACTIVE","statusCode":409}'; + expect(describeChatError(body, t).title).toBe( + "The agent is already answering", + ); + // Never leaks the raw code as the detail. + expect(describeChatError(body, t).detail).not.toContain("A_RUN_ALREADY_ACTIVE"); + }); + + it("classifies SUPERSEDE_TARGET_MISMATCH (409) as run-changed", () => { + const body = + '{"message":"active run does not match the supersede target","code":"SUPERSEDE_TARGET_MISMATCH","runId":"run-x","statusCode":409}'; + expect(describeChatError(body, t).title).toBe( + "Couldn't interrupt — the run changed", + ); + }); + + it("classifies SUPERSEDE_TIMEOUT (409) as couldn't-interrupt-in-time", () => { + const body = + '{"message":"the run did not settle within the supersede window","code":"SUPERSEDE_TIMEOUT","statusCode":409}'; + expect(describeChatError(body, t).title).toBe("Couldn't interrupt in time"); + }); + + it("classifies SUPERSEDE_INVALID (409) as couldn't-interrupt-that-run", () => { + const body = + '{"message":"supervise requires chatId","code":"SUPERSEDE_INVALID","statusCode":409}'; + expect(describeChatError(body, t).title).toBe( + "Couldn't interrupt that run", + ); + }); + + it("ORDER GUARD: A_RUN_ALREADY_ACTIVE wins over any generic status branch", () => { + // Even though the body could superficially look 4xx-ish, the code branch runs + // first, so it is never mislabeled by a generic status heading. + const body = + '{"message":"conflict","code":"A_RUN_ALREADY_ACTIVE","statusCode":409}'; + const view = describeChatError(body, t); + expect(view.title).not.toBe("Something went wrong"); + expect(view.title).not.toBe("AI provider not configured"); + }); + it("classifies a dropped connection (ECONNRESET) as a lost-connection error", () => { expect( describeChatError("Cannot connect to API: read ECONNRESET", t).title, diff --git a/apps/client/src/features/ai-chat/utils/error-message.ts b/apps/client/src/features/ai-chat/utils/error-message.ts index 2d59c177..897bb629 100644 --- a/apps/client/src/features/ai-chat/utils/error-message.ts +++ b/apps/client/src/features/ai-chat/utils/error-message.ts @@ -39,6 +39,44 @@ export function describeChatError( }; } + // #488 commit 5: the #487 concurrency-gate / supersede 409s. These arrive as a + // ConflictException(object) body carrying a `code` (and statusCode 409). They + // MUST be classified by `code` STRICTLY BEFORE any generic status branch, or the + // user sees the raw JSON `{"code":"A_RUN_ALREADY_ACTIVE",…}`. The code strings + // are the real #487 server contract (ai-chat.controller.ts) — do not invent. + if (/"code"\s*:\s*"A_RUN_ALREADY_ACTIVE"/.test(msg)) { + return { + title: t("The agent is already answering"), + detail: t( + "This chat already has a run in progress. Wait for it to finish, or interrupt it and send now.", + ), + }; + } + if (/"code"\s*:\s*"SUPERSEDE_TARGET_MISMATCH"/.test(msg)) { + return { + title: t("Couldn't interrupt — the run changed"), + detail: t( + "The run you tried to interrupt is no longer the active one. Check the latest answer and try again.", + ), + }; + } + if (/"code"\s*:\s*"SUPERSEDE_TIMEOUT"/.test(msg)) { + return { + title: t("Couldn't interrupt in time"), + detail: t( + "The previous run didn't stop in time. Nothing was sent — try sending again.", + ), + }; + } + if (/"code"\s*:\s*"SUPERSEDE_INVALID"/.test(msg)) { + return { + title: t("Couldn't interrupt that run"), + detail: t( + "The run to interrupt doesn't belong to this chat. Reload and try again.", + ), + }; + } + if (/"statusCode"\s*:\s*403\b/.test(msg)) { return { title: t("AI chat is disabled"), From d533daa45f218e34381417c5caea83de332a77eb Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 06:04:00 +0300 Subject: [PATCH 32/41] =?UTF-8?q?fix(client):=20=D0=BE=D0=B1=D1=80=D1=8B?= =?UTF-8?q?=D0=B2=20SSE=20=D0=B4=D0=BE=20=D0=BF=D0=B5=D1=80=D0=B2=D0=BE?= =?UTF-8?q?=D0=B3=D0=BE=20=D0=BA=D0=B0=D0=B4=D1=80=D0=B0=20=D0=B0=D1=81?= =?UTF-8?q?=D1=81=D0=B8=D1=81=D1=82=D0=B5=D0=BD=D1=82=D0=B0=20=E2=86=92=20?= =?UTF-8?q?=D1=80=D0=B0=D0=BD=20=D0=BF=D0=BE=D0=B4=D1=85=D0=B2=D0=B0=D1=82?= =?UTF-8?q?=D1=8B=D0=B2=D0=B0=D0=B5=D1=82=D1=81=D1=8F=20(#488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Коммит 2. Раньше вход в reconnect требовал `message?.role === "assistant"`, и обрыв в setup-фазе (до первого кадра ассистента, включая сборку MCP-тулсета с дедлайном до 60 c) не давал НИ реконнекта, НИ поллинга — а detached-ран продолжал писать в страницы (тихая дыра целостности). Правка: вход в reconnect по РАН-ФАКТУ (активный detached-ран), а не по наличию assistant-сообщения. В autonomous-режиме ран активен весь ход, поэтому здесь сигнал run-факта — сам autonomousRunsEnabled; более богатый серверный run-факт (POST /run / runId из start-метаданных) смоделирован и покрыт тестами в FSM (run-fsm.ts FINISH_DISCONNECT по ctx.runFact) и приезжает с полной миграцией компонента. При отсутствии assistant-строки reconnect идёт БЕЗ strip/anchor (простой live-attach — на экране нечего перестраивать). Тест: «обрыв до первого кадра → баннер reconnect + resumeStream + attach без anchor», плюс FSM-переход (уже зелёный в коммите 1). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/chat-thread.test.tsx | 29 ++++++++ .../ai-chat/components/chat-thread.tsx | 73 ++++++++++++------- 2 files changed, 76 insertions(+), 26 deletions(-) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index 0cebf8e6..c6b8b65a 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -1122,6 +1122,35 @@ describe("ChatThread — live reconnect after isDisconnect (#430)", () => { ); }); + // #488 commit 2: a break during the SETUP phase — before the first assistant + // frame (onFinish carries no assistant message) — must STILL reconnect, because + // a detached autonomous run keeps writing to pages. The old gate required + // `message?.role === "assistant"`, so this fell through to neither reconnect nor + // poll. With no anchor row, the attach is a PLAIN live attach (no strip/anchor). + it("#488 commit 2: disconnect BEFORE the first assistant frame reconnects with no anchor", () => { + renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); + expect(h.state.resumeStream).not.toHaveBeenCalled(); + act(() => { + h.state.onFinish?.({ + message: undefined, + isAbort: false, + isDisconnect: true, + isError: false, + }); + }); + // The reconnect banner shows (not the dead terminal "connection lost" notice). + expect(screen.getByText(/reconnecting/i)).toBeTruthy(); + expect( + screen.queryByText("Connection lost — the answer was interrupted."), + ).toBeNull(); + advanceToAttempt(1); + expect(h.state.resumeStream).toHaveBeenCalledTimes(1); + // No anchor -> a plain attach URL (no expect=live&anchor pinning a row). + expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe( + "/api/ai-chat/runs/c1/stream", + ); + }); + it("strips the pinned live row before replay so content is NOT duplicated", () => { renderLiveThenDisconnect(); advanceToAttempt(1); diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index 4c3662f7..544e5f3f 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -594,30 +594,49 @@ export default function ChatThread({ ); } } - // (2b) #430: a LIVE (non-resumed) detached run whose SSE just dropped. The - // server run keeps executing, so instead of a dead "Lost connection" banner - // start a reconnect sequence: pin the CURRENT streaming assistant row as the - // strip/anchor (the live tail is the already-shown partial in `messages`, not - // a persistent row) and re-attach to the live tail via the resumable machinery. - const startedReconnect = + // (2b) #430/#488: a LIVE (non-resumed) detached run whose SSE just dropped. + // The server run keeps executing, so instead of a dead "Lost connection" + // banner start a reconnect sequence and re-attach to the live tail via the + // resumable machinery. + // + // #488 commit 2 — enter reconnect by the RUN-FACT (an active detached run), + // NOT by the presence of an assistant message. The old gate additionally + // required `message?.role === "assistant"`, so a break during the SETUP phase + // — before the first assistant frame, INCLUDING an up-to-60s MCP-toolset + // assembly — fell through to neither reconnect NOR poll while the detached + // run kept writing to pages (a silent data-integrity hole). In autonomous + // mode a run is active for the whole turn, so `autonomousRunsEnabled` is the + // run-fact signal here; the richer server run-fact (POST /run / start-metadata + // runId) is modelled + tested in the FSM (run-fsm.ts `FINISH_DISCONNECT`, + // gated on `ctx.runFact`) and lands with the full component migration. + // + // When there IS an assistant row, pin it as the strip/anchor so the live + // replay rebuilds it without duplicating parts; when there is NONE (the + // pre-first-frame break), reconnect WITHOUT an anchor (a plain live attach — + // there is nothing on screen to rebuild). + const canReconnect = isDisconnect && !wasResumed && autonomousRunsEnabled === true && - mountedRef.current && - message?.role === "assistant" && - typeof message.id === "string"; - if (startedReconnect) { - beginReconnect({ - id: message.id, - role: "assistant", - content: "", - status: "streaming", - createdAt: new Date().toISOString(), - // Preserve the partial parts so a 204 restore (onNoActiveStream) re-shows - // what was on screen while the degraded poll catches the run up to - // terminal (rowToUiMessage prefers metadata.parts). - metadata: { parts: message.parts }, - }); + mountedRef.current; + if (canReconnect) { + const hasAnchor = + message?.role === "assistant" && typeof message.id === "string"; + beginReconnect( + hasAnchor + ? { + id: message.id, + role: "assistant", + content: "", + status: "streaming", + createdAt: new Date().toISOString(), + // Preserve the partial parts so a 204 restore (onNoActiveStream) + // re-shows what was on screen while the degraded poll catches the + // run up to terminal (rowToUiMessage prefers metadata.parts). + metadata: { parts: message.parts }, + } + : null, + ); } // (3) Standard branches. // Forward the authoritative server chatId (streamed on the assistant @@ -632,7 +651,7 @@ export default function ChatThread({ // "connection lost" notice — the reconnect banner takes over (#430). if (isError) setStopNotice(null); else if (isAbort) setStopNotice("manual"); - else if (isDisconnect) setStopNotice(startedReconnect ? null : "disconnect"); + else if (isDisconnect) setStopNotice(canReconnect ? null : "disconnect"); else setStopNotice(null); // A resumed turn NEVER flushes the queue (invariant 7): skip BOTH the // flush-on-abort branch and the plain flush. The local streamer is the only @@ -790,13 +809,15 @@ export default function ChatThread({ [clearReconnectTimer, setReconnectStatePair, attemptReconnectOnce], ); - // Start a fresh reconnect sequence, pinning `anchorRow` (the live run's assistant - // row) as the strip/anchor reused by every attempt. + // Start a fresh reconnect sequence. `anchorRow` (the live run's assistant row) is + // pinned as the strip/anchor reused by every attempt; `null` (a pre-first-frame + // break, #488 commit 2) means there is nothing on screen to rebuild, so attach + // WITHOUT a strip/anchor (a plain live attach to whatever the run is emitting). const beginReconnect = useCallback( - (anchorRow: IAiChatMessageRow) => { + (anchorRow: IAiChatMessageRow | null) => { if (!autonomousRunsEnabled || !mountedRef.current) return; strippedRowRef.current = anchorRow; - stripRef.current = true; + stripRef.current = anchorRow !== null; scheduleReconnectAttempt(1); }, [autonomousRunsEnabled, scheduleReconnectAttempt], From 77e20ecb41df6b5088b005fbe72ff90691fd9230 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 06:44:10 +0300 Subject: [PATCH 33/41] =?UTF-8?q?fix(client):=20=D0=BC=D0=B8=D0=B3=D1=80?= =?UTF-8?q?=D0=B0=D1=86=D0=B8=D1=8F=20chat-thread=20=D0=BD=D0=B0=20FSM=20?= =?UTF-8?q?=E2=80=94=20reconnect=C3=97N,=20stalled,=20supersede=20(#488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Полная миграция chat-thread.tsx на автомат run-fsm: 13 ref-флагов жизненного цикла resume/reconnect/poll/ownership УБРАНЫ (карта ref'ов в run-fsm.spec.md, колонка pending пуста). Коммиты 3/4/5 приезжают одной атомарной миграцией — три фикса делят одну модель состояний (раздельные коммиты давали бы несобираемые промежуточные состояния, что противоречит смыслу единого автомата). Коммит 3 — повторные циклы reconnect: attached→reconnecting разрешён многократно. Различие «live-follow (лестница reconnect) vs mount-resume» вынесено в ctx-поле liveFollow (НЕ новый ref — это и есть смысл FSM): live-follow-обрыв перезаходит в лестницу (сброс счётчика после успешного re-attach), mount-resume-обрыв уходит в poll. Тест «два обрыва подряд → два цикла». Коммит 4 — (a) polling→stalled по idle-капу (баннер+Retry вместо тихого «вечно-полуготового»); кап переехал в тред (idleCapTimerRef, effect-owned, не флаг), окно теперь тупо поллит по armed-флагу. (b) resume армится ТОЛЬКО при серверном подтверждении активного рана: streaming-tail (статус) или POST /run для user-tail — чат без активного рана больше не порождает ~240 req/10мин. Тесты: stalled-баннер; user-tail с/без активного рана. Коммит 5 (supersede) — удалены SUPERSEDE_RETRY_DELAYS_MS/isRunAlreadyActive/ supersedeRetryRef (клиентская лестница ретраев). «Прервать и отправить» идёт через FSM superseding → POST /stream {supersede:{runId}} (runId из start-метаданных, extractRunId). Транспорт разбирает CAS-исход: ok→SUPERSEDE_READY (новый стрим), 409 MISMATCH→verify через /run, TIMEOUT/INVALID→классифицированная ошибка без авто-ретрая; голый 409 A_RUN_ALREADY_ACTIVE→RUN_ALREADY_ACTIVE. pendingSupersedeRef (send-плумбинг data) — единственная замена трёх удалённых one-shot'ов. Инвариант epoch (I1) гейтит каждый command-outcome (attach/reconnect/supersede/ postRun): устаревшее поколение колбэка отбрасывается редьюсером; DISPOSE на unmount инкрементит epoch. mountedRef оставлен как React-liveness (ортогонален lifecycle). Тесты: FSM 37 переходов; chat-thread 35 (переписан на FSM-переходы); все зелёные. E2E (реальный SSE/reconnect/supersede через редиплой) — на staging QA. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/ai-chat-window.tsx | 58 +- .../ai-chat/components/chat-thread.test.tsx | 1027 ++++--------- .../ai-chat/components/chat-thread.tsx | 1358 +++++++---------- .../features/ai-chat/state/run-fsm.spec.md | 75 +- .../features/ai-chat/state/run-fsm.test.ts | 41 +- .../src/features/ai-chat/state/run-fsm.ts | 96 +- 6 files changed, 1035 insertions(+), 1620 deletions(-) diff --git a/apps/client/src/features/ai-chat/components/ai-chat-window.tsx b/apps/client/src/features/ai-chat/components/ai-chat-window.tsx index 5e34fbf3..4b8b07c0 100644 --- a/apps/client/src/features/ai-chat/components/ai-chat-window.tsx +++ b/apps/client/src/features/ai-chat/components/ai-chat-window.tsx @@ -86,19 +86,11 @@ const MIN_HEIGHT = 400; // Margin kept between the window and the viewport edges while dragging. const EDGE_MARGIN = 8; -// #184 phase 1.5 / #430: backstop for the degraded-poll fallback. The poll is -// armed when a resume attempt could not attach to the live run and disarmed by the -// thread on settle / local stream; this cap is the ONLY backstop against an endless -// tick (a stuck 'streaming' row before the boot-sweep, or a user-tail 204 with no -// run). -// -// #430: measured from RUN ACTIVITY, not from arm-time. A real autonomous run takes -// 11-25 min — longer than a fixed 10-min-from-start cap, which used to cut the poll -// off mid-run. Instead we cap on INACTIVITY: keep polling as long as the run is -// still making progress (its persisted rows keep changing), and only give up after -// this long with NO new activity. A genuinely stuck run produces no row changes, so -// the idle cap still bounds it; a long-but-progressing run polls to completion. -const DEGRADED_POLL_IDLE_MAX_MS = 10 * 60_000; +// #184 phase 1.5 / #430 / #488: the degraded-poll fallback. The window owns only +// a DUMB 2.5s timer, gated by an armed flag; the THREAD's run-lifecycle FSM owns +// arm/disarm AND the inactivity cap that turns a stuck run into a `stalled` banner +// (#488 commit 4a — the cap moved into the thread so polling->stalled is a single +// FSM transition; the window no longer silently stops polling at the cap). /** Compact token formatter: 1.2M / 3.4k / 950. */ function formatTokens(n: number): string { @@ -259,17 +251,13 @@ export default function AiChatWindow() { [roles], ); - // #184 phase 1.5: degraded-poll fallback (replaces the F4/F5/F7 latches). When - // ChatThread could not attach to a still-running run it arms this via - // onResumeFallback(true); the thread disarms it on settle / local stream. The - // window only OWNS the timer (armedAtRef stamps when it was armed for the cap). + // #184 phase 1.5 / #488: degraded-poll fallback. ChatThread's FSM arms this via + // onResumeFallback(true) when it enters a poll-bearing recovery (attach 204 / + // starved finish / stop) and disarms it on settle / local stream / stalled. The + // window owns ONLY the dumb 2.5s timer; the THREAD owns arm/disarm AND the + // inactivity cap (a stuck run -> the thread's `stalled` banner disarms this). const [degradedPoll, setDegradedPoll] = useState(false); - // #430: timestamp of the LAST run activity while the poll is armed — stamped on - // arm and re-stamped whenever the polled rows change (see the effect below). The - // idle cap is measured from this, so a long-but-progressing run keeps polling. - const lastActivityAtRef = useRef(0); const onResumeFallback = useCallback((active: boolean): void => { - if (active) lastActivityAtRef.current = Date.now(); setDegradedPoll(active); }, []); // Reset the degraded poll whenever the open chat changes: it is scoped to the @@ -281,33 +269,17 @@ export default function AiChatWindow() { const { data: messageRows, isLoading: messagesLoading } = useAiChatMessagesQuery( activeChatId ?? undefined, - // DELIBERATELY DUMB (invariant 8 / task 2.4): poll every 2.5s while armed - // and while the run is still active (#430: under the INACTIVITY cap, not a - // fixed-from-start cap); otherwise off. NO error checks (TanStack v5 resets - // fetchFailureCount each fetch, so consecutive errors are not expressible — - // and the poll must survive a server restart) and NO tail checks (the - // settled/local-stream semantics live in ChatThread, which disarms via - // onResumeFallback(false)). The idle cap is the only backstop. - () => - degradedPoll === true && - Date.now() - lastActivityAtRef.current < DEGRADED_POLL_IDLE_MAX_MS - ? 2500 - : false, + // DELIBERATELY DUMB: poll every 2.5s WHILE ARMED, otherwise off. NO error + // checks (TanStack resets fetchFailureCount each fetch; the poll must survive + // a server restart), NO tail checks, NO cap here — the settled/stalled/idle-cap + // semantics all live in ChatThread's FSM, which disarms via onResumeFallback. + () => (degradedPoll === true ? 2500 : false), // #344: gate on windowOpen too — no message history is fetched (and no // degraded poll runs) while the window is closed; it loads when the window // opens with an active chat. windowOpen, ); - // #430: re-stamp the activity clock whenever the polled rows change while the - // poll is armed. TanStack keeps the same `messageRows` reference across refetches - // that return deep-equal data (structural sharing), so a new reference means the - // run genuinely progressed — which extends the inactivity cap above. A stuck run - // yields no reference change, so the cap eventually fires and stops the poll. - useEffect(() => { - if (degradedPoll) lastActivityAtRef.current = Date.now(); - }, [degradedPoll, messageRows]); - // #184 reconnect-and-live-follow. Whether detached agent runs are enabled for // this workspace. When the feature is off no runs are ever created, so the // resume attempt would only ever 204; gating ChatThread's resume on it avoids a diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index c6b8b65a..9aaaed60 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -15,12 +15,12 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; const h = vi.hoisted(() => ({ state: { status: "streaming" as string, + messages: [] as unknown[], onFinish: null as null | ((arg: Record) => void), sendMessage: vi.fn(), stop: vi.fn(), setMessages: vi.fn(), resumeStream: vi.fn(), - // The messages array useChat was seeded with (to assert strip/seed behavior). seededMessages: null as null | unknown[], transport: null as null | { prepareSendMessagesRequest?: (arg: { @@ -33,11 +33,17 @@ const h = vi.hoisted(() => ({ init?: { method?: string; body?: unknown }, ) => Promise; }, + // getRun mock (POST /ai-chat/run run-fact). Default: no active run. + getRun: vi.fn( + async (_chatId?: string) => + ({ run: null, message: null }) as { + run: { id: string; status: string } | null; + message: unknown; + }, + ), }, })); -// Mock useChat: capture onFinish + seeded messages, return the spies and the -// controllable status. vi.mock("@ai-sdk/react", () => ({ useChat: (opts: { messages?: unknown[]; @@ -46,7 +52,7 @@ vi.mock("@ai-sdk/react", () => ({ h.state.onFinish = opts.onFinish ?? null; h.state.seededMessages = opts.messages ?? null; return { - messages: [], + messages: h.state.messages, sendMessage: h.state.sendMessage, status: h.state.status, stop: h.state.stop, @@ -57,8 +63,6 @@ vi.mock("@ai-sdk/react", () => ({ }, })); -// Mock "ai": deterministic ids + a transport that records its options so the test -// can invoke prepareSendMessagesRequest / prepareReconnectToStreamRequest / fetch. vi.mock("ai", () => { let counter = 0; return { @@ -71,15 +75,16 @@ vi.mock("ai", () => { }; }); -// Keep the ai-chat-query import light: ChatThread only needs the messages RQ key, -// so stub the module to avoid pulling axios / i18n transitively. vi.mock("@/features/ai-chat/queries/ai-chat-query.ts", () => ({ AI_CHAT_MESSAGES_RQ_KEY: (chatId: string) => ["ai-chat-messages", chatId], })); -// Stub the heavy children: MessageList (markdown/render) and ChatInput (the -// composer). The ChatInput stub exposes a button that queues a message, the only -// interaction this test needs to populate the queue while "streaming". +// The run-fact service (POST /ai-chat/run). Mocked so a user-tail mount and the +// supersede verify do not hit axios. +vi.mock("@/features/ai-chat/services/ai-chat-service.ts", () => ({ + getRun: (chatId: string) => h.state.getRun(chatId), +})); + vi.mock("@/features/ai-chat/components/message-list.tsx", () => ({ default: () =>
, })); @@ -87,14 +92,19 @@ vi.mock("@/features/ai-chat/components/chat-input.tsx", () => ({ default: ({ onQueue, onStop, + onSend, }: { onQueue: (text: string) => void; onStop: () => void; + onSend: (text: string) => void; }) => ( <> + @@ -145,6 +155,7 @@ function renderThread(props?: { function resetState() { h.state.status = "streaming"; + h.state.messages = []; h.state.onFinish = null; h.state.seededMessages = null; h.state.transport = null; @@ -152,70 +163,120 @@ function resetState() { h.state.stop.mockClear(); h.state.setMessages.mockClear(); h.state.resumeStream.mockClear(); + h.state.getRun.mockReset(); + h.state.getRun.mockResolvedValue({ run: null, message: null }); } -describe("ChatThread — send now (#198)", () => { +const streamingTail = () => [ + row("u1", "user", undefined, "hi"), + row("a1", "assistant", "streaming", "partial"), +]; +const settledTail = () => [ + row("u1", "user", undefined, "hi"), + row("a1", "assistant", "succeeded", "done"), +]; +const userTail = () => [row("u1", "user", undefined, "hi")]; + +// ----------------------------------------------------------------------------- +// Send now (#198) — local send + #488 commit 5 CAS supersede. +// ----------------------------------------------------------------------------- +describe("ChatThread — send now", () => { beforeEach(resetState); + afterEach(cleanup); - it("aborts the current turn and resends the queued message on the abort", () => { - renderThread(); - - fireEvent.click(screen.getByTestId("queue-btn")); - const sendNowBtn = screen.getByLabelText("Send now"); - expect(sendNowBtn).toBeTruthy(); - - fireEvent.click(sendNowBtn); - expect(h.state.stop).toHaveBeenCalledTimes(1); - expect(h.state.sendMessage).not.toHaveBeenCalled(); - - act(() => { - h.state.onFinish?.({ - message: { id: "a", role: "assistant", parts: [] }, - isAbort: true, - isDisconnect: false, - isError: false, - }); - }); - expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); - }); - - it("tags exactly the next send as interrupted (one-shot flag)", () => { - renderThread(); - fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); - - const prep = h.state.transport!.prepareSendMessagesRequest!; - expect(prep({ messages: [], body: {} }).body.interrupted).toBe(true); - expect(prep({ messages: [], body: {} }).body.interrupted).toBe(false); - }); - - it("sends immediately without an interrupt when not streaming", () => { + it("sends immediately (no interrupt) when not streaming", () => { h.state.status = "ready"; - renderThread(); - + renderThread({ initialRows: settledTail() }); fireEvent.click(screen.getByTestId("queue-btn")); fireEvent.click(screen.getByLabelText("Send now")); - expect(h.state.stop).not.toHaveBeenCalled(); expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); + }); + + it("#488 commit 5: autonomous live sendNow with a known runId POSTs a supersede body", () => { + // A streaming assistant message carrying the start-metadata runId -> the FSM + // adopts the run-fact; sendNow then interrupts via the server CAS. + h.state.status = "streaming"; + h.state.messages = [ + { + id: "a1", + role: "assistant", + parts: [{ type: "text", text: "x" }], + metadata: { runId: "run-1" }, + }, + ]; + renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); + fireEvent.click(screen.getByTestId("queue-btn")); + fireEvent.click(screen.getByLabelText("Send now")); + // The message is sent... + expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); + // ...and the NEXT POST carries supersede:{runId} (read-and-cleared once). const prep = h.state.transport!.prepareSendMessagesRequest!; - expect(prep({ messages: [], body: {} }).body.interrupted).toBe(false); + const body = prep({ messages: [], body: {} }).body as Record; + expect(body.supersede).toEqual({ runId: "run-1" }); + // One-shot: a subsequent request has no supersede. + expect(prep({ messages: [], body: {} }).body.supersede).toBeUndefined(); + }); + + it("#488 commit 5: a supersede POST that 409s SUPERSEDE_TIMEOUT is returned as-is (NO retry ladder)", async () => { + h.state.status = "streaming"; + h.state.messages = [ + { + id: "a1", + role: "assistant", + parts: [{ type: "text", text: "x" }], + metadata: { runId: "run-1" }, + }, + ]; + renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); + fireEvent.click(screen.getByTestId("queue-btn")); + fireEvent.click(screen.getByLabelText("Send now")); // -> superseding, arms body + // Consume the supersede body (as the SDK would) so the POST is the CAS one. + h.state.transport!.prepareSendMessagesRequest!({ messages: [], body: {} }); + + const fetchMock = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ code: "SUPERSEDE_TIMEOUT" }), { + status: 409, + }), + ); + vi.stubGlobal("fetch", fetchMock); + let res!: Response; + await act(async () => { + res = (await h.state.transport!.fetch!("http://x", { + method: "POST", + body: "{}", + })) as Response; + }); + // Exactly ONE fetch (the old bounded 409 retry ladder is gone). + expect(fetchMock).toHaveBeenCalledTimes(1); + expect(res.status).toBe(409); + }); + + it("Send now is HIDDEN while observing a resumed run and VISIBLE on a local stream", () => { + // Resumed (mount attach) -> observer -> hidden. + renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); + fireEvent.click(screen.getByTestId("queue-btn")); + expect(screen.queryByLabelText("Send now")).toBeNull(); + // Local stream (no resume) -> visible. + cleanup(); + resetState(); + renderThread({ initialRows: [] }); + fireEvent.click(screen.getByTestId("queue-btn")); + expect(screen.getByLabelText("Send now")).toBeTruthy(); }); }); -// #486: the final onFinish -> flushNext() must be gated on the live-mount flag. -// A clean onFinish can land AFTER the thread unmounts (New-chat / chat-switch -// mid-stream — the async attach/resume settles late); flushing then dequeues and -// re-POSTs a queued message from an abandoned thread (a "ghost" send). +// ----------------------------------------------------------------------------- +// onFinish flush gated on mount (#486). +// ----------------------------------------------------------------------------- describe("ChatThread — onFinish flush gated on mount (#486)", () => { beforeEach(resetState); afterEach(cleanup); - it("a clean onFinish WHILE MOUNTED flushes the queued message (control)", () => { + it("a clean onFinish WHILE MOUNTED flushes the queued message", () => { renderThread(); - fireEvent.click(screen.getByTestId("queue-btn")); // enqueue "queued text" + fireEvent.click(screen.getByTestId("queue-btn")); expect(h.state.sendMessage).not.toHaveBeenCalled(); - act(() => { h.state.onFinish?.({ message: { id: "a", role: "assistant", parts: [] }, @@ -224,18 +285,14 @@ describe("ChatThread — onFinish flush gated on mount (#486)", () => { isError: false, }); }); - // Mounted: the queue flushes normally. expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); }); it("a clean onFinish AFTER unmount does NOT flush (no ghost send)", () => { const { unmount } = renderThread(); - fireEvent.click(screen.getByTestId("queue-btn")); // enqueue "queued text" + fireEvent.click(screen.getByTestId("queue-btn")); h.state.sendMessage.mockClear(); - - // Chat switched away mid-stream: the streamer unmounts... unmount(); - // ...and a late, clean onFinish lands on the abandoned thread. act(() => { h.state.onFinish?.({ message: { id: "a", role: "assistant", parts: [] }, @@ -244,252 +301,80 @@ describe("ChatThread — onFinish flush gated on mount (#486)", () => { isError: false, }); }); - // Gated on mountedRef: NOTHING is sent from the dead thread. expect(h.state.sendMessage).not.toHaveBeenCalled(); }); }); -// #396: in autonomous mode a live sendNow must additionally request the -// AUTHORITATIVE server stop of the detached run (a local abort is only a client -// disconnect the server ignores) and arm a bounded 409 retry so the re-POST -// converges once the one-active-run slot frees. Legacy mode is unchanged. -describe("ChatThread — send now server-stop + supersede retry (#396)", () => { +// ----------------------------------------------------------------------------- +// Turn-end decision (onFinish). +// ----------------------------------------------------------------------------- +describe("ChatThread — turn-end decision (onFinish)", () => { beforeEach(resetState); afterEach(cleanup); - // A settled assistant tail => no mount resume (attemptResumeRef false), so the - // "Send now" button is visible for the NEW local streaming turn while - // autonomous runs are enabled. - const settledTail = () => [ - row("u1", "user", undefined, "hi"), - row("a1", "assistant", "succeeded", "done"), - ]; - - it("autonomous: sendNow during a live stream calls onServerStop with the chat id", () => { - const { onServerStop } = renderThread({ - autonomousRunsEnabled: true, - initialRows: settledTail(), - }); + function finishWith(flags: { + isAbort?: boolean; + isDisconnect?: boolean; + isError?: boolean; + }) { + const { onTurnFinished } = renderThread(); fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); - - expect(h.state.stop).toHaveBeenCalledTimes(1); - expect(onServerStop).toHaveBeenCalledWith("c1"); - }); - - it("legacy (autonomous off): sendNow does NOT call onServerStop and does NOT retry the send", async () => { - const { onServerStop } = renderThread({ - autonomousRunsEnabled: false, - initialRows: settledTail(), - }); - fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); - expect(onServerStop).not.toHaveBeenCalled(); - - // The supersede retry must NOT be armed: a POST that 409s is returned as-is - // (single fetch, no retry). - const fetchMock = vi - .fn() - .mockResolvedValue( - new Response(JSON.stringify({ code: "A_RUN_ALREADY_ACTIVE" }), { - status: 409, - }), - ); - vi.stubGlobal("fetch", fetchMock); - let res!: Response; - await act(async () => { - res = (await h.state.transport!.fetch!("http://x", { - method: "POST", - body: "{}", - })) as Response; - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(res.status).toBe(409); - }); - - it("armed supersede send retries 409 A_RUN_ALREADY_ACTIVE and succeeds once the slot frees", async () => { - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); - // Arm the retry by performing a live sendNow (autonomous branch sets the ref). - fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); - - const fetchMock = vi - .fn() - // First POST: the old detached run still holds the slot -> 409. - .mockResolvedValueOnce( - new Response(JSON.stringify({ code: "A_RUN_ALREADY_ACTIVE" }), { - status: 409, - }), - ) - // Retry: the server stop settled the old run -> 200. - .mockResolvedValueOnce(new Response("ok", { status: 200 })); - vi.stubGlobal("fetch", fetchMock); - - let res!: Response; - await act(async () => { - res = (await h.state.transport!.fetch!("http://x", { - method: "POST", - body: "{}", - })) as Response; - }); - expect(fetchMock).toHaveBeenCalledTimes(2); - expect(res.status).toBe(200); - }); - - it("supersede retry is one-shot: a later send (ref cleared) does NOT retry a 409", async () => { - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); - fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); // arms the one-shot - - // First armed send: immediately succeeds, consuming the arm. - let fetchMock = vi - .fn() - .mockResolvedValue(new Response("ok", { status: 200 })); - vi.stubGlobal("fetch", fetchMock); - await act(async () => { - await h.state.transport!.fetch!("http://x", { - method: "POST", - body: "{}", - }); - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - - // A subsequent send is NOT armed -> a 409 is returned as-is (no retry). - fetchMock = vi.fn().mockResolvedValue( - new Response(JSON.stringify({ code: "A_RUN_ALREADY_ACTIVE" }), { - status: 409, - }), - ); - vi.stubGlobal("fetch", fetchMock); - let res!: Response; - await act(async () => { - res = (await h.state.transport!.fetch!("http://x", { - method: "POST", - body: "{}", - })) as Response; - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(res.status).toBe(409); - }); - - it("supersede retry is bounded: exhaustion surfaces the 409 error", async () => { - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); - fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); - - // Every attempt 409s -> after 4 attempts the last 409 surfaces. - const fetchMock = vi.fn().mockResolvedValue( - new Response(JSON.stringify({ code: "A_RUN_ALREADY_ACTIVE" }), { - status: 409, - }), - ); - vi.stubGlobal("fetch", fetchMock); - let res!: Response; - await act(async () => { - res = (await h.state.transport!.fetch!("http://x", { - method: "POST", - body: "{}", - })) as Response; - }); - // 4 attempts total (1 immediate + 3 backoff retries), then give up. - expect(fetchMock).toHaveBeenCalledTimes(4); - expect(res.status).toBe(409); - }); - - it("armed supersede send does NOT retry a non-409 status", async () => { - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); - fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); - const fetchMock = vi - .fn() - .mockResolvedValue(new Response("boom", { status: 500 })); - vi.stubGlobal("fetch", fetchMock); - let res!: Response; - await act(async () => { - res = (await h.state.transport!.fetch!("http://x", { - method: "POST", - body: "{}", - })) as Response; - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(res.status).toBe(500); - }); - - // Strand-path regression: sendNow arms the supersede retry, but if the promoted - // head is removed before the abort's onFinish lands, flushNext() sends nothing - // (returns false) and NO re-POST consumes the arm. The arm must be disarmed on - // that no-send branch so the NEXT unrelated NORMAL send does not inherit it and - // silently retry a genuine 409 (e.g. a legitimate two-tab conflict) 4x instead - // of surfacing it immediately. - it("strand-path: a stranded supersede arm (flushNext no-send) does NOT retry a later normal 409", async () => { - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); - // Arm the retry via a live autonomous sendNow (promotes the head + arms). - fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); - - // Remove the promoted head BEFORE the abort lands, so flushNext() returns - // false (no POST) and the arm would strand without the disarm fix. - fireEvent.click(screen.getByLabelText("Remove queued message")); - - // The abort's onFinish now takes the flushOnAbortRef branch, calls flushNext() - // which finds an empty queue and returns false -> the no-send disarm must run. act(() => { h.state.onFinish?.({ - message: { id: "a1", role: "assistant", parts: [] }, - isAbort: true, + message: { id: "a", role: "assistant", parts: [] }, + isAbort: false, isDisconnect: false, isError: false, + ...flags, }); }); - // No re-POST was sent (nothing to flush). - expect(h.state.sendMessage).not.toHaveBeenCalled(); + return { onTurnFinished }; + } - // A subsequent NORMAL send that 409s must be returned as-is (exactly 1 fetch): - // the stranded arm must NOT cause the genuine 409 to be retried. - const fetchMock = vi - .fn() - .mockResolvedValue( - new Response(JSON.stringify({ code: "A_RUN_ALREADY_ACTIVE" }), { - status: 409, - }), - ); - vi.stubGlobal("fetch", fetchMock); - let res!: Response; - await act(async () => { - res = (await h.state.transport!.fetch!("http://x", { - method: "POST", - body: "{}", - })) as Response; - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(res.status).toBe(409); + it("CONTINUES — flushes the next queued message on a clean finish", () => { + finishWith({}); + expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); + expect(screen.queryByText("Response stopped.")).toBeNull(); }); - it("armed supersede send does NOT retry a 409 with a different (non-A_RUN_ALREADY_ACTIVE) body", async () => { - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); - fireEvent.click(screen.getByTestId("queue-btn")); - fireEvent.click(screen.getByLabelText("Send now")); - const fetchMock = vi.fn().mockResolvedValue( - new Response(JSON.stringify({ code: "SOMETHING_ELSE" }), { - status: 409, - }), - ); - vi.stubGlobal("fetch", fetchMock); - let res!: Response; - await act(async () => { - res = (await h.state.transport!.fetch!("http://x", { - method: "POST", - body: "{}", - })) as Response; - }); - expect(fetchMock).toHaveBeenCalledTimes(1); - expect(res.status).toBe(409); + it("ENDS — keeps the queue on a user abort and shows the stopped notice", () => { + finishWith({ isAbort: true }); + expect(h.state.sendMessage).not.toHaveBeenCalled(); + expect(screen.getByText("Response stopped.")).toBeTruthy(); + }); + + it("ENDS — keeps the queue on a disconnect (non-autonomous) and shows the notice", () => { + finishWith({ isDisconnect: true }); + expect(h.state.sendMessage).not.toHaveBeenCalled(); + expect( + screen.getByText("Connection lost — the answer was interrupted."), + ).toBeTruthy(); + }); + + it("ENDS — keeps the queue on a stream error (no notice)", () => { + finishWith({ isError: true }); + expect(h.state.sendMessage).not.toHaveBeenCalled(); + expect(screen.queryByText("Response stopped.")).toBeNull(); + }); + + it("notifies the parent on EVERY terminal outcome", () => { + for (const flags of [ + {}, + { isAbort: true }, + { isDisconnect: true }, + { isError: true }, + ]) { + cleanup(); + resetState(); + const { onTurnFinished } = finishWith(flags); + expect(onTurnFinished).toHaveBeenCalled(); + } }); }); -// #388: the editor selection is snapshotted at send time and nested inside -// openPage on the wire. The getter is read live from a ref, so each send ships a -// fresh snapshot. +// ----------------------------------------------------------------------------- +// editor selection wiring (#388). +// ----------------------------------------------------------------------------- describe("ChatThread — editor selection wiring (#388)", () => { beforeEach(resetState); afterEach(cleanup); @@ -518,7 +403,7 @@ describe("ChatThread — editor selection wiring (#388)", () => { ); } - it("nests the snapshot from the getter into openPage.selection at send time", () => { + it("nests the snapshot into openPage.selection at send time", () => { const selection = { text: "fix this", blockIds: ["b1"], before: "a " }; renderWithSelection({ openPage: { id: "p1", title: "Doc" }, @@ -545,131 +430,36 @@ describe("ChatThread — editor selection wiring (#388)", () => { expect(openPage).toEqual({ id: "p1", title: "Doc", selection: null }); }); - it("does not send selection at all on a non-page route (openPage null)", () => { + it("does not consult the getter on a non-page route (openPage null)", () => { const getter = vi.fn(() => ({ text: "sel" })); renderWithSelection({ openPage: null, getEditorSelection: getter }); const prep = h.state.transport!.prepareSendMessagesRequest!; expect(prep({ messages: [], body: {} }).body.openPage).toBeNull(); - // The getter must not even be consulted when there is no page. expect(getter).not.toHaveBeenCalled(); }); }); -describe("ChatThread — turn-end decision (onFinish)", () => { +// ----------------------------------------------------------------------------- +// Resume (attach) machinery (#184) + #488 commit 4b run-fact gating. +// ----------------------------------------------------------------------------- +describe("ChatThread — resume (attach) machinery", () => { beforeEach(resetState); - - function finishWith(flags: { - isAbort?: boolean; - isDisconnect?: boolean; - isError?: boolean; - }) { - cleanup(); - h.state.sendMessage.mockClear(); - const { onTurnFinished } = renderThread(); - fireEvent.click(screen.getByTestId("queue-btn")); - act(() => { - h.state.onFinish?.({ - message: { id: "a", role: "assistant", parts: [] }, - isAbort: false, - isDisconnect: false, - isError: false, - ...flags, - }); - }); - return { onTurnFinished }; - } - - it("CONTINUES — flushes the next queued message on a clean finish", () => { - finishWith({}); - expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); - expect(screen.queryByText("Response stopped.")).toBeNull(); - }); - - it("ENDS — keeps the queue intact on a user abort and shows the stopped notice", () => { - finishWith({ isAbort: true }); - expect(h.state.sendMessage).not.toHaveBeenCalled(); - expect(screen.getByText("Response stopped.")).toBeTruthy(); - }); - - it("ENDS — keeps the queue intact on a disconnect and shows the connection-lost notice", () => { - finishWith({ isDisconnect: true }); - expect(h.state.sendMessage).not.toHaveBeenCalled(); - expect( - screen.getByText("Connection lost — the answer was interrupted."), - ).toBeTruthy(); - }); - - it("ENDS — keeps the queue intact on a stream error (no auto-retry, no stopped notice)", () => { - finishWith({ isError: true }); - expect(h.state.sendMessage).not.toHaveBeenCalled(); - expect(screen.queryByText("Response stopped.")).toBeNull(); - }); - - it("notifies the parent on EVERY terminal outcome", () => { - for (const flags of [ - {}, - { isAbort: true }, - { isDisconnect: true }, - { isError: true }, - ]) { - const { onTurnFinished } = finishWith(flags); - expect(onTurnFinished).toHaveBeenCalled(); - } - }); -}); - -// #184 phase 1.5: the resumable-SSE client. A reopened tab resumes the live run -// via the SDK's reconnect transport (attach: replay + tail) instead of polling. -describe("ChatThread — resume (attach) machinery (#184)", () => { - const streamingTail = () => [ - row("u1", "user", undefined, "hi"), - row("a1", "assistant", "streaming", "partial"), - ]; - const settledTail = () => [ - row("u1", "user", undefined, "hi"), - row("a1", "assistant", "succeeded", "done"), - ]; - const userTail = () => [row("u1", "user", undefined, "hi")]; - - const visibleMsg = { - id: "a1", - role: "assistant", - parts: [{ type: "text", text: "streamed answer" }], - }; - const emptyMsg = { id: "a1", role: "assistant", parts: [] }; - - beforeEach(resetState); - // NOTE: do NOT vi.unstubAllGlobals() here — vitest.setup.ts installs - // matchMedia/localStorage via vi.stubGlobal and unstubbing wipes them for the - // rest of the file. Fetch is re-stubbed per test that needs it. afterEach(cleanup); - it("resumes on mount only when the flag is on, chatId is set, and the tail is not a settled assistant", () => { - // streaming tail -> resume + it("resumes on mount for a STREAMING tail (the streaming status is the run-fact)", () => { renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); expect(h.state.resumeStream).toHaveBeenCalledTimes(1); + }); - // user tail -> resume (the assistant row may not be seeded yet) - cleanup(); - h.state.resumeStream.mockClear(); - renderThread({ autonomousRunsEnabled: true, initialRows: userTail() }); - expect(h.state.resumeStream).toHaveBeenCalledTimes(1); - - // settled assistant tail -> NO resume - cleanup(); - h.state.resumeStream.mockClear(); + it("does NOT resume for a settled tail, flag off, or no chatId", () => { renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); expect(h.state.resumeStream).not.toHaveBeenCalled(); - - // flag off -> NO resume cleanup(); - h.state.resumeStream.mockClear(); + resetState(); renderThread({ autonomousRunsEnabled: false, initialRows: streamingTail() }); expect(h.state.resumeStream).not.toHaveBeenCalled(); - - // no chatId -> NO resume cleanup(); - h.state.resumeStream.mockClear(); + resetState(); renderThread({ autonomousRunsEnabled: true, chatId: null, @@ -678,61 +468,87 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { expect(h.state.resumeStream).not.toHaveBeenCalled(); }); - it("strips the streaming tail from the seed, but keeps a user tail whole", () => { - renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); - // 2 rows in, streaming tail stripped -> 1 seeded message. - expect(h.state.seededMessages).toHaveLength(1); - - cleanup(); + it("#488 commit 4b: a USER tail resumes ONLY when POST /run confirms an active run", async () => { + h.state.getRun.mockResolvedValue({ + run: { id: "run-1", status: "running" }, + message: null, + }); + renderThread({ autonomousRunsEnabled: true, initialRows: userTail() }); + await act(async () => { + await Promise.resolve(); + }); + expect(h.state.getRun).toHaveBeenCalledWith("c1"); + expect(h.state.resumeStream).toHaveBeenCalledTimes(1); + }); + + it("#488 commit 4b: a USER tail with NO active run does NOT resume and does NOT arm the poll", async () => { + h.state.getRun.mockResolvedValue({ run: null, message: null }); + const { onResumeFallback } = renderThread({ + autonomousRunsEnabled: true, + initialRows: userTail(), + }); + await act(async () => { + await Promise.resolve(); + }); + expect(h.state.resumeStream).not.toHaveBeenCalled(); + expect(onResumeFallback).not.toHaveBeenCalledWith(true); + }); + + it("strips the streaming tail from the seed, keeps a user tail whole", () => { + renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); + expect(h.state.seededMessages).toHaveLength(1); + cleanup(); + resetState(); renderThread({ autonomousRunsEnabled: true, initialRows: userTail() }); - // user tail is not stripped. expect(h.state.seededMessages).toHaveLength(1); }); - it("builds the attach URL with expect=live&anchor only when the streaming tail was stripped", () => { + it("builds the attach URL with expect=live&anchor only for a stripped streaming tail", () => { renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe( "/api/ai-chat/runs/c1/stream?expect=live&anchor=a1", ); - cleanup(); + resetState(); renderThread({ autonomousRunsEnabled: true, initialRows: userTail() }); expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe( "/api/ai-chat/runs/c1/stream", ); }); - async function fetch204() { + async function attachFetch(response: unknown, reject = false) { vi.stubGlobal( "fetch", - vi.fn().mockResolvedValue({ status: 204, ok: false }), + reject + ? vi.fn().mockRejectedValue(response) + : vi.fn().mockResolvedValue(response), ); await act(async () => { - await h.state.transport!.fetch!("http://x", { method: "GET" }); + await h.state + .transport!.fetch!("http://x", { method: "GET" }) + .catch(() => undefined); }); } - it("204 on a user tail: no crash, no restore, reconcile+invalidate, onResumeFallback(true)", async () => { - const { onResumeFallback, invalidateSpy } = renderThread({ - autonomousRunsEnabled: true, - initialRows: userTail(), - }); - await fetch204(); - // No stripped row -> no restore merge. - expect(h.state.setMessages).not.toHaveBeenCalled(); - expect(invalidateSpy).toHaveBeenCalledWith({ - queryKey: ["ai-chat-messages", "c1"], - }); - expect(onResumeFallback).toHaveBeenCalledWith(true); - }); - it("204 on a streaming tail: restore + invalidate + onResumeFallback(true)", async () => { const { onResumeFallback, invalidateSpy } = renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail(), }); - await fetch204(); - // Stripped row is restored to the store. + await attachFetch({ status: 204, ok: false }); + expect(h.state.setMessages).toHaveBeenCalledTimes(1); // restore + expect(invalidateSpy).toHaveBeenCalledWith({ + queryKey: ["ai-chat-messages", "c1"], + }); + expect(onResumeFallback).toHaveBeenCalledWith(true); + }); + + it("F7 restart-survival: a 500 attach failure restores the row AND arms the poll", async () => { + const { onResumeFallback, invalidateSpy } = renderThread({ + autonomousRunsEnabled: true, + initialRows: streamingTail(), + }); + await attachFetch({ status: 500, ok: false }); expect(h.state.setMessages).toHaveBeenCalledTimes(1); expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["ai-chat-messages", "c1"], @@ -740,39 +556,12 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { expect(onResumeFallback).toHaveBeenCalledWith(true); }); - it("F7 restart-survival: a 500 attach failure restores the stripped row AND arms the poll (not lost)", async () => { + it("F7 restart-survival: a network throw restores the row AND arms the poll", async () => { const { onResumeFallback, invalidateSpy } = renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail(), }); - vi.stubGlobal( - "fetch", - vi.fn().mockResolvedValue({ status: 500, ok: false }), - ); - await act(async () => { - await h.state.transport!.fetch!("http://x", { method: "GET" }); - }); - expect(h.state.setMessages).toHaveBeenCalledTimes(1); // stripped row restored - expect(invalidateSpy).toHaveBeenCalledWith({ - queryKey: ["ai-chat-messages", "c1"], - }); - expect(onResumeFallback).toHaveBeenCalledWith(true); // degraded poll armed - }); - - it("F7 restart-survival: a network throw restores the stripped row AND arms the poll", async () => { - const { onResumeFallback, invalidateSpy } = renderThread({ - autonomousRunsEnabled: true, - initialRows: streamingTail(), - }); - vi.stubGlobal( - "fetch", - vi.fn().mockRejectedValue(new Error("network down")), - ); - await act(async () => { - await h.state - .transport!.fetch!("http://x", { method: "GET" }) - .catch(() => undefined); // the wrapper rethrows; swallow here - }); + await attachFetch(new Error("network down"), true); expect(h.state.setMessages).toHaveBeenCalledTimes(1); expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["ai-chat-messages", "c1"], @@ -780,7 +569,7 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { expect(onResumeFallback).toHaveBeenCalledWith(true); }); - it("unmount during a pending attach aborts the controller and gates late callbacks", async () => { + it("unmount during a pending attach aborts the controller and gates late callbacks (epoch I1)", async () => { const { onResumeFallback, invalidateSpy, unmount } = renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail(), @@ -798,55 +587,33 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { }); }), ); - // Kick a reconnect GET (stays pending). let pending!: Promise; act(() => { pending = h.state.transport!.fetch!("http://x", { method: "GET" }); }); - // Unmount: the cleanup aborts the in-flight attach. - unmount(); + unmount(); // DISPOSE aborts the attach + bumps the epoch expect(abortSeen).toBe(true); - // A late 204 landing after unmount must NOT arm a poll / invalidate the (now - // different) chat. onResumeFallback.mockClear(); invalidateSpy.mockClear(); await act(async () => { resolveFetch({ status: 204, ok: false }); await pending; }); + // The stale (superseded-epoch) outcome is dropped. expect(onResumeFallback).not.toHaveBeenCalledWith(true); expect(invalidateSpy).not.toHaveBeenCalled(); }); - it("a resume fetch error clears resumedTurn so the next local turn flushes the queue", async () => { - renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); - h.state.status = "ready"; - vi.stubGlobal( - "fetch", - vi.fn().mockResolvedValue({ status: 500, ok: false }), - ); - await act(async () => { - await h.state.transport!.fetch!("http://x", { method: "GET" }); - }); - // Queue then clean-finish: suppression was cleared, so the queue flushes. - fireEvent.click(screen.getByTestId("queue-btn")); - act(() => { - h.state.onFinish?.({ - message: visibleMsg, - isAbort: false, - isDisconnect: false, - isError: false, - }); - }); - expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); - }); - - it("a resumed turn's onFinish does NOT flush the queue", () => { + it("a resumed (observer) turn's onFinish does NOT flush the queue", () => { renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); fireEvent.click(screen.getByTestId("queue-btn")); act(() => { h.state.onFinish?.({ - message: visibleMsg, + message: { + id: "a1", + role: "assistant", + parts: [{ type: "text", text: "streamed answer" }], + }, isAbort: false, isDisconnect: false, isError: false, @@ -855,7 +622,7 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { expect(h.state.sendMessage).not.toHaveBeenCalled(); }); - it("a healthy resumed finish (visible content) arms nothing and keeps the store", () => { + it("an empty resumed message (starved replay) restores the row AND arms the poll", () => { h.state.status = "ready"; const { onResumeFallback } = renderThread({ autonomousRunsEnabled: true, @@ -865,53 +632,7 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { onResumeFallback.mockClear(); act(() => { h.state.onFinish?.({ - message: visibleMsg, - isAbort: false, - isDisconnect: false, - isError: false, - }); - }); - // No restore (would clobber the fuller streamed message), no poll arm. - expect(h.state.setMessages).not.toHaveBeenCalled(); - expect(onResumeFallback).not.toHaveBeenCalledWith(true); - }); - - it("isDisconnect WITH visible content arms the poll but does NOT restore", () => { - h.state.status = "ready"; - const { onResumeFallback, invalidateSpy } = renderThread({ - autonomousRunsEnabled: true, - initialRows: streamingTail(), - }); - h.state.setMessages.mockClear(); - onResumeFallback.mockClear(); - invalidateSpy.mockClear(); - act(() => { - h.state.onFinish?.({ - message: visibleMsg, - isAbort: false, - isDisconnect: true, - isError: false, - }); - }); - expect(onResumeFallback).toHaveBeenCalledWith(true); - expect(invalidateSpy).toHaveBeenCalledWith({ - queryKey: ["ai-chat-messages", "c1"], - }); - // Restore forbidden: the on-screen partial must not roll back. - expect(h.state.setMessages).not.toHaveBeenCalled(); - }); - - it("an empty resumed message (starved replay) restores the stripped row AND arms the poll", () => { - h.state.status = "ready"; - const { onResumeFallback } = renderThread({ - autonomousRunsEnabled: true, - initialRows: streamingTail(), - }); - h.state.setMessages.mockClear(); - onResumeFallback.mockClear(); - act(() => { - h.state.onFinish?.({ - message: emptyMsg, + message: { id: "a1", role: "assistant", parts: [] }, isAbort: false, isDisconnect: false, isError: false, @@ -921,67 +642,11 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { expect(onResumeFallback).toHaveBeenCalledWith(true); // arm }); - it("degraded-merge: merges the tail per initialRows update, and settles disarm the poll", async () => { - h.state.status = "ready"; - const { rerender, onResumeFallback } = renderResumable(streamingTail()); - // Arm reconcile via a 204. - vi.stubGlobal( - "fetch", - vi.fn().mockResolvedValue({ status: 204, ok: false }), - ); - await act(async () => { - await h.state.transport!.fetch!("http://x", { method: "GET" }); - }); - h.state.setMessages.mockClear(); - onResumeFallback.mockClear(); - - // A streaming-tail update: merge, poll stays armed. - rerender([ - row("u1", "user", undefined, "hi"), - row("a1", "assistant", "streaming", "step 1\nstep 2"), - ]); - expect(h.state.setMessages).toHaveBeenCalledTimes(1); - expect(onResumeFallback).not.toHaveBeenCalledWith(false); - - // A settled-tail update: merge + disarm. - h.state.setMessages.mockClear(); - rerender([ - row("u1", "user", undefined, "hi"), - row("a1", "assistant", "succeeded", "final"), - ]); - expect(h.state.setMessages).toHaveBeenCalledTimes(1); - expect(onResumeFallback).toHaveBeenCalledWith(false); - }); - - it("a local stream disarms both the merge and the poll", () => { - h.state.status = "streaming"; - const { rerender, onResumeFallback } = renderResumable(streamingTail()); - onResumeFallback.mockClear(); - // A re-render while streaming: the reconciliation effect disarms. - rerender(streamingTail()); - expect(onResumeFallback).toHaveBeenCalledWith(false); - }); - - it("Send now is hidden on a resumed turn but visible on a local stream", () => { - // Resumed turn: hidden. - renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); - fireEvent.click(screen.getByTestId("queue-btn")); - expect(screen.queryByLabelText("Send now")).toBeNull(); - - // Local streaming turn (no resume): visible. - cleanup(); - resetState(); - renderThread({ initialRows: [] }); - fireEvent.click(screen.getByTestId("queue-btn")); - expect(screen.getByLabelText("Send now")).toBeTruthy(); - }); - - it("handleStop aborts the attach controller and calls onServerStop", async () => { + it("handleStop aborts the attach controller and calls onServerStop", () => { const { onServerStop } = renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail(), }); - // Establish an attach controller via a (pending) reconnect GET. let abortSeen = false; vi.stubGlobal( "fetch", @@ -989,7 +654,7 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { init.signal?.addEventListener("abort", () => { abortSeen = true; }); - return new Promise(() => undefined); // never resolves + return new Promise(() => undefined); }), ); act(() => { @@ -1001,45 +666,10 @@ describe("ChatThread — resume (attach) machinery (#184)", () => { }); }); -// Helper: render a resumable thread and expose a rerender that only swaps -// initialRows (the degraded-merge effect depends on it). -function renderResumable(initialRows: IAiChatMessageRow[]) { - const onResumeFallback = vi.fn(); - const queryClient = new QueryClient({ - defaultOptions: { queries: { retry: false } }, - }); - const Wrapper = ({ rows }: { rows: IAiChatMessageRow[] }) => ( - - - - - - ); - const view = render(); - const rerender = (rows: IAiChatMessageRow[]) => - act(() => view.rerender()); - return { rerender, onResumeFallback }; -} - -// #430: auto-reconnect to a DETACHED run after a LIVE SSE disconnect. The mount -// path only resumes on mount/reload; these cover the missing trigger — a live -// `isDisconnect` on onFinish must (backoff-)re-attach WITHOUT a reload, pin+strip -// the live row to avoid duplicates, fall back to the degraded poll on a 204, and -// exhaust to a manual Retry. -describe("ChatThread — live reconnect after isDisconnect (#430)", () => { - // A LIVE local turn that just dropped: the settled tail existed before, and the - // partial assistant row lives only in `messages` (not persisted as a tail). - const settledTail = () => [ - row("u1", "user", undefined, "hi"), - row("a1", "assistant", "succeeded", "done"), - ]; - // The partial assistant message onFinish hands us for the dropped LIVE turn. +// ----------------------------------------------------------------------------- +// Live reconnect (#430) + #488 commits 2/3 + stalled (4a). Fake timers. +// ----------------------------------------------------------------------------- +describe("ChatThread — live reconnect + stalled", () => { const liveMsg = { id: "a2", role: "assistant", @@ -1048,8 +678,6 @@ describe("ChatThread — live reconnect after isDisconnect (#430)", () => { beforeEach(() => { resetState(); - // status "ready": with a live disconnect the mock is not streaming, so the - // status==="streaming" auto-clear effect stays out of the way. h.state.status = "ready"; vi.useFakeTimers(); }); @@ -1058,144 +686,92 @@ describe("ChatThread — live reconnect after isDisconnect (#430)", () => { cleanup(); }); - // Render a NON-resuming mount (settled tail -> no mount resume) with autonomous - // runs on, then simulate a live disconnect via onFinish. - function renderLiveThenDisconnect() { - const view = renderThread({ - autonomousRunsEnabled: true, - initialRows: settledTail(), - }); - // The settled tail must NOT have triggered a mount resume. - expect(h.state.resumeStream).not.toHaveBeenCalled(); + function disconnect(message: unknown = liveMsg) { act(() => { h.state.onFinish?.({ - message: liveMsg, + message, isAbort: false, isDisconnect: true, isError: false, }); }); + } + function renderLive() { + const view = renderThread({ + autonomousRunsEnabled: true, + initialRows: settledTail(), + }); + expect(h.state.resumeStream).not.toHaveBeenCalled(); return view; } - - // Fire the pending (scheduled) attempt for `attempt` (backoff = 1s,2s,4s,...). function advanceToAttempt(attempt: number) { act(() => { vi.advanceTimersByTime(1000 * 2 ** (attempt - 1)); }); } - - // Simulate the reconnect GET returning 204 (nothing live) so the transport's - // no-active-stream recovery runs. - async function reconnect204() { - vi.stubGlobal( - "fetch", - vi.fn().mockResolvedValue({ status: 204, ok: false }), - ); + async function reconnect(response: unknown) { + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(response)); await act(async () => { await h.state.transport!.fetch!("http://x", { method: "GET" }); }); } - // Simulate the reconnect GET returning a live 2xx stream. - async function reconnect200() { - vi.stubGlobal( - "fetch", - vi.fn().mockResolvedValue({ status: 200, ok: true }), - ); - await act(async () => { - await h.state.transport!.fetch!("http://x", { method: "GET" }); - }); - } - - it("calls resumeStream POST-mount (a live disconnect triggers a backoff reconnect)", () => { - renderLiveThenDisconnect(); - // The banner shows immediately; the attach itself fires after the first backoff. + it("a live disconnect starts a backoff reconnect (banner + resumeStream after backoff)", () => { + renderLive(); + disconnect(); expect(screen.getByText(/reconnecting/i)).toBeTruthy(); expect(h.state.resumeStream).not.toHaveBeenCalled(); advanceToAttempt(1); - // resumeStream is now called AFTER mount — the bug was it only ever fired once - // on mount. The reconnect URL pins expect=live&anchor to OUR run. expect(h.state.resumeStream).toHaveBeenCalledTimes(1); expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe( "/api/ai-chat/runs/c1/stream?expect=live&anchor=a2", ); }); - // #488 commit 2: a break during the SETUP phase — before the first assistant - // frame (onFinish carries no assistant message) — must STILL reconnect, because - // a detached autonomous run keeps writing to pages. The old gate required - // `message?.role === "assistant"`, so this fell through to neither reconnect nor - // poll. With no anchor row, the attach is a PLAIN live attach (no strip/anchor). - it("#488 commit 2: disconnect BEFORE the first assistant frame reconnects with no anchor", () => { - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); - expect(h.state.resumeStream).not.toHaveBeenCalled(); - act(() => { - h.state.onFinish?.({ - message: undefined, - isAbort: false, - isDisconnect: true, - isError: false, - }); - }); - // The reconnect banner shows (not the dead terminal "connection lost" notice). + it("#488 commit 2: a disconnect BEFORE the first assistant frame reconnects with NO anchor", () => { + renderLive(); + disconnect(null); // no assistant message yet (pre-first-frame break) expect(screen.getByText(/reconnecting/i)).toBeTruthy(); expect( screen.queryByText("Connection lost — the answer was interrupted."), ).toBeNull(); advanceToAttempt(1); expect(h.state.resumeStream).toHaveBeenCalledTimes(1); - // No anchor -> a plain attach URL (no expect=live&anchor pinning a row). expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe( "/api/ai-chat/runs/c1/stream", ); }); - it("strips the pinned live row before replay so content is NOT duplicated", () => { - renderLiveThenDisconnect(); - advanceToAttempt(1); - // The attempt strips the anchor row from the store (the live replay rebuilds - // it). Apply the setMessages updater to prove it removes exactly the anchor. - const updater = h.state.setMessages.mock.calls.at(-1)![0] as ( - prev: { id: string }[], - ) => { id: string }[]; - expect(updater([{ id: "u1" }, { id: "a2" }])).toEqual([{ id: "u1" }]); - }); - it("a live re-attach (2xx) clears the reconnect banner", async () => { - renderLiveThenDisconnect(); + renderLive(); + disconnect(); advanceToAttempt(1); - await reconnect200(); + await reconnect({ status: 200, ok: true }); expect(screen.queryByText(/reconnecting/i)).toBeNull(); }); it("a 204 arms the degraded poll and backs off to the next attempt", async () => { - const { onResumeFallback } = renderLiveThenDisconnect(); + const { onResumeFallback } = renderLive(); + disconnect(); advanceToAttempt(1); expect(h.state.resumeStream).toHaveBeenCalledTimes(1); - await reconnect204(); - // Fallback engaged: the degraded poll is armed (204 -> onNoActiveStream). + await reconnect({ status: 204, ok: false }); expect(onResumeFallback).toHaveBeenCalledWith(true); - // Still reconnecting — the banner advanced to attempt 2/5. expect(screen.getByText(/reconnecting.*2\/5/i)).toBeTruthy(); - // The next backoff fires attempt 2 (another resumeStream). advanceToAttempt(2); expect(h.state.resumeStream).toHaveBeenCalledTimes(2); }); it("exhausts the attempt limit into a manual Retry, which restarts the sequence", async () => { - renderLiveThenDisconnect(); - // Drive all 5 attempts, each failing with a 204. + renderLive(); + disconnect(); for (let n = 1; n <= 5; n++) { advanceToAttempt(n); expect(h.state.resumeStream).toHaveBeenCalledTimes(n); - await reconnect204(); + await reconnect({ status: 204, ok: false }); } - // The 5th 204 exhausted the cap -> the manual Retry replaces the banner. expect(screen.queryByText(/reconnecting/i)).toBeNull(); const retry = screen.getByText("Retry"); - expect(retry).toBeTruthy(); - // Retry fires attempt 1 immediately (no backoff) — a 6th resumeStream. act(() => { fireEvent.click(retry); }); @@ -1203,22 +779,43 @@ describe("ChatThread — live reconnect after isDisconnect (#430)", () => { expect(screen.getByText(/reconnecting/i)).toBeTruthy(); }); + it("#488 commit 3: two breaks in a row produce two reconnect cycles", async () => { + renderLive(); + // First break -> reconnect -> re-attach live. + disconnect(); + advanceToAttempt(1); + expect(h.state.resumeStream).toHaveBeenCalledTimes(1); + await reconnect({ status: 200, ok: true }); + expect(screen.queryByText(/reconnecting/i)).toBeNull(); + // The re-attached observer stream drops AGAIN -> a SECOND reconnect cycle + // (the old one-shot !wasResumed gate sent this to silent poll). + disconnect(); + expect(screen.getByText(/reconnecting/i)).toBeTruthy(); + advanceToAttempt(1); + expect(h.state.resumeStream).toHaveBeenCalledTimes(2); + }); + it("does NOT reconnect when autonomous runs are disabled", () => { renderThread({ autonomousRunsEnabled: false, initialRows: settledTail() }); - act(() => { - h.state.onFinish?.({ - message: liveMsg, - isAbort: false, - isDisconnect: true, - isError: false, - }); - }); + disconnect(); expect(screen.queryByText(/reconnecting/i)).toBeNull(); - // The terminal "connection lost" notice is shown instead (unchanged behavior). expect( screen.getByText("Connection lost — the answer was interrupted."), ).toBeTruthy(); advanceToAttempt(1); expect(h.state.resumeStream).not.toHaveBeenCalled(); }); + + it("#488 commit 4a: the poll idle cap surfaces a stalled banner + Retry (not silent)", async () => { + renderLive(); + disconnect(); + advanceToAttempt(1); + await reconnect({ status: 204, ok: false }); // arms the poll (reconnecting) + // No activity for the whole idle cap -> stalled. + act(() => { + vi.advanceTimersByTime(10 * 60_000); + }); + expect(screen.getByText(/the run stopped responding/i)).toBeTruthy(); + expect(screen.getByText("Retry")).toBeTruthy(); + }); }); diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index 544e5f3f..8a4d72cb 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -34,7 +34,10 @@ import { shouldResetRolePicked, } from "@/features/ai-chat/utils/role-launch.ts"; import { describeChatError } from "@/features/ai-chat/utils/error-message.ts"; -import { extractServerChatId } from "@/features/ai-chat/utils/adopt-chat-id.ts"; +import { + extractServerChatId, + extractRunId, +} from "@/features/ai-chat/utils/adopt-chat-id.ts"; import { assistantMessageHasVisibleContent } from "@/features/ai-chat/utils/message-content.ts"; import { isStreamingTail, @@ -42,6 +45,7 @@ import { seedRows, mergeById, } from "@/features/ai-chat/utils/resume-helpers.ts"; +import { getRun } from "@/features/ai-chat/services/ai-chat-service.ts"; import { AI_CHAT_MESSAGES_RQ_KEY } from "@/features/ai-chat/queries/ai-chat-query.ts"; import type { EditorSelectionContext } from "@/features/editor/utils/get-editor-selection.ts"; import { @@ -51,6 +55,14 @@ import { removeQueuedById, type QueuedMessage, } from "@/features/ai-chat/utils/queue-helpers.ts"; +import { + reduce, + initialMachine, + RECONNECT_MAX_ATTEMPTS, + type Machine, + type Event as RunEvent, + type Effect as RunEffect, +} from "@/features/ai-chat/state/run-fsm.ts"; import classes from "@/features/ai-chat/components/ai-chat.module.css"; // Throttle how often the streamed `messages` state triggers a re-render. Without @@ -61,42 +73,32 @@ import classes from "@/features/ai-chat/components/ai-chat.module.css"; // from the token rate. const STREAM_THROTTLE_MS = 50; -// #430: auto-reconnect after a LIVE SSE disconnect of a DETACHED (autonomous) run. -// The run keeps executing server-side, so instead of a dead "Lost connection" -// banner we re-attach to the live tail through the SAME resumable machinery the -// mount path uses. Attempts back off exponentially and are capped; on exhaustion -// the user gets a manual Retry (the degraded poll keeps catching up underneath). -const RECONNECT_MAX_ATTEMPTS = 5; -// Backoff before attempt N (1-based): 1s, 2s, 4s, 8s, 16s. -const RECONNECT_BASE_DELAY_MS = 1000; +// #488 commit 4a: how long the degraded poll may run with NO new run activity +// (its persisted rows unchanged) before the FSM declares the run STALLED and +// surfaces a banner + Retry — instead of the old silent `refetchInterval -> false` +// ("forever half-done answer"). Measured from INACTIVITY, so a long-but-progressing +// run keeps polling; only a genuinely stuck run trips it. This cap lives in the +// THREAD now (the FSM owns polling->stalled); the window just polls while armed. +const DEGRADED_POLL_IDLE_MAX_MS = 10 * 60_000; -// #396: bounded retry for the "Interrupt and send now" re-send when it races the -// authoritative server stop of the just-superseded detached run. The re-POST can -// arrive before the old run has released the one-active-run slot, so the server -// returns 409 A_RUN_ALREADY_ACTIVE. The server stop guarantees the slot frees, so -// a few short backoffs converge. 4 total attempts: attempt 1 fires immediately, -// then these are the waits BEFORE attempts 2, 3 and 4 (150ms, 300ms, 600ms). If -// all 4 attempts 409, the last 409 surfaces (the banner) — acceptable per #396. -const SUPERSEDE_RETRY_DELAYS_MS = [150, 300, 600]; -// The server error code that means "another run is already active for this chat". -const A_RUN_ALREADY_ACTIVE = "A_RUN_ALREADY_ACTIVE"; +/** The #487 active (non-terminal) run statuses — mirrors the server's + * ACTIVE_RUN_STATUSES. A run-fact is "active" only for these. */ +function isActiveRunStatus(status: string | null | undefined): boolean { + return status === "pending" || status === "running"; +} -/** - * #396: defensively decide whether a 409 response is the one-active-run gate - * rejection (code A_RUN_ALREADY_ACTIVE) vs. some other 409. Reads a CLONE so the - * original response body stays intact for the caller when it is returned as-is. - * Any parse failure or unexpected shape => false (do NOT retry). - */ -async function isRunAlreadyActive(response: Response): Promise { +/** Read the `{ code, runId }` off a JSON error response WITHOUT consuming the + * original body (reads a clone), so the caller can still return the Response + * untouched to the SDK. Any parse failure => empty. */ +async function read409(response: Response): Promise<{ code?: string; runId?: string }> { try { - const body = (await response.clone().json()) as unknown; - return ( - typeof body === "object" && - body !== null && - (body as { code?: unknown }).code === A_RUN_ALREADY_ACTIVE - ); + const b = (await response.clone().json()) as { code?: unknown; runId?: unknown }; + return { + code: typeof b?.code === "string" ? b.code : undefined, + runId: typeof b?.runId === "string" ? b.runId : undefined, + }; } catch { - return false; + return {}; } } @@ -150,21 +152,18 @@ interface ChatThreadProps { * which fires only at the terminal outcome. */ onServerChatId?: (serverChatId?: string) => void; /** #184 phase 1.5: arm/disarm the parent's degraded-poll fallback for THIS - * chat's window. Called `true` when a resume attempt could not attach to the - * live run (attach 204 / starved-or-torn resumed finish), so the window starts - * a dumb timed poll of the message history to follow the detached run to settle; - * called `false` the moment a local stream starts or the terminal settled row is - * merged (invariant 8). The window owns the timer + its 10-min cap. */ + * chat's window. Called `true` when the FSM enters a poll-bearing recovery + * (attach 204 / starved-or-torn resumed finish / a stop), `false` the moment a + * local stream starts or the run settles. The window owns only the dumb 2.5s + * timer; the THREAD's FSM owns arm/disarm + the stalled cap (#488). */ onResumeFallback?: (active: boolean) => void; /** #184: whether detached/autonomous agent runs are enabled for this workspace. * When true the Stop button must additionally hit the AUTHORITATIVE server stop * (via onServerStop) — aborting only the local SSE is just a client disconnect, * which the server deliberately ignores, so the detached run would keep going. */ autonomousRunsEnabled?: boolean; - /** #184: request the server-side stop of this chat's active run (the parent owns - * the endpoint call + the "stopping" latch that keeps observer-polling from - * immediately re-streaming the stopping run's output). Called with the resolved - * chat id when the user presses Stop in autonomous mode. */ + /** #184: request the server-side stop of this chat's active run. Called with the + * resolved chat id when the user presses Stop in autonomous mode. */ onServerStop?: (chatId: string) => void; } @@ -199,6 +198,14 @@ function rowToUiMessage(row: IAiChatMessageRow): UIMessage { * Owns the AI SDK `useChat` lifecycle for ONE chat. The parent remounts this * with a `key` when the selected chat changes, so initial messages re-seed * cleanly (the v6 transport-based hook keeps its state per mount). + * + * #488: the resume/reconnect/poll/stop/supersede lifecycle is driven by the pure + * FSM in `state/run-fsm.ts` (see `run-fsm.spec.md`). The component is the RUNTIME: + * it dispatches typed events (from SDK callbacks / HTTP outcomes) and executes the + * reducer's command EFFECTS (attach GET, POST /run, POST /stop, POST /stream + * supersede, backoff timers, poll arm/disarm). The FSM lives in this thread (not + * the window) so a late SDK callback dies with the owner (#161). The one-shot-ref + * zoo is gone: the epoch counter (I1) drops stale-generation outcomes. */ export default function ChatThread({ chatId, @@ -219,186 +226,204 @@ export default function ChatThread({ const { t } = useTranslation(); const queryClient = useQueryClient(); - // resume machinery refs (#184 phase 1.5) - const attachAbortRef = useRef(null); - const reconcileTailRef = useRef(false); - const noStreamHandledRef = useRef(false); - const onNoActiveStreamRef = useRef<(() => void) | null>(null); - // #430: called from the transport's reconnect-GET success branch when a live - // stream re-attached (2xx, not 204) — clears the reconnect banner. Kept in a ref - // because the transport's fetch closure (useMemo([])) reads it live. - const onReconnectAttachedRef = useRef<(() => void) | null>(null); - // Live mount flag. The attach GET and the resumed `onFinish` are async and can - // land AFTER this thread unmounts (the parent remounts per chat via `key`); with - // chatIdRef then pointing at the NEW chat, an ungated late callback would arm a - // spurious poll + foreign invalidation on the newly-opened chat. Every parent- - // facing resume side-effect is gated on this. - const mountedRef = useRef(true); - const [resumedTurn, setResumedTurn] = useState(false); - const resumedTurnRef = useRef(false); - // Identity-stable pair setter (bare useState setter + ref write): it is closed - // over by the transport useMemo([]), so it MUST NOT capture state. - const setResumedTurnPair = useCallback((v: boolean) => { - resumedTurnRef.current = v; - setResumedTurn(v); - }, []); + // --- The run-lifecycle FSM (source of truth) ----------------------------- + // Stored in state (drives UI re-render on phase/ctx change) + mirrored in a ref + // for the transport/timer closures (useMemo([])-stable) that read it live. + const [machine, setMachine] = useState(() => initialMachine()); + const machineRef = useRef(machine); + // I1: the current generation. Command-transitions bump it; async OUTCOME events + // carry the epoch they were issued under; a mismatch is dropped by the reducer. + const epochRef = useRef(0); + // The generation an in-flight attach GET was issued under (for stamping its + // 2xx/204/throw outcome). Replaces the one-shot `noStreamHandledRef` guard. + const pendingAttachEpochRef = useRef(0); + // Stable dispatch handle for closures defined before `dispatch` (transport, + // timers) — assigned once `dispatch` exists below. + const dispatchRef = useRef<(e: RunEvent) => void>(() => undefined); - // Mount-time resume gating (in refs — computed once for this mount; the parent - // remounts per chat via `key`). - // - // Attempt resume for any non-settled tail: a streaming tail (strip + expect - // live replay) or a user tail (the run may exist but its assistant row is not - // seeded yet — attach to the pre-opened registry entry and wait for frames). - // A settled assistant tail must NEVER resume: replaying a finished run into a - // store that already contains its message duplicates parts (SDK text-start - // always pushes a new part). + // --- Effect-dependency refs (live values for the stable effect runner) ----- + // These are useChat/prop values that change identity across renders; the effect + // runner (useCallback([])) reads them live so it never captures a stale closure. + const resumeStreamRef = useRef<(() => Promise | void) | null>(null); + const setMessagesRef = useRef< + ((updater: (prev: UIMessage[]) => UIMessage[]) => void) | null + >(null); + const sendMessageRef = useRef<((m: { text: string }) => void) | null>(null); + const stopFnRef = useRef<(() => void) | null>(null); + const onResumeFallbackRef = useRef(onResumeFallback); + onResumeFallbackRef.current = onResumeFallback; + const onServerStopRef = useRef(onServerStop); + onServerStopRef.current = onServerStop; + + // Live mount flag: the attach GET / a resumed onFinish are async and can land + // AFTER unmount (the parent remounts per chat). The epoch (I1) drops stale FSM + // OUTCOMES, but the imperative onFinish side-effects (parent flush, invalidate) + // still need a plain React-liveness bit — orthogonal to the run-lifecycle, so it + // is NOT one of the lifecycle flags the FSM replaced. + const mountedRef = useRef(true); + + // attachStrategy DATA (behind the resumeStream effect; #491 swaps it to tail-only + // WITHOUT touching the FSM). The controller is effect-owned (aborted in cleanup, + // I5). `stripRef`/`strippedRowRef` are the current full-replay+strip anchor. + const attachAbortRef = useRef(null); const stripRef = useRef(chatId !== null && isStreamingTail(initialRows ?? [])); - const attemptResumeRef = useRef( - autonomousRunsEnabled === true && - chatId !== null && - !isSettledAssistantTail(initialRows ?? []), - ); const strippedRowRef = useRef( stripRef.current ? (initialRows ?? [])[initialRows!.length - 1] : null, ); + // Effect-owned backoff timers (not lifecycle flags): the reconnect ladder and the + // stalled inactivity cap. Cleared by the cancelReconnect effect / the cap effect. + const reconnectTimerRef = useRef | null>(null); + const idleCapTimerRef = useRef | null>(null); const initialMessages = useMemo( () => seedRows( initialRows ?? [], - attemptResumeRef.current && stripRef.current, + stripRef.current && autonomousRunsEnabled === true, ).map(rowToUiMessage), [initialRows], ); - // The server resolves/creates the chat from the `chatId` in the request body. - // A new chat starts as null; we keep the id in a ref so the SAME hook instance - // can keep streaming to a chat once it exists (the parent adopts the id on - // finish, but within this mount the body carries whatever we know). + // Identity/data mirrors (NOT lifecycle flags — they stay as data). const chatIdRef = useRef(chatId); chatIdRef.current = chatId; - - // Keep the currently-open page in a ref, updated each render, so the LATEST - // open page is sent on every send WITHOUT re-creating the `useMemo([])`-stable - // transport (and thus without re-creating the useChat store mid-stream — see - // the `chatStoreId` note below). Read live inside `prepareSendMessagesRequest`. const openPageRef = useRef(openPage ?? null); openPageRef.current = openPage ?? null; - - // Keep the selection snapshotter in a ref, same rationale as openPageRef: the - // transport useMemo([]) closes it over, so prop-identity churn must not matter. - // Called at send time inside prepareSendMessagesRequest (#388). const getEditorSelectionRef = useRef< (() => EditorSelectionContext | null) | undefined >(getEditorSelection); getEditorSelectionRef.current = getEditorSelection; - - // Keep the selected role id in a ref, same rationale as openPageRef. Only the - // FIRST request of a brand-new chat uses it (the server persists it then and - // ignores it for existing chats), but sending it on every send is harmless. const roleIdRef = useRef(roleId ?? null); roleIdRef.current = roleId ?? null; - - // Stable `useChat` store key for the lifetime of THIS mount. - // - // CRITICAL: `useChat` (@ai-sdk/react) re-creates its internal `Chat` store - // whenever the `id` option no longer equals the store's current id - // (`"id" in options && chatRef.current.id !== options.id`). For a brand-new - // chat (`chatId === null`) we previously passed `id: undefined`; the store - // then generated its OWN random id internally, so `store.id !== undefined` - // stayed true on EVERY render and the store was re-created on every render — - // wiping the optimistic user message, the "submitted" status, and every - // streamed delta until the turn fully finished (then the parent adopts the - // new chat id and remounts with the persisted history, making everything - // "appear at once"). Passing a STABLE non-undefined id keeps one store for - // the whole turn, so the user message shows immediately and tokens stream - // live. This id is purely the client store key; the server still resolves the - // real chat from `chatId` in the request body (see `prepareSendMessagesRequest`). - // The id only needs to be stable per mount — the parent remounts this via - // `key` on chat switch, which re-seeds cleanly. + // Stable useChat store key for this mount (see the long #137/#174 note that used + // to live here — recreating the store mid-stream wipes the live turn). const stableIdRef = useRef(chatId ?? `new-${generateId()}`); - // Stable for the LIFETIME of this mount. When a brand-new chat adopts its - // server id, the parent now updates the `chatId` prop WITHOUT remounting this - // thread, so the store id must NOT follow `chatId`: recreating the useChat - // store would wipe the live (just-finished) turn. The server still resolves - // the real chat from `chatId` in the request body (see chatIdRef / - // prepareSendMessagesRequest), so this purely-client store key can stay fixed. const chatStoreId = stableIdRef.current; - // Pending messages the user composed WHILE a turn was streaming. They are sent - // automatically, FIFO, on successful turn completion (`onFinish`). The queue is - // LOCAL state so it is scoped to this conversation: it is cleared when the user - // deliberately switches chat / starts a new chat (the parent remounts this via - // `key`), but it SURVIVES in-place new-chat id adoption (no remount), so a - // message queued during a brand-new chat's first turn is not lost. On Stop or - // error the queue is intentionally preserved (onFinish does not fire then) so - // the user decides what to do with the pending messages. + // Pending messages composed while a turn streams; flushed FIFO on a clean finish. const [queued, setQueued] = useState([]); - // Mirror the queue in a ref so the `onFinish` flush always reads the latest - // queue without a stale closure; `setQueue` updates BOTH the ref and the state. const queuedRef = useRef([]); const setQueue = useCallback((next: QueuedMessage[]) => { queuedRef.current = next; setQueued(next); }, []); - // Capture the latest `sendMessage` (returned by useChat below) so the flush - // helper can call the current instance from the stable `onFinish` callback. - const sendMessageRef = useRef<((m: { text: string }) => void) | null>(null); + // #488 commit 5: the runId to inject into the NEXT POST as `supersede:{runId}` + // (read-and-cleared by prepareSendMessagesRequest). This is send-plumbing DATA + // (like chatIdRef/roleIdRef), the single replacement for the three REMOVED + // one-shot flags flushOnAbortRef/interruptNextSendRef/supersedeRetryRef. + const pendingSupersedeRef = useRef(null); - // "Send now" single-flight flags. Kept in refs (not state) so they are read - // inside the stable `onFinish` callback and the transport closure WITHOUT a - // re-render or a stale closure. Both are one-shot (read-and-clear). - // - flushOnAbortRef: flush the promoted head on the abort WE triggered, even - // though an aborted turn normally keeps the queue intact. - // - interruptNextSendRef: tag the next send as a user interrupt so the server - // injects the "your previous answer was interrupted" note for that turn only. - const flushOnAbortRef = useRef(false); - const interruptNextSendRef = useRef(false); + // --- Effect runner: executes the reducer's command effects --------------- + const runEffect = useCallback( + (eff: RunEffect, epoch: number) => { + switch (eff.type) { + case "resumeStream": { + // The attach GET. Stamp the outcome's generation (I1). A reconnect + // attempt filters the pinned live row from the store first (the mount + // seed already stripped it), so the live replay's text-start rebuilds it + // without duplicating parts (#430). + pendingAttachEpochRef.current = epoch; + if (machineRef.current.phase.name === "reconnecting") { + const anchor = strippedRowRef.current; + if (anchor) + setMessagesRef.current?.((prev) => + prev.filter((m) => m.id !== anchor.id), + ); + } + void resumeStreamRef.current?.(); + break; + } + case "scheduleReconnect": { + if (reconnectTimerRef.current) clearTimeout(reconnectTimerRef.current); + const attempt = eff.attempt; + const scheduledEpoch = epoch; + reconnectTimerRef.current = setTimeout(() => { + dispatchRef.current({ + type: "RECONNECT_ATTEMPT", + attempt, + epoch: scheduledEpoch, + }); + }, eff.delayMs); + break; + } + case "cancelReconnect": + if (reconnectTimerRef.current) { + clearTimeout(reconnectTimerRef.current); + reconnectTimerRef.current = null; + } + break; + case "armPoll": + onResumeFallbackRef.current?.(true); + break; + case "disarmPoll": + onResumeFallbackRef.current?.(false); + break; + case "abortAttach": + attachAbortRef.current?.abort(); + break; + case "stopRun": + // Authoritative stop. When the chat id is not known yet (brand-new chat's + // first moment), it is deferred: the chat-id adoption effect fires + // onServerStop the moment the id lands while still in `stopping`. + if (chatIdRef.current) + onServerStopRef.current?.(chatIdRef.current); + break; + case "postRun": { + // (Re)establish / verify the run-fact from POST /ai-chat/run. Stamp the + // RUN_FACT outcome with the issuing generation so a superseded verify is + // dropped (I1). + const cid = chatIdRef.current; + if (!cid) break; + const ep = epoch; + void getRun(cid) + .then((res) => { + const active = + res.run && isActiveRunStatus(res.run.status) + ? { runId: res.run.id } + : null; + dispatchRef.current({ type: "RUN_FACT", runFact: active, epoch: ep }); + }) + .catch(() => undefined); + break; + } + case "supersede": + // Arm the next POST to carry `supersede:{runId}`; the send itself is + // triggered by the caller (sendNow) via sendMessage. + pendingSupersedeRef.current = eff.targetRunId; + break; + } + }, + // Reads only refs + the stable queryClient — safe as an empty-dep callback. + [], + ); - // #396: one-shot arm for the bounded 409 A_RUN_ALREADY_ACTIVE retry on the - // "Interrupt and send now" re-send in autonomous mode. sendNow triggers the - // authoritative server stop of the detached run, but that stop and the - // onFinish->flushNext re-POST race: the new POST can hit the one-active-run - // gate before the old detached run has settled, yielding a spurious 409. When - // this ref is armed, the transport's send path retries that 409 with a short - // bounded backoff (the server stop guarantees convergence). A normal send (ref - // not armed) must STILL fail a 409 instantly (e.g. a genuine two-tab conflict). - // - // INVARIANT: sendNow arms this only to be consumed by the ONE re-POST that - // flushNext fires from onFinish. But that re-POST does not always happen (the - // promoted head may be gone, the finish may be a resumed turn, or the arm may - // race a stale finish). To keep the arm strictly one-shot it is disarmed on - // EVERY path where the paired interrupt one-shots (flushOnAbortRef / - // interruptNextSendRef) are cleared without a POST: the transport POST branch - // consumes it (read-and-clear), the onFinish `!flushNext()` no-send branch - // clears it, and the isStreaming-defuse effect clears it symmetrically. So it - // can never leak into a later, unrelated send and retry that send's genuine 409. - const supersedeRetryRef = useRef(false); + // Dispatch: reduce, run effects, re-render on a real state change. + const dispatch = useCallback( + (event: RunEvent) => { + const prev = machineRef.current; + const next = reduce(prev, event); + machineRef.current = next; + epochRef.current = next.ctx.epoch; + if (next.phase !== prev.phase || next.ctx !== prev.ctx) setMachine(next); + for (const eff of next.effects) runEffect(eff, next.ctx.epoch); + }, + [runEffect], + ); + dispatchRef.current = dispatch; - // #234 F5: the user pressed Stop while streaming a BRAND-NEW chat whose server - // chat id has not been adopted yet (the `start` chunk carrying it hadn't landed - // when Stop was pressed). A local SSE abort alone does NOT stop the DETACHED - // autonomous run — it keeps burning tokens and WRITING TO PAGES — so we cannot - // just no-op. We latch the stop as PENDING and fire the authoritative server - // stop the moment onServerChatId adopts the id (below). Read-and-cleared there; - // also defused on every new turn start so it can never fire against a later, - // unrelated turn's run. - const stopPendingRef = useRef(false); - - // FIFO dequeue + send the next queued message (no-op when the queue is empty). - // Returns whether a message was actually sent, so callers can tell an empty - // dequeue (nothing to flush) from a real send. + // FIFO dequeue + local send of the next queued message (no-op when empty). + const localSend = useCallback((text: string) => { + dispatchRef.current({ type: "SEND_LOCAL" }); + sendMessageRef.current?.({ text }); + }, []); const flushNext = useCallback(() => { const { head, rest } = dequeue(queuedRef.current); if (!head) return false; setQueue(rest); - // Local send: clear any resume-suppression flag so this genuine local turn's - // onFinish flushes normally (invariant 8). - setResumedTurnPair(false); - sendMessageRef.current?.({ text: head.text }); + localSend(head.text); return true; - }, [setQueue, setResumedTurnPair]); + }, [setQueue, localSend]); const enqueue = useCallback( (text: string) => { @@ -419,12 +444,11 @@ export default function ChatThread({ api: "/api/ai-chat/stream", credentials: "include", prepareReconnectToStreamRequest: () => ({ - // SDK default URL uses the useChat STORE id — always build from the real chat id. - // ?expect=live&anchor= ONLY when we stripped a streaming tail: expect=live - // is the only case where a finished-retained replay is safe (the row is stripped, - // replay rebuilds it), and the anchor pins the replay to OUR run — a mismatching - // (newer) run must 204 into the restore+poll path instead of replaying a foreign - // transcript into this store. + // Build the attach URL from the REAL chat id. ?expect=live&anchor= + // only when a streaming tail was stripped: expect=live opts into a + // finished-retained replay (safe only because the row is stripped and the + // replay rebuilds it), and the anchor pins the replay to OUR run — a + // mismatching (newer) run 204s into the restore+poll path instead. api: `/api/ai-chat/runs/${chatIdRef.current}/stream${ stripRef.current ? `?expect=live&anchor=${strippedRowRef.current!.id}` @@ -433,102 +457,77 @@ export default function ChatThread({ }), fetch: async (input: RequestInfo | URL, init: RequestInit = {}) => { if ((init.method ?? "GET") !== "GET") { - // Send path (POST). #396: read-and-clear the one-shot supersede arm - // here so it is strictly scoped to THIS send. When unarmed, behave - // exactly as before — a single fetch, a 409 surfaces instantly (a - // genuine two-tab conflict must NOT be retried). - const supersede = supersedeRetryRef.current; - supersedeRetryRef.current = false; - if (!supersede) return fetch(input, init); - // Buffer a ReadableStream body once so each retry can replay it. - // DefaultChatTransport sends the body as a JSON STRING (replayable as - // is), but guard defensively in case a future SDK streams it. - let sendInit = init; - if (init.body instanceof ReadableStream) { - const buffered = await new Response(init.body).arrayBuffer(); - sendInit = { ...init, body: buffered }; - } - // Bounded retry: attempt 1 fires immediately, then wait between - // attempts per SUPERSEDE_RETRY_DELAYS_MS. Retry ONLY on a real - // 409 A_RUN_ALREADY_ACTIVE; any other status/body is returned as-is. - for (let attempt = 0; ; attempt++) { - const response = await fetch(input, sendInit); - if ( - response.status !== 409 || - attempt >= SUPERSEDE_RETRY_DELAYS_MS.length || - !(await isRunAlreadyActive(response)) - ) { - return response; + // Send path (POST). #488 commit 5: NO client 409 retry ladder anymore + // (the CAS supersede replaces it). We DO peek 409/CAS outcomes to drive + // the FSM, then return the Response untouched to the SDK (a 409 body is + // surfaced by the classified error banner via error-message.ts). + const wasSupersede = + machineRef.current.phase.name === "superseding"; + const ep = epochRef.current; + const response = await fetch(input, init); + if (wasSupersede) { + if (response.ok) { + dispatchRef.current({ type: "SUPERSEDE_READY", epoch: ep }); + } else if (response.status === 409) { + const { code, runId } = await read409(response); + if (code === "SUPERSEDE_TARGET_MISMATCH") + dispatchRef.current({ + type: "SUPERSEDE_MISMATCH", + currentRunId: runId, + epoch: ep, + }); + else if (code === "SUPERSEDE_TIMEOUT") + dispatchRef.current({ type: "SUPERSEDE_TIMEOUT", epoch: ep }); + else if (code === "SUPERSEDE_INVALID") + dispatchRef.current({ type: "SUPERSEDE_INVALID", epoch: ep }); } - // The old detached run has not released the one-active-run slot - // yet; the server stop we requested guarantees it will, so back off - // and re-POST (the 409 fired before the user message was persisted, - // so re-POSTing is safe — no duplicate rows). - await new Promise((r) => - setTimeout(r, SUPERSEDE_RETRY_DELAYS_MS[attempt]), - ); + } else if (response.status === 409) { + const { code } = await read409(response); + if (code === "A_RUN_ALREADY_ACTIVE") + dispatchRef.current({ type: "RUN_ALREADY_ACTIVE" }); } + return response; } - // Reconnect GET: the SDK passes no AbortSignal, so wire our own controller - // for observer Stop / unmount abort. + // Reconnect/attach GET: the SDK passes no AbortSignal, so wire our own + // controller for observer Stop / unmount abort (effect-owned, I5). const controller = new AbortController(); attachAbortRef.current = controller; + const ep = pendingAttachEpochRef.current; + const wasReconnecting = + machineRef.current.phase.name === "reconnecting"; try { const response = await fetch(input, { ...init, signal: controller.signal, }); - // No onFinish will come for a 204 (silent no-op) OR any non-2xx - // (5xx/502 — a server restart mid-attach). Both run the same - // no-active-stream recovery: restore the stripped row, invalidate, and - // arm the degraded poll (idempotent via noStreamHandledRef; its part-d - // also clears the resumedTurn flag). This is the restart-survival path - // the removed F7 latch used to guard — a transient attach failure must - // NOT drop the in-progress row or stop tracking the durable run. if (response.status === 204 || !response.ok) - onNoActiveStreamRef.current?.(); - // #430: a 2xx stream re-attached (live tail or finished-replay). Signal - // the reconnect controller to clear its banner. No-op outside an active - // reconnect sequence (e.g. the mount attach), so it is safe here. - else onReconnectAttachedRef.current?.(); + handleAttachOutcome(ep, wasReconnecting, false); + else handleAttachOutcome(ep, wasReconnecting, true); return response; } catch (err) { - // Network throw: same no-onFinish recovery, then rethrow so the SDK - // still surfaces the error to its own machinery. - onNoActiveStreamRef.current?.(); + handleAttachOutcome(ep, wasReconnecting, false); throw err; } }, - // Inject the chat id and the currently-open page alongside the useChat - // messages so the server can resolve an existing chat (or create one - // when null) and tell the agent which page "this page" refers to. Both - // are read live from refs so changing chats/pages does NOT recreate the - // transport. `openPage` is null on a non-page route. prepareSendMessagesRequest: ({ messages, body }) => { - // Read-and-clear the interrupt flag so the "you were interrupted" note - // is carried by ONLY this request (the one resending the promoted - // message right after we aborted the previous turn). The server still - // confirms it against history before acting on it. - const interrupted = interruptNextSendRef.current; - interruptNextSendRef.current = false; // one-shot + // #488 commit 5: read-and-clear the supersede runId so `supersede:{runId}` + // is carried by ONLY this request (the CAS "interrupt and send now"). + const supersedeRunId = pendingSupersedeRef.current; + pendingSupersedeRef.current = null; return { body: { ...body, chatId: chatIdRef.current, - // Attach the live editor selection to the open-page context at send - // time — "this"/"here" in the user's message means THIS selection. - // Nested inside openPage so it dies with the page when the server - // rejects the page id (#388). Null when nothing is selected. openPage: openPageRef.current ? { ...openPageRef.current, selection: getEditorSelectionRef.current?.() ?? null, } : null, - // Honoured by the server only when creating a new chat; null => - // universal assistant. roleId: roleIdRef.current, - interrupted, + ...(supersedeRunId + ? { supersede: { runId: supersedeRunId } } + : {}), messages, }, }; @@ -537,6 +536,37 @@ export default function ChatThread({ [], ); + // Attach GET outcome -> FSM event. The epoch guard replaces BOTH the one-shot + // 204 guard (noStreamHandledRef) and the unmount gate: a stale/superseded or + // post-DISPOSE outcome is dropped (I1). For a NONE outcome the attachStrategy + // recovery (restore the stripped row + invalidate for a fresh poll) runs first. + const handleAttachOutcome = useCallback( + (ep: number, wasReconnecting: boolean, live: boolean) => { + if (ep !== epochRef.current) return; // stale generation — drop + if (live) { + dispatchRef.current( + wasReconnecting + ? { type: "RECONNECT_ATTACHED", epoch: ep } + : { type: "ATTACH_LIVE", epoch: ep }, + ); + return; + } + if (strippedRowRef.current) + setMessagesRef.current?.((prev) => + mergeById(prev, rowToUiMessage(strippedRowRef.current!)), + ); + queryClient.invalidateQueries({ + queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), + }); + dispatchRef.current( + wasReconnecting + ? { type: "RECONNECT_NONE", epoch: ep } + : { type: "ATTACH_NONE", epoch: ep }, + ); + }, + [queryClient], + ); + const { messages, sendMessage, @@ -546,579 +576,318 @@ export default function ChatThread({ setMessages, resumeStream, } = useChat({ - // Stable per-mount key. Existing chats use their real id; new chats use a - // generated client id (never `undefined`) so the store is NOT re-created on - // every render mid-stream (see `chatStoreId` above). id: chatStoreId, messages: initialMessages, transport, - // See STREAM_THROTTLE_MS — bounds re-render/markdown-reparse frequency. experimental_throttle: STREAM_THROTTLE_MS, - // `onFinish` (ai@6 useChat) fires from a `finally` on EVERY terminal outcome - // — success, user Stop/abort (`isAbort`), network drop (`isDisconnect`), and - // stream error (`isError`). Keep calling `onTurnFinished()` on all of them - // (chat-list refresh + new-chat id adoption must happen even on a failed - // first turn), but flush the pending queue ONLY on a clean finish: auto- - // sending after the user hit Stop — or blindly retrying after a failure — - // would be wrong, so on Stop/disconnect/error the queue is left intact for - // the user to decide. onFinish: ({ message, isAbort, isDisconnect, isError }) => { - // (1) Capture whether THIS finish belongs to a resumed (attach) turn and - // immediately clear the flag so it can never suppress a LATER local turn. - const wasResumed = resumedTurnRef.current; - setResumedTurnPair(false); - // (2) Recovery after a starved/torn resumed finish (invariant 9). The arm - // and the stripped-row restore are gated DIFFERENTLY. Skip entirely once - // unmounted (an abort-triggered onFinish landing after a chat switch must - // not arm a poll / invalidate on the new chat). - if (wasResumed && mountedRef.current) { - const hasVisibleContent = assistantMessageHasVisibleContent(message); - // ARM the reconcile + degraded poll when the resumed message carries no - // visible content (starved replay) OR the connection dropped mid-run — in - // both cases the poll must drive the row to its real terminal state. - if (isDisconnect || !hasVisibleContent) { - reconcileTailRef.current = true; - queryClient.invalidateQueries({ - queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), - }); - onResumeFallback?.(true); - } - // RESTORE the stripped streaming row ONLY when the resumed message has no - // visible content. On isDisconnect WITH visible content restore is - // FORBIDDEN: the live stream may have advanced far past the mount-time - // snapshot, so restoring would clobber on-screen content (invariant 9) — - // the arm above suffices, the poll reaches the true terminal. - if (!hasVisibleContent && strippedRowRef.current) { - setMessages((prev) => - mergeById(prev, rowToUiMessage(strippedRowRef.current!)), - ); - } - } - // (2b) #430/#488: a LIVE (non-resumed) detached run whose SSE just dropped. - // The server run keeps executing, so instead of a dead "Lost connection" - // banner start a reconnect sequence and re-attach to the live tail via the - // resumable machinery. - // - // #488 commit 2 — enter reconnect by the RUN-FACT (an active detached run), - // NOT by the presence of an assistant message. The old gate additionally - // required `message?.role === "assistant"`, so a break during the SETUP phase - // — before the first assistant frame, INCLUDING an up-to-60s MCP-toolset - // assembly — fell through to neither reconnect NOR poll while the detached - // run kept writing to pages (a silent data-integrity hole). In autonomous - // mode a run is active for the whole turn, so `autonomousRunsEnabled` is the - // run-fact signal here; the richer server run-fact (POST /run / start-metadata - // runId) is modelled + tested in the FSM (run-fsm.ts `FINISH_DISCONNECT`, - // gated on `ctx.runFact`) and lands with the full component migration. - // - // When there IS an assistant row, pin it as the strip/anchor so the live - // replay rebuilds it without duplicating parts; when there is NONE (the - // pre-first-frame break), reconnect WITHOUT an anchor (a plain live attach — - // there is nothing on screen to rebuild). - const canReconnect = - isDisconnect && - !wasResumed && - autonomousRunsEnabled === true && - mountedRef.current; - if (canReconnect) { - const hasAnchor = - message?.role === "assistant" && typeof message.id === "string"; - beginReconnect( - hasAnchor - ? { - id: message.id, - role: "assistant", - content: "", - status: "streaming", - createdAt: new Date().toISOString(), - // Preserve the partial parts so a 204 restore (onNoActiveStream) - // re-shows what was on screen while the degraded poll catches the - // run up to terminal (rowToUiMessage prefers metadata.parts). - metadata: { parts: message.parts }, - } - : null, - ); - } - // (3) Standard branches. - // Forward the authoritative server chatId (streamed on the assistant - // message metadata) so the parent adopts the REAL created chat id for a new - // chat — see adopt-chat-id.ts for the full #137 design. `threadKey` lets the - // session ignore this finish if it belongs to a thread abandoned by New chat - // mid-stream (#161). + // Ownership (I2) is the FSM ctx: a resumed/attached/reconnected turn is an + // OBSERVER; a local send is the owner. The queue flushes ONLY under local + // ownership; an observer never flushes (invariant 7). + const wasObserver = machineRef.current.ctx.ownership === "observer"; + // Notify the parent on EVERY terminal outcome (threadKey-guarded downstream + // for #161); fires even while unmounting. onTurnFinished(extractServerChatId(message), threadKey); - // Show a neutral "stopped" marker for an aborted turn; the red error banner - // (via `error`) already covers isError, and a clean finish clears any marker. - // On a live disconnect that STARTED a reconnect, suppress the terminal - // "connection lost" notice — the reconnect banner takes over (#430). - if (isError) setStopNotice(null); - else if (isAbort) setStopNotice("manual"); - else if (isDisconnect) setStopNotice(canReconnect ? null : "disconnect"); - else setStopNotice(null); - // A resumed turn NEVER flushes the queue (invariant 7): skip BOTH the - // flush-on-abort branch and the plain flush. The local streamer is the only - // tab that owns the queue. - if (wasResumed) return; - // "Send now": WE triggered this abort to interrupt the current turn and - // immediately send the promoted head. Flush it even though the turn was - // aborted (the normal abort path below keeps the queue intact). The - // interrupt note travels with this send via interruptNextSendRef. - if (flushOnAbortRef.current) { - flushOnAbortRef.current = false; - // Suppress the "Response stopped." flash for an intentional interrupt. + + if (isError) { + dispatch({ type: "FINISH_ERROR", kind: "stream" }); setStopNotice(null); - // If the promoted head vanished (e.g. the user removed it before the - // abort landed) flushNext sends nothing — clear the one-shot interrupt - // tag AND the #396 supersede arm so neither can leak onto the next - // unrelated send (no re-POST will consume the arm here). On a real send - // the tag is consumed by prepareSendMessagesRequest and the arm by the - // transport POST branch, so both stay untouched then. - if (!flushNext()) { - interruptNextSendRef.current = false; - supersedeRetryRef.current = false; + return; + } + if (isAbort) { + // A user Stop / an interrupt abort finished. The FSM stopping/idle exit is + // by DATA (this terminal outcome, I4). + dispatch({ type: "FINISH_ABORT" }); + setStopNotice("manual"); + return; + } + // A missing message (a pre-first-frame break) has no visible content. + const msgHasVisible = message + ? assistantMessageHasVisibleContent(message) + : false; + if (isDisconnect) { + if (wasObserver) { + // A resumed/attached OBSERVER stream dropped. Recover via the degraded + // poll (restore the stripped row only when there is no visible content; + // never clobber a fuller on-screen tail, invariant 9). The FSM decides + // reconnect-vs-poll from liveFollow (a live-follow drop reconnects again, + // #488 commit 3; a mount-resume drop polls). + if (mountedRef.current) { + const hasVisible = msgHasVisible; + if (!hasVisible && strippedRowRef.current) + setMessages((prev) => + mergeById(prev, rowToUiMessage(strippedRowRef.current!)), + ); + queryClient.invalidateQueries({ + queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), + }); + dispatch({ + type: "FINISH_DISCONNECT", + hasVisibleContent: hasVisible, + }); + } + setStopNotice(null); + return; + } + // A LOCAL live turn dropped. #488 commit 2: recover by the RUN-FACT, not by + // the presence of an assistant message — a setup-phase break (before the + // first frame) still leaves a detached run writing to pages. In autonomous + // mode a run is active for the whole turn, so seed the run-fact from the + // start-metadata runId when known, else a sentinel (the attach GET goes by + // chatId, not runId). Pin the assistant row as the strip/anchor when present. + if (autonomousRunsEnabled === true && mountedRef.current) { + const hasAnchor = + message?.role === "assistant" && typeof message.id === "string"; + if (hasAnchor) { + strippedRowRef.current = { + id: message.id, + role: "assistant", + content: "", + status: "streaming", + createdAt: new Date().toISOString(), + metadata: { parts: message.parts }, + }; + stripRef.current = true; + } else { + strippedRowRef.current = null; + stripRef.current = false; + } + dispatch({ + type: "RUN_FACT", + runFact: { runId: extractRunId(message) ?? "pending" }, + }); + dispatch({ + type: "FINISH_DISCONNECT", + hasVisibleContent: msgHasVisible, + }); + setStopNotice(null); + } else { + dispatch({ type: "FINISH_DISCONNECT", hasVisibleContent: false }); + setStopNotice("disconnect"); } return; } - if (isAbort || isDisconnect || isError) return; - // Gate the final flush on the live-mount flag (#486): a clean onFinish can - // land AFTER this thread unmounted (a New-chat / chat-switch mid-stream — - // the async attach/resume settles late). Flushing then dequeues and POSTs a - // queued message from an abandoned thread — a "ghost" send / ghost chat. - // Every other queue side effect already guards on mountedRef; this last one - // was the gap. + // Clean finish. + if (wasObserver) { + if (mountedRef.current) { + const hasVisible = msgHasVisible; + if (!hasVisible) { + // Starved replay: restore the stripped row + poll to the real terminal. + if (strippedRowRef.current) + setMessages((prev) => + mergeById(prev, rowToUiMessage(strippedRowRef.current!)), + ); + queryClient.invalidateQueries({ + queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), + }); + dispatch({ type: "STREAM_INCOMPLETE", reason: "starved" }); + } else { + // Healthy resumed finish — nothing to restore/arm, just settle. + dispatch({ type: "FINISH_CLEAN" }); + } + } + setStopNotice(null); + return; + } + // Local clean finish: settle + flush the queue (gated on liveness, #486). + dispatch({ type: "FINISH_CLEAN" }); + setStopNotice(null); if (mountedRef.current) flushNext(); }, - // `onError` runs in addition to `onFinish` (which ai@6 also calls on error). - // Log the raw failure here for devtools; the UI shows a friendly classified - // banner via `error` below. We still call `onTurnFinished()` with NO server id - // (idempotent with the onFinish call): for a brand-new chat that ARMS the - // bounded list-refetch fallback (adopt the single newly-appeared chat once the - // refetch lands); for an existing chat it just refreshes the chat list - // immediately rather than after a manual refresh. onError: (streamError) => { - // Surface the raw failure in the browser console (devtools) for debugging; - // the UI separately shows a friendly classified banner (see errorView). console.error("AI chat stream error:", streamError); onTurnFinished(undefined, threadKey); }, }); - // Keep the flush helper pointed at the latest sendMessage instance. + // Publish the live useChat handles to the effect-runner refs. + resumeStreamRef.current = resumeStream; + setMessagesRef.current = setMessages as unknown as ( + updater: (prev: UIMessage[]) => UIMessage[], + ) => void; sendMessageRef.current = sendMessage; + stopFnRef.current = stop; - // Mirror the live turn status in a ref so event handlers (sendNow) branch on the - // CURRENT status rather than a value captured in a stale render closure — a turn - // can finish between render and click, and arming the interrupt refs against a - // no-op stop() would leave them set to leak into a later, unrelated Stop. const statusRef = useRef(status); statusRef.current = status; - // EARLY chat-id adoption (#174): the server streams the authoritative chat id - // on the assistant message metadata at the `start` chunk (message.metadata. - // chatId — see adopt-chat-id.ts / chatStreamMetadata). Forward it to the parent - // AS SOON AS it appears (mid-stream), so a brand-new chat adopts its real id - // WHILE the first turn is still streaming and activeChatId-gated affordances - // (the Copy/export button) light up immediately, instead of only at onFinish. - // Keyed by the last-seen id so we forward each distinct id exactly once. The - // parent's onServerChatId is idempotent and a no-op once the chat has an id. + // EARLY chat-id (#174) + runId (#488) adoption from the streaming assistant + // message's start metadata. Forward the chat id once; adopt the runId into the + // FSM run-fact; and (#234 F5) fire a DEFERRED server stop the moment the id lands + // while still `stopping` — no stopPendingRef needed (the FSM phase is the latch). const lastForwardedChatIdRef = useRef(undefined); useEffect(() => { - if (!onServerChatId) return; const tail = messages[messages.length - 1]; if (tail?.role !== "assistant") return; const serverChatId = extractServerChatId(tail); - if (!serverChatId || serverChatId === lastForwardedChatIdRef.current) - return; - lastForwardedChatIdRef.current = serverChatId; - onServerChatId(serverChatId); - // #234 F5: if Stop was pressed before the id was known, the authoritative - // server stop was deferred to this adoption point — fire it now with the - // just-adopted id. One-shot (read-and-clear) so it can't fire twice. - if (stopPendingRef.current) { - stopPendingRef.current = false; - onServerStop?.(serverChatId); + if ( + onServerChatId && + serverChatId && + serverChatId !== lastForwardedChatIdRef.current + ) { + lastForwardedChatIdRef.current = serverChatId; + onServerChatId(serverChatId); + if (machineRef.current.phase.name === "stopping") + onServerStopRef.current?.(serverChatId); } - }, [messages, onServerChatId, onServerStop]); + const runId = extractRunId(tail); + if (runId && machineRef.current.ctx.runFact?.runId !== runId) + dispatch({ type: "RUN_FACT", runFact: { runId } }); + }, [messages, onServerChatId, dispatch]); - // Live "turn was interrupted" marker for the CURRENT session. The red error - // banner (driven by `error`) covers the error case; this covers an aborted - // turn, distinguishing a manual Stop (`isAbort`) from a dropped connection - // (`isDisconnect`) — a distinction only available live (the server persists - // both as finishReason 'aborted'). Cleared when the next turn starts. + // Live "turn was interrupted" marker (a manual Stop vs a dropped connection — a + // distinction only available live). Cleared when the next turn starts. const [stopNotice, setStopNotice] = useState( null, ); const isStreaming = status === "submitted" || status === "streaming"; + const phase = machine.phase; + const isObserver = machine.ctx.ownership === "observer"; - // #430: live-disconnect reconnect controller. `null` = idle; `{ trying, attempt }` - // = a backoff sequence is running (drives the "reconnecting… (N/max)" banner); - // `{ failed }` = attempts exhausted (drives the manual Retry). Mirrored into a ref - // so the transport/onNoActiveStream closures branch on the LIVE value. - type ReconnectState = - | null - | { phase: "trying"; attempt: number } - | { phase: "failed" }; - const [reconnectState, setReconnectState] = useState(null); - const reconnectStateRef = useRef(null); - const setReconnectStatePair = useCallback((s: ReconnectState) => { - reconnectStateRef.current = s; - setReconnectState(s); - }, []); - const reconnectTimerRef = useRef | null>(null); - const clearReconnectTimer = useCallback(() => { - if (reconnectTimerRef.current) { - clearTimeout(reconnectTimerRef.current); - reconnectTimerRef.current = null; - } - }, []); - - // One reconnect attempt — MIRRORS the mount strip/anchor path for the LIVE case. - // beginReconnect pinned strippedRowRef/stripRef to the run's assistant row, so: - // - remove that row from the store (the mount path strips it from the SEED; here - // it is already shown, so filter it out) — the live replay's `text-start` then - // rebuilds it without DUPLICATING parts (the main dedup risk, #430); - // - reset the one-shot 204 guard so onNoActiveStream can fire for THIS attempt; - // - mark the turn resumed (invariant 7/8) so onFinish runs the recovery block and - // never flushes the queue; - // - resumeStream() -> prepareReconnectToStreamRequest builds - // ?expect=live&anchor=, pinning the replay to OUR run (invariant 6). - const attemptReconnectOnce = useCallback( - (attempt: number) => { - if (!mountedRef.current) return; - const anchor = strippedRowRef.current; - if (anchor) { - setMessages((prev) => prev.filter((m) => m.id !== anchor.id)); - } - noStreamHandledRef.current = false; - setResumedTurnPair(true); - setReconnectStatePair({ phase: "trying", attempt }); - void resumeStream(); - }, - [setMessages, setResumedTurnPair, setReconnectStatePair, resumeStream], - ); - - // Schedule attempt `attempt` after an exponential backoff. - const scheduleReconnectAttempt = useCallback( - (attempt: number) => { - clearReconnectTimer(); - setReconnectStatePair({ phase: "trying", attempt }); - reconnectTimerRef.current = setTimeout( - () => attemptReconnectOnce(attempt), - RECONNECT_BASE_DELAY_MS * 2 ** (attempt - 1), - ); - }, - [clearReconnectTimer, setReconnectStatePair, attemptReconnectOnce], - ); - - // Start a fresh reconnect sequence. `anchorRow` (the live run's assistant row) is - // pinned as the strip/anchor reused by every attempt; `null` (a pre-first-frame - // break, #488 commit 2) means there is nothing on screen to rebuild, so attach - // WITHOUT a strip/anchor (a plain live attach to whatever the run is emitting). - const beginReconnect = useCallback( - (anchorRow: IAiChatMessageRow | null) => { - if (!autonomousRunsEnabled || !mountedRef.current) return; - strippedRowRef.current = anchorRow; - stripRef.current = anchorRow !== null; - scheduleReconnectAttempt(1); - }, - [autonomousRunsEnabled, scheduleReconnectAttempt], - ); - - // Manual Retry (shown once attempts are exhausted): restart at attempt 1 and fire - // immediately (the user asked for it now — no backoff). - const retryReconnect = useCallback(() => { - clearReconnectTimer(); - attemptReconnectOnce(1); - }, [clearReconnectTimer, attemptReconnectOnce]); - - // Live SSE re-attached (the reconnect GET returned a 2xx stream): clear the - // banner + any pending backoff. No-op outside a sequence (e.g. the mount attach). - const onReconnectAttached = useCallback(() => { - if (!mountedRef.current || !reconnectStateRef.current) return; - clearReconnectTimer(); - setReconnectStatePair(null); - }, [clearReconnectTimer, setReconnectStatePair]); - onReconnectAttachedRef.current = onReconnectAttached; - - // The reconnect GET could not attach (204 / error). onNoActiveStream has already - // armed the degraded poll (the robust fallback that drives the row to terminal - // from the DB), so this only decides the LIVE-attach retry: back off and try - // again up to the cap, else surface the manual Retry. - const onReconnectNoStream = useCallback(() => { - const s = reconnectStateRef.current; - if (s?.phase !== "trying") return; - if (s.attempt < RECONNECT_MAX_ATTEMPTS) - scheduleReconnectAttempt(s.attempt + 1); - else setReconnectStatePair({ phase: "failed" }); - }, [scheduleReconnectAttempt, setReconnectStatePair]); - - // 204-handler (`onNoActiveStream`): the attach returned 204 — nothing live to - // resume (overflow / begin-failure / after retention / anchor-mismatch). One- - // shot via noStreamHandledRef (we do NOT null onNoActiveStreamRef). Exactly four - // parts. Kept in a ref (read by the transport's fetch closure) and refreshed - // each render below. - const onNoActiveStream = useCallback(() => { - // A late attach outcome after unmount must not arm a poll / invalidate on the - // now-different chat this thread's refs were reused for. - if (!mountedRef.current) return; - if (noStreamHandledRef.current) return; - noStreamHandledRef.current = true; - // (a) Restore the stripped streaming row to the store — ONLY when we actually - // stripped one (a user-tail 204 does NOT reach here with a stripped row, so do - // not dereference null). - if (strippedRowRef.current) { - setMessages((prev) => - mergeById(prev, rowToUiMessage(strippedRowRef.current!)), - ); - } - // (b) Reconcile the tail from the message history + invalidate it so the - // degraded poll starts from a fresh fetch. - reconcileTailRef.current = true; - queryClient.invalidateQueries({ - queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), - }); - // (c) Arm the degraded poll (a dumb timer with a 10-min cap in the window); - // the thread disarms it via onResumeFallback(false) on settle / local stream. - onResumeFallback?.(true); - // (d) 204 means onFinish will NOT fire — clear the suppression flag so it - // cannot swallow the NEXT local turn's queue flush. - setResumedTurnPair(false); - // (e) #430: if this 204/error landed during a live-disconnect reconnect - // sequence, back off and retry the live attach (or give up to the manual - // Retry). The degraded poll armed in (c) is the fallback either way. - onReconnectNoStream(); - }, [ - setMessages, - queryClient, - onResumeFallback, - setResumedTurnPair, - onReconnectNoStream, - ]); - onNoActiveStreamRef.current = onNoActiveStream; - - // Mount effect: kick off the resume attempt for a non-settled tail. Marking the - // turn as resumed BEFORE resumeStream so onFinish (invariant 7/8) sees it. + // Mount effect: arm the resume attempt — ONLY on a server-confirmed active run + // (#488 commit 4b). A STREAMING tail IS the run-fact (status streaming) -> attach + // now. A USER tail (ended on a user message) may have NO active run, so confirm + // via POST /run first: a chat with no active run must NOT arm a poll (the old code + // attached -> 204 -> ~240 req/10min storm). A SETTLED assistant tail never resumes. useEffect(() => { - // Re-arm on (re)mount — StrictMode dev-mounts twice, and the cleanup below - // flips this false between the two. mountedRef.current = true; - if (attemptResumeRef.current) { - setResumedTurnPair(true); - void resumeStream(); + if (autonomousRunsEnabled === true && chatId !== null) { + const rows = initialRows ?? []; + if (isStreamingTail(rows)) { + dispatch({ type: "ATTACH_START" }); + } else if (!isSettledAssistantTail(rows)) { + void getRun(chatId) + .then((res) => { + if (!mountedRef.current) return; + if (res.run && isActiveRunStatus(res.run.status)) + dispatch({ type: "ATTACH_START", runId: res.run.id }); + }) + .catch(() => undefined); + } } - // Unmount: mark unmounted (gates late attach/onFinish side-effects) and abort - // the in-flight attach GET so its callbacks don't fire against the next chat. return () => { mountedRef.current = false; - attachAbortRef.current?.abort(); - // #430: drop any pending reconnect backoff so it can't fire against the next - // chat this thread's refs are reused for. - if (reconnectTimerRef.current) clearTimeout(reconnectTimerRef.current); + dispatch({ type: "DISPOSE" }); // aborts attach + timers, bumps epoch (I5) }; // Mount-only by design; the parent remounts per chat via `key`. // eslint-disable-next-line react-hooks/exhaustive-deps }, []); - // Reconciliation + degraded-merge (invariant 8). Deps are EXACTLY - // [initialRows, isStreaming, setMessages]. + // Reconciliation + degraded-merge (invariant 8): while a poll-bearing recovery is + // active (polling / reconnecting / stopping), merge the polled assistant tail on + // every initialRows update and settle to terminal via POLL_TERMINAL. useEffect(() => { - // A local stream owns the view: disarm BOTH the merge and the window poll. - if (isStreaming) { - reconcileTailRef.current = false; - onResumeFallback?.(false); - return; - } - if (!reconcileTailRef.current) return; + if (isStreaming) return; // a local stream owns the view + const p = machineRef.current.phase.name; + if (p !== "polling" && p !== "reconnecting" && p !== "stopping") return; const rows = initialRows ?? []; const tail = rows[rows.length - 1]; if (!tail || tail.role !== "assistant") return; - // Merge the polled assistant tail on EVERY initialRows update — while the - // degraded poll is active this IS the live per-step progress. setMessages((prev) => mergeById(prev, rowToUiMessage(tail))); - // Anchor-mismatch coherence: when we restored a stripped streaming row A but a - // DIFFERENT run's row B is now the tail (A finished, B replaced the registry - // entry, so the attach 204'd), A would otherwise linger forever as an orphan - // jumping-dots row over the real run. Settle it from fresh history (where A is - // now persisted) so no phantom row survives. No-op in the common case where A - // IS the tail (id match). + // Anchor-mismatch coherence: a restored stripped row A that a DIFFERENT run's + // row B has replaced as the tail would linger as an orphan — settle A from + // fresh history so no phantom row survives. const stripped = strippedRowRef.current; if (stripped && stripped.id !== tail.id) { const historical = rows.find((r) => r.id === stripped.id); if (historical) setMessages((prev) => mergeById(prev, rowToUiMessage(historical))); } - // Settled: the terminal merge is done — disarm the flag AND the window poll - // explicitly (the window only has a time cap, it will not disarm itself). - if (tail.status !== "streaming") { - reconcileTailRef.current = false; - onResumeFallback?.(false); - // #430: the run reached its terminal state via the degraded poll — there is - // no live tail left to reconnect to, so drop any reconnect banner / Retry. - clearReconnectTimer(); - setReconnectStatePair(null); - } - // onResumeFallback intentionally omitted (parent-stable callback); deps are - // fixed by the resume design. + if (tail.status !== "streaming") dispatch({ type: "POLL_TERMINAL" }); // eslint-disable-next-line react-hooks/exhaustive-deps }, [initialRows, isStreaming, setMessages]); - // #430: a real stream is live again — the reconnect re-attached to the live tail - // (status -> "streaming") OR the user started a new local turn. Either way clear - // the reconnect banner + any pending backoff. Gated on "streaming" (not the - // broader "submitted") so a still-pending attach GET does not clear prematurely. + // #488 commit 4a: the stalled inactivity cap. While polling/reconnecting, if no + // new run activity (initialRows unchanged) for DEGRADED_POLL_IDLE_MAX_MS, fire + // POLL_IDLE_CAP -> the FSM goes `stalled` (banner + Retry) instead of silent. + // Reset on every activity (initialRows change) and on phase change. useEffect(() => { - if (status === "streaming") { - clearReconnectTimer(); - setReconnectStatePair(null); + if (idleCapTimerRef.current) { + clearTimeout(idleCapTimerRef.current); + idleCapTimerRef.current = null; } - }, [status, clearReconnectTimer, setReconnectStatePair]); + const p = phase.name; + if (p !== "polling" && p !== "reconnecting") return; + idleCapTimerRef.current = setTimeout(() => { + dispatchRef.current({ type: "POLL_IDLE_CAP" }); + }, DEGRADED_POLL_IDLE_MAX_MS); + return () => { + if (idleCapTimerRef.current) clearTimeout(idleCapTimerRef.current); + }; + }, [phase, initialRows]); - // "Send now" on a queued message: interrupt the current turn and immediately - // send THIS message, keeping the agent's partial output. Other queued messages - // stay queued and flush normally after the new turn. Reuses the existing - // queue/flush machinery: promote the target to the head, then abort — the - // onFinish flush-on-abort branch sends exactly that head, tagged as an - // interrupt so the server notes the previous answer was cut off. - const sendNow = useCallback( - (id: string) => { - // Branch on the LIVE status (statusRef), NOT the closure-captured isStreaming: - // the turn may have finished between this render and the click, in which case - // stop() is a no-op and arming the interrupt refs would strand them for a - // later, unrelated Stop. Reading the ref always sees the current status. - const liveStreaming = - statusRef.current === "submitted" || statusRef.current === "streaming"; - if (liveStreaming) { - // Promote to head so the onFinish -> flushNext path sends exactly it. - setQueue(promoteToHead(queuedRef.current, id)); - flushOnAbortRef.current = true; - interruptNextSendRef.current = true; - // #396: in autonomous mode the turn is a DETACHED run — a local stop() - // is only a client disconnect the server ignores, so the run keeps going. - // The onFinish->flushNext re-POST would then hit the one-active-run gate - // and get a spurious 409 A_RUN_ALREADY_ACTIVE. Mirror handleStop: request - // the AUTHORITATIVE server stop so the detached run settles, and arm the - // one-shot bounded 409 retry BEFORE stop() so the re-send converges once - // the slot frees. Read chatId live from chatIdRef (adopted at the `start` - // chunk). If it is not known yet (brand-new chat, first moment of its - // first turn), defer the server stop via stopPendingRef exactly as - // handleStop does — the onServerChatId adoption effect fires it once the - // id lands; the retry stays armed so the re-send still converges then. - if (autonomousRunsEnabled) { - supersedeRetryRef.current = true; // arm the bounded 409 retry - if (chatIdRef.current) { - onServerStop?.(chatIdRef.current); - } else { - // Same #234-F5 sub-window limitation documented in handleStop: if the - // local abort below cancels the reader before the `start` chunk lands, - // the adoption effect never runs and the deferred stop never fires. Not - // a regression; at minimum we don't strand refs (the isStreaming effect - // defuses stopPendingRef on the next turn start). - stopPendingRef.current = true; - } - } - stop(); // -> onFinish({ isAbort: true }) flushes the promoted head - } else { - // Nothing to interrupt: just send it now (no interrupt note). - const msg = queuedRef.current.find((m) => m.id === id); - if (!msg) return; - setQueue(removeQueuedById(queuedRef.current, id)); - // Local send: clear any resume-suppression flag (invariant 8). - setResumedTurnPair(false); - sendMessageRef.current?.({ text: msg.text }); - } - }, - [setQueue, stop, setResumedTurnPair, autonomousRunsEnabled, onServerStop], - ); - - // Stop the current turn. ALWAYS abort the local SSE (`stop()`) so the composer - // returns to idle immediately. In AUTONOMOUS mode the turn is a DETACHED run: - // aborting the local SSE is only a client disconnect, which the server ignores, - // so the run would keep executing — we ADDITIONALLY request the authoritative - // server-side stop (the parent owns that call + the "stopping" latch that keeps - // observer-polling from re-streaming the stopping run's output). The chat id is - // read live from chatIdRef (adopted early at the stream's `start` chunk); if it - // is not known yet — a brand-new chat in the first moment of its first turn — - // only the local abort happens (there is no server-side run handle to stop yet). - const handleStop = useCallback(() => { - // Abort the resume/attach GET first: the SDK does not pass it a signal, so an - // observer's Stop would otherwise leave the attach fetch running. - attachAbortRef.current?.abort(); - stop(); - // #430: pressing Stop also cancels an in-progress reconnect sequence. - clearReconnectTimer(); - setReconnectStatePair(null); - if (!autonomousRunsEnabled) return; - if (chatIdRef.current) { - onServerStop?.(chatIdRef.current); - } else { - // #234 F5: no chat id yet (brand-new chat in the first moment of its first - // turn, before the `start` chunk adopted the id). Latch the stop as pending; - // the onServerChatId adoption effect fires the deferred server stop as soon - // as the id appears, so the detached run is still authoritatively stopped - // instead of left running by a silent local-only abort. - // - // KNOWN LIMITATION (#234 F5 review): `stop()` above has already aborted the - // local SSE reader. In the rare sub-window where Stop is pressed while still - // `submitted` (request sent, not one chunk read yet), that abort can cancel - // the reader BEFORE the `start` chunk is applied to `messages`, so the - // adoption effect never runs and this pending stop never fires. The detached - // run then keeps going for that turn. This is not a regression (the pre-fix - // behavior sent no server stop at all); closing it fully would require - // deferring the local abort until adoption, which is riskier and out of scope - // for this fix. Documented so a future change can address the abort-ordering. - stopPendingRef.current = true; - } - }, [ - stop, - autonomousRunsEnabled, - onServerStop, - clearReconnectTimer, - setReconnectStatePair, - ]); - - // Clear the stopped marker as soon as a new turn begins streaming, and drop any - // stale "Send now" interrupt flags. On the legit interrupt path both refs are - // already consumed synchronously (onFinish + prepareSendMessagesRequest) before - // this effect runs, so clearing here is a no-op for it; its purpose is to defuse - // the race where a flag was armed but the expected abort never fired (the turn - // finished in the same tick as the click), so it cannot leak into a later turn. + // Clear the stopped marker as soon as a new turn begins streaming. useEffect(() => { - if (isStreaming) { - setStopNotice(null); - flushOnAbortRef.current = false; - interruptNextSendRef.current = false; - // #396: symmetric with the other one-shot interrupt flags — defuse a stale - // supersede arm that was set but whose expected re-POST never fired (the - // turn finished in the same tick as the click, or the promoted head was - // gone), so it can never leak into this (or a later) turn's send and retry - // that send's genuine 409. A legit arm is consumed by the transport POST - // branch before this new turn streams, so this does not clobber it. - supersedeRetryRef.current = false; - // #234 F5: a new turn is starting — drop any pending deferred-stop from a - // previous turn that never adopted an id, so it can never fire against this - // (or a later) unrelated turn's run. A deferred stop for the CURRENT turn is - // set AFTER this effect (on the Stop click), so this does not clobber it. - stopPendingRef.current = false; - } + if (isStreaming) setStopNotice(null); }, [isStreaming]); - // Classify the turn error into a heading + detail so the banner names the cause - // (connection reset, timeout, rate limit, context overflow, quota, ...) instead - // of a generic "Something went wrong". Computed here (not only in the JSX) so - // the SAME on-screen banner text can be mirrored into the export (issue #160). + // "Send now" on a queued message: #488 commit 5 — interrupt the active run and + // send THIS message via the server CAS supersede (POST /stream {supersede:{runId}}) + // when the run id is known. No local stop/re-POST dance, no client retry ladder. + const sendNow = useCallback( + (id: string) => { + const msg = queuedRef.current.find((m) => m.id === id); + if (!msg) return; + const liveStreaming = + statusRef.current === "submitted" || statusRef.current === "streaming"; + const runId = machineRef.current.ctx.runFact?.runId; + if ( + liveStreaming && + autonomousRunsEnabled === true && + runId && + runId !== "pending" + ) { + // CAS supersede: the server stops the old run + starts this one atomically. + setQueue(removeQueuedById(queuedRef.current, id)); + dispatch({ type: "SUPERSEDE_REQUESTED", targetRunId: runId }); + sendMessageRef.current?.({ text: msg.text }); // POST carries supersede body + return; + } + if (liveStreaming) { + // No CAS possible (legacy without a run, or the runId not adopted yet): + // promote to head and abort; the local turn's abort finishes and the queue + // holds the promoted head for the user (a blind re-POST would 409 -> the + // classified "already answering — interrupt and send" banner guides them). + setQueue(promoteToHead(queuedRef.current, id)); + stopFnRef.current?.(); + return; + } + // Nothing to interrupt: send it now. + setQueue(removeQueuedById(queuedRef.current, id)); + localSend(msg.text); + }, + [setQueue, autonomousRunsEnabled, dispatch, localSend], + ); + + // Stop the current turn. Abort the local SSE + the attach GET; in AUTONOMOUS mode + // additionally request the AUTHORITATIVE server stop (a local abort is only a + // client disconnect the server ignores). The FSM `stopping` exits by DATA (I4): + // the local turn's onFinish (FINISH_ABORT) or the poll reaching terminal. + const handleStop = useCallback(() => { + stopFnRef.current?.(); + if (autonomousRunsEnabled) { + dispatch({ type: "STOP_REQUESTED" }); + } else { + // Legacy: no server run to stop from the client (the server's onClose issues + // requestStop on disconnect). Just reset the FSM recovery. + attachAbortRef.current?.abort(); + if (reconnectTimerRef.current) clearTimeout(reconnectTimerRef.current); + dispatch({ type: "FINISH_ABORT" }); + } + }, [autonomousRunsEnabled, dispatch]); + + // Manual Retry from the reconnect-failed OR stalled banner. + const onRetry = useCallback(() => { + dispatch({ type: "RETRY" }); + }, [dispatch]); + + // Classify the turn error into a heading + detail (connection reset, timeout, the + // #487 409 codes, ...). Computed here so the SAME text can mirror into the export. const errorView = error ? describeChatError(error.message ?? "", t) : null; - // A role was picked with autoStart=false: the role is bound but NOTHING was - // sent, so chatId stays null and the empty state would keep showing the cards. - // This flag hides the cards and reveals the composer (with the role indicated) - // so the user can type the first message themselves. roleIdRef is already set, - // so that first manual message carries the roleId. + // Role-picker empty state (unchanged from #149). const [rolePickedNoSend, setRolePickedNoSend] = useState(false); - - // Clicking a role card always binds the role to THIS new chat. Whether it also - // auto-starts the conversation is per-role (autoStart). roleIdRef is set - // synchronously here because the parent's selectedRoleId state update would - // only reach roleIdRef on the next render — after this synchronous sendMessage - // has already read it. const handleRolePick = (role: IAiRole): void => { roleIdRef.current = role.id; onRolePicked?.(role); @@ -1127,20 +896,11 @@ export default function ChatThread({ t("Take a look at the current document"), ); if (launch !== null) { - sendMessage({ text: launch }); + localSend(launch); } else { - // autoStart=false -> bind only: hide the cards, show the composer. setRolePickedNoSend(true); } }; - // Reset the "picked, not sent" flag when the thread returns to a truly empty, - // role-less state — e.g. the user hit "New chat" after picking an autoStart=false - // role. That path clears the parent's selectedRoleId (roleId -> null) but leaves - // chatId null, so the thread never remounts and the flag would stay set, hiding - // the cards forever. A picked-and-bound role keeps roleId non-null, so the cards - // correctly stay hidden then. Render-phase reset (React "adjust state on prop - // change"): one-shot — it re-renders with the flag false and the guard no longer - // matches, so it cannot loop. (Review of #149.) if (shouldResetRolePicked(chatId, roleId, rolePickedNoSend)) { setRolePickedNoSend(false); } @@ -1160,29 +920,19 @@ export default function ChatThread({ /> {errorView ? ( - - ) : reconnectState ? ( - // #430: while auto-reconnecting to a detached run's live tail, show progress - // instead of a dead "Lost connection" banner; once attempts are exhausted, - // offer a manual Retry (the degraded poll keeps catching up underneath). - + + ) : phase.name === "reconnecting" ? ( + // #430/#488: while auto-reconnecting to a detached run's live tail, show + // progress; once attempts are exhausted, offer a manual Retry (the degraded + // poll keeps catching up underneath). + - {reconnectState.phase === "trying" ? ( + {!phase.failed ? ( <> {t("Connection lost — reconnecting…")} - {` (${reconnectState.attempt}/${RECONNECT_MAX_ATTEMPTS})`} + {` (${phase.attempt}/${RECONNECT_MAX_ATTEMPTS})`} ) : ( @@ -1194,7 +944,7 @@ export default function ChatThread({ size="compact-xs" variant="light" color="gray" - onClick={retryReconnect} + onClick={onRetry} > {t("Retry")} @@ -1202,6 +952,24 @@ export default function ChatThread({ )} + ) : phase.name === "stalled" ? ( + // #488 commit 4a: the degraded poll hit the inactivity cap — instead of a + // silent "forever half-done answer", tell the user and offer Retry. + + + + {t("The answer may be incomplete — the run stopped responding.")} + + + + ) : stopNotice ? ( {m.text} - {/* "Send now" (interrupt) is hidden on a RESUMED turn: a local - stop() does not abort the resumed attach fetch, so the click - would be swallowed while flushOnAbortRef would fire minutes - later on the natural finish. Only the remove affordance stays. */} - {!resumedTurn && ( + {/* "Send now" (interrupt) is hidden while OBSERVING a detached run + (ownership observer): a local stop() does not abort the attach + fetch, so the interrupt would be swallowed. Only the remove + affordance stays. */} + {!isObserver && ( )} { - // Local send: clear any resume-suppression flag (invariant 8). - setResumedTurnPair(false); - sendMessage({ text }); - }} + onSend={(text) => localSend(text)} onQueue={enqueue} onStop={handleStop} isStreaming={isStreaming} diff --git a/apps/client/src/features/ai-chat/state/run-fsm.spec.md b/apps/client/src/features/ai-chat/state/run-fsm.spec.md index d7c7557c..f6a84e01 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.spec.md +++ b/apps/client/src/features/ai-chat/state/run-fsm.spec.md @@ -18,7 +18,10 @@ the observable property) — see `run-fsm.test.ts`. Phases: `idle | sending | streaming | attaching | reconnecting(attempt,failed) | polling(reason) | stalled | stopping | superseding | error(kind)`. -Context (orthogonal): `epoch`, `ownership: local|observer`, `runFact: {runId}|null`. +Context (orthogonal): `epoch`, `ownership: local|observer`, `runFact: {runId}|null`, +`liveFollow` (are we following a live run we locally streamed — the reconnect +ladder — vs a one-shot mount-attach resume? both are `observer`, but a live-follow +drop RE-ENTERS the ladder (#488 commit 3) while a mount-resume drop polls). Legend: **†** = command-transition (bumps `epoch`, I1). Effects in `[…]`. @@ -28,8 +31,10 @@ Legend: **†** = command-transition (bumps `epoch`, I1). Effects in `[…]`. | `STREAM_START{runId}` (SDK `start` metadata) | sending, attaching, reconnecting, superseding | streaming | `[cancelReconnect, disarmPoll]`, runFact←runId | | `FINISH_CLEAN` (onFinish clean) | streaming, … | idle | `[disarmPoll, cancelReconnect]`, runFact←null | | `FINISH_ABORT` (onFinish isAbort) | streaming, stopping | idle | `[disarmPoll, cancelReconnect]`, runFact←null (I4 exits stopping by this DATA) | -| `FINISH_DISCONNECT{hasVisibleContent}` (onFinish isDisconnect) | streaming | reconnecting(1) **†** *iff runFact* | `[scheduleReconnect(1)]` (+`armPoll(disconnect-visible)` if visible), ownership=observer | -| `FINISH_DISCONNECT` (no runFact) | streaming | idle | runFact←null (plain terminal "connection lost") | +| `FINISH_DISCONNECT` (observer, NOT liveFollow) | streaming(observer) | polling(disconnect-visible) | `[armPoll]` (a mount-resume drop polls) | +| `FINISH_DISCONNECT{hasVisibleContent}` (local drop OR liveFollow) | streaming | reconnecting(1) **†** *iff runFact\|liveFollow* | `[scheduleReconnect(1)]` (+`armPoll` if visible), ownership=observer, liveFollow=true (commit 3: repeatable) | +| `FINISH_DISCONNECT` (no runFact, not liveFollow) | streaming | idle | runFact←null (plain terminal "connection lost") | +| `STREAM_INCOMPLETE{reason}` (observer starved/torn clean finish) | streaming(observer) | polling(reason) | `[armPoll(reason)]` | | `FINISH_ERROR{kind}` (onFinish isError) | any | error(kind) | `[disarmPoll, cancelReconnect]`, runFact←null | | `ATTACH_START{runId}` (mount resume) | idle | attaching **†** | `[resumeStream]`, ownership=observer, runFact←runId | | `ATTACH_LIVE` (attach GET 2xx) | attaching | streaming | — | @@ -74,44 +79,40 @@ disconnects) carry no epoch. --- -## 2. Ref-map — every `chat-thread.tsx` ref → its new home +## 2. Ref-map — every `chat-thread.tsx` ref → its new home (MIGRATION RESOLVED) -(develop@363f20ab counted "26"; the branch already collapsed one during #486/#487, -so 25 refs are present today. Each is classified below; the run-lifecycle FLAGS -move into the FSM, the identity/data mirrors STAY as data — the post-merge rule -forbids only new **lifecycle-flag** refs.) +The migration is COMPLETE: the 13 run-lifecycle FLAGS below are GONE from +`chat-thread.tsx` (collapsed into FSM phase/ctx/effects, or deleted). What remains +are identity/data mirrors, effect-owned controllers/timers, and ONE React-liveness +bit — none of which is a run-lifecycle flag, so the post-merge "no new flags" rule +holds. **Pending column: empty.** -| # | Ref | Classification | Notes | +| # | Old ref | Resolved to | Where now | |---|---|---|---| -| 1 | `reconcileTailRef` | **FSM** (polling phase) | "poll is driving the tail" is `phase=polling` | -| 2 | `noStreamHandledRef` | **FSM ctx (epoch)** | one-shot 204 guard → epoch drops the stale second outcome | -| 3 | `onNoActiveStreamRef` | **FSM effect** | the transport dispatches `ATTACH_NONE`; no ref-callback | -| 4 | `onReconnectAttachedRef` | **FSM effect** | transport dispatches `RECONNECT_ATTACHED` / `ATTACH_LIVE` | -| 5 | `resumedTurnRef` | **FSM ctx (ownership)** | `ownership==='observer'` ⇒ resumed turn ⇒ never flush | -| 6 | `reconnectStateRef` | **FSM** (reconnecting phase) | `{trying,attempt}`/`{failed}` = `reconnecting(attempt,failed)` | -| 7 | `reconnectTimerRef` | **FSM effect** | `scheduleReconnect`/`cancelReconnect` own the timer | -| 8 | `flushOnAbortRef` | **FSM** (superseding/queue) | flush-on-abort is the superseding→READY / interrupt transition | -| 9 | `interruptNextSendRef` | **FSM ctx** (interrupt tag) | tag carried by the supersede/interrupt transition, one-shot via epoch | -| 10 | `supersedeRetryRef` | **REMOVED** (commit 5) | the client 409 retry ladder is deleted; CAS supersede replaces it | -| 11 | `stopPendingRef` | **FSM** (stopping deferral) | deferred stop is a `STOP_REQUESTED` pended on run-fact adoption | -| 12 | `mountedRef` | **FSM ctx (epoch)** + `DISPOSE` | unmount → `DISPOSE` bumps epoch; late callbacks dropped by I1 | -| 13 | `attemptResumeRef` | **FSM** (ATTACH_START decision) | armed ONLY on a server-confirmed run-fact (commit 4b) | -| 14 | `stripRef` | **data** (attachStrategy) | strip+replay strategy detail; effect-owned | -| 15 | `strippedRowRef` | **data** (attachStrategy) | the anchor row; effect-owned, aborts in cleanup | -| 16 | `attachAbortRef` | **FSM effect** (`abortAttach`) | controller owned by the attach effect, aborted in cleanup (I5) | -| 17 | `chatIdRef` | **data** (identity mirror) | stays; live chat id for the transport body | -| 18 | `openPageRef` | **data** | stays; live open-page for the send body | -| 19 | `getEditorSelectionRef` | **data** | stays; live selection snapshotter | -| 20 | `roleIdRef` | **data** | stays; live role id for the first send | -| 21 | `stableIdRef` | **data** | stays; the useChat store key (mount-stable) | -| 22 | `queuedRef` | **data** (queue) | stays; the queue is a data structure (decision #8) | -| 23 | `sendMessageRef` | **data** | stays; latest `sendMessage` for the flush | -| 24 | `statusRef` | **data** | stays; live SDK status mirror | -| 25 | `lastForwardedChatIdRef` | **data** (one-shot forward) | stays; dedupes the chat-id forward | +| 1 | `reconcileTailRef` | **FSM phase** | reconcile-merge gated on `phase ∈ {polling, reconnecting, stopping}` | +| 2 | `noStreamHandledRef` | **FSM epoch (I1)** | the attach outcome's epoch guard drops the stale/second outcome | +| 3 | `onNoActiveStreamRef` | **FSM event** | transport → `handleAttachOutcome` dispatches `ATTACH_NONE`/`RECONNECT_NONE` | +| 4 | `onReconnectAttachedRef` | **FSM event** | transport dispatches `ATTACH_LIVE` / `RECONNECT_ATTACHED` | +| 5 | `resumedTurnRef` + `resumedTurn` state | **FSM ctx `ownership`** | `ownership==='observer'` ⇒ never flush; hides "Send now" | +| 6 | `reconnectStateRef` + `reconnectState` state | **FSM phase** | `reconnecting(attempt,failed)` renders the banner | +| 7 | `reconnectTimerRef` | **effect-owned timer** | owned by `scheduleReconnect`/`cancelReconnect` effects (not a flag) | +| 8 | `flushOnAbortRef` | **DELETED** | the stop→flush dance is replaced by the CAS supersede (commit 5) | +| 9 | `interruptNextSendRef` | **DELETED** | the server injects the interrupt note from the supersede itself | +| 10 | `supersedeRetryRef` | **DELETED** (commit 5) | the client 409 retry ladder is gone; CAS supersede replaces it | +| 11 | `stopPendingRef` | **FSM phase `stopping`** | the deferred stop fires from the chat-id adoption effect while `stopping` | +| 12 | `mountedRef` | **retained (React liveness)** | orthogonal to run-lifecycle; gates imperative onFinish side-effects post-unmount. Epoch (I1) handles stale COMMAND-outcomes; DISPOSE bumps it | +| 13 | `attemptResumeRef` | **FSM `ATTACH_START` + run-fact** | mount arms attach ONLY on a confirmed active run (commit 4b: streaming-tail status, or POST /run for a user tail) | +| 14 | `stripRef` | **data** (attachStrategy) | strip+replay detail; the `resumeStream` effect reads it | +| 15 | `strippedRowRef` | **data** (attachStrategy) | the anchor row | +| 16 | `attachAbortRef` | **effect-owned controller** | aborted by the `abortAttach` effect in cleanup (I5) | +| 17–25 | `chatIdRef`, `openPageRef`, `getEditorSelectionRef`, `roleIdRef`, `stableIdRef`, `queuedRef`, `sendMessageRef`, `statusRef`, `lastForwardedChatIdRef` | **data** (identity/send mirrors) | unchanged — not lifecycle flags | +| NEW | `pendingSupersedeRef` | **data** (send-plumbing) | the runId injected into the next `POST /stream {supersede}`; the single replacement for the 3 DELETED one-shots (#8/#9/#10) — net −2 refs | +| NEW | `idleCapTimerRef` | **effect-owned timer** | the stalled inactivity cap → `POLL_IDLE_CAP` (commit 4a); not a flag | -Run-lifecycle FLAGS eliminated by the FSM: #1–#13 (10 collapse into phase/ctx/effects; -#10 is deleted). Identity/data mirrors (#14–#25 minus the two attach-strategy/effect -items) intentionally stay — they are not lifecycle flags. +Net: the 13 lifecycle flags (#1–#13) are eliminated (7 → FSM phase/ctx/epoch/event, +3 deleted, `reconnectTimerRef`/`attachAbortRef` become effect-owned controllers, +`mountedRef` retained as React liveness). Two effect-owned timers + one send-plumbing +data ref are added — none is a boolean lifecycle latch. --- diff --git a/apps/client/src/features/ai-chat/state/run-fsm.test.ts b/apps/client/src/features/ai-chat/state/run-fsm.test.ts index e297095b..62833afe 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.test.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.test.ts @@ -14,7 +14,10 @@ function run(m: Machine, ...events: Event[]): Machine { return events.reduce(reduce, m); } function withRunFact(runId = "run-1"): Machine { - return { ...initialMachine(), ctx: { epoch: 0, ownership: "local", runFact: { runId } } }; + return { + ...initialMachine(), + ctx: { epoch: 0, ownership: "local", runFact: { runId }, liveFollow: false }, + }; } function effectTypes(m: Machine): string[] { return m.effects.map((e) => e.type); @@ -140,6 +143,38 @@ describe("run-fsm — commit 3: repeated reconnect cycles", () => { expect(hasEffect(m, "scheduleReconnect")).toBe(true); }); + it("a MOUNT-attach observer drop falls to POLL, not the reconnect ladder", () => { + // Distinguishes commit 3 from a one-shot resume: an observer that never + // live-followed (liveFollow false) polls on a drop. + let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); + m = reduce(m, { type: "ATTACH_LIVE", epoch: m.ctx.epoch }); + expect(m.ctx.ownership).toBe("observer"); + expect(m.ctx.liveFollow).toBe(false); + m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: true, epoch: m.ctx.epoch }); + expect(m.phase.name).toBe("polling"); + expect(hasEffect(m, "armPoll")).toBe(true); + }); + + it("STREAM_INCOMPLETE (observer starved/torn finish) → polling", () => { + let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); + m = reduce(m, { type: "ATTACH_LIVE", epoch: m.ctx.epoch }); + m = reduce(m, { type: "STREAM_INCOMPLETE", reason: "starved", epoch: m.ctx.epoch }); + expect(m.phase).toEqual({ name: "polling", reason: "starved" }); + expect(hasEffect(m, "armPoll")).toBe(true); + }); + + it("liveFollow is set on the first local drop and kept across a re-attach", () => { + let m = withRunFact("run-3"); + m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch }); + expect(m.ctx.liveFollow).toBe(true); + m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: 1, epoch: m.ctx.epoch }); + m = reduce(m, { type: "RECONNECT_ATTACHED", epoch: m.ctx.epoch }); + expect(m.ctx.liveFollow).toBe(true); // kept — so a second drop reconnects + // A clean finish clears it. + m = reduce(m, { type: "FINISH_CLEAN", epoch: m.ctx.epoch }); + expect(m.ctx.liveFollow).toBe(false); + }); + it("RECONNECT_NONE backs off through the ladder, then fails at the cap", () => { let m = withRunFact("run-3"); m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: false, epoch: m.ctx.epoch }); @@ -276,8 +311,8 @@ describe("run-fsm — commit 5: supersede CAS + error classification", () => { }); it("RUN_SUPERSEDED (observer's run killed) → attaching + postRun(observer-follow)", () => { - const observer = { ...withRunFact("run-dead"), ctx: { epoch: 0, ownership: "observer" as const, runFact: { runId: "run-dead" } } }; - const streaming: Machine = { phase: { name: "streaming" }, ctx: observer.ctx, effects: [] }; + const ctx = { epoch: 0, ownership: "observer" as const, runFact: { runId: "run-dead" }, liveFollow: false }; + const streaming: Machine = { phase: { name: "streaming" }, ctx, effects: [] }; const m = reduce(streaming, { type: "RUN_SUPERSEDED" }); expect(m.phase.name).toBe("attaching"); expect(m.effects.find((e) => e.type === "postRun")).toEqual({ diff --git a/apps/client/src/features/ai-chat/state/run-fsm.ts b/apps/client/src/features/ai-chat/state/run-fsm.ts index 89348384..a9ff9cca 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.ts @@ -87,6 +87,16 @@ export interface Ctx { ownership: Ownership; /** I3: the server-confirmed active run. */ runFact: RunFact; + /** + * Are we FOLLOWING a live run we were locally streaming (the reconnect ladder), + * as opposed to a one-shot mount-attach resume? Both are `ownership: 'observer'`, + * but they recover DIFFERENTLY on a drop: a live-follow drop RE-ENTERS the + * reconnect ladder (#488 commit 3 — the second break after a successful re-attach + * must reconnect again, not fall to silent poll), while a mount-resume drop falls + * to the degraded poll. This is the ctx bit that separates the two WITHOUT a new + * component ref (it is why commit 3 needs the FSM, not a surgical patch). + */ + liveFollow: boolean; } export interface Machine { @@ -131,6 +141,10 @@ export type Event = // -- local turn -- | { type: "SEND_LOCAL" } | { type: "STREAM_START"; runId?: string; epoch?: number } + /** An OBSERVER's attached stream ended WITHOUT reaching terminal (a starved + * clean replay, or a torn resume) — fall to the degraded poll to drive the row + * to its real terminal state. (A live-follow drop uses FINISH_DISCONNECT.) */ + | { type: "STREAM_INCOMPLETE"; reason: PollReason; epoch?: number } | { type: "FINISH_CLEAN"; epoch?: number } | { type: "FINISH_ABORT"; epoch?: number } | { type: "FINISH_DISCONNECT"; hasVisibleContent: boolean; epoch?: number } @@ -182,7 +196,7 @@ export function reconnectDelayMs(attempt: number): number { export function initialMachine(overrides?: Partial): Machine { return { phase: { name: "idle" }, - ctx: { epoch: 0, ownership: "local", runFact: null, ...overrides }, + ctx: { epoch: 0, ownership: "local", runFact: null, liveFollow: false, ...overrides }, effects: [], }; } @@ -241,9 +255,16 @@ export function reduce(m: Machine, event: Event): Machine { m, { name: "sending" }, [{ type: "cancelReconnect" }, { type: "disarmPoll" }], - { ownership: "local" }, + { ownership: "local", liveFollow: false }, ); + case "STREAM_INCOMPLETE": + // An OBSERVER's attached stream ended incomplete (starved / torn) — follow + // the run to terminal via the degraded poll. + return to(m, { name: "polling", reason: event.reason }, { + effects: [{ type: "armPoll", reason: event.reason }], + }); + case "STREAM_START": { // First frame arrived. Adopt the run-fact runId if present. sending -> // streaming; a reconnect/attach that just went live also lands here. @@ -259,7 +280,7 @@ export function reduce(m: Machine, event: Event): Machine { // idle. (The queue flush is a component concern gated by ownership; the // FSM only models the phase.) return to(m, { name: "idle" }, { - ctx: { runFact: null }, + ctx: { runFact: null, liveFollow: false }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); @@ -267,34 +288,44 @@ export function reduce(m: Machine, event: Event): Machine { // A user Stop / intentional abort finished. If we were stopping, the // terminal data has now arrived (I4) — go idle. The run-fact is cleared. return to(m, { name: "idle" }, { - ctx: { runFact: null }, + ctx: { runFact: null, liveFollow: false }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); case "FINISH_DISCONNECT": - // A LIVE SSE drop. If a run is (or may be) active, recover: - // - visible content already on screen -> keep it, poll to terminal - // (a full replay could clobber the fuller live tail), AND begin the - // live re-attach ladder; + // A LIVE SSE drop. Recovery depends on WHO we are (I2 + liveFollow): + // - a mount-attach OBSERVER (a one-shot resume, NOT live-follow) that drops + // -> the degraded poll drives the row to terminal from the DB. + if (m.ctx.ownership === "observer" && !m.ctx.liveFollow) { + return to(m, { name: "polling", reason: "disconnect-visible" }, { + effects: [{ type: "armPoll", reason: "disconnect-visible" }], + }); + } + // - a LOCAL live turn (first drop) OR a live-follow re-attach (a SUBSEQUENT + // drop) -> (re-)enter the reconnect ladder. #488 commit 3: allowed + // REPEATEDLY — `liveFollow` is kept across a successful re-attach, so the + // second break reconnects again instead of falling to silent poll. + // #488 commit 2: gated on the RUN-FACT (or an existing live-follow), NOT on + // the presence of an assistant message — a setup-phase break still recovers. + // - visible content already on screen -> keep it, ALSO poll to terminal + // (a full replay could clobber the fuller live tail); // - no visible content -> the reconnect ladder rebuilds it. - // #488 commit 2: recovery is gated on the RUN-FACT, not on the presence - // of an assistant message — a break during the setup phase (before the - // first assistant frame) still has an active detached run to pick up. - if (m.ctx.runFact) { + if (m.ctx.runFact || m.ctx.liveFollow) { const effects: Effect[] = [ { type: "scheduleReconnect", attempt: 1, delayMs: reconnectDelayMs(1) }, ]; if (event.hasVisibleContent) effects.push({ type: "armPoll", reason: "disconnect-visible" }); return command(m, { name: "reconnecting", attempt: 1, failed: false }, effects, { ownership: "observer", + liveFollow: true, }); } // No run to recover: a plain disconnect. Surface the terminal notice. - return to(m, { name: "idle" }, { ctx: { runFact: null } }); + return to(m, { name: "idle" }, { ctx: { runFact: null, liveFollow: false } }); case "FINISH_ERROR": return to(m, { name: "error", kind: event.kind }, { - ctx: { runFact: null }, + ctx: { runFact: null, liveFollow: false }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); @@ -399,7 +430,7 @@ export function reduce(m: Machine, event: Event): Machine { // idle and disarm everything (I4: this is a DATA-driven exit, incl. exit // from `stopping`). return to(m, { name: "idle" }, { - ctx: { runFact: null }, + ctx: { runFact: null, liveFollow: false }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); @@ -424,7 +455,7 @@ export function reduce(m: Machine, event: Event): Machine { m.phase.name === "stopping" ) { return to(m, { name: "idle" }, { - ctx: { runFact: null }, + ctx: { runFact: null, liveFollow: false }, effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }], }); } @@ -438,12 +469,20 @@ export function reduce(m: Machine, event: Event): Machine { // ---- stop ---------------------------------------------------------- case "STOP_REQUESTED": // Authoritative stop of a detached run. Enter `stopping` and fire stopRun + - // abort the local/attach reader. Exit is by DATA (I4), never by the HTTP - // response — so no transition keys off stopRun's return. + // abort the local/attach reader. ALSO arm the poll so the terminal row is + // observed — the exit is by DATA (I4: a terminal row / negative run-fact), + // never by the stopRun HTTP response (which returns after abort, before + // finalization). For a local turn the onFinish FINISH_ABORT exits first and + // disarms; for an observer the poll drives to the aborted terminal. return command( m, { name: "stopping" }, - [{ type: "stopRun" }, { type: "abortAttach" }, { type: "cancelReconnect" }], + [ + { type: "stopRun" }, + { type: "abortAttach" }, + { type: "cancelReconnect" }, + { type: "armPoll", reason: "attach-none" }, + ], ); // ---- supersede (CAS) ---------------------------------------------- @@ -460,7 +499,9 @@ export function reduce(m: Machine, event: Event): Machine { // CAS succeeded (old run stopped/settled, slot taken, new run begun). We // are now the local streamer of the NEW run. Adopt its runId if provided. const runFact = event.runId ? { runId: event.runId } : m.ctx.runFact; - return to(m, { name: "streaming" }, { ctx: { ownership: "local", runFact } }); + return to(m, { name: "streaming" }, { + ctx: { ownership: "local", runFact, liveFollow: false }, + }); } case "SUPERSEDE_MISMATCH": @@ -497,11 +538,16 @@ export function reduce(m: Machine, event: Event): Machine { case "DISPOSE": // Unmount: abort in-flight controllers, drop timers, and bump the epoch so // NO late callback can drive this (now dead) machine (I5). - return command(m, { name: "idle" }, [ - { type: "abortAttach" }, - { type: "cancelReconnect" }, - { type: "disarmPoll" }, - ]); + return command( + m, + { name: "idle" }, + [ + { type: "abortAttach" }, + { type: "cancelReconnect" }, + { type: "disarmPoll" }, + ], + { liveFollow: false }, + ); default: { // Exhaustiveness guard. From 4b78e336a29af53a01001ac18dc63ea719a7451d Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:23:16 +0300 Subject: [PATCH 34/41] =?UTF-8?q?fix(client):=20=D0=B2=D0=BD=D1=83=D1=82?= =?UTF-8?q?=D1=80=D0=B5=D0=BD=D0=BD=D0=B5=D0=B5=20=D1=80=D0=B5=D0=B2=D1=8C?= =?UTF-8?q?=D1=8E=20=D0=BC=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D0=B8=20FSM?= =?UTF-8?q?=20=E2=80=94=20F1..F4=20(#488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 [correctness] — supersede: перекрытие стримов рушило свежий run. ai@6 AbstractChat.makeRequest в finally читает и обнуляет общий activeResponse, поэтому параллельные стрим-A и стрим-B корёжат друг друга, а незастемпленный onFinish мёртвого A уводил ЖИВОЙ новый run в ложный reconnect / сбрасывал runFact. Фикс: (1) FINISH_*/STREAM_INCOMPLETE штампуются per-stream generation (turnEpochRef, взводится в момент старта стрима) → фильтр I1 отбрасывает финиш чужого поколения; (2) sendNow-supersede АБОРТИТ A и стартует B только из onFinish A (микротаск, после того как finally A обнулил activeResponse) — гарантия отсутствия перекрытия. Тест на поздний isDisconnect A после SUPERSEDE_REQUESTED: машина НЕ уходит в reconnect, B отправлен. MUTATION-VERIFY: снятие epoch-штампа у FINISH_DISCONNECT → тест краснеет («Connection lost — reconnecting»). F2 [correctness] — гонка mount getRun→ATTACH_START с локальным send. Редьюсер ATTACH_START теперь игнорирует любую не-idle фазу, поэтому поздний резолв getRun не перехватывает начавшийся локальный турн в observer-attach. Тест на гонку. F3 [ghost feature] — RUN_SUPERSEDED объявлен+покрыт тестом, но НИКОГДА не диспатчился. Удалён (событие+обработчик+тест+postRun-reason observer-follow) как сознательный scope-cut: наблюдатель убитого supersede-рана и так следует за новым через деградированный поллинг (свежие строки истории, независимо от runId). F4 [hygiene] — мёртвые события. STREAM_START подключён (первый ассистент-фрейм локального турна: sending→streaming, спека↔код совпали). RECONNECT_BEGIN и POLL_ACTIVITY удалены (не диспатчились). Множество событий редьюсера = множеству диспатчащихся; редьюсер тотален. Тесты: FSM 35 + chat-thread 37 (+error 26 +adopt 16) = 114 зелёных; tsc ai-chat 0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/chat-thread.test.tsx | 80 ++++++++++++++-- .../ai-chat/components/chat-thread.tsx | 93 ++++++++++++++++--- .../features/ai-chat/state/run-fsm.spec.md | 27 ++++-- .../features/ai-chat/state/run-fsm.test.ts | 23 +++-- .../src/features/ai-chat/state/run-fsm.ts | 40 ++------ 5 files changed, 195 insertions(+), 68 deletions(-) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index 9aaaed60..137c993f 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -193,9 +193,9 @@ describe("ChatThread — send now", () => { expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); }); - it("#488 commit 5: autonomous live sendNow with a known runId POSTs a supersede body", () => { - // A streaming assistant message carrying the start-metadata runId -> the FSM - // adopts the run-fact; sendNow then interrupts via the server CAS. + // A streaming assistant message carrying the start-metadata runId -> the FSM + // adopts the run-fact so sendNow can interrupt via the server CAS. + const withRunId = () => { h.state.status = "streaming"; h.state.messages = [ { @@ -205,19 +205,58 @@ describe("ChatThread — send now", () => { metadata: { runId: "run-1" }, }, ]; + }; + + it("#488 commit 5 / F1: sendNow ABORTS stream A first, then sends B (CAS) only after A finalizes", async () => { + withRunId(); renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); fireEvent.click(screen.getByTestId("queue-btn")); fireEvent.click(screen.getByLabelText("Send now")); - // The message is sent... + // F1: A is ABORTED, and B is NOT sent yet (no overlap). + expect(h.state.stop).toHaveBeenCalledTimes(1); + expect(h.state.sendMessage).not.toHaveBeenCalled(); + // A's onFinish fires -> B is scheduled on a microtask (after A finalizes). + await act(async () => { + h.state.onFinish?.({ + message: { id: "a1", role: "assistant", parts: [] }, + isAbort: true, + isDisconnect: false, + isError: false, + }); + await Promise.resolve(); + }); expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); - // ...and the NEXT POST carries supersede:{runId} (read-and-cleared once). + // B's POST carries supersede:{runId} (read-and-cleared once). const prep = h.state.transport!.prepareSendMessagesRequest!; - const body = prep({ messages: [], body: {} }).body as Record; - expect(body.supersede).toEqual({ runId: "run-1" }); - // One-shot: a subsequent request has no supersede. + expect(prep({ messages: [], body: {} }).body.supersede).toEqual({ + runId: "run-1", + }); expect(prep({ messages: [], body: {} }).body.supersede).toBeUndefined(); }); + it("#488 F1: a superseded stream's LATE disconnect does NOT falsely reconnect (epoch stamp drops it)", async () => { + // MUTATION-VERIFY: drop `epoch: stampEpoch` from the supersede-branch + // FINISH_DISCONNECT dispatch and this goes red (A's disconnect -> reconnecting). + withRunId(); + renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); + fireEvent.click(screen.getByTestId("queue-btn")); + fireEvent.click(screen.getByLabelText("Send now")); // -> superseding, aborts A + // A ends via isDisconnect (the server CAS closed it). The OLD overlap bug would + // route the LIVE new run into a false reconnect banner. + await act(async () => { + h.state.onFinish?.({ + message: { id: "a1", role: "assistant", parts: [{ type: "text", text: "x" }] }, + isAbort: false, + isDisconnect: true, + isError: false, + }); + await Promise.resolve(); + }); + expect(screen.queryByText(/reconnecting/i)).toBeNull(); + // ...and B was still sent. + expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); + }); + it("#488 commit 5: a supersede POST that 409s SUPERSEDE_TIMEOUT is returned as-is (NO retry ladder)", async () => { h.state.status = "streaming"; h.state.messages = [ @@ -494,6 +533,31 @@ describe("ChatThread — resume (attach) machinery", () => { expect(onResumeFallback).not.toHaveBeenCalledWith(true); }); + it("#488 F2: a local send DURING the mount getRun round-trip is NOT hijacked by the late attach", async () => { + // getRun stays pending while the user sends locally; its late resolve would + // otherwise ATTACH_START and flip the local turn into an observer-attach. + let resolveGetRun!: (v: { + run: { id: string; status: string } | null; + message: unknown; + }) => void; + h.state.getRun.mockReturnValue( + new Promise((r) => { + resolveGetRun = r; + }), + ); + renderThread({ autonomousRunsEnabled: true, initialRows: userTail() }); + // Local send in flight of getRun -> SEND_LOCAL (phase sending, ownership local). + fireEvent.click(screen.getByTestId("send-btn")); + expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "typed text" }); + // getRun resolves with an ACTIVE run -> the F2 guard ignores ATTACH_START. + await act(async () => { + resolveGetRun({ run: { id: "run-1", status: "running" }, message: null }); + await Promise.resolve(); + }); + // The local turn was NOT hijacked into a resume/attach. + expect(h.state.resumeStream).not.toHaveBeenCalled(); + }); + it("strips the streaming tail from the seed, keeps a user tail whole", () => { renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); expect(h.state.seededMessages).toHaveLength(1); diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index 8a4d72cb..946e466c 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -313,6 +313,18 @@ export default function ChatThread({ // (like chatIdRef/roleIdRef), the single replacement for the three REMOVED // one-shot flags flushOnAbortRef/interruptNextSendRef/supersedeRetryRef. const pendingSupersedeRef = useRef(null); + // #488 F1: the interrupt-and-send text, held until the superseded stream A has + // FULLY finalized (its SDK `finally` cleared `activeResponse`) — only THEN is + // stream B started, so A cannot corrupt B (ai@6 AbstractChat.makeRequest reads + // `this.activeResponse.state.message` in its finally and then nulls it, so an + // overlapping B is clobbered). Send-plumbing DATA, not a lifecycle flag. + const pendingSupersedeTextRef = useRef(null); + // #488 F1: the epoch under which the CURRENTLY-OWNED stream started, used to + // STAMP its onFinish (I1). A superseded/dead stream's late finish carries an OLD + // generation and is dropped by the reducer, so it cannot drive the live machine + // into a false reconnect or reset its run-fact. Set at each honored stream start + // (local send, resume/reconnect attach, the supersede B-send). + const turnEpochRef = useRef(0); // --- Effect runner: executes the reducer's command effects --------------- const runEffect = useCallback( @@ -324,6 +336,9 @@ export default function ChatThread({ // seed already stripped it), so the live replay's text-start rebuilds it // without duplicating parts (#430). pendingAttachEpochRef.current = epoch; + // The resumed stream's onFinish is stamped with THIS attach generation + // (F1), so a superseded attempt's late finish is dropped. + turnEpochRef.current = epoch; if (machineRef.current.phase.name === "reconnecting") { const anchor = strippedRowRef.current; if (anchor) @@ -415,6 +430,8 @@ export default function ChatThread({ // FIFO dequeue + local send of the next queued message (no-op when empty). const localSend = useCallback((text: string) => { dispatchRef.current({ type: "SEND_LOCAL" }); + // F1: this local stream's onFinish is stamped with the just-bumped generation. + turnEpochRef.current = epochRef.current; sendMessageRef.current?.({ text }); }, []); const flushNext = useCallback(() => { @@ -581,6 +598,10 @@ export default function ChatThread({ transport, experimental_throttle: STREAM_THROTTLE_MS, onFinish: ({ message, isAbort, isDisconnect, isError }) => { + // #488 F1: STAMP this finish with the generation the stream STARTED under + // (I1). A superseded/dead stream's late finish carries an OLD generation and + // is dropped by the reducer, so it cannot drive the live machine. + const stampEpoch = turnEpochRef.current; // Ownership (I2) is the FSM ctx: a resumed/attached/reconnected turn is an // OBSERVER; a local send is the owner. The queue flushes ONLY under local // ownership; an observer never flushes (invariant 7). @@ -589,22 +610,50 @@ export default function ChatThread({ // for #161); fires even while unmounting. onTurnFinished(extractServerChatId(message), threadKey); + // A missing message (a pre-first-frame break) has no visible content. + const msgHasVisible = message + ? assistantMessageHasVisibleContent(message) + : false; + + // #488 F1: the SUPERSEDED stream A just finalized (we are still `superseding` + // and stream B has not been sent yet). Record A's terminal outcome STAMPED — + // I1 drops it (superseding bumped the epoch), so A cannot drive a false + // reconnect / reset the run-fact — then start B NOW that A is fully done. B is + // deferred to a microtask so A's SDK `finally` (`activeResponse = void 0`) + // runs BEFORE B's makeRequest sets `activeResponse`, else the dying A clobbers + // B (ai@6). This is the no-overlap guarantee the CAS supersede needs. + if ( + machineRef.current.phase.name === "superseding" && + pendingSupersedeTextRef.current !== null + ) { + if (isError) dispatch({ type: "FINISH_ERROR", kind: "stream", epoch: stampEpoch }); + else if (isAbort) dispatch({ type: "FINISH_ABORT", epoch: stampEpoch }); + else if (isDisconnect) + dispatch({ type: "FINISH_DISCONNECT", hasVisibleContent: msgHasVisible, epoch: stampEpoch }); + else dispatch({ type: "FINISH_CLEAN", epoch: stampEpoch }); + const text = pendingSupersedeTextRef.current; + pendingSupersedeTextRef.current = null; + setStopNotice(null); + queueMicrotask(() => { + if (!mountedRef.current) return; + turnEpochRef.current = epochRef.current; // B's (superseding) generation + sendMessageRef.current?.({ text }); + }); + return; + } + if (isError) { - dispatch({ type: "FINISH_ERROR", kind: "stream" }); + dispatch({ type: "FINISH_ERROR", kind: "stream", epoch: stampEpoch }); setStopNotice(null); return; } if (isAbort) { // A user Stop / an interrupt abort finished. The FSM stopping/idle exit is // by DATA (this terminal outcome, I4). - dispatch({ type: "FINISH_ABORT" }); + dispatch({ type: "FINISH_ABORT", epoch: stampEpoch }); setStopNotice("manual"); return; } - // A missing message (a pre-first-frame break) has no visible content. - const msgHasVisible = message - ? assistantMessageHasVisibleContent(message) - : false; if (isDisconnect) { if (wasObserver) { // A resumed/attached OBSERVER stream dropped. Recover via the degraded @@ -624,6 +673,7 @@ export default function ChatThread({ dispatch({ type: "FINISH_DISCONNECT", hasVisibleContent: hasVisible, + epoch: stampEpoch, }); } setStopNotice(null); @@ -659,10 +709,15 @@ export default function ChatThread({ dispatch({ type: "FINISH_DISCONNECT", hasVisibleContent: msgHasVisible, + epoch: stampEpoch, }); setStopNotice(null); } else { - dispatch({ type: "FINISH_DISCONNECT", hasVisibleContent: false }); + dispatch({ + type: "FINISH_DISCONNECT", + hasVisibleContent: false, + epoch: stampEpoch, + }); setStopNotice("disconnect"); } return; @@ -680,17 +735,17 @@ export default function ChatThread({ queryClient.invalidateQueries({ queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), }); - dispatch({ type: "STREAM_INCOMPLETE", reason: "starved" }); + dispatch({ type: "STREAM_INCOMPLETE", reason: "starved", epoch: stampEpoch }); } else { // Healthy resumed finish — nothing to restore/arm, just settle. - dispatch({ type: "FINISH_CLEAN" }); + dispatch({ type: "FINISH_CLEAN", epoch: stampEpoch }); } } setStopNotice(null); return; } // Local clean finish: settle + flush the queue (gated on liveness, #486). - dispatch({ type: "FINISH_CLEAN" }); + dispatch({ type: "FINISH_CLEAN", epoch: stampEpoch }); setStopNotice(null); if (mountedRef.current) flushNext(); }, @@ -730,9 +785,16 @@ export default function ChatThread({ if (machineRef.current.phase.name === "stopping") onServerStopRef.current?.(serverChatId); } + // #488 F4: the FIRST assistant frame of a LOCAL turn moves the FSM + // `sending -> streaming` (and adopts the runId), matching the spec's + // STREAM_START transition. Later frames / observer turns (already streaming) + // just refresh the run-fact. const runId = extractRunId(tail); - if (runId && machineRef.current.ctx.runFact?.runId !== runId) + if (machineRef.current.phase.name === "sending") { + dispatch({ type: "STREAM_START", runId, epoch: epochRef.current }); + } else if (runId && machineRef.current.ctx.runFact?.runId !== runId) { dispatch({ type: "RUN_FACT", runFact: { runId } }); + } }, [messages, onServerChatId, dispatch]); // Live "turn was interrupted" marker (a manual Stop vs a dropped connection — a @@ -839,9 +901,16 @@ export default function ChatThread({ runId !== "pending" ) { // CAS supersede: the server stops the old run + starts this one atomically. + // #488 F1: do NOT start stream B synchronously here — the local stream A is + // still live, and overlapping streams corrupt each other in ai@6 (A's + // `finally` reads/ nulls the shared `activeResponse`). Instead ABORT A now + // and stash the text; A's onFinish (dropped by the epoch stamp) starts B in + // a microtask, AFTER A is fully finalized (no overlap). The POST then + // carries the supersede body (pendingSupersedeRef, armed by the effect). setQueue(removeQueuedById(queuedRef.current, id)); + pendingSupersedeTextRef.current = msg.text; dispatch({ type: "SUPERSEDE_REQUESTED", targetRunId: runId }); - sendMessageRef.current?.({ text: msg.text }); // POST carries supersede body + stopFnRef.current?.(); // abort A -> its onFinish sends B return; } if (liveStreaming) { diff --git a/apps/client/src/features/ai-chat/state/run-fsm.spec.md b/apps/client/src/features/ai-chat/state/run-fsm.spec.md index f6a84e01..c4302e07 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.spec.md +++ b/apps/client/src/features/ai-chat/state/run-fsm.spec.md @@ -36,36 +36,47 @@ Legend: **†** = command-transition (bumps `epoch`, I1). Effects in `[…]`. | `FINISH_DISCONNECT` (no runFact, not liveFollow) | streaming | idle | runFact←null (plain terminal "connection lost") | | `STREAM_INCOMPLETE{reason}` (observer starved/torn clean finish) | streaming(observer) | polling(reason) | `[armPoll(reason)]` | | `FINISH_ERROR{kind}` (onFinish isError) | any | error(kind) | `[disarmPoll, cancelReconnect]`, runFact←null | -| `ATTACH_START{runId}` (mount resume) | idle | attaching **†** | `[resumeStream]`, ownership=observer, runFact←runId | +| `STREAM_START{runId}` (first assistant frame of a local turn) | sending | streaming | runFact←runId, `[cancelReconnect, disarmPoll]` | +| `ATTACH_START{runId}` (mount resume) | **idle only** (F2) | attaching **†** | `[resumeStream]`, ownership=observer, runFact←runId; ignored from any non-idle phase | | `ATTACH_LIVE` (attach GET 2xx) | attaching | streaming | — | | `ATTACH_NONE` (attach GET 204/err/throw) | attaching | polling(attach-none) | `[armPoll(attach-none)]` | -| `RECONNECT_BEGIN` | streaming, idle | reconnecting(1) **†** *iff runFact* | `[scheduleReconnect(1)]`, ownership=observer | | `RECONNECT_ATTEMPT{n}` (backoff timer) | reconnecting | reconnecting(n) **†** | `[resumeStream]` | | `RECONNECT_ATTACHED` (reconnect GET 2xx) | reconnecting | streaming | `[cancelReconnect, disarmPoll]` — **counter reset** (commit 3) | | `RECONNECT_NONE` (reconnect GET 204/err), attempt { expect(m.phase).toEqual({ name: "error", kind: "run-already-active" }); expect(m.effects).toEqual([]); }); +}); - it("RUN_SUPERSEDED (observer's run killed) → attaching + postRun(observer-follow)", () => { - const ctx = { epoch: 0, ownership: "observer" as const, runFact: { runId: "run-dead" }, liveFollow: false }; - const streaming: Machine = { phase: { name: "streaming" }, ctx, effects: [] }; - const m = reduce(streaming, { type: "RUN_SUPERSEDED" }); +// #488 F2 — a late mount `getRun → ATTACH_START` must not hijack a local turn. +describe("run-fsm — F2: ATTACH_START only from idle", () => { + it("ATTACH_START from a local `sending` turn is ignored (no observer hijack)", () => { + const sending = reduce(initialMachine(), { type: "SEND_LOCAL" }); // idle -> sending, local + const m = reduce(sending, { type: "ATTACH_START", runId: "r" }); + expect(m.phase.name).toBe("sending"); + expect(m.ctx.ownership).toBe("local"); // NOT flipped to observer + expect(m.effects).toEqual([]); // no resumeStream + }); + + it("ATTACH_START from idle attaches as normal", () => { + const m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); expect(m.phase.name).toBe("attaching"); - expect(m.effects.find((e) => e.type === "postRun")).toEqual({ - type: "postRun", - reason: "observer-follow", - }); + expect(m.ctx.ownership).toBe("observer"); + expect(hasEffect(m, "resumeStream")).toBe(true); }); }); diff --git a/apps/client/src/features/ai-chat/state/run-fsm.ts b/apps/client/src/features/ai-chat/state/run-fsm.ts index a9ff9cca..6239b450 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.ts @@ -113,7 +113,7 @@ export interface Machine { export type Effect = /** POST /run to (re)establish or verify the run-fact. `reason` is diagnostic. */ - | { type: "postRun"; reason: "mount" | "verify" | "observer-follow" } + | { type: "postRun"; reason: "mount" | "verify" } /** Trigger the SDK `resumeStream()` (attach GET via prepareReconnectToStream). */ | { type: "resumeStream" } /** Schedule a reconnect attempt after a backoff, then dispatch RECONNECT_ATTEMPT. */ @@ -153,15 +153,12 @@ export type Event = | { type: "ATTACH_START"; runId?: string } | { type: "ATTACH_LIVE"; epoch?: number } | { type: "ATTACH_NONE"; epoch?: number } - // -- reconnect after a live disconnect -- - /** #488 commit 2: entered by the RUN-FACT, not by an assistant message. */ - | { type: "RECONNECT_BEGIN" } + // -- reconnect after a live disconnect (entered by FINISH_DISCONNECT, #488 c2) -- | { type: "RECONNECT_ATTEMPT"; attempt: number; epoch?: number } | { type: "RECONNECT_ATTACHED"; epoch?: number } | { type: "RECONNECT_NONE"; epoch?: number } | { type: "RETRY" } // -- degraded poll -- - | { type: "POLL_ACTIVITY" } | { type: "POLL_TERMINAL" } | { type: "POLL_IDLE_CAP" } // -- run-fact (server-confirmed active run) -- @@ -175,10 +172,6 @@ export type Event = | { type: "SUPERSEDE_TIMEOUT"; epoch?: number } | { type: "SUPERSEDE_INVALID"; epoch?: number } | { type: "RUN_ALREADY_ACTIVE" } - /** An observer's attached run was aborted because it was superseded server-side: - * ask for the latest run and follow it (else an observer tab freezes on the - * killed run). */ - | { type: "RUN_SUPERSEDED" } // -- lifecycle -- | { type: "DISPOSE" }; @@ -332,6 +325,12 @@ export function reduce(m: Machine, event: Event): Machine { // ---- mount attach (resume) ---------------------------------------- case "ATTACH_START": // A reopened tab attaches to a still-running run: observer ownership. + // #488 F2: ONLY from idle. The mount `getRun` round-trip resolves async, and + // a local send may have started meanwhile (phase `sending`, ownership local); + // a late ATTACH_START must NOT hijack that local turn into an observer-attach + // (queue would stop flushing, "Send now" would hide). Guarding in the reducer + // covers every dispatch source. + if (m.phase.name !== "idle") return stay(m); return command(m, { name: "attaching" }, [{ type: "resumeStream" }], { ownership: "observer", runFact: event.runId ? { runId: event.runId } : m.ctx.runFact, @@ -351,17 +350,6 @@ export function reduce(m: Machine, event: Event): Machine { }); // ---- reconnect after a live disconnect ---------------------------- - case "RECONNECT_BEGIN": - // Explicit begin (used when the runtime decides to reconnect from a signal - // other than FINISH_DISCONNECT). Requires a run-fact (I3). - if (!m.ctx.runFact) return stay(m); - return command( - m, - { name: "reconnecting", attempt: 1, failed: false }, - [{ type: "scheduleReconnect", attempt: 1, delayMs: reconnectDelayMs(1) }], - { ownership: "observer" }, - ); - case "RECONNECT_ATTEMPT": // A scheduled backoff fired — fire the attach GET. epoch++ so the previous // attempt's late outcome cannot drive this one. @@ -420,11 +408,6 @@ export function reduce(m: Machine, event: Event): Machine { return stay(m); // ---- degraded poll ------------------------------------------------- - case "POLL_ACTIVITY": - // The polled rows changed (progress). No phase change — the runtime resets - // its inactivity clock. Only meaningful while a poll-bearing phase is active. - return stay(m); - case "POLL_TERMINAL": // The run reached a terminal row via the poll (or the reconcile merge). Go // idle and disarm everything (I4: this is a DATA-driven exit, incl. exit @@ -527,13 +510,6 @@ export function reduce(m: Machine, event: Event): Machine { // offers "interrupt and send" (supersede) instead. return to(m, { name: "error", kind: "run-already-active" }); - case "RUN_SUPERSEDED": - // An observer's attached run was killed by a server-side supersede. Ask for - // the latest run and follow it, instead of freezing on the dead run. - return command(m, { name: "attaching" }, [{ type: "postRun", reason: "observer-follow" }], { - ownership: "observer", - }); - // ---- lifecycle ----------------------------------------------------- case "DISPOSE": // Unmount: abort in-flight controllers, drop timers, and bump the epoch so From 1685b3233357506245ebe37088dd42f775d30196 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 07:43:58 +0300 Subject: [PATCH 35/41] =?UTF-8?q?fix(client):=20=D1=80=D0=B5-=D1=80=D0=B5?= =?UTF-8?q?=D0=B2=D1=8C=D1=8E=20F1=20=E2=80=94=20=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=B0=20=D0=B7=D0=B0=D0=B4=D0=B5=D1=82=D1=8B=D1=85=20Sto?= =?UTF-8?q?p=20=D0=B8=20supersede-=D1=82=D0=B0=D0=B9=D0=BC=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3=D0=BE=D0=B2=20(#488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MEDIUM — эпоха-штамп из F1 сломал выход из `stopping` для локального Stop. STOP_REQUESTED бампит epoch (E1→E2), а onFinish аборчиваемого стрима стампится ПРЕД-стоп-эпохой E1 (start-эпоха стрима), поэтому фильтр I1 дропал FINISH_ABORT и машина зависала в `stopping` навсегда (idle-cap покрывает только polling/ reconnecting). Фикс в редьюсере: ЛЮБОЙ finish (`FINISH_*`/`STREAM_INCOMPLETE`) в фазе `stopping` честится и выводит `stopping→idle` МИНУЯ epoch-фильтр — у обычного Stop нет преемника, финиш аборта и есть ожидаемое завершение (I4). Для `superseding` фильтр сохранён (это и есть F1-drop). Тест переписан на ПРЕД-стоп-эпоху E1 (реальная проводка); MUTATION-VERIFY: снятие honor-in-stopping → зависание → красный. LOW-1 — supersede терял B, если A уже settled, а statusRef ещё «streaming». Ливнесс в sendNow теперь берётся из ФАЗЫ FSM (machineRef, обновляется onFinish'ем СИНХРОННО), а не из отрендеренного statusRef: settled-A (фаза idle) → B шлётся немедленно, без аборта мёртвого стрима и залипания в `superseding`. statusRef удалён (больше не нужен). Тест на под-кадровое окно. LOW-2 — двойной «Send now» в окне аборта A перезаписывал pendingSupersedeTextRef. Второй клик при уже летящем supersede (pendingText взведён / фаза `superseding`) — NO-OP: сообщение остаётся в очереди, ничего не теряется/не перетирается. Тест. Тесты: FSM 36 + chat-thread 39 (+error 26 +adopt 16) = 117 зелёных; tsc ai-chat 0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/chat-thread.test.tsx | 73 ++++++++++++++----- .../ai-chat/components/chat-thread.tsx | 47 +++++++----- .../features/ai-chat/state/run-fsm.spec.md | 10 ++- .../features/ai-chat/state/run-fsm.test.ts | 22 +++++- .../src/features/ai-chat/state/run-fsm.ts | 32 +++++++- 5 files changed, 141 insertions(+), 43 deletions(-) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index 137c993f..2c5e6ce2 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -193,9 +193,11 @@ describe("ChatThread — send now", () => { expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); }); - // A streaming assistant message carrying the start-metadata runId -> the FSM - // adopts the run-fact so sendNow can interrupt via the server CAS. - const withRunId = () => { + // Drive a REAL live LOCAL stream with an active run-fact: a runId-bearing + // assistant message (mount adoption -> RUN_FACT) + a local send (SEND_LOCAL -> + // FSM `sending`, ownership local). sendNow reads LIVENESS off the FSM phase + // (LOW-1), so a faked SDK status alone is not enough — the stream must be real. + function startLocalStreamWithRun() { h.state.status = "streaming"; h.state.messages = [ { @@ -205,11 +207,17 @@ describe("ChatThread — send now", () => { metadata: { runId: "run-1" }, }, ]; - }; + const view = renderThread({ + autonomousRunsEnabled: true, + initialRows: settledTail(), + }); + fireEvent.click(screen.getByTestId("send-btn")); // SEND_LOCAL -> FSM `sending` + h.state.sendMessage.mockClear(); + return view; + } it("#488 commit 5 / F1: sendNow ABORTS stream A first, then sends B (CAS) only after A finalizes", async () => { - withRunId(); - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); + startLocalStreamWithRun(); fireEvent.click(screen.getByTestId("queue-btn")); fireEvent.click(screen.getByLabelText("Send now")); // F1: A is ABORTED, and B is NOT sent yet (no overlap). @@ -237,8 +245,7 @@ describe("ChatThread — send now", () => { it("#488 F1: a superseded stream's LATE disconnect does NOT falsely reconnect (epoch stamp drops it)", async () => { // MUTATION-VERIFY: drop `epoch: stampEpoch` from the supersede-branch // FINISH_DISCONNECT dispatch and this goes red (A's disconnect -> reconnecting). - withRunId(); - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); + startLocalStreamWithRun(); fireEvent.click(screen.getByTestId("queue-btn")); fireEvent.click(screen.getByLabelText("Send now")); // -> superseding, aborts A // A ends via isDisconnect (the server CAS closed it). The OLD overlap bug would @@ -258,16 +265,7 @@ describe("ChatThread — send now", () => { }); it("#488 commit 5: a supersede POST that 409s SUPERSEDE_TIMEOUT is returned as-is (NO retry ladder)", async () => { - h.state.status = "streaming"; - h.state.messages = [ - { - id: "a1", - role: "assistant", - parts: [{ type: "text", text: "x" }], - metadata: { runId: "run-1" }, - }, - ]; - renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); + startLocalStreamWithRun(); fireEvent.click(screen.getByTestId("queue-btn")); fireEvent.click(screen.getByLabelText("Send now")); // -> superseding, arms body // Consume the supersede body (as the SDK would) so the POST is the CAS one. @@ -291,6 +289,45 @@ describe("ChatThread — send now", () => { expect(res.status).toBe(409); }); + it("#488 LOW-1: sendNow when A has ALREADY settled (statusRef stale) sends immediately, not stuck superseding", () => { + startLocalStreamWithRun(); // FSM `sending`, mock status "streaming" + // A settles cleanly -> FSM `idle`. The mock status stays "streaming" (render- + // lagged), which the OLD statusRef gate would misread as "still live". + act(() => { + h.state.onFinish?.({ + message: { id: "a1", role: "assistant", parts: [{ type: "text", text: "done" }] }, + isAbort: false, + isDisconnect: false, + isError: false, + }); + }); + h.state.sendMessage.mockClear(); + h.state.stop.mockClear(); + fireEvent.click(screen.getByTestId("queue-btn")); + fireEvent.click(screen.getByLabelText("Send now")); + // The FSM phase (idle) is authoritative -> B is sent IMMEDIATELY, not routed + // into an aborting supersede that would strand the machine (A's second onFinish + // never comes) and lose the message. + expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" }); + expect(h.state.stop).not.toHaveBeenCalled(); + }); + + it("#488 LOW-2: a second Send now while a supersede is in flight is a no-op (first message not lost)", () => { + startLocalStreamWithRun(); + fireEvent.click(screen.getByTestId("queue-btn")); // X + fireEvent.click(screen.getByTestId("queue-btn")); // Y + expect(screen.getAllByLabelText("Send now")).toHaveLength(2); + fireEvent.click(screen.getAllByLabelText("Send now")[0]); // supersede X -> superseding + expect(h.state.stop).toHaveBeenCalledTimes(1); + // Y is still queued; a second Send now must NOT clobber X or re-abort. + const remaining = screen.getAllByLabelText("Send now"); + expect(remaining).toHaveLength(1); + fireEvent.click(remaining[0]); + expect(h.state.stop).toHaveBeenCalledTimes(1); // no second abort + // Y is still queued (kept, not lost, not sent). + expect(screen.getAllByLabelText("Remove queued message")).toHaveLength(1); + }); + it("Send now is HIDDEN while observing a resumed run and VISIBLE on a local stream", () => { // Resumed (mount attach) -> observer -> hidden. renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index 946e466c..bd0b513c 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -763,9 +763,6 @@ export default function ChatThread({ sendMessageRef.current = sendMessage; stopFnRef.current = stop; - const statusRef = useRef(status); - statusRef.current = status; - // EARLY chat-id (#174) + runId (#488) adoption from the streaming assistant // message's start metadata. Forward the chat id once; adopt the runId into the // FSM run-fact; and (#234 F5) fire a DEFERRED server stop the moment the id lands @@ -891,29 +888,43 @@ export default function ChatThread({ (id: string) => { const msg = queuedRef.current.find((m) => m.id === id); if (!msg) return; - const liveStreaming = - statusRef.current === "submitted" || statusRef.current === "streaming"; - const runId = machineRef.current.ctx.runFact?.runId; + // #488 LOW-2: a supersede is already in flight (stream A aborted, B not sent + // yet). A second "Send now" must NOT clobber the pending message nor start a + // competing send. Least-surprising choice: this click is a NO-OP and the + // message stays queued — the user can send it once B has started (nothing is + // lost). (Guarding on both the pending text and the FSM phase.) if ( - liveStreaming && - autonomousRunsEnabled === true && - runId && - runId !== "pending" + pendingSupersedeTextRef.current !== null || + machineRef.current.phase.name === "superseding" ) { + return; + } + // #488 LOW-1: decide "is stream A still live" from the FSM phase (machineRef), + // which onFinish updates SYNCHRONOUSLY — NOT the render-lagged statusRef. In + // the sub-frame window where A has already fired onFinish (FSM -> idle) but + // statusRef still reads "streaming", treating A as live would abort a dead + // stream (no second onFinish -> B never sent, message lost). The FSM phase + // closes that window: a settled A reads as not-live -> B is sent immediately. + const p = machineRef.current.phase.name; + const aLiveLocal = + (p === "sending" || p === "streaming") && + machineRef.current.ctx.ownership === "local"; + const runId = machineRef.current.ctx.runFact?.runId; + if (aLiveLocal && autonomousRunsEnabled === true && runId && runId !== "pending") { // CAS supersede: the server stops the old run + starts this one atomically. - // #488 F1: do NOT start stream B synchronously here — the local stream A is - // still live, and overlapping streams corrupt each other in ai@6 (A's - // `finally` reads/ nulls the shared `activeResponse`). Instead ABORT A now - // and stash the text; A's onFinish (dropped by the epoch stamp) starts B in - // a microtask, AFTER A is fully finalized (no overlap). The POST then - // carries the supersede body (pendingSupersedeRef, armed by the effect). + // #488 F1: do NOT start stream B synchronously — the local stream A is still + // live, and overlapping streams corrupt each other in ai@6 (A's `finally` + // reads then nulls the shared `activeResponse`). ABORT A now and stash the + // text; A's onFinish (dropped by the epoch stamp) starts B in a microtask, + // AFTER A is fully finalized (no overlap). The POST then carries the + // supersede body (pendingSupersedeRef, armed by the effect). setQueue(removeQueuedById(queuedRef.current, id)); pendingSupersedeTextRef.current = msg.text; dispatch({ type: "SUPERSEDE_REQUESTED", targetRunId: runId }); stopFnRef.current?.(); // abort A -> its onFinish sends B return; } - if (liveStreaming) { + if (aLiveLocal) { // No CAS possible (legacy without a run, or the runId not adopted yet): // promote to head and abort; the local turn's abort finishes and the queue // holds the promoted head for the user (a blind re-POST would 409 -> the @@ -922,7 +933,7 @@ export default function ChatThread({ stopFnRef.current?.(); return; } - // Nothing to interrupt: send it now. + // Nothing live to interrupt (idle, or A already settled): send it now. setQueue(removeQueuedById(queuedRef.current, id)); localSend(msg.text); }, diff --git a/apps/client/src/features/ai-chat/state/run-fsm.spec.md b/apps/client/src/features/ai-chat/state/run-fsm.spec.md index c4302e07..15c504f5 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.spec.md +++ b/apps/client/src/features/ai-chat/state/run-fsm.spec.md @@ -59,7 +59,15 @@ Legend: **†** = command-transition (bumps `epoch`, I1). Effects in `[…]`. | `RUN_ALREADY_ACTIVE` (409 A_RUN_ALREADY_ACTIVE, plain POST) | sending | error(run-already-active) | — (composer offers supersede; NO auto-retry) | | `DISPOSE` (unmount) | any | idle **†** | `[abortAttach, cancelReconnect, disarmPoll]` (I1/I5 — epoch++ kills late callbacks) | -**Epoch filter (I1):** the reducer FIRST drops any event carrying an `epoch` that +**`stopping` honors any finish (re-review MEDIUM):** BEFORE the epoch filter, a +stream finish (`FINISH_*`/`STREAM_INCOMPLETE`) arriving in phase `stopping` exits +`stopping -> idle` regardless of generation. A plain Stop has no successor stream, +so the aborted stream's finish IS the expected end (I4 exit by data) — and it +carries the PRE-stop generation (STOP_REQUESTED bumped the epoch), so the filter +would otherwise strand the machine in `stopping` (no idle-cap covers it). The filter +stays in force for `superseding` (that is the F1 supersede drop). + +**Epoch filter (I1):** the reducer then drops any event carrying an `epoch` that does not equal the current `ctx.epoch`. Outcome events (`STREAM_START`, `ATTACH_*`, `RECONNECT_*`, `SUPERSEDE_*`, **`FINISH_*`/`STREAM_INCOMPLETE`**, `RUN_FACT`) are stamped with the generation the corresponding STREAM started under (the runtime diff --git a/apps/client/src/features/ai-chat/state/run-fsm.test.ts b/apps/client/src/features/ai-chat/state/run-fsm.test.ts index c924bf9f..9155229f 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.test.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.test.ts @@ -336,14 +336,28 @@ describe("run-fsm — stop (I4: exit by data)", () => { expect(effectTypes(m)).toEqual(expect.arrayContaining(["stopRun", "abortAttach"])); }); - it("stopping exits on a terminal FINISH_ABORT (data), not on the stopRun response", () => { - let m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); - // No transition keys off the HTTP result — only the terminal finish moves us. - m = reduce(m, { type: "FINISH_ABORT", epoch: m.ctx.epoch }); + it("stopping exits on the aborted stream's finish carrying the PRE-STOP epoch", () => { + // MEDIUM (#488 re-review): STOP_REQUESTED is a command that BUMPS the epoch, but + // the runtime stamps the aborted stream's onFinish with the stream's START (pre- + // stop) generation — exactly what the component sends. `stopping` must HONOR + // that finish regardless of generation (no idle-cap covers `stopping`). + // MUTATION-VERIFY: remove the honor-in-`stopping` branch and this hangs in + // `stopping` (the epoch filter drops the pre-stop finish) -> red. + const preStopEpoch = withRunFact().ctx.epoch; // E1 (the stream's start epoch) + let m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); // E1 -> E2, stopping + expect(m.ctx.epoch).toBe(preStopEpoch + 1); + m = reduce(m, { type: "FINISH_ABORT", epoch: preStopEpoch }); // NOT the current epoch expect(m.phase.name).toBe("idle"); expect(m.ctx.runFact).toBeNull(); }); + it("stopping exits on a clean finish carrying the pre-stop epoch too", () => { + const preStopEpoch = withRunFact().ctx.epoch; + let m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); + m = reduce(m, { type: "FINISH_CLEAN", epoch: preStopEpoch }); + expect(m.phase.name).toBe("idle"); + }); + it("stopping exits on a negative run-fact (data)", () => { let m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); m = reduce(m, { type: "RUN_FACT", runFact: null, epoch: m.ctx.epoch }); diff --git a/apps/client/src/features/ai-chat/state/run-fsm.ts b/apps/client/src/features/ai-chat/state/run-fsm.ts index 6239b450..2ca554b2 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.ts @@ -232,7 +232,33 @@ function command( // The pure reducer. // --------------------------------------------------------------------------- +/** The terminal stream-finish events (one turn's stream ended). */ +function isFinishEvent(event: Event): boolean { + return ( + event.type === "FINISH_ABORT" || + event.type === "FINISH_CLEAN" || + event.type === "FINISH_DISCONNECT" || + event.type === "FINISH_ERROR" || + event.type === "STREAM_INCOMPLETE" + ); +} + export function reduce(m: Machine, event: Event): Machine { + // MEDIUM (#488 re-review): honor ANY stream finish in `stopping` regardless of + // generation. A plain user Stop has NO successor stream — the aborted stream's + // finish IS the expected end of the stop, so exit `stopping -> idle` by that DATA + // (I4). The epoch filter below must NOT drop it: STOP_REQUESTED bumped the epoch, + // but the finish carries the PRE-stop generation (the runtime stamps it with the + // stream's start epoch), so I1 would otherwise strand the machine in `stopping` + // forever (no idle-cap covers `stopping`). The epoch filter stays in force for + // `superseding` (a successor B owns) — that is the F1 supersede drop. + if (m.phase.name === "stopping" && isFinishEvent(event)) { + return to(m, { name: "idle" }, { + ctx: { runFact: null, liveFollow: false }, + effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], + }); + } + // I1: drop a stale outcome (an event issued under a superseded epoch). if ("epoch" in event && event.epoch !== undefined && event.epoch !== m.ctx.epoch) { return stay(m); @@ -455,8 +481,10 @@ export function reduce(m: Machine, event: Event): Machine { // abort the local/attach reader. ALSO arm the poll so the terminal row is // observed — the exit is by DATA (I4: a terminal row / negative run-fact), // never by the stopRun HTTP response (which returns after abort, before - // finalization). For a local turn the onFinish FINISH_ABORT exits first and - // disarms; for an observer the poll drives to the aborted terminal. + // finalization). For a local turn the aborted stream's onFinish (ANY finish) + // is HONORED in `stopping` at the top of reduce() — regardless of generation + // — and exits to idle; the armed poll is the fallback for an observer stop + // with no local onFinish. return command( m, { name: "stopping" }, From 60205398bb9c552d8027d3ede4b11b9dea7535ec Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 08:25:29 +0300 Subject: [PATCH 36/41] =?UTF-8?q?test(ai-chat):=20=D0=B7=D0=B0=D1=81=D1=82?= =?UTF-8?q?=D0=B0=D0=B1=D0=B8=D1=82=D1=8C=20findAllByChat=20=D0=B2=20lifec?= =?UTF-8?q?ycle-=D0=BC=D0=BE=D0=BA=D0=B5=20=D0=BF=D0=BE=D0=B4=20convert-be?= =?UTF-8?q?fore-insert=20(#489)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Коммит 1 (#489) прогоняет загрузку истории + convertToModelMessages ДО insert user-строки (convert-before-insert, чтобы ретрай не плодил дубли), поэтому stream() теперь зовёт реальный метод репо findAllByChat перед insert. Хэнд-роллед мок в тесте «exception after beginRun → settled to error» стабил только insert, и тест ловил «findAllByChat is not a function» вместо «insert boom». Порядок инварианта НЕ изменился: и findAllByChat, и insert идут ПОСЛЕ beginRun (reconcileChat между ними — best-effort, глотает ошибку), так что бросок по- прежнему происходит ПОСЛЕ начала рана и обязан сеттлить ран в error. Застабил findAllByChat → [] (реальный репо-метод, см. ai-chat.controller.ts), тест снова доходит до insert boom и проверяет settle-to-error. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../src/core/ai-chat/ai-chat.service.lifecycle.spec.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/apps/server/src/core/ai-chat/ai-chat.service.lifecycle.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.lifecycle.spec.ts index a6f3e75c..22a721e8 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.lifecycle.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.lifecycle.spec.ts @@ -97,8 +97,14 @@ describe('AiChatService.stream run-lifecycle safety net (#184)', () => { }; const runService = new AiChatRunService(runRepo as never, { isCloud: () => false } as never); - // The user-message insert (the first bare await after beginRun) throws. + // The user-message insert throws. #489 runs the history load + convert BEFORE + // the insert (convert-before-insert, so a retry cannot duplicate the user row), + // so `findAllByChat` (a real repo method) is now called first — stub it to an + // empty history so the flow reaches the insert. Both awaits are AFTER beginRun, + // so the "exception after beginRun -> settled to error" invariant is unchanged; + // the throw point simply moved from insert to a later insert after a no-op load. const aiChatMessageRepo = { + findAllByChat: jest.fn().mockResolvedValue([]), insert: jest.fn().mockRejectedValue(new Error('insert boom')), }; const aiChatRepo = { From 791a707eb6eec5539fb87f2e5f565a25d9cbf229 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 09:11:32 +0300 Subject: [PATCH 37/41] =?UTF-8?q?docs(ai-chat):=20=D0=B7=D0=B0=D0=B4=D0=BE?= =?UTF-8?q?=D0=BA=D1=83=D0=BC=D0=B5=D0=BD=D1=82=D0=B8=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D0=BF=D0=BE=D0=B2=D0=B5=D1=80=D1=85=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D1=8C=20#489=20=E2=80=94=20env-var,=20=D1=83?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D1=80=D0=B5=D0=B2=D1=88=D0=B0=D1=8F=20=D0=B7?= =?UTF-8?q?=D0=B0=D0=BC=D0=B5=D1=82=D0=BA=D0=B0,=20CHANGELOG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 (ревью #508): - .env.example: новая AI_MCP_SSE_BODY_TIMEOUT_MS (дефолт 600000, 10 мин) — bodyTimeout SSE-транспорта external-MCP; idle между тул-вызовами легитимен. - .env.example: поправлена ставшая ложной заметка про AI_MCP_STREAM_TIMEOUT_MS (SSE-idle-между-вызовами больше не режется им — с #489 это отдельная var; 1-мин silence остаётся только для HTTP/headers и одиночного залипшего вызова). - CHANGELOG [Unreleased]/Fixed: битый part больше не 500-ит каждый ход + не плодит дубль user-строки; MCP-транспорт-дропы восстанавливаются in-run (readOnly ретраится раз, write никогда) + новая env-var. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 21 ++++++++++++++++++--- CHANGELOG.md | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.env.example b/.env.example index f3acb7ba..6d7dec43 100644 --- a/.env.example +++ b/.env.example @@ -225,11 +225,26 @@ MCP_DOCMOST_PASSWORD= # Silence timeout (ms) for EXTERNAL-MCP transport ONLY (not the chat provider). # Tighter than AI_STREAM_TIMEOUT_MS so a byte-silent/hung MCP server is broken in -# ~1 min instead of 15. Note it also cuts a legitimately long but byte-silent -# single tool call (a slow crawl that emits nothing until done) and an SSE -# transport idling >1 min BETWEEN tool calls. Default 60000 (1 min). +# ~1 min instead of 15. It cuts a legitimately long but byte-silent single tool +# call (a slow crawl that emits nothing until done) on the HTTP (streamable) +# transport, which opens a fresh request per call. The SSE transport — one +# long-lived body across many calls — is NO LONGER governed by this timeout +# (as of #489): its idle-BETWEEN-calls window has its own, raised bodyTimeout, +# AI_MCP_SSE_BODY_TIMEOUT_MS below. Default 60000 (1 min). # AI_MCP_STREAM_TIMEOUT_MS=60000 +# bodyTimeout (ms) for the EXTERNAL-MCP SSE transport ONLY (#489). The SSE +# transport holds ONE response body open across many tool calls, so undici's +# bodyTimeout (time between body bytes) counts the LEGITIMATE silence BETWEEN the +# model's tool calls, not just a hung single call. At the tight 1-min silence +# timeout above, a normal >1-min gap between calls would break the SSE socket and +# the cache would serve a dead client until TTL — so the SSE transport gets its +# OWN, RAISED bodyTimeout. A single stuck call is still bounded by the per-call +# cap (AI_MCP_CALL_TIMEOUT_MS), and a socket that does break is healed by the +# in-run transport-error retry. The HTTP (streamable) transport keeps the tight +# timeout. Default 600000 (10 min). +# AI_MCP_SSE_BODY_TIMEOUT_MS=600000 + # Total wall-clock cap (ms) for ONE external MCP tool call (app-level, not # transport). Aborts a tool that keeps the socket warm (SSE heartbeats / trickle) # but never returns a result — which the silence timeout above never breaks. diff --git a/CHANGELOG.md b/CHANGELOG.md index 30d27413..adecf01f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -336,6 +336,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **A chat with one malformed message part no longer 500s on every turn, and a + failed send no longer duplicates the user's message.** Incoming client parts + are now whitelisted to `text` (a forged tool-result part can no longer reach + the persisted history or the model context), and the turn is converted BEFORE + the user row is inserted, so a mid-flight failure cannot leave a duplicate + user row that a retry then compounds. A single part that still fails to convert + degrades to a `[tool context omitted]` marker on that one row instead of + bricking the whole chat. (#489) +- **A transport drop to an external MCP server now heals within the same turn.** + On an undici transport error, a read-only MCP tool reconnects its server and + retries once within the run; a write is never auto-retried (it may already have + applied). One flapping server no longer nulls the shared client cache, so other + servers' cached clients are untouched. The SSE transport also gets a raised + body-timeout so a legitimate >1-min idle between the model's tool calls no + longer breaks a long-lived SSE socket (new `AI_MCP_SSE_BODY_TIMEOUT_MS`, default + 10 min; see `.env.example`). (#489) + - **The server no longer runs out of heap during long autonomous agent runs.** A new pnpm patch on `ai@6.0.134` stops the SDK from building a cumulative snapshot of the ENTIRE turn text on every streamed text-delta when no output From c7a38e274e5bed2d4351ca32dc88c50799bea52f Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 09:37:07 +0300 Subject: [PATCH 38/41] =?UTF-8?q?fix(client):=20=D1=80=D0=B5=D0=B0=D0=BB?= =?UTF-8?q?=D1=8C=D0=BD=D1=8B=D0=B9=20=D0=BE=D0=B1=D1=80=D1=8B=D0=B2=20SSE?= =?UTF-8?q?=20=E2=86=92=20reconnect-=D0=BB=D0=B5=D1=81=D0=B5=D0=BD=D0=BA?= =?UTF-8?q?=D0=B0,=20=D0=BD=D0=B5=20=D1=82=D0=B5=D1=80=D0=BC=D0=B8=D0=BD?= =?UTF-8?q?=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D0=B9=20error=20(#488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Браузерный QA поймал баг, который юнит-тесты пропускали из-за вакуумной SDK-формы. Премиса (подтверждена по ai@6.0.207 AbstractChat.makeRequest, catch ~13763): при сетевом дропе SDK ставит isError=true БЕЗУСЛОВНО, а isError=true → isDisconnect=true ставится РЯДОМ (только для fetch/network TypeError). Т.е. реальный обрыв ВСЕГДА даёт { isError:true, isDisconnect:true }; форма { isDisconnect:true, isError:false } SDK'ом НЕ эмитится. Баг: в onFinish проверка `if (isError) return` стояла РАНЬШЕ ветки isDisconnect → реальный дроп уходил в терминальный error-баннер, а FINISH_DISCONNECT (единственный вход в reconnect-лесенку) не диспатчился НИКОГДА. Сценарии 1/2 (commit 2 «обрыв до первого кадра» и commit 3 «два обрыва») в браузере не работали. Фикс: маршрутизация по disconnect ПЕРВЫМ: isDisconnect → FINISH_DISCONNECT (reconnect); НЕ-disconnect error (isError && !isDisconnect, напр. провайдерский 500) → FINISH_ERROR (терминал); затем isAbort; затем clean. Порядок веток supersede- блока тоже disconnect-first (для консистентности; там всё равно всё дропается I1). Инварианты сохранены: epoch-штамп turnEpochRef на всех FINISH_*; honor-in-stopping в редьюсере честит ЛЮБОЙ финиш в фазе stopping → на пути Stop дроп-финиш уходит в idle, а не в ложный reconnect (новый тест). F1 supersede-drop чужого поколения цел. Тесты сделаны НЕ вакуумными: дроп подаётся реальной формой { isError:true, isDisconnect:true }, терминальная ошибка — { isError:true, isDisconnect:false }. MUTATION-VERIFY: багованный isError-first порядок → 6 reconnect-тестов краснеют (нет баннера «reconnecting»); с фиксом зелены. Полный ai-chat 35 файлов / 377 / 0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/chat-thread.test.tsx | 42 +++++++++++++++---- .../ai-chat/components/chat-thread.tsx | 42 ++++++++++++------- 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index 2c5e6ce2..ee409e6a 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -248,14 +248,15 @@ describe("ChatThread — send now", () => { startLocalStreamWithRun(); fireEvent.click(screen.getByTestId("queue-btn")); fireEvent.click(screen.getByLabelText("Send now")); // -> superseding, aborts A - // A ends via isDisconnect (the server CAS closed it). The OLD overlap bug would - // route the LIVE new run into a false reconnect banner. + // A ends via a REAL network drop { isError:true, isDisconnect:true } (the server + // CAS closed it). The OLD overlap bug would route the LIVE new run into a false + // reconnect banner. await act(async () => { h.state.onFinish?.({ message: { id: "a1", role: "assistant", parts: [{ type: "text", text: "x" }] }, isAbort: false, isDisconnect: true, - isError: false, + isError: true, }); await Promise.resolve(); }); @@ -328,6 +329,23 @@ describe("ChatThread — send now", () => { expect(screen.getAllByLabelText("Remove queued message")).toHaveLength(1); }); + it("Stop then a REAL network-drop finish exits to idle (honor-in-stopping), NOT a false reconnect", () => { + // Regression for the disconnect-first reorder: on the STOP path, even a drop- + // form finish { isError:true, isDisconnect:true } arriving in `stopping` must be + // HONORED (reducer) and exit to idle — it must NOT enter the reconnect ladder. + startLocalStreamWithRun(); // live local stream, autonomous + fireEvent.click(screen.getByLabelText("Stop")); // STOP_REQUESTED -> stopping + act(() => { + h.state.onFinish?.({ + message: { id: "a1", role: "assistant", parts: [] }, + isAbort: false, + isDisconnect: true, + isError: true, + }); + }); + expect(screen.queryByText(/reconnecting/i)).toBeNull(); + }); + it("Send now is HIDDEN while observing a resumed run and VISIBLE on a local stream", () => { // Resumed (mount attach) -> observer -> hidden. renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); @@ -419,15 +437,18 @@ describe("ChatThread — turn-end decision (onFinish)", () => { expect(screen.getByText("Response stopped.")).toBeTruthy(); }); - it("ENDS — keeps the queue on a disconnect (non-autonomous) and shows the notice", () => { - finishWith({ isDisconnect: true }); + it("ENDS — keeps the queue on a REAL disconnect (non-autonomous) and shows the notice", () => { + // Real SDK drop form: { isError:true, isDisconnect:true }. With the buggy + // isError-first order this would fall to the terminal error branch (no notice). + finishWith({ isDisconnect: true, isError: true }); expect(h.state.sendMessage).not.toHaveBeenCalled(); expect( screen.getByText("Connection lost — the answer was interrupted."), ).toBeTruthy(); }); - it("ENDS — keeps the queue on a stream error (no notice)", () => { + it("ENDS — keeps the queue on a NON-disconnect stream error (no notice)", () => { + // A provider error is { isError:true, isDisconnect:false } — terminal. finishWith({ isError: true }); expect(h.state.sendMessage).not.toHaveBeenCalled(); expect(screen.queryByText("Response stopped.")).toBeNull(); @@ -437,7 +458,7 @@ describe("ChatThread — turn-end decision (onFinish)", () => { for (const flags of [ {}, { isAbort: true }, - { isDisconnect: true }, + { isDisconnect: true, isError: true }, { isError: true }, ]) { cleanup(); @@ -787,13 +808,18 @@ describe("ChatThread — live reconnect + stalled", () => { cleanup(); }); + // A REAL live SSE drop. ai@6.0.207 emits BOTH { isError:true, isDisconnect:true } + // for a network TypeError — NOT the { isError:false } form the old tests fed. This + // is the form browser QA hit; with the buggy isError-first routing these tests go + // red (a real drop would surface the terminal error banner, not the reconnect + // ladder). MUTATION-VERIFY of the disconnect-first fix. function disconnect(message: unknown = liveMsg) { act(() => { h.state.onFinish?.({ message, isAbort: false, isDisconnect: true, - isError: false, + isError: true, }); }); } diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index bd0b513c..3ec5930b 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -626,10 +626,13 @@ export default function ChatThread({ machineRef.current.phase.name === "superseding" && pendingSupersedeTextRef.current !== null ) { - if (isError) dispatch({ type: "FINISH_ERROR", kind: "stream", epoch: stampEpoch }); - else if (isAbort) dispatch({ type: "FINISH_ABORT", epoch: stampEpoch }); - else if (isDisconnect) + // Disconnect-first (see the routing note below): a real drop is + // { isError:true, isDisconnect:true }. All of these are dropped by I1 here + // (superseding bumped the epoch) — the order only mirrors the main routing. + if (isDisconnect) dispatch({ type: "FINISH_DISCONNECT", hasVisibleContent: msgHasVisible, epoch: stampEpoch }); + else if (isError) dispatch({ type: "FINISH_ERROR", kind: "stream", epoch: stampEpoch }); + else if (isAbort) dispatch({ type: "FINISH_ABORT", epoch: stampEpoch }); else dispatch({ type: "FINISH_CLEAN", epoch: stampEpoch }); const text = pendingSupersedeTextRef.current; pendingSupersedeTextRef.current = null; @@ -642,18 +645,14 @@ export default function ChatThread({ return; } - if (isError) { - dispatch({ type: "FINISH_ERROR", kind: "stream", epoch: stampEpoch }); - setStopNotice(null); - return; - } - if (isAbort) { - // A user Stop / an interrupt abort finished. The FSM stopping/idle exit is - // by DATA (this terminal outcome, I4). - dispatch({ type: "FINISH_ABORT", epoch: stampEpoch }); - setStopNotice("manual"); - return; - } + // #488 (browser QA): route DISCONNECT FIRST. In ai@6.0.207 a real network drop + // yields BOTH `{ isError:true, isDisconnect:true }` — the SDK sets `isError` + // unconditionally in its catch and `isDisconnect` only ALONGSIDE it (for a + // fetch/network TypeError). Checking `isError` first therefore sent EVERY real + // drop to the terminal error banner and NEVER entered the reconnect ladder + // (`FINISH_DISCONNECT` is its only entry). A disconnect — the detached run + // keeps executing server-side — must win; only a NON-disconnect error (a + // provider 500, `{ isError:true, isDisconnect:false }`) is terminal. if (isDisconnect) { if (wasObserver) { // A resumed/attached OBSERVER stream dropped. Recover via the degraded @@ -722,6 +721,19 @@ export default function ChatThread({ } return; } + // A NON-disconnect stream error (a provider 500 etc.) -> terminal error banner. + if (isError) { + dispatch({ type: "FINISH_ERROR", kind: "stream", epoch: stampEpoch }); + setStopNotice(null); + return; + } + if (isAbort) { + // A user Stop / an interrupt abort finished. The FSM stopping/idle exit is + // by DATA (this terminal outcome, I4 — honored in `stopping` by the reducer). + dispatch({ type: "FINISH_ABORT", epoch: stampEpoch }); + setStopNotice("manual"); + return; + } // Clean finish. if (wasObserver) { if (mountedRef.current) { From 803bbd40b4e09d5da39cf53f5e2c997e305a3cc1 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 09:55:39 +0300 Subject: [PATCH 39/41] =?UTF-8?q?fix(client):=20=D1=84=D0=B0=D0=B7=D0=B0?= =?UTF-8?q?=20FSM=20=D1=80=D0=B5=D1=88=D0=B0=D0=B5=D1=82=20=D0=B1=D0=B0?= =?UTF-8?q?=D0=BD=D0=BD=D0=B5=D1=80=20=E2=80=94=20reconnect=20=D0=BD=D0=B5?= =?UTF-8?q?=20=D0=BC=D0=B0=D1=81=D0=BA=D0=B8=D1=80=D1=83=D0=B5=D1=82=D1=81?= =?UTF-8?q?=D1=8F=20=D0=BE=D1=81=D1=82=D0=B0=D1=82=D0=BE=D1=87=D0=BD=D1=8B?= =?UTF-8?q?=D0=BC=20error=20(#488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Браузерный QA: маршрутизация disconnect-first работает (лесенка реально стартует), но пользователь всё равно видел терминальный «Lost connection… reload», а не «reconnecting… (N/5)». Трасса (подтверждена по коду): рендер `{errorView ? : phase === "reconnecting" ? ...}` даёт errorView ПРИОРИТЕТ над recovery-фазой. errorView = `error && describeChatError(...)`, где `error` — из useChat. Реальный ai@6 при isError выставляет useChat `error`, а дроп ВСЕГДА isError+isDisconnect → после починки маршрутизации FSM уходит в `reconnecting`, но `error` остаётся выставленным → errorView перекрывает reconnect-баннер. Тот же класс вакуум-мока, что и с isDisconnect, но на поле `error` (мок хардкодил error:null → в тестах маски не было). Фикс рендера: фаза FSM — источник истины. Терминальный errorView показывается ТОЛЬКО когда FSM реально терминален: `showError = errorView && phase === "error"`. В recovery-фазах (reconnecting/polling/stalled/superseding/stopping) выигрывает recovery-баннер (или контент стрима). Классифицированные 409 (#487) целы: supersede/ gate-409 ставят FSM в error(kind) → errorView показывается там, где должен. Мок useChat сделан реалистичным СИСТЕМНО: добавлено поле h.state.error, мок его возвращает; при любом isError-финише (дроп или провайдерская ошибка) тест ставит error в реальную форму, зеркаля связку SDK. MUTATION-VERIFY: откат рендер-гейта (errorView-first) → 8 reconnect/stalled-тестов краснеют (баннер маскируется); с гейтом — зелёные. Плюс отдельный тест «reconnect-баннер виден, не замаскирован». Всё прошлое цело: disconnect-first, epoch-штамп, honor-in-stopping, supersede- исходы, stalled/no-poll. Полный ai-chat 35 файлов / 378 / 0; tsc ai-chat 0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/chat-thread.test.tsx | 43 +++++++++++++++---- .../ai-chat/components/chat-thread.tsx | 14 +++++- 2 files changed, 48 insertions(+), 9 deletions(-) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index ee409e6a..8a507076 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -16,6 +16,10 @@ const h = vi.hoisted(() => ({ state: { status: "streaming" as string, messages: [] as unknown[], + // Mirrors the SDK: on an isError finish (incl. a network drop, which is ALWAYS + // isError+isDisconnect) the SDK ALSO sets useChat `error`. Realistic tests set + // this so the banner-priority (errorView vs recovery phase) is actually exercised. + error: null as null | { message: string }, onFinish: null as null | ((arg: Record) => void), sendMessage: vi.fn(), stop: vi.fn(), @@ -56,7 +60,7 @@ vi.mock("@ai-sdk/react", () => ({ sendMessage: h.state.sendMessage, status: h.state.status, stop: h.state.stop, - error: null, + error: h.state.error, setMessages: h.state.setMessages, resumeStream: h.state.resumeStream, }; @@ -156,6 +160,7 @@ function renderThread(props?: { function resetState() { h.state.status = "streaming"; h.state.messages = []; + h.state.error = null; h.state.onFinish = null; h.state.seededMessages = null; h.state.transport = null; @@ -248,9 +253,10 @@ describe("ChatThread — send now", () => { startLocalStreamWithRun(); fireEvent.click(screen.getByTestId("queue-btn")); fireEvent.click(screen.getByLabelText("Send now")); // -> superseding, aborts A - // A ends via a REAL network drop { isError:true, isDisconnect:true } (the server - // CAS closed it). The OLD overlap bug would route the LIVE new run into a false - // reconnect banner. + // A ends via a REAL network drop { isError:true, isDisconnect:true } + error set + // (the server CAS closed it). The OLD overlap bug would route the LIVE new run + // into a false reconnect banner. + h.state.error = { message: "Failed to fetch" }; await act(async () => { h.state.onFinish?.({ message: { id: "a1", role: "assistant", parts: [{ type: "text", text: "x" }] }, @@ -335,6 +341,7 @@ describe("ChatThread — send now", () => { // HONORED (reducer) and exit to idle — it must NOT enter the reconnect ladder. startLocalStreamWithRun(); // live local stream, autonomous fireEvent.click(screen.getByLabelText("Stop")); // STOP_REQUESTED -> stopping + h.state.error = { message: "Failed to fetch" }; act(() => { h.state.onFinish?.({ message: { id: "a1", role: "assistant", parts: [] }, @@ -413,6 +420,10 @@ describe("ChatThread — turn-end decision (onFinish)", () => { }) { const { onTurnFinished } = renderThread(); fireEvent.click(screen.getByTestId("queue-btn")); + // Mirror the SDK: any isError finish (a provider error or a drop) also sets + // useChat `error` (a drop message triggers the connection-lost classification). + if (flags.isError) + h.state.error = { message: flags.isDisconnect ? "Failed to fetch" : "500: boom" }; act(() => { h.state.onFinish?.({ message: { id: "a", role: "assistant", parts: [] }, @@ -809,11 +820,13 @@ describe("ChatThread — live reconnect + stalled", () => { }); // A REAL live SSE drop. ai@6.0.207 emits BOTH { isError:true, isDisconnect:true } - // for a network TypeError — NOT the { isError:false } form the old tests fed. This - // is the form browser QA hit; with the buggy isError-first routing these tests go - // red (a real drop would surface the terminal error banner, not the reconnect - // ladder). MUTATION-VERIFY of the disconnect-first fix. + // for a network TypeError AND sets useChat `error` — NOT the { isError:false, + // error:null } form the old tests fed. This is the form browser QA hit; with the + // buggy isError-first routing OR without the errorView render-gate these tests go + // red (a real drop surfaces the terminal error banner, masking the reconnect + // ladder). MUTATION-VERIFY of disconnect-first + the errorView phase-gate. function disconnect(message: unknown = liveMsg) { + h.state.error = { message: "Failed to fetch" }; // the SDK sets error on the drop act(() => { h.state.onFinish?.({ message, @@ -855,6 +868,20 @@ describe("ChatThread — live reconnect + stalled", () => { ); }); + it("#488 (browser QA): the reconnect banner is SHOWN, not masked by the residual useChat error", () => { + // The drop sets useChat `error` (real SDK), and the terminal errorView describes + // it ("Lost connection to the server"). The FSM phase-gate must let the + // `reconnecting` banner WIN over that residual error. MUTATION-VERIFY: revert the + // errorView phase-gate (show errorView whenever error is set) and the terminal + // banner masks "reconnecting…" -> red. + renderLive(); + disconnect(); + expect(h.state.error).not.toBeNull(); // the SDK error IS set during recovery + expect(screen.getByText(/reconnecting/i)).toBeTruthy(); + // The terminal "Lost connection… reload" banner must NOT be showing. + expect(screen.queryByText(/reload and try again/i)).toBeNull(); + }); + it("#488 commit 2: a disconnect BEFORE the first assistant frame reconnects with NO anchor", () => { renderLive(); disconnect(null); // no assistant message yet (pre-first-frame break) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index 3ec5930b..a82fbf77 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -978,6 +978,18 @@ export default function ChatThread({ // #487 409 codes, ...). Computed here so the SAME text can mirror into the export. const errorView = error ? describeChatError(error.message ?? "", t) : null; + // #488 (browser QA): the FSM PHASE is the source of truth for WHICH banner to + // show. A real network drop leaves useChat `error` SET even after the FSM has + // moved to `reconnecting` — in ai@6 a drop is always `{isError:true, + // isDisconnect:true}` and the SDK sets `error` alongside `isError`. So the + // terminal error banner must show ONLY when the FSM is actually TERMINAL + // (`error(kind)`); during recovery (reconnecting / polling / stalled / + // superseding / stopping) the recovery banner — or the streaming content — wins + // over the residual `error`, otherwise the terminal "Lost connection… reload" + // banner masks "reconnecting… (N/5)". This still surfaces the classified #487 409 + // errors: a supersede/gate 409 lands the FSM in `error(kind)`, so it shows then. + const showError = errorView !== null && phase.name === "error"; + // Role-picker empty state (unchanged from #149). const [rolePickedNoSend, setRolePickedNoSend] = useState(false); const handleRolePick = (role: IAiRole): void => { @@ -1011,7 +1023,7 @@ export default function ChatThread({ assistantName={assistantName} /> - {errorView ? ( + {showError && errorView ? ( ) : phase.name === "reconnecting" ? ( // #430/#488: while auto-reconnecting to a detached run's live tail, show From 080dd82023f45b7c0ce6750f9ddd51779c5a0c77 Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 11:07:02 +0300 Subject: [PATCH 40/41] =?UTF-8?q?fix(client):=20=D1=80=D0=B5-=D1=80=D0=B5?= =?UTF-8?q?=D0=B2=D1=8C=D1=8E=20#509=20=E2=80=94=205=20=D0=BF=D1=83=D0=BD?= =?UTF-8?q?=D0=BA=D1=82=D0=BE=D0=B2=20Do-=D0=BB=D0=B8=D1=81=D1=82=D0=B0=20?= =?UTF-8?q?(=D1=81=D1=82=D0=B0=D0=B1=D0=B8=D0=BB=D1=8C=D0=BD=D0=BE=D1=81?= =?UTF-8?q?=D1=82=D1=8C/=D1=80=D0=B5=D0=B3=D1=80=D0=B5=D1=81=D1=81=D0=B8?= =?UTF-8?q?=D0=B8)=20(#488)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 [stability] Позитивные attach-исходы гвардятся по ИСХОДНОЙ фазе. Одного epoch- фильтра мало: POLL_TERMINAL использует to() (epoch не инкрементит) и не шлёт abortAttach, поэтому медленный GET, вернувший live 2xx уже ПОСЛЕ того как армленный poll увёл машину в idle, воскрешал осевший ран в фантомный streaming. RECONNECT_ ATTACHED теперь bail'ит если фаза != reconnecting; ATTACH_LIVE/ATTACH_NONE — если != attaching. Тест на гонку (POLL_TERMINAL до RECONNECT_ATTACHED → idle) + mutation. 2 [regressions] ownership сбрасывается в "local" на ВСЕХ терминальных переходах (FINISH_CLEAN/ABORT/ERROR, POLL_TERMINAL, RUN_FACT{null}→idle, honor-in-stopping, disconnect→idle). Иначе observer-attach + очередь + clean-финиш → idle, но ownership навсегда observer → «Send now» скрыт при свободном композере. Безопасно для I2: рантайм захватывает wasObserver из machineRef ДО dispatch. Тест + mutation. 3 [coverage] Тест happy-path CAS-supersede: SUPERSEDE_READY-dispatch (200-исход transport.fetch) раньше НЕ исполнялся ни в одном тесте — риск залипания в superseding на весь стрим B. Тест вводит в superseding, гонит A.onFinish→B, затем POST 200 → SUPERSEDE_READY → streaming, проверяет повторный supersede (не залип). Сиблинги: 409 SUPERSEDE_TARGET_MISMATCH → getRun/verify; plain-409 A_RUN_ALREADY_ACTIVE → классифицированный баннер. Mutation (no-op READY → красный). 4 [regressions] Inactivity-бэкстоп для poll, армленного в stopping. STOP_REQUESTED армит poll и входит в stopping, но idle-cap покрывал только polling/reconnecting → observer-стоп без SDK-стрима и без серверного терминала поллил БД вечно. Добавлен stopping в фаза-чек эффекта + переход POLL_IDLE_CAP: stopping→idle+disarm (НЕ stalled — Stop уже нажат). FSM + компонент-тест (idle-cap→disarm) + mutation. 5 [docs] Счёт §2 «Net»: 8→FSM (#1-6,#11,#13) + 3 deleted + reconnectTimerRef (effect-owned) + mountedRef (retained) = 13; attachAbortRef вне набора #1-13. Не трогал DROP-блок (мёртвый FINISH_* в superseding, ErrorKind.kind, неиспользуемые enum-варианты, epochRef-зеркало). Всё прошлое цело (disconnect-first, epoch, honor- in-stopping, render-gate, supersede). Полный ai-chat 35 файлов / 388 / 0; tsc 0. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/chat-thread.test.tsx | 117 ++++++++++++++++++ .../ai-chat/components/chat-thread.tsx | 4 +- .../features/ai-chat/state/run-fsm.spec.md | 10 +- .../features/ai-chat/state/run-fsm.test.ts | 69 +++++++++++ .../src/features/ai-chat/state/run-fsm.ts | 50 ++++++-- 5 files changed, 236 insertions(+), 14 deletions(-) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index 8a507076..e15f8030 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -353,6 +353,102 @@ describe("ChatThread — send now", () => { expect(screen.queryByText(/reconnecting/i)).toBeNull(); }); + it("#488 review-2: an observer turn's clean finish re-exposes 'Send now' (ownership reset to local)", () => { + // Mount attach -> observer. The queued message shows only "Remove" while + // observing; once the attached run finishes CLEAN the FSM resets ownership to + // local, so "Send now" becomes available again (composer is free). + // MUTATION-VERIFY: drop `ownership:"local"` from FINISH_CLEAN -> stays observer + // -> "Send now" stays hidden -> red. + renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); + fireEvent.click(screen.getByTestId("queue-btn")); + expect(screen.queryByLabelText("Send now")).toBeNull(); // observer -> hidden + act(() => { + h.state.onFinish?.({ + message: { id: "a1", role: "assistant", parts: [{ type: "text", text: "done" }] }, + isAbort: false, + isDisconnect: false, + isError: false, + }); + }); + expect(screen.getByLabelText("Send now")).toBeTruthy(); // ownership local again + }); + + it("#488 review-3: a successful CAS supersede POST (200) exits `superseding` (Send now works again)", async () => { + // Happy-path of the transport 409/CAS block: SUPERSEDE_READY was never executed + // in tests (reviewer confirmed no-op mutation stayed green). If it does not fire, + // the machine stays `superseding` for the whole B stream and "Send now" is dead. + startLocalStreamWithRun(); + fireEvent.click(screen.getByTestId("queue-btn")); // X + fireEvent.click(screen.getByLabelText("Send now")); // -> superseding, stop() #1 + expect(h.state.stop).toHaveBeenCalledTimes(1); + // A's onFinish sends B (clears the pending-supersede text) — the realistic + // no-overlap sequence. + await act(async () => { + h.state.onFinish?.({ + message: { id: "a1", role: "assistant", parts: [] }, + isAbort: true, + isDisconnect: false, + isError: false, + }); + await Promise.resolve(); + }); + // B's CAS POST returns 200 -> SUPERSEDE_READY -> streaming. + vi.stubGlobal("fetch", vi.fn().mockResolvedValue(new Response("{}", { status: 200 }))); + await act(async () => { + await h.state.transport!.fetch!("http://x", { method: "POST", body: "{}" }); + }); + // We LEFT `superseding`: a fresh Send now supersedes AGAIN (still stuck in + // superseding -> LOW-2 guard would no-op it). MUTATION-VERIFY: no-op + // SUPERSEDE_READY -> stuck superseding -> stop stays at 1 -> red. + fireEvent.click(screen.getByTestId("queue-btn")); // Y + fireEvent.click(screen.getByLabelText("Send now")); + expect(h.state.stop).toHaveBeenCalledTimes(2); + }); + + it("#488 review-3 sibling: a 409 SUPERSEDE_TARGET_MISMATCH fires the /run verify", async () => { + startLocalStreamWithRun(); + fireEvent.click(screen.getByTestId("queue-btn")); + fireEvent.click(screen.getByLabelText("Send now")); // -> superseding + h.state.getRun.mockClear(); + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ code: "SUPERSEDE_TARGET_MISMATCH", runId: "run-x" }), + { status: 409 }, + ), + ), + ); + await act(async () => { + await h.state.transport!.fetch!("http://x", { method: "POST", body: "{}" }); + }); + // SUPERSEDE_MISMATCH -> error(supersede-mismatch) + postRun(verify) -> getRun. + expect(h.state.getRun).toHaveBeenCalledWith("c1"); + }); + + it("#488 review-3 sibling: a plain 409 A_RUN_ALREADY_ACTIVE shows the classified banner", async () => { + renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); + // The SDK sets useChat error to the 409 body on the failed POST. + h.state.error = { + message: + '{"message":"active","code":"A_RUN_ALREADY_ACTIVE","statusCode":409}', + }; + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + new Response(JSON.stringify({ code: "A_RUN_ALREADY_ACTIVE" }), { + status: 409, + }), + ), + ); + await act(async () => { + await h.state.transport!.fetch!("http://x", { method: "POST", body: "{}" }); + }); + // RUN_ALREADY_ACTIVE -> phase error(kind) -> the phase-gate lets errorView show + // the classified banner. + expect(screen.getByText("The agent is already answering")).toBeTruthy(); + }); + it("Send now is HIDDEN while observing a resumed run and VISIBLE on a local stream", () => { // Resumed (mount attach) -> observer -> hidden. renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); @@ -972,4 +1068,25 @@ describe("ChatThread — live reconnect + stalled", () => { expect(screen.getByText(/the run stopped responding/i)).toBeTruthy(); expect(screen.getByText("Retry")).toBeTruthy(); }); + + it("#488 review-4: an observer-stop's armed poll is bounded by the idle cap (exits to idle, disarm)", () => { + // STOP_REQUESTED arms the poll and enters `stopping`; an observer stop has no + // SDK stream to fire onFinish, and the server stop may never drive the run + // terminal. Without a backstop the DB would poll forever. The idle cap must give + // `stopping` a bounded exit -> idle + disarm (NOT stalled). MUTATION-VERIFY: drop + // `stopping` from the idle-cap effect / the POLL_IDLE_CAP branch -> no disarm. + const { onResumeFallback } = renderThread({ + autonomousRunsEnabled: true, + initialRows: streamingTail(), // mount attach -> observer + }); + onResumeFallback.mockClear(); + fireEvent.click(screen.getByLabelText("Stop")); // STOP_REQUESTED -> stopping + armPoll + expect(onResumeFallback).toHaveBeenCalledWith(true); + onResumeFallback.mockClear(); + // No terminal row ever arrives; the inactivity cap bounds it. + act(() => { + vi.advanceTimersByTime(10 * 60_000); + }); + expect(onResumeFallback).toHaveBeenCalledWith(false); // POLL_IDLE_CAP -> idle -> disarm + }); }); diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index a82fbf77..24db89b6 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -878,8 +878,10 @@ export default function ChatThread({ clearTimeout(idleCapTimerRef.current); idleCapTimerRef.current = null; } + // Review #4: `stopping` also arms the poll and needs a bounded exit (the FSM + // maps POLL_IDLE_CAP from `stopping` -> idle, not stalled). const p = phase.name; - if (p !== "polling" && p !== "reconnecting") return; + if (p !== "polling" && p !== "reconnecting" && p !== "stopping") return; idleCapTimerRef.current = setTimeout(() => { dispatchRef.current({ type: "POLL_IDLE_CAP" }); }, DEGRADED_POLL_IDLE_MAX_MS); diff --git a/apps/client/src/features/ai-chat/state/run-fsm.spec.md b/apps/client/src/features/ai-chat/state/run-fsm.spec.md index 15c504f5..7287012a 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.spec.md +++ b/apps/client/src/features/ai-chat/state/run-fsm.spec.md @@ -128,10 +128,12 @@ holds. **Pending column: empty.** | NEW | `pendingSupersedeRef` | **data** (send-plumbing) | the runId injected into the next `POST /stream {supersede}`; the single replacement for the 3 DELETED one-shots (#8/#9/#10) — net −2 refs | | NEW | `idleCapTimerRef` | **effect-owned timer** | the stalled inactivity cap → `POLL_IDLE_CAP` (commit 4a); not a flag | -Net: the 13 lifecycle flags (#1–#13) are eliminated (7 → FSM phase/ctx/epoch/event, -3 deleted, `reconnectTimerRef`/`attachAbortRef` become effect-owned controllers, -`mountedRef` retained as React liveness). Two effect-owned timers + one send-plumbing -data ref are added — none is a boolean lifecycle latch. +Net: the 13 lifecycle flags (#1–#13) are eliminated: **8** → FSM phase/ctx/epoch/event +(#1–#6, #11, #13), **3** deleted (#8/#9/#10), **`reconnectTimerRef` (#7)** becomes an +effect-owned controller, and **`mountedRef` (#12)** is retained as React liveness +(8 + 3 + 1 + 1 = 13). (`attachAbortRef` (#16) is outside the #1–#13 set — it was +already an effect-owned controller.) Two effect-owned timers + one send-plumbing data +ref are added — none is a boolean lifecycle latch. --- diff --git a/apps/client/src/features/ai-chat/state/run-fsm.test.ts b/apps/client/src/features/ai-chat/state/run-fsm.test.ts index 9155229f..d01913db 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.test.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.test.ts @@ -363,6 +363,75 @@ describe("run-fsm — stop (I4: exit by data)", () => { m = reduce(m, { type: "RUN_FACT", runFact: null, epoch: m.ctx.epoch }); expect(m.phase.name).toBe("idle"); }); + + // Review #4: `stopping` arms the poll but had no inactivity backstop. + it("review-4: POLL_IDLE_CAP in `stopping` exits to idle (bounded), NOT stalled", () => { + let m = reduce(withRunFact(), { type: "STOP_REQUESTED" }); + expect(m.phase.name).toBe("stopping"); + expect(hasEffect(m, "armPoll")).toBe(true); + // MUTATION-VERIFY: drop the `stopping` branch in POLL_IDLE_CAP and this hangs + // in `stopping` (poll forever) -> red. + m = reduce(m, { type: "POLL_IDLE_CAP" }); + expect(m.phase.name).toBe("idle"); + expect(hasEffect(m, "disarmPoll")).toBe(true); + expect(m.ctx.ownership).toBe("local"); + }); +}); + +// Review #1: positive attach outcomes must be guarded by the SOURCE phase — the +// epoch filter alone is insufficient because POLL_TERMINAL uses to() (no epoch +// bump) and does not abort the in-flight GET. +describe("run-fsm — review-1: attach outcomes guarded by source phase", () => { + it("a late RECONNECT_ATTACHED after POLL_TERMINAL stays idle (no phantom streaming)", () => { + let m = withRunFact("run-1"); + m = reduce(m, { type: "FINISH_DISCONNECT", hasVisibleContent: true, epoch: m.ctx.epoch }); + m = reduce(m, { type: "RECONNECT_ATTEMPT", attempt: 1, epoch: m.ctx.epoch }); // attach GET + const epoch = m.ctx.epoch; + // The armed degraded poll reaches the terminal row FIRST (epoch unchanged). + m = reduce(m, { type: "POLL_TERMINAL" }); + expect(m.phase.name).toBe("idle"); + expect(m.ctx.epoch).toBe(epoch); // POLL_TERMINAL did NOT bump the epoch + // The slow GET returns live 2xx under the SAME epoch — must NOT resurrect. + m = reduce(m, { type: "RECONNECT_ATTACHED", epoch }); + expect(m.phase.name).toBe("idle"); + }); + + it("a late ATTACH_LIVE / ATTACH_NONE after leaving `attaching` is ignored", () => { + let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); + const epoch = m.ctx.epoch; + m = reduce(m, { type: "ATTACH_NONE", epoch }); // attaching -> polling + m = reduce(m, { type: "POLL_TERMINAL" }); // -> idle (epoch unchanged) + expect(m.phase.name).toBe("idle"); + m = reduce(m, { type: "ATTACH_LIVE", epoch }); // late 2xx, same epoch + expect(m.phase.name).toBe("idle"); + // And a late ATTACH_NONE (not `attaching`) is a no-op too. + m = reduce(m, { type: "ATTACH_NONE", epoch }); + expect(m.phase.name).toBe("idle"); + }); +}); + +// Review #2: every terminal transition resets ownership to local. +describe("run-fsm — review-2: terminal transitions reset ownership to local", () => { + const observer = (): Machine => { + let m = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); + m = reduce(m, { type: "ATTACH_LIVE", epoch: m.ctx.epoch }); + expect(m.ctx.ownership).toBe("observer"); + return m; + }; + it("FINISH_CLEAN resets ownership", () => { + const m = reduce(observer(), { type: "FINISH_CLEAN", epoch: observer().ctx.epoch }); + expect(m.ctx.ownership).toBe("local"); + }); + it("FINISH_ERROR / POLL_TERMINAL / RUN_FACT(null) reset ownership", () => { + let o = observer(); + expect(reduce(o, { type: "FINISH_ERROR", kind: "stream", epoch: o.ctx.epoch }).ctx.ownership).toBe("local"); + // POLL_TERMINAL from an observer polling phase + let p = reduce(observer(), { type: "STREAM_INCOMPLETE", reason: "starved", epoch: observer().ctx.epoch }); + expect(reduce(p, { type: "POLL_TERMINAL" }).ctx.ownership).toBe("local"); + // RUN_FACT(null) from an observer attaching phase + let a = reduce(initialMachine(), { type: "ATTACH_START", runId: "r" }); + expect(reduce(a, { type: "RUN_FACT", runFact: null, epoch: a.ctx.epoch }).ctx.ownership).toBe("local"); + }); }); describe("run-fsm — ownership (I2) is context, orthogonal to phase", () => { diff --git a/apps/client/src/features/ai-chat/state/run-fsm.ts b/apps/client/src/features/ai-chat/state/run-fsm.ts index 2ca554b2..34c858b2 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.ts @@ -254,7 +254,9 @@ export function reduce(m: Machine, event: Event): Machine { // `superseding` (a successor B owns) — that is the F1 supersede drop. if (m.phase.name === "stopping" && isFinishEvent(event)) { return to(m, { name: "idle" }, { - ctx: { runFact: null, liveFollow: false }, + // Reset ownership to local on this terminal transition (review #2): otherwise + // an observer-stop leaves ownership 'observer' and hides "Send now" forever. + ctx: { runFact: null, liveFollow: false, ownership: "local" }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); } @@ -297,9 +299,10 @@ export function reduce(m: Machine, event: Event): Machine { case "FINISH_CLEAN": // A clean terminal outcome. The run is done — clear the run-fact and go // idle. (The queue flush is a component concern gated by ownership; the - // FSM only models the phase.) + // FSM only models the phase.) Review #2: reset ownership to local so a + // just-finished observer-attach turn re-exposes "Send now" for the queue. return to(m, { name: "idle" }, { - ctx: { runFact: null, liveFollow: false }, + ctx: { runFact: null, liveFollow: false, ownership: "local" }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); @@ -307,7 +310,7 @@ export function reduce(m: Machine, event: Event): Machine { // A user Stop / intentional abort finished. If we were stopping, the // terminal data has now arrived (I4) — go idle. The run-fact is cleared. return to(m, { name: "idle" }, { - ctx: { runFact: null, liveFollow: false }, + ctx: { runFact: null, liveFollow: false, ownership: "local" }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); @@ -340,11 +343,13 @@ export function reduce(m: Machine, event: Event): Machine { }); } // No run to recover: a plain disconnect. Surface the terminal notice. - return to(m, { name: "idle" }, { ctx: { runFact: null, liveFollow: false } }); + return to(m, { name: "idle" }, { + ctx: { runFact: null, liveFollow: false, ownership: "local" }, + }); case "FINISH_ERROR": return to(m, { name: "error", kind: event.kind }, { - ctx: { runFact: null, liveFollow: false }, + ctx: { runFact: null, liveFollow: false, ownership: "local" }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); @@ -364,6 +369,12 @@ export function reduce(m: Machine, event: Event): Machine { case "ATTACH_LIVE": // The attach GET returned a live 2xx stream — follow it as an observer. + // Review #1: guard by SOURCE phase. The epoch filter alone is not enough — a + // POLL_TERMINAL uses to() (no epoch bump) and does not abort the in-flight + // GET, so a slow 2xx landing after the machine already left `attaching` (e.g. + // the armed poll saw the terminal row -> idle) would resurrect a settled run + // into a phantom `streaming`. Only enter streaming FROM `attaching`. + if (m.phase.name !== "attaching") return stay(m); return to(m, { name: "streaming" }); case "ATTACH_NONE": @@ -371,6 +382,9 @@ export function reduce(m: Machine, event: Event): Machine { // follow the run to terminal from the DB. This is a soft-negative run-fact // (204 on a non-stripped path is authoritative-negative; the runtime may // pass a RUN_FACT null separately). Keep the run-fact as-is here. + // Review #1: guard by source phase for consistency (a late outcome after the + // machine already left `attaching` must not re-arm a poll). + if (m.phase.name !== "attaching") return stay(m); return to(m, { name: "polling", reason: "attach-none" }, { effects: [{ type: "armPoll", reason: "attach-none" }], }); @@ -391,6 +405,12 @@ export function reduce(m: Machine, event: Event): Machine { // attempt counter is dropped, so a LATER disconnect can start a fresh // ladder from attempt 1 (the old one-shot `!wasResumed` gate forbade a // second cycle, sending the second break to silent poll). + // Review #1: guard by SOURCE phase. The armed degraded poll can reach the + // terminal row (POLL_TERMINAL -> idle, via to(), NO epoch bump, GET not + // aborted) BEFORE a slow reconnect GET returns 2xx; without this guard that + // late RECONNECT_ATTACHED (same epoch) would resurrect a settled run into a + // phantom `streaming`. Only re-enter streaming FROM `reconnecting`. + if (m.phase.name !== "reconnecting") return stay(m); return to(m, { name: "streaming" }, { effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }], }); @@ -437,13 +457,24 @@ export function reduce(m: Machine, event: Event): Machine { case "POLL_TERMINAL": // The run reached a terminal row via the poll (or the reconcile merge). Go // idle and disarm everything (I4: this is a DATA-driven exit, incl. exit - // from `stopping`). + // from `stopping`). Review #2: reset ownership to local. return to(m, { name: "idle" }, { - ctx: { runFact: null, liveFollow: false }, + ctx: { runFact: null, liveFollow: false, ownership: "local" }, effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], }); case "POLL_IDLE_CAP": + // Review #4: `stopping` also arms the poll (STOP_REQUESTED) but has NO other + // backstop — an observer-stop with no SDK stream to fire onFinish, whose + // server stop never drives the run terminal, would poll the DB forever. Give + // it a bounded exit: cap -> idle + disarm (NOT `stalled`; Stop was already + // pressed, so there is nothing for the user to retry). + if (m.phase.name === "stopping") { + return to(m, { name: "idle" }, { + ctx: { runFact: null, liveFollow: false, ownership: "local" }, + effects: [{ type: "disarmPoll" }, { type: "cancelReconnect" }], + }); + } // #488 commit 4a: the poll hit the inactivity cap. Instead of going SILENT // (the old "forever half-done answer"), surface a stalled banner + Retry. if (m.phase.name !== "polling" && m.phase.name !== "reconnecting") return stay(m); @@ -464,7 +495,8 @@ export function reduce(m: Machine, event: Event): Machine { m.phase.name === "stopping" ) { return to(m, { name: "idle" }, { - ctx: { runFact: null, liveFollow: false }, + // Review #2: reset ownership to local on this terminal transition. + ctx: { runFact: null, liveFollow: false, ownership: "local" }, effects: [{ type: "cancelReconnect" }, { type: "disarmPoll" }], }); } From 0503e8b4b160b5e1fcde557ad422f5e1dababc1c Mon Sep 17 00:00:00 2001 From: agent_coder Date: Sat, 11 Jul 2026 19:23:10 +0300 Subject: [PATCH 41/41] =?UTF-8?q?fix(ai-chat):=20W1=20=E2=80=94=20=D0=BA?= =?UTF-8?q?=D0=BB=D0=B8=D0=B5=D0=BD=D1=82=20=D1=87=D0=B8=D1=82=D0=B0=D0=B5?= =?UTF-8?q?=D1=82=20409-=D0=BF=D0=BE=D0=BB=D0=B5=20activeRunId=20(=D0=B1?= =?UTF-8?q?=D1=8B=D0=BB=D0=BE=20runId=3Dundefined)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ревью волны 1 (agent_vscode): сервер эмитит id текущего рана в activeRunId на обоих 409-бранчах (SUPERSEDE_TARGET_MISMATCH, A_RUN_ALREADY_ACTIVE), а клиентский read409 читал runId → SUPERSEDE_MISMATCH{currentRunId} всегда undefined: быстрый supersede-хинт мёртв в проде, а клиентские тесты ложно-зелёные (мокали поле runId, которого сервер не шлёт). - read409 читает activeRunId (undefined-safe); оба мока — на реальную форму + ассерт на усвоенный currentRunId (mutation: revert→runId краснит тесты). - S4 (groundwork): activeRunId из A_RUN_ALREADY_ACTIVE поглощается в runFact (to(), без бампа эпохи/ownership — инварианты #488 целы). Полная обвязка supersede чужой вкладки из фазы error требует расширения gate в sendNow — отдельным follow-up; сейчас это безопасная заготовка, не рабочая фича. - spec.md: 2 строки контракт-таблицы приведены к activeRunId. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../ai-chat/components/chat-thread.test.tsx | 82 ++++++++++++++++++- .../ai-chat/components/chat-thread.tsx | 27 +++--- .../features/ai-chat/state/run-fsm.spec.md | 6 +- .../features/ai-chat/state/run-fsm.test.ts | 21 +++++ .../src/features/ai-chat/state/run-fsm.ts | 11 ++- .../ai-chat/utils/error-message.test.ts | 4 +- 6 files changed, 133 insertions(+), 18 deletions(-) diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index e15f8030..ba4dcb7c 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -414,7 +414,9 @@ describe("ChatThread — send now", () => { "fetch", vi.fn().mockResolvedValue( new Response( - JSON.stringify({ code: "SUPERSEDE_TARGET_MISMATCH", runId: "run-x" }), + // REAL server body shape: the current run id is `activeRunId`, NOT `runId` + // (see ai-chat.controller.ts SUPERSEDE_TARGET_MISMATCH branch). + JSON.stringify({ code: "SUPERSEDE_TARGET_MISMATCH", activeRunId: "run-x" }), { status: 409 }, ), ), @@ -426,6 +428,84 @@ describe("ChatThread — send now", () => { expect(h.state.getRun).toHaveBeenCalledWith("c1"); }); + it("#497/W1: the mismatch ABSORBS the server's activeRunId into runFact (fast hint is live) — a follow-up Send now CAS-targets it", async () => { + // The 409 body's current run id is `activeRunId`; read409 must feed THAT into + // SUPERSEDE_MISMATCH{currentRunId} -> runFact, else the fast hint is undefined. + // Observe the absorbed fact via the NEXT CAS supersede body. Keep the verify + // getRun PENDING so it cannot overwrite the absorbed fact with its own result. + h.state.getRun.mockReturnValue(new Promise(() => {})); // verify never resolves + startLocalStreamWithRun(); // runFact run-1, sending, local, autonomous + fireEvent.click(screen.getByTestId("queue-btn")); // X + fireEvent.click(screen.getByLabelText("Send now")); // -> superseding (target run-1) + // A's onFinish sends B and CLEARS the pending-supersede text (no-overlap). + await act(async () => { + h.state.onFinish?.({ + message: { id: "a1", role: "assistant", parts: [] }, + isAbort: true, + isDisconnect: false, + isError: false, + }); + await Promise.resolve(); + }); + // B's CAS POST -> 409 SUPERSEDE_TARGET_MISMATCH with the REAL field `activeRunId`. + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ code: "SUPERSEDE_TARGET_MISMATCH", activeRunId: "run-x" }), + { status: 409 }, + ), + ), + ); + await act(async () => { + await h.state.transport!.fetch!("http://x", { method: "POST", body: "{}" }); + }); + // runFact is now the absorbed "run-x". SEND_LOCAL preserves it; a fresh Send now + // then CAS-supersedes THAT run — surfacing runFact through the supersede body. + fireEvent.click(screen.getByTestId("send-btn")); // SEND_LOCAL -> sending, local + fireEvent.click(screen.getByTestId("queue-btn")); // Y + fireEvent.click(screen.getByLabelText("Send now")); // CAS -> arms pendingSupersede + const { body } = h.state.transport!.prepareSendMessagesRequest!({ + messages: [], + body: {}, + }); + // MUTATION-VERIFY: revert read409 to `runId` -> currentRunId undefined -> runFact + // stays "run-1" -> the CAS targets "run-1" -> this assertion reddens. + expect((body.supersede as { runId?: string } | undefined)?.runId).toBe("run-x"); + }); + + it("#497/S4: a plain 409 A_RUN_ALREADY_ACTIVE absorbs activeRunId into runFact so Send now CAS-targets the foreign run", async () => { + startLocalStreamWithRun(); // sending, local, autonomous, runFact run-1 + // A plain (non-supersede) POST hits the one-active-run gate. The FSM must adopt + // the server's activeRunId as the run-fact — NOT stay blind. + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ code: "A_RUN_ALREADY_ACTIVE", activeRunId: "run-foreign" }), + { status: 409 }, + ), + ), + ); + // Drive a NON-supersede POST (phase is `sending`, not `superseding`). + await act(async () => { + await h.state.transport!.fetch!("http://x", { method: "POST", body: "{}" }); + }); + // runFact is now "run-foreign". SEND_LOCAL preserves it; Send now CAS-targets it. + fireEvent.click(screen.getByTestId("send-btn")); // SEND_LOCAL -> sending, local + fireEvent.click(screen.getByTestId("queue-btn")); // Y + fireEvent.click(screen.getByLabelText("Send now")); // CAS -> arms pendingSupersede + const { body } = h.state.transport!.prepareSendMessagesRequest!({ + messages: [], + body: {}, + }); + // MUTATION-VERIFY: drop the activeRunId threading (or read the wrong field) -> + // runFact stays "run-1" -> the CAS targets "run-1" -> this assertion reddens. + expect((body.supersede as { runId?: string } | undefined)?.runId).toBe( + "run-foreign", + ); + }); + it("#488 review-3 sibling: a plain 409 A_RUN_ALREADY_ACTIVE shows the classified banner", async () => { renderThread({ autonomousRunsEnabled: true, initialRows: settledTail() }); // The SDK sets useChat error to the 409 body on the failed POST. diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index 24db89b6..e213da32 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -87,15 +87,19 @@ function isActiveRunStatus(status: string | null | undefined): boolean { return status === "pending" || status === "running"; } -/** Read the `{ code, runId }` off a JSON error response WITHOUT consuming the - * original body (reads a clone), so the caller can still return the Response - * untouched to the SDK. Any parse failure => empty. */ -async function read409(response: Response): Promise<{ code?: string; runId?: string }> { +/** Read the `{ code, activeRunId }` off a JSON error response WITHOUT consuming + * the original body (reads a clone), so the caller can still return the Response + * untouched to the SDK. `activeRunId` is the server's field for the run currently + * active on the chat — it is the name emitted on BOTH 409 branches + * (SUPERSEDE_TARGET_MISMATCH and A_RUN_ALREADY_ACTIVE, see ai-chat.controller.ts). + * Reading `runId` here (the field the server never sends) silently yields + * `undefined`. Any parse failure => empty. */ +async function read409(response: Response): Promise<{ code?: string; activeRunId?: string }> { try { - const b = (await response.clone().json()) as { code?: unknown; runId?: unknown }; + const b = (await response.clone().json()) as { code?: unknown; activeRunId?: unknown }; return { code: typeof b?.code === "string" ? b.code : undefined, - runId: typeof b?.runId === "string" ? b.runId : undefined, + activeRunId: typeof b?.activeRunId === "string" ? b.activeRunId : undefined, }; } catch { return {}; @@ -486,11 +490,11 @@ export default function ChatThread({ if (response.ok) { dispatchRef.current({ type: "SUPERSEDE_READY", epoch: ep }); } else if (response.status === 409) { - const { code, runId } = await read409(response); + const { code, activeRunId } = await read409(response); if (code === "SUPERSEDE_TARGET_MISMATCH") dispatchRef.current({ type: "SUPERSEDE_MISMATCH", - currentRunId: runId, + currentRunId: activeRunId, epoch: ep, }); else if (code === "SUPERSEDE_TIMEOUT") @@ -499,9 +503,12 @@ export default function ChatThread({ dispatchRef.current({ type: "SUPERSEDE_INVALID", epoch: ep }); } } else if (response.status === 409) { - const { code } = await read409(response); + const { code, activeRunId } = await read409(response); if (code === "A_RUN_ALREADY_ACTIVE") - dispatchRef.current({ type: "RUN_ALREADY_ACTIVE" }); + // S4: thread the server's activeRunId into the event so the FSM can + // adopt it as the run-fact — a later "Send now" then CAS-supersedes + // that (possibly foreign-tab) run instead of a blind promote+abort. + dispatchRef.current({ type: "RUN_ALREADY_ACTIVE", activeRunId }); } return response; } diff --git a/apps/client/src/features/ai-chat/state/run-fsm.spec.md b/apps/client/src/features/ai-chat/state/run-fsm.spec.md index 7287012a..4f70f744 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.spec.md +++ b/apps/client/src/features/ai-chat/state/run-fsm.spec.md @@ -56,7 +56,7 @@ Legend: **†** = command-transition (bumps `epoch`, I1). Effects in `[…]`. | `SUPERSEDE_MISMATCH{currentRunId}` (409 SUPERSEDE_TARGET_MISMATCH) | superseding | error(supersede-mismatch) | `[postRun(verify)]`, runFact←currentRunId | | `SUPERSEDE_TIMEOUT` (409 SUPERSEDE_TIMEOUT) | superseding | error(supersede-timeout) | — (composer keeps text; no auto-retry) | | `SUPERSEDE_INVALID` (409 SUPERSEDE_INVALID) | superseding | error(supersede-invalid) | — | -| `RUN_ALREADY_ACTIVE` (409 A_RUN_ALREADY_ACTIVE, plain POST) | sending | error(run-already-active) | — (composer offers supersede; NO auto-retry) | +| `RUN_ALREADY_ACTIVE{activeRunId}` (409 A_RUN_ALREADY_ACTIVE, plain POST) | sending | error(run-already-active) | runFact←activeRunId (composer offers supersede; NO auto-retry) | | `DISPOSE` (unmount) | any | idle **†** | `[abortAttach, cancelReconnect, disarmPoll]` (I1/I5 — epoch++ kills late callbacks) | **`stopping` honors any finish (re-review MEDIUM):** BEFORE the epoch filter, a @@ -90,8 +90,8 @@ share exactly the dispatched event set. | Server response | Event dispatched | error kind → banner | |---|---|---| -| 409 `A_RUN_ALREADY_ACTIVE` (plain POST) | `RUN_ALREADY_ACTIVE` | run-already-active → "already answering / interrupt & send" | -| 409 `SUPERSEDE_TARGET_MISMATCH` (+ body.runId) | `SUPERSEDE_MISMATCH{currentRunId}` | supersede-mismatch → verify via /run | +| 409 `A_RUN_ALREADY_ACTIVE` (+ body.activeRunId) | `RUN_ALREADY_ACTIVE{activeRunId}` | run-already-active → "already answering / interrupt & send" | +| 409 `SUPERSEDE_TARGET_MISMATCH` (+ body.activeRunId) | `SUPERSEDE_MISMATCH{currentRunId}` | supersede-mismatch → verify via /run | | 409 `SUPERSEDE_TIMEOUT` | `SUPERSEDE_TIMEOUT` | supersede-timeout → "couldn't interrupt in time, resend" | | 409 `SUPERSEDE_INVALID` | `SUPERSEDE_INVALID` | supersede-invalid → "couldn't interrupt this run" | | 503 `A_RUN_BEGIN_FAILED` | `FINISH_ERROR{begin-failed}` | begin-failed → "could not start, temporary" | diff --git a/apps/client/src/features/ai-chat/state/run-fsm.test.ts b/apps/client/src/features/ai-chat/state/run-fsm.test.ts index d01913db..9ff49acb 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.test.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.test.ts @@ -309,6 +309,27 @@ describe("run-fsm — commit 5: supersede CAS + error classification", () => { expect(m.phase).toEqual({ name: "error", kind: "run-already-active" }); expect(m.effects).toEqual([]); }); + + it("#497/S4: RUN_ALREADY_ACTIVE{activeRunId} ADOPTS the server's active run as the run-fact", () => { + // The server sends `activeRunId` so a later supersede can TARGET that run + // instead of a blind promote+abort. Absorb it into runFact. + const m = reduce(run(initialMachine(), { type: "SEND_LOCAL" }), { + type: "RUN_ALREADY_ACTIVE", + activeRunId: "run-foreign", + }); + expect(m.phase).toEqual({ name: "error", kind: "run-already-active" }); + expect(m.ctx.runFact).toEqual({ runId: "run-foreign" }); + expect(m.effects).toEqual([]); + }); + + it("#497/S4: RUN_ALREADY_ACTIVE without an activeRunId keeps the prior run-fact", () => { + const seeded = reduce(run(initialMachine(), { type: "SEND_LOCAL" }), { + type: "RUN_FACT", + runFact: { runId: "run-prior" }, + }); + const m = reduce(seeded, { type: "RUN_ALREADY_ACTIVE" }); + expect(m.ctx.runFact).toEqual({ runId: "run-prior" }); + }); }); // #488 F2 — a late mount `getRun → ATTACH_START` must not hijack a local turn. diff --git a/apps/client/src/features/ai-chat/state/run-fsm.ts b/apps/client/src/features/ai-chat/state/run-fsm.ts index 34c858b2..b83bbfc5 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.ts +++ b/apps/client/src/features/ai-chat/state/run-fsm.ts @@ -171,7 +171,7 @@ export type Event = | { type: "SUPERSEDE_MISMATCH"; currentRunId?: string; epoch?: number } | { type: "SUPERSEDE_TIMEOUT"; epoch?: number } | { type: "SUPERSEDE_INVALID"; epoch?: number } - | { type: "RUN_ALREADY_ACTIVE" } + | { type: "RUN_ALREADY_ACTIVE"; activeRunId?: string } // -- lifecycle -- | { type: "DISPOSE" }; @@ -567,8 +567,13 @@ export function reduce(m: Machine, event: Event): Machine { case "RUN_ALREADY_ACTIVE": // A plain POST hit the one-active-run gate. NO auto-retry — the composer - // offers "interrupt and send" (supersede) instead. - return to(m, { name: "error", kind: "run-already-active" }); + // offers "interrupt and send" (supersede) instead. #497/S4: adopt the + // server's activeRunId as the run-fact so that supersede can TARGET the + // (possibly foreign-tab) active run via the CAS, rather than a blind + // promote+abort that just 409s again. A stale/absent id keeps the prior fact. + return to(m, { name: "error", kind: "run-already-active" }, { + ctx: { runFact: event.activeRunId ? { runId: event.activeRunId } : m.ctx.runFact }, + }); // ---- lifecycle ----------------------------------------------------- case "DISPOSE": diff --git a/apps/client/src/features/ai-chat/utils/error-message.test.ts b/apps/client/src/features/ai-chat/utils/error-message.test.ts index 594503f5..04d94225 100644 --- a/apps/client/src/features/ai-chat/utils/error-message.test.ts +++ b/apps/client/src/features/ai-chat/utils/error-message.test.ts @@ -56,8 +56,10 @@ describe("describeChatError", () => { }); it("classifies SUPERSEDE_TARGET_MISMATCH (409) as run-changed", () => { + // Real server body shape: the current run id is `activeRunId` (NOT `runId`) — + // see ai-chat.controller.ts. describeChatError classifies off `code` only. const body = - '{"message":"active run does not match the supersede target","code":"SUPERSEDE_TARGET_MISMATCH","runId":"run-x","statusCode":409}'; + '{"message":"active run does not match the supersede target","code":"SUPERSEDE_TARGET_MISMATCH","activeRunId":"run-x","statusCode":409}'; expect(describeChatError(body, t).title).toBe( "Couldn't interrupt — the run changed", );