Compare commits
20 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 574267de06 | |||
| f46d89eafb | |||
| ee33a293b9 | |||
| 86830b860d | |||
| d0d2a7880f | |||
| 9acbc07f7d | |||
| a0eb3131a6 | |||
| 50bb086edf | |||
| f2ad0121a5 | |||
| 2194f423a1 | |||
| 5a6009c750 | |||
| 9685074237 | |||
| 22f687c39e | |||
| 0d4f719f47 | |||
| 572f0a2ab9 | |||
| 72c2d1687e | |||
| 96faa28220 | |||
| c9293e316b | |||
| 654ba9f249 | |||
| 9120ad3b2d |
@@ -86,11 +86,19 @@ const MIN_HEIGHT = 400;
|
|||||||
// Margin kept between the window and the viewport edges while dragging.
|
// Margin kept between the window and the viewport edges while dragging.
|
||||||
const EDGE_MARGIN = 8;
|
const EDGE_MARGIN = 8;
|
||||||
|
|
||||||
// #184 phase 1.5: hard cap on the degraded-poll fallback. The poll is armed when
|
// #184 phase 1.5 / #430: backstop for the degraded-poll fallback. The poll is
|
||||||
// a resume attempt could not attach to the live run and disarmed by the thread on
|
// armed when a resume attempt could not attach to the live run and disarmed by the
|
||||||
// settle / local stream; this cap is the ONLY backstop against an endless tick
|
// thread on settle / local stream; this cap is the ONLY backstop against an endless
|
||||||
// (a stuck 'streaming' row before the boot-sweep, or a user-tail 204 with no run).
|
// tick (a stuck 'streaming' row before the boot-sweep, or a user-tail 204 with no
|
||||||
const DEGRADED_POLL_MAX_MS = 10 * 60_000;
|
// run).
|
||||||
|
//
|
||||||
|
// #430: measured from RUN ACTIVITY, not from arm-time. A real autonomous run takes
|
||||||
|
// 11-25 min — longer than a fixed 10-min-from-start cap, which used to cut the poll
|
||||||
|
// off mid-run. Instead we cap on INACTIVITY: keep polling as long as the run is
|
||||||
|
// still making progress (its persisted rows keep changing), and only give up after
|
||||||
|
// this long with NO new activity. A genuinely stuck run produces no row changes, so
|
||||||
|
// the idle cap still bounds it; a long-but-progressing run polls to completion.
|
||||||
|
const DEGRADED_POLL_IDLE_MAX_MS = 10 * 60_000;
|
||||||
|
|
||||||
/** Compact token formatter: 1.2M / 3.4k / 950. */
|
/** Compact token formatter: 1.2M / 3.4k / 950. */
|
||||||
function formatTokens(n: number): string {
|
function formatTokens(n: number): string {
|
||||||
@@ -254,9 +262,12 @@ export default function AiChatWindow() {
|
|||||||
// onResumeFallback(true); the thread disarms it on settle / local stream. The
|
// onResumeFallback(true); the thread disarms it on settle / local stream. The
|
||||||
// window only OWNS the timer (armedAtRef stamps when it was armed for the cap).
|
// window only OWNS the timer (armedAtRef stamps when it was armed for the cap).
|
||||||
const [degradedPoll, setDegradedPoll] = useState(false);
|
const [degradedPoll, setDegradedPoll] = useState(false);
|
||||||
const armedAtRef = useRef(0);
|
// #430: timestamp of the LAST run activity while the poll is armed — stamped on
|
||||||
|
// arm and re-stamped whenever the polled rows change (see the effect below). The
|
||||||
|
// idle cap is measured from this, so a long-but-progressing run keeps polling.
|
||||||
|
const lastActivityAtRef = useRef(0);
|
||||||
const onResumeFallback = useCallback((active: boolean): void => {
|
const onResumeFallback = useCallback((active: boolean): void => {
|
||||||
if (active) armedAtRef.current = Date.now();
|
if (active) lastActivityAtRef.current = Date.now();
|
||||||
setDegradedPoll(active);
|
setDegradedPoll(active);
|
||||||
}, []);
|
}, []);
|
||||||
// Reset the degraded poll whenever the open chat changes: it is scoped to the
|
// Reset the degraded poll whenever the open chat changes: it is scoped to the
|
||||||
@@ -269,18 +280,28 @@ export default function AiChatWindow() {
|
|||||||
useAiChatMessagesQuery(
|
useAiChatMessagesQuery(
|
||||||
activeChatId ?? undefined,
|
activeChatId ?? undefined,
|
||||||
// DELIBERATELY DUMB (invariant 8 / task 2.4): poll every 2.5s while armed
|
// DELIBERATELY DUMB (invariant 8 / task 2.4): poll every 2.5s while armed
|
||||||
// and under the 10-min cap; otherwise off. NO error checks (TanStack v5
|
// and while the run is still active (#430: under the INACTIVITY cap, not a
|
||||||
// resets fetchFailureCount each fetch, so consecutive errors are not
|
// fixed-from-start cap); otherwise off. NO error checks (TanStack v5 resets
|
||||||
// expressible — and the poll must survive a server restart) and NO tail
|
// fetchFailureCount each fetch, so consecutive errors are not expressible —
|
||||||
// checks (the settled/local-stream semantics live in ChatThread, which
|
// and the poll must survive a server restart) and NO tail checks (the
|
||||||
// disarms via onResumeFallback(false)). The time cap is the only backstop.
|
// settled/local-stream semantics live in ChatThread, which disarms via
|
||||||
|
// onResumeFallback(false)). The idle cap is the only backstop.
|
||||||
() =>
|
() =>
|
||||||
degradedPoll === true &&
|
degradedPoll === true &&
|
||||||
Date.now() - armedAtRef.current < DEGRADED_POLL_MAX_MS
|
Date.now() - lastActivityAtRef.current < DEGRADED_POLL_IDLE_MAX_MS
|
||||||
? 2500
|
? 2500
|
||||||
: false,
|
: false,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// #430: re-stamp the activity clock whenever the polled rows change while the
|
||||||
|
// poll is armed. TanStack keeps the same `messageRows` reference across refetches
|
||||||
|
// that return deep-equal data (structural sharing), so a new reference means the
|
||||||
|
// run genuinely progressed — which extends the inactivity cap above. A stuck run
|
||||||
|
// yields no reference change, so the cap eventually fires and stops the poll.
|
||||||
|
useEffect(() => {
|
||||||
|
if (degradedPoll) lastActivityAtRef.current = Date.now();
|
||||||
|
}, [degradedPoll, messageRows]);
|
||||||
|
|
||||||
// #184 reconnect-and-live-follow. Whether detached agent runs are enabled for
|
// #184 reconnect-and-live-follow. Whether detached agent runs are enabled for
|
||||||
// this workspace. When the feature is off no runs are ever created, so the
|
// this workspace. When the feature is off no runs are ever created, so the
|
||||||
// resume attempt would only ever 204; gating ChatThread's resume on it avoids a
|
// resume attempt would only ever 204; gating ChatThread's resume on it avoids a
|
||||||
|
|||||||
@@ -739,3 +739,170 @@ function renderResumable(initialRows: IAiChatMessageRow[]) {
|
|||||||
act(() => view.rerender(<Wrapper rows={rows} />));
|
act(() => view.rerender(<Wrapper rows={rows} />));
|
||||||
return { rerender, onResumeFallback };
|
return { rerender, onResumeFallback };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// #430: auto-reconnect to a DETACHED run after a LIVE SSE disconnect. The mount
|
||||||
|
// path only resumes on mount/reload; these cover the missing trigger — a live
|
||||||
|
// `isDisconnect` on onFinish must (backoff-)re-attach WITHOUT a reload, pin+strip
|
||||||
|
// the live row to avoid duplicates, fall back to the degraded poll on a 204, and
|
||||||
|
// exhaust to a manual Retry.
|
||||||
|
describe("ChatThread — live reconnect after isDisconnect (#430)", () => {
|
||||||
|
// A LIVE local turn that just dropped: the settled tail existed before, and the
|
||||||
|
// partial assistant row lives only in `messages` (not persisted as a tail).
|
||||||
|
const settledTail = () => [
|
||||||
|
row("u1", "user", undefined, "hi"),
|
||||||
|
row("a1", "assistant", "succeeded", "done"),
|
||||||
|
];
|
||||||
|
// The partial assistant message onFinish hands us for the dropped LIVE turn.
|
||||||
|
const liveMsg = {
|
||||||
|
id: "a2",
|
||||||
|
role: "assistant",
|
||||||
|
parts: [{ type: "text", text: "partial live answer" }],
|
||||||
|
};
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
resetState();
|
||||||
|
// status "ready": with a live disconnect the mock is not streaming, so the
|
||||||
|
// status==="streaming" auto-clear effect stays out of the way.
|
||||||
|
h.state.status = "ready";
|
||||||
|
vi.useFakeTimers();
|
||||||
|
});
|
||||||
|
afterEach(() => {
|
||||||
|
vi.useRealTimers();
|
||||||
|
cleanup();
|
||||||
|
});
|
||||||
|
|
||||||
|
// Render a NON-resuming mount (settled tail -> no mount resume) with autonomous
|
||||||
|
// runs on, then simulate a live disconnect via onFinish.
|
||||||
|
function renderLiveThenDisconnect() {
|
||||||
|
const view = renderThread({
|
||||||
|
autonomousRunsEnabled: true,
|
||||||
|
initialRows: settledTail(),
|
||||||
|
});
|
||||||
|
// The settled tail must NOT have triggered a mount resume.
|
||||||
|
expect(h.state.resumeStream).not.toHaveBeenCalled();
|
||||||
|
act(() => {
|
||||||
|
h.state.onFinish?.({
|
||||||
|
message: liveMsg,
|
||||||
|
isAbort: false,
|
||||||
|
isDisconnect: true,
|
||||||
|
isError: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return view;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fire the pending (scheduled) attempt for `attempt` (backoff = 1s,2s,4s,...).
|
||||||
|
function advanceToAttempt(attempt: number) {
|
||||||
|
act(() => {
|
||||||
|
vi.advanceTimersByTime(1000 * 2 ** (attempt - 1));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simulate the reconnect GET returning 204 (nothing live) so the transport's
|
||||||
|
// no-active-stream recovery runs.
|
||||||
|
async function reconnect204() {
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn().mockResolvedValue({ status: 204, ok: false }),
|
||||||
|
);
|
||||||
|
await act(async () => {
|
||||||
|
await h.state.transport!.fetch!("http://x", { method: "GET" });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simulate the reconnect GET returning a live 2xx stream.
|
||||||
|
async function reconnect200() {
|
||||||
|
vi.stubGlobal(
|
||||||
|
"fetch",
|
||||||
|
vi.fn().mockResolvedValue({ status: 200, ok: true }),
|
||||||
|
);
|
||||||
|
await act(async () => {
|
||||||
|
await h.state.transport!.fetch!("http://x", { method: "GET" });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
it("calls resumeStream POST-mount (a live disconnect triggers a backoff reconnect)", () => {
|
||||||
|
renderLiveThenDisconnect();
|
||||||
|
// The banner shows immediately; the attach itself fires after the first backoff.
|
||||||
|
expect(screen.getByText(/reconnecting/i)).toBeTruthy();
|
||||||
|
expect(h.state.resumeStream).not.toHaveBeenCalled();
|
||||||
|
advanceToAttempt(1);
|
||||||
|
// resumeStream is now called AFTER mount — the bug was it only ever fired once
|
||||||
|
// on mount. The reconnect URL pins expect=live&anchor to OUR run.
|
||||||
|
expect(h.state.resumeStream).toHaveBeenCalledTimes(1);
|
||||||
|
expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe(
|
||||||
|
"/api/ai-chat/runs/c1/stream?expect=live&anchor=a2",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("strips the pinned live row before replay so content is NOT duplicated", () => {
|
||||||
|
renderLiveThenDisconnect();
|
||||||
|
advanceToAttempt(1);
|
||||||
|
// The attempt strips the anchor row from the store (the live replay rebuilds
|
||||||
|
// it). Apply the setMessages updater to prove it removes exactly the anchor.
|
||||||
|
const updater = h.state.setMessages.mock.calls.at(-1)![0] as (
|
||||||
|
prev: { id: string }[],
|
||||||
|
) => { id: string }[];
|
||||||
|
expect(updater([{ id: "u1" }, { id: "a2" }])).toEqual([{ id: "u1" }]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a live re-attach (2xx) clears the reconnect banner", async () => {
|
||||||
|
renderLiveThenDisconnect();
|
||||||
|
advanceToAttempt(1);
|
||||||
|
await reconnect200();
|
||||||
|
expect(screen.queryByText(/reconnecting/i)).toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("a 204 arms the degraded poll and backs off to the next attempt", async () => {
|
||||||
|
const { onResumeFallback } = renderLiveThenDisconnect();
|
||||||
|
advanceToAttempt(1);
|
||||||
|
expect(h.state.resumeStream).toHaveBeenCalledTimes(1);
|
||||||
|
await reconnect204();
|
||||||
|
// Fallback engaged: the degraded poll is armed (204 -> onNoActiveStream).
|
||||||
|
expect(onResumeFallback).toHaveBeenCalledWith(true);
|
||||||
|
// Still reconnecting — the banner advanced to attempt 2/5.
|
||||||
|
expect(screen.getByText(/reconnecting.*2\/5/i)).toBeTruthy();
|
||||||
|
// The next backoff fires attempt 2 (another resumeStream).
|
||||||
|
advanceToAttempt(2);
|
||||||
|
expect(h.state.resumeStream).toHaveBeenCalledTimes(2);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("exhausts the attempt limit into a manual Retry, which restarts the sequence", async () => {
|
||||||
|
renderLiveThenDisconnect();
|
||||||
|
// Drive all 5 attempts, each failing with a 204.
|
||||||
|
for (let n = 1; n <= 5; n++) {
|
||||||
|
advanceToAttempt(n);
|
||||||
|
expect(h.state.resumeStream).toHaveBeenCalledTimes(n);
|
||||||
|
await reconnect204();
|
||||||
|
}
|
||||||
|
// The 5th 204 exhausted the cap -> the manual Retry replaces the banner.
|
||||||
|
expect(screen.queryByText(/reconnecting/i)).toBeNull();
|
||||||
|
const retry = screen.getByText("Retry");
|
||||||
|
expect(retry).toBeTruthy();
|
||||||
|
// Retry fires attempt 1 immediately (no backoff) — a 6th resumeStream.
|
||||||
|
act(() => {
|
||||||
|
fireEvent.click(retry);
|
||||||
|
});
|
||||||
|
expect(h.state.resumeStream).toHaveBeenCalledTimes(6);
|
||||||
|
expect(screen.getByText(/reconnecting/i)).toBeTruthy();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does NOT reconnect when autonomous runs are disabled", () => {
|
||||||
|
renderThread({ autonomousRunsEnabled: false, initialRows: settledTail() });
|
||||||
|
act(() => {
|
||||||
|
h.state.onFinish?.({
|
||||||
|
message: liveMsg,
|
||||||
|
isAbort: false,
|
||||||
|
isDisconnect: true,
|
||||||
|
isError: false,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
expect(screen.queryByText(/reconnecting/i)).toBeNull();
|
||||||
|
// The terminal "connection lost" notice is shown instead (unchanged behavior).
|
||||||
|
expect(
|
||||||
|
screen.getByText("Connection lost — the answer was interrupted."),
|
||||||
|
).toBeTruthy();
|
||||||
|
advanceToAttempt(1);
|
||||||
|
expect(h.state.resumeStream).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,7 +1,17 @@
|
|||||||
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
||||||
import { useQueryClient } from "@tanstack/react-query";
|
import { useQueryClient } from "@tanstack/react-query";
|
||||||
import { generateId } from "ai";
|
import { generateId } from "ai";
|
||||||
import { ActionIcon, Box, Group, Stack, Text, Tooltip } from "@mantine/core";
|
import {
|
||||||
|
ActionIcon,
|
||||||
|
Alert,
|
||||||
|
Box,
|
||||||
|
Button,
|
||||||
|
Group,
|
||||||
|
Loader,
|
||||||
|
Stack,
|
||||||
|
Text,
|
||||||
|
Tooltip,
|
||||||
|
} from "@mantine/core";
|
||||||
import {
|
import {
|
||||||
IconClockHour4,
|
IconClockHour4,
|
||||||
IconPlayerPlayFilled,
|
IconPlayerPlayFilled,
|
||||||
@@ -51,6 +61,15 @@ import classes from "@/features/ai-chat/components/ai-chat.module.css";
|
|||||||
// from the token rate.
|
// from the token rate.
|
||||||
const STREAM_THROTTLE_MS = 50;
|
const STREAM_THROTTLE_MS = 50;
|
||||||
|
|
||||||
|
// #430: auto-reconnect after a LIVE SSE disconnect of a DETACHED (autonomous) run.
|
||||||
|
// The run keeps executing server-side, so instead of a dead "Lost connection"
|
||||||
|
// banner we re-attach to the live tail through the SAME resumable machinery the
|
||||||
|
// mount path uses. Attempts back off exponentially and are capped; on exhaustion
|
||||||
|
// the user gets a manual Retry (the degraded poll keeps catching up underneath).
|
||||||
|
const RECONNECT_MAX_ATTEMPTS = 5;
|
||||||
|
// Backoff before attempt N (1-based): 1s, 2s, 4s, 8s, 16s.
|
||||||
|
const RECONNECT_BASE_DELAY_MS = 1000;
|
||||||
|
|
||||||
/** The page the user is currently viewing, sent as chat context. */
|
/** The page the user is currently viewing, sent as chat context. */
|
||||||
export interface OpenPageContext {
|
export interface OpenPageContext {
|
||||||
id: string;
|
id: string;
|
||||||
@@ -175,6 +194,10 @@ export default function ChatThread({
|
|||||||
const reconcileTailRef = useRef(false);
|
const reconcileTailRef = useRef(false);
|
||||||
const noStreamHandledRef = useRef(false);
|
const noStreamHandledRef = useRef(false);
|
||||||
const onNoActiveStreamRef = useRef<(() => void) | null>(null);
|
const onNoActiveStreamRef = useRef<(() => void) | null>(null);
|
||||||
|
// #430: called from the transport's reconnect-GET success branch when a live
|
||||||
|
// stream re-attached (2xx, not 204) — clears the reconnect banner. Kept in a ref
|
||||||
|
// because the transport's fetch closure (useMemo([])) reads it live.
|
||||||
|
const onReconnectAttachedRef = useRef<(() => void) | null>(null);
|
||||||
// Live mount flag. The attach GET and the resumed `onFinish` are async and can
|
// Live mount flag. The attach GET and the resumed `onFinish` are async and can
|
||||||
// land AFTER this thread unmounts (the parent remounts per chat via `key`); with
|
// land AFTER this thread unmounts (the parent remounts per chat via `key`); with
|
||||||
// chatIdRef then pointing at the NEW chat, an ungated late callback would arm a
|
// chatIdRef then pointing at the NEW chat, an ungated late callback would arm a
|
||||||
@@ -378,6 +401,10 @@ export default function ChatThread({
|
|||||||
// NOT drop the in-progress row or stop tracking the durable run.
|
// NOT drop the in-progress row or stop tracking the durable run.
|
||||||
if (response.status === 204 || !response.ok)
|
if (response.status === 204 || !response.ok)
|
||||||
onNoActiveStreamRef.current?.();
|
onNoActiveStreamRef.current?.();
|
||||||
|
// #430: a 2xx stream re-attached (live tail or finished-replay). Signal
|
||||||
|
// the reconnect controller to clear its banner. No-op outside an active
|
||||||
|
// reconnect sequence (e.g. the mount attach), so it is safe here.
|
||||||
|
else onReconnectAttachedRef.current?.();
|
||||||
return response;
|
return response;
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
// Network throw: same no-onFinish recovery, then rethrow so the SDK
|
// Network throw: same no-onFinish recovery, then rethrow so the SDK
|
||||||
@@ -481,6 +508,31 @@ export default function ChatThread({
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// (2b) #430: a LIVE (non-resumed) detached run whose SSE just dropped. The
|
||||||
|
// server run keeps executing, so instead of a dead "Lost connection" banner
|
||||||
|
// start a reconnect sequence: pin the CURRENT streaming assistant row as the
|
||||||
|
// strip/anchor (the live tail is the already-shown partial in `messages`, not
|
||||||
|
// a persistent row) and re-attach to the live tail via the resumable machinery.
|
||||||
|
const startedReconnect =
|
||||||
|
isDisconnect &&
|
||||||
|
!wasResumed &&
|
||||||
|
autonomousRunsEnabled === true &&
|
||||||
|
mountedRef.current &&
|
||||||
|
message?.role === "assistant" &&
|
||||||
|
typeof message.id === "string";
|
||||||
|
if (startedReconnect) {
|
||||||
|
beginReconnect({
|
||||||
|
id: message.id,
|
||||||
|
role: "assistant",
|
||||||
|
content: "",
|
||||||
|
status: "streaming",
|
||||||
|
createdAt: new Date().toISOString(),
|
||||||
|
// Preserve the partial parts so a 204 restore (onNoActiveStream) re-shows
|
||||||
|
// what was on screen while the degraded poll catches the run up to
|
||||||
|
// terminal (rowToUiMessage prefers metadata.parts).
|
||||||
|
metadata: { parts: message.parts },
|
||||||
|
});
|
||||||
|
}
|
||||||
// (3) Standard branches.
|
// (3) Standard branches.
|
||||||
// Forward the authoritative server chatId (streamed on the assistant
|
// Forward the authoritative server chatId (streamed on the assistant
|
||||||
// message metadata) so the parent adopts the REAL created chat id for a new
|
// message metadata) so the parent adopts the REAL created chat id for a new
|
||||||
@@ -490,9 +542,11 @@ export default function ChatThread({
|
|||||||
onTurnFinished(extractServerChatId(message), threadKey);
|
onTurnFinished(extractServerChatId(message), threadKey);
|
||||||
// Show a neutral "stopped" marker for an aborted turn; the red error banner
|
// Show a neutral "stopped" marker for an aborted turn; the red error banner
|
||||||
// (via `error`) already covers isError, and a clean finish clears any marker.
|
// (via `error`) already covers isError, and a clean finish clears any marker.
|
||||||
|
// On a live disconnect that STARTED a reconnect, suppress the terminal
|
||||||
|
// "connection lost" notice — the reconnect banner takes over (#430).
|
||||||
if (isError) setStopNotice(null);
|
if (isError) setStopNotice(null);
|
||||||
else if (isAbort) setStopNotice("manual");
|
else if (isAbort) setStopNotice("manual");
|
||||||
else if (isDisconnect) setStopNotice("disconnect");
|
else if (isDisconnect) setStopNotice(startedReconnect ? null : "disconnect");
|
||||||
else setStopNotice(null);
|
else setStopNotice(null);
|
||||||
// A resumed turn NEVER flushes the queue (invariant 7): skip BOTH the
|
// A resumed turn NEVER flushes the queue (invariant 7): skip BOTH the
|
||||||
// flush-on-abort branch and the plain flush. The local streamer is the only
|
// flush-on-abort branch and the plain flush. The local streamer is the only
|
||||||
@@ -579,6 +633,106 @@ export default function ChatThread({
|
|||||||
|
|
||||||
const isStreaming = status === "submitted" || status === "streaming";
|
const isStreaming = status === "submitted" || status === "streaming";
|
||||||
|
|
||||||
|
// #430: live-disconnect reconnect controller. `null` = idle; `{ trying, attempt }`
|
||||||
|
// = a backoff sequence is running (drives the "reconnecting… (N/max)" banner);
|
||||||
|
// `{ failed }` = attempts exhausted (drives the manual Retry). Mirrored into a ref
|
||||||
|
// so the transport/onNoActiveStream closures branch on the LIVE value.
|
||||||
|
type ReconnectState =
|
||||||
|
| null
|
||||||
|
| { phase: "trying"; attempt: number }
|
||||||
|
| { phase: "failed" };
|
||||||
|
const [reconnectState, setReconnectState] = useState<ReconnectState>(null);
|
||||||
|
const reconnectStateRef = useRef<ReconnectState>(null);
|
||||||
|
const setReconnectStatePair = useCallback((s: ReconnectState) => {
|
||||||
|
reconnectStateRef.current = s;
|
||||||
|
setReconnectState(s);
|
||||||
|
}, []);
|
||||||
|
const reconnectTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||||
|
const clearReconnectTimer = useCallback(() => {
|
||||||
|
if (reconnectTimerRef.current) {
|
||||||
|
clearTimeout(reconnectTimerRef.current);
|
||||||
|
reconnectTimerRef.current = null;
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// One reconnect attempt — MIRRORS the mount strip/anchor path for the LIVE case.
|
||||||
|
// beginReconnect pinned strippedRowRef/stripRef to the run's assistant row, so:
|
||||||
|
// - remove that row from the store (the mount path strips it from the SEED; here
|
||||||
|
// it is already shown, so filter it out) — the live replay's `text-start` then
|
||||||
|
// rebuilds it without DUPLICATING parts (the main dedup risk, #430);
|
||||||
|
// - reset the one-shot 204 guard so onNoActiveStream can fire for THIS attempt;
|
||||||
|
// - mark the turn resumed (invariant 7/8) so onFinish runs the recovery block and
|
||||||
|
// never flushes the queue;
|
||||||
|
// - resumeStream() -> prepareReconnectToStreamRequest builds
|
||||||
|
// ?expect=live&anchor=<pinned id>, pinning the replay to OUR run (invariant 6).
|
||||||
|
const attemptReconnectOnce = useCallback(
|
||||||
|
(attempt: number) => {
|
||||||
|
if (!mountedRef.current) return;
|
||||||
|
const anchor = strippedRowRef.current;
|
||||||
|
if (anchor) {
|
||||||
|
setMessages((prev) => prev.filter((m) => m.id !== anchor.id));
|
||||||
|
}
|
||||||
|
noStreamHandledRef.current = false;
|
||||||
|
setResumedTurnPair(true);
|
||||||
|
setReconnectStatePair({ phase: "trying", attempt });
|
||||||
|
void resumeStream();
|
||||||
|
},
|
||||||
|
[setMessages, setResumedTurnPair, setReconnectStatePair, resumeStream],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Schedule attempt `attempt` after an exponential backoff.
|
||||||
|
const scheduleReconnectAttempt = useCallback(
|
||||||
|
(attempt: number) => {
|
||||||
|
clearReconnectTimer();
|
||||||
|
setReconnectStatePair({ phase: "trying", attempt });
|
||||||
|
reconnectTimerRef.current = setTimeout(
|
||||||
|
() => attemptReconnectOnce(attempt),
|
||||||
|
RECONNECT_BASE_DELAY_MS * 2 ** (attempt - 1),
|
||||||
|
);
|
||||||
|
},
|
||||||
|
[clearReconnectTimer, setReconnectStatePair, attemptReconnectOnce],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Start a fresh reconnect sequence, pinning `anchorRow` (the live run's assistant
|
||||||
|
// row) as the strip/anchor reused by every attempt.
|
||||||
|
const beginReconnect = useCallback(
|
||||||
|
(anchorRow: IAiChatMessageRow) => {
|
||||||
|
if (!autonomousRunsEnabled || !mountedRef.current) return;
|
||||||
|
strippedRowRef.current = anchorRow;
|
||||||
|
stripRef.current = true;
|
||||||
|
scheduleReconnectAttempt(1);
|
||||||
|
},
|
||||||
|
[autonomousRunsEnabled, scheduleReconnectAttempt],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Manual Retry (shown once attempts are exhausted): restart at attempt 1 and fire
|
||||||
|
// immediately (the user asked for it now — no backoff).
|
||||||
|
const retryReconnect = useCallback(() => {
|
||||||
|
clearReconnectTimer();
|
||||||
|
attemptReconnectOnce(1);
|
||||||
|
}, [clearReconnectTimer, attemptReconnectOnce]);
|
||||||
|
|
||||||
|
// Live SSE re-attached (the reconnect GET returned a 2xx stream): clear the
|
||||||
|
// banner + any pending backoff. No-op outside a sequence (e.g. the mount attach).
|
||||||
|
const onReconnectAttached = useCallback(() => {
|
||||||
|
if (!mountedRef.current || !reconnectStateRef.current) return;
|
||||||
|
clearReconnectTimer();
|
||||||
|
setReconnectStatePair(null);
|
||||||
|
}, [clearReconnectTimer, setReconnectStatePair]);
|
||||||
|
onReconnectAttachedRef.current = onReconnectAttached;
|
||||||
|
|
||||||
|
// The reconnect GET could not attach (204 / error). onNoActiveStream has already
|
||||||
|
// armed the degraded poll (the robust fallback that drives the row to terminal
|
||||||
|
// from the DB), so this only decides the LIVE-attach retry: back off and try
|
||||||
|
// again up to the cap, else surface the manual Retry.
|
||||||
|
const onReconnectNoStream = useCallback(() => {
|
||||||
|
const s = reconnectStateRef.current;
|
||||||
|
if (s?.phase !== "trying") return;
|
||||||
|
if (s.attempt < RECONNECT_MAX_ATTEMPTS)
|
||||||
|
scheduleReconnectAttempt(s.attempt + 1);
|
||||||
|
else setReconnectStatePair({ phase: "failed" });
|
||||||
|
}, [scheduleReconnectAttempt, setReconnectStatePair]);
|
||||||
|
|
||||||
// 204-handler (`onNoActiveStream`): the attach returned 204 — nothing live to
|
// 204-handler (`onNoActiveStream`): the attach returned 204 — nothing live to
|
||||||
// resume (overflow / begin-failure / after retention / anchor-mismatch). One-
|
// resume (overflow / begin-failure / after retention / anchor-mismatch). One-
|
||||||
// shot via noStreamHandledRef (we do NOT null onNoActiveStreamRef). Exactly four
|
// shot via noStreamHandledRef (we do NOT null onNoActiveStreamRef). Exactly four
|
||||||
@@ -610,7 +764,17 @@ export default function ChatThread({
|
|||||||
// (d) 204 means onFinish will NOT fire — clear the suppression flag so it
|
// (d) 204 means onFinish will NOT fire — clear the suppression flag so it
|
||||||
// cannot swallow the NEXT local turn's queue flush.
|
// cannot swallow the NEXT local turn's queue flush.
|
||||||
setResumedTurnPair(false);
|
setResumedTurnPair(false);
|
||||||
}, [setMessages, queryClient, onResumeFallback, setResumedTurnPair]);
|
// (e) #430: if this 204/error landed during a live-disconnect reconnect
|
||||||
|
// sequence, back off and retry the live attach (or give up to the manual
|
||||||
|
// Retry). The degraded poll armed in (c) is the fallback either way.
|
||||||
|
onReconnectNoStream();
|
||||||
|
}, [
|
||||||
|
setMessages,
|
||||||
|
queryClient,
|
||||||
|
onResumeFallback,
|
||||||
|
setResumedTurnPair,
|
||||||
|
onReconnectNoStream,
|
||||||
|
]);
|
||||||
onNoActiveStreamRef.current = onNoActiveStream;
|
onNoActiveStreamRef.current = onNoActiveStream;
|
||||||
|
|
||||||
// Mount effect: kick off the resume attempt for a non-settled tail. Marking the
|
// Mount effect: kick off the resume attempt for a non-settled tail. Marking the
|
||||||
@@ -628,6 +792,9 @@ export default function ChatThread({
|
|||||||
return () => {
|
return () => {
|
||||||
mountedRef.current = false;
|
mountedRef.current = false;
|
||||||
attachAbortRef.current?.abort();
|
attachAbortRef.current?.abort();
|
||||||
|
// #430: drop any pending reconnect backoff so it can't fire against the next
|
||||||
|
// chat this thread's refs are reused for.
|
||||||
|
if (reconnectTimerRef.current) clearTimeout(reconnectTimerRef.current);
|
||||||
};
|
};
|
||||||
// Mount-only by design; the parent remounts per chat via `key`.
|
// Mount-only by design; the parent remounts per chat via `key`.
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
@@ -666,12 +833,27 @@ export default function ChatThread({
|
|||||||
if (tail.status !== "streaming") {
|
if (tail.status !== "streaming") {
|
||||||
reconcileTailRef.current = false;
|
reconcileTailRef.current = false;
|
||||||
onResumeFallback?.(false);
|
onResumeFallback?.(false);
|
||||||
|
// #430: the run reached its terminal state via the degraded poll — there is
|
||||||
|
// no live tail left to reconnect to, so drop any reconnect banner / Retry.
|
||||||
|
clearReconnectTimer();
|
||||||
|
setReconnectStatePair(null);
|
||||||
}
|
}
|
||||||
// onResumeFallback intentionally omitted (parent-stable callback); deps are
|
// onResumeFallback intentionally omitted (parent-stable callback); deps are
|
||||||
// fixed by the resume design.
|
// fixed by the resume design.
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, [initialRows, isStreaming, setMessages]);
|
}, [initialRows, isStreaming, setMessages]);
|
||||||
|
|
||||||
|
// #430: a real stream is live again — the reconnect re-attached to the live tail
|
||||||
|
// (status -> "streaming") OR the user started a new local turn. Either way clear
|
||||||
|
// the reconnect banner + any pending backoff. Gated on "streaming" (not the
|
||||||
|
// broader "submitted") so a still-pending attach GET does not clear prematurely.
|
||||||
|
useEffect(() => {
|
||||||
|
if (status === "streaming") {
|
||||||
|
clearReconnectTimer();
|
||||||
|
setReconnectStatePair(null);
|
||||||
|
}
|
||||||
|
}, [status, clearReconnectTimer, setReconnectStatePair]);
|
||||||
|
|
||||||
// "Send now" on a queued message: interrupt the current turn and immediately
|
// "Send now" on a queued message: interrupt the current turn and immediately
|
||||||
// send THIS message, keeping the agent's partial output. Other queued messages
|
// send THIS message, keeping the agent's partial output. Other queued messages
|
||||||
// stay queued and flush normally after the new turn. Reuses the existing
|
// stay queued and flush normally after the new turn. Reuses the existing
|
||||||
@@ -719,6 +901,9 @@ export default function ChatThread({
|
|||||||
// observer's Stop would otherwise leave the attach fetch running.
|
// observer's Stop would otherwise leave the attach fetch running.
|
||||||
attachAbortRef.current?.abort();
|
attachAbortRef.current?.abort();
|
||||||
stop();
|
stop();
|
||||||
|
// #430: pressing Stop also cancels an in-progress reconnect sequence.
|
||||||
|
clearReconnectTimer();
|
||||||
|
setReconnectStatePair(null);
|
||||||
if (!autonomousRunsEnabled) return;
|
if (!autonomousRunsEnabled) return;
|
||||||
if (chatIdRef.current) {
|
if (chatIdRef.current) {
|
||||||
onServerStop?.(chatIdRef.current);
|
onServerStop?.(chatIdRef.current);
|
||||||
@@ -740,7 +925,13 @@ export default function ChatThread({
|
|||||||
// for this fix. Documented so a future change can address the abort-ordering.
|
// for this fix. Documented so a future change can address the abort-ordering.
|
||||||
stopPendingRef.current = true;
|
stopPendingRef.current = true;
|
||||||
}
|
}
|
||||||
}, [stop, autonomousRunsEnabled, onServerStop]);
|
}, [
|
||||||
|
stop,
|
||||||
|
autonomousRunsEnabled,
|
||||||
|
onServerStop,
|
||||||
|
clearReconnectTimer,
|
||||||
|
setReconnectStatePair,
|
||||||
|
]);
|
||||||
|
|
||||||
// Clear the stopped marker as soon as a new turn begins streaming, and drop any
|
// Clear the stopped marker as soon as a new turn begins streaming, and drop any
|
||||||
// stale "Send now" interrupt flags. On the legit interrupt path both refs are
|
// stale "Send now" interrupt flags. On the legit interrupt path both refs are
|
||||||
@@ -825,6 +1016,43 @@ export default function ChatThread({
|
|||||||
detail={errorView.detail}
|
detail={errorView.detail}
|
||||||
mb="xs"
|
mb="xs"
|
||||||
/>
|
/>
|
||||||
|
) : reconnectState ? (
|
||||||
|
// #430: while auto-reconnecting to a detached run's live tail, show progress
|
||||||
|
// instead of a dead "Lost connection" banner; once attempts are exhausted,
|
||||||
|
// offer a manual Retry (the degraded poll keeps catching up underneath).
|
||||||
|
<Alert
|
||||||
|
variant="light"
|
||||||
|
color="gray"
|
||||||
|
p="xs"
|
||||||
|
mb="xs"
|
||||||
|
style={{ flexShrink: 0 }}
|
||||||
|
>
|
||||||
|
<Group gap={8} wrap="nowrap" align="center">
|
||||||
|
{reconnectState.phase === "trying" ? (
|
||||||
|
<>
|
||||||
|
<Loader size={14} color="gray" style={{ flex: "none" }} />
|
||||||
|
<Text size="sm" lh={1.3} c="dimmed">
|
||||||
|
{t("Connection lost — reconnecting…")}
|
||||||
|
{` (${reconnectState.attempt}/${RECONNECT_MAX_ATTEMPTS})`}
|
||||||
|
</Text>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<>
|
||||||
|
<Text size="sm" lh={1.3} c="dimmed" style={{ flex: 1 }}>
|
||||||
|
{t("Couldn't reconnect to the answer.")}
|
||||||
|
</Text>
|
||||||
|
<Button
|
||||||
|
size="compact-xs"
|
||||||
|
variant="light"
|
||||||
|
color="gray"
|
||||||
|
onClick={retryReconnect}
|
||||||
|
>
|
||||||
|
{t("Retry")}
|
||||||
|
</Button>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</Group>
|
||||||
|
</Alert>
|
||||||
) : stopNotice ? (
|
) : stopNotice ? (
|
||||||
<ChatStoppedNotice
|
<ChatStoppedNotice
|
||||||
text={
|
text={
|
||||||
|
|||||||
@@ -49,19 +49,14 @@ export default function FootnoteDefinitionView(props: NodeViewProps) {
|
|||||||
className={classes.definition}
|
className={classes.definition}
|
||||||
style={{ ["--footnote-number" as any]: `"${number}"` }}
|
style={{ ["--footnote-number" as any]: `"${number}"` }}
|
||||||
>
|
>
|
||||||
{/* #146: contentDOM MUST be the first child — a non-editable marker before
|
{/* #146: contentDOM MUST be the first child — non-editable chrome before
|
||||||
it makes click hit-testing snap the caret above. Content first; the
|
it makes click hit-testing snap the caret above. Content first; the
|
||||||
marker + back-link follow in DOM and are placed left/right via CSS
|
back-link follows in DOM and is placed on the right via CSS flex. The
|
||||||
flex `order`. The second #146 mitigation lives in
|
decorative "N." number is rendered inline via the .definitionContent
|
||||||
|
::before rule (from the --footnote-number var), so no marker element
|
||||||
|
precedes the content. The second #146 mitigation lives in
|
||||||
editor-paste-handler.tsx (reflowAfterPaste). */}
|
editor-paste-handler.tsx (reflowAfterPaste). */}
|
||||||
<NodeViewContent className={classes.definitionContent} />
|
<NodeViewContent className={classes.definitionContent} />
|
||||||
<span
|
|
||||||
className={classes.definitionMarker}
|
|
||||||
contentEditable={false}
|
|
||||||
aria-hidden="true"
|
|
||||||
>
|
|
||||||
{number}.
|
|
||||||
</span>
|
|
||||||
{refCount > 1 ? (
|
{refCount > 1 ? (
|
||||||
// Multiple references -> ↩ followed by one lettered link per occurrence.
|
// Multiple references -> ↩ followed by one lettered link per occurrence.
|
||||||
<span
|
<span
|
||||||
|
|||||||
@@ -81,34 +81,34 @@
|
|||||||
.definition {
|
.definition {
|
||||||
display: flex;
|
display: flex;
|
||||||
align-items: flex-start;
|
align-items: flex-start;
|
||||||
/* Tight number→text spacing (~one space) so it reads like "1. text"
|
/* Tight spacing between the content and the trailing ↩ back-link. */
|
||||||
instead of leaving a wide gap after the period. */
|
gap: 0.3em;
|
||||||
gap: 0.4em;
|
|
||||||
padding: 2px 0;
|
padding: 2px 0;
|
||||||
|
/* Footnotes read smaller than body text (16px). Matches .listHeading. */
|
||||||
|
font-size: var(--mantine-font-size-sm);
|
||||||
}
|
}
|
||||||
|
|
||||||
.definitionMarker {
|
/* The "N." number is decorative (from the --footnote-number CSS var on the
|
||||||
order: -1; /* keep the "N." marker on the LEFT though it follows content in DOM (#146) */
|
wrapper, never in the document model) and is rendered inline at the start of
|
||||||
flex: 0 0 auto;
|
the first content line via ::before. This keeps text and wrapped lines flush
|
||||||
min-width: 1.5em;
|
to the left margin — no hanging indent — while the editable contentDOM stays
|
||||||
/* Right-align within the narrow column so the period sits next to the text
|
the FIRST DOM child (#146). */
|
||||||
and multi-digit numbers (10, 11, …) stay aligned on their right edge. */
|
|
||||||
text-align: right;
|
|
||||||
font-variant-numeric: tabular-nums;
|
|
||||||
color: var(--mantine-color-dimmed);
|
|
||||||
user-select: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.definitionContent {
|
.definitionContent {
|
||||||
flex: 1 1 auto;
|
flex: 1 1 auto;
|
||||||
min-width: 0;
|
min-width: 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
/* The inner editable paragraph inherits `.ProseMirror p { margin: 0.5em 0 }`,
|
.definitionContent > :first-child::before {
|
||||||
which pushes the first text line ~0.5em below the "N." marker (aligned to
|
content: var(--footnote-number, "?") ". ";
|
||||||
flex-start), making the number float above the text. Drop the outer margins
|
color: var(--mantine-color-dimmed);
|
||||||
so the marker and the first line share the same top edge — same approach
|
font-variant-numeric: tabular-nums;
|
||||||
used for callouts in core.css. */
|
user-select: none;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* The inner editable paragraph inherits `.ProseMirror p { margin: 0.5em 0 }`.
|
||||||
|
Drop the outer margins so the definition sits tight to the heading above and
|
||||||
|
the ::before number aligns with the top of the row — same approach used for
|
||||||
|
callouts in core.css. */
|
||||||
.definitionContent > :first-child {
|
.definitionContent > :first-child {
|
||||||
margin-top: 0;
|
margin-top: 0;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
/**
|
||||||
|
* gitmost #401 — regression test for the connect-vs-unload race in
|
||||||
|
* @hocuspocus/server 3.4.4 (patched via patches/@hocuspocus__server@3.4.4.patch).
|
||||||
|
*
|
||||||
|
* The race (unpatched): when the last client disconnects, storeDocumentHooks'
|
||||||
|
* `finally` schedules an async `unloadDocument`. That unload runs its
|
||||||
|
* `beforeUnloadDocument` hooks asynchronously and, meanwhile, records an
|
||||||
|
* in-flight promise in `this.unloadingDocuments`. In the original 3.4.4
|
||||||
|
* `createDocument`, a NEW connection arriving in that window falls straight
|
||||||
|
* through to the `loadingDocuments`/`documents` checks — it never consults
|
||||||
|
* `unloadingDocuments`. So the new connection can start loading (or reuse) a
|
||||||
|
* document while the old instance is still being torn down; the re-check inside
|
||||||
|
* unload (`shouldUnloadDocument`, which sees 0 connections because async auth
|
||||||
|
* hooks have not registered the new connection yet) then deletes/destroys the
|
||||||
|
* doc out from under the freshly-connected client → orphaned Document → later
|
||||||
|
* redis-sync takes the "doc not loaded" path → sync never completes → the
|
||||||
|
* provider hangs until its ~25s timeout.
|
||||||
|
*
|
||||||
|
* The patch: `createDocument` first awaits any in-flight
|
||||||
|
* `unloadingDocuments.get(name)` before proceeding. Once that settles, the
|
||||||
|
* decision is deterministic — either the doc was fully unloaded (gone from
|
||||||
|
* `documents`, so a clean fresh load) or the unload aborted (healthy doc still
|
||||||
|
* in `documents`, reused). The new connection can never hand-shake onto an
|
||||||
|
* about-to-be-destroyed Document.
|
||||||
|
*
|
||||||
|
* These tests exercise the REAL patched `Hocuspocus.createDocument` (the class
|
||||||
|
* is directly constructible) by seeding `unloadingDocuments` with a controllable
|
||||||
|
* in-flight unload and observing that createDocument waits for it.
|
||||||
|
*/
|
||||||
|
import { Hocuspocus } from '@hocuspocus/server';
|
||||||
|
|
||||||
|
// A promise we can resolve on demand, to model an unload that is mid-flight.
|
||||||
|
function deferred<T = void>() {
|
||||||
|
let resolve!: (v: T) => void;
|
||||||
|
const promise = new Promise<T>((r) => {
|
||||||
|
resolve = r;
|
||||||
|
});
|
||||||
|
return { promise, resolve };
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('gitmost #401 — hocuspocus createDocument awaits in-flight unload', () => {
|
||||||
|
it('does NOT start loading a new doc until the in-flight unload settles, then loads fresh', async () => {
|
||||||
|
const hp = new Hocuspocus();
|
||||||
|
const name = 'page.race';
|
||||||
|
|
||||||
|
// Observe loadDocument: on the unpatched code it is invoked synchronously
|
||||||
|
// within createDocument (before the unload settles); on the patched code it
|
||||||
|
// must be deferred until unloadingDocuments resolves.
|
||||||
|
const freshDoc = { name, __fresh: true } as any;
|
||||||
|
const loadSpy = jest
|
||||||
|
.spyOn(hp as any, 'loadDocument')
|
||||||
|
.mockResolvedValue(freshDoc);
|
||||||
|
|
||||||
|
// Model an unload in progress: an entry sits in unloadingDocuments and, when
|
||||||
|
// it completes, it removes the doc from `documents` (a real full unload).
|
||||||
|
const unload = deferred();
|
||||||
|
(hp as any).documents.set(name, { name, __dying: true });
|
||||||
|
(hp as any).unloadingDocuments.set(
|
||||||
|
name,
|
||||||
|
unload.promise.then(() => {
|
||||||
|
(hp as any).documents.delete(name);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
// Kick off a new connection's createDocument but do not await it yet.
|
||||||
|
const createPromise = (hp as any).createDocument(
|
||||||
|
name,
|
||||||
|
{},
|
||||||
|
'socket-1',
|
||||||
|
{ isAuthenticated: true, readOnly: false },
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
|
||||||
|
// Let all currently-schedulable microtasks run. The patched createDocument is
|
||||||
|
// now parked on `await unloadingDocuments.get(name)`, so loadDocument must
|
||||||
|
// NOT have been called yet, and it must NOT have returned the dying doc.
|
||||||
|
await Promise.resolve();
|
||||||
|
await Promise.resolve();
|
||||||
|
expect(loadSpy).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
// The unload completes (doc removed from `documents`).
|
||||||
|
unload.resolve();
|
||||||
|
|
||||||
|
// createDocument now proceeds: sees no existing doc → fresh load.
|
||||||
|
const doc = await createPromise;
|
||||||
|
expect(loadSpy).toHaveBeenCalledTimes(1);
|
||||||
|
expect(doc).toBe(freshDoc);
|
||||||
|
// The freshly-loaded doc is the one registered — never the dying instance.
|
||||||
|
expect((hp as any).documents.get(name)).toBe(freshDoc);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('reuses the live doc when the in-flight unload aborts (doc left in documents)', async () => {
|
||||||
|
const hp = new Hocuspocus();
|
||||||
|
const name = 'page.abort';
|
||||||
|
|
||||||
|
const loadSpy = jest.spyOn(hp as any, 'loadDocument');
|
||||||
|
|
||||||
|
// Model an unload that ABORTS (e.g. a new connection reappeared before the
|
||||||
|
// sync re-check): it settles WITHOUT deleting the doc from `documents`.
|
||||||
|
const unload = deferred();
|
||||||
|
const liveDoc = { name, __live: true } as any;
|
||||||
|
(hp as any).documents.set(name, liveDoc);
|
||||||
|
(hp as any).unloadingDocuments.set(name, unload.promise); // no-op unload
|
||||||
|
|
||||||
|
const createPromise = (hp as any).createDocument(
|
||||||
|
name,
|
||||||
|
{},
|
||||||
|
'socket-2',
|
||||||
|
{ isAuthenticated: true, readOnly: false },
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
|
||||||
|
unload.resolve();
|
||||||
|
const doc = await createPromise;
|
||||||
|
|
||||||
|
// The still-present live doc is reused; no fresh load happened.
|
||||||
|
expect(doc).toBe(liveDoc);
|
||||||
|
expect(loadSpy).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('no in-flight unload → behaves normally (fresh load)', async () => {
|
||||||
|
const hp = new Hocuspocus();
|
||||||
|
const name = 'page.normal';
|
||||||
|
const freshDoc = { name } as any;
|
||||||
|
const loadSpy = jest
|
||||||
|
.spyOn(hp as any, 'loadDocument')
|
||||||
|
.mockResolvedValue(freshDoc);
|
||||||
|
|
||||||
|
const doc = await (hp as any).createDocument(
|
||||||
|
name,
|
||||||
|
{},
|
||||||
|
'socket-3',
|
||||||
|
{ isAuthenticated: true, readOnly: false },
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(loadSpy).toHaveBeenCalledTimes(1);
|
||||||
|
expect(doc).toBe(freshDoc);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
/**
|
||||||
|
* gitmost #401 fix 2 — onLoadDocument applies the DB state directly into the
|
||||||
|
* hook's target document and returns undefined (instead of building a NEW Y.Doc
|
||||||
|
* and returning it, which made hocuspocus re-encode+apply the whole state a
|
||||||
|
* SECOND time on every cold load).
|
||||||
|
*
|
||||||
|
* These tests assert:
|
||||||
|
* - the hook mutates `data.document` in place so its content equals the DB doc,
|
||||||
|
* - onLoadDocument returns undefined (so hocuspocus keeps the mutated doc and
|
||||||
|
* does NOT run its own applyUpdate(encodeStateAsUpdate(...)) merge),
|
||||||
|
* - both the raw-ydoc branch and the json→ydoc conversion branch behave so.
|
||||||
|
*
|
||||||
|
* Returning undefined is the observable signal that the double-encode is gone
|
||||||
|
* (the old code returned a new Y.Doc, which made hocuspocus re-encode+apply the
|
||||||
|
* state a second time); we assert that contract rather than counting internal
|
||||||
|
* encode calls, which is brittle given the encodes inside toYdoc and the test's
|
||||||
|
* own `expected` fixtures.
|
||||||
|
*/
|
||||||
|
import * as Y from 'yjs';
|
||||||
|
import { Document } from '@hocuspocus/server';
|
||||||
|
import { TiptapTransformer } from '@hocuspocus/transformer';
|
||||||
|
import { PersistenceExtension } from './persistence.extension';
|
||||||
|
import { tiptapExtensions } from '../collaboration.util';
|
||||||
|
|
||||||
|
// A fresh hocuspocus Document (extends Y.Doc, adds isEmpty()) as hocuspocus
|
||||||
|
// hands to onLoadDocument on a cold load.
|
||||||
|
const freshDoc = () => new Document(`page.${PAGE_ID}`, {});
|
||||||
|
|
||||||
|
const PAGE_ID = '550e8400-e29b-41d4-a716-446655440000';
|
||||||
|
|
||||||
|
const doc = (text: string) => ({
|
||||||
|
type: 'doc',
|
||||||
|
content: [{ type: 'paragraph', content: [{ type: 'text', text }] }],
|
||||||
|
});
|
||||||
|
|
||||||
|
const jsonOf = (ydoc: Y.Doc) =>
|
||||||
|
TiptapTransformer.fromYdoc(ydoc, 'default');
|
||||||
|
|
||||||
|
describe('PersistenceExtension.onLoadDocument — #401 fix 2 (apply-into-hook-doc)', () => {
|
||||||
|
let ext: PersistenceExtension;
|
||||||
|
let pageRepo: { findById: jest.Mock };
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
pageRepo = { findById: jest.fn() };
|
||||||
|
ext = new PersistenceExtension(
|
||||||
|
pageRepo as any,
|
||||||
|
{} as any,
|
||||||
|
{} as any,
|
||||||
|
{} as any,
|
||||||
|
{} as any,
|
||||||
|
{} as any,
|
||||||
|
{} as any,
|
||||||
|
{} as any,
|
||||||
|
);
|
||||||
|
jest.spyOn(ext['logger'], 'debug').mockImplementation(() => undefined);
|
||||||
|
jest.spyOn(ext['logger'], 'warn').mockImplementation(() => undefined);
|
||||||
|
});
|
||||||
|
|
||||||
|
const load = (document: Document) =>
|
||||||
|
ext.onLoadDocument({ documentName: `page.${PAGE_ID}`, document } as any);
|
||||||
|
|
||||||
|
it('raw ydoc branch: mutates the hook doc to the DB state and returns undefined', async () => {
|
||||||
|
// Source doc representing the persisted ydoc state.
|
||||||
|
const source = TiptapTransformer.toYdoc(
|
||||||
|
doc('DB CONTENT'),
|
||||||
|
'default',
|
||||||
|
tiptapExtensions,
|
||||||
|
);
|
||||||
|
const dbState = Buffer.from(Y.encodeStateAsUpdate(source));
|
||||||
|
pageRepo.findById.mockResolvedValue({ id: PAGE_ID, ydoc: dbState });
|
||||||
|
|
||||||
|
// The hook target is a fresh empty doc (as hocuspocus supplies on cold load).
|
||||||
|
const target = freshDoc();
|
||||||
|
const result = await load(target);
|
||||||
|
|
||||||
|
// Return undefined so hocuspocus keeps `target` as-is (no second merge).
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
// The hook document now carries the DB content.
|
||||||
|
expect(jsonOf(target)).toEqual(jsonOf(source));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('json→ydoc branch: converts page.content into the hook doc and returns undefined', async () => {
|
||||||
|
pageRepo.findById.mockResolvedValue({
|
||||||
|
id: PAGE_ID,
|
||||||
|
ydoc: null,
|
||||||
|
content: doc('JSON CONTENT'),
|
||||||
|
});
|
||||||
|
|
||||||
|
const target = freshDoc();
|
||||||
|
const result = await load(target);
|
||||||
|
|
||||||
|
// Returning undefined is what keeps hocuspocus from re-encoding+applying the
|
||||||
|
// state a second time (the old code returned the doc, forcing that extra
|
||||||
|
// encode). We assert the observable contract here — the return value and the
|
||||||
|
// resulting content — rather than counting internal encode calls, which is
|
||||||
|
// brittle: toYdoc and the `expected` build below both encode too.
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
|
||||||
|
// The converted content landed in the hook document.
|
||||||
|
const expected = TiptapTransformer.toYdoc(
|
||||||
|
doc('JSON CONTENT'),
|
||||||
|
'default',
|
||||||
|
tiptapExtensions,
|
||||||
|
);
|
||||||
|
expect(jsonOf(target)).toEqual(jsonOf(expected));
|
||||||
|
});
|
||||||
|
|
||||||
|
it('live doc already non-empty: early return, no DB read', async () => {
|
||||||
|
// A hocuspocus Document carrying live content (isEmpty('default') === false).
|
||||||
|
const target = freshDoc();
|
||||||
|
const live = TiptapTransformer.toYdoc(
|
||||||
|
doc('LIVE'),
|
||||||
|
'default',
|
||||||
|
tiptapExtensions,
|
||||||
|
);
|
||||||
|
Y.applyUpdate(target, Y.encodeStateAsUpdate(live));
|
||||||
|
|
||||||
|
const result = await load(target);
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
expect(pageRepo.findById).not.toHaveBeenCalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('no persisted state: leaves the fresh empty doc untouched, returns undefined', async () => {
|
||||||
|
pageRepo.findById.mockResolvedValue({ id: PAGE_ID, ydoc: null, content: null });
|
||||||
|
const target = freshDoc();
|
||||||
|
const result = await load(target);
|
||||||
|
expect(result).toBeUndefined();
|
||||||
|
expect(target.isEmpty('default')).toBe(true);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -171,15 +171,21 @@ export class PersistenceExtension implements Extension {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// #401 fix 2 — apply the DB state DIRECTLY into the hook's target document
|
||||||
|
// (`document` === `data.document`) and return undefined. When onLoadDocument
|
||||||
|
// returns undefined, hocuspocus keeps the mutated hook document as-is; only
|
||||||
|
// when the hook RETURNS a Y.Doc does hocuspocus re-`applyUpdate(document,
|
||||||
|
// encodeStateAsUpdate(returned))` — a second full encode+apply of the whole
|
||||||
|
// (e.g. 315KB) state on every cold load. Mutating in place performs a single
|
||||||
|
// apply and avoids the throwaway `new Y.Doc()` allocation.
|
||||||
if (page.ydoc) {
|
if (page.ydoc) {
|
||||||
this.logger.debug(`ydoc loaded from db: ${pageId}`);
|
this.logger.debug(`ydoc loaded from db: ${pageId}`);
|
||||||
|
|
||||||
const doc = new Y.Doc();
|
|
||||||
const dbState = new Uint8Array(page.ydoc);
|
const dbState = new Uint8Array(page.ydoc);
|
||||||
|
|
||||||
Y.applyUpdate(doc, dbState);
|
Y.applyUpdate(document, dbState);
|
||||||
observeCollabLoad(dbState.length, (performance.now() - startedAt) / 1000);
|
observeCollabLoad(dbState.length, (performance.now() - startedAt) / 1000);
|
||||||
return doc;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// if no ydoc state in db convert json in page.content to Ydoc.
|
// if no ydoc state in db convert json in page.content to Ydoc.
|
||||||
@@ -192,18 +198,23 @@ export class PersistenceExtension implements Extension {
|
|||||||
tiptapExtensions,
|
tiptapExtensions,
|
||||||
);
|
);
|
||||||
|
|
||||||
// Reuse this single encode for the size label (do NOT add a second one).
|
// Encode the converted doc ONCE, reuse the bytes for both the size label
|
||||||
|
// and the single apply into the hook document (previously this encode's
|
||||||
|
// result was returned and hocuspocus re-encoded+applied it a second time).
|
||||||
const encoded = Y.encodeStateAsUpdate(ydoc);
|
const encoded = Y.encodeStateAsUpdate(ydoc);
|
||||||
|
Y.applyUpdate(document, encoded);
|
||||||
observeCollabLoad(
|
observeCollabLoad(
|
||||||
encoded.byteLength,
|
encoded.byteLength,
|
||||||
(performance.now() - startedAt) / 1000,
|
(performance.now() - startedAt) / 1000,
|
||||||
);
|
);
|
||||||
return ydoc;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// No persisted state: the hook document is already a fresh empty Y.Doc, so
|
||||||
|
// leave it untouched and return undefined (no re-encode of an empty doc).
|
||||||
this.logger.debug(`creating fresh ydoc: ${pageId}`);
|
this.logger.debug(`creating fresh ydoc: ${pageId}`);
|
||||||
observeCollabLoad(0, (performance.now() - startedAt) / 1000);
|
observeCollabLoad(0, (performance.now() - startedAt) / 1000);
|
||||||
return new Y.Doc();
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
async onStoreDocument(data: onStoreDocumentPayload) {
|
async onStoreDocument(data: onStoreDocumentPayload) {
|
||||||
|
|||||||
@@ -17,10 +17,24 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
|||||||
/** How long a finished entry is retained for late attach (replay + immediate end). */
|
/** How long a finished entry is retained for late attach (replay + immediate end). */
|
||||||
export const RUN_STREAM_RETAIN_FINISHED_MS = 30_000;
|
export const RUN_STREAM_RETAIN_FINISHED_MS = 30_000;
|
||||||
|
|
||||||
/** Per-run replay buffer cap. Past this the buffer is dropped (attach -> 204). */
|
/**
|
||||||
export const RUN_STREAM_MAX_BUFFER_BYTES = 4 * 1024 * 1024;
|
* Per-run replay buffer cap. Past this the buffer is dropped (attach -> 204, and
|
||||||
|
* the client falls back to its restore + degraded-poll path, #430).
|
||||||
|
*
|
||||||
|
* Raised from 4MB to 32MB (#430): marathon autonomous runs (11-25 min observed)
|
||||||
|
* stream far more than 4MB of SSE frames, so a live disconnect mid-run would find
|
||||||
|
* an already-overflowed buffer and could only degrade-poll instead of re-attaching
|
||||||
|
* to the live tail. 32MB comfortably covers those runs while staying bounded.
|
||||||
|
*
|
||||||
|
* Memory cost: this is the WORST-CASE retained size PER ACTIVE run (the buffer is
|
||||||
|
* freed on finish + retention, or dropped immediately on overflow). With the small
|
||||||
|
* number of concurrent autonomous runs a single workspace realistically has, 32MB
|
||||||
|
* each is an acceptable ceiling; the overflow->204->degraded-poll fallback remains
|
||||||
|
* the backstop for anything larger, so correctness never depends on this bound.
|
||||||
|
*/
|
||||||
|
export const RUN_STREAM_MAX_BUFFER_BYTES = 32 * 1024 * 1024;
|
||||||
|
|
||||||
// 2x the replay cap: a just-written 4MB replay burst alone can never trip the
|
// 2x the replay cap: a just-written full-replay burst alone can never trip the
|
||||||
// per-subscriber cap (see controller); only a genuinely stalled socket can.
|
// per-subscriber cap (see controller); only a genuinely stalled socket can.
|
||||||
export const SUBSCRIBER_MAX_BUFFERED_BYTES = 2 * RUN_STREAM_MAX_BUFFER_BYTES;
|
export const SUBSCRIBER_MAX_BUFFERED_BYTES = 2 * RUN_STREAM_MAX_BUFFER_BYTES;
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import {
|
|||||||
AiChatStreamRegistryService,
|
AiChatStreamRegistryService,
|
||||||
RUN_STREAM_MAX_BUFFER_BYTES,
|
RUN_STREAM_MAX_BUFFER_BYTES,
|
||||||
RUN_STREAM_RETAIN_FINISHED_MS,
|
RUN_STREAM_RETAIN_FINISHED_MS,
|
||||||
|
SUBSCRIBER_MAX_BUFFERED_BYTES,
|
||||||
RunStreamCallbacks,
|
RunStreamCallbacks,
|
||||||
} from './ai-chat-stream-registry.service';
|
} from './ai-chat-stream-registry.service';
|
||||||
|
|
||||||
@@ -210,9 +211,10 @@ describe('AiChatStreamRegistryService', () => {
|
|||||||
const att = (await registry.attach(CHAT, false, undefined, c.cb))!;
|
const att = (await registry.attach(CHAT, false, undefined, c.cb))!;
|
||||||
att.start();
|
att.start();
|
||||||
|
|
||||||
const oneMb = 'x'.repeat(1024 * 1024);
|
// Cap-relative so it survives a buffer-cap change (#430): a quarter-cap frame
|
||||||
// 5 x 1MB = 5MB > 4MB cap; the 5th frame is the one that crosses.
|
// means 5 frames comfortably exceed the replay cap; the last one crosses.
|
||||||
for (let i = 0; i < 5; i++) src.push(oneMb + i);
|
const chunk = 'x'.repeat(Math.floor(RUN_STREAM_MAX_BUFFER_BYTES / 4));
|
||||||
|
for (let i = 0; i < 5; i++) src.push(chunk + i);
|
||||||
await flush();
|
await flush();
|
||||||
|
|
||||||
const entry = (registry as any).entries.get(CHAT);
|
const entry = (registry as any).entries.get(CHAT);
|
||||||
@@ -220,7 +222,7 @@ describe('AiChatStreamRegistryService', () => {
|
|||||||
expect(entry.bytes).toBeGreaterThan(RUN_STREAM_MAX_BUFFER_BYTES);
|
expect(entry.bytes).toBeGreaterThan(RUN_STREAM_MAX_BUFFER_BYTES);
|
||||||
// The live subscriber received ALL 5 frames, including the crossing one.
|
// The live subscriber received ALL 5 frames, including the crossing one.
|
||||||
expect(c.frames).toHaveLength(5);
|
expect(c.frames).toHaveLength(5);
|
||||||
expect(c.frames[4]).toBe(oneMb + 4);
|
expect(c.frames[4]).toBe(chunk + 4);
|
||||||
|
|
||||||
// A NEW attach after overflow gets null (replay buffer is gone).
|
// A NEW attach after overflow gets null (replay buffer is gone).
|
||||||
const c2 = collector();
|
const c2 = collector();
|
||||||
@@ -240,9 +242,11 @@ describe('AiChatStreamRegistryService', () => {
|
|||||||
const attB = (await registry.attach(CHAT, false, undefined, b.cb))!;
|
const attB = (await registry.attach(CHAT, false, undefined, b.cb))!;
|
||||||
attB.start();
|
attB.start();
|
||||||
|
|
||||||
const oneMb = 'x'.repeat(1024 * 1024);
|
// Cap-relative so it survives a buffer-cap change (#430): a quarter-of-the-
|
||||||
// 9 x 1MB = 9MB > 8MB per-subscriber cap; A's pending overflows, B streams live.
|
// per-subscriber-cap frame means 5 frames exceed A's paused-pending cap while
|
||||||
for (let i = 0; i < 9; i++) src.push(oneMb + i);
|
// B streams every frame live.
|
||||||
|
const chunk = 'x'.repeat(Math.floor(SUBSCRIBER_MAX_BUFFERED_BYTES / 4));
|
||||||
|
for (let i = 0; i < 5; i++) src.push(chunk + i);
|
||||||
await flush();
|
await flush();
|
||||||
|
|
||||||
const entry = (registry as any).entries.get(CHAT);
|
const entry = (registry as any).entries.get(CHAT);
|
||||||
@@ -250,7 +254,7 @@ describe('AiChatStreamRegistryService', () => {
|
|||||||
expect(entry.subscribers.size).toBe(1);
|
expect(entry.subscribers.size).toBe(1);
|
||||||
expect(a.frames).toEqual([]); // paused + overflowed: nothing was delivered
|
expect(a.frames).toEqual([]); // paused + overflowed: nothing was delivered
|
||||||
// B received every frame live (delivery unaffected by A's overflow).
|
// B received every frame live (delivery unaffected by A's overflow).
|
||||||
expect(b.frames).toHaveLength(9);
|
expect(b.frames).toHaveLength(5);
|
||||||
|
|
||||||
// A's start() (arriving late) degrades to an immediate end, not a partial replay.
|
// A's start() (arriving late) degrades to an immediate end, not a partial replay.
|
||||||
attA.start();
|
attA.start();
|
||||||
|
|||||||
@@ -148,6 +148,53 @@ describe('assistantParts', () => {
|
|||||||
expect(toolPart).not.toHaveProperty('output');
|
expect(toolPart).not.toHaveProperty('output');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('replays the REAL error text for a THROWN tool (tool-error part)', () => {
|
||||||
|
const steps = [
|
||||||
|
{
|
||||||
|
text: '',
|
||||||
|
toolCalls: [
|
||||||
|
{ toolCallId: 'c7', toolName: 'editPageText', input: { id: 'p1' } },
|
||||||
|
],
|
||||||
|
// A thrown tool is a `tool-error` content part; toolResults holds only
|
||||||
|
// successes and stays empty for this call.
|
||||||
|
toolResults: [],
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: 'tool-error',
|
||||||
|
toolCallId: 'c7',
|
||||||
|
toolName: 'editPageText',
|
||||||
|
input: { id: 'p1' },
|
||||||
|
error: new Error('page is locked'),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const parts = assistantParts(steps, '') as AnyPart[];
|
||||||
|
const toolPart = parts.find((p) => p.type === 'tool-editPageText');
|
||||||
|
expect(toolPart).toBeDefined();
|
||||||
|
expect(toolPart!.state).toBe('output-error');
|
||||||
|
// The REAL error is replayed, NOT the 'Tool call did not complete.' placeholder.
|
||||||
|
expect(toolPart!.errorText).toBe('page is locked');
|
||||||
|
expect(toolPart).not.toHaveProperty('output');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('keeps the placeholder ONLY for a call with neither result nor tool-error', () => {
|
||||||
|
const steps = [
|
||||||
|
{
|
||||||
|
text: '',
|
||||||
|
toolCalls: [
|
||||||
|
{ toolCallId: 'c8', toolName: 'insertNode', input: { node: {} } },
|
||||||
|
],
|
||||||
|
toolResults: [],
|
||||||
|
content: [], // aborted mid-step: no result AND no tool-error
|
||||||
|
},
|
||||||
|
];
|
||||||
|
const parts = assistantParts(steps, '') as AnyPart[];
|
||||||
|
const toolPart = parts.find((p) => p.type === 'tool-insertNode');
|
||||||
|
expect(toolPart!.state).toBe('output-error');
|
||||||
|
expect(toolPart!.errorText).toBe('Tool call did not complete.');
|
||||||
|
});
|
||||||
|
|
||||||
it('skips malformed tool-calls (missing toolName or toolCallId)', () => {
|
it('skips malformed tool-calls (missing toolName or toolCallId)', () => {
|
||||||
const steps = [
|
const steps = [
|
||||||
{
|
{
|
||||||
@@ -195,6 +242,45 @@ describe('serializeSteps', () => {
|
|||||||
expect(trace[0]).toEqual({ toolName: 'getPage', input: { id: 'p1' } });
|
expect(trace[0]).toEqual({ toolName: 'getPage', input: { id: 'p1' } });
|
||||||
expect(trace[1]).toEqual({ toolName: 'getPage', output: { title: 'T' } });
|
expect(trace[1]).toEqual({ toolName: 'getPage', output: { title: 'T' } });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('records a THROWN tool failure (tool-error part) with its error message', () => {
|
||||||
|
const trace = serializeSteps([
|
||||||
|
{
|
||||||
|
toolCalls: [{ toolName: 'editPageText', input: { id: 'p1' } }],
|
||||||
|
toolResults: [],
|
||||||
|
content: [
|
||||||
|
{
|
||||||
|
type: 'tool-error',
|
||||||
|
toolName: 'editPageText',
|
||||||
|
error: new Error('page is locked'),
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
]) as Array<Record<string, unknown>>;
|
||||||
|
// The call element is followed by a paired error element (mirroring how a
|
||||||
|
// successful result is appended), so the failure survives in the trace.
|
||||||
|
expect(trace).toHaveLength(2);
|
||||||
|
expect(trace[0]).toEqual({ toolName: 'editPageText', input: { id: 'p1' } });
|
||||||
|
expect(trace[1]).toEqual({
|
||||||
|
toolName: 'editPageText',
|
||||||
|
error: 'page is locked',
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it('truncates a very long tool-error message to the tool-output limit', () => {
|
||||||
|
const long = 'x'.repeat(5000);
|
||||||
|
const trace = serializeSteps([
|
||||||
|
{
|
||||||
|
toolCalls: [{ toolName: 'editPageText', input: {} }],
|
||||||
|
toolResults: [],
|
||||||
|
content: [{ type: 'tool-error', toolName: 'editPageText', error: long }],
|
||||||
|
},
|
||||||
|
]) as Array<Record<string, unknown>>;
|
||||||
|
const errorText = trace[1].error as string;
|
||||||
|
// Truncated (not the full 5000 chars) and carries the omission marker.
|
||||||
|
expect(errorText.length).toBeLessThan(long.length);
|
||||||
|
expect(errorText).toContain('chars omitted');
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe('rowToUiMessage', () => {
|
describe('rowToUiMessage', () => {
|
||||||
|
|||||||
@@ -1637,6 +1637,17 @@ type StepLike = {
|
|||||||
toolName?: string;
|
toolName?: string;
|
||||||
output?: unknown;
|
output?: unknown;
|
||||||
}>;
|
}>;
|
||||||
|
// ai@6.0.134: a tool that THREW surfaces as a `tool-error` content part
|
||||||
|
// ({ type:'tool-error', toolCallId, toolName, input, error }), NOT as a
|
||||||
|
// `toolResults` entry (which holds only successes). Read from here so failed
|
||||||
|
// calls are persisted with their real error instead of being dropped.
|
||||||
|
content?: ReadonlyArray<{
|
||||||
|
type?: string;
|
||||||
|
toolCallId?: string;
|
||||||
|
toolName?: string;
|
||||||
|
input?: unknown;
|
||||||
|
error?: unknown;
|
||||||
|
}>;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -1739,6 +1750,26 @@ function compactValue(value: unknown, depth: number): unknown {
|
|||||||
return value;
|
return value;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Extract a bounded string message from a `tool-error` part's `error` field for
|
||||||
|
* persistence and history replay. The field may be an `Error`, a string, or an
|
||||||
|
* arbitrary object, so pull a message robustly. The result is passed through
|
||||||
|
* `compactValue` so a very long error honors the SAME truncation limits the file
|
||||||
|
* already applies to tool outputs (no new limit is introduced here).
|
||||||
|
*/
|
||||||
|
function normalizeToolError(error: unknown): string {
|
||||||
|
const message =
|
||||||
|
error instanceof Error
|
||||||
|
? error.message
|
||||||
|
: typeof error === 'string'
|
||||||
|
? error
|
||||||
|
: error != null &&
|
||||||
|
typeof (error as { message?: unknown }).message === 'string'
|
||||||
|
? (error as { message: string }).message
|
||||||
|
: String(error);
|
||||||
|
return compactValue(message, 0) as string;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Rebuild the FULL UIMessage `parts` for an assistant turn from the SDK steps,
|
* Rebuild the FULL UIMessage `parts` for an assistant turn from the SDK steps,
|
||||||
* so multi-turn history replays prior tool-calls/results to the model (not just
|
* so multi-turn history replays prior tool-calls/results to the model (not just
|
||||||
@@ -1771,6 +1802,14 @@ export function assistantParts(
|
|||||||
for (const r of step.toolResults ?? []) {
|
for (const r of step.toolResults ?? []) {
|
||||||
if (r.toolCallId) resultsById.set(r.toolCallId, r.output);
|
if (r.toolCallId) resultsById.set(r.toolCallId, r.output);
|
||||||
}
|
}
|
||||||
|
// Index this step's THROWN tool failures (ai@6 `tool-error` content parts)
|
||||||
|
// by tool call id, so a call that failed replays with its real error text.
|
||||||
|
const errorsById = new Map<string, unknown>();
|
||||||
|
for (const part of step.content ?? []) {
|
||||||
|
if (part.type === 'tool-error' && part.toolCallId) {
|
||||||
|
errorsById.set(part.toolCallId, part.error);
|
||||||
|
}
|
||||||
|
}
|
||||||
for (const call of step.toolCalls ?? []) {
|
for (const call of step.toolCalls ?? []) {
|
||||||
if (!call.toolName || !call.toolCallId) continue;
|
if (!call.toolName || !call.toolCallId) continue;
|
||||||
const hasResult = resultsById.has(call.toolCallId);
|
const hasResult = resultsById.has(call.toolCallId);
|
||||||
@@ -1783,9 +1822,21 @@ export function assistantParts(
|
|||||||
input: call.input,
|
input: call.input,
|
||||||
output: compactToolOutput(resultsById.get(call.toolCallId)),
|
output: compactToolOutput(resultsById.get(call.toolCallId)),
|
||||||
});
|
});
|
||||||
|
} else if (errorsById.has(call.toolCallId)) {
|
||||||
|
// The tool THREW: replay the REAL error so the model on the next turn
|
||||||
|
// knows WHY the call failed (and does not blindly repeat it). An
|
||||||
|
// output-error round-trips through convertToModelMessages as a balanced
|
||||||
|
// tool-call + tool-result, keeping the rebuilt history valid.
|
||||||
|
parts.push({
|
||||||
|
type: `tool-${call.toolName}`,
|
||||||
|
toolCallId: call.toolCallId,
|
||||||
|
state: 'output-error',
|
||||||
|
input: call.input,
|
||||||
|
errorText: normalizeToolError(errorsById.get(call.toolCallId)),
|
||||||
|
});
|
||||||
} else {
|
} else {
|
||||||
// No paired result (e.g. aborted mid-step). Persisting a bare
|
// No paired result AND no tool-error (e.g. aborted mid-step). Persisting
|
||||||
// tool-call (input-available) would replay as an unpaired call and
|
// a bare tool-call (input-available) would replay as an unpaired call and
|
||||||
// throw MissingToolResultsError on the next turn (convertToModelMessages
|
// throw MissingToolResultsError on the next turn (convertToModelMessages
|
||||||
// emits no tool-result for it). Emit a SYNTHETIC paired result instead:
|
// emits no tool-result for it). Emit a SYNTHETIC paired result instead:
|
||||||
// an output-error round-trips through convertToModelMessages as a
|
// an output-error round-trips through convertToModelMessages as a
|
||||||
@@ -2021,10 +2072,19 @@ export function serializeSteps(
|
|||||||
steps: ReadonlyArray<{
|
steps: ReadonlyArray<{
|
||||||
toolCalls?: ReadonlyArray<{ toolName?: string; input?: unknown }>;
|
toolCalls?: ReadonlyArray<{ toolName?: string; input?: unknown }>;
|
||||||
toolResults?: ReadonlyArray<{ toolName?: string; output?: unknown }>;
|
toolResults?: ReadonlyArray<{ toolName?: string; output?: unknown }>;
|
||||||
|
content?: ReadonlyArray<{
|
||||||
|
type?: string;
|
||||||
|
toolName?: string;
|
||||||
|
error?: unknown;
|
||||||
|
}>;
|
||||||
}>,
|
}>,
|
||||||
): unknown {
|
): unknown {
|
||||||
const calls: Array<{ toolName?: string; input?: unknown; output?: unknown }> =
|
const calls: Array<{
|
||||||
[];
|
toolName?: string;
|
||||||
|
input?: unknown;
|
||||||
|
output?: unknown;
|
||||||
|
error?: string;
|
||||||
|
}> = [];
|
||||||
for (const step of steps ?? []) {
|
for (const step of steps ?? []) {
|
||||||
for (const call of step.toolCalls ?? []) {
|
for (const call of step.toolCalls ?? []) {
|
||||||
calls.push({ toolName: call.toolName, input: call.input });
|
calls.push({ toolName: call.toolName, input: call.input });
|
||||||
@@ -2032,6 +2092,18 @@ export function serializeSteps(
|
|||||||
for (const r of step.toolResults ?? []) {
|
for (const r of step.toolResults ?? []) {
|
||||||
calls.push({ toolName: r.toolName, output: compactToolOutput(r.output) });
|
calls.push({ toolName: r.toolName, output: compactToolOutput(r.output) });
|
||||||
}
|
}
|
||||||
|
// ai@6 surfaces a THROWN tool failure as a `tool-error` content part, NOT as
|
||||||
|
// a `toolResults` entry. Record it as its own paired element (mirroring how a
|
||||||
|
// successful result is appended) so the failure and its reason survive in the
|
||||||
|
// trace instead of leaving an orphaned call with no result.
|
||||||
|
for (const part of step.content ?? []) {
|
||||||
|
if (part.type === 'tool-error') {
|
||||||
|
calls.push({
|
||||||
|
toolName: part.toolName,
|
||||||
|
error: normalizeToolError(part.error),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return calls.length > 0 ? calls : null;
|
return calls.length > 0 ? calls : null;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -729,6 +729,35 @@ export class AiChatToolsService {
|
|||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
|
|
||||||
|
// Schema + description live in @docmost/mcp's SHARED_TOOL_SPECS (#423).
|
||||||
|
// meta.hash in the result is the baseHash drawioUpdate requires.
|
||||||
|
drawioGet: sharedTool(
|
||||||
|
sharedToolSpecs.drawioGet,
|
||||||
|
async ({ pageId, node, format }) =>
|
||||||
|
await client.drawioGet(pageId, node, format ?? 'xml'),
|
||||||
|
),
|
||||||
|
|
||||||
|
// Schema + description live in @docmost/mcp's SHARED_TOOL_SPECS (#423).
|
||||||
|
// The flat schema fields are regrouped into the client's `where` object.
|
||||||
|
drawioCreate: sharedTool(
|
||||||
|
sharedToolSpecs.drawioCreate,
|
||||||
|
async ({ pageId, xml, position, anchorNodeId, anchorText, title }) =>
|
||||||
|
await client.drawioCreate(
|
||||||
|
pageId,
|
||||||
|
{ position, anchorNodeId, anchorText },
|
||||||
|
xml,
|
||||||
|
title,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
|
||||||
|
// Schema + description live in @docmost/mcp's SHARED_TOOL_SPECS (#423).
|
||||||
|
// baseHash is the optimistic lock: mismatch => structured conflict error.
|
||||||
|
drawioUpdate: sharedTool(
|
||||||
|
sharedToolSpecs.drawioUpdate,
|
||||||
|
async ({ pageId, node, xml, baseHash }) =>
|
||||||
|
await client.drawioUpdate(pageId, node, xml, baseHash),
|
||||||
|
),
|
||||||
|
|
||||||
// Schema + description now live in @docmost/mcp's SHARED_TOOL_SPECS (#294).
|
// Schema + description now live in @docmost/mcp's SHARED_TOOL_SPECS (#294).
|
||||||
// The table reference parameter was unified to `table` (was `tableRef`).
|
// The table reference parameter was unified to `table` (was `tableRef`).
|
||||||
tableInsertRow: sharedTool(
|
tableInsertRow: sharedTool(
|
||||||
|
|||||||
@@ -168,6 +168,32 @@ export interface DocmostClientLike {
|
|||||||
url: string,
|
url: string,
|
||||||
opts?: { align?: 'left' | 'center' | 'right'; alt?: string },
|
opts?: { align?: 'left' | 'center' | 'right'; alt?: string },
|
||||||
): Promise<Record<string, unknown>>;
|
): Promise<Record<string, unknown>>;
|
||||||
|
// --- draw.io diagrams (#423, stage 1) ---
|
||||||
|
// Read a diagram as decoded mxGraph XML (default) or the raw .drawio.svg.
|
||||||
|
// meta.hash is the optimistic-lock key drawioUpdate expects as baseHash.
|
||||||
|
drawioGet(
|
||||||
|
pageId: string,
|
||||||
|
node: string,
|
||||||
|
format?: 'xml' | 'svg',
|
||||||
|
): Promise<Record<string, unknown>>;
|
||||||
|
// Lint mxGraph XML, build the .drawio.svg attachment and insert a drawio node.
|
||||||
|
drawioCreate(
|
||||||
|
pageId: string,
|
||||||
|
where: {
|
||||||
|
position: 'before' | 'after' | 'append';
|
||||||
|
anchorNodeId?: string;
|
||||||
|
anchorText?: string;
|
||||||
|
},
|
||||||
|
xml: string,
|
||||||
|
title?: string,
|
||||||
|
): Promise<Record<string, unknown>>;
|
||||||
|
// Optimistic-locked full replacement of a diagram (baseHash from drawioGet).
|
||||||
|
drawioUpdate(
|
||||||
|
pageId: string,
|
||||||
|
node: string,
|
||||||
|
xml: string,
|
||||||
|
baseHash: string,
|
||||||
|
): Promise<Record<string, unknown>>;
|
||||||
tableInsertRow(
|
tableInsertRow(
|
||||||
pageId: string,
|
pageId: string,
|
||||||
tableRef: string,
|
tableRef: string,
|
||||||
|
|||||||
+76
-38
@@ -13,12 +13,14 @@ Read the **Gotchas** section before you trust any error count.
|
|||||||
- Agent chats live in Postgres, DB `docmost`, tables `ai_chat_*`.
|
- Agent chats live in Postgres, DB `docmost`, tables `ai_chat_*`.
|
||||||
- Each tool invocation is stored as **two** array elements (a `tool-call` part and
|
- Each tool invocation is stored as **two** array elements (a `tool-call` part and
|
||||||
a `tool-result` part), so naive counting double-counts.
|
a `tool-result` part), so naive counting double-counts.
|
||||||
- **A tool that *throws* writes no result part at all.** Its error text is nowhere
|
- **A tool that *throws* writes no result part.** Since the #407 fix its error is
|
||||||
in the DB — not in `tool_calls`, `content`, or `metadata`. It is shown live in
|
persisted as a dedicated `{toolName, error}` element in `tool_calls` (queryable +
|
||||||
the UI only. So `isError` / `success=false` scans under-report by design.
|
replayed to the model). **Rows written before #407 still drop it** — the error is
|
||||||
- To find where agents fail you need **three** sources: (1) soft-failure markers in
|
nowhere in the DB and shows only in the live UI. So `isError` / `success=false`
|
||||||
`tool_calls`, (2) the orphan-gap proxy for thrown errors, (3) server logs / the
|
scans under-report by design, and pre-#407 thrown errors are invisible.
|
||||||
live UI for the actual error text.
|
- To find where agents fail: (1) soft-failure markers in `tool_calls`, (2) the new
|
||||||
|
`error` field for thrown errors (new rows) / the orphan-gap proxy (old rows),
|
||||||
|
(3) server logs / the live UI for full stack traces beyond the truncated message.
|
||||||
|
|
||||||
## Where the data lives
|
## Where the data lives
|
||||||
|
|
||||||
@@ -61,13 +63,17 @@ index 0: { "toolName": "getPage", "input": { "pageId": "…" } } ← tool-ca
|
|||||||
index 1: { "toolName": "getPage", "output": { … } } ← tool-result (has output, NO input)
|
index 1: { "toolName": "getPage", "output": { … } } ← tool-result (has output, NO input)
|
||||||
```
|
```
|
||||||
|
|
||||||
The **only** keys that ever appear on an element are `toolName`, `input`, `output`.
|
The keys that appear on an element are `toolName`, `input`, `output`, and — for a
|
||||||
There is no `state`, no `errorText`, no `type`. Consequences:
|
**thrown** failure on rows written after the #407 fix — `error` (the tool's error
|
||||||
|
message; see the "Hard failures" section below). There is no `state`, no `errorText`,
|
||||||
|
no `type`. On pre-#407 rows a thrown failure has NO paired result element at all
|
||||||
|
(silent orphan). Consequences:
|
||||||
|
|
||||||
1. **Real invocation count = elements that have `output`.** Counting every element
|
1. **Real invocation count = elements that have `output` or `error`.** Counting every
|
||||||
double-counts (you get ~2× and a spurious "~50% of every tool has no output").
|
element double-counts (you get ~2× and a spurious "~50% of every tool has no output").
|
||||||
2. **Pairing:** a successful call = a `tool-call` part followed by its `tool-result`
|
2. **Pairing:** a call = a `tool-call` part followed by its result part. A success
|
||||||
part. Both carry `toolName`, so you can group by tool on either.
|
carries `output`; a thrown failure (post-#407) carries `error` instead. Both carry
|
||||||
|
`toolName`, so you can group by tool on either.
|
||||||
|
|
||||||
## The two classes of failure (and which the DB can see)
|
## The two classes of failure (and which the DB can see)
|
||||||
|
|
||||||
@@ -85,26 +91,38 @@ These are visible in the `tool-result` `output`. The marker differs per tool:
|
|||||||
Note `editPageText` returns `failed: []` on success — filtering on the *presence*
|
Note `editPageText` returns `failed: []` on success — filtering on the *presence*
|
||||||
of the key gives false positives; filter on **non-empty**.
|
of the key gives false positives; filter on **non-empty**.
|
||||||
|
|
||||||
### 2. Hard failures — tool THREW → NOT PERSISTED ❌ (the trap)
|
### 2. Hard failures — tool THREW → NOW PERSISTED ✅ (since the #407 fix)
|
||||||
|
|
||||||
When a tool throws (the classic one is `patchNode` / `insertNode` / `tableUpdateCell`
|
When a tool throws (the classic one is `patchNode` / `insertNode` / `tableUpdateCell`
|
||||||
→ `Failed to encode document to Yjs (fromJSON): Unknown node type: undefined`), the
|
→ `Failed to encode document to Yjs (fromJSON): Unknown node type: undefined`), the
|
||||||
runtime writes **no `tool-result` part**. The orphaned `tool-call` part stays, but
|
runtime still writes **no `tool-result` part** — the failure is an ai@6 `tool-error`
|
||||||
the error text is **nowhere in the DB**. It is streamed to the UI live and (until
|
content part instead. **Since the #407 fix, that error is persisted**: `serializeSteps`
|
||||||
rotation) to server logs — that is it.
|
appends a dedicated element `{toolName, error: "<message>"}` right after the failed
|
||||||
|
call, mirroring how a successful `{toolName, output}` element is appended. So a thrown
|
||||||
|
error now leaves a queryable `error` field carrying its (truncated) reason, and the
|
||||||
|
same real text is replayed to the model on the next turn (an `output-error` part with
|
||||||
|
the real `errorText`, no longer the `'Tool call did not complete.'` placeholder).
|
||||||
|
|
||||||
So any query like `count(*) FILTER (WHERE output.success = false)` will happily
|
**Cutover caveat — old rows keep the old blind shape.** Rows written **before** this
|
||||||
return **0** for `patchNode` even when the chat is visibly full of red failures.
|
change have the two-part shape (`call` + `output` only) and simply **drop** thrown
|
||||||
That is survivorship bias, not reliability.
|
errors, leaving a silent **orphan** (a `call` with no `output` *and* no `error`). Rows
|
||||||
|
written **after** the fix additionally carry the `error` element. So:
|
||||||
|
|
||||||
The only DB-side proxy for a thrown error is an **orphan**: a `tool-call` part with
|
- **New rows:** query the `error` field directly (see the hard-error query below) — no
|
||||||
no matching `tool-result`. Caveat: orphans also appear when a run is **aborted**
|
orphan heuristic needed for thrown failures.
|
||||||
mid-flight (server restart), so a high-volume tool (`createComment`, `searchInPage`,
|
- **Old rows (pre-#407):** the only DB-side proxy is still an **orphan**: a `tool-call`
|
||||||
`Search_web_search`) shows orphans from aborts, not from real errors. Treat the
|
part with no matching `tool-result` *and* no `error`. Orphans also appear when a run
|
||||||
orphan gap as an *upper bound* on hard errors, and cross-check the tool: a gap on a
|
is **aborted** mid-flight (server restart), so a high-volume tool (`createComment`,
|
||||||
structural editor (`patchNode`, `insertNode`, `updatePageJson`, `transformPage`) is
|
`searchInPage`, `Search_web_search`) shows orphans from aborts, not real errors on
|
||||||
|
old rows. Treat the orphan gap as an *upper bound*, and cross-check the tool: a gap on
|
||||||
|
a structural editor (`patchNode`, `insertNode`, `updatePageJson`, `transformPage`) is
|
||||||
almost certainly a thrown Yjs-encode error; a gap on `createComment` is mostly aborts.
|
almost certainly a thrown Yjs-encode error; a gap on `createComment` is mostly aborts.
|
||||||
|
|
||||||
|
A note on the aborted-call fallback: a call with **neither** a result **nor** a
|
||||||
|
`tool-error` (genuinely interrupted mid-step) still replays with the
|
||||||
|
`'Tool call did not complete.'` placeholder and persists as an orphan — that path is
|
||||||
|
unchanged, and is distinct from a real thrown error, which now carries `error`.
|
||||||
|
|
||||||
### 3. Run-level failures → `ai_chat_runs`
|
### 3. Run-level failures → `ai_chat_runs`
|
||||||
|
|
||||||
`status` ∈ `succeeded | aborted | failed | running`; `error` holds the text. Seen in
|
`status` ∈ `succeeded | aborted | failed | running`; `error` holds the text. Seen in
|
||||||
@@ -164,14 +182,28 @@ WHERE jsonb_typeof(o->'failed') = 'array'
|
|||||||
GROUP BY 1 ORDER BY 2 DESC;
|
GROUP BY 1 ORDER BY 2 DESC;
|
||||||
```
|
```
|
||||||
|
|
||||||
**Hard-error proxy — orphan gap per tool, WITH a spread column** (call parts minus
|
**Hard errors — persisted `error` field per tool (NEW rows, since #407)** — thrown
|
||||||
result parts, plus how many distinct chats the gap is spread across):
|
tool failures now carry their real reason, so query them directly:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT elem->>'toolName' AS tool, count(*) AS thrown_errors,
|
||||||
|
min(elem->>'error') AS sample_error
|
||||||
|
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||||
|
WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'error'
|
||||||
|
GROUP BY 1 ORDER BY 2 DESC;
|
||||||
|
```
|
||||||
|
|
||||||
|
**Hard-error proxy for OLD rows (pre-#407) — orphan gap per tool, WITH a spread column**
|
||||||
|
(call parts minus result parts, plus how many distinct chats the gap is spread across).
|
||||||
|
This covers rows written before thrown errors were persisted; on new rows a thrown
|
||||||
|
failure now has its own `error` element (use the query above) and an orphan means only
|
||||||
|
a genuinely aborted mid-step call:
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
WITH parts AS (
|
WITH parts AS (
|
||||||
SELECT m.chat_id, elem->>'toolName' AS tool,
|
SELECT m.chat_id, elem->>'toolName' AS tool,
|
||||||
(elem ? 'input' AND NOT (elem ? 'output')) AS is_call,
|
(elem ? 'input' AND NOT (elem ? 'output')) AS is_call,
|
||||||
(elem ? 'output') AS is_result
|
(elem ? 'output' OR elem ? 'error') AS is_result
|
||||||
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem
|
||||||
WHERE jsonb_typeof(m.tool_calls) = 'array' AND m.role = 'assistant'
|
WHERE jsonb_typeof(m.tool_calls) = 'array' AND m.role = 'assistant'
|
||||||
),
|
),
|
||||||
@@ -188,8 +220,12 @@ HAVING sum(gap) FILTER (WHERE gap > 0) > 0
|
|||||||
ORDER BY missing_results DESC;
|
ORDER BY missing_results DESC;
|
||||||
```
|
```
|
||||||
|
|
||||||
**`missing_results` mixes thrown errors AND aborted/interrupted runs — you cannot
|
The `is_result` predicate counts an `error` element as a paired result too, so on new
|
||||||
split them from `output` alone** (a positional "what follows the orphan" heuristic
|
rows a persisted thrown error no longer inflates the orphan gap; a remaining gap is an
|
||||||
|
aborted/interrupted call.
|
||||||
|
|
||||||
|
**On OLD rows, `missing_results` mixes thrown errors AND aborted/interrupted runs — you
|
||||||
|
cannot split them from `output` alone** (a positional "what follows the orphan" heuristic
|
||||||
breaks on parallel tool batches, which persist as `call,call,…,result,result`). Use
|
breaks on parallel tool batches, which persist as `call,call,…,result,result`). Use
|
||||||
`chats_spread` to disambiguate:
|
`chats_spread` to disambiguate:
|
||||||
|
|
||||||
@@ -244,18 +280,20 @@ docker compose -p gitmost logs -f --tail=100 # whole stack
|
|||||||
```
|
```
|
||||||
|
|
||||||
Logging is `json-file`, `max-size=10m max-file=5` → ~50 MB retained, then rotated,
|
Logging is `json-file`, `max-size=10m max-file=5` → ~50 MB retained, then rotated,
|
||||||
and **wiped on container recreate**. So thrown-tool error text is only reliably
|
and **wiped on container recreate**. Since the #407 fix, thrown-tool error text is
|
||||||
caught **in real time** (or in the live chat UI, which renders the failed part with
|
**persisted in the `error` field** of `tool_calls` (see the hard-error query above), so
|
||||||
its message). There is no durable, queryable store of hard tool errors today — if you
|
you no longer depend on live logs for it. Logs/live UI remain useful for **pre-#407
|
||||||
need one, that is a feature to add (persist `output-error` parts, or emit a
|
rows** (whose thrown errors were dropped) and for full stack traces beyond the
|
||||||
`tool_calls_total{tool,status}` metric to VictoriaMetrics).
|
truncated stored message. A per-tool `tool_calls_total{tool,status}` metric to
|
||||||
|
VictoriaMetrics is still a possible future add for aggregate dashboards.
|
||||||
|
|
||||||
## Gotchas checklist
|
## Gotchas checklist
|
||||||
|
|
||||||
- [ ] Counting every `tool_calls` element → **2× overcount**. Count elements with `output`.
|
- [ ] Counting every `tool_calls` element → **overcount**. Count `output` elements; add `error` elements for thrown failures (new rows), but don't count both as invocations.
|
||||||
- [ ] `isError` / `success=false` ≈ 0 does **not** mean "no errors" — thrown errors aren't persisted.
|
- [ ] `isError` / `success=false` ≈ 0 does **not** mean "no errors" — thrown errors are a separate `error` element (new rows) or dropped entirely (pre-#407 rows).
|
||||||
|
- [ ] Thrown errors persist only on rows written **after the #407 fix** — pre-#407 rows still drop them (orphan only). Mind the cutover when trending over time.
|
||||||
- [ ] `editPageText.failed` is `[]` on success — test for **non-empty**, not presence.
|
- [ ] `editPageText.failed` is `[]` on success — test for **non-empty**, not presence.
|
||||||
- [ ] Orphan gap mixes thrown errors **and** aborted runs — split by tool before concluding.
|
- [ ] Orphan gap on OLD rows mixes thrown errors **and** aborted runs — split by tool. On NEW rows a thrown error is its own `error` element, so a gap ≈ aborted call.
|
||||||
- [ ] `aborted` runs = server restarts, `failed` runs = provider overload — not agent mistakes.
|
- [ ] `aborted` runs = server restarts, `failed` runs = provider overload — not agent mistakes.
|
||||||
- [ ] Never dump a raw `tool_calls` cell — it can be hundreds of KB.
|
- [ ] Never dump a raw `tool_calls` cell — it can be hundreds of KB.
|
||||||
- [ ] Logs are ephemeral (≤50 MB, wiped on recreate) — grab hard-error text live.
|
- [ ] Logs are ephemeral (≤50 MB, wiped on recreate) — grab hard-error text live.
|
||||||
|
|||||||
+2
-1
@@ -97,7 +97,8 @@
|
|||||||
"patchedDependencies": {
|
"patchedDependencies": {
|
||||||
"scimmy@1.3.5": "patches/scimmy@1.3.5.patch",
|
"scimmy@1.3.5": "patches/scimmy@1.3.5.patch",
|
||||||
"yjs@13.6.30": "patches/yjs@13.6.30.patch",
|
"yjs@13.6.30": "patches/yjs@13.6.30.patch",
|
||||||
"ai@6.0.134": "patches/ai@6.0.134.patch"
|
"ai@6.0.134": "patches/ai@6.0.134.patch",
|
||||||
|
"@hocuspocus/server@3.4.4": "patches/@hocuspocus__server@3.4.4.patch"
|
||||||
},
|
},
|
||||||
"overrides": {
|
"overrides": {
|
||||||
"prosemirror-changeset": "2.4.0",
|
"prosemirror-changeset": "2.4.0",
|
||||||
|
|||||||
@@ -52,6 +52,7 @@
|
|||||||
"form-data": "^4.0.0",
|
"form-data": "^4.0.0",
|
||||||
"jsdom": "^27.4.0",
|
"jsdom": "^27.4.0",
|
||||||
"marked": "^17.0.1",
|
"marked": "^17.0.1",
|
||||||
|
"pako": "^2.0.3",
|
||||||
"re2": "^1.21.0",
|
"re2": "^1.21.0",
|
||||||
"ws": "^8.19.0",
|
"ws": "^8.19.0",
|
||||||
"y-prosemirror": "1.3.7",
|
"y-prosemirror": "1.3.7",
|
||||||
|
|||||||
+708
-217
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,11 @@ import { SHARED_TOOL_SPECS, SharedToolSpec } from "./tool-specs.js";
|
|||||||
export { DocmostClient } from "./client.js";
|
export { DocmostClient } from "./client.js";
|
||||||
export type { DocmostMcpConfig } from "./client.js";
|
export type { DocmostMcpConfig } from "./client.js";
|
||||||
|
|
||||||
|
// Teardown for the live per-page CollabSession cache (issue #400). An embedding
|
||||||
|
// HTTP host (the gitmost NestJS server) should call this from its own shutdown
|
||||||
|
// hook so no cached collab provider outlives the process.
|
||||||
|
export { destroyAllSessions } from "./lib/collab-session.js";
|
||||||
|
|
||||||
// Re-export the zod-agnostic shared tool-spec registry so the in-app AI-SDK
|
// Re-export the zod-agnostic shared tool-spec registry so the in-app AI-SDK
|
||||||
// service can read it off the loaded module (it cannot import the ESM package's
|
// service can read it off the loaded module (it cannot import the ESM package's
|
||||||
// internals directly; it goes through loadDocmostMcp()).
|
// internals directly; it goes through loadDocmostMcp()).
|
||||||
@@ -47,7 +52,7 @@ const VERSION = packageJson.version;
|
|||||||
export const SERVER_INSTRUCTIONS =
|
export const SERVER_INSTRUCTIONS =
|
||||||
"Docmost editing guide — choose the tool by intent.\n" +
|
"Docmost editing guide — choose the tool by intent.\n" +
|
||||||
"READ: find a page -> search (workspace-wide full-text); list -> list_pages / list_spaces. Locate blocks and their ids CHEAPLY -> get_outline (compact top-level map; start here, not get_page_json). One block's subtree -> get_node (by attrs.id, or \"#<index>\" for tables, which carry no id). Find every occurrence of a string/regex ON a page (and where each is) -> search_in_page, NOT block-by-block get_node — it returns each hit's node ref + block index + context for a targeted comment. Whole page -> get_page (Markdown, lossy; inline <span data-comment-id> tags are comment anchors — markup, not text) or get_page_json (lossless ProseMirror with block ids). Hand a huge page (with images) to an external consumer without pulling it through the model context -> stash_page (returns a short-lived anonymous URL).\n" +
|
"READ: find a page -> search (workspace-wide full-text); list -> list_pages / list_spaces. Locate blocks and their ids CHEAPLY -> get_outline (compact top-level map; start here, not get_page_json). One block's subtree -> get_node (by attrs.id, or \"#<index>\" for tables, which carry no id). Find every occurrence of a string/regex ON a page (and where each is) -> search_in_page, NOT block-by-block get_node — it returns each hit's node ref + block index + context for a targeted comment. Whole page -> get_page (Markdown, lossy; inline <span data-comment-id> tags are comment anchors — markup, not text) or get_page_json (lossless ProseMirror with block ids). Hand a huge page (with images) to an external consumer without pulling it through the model context -> stash_page (returns a short-lived anonymous URL).\n" +
|
||||||
"EDIT: fix wording/typos/numbers -> edit_page_text (find/replace inside blocks, no node id needed). Change ONE block (paragraph/heading/callout/etc.) structurally -> patch_node (by attrs.id from get_outline). Add a block -> insert_node (before/after a block by attrs.id or by anchor text, or append). Remove a block -> delete_node (by attrs.id). Tables -> table_get / table_update_cell / table_insert_row / table_delete_row (address by \"#<index>\" from get_outline; table nodes have no attrs.id). Images -> insert_image (add from a web URL) / replace_image (swap an existing image). Footnotes -> insert_footnote. Bulk/structural rewrite -> update_page_json (full ProseMirror replace; prefer the granular tools above to avoid resending the whole ~100KB+ document). Complex/scripted rewrite (multiple coordinated edits, renumbering) -> docmost_transform: write a JS `(doc, ctx) => doc` transform, preview the diff with dryRun (default), then apply with dryRun:false; ctx.helpers includes commentsToFootnotes for turning inline comments into numbered footnotes.\n" +
|
"EDIT: fix wording/typos/numbers -> edit_page_text (find/replace inside blocks, no node id needed). Change ONE block (paragraph/heading/callout/etc.) structurally -> patch_node (by attrs.id from get_outline). Add a block -> insert_node (before/after a block by attrs.id or by anchor text, or append). Remove a block -> delete_node (by attrs.id). Tables -> table_get / table_update_cell / table_insert_row / table_delete_row (address by \"#<index>\" from get_outline; table nodes have no attrs.id). Images -> insert_image (add from a web URL) / replace_image (swap an existing image). Draw.io diagrams -> drawio_create (create from mxGraph XML and insert), drawio_get (read a diagram as mxGraph XML + a hash), drawio_update (replace a diagram; pass the hash from drawio_get as baseHash for optimistic locking). Footnotes -> insert_footnote. Bulk/structural rewrite -> update_page_json (full ProseMirror replace; prefer the granular tools above to avoid resending the whole ~100KB+ document). Complex/scripted rewrite (multiple coordinated edits, renumbering) -> docmost_transform: write a JS `(doc, ctx) => doc` transform, preview the diff with dryRun (default), then apply with dryRun:false; ctx.helpers includes commentsToFootnotes for turning inline comments into numbered footnotes.\n" +
|
||||||
"PAGES: new -> create_page (Markdown). Rename (title only) -> rename_page. Move -> move_page. Delete -> delete_page (SOFT delete — the page goes to trash and is restorable; nothing is permanent). Copy/replace a page's whole content from another page (server-side, no document through the model) -> copy_page_content. Sharing -> share_page / unshare_page / list_shares; share_page makes the page PUBLICLY accessible — do it only when explicitly asked.\n" +
|
"PAGES: new -> create_page (Markdown). Rename (title only) -> rename_page. Move -> move_page. Delete -> delete_page (SOFT delete — the page goes to trash and is restorable; nothing is permanent). Copy/replace a page's whole content from another page (server-side, no document through the model) -> copy_page_content. Sharing -> share_page / unshare_page / list_shares; share_page makes the page PUBLICLY accessible — do it only when explicitly asked.\n" +
|
||||||
"COMMENTS: create_comment is always inline and requires an EXACT selection — contiguous text from a single block, <=250 chars (fails rather than leaving an unanchored comment); reply to a thread via parentCommentId. Propose a concrete text fix for one-click human approval -> create_comment with suggestedText (the exact plain-text replacement for the selection; the selection must then be UNIQUE in the page — extend it with context if needed); prefer this over editing directly when the change is subjective or needs the author's sign-off. Manage -> list_comments, update_comment, resolve_comment (resolve/reopen, reversible — prefer over delete to close), delete_comment, check_new_comments.\n" +
|
"COMMENTS: create_comment is always inline and requires an EXACT selection — contiguous text from a single block, <=250 chars (fails rather than leaving an unanchored comment); reply to a thread via parentCommentId. Propose a concrete text fix for one-click human approval -> create_comment with suggestedText (the exact plain-text replacement for the selection; the selection must then be UNIQUE in the page — extend it with context if needed); prefer this over editing directly when the change is subjective or needs the author's sign-off. Manage -> list_comments, update_comment, resolve_comment (resolve/reopen, reversible — prefer over delete to close), delete_comment, check_new_comments.\n" +
|
||||||
"HISTORY: review what changed -> diff_page_versions (a historyId vs current, or two versions). List saved versions -> list_page_history. Undo a bad edit -> restore_page_version (writes a past version back as current; itself revertible). Lossless markdown round-trip (download, edit, re-upload, incl. comment anchors) -> export_page_markdown / import_page_markdown.";
|
"HISTORY: review what changed -> diff_page_versions (a historyId vs current, or two versions). List saved versions -> list_page_history. Undo a bad edit -> restore_page_version (writes a past version back as current; itself revertible). Lossless markdown round-trip (download, edit, re-upload, incl. comment anchors) -> export_page_markdown / import_page_markdown.";
|
||||||
@@ -460,6 +465,38 @@ registerShared(
|
|||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Tool: drawio_get — read a draw.io diagram as mxGraph XML (or the raw SVG).
|
||||||
|
registerShared(
|
||||||
|
SHARED_TOOL_SPECS.drawioGet,
|
||||||
|
async ({ pageId, node, format }) => {
|
||||||
|
const result = await docmostClient.drawioGet(pageId, node, format ?? "xml");
|
||||||
|
return jsonContent(result);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// Tool: drawio_create — lint mxGraph XML, build the .drawio.svg, insert a node.
|
||||||
|
registerShared(
|
||||||
|
SHARED_TOOL_SPECS.drawioCreate,
|
||||||
|
async ({ pageId, xml, position, anchorNodeId, anchorText, title }) => {
|
||||||
|
const result = await docmostClient.drawioCreate(
|
||||||
|
pageId,
|
||||||
|
{ position, anchorNodeId, anchorText },
|
||||||
|
xml,
|
||||||
|
title,
|
||||||
|
);
|
||||||
|
return jsonContent(result);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
// Tool: drawio_update — optimistic-locked full replacement of a diagram.
|
||||||
|
registerShared(
|
||||||
|
SHARED_TOOL_SPECS.drawioUpdate,
|
||||||
|
async ({ pageId, node, xml, baseHash }) => {
|
||||||
|
const result = await docmostClient.drawioUpdate(pageId, node, xml, baseHash);
|
||||||
|
return jsonContent(result);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
// Tool: share_page
|
// Tool: share_page
|
||||||
// Schema + description now live in the shared registry (#294). The execute body
|
// Schema + description now live in the shared registry (#294). The execute body
|
||||||
// keeps this transport's own `searchIndexing ?? true` default.
|
// keeps this transport's own `searchIndexing ?? true` default.
|
||||||
|
|||||||
@@ -0,0 +1,668 @@
|
|||||||
|
import { HocuspocusProvider } from "@hocuspocus/provider";
|
||||||
|
import { TiptapTransformer } from "@hocuspocus/transformer";
|
||||||
|
import * as Y from "yjs";
|
||||||
|
import WebSocket from "ws";
|
||||||
|
import {
|
||||||
|
buildCollabWsUrl,
|
||||||
|
applyDocToFragment,
|
||||||
|
MutationResult,
|
||||||
|
} from "./collaboration.js";
|
||||||
|
import { summarizeChange } from "./diff.js";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Live per-page collaboration session cache (issue #400).
|
||||||
|
*
|
||||||
|
* The one-shot write path (collaboration.mutatePageContent /
|
||||||
|
* client.mutateLiveContentUnlocked) used to open a NEW HocuspocusProvider, run
|
||||||
|
* the full connect -> auth -> onLoadDocument -> initial-sync handshake, apply a
|
||||||
|
* single edit, wait for persistence, and then `provider.destroy()` — for EVERY
|
||||||
|
* content mutation. Disconnecting after every edit means that once the pause
|
||||||
|
* between calls exceeds the server's write debounce, the server does a full
|
||||||
|
* store -> unload -> reload per cell, causing 25s connect timeouts and
|
||||||
|
* event-loop lag under a burst of edits on one page.
|
||||||
|
*
|
||||||
|
* This module keeps ONE live provider + ydoc per (wsUrl, pageId, token) alive
|
||||||
|
* across a SERIES of edits. While the provider stays connected the server never
|
||||||
|
* enters store -> unload -> reload, its debounce coalesces N writes into 1-2
|
||||||
|
* stores, and the repeated auth/load/initial-sync disappears.
|
||||||
|
*
|
||||||
|
* The synchronous read -> transform -> write section and the per-edit
|
||||||
|
* persistence-ack logic are preserved VERBATIM from the one-shot machine — the
|
||||||
|
* only change is that they run on a persistent provider instead of a throwaway
|
||||||
|
* one. See CollabSession.mutate.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** Time we wait for the initial handshake/sync before giving up. */
|
||||||
|
const CONNECT_TIMEOUT_MS = 25000;
|
||||||
|
/** Time we wait for the server to acknowledge our write before giving up. */
|
||||||
|
const PERSIST_TIMEOUT_MS = 20000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Tunables, read fresh from the environment on every acquire so tests (and a
|
||||||
|
* live rollback) can change them without reloading the module. Mirrors how
|
||||||
|
* http.ts parses MCP_SESSION_IDLE_MS.
|
||||||
|
* - MCP_COLLAB_SESSION_IDLE_MS: idle TTL, reset after every op. Default 60s.
|
||||||
|
* 0 (or negative) DISABLES the cache — every op opens its own provider and
|
||||||
|
* destroys it after the op, i.e. the exact legacy per-op-provider behavior
|
||||||
|
* (the rollback path).
|
||||||
|
* - MCP_COLLAB_SESSION_MAX_AGE_MS: hard lifetime checked at acquire; bounds
|
||||||
|
* the permission-staleness window. Default 10 min.
|
||||||
|
* - MCP_COLLAB_SESSION_MAX_ENTRIES: registry cap; the least-recently-used
|
||||||
|
* session is destroy-evicted when the cap is reached. Default 32.
|
||||||
|
*/
|
||||||
|
interface SessionConfig {
|
||||||
|
idleMs: number;
|
||||||
|
maxAgeMs: number;
|
||||||
|
maxEntries: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseEnvInt(value: string | undefined, fallback: number): number {
|
||||||
|
const parsed = parseInt(value ?? "", 10);
|
||||||
|
return Number.isFinite(parsed) ? parsed : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readConfig(): SessionConfig {
|
||||||
|
// idleMs: allow 0 (disable). A malformed value falls back to the default.
|
||||||
|
const idleRaw = parseInt(process.env.MCP_COLLAB_SESSION_IDLE_MS ?? "", 10);
|
||||||
|
const idleMs = Number.isFinite(idleRaw) ? Math.max(0, idleRaw) : 60 * 1000;
|
||||||
|
const maxAgeMs = Math.max(
|
||||||
|
0,
|
||||||
|
parseEnvInt(process.env.MCP_COLLAB_SESSION_MAX_AGE_MS, 10 * 60 * 1000),
|
||||||
|
);
|
||||||
|
const maxEntriesRaw = parseEnvInt(
|
||||||
|
process.env.MCP_COLLAB_SESSION_MAX_ENTRIES,
|
||||||
|
32,
|
||||||
|
);
|
||||||
|
const maxEntries = maxEntriesRaw > 0 ? maxEntriesRaw : 32;
|
||||||
|
return { idleMs, maxAgeMs, maxEntries };
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The subset of HocuspocusProvider this module depends on, so the provider can
|
||||||
|
* be replaced with a fake in unit tests (there is no server in the test env).
|
||||||
|
*/
|
||||||
|
export interface CollabProviderLike {
|
||||||
|
synced: boolean;
|
||||||
|
unsyncedChanges: number;
|
||||||
|
destroy(): void;
|
||||||
|
on(event: "unsyncedChanges", handler: (data: { number: number }) => void): void;
|
||||||
|
off(event: "unsyncedChanges", handler: (data: { number: number }) => void): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The configuration object passed to the provider factory. */
|
||||||
|
export interface CollabProviderConfig {
|
||||||
|
url: string;
|
||||||
|
name: string;
|
||||||
|
document: Y.Doc;
|
||||||
|
token: string;
|
||||||
|
WebSocketPolyfill: unknown;
|
||||||
|
onConnect: () => void;
|
||||||
|
onSynced: () => void;
|
||||||
|
onDisconnect: () => void;
|
||||||
|
onClose: () => void;
|
||||||
|
onAuthenticationFailed: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type CollabProviderFactory = (
|
||||||
|
config: CollabProviderConfig,
|
||||||
|
) => CollabProviderLike;
|
||||||
|
|
||||||
|
const defaultProviderFactory: CollabProviderFactory = (config) =>
|
||||||
|
// @ts-ignore - WebSocketPolyfill is required for the Node.js environment.
|
||||||
|
new HocuspocusProvider(config) as unknown as CollabProviderLike;
|
||||||
|
|
||||||
|
let providerFactory: CollabProviderFactory = defaultProviderFactory;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* TEST SEAM: swap the provider factory (pass null to restore the real one).
|
||||||
|
* Not part of the public API — used only by the unit tests, which cannot reach
|
||||||
|
* a real collaboration server.
|
||||||
|
*/
|
||||||
|
export function __setCollabProviderFactory(
|
||||||
|
factory: CollabProviderFactory | null,
|
||||||
|
): void {
|
||||||
|
providerFactory = factory ?? defaultProviderFactory;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Optional per-acquire hooks (metrics), passed through from the call site. */
|
||||||
|
export interface AcquireOptions {
|
||||||
|
/** Invoked when the initial connect handshake times out (CONNECT_TIMEOUT_MS). */
|
||||||
|
onConnectTimeout?: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
type SessionState = "connecting" | "ready" | "dead";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* One live provider + ydoc for a single (wsUrl, pageId, token) triple.
|
||||||
|
*
|
||||||
|
* Lifecycle: connecting -> ready -> dead. A session becomes `dead` on the first
|
||||||
|
* disconnect/close/auth-failure at ANY time, on an idle/eviction/max-age
|
||||||
|
* teardown, or on an explicit destroy(); death is terminal and removes the
|
||||||
|
* session from the registry so the next acquire opens a fresh one. We never use
|
||||||
|
* the provider's auto-reconnect — destroying on the first disconnect closes the
|
||||||
|
* "reconnect drove unsyncedChanges to 0 without retransmitting our write" class
|
||||||
|
* of false success.
|
||||||
|
*/
|
||||||
|
export class CollabSession {
|
||||||
|
readonly key: string;
|
||||||
|
readonly pageId: string;
|
||||||
|
readonly wsUrl: string;
|
||||||
|
readonly token: string;
|
||||||
|
readonly createdAt: number;
|
||||||
|
state: SessionState = "connecting";
|
||||||
|
/**
|
||||||
|
* Set true on disconnect/close/auth-failure so a reconnect-driven
|
||||||
|
* unsyncedChanges->0 cannot be mistaken for a successful persist of our
|
||||||
|
* write (preserved verbatim from the one-shot machine).
|
||||||
|
*/
|
||||||
|
connectionLost = false;
|
||||||
|
|
||||||
|
provider: CollabProviderLike | undefined;
|
||||||
|
private readonly ydoc: Y.Doc;
|
||||||
|
private readonly cfg: SessionConfig;
|
||||||
|
/**
|
||||||
|
* Ephemeral sessions (cache disabled, MCP_COLLAB_SESSION_IDLE_MS<=0) are never
|
||||||
|
* registered and self-destroy after their single op — the legacy
|
||||||
|
* provider-per-op behavior.
|
||||||
|
*/
|
||||||
|
private readonly ephemeral: boolean;
|
||||||
|
private readonly opts: AcquireOptions | undefined;
|
||||||
|
|
||||||
|
private dead = false;
|
||||||
|
private connectTimer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
private idleTimer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
private openPromise: Promise<void> | undefined;
|
||||||
|
private openResolve: (() => void) | undefined;
|
||||||
|
private openReject: ((err: Error) => void) | undefined;
|
||||||
|
private openSettled = false;
|
||||||
|
/**
|
||||||
|
* The rejector of the CURRENT in-flight mutate, if any. A disconnect/close/
|
||||||
|
* auth-failure or timeout at ANY time rejects the in-flight op through this
|
||||||
|
* with the SAME error text the one-shot machine emitted.
|
||||||
|
*/
|
||||||
|
private inflightReject: ((err: Error) => void) | undefined;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
key: string,
|
||||||
|
pageId: string,
|
||||||
|
wsUrl: string,
|
||||||
|
token: string,
|
||||||
|
cfg: SessionConfig,
|
||||||
|
ephemeral: boolean,
|
||||||
|
opts: AcquireOptions | undefined,
|
||||||
|
) {
|
||||||
|
this.key = key;
|
||||||
|
this.pageId = pageId;
|
||||||
|
this.wsUrl = wsUrl;
|
||||||
|
this.token = token;
|
||||||
|
this.cfg = cfg;
|
||||||
|
this.ephemeral = ephemeral;
|
||||||
|
this.opts = opts;
|
||||||
|
this.createdAt = Date.now();
|
||||||
|
this.ydoc = new Y.Doc();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A cached session may be reused only when it is fully ready, still synced,
|
||||||
|
* has not lost its connection, and has not exceeded its max age (invariant 5
|
||||||
|
* "validate on reuse" + the max-age acquire check).
|
||||||
|
*/
|
||||||
|
isReusable(): boolean {
|
||||||
|
return (
|
||||||
|
!this.dead &&
|
||||||
|
this.state === "ready" &&
|
||||||
|
!this.connectionLost &&
|
||||||
|
!!this.provider &&
|
||||||
|
this.provider.synced === true &&
|
||||||
|
Date.now() - this.createdAt < this.cfg.maxAgeMs
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect and wait for the initial sync (onSynced) within CONNECT_TIMEOUT_MS.
|
||||||
|
* Idempotent: repeated calls return the same in-flight/settled promise.
|
||||||
|
*/
|
||||||
|
open(): Promise<void> {
|
||||||
|
if (this.openPromise) return this.openPromise;
|
||||||
|
this.openPromise = new Promise<void>((resolve, reject) => {
|
||||||
|
this.openResolve = resolve;
|
||||||
|
this.openReject = reject;
|
||||||
|
|
||||||
|
this.connectTimer = setTimeout(() => {
|
||||||
|
// The 25s connect timeout: the collab connection never became ready.
|
||||||
|
this.opts?.onConnectTimeout?.();
|
||||||
|
this.teardown(
|
||||||
|
new Error("Connection timeout to collaboration server"),
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
}, CONNECT_TIMEOUT_MS);
|
||||||
|
|
||||||
|
if (process.env.DEBUG)
|
||||||
|
console.error(`Connecting to WebSocket: ${this.wsUrl}`);
|
||||||
|
|
||||||
|
this.provider = providerFactory({
|
||||||
|
url: this.wsUrl,
|
||||||
|
name: `page.${this.pageId}`,
|
||||||
|
document: this.ydoc,
|
||||||
|
token: this.token,
|
||||||
|
WebSocketPolyfill: WebSocket,
|
||||||
|
onConnect: () => {
|
||||||
|
if (process.env.DEBUG) console.error("WS Connect");
|
||||||
|
},
|
||||||
|
// An unexpected disconnect/close at ANY time (during the connect-wait,
|
||||||
|
// between edits, or during a persistence wait) makes the session dead:
|
||||||
|
// surface it now instead of hanging, reject any in-flight op with the
|
||||||
|
// same error text as the one-shot machine, and remove ourselves from
|
||||||
|
// the registry so the next acquire opens fresh. `teardown` is idempotent
|
||||||
|
// so the onClose our own destroy() triggers is a harmless no-op.
|
||||||
|
onDisconnect: () => {
|
||||||
|
if (process.env.DEBUG) console.error("WS Disconnect");
|
||||||
|
this.teardown(
|
||||||
|
new Error(
|
||||||
|
"Collaboration connection closed before the update was persisted/synced",
|
||||||
|
),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
onClose: () => {
|
||||||
|
if (process.env.DEBUG) console.error("WS Close");
|
||||||
|
this.teardown(
|
||||||
|
new Error(
|
||||||
|
"Collaboration connection closed before the update was persisted/synced",
|
||||||
|
),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
onSynced: () => {
|
||||||
|
if (this.dead || this.openSettled) return;
|
||||||
|
if (process.env.DEBUG) console.error("Connected and synced!");
|
||||||
|
if (this.connectTimer) {
|
||||||
|
clearTimeout(this.connectTimer);
|
||||||
|
this.connectTimer = undefined;
|
||||||
|
}
|
||||||
|
this.state = "ready";
|
||||||
|
this.openSettled = true;
|
||||||
|
this.openResolve?.();
|
||||||
|
},
|
||||||
|
onAuthenticationFailed: () => {
|
||||||
|
this.teardown(
|
||||||
|
new Error("Authentication failed for collaboration connection"),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
});
|
||||||
|
return this.openPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Run one atomic read -> transform -> write against the LIVE doc and wait for
|
||||||
|
* the server to acknowledge the write.
|
||||||
|
*
|
||||||
|
* INVARIANT 1 (read->write atomicity): between `TiptapTransformer.fromYdoc`
|
||||||
|
* and `applyDocToFragment` there is NO `await`. Yjs applies remote updates
|
||||||
|
* only when the event loop yields, so this synchronous block sees a consistent
|
||||||
|
* live doc and no concurrent human edit can interleave and be clobbered —
|
||||||
|
* exactly as in the one-shot onSynced code, just on a persistent provider.
|
||||||
|
*
|
||||||
|
* INVARIANT 2 (per-edit ack): after the write, resolve immediately if
|
||||||
|
* unsyncedChanges is already 0, else wait for the unsyncedChanges->0 event
|
||||||
|
* (PERSIST_TIMEOUT_MS), guarded by connectionLost so a reconnect handshake
|
||||||
|
* cannot report a false success.
|
||||||
|
*
|
||||||
|
* CONCURRENCY: not safe to invoke concurrently on ONE session — the caller
|
||||||
|
* MUST serialize (hold the per-page lock), mirroring acquireCollabSession.
|
||||||
|
* The in-flight op is tracked in a single `inflightReject` field, so an
|
||||||
|
* overlapping second call would clobber the first's rejector and leave it
|
||||||
|
* hanging on disconnect. A fail-fast guard below rejects the overlap instead.
|
||||||
|
* Sequential (awaited) mutates are fine: localFinish clears inflightReject
|
||||||
|
* before the promise settles, so the guard is clear by the time the next runs.
|
||||||
|
*/
|
||||||
|
mutate(
|
||||||
|
transform: (liveDoc: any) => any | null,
|
||||||
|
): Promise<MutationResult> {
|
||||||
|
// Belt-and-suspenders (acquire already validated): refuse to write on a
|
||||||
|
// session that is not in a live, synced, ready state.
|
||||||
|
if (
|
||||||
|
this.dead ||
|
||||||
|
this.state !== "ready" ||
|
||||||
|
this.connectionLost ||
|
||||||
|
!this.provider ||
|
||||||
|
this.provider.synced !== true
|
||||||
|
) {
|
||||||
|
return Promise.reject(
|
||||||
|
new Error("Collaboration session is not in a ready state"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fail-fast on concurrent use: a second overlapping mutate would overwrite
|
||||||
|
// the first's inflightReject, so a disconnect would only reject the second
|
||||||
|
// and hang the first until PERSIST_TIMEOUT_MS. Reject the overlap WITHOUT
|
||||||
|
// touching the in-flight op's state (no localFinish/teardown here).
|
||||||
|
if (this.inflightReject) {
|
||||||
|
return Promise.reject(
|
||||||
|
new Error(
|
||||||
|
"mutate already in-flight; caller must serialize (hold the page lock)",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise<MutationResult>((resolve, reject) => {
|
||||||
|
let settled = false;
|
||||||
|
let persistTimer: ReturnType<typeof setTimeout> | undefined;
|
||||||
|
let unsyncedHandler:
|
||||||
|
| ((data: { number: number }) => void)
|
||||||
|
| undefined;
|
||||||
|
// The verifiable result resolved on every success/abort path. Set on
|
||||||
|
// abort (no-op report) and after a real write (computed change report).
|
||||||
|
let mutationResult: MutationResult;
|
||||||
|
|
||||||
|
const localFinish = (err: Error | null, value?: MutationResult) => {
|
||||||
|
if (settled) return;
|
||||||
|
settled = true;
|
||||||
|
if (persistTimer) clearTimeout(persistTimer);
|
||||||
|
if (unsyncedHandler && this.provider) {
|
||||||
|
try {
|
||||||
|
this.provider.off("unsyncedChanges", unsyncedHandler);
|
||||||
|
} catch (e) {}
|
||||||
|
}
|
||||||
|
this.inflightReject = undefined;
|
||||||
|
if (err) reject(err);
|
||||||
|
else resolve(value as MutationResult);
|
||||||
|
// Post-settle lifecycle: an ephemeral (cache-disabled) session dies with
|
||||||
|
// its single op; a cached session that is still alive re-arms its idle
|
||||||
|
// TTL so the clock starts from the LAST op.
|
||||||
|
if (this.ephemeral) {
|
||||||
|
this.destroy("ephemeral op complete");
|
||||||
|
} else if (!this.dead) {
|
||||||
|
this.armIdle();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Register so a disconnect/close/auth-failure/teardown rejects THIS op
|
||||||
|
// with the connection-loss error text. localFinish's `settled` guard makes
|
||||||
|
// a racing teardown + normal resolve safe (first one wins).
|
||||||
|
this.inflightReject = (e: Error) => localFinish(e);
|
||||||
|
|
||||||
|
// Resolve once the server acknowledges our update: the provider increments
|
||||||
|
// unsyncedChanges when the local update is sent and decrements it on the
|
||||||
|
// server's SyncStatus(applied=true); reaching 0 means the authoritative
|
||||||
|
// in-memory ydoc on the server now contains our write.
|
||||||
|
const waitForPersistence = () => {
|
||||||
|
if (settled) return;
|
||||||
|
// A missing provider is a failure, not a success: without it the write
|
||||||
|
// can never have been acknowledged.
|
||||||
|
if (!this.provider) {
|
||||||
|
localFinish(new Error("collab provider gone before persistence"));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (this.provider.unsyncedChanges === 0) {
|
||||||
|
localFinish(null, mutationResult);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
persistTimer = setTimeout(() => {
|
||||||
|
localFinish(
|
||||||
|
new Error(
|
||||||
|
"Timeout waiting for collaboration server to persist the update",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}, PERSIST_TIMEOUT_MS);
|
||||||
|
unsyncedHandler = (data: { number: number }) => {
|
||||||
|
// Only treat unsyncedChanges->0 as success when the connection is
|
||||||
|
// still up. A transient disconnect + reconnect handshake can drive the
|
||||||
|
// counter back to 0 without our write being re-transmitted; in that
|
||||||
|
// case let the disconnect/close error win instead.
|
||||||
|
if (data.number === 0 && !this.connectionLost) {
|
||||||
|
localFinish(null, mutationResult);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
this.provider.on("unsyncedChanges", unsyncedHandler);
|
||||||
|
};
|
||||||
|
|
||||||
|
// CRITICAL: everything between reading the live doc and writing it back
|
||||||
|
// must stay synchronous (no await). While the JS event loop is not
|
||||||
|
// yielded, no incoming remote update can interleave, so any already-synced
|
||||||
|
// concurrent edits are preserved in liveDoc.
|
||||||
|
let newDoc: any;
|
||||||
|
let beforeDoc: any;
|
||||||
|
try {
|
||||||
|
let liveDoc = TiptapTransformer.fromYdoc(this.ydoc, "default");
|
||||||
|
if (
|
||||||
|
!liveDoc ||
|
||||||
|
typeof liveDoc !== "object" ||
|
||||||
|
!Array.isArray(liveDoc.content)
|
||||||
|
) {
|
||||||
|
liveDoc = { type: "doc", content: [] };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot the before-doc for the change report. Docs are
|
||||||
|
// JSON-serializable, so this is a safe deep clone.
|
||||||
|
beforeDoc = JSON.parse(JSON.stringify(liveDoc));
|
||||||
|
|
||||||
|
newDoc = transform(liveDoc);
|
||||||
|
|
||||||
|
if (newDoc == null) {
|
||||||
|
// Transform aborted — write nothing, return the live doc with a no-op
|
||||||
|
// change report.
|
||||||
|
mutationResult = {
|
||||||
|
doc: liveDoc,
|
||||||
|
verify: {
|
||||||
|
changed: false,
|
||||||
|
textInserted: 0,
|
||||||
|
textDeleted: 0,
|
||||||
|
blocksChanged: 0,
|
||||||
|
marks: {},
|
||||||
|
summary: "no changes (transform aborted)",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
localFinish(null, mutationResult);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Structural diff into the live fragment (issue #152): preserves the Yjs
|
||||||
|
// ids of unchanged nodes, so an open editor's cursor is not yanked to the
|
||||||
|
// end of the document on every agent write.
|
||||||
|
applyDocToFragment(this.ydoc, newDoc);
|
||||||
|
} catch (e) {
|
||||||
|
// Includes errors thrown by transform (e.g. "afterText not found",
|
||||||
|
// "text not found"): propagate them verbatim to the caller.
|
||||||
|
localFinish(e instanceof Error ? e : new Error(String(e)));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compute the verifiable change report AFTER the transact write: it only
|
||||||
|
// needs the JSON before/after, so it cannot affect the atomic read->write
|
||||||
|
// window, and summarizeChange never throws.
|
||||||
|
mutationResult = {
|
||||||
|
doc: newDoc,
|
||||||
|
verify: summarizeChange(beforeDoc, newDoc),
|
||||||
|
};
|
||||||
|
if (process.env.DEBUG)
|
||||||
|
console.error("Content written, waiting for server to persist...");
|
||||||
|
waitForPersistence();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** (Re)arm the idle TTL so the clock starts from the most recent activity. */
|
||||||
|
armIdle(): void {
|
||||||
|
if (this.dead || this.ephemeral) return;
|
||||||
|
if (this.idleTimer) clearTimeout(this.idleTimer);
|
||||||
|
if (this.cfg.idleMs > 0) {
|
||||||
|
this.idleTimer = setTimeout(() => {
|
||||||
|
this.destroy("idle timeout");
|
||||||
|
}, this.cfg.idleMs);
|
||||||
|
// Never let the idle timer keep the process alive.
|
||||||
|
(this.idleTimer as any).unref?.();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Idempotent teardown: mark dead, clear timers, remove from the registry, fail
|
||||||
|
* any pending open/in-flight op, and destroy the provider. `inflightError` is
|
||||||
|
* the error a pending open or in-flight op is rejected with; `connectionLoss`
|
||||||
|
* marks the session as connection-lost so the ack guard cannot report a false
|
||||||
|
* success on a racing unsyncedChanges->0.
|
||||||
|
*/
|
||||||
|
private teardown(inflightError: Error | null, connectionLoss: boolean): void {
|
||||||
|
if (this.dead) return;
|
||||||
|
this.dead = true;
|
||||||
|
this.state = "dead";
|
||||||
|
if (connectionLoss) this.connectionLost = true;
|
||||||
|
|
||||||
|
if (this.connectTimer) {
|
||||||
|
clearTimeout(this.connectTimer);
|
||||||
|
this.connectTimer = undefined;
|
||||||
|
}
|
||||||
|
if (this.idleTimer) {
|
||||||
|
clearTimeout(this.idleTimer);
|
||||||
|
this.idleTimer = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove ourselves from the registry (only if we are still the live entry —
|
||||||
|
// a re-open under the same key must not be evicted by our teardown).
|
||||||
|
if (sessions.get(this.key) === this) {
|
||||||
|
sessions.delete(this.key);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fail a pending open() and any in-flight mutate with the terminal error.
|
||||||
|
if (!this.openSettled) {
|
||||||
|
this.openSettled = true;
|
||||||
|
this.openReject?.(
|
||||||
|
inflightError ?? new Error("Collaboration session destroyed"),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (this.inflightReject) {
|
||||||
|
const rej = this.inflightReject;
|
||||||
|
this.inflightReject = undefined;
|
||||||
|
rej(inflightError ?? new Error("Collaboration session destroyed"));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.provider) {
|
||||||
|
try {
|
||||||
|
this.provider.destroy();
|
||||||
|
} catch (e) {}
|
||||||
|
this.provider = undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Public idempotent teardown used by the acquire/eviction paths and by a
|
||||||
|
* caller that wants the session dropped after a failed op ("next call
|
||||||
|
* reconnects fresh").
|
||||||
|
*/
|
||||||
|
destroy(reason: string): void {
|
||||||
|
if (this.dead) return;
|
||||||
|
if (process.env.DEBUG)
|
||||||
|
console.error(`Destroying collab session ${this.pageId}: ${reason}`);
|
||||||
|
this.teardown(new Error(`Collaboration session destroyed: ${reason}`), false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** key = wsUrl + pageId + collabToken (identity isolation: invariant 4). */
|
||||||
|
const sessions = new Map<string, CollabSession>();
|
||||||
|
|
||||||
|
function sessionKey(wsUrl: string, pageId: string, token: string): string {
|
||||||
|
// The token is part of the key so sessions are NEVER shared between different
|
||||||
|
// users' MCP sessions (HTTP mode), and a token rotation makes a new entry
|
||||||
|
// while the old one idles out.
|
||||||
|
return `${wsUrl} | ||||||