fix(ai-chat): MCP-кэш — in-run восстановление, ретраи только для readOnly (#489)
Кэш внешних MCP-клиентов не делал health-check/reconnect-on-error: после
разрыва SSE-транспорта (bodyTimeout режет при тишине МЕЖДУ вызовами) отдавал
труп до конца TTL, а модель жгла шаги на ретраях ВНУТРИ текущего рана.
- Новое поле writeClass ('readOnly'|'write') на КАЖДОМ SHARED_TOOL_SPEC +
registration-time assert (и compile-time через satisfies). Все 48 спеков
расклассифицированы: чтения → readOnly, любая мутация страницы/коммента/шэра/
диаграммы → write. Экспортированы SHARED_TOOL_WRITE_CLASS + isRetryableWriteClass.
- Per-run обёртка восстановления транспорта: при транспортной ошибке readOnly-тул
реконнектит свой сервер и ретраит РОВНО 1 раз ВНУТРИ рана; write-тул НЕ
авторетраится (indeterminate — «могло примениться, проверь», класс инцидента
#435). CAS-своп байндинга по identity (проигравший конкурентный вызов ретраит
на текущем клиенте, не минтит второй). Лизы не освобождаются mid-run — ран
копит set (старая+новая) и релизит на turn-end.
- Проверка abortSignal ПЕРЕД ретраем И ПЕРЕД чеканкой свежего клиента; per-call
cap покрывает оба attempt'а + connect.
- Классификация транспортной ошибки по РЕАЛЬНЫМ шейпам undici (SocketError/
BodyTimeoutError, cause-цепочка), не по мок-ошибкам.
- Отдельный, поднятый bodyTimeout для SSE-транспорта MCP (тишина между вызовами
легальна) — DEFAULT 10 мин, AI_MCP_SSE_BODY_TIMEOUT_MS.
write-class map грузится в mcp-clients лениво через dynamic import (пакет ESM),
type-only импорт — без static require ESM из commonjs.
Тесты на РЕАЛЬНЫХ error-шейпах: «повтор после обрыва ВНУТРИ рана получает живой
клиент», «write-тул не авторетраится», «ретрай после Stop не происходит»,
+ writeClass-контракт в mcp node --test.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,214 @@
|
||||
import { errors } from 'undici';
|
||||
import {
|
||||
McpClientsService,
|
||||
isRetryableConnectError,
|
||||
} from './mcp-clients.service';
|
||||
|
||||
/**
|
||||
* #489 — external-MCP in-run transport recovery.
|
||||
*
|
||||
* The transport-error classification + retry gate are exercised against the REAL
|
||||
* undici error CLASSES prod throws (`errors.SocketError` / `errors.BodyTimeoutError`,
|
||||
* carrying the true `UND_ERR_*` codes and class names), wrapped EXACTLY as undici's
|
||||
* `fetch` wraps them — a `TypeError('fetch failed'|'terminated')` whose `.cause` is
|
||||
* the undici error. These are the real classes, not hand-rolled `{code:'...'}`
|
||||
* mocks: constructing the genuine class is what makes this a faithful test of the
|
||||
* prod predicate (epic root-cause #4 — a mock-shaped predicate would leave the
|
||||
* evict/retry path silently dead in production while CI stays green). We construct
|
||||
* rather than drive a live fetch because Jest's environment degrades the live-fetch
|
||||
* error to a generic `Error` cause (no undici code), which would NOT be the prod
|
||||
* shape.
|
||||
*/
|
||||
|
||||
/** A REAL undici socket reset, wrapped as fetch wraps it. */
|
||||
function realSocketResetError(): unknown {
|
||||
const err = new TypeError('fetch failed');
|
||||
(err as { cause?: unknown }).cause = new errors.SocketError('other side closed');
|
||||
return err;
|
||||
}
|
||||
|
||||
/** A REAL undici body timeout, wrapped as fetch wraps it. */
|
||||
function realBodyTimeoutError(): unknown {
|
||||
const err = new TypeError('terminated');
|
||||
(err as { cause?: unknown }).cause = new errors.BodyTimeoutError();
|
||||
return err;
|
||||
}
|
||||
|
||||
type FakeServer = {
|
||||
id: string;
|
||||
name: string;
|
||||
transport: 'http' | 'sse';
|
||||
url: string;
|
||||
headersEnc: string | null;
|
||||
toolAllowlist: string[] | null;
|
||||
instructions: string | null;
|
||||
};
|
||||
|
||||
const server = (over: Partial<FakeServer> = {}): FakeServer => ({
|
||||
id: 's1',
|
||||
name: 'srv',
|
||||
transport: 'http',
|
||||
url: 'http://example.test/mcp',
|
||||
headersEnc: null,
|
||||
toolAllowlist: null,
|
||||
instructions: null,
|
||||
...over,
|
||||
});
|
||||
|
||||
function buildService(servers: FakeServer[]) {
|
||||
const repo = { listEnabled: jest.fn().mockResolvedValue(servers) };
|
||||
const service = new McpClientsService(repo as never, {} as never);
|
||||
// Seed a DETERMINISTIC write-class map so the retry gate is controlled here
|
||||
// (the production map loads from @docmost/mcp via a dynamic ESM import). getPage
|
||||
// is a read, patchNode is a write — the real classifications.
|
||||
(
|
||||
service as unknown as { writeClassMapPromise: Promise<unknown> }
|
||||
).writeClassMapPromise = Promise.resolve({
|
||||
getPage: 'readOnly',
|
||||
patchNode: 'write',
|
||||
});
|
||||
return { service, repo };
|
||||
}
|
||||
|
||||
/** Spy the private `connect` so each call yields a controlled fake client whose
|
||||
* single tool's execute is the supplied function. Returns the connect spy. */
|
||||
function stubConnect(
|
||||
service: McpClientsService,
|
||||
toolName: string,
|
||||
execs: Array<(...a: unknown[]) => Promise<unknown>>,
|
||||
) {
|
||||
let n = 0;
|
||||
return jest
|
||||
.spyOn(
|
||||
service as unknown as { connect: (s: FakeServer) => Promise<unknown> },
|
||||
'connect',
|
||||
)
|
||||
.mockImplementation(async () => {
|
||||
const exec = execs[Math.min(n, execs.length - 1)];
|
||||
n += 1;
|
||||
return {
|
||||
tools: async () => ({ [toolName]: { description: 'x', execute: exec } }),
|
||||
close: jest.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
const opts = (abortSignal?: AbortSignal) =>
|
||||
({ toolCallId: 't', messages: [], abortSignal }) as never;
|
||||
|
||||
describe('isRetryableConnectError (#489, REAL error shapes)', () => {
|
||||
it('classifies a real undici socket reset and body timeout as retryable', async () => {
|
||||
const socketErr = await realSocketResetError();
|
||||
const bodyErr = await realBodyTimeoutError();
|
||||
expect(isRetryableConnectError(socketErr)).toBe(true);
|
||||
expect(isRetryableConnectError(bodyErr)).toBe(true);
|
||||
// Unwraps a wrapped cause chain (e.g. an MCPClientError around the socket err).
|
||||
const wrapped = new Error('mcp call failed');
|
||||
(wrapped as { cause?: unknown }).cause = socketErr;
|
||||
expect(isRetryableConnectError(wrapped)).toBe(true);
|
||||
});
|
||||
|
||||
it('does NOT classify an application-level error as a transport break', () => {
|
||||
expect(isRetryableConnectError(new Error('validation failed'))).toBe(false);
|
||||
expect(isRetryableConnectError({ name: 'HttpError', status: 400 })).toBe(false);
|
||||
expect(isRetryableConnectError(undefined)).toBe(false);
|
||||
expect(isRetryableConnectError('boom')).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('McpClientsService in-run transport recovery (#489)', () => {
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('a readOnly tool whose transport breaks reconnects and retries WITHIN the same run', async () => {
|
||||
const realErr = await realSocketResetError();
|
||||
const { service } = buildService([server()]);
|
||||
const first = jest.fn().mockRejectedValue(realErr);
|
||||
const second = jest.fn().mockResolvedValue({ ok: true });
|
||||
const connectSpy = stubConnect(service, 'getPage', [first, second]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-1');
|
||||
const tool = toolset.tools['srv_getPage'];
|
||||
const result = await (tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(),
|
||||
);
|
||||
|
||||
// The repeat call within the run got a LIVE client and succeeded.
|
||||
expect(result).toEqual({ ok: true });
|
||||
expect(first).toHaveBeenCalledTimes(1);
|
||||
expect(second).toHaveBeenCalledTimes(1);
|
||||
// Exactly one reconnect was minted (initial build connect + one recovery).
|
||||
expect(connectSpy).toHaveBeenCalledTimes(2);
|
||||
// The run accumulated BOTH leases (old + reconnected) — released together at end.
|
||||
expect(toolset.clients).toHaveLength(2);
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
|
||||
it('a WRITE tool does NOT auto-retry on a transport error (indeterminate)', async () => {
|
||||
const realErr = await realSocketResetError();
|
||||
const { service } = buildService([server()]);
|
||||
const exec = jest.fn().mockRejectedValue(realErr);
|
||||
const connectSpy = stubConnect(service, 'patchNode', [exec]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-2');
|
||||
const tool = toolset.tools['srv_patchNode'];
|
||||
await expect(
|
||||
(tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(),
|
||||
),
|
||||
).rejects.toThrow(/MAY have already applied/);
|
||||
|
||||
// Called exactly once — NO blind retry (avoids double-apply, the #435 class).
|
||||
expect(exec).toHaveBeenCalledTimes(1);
|
||||
// No fresh connection was minted for a write.
|
||||
expect(connectSpy).toHaveBeenCalledTimes(1);
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
|
||||
it('does NOT retry (or reconnect) after the run is aborted (Stop)', async () => {
|
||||
const realErr = await realSocketResetError();
|
||||
const { service } = buildService([server()]);
|
||||
const controller = new AbortController();
|
||||
// The transport error arrives, but the run was Stopped in the same tick.
|
||||
const first = jest.fn().mockImplementation(async () => {
|
||||
controller.abort();
|
||||
throw realErr;
|
||||
});
|
||||
const second = jest.fn().mockResolvedValue({ ok: true });
|
||||
const connectSpy = stubConnect(service, 'getPage', [first, second]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-3');
|
||||
const tool = toolset.tools['srv_getPage'];
|
||||
await expect(
|
||||
(tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(controller.signal),
|
||||
),
|
||||
).rejects.toBeDefined();
|
||||
|
||||
// getPage IS readOnly, but the Stop blocks the retry — no second call, no mint.
|
||||
expect(second).not.toHaveBeenCalled();
|
||||
expect(connectSpy).toHaveBeenCalledTimes(1);
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
|
||||
it('an app-level (non-transport) tool error is surfaced verbatim, never retried', async () => {
|
||||
const { service } = buildService([server()]);
|
||||
const appErr = new Error('tool says: bad input');
|
||||
const exec = jest.fn().mockRejectedValue(appErr);
|
||||
const connectSpy = stubConnect(service, 'getPage', [exec]);
|
||||
|
||||
const toolset = await service.toolsFor('ws-4');
|
||||
const tool = toolset.tools['srv_getPage'];
|
||||
await expect(
|
||||
(tool.execute as (a: unknown, o: unknown) => Promise<unknown>)(
|
||||
{ pageId: 'p' },
|
||||
opts(),
|
||||
),
|
||||
).rejects.toThrow('tool says: bad input');
|
||||
expect(exec).toHaveBeenCalledTimes(1);
|
||||
expect(connectSpy).toHaveBeenCalledTimes(1); // no reconnect for an app error
|
||||
await Promise.all(toolset.clients.map((c) => c.close()));
|
||||
});
|
||||
});
|
||||
@@ -106,8 +106,11 @@ describe('McpClientsService.decryptHeaders', () => {
|
||||
|
||||
describe('McpClientsService.guardedFetch (SSRF per-request guard)', () => {
|
||||
// The bound guardedFetch closure lives on the instance as a private field.
|
||||
// #489 split it into per-transport HTTP/SSE bindings (they differ only in the
|
||||
// dispatcher's bodyTimeout); the SSRF guard is identical, so testing the HTTP
|
||||
// one is sufficient.
|
||||
const guardedFetchOf = (service: McpClientsService) =>
|
||||
(service as unknown as { guardedFetch: typeof fetch }).guardedFetch;
|
||||
(service as unknown as { guardedFetchHttp: typeof fetch }).guardedFetchHttp;
|
||||
|
||||
let fetchSpy: jest.SpiedFunction<typeof fetch>;
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { isIP } from 'node:net';
|
||||
import { lookup as dnsLookup, type LookupAddress } from 'node:dns';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { type Tool, type ToolCallOptions } from 'ai';
|
||||
import { createMCPClient } from '@ai-sdk/mcp';
|
||||
@@ -10,9 +11,29 @@ import {
|
||||
streamingDispatcherOptions,
|
||||
mcpStreamTimeoutMs,
|
||||
mcpCallTimeoutMs,
|
||||
mcpSseBodyTimeoutMs,
|
||||
} from '../../../integrations/ai/ai-streaming-fetch';
|
||||
import { SecretBoxService } from '../../../integrations/crypto/secret-box';
|
||||
import { isUrlAllowed, isIpAllowed } from './ssrf-guard';
|
||||
// TYPE-ONLY (erased at compile): @docmost/mcp is ESM-only and cannot be a runtime
|
||||
// `require()` from this commonjs module (same constraint as docmost-client.loader).
|
||||
// The write-class MAP is loaded lazily via the dynamic-import trick below.
|
||||
import type { ToolWriteClass } from '@docmost/mcp';
|
||||
|
||||
// TS(commonjs) downlevels a literal `import()` to `require()`, which cannot load
|
||||
// the ESM-only @docmost/mcp. Indirect through Function so the real dynamic
|
||||
// `import()` survives compilation (same trick as docmost-client.loader.ts).
|
||||
const esmImport = new Function(
|
||||
'specifier',
|
||||
'return import(specifier)',
|
||||
) as (specifier: string) => Promise<unknown>;
|
||||
|
||||
/** Local read-only predicate — avoids a value import of the ESM-only package.
|
||||
* Only a pure read is retry-safe after a transport break (a write is
|
||||
* indeterminate). Kept in lockstep with @docmost/mcp's isRetryableWriteClass. */
|
||||
function isReadOnlyWriteClass(writeClass: ToolWriteClass | undefined): boolean {
|
||||
return writeClass === 'readOnly';
|
||||
}
|
||||
|
||||
/** A closable external MCP client handle. */
|
||||
export interface Closable {
|
||||
@@ -81,12 +102,52 @@ const MAX_TOOL_NAME_LENGTH = 64;
|
||||
* close until the turn releases it, so a TTL expiry mid-turn never closes a
|
||||
* client a stream is still executing against.
|
||||
*/
|
||||
/**
|
||||
* Where a merged (namespaced) tool came from, so the per-run recovery wrapper
|
||||
* (#489) can, on a transport error, reconnect THAT server and re-resolve the SAME
|
||||
* underlying tool by its raw name. `writeClass` gates the single auto-retry (a
|
||||
* read is retry-safe; a write is indeterminate). `serverIndex` indexes the
|
||||
* entry's `servers` array (which server config to reconnect).
|
||||
*/
|
||||
interface ToolProvenance {
|
||||
serverIndex: number;
|
||||
rawName: string;
|
||||
writeClass: ToolWriteClass | undefined;
|
||||
}
|
||||
|
||||
/** A live reconnected server (its fresh client + raw call-timeout-wrapped tools). */
|
||||
interface RecoveredServerState {
|
||||
client: McpClient;
|
||||
tools: Record<string, Tool>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-run, per-server recovery binding (#489). `current` is the server's LIVE
|
||||
* target for this run: `null` means "use the ORIGINAL cached client/template";
|
||||
* a non-null value is a reconnected throwaway client all this server's tools now
|
||||
* call. `reconnecting` dedupes concurrent reconnects so only ONE fresh client is
|
||||
* minted per death (a losing concurrent call awaits it and retries on the SAME
|
||||
* new client — the CAS-by-identity rule).
|
||||
*/
|
||||
interface ServerBinding {
|
||||
current: RecoveredServerState | null;
|
||||
reconnecting?: Promise<RecoveredServerState>;
|
||||
}
|
||||
|
||||
interface CacheEntry {
|
||||
tools: Record<string, Tool>;
|
||||
clients: McpClient[];
|
||||
outcomes: ServerOutcome[];
|
||||
/** Prompt guidance for qualifying servers (see McpServerInstruction). */
|
||||
instructions: McpServerInstruction[];
|
||||
/**
|
||||
* The enabled server configs used to build this entry (#489), so the per-run
|
||||
* recovery wrapper can reconnect a specific server by index. Parallel to the
|
||||
* indices referenced by {@link toolMeta}.
|
||||
*/
|
||||
servers: AiMcpServer[];
|
||||
/** merged-tool-key -> provenance (#489), for the per-run recovery wrapper. */
|
||||
toolMeta: Record<string, ToolProvenance>;
|
||||
expiresAt: number;
|
||||
/** Active leases (turns currently using these clients). */
|
||||
refCount: number;
|
||||
@@ -120,20 +181,65 @@ export class McpClientsService {
|
||||
*/
|
||||
private readonly cache = new Map<string, Promise<CacheEntry>>();
|
||||
/**
|
||||
* A single shared SSRF-pinned dispatcher for ALL outbound external-MCP fetches.
|
||||
* Its custom connect.lookup runs per connection, so one instance safely guards
|
||||
* every server's connections (we never connect to an unvalidated IP).
|
||||
* SSRF-pinned dispatchers for outbound external-MCP fetches. Both use the SAME
|
||||
* custom connect.lookup (so every connection is IP-validated), but carry a
|
||||
* DIFFERENT `bodyTimeout` (#489): the HTTP (streamable) transport opens a fresh
|
||||
* request per call, so it keeps the tight silence timeout; the SSE transport
|
||||
* holds ONE long-lived body open across many calls, so a >1-min idle BETWEEN
|
||||
* calls is LEGITIMATE and must not break the socket — it gets a much larger
|
||||
* bodyTimeout. (headersTimeout stays tight on both.)
|
||||
*/
|
||||
private readonly dispatcher: Dispatcher = buildPinnedDispatcher();
|
||||
/** guardedFetch bound to the pinned dispatcher; reused by every transport. */
|
||||
private readonly guardedFetch: typeof fetch = (input, init) =>
|
||||
guardedFetch(this.dispatcher, input, init);
|
||||
private readonly dispatcherHttp: Dispatcher = buildPinnedDispatcher(
|
||||
mcpStreamTimeoutMs(),
|
||||
);
|
||||
private readonly dispatcherSse: Dispatcher = buildPinnedDispatcher(
|
||||
mcpSseBodyTimeoutMs(),
|
||||
);
|
||||
/** guardedFetch bound to each dispatcher; picked by transport type in connect(). */
|
||||
private readonly guardedFetchHttp: typeof fetch = (input, init) =>
|
||||
guardedFetch(this.dispatcherHttp, input, init);
|
||||
private readonly guardedFetchSse: typeof fetch = (input, init) =>
|
||||
guardedFetch(this.dispatcherSse, input, init);
|
||||
|
||||
/**
|
||||
* Memoized write-class map (#489), loaded lazily from @docmost/mcp via the
|
||||
* dynamic-import trick. Keyed by tool name (=== mcpName). A tool NOT in the map
|
||||
* (any third-party external MCP tool) classifies as `undefined` -> treated as a
|
||||
* write by the retry gate (the safe default: never blind-retry an unknown tool).
|
||||
* On any load failure the map is `{}` (every tool -> no auto-retry), so a
|
||||
* missing/older @docmost/mcp build only DISABLES retries, never mis-retries.
|
||||
*/
|
||||
private writeClassMapPromise: Promise<Record<string, ToolWriteClass>> | null =
|
||||
null;
|
||||
|
||||
constructor(
|
||||
private readonly repo: AiMcpServerRepo,
|
||||
private readonly secretBox: SecretBoxService,
|
||||
) {}
|
||||
|
||||
/** Lazily load + memoize the shared write-class map (see the field doc). */
|
||||
private getWriteClassMap(): Promise<Record<string, ToolWriteClass>> {
|
||||
if (!this.writeClassMapPromise) {
|
||||
this.writeClassMapPromise = (async () => {
|
||||
try {
|
||||
const entry = require.resolve('@docmost/mcp');
|
||||
const mod = (await esmImport(pathToFileURL(entry).href)) as {
|
||||
SHARED_TOOL_WRITE_CLASS?: Record<string, ToolWriteClass>;
|
||||
};
|
||||
return mod.SHARED_TOOL_WRITE_CLASS ?? {};
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`Could not load MCP write-class map (auto-retry disabled): ${shortError(
|
||||
err,
|
||||
)}`,
|
||||
);
|
||||
return {};
|
||||
}
|
||||
})();
|
||||
}
|
||||
return this.writeClassMapPromise;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build (or reuse a cached) external toolset for a workspace. Returns the
|
||||
* merged tools, the open client handles to release, and per-server outcomes.
|
||||
@@ -162,11 +268,37 @@ export class McpClientsService {
|
||||
}
|
||||
},
|
||||
};
|
||||
// One release handle drives the whole leased entry; closing it releases all
|
||||
// underlying clients together (they share the same lease lifecycle).
|
||||
|
||||
// #489: the run accumulates a SET of leases — the primary cache lease PLUS any
|
||||
// throwaway client minted by an in-run transport-recovery reconnect. They are
|
||||
// NEVER released mid-run (releasing a swapped-out client while a concurrent
|
||||
// in-flight call still holds it would INDUCE a second failure); the caller
|
||||
// releases the WHOLE set together at turn-end. A recovery reconnect pushes its
|
||||
// lease onto this live array, which the consumer closes over.
|
||||
const leaseSet: Closable[] = [release];
|
||||
|
||||
// #489: per-RUN transport-recovery binding, one per server, SHARED by all of
|
||||
// that server's tools so a swap by one call is seen by the next (CAS by
|
||||
// identity). Kept per-run (here, not in the cached entry) because the binding
|
||||
// + lease-set state is per-run.
|
||||
const bindings = new Map<number, ServerBinding>();
|
||||
const capMs = mcpCallTimeoutMs();
|
||||
|
||||
// Wrap each cached tool with the recovery layer. On a transport error a
|
||||
// declared readOnly tool reconnects its server and retries ONCE; a write is
|
||||
// never blind-retried (indeterminate — may have applied before the reset). A
|
||||
// tool without provenance (a minimal stub entry in a test) passes through raw.
|
||||
const tools: Record<string, Tool> = {};
|
||||
for (const [key, tool] of Object.entries(entry.tools)) {
|
||||
const meta = entry.toolMeta?.[key];
|
||||
tools[key] = meta
|
||||
? this.wrapWithTransportRecovery(entry, meta, tool, leaseSet, bindings, capMs)
|
||||
: tool;
|
||||
}
|
||||
|
||||
return {
|
||||
tools: entry.tools,
|
||||
clients: [release],
|
||||
tools,
|
||||
clients: leaseSet,
|
||||
outcomes: entry.outcomes,
|
||||
instructions: entry.instructions,
|
||||
};
|
||||
@@ -254,6 +386,11 @@ export class McpClientsService {
|
||||
// Per-call total wall-clock cap, read once for this build (env-overridable).
|
||||
const callTimeoutMs = mcpCallTimeoutMs();
|
||||
const instructions: McpServerInstruction[] = [];
|
||||
// merged-key -> provenance for the per-run recovery wrapper (#489).
|
||||
const toolMeta: Record<string, ToolProvenance> = {};
|
||||
// Shared write-class map (#489): classifies each merged tool by its raw name
|
||||
// so the recovery wrapper knows which tools are retry-safe reads.
|
||||
const writeClassMap = await this.getWriteClassMap();
|
||||
|
||||
// Per-server connect+tools result, still tagged with its server so the merge
|
||||
// below can be applied in the SAME order as `servers` (see the parallel note).
|
||||
@@ -332,6 +469,9 @@ export class McpClientsService {
|
||||
result.guarded,
|
||||
server.name,
|
||||
server.id,
|
||||
toolMeta,
|
||||
i,
|
||||
writeClassMap,
|
||||
);
|
||||
outcomes.push({ name: server.name, ok: true });
|
||||
// Include this server's guidance ONLY when it actually contributed at
|
||||
@@ -353,6 +493,8 @@ export class McpClientsService {
|
||||
clients,
|
||||
outcomes,
|
||||
instructions,
|
||||
servers,
|
||||
toolMeta,
|
||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||
refCount: 0,
|
||||
evicted: false,
|
||||
@@ -379,18 +521,30 @@ export class McpClientsService {
|
||||
picked: Record<string, Tool>,
|
||||
serverName: string,
|
||||
serverId: string,
|
||||
toolMeta: Record<string, ToolProvenance>,
|
||||
serverIndex: number,
|
||||
writeClassMap: Record<string, ToolWriteClass>,
|
||||
): { count: number; prefix: string } {
|
||||
let count = 0;
|
||||
for (const [name, tool] of Object.entries(namespace(picked, serverName))) {
|
||||
let key = name;
|
||||
for (const { full, raw, tool } of namespace(picked, serverName)) {
|
||||
let key = full;
|
||||
if (key in target) {
|
||||
const original = key;
|
||||
key = disambiguate(name, serverId, (candidate) => candidate in target);
|
||||
key = disambiguate(full, serverId, (candidate) => candidate in target);
|
||||
this.logger.debug(
|
||||
`External MCP tool name "${original}" collided; renamed to "${key}"`,
|
||||
);
|
||||
}
|
||||
target[key] = tool;
|
||||
// Record provenance so the per-run recovery wrapper (#489) can reconnect
|
||||
// this tool's server and re-resolve it by its raw name, and gate the retry
|
||||
// on its write-class (unknown third-party tool -> undefined -> treated as a
|
||||
// write, i.e. never auto-retried).
|
||||
toolMeta[key] = {
|
||||
serverIndex,
|
||||
rawName: raw,
|
||||
writeClass: writeClassMap[raw],
|
||||
};
|
||||
count += 1;
|
||||
}
|
||||
return { count, prefix: namespacePrefix(serverName) };
|
||||
@@ -424,7 +578,10 @@ export class McpClientsService {
|
||||
// Defense in depth: re-validate the actual request host on EVERY fetch
|
||||
// AND pin the socket to a validated IP via the dispatcher's connect
|
||||
// lookup, closing the DNS-rebinding TOCTOU between check and connect.
|
||||
fetch: this.guardedFetch,
|
||||
// #489: the SSE transport uses the raised-bodyTimeout dispatcher (idle
|
||||
// between calls is legit); HTTP uses the tight one.
|
||||
fetch:
|
||||
transportType === 'sse' ? this.guardedFetchSse : this.guardedFetchHttp,
|
||||
},
|
||||
})) as unknown as McpClient;
|
||||
return client;
|
||||
@@ -505,6 +662,168 @@ export class McpClientsService {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap one merged external tool with the per-run transport-recovery layer (#489).
|
||||
*
|
||||
* attempt 1 runs on the server's CURRENT binding (the cached client, or a client
|
||||
* a sibling tool already reconnected this run). On a REAL transport error
|
||||
* (undici/@ai-sdk socket/body-timeout shapes — {@link isRetryableConnectError},
|
||||
* NOT a mock) and ONLY for a declared readOnly tool, it reconnects the server
|
||||
* and retries EXACTLY ONCE on the fresh client; a write is surfaced as an
|
||||
* indeterminate error (it may have applied before the reset — never
|
||||
* blind-retried). A single per-call cap bounds BOTH attempts + the reconnect,
|
||||
* and the run's abort signal is checked before the retry AND before minting a
|
||||
* fresh connection (no connection is opened for a stopped run).
|
||||
*/
|
||||
private wrapWithTransportRecovery(
|
||||
entry: CacheEntry,
|
||||
meta: ToolProvenance,
|
||||
template: Tool,
|
||||
leaseSet: Closable[],
|
||||
bindings: Map<number, ServerBinding>,
|
||||
capMs: number,
|
||||
): Tool {
|
||||
const original = template.execute;
|
||||
if (typeof original !== 'function') return template;
|
||||
const service = this;
|
||||
const { serverIndex, rawName, writeClass } = meta;
|
||||
|
||||
let binding = bindings.get(serverIndex);
|
||||
if (!binding) {
|
||||
binding = { current: null };
|
||||
bindings.set(serverIndex, binding);
|
||||
}
|
||||
const boundBinding = binding;
|
||||
|
||||
const execute = async (args: unknown, options: ToolCallOptions) => {
|
||||
// The per-call cap governs the WHOLE sequence (attempt1 + reconnect +
|
||||
// attempt2). Compose it with the run's abort signal so a Stop or the cap
|
||||
// ends any awaited call — @ai-sdk/mcp does not settle on abort, so we RACE.
|
||||
const capController = new AbortController();
|
||||
const capTimer = setTimeout(() => {
|
||||
capController.abort(new Error(`MCP tool call timed out after ${capMs}ms`));
|
||||
}, capMs);
|
||||
capTimer.unref?.();
|
||||
const runSignal = options?.abortSignal;
|
||||
const composed = runSignal
|
||||
? AbortSignal.any([runSignal, capController.signal])
|
||||
: capController.signal;
|
||||
const stopped = () => runSignal?.aborted === true || capController.signal.aborted;
|
||||
|
||||
const callOn = async (
|
||||
exec: NonNullable<Tool['execute']>,
|
||||
): Promise<unknown> => {
|
||||
const aborted = new Promise<never>((_, reject) => {
|
||||
const fail = () => reject(abortReason(composed));
|
||||
if (composed.aborted) fail();
|
||||
else composed.addEventListener('abort', fail, { once: true });
|
||||
});
|
||||
return Promise.race([exec(args, { ...options, abortSignal: composed }), aborted]);
|
||||
};
|
||||
|
||||
const execFor = (
|
||||
state: RecoveredServerState | null,
|
||||
): NonNullable<Tool['execute']> | undefined =>
|
||||
state ? (state.tools[rawName]?.execute as NonNullable<Tool['execute']>) : original;
|
||||
|
||||
try {
|
||||
// Snapshot the target BEFORE the call so a swap by a concurrent call is
|
||||
// detected by identity in the catch.
|
||||
const attemptState = boundBinding.current;
|
||||
const attemptExec = execFor(attemptState);
|
||||
if (typeof attemptExec !== 'function') {
|
||||
throw new Error(`external MCP tool "${rawName}" is not callable`);
|
||||
}
|
||||
try {
|
||||
return await callOn(attemptExec);
|
||||
} catch (err) {
|
||||
// Never retry on a Stop or an exhausted cap.
|
||||
if (stopped()) throw err;
|
||||
// Only a genuine transport break is a recovery candidate.
|
||||
if (!isRetryableConnectError(err)) throw err;
|
||||
// A write tool is INDETERMINATE on a transport error (may have applied
|
||||
// before the reset) — surface that; do NOT auto-retry (double-apply is
|
||||
// the #435 incident class).
|
||||
if (!isReadOnlyWriteClass(writeClass)) {
|
||||
throw new Error(
|
||||
`external MCP tool "${rawName}" hit a transport error and MAY have already ` +
|
||||
`applied on the server — not retried automatically; verify state before ` +
|
||||
`retrying. (${shortError(err)})`,
|
||||
);
|
||||
}
|
||||
// Abort check BEFORE minting a fresh connection.
|
||||
if (stopped()) throw err;
|
||||
// CAS-swap by IDENTITY: mint+swap only if nobody swapped since this
|
||||
// call's snapshot; a losing concurrent call awaits the same reconnect
|
||||
// and retries on the SAME fresh client.
|
||||
let target: RecoveredServerState;
|
||||
if (boundBinding.current === attemptState) {
|
||||
if (!boundBinding.reconnecting) {
|
||||
boundBinding.reconnecting = (async () => {
|
||||
const server = entry.servers[serverIndex];
|
||||
const fresh = await service.reconnectServer(server, capMs);
|
||||
leaseSet.push(fresh.lease); // accumulate; released at turn-end
|
||||
boundBinding.current = fresh.state;
|
||||
return fresh.state;
|
||||
})();
|
||||
// Clear the in-flight marker once it settles (success or failure) so
|
||||
// a LATER death of the new client can reconnect again.
|
||||
void boundBinding.reconnecting.then(
|
||||
() => (boundBinding.reconnecting = undefined),
|
||||
() => (boundBinding.reconnecting = undefined),
|
||||
);
|
||||
}
|
||||
target = await boundBinding.reconnecting;
|
||||
} else {
|
||||
target = boundBinding.current as RecoveredServerState;
|
||||
}
|
||||
// Abort check BEFORE the retry.
|
||||
if (stopped()) throw err;
|
||||
const retryExec = execFor(target);
|
||||
if (typeof retryExec !== 'function') throw err;
|
||||
return await callOn(retryExec);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(capTimer);
|
||||
}
|
||||
};
|
||||
return { ...template, execute } as unknown as Tool;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconnect ONE server for an in-run recovery (#489): open a fresh client and
|
||||
* list+wrap its tools. The throwaway client is NOT cached — it is owned by the
|
||||
* RUN via the returned lease (closed at turn-end), independent of the shared
|
||||
* cache entry (whose TTL rebuild heals future turns). On a failure the fresh
|
||||
* client is closed so its socket never leaks.
|
||||
*/
|
||||
private async reconnectServer(
|
||||
server: AiMcpServer,
|
||||
capMs: number,
|
||||
): Promise<{ state: RecoveredServerState; lease: Closable }> {
|
||||
const client = await this.connectWithTimeout(server, CONNECT_TIMEOUT_MS);
|
||||
let tools: Record<string, Tool>;
|
||||
try {
|
||||
const raw = await withTimeout(client.tools(), CONNECT_TIMEOUT_MS);
|
||||
const allow = server.toolAllowlist;
|
||||
const picked =
|
||||
Array.isArray(allow) && allow.length > 0 ? pick(raw, allow) : raw;
|
||||
tools = wrapToolsWithCallTimeout(picked, capMs);
|
||||
} catch (err) {
|
||||
void client.close().catch(() => undefined);
|
||||
throw err;
|
||||
}
|
||||
let released = false;
|
||||
const lease: Closable = {
|
||||
close: async () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
await client.close().catch(() => undefined);
|
||||
},
|
||||
};
|
||||
return { state: { client, tools }, lease };
|
||||
}
|
||||
|
||||
/** Mark an entry evicted; close its clients now if nothing is leasing them. */
|
||||
private evict(entry: CacheEntry): void {
|
||||
clearTimeout(entry.timer);
|
||||
@@ -554,22 +873,21 @@ export function validateResolvedAddresses(addrs: readonly LookupAddress[]): {
|
||||
* certificate validation still uses the real hostname (we never rewrite the URL
|
||||
* to an IP literal).
|
||||
*/
|
||||
function buildPinnedDispatcher(): Agent {
|
||||
// External-MCP traffic uses a DEDICATED, shorter silence timeout
|
||||
function buildPinnedDispatcher(bodyTimeoutMs: number): Agent {
|
||||
// External-MCP traffic uses a DEDICATED, shorter HEADERS silence timeout
|
||||
// (`AI_MCP_STREAM_TIMEOUT_MS`, default 1 min) — deliberately tighter than the
|
||||
// chat provider's 15-min `streamTimeoutMs()` — so a byte-silent/hung MCP
|
||||
// upstream is broken in ~1 min instead of 15. We keep the keep-alive options
|
||||
// from `streamingDispatcherOptions()` but OVERRIDE headers/body timeouts.
|
||||
// Accepted trade-off: a legitimately long but byte-silent single tool call,
|
||||
// and an SSE transport idling >1 min BETWEEN tool calls, are also cut here; the
|
||||
// per-call total cap (wrapToolsWithCallTimeout, `AI_MCP_CALL_TIMEOUT_MS`) is the
|
||||
// complementary guard for chatty-but-stuck calls that keep the socket warm yet
|
||||
// never return.
|
||||
const mcpSilenceMs = mcpStreamTimeoutMs();
|
||||
// from `streamingDispatcherOptions()` but OVERRIDE the timeouts. `bodyTimeout`
|
||||
// is passed in per-transport (#489): tight for HTTP (fresh request per call),
|
||||
// raised for SSE (one long-lived body across calls — idle BETWEEN calls is
|
||||
// legit). The per-call total cap (`AI_MCP_CALL_TIMEOUT_MS`) is the complementary
|
||||
// guard for chatty-but-stuck calls that keep the socket warm yet never return.
|
||||
const headersMs = mcpStreamTimeoutMs();
|
||||
return new Agent({
|
||||
...streamingDispatcherOptions(),
|
||||
headersTimeout: mcpSilenceMs,
|
||||
bodyTimeout: mcpSilenceMs,
|
||||
headersTimeout: headersMs,
|
||||
bodyTimeout: bodyTimeoutMs,
|
||||
connect: {
|
||||
lookup: (hostname, _options, callback) => {
|
||||
// Always resolve ALL addresses ourselves; do not trust the caller's
|
||||
@@ -669,18 +987,22 @@ function pick(
|
||||
function namespace(
|
||||
tools: Record<string, Tool>,
|
||||
serverName: string,
|
||||
): Record<string, Tool> {
|
||||
): Array<{ full: string; raw: string; tool: Tool }> {
|
||||
const prefix = namespacePrefix(serverName);
|
||||
const out: Record<string, Tool> = {};
|
||||
const out: Array<{ full: string; raw: string; tool: Tool }> = [];
|
||||
const taken: Record<string, true> = {};
|
||||
for (const [name, t] of Object.entries(tools)) {
|
||||
const safe = sanitizeName(name);
|
||||
let full = capName(`${prefix}_${safe}`);
|
||||
// Duplicate names within ONE server can still collide after sanitize/
|
||||
// truncate — suffix-disambiguate so the second tool is not overwritten.
|
||||
if (full in out) {
|
||||
full = disambiguate(full, '', (candidate) => candidate in out);
|
||||
if (full in taken) {
|
||||
full = disambiguate(full, '', (candidate) => candidate in taken);
|
||||
}
|
||||
out[full] = t;
|
||||
taken[full] = true;
|
||||
// Keep the RAW (un-namespaced) name alongside the merged key so the per-run
|
||||
// recovery wrapper (#489) can re-resolve the same tool on a fresh client.
|
||||
out.push({ full, raw: name, tool: t });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -804,6 +1126,69 @@ export function wrapToolWithCallTimeout(tool: Tool, ms: number): Tool {
|
||||
return { ...tool, execute } as unknown as Tool;
|
||||
}
|
||||
|
||||
/**
|
||||
* undici / Node network error CODES that mean the connection broke (not an
|
||||
* application-level error) — a transient transport failure a readOnly call may
|
||||
* safely retry after reconnecting. Matched against the REAL error shapes (#489):
|
||||
* a socket reset surfaces as `TypeError: fetch failed` whose `.cause` is an
|
||||
* undici `SocketError { code:'UND_ERR_SOCKET' }`; a body-timeout as
|
||||
* `TypeError: terminated` whose `.cause` is `BodyTimeoutError`. Classifying by
|
||||
* these real codes/names (not by mock errors) is essential — a mock-shaped
|
||||
* predicate would leave eviction silently dead in production while CI is green.
|
||||
*/
|
||||
const RETRYABLE_TRANSPORT_ERROR_CODES: ReadonlySet<string> = new Set([
|
||||
'ECONNRESET',
|
||||
'ECONNREFUSED',
|
||||
'ECONNABORTED',
|
||||
'EPIPE',
|
||||
'ETIMEDOUT',
|
||||
'EAI_AGAIN',
|
||||
'ENETUNREACH',
|
||||
'EHOSTUNREACH',
|
||||
'UND_ERR_SOCKET',
|
||||
'UND_ERR_CONNECT_TIMEOUT',
|
||||
'UND_ERR_HEADERS_TIMEOUT',
|
||||
'UND_ERR_BODY_TIMEOUT',
|
||||
'UND_ERR_CLOSED',
|
||||
'UND_ERR_DESTROYED',
|
||||
]);
|
||||
|
||||
/** undici error CLASS names for the same transport-break conditions. */
|
||||
const RETRYABLE_TRANSPORT_ERROR_NAMES: ReadonlySet<string> = new Set([
|
||||
'SocketError',
|
||||
'BodyTimeoutError',
|
||||
'HeadersTimeoutError',
|
||||
'ConnectTimeoutError',
|
||||
'ClientClosedError',
|
||||
'ClientDestroyedError',
|
||||
]);
|
||||
|
||||
/**
|
||||
* Whether `err` is a retryable TRANSPORT break (a broken socket / body timeout),
|
||||
* classified by the REAL undici/@ai-sdk error shapes (#489). undici surfaces a
|
||||
* reset as `TypeError('fetch failed'|'terminated')` with the real error in
|
||||
* `.cause`, and @ai-sdk/mcp may wrap it again in an `MCPClientError` (cause
|
||||
* chain), so we walk `.cause` (bounded depth) checking `.code` and `.name`. An
|
||||
* app-level tool error (a 4xx, a validation failure) is NOT retryable and returns
|
||||
* false — only a connection-level failure heals with a reconnect.
|
||||
*/
|
||||
export function isRetryableConnectError(err: unknown, depth = 0): boolean {
|
||||
if (!err || typeof err !== 'object' || depth > 6) return false;
|
||||
const e = err as {
|
||||
code?: unknown;
|
||||
name?: unknown;
|
||||
cause?: unknown;
|
||||
};
|
||||
if (typeof e.code === 'string' && RETRYABLE_TRANSPORT_ERROR_CODES.has(e.code)) {
|
||||
return true;
|
||||
}
|
||||
if (typeof e.name === 'string' && RETRYABLE_TRANSPORT_ERROR_NAMES.has(e.name)) {
|
||||
return true;
|
||||
}
|
||||
if (e.cause != null) return isRetryableConnectError(e.cause, depth + 1);
|
||||
return false;
|
||||
}
|
||||
|
||||
/** The signal's reason as an Error (informative thrown value on abort/timeout). */
|
||||
function abortReason(signal: AbortSignal): Error {
|
||||
const r = signal.reason;
|
||||
|
||||
@@ -129,6 +129,12 @@ const DEFAULT_MCP_STREAM_TIMEOUT_MS = 60_000;
|
||||
/** Default total wall-clock cap for ONE external MCP tool call (2 min). */
|
||||
const DEFAULT_MCP_CALL_TIMEOUT_MS = 120_000;
|
||||
|
||||
/**
|
||||
* Default `bodyTimeout` for the EXTERNAL-MCP SSE transport (10 min) — #489.
|
||||
* Deliberately much LARGER than {@link DEFAULT_MCP_STREAM_TIMEOUT_MS}.
|
||||
*/
|
||||
const DEFAULT_MCP_SSE_BODY_TIMEOUT_MS = 600_000;
|
||||
|
||||
/**
|
||||
* SILENCE timeout (ms) for EXTERNAL-MCP transport ONLY. Override with
|
||||
* `AI_MCP_STREAM_TIMEOUT_MS`; a missing/invalid/non-positive value falls back to
|
||||
@@ -164,6 +170,26 @@ export function mcpCallTimeoutMs(): number {
|
||||
return positiveEnv('AI_MCP_CALL_TIMEOUT_MS', DEFAULT_MCP_CALL_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* `bodyTimeout` (ms) for the EXTERNAL-MCP **SSE** transport ONLY — #489. Override
|
||||
* with `AI_MCP_SSE_BODY_TIMEOUT_MS`; a missing/invalid/non-positive value falls
|
||||
* back to {@link DEFAULT_MCP_SSE_BODY_TIMEOUT_MS} (10 min).
|
||||
*
|
||||
* The SSE transport holds ONE long-lived response body open across many tool
|
||||
* calls, so undici's `bodyTimeout` (time between body bytes) counts the LEGITIMATE
|
||||
* silence BETWEEN calls, not just a hung single call. At the tight HTTP silence
|
||||
* timeout ({@link mcpStreamTimeoutMs}, 1 min) a normal >1-min gap between the
|
||||
* model's tool calls would break the SSE socket, and the cache would then serve a
|
||||
* dead client until TTL. So the SSE transport gets its OWN, RAISED bodyTimeout;
|
||||
* the per-call total cap ({@link mcpCallTimeoutMs}) still bounds a single stuck
|
||||
* call, and the app-level transport-error retry heals a socket that does break.
|
||||
* The HTTP (streamable) transport keeps the tight timeout — it opens a fresh
|
||||
* request per call, so idle-between-calls does not apply there.
|
||||
*/
|
||||
export function mcpSseBodyTimeoutMs(): number {
|
||||
return positiveEnv('AI_MCP_SSE_BODY_TIMEOUT_MS', DEFAULT_MCP_SSE_BODY_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
/**
|
||||
* undici `Agent` options for streaming AI traffic — the (generous, finite)
|
||||
* silence timeouts plus the keep-alive recycle window. Shared by the chat
|
||||
|
||||
Reference in New Issue
Block a user