From 2fe4ca85375c3892873606ff07838d5e0cdc43b9 Mon Sep 17 00:00:00 2001 From: claude code agent 227 Date: Sun, 28 Jun 2026 15:13:11 +0300 Subject: [PATCH] feat(sandbox): in-RAM blob sandbox for out-of-band page transfer (#243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an ephemeral, process-local blob store so the in-app agent (and the embedded MCP) can hand a large page document and its images to an external consumer WITHOUT routing the bytes through the model context or Docmost auth. - SandboxStore (@Injectable singleton): Map in RAM only. put() picks a per-blob cap by mime (image vs doc), enforces a total-bytes RAM guard with oldest-first eviction, and stamps a TTL; get() lazily expires. sha256 computed at put() doubles as the strong ETag. An unref'd sweep interval clears expired entries and is cleared on destroy. - GET /api/sb/:uuid anonymous controller: serves raw bytes with Content-Type, Content-Length and ETag=sha256; 404 on missing/expired/non-UUID (anti- traversal), 304 on a matching If-None-Match. No tokens, no 401 — the capability is the unguessable UUID + short TTL + TLS. Auth-exempt the same way as /api/files/public (no JwtAuthGuard) plus an /api/sb entry in main.ts's workspace-resolution preHandler so a remote consumer with no workspace host is not rejected. - stash_page tool in both layers (MCP resource_link + in-app {uri,size,sha256, images}). client.stashPage serializes the get_page_json shape, mirrors every INTERNAL file/image src (type-agnostic, covers drawio/excalidraw/video/file) into the sandbox under Docmost auth and rewrites src to the sandbox URL; external http(s) srcs are left untouched; dedup by src; a failed image fetch is counted, never aborts the doc. - SANDBOX_PUBLIC_URL / SANDBOX_TTL_MS / SANDBOX_MAX_BYTES / SANDBOX_MAX_IMAGE_BYTES / SANDBOX_MAX_TOTAL_BYTES wired through the environment service + validation + .env.example. - SandboxModule (@Global) provides the shared store to the controller, McpService and AiChatToolsService (same instance for put and get). Tests: SandboxStore (round-trip, sha256, TTL lazy + sweep, caps, eviction), SandboxController (200+ETag+CT+CL, 404 missing/expired/non-UUID, 304), and a mock-HTTP stashPage test (mirror+rewrite internal, keep external, dedup, failed image counted, returns only a link). Interoperates with the vvzvlad/habr-mcp consumer's anonymous-GET + sha256-ETag + resource_link contract. Co-Authored-By: Claude Opus 4.8 (1M context) --- .env.example | 16 ++ apps/server/src/app.module.ts | 2 + .../tools/ai-chat-tools.service.spec.ts | 16 ++ .../ai-chat/tools/ai-chat-tools.service.ts | 24 +++ .../ai-chat/tools/docmost-client.loader.ts | 16 ++ .../environment/environment.service.ts | 48 +++++ .../environment/environment.validation.ts | 30 +++ .../src/integrations/mcp/mcp-auth.helpers.ts | 16 +- .../mcp/mcp-basic-login-gate.spec.ts | 1 + .../src/integrations/mcp/mcp.service.ts | 25 ++- .../sandbox/sandbox.controller.spec.ts | 124 ++++++++++++ .../sandbox/sandbox.controller.ts | 94 +++++++++ .../integrations/sandbox/sandbox.module.ts | 19 ++ .../sandbox/sandbox.store.spec.ts | 133 +++++++++++++ .../src/integrations/sandbox/sandbox.store.ts | 129 ++++++++++++ apps/server/src/main.ts | 4 + packages/mcp/build/client.js | 106 ++++++++++ packages/mcp/build/index.js | 20 ++ packages/mcp/build/lib/internal-file-urls.js | 55 ++++++ packages/mcp/build/tool-specs.js | 20 ++ packages/mcp/src/client.ts | 134 +++++++++++++ packages/mcp/src/index.ts | 25 +++ packages/mcp/src/lib/internal-file-urls.ts | 54 ++++++ packages/mcp/src/tool-specs.ts | 22 +++ packages/mcp/test/mock/stash-page.test.mjs | 183 ++++++++++++++++++ 25 files changed, 1312 insertions(+), 4 deletions(-) create mode 100644 apps/server/src/integrations/sandbox/sandbox.controller.spec.ts create mode 100644 apps/server/src/integrations/sandbox/sandbox.controller.ts create mode 100644 apps/server/src/integrations/sandbox/sandbox.module.ts create mode 100644 apps/server/src/integrations/sandbox/sandbox.store.spec.ts create mode 100644 apps/server/src/integrations/sandbox/sandbox.store.ts create mode 100644 packages/mcp/build/lib/internal-file-urls.js create mode 100644 packages/mcp/src/lib/internal-file-urls.ts create mode 100644 packages/mcp/test/mock/stash-page.test.mjs diff --git a/.env.example b/.env.example index 7407e629..2c59018a 100644 --- a/.env.example +++ b/.env.example @@ -124,6 +124,22 @@ MCP_DOCMOST_PASSWORD= # MCP_TOKEN= # MCP_SESSION_IDLE_MS=1800000 # +# BLOB SANDBOX (stash_page). An in-RAM, process-local store that hands large page +# content + images to an external consumer WITHOUT bloating the model context or +# requiring Docmost auth. The stash_page tool serializes a page, mirrors its +# internal images into the store, and returns ONLY a short anonymous URL; the +# consumer fetches blobs via `GET /api/sb/` (no token — the capability is +# the unguessable UUID + short TTL + TLS). Blobs are RAM-only and cleared on +# restart. ETag = the blob's sha256 (integrity check). +# SANDBOX_PUBLIC_URL is the base used to build those URLs; it MUST be reachable +# by the consumer (do NOT use a loopback address if the consumer is remote). +# Defaults to APP_URL when unset. +# SANDBOX_PUBLIC_URL=https://docs.example.com +# SANDBOX_TTL_MS=3600000 +# SANDBOX_MAX_BYTES=8388608 +# SANDBOX_MAX_IMAGE_BYTES=20971520 +# SANDBOX_MAX_TOTAL_BYTES=134217728 +# # AI-AGENT ATTRIBUTION (comments/pages written via MCP are badged as "AI"): # attribution is driven by a per-user `is_agent` flag on the users row. There is # NO admin UI/API for it — set it out-of-band with SQL. Use a DEDICATED service diff --git a/apps/server/src/app.module.ts b/apps/server/src/app.module.ts index 926d5802..e257fe9b 100644 --- a/apps/server/src/app.module.ts +++ b/apps/server/src/app.module.ts @@ -28,6 +28,7 @@ import { ClsModule } from 'nestjs-cls'; import { NoopAuditModule } from './integrations/audit/audit.module'; import { ThrottleModule } from './integrations/throttle/throttle.module'; import { McpModule } from './integrations/mcp/mcp.module'; +import { SandboxModule } from './integrations/sandbox/sandbox.module'; import { AiModule } from './integrations/ai/ai.module'; import { AiChatModule } from './core/ai-chat/ai-chat.module'; @@ -89,6 +90,7 @@ try { TelemetryModule, ThrottleModule, McpModule, + SandboxModule, AiModule, AiChatModule, ...enterpriseModules, diff --git a/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.spec.ts b/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.spec.ts index ebf1cb6a..0e695905 100644 --- a/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.spec.ts +++ b/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.spec.ts @@ -63,6 +63,10 @@ describe('AiChatToolsService deletePage guardrail (H4)', () => { {} as never, {} as never, {} as never, + // environmentService + sandboxStore (only used by the stash tool closure, + // which these tests do not execute). + {} as never, + {} as never, ); }); @@ -175,6 +179,10 @@ describe('AiChatToolsService expanded toolset guardrails', () => { {} as never, {} as never, {} as never, + // environmentService + sandboxStore (only used by the stash tool closure, + // which these tests do not execute). + {} as never, + {} as never, ); }); @@ -290,6 +298,10 @@ describe('AiChatToolsService node-arg JSON-string coercion', () => { {} as never, {} as never, {} as never, + // environmentService + sandboxStore (only used by the stash tool closure, + // which these tests do not execute). + {} as never, + {} as never, ); }); @@ -440,6 +452,10 @@ describe('AiChatToolsService model-friendly input validation (#190)', () => { {} as never, {} as never, {} as never, + // environmentService + sandboxStore (only used by the stash tool closure, + // which these tests do not execute). + {} as never, + {} as never, ); }); diff --git a/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts b/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts index 377d4036..ae17be4d 100644 --- a/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts +++ b/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts @@ -16,6 +16,8 @@ import { import { resolveCurrentPageResult } from './current-page.util'; import { parseNodeArg } from './parse-node-arg'; import { modelFriendlyInput } from './model-friendly-input'; +import { EnvironmentService } from '../../../integrations/environment/environment.service'; +import { SandboxStore } from '../../../integrations/sandbox/sandbox.store'; /** * Per-user, per-request adapter that exposes Docmost READ operations to the @@ -41,6 +43,9 @@ export class AiChatToolsService { private readonly pageEmbeddingRepo: PageEmbeddingRepo, private readonly spaceMemberRepo: SpaceMemberRepo, private readonly pagePermissionRepo: PagePermissionRepo, + private readonly environmentService: EnvironmentService, + // Shared singleton in-RAM blob store backing the stash tool. + private readonly sandboxStore: SandboxStore, ) {} async forUser( @@ -86,11 +91,22 @@ export class AiChatToolsService { aiChatId, }); + // Bind the stash tool to the shared in-RAM SandboxStore and compose the + // anonymous public URL here (the MCP package never touches env or the + // store). put() returns the read URL + sha256/size; sha256 is also the + // blob's ETag for integrity. + const sandboxPut = (buf: Buffer, mime: string) => { + const stored = this.sandboxStore.put(buf, mime); + const base = this.environmentService.getSandboxPublicUrl(); + return { uri: `${base}/api/sb/${stored.id}`, sha256: stored.sha256, size: stored.size }; + }; + const { DocmostClient, sharedToolSpecs } = await loadDocmostMcp(); const client: DocmostClientLike = new DocmostClient({ apiUrl, getToken, getCollabToken, + sandbox: { put: sandboxPut }, }); // Build an ai-SDK tool from a shared, zod-agnostic spec. The spec owns the @@ -625,6 +641,14 @@ export class AiChatToolsService { async ({ pageId, edits }) => await client.editPageText(pageId, edits), ), + // Returns ONLY the short link object — never the document body — so a + // large page can be handed to an external consumer without bloating + // context. + stashPage: sharedTool( + sharedToolSpecs.stashPage, + async ({ pageId }) => await client.stashPage(pageId), + ), + patchNode: tool({ description: 'Replace a single content block (by id) with a new ProseMirror ' + diff --git a/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts b/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts index 5b740cfe..17ec49c3 100644 --- a/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts +++ b/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts @@ -154,6 +154,14 @@ export interface DocmostClientLike { commentId: string, resolved: boolean, ): Promise>; + // Serialize a page + mirror its internal images into the blob sandbox; returns + // ONLY a short anonymous URL (the body never enters the model context). + stashPage(pageId: string): Promise<{ + uri: string; + sha256: string; + size: number; + images: { mirrored: number; failed: number }; + }>; } export type DocmostClientConfig = { @@ -161,6 +169,14 @@ export type DocmostClientConfig = { getToken: () => Promise; // Provenance collab-token provider for content mutations (signed agent claim). getCollabToken?: () => Promise; + // Optional blob-sandbox sink for the stash tool. `put` stores a blob in the + // host's in-RAM SandboxStore and returns the anonymous read URL + integrity. + sandbox?: { + put: ( + buf: Buffer, + mime: string, + ) => { uri: string; sha256: string; size: number }; + }; }; export interface DocmostClientCtor { diff --git a/apps/server/src/integrations/environment/environment.service.ts b/apps/server/src/integrations/environment/environment.service.ts index 24081b38..51c94ec2 100644 --- a/apps/server/src/integrations/environment/environment.service.ts +++ b/apps/server/src/integrations/environment/environment.service.ts @@ -332,4 +332,52 @@ export class EnvironmentService { .map((o) => o.trim()) .filter(Boolean); } + + // --- Blob sandbox (in-RAM ephemeral blob transfer; see SandboxModule) --- + + // Base URL the sandbox `uri` is built from. It MUST be reachable over the + // network by the external consumer that fetches the blobs (not a loopback + // address if that consumer is remote). Falls back to APP_URL when unset so a + // single-host deployment works out of the box; set it explicitly when the + // consumer lives on another host. + getSandboxPublicUrl(): string { + const raw = + this.configService.get('SANDBOX_PUBLIC_URL') || this.getAppUrl(); + // Drop any trailing slash so `${base}/api/sb/${id}` never doubles up. + return raw.replace(/\/+$/, ''); + } + + // Blob time-to-live. Default 1h. The unguessable UUID + this short TTL + TLS + // are the whole capability model (no tokens). + getSandboxTtlMs(): number { + return parseInt( + this.configService.get('SANDBOX_TTL_MS', '3600000'), + 10, + ); + } + + // Per-blob cap for non-image blobs (the serialized document). Default 8 MiB. + getSandboxMaxBytes(): number { + return parseInt( + this.configService.get('SANDBOX_MAX_BYTES', '8388608'), + 10, + ); + } + + // Per-blob cap for mirrored image blobs. Default 20 MiB. + getSandboxMaxImageBytes(): number { + return parseInt( + this.configService.get('SANDBOX_MAX_IMAGE_BYTES', '20971520'), + 10, + ); + } + + // RAM guard: total bytes the whole store may hold. Default 128 MiB. On + // overflow the store evicts oldest entries to make room. + getSandboxMaxTotalBytes(): number { + return parseInt( + this.configService.get('SANDBOX_MAX_TOTAL_BYTES', '134217728'), + 10, + ); + } } diff --git a/apps/server/src/integrations/environment/environment.validation.ts b/apps/server/src/integrations/environment/environment.validation.ts index ef3c420c..476bc81c 100644 --- a/apps/server/src/integrations/environment/environment.validation.ts +++ b/apps/server/src/integrations/environment/environment.validation.ts @@ -2,6 +2,7 @@ import { IsIn, IsNotEmpty, IsNotIn, + IsNumberString, IsOptional, IsString, IsUrl, @@ -170,6 +171,35 @@ export class EnvironmentVariables { }, ) CLICKHOUSE_URL: string; + + // --- Blob sandbox (in-RAM ephemeral blob transfer; see SandboxModule) --- + + @IsOptional() + @ValidateIf((obj) => obj.SANDBOX_PUBLIC_URL != '' && obj.SANDBOX_PUBLIC_URL != null) + @IsUrl( + { protocols: ['http', 'https'], require_tld: false }, + { + message: + 'SANDBOX_PUBLIC_URL must be a valid http(s) URL reachable by the external blob consumer', + }, + ) + SANDBOX_PUBLIC_URL: string; + + @IsOptional() + @IsNumberString({}, { message: 'SANDBOX_TTL_MS must be an integer (milliseconds)' }) + SANDBOX_TTL_MS: string; + + @IsOptional() + @IsNumberString({}, { message: 'SANDBOX_MAX_BYTES must be an integer (bytes)' }) + SANDBOX_MAX_BYTES: string; + + @IsOptional() + @IsNumberString({}, { message: 'SANDBOX_MAX_IMAGE_BYTES must be an integer (bytes)' }) + SANDBOX_MAX_IMAGE_BYTES: string; + + @IsOptional() + @IsNumberString({}, { message: 'SANDBOX_MAX_TOTAL_BYTES must be an integer (bytes)' }) + SANDBOX_MAX_TOTAL_BYTES: string; } export function validate(config: Record) { diff --git a/apps/server/src/integrations/mcp/mcp-auth.helpers.ts b/apps/server/src/integrations/mcp/mcp-auth.helpers.ts index f71dff9a..24e60206 100644 --- a/apps/server/src/integrations/mcp/mcp-auth.helpers.ts +++ b/apps/server/src/integrations/mcp/mcp-auth.helpers.ts @@ -131,10 +131,20 @@ export class FailedLoginLimiter { } // The per-session DocmostMcpConfig shape understood by @docmost/mcp: either the -// service-account credentials variant OR the per-user getToken variant. -export type DocmostMcpConfig = +// service-account credentials variant OR the per-user getToken variant. The +// optional `sandbox` sink (blob store for the stash tool) is common to both and +// injected by McpService after the auth decision. +export type DocmostMcpConfig = ( | { apiUrl: string; email: string; password: string } - | { apiUrl: string; getToken: () => Promise }; + | { apiUrl: string; getToken: () => Promise } +) & { + sandbox?: { + put: ( + buf: Buffer, + mime: string, + ) => { uri: string; sha256: string; size: number }; + }; +}; export interface ResolvedMcpAuth { config: DocmostMcpConfig; diff --git a/apps/server/src/integrations/mcp/mcp-basic-login-gate.spec.ts b/apps/server/src/integrations/mcp/mcp-basic-login-gate.spec.ts index 351b467b..416470da 100644 --- a/apps/server/src/integrations/mcp/mcp-basic-login-gate.spec.ts +++ b/apps/server/src/integrations/mcp/mcp-basic-login-gate.spec.ts @@ -116,6 +116,7 @@ function makeService(opts: { undefined as never, // userRepo undefined as never, // userSessionRepo moduleRef as never, // moduleRef (read by the MFA branch) + undefined as never, // sandboxStore (unused by the login-gate path) ); // Stop the constructor's unref'd sweep timer leaking across tests. service.onModuleDestroy(); diff --git a/apps/server/src/integrations/mcp/mcp.service.ts b/apps/server/src/integrations/mcp/mcp.service.ts index 637f3e56..af8719ec 100644 --- a/apps/server/src/integrations/mcp/mcp.service.ts +++ b/apps/server/src/integrations/mcp/mcp.service.ts @@ -30,6 +30,7 @@ import { DocmostMcpConfig, ResolvedMcpAuth, } from './mcp-auth.helpers'; +import { SandboxStore } from '../sandbox/sandbox.store'; // Minimal shape of the embedded MCP HTTP handler exported by @docmost/mcp/http. interface McpHttpHandler { @@ -99,6 +100,8 @@ export class McpService implements OnModuleDestroy { private readonly userRepo: UserRepo, private readonly userSessionRepo: UserSessionRepo, private readonly moduleRef: ModuleRef, + // Shared singleton in-RAM blob store backing the stash tool. + private readonly sandboxStore: SandboxStore, ) { this.sweepTimer = setInterval(() => { try { @@ -115,6 +118,23 @@ export class McpService implements OnModuleDestroy { clearInterval(this.sweepTimer); } + // Bind the stash tool to the shared in-RAM SandboxStore and compose the + // anonymous public URL (the MCP package owns neither env nor the store). + // put() returns the read URL + sha256/size; sha256 is also the blob ETag. + private buildSandboxConfig(): DocmostMcpConfig['sandbox'] { + return { + put: (buf: Buffer, mime: string) => { + const stored = this.sandboxStore.put(buf, mime); + const base = this.environmentService.getSandboxPublicUrl(); + return { + uri: `${base}/api/sb/${stored.id}`, + sha256: stored.sha256, + size: stored.size, + }; + }, + }; + } + // Service account the embedded MCP uses to talk back to this Docmost // instance over loopback REST + the collaboration WebSocket. Now OPTIONAL: // it is only a fallback when no per-user Basic/Bearer credentials are sent. @@ -326,7 +346,10 @@ export class McpService implements OnModuleDestroy { // Should never happen: handle() always stashes before delegating. throw new UnauthorizedException('MCP authentication missing.'); } - return resolved.config; + // Inject the blob-sandbox sink after the auth decision so stash_page + // can store blobs in the shared in-RAM store regardless of which + // credential variant resolved. + return { ...resolved.config, sandbox: this.buildSandboxConfig() }; }, { identify: (req: IncomingMessage) => { diff --git a/apps/server/src/integrations/sandbox/sandbox.controller.spec.ts b/apps/server/src/integrations/sandbox/sandbox.controller.spec.ts new file mode 100644 index 00000000..fb18f555 --- /dev/null +++ b/apps/server/src/integrations/sandbox/sandbox.controller.spec.ts @@ -0,0 +1,124 @@ +import { SandboxController } from './sandbox.controller'; +import { SandboxEntry } from './sandbox.store'; + +// Capturing fake of the FastifyReply surface the controller uses: +// status()/header()/headers()/send(), all chainable. +function makeRes() { + const sent: { status: number; headers: Record; body: any } = { + status: 200, + headers: {}, + body: undefined, + }; + const res: any = { + status(code: number) { + sent.status = code; + return res; + }, + header(key: string, value: any) { + sent.headers[key.toLowerCase()] = value; + return res; + }, + headers(obj: Record) { + for (const k of Object.keys(obj)) sent.headers[k.toLowerCase()] = obj[k]; + return res; + }, + send(body?: any) { + sent.body = body; + return res; + }, + _sent: sent, + }; + return res; +} + +function makeReq(headers: Record = {}) { + return { headers } as any; +} + +const VALID_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'; + +function entry(buf: Buffer, mime: string, sha256: string): SandboxEntry { + return { buf, mime, sha256, expiresAt: Date.now() + 60_000 }; +} + +describe('SandboxController', () => { + it('serves 200 with body, Content-Type, Content-Length and sha256 ETag', async () => { + const buf = Buffer.from('{"ok":true}', 'utf8'); + const sha = 'a'.repeat(64); + const store = { get: jest.fn().mockReturnValue(entry(buf, 'application/json', sha)) }; + const controller = new SandboxController(store as any); + const res = makeRes(); + + await controller.get(VALID_ID, makeReq(), res); + + expect(store.get).toHaveBeenCalledWith(VALID_ID); + expect(res._sent.status).toBe(200); + expect(res._sent.headers['content-type']).toBe('application/json'); + expect(res._sent.headers['content-length']).toBe(buf.length); + expect(res._sent.headers['etag']).toBe(`"${sha}"`); + expect(res._sent.body).toBe(buf); + }); + + it('returns 404 for a missing/expired blob', async () => { + const store = { get: jest.fn().mockReturnValue(undefined) }; + const controller = new SandboxController(store as any); + const res = makeRes(); + + await controller.get(VALID_ID, makeReq(), res); + + expect(res._sent.status).toBe(404); + expect(res._sent.body).toBeUndefined(); + }); + + it('returns 404 for a non-UUID id WITHOUT touching the store (anti-traversal)', async () => { + const store = { get: jest.fn() }; + const controller = new SandboxController(store as any); + const res = makeRes(); + + await controller.get('../../etc/passwd', makeReq(), res); + + expect(store.get).not.toHaveBeenCalled(); + expect(res._sent.status).toBe(404); + }); + + it('returns 304 (no body) when If-None-Match matches the ETag', async () => { + const sha = 'b'.repeat(64); + const store = { + get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)), + }; + const controller = new SandboxController(store as any); + const res = makeRes(); + + await controller.get(VALID_ID, makeReq({ 'if-none-match': `"${sha}"` }), res); + + expect(res._sent.status).toBe(304); + expect(res._sent.body).toBeUndefined(); + expect(res._sent.headers['etag']).toBe(`"${sha}"`); + }); + + it('accepts a bare (unquoted) sha256 in If-None-Match too', async () => { + const sha = 'c'.repeat(64); + const store = { + get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)), + }; + const controller = new SandboxController(store as any); + const res = makeRes(); + + await controller.get(VALID_ID, makeReq({ 'if-none-match': sha }), res); + + expect(res._sent.status).toBe(304); + }); + + it('serves 200 when If-None-Match does NOT match', async () => { + const sha = 'd'.repeat(64); + const store = { + get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)), + }; + const controller = new SandboxController(store as any); + const res = makeRes(); + + await controller.get(VALID_ID, makeReq({ 'if-none-match': '"stale"' }), res); + + expect(res._sent.status).toBe(200); + }); +}); diff --git a/apps/server/src/integrations/sandbox/sandbox.controller.ts b/apps/server/src/integrations/sandbox/sandbox.controller.ts new file mode 100644 index 00000000..615e2c60 --- /dev/null +++ b/apps/server/src/integrations/sandbox/sandbox.controller.ts @@ -0,0 +1,94 @@ +import { Controller, Get, Param, Req, Res } from '@nestjs/common'; +import { FastifyReply, FastifyRequest } from 'fastify'; +import { SandboxStore } from './sandbox.store'; + +// Strict UUID v-agnostic shape. This is anti-traversal / input hygiene (so `:id` +// can never be a path like `../...`), NOT authorization — the capability is the +// unguessable id itself plus the short TTL plus TLS. +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +/** + * Anonymous read endpoint for the in-RAM blob sandbox. + * + * Mounted under the global `/api` prefix as `GET /api/sb/:id`. It carries NO + * `@UseGuards(JwtAuthGuard)`, so — exactly like the public attachment route + * `GET /api/files/public/...` — it is exempt from Docmost session auth. The + * route is ALSO listed in the workspace-resolution preHandler's excludedPaths + * in main.ts so a request from a remote consumer (which carries no workspace + * host) is not rejected with "Workspace not found". + * + * It only ever serves blobs looked up from the SandboxStore by a validated + * UUID; `:id` is never used as a filesystem path, so there is no traversal + * surface. Never returns tokens, never 401s. + */ +@Controller('sb') +export class SandboxController { + constructor(private readonly store: SandboxStore) {} + + @Get(':id') + async get( + @Param('id') id: string, + @Req() req: FastifyRequest, + @Res() res: FastifyReply, + ): Promise { + // Non-UUID id (including any traversal attempt) → 404 before touching the + // store. No stack trace leaks out. + if (!UUID_RE.test(id)) { + res.status(404).send(); + return; + } + + const entry = this.store.get(id); + if (!entry) { + // Missing or expired — indistinguishable to the caller, by design. + res.status(404).send(); + return; + } + + // Strong validator: quoted sha256, no W/ weak prefix. Same value computed + // at put() time, so an external consumer can detect a truncated/corrupted + // body — the original bug this whole channel exists to fix. + const etag = `"${entry.sha256}"`; + + // Conditional request: an exact ETag match → 304 with no body. The blob is + // immutable, so the validator is stable for the blob's whole lifetime. + if (this.ifNoneMatchMatches(req.headers['if-none-match'], entry.sha256)) { + res.status(304).header('ETag', etag).send(); + return; + } + + const ttlSeconds = Math.max( + 0, + Math.floor((entry.expiresAt - Date.now()) / 1000), + ); + + // Use @Res() + res.send(Buffer) with an explicit Content-Type so the binary + // body bypasses the global JSON response transform/serializer. + res + .status(200) + .headers({ + 'Content-Type': entry.mime, + 'Content-Length': entry.buf.length, + ETag: etag, + // Capability URL — keep it out of shared caches; immutable for its TTL. + 'Cache-Control': `private, max-age=${ttlSeconds}, immutable`, + }) + .send(entry.buf); + } + + // Accept the consumer's If-None-Match whether it sends the quoted ETag, a bare + // sha256, a weak "W/"-prefixed validator, or a comma-separated list. + private ifNoneMatchMatches( + header: string | string[] | undefined, + sha256: string, + ): boolean { + if (!header) return false; + const raw = Array.isArray(header) ? header.join(',') : header; + if (raw.trim() === '*') return true; + return raw + .split(',') + .map((t) => t.trim().replace(/^W\//, '').replace(/^"|"$/g, '')) + .some((t) => t === sha256); + } +} diff --git a/apps/server/src/integrations/sandbox/sandbox.module.ts b/apps/server/src/integrations/sandbox/sandbox.module.ts new file mode 100644 index 00000000..6b20bf6c --- /dev/null +++ b/apps/server/src/integrations/sandbox/sandbox.module.ts @@ -0,0 +1,19 @@ +import { Global, Module } from '@nestjs/common'; +import { SandboxController } from './sandbox.controller'; +import { SandboxStore } from './sandbox.store'; + +/** + * In-RAM blob sandbox: a SINGLE shared SandboxStore (the @Injectable singleton) + * is written to by the stash tool (via McpService / AiChatToolsService) and read + * back by the anonymous SandboxController. Marked @Global so the same store + * instance is injectable everywhere without import churn — put() and get() MUST + * hit the same Map. EnvironmentService (caps/TTL/public URL) is provided by the + * global EnvironmentModule. + */ +@Global() +@Module({ + controllers: [SandboxController], + providers: [SandboxStore], + exports: [SandboxStore], +}) +export class SandboxModule {} diff --git a/apps/server/src/integrations/sandbox/sandbox.store.spec.ts b/apps/server/src/integrations/sandbox/sandbox.store.spec.ts new file mode 100644 index 00000000..d4e81706 --- /dev/null +++ b/apps/server/src/integrations/sandbox/sandbox.store.spec.ts @@ -0,0 +1,133 @@ +import { createHash } from 'node:crypto'; +import { SandboxStore } from './sandbox.store'; + +// Build a minimal EnvironmentService stub with overridable caps/TTL. +function makeEnv( + overrides: Partial<{ + ttlMs: number; + maxBytes: number; + maxImageBytes: number; + maxTotalBytes: number; + }> = {}, +) { + const cfg = { + ttlMs: 3_600_000, + maxBytes: 8_388_608, + maxImageBytes: 20_971_520, + maxTotalBytes: 134_217_728, + ...overrides, + }; + return { + getSandboxTtlMs: () => cfg.ttlMs, + getSandboxMaxBytes: () => cfg.maxBytes, + getSandboxMaxImageBytes: () => cfg.maxImageBytes, + getSandboxMaxTotalBytes: () => cfg.maxTotalBytes, + getSandboxPublicUrl: () => 'https://example.test', + } as any; +} + +const UUID_RE = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +describe('SandboxStore', () => { + let store: SandboxStore; + + afterEach(() => { + // Clear the unref'd sweep interval so it never leaks across tests. + store?.onModuleDestroy(); + jest.useRealTimers(); + }); + + it('put/get round-trips the exact bytes + mime and returns a UUID id', () => { + store = new SandboxStore(makeEnv()); + const buf = Buffer.from('{"type":"doc","content":[]}', 'utf8'); + + const res = store.put(buf, 'application/json'); + expect(res.id).toMatch(UUID_RE); + expect(res.size).toBe(buf.length); + + const entry = store.get(res.id); + expect(entry).toBeDefined(); + expect(entry!.buf.equals(buf)).toBe(true); + expect(entry!.mime).toBe('application/json'); + }); + + it('computes sha256 over the body (matches a manual digest)', () => { + store = new SandboxStore(makeEnv()); + const buf = Buffer.from('hello sandbox', 'utf8'); + const expected = createHash('sha256').update(buf).digest('hex'); + + const res = store.put(buf, 'text/plain'); + expect(res.sha256).toBe(expected); + expect(store.get(res.id)!.sha256).toBe(expected); + }); + + it('returns undefined for a missing id', () => { + store = new SandboxStore(makeEnv()); + expect(store.get('11111111-1111-1111-1111-111111111111')).toBeUndefined(); + }); + + it('lazily expires entries past the TTL (get returns undefined)', () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2026-01-01T00:00:00Z')); + store = new SandboxStore(makeEnv({ ttlMs: 1000 })); + const res = store.put(Buffer.from('x'), 'text/plain'); + + expect(store.get(res.id)).toBeDefined(); + jest.setSystemTime(new Date('2026-01-01T00:00:02Z')); // +2s > 1s TTL + expect(store.get(res.id)).toBeUndefined(); + // Eviction also frees the byte accounting. + expect(store.bytes).toBe(0); + }); + + it('background sweep drops expired entries without a get()', () => { + jest.useFakeTimers(); + jest.setSystemTime(new Date('2026-01-01T00:00:00Z')); + store = new SandboxStore(makeEnv({ ttlMs: 1000 })); + store.put(Buffer.from('x'), 'text/plain'); + expect(store.size).toBe(1); + + jest.setSystemTime(new Date('2026-01-01T00:01:30Z')); // past TTL + jest.advanceTimersByTime(60_000); // fire the sweep interval + expect(store.size).toBe(0); + }); + + it('rejects a non-image blob over SANDBOX_MAX_BYTES', () => { + store = new SandboxStore(makeEnv({ maxBytes: 16 })); + expect(() => store.put(Buffer.alloc(17), 'application/json')).toThrow( + /per-blob cap/, + ); + }); + + it('uses the larger image cap for image/* blobs', () => { + // 100 bytes exceeds the doc cap (16) but fits the image cap (1024). + store = new SandboxStore(makeEnv({ maxBytes: 16, maxImageBytes: 1024 })); + expect(() => store.put(Buffer.alloc(100), 'image/png')).not.toThrow(); + // SVG counts as an image too. + expect(() => store.put(Buffer.alloc(100), 'image/svg+xml')).not.toThrow(); + }); + + it('evicts oldest entries when the total cap would be exceeded', () => { + // Total cap 250 bytes; each blob 100 bytes -> only 2 fit at a time. + store = new SandboxStore( + makeEnv({ maxTotalBytes: 250, maxBytes: 1024 }), + ); + const a = store.put(Buffer.alloc(100), 'application/json'); + const b = store.put(Buffer.alloc(100), 'application/json'); + const c = store.put(Buffer.alloc(100), 'application/json'); // evicts a + + expect(store.get(a.id)).toBeUndefined(); // oldest evicted + expect(store.get(b.id)).toBeDefined(); + expect(store.get(c.id)).toBeDefined(); + expect(store.bytes).toBeLessThanOrEqual(250); + }); + + it('rejects a single blob larger than the whole total cap', () => { + store = new SandboxStore( + makeEnv({ maxTotalBytes: 50, maxBytes: 1024 }), + ); + expect(() => store.put(Buffer.alloc(100), 'application/json')).toThrow( + /total store cap/, + ); + }); +}); diff --git a/apps/server/src/integrations/sandbox/sandbox.store.ts b/apps/server/src/integrations/sandbox/sandbox.store.ts new file mode 100644 index 00000000..a833e8d7 --- /dev/null +++ b/apps/server/src/integrations/sandbox/sandbox.store.ts @@ -0,0 +1,129 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import { createHash, randomUUID } from 'node:crypto'; +import { EnvironmentService } from '../environment/environment.service'; + +// In-RAM, process-local blob store. No disk, no DB. Ephemeral by design: a +// restart empties it. A blob is addressed by an unguessable randomUUID() which +// IS the read capability — there are NO tokens. Each blob is immutable (its id +// never maps to changing content), so its sha256 is a perfect strong ETag. +export interface SandboxEntry { + buf: Buffer; + mime: string; + sha256: string; + expiresAt: number; +} + +export interface SandboxPutResult { + id: string; + sha256: string; + size: number; + expiresAt: number; +} + +@Injectable() +export class SandboxStore implements OnModuleDestroy { + private readonly logger = new Logger(SandboxStore.name); + // Map preserves insertion order, so the first key is the oldest entry — used + // for FIFO eviction when the total-bytes RAM guard is exceeded. + private readonly map = new Map(); + private totalBytes = 0; + + // Background sweep clears expired entries so never-fetched blobs do not linger + // until the next get(). unref()'d so it never holds the event loop open; + // cleared on module destroy. Mirrors the sweepTimer pattern in + // integrations/mcp/mcp.service.ts and packages/mcp/src/http.ts. + private readonly sweepIntervalMs = 60_000; + private readonly sweepTimer: NodeJS.Timeout; + + constructor(private readonly environmentService: EnvironmentService) { + this.sweepTimer = setInterval(() => { + try { + this.sweep(); + } catch (err) { + this.logger.error('Sandbox sweep failed', err as Error); + } + }, this.sweepIntervalMs); + this.sweepTimer.unref?.(); + } + + onModuleDestroy(): void { + clearInterval(this.sweepTimer); + } + + /** + * Store a blob and return its read capability id + integrity metadata. The + * per-blob cap is chosen by mime (images get the larger image cap), and the + * total-store RAM guard evicts oldest entries to make room. Throws a clear + * error when a single blob cannot fit even after eviction. Blob bodies are + * never logged. + */ + put(buf: Buffer, mime: string): SandboxPutResult { + const perBlobCap = mime.startsWith('image/') + ? this.environmentService.getSandboxMaxImageBytes() + : this.environmentService.getSandboxMaxBytes(); + if (buf.length > perBlobCap) { + throw new Error( + `Sandbox blob of ${buf.length} bytes exceeds the ${perBlobCap}-byte per-blob cap`, + ); + } + + const maxTotal = this.environmentService.getSandboxMaxTotalBytes(); + if (buf.length > maxTotal) { + throw new Error( + `Sandbox blob of ${buf.length} bytes exceeds the total store cap of ${maxTotal} bytes`, + ); + } + + // Drop expired entries first, then evict oldest until the new blob fits. + this.sweep(); + while (this.totalBytes + buf.length > maxTotal && this.map.size > 0) { + const oldest = this.map.keys().next().value as string; + this.evict(oldest); + } + + const id = randomUUID(); + const sha256 = createHash('sha256').update(buf).digest('hex'); + const expiresAt = Date.now() + this.environmentService.getSandboxTtlMs(); + this.map.set(id, { buf, mime, sha256, expiresAt }); + this.totalBytes += buf.length; + return { id, sha256, size: buf.length, expiresAt }; + } + + /** Returns the entry, or undefined if missing OR expired (lazy expiry). */ + get(id: string): SandboxEntry | undefined { + const entry = this.map.get(id); + if (!entry) return undefined; + if (entry.expiresAt <= Date.now()) { + this.evict(id); + return undefined; + } + return entry; + } + + /** Current number of live entries (test/diagnostic helper). */ + get size(): number { + return this.map.size; + } + + /** Current total bytes held (test/diagnostic helper). */ + get bytes(): number { + return this.totalBytes; + } + + private evict(id: string): void { + const entry = this.map.get(id); + if (entry) { + this.totalBytes -= entry.buf.length; + this.map.delete(id); + } + } + + private sweep(): void { + const now = Date.now(); + for (const [id, entry] of this.map) { + if (entry.expiresAt <= now) { + this.evict(id); + } + } + } +} diff --git a/apps/server/src/main.ts b/apps/server/src/main.ts index 1fb140c1..5ae036fb 100644 --- a/apps/server/src/main.ts +++ b/apps/server/src/main.ts @@ -126,6 +126,10 @@ async function bootstrap() { '/api/workspace/create', '/api/workspace/joined', '/api/workspace/find-by-email', + // Anonymous in-RAM blob sandbox: a remote consumer fetches blobs by an + // unguessable UUID without any workspace host context, so the + // workspace-resolution gate must not apply. + '/api/sb', ]; if ( diff --git a/packages/mcp/build/client.js b/packages/mcp/build/client.js index 082f8e68..74e42c87 100644 --- a/packages/mcp/build/client.js +++ b/packages/mcp/build/client.js @@ -7,6 +7,7 @@ import { TiptapTransformer } from "@hocuspocus/transformer"; import * as Y from "yjs"; import WebSocket from "ws"; import { convertProseMirrorToMarkdown } from "./lib/markdown-converter.js"; +import { collectInternalFileNodes, normalizeFileUrl, } from "./lib/internal-file-urls.js"; import { updatePageContentRealtime, replacePageContent, markdownToProseMirror, markdownToProseMirrorCanonical, mutatePageContent, buildCollabWsUrl, assertYjsEncodable, applyDocToFragment, } from "./lib/collaboration.js"; import { footnoteWarningsField } from "./lib/footnote-analyze.js"; import { buildPageTree } from "./lib/tree.js"; @@ -51,6 +52,8 @@ export class DocmostClient { // its token instead of calling POST /auth/collab-token; on a 401/403 it is // re-invoked once. Used by the internal agent to carry signed provenance. getCollabTokenFn = null; + // Optional blob-sandbox sink for the stash tool. Null when not configured. + sandboxPut = null; // In-flight login dedup: when the token expires, the 401 interceptor, // ensureAuthenticated, getCollabTokenWithReauth and the two multipart retries // can all call login() at once. Memoizing a single promise collapses that @@ -77,6 +80,9 @@ export class DocmostClient { if (config.getCollabToken) { this.getCollabTokenFn = config.getCollabToken; } + if (config.sandbox) { + this.sandboxPut = config.sandbox.put; + } this.client = axios.create({ baseURL: this.apiUrl, // Default request timeout so a hung connection cannot wedge a per-page @@ -605,6 +611,106 @@ export class DocmostClient { content: data.content || { type: "doc", content: [] }, }; } + /** + * Fetch an INTERNAL Docmost file (authed loopback) for sandbox mirroring. + * `src` is normalized to `/api/files//`; `this.client.baseURL` + * already ends in `/api`, so we strip the leading `/api` and request the + * relative path with the client's Authorization header. Returns the raw bytes + * and the response Content-Type (mime), defaulting to octet-stream. + * + * The fetch is size-bounded (hard 64 MiB ceiling) purely to protect memory; + * the authoritative per-blob cap is enforced by the sandbox `put`. The leading + * `/api` strip means this never escapes the Docmost API base. + */ + async fetchInternalFile(src) { + const HARD_CEILING = 64 * 1024 * 1024; // 64 MiB memory guard + const relPath = src.replace(/^\/api/, ""); + const response = await this.client.get(relPath, { + responseType: "arraybuffer", + timeout: 30000, + maxContentLength: HARD_CEILING, + maxBodyLength: HARD_CEILING, + }); + const buffer = Buffer.from(response.data); + if (buffer.length === 0) { + throw new Error(`Empty file response from "${src}"`); + } + const rawCt = response.headers?.["content-type"]; + const mime = typeof rawCt === "string" && rawCt.length > 0 + ? rawCt.split(";")[0].trim().toLowerCase() + : "application/octet-stream"; + return { buffer, mime }; + } + /** + * Stash a page's full content into the in-RAM blob sandbox and return ONLY a + * short anonymous URL — the body never enters the model context (this is the + * whole point: ~30KB+ ProseMirror docs blow the model context if passed as a + * tool argument). Every INTERNAL file/image src (the type-agnostic criterion, + * so drawio/excalidraw/video/file nodes are covered too) is mirrored into the + * sandbox and its `src` rewritten to the sandbox URL, so an external consumer + * can fetch the images anonymously. External http(s) srcs are left untouched. + * + * Blobs live in RAM with a short TTL and are cleared on restart — consume the + * URLs within the TTL and one uptime. A failed image fetch never aborts the + * doc: the original src is kept and the failure counted. + * + * Returns { uri, sha256, size, images:{mirrored, failed} }. `uri` and `sha256` + * are for the document blob; `sha256` is also the blob's ETag (integrity). + */ + async stashPage(pageId) { + if (!this.sandboxPut) { + throw new Error("stash_page is unavailable: the blob sandbox is not configured on this server"); + } + await this.ensureAuthenticated(); + // Stash the SAME shape get_page_json returns (id/title/.../content), with a + // deep clone so the rewrite never mutates anything shared. + const pageJson = await this.getPageJson(pageId); + const cloned = structuredClone(pageJson); + // Group internal-file nodes by normalized src so each unique resource is + // fetched + stored ONCE (dedup), and every node sharing that src points at + // the one sandbox blob. + const bySrc = new Map(); + for (const node of collectInternalFileNodes(cloned.content)) { + const src = normalizeFileUrl(String(node.attrs.src)); + const group = bySrc.get(src); + if (group) + group.push(node); + else + bySrc.set(src, [node]); + } + let mirrored = 0; + let failed = 0; + const MAX_CONCURRENCY = 5; + const groups = [...bySrc.entries()]; + for (let i = 0; i < groups.length; i += MAX_CONCURRENCY) { + const batch = groups.slice(i, i + MAX_CONCURRENCY); + await Promise.all(batch.map(async ([src, nodes]) => { + try { + const { buffer, mime } = await this.fetchInternalFile(src); + // put may throw if the blob exceeds the per-blob/total caps. + const stored = this.sandboxPut(buffer, mime); + for (const node of nodes) + node.attrs.src = stored.uri; + mirrored++; + } + catch (err) { + // One bad/oversized image must not abort the document. + failed++; + if (process.env.DEBUG) { + console.error(`stash_page: failed to mirror "${src}":`, err); + } + } + })); + } + const docBuf = Buffer.from(JSON.stringify(cloned), "utf8"); + const stored = this.sandboxPut(docBuf, "application/json"); + return { + uri: stored.uri, + sha256: stored.sha256, + size: stored.size, + images: { mirrored, failed }, + }; + } /** * Compact outline of a page's top-level blocks (no full document body). * Cheap way to locate sections/tables and grab block ids before drilling in diff --git a/packages/mcp/build/index.js b/packages/mcp/build/index.js index edcad9e6..c684a479 100644 --- a/packages/mcp/build/index.js +++ b/packages/mcp/build/index.js @@ -285,6 +285,26 @@ export function createDocmostMcpServer(config) { const result = await docmostClient.editPageText(pageId, edits); return jsonContent(result); }); + // Tool: stash_page — returns a resource_link (NOT embedded text) so the doc + // body never enters the model context. Registered directly (not via + // registerShared) because that helper only emits text content. + server.registerTool(SHARED_TOOL_SPECS.stashPage.mcpName, { + description: SHARED_TOOL_SPECS.stashPage.description, + inputSchema: SHARED_TOOL_SPECS.stashPage.buildShape(z), + }, async ({ pageId }) => { + const result = await docmostClient.stashPage(pageId); + return { + content: [ + { + type: "resource_link", + uri: result.uri, + name: "page.json", + mimeType: "application/json", + size: result.size, + }, + ], + }; + }); // Tool: patch_node server.registerTool("patch_node", { description: "Replaces a single block identified by its attrs.id WITHOUT resending the " + diff --git a/packages/mcp/build/lib/internal-file-urls.js b/packages/mcp/build/lib/internal-file-urls.js new file mode 100644 index 00000000..4eab311d --- /dev/null +++ b/packages/mcp/build/lib/internal-file-urls.js @@ -0,0 +1,55 @@ +// Detection + collection of INTERNAL Docmost file URLs inside a ProseMirror doc. +// +// An internal file URL is a relative path served by Docmost's authenticated +// attachment route (`GET /api/files/:fileId/:fileName`). It is useless to an +// external consumer (relative + needs a Docmost session), so the stash tool +// mirrors every such resource into the blob sandbox and rewrites its `src`. +// +// The criterion is "internal file URL", NOT the node TYPE: image, drawio, +// excalidraw, video and file nodes all carry such a `src`, so a type-agnostic +// walker covers them all. External http(s) srcs (CDNs) are left untouched. +// +// Mirrors editor-ext's isInternalFileUrl / normalizeFileUrl (kept as a local +// dup so the ESM mcp package does not depend on the editor-ext build). +export function isInternalFileUrl(url) { + if (typeof url !== "string") + return false; + const normalized = url.trim(); + return (normalized.startsWith("/api/files/") || normalized.startsWith("/files/")); +} +/** Normalize a bare `/files/...` src to the canonical `/api/files/...` form. */ +export function normalizeFileUrl(src) { + const trimmed = src.trim(); + if (trimmed.startsWith("/files/")) + return "/api" + trimmed; + return trimmed; +} +/** + * Recursively collect every node whose `attrs.src` is an internal file URL. + * Returns references to the live nodes (so the caller can rewrite `attrs.src` + * in place on its clone). Descends `content` arrays, covering callouts, tables, + * details and any other nested container. + */ +export function collectInternalFileNodes(doc) { + const out = []; + const visit = (node) => { + if (!node) + return; + if (Array.isArray(node)) { + for (const child of node) + visit(child); + return; + } + if (typeof node !== "object") + return; + if (node.attrs && isInternalFileUrl(node.attrs.src)) { + out.push(node); + } + if (Array.isArray(node.content)) { + for (const child of node.content) + visit(child); + } + }; + visit(doc); + return out; +} diff --git a/packages/mcp/build/tool-specs.js b/packages/mcp/build/tool-specs.js index d834e657..7b9e2f19 100644 --- a/packages/mcp/build/tool-specs.js +++ b/packages/mcp/build/tool-specs.js @@ -209,4 +209,24 @@ export const SHARED_TOOL_SPECS = { .describe('List of find/replace operations, applied in order'), }), }, + // --- hand a large page to an external consumer without bloating context --- + stashPage: { + mcpName: 'stash_page', + inAppKey: 'stashPage', + description: 'Serialize a whole page (the full ProseMirror JSON, as get_page_json ' + + 'returns) into an ephemeral in-memory blob and return ONLY a short ' + + 'anonymous URL to it — the body NEVER enters the model context, so this ' + + 'is the way to hand a large page (or its images) to an external consumer ' + + 'without truncation. Every internal file/image attachment is mirrored ' + + 'into the same sandbox and its src rewritten to a sandbox URL, so the ' + + 'consumer can fetch the images anonymously too; external http(s) images ' + + 'are left untouched. Returns { uri, size, sha256, images:{mirrored, ' + + 'failed} }. Integrity: the blob is served with ETag = its sha256, so a ' + + 'truncated/corrupted fetch is detectable. Blobs are RAM-only: they expire ' + + 'after a short TTL (~1h) and are cleared on restart — consume the URL ' + + 'within the TTL and one uptime, or re-stash.', + buildShape: (z) => ({ + pageId: z.string().min(1), + }), + }, }; diff --git a/packages/mcp/src/client.ts b/packages/mcp/src/client.ts index 181c7e79..1dfd58e4 100644 --- a/packages/mcp/src/client.ts +++ b/packages/mcp/src/client.ts @@ -13,6 +13,10 @@ import { TiptapTransformer } from "@hocuspocus/transformer"; import * as Y from "yjs"; import WebSocket from "ws"; import { convertProseMirrorToMarkdown } from "./lib/markdown-converter.js"; +import { + collectInternalFileNodes, + normalizeFileUrl, +} from "./lib/internal-file-urls.js"; import { updatePageContentRealtime, replacePageContent, @@ -102,6 +106,14 @@ const MIME_TO_EXT: Record = { * Housed here (not in index.ts) so client.ts has no type dependency on index.ts; * index.ts re-exports it for the package's public surface. */ +// Sink the stash tool writes blobs into. The host app binds this to its in-RAM +// SandboxStore and composes the public `uri` (the package never sees the store +// or any env). `put` returns the anonymous read URL plus integrity metadata. +export type SandboxPut = ( + buf: Buffer, + mime: string, +) => { uri: string; sha256: string; size: number }; + export type DocmostMcpConfig = { apiUrl: string } & ( | { email: string; password: string } | { getToken: () => Promise } // returns a BARE JWT; the client adds "Bearer " @@ -109,6 +121,9 @@ export type DocmostMcpConfig = { apiUrl: string } & ( // Optional collab-token provider (returns a ready collab JWT). Common to // both branches; see the type doc above. getCollabToken?: () => Promise; + // Optional blob sandbox sink. Present only where the stash tool is wired; + // when absent, stash_page throws a clear "not configured" error. + sandbox?: { put: SandboxPut }; }; export class DocmostClient { @@ -126,6 +141,8 @@ export class DocmostClient { // its token instead of calling POST /auth/collab-token; on a 401/403 it is // re-invoked once. Used by the internal agent to carry signed provenance. private getCollabTokenFn: (() => Promise) | null = null; + // Optional blob-sandbox sink for the stash tool. Null when not configured. + private sandboxPut: SandboxPut | null = null; // In-flight login dedup: when the token expires, the 401 interceptor, // ensureAuthenticated, getCollabTokenWithReauth and the two multipart retries // can all call login() at once. Memoizing a single promise collapses that @@ -165,6 +182,9 @@ export class DocmostClient { if (config.getCollabToken) { this.getCollabTokenFn = config.getCollabToken; } + if (config.sandbox) { + this.sandboxPut = config.sandbox.put; + } this.client = axios.create({ baseURL: this.apiUrl, // Default request timeout so a hung connection cannot wedge a per-page @@ -767,6 +787,120 @@ export class DocmostClient { }; } + /** + * Fetch an INTERNAL Docmost file (authed loopback) for sandbox mirroring. + * `src` is normalized to `/api/files//`; `this.client.baseURL` + * already ends in `/api`, so we strip the leading `/api` and request the + * relative path with the client's Authorization header. Returns the raw bytes + * and the response Content-Type (mime), defaulting to octet-stream. + * + * The fetch is size-bounded (hard 64 MiB ceiling) purely to protect memory; + * the authoritative per-blob cap is enforced by the sandbox `put`. The leading + * `/api` strip means this never escapes the Docmost API base. + */ + private async fetchInternalFile( + src: string, + ): Promise<{ buffer: Buffer; mime: string }> { + const HARD_CEILING = 64 * 1024 * 1024; // 64 MiB memory guard + const relPath = src.replace(/^\/api/, ""); + const response = await this.client.get(relPath, { + responseType: "arraybuffer", + timeout: 30000, + maxContentLength: HARD_CEILING, + maxBodyLength: HARD_CEILING, + }); + const buffer = Buffer.from(response.data); + if (buffer.length === 0) { + throw new Error(`Empty file response from "${src}"`); + } + const rawCt = response.headers?.["content-type"]; + const mime = + typeof rawCt === "string" && rawCt.length > 0 + ? rawCt.split(";")[0].trim().toLowerCase() + : "application/octet-stream"; + return { buffer, mime }; + } + + /** + * Stash a page's full content into the in-RAM blob sandbox and return ONLY a + * short anonymous URL — the body never enters the model context (this is the + * whole point: ~30KB+ ProseMirror docs blow the model context if passed as a + * tool argument). Every INTERNAL file/image src (the type-agnostic criterion, + * so drawio/excalidraw/video/file nodes are covered too) is mirrored into the + * sandbox and its `src` rewritten to the sandbox URL, so an external consumer + * can fetch the images anonymously. External http(s) srcs are left untouched. + * + * Blobs live in RAM with a short TTL and are cleared on restart — consume the + * URLs within the TTL and one uptime. A failed image fetch never aborts the + * doc: the original src is kept and the failure counted. + * + * Returns { uri, sha256, size, images:{mirrored, failed} }. `uri` and `sha256` + * are for the document blob; `sha256` is also the blob's ETag (integrity). + */ + async stashPage(pageId: string): Promise<{ + uri: string; + sha256: string; + size: number; + images: { mirrored: number; failed: number }; + }> { + if (!this.sandboxPut) { + throw new Error( + "stash_page is unavailable: the blob sandbox is not configured on this server", + ); + } + await this.ensureAuthenticated(); + + // Stash the SAME shape get_page_json returns (id/title/.../content), with a + // deep clone so the rewrite never mutates anything shared. + const pageJson = await this.getPageJson(pageId); + const cloned: any = structuredClone(pageJson); + + // Group internal-file nodes by normalized src so each unique resource is + // fetched + stored ONCE (dedup), and every node sharing that src points at + // the one sandbox blob. + const bySrc = new Map(); + for (const node of collectInternalFileNodes(cloned.content)) { + const src = normalizeFileUrl(String(node.attrs.src)); + const group = bySrc.get(src); + if (group) group.push(node); + else bySrc.set(src, [node]); + } + + let mirrored = 0; + let failed = 0; + const MAX_CONCURRENCY = 5; + const groups = [...bySrc.entries()]; + for (let i = 0; i < groups.length; i += MAX_CONCURRENCY) { + const batch = groups.slice(i, i + MAX_CONCURRENCY); + await Promise.all( + batch.map(async ([src, nodes]) => { + try { + const { buffer, mime } = await this.fetchInternalFile(src); + // put may throw if the blob exceeds the per-blob/total caps. + const stored = this.sandboxPut!(buffer, mime); + for (const node of nodes) node.attrs.src = stored.uri; + mirrored++; + } catch (err) { + // One bad/oversized image must not abort the document. + failed++; + if (process.env.DEBUG) { + console.error(`stash_page: failed to mirror "${src}":`, err); + } + } + }), + ); + } + + const docBuf = Buffer.from(JSON.stringify(cloned), "utf8"); + const stored = this.sandboxPut(docBuf, "application/json"); + return { + uri: stored.uri, + sha256: stored.sha256, + size: stored.size, + images: { mirrored, failed }, + }; + } + /** * Compact outline of a page's top-level blocks (no full document body). * Cheap way to locate sections/tables and grab block ids before drilling in diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index db29f143..bc594a28 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -408,6 +408,31 @@ registerShared(SHARED_TOOL_SPECS.editPageText, async ({ pageId, edits }) => { return jsonContent(result); }); +// Tool: stash_page — returns a resource_link (NOT embedded text) so the doc +// body never enters the model context. Registered directly (not via +// registerShared) because that helper only emits text content. +server.registerTool( + SHARED_TOOL_SPECS.stashPage.mcpName, + { + description: SHARED_TOOL_SPECS.stashPage.description, + inputSchema: SHARED_TOOL_SPECS.stashPage.buildShape!(z), + }, + async ({ pageId }: { pageId: string }) => { + const result = await docmostClient.stashPage(pageId); + return { + content: [ + { + type: "resource_link" as const, + uri: result.uri, + name: "page.json", + mimeType: "application/json", + size: result.size, + }, + ], + }; + }, +); + // Tool: patch_node server.registerTool( "patch_node", diff --git a/packages/mcp/src/lib/internal-file-urls.ts b/packages/mcp/src/lib/internal-file-urls.ts new file mode 100644 index 00000000..e201e1e0 --- /dev/null +++ b/packages/mcp/src/lib/internal-file-urls.ts @@ -0,0 +1,54 @@ +// Detection + collection of INTERNAL Docmost file URLs inside a ProseMirror doc. +// +// An internal file URL is a relative path served by Docmost's authenticated +// attachment route (`GET /api/files/:fileId/:fileName`). It is useless to an +// external consumer (relative + needs a Docmost session), so the stash tool +// mirrors every such resource into the blob sandbox and rewrites its `src`. +// +// The criterion is "internal file URL", NOT the node TYPE: image, drawio, +// excalidraw, video and file nodes all carry such a `src`, so a type-agnostic +// walker covers them all. External http(s) srcs (CDNs) are left untouched. +// +// Mirrors editor-ext's isInternalFileUrl / normalizeFileUrl (kept as a local +// dup so the ESM mcp package does not depend on the editor-ext build). + +export function isInternalFileUrl(url: unknown): boolean { + if (typeof url !== "string") return false; + const normalized = url.trim(); + return ( + normalized.startsWith("/api/files/") || normalized.startsWith("/files/") + ); +} + +/** Normalize a bare `/files/...` src to the canonical `/api/files/...` form. */ +export function normalizeFileUrl(src: string): string { + const trimmed = src.trim(); + if (trimmed.startsWith("/files/")) return "/api" + trimmed; + return trimmed; +} + +/** + * Recursively collect every node whose `attrs.src` is an internal file URL. + * Returns references to the live nodes (so the caller can rewrite `attrs.src` + * in place on its clone). Descends `content` arrays, covering callouts, tables, + * details and any other nested container. + */ +export function collectInternalFileNodes(doc: unknown): any[] { + const out: any[] = []; + const visit = (node: any): void => { + if (!node) return; + if (Array.isArray(node)) { + for (const child of node) visit(child); + return; + } + if (typeof node !== "object") return; + if (node.attrs && isInternalFileUrl(node.attrs.src)) { + out.push(node); + } + if (Array.isArray(node.content)) { + for (const child of node.content) visit(child); + } + }; + visit(doc); + return out; +} diff --git a/packages/mcp/src/tool-specs.ts b/packages/mcp/src/tool-specs.ts index 8f689c64..7a19cd66 100644 --- a/packages/mcp/src/tool-specs.ts +++ b/packages/mcp/src/tool-specs.ts @@ -266,4 +266,26 @@ export const SHARED_TOOL_SPECS = { .describe('List of find/replace operations, applied in order'), }), }, + + // --- hand a large page to an external consumer without bloating context --- + stashPage: { + mcpName: 'stash_page', + inAppKey: 'stashPage', + description: + 'Serialize a whole page (the full ProseMirror JSON, as get_page_json ' + + 'returns) into an ephemeral in-memory blob and return ONLY a short ' + + 'anonymous URL to it — the body NEVER enters the model context, so this ' + + 'is the way to hand a large page (or its images) to an external consumer ' + + 'without truncation. Every internal file/image attachment is mirrored ' + + 'into the same sandbox and its src rewritten to a sandbox URL, so the ' + + 'consumer can fetch the images anonymously too; external http(s) images ' + + 'are left untouched. Returns { uri, size, sha256, images:{mirrored, ' + + 'failed} }. Integrity: the blob is served with ETag = its sha256, so a ' + + 'truncated/corrupted fetch is detectable. Blobs are RAM-only: they expire ' + + 'after a short TTL (~1h) and are cleared on restart — consume the URL ' + + 'within the TTL and one uptime, or re-stash.', + buildShape: (z) => ({ + pageId: z.string().min(1), + }), + }, } satisfies Record; diff --git a/packages/mcp/test/mock/stash-page.test.mjs b/packages/mcp/test/mock/stash-page.test.mjs new file mode 100644 index 00000000..126caab6 --- /dev/null +++ b/packages/mcp/test/mock/stash-page.test.mjs @@ -0,0 +1,183 @@ +// Mock-HTTP test for DocmostClient.stashPage: a local http server stands in for +// Docmost so the whole flow stays deterministic and offline. Asserts the tool +// (1) serializes the page into the sandbox and returns ONLY a link (uri + sha256 +// + size), never the body; (2) mirrors INTERNAL image srcs into the sandbox and +// rewrites them to the sandbox uri; (3) leaves EXTERNAL http(s) srcs untouched; +// (4) de-duplicates a repeated internal src to a single blob; (5) counts a +// failed image fetch without aborting the document. +import { test, after } from "node:test"; +import assert from "node:assert/strict"; +import http from "node:http"; +import { createHash } from "node:crypto"; +import { DocmostClient } from "../../build/client.js"; + +function readBody(req) { + return new Promise((resolve) => { + let raw = ""; + req.on("data", (c) => (raw += c)); + req.on("end", () => resolve(raw)); + }); +} + +function startServer(handler) { + return new Promise((resolve) => { + const server = http.createServer(handler); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address(); + resolve({ server, baseURL: `http://127.0.0.1:${port}/api` }); + }); + }); +} + +const openServers = []; +async function spawn(handler) { + const { server, baseURL } = await startServer(handler); + openServers.push(server); + return baseURL; +} +after(async () => { + await Promise.all(openServers.map((s) => new Promise((r) => s.close(r)))); +}); + +// In-memory sandbox sink mirroring the host binding: store the blob, return a +// uri + sha256 + size. Records every put so the test can inspect what was +// stashed (and verify the doc body never leaves via the return value). +function makeSandbox() { + const puts = []; + return { + puts, + put(buf, mime) { + const sha256 = createHash("sha256").update(buf).digest("hex"); + const id = `id-${puts.length}`; + puts.push({ buf, mime, sha256 }); + return { uri: `https://sb.test/api/sb/${id}`, sha256, size: buf.length }; + }, + }; +} + +const IMAGE_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); // "PNG" header-ish + +function pageDoc() { + return { + type: "doc", + content: [ + { + type: "image", + attrs: { src: "/api/files/att-1/pic.png", attachmentId: "att-1", width: 100 }, + }, + // Same internal src again -> must dedup to ONE blob, both rewritten. + { + type: "image", + attrs: { src: "/api/files/att-1/pic.png", attachmentId: "att-1", width: 50 }, + }, + // External CDN image -> must be left untouched. + { + type: "image", + attrs: { src: "https://cdn.example.com/remote.png" }, + }, + ], + }; +} + +// Build a client wired to a server that logs in, serves the page, and serves the +// internal file bytes. `fileStatus` lets a test force the file fetch to fail. +async function buildClient(sandbox, { fileStatus = 200 } = {}) { + const baseURL = await spawn(async (req, res) => { + await readBody(req); + if (req.url === "/api/auth/login") { + res.writeHead(200, { + "Content-Type": "application/json", + "Set-Cookie": "authToken=tok; HttpOnly", + }); + res.end(JSON.stringify({ token: "tok" })); + return; + } + if (req.url === "/api/pages/info") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ data: { id: "page-1", title: "T", content: pageDoc() } })); + return; + } + if (req.url.startsWith("/api/files/att-1/")) { + if (fileStatus !== 200) { + res.writeHead(fileStatus); + res.end(); + return; + } + res.writeHead(200, { "Content-Type": "image/png" }); + res.end(IMAGE_BYTES); + return; + } + res.writeHead(404); + res.end(); + }); + return new DocmostClient({ + apiUrl: baseURL, + email: "u@example.com", + password: "pw", + sandbox: { put: (buf, mime) => sandbox.put(buf, mime) }, + }); +} + +test("stashPage stores the doc + mirrors/rewrites internal images, returns only a link", async () => { + const sandbox = makeSandbox(); + const client = await buildClient(sandbox); + + const result = await client.stashPage("page-1"); + + // Returns ONLY a link shape — never the document body. + assert.equal(typeof result.uri, "string"); + assert.match(result.uri, /^https:\/\/sb\.test\/api\/sb\//); + assert.equal(typeof result.sha256, "string"); + assert.equal(typeof result.size, "number"); + assert.ok(!("doc" in result) && !("content" in result) && !("body" in result)); + assert.deepEqual(result.images, { mirrored: 1, failed: 0 }); + + // One image blob (dedup) + one doc blob = 2 puts. + assert.equal(sandbox.puts.length, 2); + const imagePut = sandbox.puts[0]; + const docPut = sandbox.puts[1]; + assert.equal(imagePut.mime, "image/png"); + assert.ok(imagePut.buf.equals(IMAGE_BYTES)); + assert.equal(docPut.mime, "application/json"); + + // The returned uri/sha256 are the DOCUMENT blob's. + assert.equal(result.sha256, docPut.sha256); + + // Inspect the stashed document: internal srcs rewritten, external untouched. + const stashed = JSON.parse(docPut.buf.toString("utf8")); + const imgs = stashed.content.content.filter((n) => n.type === "image"); + assert.equal(imgs[0].attrs.src, "https://sb.test/api/sb/id-0"); + assert.equal(imgs[1].attrs.src, "https://sb.test/api/sb/id-0"); // same blob (dedup) + assert.equal(imgs[2].attrs.src, "https://cdn.example.com/remote.png"); // external kept +}); + +test("stashPage counts a failed image fetch without aborting the document", async () => { + const sandbox = makeSandbox(); + const client = await buildClient(sandbox, { fileStatus: 500 }); + + const result = await client.stashPage("page-1"); + + assert.deepEqual(result.images, { mirrored: 0, failed: 1 }); + // Only the doc blob was stored (image fetch failed). + assert.equal(sandbox.puts.length, 1); + assert.equal(sandbox.puts[0].mime, "application/json"); + + // The failed internal src is LEFT as-is so nothing is silently dropped. + const stashed = JSON.parse(sandbox.puts[0].buf.toString("utf8")); + const imgs = stashed.content.content.filter((n) => n.type === "image"); + assert.equal(imgs[0].attrs.src, "/api/files/att-1/pic.png"); +}); + +test("stashPage throws a clear error when no sandbox is configured", async () => { + const baseURL = await spawn(async (req, res) => { + await readBody(req); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({})); + }); + const client = new DocmostClient({ + apiUrl: baseURL, + email: "u@example.com", + password: "pw", + }); + await assert.rejects(() => client.stashPage("page-1"), /not configured/); +});