feat(sandbox): in-RAM blob sandbox for out-of-band page transfer (#243)

Add an ephemeral, process-local blob store so the in-app agent (and the
embedded MCP) can hand a large page document and its images to an external
consumer WITHOUT routing the bytes through the model context or Docmost auth.

- SandboxStore (@Injectable singleton): Map<uuid,{buf,mime,sha256,expiresAt}>
  in RAM only. put() picks a per-blob cap by mime (image vs doc), enforces a
  total-bytes RAM guard with oldest-first eviction, and stamps a TTL; get()
  lazily expires. sha256 computed at put() doubles as the strong ETag. An
  unref'd sweep interval clears expired entries and is cleared on destroy.
- GET /api/sb/:uuid anonymous controller: serves raw bytes with Content-Type,
  Content-Length and ETag=sha256; 404 on missing/expired/non-UUID (anti-
  traversal), 304 on a matching If-None-Match. No tokens, no 401 — the
  capability is the unguessable UUID + short TTL + TLS. Auth-exempt the same
  way as /api/files/public (no JwtAuthGuard) plus an /api/sb entry in main.ts's
  workspace-resolution preHandler so a remote consumer with no workspace host
  is not rejected.
- stash_page tool in both layers (MCP resource_link + in-app {uri,size,sha256,
  images}). client.stashPage serializes the get_page_json shape, mirrors every
  INTERNAL file/image src (type-agnostic, covers drawio/excalidraw/video/file)
  into the sandbox under Docmost auth and rewrites src to the sandbox URL;
  external http(s) srcs are left untouched; dedup by src; a failed image fetch
  is counted, never aborts the doc.
- SANDBOX_PUBLIC_URL / SANDBOX_TTL_MS / SANDBOX_MAX_BYTES /
  SANDBOX_MAX_IMAGE_BYTES / SANDBOX_MAX_TOTAL_BYTES wired through the
  environment service + validation + .env.example.
- SandboxModule (@Global) provides the shared store to the controller,
  McpService and AiChatToolsService (same instance for put and get).

Tests: SandboxStore (round-trip, sha256, TTL lazy + sweep, caps, eviction),
SandboxController (200+ETag+CT+CL, 404 missing/expired/non-UUID, 304), and a
mock-HTTP stashPage test (mirror+rewrite internal, keep external, dedup, failed
image counted, returns only a link). Interoperates with the vvzvlad/habr-mcp
consumer's anonymous-GET + sha256-ETag + resource_link contract.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
claude code agent 227
2026-06-28 15:13:11 +03:00
parent 106df7c907
commit 2fe4ca8537
25 changed files with 1312 additions and 4 deletions

View File

@@ -124,6 +124,22 @@ MCP_DOCMOST_PASSWORD=
# MCP_TOKEN=
# MCP_SESSION_IDLE_MS=1800000
#
# BLOB SANDBOX (stash_page). An in-RAM, process-local store that hands large page
# content + images to an external consumer WITHOUT bloating the model context or
# requiring Docmost auth. The stash_page tool serializes a page, mirrors its
# internal images into the store, and returns ONLY a short anonymous URL; the
# consumer fetches blobs via `GET /api/sb/<uuid>` (no token — the capability is
# the unguessable UUID + short TTL + TLS). Blobs are RAM-only and cleared on
# restart. ETag = the blob's sha256 (integrity check).
# SANDBOX_PUBLIC_URL is the base used to build those URLs; it MUST be reachable
# by the consumer (do NOT use a loopback address if the consumer is remote).
# Defaults to APP_URL when unset.
# SANDBOX_PUBLIC_URL=https://docs.example.com
# SANDBOX_TTL_MS=3600000
# SANDBOX_MAX_BYTES=8388608
# SANDBOX_MAX_IMAGE_BYTES=20971520
# SANDBOX_MAX_TOTAL_BYTES=134217728
#
# AI-AGENT ATTRIBUTION (comments/pages written via MCP are badged as "AI"):
# attribution is driven by a per-user `is_agent` flag on the users row. There is
# NO admin UI/API for it — set it out-of-band with SQL. Use a DEDICATED service

View File

@@ -28,6 +28,7 @@ import { ClsModule } from 'nestjs-cls';
import { NoopAuditModule } from './integrations/audit/audit.module';
import { ThrottleModule } from './integrations/throttle/throttle.module';
import { McpModule } from './integrations/mcp/mcp.module';
import { SandboxModule } from './integrations/sandbox/sandbox.module';
import { AiModule } from './integrations/ai/ai.module';
import { AiChatModule } from './core/ai-chat/ai-chat.module';
@@ -89,6 +90,7 @@ try {
TelemetryModule,
ThrottleModule,
McpModule,
SandboxModule,
AiModule,
AiChatModule,
...enterpriseModules,

View File

@@ -63,6 +63,10 @@ describe('AiChatToolsService deletePage guardrail (H4)', () => {
{} as never,
{} as never,
{} as never,
// environmentService + sandboxStore (only used by the stash tool closure,
// which these tests do not execute).
{} as never,
{} as never,
);
});
@@ -175,6 +179,10 @@ describe('AiChatToolsService expanded toolset guardrails', () => {
{} as never,
{} as never,
{} as never,
// environmentService + sandboxStore (only used by the stash tool closure,
// which these tests do not execute).
{} as never,
{} as never,
);
});
@@ -290,6 +298,10 @@ describe('AiChatToolsService node-arg JSON-string coercion', () => {
{} as never,
{} as never,
{} as never,
// environmentService + sandboxStore (only used by the stash tool closure,
// which these tests do not execute).
{} as never,
{} as never,
);
});
@@ -440,6 +452,10 @@ describe('AiChatToolsService model-friendly input validation (#190)', () => {
{} as never,
{} as never,
{} as never,
// environmentService + sandboxStore (only used by the stash tool closure,
// which these tests do not execute).
{} as never,
{} as never,
);
});

View File

@@ -16,6 +16,8 @@ import {
import { resolveCurrentPageResult } from './current-page.util';
import { parseNodeArg } from './parse-node-arg';
import { modelFriendlyInput } from './model-friendly-input';
import { EnvironmentService } from '../../../integrations/environment/environment.service';
import { SandboxStore } from '../../../integrations/sandbox/sandbox.store';
/**
* Per-user, per-request adapter that exposes Docmost READ operations to the
@@ -41,6 +43,9 @@ export class AiChatToolsService {
private readonly pageEmbeddingRepo: PageEmbeddingRepo,
private readonly spaceMemberRepo: SpaceMemberRepo,
private readonly pagePermissionRepo: PagePermissionRepo,
private readonly environmentService: EnvironmentService,
// Shared singleton in-RAM blob store backing the stash tool.
private readonly sandboxStore: SandboxStore,
) {}
async forUser(
@@ -86,11 +91,22 @@ export class AiChatToolsService {
aiChatId,
});
// Bind the stash tool to the shared in-RAM SandboxStore and compose the
// anonymous public URL here (the MCP package never touches env or the
// store). put() returns the read URL + sha256/size; sha256 is also the
// blob's ETag for integrity.
const sandboxPut = (buf: Buffer, mime: string) => {
const stored = this.sandboxStore.put(buf, mime);
const base = this.environmentService.getSandboxPublicUrl();
return { uri: `${base}/api/sb/${stored.id}`, sha256: stored.sha256, size: stored.size };
};
const { DocmostClient, sharedToolSpecs } = await loadDocmostMcp();
const client: DocmostClientLike = new DocmostClient({
apiUrl,
getToken,
getCollabToken,
sandbox: { put: sandboxPut },
});
// Build an ai-SDK tool from a shared, zod-agnostic spec. The spec owns the
@@ -625,6 +641,14 @@ export class AiChatToolsService {
async ({ pageId, edits }) => await client.editPageText(pageId, edits),
),
// Returns ONLY the short link object — never the document body — so a
// large page can be handed to an external consumer without bloating
// context.
stashPage: sharedTool(
sharedToolSpecs.stashPage,
async ({ pageId }) => await client.stashPage(pageId),
),
patchNode: tool({
description:
'Replace a single content block (by id) with a new ProseMirror ' +

View File

@@ -154,6 +154,14 @@ export interface DocmostClientLike {
commentId: string,
resolved: boolean,
): Promise<Record<string, unknown>>;
// Serialize a page + mirror its internal images into the blob sandbox; returns
// ONLY a short anonymous URL (the body never enters the model context).
stashPage(pageId: string): Promise<{
uri: string;
sha256: string;
size: number;
images: { mirrored: number; failed: number };
}>;
}
export type DocmostClientConfig = {
@@ -161,6 +169,14 @@ export type DocmostClientConfig = {
getToken: () => Promise<string>;
// Provenance collab-token provider for content mutations (signed agent claim).
getCollabToken?: () => Promise<string>;
// Optional blob-sandbox sink for the stash tool. `put` stores a blob in the
// host's in-RAM SandboxStore and returns the anonymous read URL + integrity.
sandbox?: {
put: (
buf: Buffer,
mime: string,
) => { uri: string; sha256: string; size: number };
};
};
export interface DocmostClientCtor {

View File

@@ -332,4 +332,52 @@ export class EnvironmentService {
.map((o) => o.trim())
.filter(Boolean);
}
// --- Blob sandbox (in-RAM ephemeral blob transfer; see SandboxModule) ---
// Base URL the sandbox `uri` is built from. It MUST be reachable over the
// network by the external consumer that fetches the blobs (not a loopback
// address if that consumer is remote). Falls back to APP_URL when unset so a
// single-host deployment works out of the box; set it explicitly when the
// consumer lives on another host.
getSandboxPublicUrl(): string {
const raw =
this.configService.get<string>('SANDBOX_PUBLIC_URL') || this.getAppUrl();
// Drop any trailing slash so `${base}/api/sb/${id}` never doubles up.
return raw.replace(/\/+$/, '');
}
// Blob time-to-live. Default 1h. The unguessable UUID + this short TTL + TLS
// are the whole capability model (no tokens).
getSandboxTtlMs(): number {
return parseInt(
this.configService.get<string>('SANDBOX_TTL_MS', '3600000'),
10,
);
}
// Per-blob cap for non-image blobs (the serialized document). Default 8 MiB.
getSandboxMaxBytes(): number {
return parseInt(
this.configService.get<string>('SANDBOX_MAX_BYTES', '8388608'),
10,
);
}
// Per-blob cap for mirrored image blobs. Default 20 MiB.
getSandboxMaxImageBytes(): number {
return parseInt(
this.configService.get<string>('SANDBOX_MAX_IMAGE_BYTES', '20971520'),
10,
);
}
// RAM guard: total bytes the whole store may hold. Default 128 MiB. On
// overflow the store evicts oldest entries to make room.
getSandboxMaxTotalBytes(): number {
return parseInt(
this.configService.get<string>('SANDBOX_MAX_TOTAL_BYTES', '134217728'),
10,
);
}
}

View File

@@ -2,6 +2,7 @@ import {
IsIn,
IsNotEmpty,
IsNotIn,
IsNumberString,
IsOptional,
IsString,
IsUrl,
@@ -170,6 +171,35 @@ export class EnvironmentVariables {
},
)
CLICKHOUSE_URL: string;
// --- Blob sandbox (in-RAM ephemeral blob transfer; see SandboxModule) ---
@IsOptional()
@ValidateIf((obj) => obj.SANDBOX_PUBLIC_URL != '' && obj.SANDBOX_PUBLIC_URL != null)
@IsUrl(
{ protocols: ['http', 'https'], require_tld: false },
{
message:
'SANDBOX_PUBLIC_URL must be a valid http(s) URL reachable by the external blob consumer',
},
)
SANDBOX_PUBLIC_URL: string;
@IsOptional()
@IsNumberString({}, { message: 'SANDBOX_TTL_MS must be an integer (milliseconds)' })
SANDBOX_TTL_MS: string;
@IsOptional()
@IsNumberString({}, { message: 'SANDBOX_MAX_BYTES must be an integer (bytes)' })
SANDBOX_MAX_BYTES: string;
@IsOptional()
@IsNumberString({}, { message: 'SANDBOX_MAX_IMAGE_BYTES must be an integer (bytes)' })
SANDBOX_MAX_IMAGE_BYTES: string;
@IsOptional()
@IsNumberString({}, { message: 'SANDBOX_MAX_TOTAL_BYTES must be an integer (bytes)' })
SANDBOX_MAX_TOTAL_BYTES: string;
}
export function validate(config: Record<string, any>) {

View File

@@ -131,10 +131,20 @@ export class FailedLoginLimiter {
}
// The per-session DocmostMcpConfig shape understood by @docmost/mcp: either the
// service-account credentials variant OR the per-user getToken variant.
export type DocmostMcpConfig =
// service-account credentials variant OR the per-user getToken variant. The
// optional `sandbox` sink (blob store for the stash tool) is common to both and
// injected by McpService after the auth decision.
export type DocmostMcpConfig = (
| { apiUrl: string; email: string; password: string }
| { apiUrl: string; getToken: () => Promise<string> };
| { apiUrl: string; getToken: () => Promise<string> }
) & {
sandbox?: {
put: (
buf: Buffer,
mime: string,
) => { uri: string; sha256: string; size: number };
};
};
export interface ResolvedMcpAuth {
config: DocmostMcpConfig;

View File

@@ -116,6 +116,7 @@ function makeService(opts: {
undefined as never, // userRepo
undefined as never, // userSessionRepo
moduleRef as never, // moduleRef (read by the MFA branch)
undefined as never, // sandboxStore (unused by the login-gate path)
);
// Stop the constructor's unref'd sweep timer leaking across tests.
service.onModuleDestroy();

View File

@@ -30,6 +30,7 @@ import {
DocmostMcpConfig,
ResolvedMcpAuth,
} from './mcp-auth.helpers';
import { SandboxStore } from '../sandbox/sandbox.store';
// Minimal shape of the embedded MCP HTTP handler exported by @docmost/mcp/http.
interface McpHttpHandler {
@@ -99,6 +100,8 @@ export class McpService implements OnModuleDestroy {
private readonly userRepo: UserRepo,
private readonly userSessionRepo: UserSessionRepo,
private readonly moduleRef: ModuleRef,
// Shared singleton in-RAM blob store backing the stash tool.
private readonly sandboxStore: SandboxStore,
) {
this.sweepTimer = setInterval(() => {
try {
@@ -115,6 +118,23 @@ export class McpService implements OnModuleDestroy {
clearInterval(this.sweepTimer);
}
// Bind the stash tool to the shared in-RAM SandboxStore and compose the
// anonymous public URL (the MCP package owns neither env nor the store).
// put() returns the read URL + sha256/size; sha256 is also the blob ETag.
private buildSandboxConfig(): DocmostMcpConfig['sandbox'] {
return {
put: (buf: Buffer, mime: string) => {
const stored = this.sandboxStore.put(buf, mime);
const base = this.environmentService.getSandboxPublicUrl();
return {
uri: `${base}/api/sb/${stored.id}`,
sha256: stored.sha256,
size: stored.size,
};
},
};
}
// Service account the embedded MCP uses to talk back to this Docmost
// instance over loopback REST + the collaboration WebSocket. Now OPTIONAL:
// it is only a fallback when no per-user Basic/Bearer credentials are sent.
@@ -326,7 +346,10 @@ export class McpService implements OnModuleDestroy {
// Should never happen: handle() always stashes before delegating.
throw new UnauthorizedException('MCP authentication missing.');
}
return resolved.config;
// Inject the blob-sandbox sink after the auth decision so stash_page
// can store blobs in the shared in-RAM store regardless of which
// credential variant resolved.
return { ...resolved.config, sandbox: this.buildSandboxConfig() };
},
{
identify: (req: IncomingMessage) => {

View File

@@ -0,0 +1,124 @@
import { SandboxController } from './sandbox.controller';
import { SandboxEntry } from './sandbox.store';
// Capturing fake of the FastifyReply surface the controller uses:
// status()/header()/headers()/send(), all chainable.
function makeRes() {
const sent: { status: number; headers: Record<string, any>; body: any } = {
status: 200,
headers: {},
body: undefined,
};
const res: any = {
status(code: number) {
sent.status = code;
return res;
},
header(key: string, value: any) {
sent.headers[key.toLowerCase()] = value;
return res;
},
headers(obj: Record<string, any>) {
for (const k of Object.keys(obj)) sent.headers[k.toLowerCase()] = obj[k];
return res;
},
send(body?: any) {
sent.body = body;
return res;
},
_sent: sent,
};
return res;
}
function makeReq(headers: Record<string, any> = {}) {
return { headers } as any;
}
const VALID_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
function entry(buf: Buffer, mime: string, sha256: string): SandboxEntry {
return { buf, mime, sha256, expiresAt: Date.now() + 60_000 };
}
describe('SandboxController', () => {
it('serves 200 with body, Content-Type, Content-Length and sha256 ETag', async () => {
const buf = Buffer.from('{"ok":true}', 'utf8');
const sha = 'a'.repeat(64);
const store = { get: jest.fn().mockReturnValue(entry(buf, 'application/json', sha)) };
const controller = new SandboxController(store as any);
const res = makeRes();
await controller.get(VALID_ID, makeReq(), res);
expect(store.get).toHaveBeenCalledWith(VALID_ID);
expect(res._sent.status).toBe(200);
expect(res._sent.headers['content-type']).toBe('application/json');
expect(res._sent.headers['content-length']).toBe(buf.length);
expect(res._sent.headers['etag']).toBe(`"${sha}"`);
expect(res._sent.body).toBe(buf);
});
it('returns 404 for a missing/expired blob', async () => {
const store = { get: jest.fn().mockReturnValue(undefined) };
const controller = new SandboxController(store as any);
const res = makeRes();
await controller.get(VALID_ID, makeReq(), res);
expect(res._sent.status).toBe(404);
expect(res._sent.body).toBeUndefined();
});
it('returns 404 for a non-UUID id WITHOUT touching the store (anti-traversal)', async () => {
const store = { get: jest.fn() };
const controller = new SandboxController(store as any);
const res = makeRes();
await controller.get('../../etc/passwd', makeReq(), res);
expect(store.get).not.toHaveBeenCalled();
expect(res._sent.status).toBe(404);
});
it('returns 304 (no body) when If-None-Match matches the ETag', async () => {
const sha = 'b'.repeat(64);
const store = {
get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
};
const controller = new SandboxController(store as any);
const res = makeRes();
await controller.get(VALID_ID, makeReq({ 'if-none-match': `"${sha}"` }), res);
expect(res._sent.status).toBe(304);
expect(res._sent.body).toBeUndefined();
expect(res._sent.headers['etag']).toBe(`"${sha}"`);
});
it('accepts a bare (unquoted) sha256 in If-None-Match too', async () => {
const sha = 'c'.repeat(64);
const store = {
get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
};
const controller = new SandboxController(store as any);
const res = makeRes();
await controller.get(VALID_ID, makeReq({ 'if-none-match': sha }), res);
expect(res._sent.status).toBe(304);
});
it('serves 200 when If-None-Match does NOT match', async () => {
const sha = 'd'.repeat(64);
const store = {
get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
};
const controller = new SandboxController(store as any);
const res = makeRes();
await controller.get(VALID_ID, makeReq({ 'if-none-match': '"stale"' }), res);
expect(res._sent.status).toBe(200);
});
});

View File

@@ -0,0 +1,94 @@
import { Controller, Get, Param, Req, Res } from '@nestjs/common';
import { FastifyReply, FastifyRequest } from 'fastify';
import { SandboxStore } from './sandbox.store';
// Strict UUID v-agnostic shape. This is anti-traversal / input hygiene (so `:id`
// can never be a path like `../...`), NOT authorization — the capability is the
// unguessable id itself plus the short TTL plus TLS.
const UUID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
/**
* Anonymous read endpoint for the in-RAM blob sandbox.
*
* Mounted under the global `/api` prefix as `GET /api/sb/:id`. It carries NO
* `@UseGuards(JwtAuthGuard)`, so — exactly like the public attachment route
* `GET /api/files/public/...` — it is exempt from Docmost session auth. The
* route is ALSO listed in the workspace-resolution preHandler's excludedPaths
* in main.ts so a request from a remote consumer (which carries no workspace
* host) is not rejected with "Workspace not found".
*
* It only ever serves blobs looked up from the SandboxStore by a validated
* UUID; `:id` is never used as a filesystem path, so there is no traversal
* surface. Never returns tokens, never 401s.
*/
@Controller('sb')
export class SandboxController {
constructor(private readonly store: SandboxStore) {}
@Get(':id')
async get(
@Param('id') id: string,
@Req() req: FastifyRequest,
@Res() res: FastifyReply,
): Promise<void> {
// Non-UUID id (including any traversal attempt) → 404 before touching the
// store. No stack trace leaks out.
if (!UUID_RE.test(id)) {
res.status(404).send();
return;
}
const entry = this.store.get(id);
if (!entry) {
// Missing or expired — indistinguishable to the caller, by design.
res.status(404).send();
return;
}
// Strong validator: quoted sha256, no W/ weak prefix. Same value computed
// at put() time, so an external consumer can detect a truncated/corrupted
// body — the original bug this whole channel exists to fix.
const etag = `"${entry.sha256}"`;
// Conditional request: an exact ETag match → 304 with no body. The blob is
// immutable, so the validator is stable for the blob's whole lifetime.
if (this.ifNoneMatchMatches(req.headers['if-none-match'], entry.sha256)) {
res.status(304).header('ETag', etag).send();
return;
}
const ttlSeconds = Math.max(
0,
Math.floor((entry.expiresAt - Date.now()) / 1000),
);
// Use @Res() + res.send(Buffer) with an explicit Content-Type so the binary
// body bypasses the global JSON response transform/serializer.
res
.status(200)
.headers({
'Content-Type': entry.mime,
'Content-Length': entry.buf.length,
ETag: etag,
// Capability URL — keep it out of shared caches; immutable for its TTL.
'Cache-Control': `private, max-age=${ttlSeconds}, immutable`,
})
.send(entry.buf);
}
// Accept the consumer's If-None-Match whether it sends the quoted ETag, a bare
// sha256, a weak "W/"-prefixed validator, or a comma-separated list.
private ifNoneMatchMatches(
header: string | string[] | undefined,
sha256: string,
): boolean {
if (!header) return false;
const raw = Array.isArray(header) ? header.join(',') : header;
if (raw.trim() === '*') return true;
return raw
.split(',')
.map((t) => t.trim().replace(/^W\//, '').replace(/^"|"$/g, ''))
.some((t) => t === sha256);
}
}

View File

@@ -0,0 +1,19 @@
import { Global, Module } from '@nestjs/common';
import { SandboxController } from './sandbox.controller';
import { SandboxStore } from './sandbox.store';
/**
* In-RAM blob sandbox: a SINGLE shared SandboxStore (the @Injectable singleton)
* is written to by the stash tool (via McpService / AiChatToolsService) and read
* back by the anonymous SandboxController. Marked @Global so the same store
* instance is injectable everywhere without import churn — put() and get() MUST
* hit the same Map. EnvironmentService (caps/TTL/public URL) is provided by the
* global EnvironmentModule.
*/
@Global()
@Module({
controllers: [SandboxController],
providers: [SandboxStore],
exports: [SandboxStore],
})
export class SandboxModule {}

View File

@@ -0,0 +1,133 @@
import { createHash } from 'node:crypto';
import { SandboxStore } from './sandbox.store';
// Build a minimal EnvironmentService stub with overridable caps/TTL.
function makeEnv(
overrides: Partial<{
ttlMs: number;
maxBytes: number;
maxImageBytes: number;
maxTotalBytes: number;
}> = {},
) {
const cfg = {
ttlMs: 3_600_000,
maxBytes: 8_388_608,
maxImageBytes: 20_971_520,
maxTotalBytes: 134_217_728,
...overrides,
};
return {
getSandboxTtlMs: () => cfg.ttlMs,
getSandboxMaxBytes: () => cfg.maxBytes,
getSandboxMaxImageBytes: () => cfg.maxImageBytes,
getSandboxMaxTotalBytes: () => cfg.maxTotalBytes,
getSandboxPublicUrl: () => 'https://example.test',
} as any;
}
const UUID_RE =
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
describe('SandboxStore', () => {
let store: SandboxStore;
afterEach(() => {
// Clear the unref'd sweep interval so it never leaks across tests.
store?.onModuleDestroy();
jest.useRealTimers();
});
it('put/get round-trips the exact bytes + mime and returns a UUID id', () => {
store = new SandboxStore(makeEnv());
const buf = Buffer.from('{"type":"doc","content":[]}', 'utf8');
const res = store.put(buf, 'application/json');
expect(res.id).toMatch(UUID_RE);
expect(res.size).toBe(buf.length);
const entry = store.get(res.id);
expect(entry).toBeDefined();
expect(entry!.buf.equals(buf)).toBe(true);
expect(entry!.mime).toBe('application/json');
});
it('computes sha256 over the body (matches a manual digest)', () => {
store = new SandboxStore(makeEnv());
const buf = Buffer.from('hello sandbox', 'utf8');
const expected = createHash('sha256').update(buf).digest('hex');
const res = store.put(buf, 'text/plain');
expect(res.sha256).toBe(expected);
expect(store.get(res.id)!.sha256).toBe(expected);
});
it('returns undefined for a missing id', () => {
store = new SandboxStore(makeEnv());
expect(store.get('11111111-1111-1111-1111-111111111111')).toBeUndefined();
});
it('lazily expires entries past the TTL (get returns undefined)', () => {
jest.useFakeTimers();
jest.setSystemTime(new Date('2026-01-01T00:00:00Z'));
store = new SandboxStore(makeEnv({ ttlMs: 1000 }));
const res = store.put(Buffer.from('x'), 'text/plain');
expect(store.get(res.id)).toBeDefined();
jest.setSystemTime(new Date('2026-01-01T00:00:02Z')); // +2s > 1s TTL
expect(store.get(res.id)).toBeUndefined();
// Eviction also frees the byte accounting.
expect(store.bytes).toBe(0);
});
it('background sweep drops expired entries without a get()', () => {
jest.useFakeTimers();
jest.setSystemTime(new Date('2026-01-01T00:00:00Z'));
store = new SandboxStore(makeEnv({ ttlMs: 1000 }));
store.put(Buffer.from('x'), 'text/plain');
expect(store.size).toBe(1);
jest.setSystemTime(new Date('2026-01-01T00:01:30Z')); // past TTL
jest.advanceTimersByTime(60_000); // fire the sweep interval
expect(store.size).toBe(0);
});
it('rejects a non-image blob over SANDBOX_MAX_BYTES', () => {
store = new SandboxStore(makeEnv({ maxBytes: 16 }));
expect(() => store.put(Buffer.alloc(17), 'application/json')).toThrow(
/per-blob cap/,
);
});
it('uses the larger image cap for image/* blobs', () => {
// 100 bytes exceeds the doc cap (16) but fits the image cap (1024).
store = new SandboxStore(makeEnv({ maxBytes: 16, maxImageBytes: 1024 }));
expect(() => store.put(Buffer.alloc(100), 'image/png')).not.toThrow();
// SVG counts as an image too.
expect(() => store.put(Buffer.alloc(100), 'image/svg+xml')).not.toThrow();
});
it('evicts oldest entries when the total cap would be exceeded', () => {
// Total cap 250 bytes; each blob 100 bytes -> only 2 fit at a time.
store = new SandboxStore(
makeEnv({ maxTotalBytes: 250, maxBytes: 1024 }),
);
const a = store.put(Buffer.alloc(100), 'application/json');
const b = store.put(Buffer.alloc(100), 'application/json');
const c = store.put(Buffer.alloc(100), 'application/json'); // evicts a
expect(store.get(a.id)).toBeUndefined(); // oldest evicted
expect(store.get(b.id)).toBeDefined();
expect(store.get(c.id)).toBeDefined();
expect(store.bytes).toBeLessThanOrEqual(250);
});
it('rejects a single blob larger than the whole total cap', () => {
store = new SandboxStore(
makeEnv({ maxTotalBytes: 50, maxBytes: 1024 }),
);
expect(() => store.put(Buffer.alloc(100), 'application/json')).toThrow(
/total store cap/,
);
});
});

View File

@@ -0,0 +1,129 @@
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
import { createHash, randomUUID } from 'node:crypto';
import { EnvironmentService } from '../environment/environment.service';
// In-RAM, process-local blob store. No disk, no DB. Ephemeral by design: a
// restart empties it. A blob is addressed by an unguessable randomUUID() which
// IS the read capability — there are NO tokens. Each blob is immutable (its id
// never maps to changing content), so its sha256 is a perfect strong ETag.
export interface SandboxEntry {
buf: Buffer;
mime: string;
sha256: string;
expiresAt: number;
}
export interface SandboxPutResult {
id: string;
sha256: string;
size: number;
expiresAt: number;
}
@Injectable()
export class SandboxStore implements OnModuleDestroy {
private readonly logger = new Logger(SandboxStore.name);
// Map preserves insertion order, so the first key is the oldest entry — used
// for FIFO eviction when the total-bytes RAM guard is exceeded.
private readonly map = new Map<string, SandboxEntry>();
private totalBytes = 0;
// Background sweep clears expired entries so never-fetched blobs do not linger
// until the next get(). unref()'d so it never holds the event loop open;
// cleared on module destroy. Mirrors the sweepTimer pattern in
// integrations/mcp/mcp.service.ts and packages/mcp/src/http.ts.
private readonly sweepIntervalMs = 60_000;
private readonly sweepTimer: NodeJS.Timeout;
constructor(private readonly environmentService: EnvironmentService) {
this.sweepTimer = setInterval(() => {
try {
this.sweep();
} catch (err) {
this.logger.error('Sandbox sweep failed', err as Error);
}
}, this.sweepIntervalMs);
this.sweepTimer.unref?.();
}
onModuleDestroy(): void {
clearInterval(this.sweepTimer);
}
/**
* Store a blob and return its read capability id + integrity metadata. The
* per-blob cap is chosen by mime (images get the larger image cap), and the
* total-store RAM guard evicts oldest entries to make room. Throws a clear
* error when a single blob cannot fit even after eviction. Blob bodies are
* never logged.
*/
put(buf: Buffer, mime: string): SandboxPutResult {
const perBlobCap = mime.startsWith('image/')
? this.environmentService.getSandboxMaxImageBytes()
: this.environmentService.getSandboxMaxBytes();
if (buf.length > perBlobCap) {
throw new Error(
`Sandbox blob of ${buf.length} bytes exceeds the ${perBlobCap}-byte per-blob cap`,
);
}
const maxTotal = this.environmentService.getSandboxMaxTotalBytes();
if (buf.length > maxTotal) {
throw new Error(
`Sandbox blob of ${buf.length} bytes exceeds the total store cap of ${maxTotal} bytes`,
);
}
// Drop expired entries first, then evict oldest until the new blob fits.
this.sweep();
while (this.totalBytes + buf.length > maxTotal && this.map.size > 0) {
const oldest = this.map.keys().next().value as string;
this.evict(oldest);
}
const id = randomUUID();
const sha256 = createHash('sha256').update(buf).digest('hex');
const expiresAt = Date.now() + this.environmentService.getSandboxTtlMs();
this.map.set(id, { buf, mime, sha256, expiresAt });
this.totalBytes += buf.length;
return { id, sha256, size: buf.length, expiresAt };
}
/** Returns the entry, or undefined if missing OR expired (lazy expiry). */
get(id: string): SandboxEntry | undefined {
const entry = this.map.get(id);
if (!entry) return undefined;
if (entry.expiresAt <= Date.now()) {
this.evict(id);
return undefined;
}
return entry;
}
/** Current number of live entries (test/diagnostic helper). */
get size(): number {
return this.map.size;
}
/** Current total bytes held (test/diagnostic helper). */
get bytes(): number {
return this.totalBytes;
}
private evict(id: string): void {
const entry = this.map.get(id);
if (entry) {
this.totalBytes -= entry.buf.length;
this.map.delete(id);
}
}
private sweep(): void {
const now = Date.now();
for (const [id, entry] of this.map) {
if (entry.expiresAt <= now) {
this.evict(id);
}
}
}
}

View File

@@ -126,6 +126,10 @@ async function bootstrap() {
'/api/workspace/create',
'/api/workspace/joined',
'/api/workspace/find-by-email',
// Anonymous in-RAM blob sandbox: a remote consumer fetches blobs by an
// unguessable UUID without any workspace host context, so the
// workspace-resolution gate must not apply.
'/api/sb',
];
if (

View File

@@ -7,6 +7,7 @@ import { TiptapTransformer } from "@hocuspocus/transformer";
import * as Y from "yjs";
import WebSocket from "ws";
import { convertProseMirrorToMarkdown } from "./lib/markdown-converter.js";
import { collectInternalFileNodes, normalizeFileUrl, } from "./lib/internal-file-urls.js";
import { updatePageContentRealtime, replacePageContent, markdownToProseMirror, markdownToProseMirrorCanonical, mutatePageContent, buildCollabWsUrl, assertYjsEncodable, applyDocToFragment, } from "./lib/collaboration.js";
import { footnoteWarningsField } from "./lib/footnote-analyze.js";
import { buildPageTree } from "./lib/tree.js";
@@ -51,6 +52,8 @@ export class DocmostClient {
// its token instead of calling POST /auth/collab-token; on a 401/403 it is
// re-invoked once. Used by the internal agent to carry signed provenance.
getCollabTokenFn = null;
// Optional blob-sandbox sink for the stash tool. Null when not configured.
sandboxPut = null;
// In-flight login dedup: when the token expires, the 401 interceptor,
// ensureAuthenticated, getCollabTokenWithReauth and the two multipart retries
// can all call login() at once. Memoizing a single promise collapses that
@@ -77,6 +80,9 @@ export class DocmostClient {
if (config.getCollabToken) {
this.getCollabTokenFn = config.getCollabToken;
}
if (config.sandbox) {
this.sandboxPut = config.sandbox.put;
}
this.client = axios.create({
baseURL: this.apiUrl,
// Default request timeout so a hung connection cannot wedge a per-page
@@ -605,6 +611,106 @@ export class DocmostClient {
content: data.content || { type: "doc", content: [] },
};
}
/**
* Fetch an INTERNAL Docmost file (authed loopback) for sandbox mirroring.
* `src` is normalized to `/api/files/<id>/<file>`; `this.client.baseURL`
* already ends in `/api`, so we strip the leading `/api` and request the
* relative path with the client's Authorization header. Returns the raw bytes
* and the response Content-Type (mime), defaulting to octet-stream.
*
* The fetch is size-bounded (hard 64 MiB ceiling) purely to protect memory;
* the authoritative per-blob cap is enforced by the sandbox `put`. The leading
* `/api` strip means this never escapes the Docmost API base.
*/
async fetchInternalFile(src) {
const HARD_CEILING = 64 * 1024 * 1024; // 64 MiB memory guard
const relPath = src.replace(/^\/api/, "");
const response = await this.client.get(relPath, {
responseType: "arraybuffer",
timeout: 30000,
maxContentLength: HARD_CEILING,
maxBodyLength: HARD_CEILING,
});
const buffer = Buffer.from(response.data);
if (buffer.length === 0) {
throw new Error(`Empty file response from "${src}"`);
}
const rawCt = response.headers?.["content-type"];
const mime = typeof rawCt === "string" && rawCt.length > 0
? rawCt.split(";")[0].trim().toLowerCase()
: "application/octet-stream";
return { buffer, mime };
}
/**
* Stash a page's full content into the in-RAM blob sandbox and return ONLY a
* short anonymous URL — the body never enters the model context (this is the
* whole point: ~30KB+ ProseMirror docs blow the model context if passed as a
* tool argument). Every INTERNAL file/image src (the type-agnostic criterion,
* so drawio/excalidraw/video/file nodes are covered too) is mirrored into the
* sandbox and its `src` rewritten to the sandbox URL, so an external consumer
* can fetch the images anonymously. External http(s) srcs are left untouched.
*
* Blobs live in RAM with a short TTL and are cleared on restart — consume the
* URLs within the TTL and one uptime. A failed image fetch never aborts the
* doc: the original src is kept and the failure counted.
*
* Returns { uri, sha256, size, images:{mirrored, failed} }. `uri` and `sha256`
* are for the document blob; `sha256` is also the blob's ETag (integrity).
*/
async stashPage(pageId) {
if (!this.sandboxPut) {
throw new Error("stash_page is unavailable: the blob sandbox is not configured on this server");
}
await this.ensureAuthenticated();
// Stash the SAME shape get_page_json returns (id/title/.../content), with a
// deep clone so the rewrite never mutates anything shared.
const pageJson = await this.getPageJson(pageId);
const cloned = structuredClone(pageJson);
// Group internal-file nodes by normalized src so each unique resource is
// fetched + stored ONCE (dedup), and every node sharing that src points at
// the one sandbox blob.
const bySrc = new Map();
for (const node of collectInternalFileNodes(cloned.content)) {
const src = normalizeFileUrl(String(node.attrs.src));
const group = bySrc.get(src);
if (group)
group.push(node);
else
bySrc.set(src, [node]);
}
let mirrored = 0;
let failed = 0;
const MAX_CONCURRENCY = 5;
const groups = [...bySrc.entries()];
for (let i = 0; i < groups.length; i += MAX_CONCURRENCY) {
const batch = groups.slice(i, i + MAX_CONCURRENCY);
await Promise.all(batch.map(async ([src, nodes]) => {
try {
const { buffer, mime } = await this.fetchInternalFile(src);
// put may throw if the blob exceeds the per-blob/total caps.
const stored = this.sandboxPut(buffer, mime);
for (const node of nodes)
node.attrs.src = stored.uri;
mirrored++;
}
catch (err) {
// One bad/oversized image must not abort the document.
failed++;
if (process.env.DEBUG) {
console.error(`stash_page: failed to mirror "${src}":`, err);
}
}
}));
}
const docBuf = Buffer.from(JSON.stringify(cloned), "utf8");
const stored = this.sandboxPut(docBuf, "application/json");
return {
uri: stored.uri,
sha256: stored.sha256,
size: stored.size,
images: { mirrored, failed },
};
}
/**
* Compact outline of a page's top-level blocks (no full document body).
* Cheap way to locate sections/tables and grab block ids before drilling in

View File

@@ -285,6 +285,26 @@ export function createDocmostMcpServer(config) {
const result = await docmostClient.editPageText(pageId, edits);
return jsonContent(result);
});
// Tool: stash_page — returns a resource_link (NOT embedded text) so the doc
// body never enters the model context. Registered directly (not via
// registerShared) because that helper only emits text content.
server.registerTool(SHARED_TOOL_SPECS.stashPage.mcpName, {
description: SHARED_TOOL_SPECS.stashPage.description,
inputSchema: SHARED_TOOL_SPECS.stashPage.buildShape(z),
}, async ({ pageId }) => {
const result = await docmostClient.stashPage(pageId);
return {
content: [
{
type: "resource_link",
uri: result.uri,
name: "page.json",
mimeType: "application/json",
size: result.size,
},
],
};
});
// Tool: patch_node
server.registerTool("patch_node", {
description: "Replaces a single block identified by its attrs.id WITHOUT resending the " +

View File

@@ -0,0 +1,55 @@
// Detection + collection of INTERNAL Docmost file URLs inside a ProseMirror doc.
//
// An internal file URL is a relative path served by Docmost's authenticated
// attachment route (`GET /api/files/:fileId/:fileName`). It is useless to an
// external consumer (relative + needs a Docmost session), so the stash tool
// mirrors every such resource into the blob sandbox and rewrites its `src`.
//
// The criterion is "internal file URL", NOT the node TYPE: image, drawio,
// excalidraw, video and file nodes all carry such a `src`, so a type-agnostic
// walker covers them all. External http(s) srcs (CDNs) are left untouched.
//
// Mirrors editor-ext's isInternalFileUrl / normalizeFileUrl (kept as a local
// dup so the ESM mcp package does not depend on the editor-ext build).
export function isInternalFileUrl(url) {
if (typeof url !== "string")
return false;
const normalized = url.trim();
return (normalized.startsWith("/api/files/") || normalized.startsWith("/files/"));
}
/** Normalize a bare `/files/...` src to the canonical `/api/files/...` form. */
export function normalizeFileUrl(src) {
const trimmed = src.trim();
if (trimmed.startsWith("/files/"))
return "/api" + trimmed;
return trimmed;
}
/**
* Recursively collect every node whose `attrs.src` is an internal file URL.
* Returns references to the live nodes (so the caller can rewrite `attrs.src`
* in place on its clone). Descends `content` arrays, covering callouts, tables,
* details and any other nested container.
*/
export function collectInternalFileNodes(doc) {
const out = [];
const visit = (node) => {
if (!node)
return;
if (Array.isArray(node)) {
for (const child of node)
visit(child);
return;
}
if (typeof node !== "object")
return;
if (node.attrs && isInternalFileUrl(node.attrs.src)) {
out.push(node);
}
if (Array.isArray(node.content)) {
for (const child of node.content)
visit(child);
}
};
visit(doc);
return out;
}

View File

@@ -209,4 +209,24 @@ export const SHARED_TOOL_SPECS = {
.describe('List of find/replace operations, applied in order'),
}),
},
// --- hand a large page to an external consumer without bloating context ---
stashPage: {
mcpName: 'stash_page',
inAppKey: 'stashPage',
description: 'Serialize a whole page (the full ProseMirror JSON, as get_page_json ' +
'returns) into an ephemeral in-memory blob and return ONLY a short ' +
'anonymous URL to it — the body NEVER enters the model context, so this ' +
'is the way to hand a large page (or its images) to an external consumer ' +
'without truncation. Every internal file/image attachment is mirrored ' +
'into the same sandbox and its src rewritten to a sandbox URL, so the ' +
'consumer can fetch the images anonymously too; external http(s) images ' +
'are left untouched. Returns { uri, size, sha256, images:{mirrored, ' +
'failed} }. Integrity: the blob is served with ETag = its sha256, so a ' +
'truncated/corrupted fetch is detectable. Blobs are RAM-only: they expire ' +
'after a short TTL (~1h) and are cleared on restart — consume the URL ' +
'within the TTL and one uptime, or re-stash.',
buildShape: (z) => ({
pageId: z.string().min(1),
}),
},
};

View File

@@ -13,6 +13,10 @@ import { TiptapTransformer } from "@hocuspocus/transformer";
import * as Y from "yjs";
import WebSocket from "ws";
import { convertProseMirrorToMarkdown } from "./lib/markdown-converter.js";
import {
collectInternalFileNodes,
normalizeFileUrl,
} from "./lib/internal-file-urls.js";
import {
updatePageContentRealtime,
replacePageContent,
@@ -102,6 +106,14 @@ const MIME_TO_EXT: Record<string, string> = {
* Housed here (not in index.ts) so client.ts has no type dependency on index.ts;
* index.ts re-exports it for the package's public surface.
*/
// Sink the stash tool writes blobs into. The host app binds this to its in-RAM
// SandboxStore and composes the public `uri` (the package never sees the store
// or any env). `put` returns the anonymous read URL plus integrity metadata.
export type SandboxPut = (
buf: Buffer,
mime: string,
) => { uri: string; sha256: string; size: number };
export type DocmostMcpConfig = { apiUrl: string } & (
| { email: string; password: string }
| { getToken: () => Promise<string> } // returns a BARE JWT; the client adds "Bearer "
@@ -109,6 +121,9 @@ export type DocmostMcpConfig = { apiUrl: string } & (
// Optional collab-token provider (returns a ready collab JWT). Common to
// both branches; see the type doc above.
getCollabToken?: () => Promise<string>;
// Optional blob sandbox sink. Present only where the stash tool is wired;
// when absent, stash_page throws a clear "not configured" error.
sandbox?: { put: SandboxPut };
};
export class DocmostClient {
@@ -126,6 +141,8 @@ export class DocmostClient {
// its token instead of calling POST /auth/collab-token; on a 401/403 it is
// re-invoked once. Used by the internal agent to carry signed provenance.
private getCollabTokenFn: (() => Promise<string>) | null = null;
// Optional blob-sandbox sink for the stash tool. Null when not configured.
private sandboxPut: SandboxPut | null = null;
// In-flight login dedup: when the token expires, the 401 interceptor,
// ensureAuthenticated, getCollabTokenWithReauth and the two multipart retries
// can all call login() at once. Memoizing a single promise collapses that
@@ -165,6 +182,9 @@ export class DocmostClient {
if (config.getCollabToken) {
this.getCollabTokenFn = config.getCollabToken;
}
if (config.sandbox) {
this.sandboxPut = config.sandbox.put;
}
this.client = axios.create({
baseURL: this.apiUrl,
// Default request timeout so a hung connection cannot wedge a per-page
@@ -767,6 +787,120 @@ export class DocmostClient {
};
}
/**
* Fetch an INTERNAL Docmost file (authed loopback) for sandbox mirroring.
* `src` is normalized to `/api/files/<id>/<file>`; `this.client.baseURL`
* already ends in `/api`, so we strip the leading `/api` and request the
* relative path with the client's Authorization header. Returns the raw bytes
* and the response Content-Type (mime), defaulting to octet-stream.
*
* The fetch is size-bounded (hard 64 MiB ceiling) purely to protect memory;
* the authoritative per-blob cap is enforced by the sandbox `put`. The leading
* `/api` strip means this never escapes the Docmost API base.
*/
private async fetchInternalFile(
src: string,
): Promise<{ buffer: Buffer; mime: string }> {
const HARD_CEILING = 64 * 1024 * 1024; // 64 MiB memory guard
const relPath = src.replace(/^\/api/, "");
const response = await this.client.get(relPath, {
responseType: "arraybuffer",
timeout: 30000,
maxContentLength: HARD_CEILING,
maxBodyLength: HARD_CEILING,
});
const buffer = Buffer.from(response.data);
if (buffer.length === 0) {
throw new Error(`Empty file response from "${src}"`);
}
const rawCt = response.headers?.["content-type"];
const mime =
typeof rawCt === "string" && rawCt.length > 0
? rawCt.split(";")[0].trim().toLowerCase()
: "application/octet-stream";
return { buffer, mime };
}
/**
* Stash a page's full content into the in-RAM blob sandbox and return ONLY a
* short anonymous URL — the body never enters the model context (this is the
* whole point: ~30KB+ ProseMirror docs blow the model context if passed as a
* tool argument). Every INTERNAL file/image src (the type-agnostic criterion,
* so drawio/excalidraw/video/file nodes are covered too) is mirrored into the
* sandbox and its `src` rewritten to the sandbox URL, so an external consumer
* can fetch the images anonymously. External http(s) srcs are left untouched.
*
* Blobs live in RAM with a short TTL and are cleared on restart — consume the
* URLs within the TTL and one uptime. A failed image fetch never aborts the
* doc: the original src is kept and the failure counted.
*
* Returns { uri, sha256, size, images:{mirrored, failed} }. `uri` and `sha256`
* are for the document blob; `sha256` is also the blob's ETag (integrity).
*/
async stashPage(pageId: string): Promise<{
uri: string;
sha256: string;
size: number;
images: { mirrored: number; failed: number };
}> {
if (!this.sandboxPut) {
throw new Error(
"stash_page is unavailable: the blob sandbox is not configured on this server",
);
}
await this.ensureAuthenticated();
// Stash the SAME shape get_page_json returns (id/title/.../content), with a
// deep clone so the rewrite never mutates anything shared.
const pageJson = await this.getPageJson(pageId);
const cloned: any = structuredClone(pageJson);
// Group internal-file nodes by normalized src so each unique resource is
// fetched + stored ONCE (dedup), and every node sharing that src points at
// the one sandbox blob.
const bySrc = new Map<string, any[]>();
for (const node of collectInternalFileNodes(cloned.content)) {
const src = normalizeFileUrl(String(node.attrs.src));
const group = bySrc.get(src);
if (group) group.push(node);
else bySrc.set(src, [node]);
}
let mirrored = 0;
let failed = 0;
const MAX_CONCURRENCY = 5;
const groups = [...bySrc.entries()];
for (let i = 0; i < groups.length; i += MAX_CONCURRENCY) {
const batch = groups.slice(i, i + MAX_CONCURRENCY);
await Promise.all(
batch.map(async ([src, nodes]) => {
try {
const { buffer, mime } = await this.fetchInternalFile(src);
// put may throw if the blob exceeds the per-blob/total caps.
const stored = this.sandboxPut!(buffer, mime);
for (const node of nodes) node.attrs.src = stored.uri;
mirrored++;
} catch (err) {
// One bad/oversized image must not abort the document.
failed++;
if (process.env.DEBUG) {
console.error(`stash_page: failed to mirror "${src}":`, err);
}
}
}),
);
}
const docBuf = Buffer.from(JSON.stringify(cloned), "utf8");
const stored = this.sandboxPut(docBuf, "application/json");
return {
uri: stored.uri,
sha256: stored.sha256,
size: stored.size,
images: { mirrored, failed },
};
}
/**
* Compact outline of a page's top-level blocks (no full document body).
* Cheap way to locate sections/tables and grab block ids before drilling in

View File

@@ -408,6 +408,31 @@ registerShared(SHARED_TOOL_SPECS.editPageText, async ({ pageId, edits }) => {
return jsonContent(result);
});
// Tool: stash_page — returns a resource_link (NOT embedded text) so the doc
// body never enters the model context. Registered directly (not via
// registerShared) because that helper only emits text content.
server.registerTool(
SHARED_TOOL_SPECS.stashPage.mcpName,
{
description: SHARED_TOOL_SPECS.stashPage.description,
inputSchema: SHARED_TOOL_SPECS.stashPage.buildShape!(z),
},
async ({ pageId }: { pageId: string }) => {
const result = await docmostClient.stashPage(pageId);
return {
content: [
{
type: "resource_link" as const,
uri: result.uri,
name: "page.json",
mimeType: "application/json",
size: result.size,
},
],
};
},
);
// Tool: patch_node
server.registerTool(
"patch_node",

View File

@@ -0,0 +1,54 @@
// Detection + collection of INTERNAL Docmost file URLs inside a ProseMirror doc.
//
// An internal file URL is a relative path served by Docmost's authenticated
// attachment route (`GET /api/files/:fileId/:fileName`). It is useless to an
// external consumer (relative + needs a Docmost session), so the stash tool
// mirrors every such resource into the blob sandbox and rewrites its `src`.
//
// The criterion is "internal file URL", NOT the node TYPE: image, drawio,
// excalidraw, video and file nodes all carry such a `src`, so a type-agnostic
// walker covers them all. External http(s) srcs (CDNs) are left untouched.
//
// Mirrors editor-ext's isInternalFileUrl / normalizeFileUrl (kept as a local
// dup so the ESM mcp package does not depend on the editor-ext build).
export function isInternalFileUrl(url: unknown): boolean {
if (typeof url !== "string") return false;
const normalized = url.trim();
return (
normalized.startsWith("/api/files/") || normalized.startsWith("/files/")
);
}
/** Normalize a bare `/files/...` src to the canonical `/api/files/...` form. */
export function normalizeFileUrl(src: string): string {
const trimmed = src.trim();
if (trimmed.startsWith("/files/")) return "/api" + trimmed;
return trimmed;
}
/**
* Recursively collect every node whose `attrs.src` is an internal file URL.
* Returns references to the live nodes (so the caller can rewrite `attrs.src`
* in place on its clone). Descends `content` arrays, covering callouts, tables,
* details and any other nested container.
*/
export function collectInternalFileNodes(doc: unknown): any[] {
const out: any[] = [];
const visit = (node: any): void => {
if (!node) return;
if (Array.isArray(node)) {
for (const child of node) visit(child);
return;
}
if (typeof node !== "object") return;
if (node.attrs && isInternalFileUrl(node.attrs.src)) {
out.push(node);
}
if (Array.isArray(node.content)) {
for (const child of node.content) visit(child);
}
};
visit(doc);
return out;
}

View File

@@ -266,4 +266,26 @@ export const SHARED_TOOL_SPECS = {
.describe('List of find/replace operations, applied in order'),
}),
},
// --- hand a large page to an external consumer without bloating context ---
stashPage: {
mcpName: 'stash_page',
inAppKey: 'stashPage',
description:
'Serialize a whole page (the full ProseMirror JSON, as get_page_json ' +
'returns) into an ephemeral in-memory blob and return ONLY a short ' +
'anonymous URL to it — the body NEVER enters the model context, so this ' +
'is the way to hand a large page (or its images) to an external consumer ' +
'without truncation. Every internal file/image attachment is mirrored ' +
'into the same sandbox and its src rewritten to a sandbox URL, so the ' +
'consumer can fetch the images anonymously too; external http(s) images ' +
'are left untouched. Returns { uri, size, sha256, images:{mirrored, ' +
'failed} }. Integrity: the blob is served with ETag = its sha256, so a ' +
'truncated/corrupted fetch is detectable. Blobs are RAM-only: they expire ' +
'after a short TTL (~1h) and are cleared on restart — consume the URL ' +
'within the TTL and one uptime, or re-stash.',
buildShape: (z) => ({
pageId: z.string().min(1),
}),
},
} satisfies Record<string, SharedToolSpec>;

View File

@@ -0,0 +1,183 @@
// Mock-HTTP test for DocmostClient.stashPage: a local http server stands in for
// Docmost so the whole flow stays deterministic and offline. Asserts the tool
// (1) serializes the page into the sandbox and returns ONLY a link (uri + sha256
// + size), never the body; (2) mirrors INTERNAL image srcs into the sandbox and
// rewrites them to the sandbox uri; (3) leaves EXTERNAL http(s) srcs untouched;
// (4) de-duplicates a repeated internal src to a single blob; (5) counts a
// failed image fetch without aborting the document.
import { test, after } from "node:test";
import assert from "node:assert/strict";
import http from "node:http";
import { createHash } from "node:crypto";
import { DocmostClient } from "../../build/client.js";
function readBody(req) {
return new Promise((resolve) => {
let raw = "";
req.on("data", (c) => (raw += c));
req.on("end", () => resolve(raw));
});
}
function startServer(handler) {
return new Promise((resolve) => {
const server = http.createServer(handler);
server.listen(0, "127.0.0.1", () => {
const { port } = server.address();
resolve({ server, baseURL: `http://127.0.0.1:${port}/api` });
});
});
}
const openServers = [];
async function spawn(handler) {
const { server, baseURL } = await startServer(handler);
openServers.push(server);
return baseURL;
}
after(async () => {
await Promise.all(openServers.map((s) => new Promise((r) => s.close(r))));
});
// In-memory sandbox sink mirroring the host binding: store the blob, return a
// uri + sha256 + size. Records every put so the test can inspect what was
// stashed (and verify the doc body never leaves via the return value).
function makeSandbox() {
const puts = [];
return {
puts,
put(buf, mime) {
const sha256 = createHash("sha256").update(buf).digest("hex");
const id = `id-${puts.length}`;
puts.push({ buf, mime, sha256 });
return { uri: `https://sb.test/api/sb/${id}`, sha256, size: buf.length };
},
};
}
const IMAGE_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); // "PNG" header-ish
function pageDoc() {
return {
type: "doc",
content: [
{
type: "image",
attrs: { src: "/api/files/att-1/pic.png", attachmentId: "att-1", width: 100 },
},
// Same internal src again -> must dedup to ONE blob, both rewritten.
{
type: "image",
attrs: { src: "/api/files/att-1/pic.png", attachmentId: "att-1", width: 50 },
},
// External CDN image -> must be left untouched.
{
type: "image",
attrs: { src: "https://cdn.example.com/remote.png" },
},
],
};
}
// Build a client wired to a server that logs in, serves the page, and serves the
// internal file bytes. `fileStatus` lets a test force the file fetch to fail.
async function buildClient(sandbox, { fileStatus = 200 } = {}) {
const baseURL = await spawn(async (req, res) => {
await readBody(req);
if (req.url === "/api/auth/login") {
res.writeHead(200, {
"Content-Type": "application/json",
"Set-Cookie": "authToken=tok; HttpOnly",
});
res.end(JSON.stringify({ token: "tok" }));
return;
}
if (req.url === "/api/pages/info") {
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({ data: { id: "page-1", title: "T", content: pageDoc() } }));
return;
}
if (req.url.startsWith("/api/files/att-1/")) {
if (fileStatus !== 200) {
res.writeHead(fileStatus);
res.end();
return;
}
res.writeHead(200, { "Content-Type": "image/png" });
res.end(IMAGE_BYTES);
return;
}
res.writeHead(404);
res.end();
});
return new DocmostClient({
apiUrl: baseURL,
email: "u@example.com",
password: "pw",
sandbox: { put: (buf, mime) => sandbox.put(buf, mime) },
});
}
test("stashPage stores the doc + mirrors/rewrites internal images, returns only a link", async () => {
const sandbox = makeSandbox();
const client = await buildClient(sandbox);
const result = await client.stashPage("page-1");
// Returns ONLY a link shape — never the document body.
assert.equal(typeof result.uri, "string");
assert.match(result.uri, /^https:\/\/sb\.test\/api\/sb\//);
assert.equal(typeof result.sha256, "string");
assert.equal(typeof result.size, "number");
assert.ok(!("doc" in result) && !("content" in result) && !("body" in result));
assert.deepEqual(result.images, { mirrored: 1, failed: 0 });
// One image blob (dedup) + one doc blob = 2 puts.
assert.equal(sandbox.puts.length, 2);
const imagePut = sandbox.puts[0];
const docPut = sandbox.puts[1];
assert.equal(imagePut.mime, "image/png");
assert.ok(imagePut.buf.equals(IMAGE_BYTES));
assert.equal(docPut.mime, "application/json");
// The returned uri/sha256 are the DOCUMENT blob's.
assert.equal(result.sha256, docPut.sha256);
// Inspect the stashed document: internal srcs rewritten, external untouched.
const stashed = JSON.parse(docPut.buf.toString("utf8"));
const imgs = stashed.content.content.filter((n) => n.type === "image");
assert.equal(imgs[0].attrs.src, "https://sb.test/api/sb/id-0");
assert.equal(imgs[1].attrs.src, "https://sb.test/api/sb/id-0"); // same blob (dedup)
assert.equal(imgs[2].attrs.src, "https://cdn.example.com/remote.png"); // external kept
});
test("stashPage counts a failed image fetch without aborting the document", async () => {
const sandbox = makeSandbox();
const client = await buildClient(sandbox, { fileStatus: 500 });
const result = await client.stashPage("page-1");
assert.deepEqual(result.images, { mirrored: 0, failed: 1 });
// Only the doc blob was stored (image fetch failed).
assert.equal(sandbox.puts.length, 1);
assert.equal(sandbox.puts[0].mime, "application/json");
// The failed internal src is LEFT as-is so nothing is silently dropped.
const stashed = JSON.parse(sandbox.puts[0].buf.toString("utf8"));
const imgs = stashed.content.content.filter((n) => n.type === "image");
assert.equal(imgs[0].attrs.src, "/api/files/att-1/pic.png");
});
test("stashPage throws a clear error when no sandbox is configured", async () => {
const baseURL = await spawn(async (req, res) => {
await readBody(req);
res.writeHead(200, { "Content-Type": "application/json" });
res.end(JSON.stringify({}));
});
const client = new DocmostClient({
apiUrl: baseURL,
email: "u@example.com",
password: "pw",
});
await assert.rejects(() => client.stashPage("page-1"), /not configured/);
});