feat(sandbox): in-RAM blob sandbox for out-of-band page transfer (#243)
Add an ephemeral, process-local blob store so the in-app agent (and the
embedded MCP) can hand a large page document and its images to an external
consumer WITHOUT routing the bytes through the model context or Docmost auth.
- SandboxStore (@Injectable singleton): Map<uuid,{buf,mime,sha256,expiresAt}>
in RAM only. put() picks a per-blob cap by mime (image vs doc), enforces a
total-bytes RAM guard with oldest-first eviction, and stamps a TTL; get()
lazily expires. sha256 computed at put() doubles as the strong ETag. An
unref'd sweep interval clears expired entries and is cleared on destroy.
- GET /api/sb/:uuid anonymous controller: serves raw bytes with Content-Type,
Content-Length and ETag=sha256; 404 on missing/expired/non-UUID (anti-
traversal), 304 on a matching If-None-Match. No tokens, no 401 — the
capability is the unguessable UUID + short TTL + TLS. Auth-exempt the same
way as /api/files/public (no JwtAuthGuard) plus an /api/sb entry in main.ts's
workspace-resolution preHandler so a remote consumer with no workspace host
is not rejected.
- stash_page tool in both layers (MCP resource_link + in-app {uri,size,sha256,
images}). client.stashPage serializes the get_page_json shape, mirrors every
INTERNAL file/image src (type-agnostic, covers drawio/excalidraw/video/file)
into the sandbox under Docmost auth and rewrites src to the sandbox URL;
external http(s) srcs are left untouched; dedup by src; a failed image fetch
is counted, never aborts the doc.
- SANDBOX_PUBLIC_URL / SANDBOX_TTL_MS / SANDBOX_MAX_BYTES /
SANDBOX_MAX_IMAGE_BYTES / SANDBOX_MAX_TOTAL_BYTES wired through the
environment service + validation + .env.example.
- SandboxModule (@Global) provides the shared store to the controller,
McpService and AiChatToolsService (same instance for put and get).
Tests: SandboxStore (round-trip, sha256, TTL lazy + sweep, caps, eviction),
SandboxController (200+ETag+CT+CL, 404 missing/expired/non-UUID, 304), and a
mock-HTTP stashPage test (mirror+rewrite internal, keep external, dedup, failed
image counted, returns only a link). Interoperates with the vvzvlad/habr-mcp
consumer's anonymous-GET + sha256-ETag + resource_link contract.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
16
.env.example
16
.env.example
@@ -124,6 +124,22 @@ MCP_DOCMOST_PASSWORD=
|
||||
# MCP_TOKEN=
|
||||
# MCP_SESSION_IDLE_MS=1800000
|
||||
#
|
||||
# BLOB SANDBOX (stash_page). An in-RAM, process-local store that hands large page
|
||||
# content + images to an external consumer WITHOUT bloating the model context or
|
||||
# requiring Docmost auth. The stash_page tool serializes a page, mirrors its
|
||||
# internal images into the store, and returns ONLY a short anonymous URL; the
|
||||
# consumer fetches blobs via `GET /api/sb/<uuid>` (no token — the capability is
|
||||
# the unguessable UUID + short TTL + TLS). Blobs are RAM-only and cleared on
|
||||
# restart. ETag = the blob's sha256 (integrity check).
|
||||
# SANDBOX_PUBLIC_URL is the base used to build those URLs; it MUST be reachable
|
||||
# by the consumer (do NOT use a loopback address if the consumer is remote).
|
||||
# Defaults to APP_URL when unset.
|
||||
# SANDBOX_PUBLIC_URL=https://docs.example.com
|
||||
# SANDBOX_TTL_MS=3600000
|
||||
# SANDBOX_MAX_BYTES=8388608
|
||||
# SANDBOX_MAX_IMAGE_BYTES=20971520
|
||||
# SANDBOX_MAX_TOTAL_BYTES=134217728
|
||||
#
|
||||
# AI-AGENT ATTRIBUTION (comments/pages written via MCP are badged as "AI"):
|
||||
# attribution is driven by a per-user `is_agent` flag on the users row. There is
|
||||
# NO admin UI/API for it — set it out-of-band with SQL. Use a DEDICATED service
|
||||
|
||||
@@ -28,6 +28,7 @@ import { ClsModule } from 'nestjs-cls';
|
||||
import { NoopAuditModule } from './integrations/audit/audit.module';
|
||||
import { ThrottleModule } from './integrations/throttle/throttle.module';
|
||||
import { McpModule } from './integrations/mcp/mcp.module';
|
||||
import { SandboxModule } from './integrations/sandbox/sandbox.module';
|
||||
import { AiModule } from './integrations/ai/ai.module';
|
||||
import { AiChatModule } from './core/ai-chat/ai-chat.module';
|
||||
|
||||
@@ -89,6 +90,7 @@ try {
|
||||
TelemetryModule,
|
||||
ThrottleModule,
|
||||
McpModule,
|
||||
SandboxModule,
|
||||
AiModule,
|
||||
AiChatModule,
|
||||
...enterpriseModules,
|
||||
|
||||
@@ -63,6 +63,10 @@ describe('AiChatToolsService deletePage guardrail (H4)', () => {
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
// environmentService + sandboxStore (only used by the stash tool closure,
|
||||
// which these tests do not execute).
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -175,6 +179,10 @@ describe('AiChatToolsService expanded toolset guardrails', () => {
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
// environmentService + sandboxStore (only used by the stash tool closure,
|
||||
// which these tests do not execute).
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -290,6 +298,10 @@ describe('AiChatToolsService node-arg JSON-string coercion', () => {
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
// environmentService + sandboxStore (only used by the stash tool closure,
|
||||
// which these tests do not execute).
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
});
|
||||
|
||||
@@ -440,6 +452,10 @@ describe('AiChatToolsService model-friendly input validation (#190)', () => {
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
// environmentService + sandboxStore (only used by the stash tool closure,
|
||||
// which these tests do not execute).
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
@@ -16,6 +16,8 @@ import {
|
||||
import { resolveCurrentPageResult } from './current-page.util';
|
||||
import { parseNodeArg } from './parse-node-arg';
|
||||
import { modelFriendlyInput } from './model-friendly-input';
|
||||
import { EnvironmentService } from '../../../integrations/environment/environment.service';
|
||||
import { SandboxStore } from '../../../integrations/sandbox/sandbox.store';
|
||||
|
||||
/**
|
||||
* Per-user, per-request adapter that exposes Docmost READ operations to the
|
||||
@@ -41,6 +43,9 @@ export class AiChatToolsService {
|
||||
private readonly pageEmbeddingRepo: PageEmbeddingRepo,
|
||||
private readonly spaceMemberRepo: SpaceMemberRepo,
|
||||
private readonly pagePermissionRepo: PagePermissionRepo,
|
||||
private readonly environmentService: EnvironmentService,
|
||||
// Shared singleton in-RAM blob store backing the stash tool.
|
||||
private readonly sandboxStore: SandboxStore,
|
||||
) {}
|
||||
|
||||
async forUser(
|
||||
@@ -86,11 +91,22 @@ export class AiChatToolsService {
|
||||
aiChatId,
|
||||
});
|
||||
|
||||
// Bind the stash tool to the shared in-RAM SandboxStore and compose the
|
||||
// anonymous public URL here (the MCP package never touches env or the
|
||||
// store). put() returns the read URL + sha256/size; sha256 is also the
|
||||
// blob's ETag for integrity.
|
||||
const sandboxPut = (buf: Buffer, mime: string) => {
|
||||
const stored = this.sandboxStore.put(buf, mime);
|
||||
const base = this.environmentService.getSandboxPublicUrl();
|
||||
return { uri: `${base}/api/sb/${stored.id}`, sha256: stored.sha256, size: stored.size };
|
||||
};
|
||||
|
||||
const { DocmostClient, sharedToolSpecs } = await loadDocmostMcp();
|
||||
const client: DocmostClientLike = new DocmostClient({
|
||||
apiUrl,
|
||||
getToken,
|
||||
getCollabToken,
|
||||
sandbox: { put: sandboxPut },
|
||||
});
|
||||
|
||||
// Build an ai-SDK tool from a shared, zod-agnostic spec. The spec owns the
|
||||
@@ -625,6 +641,14 @@ export class AiChatToolsService {
|
||||
async ({ pageId, edits }) => await client.editPageText(pageId, edits),
|
||||
),
|
||||
|
||||
// Returns ONLY the short link object — never the document body — so a
|
||||
// large page can be handed to an external consumer without bloating
|
||||
// context.
|
||||
stashPage: sharedTool(
|
||||
sharedToolSpecs.stashPage,
|
||||
async ({ pageId }) => await client.stashPage(pageId),
|
||||
),
|
||||
|
||||
patchNode: tool({
|
||||
description:
|
||||
'Replace a single content block (by id) with a new ProseMirror ' +
|
||||
|
||||
@@ -154,6 +154,14 @@ export interface DocmostClientLike {
|
||||
commentId: string,
|
||||
resolved: boolean,
|
||||
): Promise<Record<string, unknown>>;
|
||||
// Serialize a page + mirror its internal images into the blob sandbox; returns
|
||||
// ONLY a short anonymous URL (the body never enters the model context).
|
||||
stashPage(pageId: string): Promise<{
|
||||
uri: string;
|
||||
sha256: string;
|
||||
size: number;
|
||||
images: { mirrored: number; failed: number };
|
||||
}>;
|
||||
}
|
||||
|
||||
export type DocmostClientConfig = {
|
||||
@@ -161,6 +169,14 @@ export type DocmostClientConfig = {
|
||||
getToken: () => Promise<string>;
|
||||
// Provenance collab-token provider for content mutations (signed agent claim).
|
||||
getCollabToken?: () => Promise<string>;
|
||||
// Optional blob-sandbox sink for the stash tool. `put` stores a blob in the
|
||||
// host's in-RAM SandboxStore and returns the anonymous read URL + integrity.
|
||||
sandbox?: {
|
||||
put: (
|
||||
buf: Buffer,
|
||||
mime: string,
|
||||
) => { uri: string; sha256: string; size: number };
|
||||
};
|
||||
};
|
||||
|
||||
export interface DocmostClientCtor {
|
||||
|
||||
@@ -332,4 +332,52 @@ export class EnvironmentService {
|
||||
.map((o) => o.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
// --- Blob sandbox (in-RAM ephemeral blob transfer; see SandboxModule) ---
|
||||
|
||||
// Base URL the sandbox `uri` is built from. It MUST be reachable over the
|
||||
// network by the external consumer that fetches the blobs (not a loopback
|
||||
// address if that consumer is remote). Falls back to APP_URL when unset so a
|
||||
// single-host deployment works out of the box; set it explicitly when the
|
||||
// consumer lives on another host.
|
||||
getSandboxPublicUrl(): string {
|
||||
const raw =
|
||||
this.configService.get<string>('SANDBOX_PUBLIC_URL') || this.getAppUrl();
|
||||
// Drop any trailing slash so `${base}/api/sb/${id}` never doubles up.
|
||||
return raw.replace(/\/+$/, '');
|
||||
}
|
||||
|
||||
// Blob time-to-live. Default 1h. The unguessable UUID + this short TTL + TLS
|
||||
// are the whole capability model (no tokens).
|
||||
getSandboxTtlMs(): number {
|
||||
return parseInt(
|
||||
this.configService.get<string>('SANDBOX_TTL_MS', '3600000'),
|
||||
10,
|
||||
);
|
||||
}
|
||||
|
||||
// Per-blob cap for non-image blobs (the serialized document). Default 8 MiB.
|
||||
getSandboxMaxBytes(): number {
|
||||
return parseInt(
|
||||
this.configService.get<string>('SANDBOX_MAX_BYTES', '8388608'),
|
||||
10,
|
||||
);
|
||||
}
|
||||
|
||||
// Per-blob cap for mirrored image blobs. Default 20 MiB.
|
||||
getSandboxMaxImageBytes(): number {
|
||||
return parseInt(
|
||||
this.configService.get<string>('SANDBOX_MAX_IMAGE_BYTES', '20971520'),
|
||||
10,
|
||||
);
|
||||
}
|
||||
|
||||
// RAM guard: total bytes the whole store may hold. Default 128 MiB. On
|
||||
// overflow the store evicts oldest entries to make room.
|
||||
getSandboxMaxTotalBytes(): number {
|
||||
return parseInt(
|
||||
this.configService.get<string>('SANDBOX_MAX_TOTAL_BYTES', '134217728'),
|
||||
10,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import {
|
||||
IsIn,
|
||||
IsNotEmpty,
|
||||
IsNotIn,
|
||||
IsNumberString,
|
||||
IsOptional,
|
||||
IsString,
|
||||
IsUrl,
|
||||
@@ -170,6 +171,35 @@ export class EnvironmentVariables {
|
||||
},
|
||||
)
|
||||
CLICKHOUSE_URL: string;
|
||||
|
||||
// --- Blob sandbox (in-RAM ephemeral blob transfer; see SandboxModule) ---
|
||||
|
||||
@IsOptional()
|
||||
@ValidateIf((obj) => obj.SANDBOX_PUBLIC_URL != '' && obj.SANDBOX_PUBLIC_URL != null)
|
||||
@IsUrl(
|
||||
{ protocols: ['http', 'https'], require_tld: false },
|
||||
{
|
||||
message:
|
||||
'SANDBOX_PUBLIC_URL must be a valid http(s) URL reachable by the external blob consumer',
|
||||
},
|
||||
)
|
||||
SANDBOX_PUBLIC_URL: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsNumberString({}, { message: 'SANDBOX_TTL_MS must be an integer (milliseconds)' })
|
||||
SANDBOX_TTL_MS: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsNumberString({}, { message: 'SANDBOX_MAX_BYTES must be an integer (bytes)' })
|
||||
SANDBOX_MAX_BYTES: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsNumberString({}, { message: 'SANDBOX_MAX_IMAGE_BYTES must be an integer (bytes)' })
|
||||
SANDBOX_MAX_IMAGE_BYTES: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsNumberString({}, { message: 'SANDBOX_MAX_TOTAL_BYTES must be an integer (bytes)' })
|
||||
SANDBOX_MAX_TOTAL_BYTES: string;
|
||||
}
|
||||
|
||||
export function validate(config: Record<string, any>) {
|
||||
|
||||
@@ -131,10 +131,20 @@ export class FailedLoginLimiter {
|
||||
}
|
||||
|
||||
// The per-session DocmostMcpConfig shape understood by @docmost/mcp: either the
|
||||
// service-account credentials variant OR the per-user getToken variant.
|
||||
export type DocmostMcpConfig =
|
||||
// service-account credentials variant OR the per-user getToken variant. The
|
||||
// optional `sandbox` sink (blob store for the stash tool) is common to both and
|
||||
// injected by McpService after the auth decision.
|
||||
export type DocmostMcpConfig = (
|
||||
| { apiUrl: string; email: string; password: string }
|
||||
| { apiUrl: string; getToken: () => Promise<string> };
|
||||
| { apiUrl: string; getToken: () => Promise<string> }
|
||||
) & {
|
||||
sandbox?: {
|
||||
put: (
|
||||
buf: Buffer,
|
||||
mime: string,
|
||||
) => { uri: string; sha256: string; size: number };
|
||||
};
|
||||
};
|
||||
|
||||
export interface ResolvedMcpAuth {
|
||||
config: DocmostMcpConfig;
|
||||
|
||||
@@ -116,6 +116,7 @@ function makeService(opts: {
|
||||
undefined as never, // userRepo
|
||||
undefined as never, // userSessionRepo
|
||||
moduleRef as never, // moduleRef (read by the MFA branch)
|
||||
undefined as never, // sandboxStore (unused by the login-gate path)
|
||||
);
|
||||
// Stop the constructor's unref'd sweep timer leaking across tests.
|
||||
service.onModuleDestroy();
|
||||
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
DocmostMcpConfig,
|
||||
ResolvedMcpAuth,
|
||||
} from './mcp-auth.helpers';
|
||||
import { SandboxStore } from '../sandbox/sandbox.store';
|
||||
|
||||
// Minimal shape of the embedded MCP HTTP handler exported by @docmost/mcp/http.
|
||||
interface McpHttpHandler {
|
||||
@@ -99,6 +100,8 @@ export class McpService implements OnModuleDestroy {
|
||||
private readonly userRepo: UserRepo,
|
||||
private readonly userSessionRepo: UserSessionRepo,
|
||||
private readonly moduleRef: ModuleRef,
|
||||
// Shared singleton in-RAM blob store backing the stash tool.
|
||||
private readonly sandboxStore: SandboxStore,
|
||||
) {
|
||||
this.sweepTimer = setInterval(() => {
|
||||
try {
|
||||
@@ -115,6 +118,23 @@ export class McpService implements OnModuleDestroy {
|
||||
clearInterval(this.sweepTimer);
|
||||
}
|
||||
|
||||
// Bind the stash tool to the shared in-RAM SandboxStore and compose the
|
||||
// anonymous public URL (the MCP package owns neither env nor the store).
|
||||
// put() returns the read URL + sha256/size; sha256 is also the blob ETag.
|
||||
private buildSandboxConfig(): DocmostMcpConfig['sandbox'] {
|
||||
return {
|
||||
put: (buf: Buffer, mime: string) => {
|
||||
const stored = this.sandboxStore.put(buf, mime);
|
||||
const base = this.environmentService.getSandboxPublicUrl();
|
||||
return {
|
||||
uri: `${base}/api/sb/${stored.id}`,
|
||||
sha256: stored.sha256,
|
||||
size: stored.size,
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Service account the embedded MCP uses to talk back to this Docmost
|
||||
// instance over loopback REST + the collaboration WebSocket. Now OPTIONAL:
|
||||
// it is only a fallback when no per-user Basic/Bearer credentials are sent.
|
||||
@@ -326,7 +346,10 @@ export class McpService implements OnModuleDestroy {
|
||||
// Should never happen: handle() always stashes before delegating.
|
||||
throw new UnauthorizedException('MCP authentication missing.');
|
||||
}
|
||||
return resolved.config;
|
||||
// Inject the blob-sandbox sink after the auth decision so stash_page
|
||||
// can store blobs in the shared in-RAM store regardless of which
|
||||
// credential variant resolved.
|
||||
return { ...resolved.config, sandbox: this.buildSandboxConfig() };
|
||||
},
|
||||
{
|
||||
identify: (req: IncomingMessage) => {
|
||||
|
||||
124
apps/server/src/integrations/sandbox/sandbox.controller.spec.ts
Normal file
124
apps/server/src/integrations/sandbox/sandbox.controller.spec.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
import { SandboxController } from './sandbox.controller';
|
||||
import { SandboxEntry } from './sandbox.store';
|
||||
|
||||
// Capturing fake of the FastifyReply surface the controller uses:
|
||||
// status()/header()/headers()/send(), all chainable.
|
||||
function makeRes() {
|
||||
const sent: { status: number; headers: Record<string, any>; body: any } = {
|
||||
status: 200,
|
||||
headers: {},
|
||||
body: undefined,
|
||||
};
|
||||
const res: any = {
|
||||
status(code: number) {
|
||||
sent.status = code;
|
||||
return res;
|
||||
},
|
||||
header(key: string, value: any) {
|
||||
sent.headers[key.toLowerCase()] = value;
|
||||
return res;
|
||||
},
|
||||
headers(obj: Record<string, any>) {
|
||||
for (const k of Object.keys(obj)) sent.headers[k.toLowerCase()] = obj[k];
|
||||
return res;
|
||||
},
|
||||
send(body?: any) {
|
||||
sent.body = body;
|
||||
return res;
|
||||
},
|
||||
_sent: sent,
|
||||
};
|
||||
return res;
|
||||
}
|
||||
|
||||
function makeReq(headers: Record<string, any> = {}) {
|
||||
return { headers } as any;
|
||||
}
|
||||
|
||||
const VALID_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
|
||||
|
||||
function entry(buf: Buffer, mime: string, sha256: string): SandboxEntry {
|
||||
return { buf, mime, sha256, expiresAt: Date.now() + 60_000 };
|
||||
}
|
||||
|
||||
describe('SandboxController', () => {
|
||||
it('serves 200 with body, Content-Type, Content-Length and sha256 ETag', async () => {
|
||||
const buf = Buffer.from('{"ok":true}', 'utf8');
|
||||
const sha = 'a'.repeat(64);
|
||||
const store = { get: jest.fn().mockReturnValue(entry(buf, 'application/json', sha)) };
|
||||
const controller = new SandboxController(store as any);
|
||||
const res = makeRes();
|
||||
|
||||
await controller.get(VALID_ID, makeReq(), res);
|
||||
|
||||
expect(store.get).toHaveBeenCalledWith(VALID_ID);
|
||||
expect(res._sent.status).toBe(200);
|
||||
expect(res._sent.headers['content-type']).toBe('application/json');
|
||||
expect(res._sent.headers['content-length']).toBe(buf.length);
|
||||
expect(res._sent.headers['etag']).toBe(`"${sha}"`);
|
||||
expect(res._sent.body).toBe(buf);
|
||||
});
|
||||
|
||||
it('returns 404 for a missing/expired blob', async () => {
|
||||
const store = { get: jest.fn().mockReturnValue(undefined) };
|
||||
const controller = new SandboxController(store as any);
|
||||
const res = makeRes();
|
||||
|
||||
await controller.get(VALID_ID, makeReq(), res);
|
||||
|
||||
expect(res._sent.status).toBe(404);
|
||||
expect(res._sent.body).toBeUndefined();
|
||||
});
|
||||
|
||||
it('returns 404 for a non-UUID id WITHOUT touching the store (anti-traversal)', async () => {
|
||||
const store = { get: jest.fn() };
|
||||
const controller = new SandboxController(store as any);
|
||||
const res = makeRes();
|
||||
|
||||
await controller.get('../../etc/passwd', makeReq(), res);
|
||||
|
||||
expect(store.get).not.toHaveBeenCalled();
|
||||
expect(res._sent.status).toBe(404);
|
||||
});
|
||||
|
||||
it('returns 304 (no body) when If-None-Match matches the ETag', async () => {
|
||||
const sha = 'b'.repeat(64);
|
||||
const store = {
|
||||
get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
|
||||
};
|
||||
const controller = new SandboxController(store as any);
|
||||
const res = makeRes();
|
||||
|
||||
await controller.get(VALID_ID, makeReq({ 'if-none-match': `"${sha}"` }), res);
|
||||
|
||||
expect(res._sent.status).toBe(304);
|
||||
expect(res._sent.body).toBeUndefined();
|
||||
expect(res._sent.headers['etag']).toBe(`"${sha}"`);
|
||||
});
|
||||
|
||||
it('accepts a bare (unquoted) sha256 in If-None-Match too', async () => {
|
||||
const sha = 'c'.repeat(64);
|
||||
const store = {
|
||||
get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
|
||||
};
|
||||
const controller = new SandboxController(store as any);
|
||||
const res = makeRes();
|
||||
|
||||
await controller.get(VALID_ID, makeReq({ 'if-none-match': sha }), res);
|
||||
|
||||
expect(res._sent.status).toBe(304);
|
||||
});
|
||||
|
||||
it('serves 200 when If-None-Match does NOT match', async () => {
|
||||
const sha = 'd'.repeat(64);
|
||||
const store = {
|
||||
get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
|
||||
};
|
||||
const controller = new SandboxController(store as any);
|
||||
const res = makeRes();
|
||||
|
||||
await controller.get(VALID_ID, makeReq({ 'if-none-match': '"stale"' }), res);
|
||||
|
||||
expect(res._sent.status).toBe(200);
|
||||
});
|
||||
});
|
||||
94
apps/server/src/integrations/sandbox/sandbox.controller.ts
Normal file
94
apps/server/src/integrations/sandbox/sandbox.controller.ts
Normal file
@@ -0,0 +1,94 @@
|
||||
import { Controller, Get, Param, Req, Res } from '@nestjs/common';
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { SandboxStore } from './sandbox.store';
|
||||
|
||||
// Strict UUID v-agnostic shape. This is anti-traversal / input hygiene (so `:id`
|
||||
// can never be a path like `../...`), NOT authorization — the capability is the
|
||||
// unguessable id itself plus the short TTL plus TLS.
|
||||
const UUID_RE =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
/**
|
||||
* Anonymous read endpoint for the in-RAM blob sandbox.
|
||||
*
|
||||
* Mounted under the global `/api` prefix as `GET /api/sb/:id`. It carries NO
|
||||
* `@UseGuards(JwtAuthGuard)`, so — exactly like the public attachment route
|
||||
* `GET /api/files/public/...` — it is exempt from Docmost session auth. The
|
||||
* route is ALSO listed in the workspace-resolution preHandler's excludedPaths
|
||||
* in main.ts so a request from a remote consumer (which carries no workspace
|
||||
* host) is not rejected with "Workspace not found".
|
||||
*
|
||||
* It only ever serves blobs looked up from the SandboxStore by a validated
|
||||
* UUID; `:id` is never used as a filesystem path, so there is no traversal
|
||||
* surface. Never returns tokens, never 401s.
|
||||
*/
|
||||
@Controller('sb')
|
||||
export class SandboxController {
|
||||
constructor(private readonly store: SandboxStore) {}
|
||||
|
||||
@Get(':id')
|
||||
async get(
|
||||
@Param('id') id: string,
|
||||
@Req() req: FastifyRequest,
|
||||
@Res() res: FastifyReply,
|
||||
): Promise<void> {
|
||||
// Non-UUID id (including any traversal attempt) → 404 before touching the
|
||||
// store. No stack trace leaks out.
|
||||
if (!UUID_RE.test(id)) {
|
||||
res.status(404).send();
|
||||
return;
|
||||
}
|
||||
|
||||
const entry = this.store.get(id);
|
||||
if (!entry) {
|
||||
// Missing or expired — indistinguishable to the caller, by design.
|
||||
res.status(404).send();
|
||||
return;
|
||||
}
|
||||
|
||||
// Strong validator: quoted sha256, no W/ weak prefix. Same value computed
|
||||
// at put() time, so an external consumer can detect a truncated/corrupted
|
||||
// body — the original bug this whole channel exists to fix.
|
||||
const etag = `"${entry.sha256}"`;
|
||||
|
||||
// Conditional request: an exact ETag match → 304 with no body. The blob is
|
||||
// immutable, so the validator is stable for the blob's whole lifetime.
|
||||
if (this.ifNoneMatchMatches(req.headers['if-none-match'], entry.sha256)) {
|
||||
res.status(304).header('ETag', etag).send();
|
||||
return;
|
||||
}
|
||||
|
||||
const ttlSeconds = Math.max(
|
||||
0,
|
||||
Math.floor((entry.expiresAt - Date.now()) / 1000),
|
||||
);
|
||||
|
||||
// Use @Res() + res.send(Buffer) with an explicit Content-Type so the binary
|
||||
// body bypasses the global JSON response transform/serializer.
|
||||
res
|
||||
.status(200)
|
||||
.headers({
|
||||
'Content-Type': entry.mime,
|
||||
'Content-Length': entry.buf.length,
|
||||
ETag: etag,
|
||||
// Capability URL — keep it out of shared caches; immutable for its TTL.
|
||||
'Cache-Control': `private, max-age=${ttlSeconds}, immutable`,
|
||||
})
|
||||
.send(entry.buf);
|
||||
}
|
||||
|
||||
// Accept the consumer's If-None-Match whether it sends the quoted ETag, a bare
|
||||
// sha256, a weak "W/"-prefixed validator, or a comma-separated list.
|
||||
private ifNoneMatchMatches(
|
||||
header: string | string[] | undefined,
|
||||
sha256: string,
|
||||
): boolean {
|
||||
if (!header) return false;
|
||||
const raw = Array.isArray(header) ? header.join(',') : header;
|
||||
if (raw.trim() === '*') return true;
|
||||
return raw
|
||||
.split(',')
|
||||
.map((t) => t.trim().replace(/^W\//, '').replace(/^"|"$/g, ''))
|
||||
.some((t) => t === sha256);
|
||||
}
|
||||
}
|
||||
19
apps/server/src/integrations/sandbox/sandbox.module.ts
Normal file
19
apps/server/src/integrations/sandbox/sandbox.module.ts
Normal file
@@ -0,0 +1,19 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { SandboxController } from './sandbox.controller';
|
||||
import { SandboxStore } from './sandbox.store';
|
||||
|
||||
/**
|
||||
* In-RAM blob sandbox: a SINGLE shared SandboxStore (the @Injectable singleton)
|
||||
* is written to by the stash tool (via McpService / AiChatToolsService) and read
|
||||
* back by the anonymous SandboxController. Marked @Global so the same store
|
||||
* instance is injectable everywhere without import churn — put() and get() MUST
|
||||
* hit the same Map. EnvironmentService (caps/TTL/public URL) is provided by the
|
||||
* global EnvironmentModule.
|
||||
*/
|
||||
@Global()
|
||||
@Module({
|
||||
controllers: [SandboxController],
|
||||
providers: [SandboxStore],
|
||||
exports: [SandboxStore],
|
||||
})
|
||||
export class SandboxModule {}
|
||||
133
apps/server/src/integrations/sandbox/sandbox.store.spec.ts
Normal file
133
apps/server/src/integrations/sandbox/sandbox.store.spec.ts
Normal file
@@ -0,0 +1,133 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { SandboxStore } from './sandbox.store';
|
||||
|
||||
// Build a minimal EnvironmentService stub with overridable caps/TTL.
|
||||
function makeEnv(
|
||||
overrides: Partial<{
|
||||
ttlMs: number;
|
||||
maxBytes: number;
|
||||
maxImageBytes: number;
|
||||
maxTotalBytes: number;
|
||||
}> = {},
|
||||
) {
|
||||
const cfg = {
|
||||
ttlMs: 3_600_000,
|
||||
maxBytes: 8_388_608,
|
||||
maxImageBytes: 20_971_520,
|
||||
maxTotalBytes: 134_217_728,
|
||||
...overrides,
|
||||
};
|
||||
return {
|
||||
getSandboxTtlMs: () => cfg.ttlMs,
|
||||
getSandboxMaxBytes: () => cfg.maxBytes,
|
||||
getSandboxMaxImageBytes: () => cfg.maxImageBytes,
|
||||
getSandboxMaxTotalBytes: () => cfg.maxTotalBytes,
|
||||
getSandboxPublicUrl: () => 'https://example.test',
|
||||
} as any;
|
||||
}
|
||||
|
||||
const UUID_RE =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
||||
|
||||
describe('SandboxStore', () => {
|
||||
let store: SandboxStore;
|
||||
|
||||
afterEach(() => {
|
||||
// Clear the unref'd sweep interval so it never leaks across tests.
|
||||
store?.onModuleDestroy();
|
||||
jest.useRealTimers();
|
||||
});
|
||||
|
||||
it('put/get round-trips the exact bytes + mime and returns a UUID id', () => {
|
||||
store = new SandboxStore(makeEnv());
|
||||
const buf = Buffer.from('{"type":"doc","content":[]}', 'utf8');
|
||||
|
||||
const res = store.put(buf, 'application/json');
|
||||
expect(res.id).toMatch(UUID_RE);
|
||||
expect(res.size).toBe(buf.length);
|
||||
|
||||
const entry = store.get(res.id);
|
||||
expect(entry).toBeDefined();
|
||||
expect(entry!.buf.equals(buf)).toBe(true);
|
||||
expect(entry!.mime).toBe('application/json');
|
||||
});
|
||||
|
||||
it('computes sha256 over the body (matches a manual digest)', () => {
|
||||
store = new SandboxStore(makeEnv());
|
||||
const buf = Buffer.from('hello sandbox', 'utf8');
|
||||
const expected = createHash('sha256').update(buf).digest('hex');
|
||||
|
||||
const res = store.put(buf, 'text/plain');
|
||||
expect(res.sha256).toBe(expected);
|
||||
expect(store.get(res.id)!.sha256).toBe(expected);
|
||||
});
|
||||
|
||||
it('returns undefined for a missing id', () => {
|
||||
store = new SandboxStore(makeEnv());
|
||||
expect(store.get('11111111-1111-1111-1111-111111111111')).toBeUndefined();
|
||||
});
|
||||
|
||||
it('lazily expires entries past the TTL (get returns undefined)', () => {
|
||||
jest.useFakeTimers();
|
||||
jest.setSystemTime(new Date('2026-01-01T00:00:00Z'));
|
||||
store = new SandboxStore(makeEnv({ ttlMs: 1000 }));
|
||||
const res = store.put(Buffer.from('x'), 'text/plain');
|
||||
|
||||
expect(store.get(res.id)).toBeDefined();
|
||||
jest.setSystemTime(new Date('2026-01-01T00:00:02Z')); // +2s > 1s TTL
|
||||
expect(store.get(res.id)).toBeUndefined();
|
||||
// Eviction also frees the byte accounting.
|
||||
expect(store.bytes).toBe(0);
|
||||
});
|
||||
|
||||
it('background sweep drops expired entries without a get()', () => {
|
||||
jest.useFakeTimers();
|
||||
jest.setSystemTime(new Date('2026-01-01T00:00:00Z'));
|
||||
store = new SandboxStore(makeEnv({ ttlMs: 1000 }));
|
||||
store.put(Buffer.from('x'), 'text/plain');
|
||||
expect(store.size).toBe(1);
|
||||
|
||||
jest.setSystemTime(new Date('2026-01-01T00:01:30Z')); // past TTL
|
||||
jest.advanceTimersByTime(60_000); // fire the sweep interval
|
||||
expect(store.size).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects a non-image blob over SANDBOX_MAX_BYTES', () => {
|
||||
store = new SandboxStore(makeEnv({ maxBytes: 16 }));
|
||||
expect(() => store.put(Buffer.alloc(17), 'application/json')).toThrow(
|
||||
/per-blob cap/,
|
||||
);
|
||||
});
|
||||
|
||||
it('uses the larger image cap for image/* blobs', () => {
|
||||
// 100 bytes exceeds the doc cap (16) but fits the image cap (1024).
|
||||
store = new SandboxStore(makeEnv({ maxBytes: 16, maxImageBytes: 1024 }));
|
||||
expect(() => store.put(Buffer.alloc(100), 'image/png')).not.toThrow();
|
||||
// SVG counts as an image too.
|
||||
expect(() => store.put(Buffer.alloc(100), 'image/svg+xml')).not.toThrow();
|
||||
});
|
||||
|
||||
it('evicts oldest entries when the total cap would be exceeded', () => {
|
||||
// Total cap 250 bytes; each blob 100 bytes -> only 2 fit at a time.
|
||||
store = new SandboxStore(
|
||||
makeEnv({ maxTotalBytes: 250, maxBytes: 1024 }),
|
||||
);
|
||||
const a = store.put(Buffer.alloc(100), 'application/json');
|
||||
const b = store.put(Buffer.alloc(100), 'application/json');
|
||||
const c = store.put(Buffer.alloc(100), 'application/json'); // evicts a
|
||||
|
||||
expect(store.get(a.id)).toBeUndefined(); // oldest evicted
|
||||
expect(store.get(b.id)).toBeDefined();
|
||||
expect(store.get(c.id)).toBeDefined();
|
||||
expect(store.bytes).toBeLessThanOrEqual(250);
|
||||
});
|
||||
|
||||
it('rejects a single blob larger than the whole total cap', () => {
|
||||
store = new SandboxStore(
|
||||
makeEnv({ maxTotalBytes: 50, maxBytes: 1024 }),
|
||||
);
|
||||
expect(() => store.put(Buffer.alloc(100), 'application/json')).toThrow(
|
||||
/total store cap/,
|
||||
);
|
||||
});
|
||||
});
|
||||
129
apps/server/src/integrations/sandbox/sandbox.store.ts
Normal file
129
apps/server/src/integrations/sandbox/sandbox.store.ts
Normal file
@@ -0,0 +1,129 @@
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { EnvironmentService } from '../environment/environment.service';
|
||||
|
||||
// In-RAM, process-local blob store. No disk, no DB. Ephemeral by design: a
|
||||
// restart empties it. A blob is addressed by an unguessable randomUUID() which
|
||||
// IS the read capability — there are NO tokens. Each blob is immutable (its id
|
||||
// never maps to changing content), so its sha256 is a perfect strong ETag.
|
||||
export interface SandboxEntry {
|
||||
buf: Buffer;
|
||||
mime: string;
|
||||
sha256: string;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
export interface SandboxPutResult {
|
||||
id: string;
|
||||
sha256: string;
|
||||
size: number;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class SandboxStore implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(SandboxStore.name);
|
||||
// Map preserves insertion order, so the first key is the oldest entry — used
|
||||
// for FIFO eviction when the total-bytes RAM guard is exceeded.
|
||||
private readonly map = new Map<string, SandboxEntry>();
|
||||
private totalBytes = 0;
|
||||
|
||||
// Background sweep clears expired entries so never-fetched blobs do not linger
|
||||
// until the next get(). unref()'d so it never holds the event loop open;
|
||||
// cleared on module destroy. Mirrors the sweepTimer pattern in
|
||||
// integrations/mcp/mcp.service.ts and packages/mcp/src/http.ts.
|
||||
private readonly sweepIntervalMs = 60_000;
|
||||
private readonly sweepTimer: NodeJS.Timeout;
|
||||
|
||||
constructor(private readonly environmentService: EnvironmentService) {
|
||||
this.sweepTimer = setInterval(() => {
|
||||
try {
|
||||
this.sweep();
|
||||
} catch (err) {
|
||||
this.logger.error('Sandbox sweep failed', err as Error);
|
||||
}
|
||||
}, this.sweepIntervalMs);
|
||||
this.sweepTimer.unref?.();
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
clearInterval(this.sweepTimer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a blob and return its read capability id + integrity metadata. The
|
||||
* per-blob cap is chosen by mime (images get the larger image cap), and the
|
||||
* total-store RAM guard evicts oldest entries to make room. Throws a clear
|
||||
* error when a single blob cannot fit even after eviction. Blob bodies are
|
||||
* never logged.
|
||||
*/
|
||||
put(buf: Buffer, mime: string): SandboxPutResult {
|
||||
const perBlobCap = mime.startsWith('image/')
|
||||
? this.environmentService.getSandboxMaxImageBytes()
|
||||
: this.environmentService.getSandboxMaxBytes();
|
||||
if (buf.length > perBlobCap) {
|
||||
throw new Error(
|
||||
`Sandbox blob of ${buf.length} bytes exceeds the ${perBlobCap}-byte per-blob cap`,
|
||||
);
|
||||
}
|
||||
|
||||
const maxTotal = this.environmentService.getSandboxMaxTotalBytes();
|
||||
if (buf.length > maxTotal) {
|
||||
throw new Error(
|
||||
`Sandbox blob of ${buf.length} bytes exceeds the total store cap of ${maxTotal} bytes`,
|
||||
);
|
||||
}
|
||||
|
||||
// Drop expired entries first, then evict oldest until the new blob fits.
|
||||
this.sweep();
|
||||
while (this.totalBytes + buf.length > maxTotal && this.map.size > 0) {
|
||||
const oldest = this.map.keys().next().value as string;
|
||||
this.evict(oldest);
|
||||
}
|
||||
|
||||
const id = randomUUID();
|
||||
const sha256 = createHash('sha256').update(buf).digest('hex');
|
||||
const expiresAt = Date.now() + this.environmentService.getSandboxTtlMs();
|
||||
this.map.set(id, { buf, mime, sha256, expiresAt });
|
||||
this.totalBytes += buf.length;
|
||||
return { id, sha256, size: buf.length, expiresAt };
|
||||
}
|
||||
|
||||
/** Returns the entry, or undefined if missing OR expired (lazy expiry). */
|
||||
get(id: string): SandboxEntry | undefined {
|
||||
const entry = this.map.get(id);
|
||||
if (!entry) return undefined;
|
||||
if (entry.expiresAt <= Date.now()) {
|
||||
this.evict(id);
|
||||
return undefined;
|
||||
}
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Current number of live entries (test/diagnostic helper). */
|
||||
get size(): number {
|
||||
return this.map.size;
|
||||
}
|
||||
|
||||
/** Current total bytes held (test/diagnostic helper). */
|
||||
get bytes(): number {
|
||||
return this.totalBytes;
|
||||
}
|
||||
|
||||
private evict(id: string): void {
|
||||
const entry = this.map.get(id);
|
||||
if (entry) {
|
||||
this.totalBytes -= entry.buf.length;
|
||||
this.map.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
private sweep(): void {
|
||||
const now = Date.now();
|
||||
for (const [id, entry] of this.map) {
|
||||
if (entry.expiresAt <= now) {
|
||||
this.evict(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -126,6 +126,10 @@ async function bootstrap() {
|
||||
'/api/workspace/create',
|
||||
'/api/workspace/joined',
|
||||
'/api/workspace/find-by-email',
|
||||
// Anonymous in-RAM blob sandbox: a remote consumer fetches blobs by an
|
||||
// unguessable UUID without any workspace host context, so the
|
||||
// workspace-resolution gate must not apply.
|
||||
'/api/sb',
|
||||
];
|
||||
|
||||
if (
|
||||
|
||||
@@ -7,6 +7,7 @@ import { TiptapTransformer } from "@hocuspocus/transformer";
|
||||
import * as Y from "yjs";
|
||||
import WebSocket from "ws";
|
||||
import { convertProseMirrorToMarkdown } from "./lib/markdown-converter.js";
|
||||
import { collectInternalFileNodes, normalizeFileUrl, } from "./lib/internal-file-urls.js";
|
||||
import { updatePageContentRealtime, replacePageContent, markdownToProseMirror, markdownToProseMirrorCanonical, mutatePageContent, buildCollabWsUrl, assertYjsEncodable, applyDocToFragment, } from "./lib/collaboration.js";
|
||||
import { footnoteWarningsField } from "./lib/footnote-analyze.js";
|
||||
import { buildPageTree } from "./lib/tree.js";
|
||||
@@ -51,6 +52,8 @@ export class DocmostClient {
|
||||
// its token instead of calling POST /auth/collab-token; on a 401/403 it is
|
||||
// re-invoked once. Used by the internal agent to carry signed provenance.
|
||||
getCollabTokenFn = null;
|
||||
// Optional blob-sandbox sink for the stash tool. Null when not configured.
|
||||
sandboxPut = null;
|
||||
// In-flight login dedup: when the token expires, the 401 interceptor,
|
||||
// ensureAuthenticated, getCollabTokenWithReauth and the two multipart retries
|
||||
// can all call login() at once. Memoizing a single promise collapses that
|
||||
@@ -77,6 +80,9 @@ export class DocmostClient {
|
||||
if (config.getCollabToken) {
|
||||
this.getCollabTokenFn = config.getCollabToken;
|
||||
}
|
||||
if (config.sandbox) {
|
||||
this.sandboxPut = config.sandbox.put;
|
||||
}
|
||||
this.client = axios.create({
|
||||
baseURL: this.apiUrl,
|
||||
// Default request timeout so a hung connection cannot wedge a per-page
|
||||
@@ -605,6 +611,106 @@ export class DocmostClient {
|
||||
content: data.content || { type: "doc", content: [] },
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Fetch an INTERNAL Docmost file (authed loopback) for sandbox mirroring.
|
||||
* `src` is normalized to `/api/files/<id>/<file>`; `this.client.baseURL`
|
||||
* already ends in `/api`, so we strip the leading `/api` and request the
|
||||
* relative path with the client's Authorization header. Returns the raw bytes
|
||||
* and the response Content-Type (mime), defaulting to octet-stream.
|
||||
*
|
||||
* The fetch is size-bounded (hard 64 MiB ceiling) purely to protect memory;
|
||||
* the authoritative per-blob cap is enforced by the sandbox `put`. The leading
|
||||
* `/api` strip means this never escapes the Docmost API base.
|
||||
*/
|
||||
async fetchInternalFile(src) {
|
||||
const HARD_CEILING = 64 * 1024 * 1024; // 64 MiB memory guard
|
||||
const relPath = src.replace(/^\/api/, "");
|
||||
const response = await this.client.get(relPath, {
|
||||
responseType: "arraybuffer",
|
||||
timeout: 30000,
|
||||
maxContentLength: HARD_CEILING,
|
||||
maxBodyLength: HARD_CEILING,
|
||||
});
|
||||
const buffer = Buffer.from(response.data);
|
||||
if (buffer.length === 0) {
|
||||
throw new Error(`Empty file response from "${src}"`);
|
||||
}
|
||||
const rawCt = response.headers?.["content-type"];
|
||||
const mime = typeof rawCt === "string" && rawCt.length > 0
|
||||
? rawCt.split(";")[0].trim().toLowerCase()
|
||||
: "application/octet-stream";
|
||||
return { buffer, mime };
|
||||
}
|
||||
/**
|
||||
* Stash a page's full content into the in-RAM blob sandbox and return ONLY a
|
||||
* short anonymous URL — the body never enters the model context (this is the
|
||||
* whole point: ~30KB+ ProseMirror docs blow the model context if passed as a
|
||||
* tool argument). Every INTERNAL file/image src (the type-agnostic criterion,
|
||||
* so drawio/excalidraw/video/file nodes are covered too) is mirrored into the
|
||||
* sandbox and its `src` rewritten to the sandbox URL, so an external consumer
|
||||
* can fetch the images anonymously. External http(s) srcs are left untouched.
|
||||
*
|
||||
* Blobs live in RAM with a short TTL and are cleared on restart — consume the
|
||||
* URLs within the TTL and one uptime. A failed image fetch never aborts the
|
||||
* doc: the original src is kept and the failure counted.
|
||||
*
|
||||
* Returns { uri, sha256, size, images:{mirrored, failed} }. `uri` and `sha256`
|
||||
* are for the document blob; `sha256` is also the blob's ETag (integrity).
|
||||
*/
|
||||
async stashPage(pageId) {
|
||||
if (!this.sandboxPut) {
|
||||
throw new Error("stash_page is unavailable: the blob sandbox is not configured on this server");
|
||||
}
|
||||
await this.ensureAuthenticated();
|
||||
// Stash the SAME shape get_page_json returns (id/title/.../content), with a
|
||||
// deep clone so the rewrite never mutates anything shared.
|
||||
const pageJson = await this.getPageJson(pageId);
|
||||
const cloned = structuredClone(pageJson);
|
||||
// Group internal-file nodes by normalized src so each unique resource is
|
||||
// fetched + stored ONCE (dedup), and every node sharing that src points at
|
||||
// the one sandbox blob.
|
||||
const bySrc = new Map();
|
||||
for (const node of collectInternalFileNodes(cloned.content)) {
|
||||
const src = normalizeFileUrl(String(node.attrs.src));
|
||||
const group = bySrc.get(src);
|
||||
if (group)
|
||||
group.push(node);
|
||||
else
|
||||
bySrc.set(src, [node]);
|
||||
}
|
||||
let mirrored = 0;
|
||||
let failed = 0;
|
||||
const MAX_CONCURRENCY = 5;
|
||||
const groups = [...bySrc.entries()];
|
||||
for (let i = 0; i < groups.length; i += MAX_CONCURRENCY) {
|
||||
const batch = groups.slice(i, i + MAX_CONCURRENCY);
|
||||
await Promise.all(batch.map(async ([src, nodes]) => {
|
||||
try {
|
||||
const { buffer, mime } = await this.fetchInternalFile(src);
|
||||
// put may throw if the blob exceeds the per-blob/total caps.
|
||||
const stored = this.sandboxPut(buffer, mime);
|
||||
for (const node of nodes)
|
||||
node.attrs.src = stored.uri;
|
||||
mirrored++;
|
||||
}
|
||||
catch (err) {
|
||||
// One bad/oversized image must not abort the document.
|
||||
failed++;
|
||||
if (process.env.DEBUG) {
|
||||
console.error(`stash_page: failed to mirror "${src}":`, err);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
const docBuf = Buffer.from(JSON.stringify(cloned), "utf8");
|
||||
const stored = this.sandboxPut(docBuf, "application/json");
|
||||
return {
|
||||
uri: stored.uri,
|
||||
sha256: stored.sha256,
|
||||
size: stored.size,
|
||||
images: { mirrored, failed },
|
||||
};
|
||||
}
|
||||
/**
|
||||
* Compact outline of a page's top-level blocks (no full document body).
|
||||
* Cheap way to locate sections/tables and grab block ids before drilling in
|
||||
|
||||
@@ -285,6 +285,26 @@ export function createDocmostMcpServer(config) {
|
||||
const result = await docmostClient.editPageText(pageId, edits);
|
||||
return jsonContent(result);
|
||||
});
|
||||
// Tool: stash_page — returns a resource_link (NOT embedded text) so the doc
|
||||
// body never enters the model context. Registered directly (not via
|
||||
// registerShared) because that helper only emits text content.
|
||||
server.registerTool(SHARED_TOOL_SPECS.stashPage.mcpName, {
|
||||
description: SHARED_TOOL_SPECS.stashPage.description,
|
||||
inputSchema: SHARED_TOOL_SPECS.stashPage.buildShape(z),
|
||||
}, async ({ pageId }) => {
|
||||
const result = await docmostClient.stashPage(pageId);
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "resource_link",
|
||||
uri: result.uri,
|
||||
name: "page.json",
|
||||
mimeType: "application/json",
|
||||
size: result.size,
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
// Tool: patch_node
|
||||
server.registerTool("patch_node", {
|
||||
description: "Replaces a single block identified by its attrs.id WITHOUT resending the " +
|
||||
|
||||
55
packages/mcp/build/lib/internal-file-urls.js
Normal file
55
packages/mcp/build/lib/internal-file-urls.js
Normal file
@@ -0,0 +1,55 @@
|
||||
// Detection + collection of INTERNAL Docmost file URLs inside a ProseMirror doc.
|
||||
//
|
||||
// An internal file URL is a relative path served by Docmost's authenticated
|
||||
// attachment route (`GET /api/files/:fileId/:fileName`). It is useless to an
|
||||
// external consumer (relative + needs a Docmost session), so the stash tool
|
||||
// mirrors every such resource into the blob sandbox and rewrites its `src`.
|
||||
//
|
||||
// The criterion is "internal file URL", NOT the node TYPE: image, drawio,
|
||||
// excalidraw, video and file nodes all carry such a `src`, so a type-agnostic
|
||||
// walker covers them all. External http(s) srcs (CDNs) are left untouched.
|
||||
//
|
||||
// Mirrors editor-ext's isInternalFileUrl / normalizeFileUrl (kept as a local
|
||||
// dup so the ESM mcp package does not depend on the editor-ext build).
|
||||
export function isInternalFileUrl(url) {
|
||||
if (typeof url !== "string")
|
||||
return false;
|
||||
const normalized = url.trim();
|
||||
return (normalized.startsWith("/api/files/") || normalized.startsWith("/files/"));
|
||||
}
|
||||
/** Normalize a bare `/files/...` src to the canonical `/api/files/...` form. */
|
||||
export function normalizeFileUrl(src) {
|
||||
const trimmed = src.trim();
|
||||
if (trimmed.startsWith("/files/"))
|
||||
return "/api" + trimmed;
|
||||
return trimmed;
|
||||
}
|
||||
/**
|
||||
* Recursively collect every node whose `attrs.src` is an internal file URL.
|
||||
* Returns references to the live nodes (so the caller can rewrite `attrs.src`
|
||||
* in place on its clone). Descends `content` arrays, covering callouts, tables,
|
||||
* details and any other nested container.
|
||||
*/
|
||||
export function collectInternalFileNodes(doc) {
|
||||
const out = [];
|
||||
const visit = (node) => {
|
||||
if (!node)
|
||||
return;
|
||||
if (Array.isArray(node)) {
|
||||
for (const child of node)
|
||||
visit(child);
|
||||
return;
|
||||
}
|
||||
if (typeof node !== "object")
|
||||
return;
|
||||
if (node.attrs && isInternalFileUrl(node.attrs.src)) {
|
||||
out.push(node);
|
||||
}
|
||||
if (Array.isArray(node.content)) {
|
||||
for (const child of node.content)
|
||||
visit(child);
|
||||
}
|
||||
};
|
||||
visit(doc);
|
||||
return out;
|
||||
}
|
||||
@@ -209,4 +209,24 @@ export const SHARED_TOOL_SPECS = {
|
||||
.describe('List of find/replace operations, applied in order'),
|
||||
}),
|
||||
},
|
||||
// --- hand a large page to an external consumer without bloating context ---
|
||||
stashPage: {
|
||||
mcpName: 'stash_page',
|
||||
inAppKey: 'stashPage',
|
||||
description: 'Serialize a whole page (the full ProseMirror JSON, as get_page_json ' +
|
||||
'returns) into an ephemeral in-memory blob and return ONLY a short ' +
|
||||
'anonymous URL to it — the body NEVER enters the model context, so this ' +
|
||||
'is the way to hand a large page (or its images) to an external consumer ' +
|
||||
'without truncation. Every internal file/image attachment is mirrored ' +
|
||||
'into the same sandbox and its src rewritten to a sandbox URL, so the ' +
|
||||
'consumer can fetch the images anonymously too; external http(s) images ' +
|
||||
'are left untouched. Returns { uri, size, sha256, images:{mirrored, ' +
|
||||
'failed} }. Integrity: the blob is served with ETag = its sha256, so a ' +
|
||||
'truncated/corrupted fetch is detectable. Blobs are RAM-only: they expire ' +
|
||||
'after a short TTL (~1h) and are cleared on restart — consume the URL ' +
|
||||
'within the TTL and one uptime, or re-stash.',
|
||||
buildShape: (z) => ({
|
||||
pageId: z.string().min(1),
|
||||
}),
|
||||
},
|
||||
};
|
||||
|
||||
@@ -13,6 +13,10 @@ import { TiptapTransformer } from "@hocuspocus/transformer";
|
||||
import * as Y from "yjs";
|
||||
import WebSocket from "ws";
|
||||
import { convertProseMirrorToMarkdown } from "./lib/markdown-converter.js";
|
||||
import {
|
||||
collectInternalFileNodes,
|
||||
normalizeFileUrl,
|
||||
} from "./lib/internal-file-urls.js";
|
||||
import {
|
||||
updatePageContentRealtime,
|
||||
replacePageContent,
|
||||
@@ -102,6 +106,14 @@ const MIME_TO_EXT: Record<string, string> = {
|
||||
* Housed here (not in index.ts) so client.ts has no type dependency on index.ts;
|
||||
* index.ts re-exports it for the package's public surface.
|
||||
*/
|
||||
// Sink the stash tool writes blobs into. The host app binds this to its in-RAM
|
||||
// SandboxStore and composes the public `uri` (the package never sees the store
|
||||
// or any env). `put` returns the anonymous read URL plus integrity metadata.
|
||||
export type SandboxPut = (
|
||||
buf: Buffer,
|
||||
mime: string,
|
||||
) => { uri: string; sha256: string; size: number };
|
||||
|
||||
export type DocmostMcpConfig = { apiUrl: string } & (
|
||||
| { email: string; password: string }
|
||||
| { getToken: () => Promise<string> } // returns a BARE JWT; the client adds "Bearer "
|
||||
@@ -109,6 +121,9 @@ export type DocmostMcpConfig = { apiUrl: string } & (
|
||||
// Optional collab-token provider (returns a ready collab JWT). Common to
|
||||
// both branches; see the type doc above.
|
||||
getCollabToken?: () => Promise<string>;
|
||||
// Optional blob sandbox sink. Present only where the stash tool is wired;
|
||||
// when absent, stash_page throws a clear "not configured" error.
|
||||
sandbox?: { put: SandboxPut };
|
||||
};
|
||||
|
||||
export class DocmostClient {
|
||||
@@ -126,6 +141,8 @@ export class DocmostClient {
|
||||
// its token instead of calling POST /auth/collab-token; on a 401/403 it is
|
||||
// re-invoked once. Used by the internal agent to carry signed provenance.
|
||||
private getCollabTokenFn: (() => Promise<string>) | null = null;
|
||||
// Optional blob-sandbox sink for the stash tool. Null when not configured.
|
||||
private sandboxPut: SandboxPut | null = null;
|
||||
// In-flight login dedup: when the token expires, the 401 interceptor,
|
||||
// ensureAuthenticated, getCollabTokenWithReauth and the two multipart retries
|
||||
// can all call login() at once. Memoizing a single promise collapses that
|
||||
@@ -165,6 +182,9 @@ export class DocmostClient {
|
||||
if (config.getCollabToken) {
|
||||
this.getCollabTokenFn = config.getCollabToken;
|
||||
}
|
||||
if (config.sandbox) {
|
||||
this.sandboxPut = config.sandbox.put;
|
||||
}
|
||||
this.client = axios.create({
|
||||
baseURL: this.apiUrl,
|
||||
// Default request timeout so a hung connection cannot wedge a per-page
|
||||
@@ -767,6 +787,120 @@ export class DocmostClient {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch an INTERNAL Docmost file (authed loopback) for sandbox mirroring.
|
||||
* `src` is normalized to `/api/files/<id>/<file>`; `this.client.baseURL`
|
||||
* already ends in `/api`, so we strip the leading `/api` and request the
|
||||
* relative path with the client's Authorization header. Returns the raw bytes
|
||||
* and the response Content-Type (mime), defaulting to octet-stream.
|
||||
*
|
||||
* The fetch is size-bounded (hard 64 MiB ceiling) purely to protect memory;
|
||||
* the authoritative per-blob cap is enforced by the sandbox `put`. The leading
|
||||
* `/api` strip means this never escapes the Docmost API base.
|
||||
*/
|
||||
private async fetchInternalFile(
|
||||
src: string,
|
||||
): Promise<{ buffer: Buffer; mime: string }> {
|
||||
const HARD_CEILING = 64 * 1024 * 1024; // 64 MiB memory guard
|
||||
const relPath = src.replace(/^\/api/, "");
|
||||
const response = await this.client.get(relPath, {
|
||||
responseType: "arraybuffer",
|
||||
timeout: 30000,
|
||||
maxContentLength: HARD_CEILING,
|
||||
maxBodyLength: HARD_CEILING,
|
||||
});
|
||||
const buffer = Buffer.from(response.data);
|
||||
if (buffer.length === 0) {
|
||||
throw new Error(`Empty file response from "${src}"`);
|
||||
}
|
||||
const rawCt = response.headers?.["content-type"];
|
||||
const mime =
|
||||
typeof rawCt === "string" && rawCt.length > 0
|
||||
? rawCt.split(";")[0].trim().toLowerCase()
|
||||
: "application/octet-stream";
|
||||
return { buffer, mime };
|
||||
}
|
||||
|
||||
/**
|
||||
* Stash a page's full content into the in-RAM blob sandbox and return ONLY a
|
||||
* short anonymous URL — the body never enters the model context (this is the
|
||||
* whole point: ~30KB+ ProseMirror docs blow the model context if passed as a
|
||||
* tool argument). Every INTERNAL file/image src (the type-agnostic criterion,
|
||||
* so drawio/excalidraw/video/file nodes are covered too) is mirrored into the
|
||||
* sandbox and its `src` rewritten to the sandbox URL, so an external consumer
|
||||
* can fetch the images anonymously. External http(s) srcs are left untouched.
|
||||
*
|
||||
* Blobs live in RAM with a short TTL and are cleared on restart — consume the
|
||||
* URLs within the TTL and one uptime. A failed image fetch never aborts the
|
||||
* doc: the original src is kept and the failure counted.
|
||||
*
|
||||
* Returns { uri, sha256, size, images:{mirrored, failed} }. `uri` and `sha256`
|
||||
* are for the document blob; `sha256` is also the blob's ETag (integrity).
|
||||
*/
|
||||
async stashPage(pageId: string): Promise<{
|
||||
uri: string;
|
||||
sha256: string;
|
||||
size: number;
|
||||
images: { mirrored: number; failed: number };
|
||||
}> {
|
||||
if (!this.sandboxPut) {
|
||||
throw new Error(
|
||||
"stash_page is unavailable: the blob sandbox is not configured on this server",
|
||||
);
|
||||
}
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// Stash the SAME shape get_page_json returns (id/title/.../content), with a
|
||||
// deep clone so the rewrite never mutates anything shared.
|
||||
const pageJson = await this.getPageJson(pageId);
|
||||
const cloned: any = structuredClone(pageJson);
|
||||
|
||||
// Group internal-file nodes by normalized src so each unique resource is
|
||||
// fetched + stored ONCE (dedup), and every node sharing that src points at
|
||||
// the one sandbox blob.
|
||||
const bySrc = new Map<string, any[]>();
|
||||
for (const node of collectInternalFileNodes(cloned.content)) {
|
||||
const src = normalizeFileUrl(String(node.attrs.src));
|
||||
const group = bySrc.get(src);
|
||||
if (group) group.push(node);
|
||||
else bySrc.set(src, [node]);
|
||||
}
|
||||
|
||||
let mirrored = 0;
|
||||
let failed = 0;
|
||||
const MAX_CONCURRENCY = 5;
|
||||
const groups = [...bySrc.entries()];
|
||||
for (let i = 0; i < groups.length; i += MAX_CONCURRENCY) {
|
||||
const batch = groups.slice(i, i + MAX_CONCURRENCY);
|
||||
await Promise.all(
|
||||
batch.map(async ([src, nodes]) => {
|
||||
try {
|
||||
const { buffer, mime } = await this.fetchInternalFile(src);
|
||||
// put may throw if the blob exceeds the per-blob/total caps.
|
||||
const stored = this.sandboxPut!(buffer, mime);
|
||||
for (const node of nodes) node.attrs.src = stored.uri;
|
||||
mirrored++;
|
||||
} catch (err) {
|
||||
// One bad/oversized image must not abort the document.
|
||||
failed++;
|
||||
if (process.env.DEBUG) {
|
||||
console.error(`stash_page: failed to mirror "${src}":`, err);
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const docBuf = Buffer.from(JSON.stringify(cloned), "utf8");
|
||||
const stored = this.sandboxPut(docBuf, "application/json");
|
||||
return {
|
||||
uri: stored.uri,
|
||||
sha256: stored.sha256,
|
||||
size: stored.size,
|
||||
images: { mirrored, failed },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact outline of a page's top-level blocks (no full document body).
|
||||
* Cheap way to locate sections/tables and grab block ids before drilling in
|
||||
|
||||
@@ -408,6 +408,31 @@ registerShared(SHARED_TOOL_SPECS.editPageText, async ({ pageId, edits }) => {
|
||||
return jsonContent(result);
|
||||
});
|
||||
|
||||
// Tool: stash_page — returns a resource_link (NOT embedded text) so the doc
|
||||
// body never enters the model context. Registered directly (not via
|
||||
// registerShared) because that helper only emits text content.
|
||||
server.registerTool(
|
||||
SHARED_TOOL_SPECS.stashPage.mcpName,
|
||||
{
|
||||
description: SHARED_TOOL_SPECS.stashPage.description,
|
||||
inputSchema: SHARED_TOOL_SPECS.stashPage.buildShape!(z),
|
||||
},
|
||||
async ({ pageId }: { pageId: string }) => {
|
||||
const result = await docmostClient.stashPage(pageId);
|
||||
return {
|
||||
content: [
|
||||
{
|
||||
type: "resource_link" as const,
|
||||
uri: result.uri,
|
||||
name: "page.json",
|
||||
mimeType: "application/json",
|
||||
size: result.size,
|
||||
},
|
||||
],
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
// Tool: patch_node
|
||||
server.registerTool(
|
||||
"patch_node",
|
||||
|
||||
54
packages/mcp/src/lib/internal-file-urls.ts
Normal file
54
packages/mcp/src/lib/internal-file-urls.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
// Detection + collection of INTERNAL Docmost file URLs inside a ProseMirror doc.
|
||||
//
|
||||
// An internal file URL is a relative path served by Docmost's authenticated
|
||||
// attachment route (`GET /api/files/:fileId/:fileName`). It is useless to an
|
||||
// external consumer (relative + needs a Docmost session), so the stash tool
|
||||
// mirrors every such resource into the blob sandbox and rewrites its `src`.
|
||||
//
|
||||
// The criterion is "internal file URL", NOT the node TYPE: image, drawio,
|
||||
// excalidraw, video and file nodes all carry such a `src`, so a type-agnostic
|
||||
// walker covers them all. External http(s) srcs (CDNs) are left untouched.
|
||||
//
|
||||
// Mirrors editor-ext's isInternalFileUrl / normalizeFileUrl (kept as a local
|
||||
// dup so the ESM mcp package does not depend on the editor-ext build).
|
||||
|
||||
export function isInternalFileUrl(url: unknown): boolean {
|
||||
if (typeof url !== "string") return false;
|
||||
const normalized = url.trim();
|
||||
return (
|
||||
normalized.startsWith("/api/files/") || normalized.startsWith("/files/")
|
||||
);
|
||||
}
|
||||
|
||||
/** Normalize a bare `/files/...` src to the canonical `/api/files/...` form. */
|
||||
export function normalizeFileUrl(src: string): string {
|
||||
const trimmed = src.trim();
|
||||
if (trimmed.startsWith("/files/")) return "/api" + trimmed;
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively collect every node whose `attrs.src` is an internal file URL.
|
||||
* Returns references to the live nodes (so the caller can rewrite `attrs.src`
|
||||
* in place on its clone). Descends `content` arrays, covering callouts, tables,
|
||||
* details and any other nested container.
|
||||
*/
|
||||
export function collectInternalFileNodes(doc: unknown): any[] {
|
||||
const out: any[] = [];
|
||||
const visit = (node: any): void => {
|
||||
if (!node) return;
|
||||
if (Array.isArray(node)) {
|
||||
for (const child of node) visit(child);
|
||||
return;
|
||||
}
|
||||
if (typeof node !== "object") return;
|
||||
if (node.attrs && isInternalFileUrl(node.attrs.src)) {
|
||||
out.push(node);
|
||||
}
|
||||
if (Array.isArray(node.content)) {
|
||||
for (const child of node.content) visit(child);
|
||||
}
|
||||
};
|
||||
visit(doc);
|
||||
return out;
|
||||
}
|
||||
@@ -266,4 +266,26 @@ export const SHARED_TOOL_SPECS = {
|
||||
.describe('List of find/replace operations, applied in order'),
|
||||
}),
|
||||
},
|
||||
|
||||
// --- hand a large page to an external consumer without bloating context ---
|
||||
stashPage: {
|
||||
mcpName: 'stash_page',
|
||||
inAppKey: 'stashPage',
|
||||
description:
|
||||
'Serialize a whole page (the full ProseMirror JSON, as get_page_json ' +
|
||||
'returns) into an ephemeral in-memory blob and return ONLY a short ' +
|
||||
'anonymous URL to it — the body NEVER enters the model context, so this ' +
|
||||
'is the way to hand a large page (or its images) to an external consumer ' +
|
||||
'without truncation. Every internal file/image attachment is mirrored ' +
|
||||
'into the same sandbox and its src rewritten to a sandbox URL, so the ' +
|
||||
'consumer can fetch the images anonymously too; external http(s) images ' +
|
||||
'are left untouched. Returns { uri, size, sha256, images:{mirrored, ' +
|
||||
'failed} }. Integrity: the blob is served with ETag = its sha256, so a ' +
|
||||
'truncated/corrupted fetch is detectable. Blobs are RAM-only: they expire ' +
|
||||
'after a short TTL (~1h) and are cleared on restart — consume the URL ' +
|
||||
'within the TTL and one uptime, or re-stash.',
|
||||
buildShape: (z) => ({
|
||||
pageId: z.string().min(1),
|
||||
}),
|
||||
},
|
||||
} satisfies Record<string, SharedToolSpec>;
|
||||
|
||||
183
packages/mcp/test/mock/stash-page.test.mjs
Normal file
183
packages/mcp/test/mock/stash-page.test.mjs
Normal file
@@ -0,0 +1,183 @@
|
||||
// Mock-HTTP test for DocmostClient.stashPage: a local http server stands in for
|
||||
// Docmost so the whole flow stays deterministic and offline. Asserts the tool
|
||||
// (1) serializes the page into the sandbox and returns ONLY a link (uri + sha256
|
||||
// + size), never the body; (2) mirrors INTERNAL image srcs into the sandbox and
|
||||
// rewrites them to the sandbox uri; (3) leaves EXTERNAL http(s) srcs untouched;
|
||||
// (4) de-duplicates a repeated internal src to a single blob; (5) counts a
|
||||
// failed image fetch without aborting the document.
|
||||
import { test, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import http from "node:http";
|
||||
import { createHash } from "node:crypto";
|
||||
import { DocmostClient } from "../../build/client.js";
|
||||
|
||||
function readBody(req) {
|
||||
return new Promise((resolve) => {
|
||||
let raw = "";
|
||||
req.on("data", (c) => (raw += c));
|
||||
req.on("end", () => resolve(raw));
|
||||
});
|
||||
}
|
||||
|
||||
function startServer(handler) {
|
||||
return new Promise((resolve) => {
|
||||
const server = http.createServer(handler);
|
||||
server.listen(0, "127.0.0.1", () => {
|
||||
const { port } = server.address();
|
||||
resolve({ server, baseURL: `http://127.0.0.1:${port}/api` });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
const openServers = [];
|
||||
async function spawn(handler) {
|
||||
const { server, baseURL } = await startServer(handler);
|
||||
openServers.push(server);
|
||||
return baseURL;
|
||||
}
|
||||
after(async () => {
|
||||
await Promise.all(openServers.map((s) => new Promise((r) => s.close(r))));
|
||||
});
|
||||
|
||||
// In-memory sandbox sink mirroring the host binding: store the blob, return a
|
||||
// uri + sha256 + size. Records every put so the test can inspect what was
|
||||
// stashed (and verify the doc body never leaves via the return value).
|
||||
function makeSandbox() {
|
||||
const puts = [];
|
||||
return {
|
||||
puts,
|
||||
put(buf, mime) {
|
||||
const sha256 = createHash("sha256").update(buf).digest("hex");
|
||||
const id = `id-${puts.length}`;
|
||||
puts.push({ buf, mime, sha256 });
|
||||
return { uri: `https://sb.test/api/sb/${id}`, sha256, size: buf.length };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const IMAGE_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); // "PNG" header-ish
|
||||
|
||||
function pageDoc() {
|
||||
return {
|
||||
type: "doc",
|
||||
content: [
|
||||
{
|
||||
type: "image",
|
||||
attrs: { src: "/api/files/att-1/pic.png", attachmentId: "att-1", width: 100 },
|
||||
},
|
||||
// Same internal src again -> must dedup to ONE blob, both rewritten.
|
||||
{
|
||||
type: "image",
|
||||
attrs: { src: "/api/files/att-1/pic.png", attachmentId: "att-1", width: 50 },
|
||||
},
|
||||
// External CDN image -> must be left untouched.
|
||||
{
|
||||
type: "image",
|
||||
attrs: { src: "https://cdn.example.com/remote.png" },
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
// Build a client wired to a server that logs in, serves the page, and serves the
|
||||
// internal file bytes. `fileStatus` lets a test force the file fetch to fail.
|
||||
async function buildClient(sandbox, { fileStatus = 200 } = {}) {
|
||||
const baseURL = await spawn(async (req, res) => {
|
||||
await readBody(req);
|
||||
if (req.url === "/api/auth/login") {
|
||||
res.writeHead(200, {
|
||||
"Content-Type": "application/json",
|
||||
"Set-Cookie": "authToken=tok; HttpOnly",
|
||||
});
|
||||
res.end(JSON.stringify({ token: "tok" }));
|
||||
return;
|
||||
}
|
||||
if (req.url === "/api/pages/info") {
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({ data: { id: "page-1", title: "T", content: pageDoc() } }));
|
||||
return;
|
||||
}
|
||||
if (req.url.startsWith("/api/files/att-1/")) {
|
||||
if (fileStatus !== 200) {
|
||||
res.writeHead(fileStatus);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { "Content-Type": "image/png" });
|
||||
res.end(IMAGE_BYTES);
|
||||
return;
|
||||
}
|
||||
res.writeHead(404);
|
||||
res.end();
|
||||
});
|
||||
return new DocmostClient({
|
||||
apiUrl: baseURL,
|
||||
email: "u@example.com",
|
||||
password: "pw",
|
||||
sandbox: { put: (buf, mime) => sandbox.put(buf, mime) },
|
||||
});
|
||||
}
|
||||
|
||||
test("stashPage stores the doc + mirrors/rewrites internal images, returns only a link", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await buildClient(sandbox);
|
||||
|
||||
const result = await client.stashPage("page-1");
|
||||
|
||||
// Returns ONLY a link shape — never the document body.
|
||||
assert.equal(typeof result.uri, "string");
|
||||
assert.match(result.uri, /^https:\/\/sb\.test\/api\/sb\//);
|
||||
assert.equal(typeof result.sha256, "string");
|
||||
assert.equal(typeof result.size, "number");
|
||||
assert.ok(!("doc" in result) && !("content" in result) && !("body" in result));
|
||||
assert.deepEqual(result.images, { mirrored: 1, failed: 0 });
|
||||
|
||||
// One image blob (dedup) + one doc blob = 2 puts.
|
||||
assert.equal(sandbox.puts.length, 2);
|
||||
const imagePut = sandbox.puts[0];
|
||||
const docPut = sandbox.puts[1];
|
||||
assert.equal(imagePut.mime, "image/png");
|
||||
assert.ok(imagePut.buf.equals(IMAGE_BYTES));
|
||||
assert.equal(docPut.mime, "application/json");
|
||||
|
||||
// The returned uri/sha256 are the DOCUMENT blob's.
|
||||
assert.equal(result.sha256, docPut.sha256);
|
||||
|
||||
// Inspect the stashed document: internal srcs rewritten, external untouched.
|
||||
const stashed = JSON.parse(docPut.buf.toString("utf8"));
|
||||
const imgs = stashed.content.content.filter((n) => n.type === "image");
|
||||
assert.equal(imgs[0].attrs.src, "https://sb.test/api/sb/id-0");
|
||||
assert.equal(imgs[1].attrs.src, "https://sb.test/api/sb/id-0"); // same blob (dedup)
|
||||
assert.equal(imgs[2].attrs.src, "https://cdn.example.com/remote.png"); // external kept
|
||||
});
|
||||
|
||||
test("stashPage counts a failed image fetch without aborting the document", async () => {
|
||||
const sandbox = makeSandbox();
|
||||
const client = await buildClient(sandbox, { fileStatus: 500 });
|
||||
|
||||
const result = await client.stashPage("page-1");
|
||||
|
||||
assert.deepEqual(result.images, { mirrored: 0, failed: 1 });
|
||||
// Only the doc blob was stored (image fetch failed).
|
||||
assert.equal(sandbox.puts.length, 1);
|
||||
assert.equal(sandbox.puts[0].mime, "application/json");
|
||||
|
||||
// The failed internal src is LEFT as-is so nothing is silently dropped.
|
||||
const stashed = JSON.parse(sandbox.puts[0].buf.toString("utf8"));
|
||||
const imgs = stashed.content.content.filter((n) => n.type === "image");
|
||||
assert.equal(imgs[0].attrs.src, "/api/files/att-1/pic.png");
|
||||
});
|
||||
|
||||
test("stashPage throws a clear error when no sandbox is configured", async () => {
|
||||
const baseURL = await spawn(async (req, res) => {
|
||||
await readBody(req);
|
||||
res.writeHead(200, { "Content-Type": "application/json" });
|
||||
res.end(JSON.stringify({}));
|
||||
});
|
||||
const client = new DocmostClient({
|
||||
apiUrl: baseURL,
|
||||
email: "u@example.com",
|
||||
password: "pw",
|
||||
});
|
||||
await assert.rejects(() => client.stashPage("page-1"), /not configured/);
|
||||
});
|
||||
Reference in New Issue
Block a user