feat(sandbox): in-RAM blob sandbox for out-of-band page transfer (#243)

Add an ephemeral, process-local blob store so the in-app agent (and the embedded MCP) can hand a large page document and its images to an external consumer WITHOUT routing the bytes through the model context or Docmost auth. - SandboxStore (@Injectable singleton): Map<uuid,{buf,mime,sha256,expiresAt}> in RAM only. put() picks a per-blob cap by mime (image vs doc), enforces a total-bytes RAM guard with oldest-first eviction, and stamps a TTL; get() lazily expires. sha256 computed at put() doubles as the strong ETag. An unref'd sweep interval clears expired entries and is cleared on destroy. - GET /api/sb/:uuid anonymous controller: serves raw bytes with Content-Type, Content-Length and ETag=sha256; 404 on missing/expired/non-UUID (anti- traversal), 304 on a matching If-None-Match. No tokens, no 401 — the capability is the unguessable UUID + short TTL + TLS. Auth-exempt the same way as /api/files/public (no JwtAuthGuard) plus an /api/sb entry in main.ts's workspace-resolution preHandler so a remote consumer with no workspace host is not rejected. - stash_page tool in both layers (MCP resource_link + in-app {uri,size,sha256, images}). client.stashPage serializes the get_page_json shape, mirrors every INTERNAL file/image src (type-agnostic, covers drawio/excalidraw/video/file) into the sandbox under Docmost auth and rewrites src to the sandbox URL; external http(s) srcs are left untouched; dedup by src; a failed image fetch is counted, never aborts the doc. - SANDBOX_PUBLIC_URL / SANDBOX_TTL_MS / SANDBOX_MAX_BYTES / SANDBOX_MAX_IMAGE_BYTES / SANDBOX_MAX_TOTAL_BYTES wired through the environment service + validation + .env.example. - SandboxModule (@Global) provides the shared store to the controller, McpService and AiChatToolsService (same instance for put and get). Tests: SandboxStore (round-trip, sha256, TTL lazy + sweep, caps, eviction), SandboxController (200+ETag+CT+CL, 404 missing/expired/non-UUID, 304), and a mock-HTTP stashPage test (mirror+rewrite internal, keep external, dedup, failed image counted, returns only a link). Interoperates with the vvzvlad/habr-mcp consumer's anonymous-GET + sha256-ETag + resource_link contract. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-28 15:13:11 +03:00
parent 106df7c907
commit 2fe4ca8537
25 changed files with 1312 additions and 4 deletions
@@ -0,0 +1,124 @@
+import { SandboxController } from './sandbox.controller';
+import { SandboxEntry } from './sandbox.store';
+
+// Capturing fake of the FastifyReply surface the controller uses:
+// status()/header()/headers()/send(), all chainable.
+function makeRes() {
+  const sent: { status: number; headers: Record<string, any>; body: any } = {
+    status: 200,
+    headers: {},
+    body: undefined,
+  };
+  const res: any = {
+    status(code: number) {
+      sent.status = code;
+      return res;
+    },
+    header(key: string, value: any) {
+      sent.headers[key.toLowerCase()] = value;
+      return res;
+    },
+    headers(obj: Record<string, any>) {
+      for (const k of Object.keys(obj)) sent.headers[k.toLowerCase()] = obj[k];
+      return res;
+    },
+    send(body?: any) {
+      sent.body = body;
+      return res;
+    },
+    _sent: sent,
+  };
+  return res;
+}
+
+function makeReq(headers: Record<string, any> = {}) {
+  return { headers } as any;
+}
+
+const VALID_ID = 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee';
+
+function entry(buf: Buffer, mime: string, sha256: string): SandboxEntry {
+  return { buf, mime, sha256, expiresAt: Date.now() + 60_000 };
+}
+
+describe('SandboxController', () => {
+  it('serves 200 with body, Content-Type, Content-Length and sha256 ETag', async () => {
+    const buf = Buffer.from('{"ok":true}', 'utf8');
+    const sha = 'a'.repeat(64);
+    const store = { get: jest.fn().mockReturnValue(entry(buf, 'application/json', sha)) };
+    const controller = new SandboxController(store as any);
+    const res = makeRes();
+
+    await controller.get(VALID_ID, makeReq(), res);
+
+    expect(store.get).toHaveBeenCalledWith(VALID_ID);
+    expect(res._sent.status).toBe(200);
+    expect(res._sent.headers['content-type']).toBe('application/json');
+    expect(res._sent.headers['content-length']).toBe(buf.length);
+    expect(res._sent.headers['etag']).toBe(`"${sha}"`);
+    expect(res._sent.body).toBe(buf);
+  });
+
+  it('returns 404 for a missing/expired blob', async () => {
+    const store = { get: jest.fn().mockReturnValue(undefined) };
+    const controller = new SandboxController(store as any);
+    const res = makeRes();
+
+    await controller.get(VALID_ID, makeReq(), res);
+
+    expect(res._sent.status).toBe(404);
+    expect(res._sent.body).toBeUndefined();
+  });
+
+  it('returns 404 for a non-UUID id WITHOUT touching the store (anti-traversal)', async () => {
+    const store = { get: jest.fn() };
+    const controller = new SandboxController(store as any);
+    const res = makeRes();
+
+    await controller.get('../../etc/passwd', makeReq(), res);
+
+    expect(store.get).not.toHaveBeenCalled();
+    expect(res._sent.status).toBe(404);
+  });
+
+  it('returns 304 (no body) when If-None-Match matches the ETag', async () => {
+    const sha = 'b'.repeat(64);
+    const store = {
+      get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
+    };
+    const controller = new SandboxController(store as any);
+    const res = makeRes();
+
+    await controller.get(VALID_ID, makeReq({ 'if-none-match': `"${sha}"` }), res);
+
+    expect(res._sent.status).toBe(304);
+    expect(res._sent.body).toBeUndefined();
+    expect(res._sent.headers['etag']).toBe(`"${sha}"`);
+  });
+
+  it('accepts a bare (unquoted) sha256 in If-None-Match too', async () => {
+    const sha = 'c'.repeat(64);
+    const store = {
+      get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
+    };
+    const controller = new SandboxController(store as any);
+    const res = makeRes();
+
+    await controller.get(VALID_ID, makeReq({ 'if-none-match': sha }), res);
+
+    expect(res._sent.status).toBe(304);
+  });
+
+  it('serves 200 when If-None-Match does NOT match', async () => {
+    const sha = 'd'.repeat(64);
+    const store = {
+      get: jest.fn().mockReturnValue(entry(Buffer.from('x'), 'application/json', sha)),
+    };
+    const controller = new SandboxController(store as any);
+    const res = makeRes();
+
+    await controller.get(VALID_ID, makeReq({ 'if-none-match': '"stale"' }), res);
+
+    expect(res._sent.status).toBe(200);
+  });
+});
@@ -0,0 +1,94 @@
+import { Controller, Get, Param, Req, Res } from '@nestjs/common';
+import { FastifyReply, FastifyRequest } from 'fastify';
+import { SandboxStore } from './sandbox.store';
+
+// Strict UUID v-agnostic shape. This is anti-traversal / input hygiene (so `:id`
+// can never be a path like `../...`), NOT authorization — the capability is the
+// unguessable id itself plus the short TTL plus TLS.
+const UUID_RE =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+/**
+ * Anonymous read endpoint for the in-RAM blob sandbox.
+ *
+ * Mounted under the global `/api` prefix as `GET /api/sb/:id`. It carries NO
+ * `@UseGuards(JwtAuthGuard)`, so — exactly like the public attachment route
+ * `GET /api/files/public/...` — it is exempt from Docmost session auth. The
+ * route is ALSO listed in the workspace-resolution preHandler's excludedPaths
+ * in main.ts so a request from a remote consumer (which carries no workspace
+ * host) is not rejected with "Workspace not found".
+ *
+ * It only ever serves blobs looked up from the SandboxStore by a validated
+ * UUID; `:id` is never used as a filesystem path, so there is no traversal
+ * surface. Never returns tokens, never 401s.
+ */
+@Controller('sb')
+export class SandboxController {
+  constructor(private readonly store: SandboxStore) {}
+
+  @Get(':id')
+  async get(
+    @Param('id') id: string,
+    @Req() req: FastifyRequest,
+    @Res() res: FastifyReply,
+  ): Promise<void> {
+    // Non-UUID id (including any traversal attempt) → 404 before touching the
+    // store. No stack trace leaks out.
+    if (!UUID_RE.test(id)) {
+      res.status(404).send();
+      return;
+    }
+
+    const entry = this.store.get(id);
+    if (!entry) {
+      // Missing or expired — indistinguishable to the caller, by design.
+      res.status(404).send();
+      return;
+    }
+
+    // Strong validator: quoted sha256, no W/ weak prefix. Same value computed
+    // at put() time, so an external consumer can detect a truncated/corrupted
+    // body — the original bug this whole channel exists to fix.
+    const etag = `"${entry.sha256}"`;
+
+    // Conditional request: an exact ETag match → 304 with no body. The blob is
+    // immutable, so the validator is stable for the blob's whole lifetime.
+    if (this.ifNoneMatchMatches(req.headers['if-none-match'], entry.sha256)) {
+      res.status(304).header('ETag', etag).send();
+      return;
+    }
+
+    const ttlSeconds = Math.max(
+      0,
+      Math.floor((entry.expiresAt - Date.now()) / 1000),
+    );
+
+    // Use @Res() + res.send(Buffer) with an explicit Content-Type so the binary
+    // body bypasses the global JSON response transform/serializer.
+    res
+      .status(200)
+      .headers({
+        'Content-Type': entry.mime,
+        'Content-Length': entry.buf.length,
+        ETag: etag,
+        // Capability URL — keep it out of shared caches; immutable for its TTL.
+        'Cache-Control': `private, max-age=${ttlSeconds}, immutable`,
+      })
+      .send(entry.buf);
+  }
+
+  // Accept the consumer's If-None-Match whether it sends the quoted ETag, a bare
+  // sha256, a weak "W/"-prefixed validator, or a comma-separated list.
+  private ifNoneMatchMatches(
+    header: string | string[] | undefined,
+    sha256: string,
+  ): boolean {
+    if (!header) return false;
+    const raw = Array.isArray(header) ? header.join(',') : header;
+    if (raw.trim() === '*') return true;
+    return raw
+      .split(',')
+      .map((t) => t.trim().replace(/^W\//, '').replace(/^"|"$/g, ''))
+      .some((t) => t === sha256);
+  }
+}
@@ -0,0 +1,19 @@
+import { Global, Module } from '@nestjs/common';
+import { SandboxController } from './sandbox.controller';
+import { SandboxStore } from './sandbox.store';
+
+/**
+ * In-RAM blob sandbox: a SINGLE shared SandboxStore (the @Injectable singleton)
+ * is written to by the stash tool (via McpService / AiChatToolsService) and read
+ * back by the anonymous SandboxController. Marked @Global so the same store
+ * instance is injectable everywhere without import churn — put() and get() MUST
+ * hit the same Map. EnvironmentService (caps/TTL/public URL) is provided by the
+ * global EnvironmentModule.
+ */
+@Global()
+@Module({
+  controllers: [SandboxController],
+  providers: [SandboxStore],
+  exports: [SandboxStore],
+})
+export class SandboxModule {}
@@ -0,0 +1,133 @@
+import { createHash } from 'node:crypto';
+import { SandboxStore } from './sandbox.store';
+
+// Build a minimal EnvironmentService stub with overridable caps/TTL.
+function makeEnv(
+  overrides: Partial<{
+    ttlMs: number;
+    maxBytes: number;
+    maxImageBytes: number;
+    maxTotalBytes: number;
+  }> = {},
+) {
+  const cfg = {
+    ttlMs: 3_600_000,
+    maxBytes: 8_388_608,
+    maxImageBytes: 20_971_520,
+    maxTotalBytes: 134_217_728,
+    ...overrides,
+  };
+  return {
+    getSandboxTtlMs: () => cfg.ttlMs,
+    getSandboxMaxBytes: () => cfg.maxBytes,
+    getSandboxMaxImageBytes: () => cfg.maxImageBytes,
+    getSandboxMaxTotalBytes: () => cfg.maxTotalBytes,
+    getSandboxPublicUrl: () => 'https://example.test',
+  } as any;
+}
+
+const UUID_RE =
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+
+describe('SandboxStore', () => {
+  let store: SandboxStore;
+
+  afterEach(() => {
+    // Clear the unref'd sweep interval so it never leaks across tests.
+    store?.onModuleDestroy();
+    jest.useRealTimers();
+  });
+
+  it('put/get round-trips the exact bytes + mime and returns a UUID id', () => {
+    store = new SandboxStore(makeEnv());
+    const buf = Buffer.from('{"type":"doc","content":[]}', 'utf8');
+
+    const res = store.put(buf, 'application/json');
+    expect(res.id).toMatch(UUID_RE);
+    expect(res.size).toBe(buf.length);
+
+    const entry = store.get(res.id);
+    expect(entry).toBeDefined();
+    expect(entry!.buf.equals(buf)).toBe(true);
+    expect(entry!.mime).toBe('application/json');
+  });
+
+  it('computes sha256 over the body (matches a manual digest)', () => {
+    store = new SandboxStore(makeEnv());
+    const buf = Buffer.from('hello sandbox', 'utf8');
+    const expected = createHash('sha256').update(buf).digest('hex');
+
+    const res = store.put(buf, 'text/plain');
+    expect(res.sha256).toBe(expected);
+    expect(store.get(res.id)!.sha256).toBe(expected);
+  });
+
+  it('returns undefined for a missing id', () => {
+    store = new SandboxStore(makeEnv());
+    expect(store.get('11111111-1111-1111-1111-111111111111')).toBeUndefined();
+  });
+
+  it('lazily expires entries past the TTL (get returns undefined)', () => {
+    jest.useFakeTimers();
+    jest.setSystemTime(new Date('2026-01-01T00:00:00Z'));
+    store = new SandboxStore(makeEnv({ ttlMs: 1000 }));
+    const res = store.put(Buffer.from('x'), 'text/plain');
+
+    expect(store.get(res.id)).toBeDefined();
+    jest.setSystemTime(new Date('2026-01-01T00:00:02Z')); // +2s > 1s TTL
+    expect(store.get(res.id)).toBeUndefined();
+    // Eviction also frees the byte accounting.
+    expect(store.bytes).toBe(0);
+  });
+
+  it('background sweep drops expired entries without a get()', () => {
+    jest.useFakeTimers();
+    jest.setSystemTime(new Date('2026-01-01T00:00:00Z'));
+    store = new SandboxStore(makeEnv({ ttlMs: 1000 }));
+    store.put(Buffer.from('x'), 'text/plain');
+    expect(store.size).toBe(1);
+
+    jest.setSystemTime(new Date('2026-01-01T00:01:30Z')); // past TTL
+    jest.advanceTimersByTime(60_000); // fire the sweep interval
+    expect(store.size).toBe(0);
+  });
+
+  it('rejects a non-image blob over SANDBOX_MAX_BYTES', () => {
+    store = new SandboxStore(makeEnv({ maxBytes: 16 }));
+    expect(() => store.put(Buffer.alloc(17), 'application/json')).toThrow(
+      /per-blob cap/,
+    );
+  });
+
+  it('uses the larger image cap for image/* blobs', () => {
+    // 100 bytes exceeds the doc cap (16) but fits the image cap (1024).
+    store = new SandboxStore(makeEnv({ maxBytes: 16, maxImageBytes: 1024 }));
+    expect(() => store.put(Buffer.alloc(100), 'image/png')).not.toThrow();
+    // SVG counts as an image too.
+    expect(() => store.put(Buffer.alloc(100), 'image/svg+xml')).not.toThrow();
+  });
+
+  it('evicts oldest entries when the total cap would be exceeded', () => {
+    // Total cap 250 bytes; each blob 100 bytes -> only 2 fit at a time.
+    store = new SandboxStore(
+      makeEnv({ maxTotalBytes: 250, maxBytes: 1024 }),
+    );
+    const a = store.put(Buffer.alloc(100), 'application/json');
+    const b = store.put(Buffer.alloc(100), 'application/json');
+    const c = store.put(Buffer.alloc(100), 'application/json'); // evicts a
+
+    expect(store.get(a.id)).toBeUndefined(); // oldest evicted
+    expect(store.get(b.id)).toBeDefined();
+    expect(store.get(c.id)).toBeDefined();
+    expect(store.bytes).toBeLessThanOrEqual(250);
+  });
+
+  it('rejects a single blob larger than the whole total cap', () => {
+    store = new SandboxStore(
+      makeEnv({ maxTotalBytes: 50, maxBytes: 1024 }),
+    );
+    expect(() => store.put(Buffer.alloc(100), 'application/json')).toThrow(
+      /total store cap/,
+    );
+  });
+});
@@ -0,0 +1,129 @@
+import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
+import { createHash, randomUUID } from 'node:crypto';
+import { EnvironmentService } from '../environment/environment.service';
+
+// In-RAM, process-local blob store. No disk, no DB. Ephemeral by design: a
+// restart empties it. A blob is addressed by an unguessable randomUUID() which
+// IS the read capability — there are NO tokens. Each blob is immutable (its id
+// never maps to changing content), so its sha256 is a perfect strong ETag.
+export interface SandboxEntry {
+  buf: Buffer;
+  mime: string;
+  sha256: string;
+  expiresAt: number;
+}
+
+export interface SandboxPutResult {
+  id: string;
+  sha256: string;
+  size: number;
+  expiresAt: number;
+}
+
+@Injectable()
+export class SandboxStore implements OnModuleDestroy {
+  private readonly logger = new Logger(SandboxStore.name);
+  // Map preserves insertion order, so the first key is the oldest entry — used
+  // for FIFO eviction when the total-bytes RAM guard is exceeded.
+  private readonly map = new Map<string, SandboxEntry>();
+  private totalBytes = 0;
+
+  // Background sweep clears expired entries so never-fetched blobs do not linger
+  // until the next get(). unref()'d so it never holds the event loop open;
+  // cleared on module destroy. Mirrors the sweepTimer pattern in
+  // integrations/mcp/mcp.service.ts and packages/mcp/src/http.ts.
+  private readonly sweepIntervalMs = 60_000;
+  private readonly sweepTimer: NodeJS.Timeout;
+
+  constructor(private readonly environmentService: EnvironmentService) {
+    this.sweepTimer = setInterval(() => {
+      try {
+        this.sweep();
+      } catch (err) {
+        this.logger.error('Sandbox sweep failed', err as Error);
+      }
+    }, this.sweepIntervalMs);
+    this.sweepTimer.unref?.();
+  }
+
+  onModuleDestroy(): void {
+    clearInterval(this.sweepTimer);
+  }
+
+  /**
+   * Store a blob and return its read capability id + integrity metadata. The
+   * per-blob cap is chosen by mime (images get the larger image cap), and the
+   * total-store RAM guard evicts oldest entries to make room. Throws a clear
+   * error when a single blob cannot fit even after eviction. Blob bodies are
+   * never logged.
+   */
+  put(buf: Buffer, mime: string): SandboxPutResult {
+    const perBlobCap = mime.startsWith('image/')
+      ? this.environmentService.getSandboxMaxImageBytes()
+      : this.environmentService.getSandboxMaxBytes();
+    if (buf.length > perBlobCap) {
+      throw new Error(
+        `Sandbox blob of ${buf.length} bytes exceeds the ${perBlobCap}-byte per-blob cap`,
+      );
+    }
+
+    const maxTotal = this.environmentService.getSandboxMaxTotalBytes();
+    if (buf.length > maxTotal) {
+      throw new Error(
+        `Sandbox blob of ${buf.length} bytes exceeds the total store cap of ${maxTotal} bytes`,
+      );
+    }
+
+    // Drop expired entries first, then evict oldest until the new blob fits.
+    this.sweep();
+    while (this.totalBytes + buf.length > maxTotal && this.map.size > 0) {
+      const oldest = this.map.keys().next().value as string;
+      this.evict(oldest);
+    }
+
+    const id = randomUUID();
+    const sha256 = createHash('sha256').update(buf).digest('hex');
+    const expiresAt = Date.now() + this.environmentService.getSandboxTtlMs();
+    this.map.set(id, { buf, mime, sha256, expiresAt });
+    this.totalBytes += buf.length;
+    return { id, sha256, size: buf.length, expiresAt };
+  }
+
+  /** Returns the entry, or undefined if missing OR expired (lazy expiry). */
+  get(id: string): SandboxEntry | undefined {
+    const entry = this.map.get(id);
+    if (!entry) return undefined;
+    if (entry.expiresAt <= Date.now()) {
+      this.evict(id);
+      return undefined;
+    }
+    return entry;
+  }
+
+  /** Current number of live entries (test/diagnostic helper). */
+  get size(): number {
+    return this.map.size;
+  }
+
+  /** Current total bytes held (test/diagnostic helper). */
+  get bytes(): number {
+    return this.totalBytes;
+  }
+
+  private evict(id: string): void {
+    const entry = this.map.get(id);
+    if (entry) {
+      this.totalBytes -= entry.buf.length;
+      this.map.delete(id);
+    }
+  }
+
+  private sweep(): void {
+    const now = Date.now();
+    for (const [id, entry] of this.map) {
+      if (entry.expiresAt <= now) {
+        this.evict(id);
+      }
+    }
+  }
+}