feat(metrics): #402 pass 1 — collab-cycle observability (load/lifecycle/connect/auth)

Extends the #355 perf-metrics registry (all behind the METRICS_PORT hard
gate — nullable instruments, no-op helpers when unset). New families:

- collab_doc_load_duration_seconds{size_bucket} — onLoadDocument timed on
  the real DB-load paths only (the already-loaded early-return is skipped).
- size_bucket added to collab_store_duration_seconds; storeDocument returns
  the ydoc byte length (reusing its single Y.encodeStateAsUpdate, no second
  encode) so onStoreDocument observes size without extra cost.
- collab_docs_open (gauge, read-on-scrape via collect() from
  hocuspocus.getDocumentsCount() — never inc/dec'd, so it can't drift) +
  collab_doc_loads_total / collab_doc_unloads_total (afterLoad/afterUnload).
- collab_connect_duration_seconds — onConnect->connected, correlated by a
  WeakMap keyed on the shared request (leak-free; the current client uses
  one socket per document).
- collab_auth_duration_seconds — wraps onAuthenticate (success and failure).
- sizeBucket() shared helper (lt64k|lt256k|lt1m|ge1m, 4 bounded values).

The mcp_tool_duration_seconds histogram + collab_connect_timeouts_total
counter helpers are registered here but wired in pass 2 (MCP callback).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

apps/server/src/integrations/metrics/metrics.constants.ts

@@ -2,6 +2,11 @@
  * Perf-metrics contract (#355). These names/labels are FIXED by the already
  * deployed scrape+dashboard infra (VictoriaMetrics scraping docmost:9464,
  * Grafana dashboards, alerts). Do NOT rename them.
+ *
+ * #402 extends the #355 table with the collab-lifecycle + MCP-tool families
+ * (grouped below). These are the fixed contract from #402; same "do not rename"
+ * rule applies. Server-side families land in Pass 1; the MCP-tool histogram is
+ * fed by the MCP callback in a later pass but its NAME is fixed here now.
  */
 export const METRIC_HTTP_REQUEST_DURATION = 'http_request_duration_seconds';
 export const METRIC_DB_QUERY_DURATION = 'db_query_duration_seconds';
@@ -9,6 +14,17 @@ export const METRIC_BULLMQ_QUEUE_DEPTH = 'bullmq_queue_depth';
 export const METRIC_BULLMQ_JOB_DURATION = 'bullmq_job_duration_seconds';
 export const METRIC_COLLAB_STORE_DURATION = 'collab_store_duration_seconds';
 
+// #402 additions — collaboration lifecycle + MCP tool timing.
+export const METRIC_COLLAB_LOAD_DURATION = 'collab_doc_load_duration_seconds';
+export const METRIC_COLLAB_DOCS_OPEN = 'collab_docs_open';
+export const METRIC_COLLAB_DOC_LOADS_TOTAL = 'collab_doc_loads_total';
+export const METRIC_COLLAB_DOC_UNLOADS_TOTAL = 'collab_doc_unloads_total';
+export const METRIC_COLLAB_CONNECT_DURATION = 'collab_connect_duration_seconds';
+export const METRIC_COLLAB_CONNECT_TIMEOUTS_TOTAL =
+  'collab_connect_timeouts_total';
+export const METRIC_COLLAB_AUTH_DURATION = 'collab_auth_duration_seconds';
+export const METRIC_MCP_TOOL_DURATION = 'mcp_tool_duration_seconds';
+
 // Histogram buckets (seconds). Chosen to give useful p50/p95/p99 resolution
 // for typical web/DB latencies without exploding series cardinality.
 export const HTTP_BUCKETS = [
@@ -23,6 +39,12 @@ export const COLLAB_BUCKETS = [
 export const JOB_BUCKETS = [
   0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120,
 ];
+// #402 — MCP tool-call latency. Same shape as COLLAB_BUCKETS but stretched to
+// 10s at the top: an MCP tool round-trip (LLM-driven doc ops) can be slower
+// than a single collab store, so keep resolution out to 10s.
+export const MCP_TOOL_BUCKETS = [
+  0.005, 0.025, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
+];
 
 /**
  * Extract the first SQL token (select/insert/update/delete/...) from a query,
@@ -58,6 +80,30 @@ export function firstSqlToken(sql: string | undefined): string {
   return KNOWN_SQL_TOKENS.has(token) ? token : 'other';
 }
 
+/**
+ * #402 — bucket a document byte size into ONE of four fixed labels for the
+ * collab load/store histograms' `size_bucket` label. Using the raw byte count
+ * would be a continuous, unbounded label; four coarse buckets keep series
+ * cardinality bounded (each of collab_doc_load / collab_store gets ×4 series).
+ *
+ * The SAME function is shared by both the load and store paths so their buckets
+ * can never drift apart. Non-finite / negative sizes collapse to the smallest
+ * bucket ('lt64k') as a safe default (they can't be legitimately huge and we
+ * must never throw here — this runs on every store/load when metrics are on).
+ */
+// Module-const thresholds, built once (not per observe).
+const SIZE_THRESHOLDS = { lt64k: 65536, lt256k: 262144, lt1m: 1048576 } as const;
+
+export function sizeBucket(
+  bytes: number | undefined | null,
+): 'lt64k' | 'lt256k' | 'lt1m' | 'ge1m' {
+  if (bytes == null || !Number.isFinite(bytes) || bytes < 0) return 'lt64k';
+  if (bytes < SIZE_THRESHOLDS.lt64k) return 'lt64k';
+  if (bytes < SIZE_THRESHOLDS.lt256k) return 'lt256k';
+  if (bytes < SIZE_THRESHOLDS.lt1m) return 'lt1m';
+  return 'ge1m';
+}
+
 /**
  * Whether an HTTP response must be EXCLUDED from http_request_duration_seconds.
  *

apps/server/src/integrations/metrics/metrics.registry.ts

@@ -1,5 +1,6 @@
 import {
   collectDefaultMetrics,
+  Counter,
   Histogram,
   Gauge,
   Registry,
@@ -9,11 +10,21 @@ import {
   DB_BUCKETS,
   HTTP_BUCKETS,
   JOB_BUCKETS,
+  MCP_TOOL_BUCKETS,
   METRIC_BULLMQ_JOB_DURATION,
   METRIC_BULLMQ_QUEUE_DEPTH,
+  METRIC_COLLAB_AUTH_DURATION,
+  METRIC_COLLAB_CONNECT_DURATION,
+  METRIC_COLLAB_CONNECT_TIMEOUTS_TOTAL,
+  METRIC_COLLAB_DOC_LOADS_TOTAL,
+  METRIC_COLLAB_DOC_UNLOADS_TOTAL,
+  METRIC_COLLAB_DOCS_OPEN,
+  METRIC_COLLAB_LOAD_DURATION,
   METRIC_COLLAB_STORE_DURATION,
   METRIC_DB_QUERY_DURATION,
   METRIC_HTTP_REQUEST_DURATION,
+  METRIC_MCP_TOOL_DURATION,
+  sizeBucket,
 } from './metrics.constants';
 
 /**
@@ -39,7 +50,24 @@ let httpHist: Histogram<'method' | 'route' | 'status'> | null = null;
 let dbHist: Histogram<'op'> | null = null;
 let queueDepthGauge: Gauge<'queue'> | null = null;
 let jobHist: Histogram<'queue'> | null = null;
-let collabHist: Histogram | null = null;
+// #402 — collab store now carries a size_bucket label (see sizeBucket()).
+let collabHist: Histogram<'size_bucket'> | null = null;
+// #402 collab-lifecycle + MCP instruments.
+let collabLoadHist: Histogram<'size_bucket'> | null = null;
+let docsOpenGauge: Gauge | null = null;
+let docLoadsCounter: Counter | null = null;
+let docUnloadsCounter: Counter | null = null;
+let connectTimeoutsCounter: Counter | null = null;
+let collabConnectHist: Histogram | null = null;
+let collabAuthHist: Histogram | null = null;
+let mcpToolHist: Histogram<'tool'> | null = null;
+
+// #402 — read-on-scrape source for collab_docs_open. The gauge is NEVER
+// inc/dec'd (that drifts under crashes/handoffs); instead its collect() callback
+// pulls the authoritative live count from here on every scrape. Registered once,
+// gated, by the collaboration gateway via registerDocsOpenSource(). Null until
+// then → the collect() is a no-op.
+let docsOpenSource: (() => number) | null = null;
 
 function init(): void {
   if (registry || !enabled) return;
@@ -82,10 +110,71 @@ function init(): void {
 
   collabHist = new Histogram({
     name: METRIC_COLLAB_STORE_DURATION,
-    help: 'Collaboration onStoreDocument duration in seconds',
+    help: 'Collaboration onStoreDocument duration in seconds, by document size bucket',
+    labelNames: ['size_bucket'],
     buckets: COLLAB_BUCKETS,
     registers: [registry],
   });
+
+  collabLoadHist = new Histogram({
+    name: METRIC_COLLAB_LOAD_DURATION,
+    help: 'Collaboration onLoadDocument DB-load duration in seconds, by document size bucket',
+    labelNames: ['size_bucket'],
+    buckets: COLLAB_BUCKETS,
+    registers: [registry],
+  });
+
+  docsOpenGauge = new Gauge({
+    name: METRIC_COLLAB_DOCS_OPEN,
+    help: 'Number of collaboration documents currently open in memory',
+    registers: [registry],
+    // Read-on-scrape: pull the live count from the registered source (the
+    // hocuspocus instance) so the value can never drift. No-op until a source
+    // is registered.
+    collect() {
+      if (docsOpenSource) this.set(docsOpenSource());
+    },
+  });
+
+  docLoadsCounter = new Counter({
+    name: METRIC_COLLAB_DOC_LOADS_TOTAL,
+    help: 'Total collaboration documents loaded into memory',
+    registers: [registry],
+  });
+
+  docUnloadsCounter = new Counter({
+    name: METRIC_COLLAB_DOC_UNLOADS_TOTAL,
+    help: 'Total collaboration documents unloaded from memory',
+    registers: [registry],
+  });
+
+  connectTimeoutsCounter = new Counter({
+    name: METRIC_COLLAB_CONNECT_TIMEOUTS_TOTAL,
+    help: 'Total collaboration connection setup timeouts',
+    registers: [registry],
+  });
+
+  collabConnectHist = new Histogram({
+    name: METRIC_COLLAB_CONNECT_DURATION,
+    help: 'Collaboration connection acceptance duration in seconds (onConnect→connected)',
+    buckets: COLLAB_BUCKETS,
+    registers: [registry],
+  });
+
+  collabAuthHist = new Histogram({
+    name: METRIC_COLLAB_AUTH_DURATION,
+    help: 'Collaboration onAuthenticate duration in seconds',
+    buckets: COLLAB_BUCKETS,
+    registers: [registry],
+  });
+
+  mcpToolHist = new Histogram({
+    name: METRIC_MCP_TOOL_DURATION,
+    help: 'MCP tool-call duration in seconds, by tool name',
+    labelNames: ['tool'],
+    buckets: MCP_TOOL_BUCKETS,
+    registers: [registry],
+  });
 }
 
 // Runs once when this module is first imported. Safe to call again (idempotent).
@@ -121,6 +210,46 @@ export function observeJobDuration(queue: string, seconds: number): void {
   jobHist?.observe({ queue }, seconds);
 }
 
-export function observeCollabStore(seconds: number): void {
-  collabHist?.observe(seconds);
+export function observeCollabStore(bytes: number, seconds: number): void {
+  collabHist?.observe({ size_bucket: sizeBucket(bytes) }, seconds);
+}
+
+export function observeCollabLoad(bytes: number, seconds: number): void {
+  collabLoadHist?.observe({ size_bucket: sizeBucket(bytes) }, seconds);
+}
+
+/**
+ * Register the live open-document count source for the collab_docs_open gauge.
+ * Called ONCE, gated by isMetricsEnabled(), by the collaboration gateway. The
+ * gauge reads this on every scrape (collect()); nothing inc/dec's it.
+ */
+export function registerDocsOpenSource(fn: () => number): void {
+  docsOpenSource = fn;
+}
+
+export function incDocLoad(): void {
+  docLoadsCounter?.inc();
+}
+
+export function incDocUnload(): void {
+  docUnloadsCounter?.inc();
+}
+
+export function incConnectTimeout(): void {
+  connectTimeoutsCounter?.inc();
+}
+
+export function observeCollabConnect(seconds: number): void {
+  collabConnectHist?.observe(seconds);
+}
+
+export function observeCollabAuth(seconds: number): void {
+  collabAuthHist?.observe(seconds);
+}
+
+export function observeMcpTool(tool: string, seconds: number): void {
+  // `tool` MUST be a bounded, registration-derived MCP tool name (the caller
+  // guarantees it comes from the registered-tool set) — never free-form input —
+  // so this label stays low-cardinality with no 'other' bucketing needed.
+  mcpToolHist?.observe({ tool }, seconds);
 }

apps/server/src/integrations/metrics/metrics.spec.ts

@@ -1,6 +1,23 @@
 import { FastifyRequest } from 'fastify';
 import { resolveRouteLabel } from './http-metrics.hook';
-import { firstSqlToken, isStreamingResponse } from './metrics.constants';
+import {
+  firstSqlToken,
+  isStreamingResponse,
+  sizeBucket,
+} from './metrics.constants';
+import {
+  getMetricsRegistry,
+  incConnectTimeout,
+  incDocLoad,
+  incDocUnload,
+  isMetricsEnabled,
+  observeCollabAuth,
+  observeCollabConnect,
+  observeCollabLoad,
+  observeCollabStore,
+  observeMcpTool,
+  registerDocsOpenSource,
+} from './metrics.registry';
 
 describe('resolveRouteLabel (histogram route label)', () => {
   it('uses the ROUTE TEMPLATE, never the raw URL', () => {
@@ -123,3 +140,67 @@ describe('firstSqlToken (bounded db label)', () => {
     expect(firstSqlToken('vacuum analyze')).toBe('other');
   });
 });
+
+describe('sizeBucket (#402 bounded size label)', () => {
+  it('maps sizes to the four fixed buckets at their boundaries', () => {
+    // Boundaries are exclusive-upper: <65536 → lt64k, etc.
+    expect(sizeBucket(0)).toBe('lt64k');
+    expect(sizeBucket(65535)).toBe('lt64k');
+    expect(sizeBucket(65536)).toBe('lt256k');
+    expect(sizeBucket(262143)).toBe('lt256k');
+    expect(sizeBucket(262144)).toBe('lt1m');
+    expect(sizeBucket(1048575)).toBe('lt1m');
+    expect(sizeBucket(1048576)).toBe('ge1m');
+    expect(sizeBucket(5_000_000)).toBe('ge1m');
+  });
+
+  it('falls back to the smallest bucket for invalid sizes', () => {
+    // Non-finite / negative / nullish collapse to the smallest, safe default.
+    expect(sizeBucket(-1)).toBe('lt64k');
+    expect(sizeBucket(NaN)).toBe('lt64k');
+    expect(sizeBucket(Infinity)).toBe('lt64k');
+    expect(sizeBucket(undefined)).toBe('lt64k');
+    expect(sizeBucket(null)).toBe('lt64k');
+  });
+
+  it('only ever returns one of the four fixed labels (bounded cardinality)', () => {
+    const labels = new Set(
+      [0, 65536, 262144, 1048576, -5, NaN].map((b) => sizeBucket(b)),
+    );
+    for (const l of labels) {
+      expect(['lt64k', 'lt256k', 'lt1m', 'ge1m']).toContain(l);
+    }
+  });
+});
+
+describe('metrics helpers are safe no-ops when METRICS_PORT is unset', () => {
+  // These specs run without METRICS_PORT, so the registry is never created and
+  // every observe/inc/set helper must be a cheap `?.` no-op that never throws.
+  beforeAll(() => {
+    // Guard the contract this suite depends on: if a CI env set METRICS_PORT,
+    // the assertions below would be meaningless, so fail loudly instead.
+    expect(process.env.METRICS_PORT).toBeUndefined();
+  });
+
+  it('reports metrics disabled and a null registry', () => {
+    expect(isMetricsEnabled()).toBe(false);
+    expect(getMetricsRegistry()).toBeNull();
+  });
+
+  it('does not throw from any #402 collab/MCP helper', () => {
+    expect(() => {
+      observeCollabLoad(123456, 0.01);
+      observeCollabStore(123456, 0.02);
+      observeCollabConnect(0.03);
+      observeCollabAuth(0.04);
+      observeMcpTool('some-tool', 0.05);
+      incDocLoad();
+      incDocUnload();
+      incConnectTimeout();
+      // Registering a source must not create the gauge or invoke the fn.
+      registerDocsOpenSource(() => {
+        throw new Error('docsOpenSource must NOT be called when disabled');
+      });
+    }).not.toThrow();
+  });
+});