Compare commits
38 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| fdc37de3e8 | |||
| 4a750c1e7f | |||
| ea7c4d7cd2 | |||
| 255024fbe2 | |||
| b934fb2292 | |||
| 14da295185 | |||
| de8f9c804c | |||
| 8ebdfe156f | |||
| 31176d5f93 | |||
| 5f0c49d47c | |||
| b2e5b51420 | |||
| 26d9a70a1c | |||
| e1ebd79f30 | |||
| fc83ea28ab | |||
| ee15278c8f | |||
| 09ab92eccf | |||
| fe5bd159c4 | |||
| f12b685698 | |||
| f6fc914c95 | |||
| 1d89cc2058 | |||
| 70a9e2a9cb | |||
| 5d8083f8ff | |||
| 3e945305c8 | |||
| 6d7dba970c | |||
| 512bcba5f3 | |||
| 5c1ab9c7b5 | |||
| 14d7b21df0 | |||
| 363f20ab75 | |||
| 3411bda2d1 | |||
| e670f7498a | |||
| 9a8671c3af | |||
| bec2156e96 | |||
| acc705de19 | |||
| a49872444e | |||
| ffc38ea2ca | |||
| 2a951df096 | |||
| 2f23cf4b65 | |||
| d287c15db4 |
@@ -124,6 +124,40 @@ MCP_DOCMOST_PASSWORD=
|
||||
# MCP_TOKEN=
|
||||
# MCP_SESSION_IDLE_MS=1800000
|
||||
#
|
||||
# --- MCP collaboration write path: concurrency + rights-staleness (#449) ------
|
||||
# MCP content writes (update_page, insert/replace nodes, comments-in-body, etc.)
|
||||
# go over the collaboration websocket and are serialized PER PAGE by an
|
||||
# in-process mutex (a module-level Map, one promise-chain per page UUID). This
|
||||
# guarantees no two MCP writes on the SAME page overlap and clobber each other.
|
||||
#
|
||||
# DEPLOY REQUIREMENT — SINGLE INSTANCE or STICKY SESSIONS. The mutex is
|
||||
# process-local. Behind a multi-replica load balancer WITHOUT sticky sessions,
|
||||
# two replicas can each "hold" the lock for the same page at the same time and
|
||||
# serialization is silently lost (concurrent full-document writes race on the
|
||||
# live Yjs fragment). Run the MCP/app as a SINGLE instance, OR pin a page's
|
||||
# traffic to one replica (sticky sessions / consistent hashing on page id). The
|
||||
# same constraint applies to the RAM-only stash_page blob store above. There is
|
||||
# deliberately no cross-process (e.g. Postgres advisory) lock yet — this is a
|
||||
# CONSCIOUS documented constraint, not an oversight (#449).
|
||||
#
|
||||
# To reduce connect-storms the write path caches ONE live collab session per
|
||||
# (wsUrl, page, token). Tunables (all optional; defaults are safe):
|
||||
# MCP_COLLAB_SESSION_IDLE_MS=60000 # idle TTL, reset per op; 0 disables cache
|
||||
# MCP_COLLAB_SESSION_MAX_ENTRIES=32 # LRU cap on cached sessions
|
||||
# MCP_COLLAB_TOKEN_TTL_MS=300000 # per-client collab-token cache (5 min)
|
||||
#
|
||||
# RIGHTS-STALENESS TRADE-OFF. A cached collab session writes under the token
|
||||
# captured at CONNECT time, and the collab-token cache reuses a token for its TTL.
|
||||
# So if a user's access to a page is REVOKED, MCP writes on an already-open
|
||||
# session may keep succeeding until the session ages out. MCP_COLLAB_SESSION_MAX_AGE_MS
|
||||
# is the HARD lifetime (checked at each acquire) that BOUNDS this window: after it,
|
||||
# the session is torn down and the next write re-auths with a fresh token, picking
|
||||
# up the revocation. Default 10 min. LOWER it to shorten the revocation lag at the
|
||||
# cost of more reconnects; RAISE it to reduce reconnects at the cost of a longer
|
||||
# stale-rights window. There is intentionally no push-based cache invalidation on
|
||||
# a rights change — this bounded window is the accepted trade-off (#449).
|
||||
# MCP_COLLAB_SESSION_MAX_AGE_MS=600000
|
||||
#
|
||||
# BLOB SANDBOX (stash_page). An in-RAM, process-local store that hands large page
|
||||
# content + images to an external consumer WITHOUT bloating the model context or
|
||||
# requiring Docmost auth. The stash_page tool serializes a page, mirrors its
|
||||
@@ -301,6 +335,20 @@ MCP_DOCMOST_PASSWORD=
|
||||
# VictoriaMetrics/Prometheus reaching it as <host>:<port>/metrics.
|
||||
# METRICS_PORT=9464
|
||||
#
|
||||
# METRICS_BIND — interface the /metrics listener binds to. DEFAULT 127.0.0.1
|
||||
# (loopback only), so the unauthenticated endpoint is NOT exposed on all
|
||||
# interfaces. If the scraper runs in a SEPARATE container and reaches this as
|
||||
# docmost:9464, set METRICS_BIND=0.0.0.0 — but then also set METRICS_TOKEN
|
||||
# and/or keep the port on a private network, since /metrics is otherwise open.
|
||||
# METRICS_BIND=127.0.0.1
|
||||
#
|
||||
# METRICS_TOKEN — optional Bearer token guarding /metrics. When set, every
|
||||
# scrape MUST send `Authorization: Bearer <token>` (others get 401). Configure
|
||||
# the scraper with the same bearer token (e.g. VictoriaMetrics/vmagent
|
||||
# `bearer_token`, Prometheus `authorization.credentials`). Leave unset only
|
||||
# when the endpoint is bound to loopback or an otherwise-trusted network.
|
||||
# METRICS_TOKEN=
|
||||
#
|
||||
# 2) CLIENT_TELEMETRY_ENABLED — the public client perf-telemetry sink.
|
||||
# OFF by default. When true, the unauthenticated POST /api/telemetry/vitals
|
||||
# endpoint is registered and browsers collect + send web-vitals / editor
|
||||
|
||||
@@ -157,6 +157,12 @@ jobs:
|
||||
- name: Build prosemirror-markdown
|
||||
run: pnpm --filter @docmost/prosemirror-markdown build
|
||||
|
||||
# docmost-client.loader.ts type-imports from @docmost/mcp (issue #446); its
|
||||
# build/ is gitignored and `test:e2e` type-checks, so build it here or tsc
|
||||
# fails with TS2307 (mirrors the e2e-mcp / mcp-server-parity jobs).
|
||||
- name: Build mcp
|
||||
run: pnpm --filter @docmost/mcp build
|
||||
|
||||
- name: Run migrations
|
||||
run: pnpm --filter ./apps/server migration:latest
|
||||
|
||||
|
||||
@@ -5,6 +5,139 @@ repository. It has two layers: **how to run a task end-to-end** (the
|
||||
sections below), and **how the codebase is built** (the technical sections
|
||||
further down, formerly in `CLAUDE.md`).
|
||||
|
||||
## ARCHITECTURAL INVARIANTS — NON-NEGOTIABLE
|
||||
|
||||
THE TEN RULES BELOW ARE HARD CONSTRAINTS. Each one was paid for with a real
|
||||
production incident or a multi-PR bug chain in THIS repository (cited inline).
|
||||
They override convenience, deadlines and "it's just a small feature". A PR that
|
||||
violates any of them MUST be rejected in review regardless of how good the rest
|
||||
of it is. If a task genuinely seems to require breaking one — STOP and raise it
|
||||
with the owner; do not code around it.
|
||||
|
||||
### 1. EVERY BUFFER, CACHE, HISTORY AND PAYLOAD HAS AN EXPLICIT SIZE BUDGET
|
||||
|
||||
Nothing accumulates unboundedly. A row/item cap is NOT a byte cap. Anything
|
||||
replayed to a model, buffered in memory, persisted per step, or refetched by a
|
||||
poll must state its budget in bytes/tokens and enforce it. Rewriting a growing
|
||||
structure in full on every increment is FORBIDDEN — append or diff instead;
|
||||
O(n²) write/serialize patterns do not pass review.
|
||||
(Paid for by: full-row rewrite on every agent step — hundreds of MB of Postgres
|
||||
writes per 50-step run, with every tool output serialized twice; unbounded
|
||||
history replay killing long chats on the provider context window; 32 MB replay
|
||||
buffers per active run.)
|
||||
|
||||
### 2. EVERYTHING LONG-RUNNING TERMINATES BY CONSTRUCTION
|
||||
|
||||
Every run / row / session / lease / subscriber / queue entry must define AT
|
||||
DESIGN TIME: its owner; every terminal state; who writes the terminal state on
|
||||
EVERY path (success, error, abort, disconnect in each phase, process restart);
|
||||
retries for the terminal write; and a periodic sweeper that does not depend on
|
||||
a reboot. A best-effort terminal write with no retry and no sweep is FORBIDDEN.
|
||||
(Paid for by: assistant rows stuck 'streaming' forever; runs stuck 'running'
|
||||
409-locking their chat until a restart — the #183/#184 follow-up chain.)
|
||||
|
||||
### 3. EVERY AWAIT IS CANCELLABLE AND DEADLINED; NEVER BLOCK THE EVENT LOOP
|
||||
|
||||
Every async step inside a request or agent turn honors the turn's AbortSignal
|
||||
AND a wall-clock deadline — including in-app tools, lock queues and pagination
|
||||
loops, not just external calls. Synchronous CPU work beyond ~50 ms goes to a
|
||||
worker_thread. Promise.race DOES NOT cancel synchronous work — using it as a
|
||||
"timeout" for sync computation is forbidden (the timer only fires after the
|
||||
event loop is free again, i.e. after the damage is done).
|
||||
(Paid for by: in-app tools ignoring abortSignal and writing pages AFTER Stop;
|
||||
the synchronous ELK layout freezing every SSE stream in the process; the
|
||||
step-0 MCP handshake hang — #397.)
|
||||
|
||||
### 4. ONE SOURCE OF TRUTH; EVERYTHING ELSE IS A REBUILDABLE CACHE
|
||||
|
||||
Postgres is the authoritative state. Every in-memory structure (registries,
|
||||
caches, client stores) must be reconstructible from the DB and treated as
|
||||
lossy. The client renders SERVER-DECLARED state — "a run is active" is a server
|
||||
fact delivered as data, never inferred from side signals (204 vs 2xx, the
|
||||
flavor of a disconnect). A new feature must name the owner of each piece of
|
||||
state before implementation starts.
|
||||
(Paid for by: the strip/restore resume machinery, silently frozen UIs and
|
||||
ghost sends after unmount — the #381→#432→#456 chain.)
|
||||
|
||||
### 5. STATE MACHINES ARE EXPLICIT — ONE-SHOT FLAGS ARE FORBIDDEN
|
||||
|
||||
A complex lifecycle (chat thread, resume/reconnect, run) lives in a named-state
|
||||
automaton (reducer / enum) where every state has an owner and a rendered
|
||||
representation — including the failure states. Adding a boolean ref that one
|
||||
callback arms and another reads-and-clears is FORBIDDEN in the AI-chat client.
|
||||
New behavior = a new named state + explicit transitions, and the interruption
|
||||
matrix (disconnect in each phase × restart × stop × supersede) is enumerated at
|
||||
design time, not discovered one incident at a time.
|
||||
(Paid for by: 26 one-shot useRef flags in chat-thread.tsx and the drip of
|
||||
"one more missing transition" across #381→#386/#389→#432→#456.)
|
||||
|
||||
### 6. NO NEW MODE FORKS; A FLAG IS FOR ROLLOUT, THEN IT DIES
|
||||
|
||||
A behavior flag that forks a code path must ship with a written sunset
|
||||
condition; stacking a new flag onto the existing matrix without deleting or
|
||||
scheduling an old one is forbidden. While a temporary fork exists, BOTH sides
|
||||
must share identical lifecycle handling (abort semantics, error listeners,
|
||||
concurrency gates) — asymmetric forks are outlawed.
|
||||
(Paid for by: legacy vs autonomous divergence — the one-active-run gate and
|
||||
the socket 'error' listener each existing on only ONE side; 2^4 flag
|
||||
combinations each with different abort semantics.)
|
||||
|
||||
### 7. NO HAND-SYNCED MIRRORS — CODEGEN OR A CI PARITY TEST, NOTHING LESS
|
||||
|
||||
Two copies of the same knowledge (schema, tool registry, glyph map, probe
|
||||
body, hash/normalize algorithm, label list) require either generation from a
|
||||
single source or a CI test that FAILS on drift. A "mirror this change over
|
||||
there" comment is NOT a guard and does not pass review.
|
||||
(Paid for by: #293 — three drifting converter copies losing data; #447 —
|
||||
REGISTRY_STAMP covering only one of the mirrored files; ~10 still-unguarded
|
||||
mirrors across the MCP layer.)
|
||||
|
||||
### 8. CACHES, HEADERS, BUFFERS AND FSM TRANSITIONS GET AN INTEGRATION TEST OF THE OBSERVABLE PROPERTY
|
||||
|
||||
A unit test of a pure helper DOES NOT COUNT for these. Test the real header on
|
||||
the real HTTP response, the real cache hit under real token sources, the real
|
||||
transition under a really-killed socket. If the observable property cannot be
|
||||
tested, the design is wrong — fix the design, not the test.
|
||||
(Paid for by: #431→#439 — a cache keyed on a fresh-per-call JWT, so it NEVER
|
||||
hit and became prod incident #435 while its unit tests stayed green; and by
|
||||
the #352→#455 immutable-cache header silently overwritten by a framework
|
||||
default AFTER the unit-tested code ran.)
|
||||
|
||||
### 9. CLIENT INPUT IS HOSTILE UNTIL VALIDATED — ALSO BEFORE PERSISTENCE
|
||||
|
||||
Anything from the browser (message parts, ids, titles, selections, flags) is
|
||||
validated/sanitized BEFORE it is persisted into a row that will later be
|
||||
replayed into a prompt, a converter or another subsystem. A poisoned row must
|
||||
never be able to permanently brick a chat or a page on every subsequent read.
|
||||
(Paid for by: unvalidated UIMessage parts persisted verbatim — one bad row
|
||||
500s the chat on every later turn; #159 client-spoofed page titles; #388
|
||||
selection re-sanitized server-side for the same reason.)
|
||||
|
||||
### 10. FAILURES ARE LOUD AND SPECIFIC; SILENT DEGRADATION IS FORBIDDEN
|
||||
|
||||
Extends the error convention below: a fire-and-forget write is allowed ONLY
|
||||
with a metric or a greppable ERROR log; a degraded mode (dead cached MCP
|
||||
client, stopped poll, exhausted retries, evicted buffer) must be VISIBLE to
|
||||
the user or the operator. A feature that can quietly stop working — a frozen
|
||||
"streaming…" UI, a poll that silently gives up, a cache serving corpses — does
|
||||
not pass review.
|
||||
(Paid for by: the degraded poll's silent 10-minute death leaving a forever-
|
||||
"streaming" answer; dead MCP clients served from cache while every external
|
||||
tool call failed; #435 being caught in minutes ONLY because metrics — #403 —
|
||||
existed.)
|
||||
|
||||
## Default skill for feature design
|
||||
|
||||
For any feature-design request — the user hands over a raw feature idea, asks
|
||||
to design or think through a feature, or to draft an issue («спроектируй»,
|
||||
«продумай фичу», «составь ишью», "design X", "write an issue for X") — invoke
|
||||
the `orchestrator-feature-designer` skill (Skill tool) BEFORE any other work.
|
||||
It is the default operating mode for design work in this repository: research
|
||||
→ design checklist (R1–R10) → forks resolved with the human → adversarial
|
||||
self-attack → filed PR-sized issues. Do not design features or write issues
|
||||
ad-hoc while this skill is available. This does not apply to non-design work
|
||||
(bug fixes, reviews, retrospectives, refactors already specified by an issue).
|
||||
|
||||
## Task lifecycle
|
||||
|
||||
### 1. Start: sync with develop
|
||||
@@ -201,7 +334,7 @@ pnpm workspace (`pnpm@10.4.0`) orchestrated by **Nx**. Four workspace packages:
|
||||
| `apps/client` | `client` | React 18 + Vite + Mantine 8 + TanStack Query + Jotai | SPA frontend |
|
||||
| `packages/editor-ext` | `@docmost/editor-ext` | Tiptap/ProseMirror | Shared Tiptap node/mark extensions, imported by both the client and the server |
|
||||
| `packages/mcp` | `@docmost/mcp` | MCP SDK, Tiptap, Yjs | Standalone MCP server, also bundled into the server at `/mcp`. Consumes the shared converter/schema from `@docmost/prosemirror-markdown` (#293) — it no longer carries its own vendored converter/schema copy |
|
||||
| `packages/prosemirror-markdown` | `@docmost/prosemirror-markdown` | Tiptap, marked, jsdom | The single, canonical ProseMirror↔Markdown converter + Docmost schema mirror (#293). Consumed by `mcp`, `git-sync`, AND `apps/server` (server-side markdown import/export, #345); there is exactly ONE copy of the converter now |
|
||||
| `packages/prosemirror-markdown` | `@docmost/prosemirror-markdown` | Tiptap, marked; jsdom (Node only) | The single, canonical ProseMirror↔Markdown converter + Docmost schema mirror (#293). Consumed by `mcp`, `git-sync`, `apps/server` (server-side markdown import/export, #345), AND `apps/client` (markdown paste/copy + AI-chat render, via the `browser` entry — native `DOMParser`, no jsdom in the client bundle, #347); there is exactly ONE copy of the converter now |
|
||||
|
||||
`build` targets are Nx-cached and dependency-ordered (`dependsOn: ["^build"]`), so `editor-ext` builds before the apps. `nx.json` sets `affected.defaultBase: main`.
|
||||
|
||||
@@ -327,7 +460,7 @@ The API server is a Fastify app with a global `/api` prefix (`main.ts` excludes
|
||||
### Client structure
|
||||
Vite SPA. Code is organized by feature under `apps/client/src/features/*` (mirrors the server domains: `page`, `space`, `comment`, `ai-chat`, `editor`, …). Conventions:
|
||||
- **TanStack Query** for server state (one `queries/` file per feature), **Jotai** atoms for local/shared UI state, **Mantine 8** + CSS modules (`*.module.css`) + `postcss-preset-mantine` for UI.
|
||||
- The editor is Tiptap; shared node/mark extensions live in `packages/editor-ext` and are imported by **both the client and the server** (collaboration, schema, `canonicalizeFootnotes`) — editor schema changes often need to be made in `editor-ext`, not just the client. Server-side markdown import/export no longer lives in `editor-ext`: it goes through the canonical converter (#345, see below). The ProseMirror↔Markdown converter and its Docmost schema mirror now live in a SINGLE package, `@docmost/prosemirror-markdown` (#293), consumed by `mcp`, `git-sync`, and `apps/server` (#345) — do NOT reintroduce a per-package copy. `editor-ext` is the upstream source of the Tiptap schema; the package's `docmost-schema.ts` mirrors it and a serializer-contract test (`packages/prosemirror-markdown/test/serializer-contract.test.ts`) guards the boundary (every schema node must have a converter case), so a drift surfaces as a failing test rather than silent divergence. For the converter's property-testing and counterexample→fixture process (P1–P4 invariants, the `PROPERTY_SEED`/`PROPERTY_NUM_RUNS` knobs, and the nightly fuzz workflow), see `packages/prosemirror-markdown/README.md`.
|
||||
- The editor is Tiptap; shared node/mark extensions live in `packages/editor-ext` and are imported by **both the client and the server** (collaboration, schema, `canonicalizeFootnotes`) — editor schema changes often need to be made in `editor-ext`, not just the client. Server-side markdown import/export no longer lives in `editor-ext`: it goes through the canonical converter (#345, see below). The ProseMirror↔Markdown converter and its Docmost schema mirror now live in a SINGLE package, `@docmost/prosemirror-markdown` (#293), consumed by `mcp`, `git-sync`, `apps/server` (#345), and `apps/client` (#347) — do NOT reintroduce a per-package copy. The client uses the package's `browser` entry (`@docmost/prosemirror-markdown/browser`): markdown paste (`markdown-clipboard.ts`), copy-as-markdown, and AI-chat rendering now all go through the canonical converter, so the hand-written `marked`/`turndown` markdown layer that used to live in `editor-ext` was deleted (#347). The browser entry runs the HTML→DOM stage on the native `DOMParser`, so jsdom stays out of the client bundle. `editor-ext` is the upstream source of the Tiptap schema; the package's `docmost-schema.ts` mirrors it and a serializer-contract test (`packages/prosemirror-markdown/test/serializer-contract.test.ts`) guards the boundary (every schema node must have a converter case), so a drift surfaces as a failing test rather than silent divergence. For the converter's property-testing and counterexample→fixture process (P1–P4 invariants, the `PROPERTY_SEED`/`PROPERTY_NUM_RUNS` knobs, and the nightly fuzz workflow), see `packages/prosemirror-markdown/README.md`.
|
||||
- API access goes through `apps/client/src/lib/api-client.ts` (axios). The `@` alias maps to `apps/client/src`.
|
||||
- Runtime config is injected at build time by `vite.config.ts` via `define` (`APP_URL`, `COLLAB_URL`, `APP_VERSION`, …) — these come from the root `.env`, not from `import.meta.env`.
|
||||
|
||||
@@ -337,7 +470,7 @@ Vite SPA. Code is organized by feature under `apps/client/src/features/*` (mirro
|
||||
- **Errors must never be swallowed or shown as generic messages.** Every caught error MUST (1) be logged in full to the console/logger — error name, message, stack, `cause`, and (for HTTP/provider failures) the status code and response body — and (2) be surfaced to the user with a *specific, human-readable explanation of what actually went wrong*, never a bare generic string like "Something went wrong" / "Could not start recording" / "Transcription failed". Include the real reason (the underlying error/provider message) in the user-facing text. On the server, wrap third-party/provider failures with `describeProviderError` (or equivalent) and rethrow as a meaningful HTTP status + message — never let them collapse into an opaque 500. On the client, `console.error(<context>, err)` the raw error AND show the extracted reason (e.g. `err.response?.data?.message`, or the error `name: message`) in the notification.
|
||||
- The version string shown in the UI comes from `APP_VERSION` (CI/Docker) or `git describe --tags --always` (local), resolved in `vite.config.ts` — not from `package.json`.
|
||||
- Server TS config is permissive (`noImplicitAny: false`, `strictNullChecks: false`, `no-explicit-any` lint disabled). Follow the existing relaxed style rather than tightening types broadly.
|
||||
- Dependency versions are heavily pinned via `pnpm.overrides` and `pnpm.patchedDependencies` (`scimmy`, `yjs`, `ai`) in the root `package.json`. Don't bump pinned/patched deps casually; the patches and overrides exist for compatibility/security reasons. The `ai@6.0.134` patch disables the SDK's O(n²) cumulative `partialOutput` accumulation when no output strategy is requested (server heap OOM on long agent runs, #184; tripwire test: `apps/server/src/integrations/ai/ai-sdk-partial-output.patch.spec.ts`) — it MUST be re-created via `pnpm patch` when bumping `ai`.
|
||||
- Dependency versions are heavily pinned via `pnpm.overrides` and `pnpm.patchedDependencies` (`scimmy`, `yjs`, `ai`) in the root `package.json`. Don't bump pinned/patched deps casually; the patches and overrides exist for compatibility/security reasons. The `ai@6.0.134` patch carries TWO independent server fixes, each with its own tripwire test: (1) it disables the SDK's O(n²) cumulative `partialOutput` accumulation when no output strategy is requested (server heap OOM on long agent runs, #184; tripwire: `apps/server/src/integrations/ai/ai-sdk-partial-output.patch.spec.ts`); (2) it fixes `writeToServerResponse`'s drain-hang — the loop awaited only `"drain"` under backpressure, so a mid-write client disconnect parked the pipe forever and leaked the reader/buffers until restart; it now races `"drain"` against `"close"`/`"error"`, cancels the reader on disconnect, and swallows the fire-and-forget read rejection (#486; tripwire: `apps/server/src/integrations/ai/ai-sdk-drain-hang.patch.spec.ts`). Both tripwires assert BOTH installed dist builds carry their patch marker. The patch MUST be re-created via `pnpm patch` when bumping `ai`.
|
||||
- **The MCP tool inventory in `SERVER_INSTRUCTIONS` is GENERATED from the registry** (`packages/mcp/src/server-instructions.ts`: `buildToolInventory()` over `SHARED_TOOL_SPECS`) and spliced into the hand-written routing prose (`ROUTING_PROSE`). So adding/renaming/removing a **shared** spec in `packages/mcp/src/tool-specs.ts` auto-updates the `<tool_inventory>` — no manual `SERVER_INSTRUCTIONS` edit needed. Only an **inline** MCP-only tool (those registered via `server.registerTool(...)` in `index.ts`, not through the registry) needs a one-line entry in `INLINE_MCP_INVENTORY`. Enforced by `packages/mcp/test/unit/tool-inventory.test.mjs`, which fails when a registered tool is missing from the generated inventory (there is no `EXCEPTIONS` opt-out anymore — every tool must appear). Update `ROUTING_PROSE` when a tool's *intent guidance* (when-to-use) changes. `packages/mcp/build/` is gitignored and rebuilt in CI/Docker via `pnpm build` (same convention as `git-sync`/`prosemirror-markdown`) — never commit it; rebuild locally after editing to run the tests.
|
||||
|
||||
## CI / release
|
||||
|
||||
@@ -115,6 +115,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
the old ProseMirror-JSON output. Released together with the `#411`/`#412`
|
||||
breaking window so external configs break exactly once. (#413)
|
||||
|
||||
- **The Prometheus `/metrics` listener now binds to `127.0.0.1` (loopback) by
|
||||
default instead of `0.0.0.0` (all interfaces).** This closes an unauthenticated
|
||||
endpoint that was previously reachable on every interface. **DEPLOY MIGRATION —
|
||||
cross-container scraping breaks silently otherwise:** if your scraper runs in a
|
||||
SEPARATE container and reaches the app as `docmost:9464` (the exact topology the
|
||||
old `0.0.0.0` hardcode served), you MUST now set `METRICS_BIND=0.0.0.0` — and,
|
||||
because that re-exposes the endpoint, also set `METRICS_TOKEN=<secret>` and
|
||||
configure the scraper with a matching Bearer token. Without `METRICS_BIND`, the
|
||||
scraper can no longer connect and metrics go dark with no error. See the
|
||||
`METRICS_BIND` / `METRICS_TOKEN` block in `.env.example` for the migration.
|
||||
Same-host (loopback) scrapers need no change. (#486)
|
||||
|
||||
### Added
|
||||
|
||||
- **Place several images side by side in a row.** A new "Inline (side by
|
||||
@@ -270,6 +282,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Changed
|
||||
|
||||
- **Client markdown paste/copy and AI-chat rendering now go through the canonical
|
||||
converter.** Pasting markdown into the editor, "Copy as markdown", the AI title
|
||||
generator, and the AI-chat markdown renderer all now use
|
||||
`@docmost/prosemirror-markdown` (via its new `browser` entry — native
|
||||
`DOMParser`, no jsdom in the client bundle) instead of the hand-written
|
||||
`marked`/`turndown` markdown layer in `editor-ext`, which was **deleted**. As a
|
||||
result, pasting canonical markdown (`^[…]` footnotes, `<!--img …-->`,
|
||||
`> [!type]` callouts, `$…$` math, `==…==` highlight, standalone `<!--subpages-->`
|
||||
comments) now produces the SAME nodes the server import produces for the same
|
||||
text. Chat/reasoning markdown now renders through the editor schema (list items
|
||||
are wrapped in `<p>`; CSS keeps them tight). (#347)
|
||||
|
||||
- **Enabling a public share no longer auto-shares the whole sub-tree.** Turning
|
||||
a page "Shared to web" now defaults to the page alone; descendant pages become
|
||||
public only when you explicitly turn on the dedicated "Include sub-pages"
|
||||
@@ -298,6 +322,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
`tee()` branch of the stream result — a ~20-step, ~28k-chunk agent run
|
||||
retained ~1.7 GB and OOM'd the 2 GB JS heap. Streaming granularity is
|
||||
unchanged; the patch must be re-created if `ai` is ever bumped. (#184)
|
||||
|
||||
- **The server no longer leaks a hung stream pipe on every mid-run client
|
||||
disconnect.** The same `ai@6.0.134` pnpm patch now also fixes the SDK's
|
||||
`writeToServerResponse`, which awaited only a `"drain"` event under
|
||||
backpressure: when a client disconnected mid-write the socket never drained, so
|
||||
the write loop parked forever, `response.end()` was unreachable, and the stream
|
||||
reader plus buffered chunks were pinned until process restart (every mid-run
|
||||
disconnect in autonomous mode leaked one). The patch races `"drain"` against
|
||||
`"close"`/`"error"`, cancels the reader and ends the response on disconnect, and
|
||||
swallows the fire-and-forget read rejection instead of crashing on an
|
||||
unhandledRejection. (#486)
|
||||
|
||||
- **A failed autonomous agent-run start no longer becomes an unstoppable ghost
|
||||
run.** When `beginRun` failed for a transient reason (e.g. a DB-pool blip),
|
||||
the turn previously continued with NO run row — invisible to `/stop`, not
|
||||
aborted on disconnect, and able to slip a second run past the one-run-per-chat
|
||||
gate, leaving an unstoppable run until restart. The turn now fails fast with an
|
||||
honest `503 A_RUN_BEGIN_FAILED` before the first byte (no orphan state), and the
|
||||
client shows a "temporary — please try again" message instead of a misleading
|
||||
"provider not configured". (#486)
|
||||
|
||||
- **A pathological draw.io graph can no longer wedge the whole server.** The ELK
|
||||
auto-layout (`layout:"elk"`) ran elkjs synchronously on the main event loop, so
|
||||
a graph at the node/edge cap blocked ALL HTTP/SSE/loopback traffic while it
|
||||
churned — and the old `setTimeout` "timeout" could never fire because the same
|
||||
thread was blocked. Layout now runs in a worker thread with the timeout enforced
|
||||
by `worker.terminate()`; the main loop stays responsive. (#486)
|
||||
|
||||
- **The `/health` Redis probe no longer leaks a client on every tick while Redis
|
||||
is down.** It built a new `ioredis` client per probe and disconnected it only on
|
||||
success, so during an outage each health tick added another forever-reconnecting
|
||||
client (an unbounded handle leak). A single long-lived probe client is now
|
||||
reused and closed on shutdown. (#486)
|
||||
- **Internal links in exported Markdown no longer lose their visible text.** A
|
||||
link whose target page name had no file extension (e.g. a bare title) was
|
||||
collapsed to empty text during export, producing an unclickable, label-less
|
||||
@@ -374,6 +431,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
share); any other value now returns the generic "not found" instead of
|
||||
serving the page. (#218)
|
||||
|
||||
- **Tool and provider error text no longer leaks to anonymous readers in the
|
||||
public-share AI chat.** A failing tool's raw error (which could carry an
|
||||
internal page title or a stack fragment) and a provider error (which bundles the
|
||||
provider `statusCode` and response body — potentially the internal baseUrl or
|
||||
model name) were streamed verbatim to the anonymous reader over SSE. Errors are
|
||||
now sanitized at the source: the share toolset collapses any unclassified tool
|
||||
error to a safe generic string (safe, classified tool messages still pass
|
||||
through for the model's self-correction), and the anonymous stream `onError`
|
||||
maps provider failures to a fixed set of neutral strings — the full detail goes
|
||||
only to the server log. A UI render gate is layered on top. (closes #394)
|
||||
|
||||
- **The Prometheus `/metrics` endpoint can now require Bearer authentication and
|
||||
is loopback-bound by default.** Previously it listened on all interfaces with no
|
||||
auth. Setting `METRICS_TOKEN` requires every scrape to present
|
||||
`Authorization: Bearer <token>` (compared in constant time), and the listener
|
||||
defaults to `127.0.0.1` (see the Breaking Changes entry for the cross-container
|
||||
migration). (#486)
|
||||
|
||||
## [0.94.0] - 2026-06-26
|
||||
|
||||
This release makes AI chat durable and fast: assistant turns are persisted to
|
||||
|
||||
+15
@@ -45,6 +45,11 @@ COPY --from=builder /app/packages/editor-ext/dist /app/packages/editor-ext/dist
|
||||
COPY --from=builder /app/packages/editor-ext/package.json /app/packages/editor-ext/package.json
|
||||
COPY --from=builder /app/packages/mcp/build /app/packages/mcp/build
|
||||
COPY --from=builder /app/packages/mcp/package.json /app/packages/mcp/package.json
|
||||
# The mcp package reads its data files (drawio-presets.json, drawio-shape-index.json.gz)
|
||||
# at runtime via `new URL("../../data/…", import.meta.url)` relative to build/lib/*.js,
|
||||
# i.e. from packages/mcp/data/. tsc emits only build/, so ship data/ explicitly or
|
||||
# drawioFromGraph and the shape catalog die with ENOENT on packages/mcp/data/*.
|
||||
COPY --from=builder /app/packages/mcp/data /app/packages/mcp/data
|
||||
# mcp now depends on @docmost/prosemirror-markdown (workspace:*) and eager-imports
|
||||
# it at runtime (the in-app ai-chat DocmostClient loads build/index.js -> lib/
|
||||
# markdown-converter.js). Ship the built package + its manifest, or the prod
|
||||
@@ -81,4 +86,14 @@ VOLUME ["/app/data/storage"]
|
||||
|
||||
EXPOSE 3000
|
||||
|
||||
# DEPLOY REQUIREMENT — SINGLE INSTANCE or STICKY SESSIONS (#449).
|
||||
# MCP content writes are serialized per page by an IN-PROCESS mutex, and the
|
||||
# stash_page blob store + cached collab sessions are RAM-only and process-local.
|
||||
# Running MULTIPLE replicas of this image behind a load balancer WITHOUT sticky
|
||||
# sessions silently breaks per-page write serialization (two replicas can lock
|
||||
# the same page at once) and makes stash_page blobs unreachable across replicas.
|
||||
# Run a SINGLE instance, or pin each page's traffic to one replica (sticky
|
||||
# sessions / consistent hashing on page id). There is deliberately no
|
||||
# cross-process lock yet — a conscious constraint. See .env.example (the "MCP
|
||||
# collaboration write path" block) and packages/mcp/README.md for details.
|
||||
CMD ["pnpm", "start"]
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
"@atlaskit/pragmatic-drag-and-drop-live-region": "1.3.4",
|
||||
"@casl/react": "5.0.1",
|
||||
"@docmost/editor-ext": "workspace:*",
|
||||
"@docmost/prosemirror-markdown": "workspace:*",
|
||||
"@excalidraw/excalidraw": "0.18.0-3a5ef40",
|
||||
"@mantine/core": "8.3.18",
|
||||
"@mantine/dates": "8.3.18",
|
||||
|
||||
@@ -55,6 +55,15 @@
|
||||
padding-inline-start: 1.4em;
|
||||
}
|
||||
|
||||
/* The canonical converter renders list items through the editor schema, which
|
||||
wraps each item's content in a <p> (listItem content is `paragraph+`). Drop
|
||||
that paragraph's block margin so list items render TIGHT (no extra vertical
|
||||
gap), matching the previous marked output — same rule already applied to
|
||||
table cells above (issue #347). */
|
||||
.markdown li p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
/* GFM tables in assistant markdown. The chat lives in a NARROW side panel, so a
|
||||
wide LLM table must scroll horizontally instead of collapsing its columns:
|
||||
`.markdown` sets `word-break: break-word`, which (with the default table
|
||||
@@ -172,6 +181,14 @@
|
||||
margin: 0 0 4px;
|
||||
}
|
||||
|
||||
/* Same as `.markdown li p` above: the canonical converter wraps every list
|
||||
item's content in a <p>, so without this each reasoning-panel list item would
|
||||
pick up `.reasoningText p`'s 4px bottom margin and render too loose. Drop it
|
||||
so Reasoning-panel lists stay tight, mirroring the pre-#347 marked output. */
|
||||
.reasoningText li p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.inputWrapper {
|
||||
flex: 0 0 auto;
|
||||
padding-top: var(--mantine-spacing-xs);
|
||||
|
||||
@@ -203,6 +203,52 @@ describe("ChatThread — send now (#198)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// #486: the final onFinish -> flushNext() must be gated on the live-mount flag.
|
||||
// A clean onFinish can land AFTER the thread unmounts (New-chat / chat-switch
|
||||
// mid-stream — the async attach/resume settles late); flushing then dequeues and
|
||||
// re-POSTs a queued message from an abandoned thread (a "ghost" send).
|
||||
describe("ChatThread — onFinish flush gated on mount (#486)", () => {
|
||||
beforeEach(resetState);
|
||||
afterEach(cleanup);
|
||||
|
||||
it("a clean onFinish WHILE MOUNTED flushes the queued message (control)", () => {
|
||||
renderThread();
|
||||
fireEvent.click(screen.getByTestId("queue-btn")); // enqueue "queued text"
|
||||
expect(h.state.sendMessage).not.toHaveBeenCalled();
|
||||
|
||||
act(() => {
|
||||
h.state.onFinish?.({
|
||||
message: { id: "a", role: "assistant", parts: [] },
|
||||
isAbort: false,
|
||||
isDisconnect: false,
|
||||
isError: false,
|
||||
});
|
||||
});
|
||||
// Mounted: the queue flushes normally.
|
||||
expect(h.state.sendMessage).toHaveBeenCalledWith({ text: "queued text" });
|
||||
});
|
||||
|
||||
it("a clean onFinish AFTER unmount does NOT flush (no ghost send)", () => {
|
||||
const { unmount } = renderThread();
|
||||
fireEvent.click(screen.getByTestId("queue-btn")); // enqueue "queued text"
|
||||
h.state.sendMessage.mockClear();
|
||||
|
||||
// Chat switched away mid-stream: the streamer unmounts...
|
||||
unmount();
|
||||
// ...and a late, clean onFinish lands on the abandoned thread.
|
||||
act(() => {
|
||||
h.state.onFinish?.({
|
||||
message: { id: "a", role: "assistant", parts: [] },
|
||||
isAbort: false,
|
||||
isDisconnect: false,
|
||||
isError: false,
|
||||
});
|
||||
});
|
||||
// Gated on mountedRef: NOTHING is sent from the dead thread.
|
||||
expect(h.state.sendMessage).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// #396: in autonomous mode a live sendNow must additionally request the
|
||||
// AUTHORITATIVE server stop of the detached run (a local abort is only a client
|
||||
// disconnect the server ignores) and arm a bounded 409 retry so the re-POST
|
||||
|
||||
@@ -659,7 +659,13 @@ export default function ChatThread({
|
||||
return;
|
||||
}
|
||||
if (isAbort || isDisconnect || isError) return;
|
||||
flushNext();
|
||||
// Gate the final flush on the live-mount flag (#486): a clean onFinish can
|
||||
// land AFTER this thread unmounted (a New-chat / chat-switch mid-stream —
|
||||
// the async attach/resume settles late). Flushing then dequeues and POSTs a
|
||||
// queued message from an abandoned thread — a "ghost" send / ghost chat.
|
||||
// Every other queue side effect already guards on mountedRef; this last one
|
||||
// was the gap.
|
||||
if (mountedRef.current) flushNext();
|
||||
},
|
||||
// `onError` runs in addition to `onFinish` (which ai@6 also calls on error).
|
||||
// Log the raw failure here for devtools; the UI shows a friendly classified
|
||||
|
||||
@@ -47,6 +47,13 @@ interface MessageItemProps {
|
||||
* agent's raw query/argument text.
|
||||
*/
|
||||
showInput?: boolean;
|
||||
/**
|
||||
* Forwarded to ToolCallCard: whether a failed tool card renders its raw
|
||||
* errorText. Defaults to true (internal chat). The public share passes false so
|
||||
* internal detail in a tool error is never painted (belt to the server-side
|
||||
* byte sanitization).
|
||||
*/
|
||||
showErrors?: boolean;
|
||||
/**
|
||||
* Neutralize internal/relative markdown links in the rendered answer (drop
|
||||
* their href so they become inert text). Defaults to false (internal chat,
|
||||
@@ -125,6 +132,7 @@ function MessageItem({
|
||||
message,
|
||||
showCitations = true,
|
||||
showInput = true,
|
||||
showErrors = true,
|
||||
neutralizeInternalLinks = false,
|
||||
assistantName,
|
||||
turnStreaming = false,
|
||||
@@ -219,6 +227,7 @@ function MessageItem({
|
||||
part={part as unknown as ToolUiPart}
|
||||
showCitations={showCitations}
|
||||
showInput={showInput}
|
||||
showErrors={showErrors}
|
||||
/>
|
||||
);
|
||||
}
|
||||
@@ -284,6 +293,7 @@ export function arePropsEqual(
|
||||
prev.signature === next.signature &&
|
||||
prev.showCitations === next.showCitations &&
|
||||
prev.showInput === next.showInput &&
|
||||
prev.showErrors === next.showErrors &&
|
||||
prev.neutralizeInternalLinks === next.neutralizeInternalLinks &&
|
||||
prev.assistantName === next.assistantName &&
|
||||
// The turn-end flip re-renders every row once (cheap, terminal event) —
|
||||
|
||||
@@ -32,6 +32,12 @@ interface MessageListProps {
|
||||
* doesn't see the agent's raw query/argument text.
|
||||
*/
|
||||
showInput?: boolean;
|
||||
/**
|
||||
* Forwarded to MessageItem -> ToolCallCard: whether a failed tool card renders
|
||||
* its raw errorText. Defaults to true (internal chat). The public share passes
|
||||
* false so internal detail in a tool error is never painted.
|
||||
*/
|
||||
showErrors?: boolean;
|
||||
/**
|
||||
* Forwarded to MessageItem: neutralize internal/relative markdown links in
|
||||
* the rendered answers (drop their href so they render as inert text).
|
||||
@@ -127,6 +133,7 @@ export default function MessageList({
|
||||
emptyState,
|
||||
showCitations = true,
|
||||
showInput = true,
|
||||
showErrors = true,
|
||||
neutralizeInternalLinks = false,
|
||||
assistantName,
|
||||
}: MessageListProps) {
|
||||
@@ -217,6 +224,7 @@ export default function MessageList({
|
||||
signature={messageSignature(message)}
|
||||
showCitations={showCitations}
|
||||
showInput={showInput}
|
||||
showErrors={showErrors}
|
||||
neutralizeInternalLinks={neutralizeInternalLinks}
|
||||
assistantName={assistantName}
|
||||
// Turn-level liveness, gated to the TAIL row: only the tail message
|
||||
|
||||
@@ -30,6 +30,16 @@ interface ToolCallCardProps {
|
||||
* the extra summary line, leaving the card (the action log) intact.
|
||||
*/
|
||||
showInput?: boolean;
|
||||
/**
|
||||
* Whether to render the tool's raw errorText on a failed call. Defaults to true
|
||||
* (the internal chat, where the operator may debug). The public share passes
|
||||
* false: a tool error string can carry internal detail (an internal page title,
|
||||
* a stack fragment, a provider message). This is the RENDER gate only — the
|
||||
* authoritative fix also sanitizes the bytes server-side (see
|
||||
* PublicShareChatToolsService.forShare), so a share reader never receives raw
|
||||
* error text over the wire, not just never sees it painted (#394).
|
||||
*/
|
||||
showErrors?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -41,6 +51,7 @@ export default function ToolCallCard({
|
||||
part,
|
||||
showCitations = true,
|
||||
showInput = true,
|
||||
showErrors = true,
|
||||
}: ToolCallCardProps) {
|
||||
const { t } = useTranslation();
|
||||
const toolName = getToolName(part);
|
||||
@@ -74,7 +85,7 @@ export default function ToolCallCard({
|
||||
</Text>
|
||||
)}
|
||||
|
||||
{state === "error" && part.errorText && (
|
||||
{state === "error" && showErrors && part.errorText && (
|
||||
<Text size="xs" c="red" mt={2}>
|
||||
{part.errorText}
|
||||
</Text>
|
||||
|
||||
@@ -33,29 +33,44 @@ describe("collapseBlankLines", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("collapseBlankLines + renderChatMarkdown (tight reasoning rendering)", () => {
|
||||
it("renders a blank-line-separated list as a TIGHT list (no <li><p>)", () => {
|
||||
describe("collapseBlankLines + renderChatMarkdown (canonical converter)", () => {
|
||||
// Chat markdown now renders through @docmost/prosemirror-markdown (issue #347):
|
||||
// the SAME converter the editor/import use. Its list items are schema-shaped —
|
||||
// each <li>'s content is wrapped in a <p> (listItem content is `paragraph+`) —
|
||||
// so the HTML always carries `<li><p>…</p></li>` regardless of blank-line
|
||||
// looseness in the source (the converter has no tight/loose distinction). The
|
||||
// visual tightness that `collapseBlankLines` used to buy is now provided by
|
||||
// CSS (`.markdown li p { margin: 0 }`), not the HTML shape.
|
||||
it("renders a blank-line-separated bullet list as a real <ul> list", () => {
|
||||
const loose =
|
||||
"Intro paragraph.\n\n- item one\n\n- item two\n\n- item three";
|
||||
const html = renderChatMarkdown(collapseBlankLines(loose), {});
|
||||
// Tight list: each <li> holds the text directly, not wrapped in a <p>.
|
||||
expect(html).toContain("<li>item one</li>");
|
||||
expect(html).not.toContain("<li><p>");
|
||||
// The list still parses as a list after the paragraph (not a paragraph+<br>).
|
||||
// Clean, un-namespaced HTML (DOMSerializer, not XMLSerializer) — no xmlns.
|
||||
expect(html).toContain("<ul>");
|
||||
expect(html).not.toMatch(/<ul[^>]*xmlns/);
|
||||
// The item text is present (inside the schema's <li><p> wrapper).
|
||||
expect(html).toContain("item one");
|
||||
// The intro paragraph renders as its own paragraph before the list.
|
||||
expect(html).toContain("<p>Intro paragraph.</p>");
|
||||
});
|
||||
|
||||
it("renders an ordered list (1. 2.) as tight after collapsing", () => {
|
||||
it("renders an ordered list (1. 2.) as a real <ol> list", () => {
|
||||
const loose = "Intro.\n\n1. first\n\n2. second";
|
||||
const html = renderChatMarkdown(collapseBlankLines(loose), {});
|
||||
expect(html).toContain("<ol>");
|
||||
expect(html).toContain("<li>first</li>");
|
||||
expect(html).not.toContain("<li><p>");
|
||||
expect(html).not.toMatch(/<ol[^>]*xmlns/);
|
||||
expect(html).toContain("first");
|
||||
expect(html).toContain("second");
|
||||
});
|
||||
|
||||
it("the loose source WOULD render <li><p> without collapsing (control)", () => {
|
||||
it("wraps list-item content in <p> (schema shape; tightness is CSS)", () => {
|
||||
// The canonical converter always wraps a list item's content in a paragraph,
|
||||
// whether or not the source had blank lines between items.
|
||||
const loose = "- a\n\n- b";
|
||||
expect(renderChatMarkdown(loose, {})).toContain("<li><p>");
|
||||
// And a "tight" source produces the identical wrapping (no distinction).
|
||||
expect(renderChatMarkdown(collapseBlankLines(loose), {})).toContain(
|
||||
"<li><p>",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -23,6 +23,25 @@ describe("describeChatError", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies an A_RUN_BEGIN_FAILED 503 as a temporary run-start failure, NOT provider-not-configured (#486)", () => {
|
||||
// The FULL real body the server writes for a beginRun failure: a
|
||||
// ServiceUnavailableException(object) whose response is serialized verbatim
|
||||
// onto the raw socket, self-describing statusCode 503 + the run-start code.
|
||||
const body =
|
||||
'{"message":"Could not start the agent run. This is usually temporary — please try again.","code":"A_RUN_BEGIN_FAILED","statusCode":503}';
|
||||
expect(describeChatError(body, t)).toEqual({
|
||||
title: "Could not start the run",
|
||||
detail:
|
||||
"The agent run could not be started. This is usually temporary — please try again.",
|
||||
});
|
||||
// ORDER GUARD: even though the body ALSO carries statusCode 503 (which the
|
||||
// generic branch matches), the A_RUN_BEGIN_FAILED branch runs first, so it is
|
||||
// never mislabeled "AI provider not configured".
|
||||
expect(describeChatError(body, t).title).not.toBe(
|
||||
"AI provider not configured",
|
||||
);
|
||||
});
|
||||
|
||||
it("classifies a dropped connection (ECONNRESET) as a lost-connection error", () => {
|
||||
expect(
|
||||
describeChatError("Cannot connect to API: read ECONNRESET", t).title,
|
||||
|
||||
@@ -24,6 +24,21 @@ export function describeChatError(
|
||||
): ChatErrorView {
|
||||
const msg = message ?? "";
|
||||
|
||||
// Our own "could not start the run" gate (A_RUN_BEGIN_FAILED, #486): a 503
|
||||
// whose body carries this code is a TEMPORARY server-side failure while
|
||||
// starting the run (e.g. a DB-pool blip), NOT an unconfigured provider. It MUST
|
||||
// be matched STRICTLY BEFORE the generic 503 branch below, which would
|
||||
// otherwise mislabel it "The AI provider is not configured" and tell the user
|
||||
// to call an admin instead of just retrying.
|
||||
if (/"code"\s*:\s*"A_RUN_BEGIN_FAILED"/.test(msg)) {
|
||||
return {
|
||||
title: t("Could not start the run"),
|
||||
detail: t(
|
||||
"The agent run could not be started. This is usually temporary — please try again.",
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
if (/"statusCode"\s*:\s*403\b/.test(msg)) {
|
||||
return {
|
||||
title: t("AI chat is disabled"),
|
||||
|
||||
@@ -1,6 +1,37 @@
|
||||
import { markdownToHtml } from "@docmost/editor-ext";
|
||||
import {
|
||||
markdownToProseMirrorSync,
|
||||
docmostExtensions,
|
||||
} from "@docmost/prosemirror-markdown/browser";
|
||||
import { getSchema } from "@tiptap/core";
|
||||
import { Node as PMNode, DOMSerializer } from "@tiptap/pm/model";
|
||||
import DOMPurify from "dompurify";
|
||||
|
||||
// The Docmost editor schema, built once. Chat markdown is rendered through the
|
||||
// SAME schema the editor/import use (issue #347), so chat output matches how the
|
||||
// page would render the same markdown.
|
||||
const chatSchema = getSchema(docmostExtensions);
|
||||
|
||||
/**
|
||||
* Markdown -> HTML for chat display, via the canonical converter. We serialize
|
||||
* the ProseMirror doc with `DOMSerializer` into a real element and read its
|
||||
* `innerHTML` (rather than `@tiptap/html`'s `generateHTML`, whose browser path
|
||||
* uses `XMLSerializer` and stamps a `xmlns` on every block) so the markup is
|
||||
* clean HTML. `li > p` wrapping is inherent to the schema (listItem content is
|
||||
* `paragraph+`); the chat CSS zeroes those paragraph margins so lists still
|
||||
* render tight.
|
||||
*/
|
||||
function markdownToChatHtml(markdown: string): string {
|
||||
const doc = markdownToProseMirrorSync(markdown);
|
||||
const node = PMNode.fromJSON(chatSchema, doc);
|
||||
const div = document.createElement("div");
|
||||
DOMSerializer.fromSchema(chatSchema).serializeFragment(
|
||||
node.content,
|
||||
{ document },
|
||||
div,
|
||||
);
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
export interface RenderChatMarkdownOptions {
|
||||
/**
|
||||
* Neutralize INTERNAL links so they render as inert text (no `href`/`target`).
|
||||
@@ -63,22 +94,32 @@ function neutralizeInternalLinksHook(node: Element): void {
|
||||
|
||||
/**
|
||||
* Render AI markdown to sanitized HTML for read-only display. We reuse the
|
||||
* app's `markdownToHtml` (the same `marked` pipeline used for paste/import) so
|
||||
* chat output matches the editor's markdown flavor, then sanitize with
|
||||
* DOMPurify — LLM output is untrusted, so it must never reach the DOM unsanitized.
|
||||
* canonical converter (issue #347): markdown -> ProseMirror JSON (the SAME
|
||||
* `markdownToProseMirrorSync` the editor paste/import path uses, so chat output
|
||||
* matches the editor's markdown flavor) -> HTML via `markdownToChatHtml`
|
||||
* (DOMSerializer), then sanitize with DOMPurify — LLM output is untrusted, so it
|
||||
* must never reach the DOM unsanitized.
|
||||
*
|
||||
* `markdownToHtml` can return `string | Promise<string>` (it has async marked
|
||||
* extensions registered). In practice plain chat markdown resolves
|
||||
* synchronously, but we guard the Promise case by returning a safe empty string
|
||||
* for that branch (the caller renders the raw text fallback instead).
|
||||
* Stays SYNCHRONOUS: both callers render inside React (a memo and a useMemo),
|
||||
* so the whole pipeline must resolve without awaiting. The converter's sync
|
||||
* entry makes that possible; on any conversion error we return "" so the caller
|
||||
* falls back to raw text (the same fallback the old Promise-guard produced).
|
||||
*/
|
||||
export function renderChatMarkdown(
|
||||
markdown: string,
|
||||
options: RenderChatMarkdownOptions = {},
|
||||
): string {
|
||||
if (!markdown) return "";
|
||||
const html = markdownToHtml(markdown);
|
||||
if (typeof html !== "string") return "";
|
||||
let html: string;
|
||||
try {
|
||||
// markdown -> canonical PM JSON -> HTML (native DOMParser in the browser;
|
||||
// jsdom is never bundled — see @docmost/prosemirror-markdown/browser).
|
||||
html = markdownToChatHtml(markdown);
|
||||
} catch {
|
||||
// Malformed/unsupported markdown must not crash the chat render; fall back
|
||||
// to raw text (empty return -> caller shows the plain-text branch).
|
||||
return "";
|
||||
}
|
||||
|
||||
if (!options.neutralizeInternalLinks) {
|
||||
// Internal chat: unchanged behavior, no hook registered.
|
||||
|
||||
@@ -0,0 +1,206 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { Editor } from "@tiptap/core";
|
||||
import { Document } from "@tiptap/extension-document";
|
||||
import { Paragraph } from "@tiptap/extension-paragraph";
|
||||
import { Text } from "@tiptap/extension-text";
|
||||
import { Bold } from "@tiptap/extension-bold";
|
||||
import { Italic } from "@tiptap/extension-italic";
|
||||
import { MarkdownClipboard } from "./markdown-clipboard";
|
||||
|
||||
/**
|
||||
* Integration coverage for the async `handlePaste` seam (issue #347). The paste
|
||||
* conversion moved to `@docmost/prosemirror-markdown`'s browser entry, whose
|
||||
* `markdownToProseMirror` is async — so `handlePaste` captures the range, claims
|
||||
* the event (returns true), and dispatches the insert on the next microtask.
|
||||
* These tests drive that path end to end on a minimal schema (a plain-markdown
|
||||
* paste whose converted nodes fit paragraph/text/bold/italic), asserting the
|
||||
* text lands with the right marks and that the raw markdown syntax is consumed
|
||||
* (recognized as markdown, not inserted literally).
|
||||
*/
|
||||
|
||||
function makeEditor() {
|
||||
const element = document.createElement("div");
|
||||
document.body.appendChild(element);
|
||||
return new Editor({
|
||||
element,
|
||||
extensions: [
|
||||
Document,
|
||||
Paragraph,
|
||||
Text,
|
||||
Bold,
|
||||
Italic,
|
||||
MarkdownClipboard.configure({ transformPastedText: true }),
|
||||
],
|
||||
content: { type: "doc", content: [{ type: "paragraph" }] },
|
||||
});
|
||||
}
|
||||
|
||||
// Locate the markdownClipboard plugin and invoke its handlePaste directly with a
|
||||
// synthetic clipboard event (jsdom has no real paste pipeline). The plugin's
|
||||
// handlePaste closes over the extension `this`, so calling it off the plugin
|
||||
// props preserves `this.editor`/`this.options`.
|
||||
function paste(editor: Editor, text: string): boolean {
|
||||
const view = editor.view;
|
||||
const plugin = view.state.plugins.find(
|
||||
(p: any) => p.props && p.spec?.key,
|
||||
) as any;
|
||||
const event = {
|
||||
clipboardData: {
|
||||
getData: (type: string) => (type === "text/plain" ? text : ""),
|
||||
},
|
||||
} as unknown as ClipboardEvent;
|
||||
// Find the specific handlePaste that belongs to the markdown clipboard plugin.
|
||||
const md = view.state.plugins.find(
|
||||
(p: any) => typeof p.props?.handlePaste === "function",
|
||||
) as any;
|
||||
return md.props.handlePaste(view, event, view.state.selection.content());
|
||||
}
|
||||
|
||||
// Flush the microtask queue so the async .then() dispatch runs.
|
||||
const flush = () => new Promise((r) => setTimeout(r, 0));
|
||||
|
||||
describe("MarkdownClipboard handlePaste (async md -> PM)", () => {
|
||||
it("converts a plain-markdown paste with bold/italic into marked text", async () => {
|
||||
const editor = makeEditor();
|
||||
const claimed = paste(editor, "hello **bold** and *italic*");
|
||||
// The paste is claimed synchronously (async insert follows).
|
||||
expect(claimed).toBe(true);
|
||||
await flush();
|
||||
|
||||
const json = editor.getJSON();
|
||||
const text = JSON.stringify(json);
|
||||
// The raw markdown asterisks are consumed (recognized), not inserted literally.
|
||||
expect(editor.getText()).not.toContain("**");
|
||||
expect(editor.getText()).toContain("bold");
|
||||
expect(editor.getText()).toContain("italic");
|
||||
// The bold/italic marks materialized.
|
||||
expect(text).toContain('"bold"');
|
||||
expect(text).toContain('"italic"');
|
||||
editor.destroy();
|
||||
});
|
||||
|
||||
it("recognizes a bullet list paste as list structure (not literal '-')", async () => {
|
||||
// A bullet list is not representable in this minimal schema, so the converter
|
||||
// output would fail PMNode.fromJSON and the catch inserts raw text. Use a
|
||||
// paste whose nodes DO fit the schema to assert the happy path instead: two
|
||||
// paragraphs separated by a blank line.
|
||||
const editor = makeEditor();
|
||||
paste(editor, "first para\n\nsecond para");
|
||||
await flush();
|
||||
const json = editor.getJSON() as any;
|
||||
const paras = (json.content || []).filter(
|
||||
(n: any) => n.type === "paragraph",
|
||||
);
|
||||
// Two paragraphs materialized from the blank-line-separated markdown.
|
||||
expect(paras.length).toBeGreaterThanOrEqual(2);
|
||||
expect(editor.getText()).toContain("first para");
|
||||
expect(editor.getText()).toContain("second para");
|
||||
editor.destroy();
|
||||
});
|
||||
|
||||
it("falls back to raw text when conversion yields nodes the schema lacks", async () => {
|
||||
// `# heading` converts to a `heading` node absent from this minimal schema,
|
||||
// so PMNode.fromJSON throws and the catch re-inserts the raw text — the user
|
||||
// never loses their clipboard content.
|
||||
const editor = makeEditor();
|
||||
paste(editor, "# a heading line");
|
||||
await flush();
|
||||
// Content is preserved (either as heading text or literal), never dropped.
|
||||
expect(editor.getText()).toContain("a heading line");
|
||||
editor.destroy();
|
||||
});
|
||||
});
|
||||
|
||||
// The async seam captures the target range synchronously, then replaces on the
|
||||
// next microtask. If the document changed under it between capture and resolve
|
||||
// (impossible in prod — same microtask — but pinned here), BOTH the success
|
||||
// (replaceRange) and the fail-open (insertText) branches must fall back to the
|
||||
// LIVE selection rather than a stale absolute range, so neither clobbers content
|
||||
// nor throws a RangeError. We force the mid-flight change by dispatching a
|
||||
// doc-mutating transaction AFTER the synchronous claim but BEFORE flushing the
|
||||
// microtask that runs the `.then`/`.catch`.
|
||||
describe("MarkdownClipboard handlePaste — doc-changed-mid-flight guard", () => {
|
||||
// Replace the whole doc with one paragraph of `text` (synchronous dispatch).
|
||||
// An empty string yields an empty paragraph (a text node may not be empty).
|
||||
function seedContent(editor: Editor, text: string) {
|
||||
editor.commands.setContent({
|
||||
type: "doc",
|
||||
content: [
|
||||
text
|
||||
? { type: "paragraph", content: [{ type: "text", text }] }
|
||||
: { type: "paragraph" },
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
it("success branch: mid-flight doc change routes the paste to the LIVE selection, never the stale range (clobber-proving)", async () => {
|
||||
// The paste captures a NON-EMPTY range {1,5} (over "AAAA"). Then, before the
|
||||
// async resolve, the doc GROWS ("MARKER" inserted at the start) and the cursor
|
||||
// is parked at the doc END. The captured {1,5} is now stale and points INTO
|
||||
// "MARKER". A WORKING guard replaces at the live (end) selection → MARKER is
|
||||
// untouched. A BROKEN guard replaces the stale {1,5} → it erases the first
|
||||
// characters of MARKER (this is what a zero-width `from==to` range could never
|
||||
// reveal, which is why the earlier version was vacuous).
|
||||
const editor = makeEditor();
|
||||
seedContent(editor, "AAAABBBB");
|
||||
editor.commands.setTextSelection({ from: 1, to: 5 }); // captured range = {1,5}
|
||||
const claimed = paste(editor, "hello **bold**");
|
||||
expect(claimed).toBe(true);
|
||||
|
||||
// Mid-flight: grow the doc and move the cursor to a KNOWN-safe end position.
|
||||
editor.view.dispatch(editor.view.state.tr.insertText("MARKER", 1));
|
||||
const end = editor.state.doc.content.size;
|
||||
editor.commands.setTextSelection({ from: end, to: end });
|
||||
await flush();
|
||||
|
||||
const text = editor.getText();
|
||||
// MARKER intact only if the guard used the live selection, not the stale range.
|
||||
expect(text).toContain("MARKER");
|
||||
expect(text).toContain("bold");
|
||||
expect(text).not.toContain("**");
|
||||
editor.destroy();
|
||||
});
|
||||
|
||||
it("fail-open branch: a mid-flight doc SHRINK makes the stale `to` out of bounds — the guard must avoid a RangeError (throw-proving)", async () => {
|
||||
// The paste captures a range {1,9} over an 8-char paragraph, then the
|
||||
// conversion FAILS (`# heading` -> a heading node the minimal schema lacks,
|
||||
// so PMNode.fromJSON throws -> the fail-open catch runs). Before the reject,
|
||||
// the doc is SHRUNK to an empty paragraph, so the captured `to` (9) is now far
|
||||
// past the doc's end. A WORKING guard inserts the raw text at the live (valid)
|
||||
// selection → "raw heading" lands. A BROKEN guard does insertText(md, 1, 9) on
|
||||
// a size-2 doc → RangeError, so the dispatch never runs and "raw heading" is
|
||||
// absent (the assertion reddens). A zero-width/growing-doc setup could never
|
||||
// push `to` out of bounds, which is why the earlier version was vacuous.
|
||||
const editor = makeEditor();
|
||||
seedContent(editor, "AAAABBBB");
|
||||
editor.commands.setTextSelection({ from: 1, to: 9 }); // captured range = {1,9}
|
||||
paste(editor, "# raw heading");
|
||||
|
||||
// Mid-flight: shrink the doc so the captured `to` = 9 is now out of bounds.
|
||||
seedContent(editor, "");
|
||||
await flush();
|
||||
|
||||
const text = editor.getText();
|
||||
// Raw text lands (via the live selection) only if the guard avoided the
|
||||
// stale, now-out-of-bounds range.
|
||||
expect(text).toContain("raw heading");
|
||||
editor.destroy();
|
||||
});
|
||||
|
||||
it("two pastes in flight: neither payload is lost (no data loss)", async () => {
|
||||
// Prod-unreachable (two paste events are separate macrotasks, and each
|
||||
// conversion resolves on a microtask before the next), but pinned here: when
|
||||
// both resolve back-to-back, the second sees the changed doc and inserts at
|
||||
// the live selection the first left — so the two payloads may INTERLEAVE, but
|
||||
// neither is dropped. We assert no data loss, not contiguity.
|
||||
const editor = makeEditor();
|
||||
paste(editor, "alphaword");
|
||||
paste(editor, "betaword");
|
||||
await flush();
|
||||
const text = editor.getText();
|
||||
// Neither payload fully dropped (interleaving may split one of them).
|
||||
expect(text).toContain("alpha");
|
||||
expect(text).toContain("beta");
|
||||
editor.destroy();
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,12 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { htmlToMarkdown } from "@docmost/editor-ext";
|
||||
// Markdown conversion now goes through the canonical package's BROWSER entry
|
||||
// (issue #347): the same converter the server import/export uses, resolved via
|
||||
// the `browser` exports condition so it runs on the native `DOMParser` (the
|
||||
// client jsdom vitest env provides one) with jsdom never bundled.
|
||||
import {
|
||||
convertProseMirrorToMarkdown,
|
||||
markdownToProseMirrorSync,
|
||||
} from "@docmost/prosemirror-markdown/browser";
|
||||
import {
|
||||
normalizeTableColumnWidths,
|
||||
classifyClipboardSelection,
|
||||
@@ -175,10 +182,13 @@ describe("classifyClipboardSelection", () => {
|
||||
|
||||
// Output-level tests for the table clipboard regression: copying a table must
|
||||
// yield a real GFM pipe table, NOT one-value-per-line concatenated cells.
|
||||
// These exercise the actual markdown produced by htmlToMarkdown (the same
|
||||
// serializer step the clipboardTextSerializer runs), so they pin the OUTPUT
|
||||
// shape that the classifier-flag tests above do not cover.
|
||||
describe("table clipboard markdown output (htmlToMarkdown)", () => {
|
||||
// These exercise the actual markdown produced by convertProseMirrorToMarkdown —
|
||||
// the same serializer step the clipboardTextSerializer now runs (issue #347) —
|
||||
// so they pin the OUTPUT shape that the classifier-flag tests above do not cover.
|
||||
// Input is ProseMirror JSON (what the copied slice serializes to), matching the
|
||||
// clipboardTextSerializer's new call: it wraps the slice content in a synthetic
|
||||
// `doc` (and the bare-rows case in a `table`) and calls the converter.
|
||||
describe("table clipboard markdown output (convertProseMirrorToMarkdown)", () => {
|
||||
// Trim each line and drop blanks so structural assertions are whitespace-robust.
|
||||
function lines(md: string): string[] {
|
||||
return md
|
||||
@@ -188,10 +198,10 @@ describe("table clipboard markdown output (htmlToMarkdown)", () => {
|
||||
}
|
||||
|
||||
// A GFM separator row like "| --- | --- |" (any number of columns), tolerant
|
||||
// of the padding turndown emits.
|
||||
// of the padding the serializer emits.
|
||||
function isSeparatorRow(line: string): boolean {
|
||||
const compact = line.replace(/\s+/g, "");
|
||||
return /^\|(?:-{3,}\|)+$/.test(compact);
|
||||
return /^\|(?::?-{2,}:?\|)+$/.test(compact);
|
||||
}
|
||||
|
||||
// Split a pipe-delimited row into trimmed cell values.
|
||||
@@ -203,42 +213,33 @@ describe("table clipboard markdown output (htmlToMarkdown)", () => {
|
||||
.map((c) => c.trim());
|
||||
}
|
||||
|
||||
it("serializes a header-less partial cell selection (bare rows) as a valid GFM pipe table", () => {
|
||||
// Mirror the serializer's `wrapBareRows` branch exactly: bare <tr> nodes are
|
||||
// wrapped in <table><tbody> and htmlToMarkdown(div.innerHTML) is called.
|
||||
// See markdown-clipboard.ts clipboardTextSerializer:
|
||||
// const table = document.createElement("table");
|
||||
// const tbody = document.createElement("tbody");
|
||||
// tbody.appendChild(fragment); table.appendChild(tbody);
|
||||
// div.appendChild(table);
|
||||
// return htmlToMarkdown(div.innerHTML);
|
||||
const div = document.createElement("div");
|
||||
const table = document.createElement("table");
|
||||
const tbody = document.createElement("tbody");
|
||||
for (const [c1, c2] of [
|
||||
["a", "b"],
|
||||
["c", "d"],
|
||||
]) {
|
||||
const tr = document.createElement("tr");
|
||||
const td1 = document.createElement("td");
|
||||
td1.textContent = c1;
|
||||
const td2 = document.createElement("td");
|
||||
td2.textContent = c2;
|
||||
tr.appendChild(td1);
|
||||
tr.appendChild(td2);
|
||||
tbody.appendChild(tr);
|
||||
}
|
||||
table.appendChild(tbody);
|
||||
div.appendChild(table);
|
||||
const cell = (t: string) => ({
|
||||
type: "tableCell",
|
||||
content: [{ type: "paragraph", content: [{ type: "text", text: t }] }],
|
||||
});
|
||||
const headerCell = (t: string) => ({
|
||||
type: "tableHeader",
|
||||
content: [{ type: "paragraph", content: [{ type: "text", text: t }] }],
|
||||
});
|
||||
const row = (nodes: any[]) => ({ type: "tableRow", content: nodes });
|
||||
|
||||
const md = htmlToMarkdown(div.innerHTML);
|
||||
it("serializes a header-less partial cell selection (bare rows) as a valid GFM pipe table", () => {
|
||||
// Mirror the serializer's `wrapBareRows` branch: bare tableRow nodes are
|
||||
// wrapped in a synthetic `table` and convertProseMirrorToMarkdown is called
|
||||
// (see markdown-clipboard.ts clipboardTextSerializer).
|
||||
const rows = [
|
||||
row([cell("a"), cell("b")]),
|
||||
row([cell("c"), cell("d")]),
|
||||
];
|
||||
const md = convertProseMirrorToMarkdown({
|
||||
type: "doc",
|
||||
content: [{ type: "table", content: rows }],
|
||||
});
|
||||
const ls = lines(md);
|
||||
|
||||
// Valid GFM: a header/data separator row is present (an empty header is
|
||||
// synthesized by the GFM turndown plugin for a header-less table — fine).
|
||||
// Valid GFM: a header/data separator row is present.
|
||||
expect(ls.some(isSeparatorRow)).toBe(true);
|
||||
// NOT the old broken "one value per line" shape: every line is pipe-delimited
|
||||
// and no line is a bare cell value on its own.
|
||||
// NOT the old broken "one value per line" shape: every line is pipe-delimited.
|
||||
expect(ls.every((l) => l.includes("|"))).toBe(true);
|
||||
expect(md).not.toMatch(/^\s*(a|b|c|d)\s*$/m);
|
||||
// The cell values land in real pipe-delimited data rows.
|
||||
@@ -248,39 +249,21 @@ describe("table clipboard markdown output (htmlToMarkdown)", () => {
|
||||
});
|
||||
|
||||
it("serializes a whole table with a header row as a proper GFM table (headline regression)", () => {
|
||||
// Mirror the serializer's non-wrap branch: the full <table> node is appended
|
||||
// directly (div.appendChild(fragment)) and htmlToMarkdown(div.innerHTML) runs.
|
||||
const div = document.createElement("div");
|
||||
const table = document.createElement("table");
|
||||
|
||||
const thead = document.createElement("thead");
|
||||
const headerRow = document.createElement("tr");
|
||||
for (const h of ["Name", "Age"]) {
|
||||
const th = document.createElement("th");
|
||||
th.textContent = h;
|
||||
headerRow.appendChild(th);
|
||||
}
|
||||
thead.appendChild(headerRow);
|
||||
table.appendChild(thead);
|
||||
|
||||
const tbody = document.createElement("tbody");
|
||||
for (const [name, age] of [
|
||||
["Alice", "30"],
|
||||
["Bob", "25"],
|
||||
]) {
|
||||
const tr = document.createElement("tr");
|
||||
const td1 = document.createElement("td");
|
||||
td1.textContent = name;
|
||||
const td2 = document.createElement("td");
|
||||
td2.textContent = age;
|
||||
tr.appendChild(td1);
|
||||
tr.appendChild(td2);
|
||||
tbody.appendChild(tr);
|
||||
}
|
||||
table.appendChild(tbody);
|
||||
div.appendChild(table);
|
||||
|
||||
const md = htmlToMarkdown(div.innerHTML);
|
||||
// Mirror the serializer's non-wrap branch: the full `table` node is the
|
||||
// slice content and convertProseMirrorToMarkdown runs on it.
|
||||
const md = convertProseMirrorToMarkdown({
|
||||
type: "doc",
|
||||
content: [
|
||||
{
|
||||
type: "table",
|
||||
content: [
|
||||
row([headerCell("Name"), headerCell("Age")]),
|
||||
row([cell("Alice"), cell("30")]),
|
||||
row([cell("Bob"), cell("25")]),
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
const ls = lines(md);
|
||||
|
||||
// Proper GFM structure: separator row + all rows pipe-delimited.
|
||||
@@ -296,3 +279,146 @@ describe("table clipboard markdown output (htmlToMarkdown)", () => {
|
||||
expect(md).not.toMatch(/^\s*(Name|Age|Alice|Bob|30|25)\s*$/m);
|
||||
});
|
||||
});
|
||||
|
||||
// #347 acceptance: pasting CANONICAL markdown yields the SAME nodes the server
|
||||
// import produces for the same text. The paste path calls markdownToProseMirror
|
||||
// (the package browser entry) — the identical converter the server import uses —
|
||||
// so asserting the converter (via the browser entry, on the native DOMParser)
|
||||
// recognizes each canon form pins the paste-parity guarantee. These forms were
|
||||
// NOT recognized by the old editor-ext marked layer the paste used before.
|
||||
describe("canonical markdown paste recognition (browser entry parity)", () => {
|
||||
// Collect every node type present in a doc (recursively).
|
||||
const collectTypes = (n: any, set = new Set<string>()): Set<string> => {
|
||||
if (!n || typeof n !== "object") return set;
|
||||
if (n.type) set.add(n.type);
|
||||
if (Array.isArray(n.content)) n.content.forEach((c) => collectTypes(c, set));
|
||||
return set;
|
||||
};
|
||||
const findNode = (n: any, type: string): any => {
|
||||
if (!n || typeof n !== "object") return undefined;
|
||||
if (n.type === type) return n;
|
||||
if (Array.isArray(n.content)) {
|
||||
for (const c of n.content) {
|
||||
const hit = findNode(c, type);
|
||||
if (hit) return hit;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const allText = (n: any): string => {
|
||||
if (!n || typeof n !== "object") return "";
|
||||
if (typeof n.text === "string") return n.text;
|
||||
if (Array.isArray(n.content)) return n.content.map(allText).join("");
|
||||
return "";
|
||||
};
|
||||
|
||||
it("^[…] inline footnote -> footnoteReference + footnotesList", () => {
|
||||
const doc = markdownToProseMirrorSync("Body^[a note here].");
|
||||
const types = collectTypes(doc);
|
||||
expect(types.has("footnoteReference")).toBe(true);
|
||||
expect(types.has("footnotesList")).toBe(true);
|
||||
expect(types.has("footnoteDefinition")).toBe(true);
|
||||
});
|
||||
|
||||
it('<!--img {…}--> attached image comment -> image with align', () => {
|
||||
const doc = markdownToProseMirrorSync(
|
||||
' <!--img {"align":"left"}-->',
|
||||
);
|
||||
const img = findNode(doc, "image");
|
||||
expect(img).toBeTruthy();
|
||||
expect(img.attrs?.align).toBe("left");
|
||||
expect(img.attrs?.src).toBe("/files/x.png");
|
||||
});
|
||||
|
||||
it("> [!type] Obsidian callout -> callout node with type", () => {
|
||||
const doc = markdownToProseMirrorSync("> [!warning]\n> be careful");
|
||||
const callout = findNode(doc, "callout");
|
||||
expect(callout).toBeTruthy();
|
||||
expect(callout.attrs?.type).toBe("warning");
|
||||
expect(allText(callout)).toContain("be careful");
|
||||
});
|
||||
|
||||
it("$…$ inline math -> mathInline node", () => {
|
||||
const doc = markdownToProseMirrorSync("Euler: $e^{i\\pi}+1=0$ done");
|
||||
const math = findNode(doc, "mathInline");
|
||||
expect(math).toBeTruthy();
|
||||
expect(math.attrs?.text).toContain("e^{i\\pi}");
|
||||
});
|
||||
|
||||
it("==…== highlight -> highlight mark", () => {
|
||||
const doc = markdownToProseMirrorSync("A ==marked== word");
|
||||
const marked = findNode(doc, "text");
|
||||
// The highlighted run carries a `highlight` mark somewhere in the doc.
|
||||
const hasHighlight = (n: any): boolean => {
|
||||
if (!n || typeof n !== "object") return false;
|
||||
if (
|
||||
n.type === "text" &&
|
||||
(n.marks || []).some((m: any) => m.type === "highlight")
|
||||
)
|
||||
return true;
|
||||
return Array.isArray(n.content) ? n.content.some(hasHighlight) : false;
|
||||
};
|
||||
expect(marked).toBeTruthy();
|
||||
expect(hasHighlight(doc)).toBe(true);
|
||||
});
|
||||
|
||||
it("<!--subpages--> standalone comment -> subpages node", () => {
|
||||
const doc = markdownToProseMirrorSync("intro\n\n<!--subpages-->\n\nafter");
|
||||
expect(collectTypes(doc).has("subpages")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// #347 negatives: plain text carrying markdown-LIKE punctuation must NOT be
|
||||
// silently converted/mangled (currency, bare `==`, a `[^1]` reference form).
|
||||
describe("plain-text paste negatives (no phantom conversion)", () => {
|
||||
const findNode = (n: any, type: string): any => {
|
||||
if (!n || typeof n !== "object") return undefined;
|
||||
if (n.type === type) return n;
|
||||
if (Array.isArray(n.content)) {
|
||||
for (const c of n.content) {
|
||||
const hit = findNode(c, type);
|
||||
if (hit) return hit;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const collectTypes = (n: any, set = new Set<string>()): Set<string> => {
|
||||
if (!n || typeof n !== "object") return set;
|
||||
if (n.type) set.add(n.type);
|
||||
if (Array.isArray(n.content)) n.content.forEach((c) => collectTypes(c, set));
|
||||
return set;
|
||||
};
|
||||
const allText = (n: any): string => {
|
||||
if (!n || typeof n !== "object") return "";
|
||||
if (typeof n.text === "string") return n.text;
|
||||
if (Array.isArray(n.content)) return n.content.map(allText).join("");
|
||||
return "";
|
||||
};
|
||||
|
||||
it("currency `$5 and $10` is NOT turned into math", () => {
|
||||
const doc = markdownToProseMirrorSync("It costs $5 and $10 total");
|
||||
expect(findNode(doc, "mathInline")).toBeFalsy();
|
||||
expect(allText(doc)).toContain("$5 and $10");
|
||||
});
|
||||
|
||||
it("a lone `==` is NOT turned into a highlight", () => {
|
||||
const doc = markdownToProseMirrorSync("compare a == b in code");
|
||||
const hasHighlight = (n: any): boolean => {
|
||||
if (!n || typeof n !== "object") return false;
|
||||
if (
|
||||
n.type === "text" &&
|
||||
(n.marks || []).some((m: any) => m.type === "highlight")
|
||||
)
|
||||
return true;
|
||||
return Array.isArray(n.content) ? n.content.some(hasHighlight) : false;
|
||||
};
|
||||
expect(hasHighlight(doc)).toBe(false);
|
||||
expect(allText(doc)).toContain("== b");
|
||||
});
|
||||
|
||||
it("a `[^1]` reference form (no `^[`) is NOT turned into a footnote", () => {
|
||||
const doc = markdownToProseMirrorSync("see note [^1] for details");
|
||||
expect(collectTypes(doc).has("footnoteReference")).toBe(false);
|
||||
expect(allText(doc)).toContain("[^1]");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,15 +1,23 @@
|
||||
// adapted from: https://github.com/aguingand/tiptap-markdown/blob/main/src/extensions/tiptap/clipboard.js - MIT
|
||||
import { Extension } from "@tiptap/core";
|
||||
import { Plugin, PluginKey, TextSelection } from "@tiptap/pm/state";
|
||||
import { DOMParser, DOMSerializer, Fragment, Slice } from "@tiptap/pm/model";
|
||||
import { DOMParser, DOMSerializer, Fragment, Slice, Node as PMNode } from "@tiptap/pm/model";
|
||||
import { find } from "linkifyjs";
|
||||
import {
|
||||
markdownToHtml,
|
||||
htmlToMarkdown,
|
||||
canonicalizeFootnotes,
|
||||
FOOTNOTES_LIST_NAME,
|
||||
FOOTNOTE_REFERENCE_NAME,
|
||||
} from "@docmost/editor-ext";
|
||||
// Markdown <-> ProseMirror conversion now lives ONLY in the canonical
|
||||
// `@docmost/prosemirror-markdown` package (issue #347). The BROWSER entry uses
|
||||
// the native `DOMParser` for its HTML->DOM stage (jsdom stays out of the client
|
||||
// bundle) while producing the SAME nodes the server import does — so a paste of
|
||||
// canonical markdown (`^[…]`, `<!--img …-->`, `> [!type]`, `$…$`, `==…==`,
|
||||
// standalone comments) is recognized identically to import.
|
||||
import {
|
||||
markdownToProseMirror,
|
||||
convertProseMirrorToMarkdown,
|
||||
} from "@docmost/prosemirror-markdown/browser";
|
||||
import type { Schema } from "@tiptap/pm/model";
|
||||
|
||||
export const MarkdownClipboard = Extension.create({
|
||||
@@ -39,25 +47,24 @@ export const MarkdownClipboard = Extension.create({
|
||||
classifyClipboardSelection(topLevelNodes);
|
||||
if (!asMarkdown) return null;
|
||||
|
||||
const div = document.createElement("div");
|
||||
const serializer = DOMSerializer.fromSchema(this.editor.schema);
|
||||
const fragment = serializer.serializeFragment(slice.content);
|
||||
|
||||
// Convert the copied selection to Markdown through the canonical
|
||||
// package (issue #347), the SAME serializer the server export uses,
|
||||
// so a copied table/list matches the on-disk markdown form. The
|
||||
// converter takes a ProseMirror `doc` JSON, so wrap the slice's
|
||||
// top-level content in a synthetic doc.
|
||||
const content = slice.content.toJSON() as any[];
|
||||
if (wrapBareRows) {
|
||||
// A partial table cell-selection serializes to bare <tr> nodes
|
||||
// (prosemirror-tables returns the whole `table` node only when the
|
||||
// entire table is selected). Bare <tr> would be foster-parented
|
||||
// away by the HTML parser inside htmlToMarkdown, so wrap them in
|
||||
// <table><tbody> first for the GFM turndown rule to detect them.
|
||||
const table = document.createElement("table");
|
||||
const tbody = document.createElement("tbody");
|
||||
tbody.appendChild(fragment);
|
||||
table.appendChild(tbody);
|
||||
div.appendChild(table);
|
||||
} else {
|
||||
div.appendChild(fragment);
|
||||
// A partial table cell-selection serializes to bare `tableRow`
|
||||
// nodes (prosemirror-tables yields the whole `table` node only for
|
||||
// a full-table selection). The converter's table case expects a
|
||||
// `table` wrapper, so wrap the bare rows in one — mirroring the old
|
||||
// <table><tbody> wrap that the HTML->markdown step needed.
|
||||
return convertProseMirrorToMarkdown({
|
||||
type: "doc",
|
||||
content: [{ type: "table", content }],
|
||||
});
|
||||
}
|
||||
return htmlToMarkdown(div.innerHTML);
|
||||
return convertProseMirrorToMarkdown({ type: "doc", content });
|
||||
},
|
||||
handlePaste: (view, event, slice) => {
|
||||
if (!event.clipboardData) {
|
||||
@@ -95,37 +102,115 @@ export const MarkdownClipboard = Extension.create({
|
||||
}
|
||||
}
|
||||
|
||||
const { tr } = view.state;
|
||||
const { from, to } = view.state.selection;
|
||||
const schema = this.editor.schema;
|
||||
// Capture the target range NOW. markdownToProseMirror RETURNS A
|
||||
// PROMISE (kept async only for the Node consumers' contract; the
|
||||
// conversion pipeline itself is synchronous), so the actual replace
|
||||
// happens on the next microtask. No user input can interleave a
|
||||
// microtask, so the state is unchanged when we dispatch — but we
|
||||
// still re-read the live state before replacing and, if the doc did
|
||||
// change under us, fall back to the live selection rather than the
|
||||
// captured (now-stale) range.
|
||||
const from = view.state.selection.from;
|
||||
const to = view.state.selection.to;
|
||||
const startDoc = view.state.doc;
|
||||
const md = text.replace(/\n+$/, "");
|
||||
|
||||
const parsed = markdownToHtml(text.replace(/\n+$/, ""));
|
||||
const body = elementFromString(parsed);
|
||||
normalizeTableColumnWidths(body);
|
||||
void markdownToProseMirror(md)
|
||||
.then((doc) => {
|
||||
if (view.isDestroyed) return;
|
||||
// Canonical PM-JSON -> HTML via the LIVE editor schema, then
|
||||
// reuse the UNCHANGED downstream seam (normalizeTableColumnWidths
|
||||
// + parseSlice + canonicalizePastedFootnotes). The JSON->HTML->
|
||||
// JSON hop is lossless (same schema both directions); it lets the
|
||||
// existing paste-insertion logic stay byte-identical — only the
|
||||
// SOURCE of the markdown conversion changed (issue #347 guardrail:
|
||||
// no converter logic in the client, only a call into the package).
|
||||
const node = PMNode.fromJSON(schema, doc);
|
||||
const div = document.createElement("div");
|
||||
DOMSerializer.fromSchema(schema).serializeFragment(
|
||||
node.content,
|
||||
{ document },
|
||||
div,
|
||||
);
|
||||
|
||||
const parsedSlice = DOMParser.fromSchema(
|
||||
this.editor.schema,
|
||||
).parseSlice(body, {
|
||||
preserveWhitespace: true,
|
||||
});
|
||||
const body = elementFromString(div.innerHTML);
|
||||
normalizeTableColumnWidths(body);
|
||||
|
||||
// A markdown paste builds its ProseMirror fragment directly (DOM ->
|
||||
// parseSlice), bypassing the editor's footnoteSyncPlugin, which never
|
||||
// reorders an existing list. So a pasted markdown block whose footnote
|
||||
// definitions are out of order (or contains orphan defs) would be
|
||||
// stored out of order. Canonicalize the self-contained pasted block so
|
||||
// its footnotes come out reference-ordered, deduped and orphan-free
|
||||
// (issue #228). See canonicalizePastedFootnotes for why this is scoped
|
||||
// to whole-block pastes that carry their own footnotesList.
|
||||
const contentNodes = canonicalizePastedFootnotes(
|
||||
parsedSlice,
|
||||
this.editor.schema,
|
||||
);
|
||||
const parsedSlice = DOMParser.fromSchema(schema).parseSlice(
|
||||
body,
|
||||
{ preserveWhitespace: true },
|
||||
);
|
||||
|
||||
tr.replaceRange(from, to, contentNodes);
|
||||
const insertEnd = tr.mapping.map(from, 1);
|
||||
tr.setSelection(TextSelection.near(tr.doc.resolve(Math.max(from, insertEnd - 2)), -1));
|
||||
tr.setMeta('paste', true)
|
||||
view.dispatch(tr);
|
||||
// A markdown paste builds its ProseMirror fragment directly (DOM
|
||||
// -> parseSlice), bypassing the editor's footnoteSyncPlugin, which
|
||||
// never reorders an existing list. So a pasted markdown block whose
|
||||
// footnote definitions are out of order (or contains orphan defs)
|
||||
// would be stored out of order. Canonicalize the self-contained
|
||||
// pasted block so its footnotes come out reference-ordered, deduped
|
||||
// and orphan-free (issue #228). See canonicalizePastedFootnotes for
|
||||
// why this is scoped to whole-block pastes that carry their own
|
||||
// footnotesList.
|
||||
const contentNodes = canonicalizePastedFootnotes(
|
||||
parsedSlice,
|
||||
schema,
|
||||
);
|
||||
|
||||
// Target the captured range (normally still valid — same
|
||||
// microtask). If the doc changed under us since capture, the
|
||||
// captured absolute from/to are stale, so fall back to the live
|
||||
// selection rather than StepMap-mapping the old range.
|
||||
const tr = view.state.tr;
|
||||
let mappedFrom = from;
|
||||
let mappedTo = to;
|
||||
if (view.state.doc !== startDoc) {
|
||||
// Defensive: if the doc changed under us, fall back to the
|
||||
// current selection rather than a stale absolute range.
|
||||
mappedFrom = view.state.selection.from;
|
||||
mappedTo = view.state.selection.to;
|
||||
}
|
||||
tr.replaceRange(mappedFrom, mappedTo, contentNodes);
|
||||
const insertEnd = tr.mapping.map(mappedFrom, 1);
|
||||
tr.setSelection(
|
||||
TextSelection.near(
|
||||
tr.doc.resolve(Math.max(mappedFrom, insertEnd - 2)),
|
||||
-1,
|
||||
),
|
||||
);
|
||||
tr.setMeta("paste", true);
|
||||
view.dispatch(tr);
|
||||
})
|
||||
.catch((err) => {
|
||||
// Fail-open: a conversion error must not swallow the paste
|
||||
// silently in a way that loses the text. We already claimed the
|
||||
// event (returned true), so re-insert the raw text as a plain
|
||||
// paragraph so the user never loses their clipboard content.
|
||||
// Log it: this catch covers BOTH the converter and the success
|
||||
// `.then` body (e.g. PMNode.fromJSON throwing on a schema drift
|
||||
// between the canonical package and the live editor schema), so a
|
||||
// silent degrade to raw text would otherwise be an invisible,
|
||||
// non-reproducible regression ("my table pasted as text").
|
||||
console.error(
|
||||
"markdown paste conversion failed, inserting raw text",
|
||||
err,
|
||||
);
|
||||
if (view.isDestroyed) return;
|
||||
const tr = view.state.tr;
|
||||
// Same guard the success path uses: if the doc changed under us
|
||||
// since the range was captured (normally never — same microtask),
|
||||
// the captured absolute from/to are stale and would throw a
|
||||
// RangeError here (an unhandled rejection on a hot paste path).
|
||||
// Fall back to the live selection instead of a stale range.
|
||||
if (view.state.doc !== startDoc) {
|
||||
const sel = view.state.selection;
|
||||
tr.insertText(md, sel.from, sel.to);
|
||||
} else {
|
||||
tr.insertText(md, from, to);
|
||||
}
|
||||
tr.setMeta("paste", true);
|
||||
view.dispatch(tr);
|
||||
});
|
||||
// Claim the paste: we insert asynchronously above.
|
||||
return true;
|
||||
},
|
||||
// Strip trailing whitespace-only paragraphs from pasted content.
|
||||
|
||||
@@ -33,10 +33,11 @@ vi.mock("@/lib/local-emitter.ts", () => ({
|
||||
default: { emit: (...args: unknown[]) => localEmitMock(...args) },
|
||||
}));
|
||||
|
||||
// htmlToMarkdown just echoes the editor HTML so each test controls the markdown
|
||||
// purely via the fake page editor's getHTML().
|
||||
vi.mock("@docmost/editor-ext", () => ({
|
||||
htmlToMarkdown: (html: string) => html,
|
||||
// convertProseMirrorToMarkdown echoes a marker carried on the fake editor's
|
||||
// getJSON() doc, so each test controls the markdown purely via the fake page
|
||||
// editor (issue #347: the hook now serializes editor JSON through the package).
|
||||
vi.mock("@docmost/prosemirror-markdown/browser", () => ({
|
||||
convertProseMirrorToMarkdown: (doc: { __md?: string }) => doc?.__md ?? "",
|
||||
}));
|
||||
|
||||
const notificationsShowMock = vi.fn();
|
||||
@@ -53,10 +54,12 @@ import { useGeneratePageTitle } from "./use-generate-page-title.ts";
|
||||
|
||||
// --- Test helpers -------------------------------------------------------------
|
||||
|
||||
function makePageEditor(pageId: string, html = "<p>content</p>"): Editor {
|
||||
function makePageEditor(pageId: string, md = "content"): Editor {
|
||||
return {
|
||||
isDestroyed: false,
|
||||
getHTML: () => html,
|
||||
// The mocked convertProseMirrorToMarkdown reads `__md` back off this doc,
|
||||
// so `md` is exactly the markdown the hook will send to the title service.
|
||||
getJSON: () => ({ type: "doc", __md: md }),
|
||||
storage: { pageId },
|
||||
} as unknown as Editor;
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import { useMutation } from "@tanstack/react-query";
|
||||
import { useAtomValue } from "jotai";
|
||||
import { notifications } from "@mantine/notifications";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { htmlToMarkdown } from "@docmost/editor-ext";
|
||||
import { convertProseMirrorToMarkdown } from "@docmost/prosemirror-markdown/browser";
|
||||
import {
|
||||
pageEditorAtom,
|
||||
titleEditorAtom,
|
||||
@@ -49,7 +49,9 @@ export function useGeneratePageTitle(pageId: string) {
|
||||
mutationFn: async () => {
|
||||
if (!pageEditor || pageEditor.isDestroyed) return;
|
||||
|
||||
const markdown = htmlToMarkdown(pageEditor.getHTML()).trim();
|
||||
// Serialize the live editor content to markdown through the canonical
|
||||
// converter (issue #347), matching the on-disk/export markdown form.
|
||||
const markdown = convertProseMirrorToMarkdown(pageEditor.getJSON()).trim();
|
||||
if (!markdown) {
|
||||
notifications.show({ message: t("The note is empty"), color: "yellow" });
|
||||
return;
|
||||
|
||||
@@ -37,7 +37,7 @@ import { useTreeMutation } from "@/features/page/tree/hooks/use-tree-mutation.ts
|
||||
import { PageWidthToggle } from "@/features/user/components/page-width-pref.tsx";
|
||||
import { Trans, useTranslation } from "react-i18next";
|
||||
import ExportModal from "@/components/common/export-modal";
|
||||
import { htmlToMarkdown } from "@docmost/editor-ext";
|
||||
import { convertProseMirrorToMarkdown } from "@docmost/prosemirror-markdown/browser";
|
||||
import {
|
||||
pageEditorAtom,
|
||||
yjsConnectionStatusAtom,
|
||||
@@ -199,8 +199,9 @@ function PageActionMenu({ readOnly }: PageActionMenuProps) {
|
||||
|
||||
const handleCopyAsMarkdown = () => {
|
||||
if (!pageEditor) return;
|
||||
const html = pageEditor.getHTML();
|
||||
const markdown = htmlToMarkdown(html);
|
||||
// Copy the page as canonical markdown through the shared converter (issue
|
||||
// #347), so "Copy as markdown" matches the server export byte-for-byte.
|
||||
const markdown = convertProseMirrorToMarkdown(pageEditor.getJSON());
|
||||
const title = page?.title ? `# ${page.title}\n\n` : "";
|
||||
clipboard.copy(`${title}${markdown}`);
|
||||
notifications.show({ message: t("Copied") });
|
||||
|
||||
@@ -168,6 +168,10 @@ export default function ShareAiWidget({
|
||||
// Anonymous reader: suppress the tool-argument summary line so the
|
||||
// agent's raw query/argument text isn't shown on the public share.
|
||||
showInput={false}
|
||||
// Anonymous reader: never paint a tool's raw errorText (it can carry
|
||||
// internal detail). This is the render gate; the bytes are also
|
||||
// sanitized server-side in PublicShareChatToolsService.forShare (#394).
|
||||
showErrors={false}
|
||||
// Anonymous reader: neutralize internal/relative links in the
|
||||
// assistant's markdown so internal UUIDs/auth-gated routes don't
|
||||
// leak as clickable links (external http(s) links are kept).
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { markdownToHtml, encodeHtmlEmbedSource } from '@docmost/editor-ext';
|
||||
import { markdownToProseMirror } from '@docmost/prosemirror-markdown';
|
||||
import { encodeHtmlEmbedSource } from '@docmost/editor-ext';
|
||||
import { htmlToJson } from '../../../collaboration/collaboration.util';
|
||||
import { hasHtmlEmbedNode, stripHtmlEmbedNodes } from './html-embed.util';
|
||||
|
||||
@@ -10,13 +11,12 @@ import { hasHtmlEmbedNode, stripHtmlEmbedNodes } from './html-embed.util';
|
||||
*
|
||||
* The block renders inside a sandboxed iframe, so this is not an XSS surface;
|
||||
* this exercises the REAL server import conversion path that ImportService uses
|
||||
* (`markdownToHtml` then `htmlToJson`; `processHTML` adds only a cheerio
|
||||
* link/iframe normalize pass which does not touch htmlEmbed divs) and asserts
|
||||
* that such a node is DETECTED and STRIPPABLE — so the share read path's
|
||||
* (`markdownToProseMirror`, the canonical converter — issue #345/#347) and
|
||||
* asserts that such a node is DETECTED and STRIPPABLE — so the share read path's
|
||||
* master-toggle strip can remove it when the workspace toggle is OFF.
|
||||
*/
|
||||
describe('htmlEmbed smuggled via the raw serialized div in imported markdown/HTML', () => {
|
||||
it('round-trips through markdownToHtml -> htmlToJson and is DETECTED (base64 data-source)', async () => {
|
||||
it('round-trips through markdownToProseMirror and is DETECTED (base64 data-source)', async () => {
|
||||
const source = '<script>steal()</script>';
|
||||
const encoded = encodeHtmlEmbedSource(source);
|
||||
const md = [
|
||||
@@ -27,12 +27,9 @@ describe('htmlEmbed smuggled via the raw serialized div in imported markdown/HTM
|
||||
'World',
|
||||
].join('\n');
|
||||
|
||||
const html = await markdownToHtml(md);
|
||||
// marked preserves the raw block-level div verbatim.
|
||||
expect(html).toContain('data-type="htmlEmbed"');
|
||||
|
||||
const json = htmlToJson(html);
|
||||
// The div parses into a real htmlEmbed node carrying the decoded source.
|
||||
// The canonical importer parses the raw block-level div into a real
|
||||
// htmlEmbed node carrying the decoded source.
|
||||
const json = await markdownToProseMirror(md);
|
||||
expect(hasHtmlEmbedNode(json)).toBe(true);
|
||||
|
||||
// Because it is detected, the share master-toggle strip can remove it.
|
||||
@@ -59,8 +56,7 @@ describe('htmlEmbed smuggled via the raw serialized div in imported markdown/HTM
|
||||
// therefore stripping) does not depend on the source being well-formed, so
|
||||
// the bypass cannot be hidden by sending a malformed data-source.
|
||||
const md = `<div data-type="htmlEmbed" data-source="<script>x</script>"></div>`;
|
||||
const html = await markdownToHtml(md);
|
||||
const json = htmlToJson(html);
|
||||
const json = await markdownToProseMirror(md);
|
||||
expect(hasHtmlEmbedNode(json)).toBe(true);
|
||||
expect(hasHtmlEmbedNode(stripHtmlEmbedNodes(json))).toBe(false);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
import {
|
||||
ConflictException,
|
||||
Logger,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { AiChatService } from './ai-chat.service';
|
||||
import { RunAlreadyActiveError } from './ai-chat-run.service';
|
||||
|
||||
/**
|
||||
* Fail-fast guard for beginRun failures (#486, commit 4).
|
||||
*
|
||||
* When runHooks.begin() rejects for a reason OTHER than RunAlreadyActiveError
|
||||
* (e.g. a DB-pool blip), the turn must NOT continue untracked. The old code
|
||||
* logged and streamed anyway, leaving a run with NO run-row: in autonomous mode
|
||||
* nobody could abort it (/stop can't see it, disconnect doesn't abort it, and the
|
||||
* one-run gate would admit a SECOND run) — an unstoppable invisible run until
|
||||
* restart. The fix throws A_RUN_BEGIN_FAILED (503) BEFORE the first byte and
|
||||
* before the user row is persisted.
|
||||
*
|
||||
* We drive `stream()` directly on a prototype instance wired with only the
|
||||
* collaborators it touches before the throw, so the assertion is on the REAL
|
||||
* control flow, not a mock of it.
|
||||
*/
|
||||
describe('AiChatService beginRun failure (#486)', () => {
|
||||
function makeService(insertSpy: jest.Mock): AiChatService {
|
||||
// Bypass the (heavy) DI constructor: exercise the real stream() method on a
|
||||
// bare prototype instance with just the fields reached before the throw.
|
||||
// `any` because the private `logger` field makes a typed intersection collapse.
|
||||
const svc = Object.create(AiChatService.prototype);
|
||||
svc.aiChatRepo = {
|
||||
// Existing chat -> no insert path; chatId is kept as-is.
|
||||
findById: jest.fn().mockResolvedValue({ id: 'chat1' }),
|
||||
};
|
||||
svc.aiChatMessageRepo = { insert: insertSpy };
|
||||
svc.logger = new Logger('test');
|
||||
return svc as AiChatService;
|
||||
}
|
||||
|
||||
const baseArgs = () => {
|
||||
const write = jest.fn();
|
||||
const res = {
|
||||
raw: { write, writableEnded: false, headersSent: false },
|
||||
};
|
||||
return {
|
||||
user: { id: 'u1' } as never,
|
||||
workspace: { id: 'w1' } as never,
|
||||
sessionId: 's1',
|
||||
// openPage undefined -> resolveOpenPageContext returns null without any DB
|
||||
// call; chatId present -> the existing-chat path.
|
||||
body: { chatId: 'chat1', messages: [] } as never,
|
||||
res: res as never,
|
||||
signal: new AbortController().signal,
|
||||
model: {} as never,
|
||||
role: null,
|
||||
write,
|
||||
};
|
||||
};
|
||||
|
||||
it('throws A_RUN_BEGIN_FAILED (503) before the first byte and before persisting the user turn', async () => {
|
||||
const insertSpy = jest.fn();
|
||||
const svc = makeService(insertSpy);
|
||||
const { write, ...args } = baseArgs();
|
||||
|
||||
const runHooks = {
|
||||
begin: jest.fn().mockRejectedValue(new Error('DB pool exhausted')),
|
||||
} as never;
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await svc.stream({ ...args, runHooks });
|
||||
} catch (e) {
|
||||
caught = e;
|
||||
}
|
||||
|
||||
expect(caught).toBeInstanceOf(ServiceUnavailableException);
|
||||
const http = caught as ServiceUnavailableException;
|
||||
expect(http.getStatus()).toBe(503);
|
||||
expect(http.getResponse()).toMatchObject({ code: 'A_RUN_BEGIN_FAILED' });
|
||||
|
||||
// Fail-fast: nothing was written to the socket and NO user message row was
|
||||
// persisted, so the turn left no orphan state to clean up.
|
||||
expect(write).not.toHaveBeenCalled();
|
||||
expect(insertSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('still maps a lost-the-race RunAlreadyActiveError to a 409, not A_RUN_BEGIN_FAILED', async () => {
|
||||
const insertSpy = jest.fn();
|
||||
const svc = makeService(insertSpy);
|
||||
const { write, ...args } = baseArgs();
|
||||
|
||||
const runHooks = {
|
||||
begin: jest.fn().mockRejectedValue(new RunAlreadyActiveError('chat1')),
|
||||
} as never;
|
||||
|
||||
let caught: unknown;
|
||||
try {
|
||||
await svc.stream({ ...args, runHooks });
|
||||
} catch (e) {
|
||||
caught = e;
|
||||
}
|
||||
|
||||
expect(caught).toBeInstanceOf(ConflictException);
|
||||
expect((caught as ConflictException).getResponse()).toMatchObject({
|
||||
code: 'A_RUN_ALREADY_ACTIVE',
|
||||
});
|
||||
expect(write).not.toHaveBeenCalled();
|
||||
expect(insertSpy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,8 @@
|
||||
import { ConflictException, Logger } from '@nestjs/common';
|
||||
import {
|
||||
ConflictException,
|
||||
Logger,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
|
||||
// Mock the AI SDK so we can PROVE no provider call is made for the turn we are
|
||||
// about to reject. The race rejection happens at runHooks.begin(), long before
|
||||
@@ -360,22 +364,22 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => {
|
||||
});
|
||||
|
||||
/**
|
||||
* F14 — the begin-failure RESILIENCE branch (the `else` of the run-race guard).
|
||||
* F14 — the begin-failure branch (the `else` of the run-race guard).
|
||||
*
|
||||
* stream() wraps runHooks.begin in try/catch with TWO branches:
|
||||
* - RunAlreadyActiveError -> 409 ConflictException (pinned above).
|
||||
* - ANY OTHER begin failure -> SWALLOW + continue UNTRACKED on the socket signal
|
||||
* (legacy fallback): it logs "...streaming without run tracking", leaves
|
||||
* `effectiveSignal = signal` (runId undefined) and serves the turn anyway.
|
||||
* - ANY OTHER begin failure -> throw ServiceUnavailableException(A_RUN_BEGIN_FAILED)
|
||||
* BEFORE the first byte (#486, commit 4).
|
||||
*
|
||||
* The contract: a transient beginRun failure (e.g. a non-unique DB error inserting
|
||||
* the run row) must STILL serve the user's turn — it must NOT re-throw and must NOT
|
||||
* be misclassified as a 409. A regression that re-threw here would break EVERY turn
|
||||
* on a begin failure with nothing to catch it. This branch is otherwise undriven by
|
||||
* any spec, so it is pinned here SEPARATELY from the 409 path: a plain begin error
|
||||
* proceeds to streamText with the SOCKET signal and still persists the user turn.
|
||||
* POLICY CHANGE (#486): the OLD contract here was "SWALLOW + stream the turn
|
||||
* UNTRACKED on the socket signal". That was reversed: an untracked run is
|
||||
* invisible to /stop, is not aborted on disconnect, and slips past the one-run
|
||||
* gate — an unstoppable ghost run in autonomous mode. Now a plain begin failure
|
||||
* FAILS the turn fast with a 503 A_RUN_BEGIN_FAILED, before any user row is
|
||||
* persisted and before streamText runs. This case is INVERTED (not deleted) so
|
||||
* the "plain begin failure" path stays explicitly pinned under the new policy.
|
||||
*/
|
||||
describe('AiChatService.stream — begin-failure resilience / legacy fallback (#184 F14)', () => {
|
||||
describe('AiChatService.stream — begin-failure fails the turn (#184 F14 / #486)', () => {
|
||||
const streamTextMock = streamText as unknown as jest.Mock;
|
||||
|
||||
function makeStreamResult() {
|
||||
@@ -455,7 +459,7 @@ describe('AiChatService.stream — begin-failure resilience / legacy fallback (#
|
||||
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('a PLAIN begin() failure (NOT RunAlreadyActiveError) does NOT 409 — it swallows, logs, and streams the turn UNTRACKED on the socket signal', async () => {
|
||||
it('a PLAIN begin() failure (NOT RunAlreadyActiveError) FAILS the turn with a 503 A_RUN_BEGIN_FAILED before the first byte — NO untracked stream (#486)', async () => {
|
||||
const errorSpy = jest
|
||||
.spyOn(Logger.prototype, 'error')
|
||||
.mockImplementation(() => undefined as never);
|
||||
@@ -487,28 +491,26 @@ describe('AiChatService.stream — begin-failure resilience / legacy fallback (#
|
||||
} as never,
|
||||
});
|
||||
|
||||
// The turn proceeds: NO throw at all (in particular NOT a 409).
|
||||
await expect(promise).resolves.toBeUndefined();
|
||||
// NEW POLICY: the turn is REJECTED with a 503 A_RUN_BEGIN_FAILED (not a 409,
|
||||
// and NOT swallowed into an untracked stream).
|
||||
await expect(promise).rejects.toBeInstanceOf(ServiceUnavailableException);
|
||||
const err = (await promise.catch(
|
||||
(e) => e,
|
||||
)) as ServiceUnavailableException;
|
||||
expect(err.getStatus()).toBe(503);
|
||||
expect(err.getResponse()).toMatchObject({ code: 'A_RUN_BEGIN_FAILED' });
|
||||
|
||||
expect(begin).toHaveBeenCalledTimes(1);
|
||||
|
||||
// The resilience branch logged the legacy-fallback warning.
|
||||
// It logged the fail-the-turn line.
|
||||
expect(errorSpy).toHaveBeenCalledWith(
|
||||
expect.stringContaining('streaming without run tracking'),
|
||||
expect.stringContaining('failing the turn'),
|
||||
expect.anything(),
|
||||
);
|
||||
|
||||
// The turn really streamed: the user message was persisted and streamText ran.
|
||||
expect(aiChatMessageRepo.insert).toHaveBeenCalled();
|
||||
expect(streamTextMock).toHaveBeenCalledTimes(1);
|
||||
|
||||
// The decisive wiring: with no run handle, the fallback uses the SOCKET signal
|
||||
// (effectiveSignal = signal, runId undefined) — not a run-bound signal. #444:
|
||||
// the signal is unioned with the degeneration controller via AbortSignal.any,
|
||||
// so assert the socket abort still reaches the turn rather than identity.
|
||||
const passed = streamTextMock.mock.calls[0][0].abortSignal as AbortSignal;
|
||||
expect(passed.aborted).toBe(false);
|
||||
socketController.abort();
|
||||
expect(passed.aborted).toBe(true);
|
||||
// Fail-fast: the turn NEVER streamed — no user row persisted, no streamText
|
||||
// call, so no orphan/untracked run was left behind.
|
||||
expect(aiChatMessageRepo.insert).not.toHaveBeenCalled();
|
||||
expect(streamTextMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
Injectable,
|
||||
Logger,
|
||||
OnModuleInit,
|
||||
ServiceUnavailableException,
|
||||
} from '@nestjs/common';
|
||||
import { FastifyReply } from 'fastify';
|
||||
import {
|
||||
@@ -55,6 +56,7 @@ import {
|
||||
import {
|
||||
isDegenerateOutput,
|
||||
truncateDegeneratedTail,
|
||||
shouldCheckDegeneration,
|
||||
} from './output-degeneration';
|
||||
|
||||
// Max agent steps per turn. One step = one model generation; a step that calls
|
||||
@@ -756,6 +758,13 @@ export class AiChatService implements OnModuleInit {
|
||||
// or violate the page_id FK on insert (this runs after res.hijack(), so a
|
||||
// DB error would break the stream).
|
||||
const originPageId: string | null = openPageContext?.id ?? null;
|
||||
// ORPHAN-ON-BEGIN-FAILURE tradeoff (#486, B3): the chat row is inserted
|
||||
// HERE, before runHooks.begin below. If begin fails (e.g. a 503 / run-slot
|
||||
// rejection) the turn aborts before the client is told this new chatId, so
|
||||
// an empty chat is left behind and a retry mints ANOTHER one. We accept this
|
||||
// over reordering: begin needs a chatId to bind the run to, and inserting
|
||||
// the chat first keeps the id stable + the FK/history-join invariants above
|
||||
// intact. Orphan empty chats are cheap and swept by normal chat cleanup.
|
||||
const chat = await this.aiChatRepo.insert({
|
||||
creatorId: user.id,
|
||||
workspaceId: workspace.id,
|
||||
@@ -797,12 +806,32 @@ export class AiChatService implements OnModuleInit {
|
||||
code: 'A_RUN_ALREADY_ACTIVE',
|
||||
});
|
||||
}
|
||||
// Any OTHER run-start failure must not break the turn — fall back to the
|
||||
// socket signal (legacy behavior) and stream anyway.
|
||||
// Any OTHER run-start failure (e.g. a DB-pool blip) must FAIL THE TURN,
|
||||
// not silently stream without a run-row. The old fallback let the turn
|
||||
// continue untracked: in autonomous mode nobody could then abort it —
|
||||
// /stop can't see a run that doesn't exist, a client disconnect doesn't
|
||||
// abort it, and the one-run-per-chat gate would let a SECOND run in. That
|
||||
// is an unstoppable, invisible run until process restart. Reject NOW,
|
||||
// BEFORE the first byte (nothing is written yet, no user row inserted, no
|
||||
// MCP lease taken), so the controller's post-hijack catch turns this
|
||||
// HttpException into an honest 503 on the raw socket. Same policy for BOTH
|
||||
// modes — #487 inherits it (no mode-branching here).
|
||||
this.logger.error(
|
||||
`Failed to begin agent run (chat ${chatId}); streaming without run tracking`,
|
||||
`Failed to begin agent run (chat ${chatId}); failing the turn`,
|
||||
err as Error,
|
||||
);
|
||||
throw new ServiceUnavailableException({
|
||||
message:
|
||||
'Could not start the agent run. This is usually temporary — please try again.',
|
||||
code: 'A_RUN_BEGIN_FAILED',
|
||||
// Self-describe the status in the body: the controller's post-hijack
|
||||
// catch writes getResponse() verbatim onto the raw socket, and an
|
||||
// object-arg HttpException does NOT inject statusCode. Without it the
|
||||
// client's 503 classifier (which reads the body JSON) could not see the
|
||||
// status. With it present, the client's A_RUN_BEGIN_FAILED branch (which
|
||||
// runs strictly before the generic-503 branch) shows "temporary, retry".
|
||||
statusCode: 503,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1080,7 +1109,6 @@ export class AiChatService implements OnModuleInit {
|
||||
const degenerationController = new AbortController();
|
||||
let degenerationDetected = false;
|
||||
let lastDegenerationCheckLen = 0;
|
||||
const DEGENERATION_CHECK_STEP = 2000;
|
||||
|
||||
// Step-granular durability (#183): create the assistant row UPFRONT in the
|
||||
// 'streaming' state (before any token), then UPDATE it as each step finishes
|
||||
@@ -1254,8 +1282,10 @@ export class AiChatService implements OnModuleInit {
|
||||
// trigger, abort the run ONCE with a distinguishable reason.
|
||||
if (
|
||||
!degenerationDetected &&
|
||||
inProgressText.length - lastDegenerationCheckLen >=
|
||||
DEGENERATION_CHECK_STEP
|
||||
shouldCheckDegeneration(
|
||||
inProgressText.length,
|
||||
lastDegenerationCheckLen,
|
||||
)
|
||||
) {
|
||||
lastDegenerationCheckLen = inProgressText.length;
|
||||
if (isDegenerateOutput(inProgressText)) {
|
||||
@@ -1275,6 +1305,13 @@ export class AiChatService implements OnModuleInit {
|
||||
// the in-progress accumulator for the next step.
|
||||
capturedSteps.push(step as StepLike);
|
||||
inProgressText = '';
|
||||
// Reset the degeneration-check watermark too (#486): it tracks a byte
|
||||
// offset INTO inProgressText, so once that resets to '' a stale (large)
|
||||
// mark makes `inProgressText.length - lastDegenerationCheckLen` go
|
||||
// negative and the throttled detector stays silent until a later step's
|
||||
// text re-grows past the old offset — a whole degenerate step could slip
|
||||
// through undetected. Zeroing it re-arms the check from the next byte.
|
||||
lastDegenerationCheckLen = 0;
|
||||
// Step-granular durability (#183): persist this finished step (its text +
|
||||
// tool calls + tool RESULTS) the moment it ends, so a process death after
|
||||
// this point still recovers the step. Not awaited here (never block the
|
||||
|
||||
@@ -1,11 +1,24 @@
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { streamText } from 'ai';
|
||||
import {
|
||||
hasRepeatedLineRun,
|
||||
hasPeriodicTail,
|
||||
isDegenerateOutput,
|
||||
truncateDegeneratedTail,
|
||||
shouldCheckDegeneration,
|
||||
DEGENERATION_CHECK_STEP,
|
||||
REPEATED_LINES_THRESHOLD,
|
||||
MIN_PERIOD_REPEATS,
|
||||
} from './output-degeneration';
|
||||
import { AiChatService } from './ai-chat.service';
|
||||
|
||||
// Mock ONLY streamText so we can capture the onChunk/onStepFinish callbacks the
|
||||
// service registers and drive them by hand; every other `ai` export the service
|
||||
// uses (convertToModelMessages, stepCountIs, …) stays real.
|
||||
jest.mock('ai', () => {
|
||||
const actual = jest.requireActual('ai');
|
||||
return { ...actual, streamText: jest.fn() };
|
||||
});
|
||||
|
||||
/**
|
||||
* Unit tests for the token-degeneration detector (#444) — the sole anti-babble
|
||||
@@ -180,3 +193,188 @@ describe('truncateDegeneratedTail', () => {
|
||||
expect(truncateDegeneratedTail(text)).toBe(text);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Throttle + step-boundary reset (#486). The stream keeps a watermark
|
||||
* (`lastDegenerationCheckLen`) that is an OFFSET into the accumulated step text.
|
||||
* On a step boundary the accumulator resets to '', so the watermark MUST reset to
|
||||
* 0 too — otherwise the throttle goes silent for the whole next step. These tests
|
||||
* pin the pure decision AND the reset property that ai-chat.service.onStepFinish
|
||||
* now enforces.
|
||||
*/
|
||||
describe('shouldCheckDegeneration (throttle) + step-boundary reset (#486)', () => {
|
||||
it('fires once the text grows a full DEGENERATION_CHECK_STEP past the mark', () => {
|
||||
expect(shouldCheckDegeneration(DEGENERATION_CHECK_STEP, 0)).toBe(true);
|
||||
expect(shouldCheckDegeneration(DEGENERATION_CHECK_STEP - 1, 0)).toBe(false);
|
||||
expect(shouldCheckDegeneration(5000, 3000)).toBe(true); // grew 2000 since mark
|
||||
expect(shouldCheckDegeneration(4000, 3000)).toBe(false); // grew only 1000
|
||||
});
|
||||
|
||||
it('BUG (no reset): a stale large watermark silences the next step', () => {
|
||||
// End of a long step: the watermark sits at 5000. The step ends and the
|
||||
// accumulator resets to '' — but if the watermark is NOT reset, a fresh short
|
||||
// degenerate burst (length 2000) never triggers a check: 2000 - 5000 < STEP.
|
||||
const staleWatermark = 5000;
|
||||
const nextStepLen = DEGENERATION_CHECK_STEP; // a fresh 2KB burst
|
||||
expect(shouldCheckDegeneration(nextStepLen, staleWatermark)).toBe(false);
|
||||
});
|
||||
|
||||
it('FIX (reset to 0): the same short degenerate burst IS checked and detected', () => {
|
||||
// onStepFinish now zeroes the watermark, so the fresh burst re-arms the check.
|
||||
const resetWatermark = 0;
|
||||
const degenerateBurst = 'loadTools.\n'.repeat(300); // real degeneration
|
||||
expect(degenerateBurst.length).toBeGreaterThanOrEqual(DEGENERATION_CHECK_STEP);
|
||||
// The throttle now fires...
|
||||
expect(
|
||||
shouldCheckDegeneration(degenerateBurst.length, resetWatermark),
|
||||
).toBe(true);
|
||||
// ...and the detector catches the loop that would otherwise stream unchecked.
|
||||
expect(isDegenerateOutput(degenerateBurst)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* BEHAVIOR guard for the ACTUAL fix (#486, ai-chat.service.onStepFinish resets
|
||||
* lastDegenerationCheckLen to 0). The pure tests above use a hard-coded
|
||||
* resetWatermark, so a REVERT of the real `lastDegenerationCheckLen = 0` line
|
||||
* would not redden any of them. This drives the REAL onChunk/onStepFinish
|
||||
* closures from stream() end to end and asserts the run is aborted when a fresh
|
||||
* degenerate burst arrives in the step AFTER a long clean step — which only
|
||||
* happens if the watermark was actually zeroed on the step boundary.
|
||||
*/
|
||||
describe('AiChatService: onStepFinish re-arms the degeneration watermark (#486)', () => {
|
||||
const streamTextMock = streamText as unknown as jest.Mock;
|
||||
|
||||
function makeRes() {
|
||||
return {
|
||||
raw: {
|
||||
writeHead: jest.fn(),
|
||||
write: jest.fn(),
|
||||
once: jest.fn(),
|
||||
on: jest.fn(),
|
||||
flushHeaders: jest.fn(),
|
||||
writableEnded: false,
|
||||
destroyed: false,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeService() {
|
||||
const aiChatRepo = {
|
||||
findById: jest.fn(async () => ({ id: 'chat-1', workspaceId: 'ws-1' })),
|
||||
insert: jest.fn(),
|
||||
};
|
||||
const aiChatMessageRepo = {
|
||||
insert: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
findAllByChat: jest.fn(async () => []),
|
||||
update: jest.fn(async () => ({ id: 'msg-1' })),
|
||||
};
|
||||
const aiSettings = { resolve: jest.fn(async () => ({})) };
|
||||
const tools = { forUser: jest.fn(async () => ({})) };
|
||||
const mcpClients = {
|
||||
toolsFor: jest.fn(async () => ({
|
||||
tools: {},
|
||||
clients: [],
|
||||
outcomes: [],
|
||||
instructions: [],
|
||||
})),
|
||||
};
|
||||
return new AiChatService(
|
||||
{} as never, // ai
|
||||
aiChatRepo as never,
|
||||
aiChatMessageRepo as never,
|
||||
{} as never, // aiChatPageSnapshotRepo
|
||||
aiSettings as never,
|
||||
tools as never,
|
||||
mcpClients as never,
|
||||
{} as never, // aiAgentRoleRepo
|
||||
{} as never, // pageRepo
|
||||
{} as never, // pageAccess
|
||||
{
|
||||
isAiChatDeferredToolsEnabled: () => false,
|
||||
// Lockdown OFF -> the degeneration guard is the active anti-babble path.
|
||||
isAiChatFinalStepLockdownEnabled: () => false,
|
||||
} as never, // environment
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
streamTextMock.mockReset();
|
||||
jest.spyOn(Logger.prototype, 'log').mockImplementation(() => undefined as never);
|
||||
jest.spyOn(Logger.prototype, 'warn').mockImplementation(() => undefined as never);
|
||||
});
|
||||
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('aborts on a fresh degenerate burst in the NEXT step (reverting the reset line reddens this)', async () => {
|
||||
let captured:
|
||||
| {
|
||||
onChunk?: (e: { chunk: { type: string; text: string } }) => void;
|
||||
onStepFinish?: (step: unknown) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
| undefined;
|
||||
streamTextMock.mockImplementation((opts: never) => {
|
||||
captured = opts;
|
||||
return {
|
||||
consumeStream: jest.fn(),
|
||||
pipeUIMessageStreamToResponse: jest.fn(),
|
||||
};
|
||||
});
|
||||
|
||||
const svc = makeService();
|
||||
await svc.stream({
|
||||
user: { id: 'user-1' } as never,
|
||||
workspace: { id: 'ws-1' } as never,
|
||||
sessionId: 'sess-1',
|
||||
body: {
|
||||
chatId: 'chat-1',
|
||||
messages: [
|
||||
{ id: 'm1', role: 'user', parts: [{ type: 'text', text: 'hi' }] },
|
||||
],
|
||||
} as never,
|
||||
res: makeRes() as never,
|
||||
signal: new AbortController().signal,
|
||||
model: {} as never,
|
||||
role: null,
|
||||
// No runHooks -> legacy path (socket signal), degeneration guard active.
|
||||
});
|
||||
|
||||
expect(streamTextMock).toHaveBeenCalledTimes(1);
|
||||
const onChunk = captured!.onChunk!;
|
||||
const onStepFinish = captured!.onStepFinish!;
|
||||
const abortSignal = captured!.abortSignal!;
|
||||
expect(abortSignal.aborted).toBe(false);
|
||||
|
||||
// STEP 1: a LONG, non-degenerate first step. Distinct lines never trip the
|
||||
// detector, but they advance the throttle watermark far past the burst size
|
||||
// that follows (to ~5x the step). This is the stale watermark that, WITHOUT
|
||||
// the reset, would silence step 2.
|
||||
let counter = 0;
|
||||
let accumulated = 0;
|
||||
while (accumulated < DEGENERATION_CHECK_STEP * 5) {
|
||||
const line = `unique clean line number ${counter++} with distinct words\n`;
|
||||
accumulated += line.length;
|
||||
onChunk({ chunk: { type: 'text-delta', text: line } });
|
||||
}
|
||||
expect(abortSignal.aborted).toBe(false); // clean step must not abort
|
||||
|
||||
// STEP BOUNDARY: the real onStepFinish resets inProgressText AND (the fix)
|
||||
// zeroes lastDegenerationCheckLen.
|
||||
onStepFinish({ text: 'a clean first step', toolCalls: [], toolResults: [] });
|
||||
|
||||
// STEP 2: a FRESH, short degenerate burst (~3.3KB). Its length is far below
|
||||
// the step-1 stale watermark (~10KB), so WITHOUT the reset the throttle stays
|
||||
// silent and this streams unchecked. WITH the reset (watermark 0) it re-arms,
|
||||
// the detector fires, and the run aborts.
|
||||
const burst = 'loadTools.\n'.repeat(300);
|
||||
expect(burst.length).toBeGreaterThanOrEqual(DEGENERATION_CHECK_STEP);
|
||||
expect(burst.length).toBeLessThan(DEGENERATION_CHECK_STEP * 5);
|
||||
onChunk({ chunk: { type: 'text-delta', text: burst } });
|
||||
|
||||
// The decisive assertion: the composed abortSignal (unioned with the
|
||||
// degeneration controller) is now aborted. Reverting `lastDegenerationCheckLen
|
||||
// = 0` in onStepFinish makes this stay false.
|
||||
expect(abortSignal.aborted).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -131,6 +131,32 @@ export function isDegenerateOutput(text: string): boolean {
|
||||
return hasRepeatedLineRun(text) || hasPeriodicTail(text);
|
||||
}
|
||||
|
||||
/**
|
||||
* How many bytes the in-progress text must grow before the (amortized) tail
|
||||
* heuristics are re-run. Shared with ai-chat.service so the throttle the stream
|
||||
* applies is the SAME one the unit test drives.
|
||||
*/
|
||||
export const DEGENERATION_CHECK_STEP = 2000;
|
||||
|
||||
/**
|
||||
* Throttle decision for the degeneration guard (#444/#486). Returns true when
|
||||
* the accumulated text has grown at least DEGENERATION_CHECK_STEP bytes past the
|
||||
* last-checked offset, so the pure rules only fire every ~2KB. Pure; the caller
|
||||
* updates its watermark to `textLen` when this returns true.
|
||||
*
|
||||
* The watermark is an offset INTO the accumulator, so when the accumulator is
|
||||
* reset to '' on a step boundary the caller MUST reset the watermark to 0 too
|
||||
* (#486). Otherwise `textLen - lastCheckLen` goes negative after the reset and
|
||||
* this returns false until a later step re-grows past the stale offset — a whole
|
||||
* degenerate step could stream unchecked.
|
||||
*/
|
||||
export function shouldCheckDegeneration(
|
||||
textLen: number,
|
||||
lastCheckLen: number,
|
||||
): boolean {
|
||||
return textLen - lastCheckLen >= DEGENERATION_CHECK_STEP;
|
||||
}
|
||||
|
||||
/**
|
||||
* Truncate a degenerated tail before persist so hundreds of KB of garbage never
|
||||
* reach the DB / replay (#444). Keeps everything up to and including the FIRST
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
// Break the editor-ext import chain (share.service -> collaboration.util ->
|
||||
// @docmost/editor-ext -> @tiptap/core) that is unresolvable in this jest env and
|
||||
// pre-existingly breaks these specs. jsonToMarkdown is never reached in these
|
||||
// tests (the tools fail before rendering markdown).
|
||||
jest.mock('../../collaboration/collaboration.util', () => ({
|
||||
jsonToMarkdown: () => '',
|
||||
}));
|
||||
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { MockLanguageModelV3, simulateReadableStream } from 'ai/test';
|
||||
import { PublicShareChatService } from './public-share-chat.service';
|
||||
import { PublicShareChatToolsService } from './tools/public-share-chat-tools.service';
|
||||
|
||||
/**
|
||||
* SECURITY integration guard for #394 (commit 5): a tool's or the provider's raw
|
||||
* error text must NOT leak to an anonymous public-share reader.
|
||||
*
|
||||
* The render gate (ToolCallCard showErrors=false) hides the text in the DOM but
|
||||
* NOT on the wire, so this test asserts on the RAW SSE BYTES the server writes —
|
||||
* exactly the channel the render gate masks. We drive the real
|
||||
* PublicShareChatService.stream() with a real share toolset (its underlying
|
||||
* services mocked to fail) and a mock model, then inspect every byte piped to the
|
||||
* fake socket.
|
||||
*/
|
||||
|
||||
// A minimal ServerResponse stand-in that records every written chunk.
|
||||
class FakeSocket {
|
||||
chunks: string[] = [];
|
||||
statusCode = 200;
|
||||
writableEnded = false;
|
||||
destroyed = false;
|
||||
headersSent = false;
|
||||
writeHead(): this {
|
||||
this.headersSent = true;
|
||||
return this;
|
||||
}
|
||||
setHeader(): void {}
|
||||
removeHeader(): void {}
|
||||
getHeader(): undefined {
|
||||
return undefined;
|
||||
}
|
||||
flushHeaders(): void {}
|
||||
write(chunk: unknown): boolean {
|
||||
this.chunks.push(
|
||||
typeof chunk === 'string' ? chunk : Buffer.from(chunk as never).toString('utf8'),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
end(chunk?: unknown): void {
|
||||
if (chunk) this.write(chunk);
|
||||
this.writableEnded = true;
|
||||
}
|
||||
on(): this {
|
||||
return this;
|
||||
}
|
||||
once(): this {
|
||||
return this;
|
||||
}
|
||||
get body(): string {
|
||||
return this.chunks.join('');
|
||||
}
|
||||
}
|
||||
|
||||
/** Mock model that issues one getSharePage tool call, then finishes with text. */
|
||||
function toolCallingModel(): MockLanguageModelV3 {
|
||||
let call = 0;
|
||||
return new MockLanguageModelV3({
|
||||
doStream: async () => {
|
||||
call++;
|
||||
if (call === 1) {
|
||||
return {
|
||||
stream: simulateReadableStream({
|
||||
chunks: [
|
||||
{ type: 'stream-start' as const, warnings: [] },
|
||||
{ type: 'tool-input-start' as const, id: 't1', toolName: 'getSharePage' },
|
||||
{ type: 'tool-input-end' as const, id: 't1' },
|
||||
{
|
||||
type: 'tool-call' as const,
|
||||
toolCallId: 't1',
|
||||
toolName: 'getSharePage',
|
||||
input: '{"pageId":"secret-page"}',
|
||||
},
|
||||
{
|
||||
type: 'finish' as const,
|
||||
finishReason: { unified: 'tool-calls' as const, raw: 'tool_calls' },
|
||||
usage: {
|
||||
inputTokens: { total: 1, noCache: undefined, cacheRead: undefined, cacheWrite: undefined },
|
||||
outputTokens: { total: 1, text: 1, reasoning: undefined },
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
}
|
||||
return {
|
||||
stream: simulateReadableStream({
|
||||
chunks: [
|
||||
{ type: 'stream-start' as const, warnings: [] },
|
||||
{ type: 'text-start' as const, id: '1' },
|
||||
{ type: 'text-delta' as const, id: '1', delta: 'Sorry.' },
|
||||
{ type: 'text-end' as const, id: '1' },
|
||||
{
|
||||
type: 'finish' as const,
|
||||
finishReason: { unified: 'stop' as const, raw: 'stop' },
|
||||
usage: {
|
||||
inputTokens: { total: 1, noCache: undefined, cacheRead: undefined, cacheWrite: undefined },
|
||||
outputTokens: { total: 1, text: 1, reasoning: undefined },
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Mock model whose stream emits a provider error carrying an internal secret. */
|
||||
function providerErrorModel(secret: string): MockLanguageModelV3 {
|
||||
return new MockLanguageModelV3({
|
||||
doStream: async () => ({
|
||||
stream: simulateReadableStream({
|
||||
chunks: [
|
||||
{ type: 'stream-start' as const, warnings: [] },
|
||||
{
|
||||
type: 'error' as const,
|
||||
error: {
|
||||
statusCode: 503,
|
||||
message: 'Service Unavailable',
|
||||
responseBody: `upstream ${secret} model=internal-gpt`,
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function makeService(toolsService: PublicShareChatToolsService): {
|
||||
svc: PublicShareChatService;
|
||||
logSpy: jest.SpyInstance;
|
||||
} {
|
||||
const svc = Object.create(PublicShareChatService.prototype);
|
||||
const logger = new Logger('test');
|
||||
const logSpy = jest.spyOn(logger, 'error').mockImplementation(() => undefined);
|
||||
jest.spyOn(logger, 'warn').mockImplementation(() => undefined);
|
||||
svc.tools = toolsService;
|
||||
svc.logger = logger;
|
||||
svc.tokenBudget = { record: jest.fn().mockResolvedValue(undefined) };
|
||||
return { svc, logSpy };
|
||||
}
|
||||
|
||||
async function runStream(
|
||||
svc: PublicShareChatService,
|
||||
model: MockLanguageModelV3,
|
||||
): Promise<FakeSocket> {
|
||||
const socket = new FakeSocket();
|
||||
await svc.stream({
|
||||
workspaceId: 'ws1',
|
||||
shareId: 'share1',
|
||||
share: { id: 'share1', pageId: 'p1', sharedPage: { id: 'p1', title: 'Docs' } },
|
||||
openedPage: null,
|
||||
messages: [
|
||||
{ id: 'm1', role: 'user', parts: [{ type: 'text', text: 'read the page' }] } as never,
|
||||
],
|
||||
res: { raw: socket } as never,
|
||||
signal: new AbortController().signal,
|
||||
model: model as never,
|
||||
role: null,
|
||||
});
|
||||
// Let the piped stream drain fully.
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
return socket;
|
||||
}
|
||||
|
||||
describe('public share chat error leak (#394)', () => {
|
||||
afterEach(() => jest.restoreAllMocks());
|
||||
|
||||
it('does NOT leak a tool\'s raw internal error to the SSE bytes (generic classified string instead)', async () => {
|
||||
const SECRET = 'INTERNAL_baseUrl_http://provider.internal:8080/v1';
|
||||
const shareService = {
|
||||
// The canonical boundary throws a RAW internal error (with a secret).
|
||||
resolveReadableSharePage: jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error(`db failed at ${SECRET} stack@line42`)),
|
||||
};
|
||||
const tools = new PublicShareChatToolsService(
|
||||
shareService as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const { svc } = makeService(tools);
|
||||
|
||||
const socket = await runStream(svc, toolCallingModel());
|
||||
|
||||
// The tool-output-error frame is present on the wire...
|
||||
expect(socket.body).toContain('tool-output-error');
|
||||
// ...but it carries ONLY the generic classified string — never the secret,
|
||||
// the raw driver message, or a stack fragment.
|
||||
expect(socket.body).toContain('The tool could not complete the request.');
|
||||
expect(socket.body).not.toContain(SECRET);
|
||||
expect(socket.body).not.toContain('stack@line42');
|
||||
expect(socket.body).not.toContain('db failed');
|
||||
});
|
||||
|
||||
it('passes a SAFE ShareToolError message (page not available) through to the bytes', async () => {
|
||||
const shareService = {
|
||||
// Not found in this share -> the tool throws the classified SAFE message.
|
||||
resolveReadableSharePage: jest.fn().mockResolvedValue(null),
|
||||
};
|
||||
const tools = new PublicShareChatToolsService(
|
||||
shareService as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const { svc } = makeService(tools);
|
||||
|
||||
const socket = await runStream(svc, toolCallingModel());
|
||||
expect(socket.body).toContain('tool-output-error');
|
||||
expect(socket.body).toContain('not available in this share');
|
||||
});
|
||||
|
||||
it('does NOT leak a provider error (statusCode + response body) to the SSE bytes', async () => {
|
||||
const SECRET = 'http://provider.internal:8080';
|
||||
const tools = new PublicShareChatToolsService(
|
||||
{} as never,
|
||||
{} as never,
|
||||
{} as never,
|
||||
);
|
||||
const { svc, logSpy } = makeService(tools);
|
||||
|
||||
const socket = await runStream(svc, providerErrorModel(SECRET));
|
||||
|
||||
// The anon sees a fixed classified string, not the provider body/baseUrl/model.
|
||||
expect(socket.body).toContain('temporarily unavailable');
|
||||
expect(socket.body).not.toContain(SECRET);
|
||||
expect(socket.body).not.toContain('internal-gpt');
|
||||
// The FULL provider detail is logged server-side only.
|
||||
const logged = logSpy.mock.calls.map((c) => String(c[0])).join('\n');
|
||||
expect(logged).toContain(SECRET);
|
||||
});
|
||||
});
|
||||
@@ -12,7 +12,10 @@ import { AiAgentRoleRepo } from '@docmost/db/repos/ai-agent-roles/ai-agent-roles
|
||||
import { AiAgentRole } from '@docmost/db/types/entity.types';
|
||||
import { AiService } from '../../integrations/ai/ai.service';
|
||||
import { AiSettingsService } from '../../integrations/ai/ai-settings.service';
|
||||
import { PublicShareChatToolsService } from './tools/public-share-chat-tools.service';
|
||||
import {
|
||||
PublicShareChatToolsService,
|
||||
ShareToolError,
|
||||
} from './tools/public-share-chat-tools.service';
|
||||
import { buildShareSystemPrompt } from './public-share-chat.prompt';
|
||||
import { roleModelOverride } from './roles/role-model-config';
|
||||
import {
|
||||
@@ -102,6 +105,30 @@ export function filterShareTranscript(messages: UIMessage[]): UIMessage[] {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Fixed, classified strings an ANONYMOUS share reader may see when the assistant
|
||||
* stream fails (#394). These reveal NOTHING about the internal provider, its
|
||||
* baseUrl, the model name, or the raw response body — unlike describeProviderError
|
||||
* (which is for the server log / the authenticated operator only). We classify by
|
||||
* HTTP status where available so the reader still gets a useful hint (retry vs.
|
||||
* give up) without any internal detail.
|
||||
*/
|
||||
export function classifyAnonStreamError(error: unknown): string {
|
||||
const status =
|
||||
typeof error === 'object' && error !== null
|
||||
? (error as { statusCode?: number }).statusCode
|
||||
: undefined;
|
||||
if (status === 429) {
|
||||
return 'The assistant is receiving too many requests right now. Please try again shortly.';
|
||||
}
|
||||
if (typeof status === 'number' && status >= 500) {
|
||||
return 'The assistant is temporarily unavailable. Please try again.';
|
||||
}
|
||||
// Any other failure (including a bare connection error with no status): a
|
||||
// single neutral line. No provider identity, no config, no response body.
|
||||
return 'The assistant could not complete your request. Please try again.';
|
||||
}
|
||||
|
||||
/**
|
||||
* Anonymous, read-only AI assistant for a single PUBLIC share tree.
|
||||
*
|
||||
@@ -318,11 +345,28 @@ export class PublicShareChatService {
|
||||
result.pipeUIMessageStreamToResponse(res.raw, {
|
||||
headers: { 'X-Accel-Buffering': 'no' },
|
||||
onError: (error: unknown) => {
|
||||
// Reuse the shared formatter so provider error formatting stays
|
||||
// unified between the log line and the streamed error message — a
|
||||
// share reader sees 402/429/503 causes consistently with the
|
||||
// authenticated path.
|
||||
return describeProviderError(error, 'AI stream error');
|
||||
// SECURITY (#394): the string this returns is written verbatim into the
|
||||
// SSE error frame delivered to an ANONYMOUS reader (for a tool failure
|
||||
// it becomes the atomic `tool-output-error` frame's errorText; for a
|
||||
// stream/provider failure, the terminal error frame).
|
||||
//
|
||||
// A ShareToolError is already a classified, safe tool message (see
|
||||
// PublicShareChatToolsService.wrapToolErrors) — pass it through so the
|
||||
// reader still gets the useful "page not available in this share" hint.
|
||||
if (error instanceof ShareToolError) {
|
||||
return error.message;
|
||||
}
|
||||
// Anything else is a provider/stream error. describeProviderError
|
||||
// bundles the provider statusCode AND response body, which can carry the
|
||||
// internal baseUrl or model name — NEVER expose that to the public. Log
|
||||
// the full detail server-side only and return a fixed classified string.
|
||||
this.logger.error(
|
||||
`Public share chat pipe error: ${describeProviderError(
|
||||
error,
|
||||
'AI stream error',
|
||||
)}`,
|
||||
);
|
||||
return classifyAnonStreamError(error);
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
@@ -808,7 +808,7 @@ describe('PublicShareChatToolsService share scoping', () => {
|
||||
};
|
||||
|
||||
await expect(getSharePage.execute({ pageId: 'p-outside' })).rejects.toThrow(
|
||||
/not part of this published share/i,
|
||||
/not available in this share/i,
|
||||
);
|
||||
// The tool delegated the resolve to the canonical boundary with the
|
||||
// forShare-scoped shareId, and returned NO content for a non-resolving page.
|
||||
@@ -841,7 +841,7 @@ describe('PublicShareChatToolsService share scoping', () => {
|
||||
|
||||
await expect(
|
||||
getSharePage.execute({ pageId: 'p-restricted' }),
|
||||
).rejects.toThrow(/not part of this published share/i);
|
||||
).rejects.toThrow(/not available in this share/i);
|
||||
// No content was ever sanitized/returned for the blocked page.
|
||||
expect(shareService.updatePublicAttachments).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -1003,7 +1003,7 @@ describe('public-share assistant boundary locks (red-team regression guards)', (
|
||||
};
|
||||
await expect(
|
||||
getSharePage.execute({ pageId: 'p-elsewhere' }),
|
||||
).rejects.toThrow(/not part of this published share/i);
|
||||
).rejects.toThrow(/not available in this share/i);
|
||||
// The forged share id is the scope the boundary re-derivation rejects against.
|
||||
expect(shareService.resolveReadableSharePage).toHaveBeenCalledWith(
|
||||
'FORGED-SHARE',
|
||||
|
||||
@@ -123,6 +123,23 @@ function __assertClientCallContract(client: DocmostClientLike): void {
|
||||
void client.drawioGet(s, s, 'xml');
|
||||
void client.drawioCreate(s, { position: 'append', anchorNodeId: s }, s, s, 'elk');
|
||||
void client.drawioUpdate(s, s, s, s, 'elk');
|
||||
// --- draw.io high-level semantic tools (#425 stage 3) ---
|
||||
void client.drawioEditCells(s, s, [{ op: 'delete', cellId: s }], s);
|
||||
void client.drawioFromGraph(
|
||||
s,
|
||||
{ position: 'append', anchorNodeId: s },
|
||||
{ nodes: [{ id: s, label: s }] },
|
||||
'LR',
|
||||
s,
|
||||
'full',
|
||||
s,
|
||||
);
|
||||
void client.drawioFromMermaid(
|
||||
s,
|
||||
{ position: 'append', anchorNodeId: s },
|
||||
s,
|
||||
s,
|
||||
);
|
||||
// --- write (comment) ---
|
||||
void client.createComment(s, s, 'inline', s, s, s);
|
||||
void client.resolveComment(s, true);
|
||||
|
||||
@@ -1,7 +1,15 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
||||
import {
|
||||
mkdtempSync,
|
||||
mkdirSync,
|
||||
writeFileSync,
|
||||
rmSync,
|
||||
readdirSync,
|
||||
statSync,
|
||||
readFileSync,
|
||||
} from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { dirname, join, relative, sep } from 'node:path';
|
||||
|
||||
import { computeSrcRegistryStamp } from './docmost-client.loader';
|
||||
|
||||
@@ -30,10 +38,14 @@ function assertStaleGuard(
|
||||
}
|
||||
}
|
||||
|
||||
// Build a throwaway `<pkg>/build/index.js` + optional `<pkg>/src/tool-specs.ts`
|
||||
// layout so `computeSrcRegistryStamp(<pkg>/build/index.js)` resolves src the same
|
||||
// way the loader does (dirname(dirname(entry))/src/tool-specs.ts).
|
||||
function makeFakePackage(toolSpecsSource: string | null): {
|
||||
// Build a throwaway `<pkg>/build/index.js` + optional `<pkg>/src/` tree so
|
||||
// `computeSrcRegistryStamp(<pkg>/build/index.js)` resolves src the same way the
|
||||
// loader does (dirname(dirname(entry))/src). Since #486 the stamp hashes the WHOLE
|
||||
// src tree, so a fixture is a { relPath: content } map. A bare string is sugar for
|
||||
// a single `tool-specs.ts`; `null` means "no src tree" (the prod no-op path).
|
||||
function makeFakePackage(
|
||||
src: string | Record<string, string> | null,
|
||||
): {
|
||||
entry: string;
|
||||
cleanup: () => void;
|
||||
} {
|
||||
@@ -42,10 +54,15 @@ function makeFakePackage(toolSpecsSource: string | null): {
|
||||
mkdirSync(buildDir, { recursive: true });
|
||||
const entry = join(buildDir, 'index.js');
|
||||
writeFileSync(entry, '// fake @docmost/mcp build entry\n', 'utf8');
|
||||
if (toolSpecsSource !== null) {
|
||||
if (src !== null) {
|
||||
const files =
|
||||
typeof src === 'string' ? { 'tool-specs.ts': src } : src;
|
||||
const srcDir = join(root, 'src');
|
||||
mkdirSync(srcDir, { recursive: true });
|
||||
writeFileSync(join(srcDir, 'tool-specs.ts'), toolSpecsSource, 'utf8');
|
||||
for (const [rel, content] of Object.entries(files)) {
|
||||
const full = join(srcDir, rel);
|
||||
mkdirSync(dirname(full), { recursive: true });
|
||||
writeFileSync(full, content, 'utf8');
|
||||
}
|
||||
}
|
||||
return { entry, cleanup: () => rmSync(root, { recursive: true, force: true }) };
|
||||
}
|
||||
@@ -93,34 +110,109 @@ describe('computeSrcRegistryStamp (#447 stale-build guard)', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// CROSS-IMPL EQUALITY (covers reviewer suggestion 2). The SAME fixed input and
|
||||
// #486 CORE (negative): an edit to a NON-tool-specs src file (client.ts) with a
|
||||
// rebuild NOT run must move the src stamp away from the built REGISTRY_STAMP, so
|
||||
// the loader's stale-check refuses. Under the old tool-specs.ts-only hash this
|
||||
// edit was invisible and a stale build/ served the old client.ts silently.
|
||||
it('a client.ts edit (no rebuild) moves the src stamp -> loader refuses (#486)', () => {
|
||||
// "Built" state: the package as it was compiled.
|
||||
const built = makeFakePackage({
|
||||
'tool-specs.ts': 'export const SPECS = 1;\n',
|
||||
'client.ts': "export const impl = 'v1';\n",
|
||||
});
|
||||
// "Dev edited src, forgot to rebuild": client.ts changed, tool-specs.ts not.
|
||||
const edited = makeFakePackage({
|
||||
'tool-specs.ts': 'export const SPECS = 1;\n',
|
||||
'client.ts': "export const impl = 'v2';\n",
|
||||
});
|
||||
try {
|
||||
const builtStamp = computeSrcRegistryStamp(built.entry);
|
||||
const editedStamp = computeSrcRegistryStamp(edited.entry);
|
||||
expect(builtStamp).not.toBeNull();
|
||||
expect(editedStamp).not.toBe(builtStamp);
|
||||
// build/ still carries builtStamp; src now hashes to editedStamp -> refuse.
|
||||
expect(() => assertStaleGuard(editedStamp, builtStamp as string)).toThrow(
|
||||
STALE_BUILD_MESSAGE,
|
||||
);
|
||||
} finally {
|
||||
built.cleanup();
|
||||
edited.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
// *.generated.ts is excluded (the codegen's own output — a fixed-point cycle
|
||||
// otherwise): its presence/content must not move the stamp.
|
||||
it('excludes *.generated.ts from the stamp', () => {
|
||||
const without = makeFakePackage({ 'tool-specs.ts': 'x\n' });
|
||||
const withGen = makeFakePackage({
|
||||
'tool-specs.ts': 'x\n',
|
||||
'registry-stamp.generated.ts': 'export const REGISTRY_STAMP = "abc";\n',
|
||||
});
|
||||
try {
|
||||
expect(computeSrcRegistryStamp(withGen.entry)).toBe(
|
||||
computeSrcRegistryStamp(without.entry),
|
||||
);
|
||||
} finally {
|
||||
without.cleanup();
|
||||
withGen.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
// CROSS-IMPL EQUALITY (covers reviewer suggestion 2). The SAME fixed tree and
|
||||
// EXPECTED hash are asserted in the mcp-side node test
|
||||
// (packages/mcp/test/unit/registry-stamp.test.mjs) against the codegen's
|
||||
// `computeRegistryStamp`. Asserting the SAME pair here against the loader's
|
||||
// `computeSrcRegistryStamp` proves both implementations normalize+hash
|
||||
// `computeSrcRegistryStamp` proves both implementations enumerate+normalize+hash
|
||||
// identically; a divergence in EITHER side reddens one of the two tests.
|
||||
it('matches the documented cross-impl hash for a fixed input', () => {
|
||||
const FIXED_INPUT = 'line1\r\nline2\n';
|
||||
const EXPECTED =
|
||||
'683376e290829b482c2655745caffa7a1dccfa10afaa62dac2b42dd6c68d0f83';
|
||||
const { entry, cleanup } = makeFakePackage(FIXED_INPUT);
|
||||
const CROSS_IMPL_TREE = {
|
||||
'tool-specs.ts': 'line1\r\nline2\n',
|
||||
'client/read.ts': 'export const R = 1;\n',
|
||||
'registry-stamp.generated.ts': 'export const REGISTRY_STAMP="ignored";\n',
|
||||
};
|
||||
const CROSS_IMPL_EXPECTED =
|
||||
'131c1b9e4e2f5a7d6cef91ca8df619822b442f52bc45ebd09474a4c1d6728616';
|
||||
|
||||
it('matches the documented cross-impl hash for a fixed tree', () => {
|
||||
const { entry, cleanup } = makeFakePackage(CROSS_IMPL_TREE);
|
||||
try {
|
||||
expect(computeSrcRegistryStamp(entry)).toBe(EXPECTED);
|
||||
expect(computeSrcRegistryStamp(entry)).toBe(CROSS_IMPL_EXPECTED);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it('the documented EXPECTED is the normalize+sha256 of the fixed input', () => {
|
||||
// Proves EXPECTED is not a magic constant but the documented computation.
|
||||
const FIXED_INPUT = 'line1\r\nline2\n';
|
||||
const normalized = FIXED_INPUT.replace(/\r\n/g, '\n').replace(/\n$/, '');
|
||||
const expected = createHash('sha256')
|
||||
.update(normalized, 'utf8')
|
||||
.digest('hex');
|
||||
const { entry, cleanup } = makeFakePackage(FIXED_INPUT);
|
||||
it('the documented EXPECTED is the enumerate+normalize+sha256 of the tree', () => {
|
||||
// Proves EXPECTED is not a magic constant but the documented computation — a
|
||||
// local re-implementation of the loader's tree walk.
|
||||
const { entry, cleanup } = makeFakePackage(CROSS_IMPL_TREE);
|
||||
try {
|
||||
expect(computeSrcRegistryStamp(entry)).toBe(expected);
|
||||
const srcDir = join(dirname(dirname(entry)), 'src');
|
||||
const collect = (dir: string): string[] => {
|
||||
const out: string[] = [];
|
||||
for (const e of readdirSync(dir)) {
|
||||
const f = join(dir, e);
|
||||
if (statSync(f).isDirectory()) out.push(...collect(f));
|
||||
else if (e.endsWith('.ts') && !e.endsWith('.generated.ts'))
|
||||
out.push(f);
|
||||
}
|
||||
return out;
|
||||
};
|
||||
const files = collect(srcDir)
|
||||
.map((abs) => ({ rel: relative(srcDir, abs).split(sep).join('/'), abs }))
|
||||
.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
|
||||
const h = createHash('sha256');
|
||||
for (const { rel, abs } of files) {
|
||||
const n = readFileSync(abs, 'utf8')
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\n$/, '');
|
||||
h.update(rel, 'utf8');
|
||||
h.update('\0', 'utf8');
|
||||
h.update(n, 'utf8');
|
||||
h.update('\0', 'utf8');
|
||||
}
|
||||
const localHash = h.digest('hex');
|
||||
expect(computeSrcRegistryStamp(entry)).toBe(localHash);
|
||||
expect(localHash).toBe(CROSS_IMPL_EXPECTED);
|
||||
} finally {
|
||||
cleanup();
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs';
|
||||
import { dirname, join, relative, sep } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import type { DocmostClient, SharedToolSpec } from '@docmost/mcp';
|
||||
|
||||
@@ -71,6 +71,10 @@ type DocmostClientMethod =
|
||||
| 'drawioGet'
|
||||
| 'drawioCreate'
|
||||
| 'drawioUpdate'
|
||||
// --- draw.io high-level semantic tools (#425 stage 3) ---
|
||||
| 'drawioEditCells'
|
||||
| 'drawioFromGraph'
|
||||
| 'drawioFromMermaid'
|
||||
// --- write (comment) ---
|
||||
| 'createComment'
|
||||
| 'resolveComment';
|
||||
@@ -187,33 +191,52 @@ interface DocmostMcpModule {
|
||||
* present. Returns the stamp string, or `null` when the source is absent (a prod
|
||||
* image ships only build/, no src/). MUST stay byte-for-byte identical to
|
||||
* packages/mcp/scripts/gen-registry-stamp.mjs's `computeRegistryStamp` so the
|
||||
* build-time and src-time hashes agree: same input file (src/tool-specs.ts), same
|
||||
* normalization (CRLF -> LF, strip a single trailing newline), same sha256.
|
||||
* build-time and src-time hashes agree: same file set (every src/**\/*.ts except
|
||||
* *.generated.ts), same POSIX-relative sort, same per-file normalization (CRLF ->
|
||||
* LF, strip a single trailing newline) with the same path+content framing, same
|
||||
* sha256. Hashing the WHOLE src tree (not just tool-specs.ts) is #486: an edit to
|
||||
* client.ts / a client/* module / comment-signal / drawio-* without a rebuild
|
||||
* must also be caught, otherwise build/ silently serves the old code.
|
||||
*
|
||||
* DEV vs PROD detection is by FILE EXISTENCE, not NODE_ENV: we resolve the
|
||||
* package's own directory from `require.resolve('@docmost/mcp')` (which points at
|
||||
* build/index.js) and look for ../src/tool-specs.ts next to it. In a dev/test
|
||||
* worktree that file exists; in a prod image (build/ only, src/ stripped) it does
|
||||
* not, so this returns null and the caller skips the check. Any error (ENOENT, a
|
||||
* bad resolve) is swallowed to null — the stale-check must NEVER break startup.
|
||||
* build/index.js) and look for ../src next to it. In a dev/test worktree that
|
||||
* directory exists; in a prod image (build/ only, src/ stripped) it does not, so
|
||||
* this returns null and the caller skips the check. Any error (ENOENT, a bad
|
||||
* resolve) is swallowed to null — the stale-check must NEVER break startup.
|
||||
*
|
||||
* Exported for unit testing (docmost-client.loader.spec.ts): the export keyword
|
||||
* is behaviourally a no-op — the module-internal caller `loadDocmostMcp` is
|
||||
* unaffected. The test drives the null (no-src) path and asserts this
|
||||
* normalize+sha256 stays identical to the codegen's `computeRegistryStamp`.
|
||||
* enumerate+normalize+sha256 stays identical to the codegen's
|
||||
* `computeRegistryStamp`.
|
||||
*/
|
||||
export function computeSrcRegistryStamp(packageEntry: string): string | null {
|
||||
try {
|
||||
// packageEntry is <pkg>/build/index.js; the source lives at <pkg>/src/.
|
||||
const toolSpecsPath = join(
|
||||
dirname(dirname(packageEntry)),
|
||||
'src',
|
||||
'tool-specs.ts',
|
||||
);
|
||||
if (!existsSync(toolSpecsPath)) return null; // prod: no src tree -> skip.
|
||||
const source = readFileSync(toolSpecsPath, 'utf8');
|
||||
const normalized = source.replace(/\r\n/g, '\n').replace(/\n$/, '');
|
||||
return createHash('sha256').update(normalized, 'utf8').digest('hex');
|
||||
const srcDir = join(dirname(dirname(packageEntry)), 'src');
|
||||
if (!existsSync(srcDir)) return null; // prod: no src tree -> skip.
|
||||
// Enumerate every src/**\/*.ts except the codegen's own *.generated.ts
|
||||
// output (including it would be a fixed-point cycle). Sort by POSIX-relative
|
||||
// path so ordering is platform-independent, then fold each file's relative
|
||||
// path + normalized content into one hash — identical to the codegen.
|
||||
const files = collectStampFiles(srcDir)
|
||||
.map((abs) => ({
|
||||
rel: relative(srcDir, abs).split(sep).join('/'),
|
||||
abs,
|
||||
}))
|
||||
.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
|
||||
const hash = createHash('sha256');
|
||||
for (const { rel, abs } of files) {
|
||||
const normalized = readFileSync(abs, 'utf8')
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\n$/, '');
|
||||
hash.update(rel, 'utf8');
|
||||
hash.update('\0', 'utf8');
|
||||
hash.update(normalized, 'utf8');
|
||||
hash.update('\0', 'utf8');
|
||||
}
|
||||
return hash.digest('hex');
|
||||
} catch {
|
||||
// Never let a resolution/read hiccup break server startup — treat as "no
|
||||
// src available" and skip the check (identical to the prod no-op path).
|
||||
@@ -221,6 +244,24 @@ export function computeSrcRegistryStamp(packageEntry: string): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively enumerate every `*.ts` under `dir`, EXCLUDING `*.generated.ts`.
|
||||
* Mirror of the codegen's `collectStampFiles` (packages/mcp/scripts/
|
||||
* gen-registry-stamp.mjs) — keep the two walk/filter rules identical.
|
||||
*/
|
||||
function collectStampFiles(dir: string): string[] {
|
||||
const out: string[] = [];
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
if (statSync(full).isDirectory()) {
|
||||
out.push(...collectStampFiles(full));
|
||||
} else if (entry.endsWith('.ts') && !entry.endsWith('.generated.ts')) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// TS with module:commonjs downlevels a literal `import()` to `require()`, which
|
||||
// cannot load the ESM-only `@docmost/mcp` package. Indirect through Function so
|
||||
// the real dynamic `import()` survives compilation and can load ESM from
|
||||
|
||||
@@ -224,7 +224,7 @@ describe('PublicShareChatToolsService.forShare', () => {
|
||||
(tools.getSharePage as unknown as ToolExec).execute({
|
||||
pageId: 'page-1',
|
||||
}),
|
||||
).rejects.toThrow('That page is not part of this published share.');
|
||||
).rejects.toThrow('The requested page is not available in this share.');
|
||||
|
||||
// No content is ever fetched/returned for a non-resolving page.
|
||||
expect(shareService.updatePublicAttachments).not.toHaveBeenCalled();
|
||||
|
||||
@@ -7,6 +7,22 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { jsonToMarkdown } from '../../../collaboration/collaboration.util';
|
||||
import { modelFriendlyInput } from './model-friendly-input';
|
||||
|
||||
/**
|
||||
* A tool error whose message is DELIBERATELY safe to expose to an anonymous
|
||||
* share reader (and to the model, for self-correction). Every OTHER thrown error
|
||||
* is treated as internal and replaced with a generic string by `wrapToolErrors`,
|
||||
* so a raw exception message — an internal page title, a DB/stack fragment, a
|
||||
* driver detail — never rides the public UI stream (#394).
|
||||
*/
|
||||
export class ShareToolError extends Error {}
|
||||
|
||||
// The only two classified strings an anonymous reader may ever see from a tool
|
||||
// failure. The specific one keeps the model's self-correction useful ("try a
|
||||
// different page"); the generic one reveals nothing about the internal fault.
|
||||
const SHARE_TOOL_ERROR_NOT_AVAILABLE =
|
||||
'The requested page is not available in this share.';
|
||||
const SHARE_TOOL_ERROR_GENERIC = 'The tool could not complete the request.';
|
||||
|
||||
/**
|
||||
* Isolated, READ-ONLY toolset for the ANONYMOUS public-share assistant.
|
||||
*
|
||||
@@ -44,7 +60,7 @@ export class PublicShareChatToolsService {
|
||||
* are NO write tools, NO comments/history, NO cross-space or external tools.
|
||||
*/
|
||||
forShare(shareId: string, workspaceId: string): Record<string, Tool> {
|
||||
return {
|
||||
return this.wrapToolErrors({
|
||||
searchSharePages: tool({
|
||||
description:
|
||||
'Search the pages of THIS published documentation share for a ' +
|
||||
@@ -96,7 +112,7 @@ export class PublicShareChatToolsService {
|
||||
execute: async ({ pageId }) => {
|
||||
const id = (pageId ?? '').trim();
|
||||
if (!id) {
|
||||
throw new Error('A pageId is required.');
|
||||
throw new ShareToolError('A pageId is required.');
|
||||
}
|
||||
// Resolve via the SINGLE canonical share-access boundary: confirms the
|
||||
// page resolves to THIS share (recursive CTE up the tree, honouring
|
||||
@@ -112,7 +128,7 @@ export class PublicShareChatToolsService {
|
||||
workspaceId,
|
||||
);
|
||||
if (!resolved) {
|
||||
throw new Error('That page is not part of this published share.');
|
||||
throw new ShareToolError(SHARE_TOOL_ERROR_NOT_AVAILABLE);
|
||||
}
|
||||
const { page } = resolved;
|
||||
|
||||
@@ -193,6 +209,57 @@ export class PublicShareChatToolsService {
|
||||
}
|
||||
},
|
||||
}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap every tool's `execute` so a THROWN error is sanitized in ONE place —
|
||||
* closing the byte leak, the render, and the model context at once (#394).
|
||||
*
|
||||
* The AI SDK surfaces a tool-execution throw as an atomic `tool-output-error`
|
||||
* frame on the v6 UI stream whose `errorText` is the thrown message; on the
|
||||
* public share that frame goes straight to an anonymous reader. Unwrapped, a
|
||||
* raw exception (an internal page title, a DB/stack fragment, a driver detail)
|
||||
* would ride that frame verbatim. Here we catch it, LOG the full detail
|
||||
* server-side only, and re-throw a CLASSIFIED, safe error: the tool's own
|
||||
* intentional ShareToolError messages pass through (they keep the model's
|
||||
* self-correction useful), everything else collapses to a generic string.
|
||||
*/
|
||||
private wrapToolErrors(
|
||||
tools: Record<string, Tool>,
|
||||
): Record<string, Tool> {
|
||||
const wrapped: Record<string, Tool> = {};
|
||||
for (const [name, t] of Object.entries(tools)) {
|
||||
const original = t.execute;
|
||||
if (typeof original !== 'function') {
|
||||
wrapped[name] = t;
|
||||
continue;
|
||||
}
|
||||
wrapped[name] = {
|
||||
...t,
|
||||
execute: async (args: unknown, options: unknown) => {
|
||||
try {
|
||||
return await (
|
||||
original as (a: unknown, o: unknown) => Promise<unknown>
|
||||
)(args, options);
|
||||
} catch (err) {
|
||||
const safe =
|
||||
err instanceof ShareToolError
|
||||
? err.message
|
||||
: SHARE_TOOL_ERROR_GENERIC;
|
||||
// Full detail to the server log ONLY — never to the anon.
|
||||
this.logger.warn(
|
||||
`Public share tool "${name}" failed: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
// This safe string is ALL that rides the tool-output-error frame,
|
||||
// becomes model context, and could be rendered — one choke point.
|
||||
throw new ShareToolError(safe);
|
||||
}
|
||||
},
|
||||
} as Tool;
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,10 +27,12 @@ import type { DocmostClientLike } from './docmost-client.loader';
|
||||
*/
|
||||
|
||||
describe('tool tier metadata (#332)', () => {
|
||||
it('core set is the documented 13 + searchInPage + insertFootnote (15)', () => {
|
||||
expect(CORE_TOOL_KEYS).toHaveLength(15);
|
||||
it('core set is the documented 13 + searchInPage + insertFootnote + getTree + getPageContext (17, #443)', () => {
|
||||
expect(CORE_TOOL_KEYS).toHaveLength(17);
|
||||
expect(CORE_TOOL_SET.has('searchInPage')).toBe(true); // #330, promoted to core
|
||||
expect(CORE_TOOL_SET.has('insertFootnote')).toBe(true); // #410, promoted to core
|
||||
expect(CORE_TOOL_SET.has('getTree')).toBe(true); // #443, promoted to core
|
||||
expect(CORE_TOOL_SET.has('getPageContext')).toBe(true); // #443, promoted to core
|
||||
// loadTools is a meta-tool, not a normal core key.
|
||||
expect(CORE_TOOL_SET.has(LOAD_TOOLS_NAME)).toBe(false);
|
||||
});
|
||||
|
||||
@@ -39,12 +39,14 @@ export interface ToolCatalogEntry {
|
||||
|
||||
/**
|
||||
* CORE (always-active) in-app tool keys — 13 frequent/tiny tools + `searchInPage`
|
||||
* (#330) + `insertFootnote` (#410). `searchInPage` is core because it is frequent
|
||||
* for the editorial roles this feature targets; `insertFootnote` is core so the
|
||||
* footnote tool is NOT hidden while its natural sibling `editPageText` is always
|
||||
* active (that asymmetry is exactly what pushed the agent to write literal
|
||||
* `^[...]`). `loadTools` is active too but is not a normal tool key (it is added
|
||||
* to activeTools separately).
|
||||
* (#330) + `insertFootnote` (#410) + `getTree`/`getPageContext` (#443).
|
||||
* `searchInPage` is core because it is frequent for the editorial roles this
|
||||
* feature targets; `insertFootnote` is core so the footnote tool is NOT hidden
|
||||
* while its natural sibling `editPageText` is always active (that asymmetry is
|
||||
* exactly what pushed the agent to write literal `^[...]`). `getTree` and
|
||||
* `getPageContext` are the single-call navigation/lookup tools — core so the
|
||||
* agent never has to loadTools just to orient itself. `loadTools` is active too
|
||||
* but is not a normal tool key (it is added to activeTools separately).
|
||||
*/
|
||||
export const CORE_TOOL_KEYS = [
|
||||
'searchPages',
|
||||
@@ -66,6 +68,11 @@ export const CORE_TOOL_KEYS = [
|
||||
// #410 insertFootnote — core so pinpoint citations to already-written text
|
||||
// don't degrade into literal `^[...]`; kept symmetric with editPageText.
|
||||
'insertFootnote',
|
||||
// #443 getTree + getPageContext — cheap single-call navigation/lookup tools
|
||||
// (the core listPages even points to getTree); core so the agent never has
|
||||
// to loadTools just to orient itself.
|
||||
'getTree',
|
||||
'getPageContext',
|
||||
] as const;
|
||||
|
||||
/** O(1) membership test for the core tier. */
|
||||
|
||||
@@ -120,3 +120,102 @@ describe('JwtStrategy — provenance derivation', () => {
|
||||
expect(req.raw.actor).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Provenance derivation on the API-KEY path (jwt.strategy.validateApiKey, #486).
|
||||
*
|
||||
* The access-token path stamped provenance; the API-key path returned early
|
||||
* WITHOUT it, so an is_agent API key's REST writes recorded no 'agent' marker.
|
||||
* The API-key payload carries no signed claim, so provenance is resolved from the
|
||||
* SERVER-SIDE user returned by ApiKeyService.validateApiKey: isAgent -> 'agent',
|
||||
* otherwise 'user'; aiChatId is always null (an API key has no ai_chats row).
|
||||
*
|
||||
* The enterprise ApiKeyService is not bundled in the OSS build, so the strategy
|
||||
* loads it through an overridable `resolveApiKeyService` seam that we stub here.
|
||||
*/
|
||||
describe('JwtStrategy — API-key provenance derivation (#486)', () => {
|
||||
function makeApiKeyStrategy(validateApiKeyImpl: (p: any) => Promise<any>) {
|
||||
const userRepo: any = { findById: jest.fn() };
|
||||
const workspaceRepo: any = { findById: jest.fn() };
|
||||
const userSessionRepo: any = { findActiveById: jest.fn() };
|
||||
const sessionActivityService: any = { trackActivity: jest.fn() };
|
||||
const environmentService: any = { getAppSecret: () => 'test-secret' };
|
||||
const moduleRef: any = {};
|
||||
|
||||
const strategy = new JwtStrategy(
|
||||
userRepo,
|
||||
workspaceRepo,
|
||||
userSessionRepo,
|
||||
sessionActivityService,
|
||||
environmentService,
|
||||
moduleRef,
|
||||
);
|
||||
// Stub the EE ApiKeyService seam (the real module is not in the OSS build).
|
||||
const validateApiKey = jest.fn(validateApiKeyImpl);
|
||||
jest
|
||||
.spyOn(strategy as any, 'resolveApiKeyService')
|
||||
.mockReturnValue({ validateApiKey });
|
||||
return { strategy, validateApiKey };
|
||||
}
|
||||
|
||||
const makeReq = () => ({ raw: {} as Record<string, any> });
|
||||
const apiKeyPayload = () => ({
|
||||
sub: 'svc-1',
|
||||
workspaceId: 'ws-1',
|
||||
apiKeyId: 'key-1',
|
||||
type: JwtType.API_KEY,
|
||||
});
|
||||
|
||||
it("stamps actor='agent' for an is_agent API key (from the validated user)", async () => {
|
||||
const validated = {
|
||||
user: { id: 'svc-1', isAgent: true },
|
||||
workspace: { id: 'ws-1' },
|
||||
};
|
||||
const { strategy, validateApiKey } = makeApiKeyStrategy(
|
||||
async () => validated,
|
||||
);
|
||||
const req = makeReq();
|
||||
|
||||
const result = await strategy.validate(req, apiKeyPayload() as any);
|
||||
|
||||
expect(validateApiKey).toHaveBeenCalledTimes(1);
|
||||
expect(req.raw.actor).toBe('agent');
|
||||
// API keys carry no internal ai_chats row -> null.
|
||||
expect(req.raw.aiChatId).toBeNull();
|
||||
// The validated auth object is returned unchanged (req.user shape preserved).
|
||||
expect(result).toBe(validated);
|
||||
});
|
||||
|
||||
it("stamps actor='user' for an ordinary (non-agent) API key", async () => {
|
||||
const { strategy } = makeApiKeyStrategy(async () => ({
|
||||
user: { id: 'u-1', isAgent: false },
|
||||
workspace: { id: 'ws-1' },
|
||||
}));
|
||||
const req = makeReq();
|
||||
|
||||
await strategy.validate(req, apiKeyPayload() as any);
|
||||
|
||||
expect(req.raw.actor).toBe('user');
|
||||
expect(req.raw.aiChatId).toBeNull();
|
||||
});
|
||||
|
||||
it('throws Unauthorized (and stamps nothing) when the EE module is missing', async () => {
|
||||
const userRepo: any = { findById: jest.fn() };
|
||||
const strategy = new JwtStrategy(
|
||||
userRepo,
|
||||
{ findById: jest.fn() } as any,
|
||||
{ findActiveById: jest.fn() } as any,
|
||||
{ trackActivity: jest.fn() } as any,
|
||||
{ getAppSecret: () => 'test-secret' } as any,
|
||||
{} as any,
|
||||
);
|
||||
// EE not bundled: the seam returns null.
|
||||
jest.spyOn(strategy as any, 'resolveApiKeyService').mockReturnValue(null);
|
||||
const req = makeReq();
|
||||
|
||||
await expect(
|
||||
strategy.validate(req, apiKeyPayload() as any),
|
||||
).rejects.toThrow(UnauthorizedException);
|
||||
expect(req.raw.actor).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -102,28 +102,49 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
|
||||
}
|
||||
|
||||
private async validateApiKey(req: any, payload: JwtApiKeyPayload) {
|
||||
let ApiKeyModule: any;
|
||||
let isApiKeyModuleReady = false;
|
||||
const apiKeyService = this.resolveApiKeyService();
|
||||
if (!apiKeyService) {
|
||||
throw new UnauthorizedException('Enterprise API Key module missing');
|
||||
}
|
||||
|
||||
const result = await apiKeyService.validateApiKey(payload);
|
||||
|
||||
// Stamp the agent-edit provenance for the API-KEY path too (#486). Unlike the
|
||||
// access-token path above, it CANNOT be resolved before this point: the
|
||||
// API-key payload carries no signed actor/aiChatId claim, and the user (with
|
||||
// its isAgent flag) is unknown until the key is validated. Claim semantics for
|
||||
// API keys: an is_agent API key (an agent service account) stamps 'agent' on
|
||||
// every REST write; an ordinary API key resolves to 'user'. An API key has no
|
||||
// internal ai_chats row, so aiChatId is always null. Derived from the
|
||||
// SERVER-SIDE user (never a client field), so an 'agent' badge is unspoofable
|
||||
// — mirroring the access-token path. Passing `null` for the claim means the
|
||||
// actor is decided solely by user.isAgent.
|
||||
const provenance = resolveProvenance((result as any)?.user, null);
|
||||
req.raw.actor = provenance.actor;
|
||||
req.raw.aiChatId = provenance.aiChatId;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the enterprise ApiKeyService, or `null` when the EE module is not
|
||||
* bundled in this build (community build). Extracted as an overridable seam so
|
||||
* the API-key provenance stamping can be unit-tested without the EE package
|
||||
* present (docmost is OSS + a separate EE bundle; `require` of the EE path
|
||||
* throws here). Any load/resolve error is treated as "module missing".
|
||||
*/
|
||||
protected resolveApiKeyService(): {
|
||||
validateApiKey: (payload: JwtApiKeyPayload) => Promise<unknown>;
|
||||
} | null {
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
ApiKeyModule = require('./../../../ee/api-key/api-key.service');
|
||||
isApiKeyModuleReady = true;
|
||||
const ApiKeyModule = require('./../../../ee/api-key/api-key.service');
|
||||
return this.moduleRef.get(ApiKeyModule.ApiKeyService, { strict: false });
|
||||
} catch (err) {
|
||||
this.logger.debug(
|
||||
'API Key module requested but enterprise module not bundled in this build',
|
||||
);
|
||||
isApiKeyModuleReady = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
if (isApiKeyModuleReady) {
|
||||
const ApiKeyService = this.moduleRef.get(ApiKeyModule.ApiKeyService, {
|
||||
strict: false,
|
||||
});
|
||||
|
||||
return ApiKeyService.validateApiKey(payload);
|
||||
}
|
||||
|
||||
throw new UnauthorizedException('Enterprise API Key module missing');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import { readFileSync } from 'fs';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { streamText } from 'ai';
|
||||
import { MockLanguageModelV3, simulateReadableStream } from 'ai/test';
|
||||
|
||||
/**
|
||||
* Regression tests for the writeToServerResponse drain-hang fix in
|
||||
* patches/ai@6.0.134.patch (#486, commit 6).
|
||||
*
|
||||
* Unpatched ai@6.0.134's writeToServerResponse awaits ONLY `once("drain")` when
|
||||
* response.write() returns false (backpressure). If the client disconnects
|
||||
* mid-write the socket never drains, so that await never resolves: the read loop
|
||||
* parks FOREVER, its `finally { response.end() }` is unreachable, and the stream
|
||||
* reader + buffered chunks are pinned until process restart. In autonomous mode
|
||||
* the run keeps producing output after the disconnect, so EVERY mid-run
|
||||
* disconnect leaks a hung pipe. The patch races drain against close/error, and on
|
||||
* a terminal socket event cancels the reader and breaks so `finally` always runs.
|
||||
*
|
||||
* This drives the REAL patched writeToServerResponse through the public
|
||||
* pipeUIMessageStreamToResponse API with a response that never drains and closes
|
||||
* mid-write — exactly the leak scenario.
|
||||
*/
|
||||
|
||||
/** A ServerResponse-like emitter whose first write() stalls (returns false) and
|
||||
* then "closes" like a disconnecting client — never firing 'drain'. */
|
||||
class DisconnectingResponse extends EventEmitter {
|
||||
ended = false;
|
||||
writeCount = 0;
|
||||
statusCode = 200;
|
||||
writableEnded = false;
|
||||
destroyed = false;
|
||||
writeHead(): this {
|
||||
return this;
|
||||
}
|
||||
setHeader(): void {}
|
||||
flushHeaders(): void {}
|
||||
write(): boolean {
|
||||
this.writeCount++;
|
||||
if (this.writeCount === 1) {
|
||||
// Simulate the client vanishing mid-write: backpressure (false) and then a
|
||||
// 'close' on the next tick, and CRUCIALLY never a 'drain'. Unpatched, the
|
||||
// loop would await drain forever here.
|
||||
setImmediate(() => this.emit('close'));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
end(): void {
|
||||
this.ended = true;
|
||||
this.writableEnded = true;
|
||||
this.emit('finish');
|
||||
}
|
||||
}
|
||||
|
||||
function makeModel() {
|
||||
return new MockLanguageModelV3({
|
||||
doStream: async () => ({
|
||||
stream: simulateReadableStream({
|
||||
chunks: [
|
||||
{ type: 'stream-start' as const, warnings: [] },
|
||||
{ type: 'text-start' as const, id: '1' },
|
||||
{ type: 'text-delta' as const, id: '1', delta: 'hello ' },
|
||||
{ type: 'text-delta' as const, id: '1', delta: 'world' },
|
||||
{ type: 'text-end' as const, id: '1' },
|
||||
{
|
||||
type: 'finish' as const,
|
||||
finishReason: { unified: 'stop' as const, raw: 'stop' },
|
||||
usage: {
|
||||
inputTokens: { total: 1, noCache: undefined, cacheRead: undefined, cacheWrite: undefined },
|
||||
outputTokens: { total: 1, text: 1, reasoning: undefined },
|
||||
},
|
||||
},
|
||||
],
|
||||
}),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
describe('ai@6.0.134 pnpm patch: writeToServerResponse drain-hang (#486)', () => {
|
||||
it('ends the response (does NOT hang) when the socket closes mid-write without draining', async () => {
|
||||
const result = streamText({ model: makeModel(), prompt: 'hi' });
|
||||
const res = new DisconnectingResponse();
|
||||
// Drain the SDK stream independently, like the production detached path.
|
||||
void result.consumeStream({ onError: () => undefined });
|
||||
result.pipeUIMessageStreamToResponse(res as never);
|
||||
|
||||
// TRIPWIRE: the patched loop exits on 'close' and runs finally -> end().
|
||||
// Unpatched, it awaits 'drain' forever and this never becomes true.
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const started = Date.now();
|
||||
const poll = setInterval(() => {
|
||||
if (res.ended) {
|
||||
clearInterval(poll);
|
||||
resolve();
|
||||
} else if (Date.now() - started > 3000) {
|
||||
clearInterval(poll);
|
||||
reject(new Error('writeToServerResponse hung: response never ended'));
|
||||
}
|
||||
}, 20);
|
||||
});
|
||||
|
||||
expect(res.ended).toBe(true);
|
||||
});
|
||||
|
||||
it('does not emit an unhandledRejection when the fire-and-forget read() throws', async () => {
|
||||
// The patch swallows read()'s rejection (fire-and-forget) with a log instead
|
||||
// of letting it surface as a process-killing unhandledRejection.
|
||||
const rejections: unknown[] = [];
|
||||
const onUnhandled = (e: unknown) => rejections.push(e);
|
||||
process.on('unhandledRejection', onUnhandled);
|
||||
// Silence the patch's diagnostic console.error for the throwing read().
|
||||
const errSpy = jest.spyOn(console, 'error').mockImplementation(() => undefined);
|
||||
try {
|
||||
const result = streamText({ model: makeModel(), prompt: 'hi' });
|
||||
const res = new DisconnectingResponse();
|
||||
void result.consumeStream({ onError: () => undefined });
|
||||
result.pipeUIMessageStreamToResponse(res as never);
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
} finally {
|
||||
process.off('unhandledRejection', onUnhandled);
|
||||
errSpy.mockRestore();
|
||||
}
|
||||
expect(rejections).toEqual([]);
|
||||
});
|
||||
|
||||
it('both installed dist builds (CJS and ESM) carry the #486 patch marker', () => {
|
||||
const cjsPath = require.resolve('ai');
|
||||
const mjsPath = cjsPath.replace(/index\.js$/, 'index.mjs');
|
||||
expect(cjsPath).toMatch(/index\.js$/);
|
||||
expect(readFileSync(cjsPath, 'utf8')).toContain('PATCH(docmost #486)');
|
||||
expect(readFileSync(mjsPath, 'utf8')).toContain('PATCH(docmost #486)');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,211 @@
|
||||
import type { HealthIndicatorService } from '@nestjs/terminus';
|
||||
import type { EnvironmentService } from '../environment/environment.service';
|
||||
|
||||
/**
|
||||
* Integration guard for the /health Redis-probe handle leak (#486, commit 2).
|
||||
*
|
||||
* The bug: `pingCheck` built `new Redis(...)` per call and only disconnected on
|
||||
* the SUCCESS path, so when Redis is DOWN every probe tick added ANOTHER
|
||||
* forever-reconnecting client — an unbounded handle/client leak for the duration
|
||||
* of the outage. The fix reuses ONE long-lived probe client.
|
||||
*
|
||||
* This is an OBSERVABLE-property test, not an assertion on a mocked return value:
|
||||
* we point the indicator at a REAL, refused TCP endpoint (a dead port) so ioredis
|
||||
* genuinely fails to connect, run many probes, and assert the number of live
|
||||
* Redis CLIENTS created stays at exactly ONE. `ioredis` is delegated to its real
|
||||
* implementation (requireActual) — only the constructor is wrapped to COUNT the
|
||||
* real clients it creates, which is precisely the leaking resource.
|
||||
*/
|
||||
import type { Redis } from 'ioredis';
|
||||
|
||||
const mockLiveClients: Redis[] = [];
|
||||
|
||||
/**
|
||||
* Fully tear a REAL ioredis client down so NO timer survives jest's 1s exit
|
||||
* window (this suite must exit cleanly WITHOUT forceExit; see #382).
|
||||
*
|
||||
* `connector.disconnect()` arms a ~12s "force-destroy the stream" `setTimeout`
|
||||
* that is cleared ONLY by the stream's 'close' event — but only when the
|
||||
* connector still holds a stream. Two problem cases:
|
||||
* - a LIVE/connecting socket: disconnect arms the timer and 'close' may lag
|
||||
* past jest's window, so we destroy the socket to make 'close' fire NOW;
|
||||
* - a client BETWEEN reconnect attempts to a dead port: the held socket is
|
||||
* ALREADY destroyed (its 'close' fired long ago), so disconnect would arm a
|
||||
* timer whose clearing 'close' can never come again. We drop that dead stream
|
||||
* reference BEFORE disconnect so the doomed timer is never armed.
|
||||
* `disconnect()` itself also clears ioredis' own reconnect backoff timer.
|
||||
*/
|
||||
type DrainableStream = { destroyed?: boolean; destroy?: () => void } | null;
|
||||
type DrainableClient = {
|
||||
removeAllListeners: (event: string) => void;
|
||||
disconnect: () => void;
|
||||
stream?: DrainableStream;
|
||||
connector?: { stream?: DrainableStream };
|
||||
};
|
||||
|
||||
async function drainClient(client: Redis): Promise<void> {
|
||||
if (!client || client.status === 'end') return;
|
||||
const c = client as unknown as DrainableClient;
|
||||
c.removeAllListeners('error');
|
||||
|
||||
// Drop an already-dead held socket so disconnect() can't arm a timer whose
|
||||
// clearing 'close' will never fire again.
|
||||
if (c.connector?.stream && c.connector.stream.destroyed) {
|
||||
c.connector.stream = null;
|
||||
}
|
||||
if (c.stream && c.stream.destroyed) {
|
||||
c.stream = null;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
let done = false;
|
||||
const finish = () => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
resolve();
|
||||
};
|
||||
client.once('end', finish);
|
||||
// reconnect=false (the default): stop the retry loop and close the socket.
|
||||
client.disconnect();
|
||||
// Force any still-live socket closed NOW so the connector's stream-destroy
|
||||
// timer clears inside jest's window instead of lagging behind a real 'close'.
|
||||
if (c.stream && !c.stream.destroyed) {
|
||||
c.stream.destroy?.();
|
||||
}
|
||||
// Fallback for a client with no live stream to emit 'end' (unref'd so it
|
||||
// can never itself hold the loop open).
|
||||
const fallback = setTimeout(finish, 500);
|
||||
(fallback as { unref?: () => void }).unref?.();
|
||||
});
|
||||
}
|
||||
|
||||
async function drainAll(): Promise<void> {
|
||||
await Promise.all(mockLiveClients.map((c) => drainClient(c)));
|
||||
}
|
||||
|
||||
jest.mock('ioredis', () => {
|
||||
const actual = jest.requireActual('ioredis');
|
||||
const RealRedis = actual.Redis ?? actual.default ?? actual;
|
||||
class CountingRedis extends RealRedis {
|
||||
constructor(...args: unknown[]) {
|
||||
super(...(args as []));
|
||||
mockLiveClients.push(this as never);
|
||||
}
|
||||
}
|
||||
return { ...actual, Redis: CountingRedis, default: CountingRedis };
|
||||
});
|
||||
|
||||
// Import AFTER the mock is registered so the class picks up the counting client.
|
||||
import { RedisHealthIndicator } from './redis.health';
|
||||
|
||||
describe('RedisHealthIndicator handle leak (#486)', () => {
|
||||
const indicatorService = {
|
||||
check: (key: string) => ({
|
||||
up: () => ({ [key]: { status: 'up' } }),
|
||||
down: (message: string) => ({ [key]: { status: 'down', message } }),
|
||||
}),
|
||||
} as unknown as HealthIndicatorService;
|
||||
|
||||
// A port with (almost certainly) nothing listening -> connection refused fast.
|
||||
const environmentService = {
|
||||
getRedisUrl: () => 'redis://127.0.0.1:6399/0',
|
||||
} as unknown as EnvironmentService;
|
||||
|
||||
let indicator: RedisHealthIndicator;
|
||||
|
||||
beforeEach(() => {
|
||||
mockLiveClients.length = 0;
|
||||
indicator = new RedisHealthIndicator(indicatorService, environmentService);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Drain (destroy socket + AWAIT 'end') every client the test created FIRST,
|
||||
// so each is fully 'end' before onModuleDestroy's disconnect runs — that way
|
||||
// no ioredis reconnect / stream-destroy timer outlives jest's exit window.
|
||||
await drainAll();
|
||||
indicator.onModuleDestroy();
|
||||
});
|
||||
|
||||
it('creates exactly ONE Redis client across many probes while Redis is DOWN', async () => {
|
||||
const N = 8;
|
||||
for (let i = 0; i < N; i++) {
|
||||
const result = await indicator.pingCheck('redis');
|
||||
// Down endpoint -> every probe reports "down" (not an unhandled crash).
|
||||
expect(result.redis.status).toBe('down');
|
||||
}
|
||||
|
||||
// THE OBSERVABLE LEAK: on the buggy code this is N (a fresh, never-cleaned
|
||||
// reconnecting client per probe). The fix reuses one shared client.
|
||||
expect(mockLiveClients).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('onModuleDestroy releases the probe client (a later probe builds a fresh one)', async () => {
|
||||
await indicator.pingCheck('redis');
|
||||
expect(mockLiveClients).toHaveLength(1);
|
||||
|
||||
indicator.onModuleDestroy();
|
||||
// A second destroy is a safe no-op (probeClient was nulled).
|
||||
indicator.onModuleDestroy();
|
||||
|
||||
// After shutdown the indicator lazily builds a NEW client on the next probe,
|
||||
// proving the old one was truly released rather than reused.
|
||||
await indicator.pingCheck('redis');
|
||||
expect(mockLiveClients).toHaveLength(2);
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Happy-path regression guard (#486, B2): the FIRST probe against a LIVE Redis
|
||||
* must report UP.
|
||||
*
|
||||
* With `lazyConnect: true` + `enableOfflineQueue: false`, a freshly-built client
|
||||
* is in the `wait` state and the socket opens lazily. If the very first `ping()`
|
||||
* is issued before an explicit `connect()`, ioredis rejects it instantly with
|
||||
* "Stream isn't writeable and enableOfflineQueue options is false" — a FALSE
|
||||
* DOWN even though Redis is alive. The fix opens the socket before the first
|
||||
* ping. This exercises a REAL ioredis client against a REAL TCP redis server
|
||||
* (not a mock), so a regression genuinely reddens it.
|
||||
*/
|
||||
describe('RedisHealthIndicator live Redis first-probe (#486, B2)', () => {
|
||||
const indicatorService = {
|
||||
check: (key: string) => ({
|
||||
up: () => ({ [key]: { status: 'up' } }),
|
||||
down: (message: string) => ({ [key]: { status: 'down', message } }),
|
||||
}),
|
||||
} as unknown as HealthIndicatorService;
|
||||
|
||||
// A REAL running redis (see the neighboring harness / CI env).
|
||||
const environmentService = {
|
||||
getRedisUrl: () => 'redis://127.0.0.1:6379/0',
|
||||
} as unknown as EnvironmentService;
|
||||
|
||||
let indicator: RedisHealthIndicator;
|
||||
|
||||
beforeEach(() => {
|
||||
mockLiveClients.length = 0;
|
||||
indicator = new RedisHealthIndicator(indicatorService, environmentService);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
// Await full socket close of every live client (see drainClient) BEFORE
|
||||
// onModuleDestroy: a real, connected ioredis client MUST be drained to 'end'
|
||||
// or its stream-destroy timer keeps the jest worker alive past the 1s window.
|
||||
await drainAll();
|
||||
indicator.onModuleDestroy();
|
||||
});
|
||||
|
||||
it('reports UP on the FIRST probe against a live Redis', async () => {
|
||||
// The VERY FIRST probe — no warm-up ping — must be UP.
|
||||
const result = await indicator.pingCheck('redis');
|
||||
expect(result.redis.status).toBe('up');
|
||||
});
|
||||
|
||||
it('stays UP on a probe AFTER onModuleDestroy re-creates the client', async () => {
|
||||
await indicator.pingCheck('redis');
|
||||
indicator.onModuleDestroy();
|
||||
// The re-created client is again in `wait`; the first ping on it must still
|
||||
// open the socket (the false-DOWN also recurs on the post-destroy path).
|
||||
const result = await indicator.pingCheck('redis');
|
||||
expect(result.redis.status).toBe('up');
|
||||
});
|
||||
});
|
||||
@@ -2,33 +2,173 @@ import {
|
||||
HealthIndicatorResult,
|
||||
HealthIndicatorService,
|
||||
} from '@nestjs/terminus';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { EnvironmentService } from '../environment/environment.service';
|
||||
import { Redis } from 'ioredis';
|
||||
|
||||
@Injectable()
|
||||
export class RedisHealthIndicator {
|
||||
export class RedisHealthIndicator implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(RedisHealthIndicator.name);
|
||||
|
||||
/**
|
||||
* ONE long-lived probe connection, reused across every /health tick. The old
|
||||
* code built `new Redis(...)` per call and only `disconnect()`d on the SUCCESS
|
||||
* path, so while Redis was DOWN every probe added a fresh, forever-reconnecting
|
||||
* client — a handle leak that grew without bound for as long as the outage (and
|
||||
* the health checker keeps polling) lasted. A single shared client keeps at most
|
||||
* ONE background reconnect loop regardless of how many probes run.
|
||||
*/
|
||||
private probeClient: Redis | null = null;
|
||||
|
||||
/**
|
||||
* How long the first-ping `connect()` may take before a probe gives up and
|
||||
* reports DOWN. A `connect()` against a truly-down Redis never settles on its
|
||||
* own (ioredis retries the socket indefinitely per its retryStrategy), so the
|
||||
* probe MUST bound it or the /health handler would hang. Kept short so a real
|
||||
* outage is reported fast; localhost/live Redis connects well within it.
|
||||
*/
|
||||
private static readonly CONNECT_TIMEOUT_MS = 2000;
|
||||
|
||||
/**
|
||||
* The single in-flight first-`connect()`, memoized so CONCURRENT probes share
|
||||
* it. k8s liveness+readiness hit /health in parallel on startup: without this,
|
||||
* probe A drives `connect()` (the client leaves the `wait` state) and probe B,
|
||||
* seeing a not-`wait`/not-`ready` client, would skip connect and fire `ping()`
|
||||
* at a still-opening socket → an instant FALSE DOWN. With the memo, B awaits
|
||||
* the SAME connect. Cleared once it settles so a later disconnect / re-create
|
||||
* starts a fresh connect.
|
||||
*/
|
||||
private connectingPromise: Promise<void> | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly healthIndicatorService: HealthIndicatorService,
|
||||
private environmentService: EnvironmentService,
|
||||
) {}
|
||||
|
||||
private getProbeClient(): Redis {
|
||||
if (!this.probeClient) {
|
||||
this.probeClient = new Redis(this.environmentService.getRedisUrl(), {
|
||||
// Constructing must never throw or eagerly connect; the first ping opens
|
||||
// the socket. This lets us build the client once and reuse it.
|
||||
lazyConnect: true,
|
||||
// A health probe must fail FAST, not queue behind a stuck reconnect: one
|
||||
// retry per request, and no offline queue so a ping while disconnected
|
||||
// rejects immediately instead of buffering commands that pile up in RAM.
|
||||
maxRetriesPerRequest: 1,
|
||||
enableOfflineQueue: false,
|
||||
});
|
||||
// ioredis emits 'error' on every failed (re)connect; with no listener that
|
||||
// surfaces as an unhandled 'error' event and can crash the process. Swallow
|
||||
// it here — pingCheck already reports health — and log at debug so a Redis
|
||||
// outage does not flood the logs.
|
||||
this.probeClient.on('error', (err) => {
|
||||
this.logger.debug(
|
||||
`Redis probe connection error: ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
return this.probeClient;
|
||||
}
|
||||
|
||||
/**
|
||||
* Open the probe socket BEFORE the first ping. `lazyConnect: true` leaves a
|
||||
* freshly-built (or post-destroy re-built) client in the `wait` state: the
|
||||
* socket is NOT open yet, so with `enableOfflineQueue: false` the very first
|
||||
* `ping()` rejects instantly with "Stream isn't writeable and
|
||||
* enableOfflineQueue options is false" even when Redis is perfectly alive — a
|
||||
* false DOWN on the happy path. We drive `connect()` ONLY from `wait`; once
|
||||
* the client is connected, ioredis owns its own (re)connect loop and a ping
|
||||
* issued while it reconnects still fast-fails to a correct DOWN (offline queue
|
||||
* stays off). A failed/timed-out connect rejects → reported DOWN, which is the
|
||||
* right signal for a truly-down Redis.
|
||||
*/
|
||||
private ensureConnected(client: Redis): Promise<void> {
|
||||
// Already open — steady state, nothing to do.
|
||||
if (client.status === 'ready') return Promise.resolve();
|
||||
// A first-connect is already in flight (possibly started by a CONCURRENT
|
||||
// probe): await the SAME one instead of racing a second connect() (ioredis
|
||||
// throws "already connecting") or firing ping() at a not-yet-open socket.
|
||||
if (this.connectingPromise) return this.connectingPromise;
|
||||
// Only DRIVE connect() from the initial `wait` state (fresh / post-destroy
|
||||
// re-created client). In any other non-ready state ioredis already owns its
|
||||
// (re)connect loop; a ping there fast-fails to a correct DOWN, so we must not
|
||||
// start a competing connect.
|
||||
if (client.status !== 'wait') return Promise.resolve();
|
||||
|
||||
const promise = this.connectWithTimeout(client).finally(() => {
|
||||
// Clear only if still ours, so a later disconnect / re-create can connect
|
||||
// again. Whether it resolved or rejected, the memo has served its window.
|
||||
if (this.connectingPromise === promise) {
|
||||
this.connectingPromise = null;
|
||||
}
|
||||
});
|
||||
this.connectingPromise = promise;
|
||||
return promise;
|
||||
}
|
||||
|
||||
private connectWithTimeout(client: Redis): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
reject(new Error('Redis probe connect timed out'));
|
||||
}, RedisHealthIndicator.CONNECT_TIMEOUT_MS);
|
||||
// Never let THIS timer alone keep the event loop (or a jest worker) alive;
|
||||
// it is cleared on settle anyway, this is belt-and-braces.
|
||||
timer.unref?.();
|
||||
// `.catch` is always attached, so a connect() that rejects AFTER we have
|
||||
// already timed out is handled here (guarded by `settled`) and never
|
||||
// surfaces as an unhandled rejection.
|
||||
client
|
||||
.connect()
|
||||
.then(() => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
})
|
||||
.catch((err) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async pingCheck(key: string): Promise<HealthIndicatorResult> {
|
||||
const indicator = this.healthIndicatorService.check(key);
|
||||
|
||||
try {
|
||||
const redis = new Redis(this.environmentService.getRedisUrl(), {
|
||||
maxRetriesPerRequest: 15,
|
||||
});
|
||||
|
||||
const redis = this.getProbeClient();
|
||||
// Open the socket before the first ping (see ensureConnected); without
|
||||
// this the first probe after (re)creation falsely reports DOWN on a live
|
||||
// Redis because lazyConnect defers the connect past the first ping.
|
||||
await this.ensureConnected(redis);
|
||||
await redis.ping();
|
||||
redis.disconnect();
|
||||
return indicator.up();
|
||||
} catch (e) {
|
||||
this.logger.error(e);
|
||||
return indicator.down(`${key} is not available`);
|
||||
}
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
if (this.probeClient) {
|
||||
// disconnect() (not quit()) tears the socket + reconnect loop down
|
||||
// immediately without waiting on a round-trip to a possibly-down server.
|
||||
// Do NOT removeAllListeners() with no event name — that would also strip
|
||||
// ioredis' OWN internal listeners and break its teardown; our 'error'
|
||||
// listener is harmless and dies with the dropped client reference.
|
||||
this.probeClient.disconnect();
|
||||
this.probeClient = null;
|
||||
}
|
||||
// Drop any in-flight first-connect memo so the NEXT client (lazily rebuilt on
|
||||
// the next probe) starts a fresh connect rather than awaiting a promise tied
|
||||
// to the client we just tore down.
|
||||
this.connectingPromise = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,8 +238,9 @@ function convertReferenceFootnotes(markdown: string): string {
|
||||
*
|
||||
* LINE-ANCHORED (the same shape the canonical parser uses in
|
||||
* prosemirror-markdown/page-file.ts): the block opens only on `---\n` at the
|
||||
* very start and closes only on a `\n---` line. The retired `markdownToHtml`
|
||||
* strip closed on the FIRST `---` ANYWHERE (an unanchored close), so a value
|
||||
* very start and closes only on a `\n---` line. The retired editor-ext
|
||||
* `markdownToHtml` front-matter strip (removed in #347) closed on the FIRST
|
||||
* `---` ANYWHERE (an unanchored close), so a value
|
||||
* containing a triple-dash (e.g. `title: Q1 --- Q2`) truncated the front-matter
|
||||
* and leaked the rest into the body. An optional leading BOM is tolerated.
|
||||
*/
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
} from './mcp-auth.helpers';
|
||||
import { JwtType } from '../../core/auth/dto/jwt-payload';
|
||||
import { CREDENTIALS_MISMATCH_MESSAGE } from '../../core/auth/auth.constants';
|
||||
import { McpService } from './mcp.service';
|
||||
|
||||
// The /mcp per-user auth decision logic is tested through the framework-free
|
||||
// `resolveMcpSessionConfig` helper that McpService delegates to. McpService
|
||||
@@ -1179,3 +1180,46 @@ describe('mapAuthResultToResponse (handle status/body mapping, refactor R2)', ()
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// #486: onModuleDestroy must ALSO tear down the live loopback CollabSessions, not
|
||||
// just clear the sweep timer — otherwise the embedded MCP's collab sockets keep
|
||||
// docs pinned open on the collab server past process exit. The teardown goes
|
||||
// through an overridable seam (destroyAllMcpSessions) so it can be spied without
|
||||
// loading the ESM-only @docmost/mcp package.
|
||||
describe('McpService.onModuleDestroy — CollabSession teardown (#486)', () => {
|
||||
function makeService(): McpService {
|
||||
// The constructor only stores its deps and starts the (unref'd) sweep timer,
|
||||
// so bare stubs suffice. onModuleDestroy clears that timer, so no leak.
|
||||
return new McpService(
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
{} as any,
|
||||
);
|
||||
}
|
||||
|
||||
it('destroys all sessions AND clears the sweep timer on shutdown', async () => {
|
||||
const svc = makeService();
|
||||
const destroy = jest.fn().mockResolvedValue(undefined);
|
||||
(svc as any).destroyAllMcpSessions = destroy;
|
||||
const clearSpy = jest.spyOn(global, 'clearInterval');
|
||||
|
||||
await svc.onModuleDestroy();
|
||||
|
||||
expect(destroy).toHaveBeenCalledTimes(1);
|
||||
expect(clearSpy).toHaveBeenCalledWith((svc as any).sweepTimer);
|
||||
clearSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('swallows a teardown failure so shutdown never throws', async () => {
|
||||
const svc = makeService();
|
||||
(svc as any).destroyAllMcpSessions = jest
|
||||
.fn()
|
||||
.mockRejectedValue(new Error('collab teardown boom'));
|
||||
|
||||
await expect(svc.onModuleDestroy()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -34,6 +34,8 @@ import {
|
||||
isMetricsEnabled,
|
||||
observeMcpTool,
|
||||
incConnectTimeout,
|
||||
incGetPageCacheHit,
|
||||
incGetPageCacheMiss,
|
||||
} from '../metrics/metrics.registry';
|
||||
|
||||
// Minimal shape of the embedded MCP HTTP handler exported by @docmost/mcp/http.
|
||||
@@ -117,10 +119,42 @@ export class McpService implements OnModuleDestroy {
|
||||
this.sweepTimer.unref?.();
|
||||
}
|
||||
|
||||
onModuleDestroy(): void {
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
clearInterval(this.sweepTimer);
|
||||
// Tear down any live loopback CollabSession providers at shutdown (#486). The
|
||||
// embedded MCP (and the in-app AI agent) open Hocuspocus collab sockets against
|
||||
// THIS process; without an explicit teardown those sessions keep their docs
|
||||
// "open" on the collab server and hold providers/buffers until they idle out,
|
||||
// so a restart can race a doc still pinned by the dying worker. Best-effort:
|
||||
// any failure is logged, never allowed to break shutdown.
|
||||
try {
|
||||
await this.destroyAllMcpSessions();
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
'MCP CollabSession teardown on shutdown failed',
|
||||
err as Error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve @docmost/mcp's `destroyAllSessions` and invoke it (#486). The live
|
||||
* CollabSession registry is a module-level singleton in the ESM package, shared
|
||||
* by every entry (`.`/`./http`), so this tears down ALL sessions regardless of
|
||||
* which surface opened them. The module is already loaded whenever MCP was used;
|
||||
* if it was never loaded (or is absent) the import + no-op is harmless.
|
||||
*
|
||||
* Held as an overridable field so a unit test can spy the teardown without
|
||||
* loading the ESM-only package or standing up the DI graph.
|
||||
*/
|
||||
private destroyAllMcpSessions: () => Promise<void> = async () => {
|
||||
const entry = require.resolve('@docmost/mcp');
|
||||
const mod = (await esmImport(pathToFileURL(entry).href)) as {
|
||||
destroyAllSessions?: () => void;
|
||||
};
|
||||
mod.destroyAllSessions?.();
|
||||
};
|
||||
|
||||
// Service account the embedded MCP uses to talk back to this Docmost
|
||||
// instance over loopback REST + the collaboration WebSocket. Now OPTIONAL:
|
||||
// it is only a fallback when no per-user Basic/Bearer credentials are sent.
|
||||
@@ -357,6 +391,10 @@ export class McpService implements OnModuleDestroy {
|
||||
observeMcpTool(labels?.tool ?? 'other', value);
|
||||
} else if (name === 'collab_connect_timeouts_total') {
|
||||
incConnectTimeout();
|
||||
} else if (name === 'mcp_getpage_cache_hits_total') {
|
||||
incGetPageCacheHit();
|
||||
} else if (name === 'mcp_getpage_cache_misses_total') {
|
||||
incGetPageCacheMiss();
|
||||
}
|
||||
}
|
||||
: undefined,
|
||||
|
||||
@@ -25,6 +25,15 @@ export const METRIC_COLLAB_CONNECT_TIMEOUTS_TOTAL =
|
||||
export const METRIC_COLLAB_AUTH_DURATION = 'collab_auth_duration_seconds';
|
||||
export const METRIC_MCP_TOOL_DURATION = 'mcp_tool_duration_seconds';
|
||||
|
||||
// #479 — getPage PM→Markdown conversion cache hit/miss counters. Emitted by the
|
||||
// MCP package via its dependency-neutral onMetric sink and routed onto these two
|
||||
// prom counters by the mcp.service onMetric callback; a >50% hit-rate is the
|
||||
// success signal for the getPage perf work. Same "do not rename" contract.
|
||||
export const METRIC_MCP_GETPAGE_CACHE_HITS_TOTAL =
|
||||
'mcp_getpage_cache_hits_total';
|
||||
export const METRIC_MCP_GETPAGE_CACHE_MISSES_TOTAL =
|
||||
'mcp_getpage_cache_misses_total';
|
||||
|
||||
// Histogram buckets (seconds). Chosen to give useful p50/p95/p99 resolution
|
||||
// for typical web/DB latencies without exploding series cardinality.
|
||||
export const HTTP_BUCKETS = [
|
||||
|
||||
@@ -24,6 +24,8 @@ import {
|
||||
METRIC_DB_QUERY_DURATION,
|
||||
METRIC_HTTP_REQUEST_DURATION,
|
||||
METRIC_MCP_TOOL_DURATION,
|
||||
METRIC_MCP_GETPAGE_CACHE_HITS_TOTAL,
|
||||
METRIC_MCP_GETPAGE_CACHE_MISSES_TOTAL,
|
||||
sizeBucket,
|
||||
} from './metrics.constants';
|
||||
|
||||
@@ -61,6 +63,9 @@ let connectTimeoutsCounter: Counter | null = null;
|
||||
let collabConnectHist: Histogram | null = null;
|
||||
let collabAuthHist: Histogram | null = null;
|
||||
let mcpToolHist: Histogram<'tool'> | null = null;
|
||||
// #479 — getPage conversion-cache hit/miss counters.
|
||||
let getPageCacheHitsCounter: Counter | null = null;
|
||||
let getPageCacheMissesCounter: Counter | null = null;
|
||||
|
||||
// #402 — read-on-scrape source for collab_docs_open. The gauge is NEVER
|
||||
// inc/dec'd (that drifts under crashes/handoffs); instead its collect() callback
|
||||
@@ -175,6 +180,18 @@ function init(): void {
|
||||
buckets: MCP_TOOL_BUCKETS,
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
getPageCacheHitsCounter = new Counter({
|
||||
name: METRIC_MCP_GETPAGE_CACHE_HITS_TOTAL,
|
||||
help: 'Total getPage PM→Markdown conversions served from the cache (skipped)',
|
||||
registers: [registry],
|
||||
});
|
||||
|
||||
getPageCacheMissesCounter = new Counter({
|
||||
name: METRIC_MCP_GETPAGE_CACHE_MISSES_TOTAL,
|
||||
help: 'Total getPage PM→Markdown conversions computed (cache misses)',
|
||||
registers: [registry],
|
||||
});
|
||||
}
|
||||
|
||||
// Runs once when this module is first imported. Safe to call again (idempotent).
|
||||
@@ -247,6 +264,14 @@ export function observeCollabAuth(seconds: number): void {
|
||||
collabAuthHist?.observe(seconds);
|
||||
}
|
||||
|
||||
export function incGetPageCacheHit(): void {
|
||||
getPageCacheHitsCounter?.inc();
|
||||
}
|
||||
|
||||
export function incGetPageCacheMiss(): void {
|
||||
getPageCacheMissesCounter?.inc();
|
||||
}
|
||||
|
||||
export function observeMcpTool(tool: string, seconds: number): void {
|
||||
// `tool` MUST be a bounded, registration-derived MCP tool name (the caller
|
||||
// guarantees it comes from the registered-tool set) — never free-form input —
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
import { get as httpGet } from 'node:http';
|
||||
import { AddressInfo } from 'node:net';
|
||||
import { createServer } from 'node:http';
|
||||
|
||||
// Drive the metrics HTTP server without the load-time METRICS_PORT gate: mock the
|
||||
// registry so isMetricsEnabled()/getMetricsRegistry() are always satisfied. What
|
||||
// we assert is observed over a REAL socket (bind address, status codes), not on
|
||||
// the mock.
|
||||
jest.mock('./metrics.registry', () => ({
|
||||
isMetricsEnabled: () => true,
|
||||
getMetricsRegistry: () => ({
|
||||
metrics: async () => '# HELP up test\nup 1\n',
|
||||
contentType: 'text/plain; version=0.0.4',
|
||||
}),
|
||||
}));
|
||||
|
||||
import {
|
||||
startMetricsServer,
|
||||
closeMetricsServer,
|
||||
resolveMetricsBind,
|
||||
resolveMetricsToken,
|
||||
} from './metrics.server';
|
||||
|
||||
/** Find a free TCP port (the metrics server requires METRICS_PORT > 0). */
|
||||
function freePort(): Promise<number> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const s = createServer();
|
||||
s.once('error', reject);
|
||||
s.listen(0, '127.0.0.1', () => {
|
||||
const p = (s.address() as AddressInfo).port;
|
||||
s.close(() => resolve(p));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Minimal GET against 127.0.0.1:port with optional Authorization header. */
|
||||
function req(
|
||||
port: number,
|
||||
headers: Record<string, string> = {},
|
||||
): Promise<{ status: number; body: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const r = httpGet(
|
||||
{ host: '127.0.0.1', port, path: '/metrics', headers },
|
||||
(res) => {
|
||||
let body = '';
|
||||
res.on('data', (c) => (body += c));
|
||||
res.on('end', () =>
|
||||
resolve({ status: res.statusCode ?? 0, body }),
|
||||
);
|
||||
},
|
||||
);
|
||||
r.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
describe('metrics server bind + auth (#486)', () => {
|
||||
const saved = {
|
||||
bind: process.env.METRICS_BIND,
|
||||
token: process.env.METRICS_TOKEN,
|
||||
port: process.env.METRICS_PORT,
|
||||
};
|
||||
|
||||
afterEach(async () => {
|
||||
await closeMetricsServer();
|
||||
process.env.METRICS_BIND = saved.bind;
|
||||
process.env.METRICS_TOKEN = saved.token;
|
||||
process.env.METRICS_PORT = saved.port;
|
||||
delete process.env.METRICS_BIND;
|
||||
delete process.env.METRICS_TOKEN;
|
||||
});
|
||||
|
||||
describe('resolveMetricsBind', () => {
|
||||
it('defaults to loopback 127.0.0.1', () => {
|
||||
delete process.env.METRICS_BIND;
|
||||
expect(resolveMetricsBind()).toBe('127.0.0.1');
|
||||
});
|
||||
it('honours the METRICS_BIND override', () => {
|
||||
process.env.METRICS_BIND = '0.0.0.0';
|
||||
expect(resolveMetricsBind()).toBe('0.0.0.0');
|
||||
});
|
||||
it('treats a blank override as unset (loopback)', () => {
|
||||
process.env.METRICS_BIND = ' ';
|
||||
expect(resolveMetricsBind()).toBe('127.0.0.1');
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveMetricsToken', () => {
|
||||
it('is null when unset', () => {
|
||||
delete process.env.METRICS_TOKEN;
|
||||
expect(resolveMetricsToken()).toBeNull();
|
||||
});
|
||||
it('returns the trimmed token when set', () => {
|
||||
process.env.METRICS_TOKEN = ' s3cret ';
|
||||
expect(resolveMetricsToken()).toBe('s3cret');
|
||||
});
|
||||
});
|
||||
|
||||
it('binds to loopback by default and serves /metrics without auth when no token', async () => {
|
||||
delete process.env.METRICS_BIND;
|
||||
delete process.env.METRICS_TOKEN;
|
||||
const port = await freePort();
|
||||
process.env.METRICS_PORT = String(port);
|
||||
|
||||
const server = startMetricsServer();
|
||||
expect(server).not.toBeNull();
|
||||
await new Promise<void>((resolve) => {
|
||||
if (server!.listening) resolve();
|
||||
else server!.once('listening', () => resolve());
|
||||
});
|
||||
// OBSERVABLE: the listener bound to loopback, not 0.0.0.0.
|
||||
expect((server!.address() as AddressInfo).address).toBe('127.0.0.1');
|
||||
|
||||
const res = await req(port);
|
||||
expect(res.status).toBe(200);
|
||||
expect(res.body).toContain('up 1');
|
||||
});
|
||||
|
||||
it('rejects unauthenticated scrapes with 401 and accepts the exact Bearer token', async () => {
|
||||
delete process.env.METRICS_BIND;
|
||||
process.env.METRICS_TOKEN = 'topsecret';
|
||||
const port = await freePort();
|
||||
process.env.METRICS_PORT = String(port);
|
||||
|
||||
const server = startMetricsServer();
|
||||
expect(server).not.toBeNull();
|
||||
|
||||
// No auth -> 401.
|
||||
const noAuth = await req(port);
|
||||
expect(noAuth.status).toBe(401);
|
||||
|
||||
// Wrong token, DIFFERENT length -> 401 (short-circuits on the length guard).
|
||||
const wrong = await req(port, { authorization: 'Bearer nope' });
|
||||
expect(wrong.status).toBe(401);
|
||||
|
||||
// Wrong token, SAME length -> 401. This drives the timingSafeEqual compare
|
||||
// itself (the length guard passes: 'Bearer topsecreX' has the same length as
|
||||
// 'Bearer topsecret'). Pins the constant-time compare: a regression that made
|
||||
// it return true would let this equal-length wrong token through — the
|
||||
// different-length case above would NOT catch that.
|
||||
const sameLen = await req(port, { authorization: 'Bearer topsecreX' });
|
||||
expect(sameLen.status).toBe(401);
|
||||
|
||||
// Correct token -> 200 with the metrics body.
|
||||
const ok = await req(port, { authorization: 'Bearer topsecret' });
|
||||
expect(ok.status).toBe(200);
|
||||
expect(ok.body).toContain('up 1');
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,27 @@
|
||||
import { createServer, Server } from 'node:http';
|
||||
import { timingSafeEqual } from 'node:crypto';
|
||||
import { Logger } from '@nestjs/common';
|
||||
import { getMetricsRegistry, isMetricsEnabled } from './metrics.registry';
|
||||
|
||||
/**
|
||||
* Constant-time compare of the presented Authorization header against the
|
||||
* expected `Bearer <token>`. This is the ONLY auth layer for the metrics
|
||||
* endpoint, so a naive `!==` would leak the token byte-by-byte via timing.
|
||||
* timingSafeEqual requires equal-length buffers, so a length mismatch short-
|
||||
* circuits to "not equal" (its own length is not itself a useful oracle: the
|
||||
* expected string length is fixed by config, not secret-derived).
|
||||
*/
|
||||
function bearerMatches(
|
||||
presented: string | undefined,
|
||||
expected: string,
|
||||
): boolean {
|
||||
if (typeof presented !== 'string') return false;
|
||||
const a = Buffer.from(presented);
|
||||
const b = Buffer.from(expected);
|
||||
if (a.length !== b.length) return false;
|
||||
return timingSafeEqual(a, b);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the Prometheus scrape endpoint on a SEPARATE port, taken from
|
||||
* `METRICS_PORT`. There is NO default port: when `METRICS_PORT` is unset the
|
||||
@@ -16,6 +36,30 @@ import { getMetricsRegistry, isMetricsEnabled } from './metrics.registry';
|
||||
*/
|
||||
let metricsServer: Server | null = null;
|
||||
|
||||
/**
|
||||
* Interface the metrics endpoint binds to. Defaults to LOOPBACK (127.0.0.1) so
|
||||
* the unauthenticated `/metrics` surface is NOT exposed on all interfaces by
|
||||
* default — the old `0.0.0.0` bind put an auth-less endpoint on every interface.
|
||||
* Deployments where the scraper runs in a SEPARATE container (and reaches this as
|
||||
* `docmost:9464`) set `METRICS_BIND=0.0.0.0`, ideally together with METRICS_TOKEN
|
||||
* and/or a private network so the port is not world-readable.
|
||||
*/
|
||||
export function resolveMetricsBind(): string {
|
||||
const raw = (process.env.METRICS_BIND ?? '').trim();
|
||||
return raw.length > 0 ? raw : '127.0.0.1';
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional Bearer token guarding `/metrics`. When `METRICS_TOKEN` is set, every
|
||||
* scrape must present `Authorization: Bearer <token>`; unset (default) leaves the
|
||||
* endpoint open (safe when bound to loopback / a trusted network). Returns the
|
||||
* trimmed token or null when unset/blank.
|
||||
*/
|
||||
export function resolveMetricsToken(): string | null {
|
||||
const raw = (process.env.METRICS_TOKEN ?? '').trim();
|
||||
return raw.length > 0 ? raw : null;
|
||||
}
|
||||
|
||||
export function startMetricsServer(): Server | null {
|
||||
if (!isMetricsEnabled()) return null;
|
||||
|
||||
@@ -31,8 +75,22 @@ export function startMetricsServer(): Server | null {
|
||||
return null;
|
||||
}
|
||||
|
||||
const bind = resolveMetricsBind();
|
||||
const token = resolveMetricsToken();
|
||||
|
||||
const server = createServer(async (req, res) => {
|
||||
if (req.method === 'GET' && req.url === '/metrics') {
|
||||
// Optional Bearer auth: reject scrapes without the exact token when one is
|
||||
// configured. This is the auth layer the old all-interfaces bind lacked.
|
||||
if (token) {
|
||||
const auth = req.headers['authorization'];
|
||||
if (!bearerMatches(auth, `Bearer ${token}`)) {
|
||||
res.statusCode = 401;
|
||||
res.setHeader('WWW-Authenticate', 'Bearer');
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
}
|
||||
try {
|
||||
const body = await register.metrics();
|
||||
res.setHeader('Content-Type', register.contentType);
|
||||
@@ -48,10 +106,14 @@ export function startMetricsServer(): Server | null {
|
||||
res.end();
|
||||
});
|
||||
|
||||
// Bind on all interfaces: the scraper (VictoriaMetrics) reaches this from
|
||||
// another container as docmost:9464. The port is not published to the host.
|
||||
server.listen(port, '0.0.0.0', () => {
|
||||
logger.log(`Metrics endpoint listening on :${port}/metrics`);
|
||||
// Bind to loopback by default so the auth-less endpoint is not exposed on all
|
||||
// interfaces. Set METRICS_BIND=0.0.0.0 (ideally with METRICS_TOKEN) when the
|
||||
// scraper runs in a separate container and reaches this as docmost:9464.
|
||||
server.listen(port, bind, () => {
|
||||
logger.log(
|
||||
`Metrics endpoint listening on ${bind}:${port}/metrics` +
|
||||
(token ? ' (Bearer auth required)' : ''),
|
||||
);
|
||||
});
|
||||
|
||||
server.on('error', (err) => {
|
||||
|
||||
@@ -10,6 +10,8 @@ import {
|
||||
incConnectTimeout,
|
||||
incDocLoad,
|
||||
incDocUnload,
|
||||
incGetPageCacheHit,
|
||||
incGetPageCacheMiss,
|
||||
isMetricsEnabled,
|
||||
observeCollabAuth,
|
||||
observeCollabConnect,
|
||||
@@ -197,6 +199,8 @@ describe('metrics helpers are safe no-ops when METRICS_PORT is unset', () => {
|
||||
incDocLoad();
|
||||
incDocUnload();
|
||||
incConnectTimeout();
|
||||
incGetPageCacheHit();
|
||||
incGetPageCacheMiss();
|
||||
// Registering a source must not create the gauge or invoke the fn.
|
||||
registerDocsOpenSource(() => {
|
||||
throw new Error('docsOpenSource must NOT be called when disabled');
|
||||
|
||||
@@ -31,9 +31,6 @@ export enum QueueJob {
|
||||
IMPORT_TASK = 'import-task',
|
||||
EXPORT_TASK = 'export-task',
|
||||
|
||||
SEARCH_REMOVE_PAGE = 'search-remove-page',
|
||||
SEARCH_REMOVE_ASSET = 'search-remove-attachment',
|
||||
SEARCH_REMOVE_FACE = 'search-remove-comment',
|
||||
TYPESENSE_FLUSH = 'typesense-flush',
|
||||
|
||||
PAGE_CREATED = 'page-created',
|
||||
|
||||
@@ -182,6 +182,7 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
{
|
||||
isAiChatDeferredToolsEnabled: () => false,
|
||||
isAiChatResumableStreamEnabled: () => true,
|
||||
isAiChatFinalStepLockdownEnabled: () => false,
|
||||
} as any,
|
||||
registry,
|
||||
);
|
||||
@@ -499,6 +500,7 @@ describe('AiChatService run-stream attach [integration]', () => {
|
||||
{
|
||||
isAiChatDeferredToolsEnabled: () => false,
|
||||
isAiChatResumableStreamEnabled: () => true,
|
||||
isAiChatFinalStepLockdownEnabled: () => false,
|
||||
} as any,
|
||||
registry,
|
||||
);
|
||||
|
||||
@@ -150,7 +150,7 @@ describe('AiChatService.stream [integration]', () => {
|
||||
{} as any, // pageAccess (idem)
|
||||
// environment (#332): keep deferred tool loading OFF for this lifecycle
|
||||
// harness so the toolset/behavior is exactly as before.
|
||||
{ isAiChatDeferredToolsEnabled: () => false } as any,
|
||||
{ isAiChatDeferredToolsEnabled: () => false, isAiChatFinalStepLockdownEnabled: () => false } as any,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -378,7 +378,7 @@ describe('AiChatService.stream [integration]', () => {
|
||||
{} as any,
|
||||
{} as any,
|
||||
// #332: deferred tool loading ON — the property under test.
|
||||
{ isAiChatDeferredToolsEnabled: () => true } as any,
|
||||
{ isAiChatDeferredToolsEnabled: () => true, isAiChatFinalStepLockdownEnabled: () => false } as any,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -11,9 +11,6 @@
|
||||
"main": "dist/index.js",
|
||||
"module": "./src/index.ts",
|
||||
"types": "dist/index.d.ts",
|
||||
"dependencies": {
|
||||
"marked": "17.0.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@vitest/coverage-v8": "4.1.6",
|
||||
"vitest": "4.1.6"
|
||||
|
||||
@@ -18,7 +18,6 @@ export * from "./lib/excalidraw";
|
||||
export * from "./lib/embed";
|
||||
export * from "./lib/html-embed/html-embed";
|
||||
export * from "./lib/mention";
|
||||
export * from "./lib/markdown";
|
||||
export * from "./lib/search-and-replace";
|
||||
export * from "./lib/embed-provider";
|
||||
export * from "./lib/subpages";
|
||||
|
||||
@@ -14,7 +14,8 @@ import {
|
||||
* ProseMirror JSON directly (never running the editor's plugins), so the
|
||||
* canonical footnote topology was never enforced on those writes. The consumers
|
||||
* of this editor-ext copy are: the server markdown/HTML import
|
||||
* (`markdownToHtml -> htmlToJson` in import.service / file-import-task.service),
|
||||
* (`markdownToProseMirror` from @docmost/prosemirror-markdown in import.service /
|
||||
* file-import-task.service),
|
||||
* `PageService` create/update (`parseProsemirrorContent` for the JSON/markdown/
|
||||
* HTML REST write paths), and the client markdown PASTE path
|
||||
* (`markdown-clipboard.ts`). (The MCP package mirrors this canonicalizer in
|
||||
|
||||
@@ -1,131 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { htmlToMarkdown } from "../markdown/utils/turndown.utils";
|
||||
import { markdownToHtml } from "../markdown/utils/marked.utils";
|
||||
import { extractFootnoteDefinitions } from "../markdown/utils/footnote.marked";
|
||||
|
||||
// HTML the editor-ext nodes render (sup[data-footnote-ref], section/div).
|
||||
const HTML =
|
||||
`<p>Water<sup data-footnote-ref data-id="fn1"></sup> and clay<sup data-footnote-ref data-id="fn2"></sup>.</p>` +
|
||||
`<section data-footnotes>` +
|
||||
`<div data-footnote-def data-id="fn1"><p>First note.</p></div>` +
|
||||
`<div data-footnote-def data-id="fn2"><p>Second note.</p></div>` +
|
||||
`</section>`;
|
||||
|
||||
describe("footnote markdown round-trip", () => {
|
||||
it("HTML -> Markdown produces pandoc footnote syntax", () => {
|
||||
const md = htmlToMarkdown(HTML);
|
||||
expect(md).toContain("[^fn1]");
|
||||
expect(md).toContain("[^fn2]");
|
||||
expect(md).toContain("[^fn1]: First note.");
|
||||
expect(md).toContain("[^fn2]: Second note.");
|
||||
});
|
||||
|
||||
it("Markdown -> HTML rebuilds the footnote nodes' HTML", async () => {
|
||||
const md = htmlToMarkdown(HTML);
|
||||
const html = await markdownToHtml(md);
|
||||
expect(html).toContain('data-footnote-ref data-id="fn1"');
|
||||
expect(html).toContain('data-footnote-ref data-id="fn2"');
|
||||
expect(html).toContain("data-footnotes");
|
||||
expect(html).toContain('data-footnote-def data-id="fn1"');
|
||||
expect(html).toContain("First note.");
|
||||
expect(html).toContain("Second note.");
|
||||
});
|
||||
|
||||
it("preserves a [^id]: line shown inside a fenced code block (not a definition)", async () => {
|
||||
// A document that DOCUMENTS footnote syntax inside a code fence. The
|
||||
// `[^demo]: ...` line is example text, not a real definition, and must
|
||||
// survive the Markdown -> HTML conversion verbatim.
|
||||
const md = [
|
||||
"Here is how footnotes look:",
|
||||
"",
|
||||
"```markdown",
|
||||
"Some text[^demo]",
|
||||
"",
|
||||
"[^demo]: this is the definition",
|
||||
"```",
|
||||
"",
|
||||
"End of doc.",
|
||||
].join("\n");
|
||||
|
||||
const html = await markdownToHtml(md);
|
||||
// The example definition line is kept inside the rendered code block.
|
||||
expect(html).toContain("[^demo]: this is the definition");
|
||||
// It did NOT get pulled out into a real footnotes section.
|
||||
expect(html).not.toContain("data-footnotes");
|
||||
expect(html).not.toContain("data-footnote-def");
|
||||
});
|
||||
|
||||
it("extractFootnoteDefinitions keeps the FIRST duplicate definition and reuses markers", () => {
|
||||
// Two definitions share id `d`, and the body has two `[^d]` markers. Under
|
||||
// the import model (#166) duplicate definition ids are FIRST-WINS: only the
|
||||
// first definition is kept; markers are NEVER rewritten, so the two `[^d]`
|
||||
// references reuse the single footnote.
|
||||
const md = [
|
||||
"See here[^d] and there[^d].",
|
||||
"",
|
||||
"[^d]: first",
|
||||
"[^d]: second",
|
||||
].join("\n");
|
||||
|
||||
const { body, section } = extractFootnoteDefinitions(md);
|
||||
|
||||
const defIds = Array.from(
|
||||
section.matchAll(/data-footnote-def data-id="([^"]+)"/g),
|
||||
).map((m) => m[1]);
|
||||
expect(defIds).toEqual(["d"]); // first-wins: one definition
|
||||
expect(section).toContain("first");
|
||||
expect(section).not.toContain("second"); // duplicate dropped
|
||||
|
||||
// Both markers stay `[^d]` (reuse) — no `d__2` minting.
|
||||
const refIds = Array.from(body.matchAll(/\[\^([^\]\s]+)\]/g)).map(
|
||||
(m) => m[1],
|
||||
);
|
||||
expect(refIds).toEqual(["d", "d"]);
|
||||
});
|
||||
|
||||
it("extractFootnoteDefinitions is DETERMINISTIC and stable (same input -> same output)", () => {
|
||||
// The output must be a pure function of the input markdown so importing the
|
||||
// same source twice (or via the editor and the MCP mirror) is identical.
|
||||
const md = [
|
||||
"See[^d] one[^d] two[^d].",
|
||||
"",
|
||||
"[^d]: first",
|
||||
"[^d]: second",
|
||||
"[^d]: third",
|
||||
].join("\n");
|
||||
|
||||
const run = () => {
|
||||
const { body, section } = extractFootnoteDefinitions(md);
|
||||
const defIds = Array.from(
|
||||
section.matchAll(/data-footnote-def data-id="([^"]+)"/g),
|
||||
).map((m) => m[1]);
|
||||
const refIds = Array.from(body.matchAll(/\[\^([^\]\s]+)\]/g)).map(
|
||||
(m) => m[1],
|
||||
);
|
||||
return { defIds, refIds };
|
||||
};
|
||||
|
||||
const a = run();
|
||||
const b = run();
|
||||
expect(a).toEqual(b);
|
||||
// First-wins: one kept definition `d`; all three reuse markers stay `d`.
|
||||
expect(a.defIds).toEqual(["d"]);
|
||||
expect(a.refIds).toEqual(["d", "d", "d"]);
|
||||
});
|
||||
|
||||
it("markdownToHtml with a reused id renders ONE shared footnote def", async () => {
|
||||
const md = [
|
||||
"See here[^d] and there[^d].",
|
||||
"",
|
||||
"[^d]: first",
|
||||
"[^d]: second",
|
||||
].join("\n");
|
||||
const html = await markdownToHtml(md);
|
||||
const defIds = Array.from(
|
||||
html.matchAll(/data-footnote-def data-id="([^"]+)"/g),
|
||||
).map((m) => m[1]);
|
||||
expect(defIds).toEqual(["d"]); // one shared definition
|
||||
expect(html).toContain("first");
|
||||
expect(html).not.toContain("second");
|
||||
});
|
||||
});
|
||||
@@ -103,8 +103,9 @@ interface CollisionPlan {
|
||||
* `X__2`, `X__3`, collision-bumped) so it survives as a distinct footnote — which,
|
||||
* having no matching reference, then falls under the normal orphan policy. It is
|
||||
* only ever dropped for lacking a reference, never for colliding. The IMPORT
|
||||
* paths (footnote.marked.ts / MCP extractFootnotes) instead apply first-wins +
|
||||
* drop + warn for duplicate definitions; that divergence is intentional — import
|
||||
* paths (@docmost/prosemirror-markdown / MCP extractFootnotes) instead apply
|
||||
* first-wins + drop + warn for duplicate definitions; that divergence is
|
||||
* intentional — import
|
||||
* is an agent-authored artifact we sanitize, the editor is live user data we must
|
||||
* not lose.
|
||||
*
|
||||
|
||||
@@ -6,8 +6,9 @@ import { deriveFootnoteId } from "./footnote-util";
|
||||
*
|
||||
* `deriveFootnoteId` lives ONLY in editor-ext now — it is used by
|
||||
* `resolveCollisions` (re-id of a duplicate definition) and `footnotePastePlugin`
|
||||
* (re-id of a pasted colliding definition). The MCP/marked import paths no longer
|
||||
* derive ids (duplicate definitions there are first-wins-dropped, #166), so there
|
||||
* (re-id of a pasted colliding definition). The MCP / @docmost/prosemirror-markdown
|
||||
* import paths no longer derive ids (duplicate definitions there are
|
||||
* first-wins-dropped, #166), so there
|
||||
* is no cross-package copy and no parity test to keep in sync. This table pins the
|
||||
* deterministic scheme so a future change to it is a conscious one.
|
||||
*/
|
||||
|
||||
@@ -63,8 +63,9 @@ export function generateFootnoteId(): string {
|
||||
* its own seen-set before requesting the next derived id.
|
||||
*
|
||||
* Used only inside editor-ext now (resolveCollisions for a re-id'd duplicate
|
||||
* DEFINITION, and footnotePastePlugin). The MCP/marked import paths no longer
|
||||
* derive ids — duplicate definitions there are first-wins-dropped (#166) — so
|
||||
* DEFINITION, and footnotePastePlugin). The MCP / @docmost/prosemirror-markdown
|
||||
* import paths no longer derive ids — duplicate definitions there are
|
||||
* first-wins-dropped (#166) — so
|
||||
* there is no cross-package copy to keep in sync. The golden table in
|
||||
* footnote-util.derive-id.test.ts pins the scheme.
|
||||
*/
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { generateJSON } from "@tiptap/html";
|
||||
import { Document } from "@tiptap/extension-document";
|
||||
import { Paragraph } from "@tiptap/extension-paragraph";
|
||||
import { Text } from "@tiptap/extension-text";
|
||||
import { htmlToMarkdown } from "../markdown/utils/turndown.utils";
|
||||
import { markdownToHtml } from "../markdown/utils/marked.utils";
|
||||
import { TiptapImage } from "./image";
|
||||
|
||||
// Minimal schema for parsing markdownToHtml output back to JSON (mirrors
|
||||
// image.spec.ts), so we can assert the recovered caption EXACTLY.
|
||||
const parseExtensions = [Document, Paragraph, Text, TiptapImage];
|
||||
|
||||
// Lossless markdown round-trip for image captions (issue #221). An image WITH a
|
||||
// caption can't be expressed as ``, so it is emitted as a raw <img>
|
||||
// (carrying data-caption) wrapped in a block <div>, the same trick the <video>
|
||||
// rule uses. marked passes the raw HTML through, so markdownToHtml keeps the
|
||||
// data-caption, and the image extension's parseHTML restores the attribute.
|
||||
describe("image caption markdown round-trip", () => {
|
||||
it("HTML -> Markdown emits a raw <img data-caption> for captioned images", () => {
|
||||
const html = `<p><img src="/files/a.png" alt="cat" data-caption="A grey cat"></p>`;
|
||||
const md = htmlToMarkdown(html);
|
||||
expect(md).toContain("data-caption=\"A grey cat\"");
|
||||
expect(md).toContain('src="/files/a.png"');
|
||||
expect(md).toContain('alt="cat"');
|
||||
// It must NOT degrade to the lossy ![]() form.
|
||||
expect(md).not.toContain("![cat]");
|
||||
});
|
||||
|
||||
it("Markdown -> HTML restores data-caption on the <img>", async () => {
|
||||
const html = `<p><img src="/files/a.png" alt="cat" data-caption="A grey cat"></p>`;
|
||||
const md = htmlToMarkdown(html);
|
||||
const back = await markdownToHtml(md);
|
||||
expect(back).toContain('data-caption="A grey cat"');
|
||||
expect(back).toContain('src="/files/a.png"');
|
||||
});
|
||||
|
||||
it("special characters in the caption survive the round-trip (escaped)", async () => {
|
||||
// The source caption is the decoded string `Tom & "Jerry"` (both an `&` and
|
||||
// a `"`). escapeHtmlAttr must encode `&` -> `&` and `"` -> `"`.
|
||||
const html = `<p><img src="/files/a.png" data-caption='Tom & "Jerry"'></p>`;
|
||||
const md = htmlToMarkdown(html);
|
||||
|
||||
// (a) The intermediate Markdown must carry the EXACT escaped attribute. This
|
||||
// fails if escapeHtmlAttr stopped escaping `"` (attribute break-out:
|
||||
// data-caption="Tom & "Jerry"") or double-encoded `&` (`&amp;`).
|
||||
expect(md).toContain('data-caption="Tom & "Jerry""');
|
||||
|
||||
const back = await markdownToHtml(md);
|
||||
expect(back).toContain("data-caption=");
|
||||
expect(back).toContain("Jerry");
|
||||
expect(back).toContain("Tom");
|
||||
|
||||
// (b) Re-parse the rendered HTML through the image extension's parseHTML and
|
||||
// assert the recovered caption is EXACTLY the original (no corruption, loss,
|
||||
// or double-encoding).
|
||||
const json = generateJSON(back, parseExtensions);
|
||||
expect(json.content?.[0]?.attrs?.caption).toBe('Tom & "Jerry"');
|
||||
});
|
||||
|
||||
it("caption-less images stay a clean  with no raw HTML", () => {
|
||||
const html = `<p><img src="/files/a.png" alt="cat"></p>`;
|
||||
const md = htmlToMarkdown(html);
|
||||
expect(md).toContain("");
|
||||
expect(md).not.toContain("data-caption");
|
||||
expect(md).not.toContain("<img");
|
||||
});
|
||||
});
|
||||
@@ -1,105 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { htmlEmbedExtension } from "./utils/html-embed.marked";
|
||||
import { markdownToHtml } from "./index";
|
||||
import { encodeHtmlEmbedSource } from "../html-embed/html-embed";
|
||||
|
||||
// CONTRACT tests for the marked block tokenizer that rebuilds an htmlEmbed node
|
||||
// from the `<!--html-embed:BASE64-->` marker (html-embed.marked.ts), plus the
|
||||
// observable round-trip through markdownToHtml.
|
||||
//
|
||||
// These pin the REAL tokenizer behaviour the import path depends on:
|
||||
// - the tokenizer rule is anchored (^) and only accepts the base64 alphabet
|
||||
// [A-Za-z0-9+/=], so a marker with non-base64 chars is NOT tokenized and
|
||||
// survives as a literal HTML comment (not silently turned into something the
|
||||
// server's strip no longer recognizes);
|
||||
// - start() reports the correct index of the next marker so marked invokes the
|
||||
// tokenizer at the right offset when a marker sits mid-document / after text;
|
||||
// - a marker with surrounding text on the SAME line is split out into its own
|
||||
// embed div while the surrounding text becomes ordinary paragraphs.
|
||||
//
|
||||
// The contract is asserted against the actual exported extension and pipeline —
|
||||
// no behaviour is invented; the expectations were read off the real tokenizer.
|
||||
|
||||
const SAMPLE = "<b>x</b>";
|
||||
const ENC = encodeHtmlEmbedSource(SAMPLE);
|
||||
|
||||
describe("htmlEmbed marked tokenizer — start()", () => {
|
||||
it("returns the index of a marker that sits mid-document", () => {
|
||||
const src = `hello world <!--html-embed:${ENC}-->`;
|
||||
expect(htmlEmbedExtension.start(src)).toBe(src.indexOf("<!--html-embed:"));
|
||||
});
|
||||
|
||||
it("returns 0 when the marker is at the very start", () => {
|
||||
expect(htmlEmbedExtension.start(`<!--html-embed:${ENC}-->`)).toBe(0);
|
||||
});
|
||||
|
||||
it("returns -1 when there is no marker", () => {
|
||||
expect(htmlEmbedExtension.start("no marker here")).toBe(-1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("htmlEmbed marked tokenizer — tokenizer()", () => {
|
||||
it("tokenizes a marker at the start of the input, capturing the base64 payload", () => {
|
||||
const token = htmlEmbedExtension.tokenizer(`<!--html-embed:${ENC}-->`);
|
||||
expect(token).toBeTruthy();
|
||||
expect(token!.type).toBe("htmlEmbed");
|
||||
expect(token!.raw).toBe(`<!--html-embed:${ENC}-->`);
|
||||
expect(token!.encoded).toBe(ENC);
|
||||
});
|
||||
|
||||
it("tokenizes an EMPTY marker (the [A-Za-z0-9+/=]* class allows zero chars)", () => {
|
||||
const token = htmlEmbedExtension.tokenizer("<!--html-embed:-->");
|
||||
expect(token).toBeTruthy();
|
||||
expect(token!.encoded).toBe("");
|
||||
expect(token!.raw).toBe("<!--html-embed:-->");
|
||||
});
|
||||
|
||||
it("does NOT tokenize when text precedes the marker (rule is anchored ^)", () => {
|
||||
// marked relies on start() to advance to the marker; the tokenizer itself
|
||||
// only matches at offset 0, so a non-anchored call returns undefined.
|
||||
expect(
|
||||
htmlEmbedExtension.tokenizer(`hello <!--html-embed:${ENC}-->`),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does NOT tokenize a marker containing a non-base64 char ('$')", () => {
|
||||
expect(
|
||||
htmlEmbedExtension.tokenizer("<!--html-embed:ab$cd-->"),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does NOT tokenize a marker containing a space", () => {
|
||||
expect(
|
||||
htmlEmbedExtension.tokenizer("<!--html-embed:ab cd-->"),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("renderer emits the embed div the node's parseHTML recognizes", () => {
|
||||
const token = htmlEmbedExtension.tokenizer(`<!--html-embed:${ENC}-->`)!;
|
||||
const html = htmlEmbedExtension.renderer(token as any);
|
||||
expect(html).toBe(
|
||||
`<div data-type="htmlEmbed" data-source="${ENC}"></div>`,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("htmlEmbed marked tokenizer — markdownToHtml round-trip", () => {
|
||||
it("splits a marker out of surrounding same-line text into its own embed div", async () => {
|
||||
const html = await markdownToHtml(`before <!--html-embed:${ENC}--> after`);
|
||||
// The marker became the embed div...
|
||||
expect(html).toContain(
|
||||
`<div data-type="htmlEmbed" data-source="${ENC}"></div>`,
|
||||
);
|
||||
// ...and the surrounding text survived as ordinary paragraph content.
|
||||
expect(html).toContain("before");
|
||||
expect(html).toContain("after");
|
||||
});
|
||||
|
||||
it("leaves a marker with non-base64 chars as a literal comment (NOT an embed div)", async () => {
|
||||
const html = await markdownToHtml("<!--html-embed:ab$cd-->");
|
||||
// It is NOT tokenized into an embed div the server would strip...
|
||||
expect(html).not.toContain('data-type="htmlEmbed"');
|
||||
// ...it passes through unchanged as a literal HTML comment.
|
||||
expect(html).toContain("<!--html-embed:ab$cd-->");
|
||||
});
|
||||
});
|
||||
@@ -1,2 +0,0 @@
|
||||
export * from "./utils/marked.utils";
|
||||
export * from "./utils/turndown.utils";
|
||||
@@ -1,112 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { markdownToHtml, htmlToMarkdown } from "./index";
|
||||
import {
|
||||
encodeHtmlEmbedSource,
|
||||
decodeHtmlEmbedSource,
|
||||
} from "../html-embed/html-embed";
|
||||
|
||||
// SECURITY (Variant C admin gate, import attack surface).
|
||||
//
|
||||
// The markdown import path is the only write path where an htmlEmbed reaches
|
||||
// the server purely from file bytes (no editor / collab socket). The marked
|
||||
// tokenizer in `html-embed.marked.ts` and the turndown rule in
|
||||
// `turndown.utils.ts` are what materialize the `<!--html-embed:BASE64-->`
|
||||
// marker into the `<div data-type="htmlEmbed" data-source="BASE64">` element
|
||||
// that the server then parses into an htmlEmbed node and the admin gate strips.
|
||||
//
|
||||
// If either the tokenizer regex or the turndown rule shape drifts, the marker
|
||||
// would either (a) stop becoming an htmlEmbed node (silently dropping admin
|
||||
// content) or (b) become some OTHER tag the server's `hasHtmlEmbedNode` no
|
||||
// longer recognizes (a strip bypass). These tests pin the marker <-> embed-div
|
||||
// contract that the server-side strip relies on. editor-ext had ZERO tests
|
||||
// before this file; this adds the runner + the round-trip coverage.
|
||||
|
||||
// The server parses the embed div by matching `data-type="htmlEmbed"` and
|
||||
// decoding `data-source`; mirror that here so the assertion is exactly what the
|
||||
// real `htmlToJson` -> htmlEmbed node parse depends on (the node's parseHTML in
|
||||
// html-embed.ts uses the same selector + decodeHtmlEmbedSource).
|
||||
const EMBED_DIV_RE = /<div[^>]*\bdata-type="htmlEmbed"[^>]*>/;
|
||||
function extractEmbedSource(html: string): string | undefined {
|
||||
const div = EMBED_DIV_RE.exec(html);
|
||||
if (!div) return undefined;
|
||||
const enc = /data-source="([^"]*)"/.exec(div[0]);
|
||||
if (!enc) return undefined;
|
||||
return decodeHtmlEmbedSource(enc[1]);
|
||||
}
|
||||
|
||||
// Replicates the server's `hasHtmlEmbedNode` decision against the embed *div*
|
||||
// (the HTML form the server immediately converts to JSON). If this matches, the
|
||||
// server's JSON-level `hasHtmlEmbedNode` will too, because htmlToJson maps this
|
||||
// exact div to an htmlEmbed node.
|
||||
function htmlHasHtmlEmbed(html: string): boolean {
|
||||
return EMBED_DIV_RE.test(html);
|
||||
}
|
||||
|
||||
describe("markdown <!--html-embed--> import round-trip", () => {
|
||||
const source = "<script>x</script>";
|
||||
|
||||
it("markdownToHtml turns the marker into an htmlEmbed div carrying the source", async () => {
|
||||
const md = "<!--html-embed:" + encodeHtmlEmbedSource(source) + "-->";
|
||||
const html = await markdownToHtml(md);
|
||||
|
||||
// The marker became the embed div the server recognizes as an htmlEmbed
|
||||
// node (so the server's hasHtmlEmbedNode would match it after htmlToJson).
|
||||
expect(htmlHasHtmlEmbed(html)).toBe(true);
|
||||
// The decoded source is the original script, intact.
|
||||
expect(extractEmbedSource(html)).toBe(source);
|
||||
// The raw script is NOT inlined into the HTML — it stays base64 in the
|
||||
// attribute (the marker itself must not be a direct injection vector).
|
||||
expect(html).not.toContain("<script>x</script>");
|
||||
});
|
||||
|
||||
it("preserves UTF-8 / special chars in the embedded source", async () => {
|
||||
const utf8 = '<script>console.log("héllo → 世界")</script>';
|
||||
const md = "<!--html-embed:" + encodeHtmlEmbedSource(utf8) + "-->";
|
||||
const html = await markdownToHtml(md);
|
||||
expect(htmlHasHtmlEmbed(html)).toBe(true);
|
||||
expect(extractEmbedSource(html)).toBe(utf8);
|
||||
});
|
||||
|
||||
it("an empty marker still produces an htmlEmbed div (empty source)", async () => {
|
||||
const html = await markdownToHtml("<!--html-embed:-->");
|
||||
expect(htmlHasHtmlEmbed(html)).toBe(true);
|
||||
expect(extractEmbedSource(html)).toBe("");
|
||||
});
|
||||
|
||||
it("round-trips htmlToMarkdown -> markdownToHtml preserving the embed marker", async () => {
|
||||
const encoded = encodeHtmlEmbedSource(source);
|
||||
// NOTE: turndown drops a *blank* (childless) element before any custom rule
|
||||
// runs, and the htmlEmbed div is normally childless. The export pipeline
|
||||
// therefore must give the rule a non-blank div to fire on; we add an inert
|
||||
// text child here to exercise the real turndown htmlEmbed rule. (A blank
|
||||
// embed div serializing to "" is asserted separately below as a documented
|
||||
// edge so this contract drift is visible.)
|
||||
const startHtml = `<div data-type="htmlEmbed" data-source="${encoded}">x</div>`;
|
||||
|
||||
// Export to markdown: the turndown rule emits the <!--html-embed:..-->
|
||||
// marker (lossless, inert in plain markdown viewers).
|
||||
const md = htmlToMarkdown(startHtml);
|
||||
expect(md).toContain("<!--html-embed:" + encoded + "-->");
|
||||
|
||||
// Re-import: the marker round-trips back into an embed div with the same
|
||||
// decoded source — this is the marker <-> embed-div contract the server's
|
||||
// import strip depends on.
|
||||
const html = await markdownToHtml(md);
|
||||
expect(htmlHasHtmlEmbed(html)).toBe(true);
|
||||
expect(extractEmbedSource(html)).toBe(source);
|
||||
});
|
||||
|
||||
it("documents that a BLANK embed div serializes to empty markdown (turndown drops childless blocks)", () => {
|
||||
const encoded = encodeHtmlEmbedSource(source);
|
||||
const blank = `<div data-type="htmlEmbed" data-source="${encoded}"></div>`;
|
||||
// This pins current behavior so a future change to the turndown rule (e.g.
|
||||
// making it fire on blank nodes) is caught rather than silently shipping.
|
||||
expect(htmlToMarkdown(blank)).toBe("");
|
||||
});
|
||||
|
||||
it("the base64 codec itself round-trips (no '<' leaks into the attribute)", () => {
|
||||
const encoded = encodeHtmlEmbedSource(source);
|
||||
expect(encoded).not.toContain("<");
|
||||
expect(decodeHtmlEmbedSource(encoded)).toBe(source);
|
||||
});
|
||||
});
|
||||
@@ -1,29 +0,0 @@
|
||||
/**
|
||||
* Flexible `basename` implementation for node and the browser
|
||||
* @see https://stackoverflow.com/a/59907288/2228771
|
||||
*/
|
||||
export function getBasename(path: string) {
|
||||
// make sure the basename is not empty, if string ends with separator
|
||||
let end = path.length - 1;
|
||||
while (path[end] === '/' || path[end] === '\\') {
|
||||
--end;
|
||||
}
|
||||
|
||||
// support mixing of Win + Unix path separators
|
||||
const i1 = path.lastIndexOf('/', end);
|
||||
const i2 = path.lastIndexOf('\\', end);
|
||||
|
||||
let start: number;
|
||||
if (i1 === -1) {
|
||||
if (i2 === -1) {
|
||||
// no separator in the whole thing
|
||||
return path;
|
||||
}
|
||||
start = i2;
|
||||
} else if (i2 === -1) {
|
||||
start = i1;
|
||||
} else {
|
||||
start = Math.max(i1, i2);
|
||||
}
|
||||
return path.substring(start + 1, end + 1);
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
/**
|
||||
* Shared pieces for the two callout tokenizers — `callout.marked.ts` (the
|
||||
* `:::type` fenced form) and `github-callout.marked.ts` (the `> [!type]` GitHub
|
||||
* alert form). Both emit the SAME callout node, so the banner type dictionary
|
||||
* and the HTML renderer live here once instead of drifting apart in two files.
|
||||
* The tokenizers themselves stay separate (different syntaxes / source matching).
|
||||
*/
|
||||
|
||||
/** The four callout banner types the editor schema supports. */
|
||||
export const CALLOUT_TYPES = ['info', 'success', 'warning', 'danger'] as const;
|
||||
|
||||
export type CalloutType = (typeof CALLOUT_TYPES)[number];
|
||||
|
||||
/**
|
||||
* Coerce an arbitrary type name onto a supported banner type, defaulting to
|
||||
* `info` for anything unrecognized (the shared fallback both tokenizers use).
|
||||
*/
|
||||
export function normalizeCalloutType(type: string): CalloutType {
|
||||
return (CALLOUT_TYPES as readonly string[]).includes(type)
|
||||
? (type as CalloutType)
|
||||
: 'info';
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a callout node to the editor's HTML shape. `body` is the already
|
||||
* markdown-parsed inner content (marked may hand back a string synchronously).
|
||||
*/
|
||||
export function renderCalloutHtml(
|
||||
type: string,
|
||||
body: string | Promise<string>,
|
||||
): string {
|
||||
return `<div data-type="callout" data-callout-type="${type}">${body}</div>`;
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import { Token, marked } from 'marked';
|
||||
import { normalizeCalloutType, renderCalloutHtml } from './callout-common.marked';
|
||||
|
||||
interface CalloutToken {
|
||||
type: 'callout';
|
||||
calloutType: string;
|
||||
text: string;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
export const calloutExtension = {
|
||||
name: 'callout',
|
||||
level: 'block',
|
||||
start(src: string) {
|
||||
return src.match(/:::/)?.index ?? -1;
|
||||
},
|
||||
tokenizer(src: string): CalloutToken | undefined {
|
||||
const rule = /^:::([a-zA-Z0-9]+)\s+([\s\S]+?):::/;
|
||||
const match = rule.exec(src);
|
||||
|
||||
if (match) {
|
||||
return {
|
||||
type: 'callout',
|
||||
calloutType: normalizeCalloutType(match[1]),
|
||||
raw: match[0],
|
||||
text: match[2].trim(),
|
||||
};
|
||||
}
|
||||
},
|
||||
renderer(token: Token) {
|
||||
const calloutToken = token as CalloutToken;
|
||||
return renderCalloutHtml(
|
||||
calloutToken.calloutType,
|
||||
marked.parse(calloutToken.text),
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -1,72 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { extractFootnoteDefinitions } from "./footnote.marked";
|
||||
|
||||
/** Pull the ordered list of `data-footnote-def` ids out of the rendered section. */
|
||||
function defIds(section: string): string[] {
|
||||
return [...section.matchAll(/data-footnote-def data-id="([^"]+)"/g)].map(
|
||||
(m) => m[1],
|
||||
);
|
||||
}
|
||||
|
||||
/** Pull the ordered list of `[^id]` markers that remain in the body. */
|
||||
function bodyMarkers(body: string): string[] {
|
||||
return [...body.matchAll(/\[\^([^\]\s]+)\]/g)].map((m) => m[1]);
|
||||
}
|
||||
|
||||
describe("extractFootnoteDefinitions: duplicate definition ids (first-wins)", () => {
|
||||
// Body has ONE `[^d]` reference but THREE `[^d]:` definitions. Under the
|
||||
// import model (#166) a duplicate definition id is FIRST-WINS: only the first
|
||||
// definition is kept; the rest are DROPPED (and surfaced by analyzeFootnotes,
|
||||
// not silently re-id'd into orphan footnotes as before). Reference markers are
|
||||
// never rewritten, so repeated references would reuse the single footnote.
|
||||
const md = ["See[^d].", "", "[^d]: a", "[^d]: b", "[^d]: c"].join("\n");
|
||||
|
||||
it("keeps only the FIRST definition for the id (first-wins)", () => {
|
||||
const { section } = extractFootnoteDefinitions(md);
|
||||
const ids = defIds(section);
|
||||
expect(ids).toEqual(["d"]);
|
||||
});
|
||||
|
||||
it("keeps the first definition's text and drops the duplicates", () => {
|
||||
const { section } = extractFootnoteDefinitions(md);
|
||||
expect(section).toContain('data-footnote-def data-id="d"><p>a</p>');
|
||||
// No derived `d__2` / `d__3` ids are emitted anymore.
|
||||
expect(section).not.toContain("d__2");
|
||||
expect(section).not.toContain("d__3");
|
||||
// The dropped duplicate texts are not in the section.
|
||||
expect(section).not.toContain("<p>b</p>");
|
||||
expect(section).not.toContain("<p>c</p>");
|
||||
});
|
||||
|
||||
it("leaves the SINGLE body marker as [^d] (markers are never rewritten)", () => {
|
||||
const { body } = extractFootnoteDefinitions(md);
|
||||
expect(bodyMarkers(body)).toEqual(["d"]);
|
||||
expect(body).toContain("See[^d].");
|
||||
// The definition lines themselves were pulled OUT of the body.
|
||||
expect(body).not.toContain("[^d]: a");
|
||||
expect(body).not.toContain("[^d]: b");
|
||||
expect(body).not.toContain("[^d]: c");
|
||||
});
|
||||
|
||||
it("does not crash and produces a well-formed footnotes section", () => {
|
||||
const { section } = extractFootnoteDefinitions(md);
|
||||
expect(section.startsWith("<section data-footnotes>")).toBe(true);
|
||||
expect(section.endsWith("</section>")).toBe(true);
|
||||
// Exactly one definition div (first-wins).
|
||||
expect([...section.matchAll(/<div data-footnote-def/g)]).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractFootnoteDefinitions: reuse (repeated references, one definition)", () => {
|
||||
// Pandoc semantics: many `[^a]` references + one `[^a]:` definition = one
|
||||
// footnote, shared. Markers are left intact so the editor numbers them as one.
|
||||
const md = ["A[^a] B[^a] C[^a].", "", "[^a]: shared note"].join("\n");
|
||||
|
||||
it("emits exactly one definition and leaves every reference marker as [^a]", () => {
|
||||
const { section, body } = extractFootnoteDefinitions(md);
|
||||
expect(defIds(section)).toEqual(["a"]);
|
||||
expect(section).toContain('data-footnote-def data-id="a"><p>shared note</p>');
|
||||
// All three reference markers stay `a` (no `a__2`/`a__3` minting).
|
||||
expect(bodyMarkers(body)).toEqual(["a", "a", "a"]);
|
||||
});
|
||||
});
|
||||
@@ -1,131 +0,0 @@
|
||||
import { marked } from "marked";
|
||||
|
||||
/**
|
||||
* Pandoc/GFM footnote support for the marked (Markdown -> HTML) pipeline.
|
||||
*
|
||||
* Two pieces:
|
||||
* - an INLINE tokenizer for `[^id]` references -> <sup data-footnote-ref
|
||||
* data-id="id"> (matches the editor-ext FootnoteReference renderHTML);
|
||||
* - a document hook (`preprocess`/`walkTokens` is awkward for collecting +
|
||||
* removing definitions, so we use a regex preprocessing step instead) that
|
||||
* pulls every `[^id]: text` definition line out of the body and appends a
|
||||
* single <section data-footnotes> with one <div data-footnote-def> per
|
||||
* definition, so the round-trip rebuilds footnotesList + footnoteDefinition.
|
||||
*
|
||||
* Every FIRST definition line is emitted — duplicate ids are first-wins (the
|
||||
* rest are dropped, and surfaced via analyzeFootnotes), and reference markers are
|
||||
* left untouched so repeated `[^a]` references reuse the one footnote (#166).
|
||||
* Orphan definitions (no matching reference) are still emitted here; the editor's
|
||||
* sync plugin reconciles the final reference/definition set (drops orphans,
|
||||
* synthesizes a single empty definition for a reference that lacks one).
|
||||
*/
|
||||
|
||||
const DEFINITION_RE = /^\[\^([^\]\s]+)\]:[ \t]*(.*)$/;
|
||||
const REFERENCE_RE = /\[\^([^\]\s]+)\]/;
|
||||
|
||||
interface FootnoteRefToken {
|
||||
type: "footnoteRef";
|
||||
raw: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
export const footnoteReferenceExtension = {
|
||||
name: "footnoteRef",
|
||||
level: "inline" as const,
|
||||
start(src: string) {
|
||||
return src.match(/\[\^/)?.index ?? -1;
|
||||
},
|
||||
tokenizer(src: string): FootnoteRefToken | undefined {
|
||||
const match = REFERENCE_RE.exec(src);
|
||||
// Only match at the very start of the remaining inline source.
|
||||
if (match && match.index === 0) {
|
||||
return {
|
||||
type: "footnoteRef",
|
||||
raw: match[0],
|
||||
id: match[1],
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
renderer(token: FootnoteRefToken) {
|
||||
return `<sup data-footnote-ref data-id="${escapeAttr(token.id)}"></sup>`;
|
||||
},
|
||||
};
|
||||
|
||||
function escapeAttr(value: string): string {
|
||||
return String(value).replace(/&/g, "&").replace(/"/g, """);
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract `[^id]: text` definition lines from the markdown body, returning the
|
||||
* cleaned body plus a rendered <section data-footnotes> (empty string when no
|
||||
* definitions). Call this BEFORE marked.parse and append the section to the
|
||||
* resulting HTML.
|
||||
*/
|
||||
export function extractFootnoteDefinitions(markdown: string): {
|
||||
body: string;
|
||||
section: string;
|
||||
} {
|
||||
const lines = markdown.split("\n");
|
||||
const bodyLines: string[] = [];
|
||||
const definitions: Array<{ id: string; text: string }> = [];
|
||||
|
||||
// Track fenced-code state so a `[^id]: ...` line that merely SHOWS footnote
|
||||
// syntax inside a ``` / ~~~ code block is left in the body verbatim and not
|
||||
// mistaken for a real definition.
|
||||
let fence: string | null = null;
|
||||
|
||||
for (const line of lines) {
|
||||
const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line);
|
||||
if (fenceMatch) {
|
||||
const marker = fenceMatch[2][0];
|
||||
if (fence === null) {
|
||||
fence = marker; // opening fence
|
||||
} else if (marker === fence) {
|
||||
fence = null; // closing fence (matching delimiter type)
|
||||
}
|
||||
bodyLines.push(line);
|
||||
continue;
|
||||
}
|
||||
|
||||
const m = fence === null ? DEFINITION_RE.exec(line) : null;
|
||||
if (m) {
|
||||
definitions.push({ id: m[1], text: m[2] });
|
||||
} else {
|
||||
bodyLines.push(line);
|
||||
}
|
||||
}
|
||||
|
||||
if (definitions.length === 0) {
|
||||
return { body: markdown, section: "" };
|
||||
}
|
||||
|
||||
// Duplicate definition ids (e.g. `[^d]: first` / `[^d]: second`): FIRST WINS,
|
||||
// the rest are DROPPED. Reference markers are left UNTOUCHED so repeated `[^a]`
|
||||
// references reuse the single footnote (Pandoc semantics, #166). This differs
|
||||
// from the live editor's never-lose policy (resolveCollisions re-ids a
|
||||
// duplicate definition into an orphan) on purpose: an import is an
|
||||
// agent-authored artifact we sanitize, and the dropped duplicate is surfaced
|
||||
// to the caller via analyzeFootnotes' `duplicateDefinitions` warning instead.
|
||||
const firstById = new Map<string, string>(); // id -> first definition text
|
||||
for (const def of definitions) {
|
||||
if (!firstById.has(def.id)) firstById.set(def.id, def.text);
|
||||
}
|
||||
|
||||
const defsHtml = [...firstById.entries()]
|
||||
.map(([id, text]) => {
|
||||
// Render the definition text as inline markdown so emphasis/links inside
|
||||
// a footnote survive the round-trip; wrap in a paragraph (the node's
|
||||
// content is paragraph+).
|
||||
const inner = marked.parseInline(text || "");
|
||||
return `<div data-footnote-def data-id="${escapeAttr(
|
||||
id,
|
||||
)}"><p>${inner}</p></div>`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
return {
|
||||
body: bodyLines.join("\n"),
|
||||
section: `<section data-footnotes>${defsHtml}</section>`,
|
||||
};
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { markdownToHtml } from "./marked.utils";
|
||||
|
||||
/**
|
||||
* Regression for issue #192: pasting a GitHub-style `> [!type]` alert produced a
|
||||
* literal `<blockquote>` containing `[!info]` instead of a callout node, because
|
||||
* only the `:::type` form was tokenized. The editor paste path runs the same
|
||||
* `markdownToHtml`, so these assertions pin the conversion at the source.
|
||||
*/
|
||||
function html(md: string): string {
|
||||
const out = markdownToHtml(md);
|
||||
if (typeof out !== "string") throw new Error("expected sync string output");
|
||||
return out;
|
||||
}
|
||||
|
||||
describe("markdownToHtml: GitHub `> [!type]` callouts", () => {
|
||||
it("converts `> [!info]` to a callout node, not a literal blockquote", () => {
|
||||
const out = html("> [!info]\n> Callout body text here");
|
||||
expect(out).toContain('data-type="callout"');
|
||||
expect(out).toContain('data-callout-type="info"');
|
||||
expect(out).toContain("Callout body text here");
|
||||
expect(out).not.toContain("[!info]");
|
||||
expect(out).not.toContain("<blockquote");
|
||||
});
|
||||
|
||||
it("maps GitHub alert aliases onto the supported banner types", () => {
|
||||
expect(html("> [!NOTE]\n> x")).toContain('data-callout-type="info"');
|
||||
expect(html("> [!TIP]\n> x")).toContain('data-callout-type="success"');
|
||||
expect(html("> [!WARNING]\n> x")).toContain('data-callout-type="warning"');
|
||||
expect(html("> [!CAUTION]\n> x")).toContain('data-callout-type="danger"');
|
||||
});
|
||||
|
||||
it("accepts the editor's own type names directly", () => {
|
||||
expect(html("> [!success]\n> x")).toContain('data-callout-type="success"');
|
||||
expect(html("> [!danger]\n> x")).toContain('data-callout-type="danger"');
|
||||
});
|
||||
|
||||
it("falls back to info for an unknown type", () => {
|
||||
expect(html("> [!bogus]\n> x")).toContain('data-callout-type="info"');
|
||||
});
|
||||
|
||||
it("preserves multi-line callout bodies", () => {
|
||||
const out = html("> [!warning]\n> line one\n> line two");
|
||||
expect(out).toContain('data-callout-type="warning"');
|
||||
expect(out).toContain("line one");
|
||||
expect(out).toContain("line two");
|
||||
});
|
||||
|
||||
it("still converts the `:::type` form", () => {
|
||||
const out = html(":::info\nbody\n:::");
|
||||
expect(out).toContain('data-type="callout"');
|
||||
expect(out).toContain('data-callout-type="info"');
|
||||
});
|
||||
});
|
||||
@@ -1,81 +0,0 @@
|
||||
import { Token, marked } from 'marked';
|
||||
import { renderCalloutHtml } from './callout-common.marked';
|
||||
|
||||
interface GithubCalloutToken {
|
||||
type: 'githubCallout';
|
||||
calloutType: string;
|
||||
text: string;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map GitHub "alert" blockquote markers (`> [!NOTE]`, `> [!WARNING]`, …) onto
|
||||
* the four callout banner types the editor schema supports. The editor's own
|
||||
* type names (`info`/`success`/`warning`/`danger`) are also accepted directly,
|
||||
* because users paste both forms. Anything unrecognized falls back to `info`,
|
||||
* matching the `:::type` callout tokenizer.
|
||||
*/
|
||||
const GITHUB_ALERT_TYPE_MAP: Record<string, string> = {
|
||||
note: 'info',
|
||||
tip: 'success',
|
||||
important: 'info',
|
||||
warning: 'warning',
|
||||
caution: 'danger',
|
||||
info: 'info',
|
||||
success: 'success',
|
||||
danger: 'danger',
|
||||
};
|
||||
|
||||
/**
|
||||
* Tokenizer for GitHub-flavored alert callouts written as a blockquote whose
|
||||
* first line is `[!type]`:
|
||||
*
|
||||
* > [!info]
|
||||
* > body line one
|
||||
* > body line two
|
||||
*
|
||||
* Without this, the default blockquote tokenizer wins and the marker renders as
|
||||
* a literal `[!info]` inside a `<blockquote>`. The editor's paste path runs the
|
||||
* same `markdownToHtml`, so registering this here also fixes pasting the syntax
|
||||
* into the editor (issue #192), not just markdown import.
|
||||
*/
|
||||
export const githubCalloutExtension = {
|
||||
name: 'githubCallout',
|
||||
level: 'block' as const,
|
||||
start(src: string) {
|
||||
return src.match(/^ {0,3}>[ \t]*\[!/m)?.index ?? -1;
|
||||
},
|
||||
tokenizer(src: string): GithubCalloutToken | undefined {
|
||||
const rule =
|
||||
/^ {0,3}>[ \t]*\[!([a-zA-Z]+)\][^\n]*(?:\n {0,3}>[^\n]*)*(?:\n|$)/;
|
||||
const match = rule.exec(src);
|
||||
if (!match) return undefined;
|
||||
|
||||
const rawType = match[1].toLowerCase();
|
||||
const calloutType = GITHUB_ALERT_TYPE_MAP[rawType] ?? 'info';
|
||||
|
||||
const text = match[0]
|
||||
.replace(/\n+$/, '')
|
||||
.split('\n')
|
||||
// Strip the blockquote marker (`>` + optional space) from every line.
|
||||
.map((line) => line.replace(/^ {0,3}>[ \t]?/, ''))
|
||||
// Drop the `[!type]` marker that opens the first line.
|
||||
.map((line, i) => (i === 0 ? line.replace(/^\[![a-zA-Z]+\][ \t]*/, '') : line))
|
||||
.join('\n')
|
||||
.trim();
|
||||
|
||||
return {
|
||||
type: 'githubCallout',
|
||||
calloutType,
|
||||
raw: match[0],
|
||||
text,
|
||||
};
|
||||
},
|
||||
renderer(token: Token) {
|
||||
const calloutToken = token as GithubCalloutToken;
|
||||
return renderCalloutHtml(
|
||||
calloutToken.calloutType,
|
||||
marked.parse(calloutToken.text),
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -1,41 +0,0 @@
|
||||
import { Token } from "marked";
|
||||
|
||||
interface HtmlEmbedToken {
|
||||
type: "htmlEmbed";
|
||||
raw: string;
|
||||
encoded: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Marked extension that rebuilds an `htmlEmbed` node from the HTML comment
|
||||
* marker produced by the turndown rule (`<!--html-embed:<base64>-->`).
|
||||
*
|
||||
* It emits the same marker div the node's `parseHTML` recognizes, so the
|
||||
* pipeline MD -> HTML -> ProseMirror JSON restores the node (and its
|
||||
* base64 `data-source`) exactly. We do NOT expand the raw markup here; the
|
||||
* source stays base64-encoded in the attribute and is only executed by the
|
||||
* client NodeView.
|
||||
*/
|
||||
export const htmlEmbedExtension = {
|
||||
name: "htmlEmbed",
|
||||
level: "block" as const,
|
||||
start(src: string) {
|
||||
return src.indexOf("<!--html-embed:");
|
||||
},
|
||||
tokenizer(src: string): HtmlEmbedToken | undefined {
|
||||
const rule = /^<!--html-embed:([A-Za-z0-9+/=]*)-->/;
|
||||
const match = rule.exec(src);
|
||||
|
||||
if (match) {
|
||||
return {
|
||||
type: "htmlEmbed",
|
||||
raw: match[0],
|
||||
encoded: match[1] ?? "",
|
||||
};
|
||||
}
|
||||
},
|
||||
renderer(token: Token) {
|
||||
const htmlEmbedToken = token as HtmlEmbedToken;
|
||||
return `<div data-type="htmlEmbed" data-source="${htmlEmbedToken.encoded}"></div>`;
|
||||
},
|
||||
};
|
||||
@@ -1,76 +0,0 @@
|
||||
import { marked } from "marked";
|
||||
import { calloutExtension } from "./callout.marked";
|
||||
import { githubCalloutExtension } from "./github-callout.marked";
|
||||
import { mathBlockExtension } from "./math-block.marked";
|
||||
import { mathInlineExtension } from "./math-inline.marked";
|
||||
import {
|
||||
footnoteReferenceExtension,
|
||||
extractFootnoteDefinitions,
|
||||
} from "./footnote.marked";
|
||||
import { htmlEmbedExtension } from "./html-embed.marked";
|
||||
|
||||
marked.use({
|
||||
renderer: {
|
||||
list({ ordered, start, items }) {
|
||||
let body = "";
|
||||
for (const item of items) {
|
||||
body += this.listitem(item);
|
||||
}
|
||||
|
||||
if (ordered) {
|
||||
const startAttr = start !== 1 ? ` start="${start}"` : "";
|
||||
return `<ol${startAttr}>\n${body}</ol>\n`;
|
||||
}
|
||||
|
||||
const isTaskList = items.some((item) => item.task);
|
||||
const dataType = isTaskList ? ' data-type="taskList"' : "";
|
||||
return `<ul${dataType}>\n${body}</ul>\n`;
|
||||
},
|
||||
listitem({ tokens, task: isTask, checked: isChecked }) {
|
||||
const text = this.parser.parse(tokens);
|
||||
if (!isTask) {
|
||||
return `<li>${text}</li>\n`;
|
||||
}
|
||||
const checkedAttr = isChecked
|
||||
? 'data-checked="true"'
|
||||
: 'data-checked="false"';
|
||||
return `<li data-type="taskItem" ${checkedAttr}>${text}</li>\n`;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
marked.use({
|
||||
extensions: [
|
||||
calloutExtension,
|
||||
githubCalloutExtension,
|
||||
mathBlockExtension,
|
||||
mathInlineExtension,
|
||||
footnoteReferenceExtension,
|
||||
htmlEmbedExtension,
|
||||
],
|
||||
});
|
||||
|
||||
marked.setOptions({ breaks: true });
|
||||
|
||||
export function markdownToHtml(
|
||||
markdownInput: string,
|
||||
): string | Promise<string> {
|
||||
const YAML_FONT_MATTER_REGEX = /^\s*---[\s\S]*?---\s*/;
|
||||
|
||||
const markdown = markdownInput
|
||||
.replace(YAML_FONT_MATTER_REGEX, "")
|
||||
.trimStart();
|
||||
|
||||
// Pull `[^id]: ...` definition lines out of the body, render the body, then
|
||||
// append a single <section data-footnotes> so the round-trip rebuilds the
|
||||
// footnotesList + footnoteDefinition nodes.
|
||||
const { body, section } = extractFootnoteDefinitions(markdown);
|
||||
|
||||
const parsed = marked.parse(body);
|
||||
if (!section) return parsed;
|
||||
|
||||
if (typeof parsed === "string") {
|
||||
return parsed + section;
|
||||
}
|
||||
return parsed.then((html) => html + section);
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import { Token, marked } from 'marked';
|
||||
|
||||
interface MathBlockToken {
|
||||
type: 'mathBlock';
|
||||
text: string;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
export const mathBlockExtension = {
|
||||
name: 'mathBlock',
|
||||
level: 'block',
|
||||
start(src: string) {
|
||||
return src.match(/\$\$/)?.index ?? -1;
|
||||
},
|
||||
tokenizer(src: string): MathBlockToken | undefined {
|
||||
const rule = /^\$\$(?!(\$))([\s\S]+?)\$\$/;
|
||||
const match = rule.exec(src);
|
||||
|
||||
if (match) {
|
||||
return {
|
||||
type: 'mathBlock',
|
||||
raw: match[0],
|
||||
text: match[2]?.trim(),
|
||||
};
|
||||
}
|
||||
},
|
||||
renderer(token: Token) {
|
||||
const mathBlockToken = token as MathBlockToken;
|
||||
// parse to prevent escaping slashes
|
||||
const latex = marked
|
||||
.parse(mathBlockToken.text)
|
||||
.toString()
|
||||
.replace(/<(\/)?p>/g, '');
|
||||
|
||||
return `<div data-type="${mathBlockToken.type}" data-katex="true">${latex}</div>`;
|
||||
},
|
||||
};
|
||||
@@ -1,50 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { markdownToHtml } from "./marked.utils";
|
||||
|
||||
/**
|
||||
* Data-integrity regression (issue #204, Phase 2): plain prose that mentions
|
||||
* prices like `$5 and $6` must NOT be misread as inline math. The inline-math
|
||||
* tokenizer mutates a global `marked` singleton at import time
|
||||
* (`marked.utils.ts`), so math behaviour can only be exercised safely through
|
||||
* the public `markdownToHtml`; importing the tokenizer in isolation would give
|
||||
* a different, non-representative result. These assertions therefore drive the
|
||||
* real conversion path.
|
||||
*/
|
||||
function html(md: string): string {
|
||||
const out = markdownToHtml(md);
|
||||
if (typeof out !== "string") throw new Error("expected sync string output");
|
||||
return out;
|
||||
}
|
||||
|
||||
const MATH_MARKERS = ['data-type="mathInline"', 'data-katex="true"'];
|
||||
|
||||
function hasInlineMath(out: string): boolean {
|
||||
return MATH_MARKERS.some((m) => out.includes(m));
|
||||
}
|
||||
|
||||
describe("markdownToHtml: inline-math false positives", () => {
|
||||
it("does not treat prices `$5 and $6` as inline math", () => {
|
||||
const out = html("It costs $5 and $6 today.");
|
||||
expect(hasInlineMath(out)).toBe(false);
|
||||
// The text survives verbatim (no katex span swallowing it).
|
||||
expect(out).toContain("$5 and $6");
|
||||
});
|
||||
|
||||
it("does not treat a single trailing price `$5` as inline math", () => {
|
||||
const out = html("Lunch was $5.");
|
||||
expect(hasInlineMath(out)).toBe(false);
|
||||
expect(out).toContain("$5");
|
||||
});
|
||||
|
||||
it("does not treat `$5, $6, $7` (multiple prices) as inline math", () => {
|
||||
const out = html("Choose $5, $6, $7 plans.");
|
||||
expect(hasInlineMath(out)).toBe(false);
|
||||
});
|
||||
|
||||
it("STILL converts a genuine inline-math expression `$x + y$`", () => {
|
||||
// Guard the positive path so the false-positive guard above can't be
|
||||
// satisfied by simply disabling math entirely.
|
||||
const out = html("The sum $x + y$ is shown.");
|
||||
expect(hasInlineMath(out)).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,55 +0,0 @@
|
||||
import { Token, marked } from 'marked';
|
||||
|
||||
interface MathInlineToken {
|
||||
type: 'mathInline';
|
||||
text: string;
|
||||
raw: string;
|
||||
}
|
||||
|
||||
const inlineMathRegex = /^\$(?!\s)(.+?)(?<!\s)\$(?!\d)/;
|
||||
|
||||
export const mathInlineExtension = {
|
||||
name: 'mathInline',
|
||||
level: 'inline',
|
||||
start(src: string) {
|
||||
let index: number;
|
||||
let indexSrc = src;
|
||||
|
||||
while (indexSrc) {
|
||||
index = indexSrc.indexOf('$');
|
||||
if (index === -1) {
|
||||
return;
|
||||
}
|
||||
const f = index === 0 || indexSrc.charAt(index - 1) === ' ';
|
||||
if (f) {
|
||||
const possibleKatex = indexSrc.substring(index);
|
||||
if (possibleKatex.match(inlineMathRegex)) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
indexSrc = indexSrc.substring(index + 1).replace(/^\$+/, '');
|
||||
}
|
||||
},
|
||||
tokenizer(src: string): MathInlineToken | undefined {
|
||||
const match = inlineMathRegex.exec(src);
|
||||
|
||||
if (match) {
|
||||
return {
|
||||
type: 'mathInline',
|
||||
raw: match[0],
|
||||
text: match[1]?.trim(),
|
||||
};
|
||||
}
|
||||
},
|
||||
renderer(token: Token) {
|
||||
const mathInlineToken = token as MathInlineToken;
|
||||
// parse to prevent escaping slashes
|
||||
const latex = marked
|
||||
.parse(mathInlineToken.text)
|
||||
.toString()
|
||||
.replace(/<(\/)?p>/g, '');
|
||||
|
||||
return `<span data-type="${mathInlineToken.type}" data-katex="true">${latex}</span>`;
|
||||
},
|
||||
};
|
||||
@@ -1,128 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { getSchema } from "@tiptap/core";
|
||||
import { generateHTML, generateJSON } from "@tiptap/html";
|
||||
import { Document } from "@tiptap/extension-document";
|
||||
import { Paragraph } from "@tiptap/extension-paragraph";
|
||||
import { Text } from "@tiptap/extension-text";
|
||||
import { Bold } from "@tiptap/extension-bold";
|
||||
import { htmlToMarkdown } from "./turndown.utils";
|
||||
import { markdownToHtml } from "./marked.utils";
|
||||
import { Spoiler } from "../../spoiler/spoiler";
|
||||
|
||||
// The spoiler mark has no native Markdown syntax, so it is preserved losslessly
|
||||
// as raw inline HTML (`<span data-spoiler="true">…</span>`), the same approach
|
||||
// htmlEmbed uses. This test drives the full editor round-trip:
|
||||
// JSON -> HTML -> Markdown -> HTML -> JSON
|
||||
// and asserts the `spoiler` mark survives end to end. We use the same
|
||||
// getSchema + @tiptap/html generateHTML/generateJSON utilities the other
|
||||
// editor-ext schema tests use.
|
||||
|
||||
const extensions = [Document, Paragraph, Text, Bold, Spoiler];
|
||||
|
||||
function html(md: string): string {
|
||||
const out = markdownToHtml(md);
|
||||
if (typeof out !== "string") throw new Error("expected sync string output");
|
||||
return out;
|
||||
}
|
||||
|
||||
// Count text nodes carrying a `spoiler` mark anywhere in a ProseMirror JSON doc.
|
||||
function countSpoilerMarks(doc: any): number {
|
||||
let count = 0;
|
||||
const walk = (node: any) => {
|
||||
if (!node || typeof node !== "object") return;
|
||||
if (Array.isArray(node.marks)) {
|
||||
for (const mark of node.marks) {
|
||||
if (mark?.type === "spoiler") count++;
|
||||
}
|
||||
}
|
||||
if (Array.isArray(node.content)) node.content.forEach(walk);
|
||||
};
|
||||
walk(doc);
|
||||
return count;
|
||||
}
|
||||
|
||||
describe("Spoiler mark schema", () => {
|
||||
it("registers the spoiler mark in the schema", () => {
|
||||
const schema = getSchema(extensions);
|
||||
expect(schema.marks.spoiler).toBeTruthy();
|
||||
});
|
||||
|
||||
it("recovers the spoiler mark from span[data-spoiler] (HTML -> JSON)", () => {
|
||||
const json = generateJSON(
|
||||
'<p>before <span data-spoiler="true">hidden</span> after</p>',
|
||||
extensions,
|
||||
);
|
||||
expect(countSpoilerMarks(json)).toBe(1);
|
||||
});
|
||||
|
||||
it("emits data-spoiler + class on render (JSON -> HTML)", () => {
|
||||
const doc = {
|
||||
type: "doc",
|
||||
content: [
|
||||
{
|
||||
type: "paragraph",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: "hidden",
|
||||
marks: [{ type: "spoiler" }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
const out = generateHTML(doc, extensions);
|
||||
expect(out).toContain('data-spoiler="true"');
|
||||
expect(out).toContain('class="spoiler"');
|
||||
});
|
||||
});
|
||||
|
||||
describe("Spoiler Markdown round-trip is lossless", () => {
|
||||
const docWith = (textNode: any) => ({
|
||||
type: "doc",
|
||||
content: [
|
||||
{
|
||||
type: "paragraph",
|
||||
content: [{ type: "text", text: "before " }, textNode, { type: "text", text: " after" }],
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
it("preserves the spoiler mark through JSON -> MD -> HTML -> JSON", () => {
|
||||
const startDoc = docWith({
|
||||
type: "text",
|
||||
text: "hidden",
|
||||
marks: [{ type: "spoiler" }],
|
||||
});
|
||||
|
||||
// JSON -> HTML
|
||||
const html1 = generateHTML(startDoc, extensions);
|
||||
expect(html1).toContain('data-spoiler="true"');
|
||||
|
||||
// HTML -> Markdown (raw inline HTML, lossless)
|
||||
const md = htmlToMarkdown(html1);
|
||||
expect(md).toContain('<span data-spoiler="true">hidden</span>');
|
||||
|
||||
// MD -> HTML -> JSON (mark restored via parseHTML)
|
||||
const endJson = generateJSON(html(md), extensions);
|
||||
expect(countSpoilerMarks(endJson)).toBe(1);
|
||||
// The visible text survives.
|
||||
expect(JSON.stringify(endJson)).toContain("hidden");
|
||||
});
|
||||
|
||||
it("keeps the spoiler intact when it intersects a bold mark", () => {
|
||||
const startDoc = docWith({
|
||||
type: "text",
|
||||
text: "secret",
|
||||
marks: [{ type: "bold" }, { type: "spoiler" }],
|
||||
});
|
||||
|
||||
const md = htmlToMarkdown(generateHTML(startDoc, extensions));
|
||||
expect(md).toContain("data-spoiler=\"true\"");
|
||||
|
||||
const endJson = generateJSON(html(md), extensions);
|
||||
expect(countSpoilerMarks(endJson)).toBe(1);
|
||||
// Bold survives alongside the spoiler.
|
||||
expect(JSON.stringify(endJson)).toContain('"bold"');
|
||||
});
|
||||
});
|
||||
@@ -1,12 +0,0 @@
|
||||
// Map @joplin/turndown types to @types/turndown
|
||||
declare module "@joplin/turndown" {
|
||||
import TurndownService from "turndown";
|
||||
export = TurndownService;
|
||||
}
|
||||
|
||||
declare module "@joplin/turndown-plugin-gfm" {
|
||||
import TurndownService from "turndown";
|
||||
export const tables: TurndownService.Plugin;
|
||||
export const strikethrough: TurndownService.Plugin;
|
||||
export const highlightedCodeBlock: TurndownService.Plugin;
|
||||
}
|
||||
@@ -1,147 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { htmlToMarkdown } from "./turndown.utils";
|
||||
import { markdownToHtml } from "./marked.utils";
|
||||
|
||||
/**
|
||||
* #206 mdrt-2 — Markdown export must never SILENTLY drop a block. (FIXED)
|
||||
*
|
||||
* `htmlToMarkdown` (turndown) historically only registered rules for a fixed
|
||||
* set of custom nodes (callout, taskItem, details, math, iframe, htmlEmbed,
|
||||
* image, video, footnote). Any other custom node — `transclusionReference`,
|
||||
* `pageBreak`, `mention`, `status` — fell through to turndown's default
|
||||
* handling: an empty wrapper is "blank" and removed, so the block disappeared
|
||||
* from the exported Markdown with no trace, and `mention`/`status` collapsed to
|
||||
* bare text, losing their identity (data-id / data-color). The invariant
|
||||
* "never silently lose a block" was broken.
|
||||
*
|
||||
* The fix adds lossless turndown rules that re-emit each of these nodes as raw
|
||||
* HTML carrying every `data-*` attribute. Plain-Markdown viewers ignore the
|
||||
* inert tag; the import path round-trips it (`markdownToHtml` passes the raw
|
||||
* HTML through and each node's `parseHTML` rebuilds the ProseMirror node). These
|
||||
* tests assert the surviving contract (the block is preserved AND its identity
|
||||
* round-trips back through import).
|
||||
*/
|
||||
describe("htmlToMarkdown — custom nodes are preserved losslessly (#206 mdrt-2)", () => {
|
||||
const wrap = (inner: string) => `<p>before</p>${inner}<p>after</p>`;
|
||||
|
||||
it("preserves a pageBreak block on Markdown export", () => {
|
||||
const md = htmlToMarkdown(
|
||||
wrap('<div data-type="pageBreak" class="page-break"></div>'),
|
||||
);
|
||||
expect(md).toContain("before");
|
||||
expect(md).toContain("after");
|
||||
// The break survives as an inert raw-HTML tag, not silently dropped.
|
||||
expect(md).toMatch(/data-type="pageBreak"/);
|
||||
expect(md).toMatch(/page-?break/i);
|
||||
});
|
||||
|
||||
it("preserves a transclusionReference's identity on Markdown export", () => {
|
||||
const md = htmlToMarkdown(
|
||||
wrap('<div data-type="transclusionReference" data-id="abc"></div>'),
|
||||
);
|
||||
expect(md).toContain("before");
|
||||
expect(md).toContain("after");
|
||||
// The data-id (the only thing that gives the reference identity) survives.
|
||||
expect(md).toContain("abc");
|
||||
expect(md).toMatch(/data-type="transclusionReference"/);
|
||||
});
|
||||
|
||||
it("preserves a mention's data-id (stable identity) on Markdown export", () => {
|
||||
const md = htmlToMarkdown(
|
||||
'<p>hi <span data-type="mention" data-id="u1" data-label="Bob">@Bob</span> there</p>',
|
||||
);
|
||||
// The mention keeps its stable identity (data-id), not just the text.
|
||||
expect(md).toContain("u1");
|
||||
expect(md).toContain("Bob");
|
||||
expect(md).toMatch(/data-type="mention"/);
|
||||
});
|
||||
|
||||
it("preserves a status chip's color on Markdown export", () => {
|
||||
const md = htmlToMarkdown(
|
||||
'<p>s <span data-type="status" data-color="green">Done</span></p>',
|
||||
);
|
||||
// The chip's color (its identity) survives, not just the visible text.
|
||||
expect(md).toContain("green");
|
||||
expect(md).toContain("Done");
|
||||
expect(md).toMatch(/data-type="status"/);
|
||||
});
|
||||
|
||||
// The export form is only lossless if the import path can rebuild it. These
|
||||
// assert the full MD -> HTML round-trip restores the node + its attributes,
|
||||
// which is the marker <-> node contract each `parseHTML` relies on.
|
||||
describe("import round-trip (markdownToHtml restores the node)", () => {
|
||||
it("round-trips a pageBreak through export + import", async () => {
|
||||
const md = htmlToMarkdown(
|
||||
wrap('<div data-type="pageBreak" class="page-break"></div>'),
|
||||
);
|
||||
const html = await markdownToHtml(md);
|
||||
expect(html).toMatch(/<div[^>]*data-type="pageBreak"[^>]*>/);
|
||||
expect(html).toContain("before");
|
||||
expect(html).toContain("after");
|
||||
});
|
||||
|
||||
it("round-trips a transclusionReference (keeps data-id)", async () => {
|
||||
const md = htmlToMarkdown(
|
||||
wrap('<div data-type="transclusionReference" data-id="abc"></div>'),
|
||||
);
|
||||
const html = await markdownToHtml(md);
|
||||
expect(html).toMatch(/<div[^>]*data-type="transclusionReference"[^>]*>/);
|
||||
expect(html).toContain("abc");
|
||||
});
|
||||
|
||||
it("round-trips a mention (keeps data-id + data-label)", async () => {
|
||||
const md = htmlToMarkdown(
|
||||
'<p>hi <span data-type="mention" data-id="u1" data-label="Bob">@Bob</span> there</p>',
|
||||
);
|
||||
const html = await markdownToHtml(md);
|
||||
expect(html).toMatch(/<span[^>]*data-type="mention"[^>]*>/);
|
||||
expect(html).toContain("u1");
|
||||
expect(html).toContain("Bob");
|
||||
});
|
||||
|
||||
it("round-trips a status chip (keeps data-color)", async () => {
|
||||
const md = htmlToMarkdown(
|
||||
'<p>s <span data-type="status" data-color="green">Done</span></p>',
|
||||
);
|
||||
const html = await markdownToHtml(md);
|
||||
expect(html).toMatch(/<span[^>]*data-type="status"[^>]*>/);
|
||||
expect(html).toContain("green");
|
||||
});
|
||||
|
||||
// HTML special chars in an attribute value or in a node's text must be
|
||||
// ESCAPED when re-emitted as raw HTML, otherwise the exported tag is
|
||||
// malformed and `markdownToHtml`'s parser cannot restore the original value
|
||||
// (the same silent data loss this PR fixes). Dropping `<`/`>` escaping is the
|
||||
// dangerous regression: a stray `<` or `>` corrupts the tag (or injects new
|
||||
// markup), so the test data carries ALL of `&`, `"`, `<`, `>` in BOTH the
|
||||
// data-label attribute and the visible text. That fully exercises
|
||||
// escapeHtmlAttr's `&,",<,>` branches and escapeHtmlText's `&,<,>` branches
|
||||
// (escapeHtmlText leaves `"` literal); the alphanumeric-only cases above hit
|
||||
// none of them.
|
||||
it("escapes HTML special chars (& \" < >) in attrs + text and round-trips them", async () => {
|
||||
const md = htmlToMarkdown(
|
||||
`<p>hi <span data-type="mention" data-id="u1" data-label="A & <B> "C"">@A & <B> "C"</span> there</p>`,
|
||||
);
|
||||
|
||||
// (a) The exported Markdown carries a WELL-FORMED, correctly-escaped tag:
|
||||
// the attribute escapes `&`, `<`, `>` AND `"`; the text escapes `&`, `<`,
|
||||
// `>` (a `"` inside text content is legal, so it stays literal).
|
||||
expect(md).toContain('data-label="A & <B> "C""');
|
||||
expect(md).toContain('>@A & <B> "C"</span>');
|
||||
// And explicitly NOT the raw, tag-corrupting forms: a literal `<B>` (would
|
||||
// mean `<`/`>` escaping was dropped in either the attr or the text)...
|
||||
expect(md).not.toContain("<B>");
|
||||
// ...nor the malformed attribute that an unescaped `"` would produce.
|
||||
expect(md).not.toContain('data-label="A & <B> "C""');
|
||||
|
||||
// (b) Import restores the ORIGINAL (unescaped) values, attribute and text.
|
||||
const html = await markdownToHtml(md);
|
||||
const dom = new DOMParser().parseFromString(html as string, "text/html");
|
||||
const span = dom.querySelector('span[data-type="mention"]');
|
||||
expect(span).not.toBeNull();
|
||||
expect(span!.getAttribute("data-id")).toBe("u1");
|
||||
expect(span!.getAttribute("data-label")).toBe('A & <B> "C"');
|
||||
expect(span!.textContent).toBe('@A & <B> "C"');
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,488 +0,0 @@
|
||||
import * as _TurndownService from '@joplin/turndown';
|
||||
import * as TurndownPluginGfm from '@joplin/turndown-plugin-gfm';
|
||||
import { getBasename } from './basename';
|
||||
|
||||
// CJS/ESM interop: .default exists in Vite, not in NestJS
|
||||
const TurndownService = (_TurndownService as any).default || _TurndownService;
|
||||
|
||||
function sanitizeMdLinkText(value: string): string {
|
||||
return value
|
||||
.replace(/\\/g, '\\\\')
|
||||
.replace(/([\[\]!])/g, '\\$1')
|
||||
.replace(/[\r\n]+/g, ' ');
|
||||
}
|
||||
|
||||
// Tags turndown treats as void (self-closing). Footnote references render as an
|
||||
// empty <sup data-footnote-ref> whose meaning lives entirely in its data-id;
|
||||
// without marking it void, turndown's blank-node removal drops it before our
|
||||
// rule runs, losing the `[^id]` marker. Mirrors turndown's built-in list.
|
||||
const TURNDOWN_VOID_ELEMENTS = [
|
||||
'AREA', 'BASE', 'BR', 'COL', 'COMMAND', 'EMBED', 'HR', 'IMG', 'INPUT',
|
||||
'KEYGEN', 'LINK', 'META', 'PARAM', 'SOURCE', 'TRACK', 'WBR',
|
||||
];
|
||||
|
||||
function isVoidNode(node: any): boolean {
|
||||
const name = node?.nodeName?.toUpperCase?.();
|
||||
if (!name) return false;
|
||||
if (name === 'SUP' && node.hasAttribute?.('data-footnote-ref')) {
|
||||
return true;
|
||||
}
|
||||
return TURNDOWN_VOID_ELEMENTS.indexOf(name) !== -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* An empty <sup data-footnote-ref> is "blank" to turndown, which removes blank
|
||||
* inline nodes (RootNode/Node use a module-level isVoid the options cannot
|
||||
* override). To survive, inject the id as text content so the node is non-blank;
|
||||
* the footnoteReference rule then reads data-id and emits `[^id]`.
|
||||
*/
|
||||
function fillEmptyFootnoteRefs(html: string): string {
|
||||
return html.replace(
|
||||
/<sup\b([^>]*\bdata-footnote-ref\b[^>]*)>\s*<\/sup>/gi,
|
||||
(_m, attrs) => `<sup${attrs}></sup>`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* `pageBreak` and `transclusionReference` are childless atom <div>s. Like an
|
||||
* empty footnote ref (see above), turndown treats a childless block as "blank"
|
||||
* and replaces it with the blankRule BEFORE any custom rule can fire — so the
|
||||
* node disappears from the export with no trace (#206 mdrt-2). Inject a
|
||||
* zero-width space so the node is non-blank and our lossless rule runs; the
|
||||
* rule rebuilds the tag from the element's attributes, so the injected char
|
||||
* never reaches the output.
|
||||
*/
|
||||
function fillEmptyAtomBlocks(html: string): string {
|
||||
return html.replace(
|
||||
/<div\b([^>]*\bdata-type="(?:pageBreak|transclusionReference)"[^>]*)>\s*<\/div>/gi,
|
||||
(_m, attrs) => `<div${attrs}></div>`,
|
||||
);
|
||||
}
|
||||
|
||||
/** HTML-escape an attribute value so a re-emitted raw-HTML tag is well-formed. */
|
||||
function escapeHtmlAttr(value: string): string {
|
||||
return value
|
||||
.replace(/&/g, '&')
|
||||
.replace(/"/g, '"')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
}
|
||||
|
||||
/** HTML-escape text placed inside a re-emitted raw-HTML element. */
|
||||
function escapeHtmlText(value: string): string {
|
||||
return value
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>');
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize ALL of an element's attributes back to a raw-HTML attribute string
|
||||
* (leading space included). Generic on purpose: a custom node's identity lives
|
||||
* entirely in its `data-*` attributes (data-id, data-color, data-source-page-id,
|
||||
* data-transclusion-id, …), and serializing every attribute keeps the export
|
||||
* lossless regardless of which attributes a given node carries.
|
||||
*/
|
||||
function serializeAttrs(node: any): string {
|
||||
const attrs = node?.attributes;
|
||||
if (!attrs) return '';
|
||||
return Array.from(attrs as ArrayLike<{ name: string; value: string }>)
|
||||
.map((attr) => ` ${attr.name}="${escapeHtmlAttr(attr.value ?? '')}"`)
|
||||
.join('');
|
||||
}
|
||||
|
||||
export function htmlToMarkdown(html: string): string {
|
||||
const turndownService = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
codeBlockStyle: 'fenced',
|
||||
hr: '---',
|
||||
bulletListMarker: '-',
|
||||
isVoid: isVoidNode,
|
||||
});
|
||||
|
||||
turndownService.use([
|
||||
TurndownPluginGfm.tables,
|
||||
TurndownPluginGfm.strikethrough,
|
||||
TurndownPluginGfm.highlightedCodeBlock,
|
||||
taskList,
|
||||
callout,
|
||||
preserveDetail,
|
||||
listParagraph,
|
||||
orderedListItem,
|
||||
mathInline,
|
||||
mathBlock,
|
||||
iframeEmbed,
|
||||
htmlEmbed,
|
||||
spoiler,
|
||||
image,
|
||||
video,
|
||||
footnoteReference,
|
||||
footnotesList,
|
||||
pageBreak,
|
||||
transclusionReference,
|
||||
mention,
|
||||
status,
|
||||
]);
|
||||
return turndownService
|
||||
.turndown(fillEmptyAtomBlocks(fillEmptyFootnoteRefs(html)))
|
||||
.replaceAll('<br>', ' ');
|
||||
}
|
||||
|
||||
/**
|
||||
* Lossless export rules for custom nodes that have NO native Markdown syntax
|
||||
* (#206 mdrt-2). Markdown cannot represent a page break, a transclusion
|
||||
* reference, a mention's stable id, or a status chip's color — so rather than
|
||||
* letting turndown silently drop them, each rule re-emits the node as raw HTML
|
||||
* carrying every `data-*` attribute. Plain-Markdown viewers ignore the inert
|
||||
* tag, and the import path round-trips it: `markdownToHtml` passes raw HTML
|
||||
* through and each node's `parseHTML` (`div[data-type="…"]`, `span[…]`) rebuilds
|
||||
* the ProseMirror node with its attributes intact.
|
||||
*/
|
||||
function pageBreak(turndownService: _TurndownService) {
|
||||
turndownService.addRule('pageBreak', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'DIV' &&
|
||||
node.getAttribute('data-type') === 'pageBreak'
|
||||
);
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
return `\n\n<div${serializeAttrs(node)}></div>\n\n`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function transclusionReference(turndownService: _TurndownService) {
|
||||
turndownService.addRule('transclusionReference', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'DIV' &&
|
||||
node.getAttribute('data-type') === 'transclusionReference'
|
||||
);
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
return `\n\n<div${serializeAttrs(node)}></div>\n\n`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function mention(turndownService: _TurndownService) {
|
||||
turndownService.addRule('mention', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'SPAN' &&
|
||||
node.getAttribute('data-type') === 'mention'
|
||||
);
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const text = escapeHtmlText(node.textContent || '');
|
||||
return `<span${serializeAttrs(node)}>${text}</span>`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function status(turndownService: _TurndownService) {
|
||||
turndownService.addRule('status', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'SPAN' && node.getAttribute('data-type') === 'status'
|
||||
);
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const text = escapeHtmlText(node.textContent || '');
|
||||
return `<span${serializeAttrs(node)}>${text}</span>`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the `htmlEmbed` node to Markdown.
|
||||
*
|
||||
* Markdown has no native representation for an arbitrary-HTML block, so we
|
||||
* preserve the node losslessly as an HTML comment carrying the base64-encoded
|
||||
* source (the same `data-source` payload the node stores). `markdownToHtml`
|
||||
* recognizes the same marker and rebuilds the node, so the round-trip
|
||||
* MD -> HTML -> JSON keeps the source intact. The comment also keeps the raw
|
||||
* markup inert in the exported `.md` file (it does not render in plain Markdown
|
||||
* viewers).
|
||||
*/
|
||||
function htmlEmbed(turndownService: _TurndownService) {
|
||||
turndownService.addRule('htmlEmbed', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'DIV' &&
|
||||
node.getAttribute('data-type') === 'htmlEmbed'
|
||||
);
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const encoded = node.getAttribute('data-source') || '';
|
||||
return `\n\n<!--html-embed:${encoded}-->\n\n`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the `spoiler` inline mark to lossless raw inline HTML.
|
||||
*
|
||||
* Markdown has no native spoiler syntax, so we emit the same `<span
|
||||
* data-spoiler="true">…</span>` the mark renders. `marked` passes inline raw HTML
|
||||
* through untouched, and `generateJSON` restores the mark via its parseHTML, so
|
||||
* the round-trip MD -> HTML -> JSON keeps the spoiler intact. The UI-only
|
||||
* `is-revealed` state is never serialized.
|
||||
*/
|
||||
function spoiler(turndownService: _TurndownService) {
|
||||
turndownService.addRule('spoiler', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'SPAN' &&
|
||||
node.getAttribute('data-spoiler') === 'true'
|
||||
);
|
||||
},
|
||||
replacement: function (content: string) {
|
||||
return `<span data-spoiler="true">${content}</span>`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function listParagraph(turndownService: _TurndownService) {
|
||||
turndownService.addRule('paragraph', {
|
||||
filter: ['p'],
|
||||
replacement: (content: string, node: HTMLInputElement) => {
|
||||
if (node.parentElement?.nodeName === 'LI') {
|
||||
return content;
|
||||
}
|
||||
return `\n\n${content}\n\n`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function orderedListItem(turndownService: _TurndownService) {
|
||||
turndownService.addRule('orderedListItem', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return node.nodeName === 'LI' && node.getAttribute('data-type') !== 'taskItem';
|
||||
},
|
||||
replacement: (content: string, node: HTMLInputElement, options: any) => {
|
||||
const parent = node.parentNode as HTMLElement;
|
||||
if (parent.nodeName !== 'OL' && parent.nodeName !== 'UL') {
|
||||
return content;
|
||||
}
|
||||
|
||||
content = content
|
||||
.replace(/^\n+/, '')
|
||||
.replace(/\n+$/, '\n')
|
||||
.replace(/\n/gm, '\n ');
|
||||
|
||||
let prefix: string;
|
||||
if (parent.nodeName === 'OL') {
|
||||
const start = parseInt(parent.getAttribute('start') || '1', 10);
|
||||
const index = Array.prototype.indexOf.call(parent.children, node);
|
||||
prefix = `${start + index}. `;
|
||||
} else {
|
||||
prefix = `${options.bulletListMarker} `;
|
||||
}
|
||||
|
||||
return (
|
||||
prefix +
|
||||
content +
|
||||
(node.nextSibling && !/\n$/.test(content) ? '\n' : '')
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function callout(turndownService: _TurndownService) {
|
||||
turndownService.addRule('callout', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'DIV' && node.getAttribute('data-type') === 'callout'
|
||||
);
|
||||
},
|
||||
replacement: function (content: string, node: HTMLInputElement) {
|
||||
const calloutType = node.getAttribute('data-callout-type');
|
||||
return `\n\n:::${calloutType}\n${content.trim()}\n:::\n\n`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function taskList(turndownService: _TurndownService) {
|
||||
turndownService.addRule('taskListItem', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.getAttribute('data-type') === 'taskItem' &&
|
||||
node.parentNode.nodeName === 'UL'
|
||||
);
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const isChecked = node.getAttribute('data-checked') === 'true';
|
||||
const div = node.querySelector('div');
|
||||
const text = div ? div.textContent.trim() : node.textContent.trim();
|
||||
|
||||
const prefix = `- ${isChecked ? '[x]' : '[ ]'} `;
|
||||
|
||||
return (
|
||||
prefix +
|
||||
text +
|
||||
(node.nextSibling && !/\n$/.test(text) ? '\n' : '')
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function preserveDetail(turndownService: _TurndownService) {
|
||||
turndownService.addRule('preserveDetail', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return node.nodeName === 'DETAILS';
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const summary = node.querySelector(':scope > summary');
|
||||
let detailSummary = '';
|
||||
|
||||
if (summary) {
|
||||
detailSummary = `<summary>${turndownService.turndown(summary.innerHTML)}</summary>`;
|
||||
}
|
||||
|
||||
const detailsContent = Array.from(node.childNodes)
|
||||
.filter((child) => child.nodeName !== 'SUMMARY')
|
||||
.map((child) =>
|
||||
child.nodeType === 1
|
||||
? turndownService.turndown((child as HTMLElement).outerHTML)
|
||||
: child.textContent,
|
||||
)
|
||||
.join('');
|
||||
|
||||
return `\n<details>\n${detailSummary}\n\n${detailsContent}\n\n</details>\n`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function mathInline(turndownService: _TurndownService) {
|
||||
turndownService.addRule('mathInline', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'SPAN' &&
|
||||
node.getAttribute('data-type') === 'mathInline'
|
||||
);
|
||||
},
|
||||
replacement: function (content: string) {
|
||||
return `$${content}$`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function mathBlock(turndownService: _TurndownService) {
|
||||
turndownService.addRule('mathBlock', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'DIV' &&
|
||||
node.getAttribute('data-type') === 'mathBlock'
|
||||
);
|
||||
},
|
||||
replacement: function (content: string) {
|
||||
return `\n$$\n${content}\n$$\n`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function iframeEmbed(turndownService: _TurndownService) {
|
||||
turndownService.addRule('iframeEmbed', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return node.nodeName === 'IFRAME';
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const src = node.getAttribute('src');
|
||||
return '[' + src + '](' + src + ')';
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function image(turndownService: _TurndownService) {
|
||||
turndownService.addRule('image', {
|
||||
filter: 'img',
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const src = node.getAttribute('src') || '';
|
||||
if (!src) return '';
|
||||
const caption = node.getAttribute('data-caption') || '';
|
||||
if (caption) {
|
||||
// ![]() can't carry a caption, so emit a raw <img> wrapped in a block
|
||||
// <div>. marked passes it through and the image extension's parseHTML
|
||||
// restores the caption from data-caption.
|
||||
const parts = [`src="${escapeHtmlAttr(src)}"`];
|
||||
const alt = node.getAttribute('alt') || '';
|
||||
if (alt) parts.push(`alt="${escapeHtmlAttr(alt)}"`);
|
||||
parts.push(`data-caption="${escapeHtmlAttr(caption)}"`);
|
||||
return `<div><img ${parts.join(' ')}></div>`;
|
||||
}
|
||||
const alt = sanitizeMdLinkText(node.getAttribute('alt') || '');
|
||||
const title = node.getAttribute('title') || '';
|
||||
const titlePart = title ? ' "' + title.replace(/"/g, '\\"') + '"' : '';
|
||||
return '';
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Footnote reference (inline atom) -> pandoc/GFM marker `[^id]`.
|
||||
* The visible number is derived (not stored), so the id is the stable anchor.
|
||||
*/
|
||||
function footnoteReference(turndownService: _TurndownService) {
|
||||
turndownService.addRule('footnoteReference', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'SUP' && node.hasAttribute('data-footnote-ref')
|
||||
);
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const id = node.getAttribute('data-id') || '';
|
||||
return id ? `[^${id}]` : '';
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Footnotes container -> the list of `[^id]: text` definitions at the end of
|
||||
* the document (one per line). Each footnoteDefinition inside emits its own
|
||||
* `[^id]: ...` line; turndown joins them with the surrounding block spacing.
|
||||
*/
|
||||
function footnotesList(turndownService: _TurndownService) {
|
||||
turndownService.addRule('footnoteDefinition', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'DIV' && node.hasAttribute('data-footnote-def')
|
||||
);
|
||||
},
|
||||
replacement: function (content: string, node: HTMLInputElement) {
|
||||
const id = node.getAttribute('data-id') || '';
|
||||
// Collapse internal newlines so the definition stays a single MD line;
|
||||
// continuation lines are a v2 refinement.
|
||||
const text = content.replace(/\s*\n+\s*/g, ' ').trim();
|
||||
return id ? `\n[^${id}]: ${text}\n` : '';
|
||||
},
|
||||
});
|
||||
|
||||
turndownService.addRule('footnotesList', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return (
|
||||
node.nodeName === 'SECTION' && node.hasAttribute('data-footnotes')
|
||||
);
|
||||
},
|
||||
replacement: function (content: string) {
|
||||
return `\n\n${content.trim()}\n`;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function video(turndownService: _TurndownService) {
|
||||
turndownService.addRule('video', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return node.tagName === 'VIDEO';
|
||||
},
|
||||
replacement: function (_content: string, node: HTMLInputElement) {
|
||||
const src = node.getAttribute('src') || '';
|
||||
const ariaLabel = node.getAttribute('aria-label');
|
||||
const name = sanitizeMdLinkText(
|
||||
ariaLabel || getBasename(src) || src,
|
||||
);
|
||||
return '[' + name + '](' + src + ')';
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -14,10 +14,15 @@ export default defineConfig({
|
||||
provider: "v8",
|
||||
reporter: ["text-summary", "text"],
|
||||
all: false,
|
||||
// functions lowered 60 -> 57 after issue #347 removed the editor-ext
|
||||
// markdown layer (src/lib/markdown) and its image/footnote round-trip
|
||||
// specs: that markdown behavior now lives in — and is tested by —
|
||||
// @docmost/prosemirror-markdown, so the editor-ext baseline shifts down.
|
||||
// Still a real gate (a few points below the post-removal measured level).
|
||||
thresholds: {
|
||||
statements: 54,
|
||||
branches: 44,
|
||||
functions: 60,
|
||||
functions: 57,
|
||||
lines: 54,
|
||||
},
|
||||
},
|
||||
|
||||
+22
-2
@@ -293,8 +293,28 @@ so capable clients steer the model automatically.
|
||||
the debounced REST snapshot), then **reads → transforms → writes synchronously** in one
|
||||
tick so no remote update can interleave, and **waits for persistence acknowledgement**
|
||||
before returning.
|
||||
- **Per-page write serialization.** A per-`pageId` async mutex ensures two MCP writes to
|
||||
the same page never overlap; different pages never block each other.
|
||||
- **Per-page write serialization.** A per-`pageId` async mutex (keyed by the resolved
|
||||
page **UUID**, never a slugId) ensures two MCP writes to the same page never overlap;
|
||||
different pages never block each other. The lock helper fails fast if it is ever handed
|
||||
a non-UUID key, so a write path that forgot to resolve the id can never silently lock
|
||||
under a split key.
|
||||
|
||||
**Deploy requirement — single instance or sticky sessions.** This mutex is an
|
||||
in-process `Map`, and the cached collab sessions and the `stash_page` blob store are
|
||||
RAM-only and process-local. Behind a **multi-replica** load balancer **without sticky
|
||||
sessions**, two replicas can each "hold" the lock for the same page at once and per-page
|
||||
serialization is silently lost. Run the MCP/app as a **single instance**, or pin each
|
||||
page's traffic to one replica (sticky sessions / consistent hashing on the page id).
|
||||
There is deliberately no cross-process (e.g. Postgres advisory) lock yet — a conscious
|
||||
documented constraint. See the `Dockerfile` comment and the `MCP collaboration write
|
||||
path` block in `.env.example`.
|
||||
|
||||
**Rights-staleness window.** A cached collab session writes under the token captured at
|
||||
connect time (and the collab-token cache reuses a token for its TTL), so a **revoked**
|
||||
page access can lag by up to `MCP_COLLAB_SESSION_MAX_AGE_MS` (the hard session lifetime,
|
||||
default 10 min) before the next re-auth picks it up. Lower it to shorten the lag at the
|
||||
cost of more reconnects. This bounded window is an accepted trade-off; there is no
|
||||
push-based cache invalidation on a rights change.
|
||||
- **Transparent re-authentication.** Login uses email/password; expired tokens are
|
||||
refreshed automatically on the first 401/403 (covering JSON, multipart upload, and the
|
||||
collaboration-token path), with in-flight login de-duplication so a burst of calls
|
||||
|
||||
@@ -302,9 +302,27 @@ Docmost-MCP не сочетают:
|
||||
правки, которых ещё нет в дебаунс-снапшоте REST), затем **читает → трансформирует →
|
||||
пишет синхронно** в одном тике, чтобы никакое удалённое обновление не вклинилось, и
|
||||
**ждёт подтверждения сохранения** до возврата.
|
||||
- **Сериализация записи по странице.** Асинхронный мьютекс по `pageId` гарантирует, что
|
||||
две записи MCP в одну страницу никогда не пересекаются; разные страницы друг друга не
|
||||
блокируют.
|
||||
- **Сериализация записи по странице.** Асинхронный мьютекс по разрешённому **UUID**
|
||||
страницы (никогда не по slugId) гарантирует, что две записи MCP в одну страницу никогда
|
||||
не пересекаются; разные страницы друг друга не блокируют. Хелпер блокировки падает сразу
|
||||
(fail-fast), если ему передали не-UUID ключ, — путь записи, забывший разрезолвить id, не
|
||||
сможет молча взять лок под расщеплённым ключом.
|
||||
|
||||
**Требование к деплою — один инстанс или sticky-сессии.** Этот мьютекс — процесс-локальный
|
||||
`Map`, а кэш collab-сессий и хранилище `stashPage` живут только в RAM одного процесса. За
|
||||
**мультиреплика**-балансировщиком **без sticky-сессий** две реплики могут одновременно
|
||||
«держать» лок одной страницы, и сериализация по странице молча теряется. Запускайте
|
||||
MCP/приложение **одним инстансом** либо прибивайте трафик страницы к одной реплике
|
||||
(sticky-сессии / consistent hashing по id страницы). Кросс-процессной блокировки (например,
|
||||
Postgres advisory-lock) намеренно пока нет — осознанное задокументированное ограничение.
|
||||
См. комментарий в `Dockerfile` и блок `MCP collaboration write path` в `.env.example`.
|
||||
|
||||
**Окно устаревших прав.** Кэшированная collab-сессия пишет под токеном, захваченным в
|
||||
момент connect (а кэш collab-токена переиспользует токен в пределах своего TTL), поэтому
|
||||
**отозванный** доступ к странице может лагать до `MCP_COLLAB_SESSION_MAX_AGE_MS` (жёсткий
|
||||
срок жизни сессии, по умолчанию 10 мин), пока следующая переавторизация его не подхватит.
|
||||
Уменьшите значение, чтобы сократить лаг ценой большего числа переподключений. Это
|
||||
ограниченное окно — принятый trade-off; push-инвалидации кэша при смене прав нет.
|
||||
- **Прозрачная переавторизация.** Логин по email/паролю; истёкшие токены обновляются
|
||||
автоматически на первом 401/403 (покрывая JSON, multipart-загрузку и путь токена
|
||||
коллаборации), с дедупликацией параллельных логинов, так что пачка вызовов вызывает один
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
{
|
||||
"$comment": "Semantic palettes for drawioFromGraph (issue #425). DATA, not code: node `kind` -> fill/stroke slot, edge `kind` -> line-style props, per preset. The `default` node palette is the issue's base table; `dark` keeps the same hues on a dark canvas with lighter strokes/font; `colorblind-safe` maps every slot onto the Okabe-Ito qualitative palette (8 colours proven distinguishable for all common colour-vision deficiencies) so no two adjacent kinds collide. `fontColor`/`fillColor`/`strokeColor` are exact draw.io values. `edgeDefault` is the fallback line style; `group` is the (always-transparent) container stroke per preset.",
|
||||
"presets": {
|
||||
"default": {
|
||||
"canvasDark": false,
|
||||
"nodes": {
|
||||
"service": { "fillColor": "#dae8fc", "strokeColor": "#6c8ebf", "fontColor": "#000000" },
|
||||
"db": { "fillColor": "#d5e8d4", "strokeColor": "#82b366", "fontColor": "#000000" },
|
||||
"queue": { "fillColor": "#fff2cc", "strokeColor": "#d6b656", "fontColor": "#000000" },
|
||||
"gateway": { "fillColor": "#ffe6cc", "strokeColor": "#d79b00", "fontColor": "#000000" },
|
||||
"error": { "fillColor": "#f8cecc", "strokeColor": "#b85450", "fontColor": "#000000" },
|
||||
"external": { "fillColor": "#f5f5f5", "strokeColor": "#666666", "fontColor": "#333333" },
|
||||
"security": { "fillColor": "#e1d5e7", "strokeColor": "#9673a6", "fontColor": "#000000" }
|
||||
},
|
||||
"edges": {
|
||||
"sync": { "props": "" },
|
||||
"async": { "props": "dashed=1;" },
|
||||
"error": { "props": "dashed=1;strokeColor=#DD344C;" }
|
||||
},
|
||||
"edgeDefault": { "strokeColor": "#333333", "fontColor": "#333333" },
|
||||
"group": { "strokeColor": "#666666", "fontColor": "#333333" }
|
||||
},
|
||||
"dark": {
|
||||
"canvasDark": true,
|
||||
"nodes": {
|
||||
"service": { "fillColor": "#1a2a44", "strokeColor": "#7ea6e0", "fontColor": "#dae8fc" },
|
||||
"db": { "fillColor": "#1f331e", "strokeColor": "#97d077", "fontColor": "#d5e8d4" },
|
||||
"queue": { "fillColor": "#3a3218", "strokeColor": "#e5c15a", "fontColor": "#fff2cc" },
|
||||
"gateway": { "fillColor": "#3a2812", "strokeColor": "#ffb570", "fontColor": "#ffe6cc" },
|
||||
"error": { "fillColor": "#3a1c1b", "strokeColor": "#e08e8b", "fontColor": "#f8cecc" },
|
||||
"external": { "fillColor": "#2b2b2b", "strokeColor": "#999999", "fontColor": "#e0e0e0" },
|
||||
"security": { "fillColor": "#2c2338", "strokeColor": "#b39ddb", "fontColor": "#e1d5e7" }
|
||||
},
|
||||
"edges": {
|
||||
"sync": { "props": "" },
|
||||
"async": { "props": "dashed=1;" },
|
||||
"error": { "props": "dashed=1;strokeColor=#ff6b6b;" }
|
||||
},
|
||||
"edgeDefault": { "strokeColor": "#cccccc", "fontColor": "#e0e0e0" },
|
||||
"group": { "strokeColor": "#aaaaaa", "fontColor": "#e0e0e0" }
|
||||
},
|
||||
"colorblind-safe": {
|
||||
"canvasDark": false,
|
||||
"okabeIto": ["#000000", "#E69F00", "#56B4E9", "#009E73", "#F0E442", "#0072B2", "#D55E00", "#CC79A7"],
|
||||
"nodes": {
|
||||
"service": { "fillColor": "#D6E9F5", "strokeColor": "#0072B2", "fontColor": "#000000" },
|
||||
"db": { "fillColor": "#D6EFE4", "strokeColor": "#009E73", "fontColor": "#000000" },
|
||||
"queue": { "fillColor": "#FCF8CC", "strokeColor": "#F0E442", "fontColor": "#000000" },
|
||||
"gateway": { "fillColor": "#FBEBD0", "strokeColor": "#E69F00", "fontColor": "#000000" },
|
||||
"error": { "fillColor": "#F7DDCC", "strokeColor": "#D55E00", "fontColor": "#000000" },
|
||||
"external": { "fillColor": "#EDEDED", "strokeColor": "#000000", "fontColor": "#000000" },
|
||||
"security": { "fillColor": "#F3DEEB", "strokeColor": "#CC79A7", "fontColor": "#000000" }
|
||||
},
|
||||
"edges": {
|
||||
"sync": { "props": "" },
|
||||
"async": { "props": "dashed=1;" },
|
||||
"error": { "props": "dashed=1;strokeColor=#D55E00;" }
|
||||
},
|
||||
"edgeDefault": { "strokeColor": "#000000", "fontColor": "#000000" },
|
||||
"group": { "strokeColor": "#000000", "fontColor": "#000000" }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,60 +1,100 @@
|
||||
// Codegen: emit src/registry-stamp.generated.ts with a REGISTRY_STAMP hash of
|
||||
// the tool-specs REGISTRY CONTENT, so a build/ vs src/ skew (issue #447) is
|
||||
// detectable at runtime.
|
||||
// the ENTIRE src/ tree, so a build/ vs src/ skew (issue #447) is detectable at
|
||||
// runtime for ANY source file — not just tool-specs.ts.
|
||||
//
|
||||
// WHY hash the raw source text (not extracted structured data):
|
||||
// SHARED_TOOL_SPECS carries `buildShape` functions (the input SCHEMAS) which are
|
||||
// NOT serializable. The input schema is exactly one of the things that MUST stay
|
||||
// in sync between build/ and src/, so we cannot drop it from the hash. Rather
|
||||
// than probe zod with a fragile shim to reconstruct the schema shape, we hash the
|
||||
// STABLE, deterministic source TEXT of tool-specs.ts. That text fully captures
|
||||
// every field that must stay in sync — mcpName, inAppKey, description, tier,
|
||||
// catalogLine AND the buildShape bodies (input schemas) — with zero probing
|
||||
// fragility. Any edit to a spec (a renamed tool, a reworded description, a
|
||||
// changed schema field) changes the text and therefore the stamp.
|
||||
// WHY hash the whole src tree (not just tool-specs.ts): the runtime tools are
|
||||
// assembled from far more than the spec registry — client.ts, the client/*
|
||||
// domain modules, comment-signal.ts and the drawio-* helpers all ship in build/
|
||||
// and are loaded by the in-app server. Hashing ONLY tool-specs.ts meant an edit
|
||||
// to any of those (e.g. a behavioural fix in client.ts) left the stamp unchanged,
|
||||
// so a stale build/ served the OLD code silently (issue #486). Hashing every
|
||||
// src/**/*.ts closes that gap: any source edit changes the stamp.
|
||||
//
|
||||
// DETERMINISM: the hash is computed over the file bytes with line endings
|
||||
// normalized to LF and a single trailing newline stripped, so a CRLF checkout or
|
||||
// an editor's trailing-newline habit cannot make build/ and src/ disagree. No
|
||||
// Date.now / randomness. The loader's dev-only stale-check (docmost-client.loader.ts)
|
||||
// re-runs THIS SAME normalization + sha256 over src/tool-specs.ts and compares to
|
||||
// the built REGISTRY_STAMP; the two must compute identically.
|
||||
// WHY hash the raw source text (not extracted structured data): the tool input
|
||||
// SCHEMAS live as `buildShape` functions which are NOT serializable, so we cannot
|
||||
// reduce them to structured data without a fragile zod shim. Hashing the STABLE,
|
||||
// deterministic source TEXT captures every field that must stay in sync with zero
|
||||
// probing fragility. Any edit to any source file changes the text → the stamp.
|
||||
//
|
||||
// DETERMINISM: files are enumerated recursively, filtered to *.ts EXCLUDING
|
||||
// *.generated.ts (the codegen's OWN output — including it would create a
|
||||
// fixed-point cycle), and sorted by their POSIX-normalized path relative to src/
|
||||
// so the order is platform-independent. Each file contributes its relative path
|
||||
// AND its content with line endings normalized to LF and a single trailing
|
||||
// newline stripped, so a CRLF checkout or an editor's trailing-newline habit
|
||||
// cannot make build/ and src/ disagree. No Date.now / randomness. The loader's
|
||||
// dev-only stale-check (docmost-client.loader.ts) re-runs THIS SAME enumeration +
|
||||
// normalization + sha256 and compares to the built REGISTRY_STAMP; the two must
|
||||
// compute identically.
|
||||
//
|
||||
// This script runs from the `build` and `pretest` npm scripts BEFORE tsc, so
|
||||
// build/ always carries a stamp derived from the tool-specs.ts that was compiled.
|
||||
// build/ always carries a stamp derived from the src/ tree that was compiled.
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { dirname, join, relative, sep } from 'node:path';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const SRC_DIR = join(__dirname, '..', 'src');
|
||||
const TOOL_SPECS_PATH = join(SRC_DIR, 'tool-specs.ts');
|
||||
const OUT_PATH = join(SRC_DIR, 'registry-stamp.generated.ts');
|
||||
|
||||
/**
|
||||
* Deterministic stamp of the tool-specs registry content. Kept as a plain
|
||||
* function (exported) so the algorithm has a single home; the loader duplicates
|
||||
* only the tiny normalize+sha256 steps because it lives in the CJS server build
|
||||
* and cannot import this ESM script. If you change the normalization here, mirror
|
||||
* it in apps/server/src/core/ai-chat/tools/docmost-client.loader.ts.
|
||||
* Recursively enumerate every `*.ts` file under `dir`, EXCLUDING the codegen's
|
||||
* own `*.generated.ts` output (a self-referential cycle otherwise). Returns
|
||||
* absolute paths, unsorted (the caller sorts by relative path for determinism).
|
||||
* Kept as a plain exported function so the algorithm has a single home; the
|
||||
* loader duplicates it because it lives in the CJS server build and cannot import
|
||||
* this ESM script. If you change the walk/filter here, mirror it in
|
||||
* apps/server/src/core/ai-chat/tools/docmost-client.loader.ts.
|
||||
*/
|
||||
export function computeRegistryStamp(toolSpecsSource) {
|
||||
const normalized = toolSpecsSource.replace(/\r\n/g, '\n').replace(/\n$/, '');
|
||||
return createHash('sha256').update(normalized, 'utf8').digest('hex');
|
||||
export function collectStampFiles(dir) {
|
||||
const out = [];
|
||||
for (const entry of readdirSync(dir)) {
|
||||
const full = join(dir, entry);
|
||||
if (statSync(full).isDirectory()) {
|
||||
out.push(...collectStampFiles(full));
|
||||
} else if (entry.endsWith('.ts') && !entry.endsWith('.generated.ts')) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deterministic stamp of the whole src/ tree. Enumerate + sort by POSIX-relative
|
||||
* path, then fold each file's relative path AND normalized content into one
|
||||
* sha256. MUST stay byte-for-byte identical to the loader's recompute.
|
||||
*/
|
||||
export function computeRegistryStamp(srcDir) {
|
||||
const files = collectStampFiles(srcDir)
|
||||
.map((abs) => ({
|
||||
rel: relative(srcDir, abs).split(sep).join('/'),
|
||||
abs,
|
||||
}))
|
||||
.sort((a, b) => (a.rel < b.rel ? -1 : a.rel > b.rel ? 1 : 0));
|
||||
const hash = createHash('sha256');
|
||||
for (const { rel, abs } of files) {
|
||||
const normalized = readFileSync(abs, 'utf8')
|
||||
.replace(/\r\n/g, '\n')
|
||||
.replace(/\n$/, '');
|
||||
hash.update(rel, 'utf8');
|
||||
hash.update('\0', 'utf8');
|
||||
hash.update(normalized, 'utf8');
|
||||
hash.update('\0', 'utf8');
|
||||
}
|
||||
return hash.digest('hex');
|
||||
}
|
||||
|
||||
function main() {
|
||||
const source = readFileSync(TOOL_SPECS_PATH, 'utf8');
|
||||
const stamp = computeRegistryStamp(source);
|
||||
const stamp = computeRegistryStamp(SRC_DIR);
|
||||
const out =
|
||||
'// AUTO-GENERATED by scripts/gen-registry-stamp.mjs — DO NOT EDIT BY HAND.\n' +
|
||||
'// A deterministic hash of src/tool-specs.ts content (tool names, descriptions,\n' +
|
||||
'// tiers, catalog lines and input schemas). Regenerated on every build/pretest\n' +
|
||||
'// so build/ always matches the compiled src. The in-app loader recomputes this\n' +
|
||||
'// from src and refuses to run on a mismatch (issue #447). This file is\n' +
|
||||
'// gitignored and produced by the build — see .gitignore.\n' +
|
||||
'// A deterministic hash of the whole src/ tree (every src/**/*.ts except\n' +
|
||||
'// *.generated.ts). Regenerated on every build/pretest so build/ always\n' +
|
||||
'// matches the compiled src. The in-app loader recomputes this from src and\n' +
|
||||
'// refuses to run on a mismatch (issue #447/#486). This file is gitignored\n' +
|
||||
'// and produced by the build — see .gitignore.\n' +
|
||||
`export const REGISTRY_STAMP = ${JSON.stringify(stamp)};\n`;
|
||||
writeFileSync(OUT_PATH, out, 'utf8');
|
||||
// eslint-disable-next-line no-console
|
||||
|
||||
+89
-4934
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,704 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import { assertFullUuid } from "./errors.js";
|
||||
import {
|
||||
filterWorkspace,
|
||||
filterSpace,
|
||||
filterPage,
|
||||
filterComment,
|
||||
filterSearchResult,
|
||||
} from "../lib/filters.js";
|
||||
import { convertProseMirrorToMarkdown } from "../lib/markdown-converter.js";
|
||||
import {
|
||||
updatePageContentRealtime,
|
||||
replacePageContent,
|
||||
markdownToProseMirror,
|
||||
markdownToProseMirrorCanonical,
|
||||
mutatePageContent,
|
||||
assertYjsEncodable,
|
||||
MutationResult,
|
||||
} from "../lib/collaboration.js";
|
||||
import {
|
||||
replaceNodeById,
|
||||
replaceNodeByIdWithMany,
|
||||
reassignCollidingBlockIds,
|
||||
deleteNodeById,
|
||||
assertUnambiguousMatch,
|
||||
insertNodeRelative,
|
||||
insertNodesRelative,
|
||||
blockPlainText,
|
||||
buildOutline,
|
||||
getNodeByRef,
|
||||
readTable,
|
||||
insertTableRow,
|
||||
deleteTableRow,
|
||||
updateTableCell,
|
||||
findInvalidNode,
|
||||
} from "@docmost/prosemirror-markdown";
|
||||
import {
|
||||
applyAnchorInDoc,
|
||||
countAnchorMatches,
|
||||
getAnchoredText,
|
||||
resolveAnchorSelection,
|
||||
normalizeForMatch,
|
||||
} from "../lib/comment-anchor.js";
|
||||
import { closestBlockHint } from "../lib/text-normalize.js";
|
||||
import {
|
||||
blockText,
|
||||
walk,
|
||||
getList,
|
||||
insertMarkerAfter,
|
||||
setCalloutRange,
|
||||
noteItem,
|
||||
mdToInlineNodes,
|
||||
commentsToFootnotes,
|
||||
canonicalizeFootnotes,
|
||||
insertInlineFootnote,
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
|
||||
// Public method surface of CommentsMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements ICommentsMixin` fails to compile on drift.
|
||||
export interface ICommentsMixin {
|
||||
listComments(pageId: string, includeResolved?: boolean): any;
|
||||
getComment(commentId: string): any;
|
||||
createComment(pageId: string, content: string, type?: "page" | "inline", selection?: string, parentCommentId?: string, suggestedText?: string): any;
|
||||
updateComment(commentId: string, content: string): any;
|
||||
deleteComment(commentId: string): any;
|
||||
resolveComment(commentId: string, resolved: boolean): any;
|
||||
checkNewComments(spaceId: string, since: string, parentPageId?: string): any;
|
||||
}
|
||||
|
||||
export function CommentsMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & ICommentsMixin> & TBase {
|
||||
abstract class CommentsMixin extends Base implements ICommentsMixin {
|
||||
// --- Comment methods (ported from upstream PR #3 by Max Nikitin) ---
|
||||
|
||||
/**
|
||||
* Normalize a comment's `content` into a ProseMirror doc object before
|
||||
* markdown conversion. createComment/updateComment send content as a
|
||||
* JSON.stringify(...) STRING, and the server stores it as-is, so on read it
|
||||
* comes back as a string. convertProseMirrorToMarkdown returns "" for a
|
||||
* string, so parse it first (guarded — fall back to the raw value on any
|
||||
* parse failure so a non-JSON legacy value is still handled gracefully).
|
||||
*/
|
||||
protected parseCommentContent(content: any): any {
|
||||
if (typeof content !== "string") return content;
|
||||
try {
|
||||
return JSON.parse(content);
|
||||
} catch {
|
||||
return content;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List comments on a page (cursor-paginated), content as markdown.
|
||||
*
|
||||
* DEFAULT (`includeResolved = false`) hides RESOLVED THREADS WHOLESALE so the
|
||||
* agent sees only active discussions: a top-level comment with `resolvedAt`
|
||||
* set AND every reply under it (a reply of a closed thread is part of the
|
||||
* closed thread) are dropped from `items`. `resolvedThreadsHidden` reports how
|
||||
* many resolved top-level threads were hidden so the agent can re-query with
|
||||
* `includeResolved: true` to see everything. Active threads always stay.
|
||||
*
|
||||
* Returns `{ items, resolvedThreadsHidden }` (NOT a bare array) — callers that
|
||||
* need the full feed (lossless export, transformPage, checkNewComments) pass
|
||||
* `includeResolved: true` and read `.items`.
|
||||
*/
|
||||
async listComments(pageId: string, includeResolved = false) {
|
||||
await this.ensureAuthenticated();
|
||||
let allComments: any[] = [];
|
||||
let cursor: string | null = null;
|
||||
|
||||
// Hard ceiling + immovable-cursor guard (mirrors paginateAll): if /comments
|
||||
// ever stops advancing the cursor (the exact #442 drift scenario) this loop
|
||||
// would otherwise spin forever accumulating duplicates.
|
||||
const MAX_PAGES = 50;
|
||||
let truncated = false;
|
||||
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const payload: Record<string, any> = { pageId, limit: 100 };
|
||||
if (cursor) payload.cursor = cursor;
|
||||
|
||||
const response = await this.client.post("/comments", payload);
|
||||
const data = response.data.data || response.data;
|
||||
const items = data.items || [];
|
||||
allComments = allComments.concat(items);
|
||||
|
||||
// Advance strictly via the server-issued cursor. A missing nextCursor or a
|
||||
// cursor identical to the one we just sent means the end (or a server that
|
||||
// ignores our pagination param) — stop instead of re-fetching page one.
|
||||
const next: string | null = data.meta?.nextCursor || null;
|
||||
if (!next || next === cursor) break;
|
||||
cursor = next;
|
||||
|
||||
// Reaching the ceiling with a still-advancing cursor means truncation.
|
||||
if (page === MAX_PAGES - 1) truncated = true;
|
||||
}
|
||||
|
||||
if (truncated) {
|
||||
console.warn(
|
||||
`listComments: comments for "${pageId}" truncated at the ${MAX_PAGES}-page cap; more pages exist on the server`,
|
||||
);
|
||||
}
|
||||
|
||||
const mapped = allComments.map((comment: any) => {
|
||||
const markdown = comment.content
|
||||
? convertProseMirrorToMarkdown(
|
||||
this.parseCommentContent(comment.content),
|
||||
)
|
||||
: "";
|
||||
return filterComment(comment, markdown);
|
||||
});
|
||||
|
||||
if (includeResolved) {
|
||||
return { items: mapped, resolvedThreadsHidden: 0 };
|
||||
}
|
||||
|
||||
// Ids of RESOLVED top-level threads (a top-level comment has no
|
||||
// parentCommentId). A whole thread is hidden when its root is resolved.
|
||||
const resolvedRootIds = new Set(
|
||||
mapped
|
||||
.filter((c) => !c.parentCommentId && c.resolvedAt != null)
|
||||
.map((c) => c.id),
|
||||
);
|
||||
|
||||
const items = mapped.filter((c) => {
|
||||
// Hide the resolved root itself and every reply anchored to it. A reply's
|
||||
// own resolvedAt is irrelevant — its membership follows the parent thread.
|
||||
// ASSUMPTION: Docmost's comment model is FLAT — a reply's parentCommentId
|
||||
// always points at the thread ROOT (no reply-of-reply nesting), so a single
|
||||
// level of parent lookup covers a whole thread. If nested replies are ever
|
||||
// introduced, a deep reply of a resolved thread would need a root-walk here.
|
||||
if (!c.parentCommentId) return !resolvedRootIds.has(c.id);
|
||||
return !resolvedRootIds.has(c.parentCommentId);
|
||||
});
|
||||
|
||||
return { items, resolvedThreadsHidden: resolvedRootIds.size };
|
||||
}
|
||||
|
||||
|
||||
async getComment(commentId: string) {
|
||||
// Fail fast (#436): reject a truncated id before any network call.
|
||||
assertFullUuid("get_comment", "commentId", commentId);
|
||||
await this.ensureAuthenticated();
|
||||
const response = await this.client.post("/comments/info", { commentId });
|
||||
const comment = response.data.data || response.data;
|
||||
const markdown = comment.content
|
||||
? convertProseMirrorToMarkdown(this.parseCommentContent(comment.content))
|
||||
: "";
|
||||
return {
|
||||
data: filterComment(comment, markdown),
|
||||
success: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** Plain text of each TOP-LEVEL block of `doc`, for anchor-failure hints. */
|
||||
protected topLevelBlockTexts(doc: any): string[] {
|
||||
const content = doc && Array.isArray(doc.content) ? doc.content : [];
|
||||
return content
|
||||
.map((b: any) => blockPlainText(b))
|
||||
.filter((t: string) => t.length > 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* True when per-block anchoring failed but the (normalized) selection DOES
|
||||
* appear in the blocks' joined plain text — i.e. it straddles a block
|
||||
* boundary. Blocks are joined with a newline (collapsed to one space by
|
||||
* normalizeForMatch) so a selection whose parts are separated by a paragraph
|
||||
* break still matches. Callers only reach here after single-block anchoring
|
||||
* (incl. the markdown-strip fallback) has already failed.
|
||||
*/
|
||||
protected selectionSpansMultipleBlocks(
|
||||
blockTexts: string[],
|
||||
selection: string,
|
||||
): boolean {
|
||||
const normSel = normalizeForMatch(selection).norm.trim();
|
||||
if (normSel.length === 0) return false;
|
||||
const joined = normalizeForMatch(blockTexts.join("\n")).norm;
|
||||
return joined.indexOf(normSel) !== -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the actionable error for a createComment anchor MISS, porting
|
||||
* editPageText's self-correction affordances: an explicit "spans multiple
|
||||
* blocks" message when the selection straddles a block boundary, otherwise a
|
||||
* "closest block text" hint quoting the block that holds the selection's
|
||||
* longest token. `live` switches the wording between the pre-check (reading the
|
||||
* persisted page) and the post-create live-anchor failure (which rolls back).
|
||||
*/
|
||||
protected anchorNotFoundError(
|
||||
doc: any,
|
||||
selection: string,
|
||||
live: boolean,
|
||||
): Error {
|
||||
const blockTexts = this.topLevelBlockTexts(doc);
|
||||
const rolled = live ? " The comment was rolled back." : "";
|
||||
if (this.selectionSpansMultipleBlocks(blockTexts, selection)) {
|
||||
return new Error(
|
||||
"createComment: the selection spans multiple blocks; anchor on a " +
|
||||
"contiguous fragment within a SINGLE paragraph/block (<=250 chars)." +
|
||||
rolled,
|
||||
);
|
||||
}
|
||||
const where = live ? "in the live document" : "in the page";
|
||||
return new Error(
|
||||
`createComment: could not find the selection text ${where} to anchor ` +
|
||||
"the comment. Provide the EXACT contiguous text from a single " +
|
||||
"paragraph/block (<=250 chars)." +
|
||||
closestBlockHint(blockTexts, selection) +
|
||||
rolled,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an inline comment anchored to its `selection` text, or a reply.
|
||||
*
|
||||
* Top-level comments (no `parentCommentId`) are ALWAYS inline and MUST carry a
|
||||
* `selection`: the `type` argument is kept for interface compatibility but the
|
||||
* effective type is coerced to "inline". The selection has to anchor in the
|
||||
* document; if it cannot, the comment is rolled back and an error is thrown so
|
||||
* the caller is forced to supply a proper inline selection rather than leaving
|
||||
* an orphan, unanchored comment behind. Replies (parentCommentId set) inherit
|
||||
* their parent's anchor: they take NO selection and are not anchored.
|
||||
*/
|
||||
async createComment(
|
||||
pageId: string,
|
||||
content: string,
|
||||
type: "page" | "inline" = "page",
|
||||
selection?: string,
|
||||
parentCommentId?: string,
|
||||
suggestedText?: string,
|
||||
) {
|
||||
// Fail fast (#436): a provided parent id must be a full UUID before any
|
||||
// network call. Validate only when truthy — a falsy parentCommentId means
|
||||
// "top-level comment" (mirrors the isReply computation below), not a reply.
|
||||
if (parentCommentId) {
|
||||
assertFullUuid("createComment", "parentCommentId", parentCommentId);
|
||||
}
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
const isReply = !!parentCommentId;
|
||||
const hasSuggestion =
|
||||
suggestedText !== undefined && suggestedText !== null;
|
||||
// Defense in depth mirroring the server DTO/service: a suggested edit rewrites
|
||||
// the exact anchored text, so it is only meaningful on a top-level inline
|
||||
// comment that carries a selection.
|
||||
if (hasSuggestion) {
|
||||
if (isReply) {
|
||||
throw new Error(
|
||||
"createComment: a suggested edit (suggestedText) cannot be attached to a reply; it applies only to a top-level inline comment.",
|
||||
);
|
||||
}
|
||||
if (!selection || !selection.trim()) {
|
||||
throw new Error(
|
||||
"createComment: a suggested edit (suggestedText) requires a 'selection' to anchor and rewrite.",
|
||||
);
|
||||
}
|
||||
}
|
||||
// Only top-level comments are inline-anchored, so they are stored as
|
||||
// "inline". Replies carry no inline selection, so they keep the historical
|
||||
// general ("page") type — both backward-compatible and semantically correct.
|
||||
// The `type` argument is kept for interface compatibility; createComment
|
||||
// normalizes the effective type internally, so callers may pass "inline".
|
||||
const effectiveType: "page" | "inline" = isReply ? "page" : "inline";
|
||||
if (!isReply && (!selection || !selection.trim())) {
|
||||
throw new Error(
|
||||
"createComment: an inline 'selection' (exact text to anchor on) is required for a top-level comment",
|
||||
);
|
||||
}
|
||||
|
||||
// For a SUGGESTION, the value we store as the comment's `selection` must be
|
||||
// the RAW document substring the mark lands on (typographic quotes/dashes,
|
||||
// nbsp, collapsed whitespace), NOT the agent's ASCII input. The anchor is
|
||||
// placed via normalization, so when the doc was auto-converted to
|
||||
// typographic the raw substring differs from the agent input; apply-time
|
||||
// compares the stored selection to the marked doc text STRICTLY, so storing
|
||||
// the raw substring is what makes "Apply" succeed instead of a spurious 409.
|
||||
// Captured in the pre-check below (which already reads the page) and used as
|
||||
// payload.selection. Ordinary comments keep sending the raw agent selection.
|
||||
let anchoredSelection: string | null = null;
|
||||
// Set when the anchor matched only after stripping markdown from the
|
||||
// selection (the strip fallback); surfaced as a soft warning like
|
||||
// editPageText does, so a stale-markdown selection is flagged.
|
||||
let anchorNormalized = false;
|
||||
|
||||
// For a top-level comment, fail BEFORE creating anything when the selection
|
||||
// is not present in the persisted document — this avoids leaving an orphan
|
||||
// comment + notification behind. A read failure (network) is non-fatal: the
|
||||
// live anchor step below still enforces the anchoring invariant.
|
||||
if (!isReply && selection) {
|
||||
try {
|
||||
const page = await this.getPageJson(pageId);
|
||||
if (hasSuggestion) {
|
||||
// A suggestion's anchor MUST be unambiguous: applying it rewrites the
|
||||
// exact anchored text, and ordinary anchoring silently takes the first
|
||||
// occurrence, so 0 matches -> not found and >=2 -> ambiguous, both
|
||||
// rejected BEFORE creating the comment.
|
||||
const matches = countAnchorMatches(page.content, selection);
|
||||
if (matches === 0) {
|
||||
throw this.anchorNotFoundError(page.content, selection, false);
|
||||
}
|
||||
if (matches >= 2) {
|
||||
throw new Error(
|
||||
`createComment: the suggestion's selection is ambiguous — it occurs ${matches} times in the page. ` +
|
||||
"A suggested edit must anchor to a UNIQUE location; expand the selection with surrounding context " +
|
||||
"(still <=250 chars) so it appears exactly once.",
|
||||
);
|
||||
}
|
||||
// Exactly one match: capture the RAW anchored substring to store as the
|
||||
// comment selection (so apply-time equality holds). If this returns
|
||||
// null despite countAnchorMatches===1 (shouldn't happen), fall back to
|
||||
// the raw agent selection below rather than crash.
|
||||
anchoredSelection = getAnchoredText(page.content, selection);
|
||||
anchorNormalized = resolveAnchorSelection(
|
||||
page.content,
|
||||
selection,
|
||||
).normalized;
|
||||
} else {
|
||||
const resolved = resolveAnchorSelection(page.content, selection);
|
||||
if (!resolved.found) {
|
||||
throw this.anchorNotFoundError(page.content, selection, false);
|
||||
}
|
||||
anchorNormalized = resolved.normalized;
|
||||
}
|
||||
} catch (e) {
|
||||
// Rethrow our own "not found"/"ambiguous"/"spans multiple blocks" errors;
|
||||
// swallow read/network errors so the live anchor step can still try (and
|
||||
// enforce) anchoring.
|
||||
if (
|
||||
e instanceof Error &&
|
||||
(e.message.startsWith("createComment: could not find the selection") ||
|
||||
e.message.startsWith(
|
||||
"createComment: the selection spans multiple blocks",
|
||||
) ||
|
||||
e.message.startsWith(
|
||||
"createComment: the suggestion's selection is ambiguous",
|
||||
))
|
||||
) {
|
||||
throw e;
|
||||
}
|
||||
if (process.env.DEBUG) {
|
||||
console.error(
|
||||
"Pre-check getPageJson failed; deferring to live anchor step:",
|
||||
e,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Convert through the full Docmost schema. Deliberately the NON-canonicalizing
|
||||
// variant: a comment body may carry a footnote definition with no matching
|
||||
// reference, and canonicalization would drop it (data loss). See
|
||||
// markdownToProseMirror vs markdownToProseMirrorCanonical.
|
||||
const jsonContent = await markdownToProseMirror(content);
|
||||
const payload: Record<string, any> = {
|
||||
pageId,
|
||||
content: JSON.stringify(jsonContent),
|
||||
type: effectiveType,
|
||||
};
|
||||
// For a suggestion, store the RAW anchored substring (anchoredSelection) so
|
||||
// the stored selection === the text under the mark === apply-time
|
||||
// expectedText. Ordinary comments (and the null fallback) keep the raw
|
||||
// agent selection — their selection is only display/anchor and never used
|
||||
// by apply, so their behavior is unchanged.
|
||||
if (!isReply && selection)
|
||||
payload.selection = anchoredSelection ?? selection;
|
||||
if (parentCommentId) payload.parentCommentId = parentCommentId;
|
||||
// Only a top-level inline comment (with a selection) may carry a suggestion.
|
||||
if (!isReply && selection && hasSuggestion) {
|
||||
payload.suggestedText = suggestedText;
|
||||
}
|
||||
|
||||
const response = await this.client.post("/comments/create", payload);
|
||||
const comment = response.data.data || response.data;
|
||||
const markdown = comment.content
|
||||
? convertProseMirrorToMarkdown(this.parseCommentContent(comment.content))
|
||||
: content;
|
||||
const result: any = {
|
||||
data: filterComment(comment, markdown),
|
||||
success: true,
|
||||
};
|
||||
|
||||
// Replies inherit the parent's anchor: no selection, no anchoring.
|
||||
if (isReply) {
|
||||
return result;
|
||||
}
|
||||
|
||||
// Anchor the comment in the document. The /comments/create API records the
|
||||
// comment + its `selection` text, but it does NOT insert the comment MARK
|
||||
// into the page content, so without this the inline comment has no
|
||||
// highlight/anchor and is not clickable. If anchoring fails the comment is
|
||||
// rolled back (deleted) and an error is thrown — never an orphan comment.
|
||||
const newCommentId: string = comment.id;
|
||||
// Guard: a create response without an id would mean writing a comment mark
|
||||
// with commentId: undefined and a later delete of a falsy id. We have no id
|
||||
// to roll back here (nothing was created with an id), so just fail loudly.
|
||||
if (!newCommentId) {
|
||||
throw new Error(
|
||||
"createComment: the server returned no comment id, so the comment could not be anchored",
|
||||
);
|
||||
}
|
||||
let anchored = false;
|
||||
// Set inside the transform when a suggestion's live anchor is ambiguous
|
||||
// (>=2 occurrences), so the rollback path can surface the right error.
|
||||
let ambiguousInLiveDoc = false;
|
||||
// Captured inside the transform on a not-found abort, so the rollback path
|
||||
// can surface the closest-block / spans-multiple-blocks hint built from the
|
||||
// LIVE document (the pre-check page is not in scope there).
|
||||
let liveNotFoundError: Error | null = null;
|
||||
try {
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260). The
|
||||
// /comments/create REST call above keeps the agent-supplied id.
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
// Route through the mutatePage seam (not the free function) so this
|
||||
// wrapper's uniqueness gate + rollback can be unit-tested without a live
|
||||
// Hocuspocus collab socket.
|
||||
const mutation = await this.mutatePage(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
const doc =
|
||||
liveDoc && liveDoc.type === "doc"
|
||||
? liveDoc
|
||||
: { type: "doc", content: [] };
|
||||
if (hasSuggestion) {
|
||||
// Authoritative uniqueness check against the LIVE document: a
|
||||
// suggestion must anchor to EXACTLY ONE occurrence, otherwise
|
||||
// "Apply" would rewrite the wrong/ambiguous text. If the live doc
|
||||
// no longer has exactly one occurrence (it changed since the
|
||||
// pre-check), abort so the just-created comment is rolled back
|
||||
// rather than mis-anchored to the first occurrence.
|
||||
const liveCount = countAnchorMatches(doc, selection as string);
|
||||
if (liveCount !== 1) {
|
||||
ambiguousInLiveDoc = liveCount >= 2;
|
||||
if (liveCount === 0) {
|
||||
liveNotFoundError = this.anchorNotFoundError(
|
||||
doc,
|
||||
selection as string,
|
||||
true,
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (applyAnchorInDoc(doc, selection as string, newCommentId)) {
|
||||
anchored = true;
|
||||
return doc;
|
||||
}
|
||||
// Selection text not found in the LIVE document: abort the write. The
|
||||
// rollback + throw below turns this into a hard error.
|
||||
liveNotFoundError = this.anchorNotFoundError(
|
||||
doc,
|
||||
selection as string,
|
||||
true,
|
||||
);
|
||||
return null;
|
||||
},
|
||||
);
|
||||
result.verify = mutation.verify;
|
||||
} catch (e) {
|
||||
// The comment record already exists; roll it back so we never leave an
|
||||
// orphan, then rethrow the original anchoring error.
|
||||
await this.safeDeleteComment(newCommentId);
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (!anchored) {
|
||||
// Mutation aborted because the selection was not found (or, for a
|
||||
// suggestion, was ambiguous) in the live document. Roll back the comment
|
||||
// and surface a hard error.
|
||||
await this.safeDeleteComment(newCommentId);
|
||||
if (ambiguousInLiveDoc) {
|
||||
throw new Error(
|
||||
"createComment: the suggestion's selection is ambiguous in the live document (multiple occurrences); the comment was rolled back. Expand the selection with surrounding context so it is unique.",
|
||||
);
|
||||
}
|
||||
throw (
|
||||
liveNotFoundError ??
|
||||
new Error(
|
||||
"createComment: failed to anchor the comment (selection not found in the live document); the comment was rolled back",
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Soft warning (like editPageText): the selection only matched after
|
||||
// stripping markdown, so the caller likely quoted a styled fragment.
|
||||
if (anchorNormalized) {
|
||||
result.warning =
|
||||
"The selection matched only after stripping markdown syntax; the comment " +
|
||||
"was anchored on the document's plain text. Copy the selection verbatim " +
|
||||
"from getPage / searchInPage output to avoid this.";
|
||||
}
|
||||
|
||||
result.anchored = true;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort rollback of a just-created comment. Swallows any delete failure
|
||||
* (logging under DEBUG) so a failed cleanup never masks the original error.
|
||||
*/
|
||||
protected async safeDeleteComment(commentId: string): Promise<void> {
|
||||
// Defense in depth: never call the delete API with a falsy id — there is
|
||||
// nothing to roll back, and deleteComment(undefined) would hit a bad route.
|
||||
if (!commentId) return;
|
||||
try {
|
||||
await this.deleteComment(commentId);
|
||||
} catch (delErr) {
|
||||
if (process.env.DEBUG) {
|
||||
console.error(
|
||||
"Failed to roll back comment after anchoring error:",
|
||||
delErr,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
async updateComment(commentId: string, content: string) {
|
||||
// Fail fast (#436): reject a truncated id before any network call.
|
||||
assertFullUuid("updateComment", "commentId", commentId);
|
||||
await this.ensureAuthenticated();
|
||||
// NON-canonicalizing on purpose (comment body — see createComment).
|
||||
const jsonContent = await markdownToProseMirror(content);
|
||||
await this.client.post("/comments/update", {
|
||||
commentId,
|
||||
content: JSON.stringify(jsonContent),
|
||||
});
|
||||
return {
|
||||
success: true,
|
||||
commentId,
|
||||
message: "Comment updated successfully.",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
async deleteComment(commentId: string) {
|
||||
// Fail fast (#436): reject a truncated id before any network call.
|
||||
assertFullUuid("deleteComment", "commentId", commentId);
|
||||
await this.ensureAuthenticated();
|
||||
return this.client
|
||||
.post("/comments/delete", { commentId })
|
||||
.then((res) => res.data);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve or reopen a top-level comment thread (reversible — `resolved`
|
||||
* toggles the state). Only top-level comments can be resolved; the server
|
||||
* rejects resolving a reply. Hits POST /comments/resolve.
|
||||
*/
|
||||
async resolveComment(commentId: string, resolved: boolean) {
|
||||
// Fail fast (#436): reject a truncated id before any network call.
|
||||
assertFullUuid("resolveComment", "commentId", commentId);
|
||||
await this.ensureAuthenticated();
|
||||
const response = await this.client.post("/comments/resolve", {
|
||||
commentId,
|
||||
resolved,
|
||||
});
|
||||
const comment = response.data?.data ?? response.data;
|
||||
return {
|
||||
success: true,
|
||||
commentId,
|
||||
resolved,
|
||||
comment,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for new comments across pages in a space (optionally scoped to a
|
||||
* subtree): pages updated after `since` are scanned and their comments
|
||||
* filtered by createdAt > since.
|
||||
*/
|
||||
async checkNewComments(
|
||||
spaceId: string,
|
||||
since: string,
|
||||
parentPageId?: string,
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
const sinceDate = new Date(since);
|
||||
|
||||
// Reject an unparseable `since`: comparing against an Invalid Date silently
|
||||
// yields zero new comments (every `>` against NaN is false), which would
|
||||
// mask a malformed input as "nothing new" instead of erroring.
|
||||
if (Number.isNaN(sinceDate.getTime())) {
|
||||
throw new Error(
|
||||
`checkNewComments: invalid "since" date "${since}"; expected an ISO-8601 timestamp`,
|
||||
);
|
||||
}
|
||||
|
||||
// 1. Enumerate the FULL set of pages in scope via the page tree (a complete
|
||||
// page index), NOT the bounded "/pages/recent" feed which caps at ~5000
|
||||
// recent items and silently misses comments on older pages.
|
||||
//
|
||||
// Subtree scope: when parentPageId is given, the scope is that page ITSELF
|
||||
// plus every descendant. Otherwise the scope is the whole space (all roots
|
||||
// and their descendants).
|
||||
//
|
||||
// NOTE: do NOT pre-filter by page.updatedAt — creating a comment does not
|
||||
// bump it (verified on a live server), so such a filter silently misses
|
||||
// comments on pages that were not otherwise edited. The complete tree walk
|
||||
// already restricts the scope correctly, so no recent-feed allow-list is
|
||||
// needed any more.
|
||||
//
|
||||
// The subtree scope (parentPageId given) already INCLUDES the root node
|
||||
// itself: /pages/tree seeds getPageAndDescendants with id = parentPageId, so
|
||||
// no separate getPageRaw fetch for the parent is needed.
|
||||
const { pages: pagesInScope, truncated } = await this.enumerateSpacePages(
|
||||
spaceId,
|
||||
parentPageId,
|
||||
);
|
||||
|
||||
// 2. Fetch comments for each page, keep ones created after since
|
||||
const results: any[] = [];
|
||||
for (const page of pagesInScope) {
|
||||
try {
|
||||
// Full feed (incl. resolved): a "new comments since" scan reports all
|
||||
// recent activity; the active-only filter is scoped to listComments.
|
||||
const comments = (await this.listComments(page.id, true)).items;
|
||||
const newComments = comments.filter(
|
||||
(c: any) => new Date(c.createdAt) > sinceDate,
|
||||
);
|
||||
if (newComments.length > 0) {
|
||||
results.push({
|
||||
pageId: page.id,
|
||||
pageTitle: page.title,
|
||||
comments: newComments,
|
||||
});
|
||||
}
|
||||
} catch (e: any) {
|
||||
// Skip pages with errors (e.g. deleted between calls)
|
||||
}
|
||||
}
|
||||
|
||||
const totalNewComments = results.reduce(
|
||||
(sum, r) => sum + r.comments.length,
|
||||
0,
|
||||
);
|
||||
|
||||
// `truncated` is reported by enumerateSpacePages: it is true ONLY when the
|
||||
// stdio fallback BFS hit its node cap. The primary /pages/tree path is
|
||||
// uncapped, so a space with legitimately many pages is not falsely flagged.
|
||||
return {
|
||||
since,
|
||||
scope: parentPageId ? `subtree of ${parentPageId}` : `space ${spaceId}`,
|
||||
checkedPages: pagesInScope.length,
|
||||
pagesWithNewComments: results.length,
|
||||
totalNewComments,
|
||||
truncated,
|
||||
comments: results,
|
||||
};
|
||||
}
|
||||
|
||||
// --- Image upload / embedding ---
|
||||
|
||||
/** Map a Content-Type string to a supported MIME type, or null if unsupported. */
|
||||
}
|
||||
return CommentsMixin;
|
||||
}
|
||||
@@ -0,0 +1,745 @@
|
||||
// Shared client context + core seams (issue #450). The abstract base of the
|
||||
// DocmostClient mixin chain: it owns ALL shared instance state (the axios
|
||||
// client, apiUrl, auth tokens, the resolvePageId cache, the collab-token cache,
|
||||
// the sandbox/metrics sinks) and the core HTTP/auth/pagination/write seams every
|
||||
// domain module builds on. Domain modules are mixins layered on top; the final
|
||||
// DocmostClient (client.ts) assembles them. Extracted VERBATIM from the original
|
||||
// monolith — only field/seam visibility was widened from `private` to
|
||||
// `protected` so sibling mixins can reach the shared state through `this`, and
|
||||
// the cross-module methods that live in other mixins are declared `abstract`
|
||||
// here so `this.<method>` type-checks. No behaviour changed.
|
||||
import axios, { AxiosInstance } from "axios";
|
||||
import FormData from "form-data";
|
||||
import {
|
||||
updatePageContentRealtime,
|
||||
replacePageContent,
|
||||
markdownToProseMirror,
|
||||
markdownToProseMirrorCanonical,
|
||||
mutatePageContent,
|
||||
assertYjsEncodable,
|
||||
MutationResult,
|
||||
} from "../lib/collaboration.js";
|
||||
import {
|
||||
acquireCollabSession,
|
||||
isCollabAuthFailedError,
|
||||
} from "../lib/collab-session.js";
|
||||
import { withPageLock, isUuid } from "../lib/page-lock.js";
|
||||
import { getCollabToken, performLogin } from "../lib/auth-utils.js";
|
||||
import { formatDocmostAxiosError } from "./errors.js";
|
||||
import { GetPageConversionCache } from "./getpage-cache.js";
|
||||
|
||||
// A generic mixin base constructor (issue #450). Each domain mixin is a factory
|
||||
// `<T extends GConstructor<DocmostClientContext>>(Base: T) => class extends Base`
|
||||
// so the mixins compose into one prototype chain sharing this context.
|
||||
export type GConstructor<T = {}> = abstract new (...args: any[]) => T;
|
||||
|
||||
/**
|
||||
* Configuration for a DocmostClient / MCP server instance. A discriminated
|
||||
* union: either service-account credentials (email/password — the client calls
|
||||
* performLogin, powering the external /mcp HTTP endpoint and the stdio CLI) OR
|
||||
* a token getter (getToken — the client uses the returned BARE access JWT as
|
||||
* the Bearer and never calls performLogin; used for the internal per-user path).
|
||||
*
|
||||
* Both branches may ALSO carry an optional `getCollabToken` provider. When set,
|
||||
* content mutations (which go over the collaboration websocket) use the token it
|
||||
* returns INSTEAD of calling `POST /auth/collab-token`. The internal per-user
|
||||
* agent path uses this to hand the client a provenance collab token (signed
|
||||
* `actor:'agent'`+`aiChatId`), so agent content edits are attributed without a
|
||||
* spoofable client-side field. When absent the client keeps the original
|
||||
* `/auth/collab-token` path (service-account/stdio unchanged).
|
||||
*
|
||||
* Housed here (not in index.ts) so client.ts has no type dependency on index.ts;
|
||||
* index.ts re-exports it for the package's public surface.
|
||||
*/
|
||||
// Sink the stash tool writes blobs into. The host app binds this to its in-RAM
|
||||
// SandboxStore and composes the public `uri` (the package never sees the store
|
||||
// or any env). `put` returns the anonymous read URL plus integrity metadata.
|
||||
export type SandboxPut = (
|
||||
buf: Buffer,
|
||||
mime: string,
|
||||
) => { uri: string; sha256: string; size: number };
|
||||
|
||||
export type DocmostMcpConfig = { apiUrl: string } & (
|
||||
| { email: string; password: string }
|
||||
| { getToken: () => Promise<string> } // returns a BARE JWT; the client adds "Bearer "
|
||||
) & {
|
||||
// Optional collab-token provider (returns a ready collab JWT). Common to
|
||||
// both branches; see the type doc above.
|
||||
getCollabToken?: () => Promise<string>;
|
||||
// Optional blob sandbox sink. Present only where the stash tool is wired;
|
||||
// when absent, stashPage throws a clear "not configured" error. The
|
||||
// optional `has`/`evict` probes let stashPage keep its mirror counts honest
|
||||
// under the store's FIFO eviction (see stashPage); older sinks omit them.
|
||||
sandbox?: {
|
||||
put: SandboxPut;
|
||||
has?: (uri: string) => boolean;
|
||||
evict?: (uri: string) => void;
|
||||
};
|
||||
// Dependency-neutral metrics sink. When present, the client emits generic
|
||||
// (name, value, labels) samples; the HOST maps those names onto its own
|
||||
// metrics registry (the package never depends on prom-client or the server).
|
||||
// Absent in standalone/stdio mode → the client is a complete no-op here.
|
||||
onMetric?: (
|
||||
name: string,
|
||||
value: number,
|
||||
labels?: Record<string, string>,
|
||||
) => void;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Collab-token cache TTL in milliseconds (issue #435). Read fresh from the
|
||||
* environment on every mint — like collab-session.ts readConfig — so tests and a
|
||||
* live rollback can change it without reloading the module.
|
||||
*
|
||||
* Why a cache at all: the live CollabSession registry (#400/#431) keys sessions
|
||||
* on (wsUrl, pageId, collabToken) for identity isolation (invariant 4). But BOTH
|
||||
* collab-token sources mint a FRESH token per mutation — the in-app provider
|
||||
* re-signs a JWT whose iat/exp (seconds) changes every second, and the external
|
||||
* MCP POSTs /auth/collab-token each call — so the token in the key changed on
|
||||
* every op and the session was almost never reused (connect-storms, 25s
|
||||
* timeouts, zombie sessions). Caching the token per-client keeps the key stable
|
||||
* across a burst of mutations so ONE session is reused.
|
||||
*
|
||||
* Default 5 min: well under the 24h collab-token lifetime AND <= the collab
|
||||
* session max-age (10 min, MCP_COLLAB_SESSION_MAX_AGE_MS), so the
|
||||
* permission-staleness window is not widened beyond what #431 already accepted.
|
||||
* The rollback knob is an EXPLICIT 0 (or a negative number): that DISABLES the
|
||||
* cache — an exact fetch-per-call legacy path, mirroring how idleMs<=0 disables
|
||||
* the session cache. Unset OR unparseable (e.g. a typo like "5min", "abc") falls
|
||||
* back to the 5-min default with the cache ON — parseInt yields NaN, which is
|
||||
* treated as "not configured", not as "disabled". So to turn the cache off you
|
||||
* must set the value to exactly 0, not to garbage.
|
||||
*/
|
||||
function readCollabTokenTtlMs(): number {
|
||||
const raw = parseInt(process.env.MCP_COLLAB_TOKEN_TTL_MS ?? "", 10);
|
||||
return Number.isFinite(raw) ? Math.max(0, raw) : 5 * 60 * 1000;
|
||||
}
|
||||
|
||||
export abstract class DocmostClientContext {
|
||||
protected client: AxiosInstance;
|
||||
protected token: string | null = null;
|
||||
protected apiUrl: string;
|
||||
// email/password are only set on the service-account (credentials) variant;
|
||||
// null on the getToken variant (where there are no credentials to log in with).
|
||||
protected email: string | null = null;
|
||||
protected password: string | null = null;
|
||||
// Per-user token provider. When set, login() calls it to obtain a BARE access
|
||||
// JWT instead of performLogin, and the 401/403 re-auth path re-calls it.
|
||||
protected getTokenFn: (() => Promise<string>) | null = null;
|
||||
// Optional collab-token provider. When set, getCollabTokenWithReauth() returns
|
||||
// its token instead of calling POST /auth/collab-token; on a 401/403 it is
|
||||
// re-invoked once. Used by the internal agent to carry signed provenance.
|
||||
protected getCollabTokenFn: (() => Promise<string>) | null = null;
|
||||
// Optional blob-sandbox sink for the stash tool. Null when not configured.
|
||||
protected sandboxPut: SandboxPut | null = null;
|
||||
// Optional probes paired with the sink. `has` lets stashPage detect a blob
|
||||
// FIFO-evicted by a LATER put in the same stash; `evict` lets it free this
|
||||
// op's image blobs if the final doc put throws. Null when the sink omits them.
|
||||
protected sandboxHas: ((uri: string) => boolean) | null = null;
|
||||
protected sandboxEvict: ((uri: string) => void) | null = null;
|
||||
// Optional dependency-neutral metrics sink (see DocmostMcpConfig.onMetric).
|
||||
// Null on the legacy positional form and whenever the host omits it → no-op.
|
||||
protected onMetricFn:
|
||||
| ((name: string, value: number, labels?: Record<string, string>) => void)
|
||||
| null = null;
|
||||
// In-flight login dedup: when the token expires, the 401 interceptor,
|
||||
// ensureAuthenticated, getCollabTokenWithReauth and the two multipart retries
|
||||
// can all call login() at once. Memoizing a single promise collapses that
|
||||
// thundering herd into ONE /auth/login request that everyone awaits.
|
||||
protected loginPromise: Promise<void> | null = null;
|
||||
// Canonical-UUID cache for resolvePageId: maps an agent-supplied slugId to the
|
||||
// page's canonical UUID, so repeated collab edits on the same page do not
|
||||
// re-fetch /pages/info. A UUID input short-circuits before this cache (see
|
||||
// resolvePageId), so only slugId->uuid entries are stored/read here.
|
||||
protected pageIdCache = new Map<string, string>();
|
||||
|
||||
// Collab-token cache (issue #435): the last minted collab token plus the
|
||||
// wall-clock time it was minted, so a burst of content mutations reuses ONE
|
||||
// token and therefore ONE live CollabSession (whose registry key includes the
|
||||
// token — #400 invariant 4). Per-instance: a DocmostClient is built per
|
||||
// user/per chat request, so a cached token can never leak across identities.
|
||||
// Reset whenever the client's identity changes (login() / this.token cleared);
|
||||
// bypassed on a forced refresh (the 401/403 reauth path). null = no token yet.
|
||||
protected collabTokenCache: { token: string; mintedAt: number } | null = null;
|
||||
|
||||
// Content-addressed conversion cache for getPage (issue #479). Keyed on
|
||||
// (canonical pageId, updatedAt, optionsHash) -> the converted Markdown, so a
|
||||
// re-read of an UNCHANGED page skips the expensive convertProseMirrorToMarkdown
|
||||
// tree walk. Per-instance (a DocmostClient is built per user / per chat), so a
|
||||
// cached conversion can never leak across identities. See getpage-cache.ts.
|
||||
protected getPageCache = new GetPageConversionCache();
|
||||
|
||||
// Two construction forms:
|
||||
// - new DocmostClient(config) // discriminated union (current)
|
||||
// - new DocmostClient(baseURL, email, password) // legacy positional creds
|
||||
// The positional form is retained so existing callers/tests keep working; it
|
||||
// is exactly equivalent to the credentials branch of the object form.
|
||||
constructor(config: DocmostMcpConfig);
|
||||
constructor(baseURL: string, email: string, password: string);
|
||||
constructor(
|
||||
configOrBaseURL: DocmostMcpConfig | string,
|
||||
email?: string,
|
||||
password?: string,
|
||||
) {
|
||||
// Normalize the legacy positional form into the object union.
|
||||
const config: DocmostMcpConfig =
|
||||
typeof configOrBaseURL === "string"
|
||||
? { apiUrl: configOrBaseURL, email: email!, password: password! }
|
||||
: configOrBaseURL;
|
||||
|
||||
this.apiUrl = config.apiUrl;
|
||||
if ("getToken" in config) {
|
||||
// Token variant: carry the user's JWT via getToken; no credentials, so
|
||||
// login() must never call performLogin (there is nothing to log in with).
|
||||
this.getTokenFn = config.getToken;
|
||||
} else {
|
||||
// Service-account variant: behaves exactly as before (performLogin).
|
||||
this.email = config.email;
|
||||
this.password = config.password;
|
||||
}
|
||||
// Optional, available to both variants. When present, content mutations get
|
||||
// their collab token from here instead of POST /auth/collab-token.
|
||||
if (config.getCollabToken) {
|
||||
this.getCollabTokenFn = config.getCollabToken;
|
||||
}
|
||||
if (config.sandbox) {
|
||||
this.sandboxPut = config.sandbox.put;
|
||||
this.sandboxHas = config.sandbox.has ?? null;
|
||||
this.sandboxEvict = config.sandbox.evict ?? null;
|
||||
}
|
||||
// Legacy positional form carries no onMetric → null (complete no-op).
|
||||
this.onMetricFn = config.onMetric ?? null;
|
||||
this.client = axios.create({
|
||||
baseURL: this.apiUrl,
|
||||
// Default request timeout so a hung connection cannot wedge a per-page
|
||||
// lock or block the server indefinitely. Multipart uploads override this
|
||||
// with a longer per-request timeout.
|
||||
timeout: 30000,
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
// Re-authenticate transparently on a 401/403 once: the JWT authToken can
|
||||
// expire while the server is long-running, after which every cached-token
|
||||
// request would otherwise fail until a manual restart. On such a response,
|
||||
// clear the stale token, perform a fresh login, and replay the original
|
||||
// request exactly once (guarded by config._retry to avoid infinite loops;
|
||||
// the login request itself is never retried).
|
||||
this.client.interceptors.response.use(
|
||||
(response) => response,
|
||||
async (error) => {
|
||||
const config = error.config;
|
||||
const status = error.response?.status;
|
||||
const isAuthError = status === 401 || status === 403;
|
||||
const isLoginRequest =
|
||||
typeof config?.url === "string" && config.url.includes("/auth/login");
|
||||
|
||||
if (config && isAuthError && !config._retry && !isLoginRequest) {
|
||||
config._retry = true;
|
||||
// Drop the stale token + Authorization header before re-login. Also
|
||||
// clear the collab-token cache (#435): a new identity/login must not
|
||||
// keep serving a collab token minted under the old one.
|
||||
this.token = null;
|
||||
this.collabTokenCache = null;
|
||||
delete this.client.defaults.headers.common["Authorization"];
|
||||
try {
|
||||
await this.login();
|
||||
} catch (loginError) {
|
||||
// Re-login failed: surface the original error to the caller.
|
||||
return Promise.reject(error);
|
||||
}
|
||||
// Re-issue the original request with the freshly minted Bearer token.
|
||||
// Read it from the default header that login() just set, not from
|
||||
// this.token, to avoid a theoretical "Bearer null" if this.token was
|
||||
// cleared between login() resolving and this point.
|
||||
config.headers = config.headers || {};
|
||||
config.headers["Authorization"] =
|
||||
this.client.defaults.headers.common["Authorization"];
|
||||
return this.client.request(config);
|
||||
}
|
||||
|
||||
return Promise.reject(error);
|
||||
},
|
||||
);
|
||||
|
||||
// Diagnostics interceptor (issue #437). Registered AFTER the re-login
|
||||
// interceptor so a successful re-login retry (which resolves to a real
|
||||
// response) is never seen here as an error; only a genuine failure reaches
|
||||
// this rejection handler. It reformats error.message IN PLACE (see
|
||||
// formatDocmostAxiosError — kept as a mutation, not a custom Error class, so
|
||||
// the surrounding axios.isAxiosError / error.response?.status / config._retry
|
||||
// checks keep working) and re-rejects the SAME error. The _docmostFormatted
|
||||
// flag makes a re-processed retry-failure a no-op.
|
||||
this.client.interceptors.response.use(
|
||||
(response) => response,
|
||||
(error) => {
|
||||
formatDocmostAxiosError(error);
|
||||
return Promise.reject(error);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
// --- Cross-module seams (issue #450) -----------------------------------
|
||||
// A method in one domain mixin sometimes calls a PROTECTED method owned by
|
||||
// another mixin (e.g. nodes-write -> validateDocUrls in doc-validate). Those
|
||||
// callees are `protected`, so they cannot be surfaced through the public
|
||||
// per-mixin interfaces. Declaring them here on the shared base lets `this.<m>`
|
||||
// type-check across modules. Each is a stub that is ALWAYS overridden by the
|
||||
// owning mixin (layered above this base in the chain), so the body never runs;
|
||||
// it throws only to make an impossible mis-wiring loud instead of silent.
|
||||
// (The PUBLIC cross-module callees — getPage, getPageJson, listComments,
|
||||
// deleteComment, listPageHistory — arrive via the mixins' public interfaces,
|
||||
// so they are not restated here.)
|
||||
protected enumerateSpacePages(
|
||||
_spaceId: string,
|
||||
_rootPageId?: string,
|
||||
): Promise<{ pages: any[]; truncated: boolean }> {
|
||||
throw new Error("enumerateSpacePages not wired (missing ReadMixin)");
|
||||
}
|
||||
protected validateDocUrls(_node: any, _depth?: number): void {
|
||||
throw new Error("validateDocUrls not wired (missing DocValidateMixin)");
|
||||
}
|
||||
protected validateDocStructure(_node: any, _depth?: number): void {
|
||||
throw new Error("validateDocStructure not wired (missing DocValidateMixin)");
|
||||
}
|
||||
protected assertValidNodeShape(_op: string, _node: any): void {
|
||||
throw new Error("assertValidNodeShape not wired (missing DocValidateMixin)");
|
||||
}
|
||||
protected fetchInternalFile(
|
||||
_src: string,
|
||||
): Promise<{ buffer: Buffer; mime: string }> {
|
||||
throw new Error("fetchInternalFile not wired (missing StashMixin)");
|
||||
}
|
||||
protected uploadAttachmentBuffer(
|
||||
_pageId: string,
|
||||
_buffer: Buffer,
|
||||
_fileName: string,
|
||||
_mime: string,
|
||||
): Promise<{ id: string; fileName: string; fileSize: number }> {
|
||||
throw new Error("uploadAttachmentBuffer not wired (missing MediaMixin)");
|
||||
}
|
||||
protected fetchAttachmentText(_src: string): Promise<string> {
|
||||
throw new Error("fetchAttachmentText not wired (missing MediaMixin)");
|
||||
}
|
||||
// PUBLIC cross-module callees. Declared here too (as always-overridden stubs)
|
||||
// so a mixin calling e.g. `this.getPageJson` type-checks against the base —
|
||||
// the mixin's own public interface only covers its own methods. The real
|
||||
// implementations live in ReadMixin / CommentsMixin / PagesMixin and shadow
|
||||
// these on the prototype chain.
|
||||
getPage(_pageId: string): Promise<any> {
|
||||
throw new Error("getPage not wired (missing ReadMixin)");
|
||||
}
|
||||
getPageJson(_pageId: string): Promise<any> {
|
||||
throw new Error("getPageJson not wired (missing ReadMixin)");
|
||||
}
|
||||
listComments(_pageId: string, _includeResolved?: boolean): Promise<any> {
|
||||
throw new Error("listComments not wired (missing CommentsMixin)");
|
||||
}
|
||||
deleteComment(_commentId: string): Promise<any> {
|
||||
throw new Error("deleteComment not wired (missing CommentsMixin)");
|
||||
}
|
||||
listPageHistory(_pageId: string, _cursor?: string): Promise<any> {
|
||||
throw new Error("listPageHistory not wired (missing PagesMixin)");
|
||||
}
|
||||
|
||||
/** Application base URL (API URL without the /api suffix). */
|
||||
get appUrl(): string {
|
||||
return this.apiUrl.replace(/\/api\/?$/, "");
|
||||
}
|
||||
|
||||
|
||||
async login() {
|
||||
// Reuse an in-flight login if one is already running so concurrent callers
|
||||
// share a single token fetch instead of each issuing their own.
|
||||
if (!this.loginPromise) {
|
||||
// Token variant: re-fetch a BARE JWT via getToken() (there are no
|
||||
// credentials to log in with — on a 401/403 the interceptor below calls
|
||||
// login() again, which re-invokes getToken()). Credentials variant:
|
||||
// performLogin against /auth/login exactly as before.
|
||||
const fetchToken = this.getTokenFn
|
||||
? this.getTokenFn()
|
||||
: performLogin(this.apiUrl, this.email!, this.password!);
|
||||
this.loginPromise = fetchToken
|
||||
.then((token) => {
|
||||
// Guard against an empty/invalid token (e.g. a getToken provider that
|
||||
// resolves to "" or null): without this an empty token would set a
|
||||
// literal "Authorization: Bearer null"/"Bearer " header and every
|
||||
// request would 401 with a confusing error. Fail loudly instead.
|
||||
if (typeof token !== "string" || token.length === 0) {
|
||||
throw new Error("getToken returned an empty token");
|
||||
}
|
||||
this.token = token;
|
||||
// Identity (re)established: drop any collab token minted under a
|
||||
// previous identity so the #435 cache can never outlive it.
|
||||
this.collabTokenCache = null;
|
||||
this.client.defaults.headers.common["Authorization"] =
|
||||
`Bearer ${token}`;
|
||||
})
|
||||
.finally(() => {
|
||||
this.loginPromise = null;
|
||||
});
|
||||
}
|
||||
return this.loginPromise;
|
||||
}
|
||||
|
||||
|
||||
async ensureAuthenticated() {
|
||||
if (!this.token) {
|
||||
await this.login();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a collaboration token, transparently re-authenticating once on a
|
||||
* 401/403. getCollabToken() uses bare axios internally, so it is NOT covered
|
||||
* by this.client's response interceptor; this helper replicates that
|
||||
* behaviour for collab-token requests: ensure a token, try once, and on an
|
||||
* expired-token auth error perform a fresh login and retry exactly once.
|
||||
*
|
||||
* Collab-token cache (issue #435): both sources — the getCollabToken provider
|
||||
* (in-app agent) AND the REST /auth/collab-token endpoint (external MCP) — mint
|
||||
* a FRESH token per call, whose string therefore changes every op. Since the
|
||||
* live CollabSession registry keys on the token string (#400/#431 invariant 4),
|
||||
* that churned the key and defeated session reuse. So we cache the last minted
|
||||
* token per-client for readCollabTokenTtlMs() and hand it back for a burst of
|
||||
* mutations, keeping the session key stable. `forceRefresh` bypasses the cache
|
||||
* (the 401/403 reauth retry uses it, so the retry cannot be handed the same
|
||||
* stale token that just failed — otherwise reauth would be a no-op). TTL 0
|
||||
* disables the cache: exact fetch-per-call legacy behaviour.
|
||||
*/
|
||||
protected async getCollabTokenWithReauth(
|
||||
forceRefresh = false,
|
||||
): Promise<string> {
|
||||
const ttl = readCollabTokenTtlMs();
|
||||
// Serve the cached collab token while it is still fresh (identity isolation
|
||||
// is preserved: the cache is a per-instance field on a client built per
|
||||
// user/per chat request, and it is cleared on every identity change).
|
||||
if (
|
||||
!forceRefresh &&
|
||||
ttl > 0 &&
|
||||
this.collabTokenCache &&
|
||||
Date.now() - this.collabTokenCache.mintedAt < ttl
|
||||
) {
|
||||
return this.collabTokenCache.token;
|
||||
}
|
||||
|
||||
// Collab-token PROVIDER path: when a getCollabToken provider was supplied
|
||||
// (the internal agent's provenance collab token), use it instead of the
|
||||
// REST /auth/collab-token endpoint. Re-invoke it once on a 401/403 (e.g. the
|
||||
// signed token expired between content mutations in a long agent turn).
|
||||
if (this.getCollabTokenFn) {
|
||||
try {
|
||||
const token = await this.getCollabTokenFn();
|
||||
if (typeof token !== "string" || token.length === 0) {
|
||||
throw new Error("getCollabToken returned an empty token");
|
||||
}
|
||||
return this.rememberCollabToken(token, ttl);
|
||||
} catch (e) {
|
||||
// On an auth error retry EXACTLY once, forcing a refresh so the retry
|
||||
// re-invokes the provider (bypassing the cache) for a genuinely fresh
|
||||
// token. `!forceRefresh` bounds it to a single retry (no loop).
|
||||
if (this.isCollabAuthError(e) && !forceRefresh) {
|
||||
return this.getCollabTokenWithReauth(true);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
await this.ensureAuthenticated();
|
||||
try {
|
||||
const token = await getCollabToken(this.apiUrl, this.token!);
|
||||
return this.rememberCollabToken(token, ttl);
|
||||
} catch (e) {
|
||||
// getCollabToken wraps the AxiosError in a plain Error but attaches the
|
||||
// HTTP status as `.status`, so isCollabAuthError detects an auth failure
|
||||
// via either the raw AxiosError shape OR the attached status.
|
||||
if (this.isCollabAuthError(e) && !forceRefresh) {
|
||||
// Fresh login (which clears this.token AND the collab-token cache), then
|
||||
// retry exactly once with the cache bypassed via forceRefresh.
|
||||
await this.login();
|
||||
return this.getCollabTokenWithReauth(true);
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Store a freshly minted collab token in the per-client cache (issue #435) and
|
||||
* return it unchanged. No-op write when the cache is disabled (ttl<=0) or the
|
||||
* token is empty, so a disabled cache is exact fetch-per-call legacy behaviour
|
||||
* and a bad token is never cached.
|
||||
*/
|
||||
protected rememberCollabToken(token: string, ttl: number): string {
|
||||
if (ttl > 0 && typeof token === "string" && token.length > 0) {
|
||||
this.collabTokenCache = { token, mintedAt: Date.now() };
|
||||
}
|
||||
return token;
|
||||
}
|
||||
|
||||
/**
|
||||
* True when an error carries a 401/403 — either as a raw AxiosError
|
||||
* (`error.response.status`) or as the plain-Error `.status` that
|
||||
* lib/auth-utils.getCollabToken attaches after wrapping the AxiosError.
|
||||
*/
|
||||
protected isCollabAuthError(e: unknown): boolean {
|
||||
const axiosStatus = axios.isAxiosError(e) ? e.response?.status : undefined;
|
||||
const attachedStatus = (e as any)?.status;
|
||||
return (
|
||||
axiosStatus === 401 ||
|
||||
axiosStatus === 403 ||
|
||||
attachedStatus === 401 ||
|
||||
attachedStatus === 403
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a collab write and, on a Hocuspocus HANDSHAKE auth failure, self-heal
|
||||
* once (#486). Symmetric to the HTTP-401 path in getCollabTokenWithReauth: the
|
||||
* REST interceptor and login() already drop the cached collab token on a 401/
|
||||
* 403, but a rejected WEBSOCKET handshake left the stale token in the cache, so
|
||||
* every subsequent mutation kept re-presenting the same bad token for up to the
|
||||
* collab-token TTL (minutes) with no self-heal. Here, when the write rejects
|
||||
* with the tagged collab-auth error, we invalidate the cached token and retry
|
||||
* the write EXACTLY once with a force-refreshed token. Not a loop: a second
|
||||
* failure (or any non-auth error) propagates unchanged.
|
||||
*
|
||||
* `write` receives the token to use, so the retry can hand it a genuinely fresh
|
||||
* one rather than re-running with the same stale string.
|
||||
*/
|
||||
protected async writeWithCollabAuthRetry<T>(
|
||||
collabToken: string,
|
||||
write: (token: string) => Promise<T>,
|
||||
): Promise<T> {
|
||||
try {
|
||||
return await write(collabToken);
|
||||
} catch (e) {
|
||||
if (!isCollabAuthFailedError(e)) throw e;
|
||||
// The WS handshake rejected our token: drop it from the cache so it can't
|
||||
// be reused for the rest of the TTL, mint a fresh one (forceRefresh bypasses
|
||||
// the cache and re-invokes the provider/login), and retry the write once.
|
||||
this.collabTokenCache = null;
|
||||
const fresh = await this.getCollabTokenWithReauth(true);
|
||||
return await write(fresh);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to the collaboration websocket, read the live doc, apply
|
||||
* `transform`, write the result, and wait for the server to persist it —
|
||||
* WITHOUT acquiring the per-page lock.
|
||||
*
|
||||
* This mirrors collaboration.mutatePageContent EXCEPT that it does not call
|
||||
* withPageLock. It exists solely so replaceImage can hold ONE withPageLock
|
||||
* across its scan -> upload -> write sequence: the per-page mutex is NOT
|
||||
* reentrant, so calling the normal (self-locking) mutatePageContent inside an
|
||||
* outer withPageLock for the same pageId would deadlock. The caller MUST hold
|
||||
* the page lock for the whole operation; this helper assumes that invariant.
|
||||
*
|
||||
* `transform` receives the live ProseMirror doc and returns the NEW full doc
|
||||
* to write, or `null` to abort with no write. Errors thrown by `transform`
|
||||
* propagate to the caller.
|
||||
*
|
||||
* Resolves a `MutationResult { doc, verify }` mirroring mutatePageContent, so
|
||||
* every content mutator (including replaceImage) can return a verifiable
|
||||
* change report. The report is computed AFTER the atomic read->write and
|
||||
* never throws.
|
||||
*/
|
||||
protected async mutateLiveContentUnlocked(
|
||||
pageId: string,
|
||||
collabToken: string,
|
||||
transform: (liveDoc: any) => any | null,
|
||||
): Promise<MutationResult> {
|
||||
// Reuse a live CollabSession for the page (issue #400) instead of opening a
|
||||
// fresh provider per op. acquireCollabSession does NOT take the per-page
|
||||
// lock — the caller (replaceImage) already holds ONE withPageLock across its
|
||||
// scan -> upload -> write sequence, and the mutex is not reentrant, so
|
||||
// taking it here would deadlock. The synchronous read->write section and the
|
||||
// unsyncedChanges/connectionLost ack logic live in CollabSession.mutate,
|
||||
// preserved verbatim from the old inline machine (incl. the #152 structural
|
||||
// diff that keeps a live editor's cursor anchored).
|
||||
// Wrap in the collab-auth self-heal (#486): a rejected WS handshake drops the
|
||||
// cached collab token and retries once with a fresh one (the retry passes the
|
||||
// refreshed token down to acquireCollabSession via `token`).
|
||||
return this.writeWithCollabAuthRetry(collabToken, async (token) => {
|
||||
const session = await acquireCollabSession(pageId, token, this.apiUrl, {
|
||||
// Only the actual 25s collab connect timeout emits this — the connect-vs-
|
||||
// unload signal; the other failure paths must NOT emit it.
|
||||
onConnectTimeout: () =>
|
||||
this.onMetricFn?.("collab_connect_timeouts_total", 1),
|
||||
});
|
||||
try {
|
||||
return await session.mutate(transform);
|
||||
} catch (e) {
|
||||
// Drop the session on any failure so the next call reconnects fresh.
|
||||
session.destroy("mutate failed");
|
||||
throw e;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generic pagination handler for Docmost API endpoints
|
||||
*/
|
||||
async paginateAll<T = any>(
|
||||
endpoint: string,
|
||||
basePayload: Record<string, any> = {},
|
||||
limit: number = 100,
|
||||
): Promise<T[]> {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
const clampedLimit = Math.max(1, Math.min(100, limit));
|
||||
|
||||
// Hard ceiling on the number of pages to fetch: guards against a server
|
||||
// that returns a perpetually-true hasNextPage (which would otherwise loop
|
||||
// forever and accumulate duplicates).
|
||||
const MAX_PAGES = 50;
|
||||
|
||||
let cursor: string | undefined;
|
||||
let allItems: T[] = [];
|
||||
let truncated = false;
|
||||
|
||||
for (let page = 0; page < MAX_PAGES; page++) {
|
||||
const payload: Record<string, any> = {
|
||||
...basePayload,
|
||||
limit: clampedLimit,
|
||||
};
|
||||
if (cursor) payload.cursor = cursor;
|
||||
|
||||
const response = await this.client.post(endpoint, payload);
|
||||
|
||||
const data = response.data;
|
||||
const items = data.data?.items || data.items || [];
|
||||
const meta = data.data?.meta || data.meta;
|
||||
|
||||
allItems = allItems.concat(items);
|
||||
|
||||
// Advance strictly via the server-issued cursor. A missing nextCursor (or
|
||||
// hasNextPage false) means we reached the end. A cursor identical to the
|
||||
// one we just sent means the server did not understand our pagination
|
||||
// param — stop instead of re-fetching page one forever and duplicating.
|
||||
const next = meta?.hasNextPage ? meta?.nextCursor : null;
|
||||
if (!next || next === cursor) {
|
||||
// If the server still reports more pages but stopped issuing a usable
|
||||
// cursor at the ceiling, flag the result as truncated below.
|
||||
if (page === MAX_PAGES - 1 && meta?.hasNextPage) truncated = true;
|
||||
break;
|
||||
}
|
||||
cursor = next;
|
||||
|
||||
// Reaching the ceiling with more pages still available means the result
|
||||
// set is truncated.
|
||||
if (page === MAX_PAGES - 1) truncated = true;
|
||||
}
|
||||
|
||||
// If the loop stopped because it hit the MAX_PAGES ceiling while the server
|
||||
// still reported more results, the result set is truncated — warn so the
|
||||
// caller is not silently handed an incomplete list.
|
||||
if (truncated) {
|
||||
console.warn(
|
||||
`paginateAll: results from "${endpoint}" truncated at the ${MAX_PAGES}-page cap; more pages exist on the server`,
|
||||
);
|
||||
}
|
||||
|
||||
return allItems;
|
||||
}
|
||||
|
||||
|
||||
/** Raw page info including the ProseMirror JSON content and slugId. */
|
||||
async getPageRaw(pageId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
const response = await this.client.post("/pages/info", { pageId });
|
||||
return response.data?.data ?? response.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve an agent-supplied pageId to the page's CANONICAL UUID (`page.id`),
|
||||
* so every collaboration document the MCP opens is named `page.<uuid>` — the
|
||||
* SAME name the web editor always uses (`page.${page.id}`).
|
||||
*
|
||||
* The agent commonly passes a 10-char public slugId (from URLs/listings) as
|
||||
* the pageId. The web editor opens the collab doc by UUID, but the MCP used to
|
||||
* pass that slugId straight into the collab doc name (`page.<slugId>`). For one
|
||||
* DB row that produced TWO independent Yjs documents whose debounced stores
|
||||
* clobbered each other — the agent's edit was silently lost (#260).
|
||||
*
|
||||
* A UUID input short-circuits with no network round-trip. A slugId is resolved
|
||||
* once via getPageRaw and cached (both slugId->uuid and uuid->uuid), so
|
||||
* repeated edits on the same page add no extra request.
|
||||
*/
|
||||
protected async resolvePageId(pageId: string): Promise<string> {
|
||||
if (isUuid(pageId)) return pageId;
|
||||
const cached = this.pageIdCache.get(pageId);
|
||||
if (cached) return cached;
|
||||
const data = await this.getPageRaw(pageId);
|
||||
const uuid = data?.id;
|
||||
if (typeof uuid !== "string" || !uuid) {
|
||||
throw new Error(
|
||||
`Could not resolve a canonical page id for "${pageId}"`,
|
||||
);
|
||||
}
|
||||
this.pageIdCache.set(pageId, uuid);
|
||||
return uuid;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Page-locked write seam over collaboration.mutatePageContent. Production just
|
||||
* delegates; it exists as an overridable method so the insertFootnote wrapper
|
||||
* (transform abort-on-not-found + response shaping) can be unit-tested without
|
||||
* standing up a live Hocuspocus collab socket.
|
||||
*
|
||||
* SELF-RESOLVES the pageId to the canonical UUID (issue #449, "resolve-then-
|
||||
* lock"): every write must lock and key its CollabSession by the UUID, never a
|
||||
* raw slugId (#260). resolvePageId is cached/idempotent, so a caller that
|
||||
* already resolved pays no extra round-trip; centralizing it here means a
|
||||
* caller that reaches this seam with a raw slugId still locks correctly instead
|
||||
* of silently splitting the mutex key. withPageLock also asserts the key is a
|
||||
* UUID as a hard backstop.
|
||||
*/
|
||||
protected async mutatePage(
|
||||
pageId: string,
|
||||
collabToken: string,
|
||||
apiUrl: string,
|
||||
transform: (doc: any) => any,
|
||||
): Promise<{ doc?: any; verify?: any }> {
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
// #486: on a rejected collab-WS handshake, invalidate + refresh the token and
|
||||
// retry the write once (symmetric to the HTTP-401 reauth path).
|
||||
return this.writeWithCollabAuthRetry(collabToken, (token) =>
|
||||
mutatePageContent(pageUuid, token, apiUrl, transform),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-document write seam over collaboration.replacePageContent. Production
|
||||
* just delegates; it exists as an overridable method so the full-doc write
|
||||
* tools (updatePageJson, copyPageContent) can have their footnote-
|
||||
* canonicalization binding unit-tested without a live Hocuspocus collab socket.
|
||||
*
|
||||
* SELF-RESOLVES the pageId to the canonical UUID (issue #449, "resolve-then-
|
||||
* lock") for the same reason as mutatePage above — the lock/CollabSession key
|
||||
* is guaranteed canonical here, not left to the caller's discipline.
|
||||
*/
|
||||
protected async replacePage(
|
||||
pageId: string,
|
||||
doc: any,
|
||||
collabToken: string,
|
||||
apiUrl: string,
|
||||
): Promise<{ doc?: any; verify?: any }> {
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
// #486: on a rejected collab-WS handshake, invalidate + refresh the token and
|
||||
// retry the write once (symmetric to the HTTP-401 reauth path).
|
||||
return this.writeWithCollabAuthRetry(collabToken, (token) =>
|
||||
replacePageContent(pageUuid, doc, token, apiUrl),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Export a page to a single self-contained Docmost-flavoured markdown file:
|
||||
* meta block + body (with inline comment anchors + diagrams) + comment
|
||||
* threads. Lossless round-trip target; see importPageMarkdown for the inverse.
|
||||
*/
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import {
|
||||
updatePageContentRealtime,
|
||||
replacePageContent,
|
||||
markdownToProseMirror,
|
||||
markdownToProseMirrorCanonical,
|
||||
mutatePageContent,
|
||||
assertYjsEncodable,
|
||||
MutationResult,
|
||||
} from "../lib/collaboration.js";
|
||||
import {
|
||||
replaceNodeById,
|
||||
replaceNodeByIdWithMany,
|
||||
reassignCollidingBlockIds,
|
||||
deleteNodeById,
|
||||
assertUnambiguousMatch,
|
||||
insertNodeRelative,
|
||||
insertNodesRelative,
|
||||
blockPlainText,
|
||||
buildOutline,
|
||||
getNodeByRef,
|
||||
readTable,
|
||||
insertTableRow,
|
||||
deleteTableRow,
|
||||
updateTableCell,
|
||||
findInvalidNode,
|
||||
} from "@docmost/prosemirror-markdown";
|
||||
import {
|
||||
blockText,
|
||||
walk,
|
||||
getList,
|
||||
insertMarkerAfter,
|
||||
setCalloutRange,
|
||||
noteItem,
|
||||
mdToInlineNodes,
|
||||
commentsToFootnotes,
|
||||
canonicalizeFootnotes,
|
||||
insertInlineFootnote,
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
|
||||
// Public method surface of DocValidateMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements IDocValidateMixin` fails to compile on drift.
|
||||
export interface IDocValidateMixin {
|
||||
}
|
||||
|
||||
export function DocValidateMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & IDocValidateMixin> & TBase {
|
||||
abstract class DocValidateMixin extends Base implements IDocValidateMixin {
|
||||
/**
|
||||
* Validate a URL string against a scheme allowlist for a given context.
|
||||
*
|
||||
* The markdown link path enforces safe schemes via TipTap, but the raw
|
||||
* JSON path (updatePageJson) bypasses that — so this is the sanitization
|
||||
* choke point for ProseMirror JSON written directly by the caller.
|
||||
*
|
||||
* - "link": reject javascript:, vbscript:, data: (any scheme that can
|
||||
* execute or smuggle script when the href is clicked).
|
||||
* - "src": allow only http(s):, mailto:, /api/files paths, or a
|
||||
* scheme-less relative/absolute path; reject
|
||||
* javascript:/vbscript:/data:/file:.
|
||||
*/
|
||||
protected isSafeUrl(url: unknown, context: "link" | "src"): boolean {
|
||||
if (typeof url !== "string") return false;
|
||||
const trimmed = url.trim();
|
||||
if (trimmed === "") return true; // empty href/src is harmless
|
||||
|
||||
// Extract a leading "scheme:" if present. A scheme must start with a
|
||||
// letter and contain only letters/digits/+/-/. before the colon. Strip
|
||||
// whitespace and ASCII control chars first so a tab/newline embedded in
|
||||
// the scheme cannot smuggle a dangerous scheme past the check.
|
||||
const cleaned = trimmed.replace(/[\s\x00-\x1f]+/g, "");
|
||||
const schemeMatch = /^([a-zA-Z][a-zA-Z0-9+.-]*):/.exec(cleaned);
|
||||
const scheme = schemeMatch ? schemeMatch[1].toLowerCase() : null;
|
||||
|
||||
const dangerous = new Set(["javascript", "vbscript", "data", "file"]);
|
||||
|
||||
if (context === "link") {
|
||||
if (scheme === null) return true; // relative/anchor link is fine
|
||||
// For links, data: is also blocked (can carry script payloads).
|
||||
return !new Set(["javascript", "vbscript", "data"]).has(scheme);
|
||||
}
|
||||
|
||||
// context === "src"
|
||||
if (scheme === null) return true; // relative/absolute path (incl. /api/files)
|
||||
if (dangerous.has(scheme)) return false;
|
||||
return scheme === "http" || scheme === "https" || scheme === "mailto";
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively walk a ProseMirror doc and reject any unsafe URL on a link
|
||||
* mark href or on a media node's src/url. Media nodes covered: image,
|
||||
* attachment, video, plus embed (rendered as an iframe), youtube, drawio
|
||||
* and excalidraw — all of which carry a user-controlled URL that Docmost
|
||||
* renders. Throws a clear error on the first violation. A max-depth guard
|
||||
* turns an over-deep document into a clean error instead of a RangeError
|
||||
* stack overflow.
|
||||
*/
|
||||
protected validateDocUrls(node: any, depth: number = 0): void {
|
||||
const MAX_DEPTH = 200;
|
||||
if (depth > MAX_DEPTH) {
|
||||
throw new Error(
|
||||
`document nesting exceeds the maximum depth of ${MAX_DEPTH}`,
|
||||
);
|
||||
}
|
||||
if (!node || typeof node !== "object") return;
|
||||
|
||||
// Link marks on text nodes: validate the href.
|
||||
if (Array.isArray(node.marks)) {
|
||||
for (const mark of node.marks) {
|
||||
if (mark && mark.type === "link" && mark.attrs) {
|
||||
if (!this.isSafeUrl(mark.attrs.href, "link")) {
|
||||
throw new Error(`unsafe link href rejected: "${mark.attrs.href}"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Media nodes: validate src/url against the stricter src allowlist.
|
||||
// embed renders as an iframe (highest risk); youtube/drawio/excalidraw
|
||||
// likewise carry a user-controlled URL Docmost renders, so they get the
|
||||
// same scheme check as image/attachment/video.
|
||||
if (
|
||||
node.type === "image" ||
|
||||
node.type === "attachment" ||
|
||||
node.type === "video" ||
|
||||
node.type === "embed" ||
|
||||
node.type === "youtube" ||
|
||||
node.type === "drawio" ||
|
||||
node.type === "excalidraw" ||
|
||||
node.type === "audio" ||
|
||||
node.type === "pdf"
|
||||
) {
|
||||
const attrs = node.attrs || {};
|
||||
for (const key of ["src", "url"]) {
|
||||
if (attrs[key] != null && !this.isSafeUrl(attrs[key], "src")) {
|
||||
throw new Error(
|
||||
`unsafe ${node.type} ${key} rejected: "${attrs[key]}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(node.content)) {
|
||||
for (const child of node.content) {
|
||||
this.validateDocUrls(child, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively validate the STRUCTURE of a ProseMirror node (reuses the
|
||||
* recursion shape of validateDocUrls). Every node must be an object with a
|
||||
* string `type`; when present, `content` must be an array, `marks` must be
|
||||
* an array of objects each with a string `type`, and a text node's `text`
|
||||
* must be a string. Throws a clear "invalid ProseMirror document" error on
|
||||
* the first violation. A max-depth guard turns an over-deep document into a
|
||||
* clean error instead of a RangeError stack overflow.
|
||||
*/
|
||||
protected validateDocStructure(node: any, depth: number = 0): void {
|
||||
const MAX_DEPTH = 200;
|
||||
if (depth > MAX_DEPTH) {
|
||||
throw new Error(
|
||||
`invalid ProseMirror document: nesting exceeds the maximum depth of ${MAX_DEPTH}`,
|
||||
);
|
||||
}
|
||||
if (!node || typeof node !== "object" || typeof node.type !== "string") {
|
||||
throw new Error(
|
||||
"invalid ProseMirror document: every node must be an object with a string `type`",
|
||||
);
|
||||
}
|
||||
if (
|
||||
"text" in node &&
|
||||
node.type === "text" &&
|
||||
typeof node.text !== "string"
|
||||
) {
|
||||
throw new Error(
|
||||
"invalid ProseMirror document: a text node must have a string `text`",
|
||||
);
|
||||
}
|
||||
if (node.marks !== undefined) {
|
||||
if (!Array.isArray(node.marks)) {
|
||||
throw new Error(
|
||||
"invalid ProseMirror document: `marks` must be an array",
|
||||
);
|
||||
}
|
||||
for (const mark of node.marks) {
|
||||
if (
|
||||
!mark ||
|
||||
typeof mark !== "object" ||
|
||||
typeof mark.type !== "string"
|
||||
) {
|
||||
throw new Error(
|
||||
"invalid ProseMirror document: every mark must be an object with a string `type`",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (node.content !== undefined) {
|
||||
if (!Array.isArray(node.content)) {
|
||||
throw new Error(
|
||||
"invalid ProseMirror document: `content` must be an array when present",
|
||||
);
|
||||
}
|
||||
for (const child of node.content) {
|
||||
this.validateDocStructure(child, depth + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pre-write SHAPE gate (#409). Walk the WHOLE node tree with the shared
|
||||
* `findInvalidNode` and throw a rich, path-anchored error the instant a nested
|
||||
* node has an absent/unknown `type` (or an unknown mark) — the exact shape that
|
||||
* otherwise surfaces DEEP in the Yjs encode as the cryptic
|
||||
* `Unknown node type: undefined`, but only AFTER a collab session was opened
|
||||
* and a page lock taken. Calling this BEFORE `getCollabTokenWithReauth` /
|
||||
* `mutatePageContent` fails fast: no collab connection, no lock, deterministic
|
||||
* message. `op` names the tool for the message prefix (e.g. "patchNode").
|
||||
*
|
||||
* `findInvalidNode` derives its "known type" set from the very same
|
||||
* `docmostExtensions` the encode path uses, so a node this gate accepts is one
|
||||
* the encoder will accept too.
|
||||
*/
|
||||
protected assertValidNodeShape(op: string, node: any): void {
|
||||
const bad = findInvalidNode(node);
|
||||
if (bad) {
|
||||
throw new Error(`${op}: invalid node — ${bad.summary}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace page content with a raw ProseMirror JSON document (lossless) and/or
|
||||
* update its title. Both `doc` and `title` are optional, but at least one must
|
||||
* be supplied:
|
||||
* - `doc` provided -> validate + full-overwrite the body (and update the
|
||||
* title too when `title` is also given).
|
||||
* - `doc` omitted, `title` given -> title-only update; the body is NOT
|
||||
* touched/resent (no collab write happens).
|
||||
* - neither given -> throws (nothing to update).
|
||||
*/
|
||||
}
|
||||
return DocValidateMixin;
|
||||
}
|
||||
@@ -0,0 +1,707 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import { parseCells as parseDrawioCells } from "../lib/drawio-xml.js";
|
||||
import {
|
||||
replaceNodeById,
|
||||
replaceNodeByIdWithMany,
|
||||
reassignCollidingBlockIds,
|
||||
deleteNodeById,
|
||||
assertUnambiguousMatch,
|
||||
insertNodeRelative,
|
||||
insertNodesRelative,
|
||||
blockPlainText,
|
||||
buildOutline,
|
||||
getNodeByRef,
|
||||
readTable,
|
||||
insertTableRow,
|
||||
deleteTableRow,
|
||||
updateTableCell,
|
||||
findInvalidNode,
|
||||
} from "@docmost/prosemirror-markdown";
|
||||
import {
|
||||
prepareModel,
|
||||
decodeDrawioSvg,
|
||||
buildDrawioSvg,
|
||||
mxHash,
|
||||
normalizeXml,
|
||||
countUserCells,
|
||||
} from "../lib/drawio-xml.js";
|
||||
import { renderDiagramShapes } from "../lib/drawio-preview.js";
|
||||
import { applyElkLayout } from "../lib/drawio-layout.js";
|
||||
import {
|
||||
buildFromGraph,
|
||||
type Graph,
|
||||
type LayoutMode as GraphLayoutMode,
|
||||
} from "../lib/drawio-graph.js";
|
||||
import { applyCellOps, type CellOp } from "../lib/drawio-cell-ops.js";
|
||||
import { mermaidToGraph } from "../lib/drawio-mermaid.js";
|
||||
import {
|
||||
blockText,
|
||||
walk,
|
||||
getList,
|
||||
insertMarkerAfter,
|
||||
setCalloutRange,
|
||||
noteItem,
|
||||
mdToInlineNodes,
|
||||
commentsToFootnotes,
|
||||
canonicalizeFootnotes,
|
||||
insertInlineFootnote,
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
|
||||
// Public method surface of DrawioMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements IDrawioMixin` fails to compile on drift.
|
||||
export interface IDrawioMixin {
|
||||
drawioGet(pageId: string, node: string, format?: "xml" | "svg"): Promise<{ pageId: string; nodeId: string; format: "xml" | "svg"; content: string; meta: { attachmentId: string | null; title: string | null; width: number | null; height: number | null; cellCount: number; hash: string; }; }>;
|
||||
drawioCreate(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, xml: string, title?: string, layout?: "elk"): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; verify?: any; }>;
|
||||
drawioUpdate(pageId: string, node: string, xml: string, baseHash: string, layout?: "elk"): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; verify?: any; }>;
|
||||
drawioEditCells(pageId: string, node: string, operations: CellOp[], baseHash: string): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; verify?: any; }>;
|
||||
drawioFromGraph(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, graph: Graph, direction?: "LR" | "RL" | "TB" | "BT", preset?: string, layout?: GraphLayoutMode, node?: string): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; iconsResolved: number; iconsMissing: string[]; verify?: any; }>;
|
||||
drawioFromMermaid(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, mermaid: string, preset?: string): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; iconsResolved: number; iconsMissing: string[]; verify?: any; }>;
|
||||
}
|
||||
|
||||
export function DrawioMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & IDrawioMixin> & TBase {
|
||||
abstract class DrawioMixin extends Base implements IDrawioMixin {
|
||||
/**
|
||||
* Resolve a drawio node on a page by `attrs.id` or `#<index>` and return the
|
||||
* node plus its ref. Throws a clear error if the ref does not resolve to a
|
||||
* drawio node.
|
||||
*/
|
||||
protected async resolveDrawioNode(
|
||||
pageId: string,
|
||||
node: string,
|
||||
): Promise<{ node: any; ref: string }> {
|
||||
const data = await this.getPageRaw(pageId);
|
||||
const hit = getNodeByRef(
|
||||
data.content ?? { type: "doc", content: [] },
|
||||
node,
|
||||
);
|
||||
if (!hit) {
|
||||
throw new Error(
|
||||
`drawio: no node found for "${node}" on page ${pageId} (use the drawio node's attrs.id or "#<index>" from getOutline)`,
|
||||
);
|
||||
}
|
||||
if (hit.type !== "drawio") {
|
||||
throw new Error(
|
||||
`drawio: node "${node}" on page ${pageId} is a ${hit.type}, not a drawio diagram`,
|
||||
);
|
||||
}
|
||||
return { node: hit.node, ref: node };
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a drawio diagram as mxGraph XML (default) or as the raw `.drawio.svg`.
|
||||
* Runs the decode chain (base64/entity content= → drawio file → nested XML or
|
||||
* pako-inflated compressed <diagram>). The returned `hash` is the
|
||||
* optimistic-lock key for drawioUpdate.
|
||||
*/
|
||||
async drawioGet(
|
||||
pageId: string,
|
||||
node: string,
|
||||
format: "xml" | "svg" = "xml",
|
||||
): Promise<{
|
||||
pageId: string;
|
||||
nodeId: string;
|
||||
format: "xml" | "svg";
|
||||
content: string;
|
||||
meta: {
|
||||
attachmentId: string | null;
|
||||
title: string | null;
|
||||
width: number | null;
|
||||
height: number | null;
|
||||
cellCount: number;
|
||||
hash: string;
|
||||
};
|
||||
}> {
|
||||
await this.ensureAuthenticated();
|
||||
const { node: drawio } = await this.resolveDrawioNode(pageId, node);
|
||||
const attrs = drawio.attrs || {};
|
||||
const src = attrs.src;
|
||||
if (!src) {
|
||||
throw new Error(
|
||||
`drawio: node "${node}" on page ${pageId} has no src to read`,
|
||||
);
|
||||
}
|
||||
const svg = await this.fetchAttachmentText(src);
|
||||
const modelXml = decodeDrawioSvg(svg);
|
||||
const meta = {
|
||||
attachmentId: attrs.attachmentId ?? null,
|
||||
title: attrs.title ?? null,
|
||||
width: attrs.width != null ? Number(attrs.width) : null,
|
||||
height: attrs.height != null ? Number(attrs.height) : null,
|
||||
cellCount: countUserCells(modelXml),
|
||||
hash: mxHash(modelXml),
|
||||
};
|
||||
return {
|
||||
pageId,
|
||||
nodeId: attrs.id ?? node,
|
||||
format,
|
||||
content: format === "svg" ? svg : normalizeXml(modelXml),
|
||||
meta,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a drawio diagram from mxGraph XML: lint → schematic SVG preview
|
||||
* (pure TS) → build the `.drawio.svg` (createDrawioSvg contract) → create the
|
||||
* attachment → insert a `drawio` node before/after an anchor or appended.
|
||||
* `xml` is a bare `<mxGraphModel>` or a list of `<mxCell>` (the server wraps
|
||||
* it and adds the id=0/id=1 sentinels).
|
||||
*/
|
||||
async drawioCreate(
|
||||
pageId: string,
|
||||
where: {
|
||||
position: "before" | "after" | "append";
|
||||
anchorNodeId?: string;
|
||||
anchorText?: string;
|
||||
},
|
||||
xml: string,
|
||||
title?: string,
|
||||
layout?: "elk",
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
nodeId: string;
|
||||
attachmentId: string;
|
||||
warnings: string[];
|
||||
verify?: any;
|
||||
}> {
|
||||
await this.ensureAuthenticated();
|
||||
if (
|
||||
!where ||
|
||||
(where.position !== "before" &&
|
||||
where.position !== "after" &&
|
||||
where.position !== "append")
|
||||
) {
|
||||
throw new Error(
|
||||
'drawioCreate: `where.position` must be one of "before", "after", "append"',
|
||||
);
|
||||
}
|
||||
if (where.position === "before" || where.position === "after") {
|
||||
const hasId =
|
||||
typeof where.anchorNodeId === "string" && where.anchorNodeId.length > 0;
|
||||
const hasText =
|
||||
typeof where.anchorText === "string" && where.anchorText.length > 0;
|
||||
if (hasId === hasText) {
|
||||
throw new Error(
|
||||
`drawioCreate: position "${where.position}" requires exactly one of anchorNodeId or anchorText`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Optional server-side ELK auto-layout: the model declares structure with
|
||||
// rough coords, ELK computes the pixels (best-effort — returns the input
|
||||
// unchanged on any layout failure).
|
||||
const laidOutXml = layout === "elk" ? await applyElkLayout(xml) : xml;
|
||||
// Pre-write pipeline (throws a structured DrawioLintError on any violation).
|
||||
const prepared = prepareModel(laidOutXml);
|
||||
const inner = renderDiagramShapes(prepared.cells, prepared.bbox);
|
||||
const diagramTitle = title || "Page-1";
|
||||
const svg = buildDrawioSvg(prepared.modelXml, inner, prepared.bbox, diagramTitle);
|
||||
|
||||
const att = await this.uploadAttachmentBuffer(
|
||||
pageId,
|
||||
Buffer.from(svg, "utf-8"),
|
||||
"diagram.drawio.svg",
|
||||
"image/svg+xml",
|
||||
);
|
||||
|
||||
// NOTE: no `id` attribute is set here. The vendored `drawio` node schema
|
||||
// (diagramAttributes) declares no `id`, so any block id would be silently
|
||||
// dropped by PMNode.fromJSON on save and the returned handle would fail to
|
||||
// resolve. The addressable handle is the node's "#<index>" (like image/table
|
||||
// nodes), computed after the insert below.
|
||||
const drawioNode: any = {
|
||||
type: "drawio",
|
||||
attrs: {
|
||||
src: `/api/files/${att.id}/${att.fileName}`,
|
||||
attachmentId: att.id,
|
||||
width: prepared.bbox.width,
|
||||
height: prepared.bbox.height,
|
||||
align: "center",
|
||||
},
|
||||
};
|
||||
if (title) drawioNode.attrs.title = title;
|
||||
// Reuse the existing URL trust boundary (rejects unsafe src schemes).
|
||||
this.validateDocUrls(drawioNode);
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
let inserted = false;
|
||||
let insertedIndex = -1;
|
||||
const mutation = await this.mutatePage(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
inserted = false;
|
||||
insertedIndex = -1;
|
||||
const { doc: nd, inserted: ins } = insertNodeRelative(
|
||||
liveDoc,
|
||||
drawioNode,
|
||||
where,
|
||||
);
|
||||
inserted = ins;
|
||||
if (!inserted) return null; // anchor not found -> skip the write
|
||||
// Locate the freshly-inserted node to derive its "#<index>" handle. The
|
||||
// just-uploaded attachmentId is unique, so it identifies our node.
|
||||
if (Array.isArray(nd.content)) {
|
||||
insertedIndex = nd.content.findIndex(
|
||||
(b: any) =>
|
||||
b &&
|
||||
b.type === "drawio" &&
|
||||
b.attrs &&
|
||||
b.attrs.attachmentId === att.id,
|
||||
);
|
||||
}
|
||||
return nd;
|
||||
},
|
||||
);
|
||||
|
||||
if (!inserted) {
|
||||
const anchorDesc = where.anchorNodeId
|
||||
? `anchorNodeId "${where.anchorNodeId}"`
|
||||
: `anchorText "${where.anchorText}"`;
|
||||
throw new Error(
|
||||
`drawioCreate: anchor not found (${anchorDesc}) on page ${pageId}. The diagram attachment ${att.id} is now an unreferenced orphan.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (insertedIndex < 0) {
|
||||
// The node was inserted nested (e.g. inside a callout/table cell via an
|
||||
// anchor), where "#<index>" — which addresses only top-level blocks —
|
||||
// cannot reference it. drawio nodes carry no persisted id, so there is no
|
||||
// stable handle for a nested diagram.
|
||||
throw new Error(
|
||||
`drawioCreate: the diagram was inserted on page ${pageId} but not as a ` +
|
||||
`top-level block, so it has no addressable "#<index>" handle. Anchor ` +
|
||||
`on a top-level block (or append) so the diagram can be re-read.`,
|
||||
);
|
||||
}
|
||||
|
||||
// The returned handle is POSITIONAL ("#<index>"): valid for the immediate
|
||||
// create -> get/update flow, but re-resolve via getOutline if the document
|
||||
// structure changes (blocks added/removed before it shift the index).
|
||||
const nodeId = `#${insertedIndex}`;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
nodeId,
|
||||
attachmentId: att.id,
|
||||
warnings: prepared.warnings,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-replacement update of a drawio diagram. `baseHash` is MANDATORY: it is
|
||||
* compared against the hash of the diagram's CURRENT XML (from drawioGet);
|
||||
* any mismatch means a human or another agent edited the diagram after the
|
||||
* read, so the write is refused with a conflict error. On success the new
|
||||
* `.drawio.svg` is uploaded as a FRESH attachment (in-place byte overwrite is
|
||||
* avoided — some Docmost versions corrupt an attachment on overwrite, exactly
|
||||
* as replaceImage documents) and the node is repointed with new dimensions.
|
||||
*/
|
||||
async drawioUpdate(
|
||||
pageId: string,
|
||||
node: string,
|
||||
xml: string,
|
||||
baseHash: string,
|
||||
layout?: "elk",
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
nodeId: string;
|
||||
attachmentId: string;
|
||||
warnings: string[];
|
||||
verify?: any;
|
||||
}> {
|
||||
await this.ensureAuthenticated();
|
||||
if (typeof baseHash !== "string" || baseHash.length === 0) {
|
||||
throw new Error(
|
||||
"drawioUpdate: baseHash is mandatory — read the diagram with drawioGet first and pass back its meta.hash",
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve the node and read the CURRENT diagram to enforce the optimistic
|
||||
// lock before doing any write or upload.
|
||||
const { node: drawio, ref } = await this.resolveDrawioNode(pageId, node);
|
||||
const oldAttrs = drawio.attrs || {};
|
||||
const oldSrc = oldAttrs.src;
|
||||
// The returned handle is the caller-supplied reference. drawio nodes carry
|
||||
// no persisted id, so `ref` (an "#<index>" or a rare legacy attrs.id) is the
|
||||
// honest identifier to hand back.
|
||||
const nodeId = oldAttrs.id ?? ref;
|
||||
if (!oldSrc) {
|
||||
throw new Error(
|
||||
`drawioUpdate: node "${node}" on page ${pageId} has no src to compare against`,
|
||||
);
|
||||
}
|
||||
const currentSvg = await this.fetchAttachmentText(oldSrc);
|
||||
const currentHash = mxHash(decodeDrawioSvg(currentSvg));
|
||||
if (currentHash !== baseHash) {
|
||||
throw new Error(
|
||||
`drawioUpdate: conflict — the diagram changed since it was read ` +
|
||||
`(baseHash ${baseHash} != current ${currentHash}). Re-read it with drawioGet and retry.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Optional server-side ELK auto-layout (best-effort; see drawioCreate).
|
||||
const laidOutXml = layout === "elk" ? await applyElkLayout(xml) : xml;
|
||||
// Pipeline for the new content (throws a structured DrawioLintError).
|
||||
const prepared = prepareModel(laidOutXml);
|
||||
const inner = renderDiagramShapes(prepared.cells, prepared.bbox);
|
||||
const diagramTitle = oldAttrs.title || "Page-1";
|
||||
const svg = buildDrawioSvg(prepared.modelXml, inner, prepared.bbox, diagramTitle);
|
||||
|
||||
const att = await this.uploadAttachmentBuffer(
|
||||
pageId,
|
||||
Buffer.from(svg, "utf-8"),
|
||||
"diagram.drawio.svg",
|
||||
"image/svg+xml",
|
||||
);
|
||||
const newSrc = `/api/files/${att.id}/${att.fileName}`;
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
let repointed = 0;
|
||||
const repoint = (n: any) => {
|
||||
n.attrs = {
|
||||
...n.attrs,
|
||||
src: newSrc,
|
||||
attachmentId: att.id,
|
||||
width: prepared.bbox.width,
|
||||
height: prepared.bbox.height,
|
||||
};
|
||||
repointed++;
|
||||
};
|
||||
|
||||
const mutation = await this.mutatePage(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
repointed = 0;
|
||||
const doc =
|
||||
liveDoc && liveDoc.type === "doc"
|
||||
? liveDoc
|
||||
: { type: "doc", content: [] };
|
||||
if (!Array.isArray(doc.content)) doc.content = [];
|
||||
// Repoint ONLY the resolved node — never every node that happens to
|
||||
// share this attachmentId (a copied diagram is two nodes with one
|
||||
// attachmentId; keying on it would clobber both). Re-resolve the same
|
||||
// handle against the live doc and walk to its exact position.
|
||||
const hit = getNodeByRef(doc, ref);
|
||||
if (!hit || hit.type !== "drawio") return null; // vanished/changed -> skip
|
||||
let target: any = doc;
|
||||
for (const idx of hit.path) {
|
||||
if (!target || !Array.isArray(target.content)) {
|
||||
target = null;
|
||||
break;
|
||||
}
|
||||
target = target.content[idx];
|
||||
}
|
||||
if (!target || target.type !== "drawio") return null;
|
||||
repoint(target);
|
||||
if (repointed === 0) return null; // node vanished concurrently -> skip
|
||||
return doc;
|
||||
},
|
||||
);
|
||||
|
||||
if (repointed === 0) {
|
||||
return {
|
||||
success: true,
|
||||
nodeId,
|
||||
attachmentId: att.id,
|
||||
warnings: [
|
||||
...prepared.warnings,
|
||||
"target drawio node was removed concurrently; uploaded attachment is unreferenced",
|
||||
],
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
nodeId,
|
||||
attachmentId: att.id,
|
||||
warnings: prepared.warnings,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
// --- draw.io high-level semantic tools (issue #425) ---
|
||||
|
||||
/**
|
||||
* ID-based targeted edits of an existing drawio diagram (add / update / delete
|
||||
* cells) instead of resending the whole XML. Reads the CURRENT diagram, checks
|
||||
* the optimistic lock (`baseHash` is MANDATORY, exactly as drawioUpdate), applies
|
||||
* the operations to the parsed model (a `delete` CASCADES to container children
|
||||
* and to every edge whose source/target is deleted), then runs the SAME #423
|
||||
* pipeline as drawioUpdate (lint + quality warnings -> preview -> attachment ->
|
||||
* repoint the node). Ids are stable so diffs stay meaningful across edits.
|
||||
*/
|
||||
|
||||
// --- draw.io high-level semantic tools (issue #425) ---
|
||||
|
||||
/**
|
||||
* ID-based targeted edits of an existing drawio diagram (add / update / delete
|
||||
* cells) instead of resending the whole XML. Reads the CURRENT diagram, checks
|
||||
* the optimistic lock (`baseHash` is MANDATORY, exactly as drawioUpdate), applies
|
||||
* the operations to the parsed model (a `delete` CASCADES to container children
|
||||
* and to every edge whose source/target is deleted), then runs the SAME #423
|
||||
* pipeline as drawioUpdate (lint + quality warnings -> preview -> attachment ->
|
||||
* repoint the node). Ids are stable so diffs stay meaningful across edits.
|
||||
*/
|
||||
async drawioEditCells(
|
||||
pageId: string,
|
||||
node: string,
|
||||
operations: CellOp[],
|
||||
baseHash: string,
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
nodeId: string;
|
||||
attachmentId: string;
|
||||
warnings: string[];
|
||||
verify?: any;
|
||||
}> {
|
||||
await this.ensureAuthenticated();
|
||||
if (typeof baseHash !== "string" || baseHash.length === 0) {
|
||||
throw new Error(
|
||||
"drawioEditCells: baseHash is mandatory — read the diagram with drawioGet first and pass back its meta.hash",
|
||||
);
|
||||
}
|
||||
if (!Array.isArray(operations) || operations.length === 0) {
|
||||
throw new Error(
|
||||
"drawioEditCells: operations must be a non-empty array of { op, ... }",
|
||||
);
|
||||
}
|
||||
|
||||
const { node: drawio, ref } = await this.resolveDrawioNode(pageId, node);
|
||||
const oldAttrs = drawio.attrs || {};
|
||||
const oldSrc = oldAttrs.src;
|
||||
const nodeId = oldAttrs.id ?? ref;
|
||||
if (!oldSrc) {
|
||||
throw new Error(
|
||||
`drawioEditCells: node "${node}" on page ${pageId} has no src to edit`,
|
||||
);
|
||||
}
|
||||
const currentSvg = await this.fetchAttachmentText(oldSrc);
|
||||
const currentModel = decodeDrawioSvg(currentSvg);
|
||||
const currentHash = mxHash(currentModel);
|
||||
if (currentHash !== baseHash) {
|
||||
throw new Error(
|
||||
`drawioEditCells: conflict — the diagram changed since it was read ` +
|
||||
`(baseHash ${baseHash} != current ${currentHash}). Re-read it with drawioGet and retry.`,
|
||||
);
|
||||
}
|
||||
|
||||
// Apply the operations to the parsed model, then run the standard pipeline.
|
||||
const editedModel = applyCellOps(currentModel, operations);
|
||||
const prepared = prepareModel(editedModel);
|
||||
const inner = renderDiagramShapes(prepared.cells, prepared.bbox);
|
||||
const diagramTitle = oldAttrs.title || "Page-1";
|
||||
const svg = buildDrawioSvg(prepared.modelXml, inner, prepared.bbox, diagramTitle);
|
||||
|
||||
const att = await this.uploadAttachmentBuffer(
|
||||
pageId,
|
||||
Buffer.from(svg, "utf-8"),
|
||||
"diagram.drawio.svg",
|
||||
"image/svg+xml",
|
||||
);
|
||||
const newSrc = `/api/files/${att.id}/${att.fileName}`;
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
let repointed = 0;
|
||||
const mutation = await this.mutatePage(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
repointed = 0;
|
||||
const doc =
|
||||
liveDoc && liveDoc.type === "doc" ? liveDoc : { type: "doc", content: [] };
|
||||
if (!Array.isArray(doc.content)) doc.content = [];
|
||||
const hit = getNodeByRef(doc, ref);
|
||||
if (!hit || hit.type !== "drawio") return null;
|
||||
let target: any = doc;
|
||||
for (const idx of hit.path) {
|
||||
if (!target || !Array.isArray(target.content)) {
|
||||
target = null;
|
||||
break;
|
||||
}
|
||||
target = target.content[idx];
|
||||
}
|
||||
if (!target || target.type !== "drawio") return null;
|
||||
target.attrs = {
|
||||
...target.attrs,
|
||||
src: newSrc,
|
||||
attachmentId: att.id,
|
||||
width: prepared.bbox.width,
|
||||
height: prepared.bbox.height,
|
||||
};
|
||||
repointed++;
|
||||
return doc;
|
||||
},
|
||||
);
|
||||
|
||||
if (repointed === 0) {
|
||||
return {
|
||||
success: true,
|
||||
nodeId,
|
||||
attachmentId: att.id,
|
||||
warnings: [
|
||||
...prepared.warnings,
|
||||
"target drawio node was removed concurrently; uploaded attachment is unreferenced",
|
||||
],
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
nodeId,
|
||||
attachmentId: att.id,
|
||||
warnings: prepared.warnings,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The main high-level tool: build a diagram from a SEMANTIC graph (nodes with
|
||||
* a `kind`/`icon`, groups, edges) — the model never supplies coordinates or
|
||||
* style strings. The server resolves icons via the shape catalog (#424),
|
||||
* assigns palette colors from the preset, runs ELK layered layout (honouring
|
||||
* `direction` and the `layer`/`sameLayerAs`/`pinned` hints and compound groups),
|
||||
* and assembles linter-clean XML, then inserts it through the SAME create
|
||||
* pipeline as drawioCreate. `layout:"incremental"` is only meaningful when a
|
||||
* target `node` is given (it preserves that diagram's existing coordinates and
|
||||
* places only new cells); on a fresh insert it behaves like "full".
|
||||
*/
|
||||
async drawioFromGraph(
|
||||
pageId: string,
|
||||
where: {
|
||||
position: "before" | "after" | "append";
|
||||
anchorNodeId?: string;
|
||||
anchorText?: string;
|
||||
},
|
||||
graph: Graph,
|
||||
direction?: "LR" | "RL" | "TB" | "BT",
|
||||
preset?: string,
|
||||
layout?: GraphLayoutMode,
|
||||
node?: string,
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
nodeId: string;
|
||||
attachmentId: string;
|
||||
warnings: string[];
|
||||
iconsResolved: number;
|
||||
iconsMissing: string[];
|
||||
verify?: any;
|
||||
}> {
|
||||
await this.ensureAuthenticated();
|
||||
// Direction/preset supplied as separate params override the graph fields so
|
||||
// both the flat tool schema and an inline graph can set them.
|
||||
const merged: Graph = {
|
||||
...graph,
|
||||
direction: direction ?? graph.direction,
|
||||
preset: preset ?? graph.preset,
|
||||
};
|
||||
const mode: GraphLayoutMode = layout ?? "full";
|
||||
|
||||
// Incremental into an EXISTING node: read its coords so ELK preserves them,
|
||||
// and keep the full existing model so incremental MERGES (never drops) any
|
||||
// cell the new graph doesn't re-list.
|
||||
let existingCoords: Map<string, { x: number; y: number }> | undefined;
|
||||
let existingModelXml: string | undefined;
|
||||
let editExisting = false;
|
||||
let baseHash: string | undefined;
|
||||
if (node && (mode === "incremental" || mode === "none")) {
|
||||
const { node: drawio } = await this.resolveDrawioNode(pageId, node);
|
||||
const src = (drawio.attrs || {}).src;
|
||||
if (src) {
|
||||
const svg = await this.fetchAttachmentText(src);
|
||||
const model = decodeDrawioSvg(svg);
|
||||
baseHash = mxHash(model);
|
||||
existingModelXml = model;
|
||||
existingCoords = new Map();
|
||||
for (const c of parseDrawioCells(model)) {
|
||||
if (c.vertex && c.geometry.x != null && c.geometry.y != null) {
|
||||
existingCoords.set(c.id, { x: c.geometry.x, y: c.geometry.y });
|
||||
}
|
||||
}
|
||||
editExisting = true;
|
||||
}
|
||||
}
|
||||
|
||||
const built = await buildFromGraph(
|
||||
merged,
|
||||
mode,
|
||||
existingCoords,
|
||||
existingModelXml,
|
||||
);
|
||||
|
||||
if (editExisting && node && baseHash) {
|
||||
// Re-target the existing diagram: replace it with the assembled model.
|
||||
const res = await this.drawioUpdate(pageId, node, built.modelXml, baseHash);
|
||||
return {
|
||||
...res,
|
||||
iconsResolved: built.iconsResolved,
|
||||
iconsMissing: built.iconsMissing,
|
||||
};
|
||||
}
|
||||
|
||||
const res = await this.drawioCreate(pageId, where, built.modelXml);
|
||||
return {
|
||||
...res,
|
||||
iconsResolved: built.iconsResolved,
|
||||
iconsMissing: built.iconsMissing,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert a Mermaid `flowchart` to a redactable draw.io diagram via a PURE
|
||||
* parser (no Electron / draw.io CLI): mermaid text -> graph-JSON -> the
|
||||
* drawioFromGraph pipeline. Only `flowchart`/`graph` is supported (the most
|
||||
* common wiki case); other diagram types throw a clear error so the model can
|
||||
* fall back to drawioFromGraph.
|
||||
*/
|
||||
async drawioFromMermaid(
|
||||
pageId: string,
|
||||
where: {
|
||||
position: "before" | "after" | "append";
|
||||
anchorNodeId?: string;
|
||||
anchorText?: string;
|
||||
},
|
||||
mermaid: string,
|
||||
preset?: string,
|
||||
): Promise<{
|
||||
success: boolean;
|
||||
nodeId: string;
|
||||
attachmentId: string;
|
||||
warnings: string[];
|
||||
iconsResolved: number;
|
||||
iconsMissing: string[];
|
||||
verify?: any;
|
||||
}> {
|
||||
await this.ensureAuthenticated();
|
||||
const graph = mermaidToGraph(mermaid);
|
||||
if (preset) graph.preset = preset;
|
||||
return this.drawioFromGraph(pageId, where, graph, graph.direction, graph.preset);
|
||||
}
|
||||
|
||||
// --- Page history / diff / transform ---
|
||||
|
||||
/**
|
||||
* List the saved versions (history snapshots) of a page, newest first.
|
||||
* Docmost auto-snapshots on every save. Returns one cursor-paginated page of
|
||||
* results: `{ items, nextCursor }`. The history record's id field is `id`.
|
||||
*/
|
||||
}
|
||||
return DrawioMixin;
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
// Central REST error diagnostics (issues #437 + #450). SINGLE place that maps an
|
||||
// axios error to the model-facing message. Extracted verbatim from client.ts;
|
||||
// the constructor's response interceptor (see client/context.ts) routes every
|
||||
// REST call through formatDocmostAxiosError so the whole surface is uniform.
|
||||
import axios from "axios";
|
||||
|
||||
// --- Issue #437: central error diagnostics -------------------------------
|
||||
// The agent only ever sees the thrown exception's `error.message`, so a failed
|
||||
// tool must return an ACTIONABLE message (method, path, status, and the
|
||||
// server's own validation text) instead of the opaque "Request failed with
|
||||
// status code 400". These helpers + the response interceptor in the
|
||||
// constructor are the single authoritative place that text is composed.
|
||||
|
||||
// Overall cap on the composed diagnostic message so the model context stays
|
||||
// compact and a (whitelisted) server string can never blow up the text.
|
||||
const ERROR_MESSAGE_CAP = 300;
|
||||
// Only attempt to JSON.parse an arraybuffer body under this size: a larger
|
||||
// binary body is never a JSON error envelope, so parsing it just wastes memory
|
||||
// (fetchInternalFile uses responseType:"arraybuffer", so a failed file fetch
|
||||
// carries the JSON error envelope as raw bytes here).
|
||||
const ERROR_BUFFER_PARSE_CAP = 4096;
|
||||
|
||||
// Canonical 36-char UUID (8-4-4-4-12 hex). Deliberately version/variant-
|
||||
// AGNOSTIC: the ids are UUIDv7 (e.g. 019f499a-9f8c-7d68-...), so only the
|
||||
// canonical shape/length is enforced, not the version/variant nibble.
|
||||
const FULL_UUID_RE =
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
||||
|
||||
/**
|
||||
* Throw an actionable error BEFORE any network call when `value` is not a full
|
||||
* canonical UUID. Absorbs #436: a truncated/short comment id used to reach the
|
||||
* server and bounce back as an opaque 400/404 the agent could not self-correct;
|
||||
* failing fast here names the exact fix.
|
||||
*/
|
||||
export function assertFullUuid(
|
||||
tool: string,
|
||||
param: string,
|
||||
value: string,
|
||||
): void {
|
||||
if (typeof value !== "string" || !FULL_UUID_RE.test(value)) {
|
||||
throw new Error(
|
||||
`${tool}: '${param}' must be the FULL comment UUID (36 chars, e.g. ` +
|
||||
`019f499a-9f8c-7d68-b7be-ce100d7c6c56), got '${value}'. Copy the id ` +
|
||||
`verbatim from listComments / createComment output.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Keep ONLY the pathname of a request (no host, no query string, no fragment)
|
||||
// so the message never leaks a host or query params. Resolves a relative
|
||||
// config.url against config.baseURL, then discards everything but the path.
|
||||
function requestPath(config: any): string {
|
||||
const rawUrl = typeof config?.url === "string" ? config.url : "";
|
||||
const base =
|
||||
typeof config?.baseURL === "string" ? config.baseURL : undefined;
|
||||
try {
|
||||
// A dummy base makes an absolute config.url parse too; its host is dropped.
|
||||
return new URL(rawUrl, base ?? "http://localhost").pathname;
|
||||
} catch {
|
||||
// Malformed url: still strip any query/fragment manually.
|
||||
return rawUrl.split(/[?#]/)[0] || rawUrl;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose the server-facing message from `error.response.data`, using ONLY the
|
||||
* whitelisted `message`/`error` fields or the HTTP statusText. SECURITY: the
|
||||
* raw response body, headers (Authorization!) and config are NEVER read here —
|
||||
* a string/HTML body (e.g. a proxy's 502 page) is deliberately dropped in
|
||||
* favour of the statusText.
|
||||
*/
|
||||
function extractServerMessage(data: any, statusText: string): string {
|
||||
// class-validator envelope: { message: string | string[], error?: string }.
|
||||
if (
|
||||
data &&
|
||||
typeof data === "object" &&
|
||||
!Buffer.isBuffer(data) &&
|
||||
!(data instanceof ArrayBuffer)
|
||||
) {
|
||||
const msg = (data as any).message;
|
||||
if (Array.isArray(msg)) {
|
||||
const joined = msg.filter((m) => typeof m === "string").join("; ");
|
||||
if (joined) return joined;
|
||||
} else if (typeof msg === "string" && msg) {
|
||||
return msg;
|
||||
}
|
||||
const err = (data as any).error;
|
||||
if (typeof err === "string" && err) return err;
|
||||
return statusText;
|
||||
}
|
||||
|
||||
// Buffer / ArrayBuffer body: attempt a size-capped, guarded JSON.parse so a
|
||||
// failed arraybuffer fetch still surfaces the server's validation text.
|
||||
if (Buffer.isBuffer(data) || data instanceof ArrayBuffer) {
|
||||
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data);
|
||||
if (buf.length > 0 && buf.length <= ERROR_BUFFER_PARSE_CAP) {
|
||||
try {
|
||||
return extractServerMessage(JSON.parse(buf.toString("utf8")), statusText);
|
||||
} catch {
|
||||
return statusText;
|
||||
}
|
||||
}
|
||||
return statusText;
|
||||
}
|
||||
|
||||
// A raw string / HTML body is never surfaced (may echo server internals).
|
||||
return statusText;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reformat an AxiosError's `.message` IN PLACE into an actionable diagnostic:
|
||||
* `<METHOD> <path> failed (<status> <statusText>): <serverMessage>`
|
||||
* or, when the request never got a response:
|
||||
* `<METHOD> <path> failed: <code> (no response from server)`.
|
||||
*
|
||||
* Mutates the SAME error object (never a custom subclass) so the live
|
||||
* axios.isAxiosError / error.response?.status / config._retry checks around the
|
||||
* client keep working, and sets `_docmostFormatted` as a double-processing
|
||||
* guard. A no-op on a non-axios or already-formatted error.
|
||||
*/
|
||||
export function formatDocmostAxiosError(error: any): void {
|
||||
if (!error || error._docmostFormatted) return;
|
||||
if (!axios.isAxiosError(error)) return;
|
||||
|
||||
const config: any = error.config ?? {};
|
||||
const method =
|
||||
typeof config.method === "string" ? config.method.toUpperCase() : "";
|
||||
const methodPath = `${method} ${requestPath(config)}`.trim();
|
||||
const response = error.response;
|
||||
|
||||
let message: string;
|
||||
if (response) {
|
||||
const statusText =
|
||||
typeof response.statusText === "string" ? response.statusText : "";
|
||||
const serverMessage = extractServerMessage(response.data, statusText);
|
||||
message = `${methodPath} failed (${response.status} ${statusText}): ${serverMessage}`;
|
||||
// Full body only to stderr under DEBUG (parity with downloadImage).
|
||||
if (process.env.DEBUG) {
|
||||
console.error(
|
||||
"Docmost request failed; response body:",
|
||||
JSON.stringify(response.data),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// No response at all (ECONNREFUSED / ETIMEDOUT / ECONNRESET / DNS / timeout).
|
||||
// Use ONLY error.code, never the raw error.message: axios network messages
|
||||
// embed host:port ("connect ECONNREFUSED 127.0.0.1:3000", "getaddrinfo
|
||||
// ENOTFOUND host") and #437's invariant is that the host never reaches the
|
||||
// model-visible message. code is set for essentially every real no-response
|
||||
// error (ECONNREFUSED/ETIMEDOUT/ECONNRESET/ENOTFOUND/ECONNABORTED); the full
|
||||
// native message still goes to stderr under DEBUG.
|
||||
const reason = error.code ?? "network error";
|
||||
message = `${methodPath} failed: ${reason} (no response from server)`;
|
||||
if (process.env.DEBUG) {
|
||||
console.error("Docmost request failed; no response:", error.message);
|
||||
}
|
||||
}
|
||||
|
||||
if (message.length > ERROR_MESSAGE_CAP) {
|
||||
message = message.slice(0, ERROR_MESSAGE_CAP - 1) + "…";
|
||||
}
|
||||
|
||||
error.message = message;
|
||||
(error as any)._docmostFormatted = true;
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
// Content-addressed LRU cache for the PM->Markdown conversion in getPage
|
||||
// (issue #479). getPage is the dominant agent op (812 calls / 2h, p95 840ms);
|
||||
// the bulk of its cost is convertProseMirrorToMarkdown — a full ProseMirror-tree
|
||||
// walk over the page content (hundreds of KB of JSON on large pages) run on
|
||||
// EVERY read. Since agents re-read far more than they write (812 reads vs 28
|
||||
// writes in the sample), most conversions re-produce the SAME markdown from
|
||||
// UNCHANGED content. This cache skips the recomputation on a hit.
|
||||
//
|
||||
// KEY = (pageId, updatedAt, optionsHash):
|
||||
// - pageId: the page's CANONICAL UUID (resultData.id), not the agent-supplied
|
||||
// slugId — so a slugId read and a UUID read of the same page share one entry.
|
||||
// - updatedAt: comes from the SAME /pages/info response as `content`, so the
|
||||
// two are mutually consistent; a changed page yields a new updatedAt -> a new
|
||||
// key -> automatic, precise invalidation (no stale markdown is ever served).
|
||||
// - optionsHash: a stable hash of the conversion options. getPage passes
|
||||
// `{dropResolvedCommentAnchors:true}` while exportPageMarkdown passes the
|
||||
// defaults (#328) — DIFFERENT output for the same content, so the options
|
||||
// MUST be part of the key or a hit would serve the wrong variant.
|
||||
//
|
||||
// BOUNDS: evict the LEAST-recently-used entry when EITHER the entry count OR the
|
||||
// total stored bytes would exceed its cap. Large pages are hundreds of KB, so a
|
||||
// byte cap (not just a count cap) is what actually bounds memory. A Map iterates
|
||||
// in insertion order, so the first key is the LRU entry; a hit re-inserts its key
|
||||
// to move it to the most-recently-used end.
|
||||
//
|
||||
// This module is dependency-neutral (no axios/client/prom-client): a plain class
|
||||
// the shared client context owns one instance of, so the cache persists across
|
||||
// getPage calls on a single DocmostClient instance (built per user / per chat).
|
||||
|
||||
/** A stable, order-insensitive hash of the conversion options object. */
|
||||
export function hashConvertOptions(options: unknown): string {
|
||||
// JSON.stringify with SORTED keys makes the hash independent of key order, so
|
||||
// {a:1,b:2} and {b:2,a:1} collapse to one entry. undefined/null options -> a
|
||||
// fixed empty-object key, matching a caller that passes no options at all.
|
||||
if (options === undefined || options === null) return "{}";
|
||||
return stableStringify(options);
|
||||
}
|
||||
|
||||
function stableStringify(value: any): string {
|
||||
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
||||
if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
|
||||
const keys = Object.keys(value).sort();
|
||||
return `{${keys
|
||||
.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
|
||||
interface CacheEntry {
|
||||
markdown: string;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface GetPageCacheOptions {
|
||||
/** Max number of entries before LRU eviction. Default 50. */
|
||||
maxEntries?: number;
|
||||
/** Max total stored bytes before LRU eviction. Default 10 MB. */
|
||||
maxBytes?: number;
|
||||
}
|
||||
|
||||
export class GetPageConversionCache {
|
||||
private readonly maxEntries: number;
|
||||
private readonly maxBytes: number;
|
||||
// Insertion-ordered: the FIRST key is the least-recently-used entry.
|
||||
private readonly map = new Map<string, CacheEntry>();
|
||||
private totalBytes = 0;
|
||||
|
||||
constructor(opts: GetPageCacheOptions = {}) {
|
||||
// A non-positive/NaN cap is treated as "use the default", never as an
|
||||
// unbounded (or always-empty) cache — a silently unbounded cache would leak
|
||||
// memory, and an always-empty one would defeat the whole optimization.
|
||||
this.maxEntries =
|
||||
Number.isFinite(opts.maxEntries) && (opts.maxEntries as number) > 0
|
||||
? Math.floor(opts.maxEntries as number)
|
||||
: 50;
|
||||
this.maxBytes =
|
||||
Number.isFinite(opts.maxBytes) && (opts.maxBytes as number) > 0
|
||||
? Math.floor(opts.maxBytes as number)
|
||||
: 10 * 1024 * 1024;
|
||||
}
|
||||
|
||||
/** Compose the content-addressed key from its three parts. */
|
||||
static key(pageId: string, updatedAt: string, optionsHash: string): string {
|
||||
// A space separates the parts so no combination of values can collide by
|
||||
// concatenation: a canonical UUID and an ISO updatedAt never contain a
|
||||
// space, so the boundaries between the three parts are unambiguous.
|
||||
return `${pageId} ${updatedAt} ${optionsHash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Return the cached markdown for `key`, or undefined on a miss. A hit moves
|
||||
* the entry to the most-recently-used end (delete + re-set) so the LRU order
|
||||
* reflects real access, not just insertion.
|
||||
*/
|
||||
get(key: string): string | undefined {
|
||||
const entry = this.map.get(key);
|
||||
if (entry === undefined) return undefined;
|
||||
this.map.delete(key);
|
||||
this.map.set(key, entry);
|
||||
return entry.markdown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Store `markdown` under `key`, then evict LRU entries until BOTH caps hold.
|
||||
* Re-storing an existing key refreshes its value and recency (its old bytes
|
||||
* are subtracted first, so totalBytes stays exact).
|
||||
*/
|
||||
set(key: string, markdown: string): void {
|
||||
// Byte size of the stored string (UTF-8). A single entry larger than the
|
||||
// whole byte cap is still stored (so getPage always gets a hit next time),
|
||||
// then the eviction loop below simply cannot shrink below it — accepted:
|
||||
// one oversized page is bounded by the page itself, not a cache leak.
|
||||
const bytes = Buffer.byteLength(markdown, "utf8");
|
||||
const existing = this.map.get(key);
|
||||
if (existing !== undefined) {
|
||||
this.totalBytes -= existing.bytes;
|
||||
this.map.delete(key);
|
||||
}
|
||||
this.map.set(key, { markdown, bytes });
|
||||
this.totalBytes += bytes;
|
||||
this.evict();
|
||||
}
|
||||
|
||||
/** Evict the LRU entry until both the count and byte caps are satisfied. */
|
||||
private evict(): void {
|
||||
while (
|
||||
this.map.size > this.maxEntries ||
|
||||
(this.totalBytes > this.maxBytes && this.map.size > 1)
|
||||
) {
|
||||
// The first key in insertion order is the least-recently-used.
|
||||
const oldest = this.map.keys().next().value as string | undefined;
|
||||
if (oldest === undefined) break;
|
||||
const entry = this.map.get(oldest);
|
||||
this.map.delete(oldest);
|
||||
if (entry) this.totalBytes -= entry.bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/** Current entry count (test/introspection). */
|
||||
get size(): number {
|
||||
return this.map.size;
|
||||
}
|
||||
|
||||
/** Current total stored bytes (test/introspection). */
|
||||
get bytes(): number {
|
||||
return this.totalBytes;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,730 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import FormData from "form-data";
|
||||
import axios, { AxiosInstance } from "axios";
|
||||
import { basename, extname } from "path";
|
||||
import {
|
||||
updatePageContentRealtime,
|
||||
replacePageContent,
|
||||
markdownToProseMirror,
|
||||
markdownToProseMirrorCanonical,
|
||||
mutatePageContent,
|
||||
assertYjsEncodable,
|
||||
MutationResult,
|
||||
} from "../lib/collaboration.js";
|
||||
import { withPageLock, isUuid } from "../lib/page-lock.js";
|
||||
import { diffDocs, summarizeChange } from "../lib/diff.js";
|
||||
import {
|
||||
blockText,
|
||||
walk,
|
||||
getList,
|
||||
insertMarkerAfter,
|
||||
setCalloutRange,
|
||||
noteItem,
|
||||
mdToInlineNodes,
|
||||
commentsToFootnotes,
|
||||
canonicalizeFootnotes,
|
||||
insertInlineFootnote,
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
|
||||
// Supported image types, kept as two lookup tables so both a local file
|
||||
// extension and a remote Content-Type can be mapped to the same canonical set.
|
||||
const EXT_TO_MIME: Record<string, string> = {
|
||||
".png": "image/png",
|
||||
".jpg": "image/jpeg",
|
||||
".jpeg": "image/jpeg",
|
||||
".gif": "image/gif",
|
||||
".webp": "image/webp",
|
||||
".svg": "image/svg+xml",
|
||||
};
|
||||
const MIME_TO_EXT: Record<string, string> = {
|
||||
"image/png": ".png",
|
||||
"image/jpeg": ".jpg",
|
||||
"image/gif": ".gif",
|
||||
"image/webp": ".webp",
|
||||
"image/svg+xml": ".svg",
|
||||
};
|
||||
|
||||
// Public method surface of MediaMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements IMediaMixin` fails to compile on drift.
|
||||
export interface IMediaMixin {
|
||||
uploadImage(pageId: string, url: string): any;
|
||||
insertImage(pageId: string, url: string, opts?: { align?: "left" | "center" | "right"; alt?: string; replaceText?: string; afterText?: string; }): any;
|
||||
replaceImage(pageId: string, oldAttachmentId: string, url: string, opts?: { align?: "left" | "center" | "right"; alt?: string }): any;
|
||||
}
|
||||
|
||||
export function MediaMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & IMediaMixin> & TBase {
|
||||
abstract class MediaMixin extends Base implements IMediaMixin {
|
||||
// --- Image upload / embedding ---
|
||||
|
||||
/** Map a Content-Type string to a supported MIME type, or null if unsupported. */
|
||||
protected supportedImageMime(ct: string): string | null {
|
||||
return MIME_TO_EXT[ct] ? ct : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a remote image from a caller-supplied URL and resolve its bytes,
|
||||
* MIME and a filename.
|
||||
*
|
||||
* SSRF / RESOURCE TRUST BOUNDARY: the URL comes from the MCP caller and is
|
||||
* fetched BY THE SERVER, so it must be guarded before and after the request.
|
||||
* The guards mirror the local-file trust boundary in uploadImage:
|
||||
* - scheme allowlist (http/https only) — rejects file:, data:, ftp:, etc.,
|
||||
* so the caller cannot use this path to read local files or other schemes;
|
||||
* - a size cap enforced both via axios maxContentLength/maxBodyLength AND a
|
||||
* post-download buffer.length re-check (defends against a missing/lying
|
||||
* Content-Length), so a huge response cannot exhaust memory;
|
||||
* - a 30s timeout. The timeout matters because replaceImage holds the
|
||||
* per-page lock across this upload, so a hung download would wedge the
|
||||
* lock for that page.
|
||||
* We deliberately do NOT block private IP ranges: the MCP caller is already
|
||||
* trusted to read arbitrary host files via the filePath path, so the marginal
|
||||
* trust granted by fetching internal URLs is comparable, and blocking would
|
||||
* break legitimate internal-image use.
|
||||
*/
|
||||
protected async fetchRemoteImage(
|
||||
url: string,
|
||||
maxBytes: number,
|
||||
): Promise<{ buffer: Buffer; mime: string; fileName: string }> {
|
||||
// Scheme allowlist first — cheapest guard, and rejects non-http(s) schemes
|
||||
// (file:, data:, ftp:, ...) before any network request is made.
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch (e: any) {
|
||||
throw new Error(`Invalid image URL "${url}": ${e.message}`);
|
||||
}
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new Error(
|
||||
`unsupported image URL scheme "${parsed.protocol}"; only http and https are allowed`,
|
||||
);
|
||||
}
|
||||
|
||||
let response;
|
||||
try {
|
||||
response = await axios.get(url, {
|
||||
responseType: "arraybuffer",
|
||||
timeout: 30000,
|
||||
maxContentLength: maxBytes,
|
||||
maxBodyLength: maxBytes,
|
||||
headers: { Accept: "image/*" },
|
||||
});
|
||||
} catch (error) {
|
||||
// Keep the thrown message free of the raw response body (it may echo
|
||||
// server internals); surface only status/statusText. The full body is
|
||||
// logged under DEBUG for diagnostics.
|
||||
if (axios.isAxiosError(error)) {
|
||||
if (process.env.DEBUG) {
|
||||
console.error(
|
||||
"Image download failed; response body:",
|
||||
JSON.stringify(error.response?.data),
|
||||
);
|
||||
}
|
||||
throw new Error(
|
||||
`Image download failed for "${url}": ${error.response?.status ?? ""} ${error.response?.statusText ?? error.message}`.trim(),
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
// axios returns an ArrayBuffer for responseType: "arraybuffer".
|
||||
const buffer = Buffer.from(response.data);
|
||||
// Re-check the size: maxContentLength relies on Content-Length, which may be
|
||||
// absent or lie, so guard against the actual byte count too.
|
||||
if (buffer.length === 0) {
|
||||
throw new Error(`Empty image response from "${url}"`);
|
||||
}
|
||||
if (buffer.length > maxBytes) {
|
||||
throw new Error(
|
||||
`Image too large: ${buffer.length} bytes exceeds the ${maxBytes}-byte cap`,
|
||||
);
|
||||
}
|
||||
|
||||
// Resolve MIME: prefer the response Content-Type (strip any "; charset=..."
|
||||
// parameter, lowercase, trim) mapped through the supported set; if the
|
||||
// header is generic/missing/unsupported, fall back to the URL path
|
||||
// extension via the existing extension->MIME logic.
|
||||
const rawCt = response.headers?.["content-type"];
|
||||
let mime: string | null = null;
|
||||
if (typeof rawCt === "string" && rawCt.length > 0) {
|
||||
const ct = rawCt.split(";")[0].trim().toLowerCase();
|
||||
mime = this.supportedImageMime(ct);
|
||||
}
|
||||
if (!mime) {
|
||||
// Fall back to the URL path extension. Use the pathname so the query
|
||||
// string never contaminates the extension lookup.
|
||||
const ext = extname(parsed.pathname).toLowerCase();
|
||||
mime = EXT_TO_MIME[ext] ?? null;
|
||||
}
|
||||
if (!mime) {
|
||||
throw new Error(
|
||||
`cannot determine supported image type for "${url}"; supported: png, jpg, jpeg, gif, webp, svg`,
|
||||
);
|
||||
}
|
||||
|
||||
// Build a filename from the URL path basename (ignore the query string),
|
||||
// defaulting to "image" when empty, and ensure it ends with the canonical
|
||||
// extension for the resolved MIME (append it when missing/mismatched).
|
||||
const canonicalExt = MIME_TO_EXT[mime];
|
||||
let fileName = basename(parsed.pathname) || "image";
|
||||
if (extname(fileName).toLowerCase() !== canonicalExt) {
|
||||
fileName += canonicalExt;
|
||||
}
|
||||
|
||||
return { buffer, mime, fileName };
|
||||
}
|
||||
|
||||
/** Build a Docmost ProseMirror image node from an uploaded attachment. */
|
||||
protected buildImageNode(
|
||||
att: { id: string; fileName: string; fileSize?: number },
|
||||
align?: "left" | "center" | "right",
|
||||
alt?: string,
|
||||
): any {
|
||||
// Clean file URL, matching Docmost's native behaviour. No cache-busting
|
||||
// query: the server serves the bare URL correctly, and replacement creates
|
||||
// a new attachment id (a new URL) which busts caches naturally.
|
||||
const src = `/api/files/${att.id}/${att.fileName}`;
|
||||
const node: any = {
|
||||
type: "image",
|
||||
attrs: {
|
||||
src,
|
||||
attachmentId: att.id,
|
||||
// Default to null when the server omits fileSize so the attr is never
|
||||
// undefined (undefined would be dropped on serialization / break the
|
||||
// ProseMirror image schema which expects size present).
|
||||
size: att.fileSize ?? null,
|
||||
align: align || "center",
|
||||
width: null,
|
||||
},
|
||||
};
|
||||
if (alt) node.attrs.alt = alt;
|
||||
return node;
|
||||
}
|
||||
|
||||
/**
|
||||
* Download a remote image from an http(s) URL and upload it as an attachment
|
||||
* of a page, returning the attachment metadata plus a ready-to-insert
|
||||
* ProseMirror image node. Local file paths are intentionally not supported:
|
||||
* the MCP caller is a remote AI with no access to this server's filesystem.
|
||||
*/
|
||||
async uploadImage(pageId: string, url: string) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
const MAX_IMAGE_BYTES = 20 * 1024 * 1024; // 20 MiB
|
||||
|
||||
// Fetch + validate the remote image (scheme allowlist, size cap, timeout).
|
||||
// See fetchRemoteImage for the SSRF / resource trust boundary.
|
||||
const fetched = await this.fetchRemoteImage(url, MAX_IMAGE_BYTES);
|
||||
const fileBuffer = fetched.buffer;
|
||||
const mime = fetched.mime;
|
||||
const fileName = fetched.fileName;
|
||||
|
||||
// Build a FRESH FormData for every send attempt. A FormData body is a
|
||||
// single-use stream that is CONSUMED on the first send, so it cannot be
|
||||
// replayed by this.client's response interceptor (replaying a consumed
|
||||
// stream fails with 'socket hang up'). Multipart re-auth is therefore done
|
||||
// here with bare axios and an explicit one-shot 401/403 retry that rebuilds
|
||||
// the body. Field order matters: text fields must precede the file part so
|
||||
// the server reads them; the server always generates a fresh attachment id.
|
||||
const buildForm = () => {
|
||||
const form = new FormData();
|
||||
form.append("pageId", pageId);
|
||||
form.append("file", fileBuffer, {
|
||||
filename: fileName,
|
||||
contentType: mime,
|
||||
});
|
||||
return form;
|
||||
};
|
||||
|
||||
// Local name distinct from the `url` parameter (the source image URL): this
|
||||
// is the /files/upload endpoint we POST the multipart body to.
|
||||
const uploadUrl = `${this.apiUrl}/files/upload`;
|
||||
let response;
|
||||
try {
|
||||
// Call buildForm() ONCE per attempt and reuse the instance for both
|
||||
// getHeaders() and the body so the Content-Type boundary matches the body.
|
||||
const form = buildForm();
|
||||
// Read the Authorization header from this.client's defaults (set by
|
||||
// login(), only ever deleted — never set to null) instead of building
|
||||
// `Bearer ${this.token}`: a concurrent JSON 401 can null this.token
|
||||
// mid-flight, which would otherwise produce a literal "Bearer null".
|
||||
// ensureAuthenticated() above guarantees login() ran, so the default
|
||||
// header exists here. A 60s timeout keeps a hung upload from wedging the
|
||||
// per-page lock (replaceImage holds withPageLock across this call).
|
||||
response = await axios.post(uploadUrl, form, {
|
||||
headers: {
|
||||
...form.getHeaders(),
|
||||
Authorization: this.client.defaults.headers.common["Authorization"],
|
||||
},
|
||||
timeout: 60000,
|
||||
});
|
||||
} catch (error) {
|
||||
// On an expired-token auth error, re-login and retry exactly once with a
|
||||
// freshly-rebuilt FormData (the previous one was already consumed).
|
||||
if (
|
||||
axios.isAxiosError(error) &&
|
||||
(error.response?.status === 401 || error.response?.status === 403)
|
||||
) {
|
||||
await this.login();
|
||||
const form2 = buildForm();
|
||||
response = await axios.post(uploadUrl, form2, {
|
||||
headers: {
|
||||
...form2.getHeaders(),
|
||||
Authorization: this.client.defaults.headers.common["Authorization"],
|
||||
},
|
||||
timeout: 60000,
|
||||
});
|
||||
} else if (axios.isAxiosError(error)) {
|
||||
// Keep the thrown message free of the raw response body (it may echo
|
||||
// request data or server internals); surface only status/statusText.
|
||||
// The full body is logged under DEBUG for diagnostics.
|
||||
if (process.env.DEBUG) {
|
||||
console.error(
|
||||
"Image upload failed; response body:",
|
||||
JSON.stringify(error.response?.data),
|
||||
);
|
||||
}
|
||||
throw new Error(
|
||||
`Image upload failed: ${error.response?.status} ${error.response?.statusText}`,
|
||||
);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
// The attachment may arrive bare or wrapped in a { data } envelope.
|
||||
const att = response.data?.data ?? response.data;
|
||||
if (!att?.id || !att?.fileName) {
|
||||
throw new Error(
|
||||
"Unexpected /files/upload response: " + JSON.stringify(response.data),
|
||||
);
|
||||
}
|
||||
|
||||
// Some Docmost versions omit fileSize from the upload response. Fall back
|
||||
// to the fetched byte length (the bytes we just uploaded) so callers never
|
||||
// get an undefined size.
|
||||
const resolvedSize = att.fileSize ?? fileBuffer.length;
|
||||
|
||||
return {
|
||||
attachmentId: att.id,
|
||||
fileName: att.fileName,
|
||||
fileSize: resolvedSize,
|
||||
src: `/api/files/${att.id}/${att.fileName}`,
|
||||
imageNode: this.buildImageNode({ ...att, fileSize: resolvedSize }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Upload an image from a web (http/https) URL and insert it into a page in
|
||||
* one step.
|
||||
* By default the image is appended at the end. With replaceText, the first
|
||||
* top-level block whose text contains the string is replaced; with afterText,
|
||||
* the image is inserted right after the first matching block. All other
|
||||
* block ids are preserved (only one top-level block is added or swapped).
|
||||
*/
|
||||
async insertImage(
|
||||
pageId: string,
|
||||
url: string,
|
||||
opts: {
|
||||
align?: "left" | "center" | "right";
|
||||
alt?: string;
|
||||
replaceText?: string;
|
||||
afterText?: string;
|
||||
} = {},
|
||||
) {
|
||||
const up = await this.uploadImage(pageId, url);
|
||||
// Reuse the node from uploadImage (clean /api/files/<id>/<file> src), then
|
||||
// apply align/alt onto a shallow attrs copy.
|
||||
const node: any = { ...up.imageNode, attrs: { ...up.imageNode.attrs } };
|
||||
if (opts.align) node.attrs.align = opts.align;
|
||||
if (opts.alt) node.attrs.alt = opts.alt;
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260). The
|
||||
// uploadImage /files/upload call above keeps the agent-supplied id.
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Recursively collect the plain text of a top-level block.
|
||||
const blockText = (n: any): string => {
|
||||
let out = "";
|
||||
if (n.type === "text") out += n.text || "";
|
||||
for (const child of n.content || []) out += blockText(child);
|
||||
return out;
|
||||
};
|
||||
|
||||
// Insert into the LIVE synced document, not the debounced REST snapshot, so
|
||||
// concurrent edits/comments/images are preserved and parallel insertImage
|
||||
// calls (serialized by the per-page lock) each see the previous insertion.
|
||||
let placement: "replaced" | "after" | "appended" | undefined;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
const doc =
|
||||
liveDoc && liveDoc.type === "doc"
|
||||
? liveDoc
|
||||
: { type: "doc", content: [] };
|
||||
if (!Array.isArray(doc.content)) doc.content = [];
|
||||
|
||||
if (opts.replaceText) {
|
||||
// Ambiguity guard (mirrors editPageText): count matching top-level
|
||||
// blocks first, so a non-unique fragment cannot silently replace the
|
||||
// wrong block (e.g. text that also appears inside a callout/table).
|
||||
const matches = doc.content.filter((b: any) =>
|
||||
blockText(b).includes(opts.replaceText!),
|
||||
);
|
||||
if (matches.length === 0) {
|
||||
throw new Error(`replaceText not found: "${opts.replaceText}"`);
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
throw new Error(
|
||||
`replaceText "${opts.replaceText}" matches ${matches.length} blocks; use a longer unique fragment`,
|
||||
);
|
||||
}
|
||||
const idx = doc.content.findIndex((b: any) =>
|
||||
blockText(b).includes(opts.replaceText!),
|
||||
);
|
||||
// Data-loss guard: replaceText swaps the WHOLE top-level block, so if
|
||||
// the fragment only appears nested inside a container (table, callout,
|
||||
// list, blockquote) the entire structure would be destroyed. Refuse
|
||||
// when the matched block is a container rather than a leaf
|
||||
// paragraph/heading and point the caller at a safer tool.
|
||||
const CONTAINER_TYPES = new Set([
|
||||
"table",
|
||||
"callout",
|
||||
"bulletList",
|
||||
"orderedList",
|
||||
"taskList",
|
||||
"blockquote",
|
||||
]);
|
||||
const matchedBlock = doc.content[idx];
|
||||
if (matchedBlock && CONTAINER_TYPES.has(matchedBlock.type)) {
|
||||
throw new Error(
|
||||
`replaceText matched a ${matchedBlock.type} container block; replacing it would destroy the whole structure. ` +
|
||||
`Use afterText to insert near it, or updatePageJson for surgical edits.`,
|
||||
);
|
||||
}
|
||||
doc.content.splice(idx, 1, node);
|
||||
placement = "replaced";
|
||||
} else if (opts.afterText) {
|
||||
// Ambiguity guard (mirrors editPageText): refuse a non-unique fragment.
|
||||
const matches = doc.content.filter((b: any) =>
|
||||
blockText(b).includes(opts.afterText!),
|
||||
);
|
||||
if (matches.length === 0) {
|
||||
throw new Error(`afterText not found: "${opts.afterText}"`);
|
||||
}
|
||||
if (matches.length > 1) {
|
||||
throw new Error(
|
||||
`afterText "${opts.afterText}" matches ${matches.length} blocks; use a longer unique fragment`,
|
||||
);
|
||||
}
|
||||
const idx = doc.content.findIndex((b: any) =>
|
||||
blockText(b).includes(opts.afterText!),
|
||||
);
|
||||
doc.content.splice(idx + 1, 0, node);
|
||||
placement = "after";
|
||||
} else {
|
||||
doc.content.push(node);
|
||||
placement = "appended";
|
||||
}
|
||||
|
||||
return doc;
|
||||
},
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
pageId,
|
||||
attachmentId: up.attachmentId,
|
||||
src: up.src,
|
||||
placement,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace an existing image in a page with a new image fetched from a web
|
||||
* (http/https) URL. Uploads the new file as a brand-new attachment, which
|
||||
* yields a fresh clean URL that both renders correctly and busts browser
|
||||
* caches (the URL changed). Finds every image node
|
||||
* whose attrs.attachmentId === oldAttachmentId (recursively, incl. nodes nested
|
||||
* in callouts/tables) and repoints its src/attachmentId/size, preserving
|
||||
* comments, alignment and alt. Operates on the live collab document so comments
|
||||
* and concurrent edits are preserved. Throws if no matching image is found.
|
||||
*
|
||||
* The OLD attachment is left in place as an unreferenced orphan: Docmost
|
||||
* exposes NO HTTP API to delete a single content attachment (verified against
|
||||
* the attachment controller/service and by probing the live API — deletion
|
||||
* happens only by cascade when the page, space or user is removed). This is the
|
||||
* same outcome as Docmost's own editor when an image is removed/replaced.
|
||||
* In-place byte overwrite is deliberately NOT used because some Docmost
|
||||
* versions corrupt the attachment (HTTP 500) when its bytes are overwritten.
|
||||
*/
|
||||
async replaceImage(
|
||||
pageId: string,
|
||||
oldAttachmentId: string,
|
||||
url: string,
|
||||
opts: { align?: "left" | "center" | "right"; alt?: string } = {},
|
||||
) {
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260). The
|
||||
// page lock must ALSO key on the UUID so this operation serializes against
|
||||
// other writes to the same page (mutatePageContent now locks by the resolved
|
||||
// UUID too); locking by the raw slugId here would desync the mutex key and
|
||||
// reopen the TOCTOU/orphan-attachment window the lock closes. uploadImage
|
||||
// keeps the agent-supplied id (it hits REST, not the collab doc).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Hold ONE per-page lock for the WHOLE operation (scan -> upload -> write).
|
||||
// Previously the scan and the write were two separate mutatePageContent
|
||||
// calls, each acquiring + releasing the lock, with the upload happening in
|
||||
// the UNLOCKED gap between them. A concurrent op could interleave there: it
|
||||
// could remove the target image so the write pass matches nothing, leaving
|
||||
// the freshly-uploaded attachment as an un-deletable orphan (Docmost has no
|
||||
// API to delete a single content attachment). Acquiring the lock once and
|
||||
// using the non-locking collab helper inside (the per-page mutex is NOT
|
||||
// reentrant, so the self-locking mutatePageContent would deadlock here)
|
||||
// closes that TOCTOU window. uploadImage hits /files/upload over plain HTTP
|
||||
// and does not touch the page lock, so it is safe to call while held.
|
||||
return withPageLock(pageUuid, async () => {
|
||||
// STEP 1: read-only live check. Scan the live document for any image node
|
||||
// matching oldAttachmentId BEFORE uploading anything, so a wrong/stale id
|
||||
// throws without ever creating an orphan attachment.
|
||||
let matchFound = false;
|
||||
const scan = (nodes: any[]) => {
|
||||
for (const node of nodes) {
|
||||
if (!node) continue;
|
||||
if (
|
||||
node.type === "image" &&
|
||||
node.attrs &&
|
||||
node.attrs.attachmentId === oldAttachmentId
|
||||
) {
|
||||
matchFound = true;
|
||||
}
|
||||
if (Array.isArray(node.content)) scan(node.content);
|
||||
}
|
||||
};
|
||||
|
||||
await this.mutateLiveContentUnlocked(pageUuid, collabToken, (liveDoc) => {
|
||||
matchFound = false; // reset per-transform (collab may retry the read).
|
||||
const doc =
|
||||
liveDoc && liveDoc.type === "doc"
|
||||
? liveDoc
|
||||
: { type: "doc", content: [] };
|
||||
if (Array.isArray(doc.content)) scan(doc.content);
|
||||
return null; // read-only: never write on the check pass.
|
||||
});
|
||||
|
||||
if (!matchFound) {
|
||||
throw new Error(
|
||||
`replaceImage: no image with attachmentId "${oldAttachmentId}" found on page ${pageId}`,
|
||||
);
|
||||
}
|
||||
|
||||
// STEP 2: a match exists — upload the new file as a FRESH attachment (new
|
||||
// id, new clean URL) and repoint every matching node in a second pass.
|
||||
// Still inside the SAME lock, so no other op can have changed the page
|
||||
// since the scan.
|
||||
const up = await this.uploadImage(pageId, url);
|
||||
|
||||
let replaced = 0;
|
||||
|
||||
// Swap the source of one image node, preserving align/alt/title/geometry.
|
||||
const repoint = (node: any) => {
|
||||
node.attrs = {
|
||||
...node.attrs,
|
||||
src: up.src,
|
||||
attachmentId: up.attachmentId,
|
||||
// Default to null when fileSize is unknown so the attr is never
|
||||
// undefined.
|
||||
size: up.fileSize ?? null,
|
||||
};
|
||||
if (opts.align) node.attrs.align = opts.align;
|
||||
if (opts.alt !== undefined) node.attrs.alt = opts.alt;
|
||||
replaced++;
|
||||
};
|
||||
|
||||
// Recursively repoint every image node (incl. ones nested in callouts/tables).
|
||||
const walk = (nodes: any[]) => {
|
||||
for (const node of nodes) {
|
||||
if (!node) continue;
|
||||
if (
|
||||
node.type === "image" &&
|
||||
node.attrs &&
|
||||
node.attrs.attachmentId === oldAttachmentId
|
||||
) {
|
||||
repoint(node);
|
||||
}
|
||||
if (Array.isArray(node.content)) walk(node.content);
|
||||
}
|
||||
};
|
||||
|
||||
const mutation = await this.mutateLiveContentUnlocked(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
(liveDoc) => {
|
||||
// Reset per-transform so collab retries recompute cleanly (no double-count).
|
||||
replaced = 0;
|
||||
const doc =
|
||||
liveDoc && liveDoc.type === "doc"
|
||||
? liveDoc
|
||||
: { type: "doc", content: [] };
|
||||
if (!Array.isArray(doc.content)) doc.content = [];
|
||||
walk(doc.content);
|
||||
if (replaced === 0) return null; // no match -> skip the write entirely
|
||||
return doc;
|
||||
},
|
||||
);
|
||||
// KNOWN LIMITATION: a same-count image SRC swap (image count unchanged, no
|
||||
// text/mark change) may still report verify.changed === false, because the
|
||||
// text+marks+integrity-count model in summarizeChange does not inspect
|
||||
// image `src`/attachmentId attributes. That is acceptable here — the
|
||||
// replace is confirmed by `replaced` below, and verify is supplementary.
|
||||
|
||||
if (replaced === 0) {
|
||||
// The pass-1 SCAN found the target (matchFound was true) and we already
|
||||
// uploaded the new attachment, but pass-2 matched nothing — a concurrent
|
||||
// editor must have removed the node between the two passes. Do NOT throw
|
||||
// here (that would leak the just-uploaded attachment AND report failure);
|
||||
// instead report success with the upload flagged as an unreferenced
|
||||
// orphan so the caller knows. (The early throw above still covers the
|
||||
// case where pass-1 finds nothing, before any upload happens.)
|
||||
return {
|
||||
success: true,
|
||||
replaced: 0,
|
||||
pageId,
|
||||
oldAttachmentId,
|
||||
newAttachmentId: up.attachmentId,
|
||||
src: up.src,
|
||||
orphanedAttachmentId: up.attachmentId,
|
||||
warning:
|
||||
"target image was removed concurrently; uploaded attachment is unreferenced",
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
pageId,
|
||||
replaced,
|
||||
oldAttachmentId,
|
||||
newAttachmentId: up.attachmentId,
|
||||
src: up.src,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
// --- draw.io diagrams (issue #423) ---
|
||||
|
||||
/**
|
||||
* Upload a ready-made byte buffer as a page attachment via the same
|
||||
* multipart /files/upload endpoint uploadImage uses. Split out as its own
|
||||
* (overridable) seam so drawioCreate/update can upload the generated
|
||||
* `.drawio.svg` without going through the URL-fetch path, and so tests can
|
||||
* stub the network. Mirrors uploadImage's fresh-FormData + one-shot 401/403
|
||||
* re-auth handling (a FormData body is single-use, so it must be rebuilt per
|
||||
* attempt).
|
||||
*/
|
||||
|
||||
// --- draw.io diagrams (issue #423) ---
|
||||
|
||||
/**
|
||||
* Upload a ready-made byte buffer as a page attachment via the same
|
||||
* multipart /files/upload endpoint uploadImage uses. Split out as its own
|
||||
* (overridable) seam so drawioCreate/update can upload the generated
|
||||
* `.drawio.svg` without going through the URL-fetch path, and so tests can
|
||||
* stub the network. Mirrors uploadImage's fresh-FormData + one-shot 401/403
|
||||
* re-auth handling (a FormData body is single-use, so it must be rebuilt per
|
||||
* attempt).
|
||||
*/
|
||||
protected async uploadAttachmentBuffer(
|
||||
pageId: string,
|
||||
buffer: Buffer,
|
||||
fileName: string,
|
||||
mime: string,
|
||||
): Promise<{ id: string; fileName: string; fileSize: number }> {
|
||||
await this.ensureAuthenticated();
|
||||
const buildForm = () => {
|
||||
const form = new FormData();
|
||||
form.append("pageId", pageId);
|
||||
form.append("file", buffer, { filename: fileName, contentType: mime });
|
||||
return form;
|
||||
};
|
||||
const uploadUrl = `${this.apiUrl}/files/upload`;
|
||||
let response;
|
||||
try {
|
||||
const form = buildForm();
|
||||
response = await axios.post(uploadUrl, form, {
|
||||
headers: {
|
||||
...form.getHeaders(),
|
||||
Authorization: this.client.defaults.headers.common["Authorization"],
|
||||
},
|
||||
timeout: 60000,
|
||||
});
|
||||
} catch (error) {
|
||||
if (
|
||||
axios.isAxiosError(error) &&
|
||||
(error.response?.status === 401 || error.response?.status === 403)
|
||||
) {
|
||||
await this.login();
|
||||
const form2 = buildForm();
|
||||
response = await axios.post(uploadUrl, form2, {
|
||||
headers: {
|
||||
...form2.getHeaders(),
|
||||
Authorization: this.client.defaults.headers.common["Authorization"],
|
||||
},
|
||||
timeout: 60000,
|
||||
});
|
||||
} else if (axios.isAxiosError(error)) {
|
||||
if (process.env.DEBUG) {
|
||||
console.error(
|
||||
"Attachment upload failed; response body:",
|
||||
JSON.stringify(error.response?.data),
|
||||
);
|
||||
}
|
||||
throw new Error(
|
||||
`Attachment upload failed: ${error.response?.status} ${error.response?.statusText}`,
|
||||
);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const att = response.data?.data ?? response.data;
|
||||
if (!att?.id || !att?.fileName) {
|
||||
throw new Error(
|
||||
"Unexpected /files/upload response: " + JSON.stringify(response.data),
|
||||
);
|
||||
}
|
||||
return {
|
||||
id: att.id,
|
||||
fileName: att.fileName,
|
||||
fileSize: att.fileSize ?? buffer.length,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a stored `.drawio.svg` attachment as text. Overridable seam over
|
||||
* fetchInternalFile (the authed loopback fetch, which also rejects any
|
||||
* traversal/SSRF src) so drawioGet/update can read the current diagram and
|
||||
* tests can stub the bytes.
|
||||
*/
|
||||
protected async fetchAttachmentText(src: string): Promise<string> {
|
||||
const { buffer } = await this.fetchInternalFile(src);
|
||||
return buffer.toString("utf-8");
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve a drawio node on a page by `attrs.id` or `#<index>` and return the
|
||||
* node plus its ref. Throws a clear error if the ref does not resolve to a
|
||||
* drawio node.
|
||||
*/
|
||||
}
|
||||
return MediaMixin;
|
||||
}
|
||||
@@ -0,0 +1,700 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import {
|
||||
updatePageContentRealtime,
|
||||
replacePageContent,
|
||||
markdownToProseMirror,
|
||||
markdownToProseMirrorCanonical,
|
||||
mutatePageContent,
|
||||
assertYjsEncodable,
|
||||
MutationResult,
|
||||
} from "../lib/collaboration.js";
|
||||
import {
|
||||
replaceNodeById,
|
||||
replaceNodeByIdWithMany,
|
||||
reassignCollidingBlockIds,
|
||||
deleteNodeById,
|
||||
assertUnambiguousMatch,
|
||||
insertNodeRelative,
|
||||
insertNodesRelative,
|
||||
blockPlainText,
|
||||
buildOutline,
|
||||
getNodeByRef,
|
||||
readTable,
|
||||
insertTableRow,
|
||||
deleteTableRow,
|
||||
updateTableCell,
|
||||
findInvalidNode,
|
||||
} from "@docmost/prosemirror-markdown";
|
||||
import {
|
||||
importMarkdownFragment,
|
||||
canBeDocChild,
|
||||
findUnrepresentableTableAttrs,
|
||||
} from "../lib/markdown-fragment.js";
|
||||
import {
|
||||
applyTextEdits,
|
||||
TextEdit,
|
||||
TextEditResult,
|
||||
TextEditFailure,
|
||||
} from "../lib/json-edit.js";
|
||||
import {
|
||||
blockText,
|
||||
walk,
|
||||
getList,
|
||||
insertMarkerAfter,
|
||||
setCalloutRange,
|
||||
noteItem,
|
||||
mdToInlineNodes,
|
||||
commentsToFootnotes,
|
||||
canonicalizeFootnotes,
|
||||
insertInlineFootnote,
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
import { normalizeAndMergeFootnotes } from "../lib/footnote-normalize-merge.js";
|
||||
|
||||
// Public method surface of NodesWriteMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements INodesWriteMixin` fails to compile on drift.
|
||||
export interface INodesWriteMixin {
|
||||
updatePageJson(pageId: string, doc?: any, title?: string): any;
|
||||
editPageText(pageId: string, edits: TextEdit[]): any;
|
||||
patchNode(pageId: string, nodeId: string, input: { markdown?: string; node?: any }): any;
|
||||
insertNode(pageId: string, input: { markdown?: string; node?: any }, opts: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }): any;
|
||||
deleteNode(pageId: string, nodeId: string): any;
|
||||
}
|
||||
|
||||
export function NodesWriteMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & INodesWriteMixin> & TBase {
|
||||
abstract class NodesWriteMixin extends Base implements INodesWriteMixin {
|
||||
/**
|
||||
* Replace page content with a raw ProseMirror JSON document (lossless) and/or
|
||||
* update its title. Both `doc` and `title` are optional, but at least one must
|
||||
* be supplied:
|
||||
* - `doc` provided -> validate + full-overwrite the body (and update the
|
||||
* title too when `title` is also given).
|
||||
* - `doc` omitted, `title` given -> title-only update; the body is NOT
|
||||
* touched/resent (no collab write happens).
|
||||
* - neither given -> throws (nothing to update).
|
||||
*/
|
||||
async updatePageJson(pageId: string, doc?: any, title?: string) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// Title-only / no-op handling: when no document is supplied, do NOT write
|
||||
// the body. Update the title if one was given; otherwise there is nothing
|
||||
// to do, so fail loudly rather than silently no-op.
|
||||
if (doc == null) {
|
||||
if (!title) {
|
||||
throw new Error(
|
||||
"updatePageJson: nothing to update (provide content and/or title)",
|
||||
);
|
||||
}
|
||||
await this.client.post("/pages/update", { pageId, title });
|
||||
return {
|
||||
success: true,
|
||||
modified: true,
|
||||
message: "Page title updated (content left unchanged).",
|
||||
pageId,
|
||||
};
|
||||
}
|
||||
|
||||
// Validate the document shape before a full overwrite: a malformed doc
|
||||
// would otherwise silently corrupt the page (full-overwrite is the
|
||||
// documented behaviour; no optimistic-concurrency is applied here).
|
||||
if (
|
||||
typeof doc !== "object" ||
|
||||
doc.type !== "doc" ||
|
||||
!Array.isArray(doc.content)
|
||||
) {
|
||||
throw new Error(
|
||||
'content must be a ProseMirror document ({"type":"doc","content":[...]}) ' +
|
||||
"where content is an array of nodes each having a string `type`",
|
||||
);
|
||||
}
|
||||
|
||||
// Recurse the WHOLE document so a malformed nested node (e.g. a node with a
|
||||
// non-string type, a non-array content/marks, or a text node missing its
|
||||
// string text) is rejected up front rather than silently corrupting the
|
||||
// page on overwrite.
|
||||
this.validateDocStructure(doc);
|
||||
|
||||
// #409: beyond the string-`type` check above, reject a nested node whose
|
||||
// `type` is a string but NOT a known Docmost schema node (a typo/unknown
|
||||
// block) — the same `Unknown node type` the encoder throws — with a rich,
|
||||
// path-anchored message, still BEFORE any collab connection.
|
||||
this.assertValidNodeShape("updatePageJson", doc);
|
||||
|
||||
// Sanitize URLs before writing. This closes the JSON-path bypass: unlike
|
||||
// the markdown link path (which TipTap sanitizes), raw JSON could otherwise
|
||||
// inject javascript:/data: link hrefs or media srcs straight into the doc.
|
||||
this.validateDocUrls(doc);
|
||||
|
||||
// Canonicalize footnotes (idempotent): an agent-authored JSON doc cannot
|
||||
// leave footnotes out of order, orphaned, or in multiple lists — the bottom
|
||||
// list + numbering are always derived from reference order. No-op when the
|
||||
// footnotes are already canonical.
|
||||
// #419: normalize + merge glyph-forked definitions before canonicalizing.
|
||||
doc = normalizeAndMergeFootnotes(doc);
|
||||
doc = canonicalizeFootnotes(doc);
|
||||
|
||||
// Write the BODY first, then the title (#159 split-brain): a failed body
|
||||
// write (e.g. persist timeout) must not leave a new title over the old body.
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
const mutation = await this.replacePage(
|
||||
pageUuid,
|
||||
doc,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
);
|
||||
|
||||
// Body persisted successfully — now it is safe to set the title.
|
||||
if (title) {
|
||||
await this.client.post("/pages/update", { pageId, title });
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
modified: true,
|
||||
message: "Page content replaced from ProseMirror JSON.",
|
||||
pageId,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* AUTHOR-INLINE footnote insertion. The agent supplies only WHERE
|
||||
* (`anchorText`, a snippet of body text to attach the marker after) and WHAT
|
||||
* (`text`, the footnote content as markdown). Numbering and the bottom
|
||||
* `footnotesList` are derived deterministically server-side
|
||||
* (`insertInlineFootnote` -> `canonicalizeFootnotes`): the agent never sees,
|
||||
* assigns, or edits a footnote number or the list, so it CANNOT desync.
|
||||
*
|
||||
* Content DEDUP: when an existing definition has the same content, its id is
|
||||
* reused (one number, one definition, several references). The write is atomic
|
||||
* via `mutatePageContent` (single-writer, page-locked); if the anchor text is
|
||||
* not found the transform aborts with a clear error and no write happens.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Surgical text edits: find/replace inside text nodes of the live
|
||||
* document. Preserves all block ids, marks, callouts and tables.
|
||||
*/
|
||||
async editPageText(pageId: string, edits: TextEdit[]) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Apply the edits against the LIVE synced document, not the debounced REST
|
||||
// snapshot, so concurrent human edits/comments are preserved. applyTextEdits
|
||||
// records per-edit match problems in `failed` instead of throwing, and
|
||||
// applies whatever it can; we abort the write only when nothing applied.
|
||||
let results: TextEditResult[] | undefined;
|
||||
let failed: TextEditFailure[] | undefined;
|
||||
// Whether we actually wrote new content. Set inside the transform: a
|
||||
// degenerate edit (e.g. find === replace, or a batch that nets to no change)
|
||||
// can "apply" yet leave the document byte-for-byte identical, in which case
|
||||
// we must NOT write (no spurious history version) and must not claim a write
|
||||
// happened.
|
||||
let wrote = false;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
wrote = false;
|
||||
const r = applyTextEdits(liveDoc, edits);
|
||||
results = r.results;
|
||||
failed = r.failed;
|
||||
// Nothing applied -> abort the write (mutatePageContent treats a null
|
||||
// return from the transform as "write nothing").
|
||||
if (r.results.length === 0) return null;
|
||||
// Edits "applied" but produced an identical document: skip the write so
|
||||
// no new history version is created. Stable structural comparison via
|
||||
// JSON.stringify (both docs come from the same deep-copied source, so
|
||||
// key order is stable).
|
||||
if (JSON.stringify(r.doc) === JSON.stringify(liveDoc)) return null;
|
||||
wrote = true;
|
||||
return r.doc;
|
||||
},
|
||||
);
|
||||
|
||||
if ((results?.length ?? 0) === 0 && (failed?.length ?? 0) > 0) {
|
||||
// No edit applied: surface an aggregated, actionable error so the caller
|
||||
// does not mistake a no-op for a partial success.
|
||||
throw new Error(
|
||||
"editPageText: no edits were applied (nothing written). " +
|
||||
failed!.map((f) => `"${f.find}": ${f.reason}`).join("; "),
|
||||
);
|
||||
}
|
||||
|
||||
// Edits matched but produced no content change (identical document): report
|
||||
// a successful no-op — NOT a failure — and do not falsely claim a write.
|
||||
if (!wrote) {
|
||||
return {
|
||||
success: true,
|
||||
pageId,
|
||||
applied: results,
|
||||
failed,
|
||||
message: "No changes written (edits produced identical content).",
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
const result: any = {
|
||||
success: true,
|
||||
pageId,
|
||||
applied: results,
|
||||
failed,
|
||||
message:
|
||||
(failed?.length ?? 0)
|
||||
? `Applied ${results?.length ?? 0} edit(s); ${failed!.length} failed (see failed[]). Node ids and formatting preserved.`
|
||||
: "Text edits applied (node ids and formatting preserved).",
|
||||
verify: mutation.verify,
|
||||
};
|
||||
|
||||
// If any applied edit matched only after stripping markdown (the
|
||||
// normalized fallback), warn that editPageText preserved existing marks
|
||||
// and did NOT change formatting — so a caller who intended a formatting
|
||||
// change is pointed at patchNode.
|
||||
if (results?.some((r) => r.normalized === true)) {
|
||||
result.warning =
|
||||
"Some edits matched only after stripping markdown from your find string; " +
|
||||
"editPageText preserved existing marks (it did not change bold/strike/etc.). " +
|
||||
"If you intended a formatting change, use patchNode.";
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the block whose attrs.id === nodeId. Operates on the LIVE collab
|
||||
* document so comments and concurrent edits are preserved.
|
||||
*
|
||||
* Exactly one of `input.markdown` / `input.node` (#413):
|
||||
* - `markdown` (RECOMMENDED): the block is rewritten from a canonical markdown
|
||||
* fragment. The fragment may import to N blocks (a "1 -> N" splice: rewrite a
|
||||
* whole section in one call). The FIRST resulting block INHERITS the target's
|
||||
* `attrs.id` (so an existing comment anchoring the block by id survives); the
|
||||
* rest get FRESH ids. `^[...]` footnotes in the fragment are first-class:
|
||||
* their definitions merge into the page's TAIL footnote list (content-key
|
||||
* dedup + canonicalize), same machinery insertFootnote uses. REJECTED when
|
||||
* the TARGET block carries a table-cell attribute markdown cannot represent
|
||||
* (colspan/rowspan/colwidth/background) — use the table tools or `node`.
|
||||
* - `node`: a raw ProseMirror node for precise attr/mark work. The replacement
|
||||
* keeps the target id (if `node.attrs.id` is missing it is set to nodeId).
|
||||
*
|
||||
* #159 ambiguous-id semantics are unchanged: 0 matches -> "no node"; >1 matches
|
||||
* -> "ambiguous, refused" (nothing written), on BOTH paths — the markdown path
|
||||
* runs a dry `replaceNodeById` count first, so a duplicated id never splices.
|
||||
*/
|
||||
async patchNode(
|
||||
pageId: string,
|
||||
nodeId: string,
|
||||
input: { markdown?: string; node?: any },
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// XOR: exactly one of markdown / node. Both optional in the schema; the
|
||||
// runtime enforces the recommendation ("markdown for prose, node for fine
|
||||
// work") without letting an ambiguous both-or-neither call through.
|
||||
const hasMd =
|
||||
input != null &&
|
||||
typeof input.markdown === "string" &&
|
||||
input.markdown.trim() !== "";
|
||||
const hasNode = input != null && input.node != null;
|
||||
if (hasMd === hasNode) {
|
||||
throw new Error(
|
||||
"patchNode: provide exactly one of `markdown` (recommended, for prose) " +
|
||||
"or `node` (a raw ProseMirror node, for precise attr/mark work)",
|
||||
);
|
||||
}
|
||||
|
||||
if (hasMd) {
|
||||
return this.patchNodeMarkdown(pageId, nodeId, input.markdown as string);
|
||||
}
|
||||
return this.patchNodeJson(pageId, nodeId, input.node);
|
||||
}
|
||||
|
||||
/**
|
||||
* patchNode with a raw ProseMirror `node` (the pre-#413 behavior). Replaces
|
||||
* EVERY node whose attrs.id === nodeId; the swapped-in node keeps the target
|
||||
* id. #159 ambiguity refused. Split out so the markdown path can reuse the
|
||||
* shared collab/guard plumbing without a giant branch.
|
||||
*/
|
||||
protected async patchNodeJson(pageId: string, nodeId: string, node: any) {
|
||||
if (!node || typeof node !== "object" || typeof node.type !== "string") {
|
||||
throw new Error(
|
||||
"patchNode: `node` must be an object with a string `type`",
|
||||
);
|
||||
}
|
||||
// Preserve the block id WITHOUT mutating the caller's object: build a local
|
||||
// copy whose attrs.id === nodeId (so the swapped-in node keeps the id of the
|
||||
// node it replaces).
|
||||
const target = {
|
||||
...node,
|
||||
attrs: {
|
||||
...(node.attrs && typeof node.attrs === "object" ? node.attrs : {}),
|
||||
},
|
||||
};
|
||||
if (target.attrs.id == null) {
|
||||
target.attrs.id = nodeId;
|
||||
}
|
||||
|
||||
// #409: fail fast on a malformed node SHAPE (a nested child with an
|
||||
// absent/unknown `type`, e.g. a text leaf written as `{"text":"foo"}` with
|
||||
// no `"type":"text"`) BEFORE opening a collab session or taking the page
|
||||
// lock — the root-only `typeof node.type === "string"` check above never
|
||||
// sees nested children, and the encoder's `Unknown node type: undefined`
|
||||
// would otherwise only surface after the connection.
|
||||
this.assertValidNodeShape("patchNode", target);
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Track the replacement count in an outer var, reset per-transform, so a
|
||||
// collab retry recomputes it cleanly (mirrors replaceImage's pattern).
|
||||
let replaced = 0;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
replaced = 0;
|
||||
const { doc: nd, replaced: r } = replaceNodeById(
|
||||
liveDoc,
|
||||
nodeId,
|
||||
target,
|
||||
);
|
||||
replaced = r;
|
||||
// 0 matches -> skip the write. >1 matches -> the id is AMBIGUOUS: Docmost
|
||||
// duplicates block ids on copy/paste (and copyPageContent writes them
|
||||
// verbatim), so replacing "the node with id X" would silently clobber
|
||||
// EVERY duplicate (#159). Refuse: skip the write and throw below so the
|
||||
// model re-targets with a more specific anchor instead of corrupting the
|
||||
// page. Only an unambiguous single match is written.
|
||||
if (replaced !== 1) return null;
|
||||
return nd;
|
||||
},
|
||||
);
|
||||
|
||||
// 0 -> "no node"; >1 -> "ambiguous, refused" (the transform already skipped
|
||||
// the write for any count !== 1). Single shared guard (#159, #185 review).
|
||||
assertUnambiguousMatch("patchNode", "replace", replaced, nodeId, pageId);
|
||||
|
||||
return { success: true, replaced, nodeId, verify: mutation.verify };
|
||||
}
|
||||
|
||||
/**
|
||||
* patchNode with a MARKDOWN fragment (#413). Imports the fragment through the
|
||||
* canonical importer, then 1 -> N splices the resulting blocks in place of the
|
||||
* target block on the LIVE collab doc:
|
||||
* - the FIRST block inherits the target's id; the rest get FRESH ids (minted
|
||||
* by the importer/id-remap, so neighbour blocks are untouched);
|
||||
* - `^[...]` footnote definitions merge into the page's tail list;
|
||||
* - REJECTED when the target block carries a markdown-unrepresentable table
|
||||
* attr (colspan/rowspan/colwidth/background) — guarding against silent loss;
|
||||
* - #159 ambiguity is enforced by a dry `replaceNodeById` count BEFORE the
|
||||
* splice, so a duplicated id never writes.
|
||||
*/
|
||||
protected async patchNodeMarkdown(
|
||||
pageId: string,
|
||||
nodeId: string,
|
||||
markdown: string,
|
||||
) {
|
||||
// Import the fragment up front (network-free, canonical) so a bad fragment
|
||||
// fails before any collab connection or page lock.
|
||||
const { blocks, definitions } = await importMarkdownFragment(markdown);
|
||||
|
||||
// The first imported block inherits the target id; the rest keep the fresh
|
||||
// ids the importer assigned. Build the thread now so it is stable across a
|
||||
// collab retry (the transform below is pure over its inputs).
|
||||
const threaded = blocks.map((b, i) => {
|
||||
if (i !== 0) return b;
|
||||
return {
|
||||
...b,
|
||||
attrs: {
|
||||
...(b && typeof b.attrs === "object" ? b.attrs : {}),
|
||||
id: nodeId,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
// Shape-validate every imported block up front (parity with the JSON path):
|
||||
// the importer only emits schema nodes, but the check is cheap insurance and
|
||||
// yields the same rich #409 diagnostics if the schema ever drifts.
|
||||
for (const b of threaded) {
|
||||
this.assertValidNodeShape("patchNode", b);
|
||||
}
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
let replaced = 0;
|
||||
let guardAttrs: string | null = null;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
replaced = 0;
|
||||
guardAttrs = null;
|
||||
|
||||
// #159: count matches with the same recursive walk the JSON path uses;
|
||||
// only an UNAMBIGUOUS single match may write. A dry count keeps the
|
||||
// ambiguity semantics identical across both paths.
|
||||
const { replaced: count } = replaceNodeById(liveDoc, nodeId, {
|
||||
type: "paragraph",
|
||||
});
|
||||
replaced = count;
|
||||
if (count !== 1) return null;
|
||||
|
||||
// Guard against SILENT LOSS: if the target block carries a table-cell
|
||||
// attribute markdown cannot represent (colspan/rowspan/colwidth/
|
||||
// background), refuse the markdown rewrite so those attrs are not
|
||||
// dropped. Simple tables (no such attrs) rewrite fine.
|
||||
const hit = getNodeByRef(liveDoc, nodeId);
|
||||
guardAttrs = hit ? findUnrepresentableTableAttrs(hit.node) : null;
|
||||
if (guardAttrs != null) return null;
|
||||
|
||||
// Re-mint any minted block id that collides with an existing page id
|
||||
// (skip index 0: its id is intentionally the target nodeId, unique by
|
||||
// the #159 dry-count above), so the 1 -> N splice stays page-wide unique.
|
||||
reassignCollidingBlockIds(liveDoc, threaded, 0);
|
||||
|
||||
// 1 -> N splice, then merge any fragment footnote definitions into the
|
||||
// page's tail list and re-derive canonical footnote numbering.
|
||||
const { doc: spliced } = replaceNodeByIdWithMany(
|
||||
liveDoc,
|
||||
nodeId,
|
||||
threaded,
|
||||
);
|
||||
return mergeFootnoteDefinitions(spliced, definitions);
|
||||
},
|
||||
);
|
||||
|
||||
// Surface the guard rejection with an actionable message (nothing written).
|
||||
if (guardAttrs != null) {
|
||||
throw new Error(
|
||||
`patchNode: the target block has table-cell attributes markdown cannot ` +
|
||||
`represent (${guardAttrs}) — a markdown rewrite would drop them. Use ` +
|
||||
`the table tools (tableUpdateCell/tableInsertRow) or pass a raw ` +
|
||||
`ProseMirror \`node\` instead of \`markdown\`.`,
|
||||
);
|
||||
}
|
||||
|
||||
// 0 -> "no node"; >1 -> "ambiguous, refused" (the transform skipped the write
|
||||
// for any count !== 1). Shared #159 guard, identical to the JSON path.
|
||||
assertUnambiguousMatch("patchNode", "replace", replaced, nodeId, pageId);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
replaced,
|
||||
nodeId,
|
||||
blocks: threaded.length,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert content relative to an anchor (or append it at the top level).
|
||||
* Operates on the LIVE collab document so comments and concurrent edits are
|
||||
* preserved.
|
||||
*
|
||||
* Exactly one of `input.markdown` / `input.node` (#413):
|
||||
* - `markdown` (RECOMMENDED): a canonical markdown fragment. It may import to
|
||||
* SEVERAL blocks — they are inserted IN ORDER at the anchor. `^[...]`
|
||||
* footnote definitions merge into the page's tail list (same machinery as
|
||||
* insertFootnote). Every inserted block gets a fresh id.
|
||||
* - `node`: a raw ProseMirror node for precise attr/mark work, or to insert
|
||||
* table structure (a bare tableRow/tableCell/tableHeader — NOT expressible in
|
||||
* markdown, so those stay JSON-only).
|
||||
*
|
||||
* opts.position:
|
||||
* - "append": push the content at the end of the top-level content.
|
||||
* - "before"/"after": insert as a sibling of the anchor, just before/after it.
|
||||
* Exactly one of anchorNodeId / anchorText must be given; anchorNodeId
|
||||
* locates a node anywhere by attrs.id, anchorText matches the first top-level
|
||||
* block whose plain text includes it.
|
||||
*
|
||||
* Throws if the anchor cannot be found.
|
||||
*/
|
||||
async insertNode(
|
||||
pageId: string,
|
||||
input: { markdown?: string; node?: any },
|
||||
opts: {
|
||||
position: "before" | "after" | "append";
|
||||
anchorNodeId?: string;
|
||||
anchorText?: string;
|
||||
},
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// XOR: exactly one of markdown / node (both optional in the schema).
|
||||
const hasMd =
|
||||
input != null &&
|
||||
typeof input.markdown === "string" &&
|
||||
input.markdown.trim() !== "";
|
||||
const hasNode = input != null && input.node != null;
|
||||
if (hasMd === hasNode) {
|
||||
throw new Error(
|
||||
"insertNode: provide exactly one of `markdown` (recommended, for prose) " +
|
||||
"or `node` (a raw ProseMirror node, for precise attr/mark work or table structure)",
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
!opts ||
|
||||
(opts.position !== "before" &&
|
||||
opts.position !== "after" &&
|
||||
opts.position !== "append")
|
||||
) {
|
||||
throw new Error(
|
||||
'insertNode: `position` must be one of "before", "after", "append"',
|
||||
);
|
||||
}
|
||||
if (opts.position === "before" || opts.position === "after") {
|
||||
// before/after require EXACTLY ONE anchor (an id or a text fragment).
|
||||
const hasId =
|
||||
typeof opts.anchorNodeId === "string" && opts.anchorNodeId.length > 0;
|
||||
const hasText =
|
||||
typeof opts.anchorText === "string" && opts.anchorText.length > 0;
|
||||
if (hasId === hasText) {
|
||||
throw new Error(
|
||||
`insertNode: position "${opts.position}" requires exactly one of anchorNodeId or anchorText`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve the ordered list of blocks to insert plus any footnote definitions
|
||||
// to merge. The markdown path imports canonically (so an inserted block is
|
||||
// byte-identical to the same content in a full-page import); the node path is
|
||||
// a single block with no footnote merge (raw JSON `^[...]` is not touched).
|
||||
let blocks: any[];
|
||||
let definitions: any[] = [];
|
||||
if (hasMd) {
|
||||
const frag = await importMarkdownFragment(input.markdown as string);
|
||||
blocks = frag.blocks;
|
||||
definitions = frag.definitions;
|
||||
} else {
|
||||
const node = input.node;
|
||||
if (!node || typeof node !== "object" || typeof node.type !== "string") {
|
||||
throw new Error(
|
||||
"insertNode: `node` must be an object with a string `type`",
|
||||
);
|
||||
}
|
||||
blocks = [node];
|
||||
}
|
||||
|
||||
// #409: fail fast on a malformed node SHAPE (a nested child with an
|
||||
// absent/unknown `type`) BEFORE opening a collab session or taking the page
|
||||
// lock — the root-only check above never sees nested children.
|
||||
for (const b of blocks) {
|
||||
this.assertValidNodeShape("insertNode", b);
|
||||
}
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Track insertion in an outer var, reset per-transform, so a collab retry
|
||||
// recomputes it cleanly (mirrors replaceImage's pattern).
|
||||
let inserted = false;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
inserted = false;
|
||||
// Re-mint any minted block id that collides with an existing page id
|
||||
// (all inserted blocks are fresh, no skip) so the splice stays unique.
|
||||
if (hasMd) reassignCollidingBlockIds(liveDoc, blocks);
|
||||
// Single-block node path keeps `insertNodeRelative` (it owns the
|
||||
// structural table-node splicing); the markdown path uses the array
|
||||
// splice so N blocks land in order at one anchor.
|
||||
const res = hasMd
|
||||
? insertNodesRelative(liveDoc, blocks, opts)
|
||||
: insertNodeRelative(liveDoc, blocks[0], opts);
|
||||
inserted = res.inserted;
|
||||
if (!inserted) return null; // anchor not found -> skip the write entirely
|
||||
// Merge any fragment footnote definitions into the page tail list and
|
||||
// re-derive canonical numbering (no-op when there are none).
|
||||
return mergeFootnoteDefinitions(res.doc, definitions);
|
||||
},
|
||||
);
|
||||
|
||||
if (!inserted) {
|
||||
const anchorDesc = opts.anchorNodeId
|
||||
? `anchorNodeId "${opts.anchorNodeId}"`
|
||||
: `anchorText "${opts.anchorText}"`;
|
||||
// anchorText is matched against the block's literal RENDERED plain text;
|
||||
// markdown/emoji are tolerated only as a strip-and-retry fallback, so a
|
||||
// miss usually means the text differs from what's on the page.
|
||||
const hint = opts.anchorText
|
||||
? " anchorText must be the block's literal rendered plain text (no markdown wrappers or emoji); anchorNodeId from getPageJson is more reliable."
|
||||
: "";
|
||||
throw new Error(
|
||||
`insertNode: anchor not found (${anchorDesc}) on page ${pageId}.${hint}`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
inserted: true,
|
||||
position: opts.position,
|
||||
blocks: blocks.length,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove EVERY node whose attrs.id === nodeId (recursively, including nodes
|
||||
* nested in callouts/tables) from its parent content array. Operates on the
|
||||
* LIVE collab document so comments and concurrent edits are preserved.
|
||||
* Throws if no node matches.
|
||||
*/
|
||||
async deleteNode(pageId: string, nodeId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Track the deletion count in an outer var, reset per-transform, so a
|
||||
// collab retry recomputes it cleanly (mirrors replaceImage's pattern).
|
||||
let deleted = 0;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
deleted = 0;
|
||||
const { doc: nd, deleted: d } = deleteNodeById(liveDoc, nodeId);
|
||||
deleted = d;
|
||||
// 0 matches -> skip the write. >1 matches -> the id is AMBIGUOUS (block
|
||||
// ids are duplicated on copy/paste, #159): deleting "the node with id X"
|
||||
// would silently remove EVERY duplicate. Refuse: skip the write and throw
|
||||
// below so the model re-targets. Only an unambiguous single match is
|
||||
// deleted.
|
||||
if (deleted !== 1) return null;
|
||||
return nd;
|
||||
},
|
||||
);
|
||||
|
||||
// 0 -> "no node"; >1 -> "ambiguous, refused" (the transform already skipped
|
||||
// the write for any count !== 1). Single shared guard (#159, #185 review).
|
||||
assertUnambiguousMatch("deleteNode", "delete", deleted, nodeId, pageId);
|
||||
|
||||
return { success: true, deleted, nodeId, verify: mutation.verify };
|
||||
}
|
||||
|
||||
/** Build the public share URL for a page. */
|
||||
}
|
||||
return NodesWriteMixin;
|
||||
}
|
||||
@@ -0,0 +1,655 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import FormData from "form-data";
|
||||
import axios, { AxiosInstance } from "axios";
|
||||
import { convertProseMirrorToMarkdown } from "../lib/markdown-converter.js";
|
||||
import {
|
||||
updatePageContentRealtime,
|
||||
replacePageContent,
|
||||
markdownToProseMirror,
|
||||
markdownToProseMirrorCanonical,
|
||||
mutatePageContent,
|
||||
assertYjsEncodable,
|
||||
MutationResult,
|
||||
} from "../lib/collaboration.js";
|
||||
import { footnoteWarningsField } from "../lib/footnote-analyze.js";
|
||||
import {
|
||||
serializeDocmostMarkdown,
|
||||
parseDocmostMarkdown,
|
||||
} from "../lib/markdown-document.js";
|
||||
import { diffDocs, summarizeChange } from "../lib/diff.js";
|
||||
import {
|
||||
blockText,
|
||||
walk,
|
||||
getList,
|
||||
insertMarkerAfter,
|
||||
setCalloutRange,
|
||||
noteItem,
|
||||
mdToInlineNodes,
|
||||
commentsToFootnotes,
|
||||
canonicalizeFootnotes,
|
||||
insertInlineFootnote,
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
import { normalizeAndMergeFootnotes } from "../lib/footnote-normalize-merge.js";
|
||||
import vm from "node:vm";
|
||||
|
||||
// Public method surface of PagesMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements IPagesMixin` fails to compile on drift.
|
||||
export interface IPagesMixin {
|
||||
createPage(title: string, content: string, spaceId: string, parentPageId?: string): any;
|
||||
updatePage(pageId: string, content: string, title?: string): any;
|
||||
renamePage(pageId: string, title: string): any;
|
||||
movePage(pageId: string, parentPageId: string | null, position?: string): any;
|
||||
deletePage(pageId: string): any;
|
||||
sharePage(pageId: string, searchIndexing?: boolean): any;
|
||||
listShares(): any;
|
||||
unsharePage(pageId: string): any;
|
||||
exportPageMarkdown(pageId: string): Promise<string>;
|
||||
importPageMarkdown(pageId: string, fullMarkdown: string): Promise<any>;
|
||||
copyPageContent(sourcePageId: string, targetPageId: string): any;
|
||||
listPageHistory(pageId: string, cursor?: string): any;
|
||||
getPageHistory(historyId: string): any;
|
||||
restorePageVersion(historyId: string): any;
|
||||
diffPageVersions(pageId: string, from?: string, to?: string): any;
|
||||
}
|
||||
|
||||
export function PagesMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & IPagesMixin> & TBase {
|
||||
abstract class PagesMixin extends Base implements IPagesMixin {
|
||||
/**
|
||||
* Create a new page with title and content.
|
||||
* Uses the /pages/import workaround (the only endpoint accepting content),
|
||||
* then moves the page and restores the exact title: the import endpoint
|
||||
* derives the title from the FILENAME and replaces spaces with
|
||||
* underscores, so we explicitly re-set it via /pages/update afterwards.
|
||||
*/
|
||||
async createPage(
|
||||
title: string,
|
||||
content: string,
|
||||
spaceId: string,
|
||||
parentPageId?: string,
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
if (parentPageId) {
|
||||
try {
|
||||
await this.getPage(parentPageId);
|
||||
} catch (e) {
|
||||
throw new Error(`Parent page with ID ${parentPageId} not found.`);
|
||||
}
|
||||
}
|
||||
|
||||
// 1. Create content via Import (using multipart/form-data).
|
||||
// Build a FRESH FormData per send attempt: a FormData body is a single-use
|
||||
// stream consumed on the first send, so it cannot be replayed by
|
||||
// this.client's response interceptor (replay fails with 'socket hang up').
|
||||
// Multipart re-auth is therefore done here with bare axios and an explicit
|
||||
// one-shot 401/403 retry that rebuilds the body.
|
||||
const fileContent = Buffer.from(content, "utf-8");
|
||||
const buildForm = () => {
|
||||
const form = new FormData();
|
||||
form.append("spaceId", spaceId);
|
||||
form.append("file", fileContent, {
|
||||
filename: `${title || "import"}.md`,
|
||||
contentType: "text/markdown",
|
||||
});
|
||||
return form;
|
||||
};
|
||||
|
||||
const importUrl = `${this.apiUrl}/pages/import`;
|
||||
let response;
|
||||
try {
|
||||
// Call buildForm() ONCE per attempt and reuse the instance for both
|
||||
// getHeaders() and the body so the Content-Type boundary matches the body.
|
||||
const form = buildForm();
|
||||
// Read the Authorization header from this.client's defaults (set by
|
||||
// login(), only ever deleted — never set to null) instead of building
|
||||
// `Bearer ${this.token}`: a concurrent JSON 401 can null this.token
|
||||
// mid-flight, which would otherwise produce a literal "Bearer null".
|
||||
// ensureAuthenticated() above guarantees login() ran, so the default
|
||||
// header exists here.
|
||||
response = await axios.post(importUrl, form, {
|
||||
headers: {
|
||||
...form.getHeaders(),
|
||||
Authorization: this.client.defaults.headers.common["Authorization"],
|
||||
},
|
||||
timeout: 60000,
|
||||
});
|
||||
} catch (error) {
|
||||
// On an expired-token auth error, re-login and retry exactly once with a
|
||||
// freshly-rebuilt FormData (the previous one was already consumed).
|
||||
if (
|
||||
axios.isAxiosError(error) &&
|
||||
(error.response?.status === 401 || error.response?.status === 403)
|
||||
) {
|
||||
await this.login();
|
||||
const form2 = buildForm();
|
||||
response = await axios.post(importUrl, form2, {
|
||||
headers: {
|
||||
...form2.getHeaders(),
|
||||
Authorization: this.client.defaults.headers.common["Authorization"],
|
||||
},
|
||||
timeout: 60000,
|
||||
});
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
const newPageId = (response.data?.data ?? response.data).id;
|
||||
|
||||
// 2. Move to parent if needed
|
||||
if (parentPageId) {
|
||||
await this.movePage(newPageId, parentPageId);
|
||||
}
|
||||
|
||||
// 3. Restore the exact title (import mangles spaces into underscores)
|
||||
if (title) {
|
||||
await this.client.post("/pages/update", { pageId: newPageId, title });
|
||||
}
|
||||
|
||||
const page = await this.getPage(newPageId);
|
||||
// Surface non-fatal footnote problems (dangling refs, empty/duplicate
|
||||
// definitions, markers in tables) so the agent can fix its markup (#166).
|
||||
return { ...page, ...footnoteWarningsField(content) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Update a page's content from markdown and optionally its title.
|
||||
* NOTE: full re-import — block ids regenerate. For surgical changes
|
||||
* use editPageText / updatePageJson instead.
|
||||
*/
|
||||
async updatePage(pageId: string, content: string, title?: string) {
|
||||
await this.ensureAuthenticated();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260). The
|
||||
// REST /pages/update title write below keeps the agent-supplied id (the
|
||||
// server resolves a slugId there).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Write the BODY first, then the title (#159 split-brain). If the collab
|
||||
// body write fails (e.g. a persist timeout), the title must be left
|
||||
// UNTOUCHED so the page never ends up with a new title over its old body.
|
||||
// A title write failing AFTER a successful body is rarer (REST is fast) and
|
||||
// leaves correct content under a stale title — the lesser inconsistency.
|
||||
let collabToken = "";
|
||||
let mutation;
|
||||
try {
|
||||
collabToken = await this.getCollabTokenWithReauth();
|
||||
mutation = await updatePageContentRealtime(
|
||||
pageUuid,
|
||||
content,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
);
|
||||
} catch (error: any) {
|
||||
// Verbose diagnostics (incl. anything that could expose a token prefix)
|
||||
// are gated behind DEBUG; the thrown Error below carries no token data.
|
||||
if (process.env.DEBUG) {
|
||||
console.error(
|
||||
"Failed to update page content via realtime collaboration:",
|
||||
error,
|
||||
);
|
||||
const tokenPreview = collabToken
|
||||
? collabToken.substring(0, 15) + "..."
|
||||
: "null";
|
||||
console.error(`Collab token preview: ${tokenPreview}`);
|
||||
}
|
||||
throw new Error(`Failed to update page content: ${error.message}`);
|
||||
}
|
||||
|
||||
// Body persisted successfully — now it is safe to set the title.
|
||||
if (title) {
|
||||
await this.client.post("/pages/update", { pageId, title });
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
modified: true,
|
||||
message: "Page updated successfully.",
|
||||
pageId: pageId,
|
||||
verify: mutation.verify,
|
||||
// Non-fatal footnote diagnostics (#166); omitted when there are none.
|
||||
...footnoteWarningsField(content),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a URL string against a scheme allowlist for a given context.
|
||||
*
|
||||
* The markdown link path enforces safe schemes via TipTap, but the raw
|
||||
* JSON path (updatePageJson) bypasses that — so this is the sanitization
|
||||
* choke point for ProseMirror JSON written directly by the caller.
|
||||
*
|
||||
* - "link": reject javascript:, vbscript:, data: (any scheme that can
|
||||
* execute or smuggle script when the href is clicked).
|
||||
* - "src": allow only http(s):, mailto:, /api/files paths, or a
|
||||
* scheme-less relative/absolute path; reject
|
||||
* javascript:/vbscript:/data:/file:.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Rename a page (change its title only) without touching or resending its
|
||||
* content. The slug is derived from the page record, not the body, so it is
|
||||
* left intact too.
|
||||
*/
|
||||
async renamePage(pageId: string, title: string) {
|
||||
await this.ensureAuthenticated();
|
||||
await this.client.post("/pages/update", { pageId, title });
|
||||
return { success: true, pageId, title };
|
||||
}
|
||||
|
||||
/**
|
||||
* Copy the WHOLE content of one page onto another, entirely server-side: the
|
||||
* source's ProseMirror document is read and written verbatim onto the target
|
||||
* via the live collab path, so the document never passes through the model.
|
||||
*
|
||||
* Only the target's BODY is replaced — its title and slug live on the page
|
||||
* record (not in the content), so they are untouched. The source page is not
|
||||
* modified at all.
|
||||
*/
|
||||
|
||||
async movePage(
|
||||
pageId: string,
|
||||
parentPageId: string | null,
|
||||
position?: string,
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
// Docmost requires position >= 5 chars.
|
||||
const validPosition = position || "a00000";
|
||||
|
||||
return this.client
|
||||
.post("/pages/move", {
|
||||
pageId,
|
||||
parentPageId,
|
||||
position: validPosition,
|
||||
})
|
||||
.then((res) => res.data);
|
||||
}
|
||||
|
||||
|
||||
async deletePage(pageId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
return this.client
|
||||
.post("/pages/delete", { pageId })
|
||||
.then((res) => res.data);
|
||||
}
|
||||
|
||||
// --- Comment methods (ported from upstream PR #3 by Max Nikitin) ---
|
||||
|
||||
/**
|
||||
* Normalize a comment's `content` into a ProseMirror doc object before
|
||||
* markdown conversion. createComment/updateComment send content as a
|
||||
* JSON.stringify(...) STRING, and the server stores it as-is, so on read it
|
||||
* comes back as a string. convertProseMirrorToMarkdown returns "" for a
|
||||
* string, so parse it first (guarded — fall back to the raw value on any
|
||||
* parse failure so a non-JSON legacy value is still handled gracefully).
|
||||
*/
|
||||
|
||||
/** Share a page publicly (idempotent) and return the public URL. */
|
||||
async sharePage(pageId: string, searchIndexing: boolean = true) {
|
||||
await this.ensureAuthenticated();
|
||||
const response = await this.client.post("/shares/create", {
|
||||
pageId,
|
||||
includeSubPages: false,
|
||||
searchIndexing,
|
||||
});
|
||||
const share = response.data?.data ?? response.data;
|
||||
const slugId = share.page?.slugId || (await this.getPageRaw(pageId)).slugId;
|
||||
return {
|
||||
shareId: share.id,
|
||||
key: share.key,
|
||||
pageId: share.pageId,
|
||||
publicUrl: this.shareUrl(share.key, slugId),
|
||||
searchIndexing: share.searchIndexing,
|
||||
};
|
||||
}
|
||||
|
||||
/** List all public shares in the workspace with their URLs. */
|
||||
|
||||
/** Build the public share URL for a page. */
|
||||
protected shareUrl(shareKey: string, slugId: string): string {
|
||||
return `${this.appUrl}/share/${shareKey}/p/${slugId}`;
|
||||
}
|
||||
|
||||
/** Share a page publicly (idempotent) and return the public URL. */
|
||||
|
||||
/** List all public shares in the workspace with their URLs. */
|
||||
async listShares() {
|
||||
const shares = await this.paginateAll("/shares", {});
|
||||
return shares.map((s: any) => ({
|
||||
shareId: s.id,
|
||||
key: s.key,
|
||||
pageId: s.pageId,
|
||||
pageTitle: s.page?.title,
|
||||
publicUrl: s.page?.slugId ? this.shareUrl(s.key, s.page.slugId) : null,
|
||||
searchIndexing: s.searchIndexing,
|
||||
createdAt: s.createdAt,
|
||||
}));
|
||||
}
|
||||
|
||||
/** Remove the public share of a page. */
|
||||
async unsharePage(pageId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
const shares = await this.listShares();
|
||||
const share = shares.find((s: any) => s.pageId === pageId);
|
||||
if (!share) {
|
||||
throw new Error(`Page ${pageId} is not shared.`);
|
||||
}
|
||||
await this.client.post("/shares/delete", { shareId: share.shareId });
|
||||
return { success: true, removedShareId: share.shareId, pageId };
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Export a page to a single self-contained Docmost-flavoured markdown file:
|
||||
* meta block + body (with inline comment anchors + diagrams) + comment
|
||||
* threads. Lossless round-trip target; see importPageMarkdown for the inverse.
|
||||
*/
|
||||
async exportPageMarkdown(pageId: string): Promise<string> {
|
||||
await this.ensureAuthenticated();
|
||||
const page = await this.getPageRaw(pageId);
|
||||
const body = page.content ? convertProseMirrorToMarkdown(page.content) : "";
|
||||
let comments: any[] = [];
|
||||
try {
|
||||
// Lossless export: include RESOLVED threads so the export -> import
|
||||
// round-trip preserves every comment. This is exactly why the active-only
|
||||
// filter is an opt-in (default false) on listComments.
|
||||
comments = (await this.listComments(pageId, true)).items;
|
||||
} catch (e) {
|
||||
// A comments fetch failure must not lose the body; export with [] and let
|
||||
// the caller see the (empty) comments block. Log under DEBUG only.
|
||||
if (process.env.DEBUG) console.error("export: listComments failed", e);
|
||||
}
|
||||
const meta = {
|
||||
version: 1,
|
||||
pageId: page.id,
|
||||
slugId: page.slugId,
|
||||
title: page.title,
|
||||
spaceId: page.spaceId,
|
||||
parentPageId: page.parentPageId ?? null,
|
||||
};
|
||||
return serializeDocmostMarkdown(meta, body, comments);
|
||||
}
|
||||
|
||||
/**
|
||||
* Import a self-contained Docmost markdown file back into a page. Parses out
|
||||
* the meta + comments metadata blocks, converts the body to ProseMirror
|
||||
* (restoring comment marks + diagrams from their inline HTML), and replaces
|
||||
* the page content. Comment THREAD records are NOT written to the server in
|
||||
* this version — they are preserved in the file and the inline marks are
|
||||
* re-applied so the highlights survive; managing comment records stays with
|
||||
* the comment tools/UI.
|
||||
*/
|
||||
async importPageMarkdown(pageId: string, fullMarkdown: string): Promise<any> {
|
||||
await this.ensureAuthenticated();
|
||||
const { meta, body, comments } = parseDocmostMarkdown(fullMarkdown);
|
||||
// PAGE import: canonicalize footnotes (see markdownToProseMirrorCanonical).
|
||||
const doc = await markdownToProseMirrorCanonical(body);
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
const mutation = await replacePageContent(
|
||||
pageUuid,
|
||||
doc,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
);
|
||||
// Collect distinct comment ids that actually became comment marks in the doc.
|
||||
const collectCommentIds = (node: any, acc: Set<string>): Set<string> => {
|
||||
if (!node || typeof node !== "object") return acc;
|
||||
if (Array.isArray(node.marks)) {
|
||||
for (const mk of node.marks) {
|
||||
if (mk && mk.type === "comment" && mk.attrs?.commentId) {
|
||||
acc.add(mk.attrs.commentId);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (Array.isArray(node.content)) {
|
||||
for (const child of node.content) collectCommentIds(child, acc);
|
||||
}
|
||||
return acc;
|
||||
};
|
||||
// Count reflects the comment marks present in the written document, so an id
|
||||
// that only appears as inert text (e.g. inside a fenced code block) is not
|
||||
// counted because it never becomes a comment mark.
|
||||
const anchoredIds = collectCommentIds(doc, new Set<string>());
|
||||
const result: any = {
|
||||
success: true,
|
||||
pageId,
|
||||
anchoredCommentCount: anchoredIds.size,
|
||||
commentsInFile: Array.isArray(comments) ? comments.length : 0,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
// Warn (non-fatal) if the file was exported from a DIFFERENT page.
|
||||
if (meta?.pageId && meta.pageId !== pageId) {
|
||||
result.warning = `File was exported from page ${meta.pageId} but is being imported into ${pageId}.`;
|
||||
}
|
||||
// Non-fatal footnote diagnostics (#166), analyzed on the BODY (the part after
|
||||
// the docmost:meta / docmost:comments blocks) — so a `[^x]`-like token inside
|
||||
// those JSON blocks never produces a false warning, while real markers in the
|
||||
// body do. `body` comes from parseDocmostMarkdown(fullMarkdown) above.
|
||||
Object.assign(result, footnoteWarningsField(body));
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Rename a page (change its title only) without touching or resending its
|
||||
* content. The slug is derived from the page record, not the body, so it is
|
||||
* left intact too.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Copy the WHOLE content of one page onto another, entirely server-side: the
|
||||
* source's ProseMirror document is read and written verbatim onto the target
|
||||
* via the live collab path, so the document never passes through the model.
|
||||
*
|
||||
* Only the target's BODY is replaced — its title and slug live on the page
|
||||
* record (not in the content), so they are untouched. The source page is not
|
||||
* modified at all.
|
||||
*/
|
||||
async copyPageContent(sourcePageId: string, targetPageId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// A self-copy would be a no-op overwrite; reject it explicitly so a caller
|
||||
// mistake surfaces as a clear error rather than a silent round-trip.
|
||||
if (sourcePageId === targetPageId) {
|
||||
throw new Error(
|
||||
"copyPageContent: sourcePageId and targetPageId are the same page (no-op copy)",
|
||||
);
|
||||
}
|
||||
|
||||
const source = await this.getPageRaw(sourcePageId);
|
||||
const content = source?.content;
|
||||
if (
|
||||
!content ||
|
||||
typeof content !== "object" ||
|
||||
content.type !== "doc" ||
|
||||
!Array.isArray(content.content)
|
||||
) {
|
||||
throw new Error(
|
||||
`copyPageContent: source page ${sourcePageId} has no usable ProseMirror content to copy`,
|
||||
);
|
||||
}
|
||||
|
||||
// Defense-in-depth: run the same URL-scheme sanitizer the JSON write path
|
||||
// uses, so copying never lands a javascript:/data: href/src on the target
|
||||
// (parity with updatePageJson; harmless for already-stored source content).
|
||||
this.validateDocUrls(content);
|
||||
|
||||
// Defense-in-depth (#228): this is a FULL-document write, so canonicalize
|
||||
// footnotes before copying — a no-op on already-canonical source content, but
|
||||
// it guarantees a copy can never propagate a non-canonical footnote topology
|
||||
// to the target (parity with the other full-doc write paths).
|
||||
// #419: normalize + merge glyph-forked definitions before canonicalizing.
|
||||
const canonical = canonicalizeFootnotes(normalizeAndMergeFootnotes(content));
|
||||
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the TARGET collab doc by its canonical UUID, never the slugId (#260).
|
||||
const targetUuid = await this.resolvePageId(targetPageId);
|
||||
const mutation = await this.replacePage(
|
||||
targetUuid,
|
||||
canonical,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
);
|
||||
|
||||
return {
|
||||
success: true,
|
||||
sourcePageId,
|
||||
targetPageId,
|
||||
copiedNodes: canonical.content.length,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Surgical text edits: find/replace inside text nodes of the live
|
||||
* document. Preserves all block ids, marks, callouts and tables.
|
||||
*/
|
||||
|
||||
// --- Page history / diff / transform ---
|
||||
|
||||
/**
|
||||
* List the saved versions (history snapshots) of a page, newest first.
|
||||
* Docmost auto-snapshots on every save. Returns one cursor-paginated page of
|
||||
* results: `{ items, nextCursor }`. The history record's id field is `id`.
|
||||
*/
|
||||
async listPageHistory(pageId: string, cursor?: string) {
|
||||
await this.ensureAuthenticated();
|
||||
const payload: Record<string, any> = { pageId };
|
||||
if (cursor) payload.cursor = cursor;
|
||||
const response = await this.client.post("/pages/history", payload);
|
||||
const data = response.data?.data ?? response.data;
|
||||
return {
|
||||
items: data?.items ?? [],
|
||||
nextCursor: data?.meta?.nextCursor ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a single page-history version including its lossless ProseMirror
|
||||
* `content`. The version also carries pageId/title/createdAt.
|
||||
*/
|
||||
async getPageHistory(historyId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
const response = await this.client.post("/pages/history/info", {
|
||||
historyId,
|
||||
});
|
||||
return response.data?.data ?? response.data;
|
||||
}
|
||||
|
||||
/**
|
||||
* "Restore" a version: Docmost has NO restore endpoint, so we take the
|
||||
* version's `content` and write it as the page's current content via the live
|
||||
* collab path (which itself creates a new history snapshot). Returns the
|
||||
* affected pageId and the source historyId.
|
||||
*/
|
||||
async restorePageVersion(historyId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
const version = await this.getPageHistory(historyId);
|
||||
if (
|
||||
!version ||
|
||||
!version.pageId ||
|
||||
!version.content ||
|
||||
typeof version.content !== "object"
|
||||
) {
|
||||
throw new Error(
|
||||
`restorePageVersion: history ${historyId} has no usable content`,
|
||||
);
|
||||
}
|
||||
// Defense-in-depth: sanitize URLs in the restored content (parity with the
|
||||
// JSON write path) before writing it back.
|
||||
this.validateDocUrls(version.content);
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// version.pageId is the page entity id (already a UUID); resolvePageId
|
||||
// short-circuits a UUID with no round-trip, so this is defensive only (#260).
|
||||
const pageUuid = await this.resolvePageId(version.pageId);
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
() => version.content,
|
||||
);
|
||||
return {
|
||||
pageId: version.pageId,
|
||||
restoredFrom: historyId,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Diff two versions of a page and return a Docmost-equivalent change set.
|
||||
* `from`/`to` each resolve to a ProseMirror doc:
|
||||
* - null / undefined / "current" -> the page's CURRENT content;
|
||||
* - any other string -> that historyId's content.
|
||||
* Returns the diff plus the resolved version metadata for each side.
|
||||
*/
|
||||
async diffPageVersions(pageId: string, from?: string, to?: string) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
const isCurrent = (v?: string) => v == null || v === "" || v === "current";
|
||||
|
||||
const resolveSide = async (
|
||||
v?: string,
|
||||
): Promise<{ doc: any; meta: any }> => {
|
||||
if (isCurrent(v)) {
|
||||
const raw = await this.getPageRaw(pageId);
|
||||
return {
|
||||
doc: raw.content || { type: "doc", content: [] },
|
||||
meta: {
|
||||
kind: "current",
|
||||
pageId,
|
||||
title: raw.title,
|
||||
updatedAt: raw.updatedAt,
|
||||
},
|
||||
};
|
||||
}
|
||||
const version = await this.getPageHistory(v as string);
|
||||
return {
|
||||
doc: version.content || { type: "doc", content: [] },
|
||||
meta: {
|
||||
kind: "history",
|
||||
historyId: version.id,
|
||||
pageId: version.pageId,
|
||||
title: version.title,
|
||||
createdAt: version.createdAt,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const fromSide = await resolveSide(from);
|
||||
const toSide = await resolveSide(to);
|
||||
const diff = diffDocs(fromSide.doc, toSide.doc);
|
||||
return { from: fromSide.meta, to: toSide.meta, diff };
|
||||
}
|
||||
|
||||
/**
|
||||
* Edit a page by running an arbitrary user-supplied JS transform against the
|
||||
* live document, with a diff preview + page-history safety net.
|
||||
*
|
||||
* The transform string is evaluated as `(doc, ctx) => doc` inside a node:vm
|
||||
* sandbox: it gets ONLY `{ doc, ctx, structuredClone, console }` as globals,
|
||||
* a 5s timeout, and NO access to require/process/fs/network. It must return a
|
||||
* `{ type: "doc" }` node, which is validated structurally before any write.
|
||||
*
|
||||
* `ctx` exposes:
|
||||
* - comments: the page's comments (fetched before the live read);
|
||||
* - log: an array the transform can push diagnostics to (via console.log);
|
||||
* - consume(id): mark a comment id as consumed (for deleteComments);
|
||||
* - helpers: the transforms.ts primitives + commentsToFootnotes.
|
||||
*
|
||||
* Footnote convention used by the helpers: footnote markers are plain "[N]"
|
||||
* text in the body, and the notes are an orderedList under a heading whose
|
||||
* text is "Примечания переводчика".
|
||||
*
|
||||
* dryRun (default true): read the page's current content, run the transform,
|
||||
* and return `{ pushed:false, diff, log }` WITHOUT opening the collab socket.
|
||||
* Otherwise the transform runs atomically inside mutatePageContent, optionally
|
||||
* deletes consumed comments, and returns the new historyId + diff + log.
|
||||
*/
|
||||
}
|
||||
return PagesMixin;
|
||||
}
|
||||
@@ -0,0 +1,718 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import axios, { AxiosInstance } from "axios";
|
||||
import {
|
||||
filterWorkspace,
|
||||
filterSpace,
|
||||
filterPage,
|
||||
filterComment,
|
||||
filterSearchResult,
|
||||
} from "../lib/filters.js";
|
||||
import {
|
||||
convertProseMirrorToMarkdown,
|
||||
type ConvertProseMirrorToMarkdownOptions,
|
||||
} from "../lib/markdown-converter.js";
|
||||
import {
|
||||
GetPageConversionCache,
|
||||
hashConvertOptions,
|
||||
} from "./getpage-cache.js";
|
||||
import {
|
||||
collectInternalFileNodes,
|
||||
normalizeFileUrl,
|
||||
resolveInternalFilePath,
|
||||
} from "../lib/internal-file-urls.js";
|
||||
import { buildPageTree } from "../lib/tree.js";
|
||||
import {
|
||||
replaceNodeById,
|
||||
replaceNodeByIdWithMany,
|
||||
reassignCollidingBlockIds,
|
||||
deleteNodeById,
|
||||
assertUnambiguousMatch,
|
||||
insertNodeRelative,
|
||||
insertNodesRelative,
|
||||
blockPlainText,
|
||||
buildOutline,
|
||||
getNodeByRef,
|
||||
readTable,
|
||||
insertTableRow,
|
||||
deleteTableRow,
|
||||
updateTableCell,
|
||||
findInvalidNode,
|
||||
} from "@docmost/prosemirror-markdown";
|
||||
import {
|
||||
importMarkdownFragment,
|
||||
canBeDocChild,
|
||||
findUnrepresentableTableAttrs,
|
||||
} from "../lib/markdown-fragment.js";
|
||||
import { searchInDoc, SearchOptions } from "../lib/page-search.js";
|
||||
import {
|
||||
blockText,
|
||||
walk,
|
||||
getList,
|
||||
insertMarkerAfter,
|
||||
setCalloutRange,
|
||||
noteItem,
|
||||
mdToInlineNodes,
|
||||
commentsToFootnotes,
|
||||
canonicalizeFootnotes,
|
||||
insertInlineFootnote,
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
|
||||
// Public method surface of ReadMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements IReadMixin` fails to compile on drift.
|
||||
export interface IReadMixin {
|
||||
getWorkspace(): any;
|
||||
getSpaces(): any;
|
||||
listPages(spaceId?: string, limit?: number, tree?: boolean): any;
|
||||
getTree(spaceId: string, rootPageId?: string, maxDepth?: number): any;
|
||||
getPageContext(pageId: string): any;
|
||||
listSidebarPages(spaceId: string, pageId?: string): any;
|
||||
getPage(pageId: string): any;
|
||||
getPageJson(pageId: string): any;
|
||||
getOutline(pageId: string): any;
|
||||
getNode(pageId: string, nodeId: string, format?: "markdown" | "json"): any;
|
||||
searchInPage(pageId: string, query: string, opts?: SearchOptions): any;
|
||||
getTable(pageId: string, tableRef: string): any;
|
||||
search(query: string, spaceId?: string, limit?: number, opts?: { parentPageId?: string; titleOnly?: boolean }): any;
|
||||
}
|
||||
|
||||
export function ReadMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & IReadMixin> & TBase {
|
||||
abstract class ReadMixin extends Base implements IReadMixin {
|
||||
async getWorkspace() {
|
||||
await this.ensureAuthenticated();
|
||||
const response = await this.client.post("/workspace/info", {});
|
||||
return {
|
||||
data: filterWorkspace(response.data?.data ?? response.data),
|
||||
success: response.data.success,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
async getSpaces() {
|
||||
const spaces = await this.paginateAll("/spaces", {});
|
||||
return spaces.map((space) => filterSpace(space));
|
||||
}
|
||||
|
||||
/**
|
||||
* List pages in one of two modes.
|
||||
*
|
||||
* Default (`tree` false): most recent pages by updatedAt (descending),
|
||||
* bounded. Fetching the whole space can exceed MCP response/time limits on
|
||||
* large instances, so a single bounded page of results is returned (default
|
||||
* 50, max 100) via the `/pages/recent` feed.
|
||||
*
|
||||
* Tree (`tree` true): DEPRECATED — prefer `getTree`, which shares this exact
|
||||
* code path (a single `/pages/tree` request via `enumerateSpacePages` +
|
||||
* `buildPageTree`) but returns the compact `{pageId, title, children?,
|
||||
* hasChildren?}` shape and supports `rootPageId`/`maxDepth`. This tree mode is
|
||||
* kept for backward compatibility; it REQUIRES `spaceId` (a page tree is
|
||||
* scoped to one space) and IGNORES `limit` — the whole hierarchy is returned.
|
||||
* It fetches the tree via `enumerateSpacePages`, which on the fork server
|
||||
* resolves to a single `/pages/tree` request returning the whole
|
||||
* permission-filtered flat page set (soft-deleted pages excluded
|
||||
* server-side); the cursor-BFS in `enumerateSpacePages` is only a fallback for
|
||||
* stock upstream servers that lack `/pages/tree`.
|
||||
*/
|
||||
async listPages(spaceId?: string, limit: number = 50, tree: boolean = false) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
if (tree) {
|
||||
if (!spaceId) {
|
||||
throw new Error(
|
||||
"listPages: tree mode requires a spaceId (a page tree is scoped to one space). Pass spaceId, or omit tree to get the recent-pages list.",
|
||||
);
|
||||
}
|
||||
// #486: propagate `truncated` (same pattern as check_new_comments). The old
|
||||
// code dropped it, so a caller handed an INCOMPLETE tree (the stdio-fallback
|
||||
// BFS hit its node cap) had no way to know pages were missing. Return the
|
||||
// tree alongside the flag; the primary /pages/tree path is uncapped so this
|
||||
// is false there.
|
||||
const { pages, truncated } = await this.enumerateSpacePages(spaceId);
|
||||
return { tree: buildPageTree(pages), truncated };
|
||||
}
|
||||
|
||||
const clampedLimit = Math.max(1, Math.min(100, limit));
|
||||
const payload: Record<string, any> = { limit: clampedLimit, page: 1 };
|
||||
if (spaceId) payload.spaceId = spaceId;
|
||||
const response = await this.client.post("/pages/recent", payload);
|
||||
const data = response.data;
|
||||
const items = data.data?.items || data.items || [];
|
||||
return items.map((page: any) => filterPage(page));
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a space's page hierarchy (or one subtree) as a nested tree in a SINGLE
|
||||
* request — the #443 `getTree` tool. Shares its whole code path with
|
||||
* `listPages(tree:true)`: `enumerateSpacePages` issues one `POST /pages/tree`
|
||||
* (with the cursor-BFS only as a fallback for stock upstream servers that lack
|
||||
* the endpoint), then `buildPageTree` nests the flat, permission-filtered,
|
||||
* position-ordered list. No second tree fetch, no per-node BFS.
|
||||
*
|
||||
* - `rootPageId` — restrict to that page's subtree; the server seeds the CTE
|
||||
* with the page itself, so the result is exactly ONE root (the page and its
|
||||
* descendants). Omit it for the whole space.
|
||||
* - `maxDepth` — trim the response to that many levels (roots = depth 1) to
|
||||
* save tokens; the server still returns everything in one request, the cut
|
||||
* is applied in `buildPageTree` AFTER the full tree is built. A node whose
|
||||
* children were cut carries `hasChildren: true` (source of truth = the flat
|
||||
* item's server `hasChildren`) so the caller can descend with a follow-up
|
||||
* `getTree(spaceId, rootPageId=that node)` call.
|
||||
*
|
||||
* Output nodes are `{pageId, title, children?, hasChildren?}` — only the UUID
|
||||
* `pageId` is exposed (never `slugId`/`icon`/`position`). Requires `spaceId`
|
||||
* (a page tree is scoped to one space).
|
||||
*/
|
||||
async getTree(spaceId: string, rootPageId?: string, maxDepth?: number) {
|
||||
await this.ensureAuthenticated();
|
||||
if (!spaceId) {
|
||||
throw new Error(
|
||||
"getTree: spaceId is required (a page tree is scoped to one space).",
|
||||
);
|
||||
}
|
||||
const { pages } = await this.enumerateSpacePages(spaceId, rootPageId);
|
||||
return buildPageTree(pages, { shape: "getTree", maxDepth });
|
||||
}
|
||||
|
||||
/**
|
||||
* "Where am I / what's around" for a single page — the #443 `getPageContext`
|
||||
* tool. Metadata only (no page content), using exactly TWO server requests:
|
||||
*
|
||||
* 1. `POST /pages/breadcrumbs` — a recursive CTE that walks UP from the page.
|
||||
* The server returns the chain root->page order (it `.reverse()`s the
|
||||
* child-first walk before responding), INCLUDING the page itself as the
|
||||
* LAST element. So the last element is the page and everything before it
|
||||
* is the ancestor chain root->parent. This carries the page's own title
|
||||
* and spaceId, so no extra page-info fetch is needed for a UUID input.
|
||||
* 2. `listSidebarPages(spaceId, pageId)` — the page's DIRECT children,
|
||||
* cursor-paginated (a page with >20 children returns ALL of them, no
|
||||
* dupes) and in sidebar `position` order, each carrying `hasChildren`.
|
||||
*
|
||||
* The input may be a slugId (agents copy them from URLs); it is run through
|
||||
* `resolvePageId` first, exactly like the other page tools. A UUID input adds
|
||||
* no request there (short-circuit), keeping the total at two; a slugId input
|
||||
* adds one unavoidable resolve round-trip.
|
||||
*
|
||||
* INVARIANT: only the UUID `pageId` is exposed anywhere — server `id` is
|
||||
* mapped to `pageId` and `slugId` is never leaked. A nonexistent/inaccessible
|
||||
* pageId makes the server 404/403, which propagates as a clear tool error
|
||||
* (never a hollow empty object).
|
||||
*/
|
||||
async getPageContext(pageId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// Resolve a possibly-slugId input to the canonical UUID (no round-trip for a
|
||||
// UUID). Errors here (bad/inaccessible id) propagate as a clear tool error.
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Request 1: the ancestor chain, root->page, page included as the LAST item.
|
||||
const response = await this.client.post("/pages/breadcrumbs", {
|
||||
pageId: pageUuid,
|
||||
});
|
||||
const chain: any[] = (response.data?.data ?? response.data) ?? [];
|
||||
if (!Array.isArray(chain) || chain.length === 0) {
|
||||
// The endpoint always includes the page itself, so an empty chain means
|
||||
// the page is gone/inaccessible — surface a clear error, not {}.
|
||||
throw new Error(`getPageContext: page "${pageId}" not found or inaccessible`);
|
||||
}
|
||||
|
||||
// Split: the last element is the page, the rest (root->parent) are the
|
||||
// breadcrumbs. A root page has no ancestors -> breadcrumbs is [].
|
||||
const self = chain[chain.length - 1];
|
||||
const ancestors = chain.slice(0, -1);
|
||||
|
||||
const page = {
|
||||
pageId: self.id,
|
||||
title: self.title,
|
||||
spaceId: self.spaceId,
|
||||
};
|
||||
const breadcrumbs = ancestors.map((n: any) => ({
|
||||
pageId: n.id,
|
||||
title: n.title,
|
||||
}));
|
||||
|
||||
// Request 2: direct children in sidebar order, each with hasChildren.
|
||||
const childItems = await this.listSidebarPages(self.spaceId, pageUuid);
|
||||
const children = childItems.map((c: any) => ({
|
||||
pageId: c.id,
|
||||
title: c.title,
|
||||
hasChildren: Boolean(c.hasChildren),
|
||||
}));
|
||||
|
||||
return { page, breadcrumbs, children };
|
||||
}
|
||||
|
||||
/**
|
||||
* List sidebar pages for a space. With no pageId the request returns the
|
||||
* space ROOT pages; with a pageId it returns the direct CHILDREN of that
|
||||
* page. pageId is therefore optional and is only included in the POST body
|
||||
* when provided (an empty/undefined pageId would otherwise change the
|
||||
* semantics on the server).
|
||||
*/
|
||||
async listSidebarPages(spaceId: string, pageId?: string) {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// Paginate via the server-issued cursor. The server switched from OFFSET
|
||||
// (`page`) to CURSOR (`cursor`/`nextCursor`) pagination, and the global
|
||||
// ValidationPipe(whitelist:true) SILENTLY STRIPS the obsolete `page` field
|
||||
// — so the old offset loop got the SAME first page every time (with
|
||||
// hasNextPage stuck true) and dropped every child beyond the first page.
|
||||
const MAX_PAGES = 50;
|
||||
let cursor: string | undefined;
|
||||
let allItems: any[] = [];
|
||||
let truncated = false;
|
||||
|
||||
for (let i = 0; i < MAX_PAGES; i++) {
|
||||
// limit: 100 is the server-side Max; cuts request count 5x vs the default 20.
|
||||
const payload: Record<string, any> = { spaceId, limit: 100 };
|
||||
// Only send pageId when scoping to a page's children; omit it for roots.
|
||||
if (pageId) payload.pageId = pageId;
|
||||
if (cursor) payload.cursor = cursor;
|
||||
|
||||
const data = (await this.client.post("/pages/sidebar-pages", payload)).data
|
||||
?.data;
|
||||
allItems = allItems.concat(data?.items ?? []);
|
||||
|
||||
// Advance strictly via the server-issued cursor; a missing/repeated cursor
|
||||
// means the protocol drifted again — stop instead of looping on page one.
|
||||
const next = data?.meta?.hasNextPage ? data?.meta?.nextCursor : null;
|
||||
if (!next || next === cursor) break;
|
||||
cursor = next;
|
||||
|
||||
// Reaching the ceiling with more pages still available means the child
|
||||
// list is truncated (mirrors paginateAll).
|
||||
if (i === MAX_PAGES - 1) truncated = true;
|
||||
}
|
||||
|
||||
// Warn on real truncation (ceiling hit while the server still had pages) so
|
||||
// the caller is not silently handed an incomplete child list.
|
||||
if (truncated) {
|
||||
console.warn(
|
||||
`listSidebarPages: children of "${pageId ?? spaceId}" truncated at the ${MAX_PAGES}-page cap; more pages exist on the server`,
|
||||
);
|
||||
}
|
||||
|
||||
return allItems;
|
||||
}
|
||||
|
||||
/**
|
||||
* Enumerate EVERY page in a space (or in a subtree, when rootPageId is given).
|
||||
*
|
||||
* Primary path (fork server): a SINGLE `POST /pages/tree` returns the whole
|
||||
* space (or a subtree) as a flat, permission-filtered list in one request, in
|
||||
* the exact node shape buildPageTree consumes. This replaces the old
|
||||
* per-node BFS, which issued N sidebar requests and — after the server moved
|
||||
* to cursor pagination — silently lost every child past the first sidebar
|
||||
* page (the obsolete `page` param was stripped by ValidationPipe).
|
||||
*
|
||||
* The subtree variant (rootPageId given) INCLUDES the root node itself
|
||||
* (getPageAndDescendants seeds with id = rootPageId), unlike the old BFS
|
||||
* which started from the root's children.
|
||||
*
|
||||
* Fallback path (stdio mode may target STOCK upstream Docmost, which lacks
|
||||
* `/pages/tree`): on a 404/405 it falls back to the cursor-based BFS below,
|
||||
* walking direct children via the fixed cursor listSidebarPages. Safeguards:
|
||||
* a `visited` Set of page ids prevents re-processing a node (cycles /
|
||||
* duplicate references), and a hard node cap bounds pathological trees so the
|
||||
* walk always terminates.
|
||||
*
|
||||
* Returns `{ pages, truncated }`. `truncated` is true ONLY when the fallback
|
||||
* BFS stopped at its MAX_NODES cap — the primary /pages/tree path is uncapped
|
||||
* and always returns the complete set, so it never reports truncation.
|
||||
*/
|
||||
protected async enumerateSpacePages(
|
||||
spaceId: string,
|
||||
rootPageId?: string,
|
||||
): Promise<{ pages: any[]; truncated: boolean }> {
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// Single request replaces the whole BFS: /pages/tree returns the full
|
||||
// permission-filtered flat page set of a space (or a subtree) at once. This
|
||||
// path is uncapped, so it is never truncated.
|
||||
const payload = rootPageId ? { pageId: rootPageId } : { spaceId };
|
||||
try {
|
||||
const response = await this.client.post("/pages/tree", payload);
|
||||
const pages = (response.data?.data ?? response.data)?.items ?? [];
|
||||
return { pages, truncated: false };
|
||||
} catch (e: any) {
|
||||
// Only fall back when the endpoint is absent (stock upstream Docmost);
|
||||
// any other error is a genuine failure and must propagate.
|
||||
if (
|
||||
!axios.isAxiosError(e) ||
|
||||
(e.response?.status !== 404 && e.response?.status !== 405)
|
||||
) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback: cursor-based breadth-first walk via listSidebarPages.
|
||||
const MAX_NODES = 10000;
|
||||
const result: any[] = [];
|
||||
const visited = new Set<string>();
|
||||
|
||||
// Seed with the root node itself when scoping to a subtree, so its own
|
||||
// comments aren't dropped: the primary /pages/tree seeds
|
||||
// getPageAndDescendants with id = rootPageId (root included), but
|
||||
// listSidebarPages(spaceId, rootPageId) returns only the root's CHILDREN.
|
||||
// The `visited` set below prevents a double-add if the root also appears
|
||||
// among the children. getPageRaw returns a page whose id/title/spaceId are
|
||||
// exactly what buildPageTree and checkNewComments consume.
|
||||
if (rootPageId) {
|
||||
try {
|
||||
const root = await this.getPageRaw(rootPageId);
|
||||
if (root?.id) {
|
||||
result.push(root);
|
||||
visited.add(root.id);
|
||||
}
|
||||
} catch {
|
||||
// Non-fatal: if the root can't be read, fall through to children-only.
|
||||
}
|
||||
}
|
||||
|
||||
// Seed the queue with the starting level (subtree children or roots).
|
||||
const queue: any[] = await this.listSidebarPages(spaceId, rootPageId);
|
||||
|
||||
while (queue.length > 0 && result.length < MAX_NODES) {
|
||||
const node = queue.shift();
|
||||
if (!node || typeof node !== "object" || !node.id) continue;
|
||||
|
||||
// Skip already-seen ids to guard against cycles / duplicate references.
|
||||
if (visited.has(node.id)) continue;
|
||||
visited.add(node.id);
|
||||
|
||||
result.push(node);
|
||||
|
||||
if (node.hasChildren) {
|
||||
try {
|
||||
const children = await this.listSidebarPages(spaceId, node.id);
|
||||
for (const child of children) queue.push(child);
|
||||
} catch (e: any) {
|
||||
// A failure fetching one node's children must not abort the whole
|
||||
// walk: skip this branch and keep enumerating the rest.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Truncated only when the cap was hit with the queue still non-empty (real
|
||||
// truncation, not a natural end at exactly MAX_NODES).
|
||||
return {
|
||||
pages: result,
|
||||
truncated: result.length >= MAX_NODES && queue.length > 0,
|
||||
};
|
||||
}
|
||||
|
||||
/** Raw page info including the ProseMirror JSON content and slugId. */
|
||||
|
||||
/**
|
||||
* Overridable seam over convertProseMirrorToMarkdown (issue #479). Production
|
||||
* just delegates; it exists as a method so a unit test can spy on it and
|
||||
* assert the conversion is genuinely SKIPPED on a getPage cache HIT (the whole
|
||||
* point of the cache) — an ESM named import cannot be intercepted otherwise.
|
||||
*/
|
||||
protected convertPageMarkdown(
|
||||
content: any,
|
||||
options: ConvertProseMirrorToMarkdownOptions,
|
||||
): string {
|
||||
return convertProseMirrorToMarkdown(content, options);
|
||||
}
|
||||
|
||||
async getPage(pageId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
const resultData = await this.getPageRaw(pageId);
|
||||
|
||||
// Agent read: hide resolved-comment anchors so the agent sees only active
|
||||
// discussions. Active anchors are kept. (The lossless exportPageMarkdown
|
||||
// round-trip deliberately does NOT pass this flag — resolved anchors there
|
||||
// must be preserved.)
|
||||
//
|
||||
// Content-addressed conversion cache (issue #479): the PM->Markdown walk is
|
||||
// the dominant cost of this hot read op. Key on the page's canonical UUID +
|
||||
// updatedAt (both from THIS /pages/info response, so mutually consistent) +
|
||||
// a hash of the conversion options. A hit returns the cached markdown and
|
||||
// skips the walk; a miss converts and stores. The cached value is the
|
||||
// conversion output BEFORE the {{SUBPAGES}} substitution below, which uses
|
||||
// live subpage data and stays outside the cache — so the final result is
|
||||
// byte-identical to the uncached path.
|
||||
const convertOptions = { dropResolvedCommentAnchors: true };
|
||||
let content = "";
|
||||
if (resultData.content) {
|
||||
// Only cache when we have a stable identity+version for the key. Both come
|
||||
// from the same response; if either is missing (unexpected server shape),
|
||||
// fall back to converting uncached rather than keying on a partial tuple.
|
||||
const cacheable =
|
||||
typeof resultData.id === "string" &&
|
||||
typeof resultData.updatedAt === "string";
|
||||
const cacheKey = cacheable
|
||||
? GetPageConversionCache.key(
|
||||
resultData.id,
|
||||
resultData.updatedAt,
|
||||
hashConvertOptions(convertOptions),
|
||||
)
|
||||
: null;
|
||||
|
||||
const cached = cacheKey ? this.getPageCache.get(cacheKey) : undefined;
|
||||
if (cached !== undefined) {
|
||||
content = cached;
|
||||
this.onMetricFn?.("mcp_getpage_cache_hits_total", 1);
|
||||
} else {
|
||||
// Goes through the convertPageMarkdown seam (not the raw import) so a
|
||||
// test can assert the conversion is SKIPPED on a hit (issue #479 F2).
|
||||
content = this.convertPageMarkdown(resultData.content, convertOptions);
|
||||
if (cacheKey) this.getPageCache.set(cacheKey, content);
|
||||
// A non-cacheable page (missing id/updatedAt) is still a genuine
|
||||
// conversion, so it counts as a miss for an honest hit-rate.
|
||||
this.onMetricFn?.("mcp_getpage_cache_misses_total", 1);
|
||||
}
|
||||
}
|
||||
|
||||
// Always fetch subpages to provide context to the agent.
|
||||
//
|
||||
// NOT parallelizable with the page fetch (issue #479 asked to check): the
|
||||
// sidebar-pages endpoint REQUIRES spaceId in its POST body, and spaceId is
|
||||
// only known FROM this page fetch's response (resolvePageId yields the UUID
|
||||
// but never the spaceId). So `Promise.all([pageFetch, subpagesFetch])` would
|
||||
// have to invent a spaceId it does not have — the two calls are inherently
|
||||
// sequential. Correctness wins; the conversion cache above is the real speedup.
|
||||
let subpages: any[] = [];
|
||||
try {
|
||||
// `pageId` may be a slugId, but the sidebar-pages endpoint requires the
|
||||
// UUID; `resultData.id` holds the resolved UUID returned by getPageRaw.
|
||||
subpages = await this.listSidebarPages(resultData.spaceId, resultData.id);
|
||||
} catch (e: any) {
|
||||
console.warn("Failed to fetch subpages:", e);
|
||||
}
|
||||
|
||||
// Resolve subpages if the placeholder exists
|
||||
if (content && content.includes("{{SUBPAGES}}")) {
|
||||
if (subpages && subpages.length > 0) {
|
||||
const list = subpages
|
||||
.map((p: any) => `- [${p.title}](page:${p.id})`)
|
||||
.join("\n");
|
||||
content = content.replace("{{SUBPAGES}}", `### Subpages\n${list}`);
|
||||
} else {
|
||||
content = content.replace("{{SUBPAGES}}", "");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
data: filterPage(resultData, content, subpages),
|
||||
success: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** Page info + raw ProseMirror JSON content (lossless representation). */
|
||||
async getPageJson(pageId: string) {
|
||||
const data = await this.getPageRaw(pageId);
|
||||
return {
|
||||
id: data.id,
|
||||
slugId: data.slugId,
|
||||
title: data.title,
|
||||
parentPageId: data.parentPageId,
|
||||
spaceId: data.spaceId,
|
||||
updatedAt: data.updatedAt,
|
||||
content: data.content || { type: "doc", content: [] },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch an INTERNAL Docmost file (authed loopback) for sandbox mirroring.
|
||||
* `src` is normalized to `/api/files/<id>/<file>`; `this.client.baseURL`
|
||||
* already ends in `/api`, so we strip the leading `/api` and request the
|
||||
* relative path with the client's Authorization header. Returns the raw bytes
|
||||
* and the response Content-Type (mime), defaulting to octet-stream.
|
||||
*
|
||||
* The fetch is size-bounded (hard 64 MiB ceiling) purely to protect memory;
|
||||
* the authoritative per-blob cap is enforced by the sandbox `put`. The path is
|
||||
* resolved via resolveInternalFilePath, which REJECTS (throws) any traversal
|
||||
* or percent-encoded src that would let an attacker-controlled `attrs.src`
|
||||
* escape `/api/files/` and reach another internal endpoint (SSRF). That throw
|
||||
* happens before this.client.get, so a malicious src is counted as a failed
|
||||
* mirror — it never reaches the network.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Compact outline of a page's top-level blocks (no full document body).
|
||||
* Cheap way to locate sections/tables and grab block ids before drilling in
|
||||
* with getNode / patchNode / insertNode.
|
||||
*/
|
||||
async getOutline(pageId: string) {
|
||||
await this.ensureAuthenticated();
|
||||
const data = await this.getPageRaw(pageId);
|
||||
return {
|
||||
pageId,
|
||||
slugId: data.slugId,
|
||||
title: data.title,
|
||||
outline: buildOutline(data.content ?? { type: "doc", content: [] }),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a single block for editing by reference: a block id (headings/
|
||||
* paragraphs/callouts/images), or `#<index>` to select a top-level block by its
|
||||
* outline index (the only way to reach tables/rows/cells, which carry no id).
|
||||
*
|
||||
* `format` (#413):
|
||||
* - `"markdown"` (DEFAULT): serialize the block via the canonical converter
|
||||
* (`{type:"doc",content:[node]}` -> `convertProseMirrorToMarkdown`) — a read
|
||||
* "for editing": pair it with `patchNode({markdown})` to rewrite the block.
|
||||
* Comment anchors (`<span data-comment-id>`, INCLUDING resolved ones) are
|
||||
* NOT stripped here (unlike getPage): losing them on write-back would
|
||||
* orphan the thread. Returns `{ ..., format:"markdown", markdown }`.
|
||||
* - `"json"`: return the raw ProseMirror subtree as-is (lossless; the previous
|
||||
* default). Returns `{ ..., format:"json", node }`.
|
||||
*
|
||||
* AUTO fallback: a type that cannot be a document top-level child
|
||||
* (tableRow/tableCell/tableHeader, addressed by `#<index>`) is NOT expressible
|
||||
* as a standalone markdown document, so a `"markdown"` request for such a node
|
||||
* transparently falls back to JSON with an explicit `format:"json"` field. The
|
||||
* check derives from the schema's `doc` contentMatch, so it tracks the schema.
|
||||
*/
|
||||
async getNode(
|
||||
pageId: string,
|
||||
nodeId: string,
|
||||
format: "markdown" | "json" = "markdown",
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
const data = await this.getPageRaw(pageId);
|
||||
const hit = getNodeByRef(
|
||||
data.content ?? { type: "doc", content: [] },
|
||||
nodeId,
|
||||
);
|
||||
if (!hit) {
|
||||
throw new Error(
|
||||
`getNode: no node found for "${nodeId}" on page ${pageId} (use a block id from getOutline, or "#<index>" for a top-level block such as a table)`,
|
||||
);
|
||||
}
|
||||
|
||||
// JSON requested (or a non-top-level type that markdown cannot represent as a
|
||||
// standalone document): return the subtree verbatim.
|
||||
if (format === "json" || !canBeDocChild(hit.type)) {
|
||||
return {
|
||||
pageId,
|
||||
ref: nodeId,
|
||||
path: hit.path,
|
||||
type: hit.type,
|
||||
format: "json" as const,
|
||||
node: hit.node,
|
||||
};
|
||||
}
|
||||
|
||||
// Markdown: wrap the node as a one-block doc and run the canonical converter.
|
||||
// Comment anchors are DELIBERATELY preserved (converter default) so a
|
||||
// getNode(markdown) -> edit -> patchNode(markdown) round trip does not orphan
|
||||
// a comment thread; this differs from getPage, which strips them.
|
||||
const markdown = convertProseMirrorToMarkdown({
|
||||
type: "doc",
|
||||
content: [hit.node],
|
||||
});
|
||||
return {
|
||||
pageId,
|
||||
ref: nodeId,
|
||||
path: hit.path,
|
||||
type: hit.type,
|
||||
format: "markdown" as const,
|
||||
markdown,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Find every occurrence of `query` on a page IN MEMORY, over the plain text of
|
||||
* each text container (reusing the same `getPageRaw` fetch as the other read
|
||||
* tools) — no server search endpoint, no whole-document round-trip through the
|
||||
* model. Returns `{ total, truncated, matches }`; each match carries a ref for
|
||||
* getNode/patchNode (the `#<index>` form resolves with getNode but NOT
|
||||
* patchNode — see SearchMatch.nodeId), plus the top-level block index and a
|
||||
* short context window used to build a unique text `selection` for
|
||||
* createComment (createComment has no nodeId param). The pure engine
|
||||
* (`searchInDoc`) owns the traversal, glue, the RE2 ReDoS-safe regex engine
|
||||
* and the empty-query / invalid-or-unsupported-regex errors.
|
||||
*/
|
||||
async searchInPage(pageId: string, query: string, opts: SearchOptions = {}) {
|
||||
await this.ensureAuthenticated();
|
||||
const data = await this.getPageRaw(pageId);
|
||||
const result = searchInDoc(
|
||||
data.content ?? { type: "doc", content: [] },
|
||||
query,
|
||||
opts,
|
||||
);
|
||||
return { pageId, query, ...result };
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a table as a matrix. `tableRef` is `#<index>` (from getOutline) or a
|
||||
* block id of any node inside the table. Returns the cell texts plus a
|
||||
* parallel cellIds matrix (each cell's first paragraph id, or null) so a
|
||||
* caller can patchNode a cell for rich-formatted edits. Throws when no table
|
||||
* resolves for the reference.
|
||||
*/
|
||||
async getTable(pageId: string, tableRef: string) {
|
||||
await this.ensureAuthenticated();
|
||||
const data = await this.getPageRaw(pageId);
|
||||
const t = readTable(data.content ?? { type: "doc", content: [] }, tableRef);
|
||||
if (!t) {
|
||||
throw new Error(
|
||||
`tableGet: no table found for "${tableRef}" on page ${pageId} (use "#<index>" from getOutline, or a block id inside the table)`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
pageId,
|
||||
table: tableRef,
|
||||
rows: t.rows,
|
||||
cols: t.cols,
|
||||
path: t.path,
|
||||
cells: t.cells,
|
||||
cellIds: t.cellIds,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Insert a row of plain-text cells into a table on the LIVE collab document.
|
||||
* `tableRef` is `#<index>` or a block id inside the target table. `cells` is
|
||||
* padded to the table's column count (more cells than columns throws); `index`
|
||||
* is a 0-based insert position (omit/out-of-range to append). Throws when no
|
||||
* table resolves for the reference.
|
||||
*/
|
||||
|
||||
async search(
|
||||
query: string,
|
||||
spaceId?: string,
|
||||
limit?: number,
|
||||
opts: { parentPageId?: string; titleOnly?: boolean } = {},
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
// Opt into the #443 agent-lookup mode: `substring: true` turns on the hybrid
|
||||
// substring + FTS branch that returns path + snippet + score. A stock
|
||||
// upstream server strips these unknown DTO fields (whitelist:true) and
|
||||
// silently degrades to plain FTS — see the tool-registration comment.
|
||||
const payload: Record<string, any> = {
|
||||
query,
|
||||
spaceId,
|
||||
substring: true,
|
||||
};
|
||||
if (opts.parentPageId) payload.parentPageId = opts.parentPageId;
|
||||
if (opts.titleOnly) payload.titleOnly = true;
|
||||
// Clamp an optional caller-supplied limit into the lookup range (1..50)
|
||||
// before forwarding; omit it when not provided so the server default applies.
|
||||
if (limit !== undefined) {
|
||||
payload.limit = Math.max(1, Math.min(50, limit));
|
||||
}
|
||||
const response = await this.client.post("/search", payload);
|
||||
|
||||
// Normalize both response shapes: bare array and paginated { items: [...] }
|
||||
const data = response.data?.data;
|
||||
const items = Array.isArray(data) ? data : data?.items || [];
|
||||
const filteredItems = items.map((item: any) => filterSearchResult(item));
|
||||
|
||||
return {
|
||||
items: filteredItems,
|
||||
success: response.data?.success || false,
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
return ReadMixin;
|
||||
}
|
||||
@@ -0,0 +1,225 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import {
|
||||
collectInternalFileNodes,
|
||||
normalizeFileUrl,
|
||||
resolveInternalFilePath,
|
||||
} from "../lib/internal-file-urls.js";
|
||||
|
||||
// Public method surface of StashMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements IStashMixin` fails to compile on drift.
|
||||
export interface IStashMixin {
|
||||
stashPage(pageId: string): Promise<{ uri: string; sha256: string; size: number; images: { mirrored: number; failed: number }; }>;
|
||||
}
|
||||
|
||||
export function StashMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & IStashMixin> & TBase {
|
||||
abstract class StashMixin extends Base implements IStashMixin {
|
||||
/**
|
||||
* Fetch an INTERNAL Docmost file (authed loopback) for sandbox mirroring.
|
||||
* `src` is normalized to `/api/files/<id>/<file>`; `this.client.baseURL`
|
||||
* already ends in `/api`, so we strip the leading `/api` and request the
|
||||
* relative path with the client's Authorization header. Returns the raw bytes
|
||||
* and the response Content-Type (mime), defaulting to octet-stream.
|
||||
*
|
||||
* The fetch is size-bounded (hard 64 MiB ceiling) purely to protect memory;
|
||||
* the authoritative per-blob cap is enforced by the sandbox `put`. The path is
|
||||
* resolved via resolveInternalFilePath, which REJECTS (throws) any traversal
|
||||
* or percent-encoded src that would let an attacker-controlled `attrs.src`
|
||||
* escape `/api/files/` and reach another internal endpoint (SSRF). That throw
|
||||
* happens before this.client.get, so a malicious src is counted as a failed
|
||||
* mirror — it never reaches the network.
|
||||
*/
|
||||
protected async fetchInternalFile(
|
||||
src: string,
|
||||
): Promise<{ buffer: Buffer; mime: string }> {
|
||||
const HARD_CEILING = 64 * 1024 * 1024; // 64 MiB memory guard
|
||||
const relPath = resolveInternalFilePath(src);
|
||||
const response = await this.client.get(relPath, {
|
||||
responseType: "arraybuffer",
|
||||
timeout: 30000,
|
||||
maxContentLength: HARD_CEILING,
|
||||
maxBodyLength: HARD_CEILING,
|
||||
});
|
||||
const buffer = Buffer.from(response.data);
|
||||
if (buffer.length === 0) {
|
||||
throw new Error(`Empty file response from "${src}"`);
|
||||
}
|
||||
const rawCt = response.headers?.["content-type"];
|
||||
const mime =
|
||||
typeof rawCt === "string" && rawCt.length > 0
|
||||
? rawCt.split(";")[0].trim().toLowerCase()
|
||||
: "application/octet-stream";
|
||||
return { buffer, mime };
|
||||
}
|
||||
|
||||
/**
|
||||
* Stash a page's full content into the in-RAM blob sandbox and return ONLY a
|
||||
* short anonymous URL — the body never enters the model context (this is the
|
||||
* whole point: ~30KB+ ProseMirror docs blow the model context if passed as a
|
||||
* tool argument). Every INTERNAL file/image src (the type-agnostic criterion,
|
||||
* so drawio/excalidraw/video/file nodes are covered too) is mirrored into the
|
||||
* sandbox and its `src` rewritten to the sandbox URL, so an external consumer
|
||||
* can fetch the images anonymously. External http(s) srcs are left untouched.
|
||||
*
|
||||
* Blobs live in RAM with a short TTL and are cleared on restart — consume the
|
||||
* URLs within the TTL and one uptime. A failed image fetch never aborts the
|
||||
* doc: the original src is kept and the failure counted.
|
||||
*
|
||||
* Returns { uri, sha256, size, images:{mirrored, failed} }. `uri` and `sha256`
|
||||
* are for the document blob; `sha256` is also the blob's ETag (integrity).
|
||||
*/
|
||||
async stashPage(pageId: string): Promise<{
|
||||
uri: string;
|
||||
sha256: string;
|
||||
size: number;
|
||||
images: { mirrored: number; failed: number };
|
||||
}> {
|
||||
if (!this.sandboxPut) {
|
||||
throw new Error(
|
||||
"stashPage is unavailable: the blob sandbox is not configured on this server",
|
||||
);
|
||||
}
|
||||
await this.ensureAuthenticated();
|
||||
|
||||
// Stash the SAME shape getPageJson returns (id/title/.../content), with a
|
||||
// deep clone so the rewrite never mutates anything shared.
|
||||
const pageJson = await this.getPageJson(pageId);
|
||||
const cloned: any = structuredClone(pageJson);
|
||||
|
||||
// Group internal-file nodes by normalized src so each unique resource is
|
||||
// fetched + stored ONCE (dedup), and every node sharing that src points at
|
||||
// the one sandbox blob. Capture each node's ORIGINAL raw src per-node:
|
||||
// dedup groups nodes whose normalized src is equal even when their raw srcs
|
||||
// differ (e.g. `/api/files/...` vs the bare `/files/...`), so on a revert we
|
||||
// must restore each node's own original value, not the group key.
|
||||
const bySrc = new Map<string, Array<{ node: any; origSrc: string }>>();
|
||||
for (const node of collectInternalFileNodes(cloned.content)) {
|
||||
const origSrc = String(node.attrs.src);
|
||||
const src = normalizeFileUrl(origSrc);
|
||||
const entry = { node, origSrc };
|
||||
const group = bySrc.get(src);
|
||||
if (group) group.push(entry);
|
||||
else bySrc.set(src, [entry]);
|
||||
}
|
||||
|
||||
let mirrored = 0;
|
||||
let failed = 0;
|
||||
// Record every successful mirror so it can be (a) reverted if its blob gets
|
||||
// FIFO-evicted by a LATER put in this same stash, and (b) freed if the final
|
||||
// doc put throws.
|
||||
const mirrors: Array<{
|
||||
uri: string;
|
||||
entries: Array<{ node: any; origSrc: string }>;
|
||||
}> = [];
|
||||
const MAX_CONCURRENCY = 5;
|
||||
const groups = [...bySrc.entries()];
|
||||
for (let i = 0; i < groups.length; i += MAX_CONCURRENCY) {
|
||||
const batch = groups.slice(i, i + MAX_CONCURRENCY);
|
||||
await Promise.all(
|
||||
batch.map(async ([src, entries]) => {
|
||||
try {
|
||||
const { buffer, mime } = await this.fetchInternalFile(src);
|
||||
// put may throw if the blob exceeds the per-blob/total caps.
|
||||
const stored = this.sandboxPut!(buffer, mime);
|
||||
for (const entry of entries) entry.node.attrs.src = stored.uri;
|
||||
mirrors.push({ uri: stored.uri, entries });
|
||||
mirrored++;
|
||||
} catch (err) {
|
||||
// One bad/oversized image (or a rejected traversal src) must not
|
||||
// abort the document. Logged unconditionally (never the blob body),
|
||||
// matching the package's ungated console.warn convention.
|
||||
failed++;
|
||||
console.warn(
|
||||
`stashPage: failed to mirror "${src}": ${
|
||||
err instanceof Error ? err.message : String(err)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
// Revert one mirror's nodes to their original internal srcs and re-count it
|
||||
// as failed (its blob was FIFO-evicted before the doc could reference it
|
||||
// safely).
|
||||
const revertMirror = (mirror: {
|
||||
uri: string;
|
||||
entries: Array<{ node: any; origSrc: string }>;
|
||||
}) => {
|
||||
for (const entry of mirror.entries) entry.node.attrs.src = entry.origSrc;
|
||||
mirrored--;
|
||||
failed++;
|
||||
console.warn(
|
||||
`stashPage: mirrored blob ${mirror.uri} was evicted before the doc ` +
|
||||
`could safely reference it; reverted its src and counted it as failed`,
|
||||
);
|
||||
};
|
||||
|
||||
// Pre-put reconciliation: an image put earlier in THIS stash can FIFO-evict
|
||||
// an even-earlier image of the same stash. Drop those from the live set
|
||||
// first so the first serialized doc is already mostly correct.
|
||||
let liveMirrors = mirrors;
|
||||
if (this.sandboxHas) {
|
||||
liveMirrors = [];
|
||||
for (const mirror of mirrors) {
|
||||
if (this.sandboxHas(mirror.uri)) liveMirrors.push(mirror);
|
||||
else revertMirror(mirror);
|
||||
}
|
||||
}
|
||||
|
||||
// Put the document, then reconcile against eviction caused by the doc put
|
||||
// ITSELF (the doc is newest, FIFO drops oldest = this stash's images). Each
|
||||
// iteration reverts >=1 mirror, so the loop terminates (worst case: all
|
||||
// images reverted and the doc references no sandbox image URLs).
|
||||
let stored: { uri: string; sha256: string; size: number };
|
||||
for (;;) {
|
||||
const docBuf = Buffer.from(JSON.stringify(cloned), "utf8");
|
||||
let docStored: { uri: string; sha256: string; size: number };
|
||||
try {
|
||||
docStored = this.sandboxPut(docBuf, "application/json");
|
||||
} catch (err) {
|
||||
// The doc put failed (e.g. doc exceeds the cap). Free this op's image
|
||||
// blobs instead of leaking them in RAM for the whole TTL, then
|
||||
// re-throw.
|
||||
if (this.sandboxEvict) {
|
||||
for (const mirror of liveMirrors) this.sandboxEvict(mirror.uri);
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
if (!this.sandboxHas) {
|
||||
stored = docStored;
|
||||
break;
|
||||
}
|
||||
const evictedNow = liveMirrors.filter((m) => !this.sandboxHas!(m.uri));
|
||||
if (evictedNow.length === 0) {
|
||||
stored = docStored;
|
||||
break;
|
||||
}
|
||||
// The doc we just stored references now-dead blobs. Revert those nodes,
|
||||
// drop the stale doc blob, and loop to re-serialize + re-put the
|
||||
// corrected doc.
|
||||
for (const mirror of evictedNow) revertMirror(mirror);
|
||||
liveMirrors = liveMirrors.filter((m) => this.sandboxHas!(m.uri));
|
||||
if (this.sandboxEvict) this.sandboxEvict(docStored.uri);
|
||||
}
|
||||
return {
|
||||
uri: stored.uri,
|
||||
sha256: stored.sha256,
|
||||
size: stored.size,
|
||||
images: { mirrored, failed },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compact outline of a page's top-level blocks (no full document body).
|
||||
* Cheap way to locate sections/tables and grab block ids before drilling in
|
||||
* with getNode / patchNode / insertNode.
|
||||
*/
|
||||
}
|
||||
return StashMixin;
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
// Auto-split from client.ts (issue #450). Mixin over the shared client context.
|
||||
// Bodies are VERBATIM from the original DocmostClient; only the enclosing class
|
||||
// changed to a mixin factory. See client/context.ts for the shared base.
|
||||
import type { GConstructor, DocmostClientContext } from "./context.js";
|
||||
import {
|
||||
updatePageContentRealtime,
|
||||
replacePageContent,
|
||||
markdownToProseMirror,
|
||||
markdownToProseMirrorCanonical,
|
||||
mutatePageContent,
|
||||
assertYjsEncodable,
|
||||
MutationResult,
|
||||
} from "../lib/collaboration.js";
|
||||
import {
|
||||
replaceNodeById,
|
||||
replaceNodeByIdWithMany,
|
||||
reassignCollidingBlockIds,
|
||||
deleteNodeById,
|
||||
assertUnambiguousMatch,
|
||||
insertNodeRelative,
|
||||
insertNodesRelative,
|
||||
blockPlainText,
|
||||
buildOutline,
|
||||
getNodeByRef,
|
||||
readTable,
|
||||
insertTableRow,
|
||||
deleteTableRow,
|
||||
updateTableCell,
|
||||
findInvalidNode,
|
||||
} from "@docmost/prosemirror-markdown";
|
||||
import { withPageLock, isUuid } from "../lib/page-lock.js";
|
||||
import {
|
||||
blockText,
|
||||
walk,
|
||||
getList,
|
||||
insertMarkerAfter,
|
||||
setCalloutRange,
|
||||
noteItem,
|
||||
mdToInlineNodes,
|
||||
commentsToFootnotes,
|
||||
canonicalizeFootnotes,
|
||||
insertInlineFootnote,
|
||||
mergeFootnoteDefinitions,
|
||||
} from "../lib/transforms.js";
|
||||
|
||||
// Public method surface of TablesMixin (issue #450) — a NAMED type so the factory
|
||||
// return type is expressible in the emitted .d.ts (the anonymous mixin class
|
||||
// carries the base's protected shared state, which would otherwise trip TS4094).
|
||||
// Derived from the class below; `implements ITablesMixin` fails to compile on drift.
|
||||
export interface ITablesMixin {
|
||||
insertFootnote(pageId: string, anchorText: string, text: string): any;
|
||||
tableInsertRow(pageId: string, tableRef: string, cells: string[], index?: number): any;
|
||||
tableDeleteRow(pageId: string, tableRef: string, index: number): any;
|
||||
tableUpdateCell(pageId: string, tableRef: string, row: number, col: number, text: string): any;
|
||||
}
|
||||
|
||||
export function TablesMixin<TBase extends GConstructor<DocmostClientContext>>(Base: TBase): GConstructor<DocmostClientContext & ITablesMixin> & TBase {
|
||||
abstract class TablesMixin extends Base implements ITablesMixin {
|
||||
/**
|
||||
* AUTHOR-INLINE footnote insertion. The agent supplies only WHERE
|
||||
* (`anchorText`, a snippet of body text to attach the marker after) and WHAT
|
||||
* (`text`, the footnote content as markdown). Numbering and the bottom
|
||||
* `footnotesList` are derived deterministically server-side
|
||||
* (`insertInlineFootnote` -> `canonicalizeFootnotes`): the agent never sees,
|
||||
* assigns, or edits a footnote number or the list, so it CANNOT desync.
|
||||
*
|
||||
* Content DEDUP: when an existing definition has the same content, its id is
|
||||
* reused (one number, one definition, several references). The write is atomic
|
||||
* via `mutatePageContent` (single-writer, page-locked); if the anchor text is
|
||||
* not found the transform aborts with a clear error and no write happens.
|
||||
*/
|
||||
async insertFootnote(pageId: string, anchorText: string, text: string) {
|
||||
await this.ensureAuthenticated();
|
||||
if (!anchorText || !anchorText.trim()) {
|
||||
throw new Error("insertFootnote: anchorText is required");
|
||||
}
|
||||
if (text == null || `${text}`.trim() === "") {
|
||||
throw new Error("insertFootnote: text is required");
|
||||
}
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
let result: { footnoteId: string; reused: boolean } | null = null;
|
||||
const mutation = await this.mutatePage(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc: any) => {
|
||||
const r = insertInlineFootnote(liveDoc, { anchorText, text });
|
||||
if (!r.inserted) {
|
||||
// Abort the page-locked write by throwing: mutatePageContent does not
|
||||
// persist when the transform throws, so a missing anchor leaves the
|
||||
// page untouched (no partial write).
|
||||
throw new Error(
|
||||
`insertFootnote: anchor text not found: ${JSON.stringify(
|
||||
anchorText.slice(0, 80),
|
||||
)}`,
|
||||
);
|
||||
}
|
||||
result = { footnoteId: r.footnoteId, reused: r.reused };
|
||||
return r.doc;
|
||||
},
|
||||
);
|
||||
// The not-found path throws inside the transform (aborting mutatePage), so by
|
||||
// here `result` is always set.
|
||||
const r = result!;
|
||||
return {
|
||||
success: true,
|
||||
modified: true,
|
||||
pageId,
|
||||
footnoteId: r.footnoteId,
|
||||
reused: r.reused,
|
||||
message: r.reused
|
||||
? "Footnote inserted (reused an existing same-content definition)."
|
||||
: "Footnote inserted.",
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Page-locked write seam over collaboration.mutatePageContent. Production just
|
||||
* delegates; it exists as an overridable method so the insertFootnote wrapper
|
||||
* (transform abort-on-not-found + response shaping) can be unit-tested without
|
||||
* standing up a live Hocuspocus collab socket.
|
||||
*
|
||||
* SELF-RESOLVES the pageId to the canonical UUID (issue #449, "resolve-then-
|
||||
* lock"): every write must lock and key its CollabSession by the UUID, never a
|
||||
* raw slugId (#260). resolvePageId is cached/idempotent, so a caller that
|
||||
* already resolved pays no extra round-trip; centralizing it here means a
|
||||
* caller that reaches this seam with a raw slugId still locks correctly instead
|
||||
* of silently splitting the mutex key. withPageLock also asserts the key is a
|
||||
* UUID as a hard backstop.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Insert a row of plain-text cells into a table on the LIVE collab document.
|
||||
* `tableRef` is `#<index>` or a block id inside the target table. `cells` is
|
||||
* padded to the table's column count (more cells than columns throws); `index`
|
||||
* is a 0-based insert position (omit/out-of-range to append). Throws when no
|
||||
* table resolves for the reference.
|
||||
*/
|
||||
async tableInsertRow(
|
||||
pageId: string,
|
||||
tableRef: string,
|
||||
cells: string[],
|
||||
index?: number,
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
// Track insertion in an outer var, reset per-transform, so a collab retry
|
||||
// recomputes it cleanly (mirrors insertNode's pattern).
|
||||
let inserted = false;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
inserted = false;
|
||||
const { doc: nd, inserted: ins } = insertTableRow(
|
||||
liveDoc,
|
||||
tableRef,
|
||||
cells,
|
||||
index,
|
||||
);
|
||||
inserted = ins;
|
||||
if (!inserted) return null; // table not found -> skip the write entirely
|
||||
return nd;
|
||||
},
|
||||
);
|
||||
|
||||
if (!inserted) {
|
||||
throw new Error(
|
||||
`tableInsertRow: no table found for "${tableRef}" on page ${pageId} (use "#<index>" from getOutline, or a block id inside the table)`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
table: tableRef,
|
||||
inserted: true,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete the row at 0-based `index` from a table on the LIVE collab document.
|
||||
* `tableRef` is `#<index>` or a block id inside the target table. The helper's
|
||||
* out-of-range and last-row errors propagate; a missing table throws here.
|
||||
*/
|
||||
async tableDeleteRow(pageId: string, tableRef: string, index: number) {
|
||||
await this.ensureAuthenticated();
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
let deleted = false;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
deleted = false;
|
||||
const { doc: nd, deleted: del } = deleteTableRow(
|
||||
liveDoc,
|
||||
tableRef,
|
||||
index,
|
||||
);
|
||||
deleted = del;
|
||||
if (!deleted) return null; // table not found -> skip the write entirely
|
||||
return nd;
|
||||
},
|
||||
);
|
||||
|
||||
if (!deleted) {
|
||||
throw new Error(
|
||||
`tableDeleteRow: no table found for "${tableRef}" on page ${pageId} (use "#<index>" from getOutline, or a block id inside the table)`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
table: tableRef,
|
||||
deleted: true,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the plain-text content of cell `[row, col]` (0-based) in a table on the
|
||||
* LIVE collab document, replacing the cell's content with a single text
|
||||
* paragraph (the cell's first-paragraph id is preserved). `tableRef` is
|
||||
* `#<index>` or a block id inside the target table. The helper's out-of-range
|
||||
* error propagates; a missing table throws here.
|
||||
*/
|
||||
async tableUpdateCell(
|
||||
pageId: string,
|
||||
tableRef: string,
|
||||
row: number,
|
||||
col: number,
|
||||
text: string,
|
||||
) {
|
||||
await this.ensureAuthenticated();
|
||||
const collabToken = await this.getCollabTokenWithReauth();
|
||||
// Open the collab doc by the canonical UUID, never the slugId (#260).
|
||||
const pageUuid = await this.resolvePageId(pageId);
|
||||
|
||||
let updated = false;
|
||||
const mutation = await mutatePageContent(
|
||||
pageUuid,
|
||||
collabToken,
|
||||
this.apiUrl,
|
||||
(liveDoc) => {
|
||||
updated = false;
|
||||
const { doc: nd, updated: upd } = updateTableCell(
|
||||
liveDoc,
|
||||
tableRef,
|
||||
row,
|
||||
col,
|
||||
text,
|
||||
);
|
||||
updated = upd;
|
||||
if (!updated) return null; // table not found -> skip the write entirely
|
||||
return nd;
|
||||
},
|
||||
);
|
||||
|
||||
if (!updated) {
|
||||
throw new Error(
|
||||
`tableUpdateCell: no table found for "${tableRef}" on page ${pageId} (use "#<index>" from getOutline, or a block id inside the table)`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
success: true,
|
||||
table: tableRef,
|
||||
row,
|
||||
col,
|
||||
verify: mutation.verify,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new page with title and content.
|
||||
* Uses the /pages/import workaround (the only endpoint accepting content),
|
||||
* then moves the page and restores the exact title: the import endpoint
|
||||
* derives the title from the FILENAME and replaces spaces with
|
||||
* underscores, so we explicitly re-set it via /pages/update afterwards.
|
||||
*/
|
||||
}
|
||||
return TablesMixin;
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user