diff --git a/.env.example b/.env.example index 6b06ffe4..bb189adc 100644 --- a/.env.example +++ b/.env.example @@ -312,6 +312,16 @@ MCP_DOCMOST_PASSWORD= # enabled for a workspace, and the same single-instance constraint applies (the # registry is process-local). # AI_CHAT_RESUMABLE_STREAM=false +# +# Per-run replay ring cap (#491), in BYTES, for the resumable-stream registry +# above. The registry buffers the run's recent SSE tail so a reopened tab can +# attach and continue from the step it already persisted; the ring is bounded and +# rotates on every confirmed step-persist. This caps the un-persisted tail between +# rotations — an overflow evicts the oldest frames and a late attach falls back to +# 204 -> degraded poll, so correctness never depends on the size. Default 4194304 +# (4MB); a 0/invalid value falls back to the default. The per-subscriber backpressure +# cap is derived as 2x this value. Only meaningful with AI_CHAT_RESUMABLE_STREAM on. +# AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES=4194304 # --- Run lifecycle tunables (#487) --- # These govern the universal run machinery (every turn is now a first-class run, diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index f0996955..224a1e43 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -62,6 +62,38 @@ jobs: needs: [test, e2e-server, e2e-mcp, build] runs-on: ubuntu-latest timeout-minutes: 30 + # Image boot-smoke (issue #476): every other job tests code from the working + # tree, but the :develop IMAGE that watchtower pulls was never actually + # started anywhere (incident classes #353/#452/#361-boot: startup-migrator + # crash-loop, runtime module missing from the image, wrong static-asset + # headers). The services below back a smoke boot of the exact image right + # before it is pushed; a smoke failure blocks the push. + services: + postgres: + # via mirror.gcr.io (Docker Hub pull-through cache; avoids Hub anonymous + # pull rate-limit that randomly fails on shared GitHub runner IPs). + image: mirror.gcr.io/pgvector/pgvector:pg18 + env: + POSTGRES_DB: docmost + POSTGRES_USER: docmost + POSTGRES_PASSWORD: docmost + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U docmost" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + redis: + # via mirror.gcr.io (see postgres note above). + image: mirror.gcr.io/library/redis:7 + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 5s + --health-timeout 5s + --health-retries 20 steps: - name: Checkout uses: actions/checkout@v4 @@ -82,6 +114,37 @@ jobs: id: version run: echo "value=$(git describe --tags --always)" >> "$GITHUB_OUTPUT" + # Load the image into the local docker daemon so it can be booted (the + # push step below exports straight to the registry and leaves nothing + # runnable locally). CONVENTION: build-args here must stay TEXTUALLY + # IDENTICAL to the push step's build-args — same cache scope + same args + # means the layers are reused and the image we smoke IS the image we push. + - name: Build image for smoke (load, no push) + uses: docker/build-push-action@v6 + with: + context: . + platforms: linux/amd64 + build-args: | + APP_VERSION=${{ steps.version.outputs.value }} + AI_AGENT_ROLES_CATALOG_URL=https://raw.githubusercontent.com/vvzvlad/gitmost/develop/agent-roles-catalog + load: true + push: false + tags: gitmost:smoke + cache-from: type=gha,scope=develop-amd64 + + # Boot-smoke the exact image against the job services (see the comment on + # `services:` above): health (startup migrator), auth/setup, client dist + # served, immutable + brotli asset headers. Fails the job (and therefore + # the push) on any miss. + - name: Smoke the built image + run: bash scripts/ci/image-smoke.sh gitmost:smoke + + # The smoke script leaves the container running on failure precisely so + # the boot error (migration mismatch, stack trace) is diagnosable here. + - name: Dump smoke container log on failure + if: failure() + run: docker logs gitmost-smoke 2>&1 | tail -200 || true + - name: Build and push develop image uses: docker/build-push-action@v6 with: diff --git a/.github/workflows/nightly-property.yml b/.github/workflows/nightly-property.yml index 207972cc..ba5b7976 100644 --- a/.github/workflows/nightly-property.yml +++ b/.github/workflows/nightly-property.yml @@ -124,9 +124,17 @@ jobs: exit "$FAILED" # A GENUINE counterexample: fast-check printed a shrunk minimal case and its - # reproducing seed into property-output.txt. File a dedup-guarded issue whose - # title prefix is UNIQUE to counterexamples, so an infra failure (handled by - # the next step under a different title) can never poison this dedup. + # reproducing seed into property-output.txt. File a dedup-guarded issue. + # + # Dedup is keyed on a HASH of the SHRUNK COUNTEREXAMPLE (the minimal failing + # input), NOT on the issue title prefix. Keying on the prefix would let a + # single open issue swallow every OTHER counterexample (a different bug B whose + # title shares the prefix would be treated as a duplicate and stay silent until + # the first issue is closed). Hashing the shrunk example instead means two + # DIFFERENT counterexamples get two DIFFERENT issues, while a re-find of the + # SAME counterexample still dedupes onto the existing one. The infra-failure + # step (below) still keys on its own distinct title, so it can never poison + # this dedup either. - name: File counterexample issue # always() is REQUIRED: the fuzz step exits nonzero on a failing shard, # so a bare `if:` (implicitly success() && ...) would skip this step @@ -146,25 +154,48 @@ jobs: echo "No fast-check counterexample signature — infra failure, handled by the next step." exit 0 fi - TITLE="${TITLE_PREFIX} (seed=${FAIL_SEED})" + # Extract the SHRUNK counterexample block: the "Counterexample:" line(s) + # up to (but excluding) the "Shrunk N time(s)" / "Got error" line. This is + # the minimal failing INPUT and is STABLE across the different seeds/paths + # that reach the same bug — unlike the seed, path, or shrink count (which + # precede/follow this block and vary run-to-run) and unlike the whole + # output (which embeds those varying parts). Hashing THIS is what makes the + # dedup identity the bug itself rather than an incidental run detail. + CE_TEXT=$(awk '/Counterexample:/{c=1} /Shrunk [0-9]+ time|Got error/{c=0} c{print}' property-output.txt) + if [ -z "$CE_TEXT" ]; then + # No parseable shrunk block (unexpected — the signature check above + # already confirmed fast-check output). Fall back to the reproducing + # seed so we still emit a stable identity instead of silently deduping. + CE_TEXT="seed:${FAIL_SEED}" + fi + # Stable short id: first 12 hex chars of sha256 over the counterexample. + CE_HASH=$(printf '%s' "$CE_TEXT" | sha256sum | cut -c1-12) + # Machine-readable marker embedded in the issue body; the open-issue search + # below matches on it (and on the hash in the title) so identity travels + # with the issue regardless of any human title edits. + CE_MARKER="" + export CE_HASH CE_MARKER + TITLE="${TITLE_PREFIX} [${CE_HASH}] (seed=${FAIL_SEED})" - # Best-effort dedup: skip if an open issue with the counterexample title - # prefix already exists. A failure of this check must NOT block creation. + # Dedup on the counterexample hash: skip only if an OPEN issue already + # carries this exact hash (in its title or its body marker). A different + # counterexample has a different hash and is NOT deduped. A failure of this + # check must NOT block creation. EXISTING="" if EXISTING=$(curl -sS \ -H "Authorization: token ${GITHUB_TOKEN}" \ "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/issues?state=open&limit=100"); then if printf '%s' "$EXISTING" \ - | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{let a;try{a=JSON.parse(s)}catch{process.exit(1)}if(!Array.isArray(a))process.exit(1);const p=process.env.TITLE_PREFIX;process.exit(a.some(i=>typeof i.title==="string"&&i.title.startsWith(p))?0:1)})'; then - echo "An open '${TITLE_PREFIX}' issue already exists — skipping creation." + | node -e 'let s="";process.stdin.on("data",d=>s+=d).on("end",()=>{let a;try{a=JSON.parse(s)}catch{process.exit(1)}if(!Array.isArray(a))process.exit(1);const h=process.env.CE_HASH,m=process.env.CE_MARKER;process.exit(a.some(i=>(typeof i.title==="string"&&i.title.includes(h))||(typeof i.body==="string"&&i.body.includes(m)))?0:1)})'; then + echo "An open issue for counterexample ${CE_HASH} already exists — skipping creation." exit 0 fi fi # Build the JSON body with the test output SAFELY escaped (never hand- # interpolate the counterexample into JSON). - BODY_TEXT=$(printf 'A nightly property fuzz SHARD failed with a fast-check counterexample.\n\n- failing shard seed: `%s`\n- NUM_RUNS (per shard): `%s`\n- run: %s\n\nReproduce locally:\n\n```\nPROPERTY_SEED=%s PROPERTY_NUM_RUNS=%s pnpm --filter @docmost/prosemirror-markdown exec vitest run test/generative/\n```\n\nfast-check shrinks the failure to a minimal counterexample. Commit it as a permanent fixture under `packages/prosemirror-markdown/test/fixtures/counterexamples/` + a case in `counterexamples.test.ts`, then FIX the converter (do not weaken a property). See `packages/prosemirror-markdown/README.md`.\n\nTail of the test output (contains the shrunk counterexample):\n\n```\n%s\n```\n' \ - "$FAIL_SEED" "$NUM_RUNS" "$RUN_URL" "$FAIL_SEED" "$NUM_RUNS" "$(tail -n 120 property-output.txt)") + BODY_TEXT=$(printf 'A nightly property fuzz SHARD failed with a fast-check counterexample.\n\n- counterexample hash: `%s`\n- failing shard seed: `%s`\n- NUM_RUNS (per shard): `%s`\n- run: %s\n\nReproduce locally:\n\n```\nPROPERTY_SEED=%s PROPERTY_NUM_RUNS=%s pnpm --filter @docmost/prosemirror-markdown exec vitest run test/generative/\n```\n\nfast-check shrinks the failure to a minimal counterexample. Commit it as a permanent fixture under `packages/prosemirror-markdown/test/fixtures/counterexamples/` + a case in `counterexamples.test.ts`, then FIX the converter (do not weaken a property). See `packages/prosemirror-markdown/README.md`.\n\nTail of the test output (contains the shrunk counterexample):\n\n```\n%s\n```\n\n%s\n' \ + "$CE_HASH" "$FAIL_SEED" "$NUM_RUNS" "$RUN_URL" "$FAIL_SEED" "$NUM_RUNS" "$(tail -n 120 property-output.txt)" "$CE_MARKER") jq -n --arg title "$TITLE" --arg body "$BODY_TEXT" \ '{title: $title, body: $body}' > payload.json diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11cf4694..509ad6be 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,37 +25,65 @@ jobs: # filename sorts BEFORE migrations already applied on the target branch (and # thus in prod). The Kysely startup migrator rejects that as "corrupted # migrations" and crash-loops the app on boot (incident #361). This gate fails - # the PR so the migration is renamed to a current timestamp before merge. Only - # runs for pull_request events (needs a base branch to diff against). + # the PR so the migration is renamed to a current timestamp before merge. + # Runs for pull_request (diff against the base branch) AND for push (#476 + # retrospective: a DIRECT push to develop used to bypass this PR-only gate + # entirely — now the push is diffed against its `before` SHA; workflow_call + # from develop.yml inherits the caller's push event). workflow_dispatch has + # nothing to diff against and still skips the job. migration-order: - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' || github.event_name == 'push' runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Checkout (full history for the base-branch diff) + - name: Checkout (full history for the base diff) uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Added migrations must sort after the newest on the base branch + - name: Added migrations must sort after the newest on the base env: TARGET_BRANCH: ${{ github.base_ref }} + BEFORE_SHA: ${{ github.event.before }} run: | set -euo pipefail MIG_DIR="apps/server/src/database/migrations" - # checkout above already did fetch-depth:0 (full history). Fetch the base - # WITHOUT --depth (a shallow graft would truncate the base history and - # break the merge-base when the base has moved ahead of the PR merge — - # exactly the long-branch-vs-moving-base case this gate guards, #361). - git fetch --no-tags origin "$TARGET_BRANCH" - newest_on_target=$(git ls-tree -r --name-only "origin/${TARGET_BRANCH}" "$MIG_DIR" | sort | tail -1) + if [ "${GITHUB_EVENT_NAME}" = "pull_request" ]; then + # checkout above already did fetch-depth:0 (full history). Fetch the base + # WITHOUT --depth (a shallow graft would truncate the base history and + # break the merge-base when the base has moved ahead of the PR merge — + # exactly the long-branch-vs-moving-base case this gate guards, #361). + git fetch --no-tags origin "$TARGET_BRANCH" + BASE="origin/${TARGET_BRANCH}" + else + # push event: compare against the pre-push tip of the branch. + if [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then + echo "::notice::branch creation push — nothing to compare" + exit 0 + fi + if ! git cat-file -e "${BEFORE_SHA}^{commit}" 2>/dev/null; then + # The before-SHA is not in the clone (a force-push rewrote history). + # One recovery attempt — refresh every remote head (cheap: the + # checkout is already fetch-depth:0); a fetch failure aborts via + # `set -e`, which is fail-closed too. + git fetch --no-tags origin '+refs/heads/*:refs/remotes/origin/*' + fi + if ! git cat-file -e "${BEFORE_SHA}^{commit}" 2>/dev/null; then + # FAIL-CLOSED: without the before-SHA there is no base to prove the + # ordering against, and a gate whose job is to BLOCK must not guess. + echo "::error::force-push detected — verify migration order manually, then re-run via workflow_dispatch" + exit 1 + fi + BASE="$BEFORE_SHA" + fi + newest_on_target=$(git ls-tree -r --name-only "$BASE" "$MIG_DIR" | sort | tail -1) # NO `|| true`: a diff failure (e.g. an unresolved merge-base) must fail # the job CLOSED — a gate whose job is to BLOCK must never pass on error. # `set -e` above already aborts on a non-zero diff exit. - added=$(git diff --diff-filter=A --name-only "origin/${TARGET_BRANCH}...HEAD" -- "$MIG_DIR") + added=$(git diff --diff-filter=A --name-only "${BASE}...HEAD" -- "$MIG_DIR") bad=0 for f in $added; do if [[ "$f" < "$newest_on_target" || "$f" == "$newest_on_target" ]]; then - echo "::error::Migration $f sorts at or before the newest on ${TARGET_BRANCH} ($newest_on_target) — rename it with a CURRENT timestamp before merge (do not change its contents). See incident #361." + echo "::error::Migration $f sorts at or before the newest on the base ($newest_on_target) — rename it with a CURRENT timestamp before merge (do not change its contents). See incident #361." bad=1 fi done diff --git a/.gitignore b/.gitignore index 3eb7e75b..f3ce4f3e 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,10 @@ packages/mcp/build/ # is a build artifact like build/ — never committed, always fresh. packages/mcp/src/registry-stamp.generated.ts +# token-estimate compiled output (#490; built in CI/Docker via `pnpm build` / +# the server `pretest`, never committed, so src/ and prod can never diverge). +packages/token-estimate/dist/ + # Logs logs *.log diff --git a/AGENTS.md b/AGENTS.md index dd548743..5206ae90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -471,6 +471,8 @@ Vite SPA. Code is organized by feature under `apps/client/src/features/*` (mirro - The version string shown in the UI comes from `APP_VERSION` (CI/Docker) or `git describe --tags --always` (local), resolved in `vite.config.ts` — not from `package.json`. - Server TS config is permissive (`noImplicitAny: false`, `strictNullChecks: false`, `no-explicit-any` lint disabled). Follow the existing relaxed style rather than tightening types broadly. - Dependency versions are heavily pinned via `pnpm.overrides` and `pnpm.patchedDependencies` (`scimmy`, `yjs`, `ai`) in the root `package.json`. Don't bump pinned/patched deps casually; the patches and overrides exist for compatibility/security reasons. The `ai@6.0.134` patch carries TWO independent server fixes, each with its own tripwire test: (1) it disables the SDK's O(n²) cumulative `partialOutput` accumulation when no output strategy is requested (server heap OOM on long agent runs, #184; tripwire: `apps/server/src/integrations/ai/ai-sdk-partial-output.patch.spec.ts`); (2) it fixes `writeToServerResponse`'s drain-hang — the loop awaited only `"drain"` under backpressure, so a mid-write client disconnect parked the pipe forever and leaked the reader/buffers until restart; it now races `"drain"` against `"close"`/`"error"`, cancels the reader on disconnect, and swallows the fire-and-forget read rejection (#486; tripwire: `apps/server/src/integrations/ai/ai-sdk-drain-hang.patch.spec.ts`). Both tripwires assert BOTH installed dist builds carry their patch marker. The patch MUST be re-created via `pnpm patch` when bumping `ai`. + - **Upstream tracking (report the analysis upstream, don't just carry it):** both `ai` fixes and the hocuspocus one are candidates for upstreaming so we can eventually drop the local patch — the analysis is already written up in each patch's `PATCH(...)` header comments. File (a) an upstream **issue** on `vercel/ai` for the O(n²) cumulative `partialOutput` accumulation (heap OOM), (b) an upstream **issue** on `vercel/ai` for the `writeToServerResponse` drain-hang, and (c) an upstream **PR** on `@hocuspocus/server` for the connect-vs-unload race (local marker `PATCH(gitmost #401)` in `patches/@hocuspocus__server@3.4.4.patch`). Do NOT edit the patch files to add links — the patch bytes feed `patch_hash` in `pnpm-lock.yaml` (`ai@6.0.134` → `e8c599b3…`), so any content change there desyncs the lockfile pin and breaks `pnpm install`; keep upstream references here instead. + - **`ai` version is split across the monorepo and MUST be aligned deliberately, NOT casually:** the server pins `ai@6.0.134` (patched, exact — the `patchedDependencies` key forces that version), while the client declares `ai@6.0.207` (unpatched — the server-side `writeToServerResponse`/`partialOutput` fixes are dead code in the browser, so the mismatch is currently benign but is real drift). Alignment is a **planned, install-gated step**, never a bare `package.json` edit: (1) choose the target version; (2) re-create ALL THREE patch hunks (partialOutput publish-each, the `DefaultStreamTextResult` lazy-`output` wiring, and the drain-hang race) against the target dist via `pnpm patch` — the line offsets shift between versions, so the current patch WILL fail to apply as-is; (3) run a full `pnpm install` so the lockfile + new `patch_hash` regenerate together; (4) confirm both tripwire specs still find their markers. `pnpm install` FAILS HARD on an unapplied patch — that failure is the guardrail, so treat the port as a deliberate plan rather than discovering it as a deploy-time surprise. - **The MCP tool inventory in `SERVER_INSTRUCTIONS` is GENERATED from the registry** (`packages/mcp/src/server-instructions.ts`: `buildToolInventory()` over `SHARED_TOOL_SPECS`) and spliced into the hand-written routing prose (`ROUTING_PROSE`). So adding/renaming/removing a **shared** spec in `packages/mcp/src/tool-specs.ts` auto-updates the `` — no manual `SERVER_INSTRUCTIONS` edit needed. Only an **inline** MCP-only tool (those registered via `server.registerTool(...)` in `index.ts`, not through the registry) needs a one-line entry in `INLINE_MCP_INVENTORY`. Enforced by `packages/mcp/test/unit/tool-inventory.test.mjs`, which fails when a registered tool is missing from the generated inventory (there is no `EXCEPTIONS` opt-out anymore — every tool must appear). Update `ROUTING_PROSE` when a tool's *intent guidance* (when-to-use) changes. `packages/mcp/build/` is gitignored and rebuilt in CI/Docker via `pnpm build` (same convention as `git-sync`/`prosemirror-markdown`) — never commit it; rebuild locally after editing to run the tests. ## CI / release diff --git a/CHANGELOG.md b/CHANGELOG.md index adecf01f..fc7ae18d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -129,6 +129,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **A drifted comment suggestion can be re-synced instead of failing forever + with a 409.** A suggestion whose stored anchor no longer matched the live + document used to reject every apply attempt with an unrecoverable conflict; a + new resync path re-reads the live anchor so the suggestion applies against the + current text, and orphaned anchors (whose marked run was deleted) are + reconciled rather than left blocking. (#496) + - **Place several images side by side in a row.** A new "Inline (side by side)" alignment mode in the image bubble menu renders consecutive inline images as a row that wraps onto the next line on narrow screens. The row is @@ -304,6 +311,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 longer controls whether a turn is a run — it now governs **only** the browser-disconnect semantics (ON = detached/survives a disconnect; OFF = a disconnect stops the run). (#487) +- **Vendor `ai` patch: upstream-tracking + version-alignment plan documented.** + The two local `ai@6.0.134` fixes (O(n²) `partialOutput` heap-OOM; the + `writeToServerResponse` drain-hang) and the hocuspocus connect-vs-unload race + now have explicit upstream-reporting and `ai`-version-alignment steps recorded + in `AGENTS.md` (client `ai@6.0.207` vs server `ai@6.0.134`-patched drift). The + patch bytes are unchanged — they feed the lockfile `patch_hash`, so the + alignment is called out as an install-gated plan rather than a bare version + bump. No runtime change. - **Client markdown paste/copy and AI-chat rendering now go through the canonical converter.** Pasting markdown into the editor, "Copy as markdown", the AI title generator, and the AI-chat markdown renderer all now use @@ -336,6 +351,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **MCP write tools no longer report a false failure that provokes a duplicate + write.** `drawioCreate` used to throw when the diagram landed as a NESTED block + (anchored inside a callout or table cell) because there is no `#` handle + for it — but the diagram was already written, so a retry-prone agent re-created + it and produced a duplicate. It now returns success with `nodeId: null` plus a + warning that explains the write landed and how to re-read it (via + `getOutline` / `getPageJson` by `attachmentId`). Separately, when the live + collaboration-session cache hits its LRU entry cap, evicting a session whose + write is still in flight no longer rejects that write as a hard failure — it is + reported as INDETERMINATE ("the update may already have persisted; verify + before retry") so the agent re-reads instead of blind-retrying, and a + still-connecting session is no longer picked as an idle eviction victim by a + parallel acquire. (#494) +- **A long AI chat no longer bricks on the model's context window, and each turn + stops re-persisting the whole tool-output history.** Tool outputs are now + stored ONCE, in `metadata.parts`; the `tool_calls` trace keeps only per-step + outcome flags (a v2 trace shape), ending the O(N²) write amplification that + re-wrote every prior output on every step (measured on a live Postgres via the + `pg_current_wal_lsn()` delta: the trace column shrank ~3200×, the full + assistant row ~51%). The persisted record is unchanged in content — the full + history still lives in `metadata.parts`. At REPLAY time only, the history sent + to the provider is now bounded by a deterministic, prompt-cache-friendly token + budget: `floor(0.7 × chatContextWindow)` when a window is configured (no cap — + anti-brick protection, not a cost limiter), a flat 100k fallback for installs + with no window set (exactly the ones that hit terminal overflow), or off when + the window is explicitly `0`. Trimming truncates old tool outputs first, then + mechanically collapses the oldest turns, always keeping the recent turns full + and the tool-call/result pairing balanced. A provider context-overflow 400 is + now classified and used as a reactive signal: the row is stamped so the NEXT + turn re-trims aggressively (0.5×), which un-bricks a chat that just 400'd. The + client token badge and the server budgeter now share one estimator (new + `@docmost/token-estimate` package) so they can never diverge. Deferred-tool + activation is also cached in the chat metadata to avoid re-resolving it each + turn. (#490) - **A chat with one malformed message part no longer 500s on every turn, and a failed send no longer duplicates the user's message.** Incoming client parts are now whitelisted to `text` (a forged tool-result part can no longer reach @@ -352,7 +401,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 body-timeout so a legitimate >1-min idle between the model's tool calls no longer breaks a long-lived SSE socket (new `AI_MCP_SSE_BODY_TIMEOUT_MS`, default 10 min; see `.env.example`). (#489) - +- **Decisions on comment suggestions now leave a durable audit record.** + Applying or dismissing a comment suggestion hard-deletes the (childless) + subject comment, so the only surviving trace of who decided what is the audit + event — but the audit trail was wired to a Noop service that silently + swallowed every event. The trail is now DB-backed, so + `comment.suggestion_applied` / `comment.suggestion_dismissed` (and the other + comment-decision events) persist to the `audit` table and can be reviewed + after the comment is gone. A persistence failure is still swallowed with a + warning so it never breaks the originating request. (#496) +- **Applying a comment suggestion no longer strips the replaced run's inline + formatting.** The suggested text was re-inserted carrying only the comment + anchor mark, silently dropping bold/italic/code/link on the affected run; the + prevailing formatting of the replaced run is now carried onto the applied + text. (#496) +- **Markdown round-trips no longer silently drop a line that opens with a block + trigger.** When a document is exported to Markdown and re-imported (git-sync + stabilize, agent writes), a paragraph or continuation line (after a hard break) + that begins with a block marker — an ATX heading `#`, a blockquote/callout `>`, + a list marker (`-`/`*`/`+`/`N.`/`N)`), a code fence, a table `|`, a thematic + break (`---`), or a setext underline (`--`, `----`, or a lone `=`) — is now + backslash-escaped so it round-trips as text instead of being re-parsed into a + heading/list/quote/rule and losing its content. Front-matter stripping is + scoped to the import path only. (#493) - **The server no longer runs out of heap during long autonomous agent runs.** A new pnpm patch on `ai@6.0.134` stops the SDK from building a cumulative snapshot of the ENTIRE turn text on every streamed text-delta when no output @@ -469,6 +540,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 through that exact share (its own share or an ancestor `includeSubPages` share); any other value now returns the generic "not found" instead of serving the page. (#218) +- **MCP tool-allowlist semantics flipped: an empty `[]` now means deny-all + (previously it was coerced to "no restrictions").** For an external MCP server, + a stored `tool_allowlist` of `[]` now denies **every** tool of that server + (zero tools reach the agent) instead of being treated as an empty/unset filter + that allowed all of them. A corrupt or non-array stored value now **fails + closed** to deny-all rather than silently allowing everything. The admin form + no longer silently widens an existing deny-all server: leaving its tag field + empty preserves `[]` (deny-all) on save instead of NULL-ing the column to + allow-all, so a routine rename/toggle can no longer grant the agent every tool. + "No restrictions" is still expressible — a genuinely unrestricted server stores + NULL, and clearing the field on such a server keeps it NULL. Operationally + significant: audit any server that was created or left with a literal `[]`, as + it now exposes no tools until an explicit allowlist (or NULL) is set. (#476) - **Tool and provider error text no longer leaks to anonymous readers in the public-share AI chat.** A failing tool's raw error (which could carry an diff --git a/README.md b/README.md index bf6a46ae..07e9fde9 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,137 @@ start the new migrations apply on top of your existing schema (`CREATE EXTENSION existing pages are indexed on their next edit. pgvector is still required for the migration to apply at all. +## Local embeddings server + +The AI agent's semantic (RAG) search needs an **embeddings model**. Instead of paying a cloud +provider (e.g. OpenAI `text-embedding-3-*`) to embed every page, you can run a small open-weights +model yourself with Hugging Face +[Text Embeddings Inference](https://github.com/huggingface/text-embeddings-inference) (TEI), which +serves an OpenAI-compatible `/v1/embeddings` endpoint. `intfloat/multilingual-e5-small` is a good +default: multilingual, 384-dim, and comfortable on CPU (~1–2 GB RAM, 1–2 vCPU). Point Gitmost at it +under **Workspace settings → AI → Embeddings**. + +### Option A — local (same Docker network as Gitmost) + +Run TEI as a container on the network Gitmost is already on. The port is never published, so the +endpoint stays internal and needs no authentication. + +```yaml +services: + embeddings: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.9 # pin version; use a cuda-* tag for GPU + container_name: embeddings + restart: unless-stopped + networks: + - gitmost_net # same network Gitmost is on + command: + - "--model-id" + - "intfloat/multilingual-e5-small" + - "--auto-truncate" # clamp over-long inputs instead of returning 413 + volumes: + - tei-models:/data # weights are downloaded once and cached here + +networks: + gitmost_net: + external: true # the network Gitmost already uses + +volumes: + tei-models: +``` + +Gitmost settings (**Workspace settings → AI → Embeddings**): + +| Field | Value | +|-------------------|-----------------------------------| +| Model | `intfloat/multilingual-e5-small` | +| Base URL | `http://embeddings:80/v1/` | +| Embedding API key | — (leave empty) | + +> `embeddings` is the container name — Gitmost resolves it over DNS inside the Docker network. +> The port is not published, so the endpoint is reachable only by containers on that network and +> no authorization is required. + +### Option B — separate host (public via Traefik + Let's Encrypt) + +This assumes the host already runs Traefik with an ACME resolver (the example below uses +`letsEncrypt`, the `websecure` entrypoint and a shared `docker_main_net` network). Replace the +domain / network / resolver with your own. + +**DNS:** add an A record `embeddings.example.com` → the IP of your Traefik host (same +challenge / port 80 as the rest of your sites). + +```yaml +services: + embeddings: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.9 # pin version; cuda-* tag for GPU + container_name: embeddings + restart: unless-stopped + networks: + - docker_main_net # the network Traefik is attached to + command: + - "--model-id" + - "intfloat/multilingual-e5-small" + - "--auto-truncate" + - "--api-key" + - "sk-emb-REPLACE_WITH_YOUR_KEY" + volumes: + - tei-models:/data + labels: + traefik.enable: "true" + traefik.http.routers.embeddings.rule: "Host(`embeddings.example.com`)" + traefik.http.routers.embeddings.entrypoints: "websecure" + traefik.http.routers.embeddings.tls: "true" + traefik.http.routers.embeddings.tls.certresolver: "letsEncrypt" + traefik.http.routers.embeddings.service: "embeddings" + traefik.http.services.embeddings.loadbalancer.server.port: "80" + # TEI enforces the Bearer key itself; Traefik only rate-limits to protect the CPU + traefik.http.routers.embeddings.middlewares: "embeddings-rl" + traefik.http.middlewares.embeddings-rl.ratelimit.average: "20" + traefik.http.middlewares.embeddings-rl.ratelimit.burst: "40" + traefik.http.middlewares.embeddings-rl.ratelimit.period: "1s" + +networks: + docker_main_net: + external: true + +volumes: + tei-models: +``` + +Gitmost settings (**Workspace settings → AI → Embeddings**): + +| Field | Value | +|-------------------|---------------------------------------| +| Model | `intfloat/multilingual-e5-small` | +| Base URL | `https://embeddings.example.com/v1/` | +| Embedding API key | your `sk-emb-…` | + +Check it from outside: + +```bash +curl -s https://embeddings.example.com/v1/embeddings \ + -H "Authorization: Bearer sk-emb-REPLACE_WITH_YOUR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"intfloat/multilingual-e5-small","input":"query: hello"}' \ + | python3 -c 'import sys,json;print("dims:",len(json.load(sys.stdin)["data"][0]["embedding"]))' +# -> dims: 384 +``` + +### Embeddings server notes + +- **Vector dimension is 384.** If this Gitmost was previously embedded with a different model + (e.g. `text-embedding-3-large` = 3072-dim), the old pgvector rows won't match the new dimension — + clear the existing embeddings / re-index before switching. Gitmost only compares vectors of the + same dimension, so mixed-dimension rows are silently ignored rather than searched. +- **First start downloads the weights** (hundreds of MB) from `huggingface.co` into the + `tei-models` volume; every start after that reads from the volume. +- **Pin the version.** Pin the image, and optionally the model: add `--revision ` to + `command` (the sha is on the model's page on Hugging Face). +- **Air-gapped / no egress:** seed the `tei-models` volume ahead of time and add + `environment: [HF_HUB_OFFLINE=1]`. +- **GPU:** use the cuda tag of the same release (e.g. + `ghcr.io/huggingface/text-embeddings-inference:cuda-1.9`) and start the container with `gpus: all`. + ## Features - Real-time collaboration diff --git a/README.ru.md b/README.ru.md index 517c4802..13ee8498 100644 --- a/README.ru.md +++ b/README.ru.md @@ -193,6 +193,137 @@ dump/restore, существующий каталог данных переис > неизменным и бэкапьте вместе с базой данных. +## Локальный сервер эмбеддингов + +Семантическому (RAG) поиску AI-агента нужна **модель эмбеддингов**. Вместо оплаты облачного +провайдера (например, OpenAI `text-embedding-3-*`) за эмбеддинг каждой страницы можно запустить +небольшую open-weights модель у себя через Hugging Face +[Text Embeddings Inference](https://github.com/huggingface/text-embeddings-inference) (TEI) — он +отдаёт OpenAI-совместимый эндпоинт `/v1/embeddings`. Хороший дефолт — `intfloat/multilingual-e5-small`: +многоязычная, 384-мерная, комфортно работает на CPU (~1–2 ГБ RAM, 1–2 vCPU). Пропишите её в +**Настройки воркспейса → AI → Эмбеддинги**. + +### Вариант A — локально (та же Docker-сеть, что и Gitmost) + +Запустите TEI контейнером в той же сети, где уже работает Gitmost. Порт наружу не публикуется, +поэтому эндпоинт остаётся внутренним и не требует авторизации. + +```yaml +services: + embeddings: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.9 # pin version; use a cuda-* tag for GPU + container_name: embeddings + restart: unless-stopped + networks: + - gitmost_net # same network Gitmost is on + command: + - "--model-id" + - "intfloat/multilingual-e5-small" + - "--auto-truncate" # clamp over-long inputs instead of returning 413 + volumes: + - tei-models:/data # weights are downloaded once and cached here + +networks: + gitmost_net: + external: true # the network Gitmost already uses + +volumes: + tei-models: +``` + +Настройки Gitmost (**Настройки воркспейса → AI → Эмбеддинги**): + +| Поле | Значение | +|-------------------|-----------------------------------| +| Model | `intfloat/multilingual-e5-small` | +| Base URL | `http://embeddings:80/v1/` | +| Embedding API key | — (оставить пустым) | + +> `embeddings` — имя контейнера, Gitmost резолвит его по DNS внутри Docker-сети. +> Наружу порт не публикуется, эндпоинт доступен только контейнерам этой сети, поэтому +> авторизация не нужна. + +### Вариант B — на отдельном хосте (наружу через Traefik + Let's Encrypt) + +Предполагается, что на хосте уже есть Traefik с ACME-резолвером (в примере ниже — `letsEncrypt`, +entrypoint `websecure`, общая сеть `docker_main_net`). Замените домен / сеть / резолвер на свои. + +**DNS:** заведите A-запись `embeddings.example.com` → IP хоста с Traefik (тот же challenge / порт 80, +что и у остальных сайтов). + +```yaml +services: + embeddings: + image: ghcr.io/huggingface/text-embeddings-inference:cpu-1.9 # pin version; cuda-* tag for GPU + container_name: embeddings + restart: unless-stopped + networks: + - docker_main_net # the network Traefik is attached to + command: + - "--model-id" + - "intfloat/multilingual-e5-small" + - "--auto-truncate" + - "--api-key" + - "sk-emb-REPLACE_WITH_YOUR_KEY" + volumes: + - tei-models:/data + labels: + traefik.enable: "true" + traefik.http.routers.embeddings.rule: "Host(`embeddings.example.com`)" + traefik.http.routers.embeddings.entrypoints: "websecure" + traefik.http.routers.embeddings.tls: "true" + traefik.http.routers.embeddings.tls.certresolver: "letsEncrypt" + traefik.http.routers.embeddings.service: "embeddings" + traefik.http.services.embeddings.loadbalancer.server.port: "80" + # TEI enforces the Bearer key itself; Traefik only rate-limits to protect the CPU + traefik.http.routers.embeddings.middlewares: "embeddings-rl" + traefik.http.middlewares.embeddings-rl.ratelimit.average: "20" + traefik.http.middlewares.embeddings-rl.ratelimit.burst: "40" + traefik.http.middlewares.embeddings-rl.ratelimit.period: "1s" + +networks: + docker_main_net: + external: true + +volumes: + tei-models: +``` + +Настройки Gitmost (**Настройки воркспейса → AI → Эмбеддинги**): + +| Поле | Значение | +|-------------------|---------------------------------------| +| Model | `intfloat/multilingual-e5-small` | +| Base URL | `https://embeddings.example.com/v1/` | +| Embedding API key | ваш `sk-emb-…` | + +Проверка снаружи: + +```bash +curl -s https://embeddings.example.com/v1/embeddings \ + -H "Authorization: Bearer sk-emb-REPLACE_WITH_YOUR_KEY" \ + -H "Content-Type: application/json" \ + -d '{"model":"intfloat/multilingual-e5-small","input":"query: hello"}' \ + | python3 -c 'import sys,json;print("dims:",len(json.load(sys.stdin)["data"][0]["embedding"]))' +# -> dims: 384 +``` + +### Заметки про сервер эмбеддингов + +- **Размерность вектора — 384.** Если раньше этот Gitmost эмбеддился другой моделью + (например, `text-embedding-3-large` = 3072-dim), старые строки в pgvector не совпадут по + размерности — очистите существующие эмбеддинги / переиндексируйте перед переключением. Gitmost + сравнивает только вектора одной размерности, поэтому строки другой размерности не участвуют в + поиске, а не ломают его. +- **Первый старт тянет веса** (сотни МБ) с `huggingface.co` в том `tei-models`; дальше — из тома. +- **Пин версии.** Пиньте образ, а при желании и модель: добавьте в `command` `--revision ` + (sha берётся со страницы модели на Hugging Face). +- **Без egress (air-gapped):** засейте том `tei-models` заранее и добавьте + `environment: [HF_HUB_OFFLINE=1]`. +- **GPU:** возьмите cuda-тег того же релиза (например, + `ghcr.io/huggingface/text-embeddings-inference:cuda-1.9`) и запустите контейнер с `gpus: all`. + + ## Возможности - Совместная работа в реальном времени diff --git a/apps/client/package.json b/apps/client/package.json index a9fb48f7..00180fd4 100644 --- a/apps/client/package.json +++ b/apps/client/package.json @@ -22,6 +22,7 @@ "@casl/react": "5.0.1", "@docmost/editor-ext": "workspace:*", "@docmost/prosemirror-markdown": "workspace:*", + "@docmost/token-estimate": "workspace:*", "@excalidraw/excalidraw": "0.18.0-3a5ef40", "@mantine/core": "8.3.18", "@mantine/dates": "8.3.18", diff --git a/apps/client/public/locales/ru-RU/translation.json b/apps/client/public/locales/ru-RU/translation.json index ddde9041..9723e108 100644 --- a/apps/client/public/locales/ru-RU/translation.json +++ b/apps/client/public/locales/ru-RU/translation.json @@ -256,6 +256,9 @@ "Invite link": "Ссылка для приглашения", "Copy": "Копировать", "Copy to space": "Копировать в пространство", + "Copy chat": "Копировать чат", + "Dock to sidebar": "Закрепить в боковой панели", + "Undock": "Открепить", "Copied": "Скопировано", "Failed to export chat": "Не удалось экспортировать чат", "Duplicate": "Дублировать", @@ -285,6 +288,9 @@ "Alt text": "Альтернативный текст", "Describe this for accessibility.": "Опишите это для специальных возможностей.", "Add a description": "Добавить описание", + "Caption": "Подпись", + "Add a caption": "Добавить подпись", + "Shown below the image.": "Отображается под изображением.", "Justify": "По ширине", "Merge cells": "Объединить ячейки", "Split cell": "Разделить ячейку", @@ -388,22 +394,6 @@ "Quote": "Цитата", "Image": "Изображение", "Audio": "Аудио", - "Transcribe": "Транскрибировать", - "Transcribing…": "Транскрибация…", - "No speech detected": "Речь не распознана", - "Transcription failed": "Не удалось распознать речь", - "Voice dictation is not configured": "Голосовой ввод не настроен", - "Start dictation": "Начать диктовку", - "Stop recording": "Остановить запись", - "Microphone access denied": "Доступ к микрофону запрещён", - "No microphone found": "Микрофон не найден", - "Microphone is unavailable or already in use": "Микрофон недоступен или уже используется", - "Could not start recording": "Не удалось начать запись", - "Audio recording is not available in this browser/context": "Запись аудио недоступна в этом браузере/контексте", - "Dictation": "Диктовка", - "Dictation becomes available once the page finishes connecting": "Диктовка станет доступна после подключения к документу", - "No connection to the collaboration server — dictation unavailable": "Нет связи с сервером совместного редактирования — диктовка недоступна", - "This page is read-only": "Страница открыта только для чтения", "Embed PDF": "Встроить PDF", "Upload and embed a PDF file.": "Загрузите и встроите PDF-файл.", "Embed as PDF": "Встроить как PDF", @@ -419,9 +409,6 @@ "Footnote {{number}}": "Сноска {{number}}", "Go to footnote": "Перейти к сноске", "Back to reference": "Вернуться к ссылке", - "Back to references": "Вернуться к ссылкам", - "Back to reference {{label}}": "Вернуться к ссылке {{label}}", - "Empty footnote": "Пустая сноска", "Math inline": "Строчная формула", "Insert inline math equation.": "Вставить математическое выражение в строку.", "Math block": "Блок формулы", @@ -447,6 +434,9 @@ "{{count}} command available_other": "Доступно {{count}} команд", "{{count}} result available_one": "Доступен 1 результат", "{{count}} result available_other": "Доступно {{count}} результатов", + "{{count}} result found_one": "Найден {{count}} результат", + "{{count}} result found_few": "Найдено {{count}} результата", + "{{count}} result found_other": "Найдено {{count}} результатов", "Equal columns": "Равные столбцы", "Left sidebar": "Левая боковая панель", "Right sidebar": "Правая боковая панель", @@ -456,6 +446,7 @@ "Names do not match": "Названия не совпадают", "Today, {{time}}": "Сегодня, {{time}}", "Yesterday, {{time}}": "Вчера, {{time}}", + "now": "сейчас", "Space created successfully": "Пространство успешно создано", "Space updated successfully": "Пространство успешно обновлено", "Space deleted successfully": "Пространство успешно удалено", @@ -559,6 +550,7 @@ "Add 2FA method": "Добавить метод 2FA", "Backup codes": "Резервные коды", "Disable": "Отключить", + "disabled": "отключено", "Invalid verification code": "Недействительный код подтверждения", "New backup codes have been generated": "Новые резервные коды сгенерированы", "Failed to regenerate backup codes": "Не удалось заново сгенерировать резервные коды", @@ -702,62 +694,6 @@ "AI search": "Поиск ИИ", "AI Answer": "Ответ ИИ", "Ask AI": "Спросить ИИ", - "AI agent": "AI-агент", - "Take a look at the current document": "Посмотри текущий документ", - "Start automatically": "Запускать автоматически", - "When on, picking this role sends a launch message and starts the chat. When off, the role is selected and you type the first message yourself.": "Когда включено, выбор этой роли отправляет стартовое сообщение и начинает чат. Когда выключено, роль выбирается, а первое сообщение вы вводите сами.", - "Launch message": "Стартовое сообщение", - "Sent automatically when this role is picked. Leave empty to use the default text. Ignored when “Start automatically” is off.": "Отправляется автоматически при выборе этой роли. Оставьте пустым, чтобы использовать текст по умолчанию. Игнорируется, когда «Запускать автоматически» выключено.", - "AI agent is typing…": "AI-агент печатает…", - "{{name}} is typing…": "{{name}} печатает…", - "Thinking…": "Думаю…", - "Thinking… · {{count}} tokens": "Думаю… · {{count}} токенов", - "Thinking… · {{count}} tokens_one": "Думаю… · {{count}} токен", - "Thinking… · {{count}} tokens_few": "Думаю… · {{count}} токена", - "Thinking… · {{count}} tokens_many": "Думаю… · {{count}} токенов", - "Thinking · {{count}} tokens": "Размышления · {{count}} токенов", - "Thinking · {{count}} tokens_one": "Размышления · {{count}} токен", - "Thinking · {{count}} tokens_few": "Размышления · {{count}} токена", - "Thinking · {{count}} tokens_many": "Размышления · {{count}} токенов", - "Agent role": "Роль агента", - "AI chat": "AI-чат", - "AI chat is disabled for this workspace.": "AI-чат отключён для этого рабочего пространства.", - "Ask a question about this documentation.": "Задайте вопрос об этой документации.", - "Ask a question…": "Задайте вопрос…", - "Ask the AI agent anything about your workspace.": "Спросите AI-агента о чём угодно по вашему рабочему пространству.", - "Ask the AI agent…": "Спросите AI-агента…", - "Copy chat": "Копировать чат", - "Dock to sidebar": "Закрепить в боковой панели", - "Undock": "Открепить", - "Created successfully": "Успешно создано", - "Context size / model limit": "Размер контекста / лимит модели", - "Context window (tokens)": "Окно контекста (токены)", - "Shown as used / total in the chat header. Leave empty to hide the limit.": "Показывается в шапке чата как использовано / всего. Пусто — лимит скрыт.", - "Delete this chat?": "Удалить этот чат?", - "Deleted successfully": "Успешно удалено", - "AI agent «{{role}}» on behalf of {{person}}": "AI-агент «{{role}}» от имени {{person}}", - "AI agent {{name}}": "AI-агент {{name}}", - "Failed to delete chat": "Не удалось удалить чат", - "Failed to rename chat": "Не удалось переименовать чат", - "Failed": "Ошибка", - "OK · {{n}}": "OK · {{n}}", - "Test": "Тест", - "No tools available": "Инструменты недоступны", - "Available tools": "Доступные инструменты", - "Minimize": "Свернуть", - "No chats yet.": "Чатов пока нет.", - "Send": "Отправить", - "Send when the agent finishes": "Отправить, когда агент закончит", - "Queue message": "Поставить в очередь", - "Remove queued message": "Убрать из очереди", - "Send now": "Отправить сейчас", - "Interrupt and send now": "Прервать и отправить сейчас", - "Something went wrong": "Что-то пошло не так", - "Stop": "Стоп", - "The AI agent could not respond. Please try again.": "AI-агент не смог ответить. Попробуйте ещё раз.", - "The AI provider is not configured. Ask an administrator to set it up.": "AI-провайдер не настроен. Попросите администратора настроить его.", - "Universal assistant": "Универсальный ассистент", - "You": "Вы", "AI is thinking...": "ИИ обрабатывает запрос...", "Thinking": "Думаю", "Ask a question...": "Задайте вопрос...", @@ -784,8 +720,40 @@ "Manage API keys for all users in the workspace. View the API documentation for usage details.": "Управляйте API-ключами для всех пользователей в рабочем пространстве. Смотрите документацию по API для получения информации об использовании.", "View the API documentation for usage details.": "Смотрите документацию по API для получения информации об использовании.", "View the MCP documentation.": "Смотрите документацию по MCP.", - "Instructions": "Инструкции", + "AI / Models": "ИИ / Модели", + "AI / External tools (MCP)": "ИИ / Внешние инструменты (MCP)", + "Add server": "Добавить сервер", + "Edit server": "Изменить сервер", + "Delete server": "Удалить сервер", + "Are you sure you want to delete this MCP server?": "Вы уверены, что хотите удалить этот MCP-сервер?", + "No external servers configured": "Внешние серверы не настроены", + "Server name": "Имя сервера", + "Transport": "Транспорт", + "URL": "URL", + "Authorization header": "Заголовок авторизации", + "Tool allowlist": "Список разрешённых инструментов", + "Optional. Leave empty to allow all tools the server exposes.": "Необязательно. Оставьте пустым, чтобы разрешить все инструменты, которые предоставляет сервер.", "Optional guidance for the agent on how and when to use this server's tools. Injected into the system prompt. The server's tools are namespaced as \"_*\".": "Необязательное указание агенту, как и когда использовать инструменты этого сервера. Добавляется в системный промпт. Инструменты сервера именуются с префиксом «<имя сервера>_*».", + "Test": "Тест", + "Available tools": "Доступные инструменты", + "No tools available": "Инструменты недоступны", + "Failed": "Ошибка", + "OK · {{n}}": "OK · {{n}}", + "Created successfully": "Успешно создано", + "Deleted successfully": "Успешно удалено", + "Clear": "Очистить", + "Provider": "Провайдер", + "•••• set": "•••• задан", + "Clear key": "Очистить ключ", + "Base URL": "Базовый URL", + "Chat model": "Модель чата", + "Embedding model": "Модель эмбеддингов", + "System message": "Системное сообщение", + "A built-in safety framework is always appended.": "Встроенный набор правил безопасности всегда добавляется автоматически.", + "Test connection": "Проверить соединение", + "Connection successful": "Соединение установлено", + "Connection failed": "Не удалось установить соединение", + "Only workspace admins can manage AI provider settings.": "Управлять настройками провайдера ИИ могут только администраторы рабочего пространства.", "Sources": "Источники", "AI Answers not available for attachments": "Ответы ИИ недоступны для вложений", "No answer available": "Ответ недоступен", @@ -1013,6 +981,7 @@ "Try again": "Попробовать снова", "Untitled chat": "Чат без названия", "No document": "Без документа", + "You": "Вы", "What can I help you with?": "Чем я могу вам помочь?", "Are you sure you want to revoke this {{credential}}": "Вы уверены, что хотите отозвать этот {{credential}}", "Automatically provision users and groups from your identity provider via SCIM.": "Автоматически предоставляйте доступ пользователям и группам из вашего провайдера удостоверений через SCIM.", @@ -1041,6 +1010,9 @@ "Page menu": "Меню страницы", "Expand": "Развернуть", "Collapse": "Свернуть", + "Expand all": "Развернуть все", + "Collapse all": "Свернуть все", + "Couldn't expand the tree: {{reason}}": "Не удалось развернуть дерево: {{reason}}", "Comment menu": "Меню комментария", "Group menu": "Меню группы", "Show hidden breadcrumbs": "Показать скрытые хлебные крошки", @@ -1077,7 +1049,7 @@ "Search pages and spaces...": "Поиск страниц и пространств...", "No results found": "Результаты не найдены", "You don't have permission to create pages here": "У вас нет прав на создание страниц здесь", - "Chat menu": "Меню чата", + "Chat menu for {{title}}": "Меню чата для {{title}}", "API key menu": "Меню API-ключа", "Jump to comment selection": "Перейти к выбору комментария", "Slash commands": "Команды со слешем", @@ -1131,6 +1103,9 @@ "Undo": "Отменить", "Redo": "Повторить", "Backlinks": "Обратные ссылки", + "Back to references": "Вернуться к ссылкам", + "Back to reference {{label}}": "Вернуться к ссылке {{label}}", + "Empty footnote": "Пустая сноска", "Last updated by": "Последний изменивший", "Last updated": "Последнее обновление", "Stats": "Статистика", @@ -1164,6 +1139,7 @@ "Page title": "Заголовок страницы", "Page content": "Содержимое страницы", "Member actions": "Действия с участником", + "Member actions for {{name}}": "Действия с участником {{name}}", "Toggle password visibility": "Переключить видимость пароля", "Send comment": "Отправить комментарий", "Token actions": "Действия с токеном", @@ -1183,11 +1159,187 @@ "Removed from favorites": "Удалено из избранного", "Added {{name}} to favorites": "{{name}} добавлено в избранное", "Removed {{name}} from favorites": "{{name}} удалено из избранного", + "Label added": "Метка добавлена", + "Label removed": "Метка удалена", + "Image updated": "Изображение обновлено", + "Unsupported image type": "Неподдерживаемый тип изображения", + "Member deactivated": "Участник деактивирован", + "Member activated": "Участник активирован", + "Name is required": "Укажите имя", + "Name must be 40 characters or fewer": "Имя должно содержать не более 40 символов", + "Group name must be at least 2 characters": "Название группы должно содержать не менее 2 символов", + "Group name must be 100 characters or fewer": "Название группы должно содержать не более 100 символов", + "Description must be 500 characters or fewer": "Описание должно содержать не более 500 символов", + "Invalid invitation link": "Недействительная ссылка-приглашение", "Page menu for {{name}}": "Меню страницы для {{name}}", "Create subpage of {{name}}": "Создать подстраницу для {{name}}", + "AI chat": "AI-чат", + "Ask a question about this documentation.": "Задайте вопрос об этой документации.", + "Ask a question…": "Задайте вопрос…", + "Thinking…": "Думаю…", + "Thinking… · {{count}} tokens": "Думаю… · {{count}} токенов", + "Thinking… · {{count}} tokens_one": "Думаю… · {{count}} токен", + "Thinking… · {{count}} tokens_few": "Думаю… · {{count}} токена", + "Thinking… · {{count}} tokens_many": "Думаю… · {{count}} токенов", + "Thinking… · {{count}} tokens_other": "Думаю… · {{count}} токенов", + "Thinking · {{count}} tokens": "Размышления · {{count}} токенов", + "Thinking · {{count}} tokens_one": "Размышления · {{count}} токен", + "Thinking · {{count}} tokens_few": "Размышления · {{count}} токена", + "Thinking · {{count}} tokens_many": "Размышления · {{count}} токенов", + "Thinking · {{count}} tokens_other": "Размышления · {{count}} токенов", + "The assistant is unavailable right now. Please try again.": "Ассистент сейчас недоступен. Попробуйте ещё раз.", + "Public share assistant": "Ассистент публичного доступа", + "Let anonymous visitors of public shares ask an AI assistant scoped to that share's pages. You pay for the tokens.": "Позвольте анонимным посетителям публичных ссылок обращаться к ИИ-ассистенту в рамках страниц этой публикации. Токены оплачиваете вы.", + "Public assistant model": "Модель публичного ассистента", + "Defaults to the chat model": "По умолчанию используется модель чата", + "Optional cheaper model id for the public assistant. Empty uses the chat model above.": "Необязательный более дешёвый идентификатор модели для публичного ассистента. Если пусто, используется модель чата выше.", + "Assistant identity": "Личность ассистента", + "Pick an agent role whose persona the public assistant adopts. The safety rules always still apply.": "Выберите роль агента, чью личность примет публичный ассистент. Правила безопасности всегда остаются в силе.", + "Built-in assistant persona": "Встроенная личность ассистента", + "Minimize": "Свернуть", + "Context size / model limit": "Размер контекста / лимит модели", + "Context window (tokens)": "Окно контекста (токены)", + "Shown as used / total in the chat header. Leave empty to hide the limit.": "Показывается в шапке чата как использовано / всего. Пусто — лимит скрыт.", + "AI agent": "AI-агент", + "Take a look at the current document": "Посмотри текущий документ", + "AI agent is typing…": "AI-агент печатает…", + "{{name}} is typing…": "{{name}} печатает…", + "Send": "Отправить", + "Send when the agent finishes": "Отправить, когда агент закончит", + "Queue message": "Поставить в очередь", + "Remove queued message": "Убрать из очереди", + "Send now": "Отправить сейчас", + "Interrupt and send now": "Прервать и отправить сейчас", + "Stop": "Стоп", + "Response stopped.": "Ответ остановлен.", + "Connection lost — the answer was interrupted.": "Соединение потеряно — ответ был прерван.", + "Response stopped (manually or the connection dropped).": "Ответ остановлен (вручную или из-за разрыва соединения).", + "Chat menu": "Меню чата", + "No chats yet.": "Чатов пока нет.", + "Delete this chat?": "Удалить этот чат?", + "Ask the AI agent…": "Спросите AI-агента…", + "Ask the AI agent anything about your workspace.": "Спросите AI-агента о чём угодно по вашему рабочему пространству.", + "Failed to rename chat": "Не удалось переименовать чат", + "Failed to delete chat": "Не удалось удалить чат", + "Something went wrong": "Что-то пошло не так", + "AI chat is disabled for this workspace.": "AI-чат отключён для этого рабочего пространства.", + "The AI provider is not configured. Ask an administrator to set it up.": "AI-провайдер не настроен. Попросите администратора настроить его.", + "The AI agent could not respond. Please try again.": "AI-агент не смог ответить. Попробуйте ещё раз.", + "Searched pages": "Поиск по страницам", + "Read page": "Прочитана страница", + "Created page": "Создана страница", + "Updated page": "Обновлена страница", + "Renamed page": "Переименована страница", + "Moved page": "Перемещена страница", + "Deleted page (to trash)": "Удалена страница (в корзину)", + "Commented": "Добавлен комментарий", + "Resolved comment": "Комментарий решён", + "Ran tool {{name}}": "Выполнен инструмент {{name}}", + "AI agent «{{role}}» on behalf of {{person}}": "AI-агент «{{role}}» от имени {{person}}", + "AI agent {{name}}": "AI-агент {{name}}", + "Endpoints": "Эндпоинты", + "where we fetch models": "откуда мы получаем модели", + "All endpoints are OpenAI-compatible. Point the Base URL at OpenAI, OpenRouter, a local Ollama, or any self-hosted server.": "Все эндпоинты совместимы с OpenAI. Укажите в базовом URL адрес OpenAI, OpenRouter, локального Ollama или любого self-hosted сервера.", + "Chat / LLM": "Чат / LLM", + "root": "корневой", + "Semantic search": "Семантический поиск", + "Voice / STT": "Голос / STT", + "Voice dictation": "Голосовой ввод", + "Streaming dictation": "Потоковый голосовой ввод", + "Transcribe as you speak, cutting on pauses": "Транскрибирование по мере речи, с разбивкой на паузах", + "Voice dictation is not available yet.": "Голосовой ввод пока недоступен.", + "Test endpoint": "Проверить эндпоинт", + "Save and test": "Сохранить и проверить", + "Save endpoints": "Сохранить эндпоинты", + "Configured and enabled": "Настроено и включено", + "Configured but disabled": "Настроено, но отключено", + "Enabled but not configured": "Включено, но не настроено", + "Not configured": "Не настроено", + "External tools": "Внешние инструменты", + "Gitmost as MCP client": "Gitmost как MCP-клиент", + "Servers the agent calls out to.": "Серверы, к которым обращается агент.", + "MCP server": "MCP-сервер", + "expose the workspace": "открыть доступ к рабочему пространству", + "Enable MCP server": "Включить MCP-сервер", + "Exposes the workspace as an MCP server at /mcp — this provides a capability, it doesn't consume a model.": "Открывает рабочее пространство как MCP-сервер по адресу /mcp — это предоставляет возможность, а не потребляет модель.", + "Resolves to {{url}}": "Разрешается в {{url}}", + "Model": "Модель", + "Done": "Готово", + "shared prompt · safety framework appended automatically": "общий промпт · правила безопасности добавляются автоматически", + "/v1/chat/completions · root endpoint — Embeddings and Voice inherit its URL and key": "/v1/chat/completions · корневой эндпоинт — Эмбеддинги и Голос наследуют его URL и ключ", + "/v1/embeddings · embeds pages so semantic search can find them": "/v1/embeddings · создаёт эмбеддинги страниц, чтобы их находил семантический поиск", + "/v1/audio/transcriptions · works with local whisper (speaches / faster-whisper-server)": "/v1/audio/transcriptions · работает с локальным whisper (speaches / faster-whisper-server)", + "Vector search · requires pgvector": "Векторный поиск · требуется pgvector", + "Embedding API key": "API-ключ для эмбеддингов", + "Embeddings": "Эмбеддинги", + "Leave empty to use the chat API key": "Оставьте пустым, чтобы использовать API-ключ чата", + "Leave empty to use the chat base URL": "Оставьте пустым, чтобы использовать базовый URL чата", + "Reindex now": "Переиндексировать сейчас", + "Start dictation": "Начать диктовку", + "Stop recording": "Остановить запись", + "Transcribing…": "Транскрибация…", + "Microphone access denied": "Доступ к микрофону запрещён", + "No microphone found": "Микрофон не найден", + "Could not start recording": "Не удалось начать запись", + "Transcription failed": "Не удалось распознать речь", + "Transcribe": "Транскрибировать", + "No speech detected": "Речь не распознана", + "Voice dictation is not configured": "Голосовой ввод не настроен", + "Microphone is unavailable or already in use": "Микрофон недоступен или уже используется", + "Audio recording is not available in this browser/context": "Запись аудио недоступна в этом браузере/контексте", + "Dictation": "Диктовка", + "Dictation becomes available once the page finishes connecting": "Диктовка станет доступна после подключения к документу", + "No connection to the collaboration server — dictation unavailable": "Нет связи с сервером совместного редактирования — диктовка недоступна", + "This page is read-only": "Страница открыта только для чтения", + "Request format": "Формат запроса", + "How transcription requests are sent to the endpoint": "Как запросы на транскрибирование отправляются на эндпоинт", + "OpenAI-compatible (multipart/form-data)": "Совместимо с OpenAI (multipart/form-data)", + "OpenRouter (JSON, base64 audio)": "OpenRouter (JSON, аудио в base64)", "Dictation language": "Язык диктовки", "Auto-detect": "Автоопределение", "Spoken language hint sent to the transcription model. Auto-detect lets the model decide.": "Подсказка языка речи для модели транскрипции. «Автоопределение» оставляет выбор за моделью.", + "Agent role": "Роль агента", + "Universal assistant": "Универсальный ассистент", + "Add role": "Добавить роль", + "Edit role": "Изменить роль", + "Role name": "Название роли", + "e.g. Proofreader": "напр. Корректор", + "Optional. Shown as the chat badge.": "Необязательно. Отображается как значок чата.", + "Optional. A short note about what this role does.": "Необязательно. Краткое описание того, что делает эта роль.", + "Instructions": "Инструкции", + "The built-in safety framework is always added automatically.": "Встроенный набор правил безопасности всегда добавляется автоматически.", + "Model provider override": "Переопределение провайдера модели", + "Optional. Defaults to the workspace provider.": "Необязательно. По умолчанию используется провайдер рабочего пространства.", + "Model override": "Переопределение модели", + "Optional. Defaults to the workspace model.": "Необязательно. По умолчанию используется модель рабочего пространства.", + "e.g. gpt-4o-mini": "напр. gpt-4o-mini", + "If you choose a different provider, it must already be configured in AI settings.": "Если вы выбираете другого провайдера, он уже должен быть настроен в настройках ИИ.", + "Start automatically": "Запускать автоматически", + "When on, picking this role sends a launch message and starts the chat. When off, the role is selected and you type the first message yourself.": "Когда включено, выбор этой роли отправляет стартовое сообщение и начинает чат. Когда выключено, роль выбирается, а первое сообщение вы вводите сами.", + "Launch message": "Стартовое сообщение", + "Sent automatically when this role is picked. Leave empty to use the default text. Ignored when “Start automatically” is off.": "Отправляется автоматически при выборе этой роли. Оставьте пустым, чтобы использовать текст по умолчанию. Игнорируется, когда «Запускать автоматически» выключено.", + "Agent roles": "Роли агента", + "Reusable presets that shape the agent's behavior (and optionally its model). Picked when starting a new chat.": "Многоразовые пресеты, определяющие поведение агента (и, при желании, его модель). Выбираются при запуске нового чата.", + "No roles configured": "Роли не настроены", + "Delete role": "Удалить роль", + "Are you sure you want to delete this role?": "Вы уверены, что хотите удалить эту роль?", + "HTML embed": "HTML-вставка", + "Edit HTML embed": "Изменить HTML-вставку", + "HTML embed is disabled in this workspace": "HTML-вставки отключены в этом рабочем пространстве", + "Click to add HTML / CSS / JS": "Нажмите, чтобы добавить HTML / CSS / JS", + "This HTML/CSS/JS runs in a sandboxed frame and cannot access the viewer's session, cookies, or API.": "Этот HTML/CSS/JS выполняется в изолированном фрейме и не имеет доступа к сессии, cookie или API просматривающего.", + "": "", + "Height (px, blank = auto)": "Высота (px, пусто = авто)", + "advanced": "дополнительно", + "Enable HTML embed": "Включить HTML-вставки", + "Allow members to insert raw HTML/CSS/JavaScript blocks. The block renders in a sandboxed frame and cannot access the viewer's session, cookies, or API. Off by default.": "Разрешить участникам вставлять блоки с необработанным HTML/CSS/JavaScript. Блок отображается в изолированном фрейме и не имеет доступа к сессии, cookie или API просматривающего. По умолчанию выключено.", + "When enabled, any member can insert an HTML embed block. The toggle just enables or disables the block type workspace-wide.": "Когда включено, любой участник может вставить блок HTML-вставки. Переключатель просто включает или отключает этот тип блока во всём рабочем пространстве.", + "Embeds run inside a sandboxed iframe with a separate origin, so they cannot read or modify the page they are embedded in.": "Вставки выполняются в изолированном iframe с отдельным источником, поэтому они не могут читать или изменять страницу, в которую встроены.", + "Turning this off hides existing embeds (they render as a disabled placeholder) and stops serving them on public share pages.": "Отключение этой опции скрывает существующие вставки (они отображаются как отключённая заглушка) и прекращает их показ на публичных страницах.", + "Analytics / tracker": "Аналитика / трекер", + "Injected verbatim into the of PUBLIC SHARE pages only (same-origin). For analytics snippets (Google Analytics, Yandex.Metrika, etc.). Admin only.": "Вставляется дословно в только ПУБЛИЧНЫХ страниц (тот же источник). Для сниппетов аналитики (Google Analytics, Яндекс.Метрика и т. п.). Только для администраторов.", + "Go to login page": "Перейти на страницу входа", + "Move to space": "Переместить в пространство", "Float left (wrap text)": "Обтекание слева", "Float right (wrap text)": "Обтекание справа", "Inline (side by side)": "В ряд", @@ -1199,6 +1351,7 @@ "Showing {{count}} subpages_one": "Показано {{count}} подстраница", "Showing {{count}} subpages_few": "Показано {{count}} подстраницы", "Showing {{count}} subpages_many": "Показано {{count}} подстраниц", + "Showing {{count}} subpages_other": "Показано {{count}} подстраниц", "Protocol": "Протокол", "How chat requests are sent and how reasoning is surfaced": "Как отправляются запросы чата и как показывается reasoning", "OpenAI-compatible (surfaces reasoning)": "OpenAI-совместимый (показывает reasoning)", @@ -1268,7 +1421,6 @@ "Retry": "Повторить", "The catalog is empty": "Каталог пуст", "No role bundles are published for this language yet. Try switching the content language.": "Для этого языка ещё не опубликовано ни одного набора ролей. Попробуйте сменить язык контента.", - "No roles configured": "Роли не настроены", "Already up to date": "Уже актуальна", "Updated to the latest version": "Обновлено до последней версии", "This role is no longer in the catalog": "Эта роль больше не представлена в каталоге", diff --git a/apps/client/src/components/chunk-load-error-boundary.test.ts b/apps/client/src/components/chunk-load-error-boundary.test.ts index 78ecce6b..a01e4c1b 100644 --- a/apps/client/src/components/chunk-load-error-boundary.test.ts +++ b/apps/client/src/components/chunk-load-error-boundary.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { isChunkLoadError } from "./chunk-load-error-boundary"; +import { isChunkLoadError, shouldAutoReload } from "./chunk-load-error-boundary"; // The detector decides whether a caught render error is a stale-deploy chunk-404 // (→ auto-reload to fetch the new manifest) vs a genuine app error (→ generic @@ -35,3 +35,31 @@ describe("isChunkLoadError", () => { expect(isChunkLoadError(err)).toBe(false); }); }); + +// The window gate replaces the old one-shot flag: it must permit recovery across +// several deploys in one tab (each > window apart) while still stopping an infinite +// reload loop when a lazy chunk is permanently broken (a second failure < window). +describe("shouldAutoReload", () => { + const WINDOW = 5 * 60 * 1000; + const NOW = 1_000_000_000_000; + + it("allows a reload when we have never auto-reloaded", () => { + expect(shouldAutoReload(NOW, null, WINDOW)).toBe(true); + }); + + it("allows a reload when the last one was 6 minutes ago (outside the window)", () => { + expect(shouldAutoReload(NOW, NOW - 6 * 60 * 1000, WINDOW)).toBe(true); + }); + + it("blocks a reload when the last one was 1 minute ago (inside the window)", () => { + expect(shouldAutoReload(NOW, NOW - 1 * 60 * 1000, WINDOW)).toBe(false); + }); + + it("blocks a reload exactly at the window boundary (not strictly older)", () => { + expect(shouldAutoReload(NOW, NOW - WINDOW, WINDOW)).toBe(false); + }); + + it("allows a reload when the stored timestamp is unparseable (NaN)", () => { + expect(shouldAutoReload(NOW, NaN, WINDOW)).toBe(true); + }); +}); diff --git a/apps/client/src/components/chunk-load-error-boundary.tsx b/apps/client/src/components/chunk-load-error-boundary.tsx index c3e39a00..e28dfe26 100644 --- a/apps/client/src/components/chunk-load-error-boundary.tsx +++ b/apps/client/src/components/chunk-load-error-boundary.tsx @@ -2,7 +2,25 @@ import { ReactNode } from "react"; import { ErrorBoundary } from "react-error-boundary"; import { Button, Center, Stack, Text } from "@mantine/core"; -const RELOAD_FLAG = "chunk-reload-attempted"; +// sessionStorage key holding the epoch-ms timestamp of the last automatic reload. +const RELOAD_AT_KEY = "chunk-reload-at"; +// Allow at most one automatic reload per this window. A stale-deploy 404 is cured +// by a single reload, so anything inside the window is treated as a reload loop +// (permanently-broken chunk) and falls through to the manual UI. A window (rather +// than a one-shot flag) lets a SECOND deploy in the same tab's lifetime recover too. +const RELOAD_WINDOW_MS = 5 * 60 * 1000; + +// Pure window decision, unit-tested in isolation: auto-reload only if we have never +// auto-reloaded (lastReloadAt null/NaN) or the last one was strictly older than the +// window. Anything inside the window is suppressed to break an infinite reload loop. +export function shouldAutoReload( + now: number, + lastReloadAt: number | null, + windowMs: number, +): boolean { + if (lastReloadAt === null || Number.isNaN(lastReloadAt)) return true; + return now - lastReloadAt > windowMs; +} // Heuristic detection of a failed dynamic import. Since the code-splitting work, // every route (plus Aside / AiChatWindow) is React.lazy: when a new deploy @@ -24,12 +42,16 @@ export function isChunkLoadError(error: unknown): boolean { function handleError(error: unknown) { if (!isChunkLoadError(error)) return; // A stale-chunk 404 is cured by a full reload that re-fetches index.html and - // the new chunk manifest. Auto-reload once, guarding against a reload loop - // (e.g. a genuinely missing chunk) with a one-shot sessionStorage flag. If the - // flag is already set we fall through to the manual recovery UI below. + // the new chunk manifest. Auto-reload at most once per RELOAD_WINDOW_MS: this + // recovers across multiple deploys in a single tab's lifetime, yet a + // permanently-broken lazy chunk (which would loop) is stopped after the first + // reload and falls through to the manual recovery UI below. try { - if (sessionStorage.getItem(RELOAD_FLAG)) return; - sessionStorage.setItem(RELOAD_FLAG, "1"); + const raw = sessionStorage.getItem(RELOAD_AT_KEY); + const lastReloadAt = raw === null ? null : Number.parseInt(raw, 10); + const now = Date.now(); + if (!shouldAutoReload(now, lastReloadAt, RELOAD_WINDOW_MS)) return; + sessionStorage.setItem(RELOAD_AT_KEY, String(now)); } catch { // sessionStorage unavailable (private mode / disabled): skip the automatic // reload rather than risk an unguarded loop; the fallback UI still recovers. diff --git a/apps/client/src/features/ai-chat/components/ai-chat-window.tsx b/apps/client/src/features/ai-chat/components/ai-chat-window.tsx index 4b8b07c0..dbb90e53 100644 --- a/apps/client/src/features/ai-chat/components/ai-chat-window.tsx +++ b/apps/client/src/features/ai-chat/components/ai-chat-window.tsx @@ -58,8 +58,11 @@ import ConversationList from "@/features/ai-chat/components/conversation-list.ts import ChatThread from "@/features/ai-chat/components/chat-thread.tsx"; import { exportAiChat, + getAiChatMessagesDelta, stopRun, } from "@/features/ai-chat/services/ai-chat-service.ts"; +import { mergeDeltaRowsIntoPages } from "@/features/ai-chat/utils/resume-helpers.ts"; +import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.ts"; import { useChatSession } from "@/features/ai-chat/hooks/use-chat-session.ts"; import { shouldCollapseOnOutsidePointer, @@ -269,17 +272,64 @@ export default function AiChatWindow() { const { data: messageRows, isLoading: messagesLoading } = useAiChatMessagesQuery( activeChatId ?? undefined, - // DELIBERATELY DUMB: poll every 2.5s WHILE ARMED, otherwise off. NO error - // checks (TanStack resets fetchFailureCount each fetch; the poll must survive - // a server restart), NO tail checks, NO cap here — the settled/stalled/idle-cap - // semantics all live in ChatThread's FSM, which disarms via onResumeFallback. - () => (degradedPoll === true ? 2500 : false), - // #344: gate on windowOpen too — no message history is fetched (and no - // degraded poll runs) while the window is closed; it loads when the window - // opens with an active chat. + // #491: the full infinite-query no longer POLLS. It seeds the thread ONCE; the + // degraded fallback now runs a DELTA poller (below) that augments THIS cache + // idempotently, instead of refetching every page (with full parts) every 2.5s. + false, + // #344: gate on windowOpen too — no message history is fetched while the window + // is closed; it loads when the window opens with an active chat. windowOpen, ); + // #491 degraded DELTA poll. While armed (degradedPoll) and the window is open on a + // chat, poll POST /ai-chat/messages/delta every 2.5s: it returns only the rows + // CHANGED since the previous cursor (+ the run fact) in ONE round-trip. We merge + // those rows into the SAME infinite-query cache the thread reads (idempotently by + // id — the delta's overlap window re-delivers rows), so the thread's reconcile + // effect follows the detached run to its terminal row from a fraction of the wire + // cost. The run-fact settle stays the thread FSM's job (row-status reconcile), so + // we do NOT double-poll /run here. Cursor resets when the chat changes / disarms. + const deltaCursorRef = useRef(undefined); + useEffect(() => { + deltaCursorRef.current = undefined; + }, [activeChatId, degradedPoll]); + useEffect(() => { + if (!degradedPoll || !windowOpen || !activeChatId) return; + const chatId = activeChatId; + let cancelled = false; + const tick = async (): Promise => { + try { + const res = await getAiChatMessagesDelta(chatId, deltaCursorRef.current); + if (cancelled) return; + deltaCursorRef.current = res.cursor; + if (res.rows.length > 0) { + queryClient.setQueryData( + AI_CHAT_MESSAGES_RQ_KEY(chatId), + ( + old: + | { + pages: { items: IAiChatMessageRow[]; meta: unknown }[]; + pageParams: unknown[]; + } + | undefined, + ) => + old + ? { ...old, pages: mergeDeltaRowsIntoPages(old.pages, res.rows) } + : old, + ); + } + } catch { + // Transient failure (e.g. a server restart mid-run): swallow and retry on + // the next tick — the poll must survive a bounce, like the old dumb refetch. + } + }; + const id = setInterval(() => void tick(), 2500); + return () => { + cancelled = true; + clearInterval(id); + }; + }, [degradedPoll, windowOpen, activeChatId, queryClient]); + // #184 reconnect-and-live-follow. Whether detached agent runs are enabled for // this workspace. When the feature is off no runs are ever created, so the // resume attempt would only ever 204; gating ChatThread's resume on it avoids a diff --git a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx index ba4dcb7c..47e047cd 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.test.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.test.tsx @@ -172,9 +172,18 @@ function resetState() { h.state.getRun.mockResolvedValue({ run: null, message: null }); } +// #491: the streaming tail carries a persisted step frontier (metadata.stepsPersisted), +// which the tail-only attach reads as `n` in `?anchor=&n=`. Seeded WHOLE now. const streamingTail = () => [ row("u1", "user", undefined, "hi"), - row("a1", "assistant", "streaming", "partial"), + { + id: "a1", + role: "assistant", + content: "partial", + status: "streaming", + createdAt: "2026-01-01T00:00:00Z", + metadata: { stepsPersisted: 2 }, + } as IAiChatMessageRow, ]; const settledTail = () => [ row("u1", "user", undefined, "hi"), @@ -335,20 +344,24 @@ describe("ChatThread — send now", () => { expect(screen.getAllByLabelText("Remove queued message")).toHaveLength(1); }); - it("Stop then a REAL network-drop finish exits to idle (honor-in-stopping), NOT a false reconnect", () => { + it("Stop then a REAL network-drop finish exits to idle (honor-in-stopping), NOT a false reconnect", async () => { // Regression for the disconnect-first reorder: on the STOP path, even a drop- // form finish { isError:true, isDisconnect:true } arriving in `stopping` must be // HONORED (reducer) and exit to idle — it must NOT enter the reconnect ladder. startLocalStreamWithRun(); // live local stream, autonomous fireEvent.click(screen.getByLabelText("Stop")); // STOP_REQUESTED -> stopping h.state.error = { message: "Failed to fetch" }; - act(() => { + // #491: the disconnect re-seeds from persist (async getRun) before dispatching + // FINISH_DISCONNECT, which the reducer HONORS in `stopping` -> idle. Flush it. + await act(async () => { h.state.onFinish?.({ message: { id: "a1", role: "assistant", parts: [] }, isAbort: false, isDisconnect: true, isError: true, }); + await Promise.resolve(); + await Promise.resolve(); }); expect(screen.queryByText(/reconnecting/i)).toBeNull(); }); @@ -803,19 +816,24 @@ describe("ChatThread — resume (attach) machinery", () => { expect(h.state.resumeStream).not.toHaveBeenCalled(); }); - it("strips the streaming tail from the seed, keeps a user tail whole", () => { + it("#491 tail-only: seeds the streaming tail WHOLE (no strip), keeps a user tail whole", () => { renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); - expect(h.state.seededMessages).toHaveLength(1); + // MUTATION-VERIFY: re-introduce the seed-strip and this goes red — the streaming + // tail (steps 0..N-1) MUST be seeded so the SDK continuation appends the tail to + // the RIGHT message. Both rows (user + assistant) are seeded. + expect(h.state.seededMessages).toHaveLength(2); cleanup(); resetState(); renderThread({ autonomousRunsEnabled: true, initialRows: userTail() }); expect(h.state.seededMessages).toHaveLength(1); }); - it("builds the attach URL with expect=live&anchor only for a stripped streaming tail", () => { + it("#491 tail-only: builds the attach URL with ?anchor=&n= from the persisted step frontier", () => { renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail() }); + // n=2 comes from a1's metadata.stepsPersisted (MUTATION-VERIFY: hardcode n=0 and + // this fails). No `expect=live` param anymore. expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe( - "/api/ai-chat/runs/c1/stream?expect=live&anchor=a1", + "/api/ai-chat/runs/c1/stream?anchor=a1&n=2", ); cleanup(); resetState(); @@ -839,39 +857,41 @@ describe("ChatThread — resume (attach) machinery", () => { }); } - it("204 on a streaming tail: restore + invalidate + onResumeFallback(true)", async () => { + it("204 on a streaming tail: NO restore (row kept) + invalidate + onResumeFallback(true)", async () => { const { onResumeFallback, invalidateSpy } = renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail(), }); await attachFetch({ status: 204, ok: false }); - expect(h.state.setMessages).toHaveBeenCalledTimes(1); // restore + // #491 tail-only: the anchor row was never stripped, so there is NOTHING to + // restore. MUTATION-VERIFY: re-add a restore setMessages here and it goes red. + expect(h.state.setMessages).not.toHaveBeenCalled(); expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["ai-chat-messages", "c1"], }); expect(onResumeFallback).toHaveBeenCalledWith(true); }); - it("F7 restart-survival: a 500 attach failure restores the row AND arms the poll", async () => { + it("F7 restart-survival: a 500 attach failure arms the poll WITHOUT a restore", async () => { const { onResumeFallback, invalidateSpy } = renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail(), }); await attachFetch({ status: 500, ok: false }); - expect(h.state.setMessages).toHaveBeenCalledTimes(1); + expect(h.state.setMessages).not.toHaveBeenCalled(); expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["ai-chat-messages", "c1"], }); expect(onResumeFallback).toHaveBeenCalledWith(true); }); - it("F7 restart-survival: a network throw restores the row AND arms the poll", async () => { + it("F7 restart-survival: a network throw arms the poll WITHOUT a restore", async () => { const { onResumeFallback, invalidateSpy } = renderThread({ autonomousRunsEnabled: true, initialRows: streamingTail(), }); await attachFetch(new Error("network down"), true); - expect(h.state.setMessages).toHaveBeenCalledTimes(1); + expect(h.state.setMessages).not.toHaveBeenCalled(); expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["ai-chat-messages", "c1"], }); @@ -931,7 +951,7 @@ describe("ChatThread — resume (attach) machinery", () => { expect(h.state.sendMessage).not.toHaveBeenCalled(); }); - it("an empty resumed message (starved replay) restores the row AND arms the poll", () => { + it("an empty resumed message (starved replay) arms the poll WITHOUT a restore", () => { h.state.status = "ready"; const { onResumeFallback } = renderThread({ autonomousRunsEnabled: true, @@ -947,7 +967,9 @@ describe("ChatThread — resume (attach) machinery", () => { isError: false, }); }); - expect(h.state.setMessages).toHaveBeenCalledTimes(1); // restore + // #491 tail-only: the seeded steps 0..N-1 are still on screen (the SDK + // continuation never wiped them), so there is nothing to restore — just poll. + expect(h.state.setMessages).not.toHaveBeenCalled(); expect(onResumeFallback).toHaveBeenCalledWith(true); // arm }); @@ -995,24 +1017,41 @@ describe("ChatThread — live reconnect + stalled", () => { cleanup(); }); + // #491: the authoritative PERSISTED assistant row `getRun` projects on a local + // disconnect — the re-seed source. Its metadata.stepsPersisted becomes `n`. + const persistedAnchor = (steps = 3) => ({ + run: { id: "run-1", status: "running" }, + message: { + id: "a2", + role: "assistant", + content: "persisted 0..N-1", + status: "streaming", + createdAt: "2026-01-01T00:00:00Z", + metadata: { stepsPersisted: steps }, + }, + }); + // A REAL live SSE drop. ai@6.0.207 emits BOTH { isError:true, isDisconnect:true } - // for a network TypeError AND sets useChat `error` — NOT the { isError:false, - // error:null } form the old tests fed. This is the form browser QA hit; with the - // buggy isError-first routing OR without the errorView render-gate these tests go - // red (a real drop surfaces the terminal error banner, masking the reconnect - // ladder). MUTATION-VERIFY of disconnect-first + the errorView phase-gate. - function disconnect(message: unknown = liveMsg) { + // for a network TypeError AND sets useChat `error`. #491: an autonomous local drop + // now RE-SEEDS from persist (async getRun) BEFORE entering the reconnect ladder, so + // this helper is async and flushes the getRun microtask before returning. + async function disconnect(message: unknown = liveMsg) { h.state.error = { message: "Failed to fetch" }; // the SDK sets error on the drop - act(() => { + await act(async () => { h.state.onFinish?.({ message, isAbort: false, isDisconnect: true, isError: true, }); + // Flush the getRun().then re-seed + the deferred FINISH_DISCONNECT dispatch. + await Promise.resolve(); + await Promise.resolve(); }); } function renderLive() { + // The persisted-anchor read the local disconnect performs to re-seed from persist. + h.state.getRun.mockResolvedValue(persistedAnchor()); const view = renderThread({ autonomousRunsEnabled: true, initialRows: settledTail(), @@ -1032,35 +1071,80 @@ describe("ChatThread — live reconnect + stalled", () => { }); } - it("a live disconnect starts a backoff reconnect (banner + resumeStream after backoff)", () => { + it("#491: a live disconnect RE-SEEDS from persist, then backs off to reconnect with ?anchor=&n=", async () => { renderLive(); - disconnect(); + await disconnect(); + // The re-seed read the authoritative persisted row and replaced the live partial. + // MUTATION-VERIFY: skip the getRun re-seed (send `n` off the live message) and the + // n below no longer matches the PERSISTED stepsPersisted. + expect(h.state.getRun).toHaveBeenCalledWith("c1"); + expect(h.state.setMessages).toHaveBeenCalled(); // re-seeded the store from persist expect(screen.getByText(/reconnecting/i)).toBeTruthy(); expect(h.state.resumeStream).not.toHaveBeenCalled(); advanceToAttempt(1); expect(h.state.resumeStream).toHaveBeenCalledTimes(1); + // n=3 is the PERSISTED row's stepsPersisted (from getRun), NOT the live store. expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe( - "/api/ai-chat/runs/c1/stream?expect=live&anchor=a2", + "/api/ai-chat/runs/c1/stream?anchor=a2&n=3", ); }); - it("#488 (browser QA): the reconnect banner is SHOWN, not masked by the residual useChat error", () => { + it("#491 regression (#137/#161 dup): getRun REJECT on a live disconnect drops the live partial + nulls the anchor", async () => { + // The re-seed source (getRun) FAILS — a flaky-network blip (SSE + getRun both + // fail, network recovers in ~1s). The OLD .catch just re-entered the ladder with + // NO re-seed and NO filter, so the reconnect could tail-apply the registry's + // frames onto the live partial that ALREADY has those steps -> duplicated text. + renderLive(); + h.state.getRun.mockReset(); + h.state.getRun.mockRejectedValue(new Error("network")); + await disconnect(); // live partial = liveMsg (id "a2") + expect(h.state.getRun).toHaveBeenCalledWith("c1"); + // THE GUARANTEE: on the getRun failure the live partial (a2) is FILTERED from the + // store, so the reconnect can never tail-apply already-present steps onto it. + // MUTATION-VERIFY: revert the .catch fix (enterReconnect only, no filter) and no + // setMessages call removes a2 -> this reddens. + const removedLivePartial = ( + h.state.setMessages as unknown as { + mock: { calls: [unknown][] }; + } + ).mock.calls.some(([updater]) => { + if (typeof updater !== "function") return false; + const out = (updater as (p: { id: string }[]) => { id: string }[])([ + { id: "a2" }, + { id: "u1" }, + ]); + return !out.some((m) => m.id === "a2"); + }); + expect(removedLivePartial).toBe(true); + expect(screen.getByText(/reconnecting/i)).toBeTruthy(); + advanceToAttempt(1); + expect(h.state.resumeStream).toHaveBeenCalledTimes(1); + // Anchor was nulled -> replay-from-start (no params) / 204 -> poll; never a stale + // ?anchor=&n= over the live partial. + expect(h.state.transport!.prepareReconnectToStreamRequest!().api).toBe( + "/api/ai-chat/runs/c1/stream", + ); + }); + + it("#488 (browser QA): the reconnect banner is SHOWN, not masked by the residual useChat error", async () => { // The drop sets useChat `error` (real SDK), and the terminal errorView describes // it ("Lost connection to the server"). The FSM phase-gate must let the // `reconnecting` banner WIN over that residual error. MUTATION-VERIFY: revert the // errorView phase-gate (show errorView whenever error is set) and the terminal // banner masks "reconnecting…" -> red. renderLive(); - disconnect(); + await disconnect(); expect(h.state.error).not.toBeNull(); // the SDK error IS set during recovery expect(screen.getByText(/reconnecting/i)).toBeTruthy(); // The terminal "Lost connection… reload" banner must NOT be showing. expect(screen.queryByText(/reload and try again/i)).toBeNull(); }); - it("#488 commit 2: a disconnect BEFORE the first assistant frame reconnects with NO anchor", () => { + it("#488 commit 2: a disconnect BEFORE the first assistant frame reconnects with NO anchor", async () => { renderLive(); - disconnect(null); // no assistant message yet (pre-first-frame break) + // No persisted assistant row for a pre-first-frame break -> no anchor. + h.state.getRun.mockResolvedValue({ run: null, message: null }); + await disconnect(null); // no assistant message yet (pre-first-frame break) expect(screen.getByText(/reconnecting/i)).toBeTruthy(); expect( screen.queryByText("Connection lost — the answer was interrupted."), @@ -1074,7 +1158,7 @@ describe("ChatThread — live reconnect + stalled", () => { it("a live re-attach (2xx) clears the reconnect banner", async () => { renderLive(); - disconnect(); + await disconnect(); advanceToAttempt(1); await reconnect({ status: 200, ok: true }); expect(screen.queryByText(/reconnecting/i)).toBeNull(); @@ -1082,7 +1166,7 @@ describe("ChatThread — live reconnect + stalled", () => { it("a 204 arms the degraded poll and backs off to the next attempt", async () => { const { onResumeFallback } = renderLive(); - disconnect(); + await disconnect(); advanceToAttempt(1); expect(h.state.resumeStream).toHaveBeenCalledTimes(1); await reconnect({ status: 204, ok: false }); @@ -1094,7 +1178,7 @@ describe("ChatThread — live reconnect + stalled", () => { it("exhausts the attempt limit into a manual Retry, which restarts the sequence", async () => { renderLive(); - disconnect(); + await disconnect(); for (let n = 1; n <= 5; n++) { advanceToAttempt(n); expect(h.state.resumeStream).toHaveBeenCalledTimes(n); @@ -1112,22 +1196,23 @@ describe("ChatThread — live reconnect + stalled", () => { it("#488 commit 3: two breaks in a row produce two reconnect cycles", async () => { renderLive(); // First break -> reconnect -> re-attach live. - disconnect(); + await disconnect(); advanceToAttempt(1); expect(h.state.resumeStream).toHaveBeenCalledTimes(1); await reconnect({ status: 200, ok: true }); expect(screen.queryByText(/reconnecting/i)).toBeNull(); - // The re-attached observer stream drops AGAIN -> a SECOND reconnect cycle - // (the old one-shot !wasResumed gate sent this to silent poll). - disconnect(); + // The re-attached observer (live-follow) stream drops AGAIN -> a SECOND reconnect + // cycle. #491: this too re-seeds from persist before re-attaching (never tail- + // applies over the live-follow partial). + await disconnect(); expect(screen.getByText(/reconnecting/i)).toBeTruthy(); advanceToAttempt(1); expect(h.state.resumeStream).toHaveBeenCalledTimes(2); }); - it("does NOT reconnect when autonomous runs are disabled", () => { + it("does NOT reconnect when autonomous runs are disabled", async () => { renderThread({ autonomousRunsEnabled: false, initialRows: settledTail() }); - disconnect(); + await disconnect(); expect(screen.queryByText(/reconnecting/i)).toBeNull(); expect( screen.getByText("Connection lost — the answer was interrupted."), @@ -1138,7 +1223,7 @@ describe("ChatThread — live reconnect + stalled", () => { it("#488 commit 4a: the poll idle cap surfaces a stalled banner + Retry (not silent)", async () => { renderLive(); - disconnect(); + await disconnect(); advanceToAttempt(1); await reconnect({ status: 204, ok: false }); // arms the poll (reconnecting) // No activity for the whole idle cap -> stalled. diff --git a/apps/client/src/features/ai-chat/components/chat-thread.tsx b/apps/client/src/features/ai-chat/components/chat-thread.tsx index e213da32..6eb7fdaa 100644 --- a/apps/client/src/features/ai-chat/components/chat-thread.tsx +++ b/apps/client/src/features/ai-chat/components/chat-thread.tsx @@ -42,7 +42,7 @@ import { assistantMessageHasVisibleContent } from "@/features/ai-chat/utils/mess import { isStreamingTail, isSettledAssistantTail, - seedRows, + stepsPersistedOf, mergeById, } from "@/features/ai-chat/utils/resume-helpers.ts"; import { getRun } from "@/features/ai-chat/services/ai-chat-service.ts"; @@ -266,25 +266,33 @@ export default function ChatThread({ // is NOT one of the lifecycle flags the FSM replaced. const mountedRef = useRef(true); - // attachStrategy DATA (behind the resumeStream effect; #491 swaps it to tail-only - // WITHOUT touching the FSM). The controller is effect-owned (aborted in cleanup, - // I5). `stripRef`/`strippedRowRef` are the current full-replay+strip anchor. + // attachStrategy DATA (behind the resumeStream effect; #491 tail-only, WITHOUT + // touching the FSM). The controller is effect-owned (aborted in cleanup, I5). + // `anchorRef` is the PERSISTED assistant row that pins the run (server invariant + // 6) and its persisted step frontier N: it feeds `?anchor=&n=` + // so the tail-only attach returns frames for steps >= N (the seed carries 0..N-1). + // It is NOT a "stripped" row — the seed keeps every row (tail-only replaces the + // old full-replay+strip). Null when there is no streaming/active tail to resume. const attachAbortRef = useRef(null); - const stripRef = useRef(chatId !== null && isStreamingTail(initialRows ?? [])); - const strippedRowRef = useRef( - stripRef.current ? (initialRows ?? [])[initialRows!.length - 1] : null, + const anchorRef = useRef<{ id: string; stepsPersisted: number } | null>( + (() => { + if (chatId === null || !isStreamingTail(initialRows ?? [])) return null; + const rows = initialRows ?? []; + const tail = rows[rows.length - 1]; + return { id: tail.id, stepsPersisted: stepsPersistedOf(tail) }; + })(), ); // Effect-owned backoff timers (not lifecycle flags): the reconnect ladder and the // stalled inactivity cap. Cleared by the cancelReconnect effect / the cap effect. const reconnectTimerRef = useRef | null>(null); const idleCapTimerRef = useRef | null>(null); + // #491 tail-only: seed EVERY persisted row unchanged (no strip). The streaming + // tail holds steps 0..N-1; the run-stream registry's tail (steps >= N) is APPENDED + // to it by the SDK continuation (readUIMessageStream({ message })), so it must be + // present in the store for the attach to continue the RIGHT message. const initialMessages = useMemo( - () => - seedRows( - initialRows ?? [], - stripRef.current && autonomousRunsEnabled === true, - ).map(rowToUiMessage), + () => (initialRows ?? []).map(rowToUiMessage), [initialRows], ); @@ -335,21 +343,16 @@ export default function ChatThread({ (eff: RunEffect, epoch: number) => { switch (eff.type) { case "resumeStream": { - // The attach GET. Stamp the outcome's generation (I1). A reconnect - // attempt filters the pinned live row from the store first (the mount - // seed already stripped it), so the live replay's text-start rebuilds it - // without duplicating parts (#430). + // The attach GET. Stamp the outcome's generation (I1). #491 tail-only: the + // store already holds EXACTLY the persisted steps 0..N-1 (the mount seed IS + // persist; a reconnect was re-seeded from persist BEFORE FINISH_DISCONNECT + // scheduled it — see the onFinish disconnect handler), so there is nothing + // to filter here: the SDK continues that seeded message, appending the tail + // (steps >= N) without duplicating the pre-drop partial step. pendingAttachEpochRef.current = epoch; // The resumed stream's onFinish is stamped with THIS attach generation // (F1), so a superseded attempt's late finish is dropped. turnEpochRef.current = epoch; - if (machineRef.current.phase.name === "reconnecting") { - const anchor = strippedRowRef.current; - if (anchor) - setMessagesRef.current?.((prev) => - prev.filter((m) => m.id !== anchor.id), - ); - } void resumeStreamRef.current?.(); break; } @@ -464,18 +467,23 @@ export default function ChatThread({ new DefaultChatTransport({ api: "/api/ai-chat/stream", credentials: "include", - prepareReconnectToStreamRequest: () => ({ - // Build the attach URL from the REAL chat id. ?expect=live&anchor= - // only when a streaming tail was stripped: expect=live opts into a - // finished-retained replay (safe only because the row is stripped and the - // replay rebuilds it), and the anchor pins the replay to OUR run — a - // mismatching (newer) run 204s into the restore+poll path instead. - api: `/api/ai-chat/runs/${chatIdRef.current}/stream${ - stripRef.current - ? `?expect=live&anchor=${strippedRowRef.current!.id}` - : "" - }`, - }), + prepareReconnectToStreamRequest: () => { + // #491 tail-only attach URL. When there is an anchor (a streaming/active + // tail to resume) build `?anchor=&n=`: the + // server returns the TAIL — a synthetic `start` frame + frames for steps + // >= n, then live — which the SDK continuation appends to the seeded row. + // The server 204s (-> restore-noop + poll) when it cannot cover the + // frontier (overflow/rotation gap) or the anchor mismatches (a newer run). + // No anchor (a user tail / pre-first-frame break) => no params. + const anchor = anchorRef.current; + return { + api: `/api/ai-chat/runs/${chatIdRef.current}/stream${ + anchor + ? `?anchor=${anchor.id}&n=${anchor.stepsPersisted}` + : "" + }`, + }; + }, fetch: async (input: RequestInfo | URL, init: RequestInit = {}) => { if ((init.method ?? "GET") !== "GET") { // Send path (POST). #488 commit 5: NO client 409 retry ladder anymore @@ -562,8 +570,9 @@ export default function ChatThread({ // Attach GET outcome -> FSM event. The epoch guard replaces BOTH the one-shot // 204 guard (noStreamHandledRef) and the unmount gate: a stale/superseded or - // post-DISPOSE outcome is dropped (I1). For a NONE outcome the attachStrategy - // recovery (restore the stripped row + invalidate for a fresh poll) runs first. + // post-DISPOSE outcome is dropped (I1). #491 tail-only: on a NONE outcome there is + // NOTHING to restore — the anchor row was never stripped from the view (the seed + // keeps it) — so we only invalidate for a fresh poll + dispatch the FSM event. const handleAttachOutcome = useCallback( (ep: number, wasReconnecting: boolean, live: boolean) => { if (ep !== epochRef.current) return; // stale generation — drop @@ -575,10 +584,6 @@ export default function ChatThread({ ); return; } - if (strippedRowRef.current) - setMessagesRef.current?.((prev) => - mergeById(prev, rowToUiMessage(strippedRowRef.current!)), - ); queryClient.invalidateQueries({ queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), }); @@ -661,56 +666,31 @@ export default function ChatThread({ // keeps executing server-side — must win; only a NON-disconnect error (a // provider 500, `{ isError:true, isDisconnect:false }`) is terminal. if (isDisconnect) { - if (wasObserver) { - // A resumed/attached OBSERVER stream dropped. Recover via the degraded - // poll (restore the stripped row only when there is no visible content; - // never clobber a fuller on-screen tail, invariant 9). The FSM decides - // reconnect-vs-poll from liveFollow (a live-follow drop reconnects again, - // #488 commit 3; a mount-resume drop polls). - if (mountedRef.current) { - const hasVisible = msgHasVisible; - if (!hasVisible && strippedRowRef.current) - setMessages((prev) => - mergeById(prev, rowToUiMessage(strippedRowRef.current!)), - ); - queryClient.invalidateQueries({ - queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), - }); - dispatch({ - type: "FINISH_DISCONNECT", - hasVisibleContent: hasVisible, - epoch: stampEpoch, - }); - } + if (!mountedRef.current) { setStopNotice(null); return; } - // A LOCAL live turn dropped. #488 commit 2: recover by the RUN-FACT, not by - // the presence of an assistant message — a setup-phase break (before the - // first frame) still leaves a detached run writing to pages. In autonomous - // mode a run is active for the whole turn, so seed the run-fact from the - // start-metadata runId when known, else a sentinel (the attach GET goes by - // chatId, not runId). Pin the assistant row as the strip/anchor when present. - if (autonomousRunsEnabled === true && mountedRef.current) { - const hasAnchor = - message?.role === "assistant" && typeof message.id === "string"; - if (hasAnchor) { - strippedRowRef.current = { - id: message.id, - role: "assistant", - content: "", - status: "streaming", - createdAt: new Date().toISOString(), - metadata: { parts: message.parts }, - }; - stripRef.current = true; - } else { - strippedRowRef.current = null; - stripRef.current = false; - } + // No detached run to recover (legacy, non-autonomous): a plain disconnect — + // terminal notice, no reconnect. (An observer only exists in autonomous mode, + // so this is always a local turn.) + if (autonomousRunsEnabled !== true) { dispatch({ - type: "RUN_FACT", - runFact: { runId: extractRunId(message) ?? "pending" }, + type: "FINISH_DISCONNECT", + hasVisibleContent: false, + epoch: stampEpoch, + }); + setStopNotice("disconnect"); + return; + } + // A mount-resume OBSERVER (one-shot resume, NOT live-follow) drop falls to + // the degraded POLL, which merges by id — it does NOT attach, so there is + // nothing to re-seed. #491 tail-only: the anchor row was never removed from + // the view (the seed keeps it; the continuation only APPENDED), so nothing to + // restore either. The FSM routes this to `polling` (ownership observer, + // !liveFollow). + if (wasObserver && !machineRef.current.ctx.liveFollow) { + queryClient.invalidateQueries({ + queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), }); dispatch({ type: "FINISH_DISCONNECT", @@ -718,14 +698,92 @@ export default function ChatThread({ epoch: stampEpoch, }); setStopNotice(null); - } else { + return; + } + // We will (re-)ENTER THE RECONNECT LADDER (an attach): a LOCAL live turn's + // first drop, OR a live-follow observer's SUBSEQUENT drop (#488 commit 3). + // #488 commit 2: recover by the RUN-FACT, not by the presence of an assistant + // message — a setup-phase break still leaves a detached run writing to pages. + // + // #491 tail-only (THE crux): the live store holds a PARTIAL step that is AHEAD + // of the persisted boundary; tail-applying the reconnect's step frames over it + // would DUPLICATE that partial step. So entering reconnecting is ALWAYS via a + // RE-SEED FROM PERSIST — never the live store. Fetch the authoritative + // persisted assistant row (`getRun` returns the projected `message`), replace + // the live partial by id (mergeById -> the store now holds EXACTLY steps + // 0..N-1), and set the anchor to `{ id, n = stepsPersisted }`. Only AFTER the + // re-seed is applied do we enter the ladder (FINISH_DISCONNECT schedules the + // backoff) — so the attach can never tail-apply over the live partial. + const cid = chatIdRef.current; + // The live-message runId is the run-fact source (the attach GET keys on + // chatId, so a sentinel still recovers a setup-phase break). + const runId = extractRunId(message ?? undefined) ?? "pending"; + const enterReconnect = (fact: string): void => { + if (!mountedRef.current) return; + // Epoch-stamp the run-fact too (I1): the getRun rtt widens the + // onFinish->dispatch window, so a concurrent SEND_LOCAL during it must be + // able to drop this stale RUN_FACT (else it clobbers the new turn's + // runFact.runId). Consistent with the postRun RUN_FACT stamp. + dispatch({ type: "RUN_FACT", runFact: { runId: fact }, epoch: stampEpoch }); dispatch({ type: "FINISH_DISCONNECT", - hasVisibleContent: false, + hasVisibleContent: msgHasVisible, epoch: stampEpoch, }); - setStopNotice("disconnect"); + }; + // Restore the STRUCTURAL guarantee that the live partial is never the + // tail-apply base: drop the live partial from the store by id and null the + // anchor, so the reconnect replays from step 0 into a CLEAN store (a full + // rebuild) or, past any rotation, 204s -> degraded poll. Used on BOTH the + // no-persisted-row and getRun-FAILURE paths — after this there is no path + // where the attach tail-applies frames onto a row that already has them + // (the #137/#161 duplication class). + const dropLivePartialAndReplayFromStart = (): void => { + if (message?.role === "assistant" && typeof message.id === "string") { + const liveId = message.id; + setMessagesRef.current?.((prev) => + prev.filter((m) => m.id !== liveId), + ); + } + anchorRef.current = null; + }; + if (cid) { + void getRun(cid) + .then((res) => { + if (!mountedRef.current) return; + const persisted = res.message; + if (persisted && persisted.role === "assistant") { + anchorRef.current = { + id: persisted.id, + stepsPersisted: stepsPersistedOf(persisted), + }; + // Replace the live partial with the persisted row IN PLACE by id — + // the re-seed from persist. The attach's tail (steps >= N) then + // appends to a store holding EXACTLY steps 0..N-1: no duplication. + setMessages((prev) => mergeById(prev, rowToUiMessage(persisted))); + } else { + // No persisted assistant row (pre-first-frame break): drop the live + // partial + replay from start (no anchor/n) so nothing is duplicated. + dropLivePartialAndReplayFromStart(); + } + enterReconnect(res.run?.id ?? runId); + }) + .catch(() => { + if (!mountedRef.current) return; + // Persist read FAILED: we cannot re-seed from fresh persist, and a + // stale mount-time anchor over the live partial would tail-apply + // already-present steps -> duplication (a flaky-network blip: + // SSE + getRun both fail, network recovers in ~1s, the registry still + // covers from the mount frontier). Restore the removed-filter guarantee + // instead: drop the live partial + replay from start / 204 -> poll. + dropLivePartialAndReplayFromStart(); + enterReconnect(runId); + }); + } else { + dropLivePartialAndReplayFromStart(); + enterReconnect(runId); } + setStopNotice(null); return; } // A NON-disconnect stream error (a provider 500 etc.) -> terminal error banner. @@ -746,11 +804,10 @@ export default function ChatThread({ if (mountedRef.current) { const hasVisible = msgHasVisible; if (!hasVisible) { - // Starved replay: restore the stripped row + poll to the real terminal. - if (strippedRowRef.current) - setMessages((prev) => - mergeById(prev, rowToUiMessage(strippedRowRef.current!)), - ); + // Starved replay (the tail carried no new steps). #491 tail-only: the + // seeded steps 0..N-1 are still on screen (the SDK continuation never + // wiped them — `start` does not reset parts), so there is nothing to + // restore; just poll to the real terminal. queryClient.invalidateQueries({ queryKey: AI_CHAT_MESSAGES_RQ_KEY(chatIdRef.current), }); @@ -863,12 +920,12 @@ export default function ChatThread({ const tail = rows[rows.length - 1]; if (!tail || tail.role !== "assistant") return; setMessages((prev) => mergeById(prev, rowToUiMessage(tail))); - // Anchor-mismatch coherence: a restored stripped row A that a DIFFERENT run's - // row B has replaced as the tail would linger as an orphan — settle A from - // fresh history so no phantom row survives. - const stripped = strippedRowRef.current; - if (stripped && stripped.id !== tail.id) { - const historical = rows.find((r) => r.id === stripped.id); + // Anchor-mismatch coherence: if a DIFFERENT run's row B has replaced our anchor + // row A as the tail, A would linger as an orphan — reconcile A by id from FRESH + // PERSISTED history (not the pinned live row) so no phantom row survives. + const anchor = anchorRef.current; + if (anchor && anchor.id !== tail.id) { + const historical = rows.find((r) => r.id === anchor.id); if (historical) setMessages((prev) => mergeById(prev, rowToUiMessage(historical))); } diff --git a/apps/client/src/features/ai-chat/services/ai-chat-service.ts b/apps/client/src/features/ai-chat/services/ai-chat-service.ts index 34e6aa71..d5a6bfac 100644 --- a/apps/client/src/features/ai-chat/services/ai-chat-service.ts +++ b/apps/client/src/features/ai-chat/services/ai-chat-service.ts @@ -57,6 +57,31 @@ export async function stopRun( return req.data; } +/** + * Delta poll (#491): the chat's message rows changed since `cursor` (a DB-clock + * timestamp echoed from the previous poll) plus the current run fact, in ONE + * round-trip — the degraded-poll fallback's payload, replacing the old "refetch + * ALL infinite-query pages every 2.5s with full parts" poll. Omit `cursor` on the + * first poll (returns just a fresh cursor, no rows, to start the chain). The + * overlap window guarantees occasional REPEATS, so the caller MUST merge rows + * idempotently by id (mergeById). Owner-gated server-side. + */ +export async function getAiChatMessagesDelta( + chatId: string, + cursor?: string, +): Promise<{ + rows: IAiChatMessageRow[]; + cursor: string; + run: { id: string; status: string } | null; +}> { + const req = await api.post<{ + rows: IAiChatMessageRow[]; + cursor: string; + run: { id: string; status: string } | null; + }>("/ai-chat/messages/delta", { chatId, cursor }); + return req.data; +} + /** * #488: the run-fact — "is a run active on this chat?" — first-class from the * server (POST /ai-chat/run). Called on mount to seed the client FSM's run-fact diff --git a/apps/client/src/features/ai-chat/state/run-fsm.spec.md b/apps/client/src/features/ai-chat/state/run-fsm.spec.md index 4f70f744..86ae3d73 100644 --- a/apps/client/src/features/ai-chat/state/run-fsm.spec.md +++ b/apps/client/src/features/ai-chat/state/run-fsm.spec.md @@ -48,6 +48,7 @@ Legend: **†** = command-transition (bumps `epoch`, I1). Effects in `[…]`. | `RETRY` (manual, stalled banner) | stalled | polling(attach-none) **†** | `[armPoll]` | | `POLL_TERMINAL` (settled tail merged) | polling, reconnecting, stopping | idle | `[disarmPoll, cancelReconnect]`, runFact←null (I4) | | `POLL_IDLE_CAP` (inactivity cap) | polling, reconnecting | stalled | `[disarmPoll, cancelReconnect]` (commit 4a — no more silent) | +| `POLL_IDLE_CAP` (inactivity cap) | stopping | idle | `[disarmPoll, cancelReconnect]`, runFact←null (Review #4: a Stop-armed poll with no SDK/terminal backstop gets a bounded exit — NOT `stalled`, Stop was already pressed so nothing to retry) | | `RUN_FACT{null}` (POST /run → null/terminal, 204) | reconnecting/attaching/polling/stopping | idle | `[cancelReconnect, disarmPoll]`, runFact←null (I3 fresh-negative gate) | | `RUN_FACT{runId}` | any | (same) | runFact←runId (pessimism toward an attempt) | | `STOP_REQUESTED` (user Stop) | streaming, reconnecting, polling | stopping **†** | `[stopRun, abortAttach, cancelReconnect, armPoll]` (poll drives the terminal — I4 exit by data) | @@ -121,8 +122,7 @@ holds. **Pending column: empty.** | 11 | `stopPendingRef` | **FSM phase `stopping`** | the deferred stop fires from the chat-id adoption effect while `stopping` | | 12 | `mountedRef` | **retained (React liveness)** | orthogonal to run-lifecycle; gates imperative onFinish side-effects post-unmount. Epoch (I1) handles stale COMMAND-outcomes; DISPOSE bumps it | | 13 | `attemptResumeRef` | **FSM `ATTACH_START` + run-fact** | mount arms attach ONLY on a confirmed active run (commit 4b: streaming-tail status, or POST /run for a user tail) | -| 14 | `stripRef` | **data** (attachStrategy) | strip+replay detail; the `resumeStream` effect reads it | -| 15 | `strippedRowRef` | **data** (attachStrategy) | the anchor row | +| 14–15 | `anchorRef {id, stepsPersisted}` | **data** (attachStrategy) | #491 tail-only: replaced `stripRef`/`strippedRowRef`. The PERSISTED assistant row that pins the run (server invariant 6) + its step frontier N; feeds `?anchor=&n=`. No strip — the seed keeps every row; entering reconnecting re-seeds from persist | | 16 | `attachAbortRef` | **effect-owned controller** | aborted by the `abortAttach` effect in cleanup (I5) | | 17–25 | `chatIdRef`, `openPageRef`, `getEditorSelectionRef`, `roleIdRef`, `stableIdRef`, `queuedRef`, `sendMessageRef`, `statusRef`, `lastForwardedChatIdRef` | **data** (identity/send mirrors) | unchanged — not lifecycle flags | | NEW | `pendingSupersedeRef` | **data** (send-plumbing) | the runId injected into the next `POST /stream {supersede}`; the single replacement for the 3 DELETED one-shots (#8/#9/#10) — net −2 refs | @@ -151,8 +151,12 @@ message. Sources, in the order they update `ctx.runFact`: 3. **Attach outcomes:** `ATTACH_LIVE` (2xx) confirms active; a 204 on a non-stripped path is an authoritative NEGATIVE fact → the runtime dispatches `RUN_FACT{null}`, which cancels recovery (I3 fresh-negative gate). -4. **Poll (future resume-stack iteration #491):** the delta will carry the run field; - until then the poll drives to a terminal ROW, dispatched as `POLL_TERMINAL`. +4. **Poll (#491, implemented):** the degraded poll now hits the delta endpoint + (`POST /ai-chat/messages/delta`), which ALREADY carries the run fact + (`run: {id, status} | null`) alongside the changed rows. The client does NOT yet + consume that run field — it still drives to a terminal ROW (merged by id), + dispatched as `POLL_TERMINAL` — so the run field rides the wire for a future + client that settles straight off it. Pessimism rule: a stale-but-positive fact PERMITS entering recovery (attach); the 204 then cuts it. A fresh negative fact gates recovery OUT immediately. @@ -178,6 +182,9 @@ Pessimism rule: a stale-but-positive fact PERMITS entering recovery (attach); th /run) are effect-owned and aborted in cleanup (`abortAttach` on `DISPOSE`), not render-phase refs. A client abort of an already-sent POST does not cancel the server action, so disarming on unmount is safe. -- **attachStrategy** (strip+replay today) is behind the `resumeStream` effect; the - resume-stack iteration (#491) swaps it to tail-only WITHOUT touching the FSM. +- **attachStrategy** is behind the `resumeStream` effect; #491 swapped it to + tail-only (`?anchor=&n=`, `anchorRef` data) WITHOUT touching the FSM. Entering + reconnecting always re-seeds from persist; on a getRun failure the live partial + is dropped + replay-from-start so it is never the tail-apply base (no #137/#161 + duplication). - **Queue** stays a data structure; flush/interrupt decisions are transitions. diff --git a/apps/client/src/features/ai-chat/types/ai-chat.types.ts b/apps/client/src/features/ai-chat/types/ai-chat.types.ts index 2c987263..acb84de5 100644 --- a/apps/client/src/features/ai-chat/types/ai-chat.types.ts +++ b/apps/client/src/features/ai-chat/types/ai-chat.types.ts @@ -181,6 +181,12 @@ export interface IAiChatMessageRow { toolCalls?: unknown; metadata?: { parts?: UIMessage["parts"]; + // #491 step-alignment anchor: the count of FINISHED steps whose parts are in + // THIS row, written atomically with `parts` server-side (flushAssistant). The + // resume client reads it as its persisted step frontier N — the tail-only + // attach asks the run-stream registry for the frames of step N onward (the + // seed already carries steps 0..N-1). Absent on pre-#491 rows -> read as 0. + stepsPersisted?: number; // AI SDK v6 `totalUsage` persisted on assistant rows. Legacy cumulative // figure (sum of every step's usage for the turn); kept for back-compat and // as the fallback for older rows that have no `contextTokens`. diff --git a/apps/client/src/features/ai-chat/utils/count-stream-tokens.test.ts b/apps/client/src/features/ai-chat/utils/count-stream-tokens.test.ts index 6b00fbc4..819c057e 100644 --- a/apps/client/src/features/ai-chat/utils/count-stream-tokens.test.ts +++ b/apps/client/src/features/ai-chat/utils/count-stream-tokens.test.ts @@ -6,10 +6,13 @@ describe("estimateTokens", () => { expect(estimateTokens("")).toBe(0); }); - it("ceils chars/4 so any non-empty text is at least 1 token", () => { + // #490: migrated onto the shared @docmost/token-estimate module (chars/2.5, up + // from the old client-only chars/4) so the client counter and the server replay + // budgeter can never diverge. + it("ceils chars/2.5 so any non-empty text is at least 1 token", () => { expect(estimateTokens("a")).toBe(1); - expect(estimateTokens("abcd")).toBe(1); - expect(estimateTokens("abcde")).toBe(2); - expect(estimateTokens("12345678")).toBe(2); + expect(estimateTokens("ab")).toBe(1); + expect(estimateTokens("abcde")).toBe(2); // 5 / 2.5 = 2 + expect(estimateTokens("x".repeat(10))).toBe(4); // 10 / 2.5 = 4 }); }); diff --git a/apps/client/src/features/ai-chat/utils/count-stream-tokens.ts b/apps/client/src/features/ai-chat/utils/count-stream-tokens.ts index aaf99599..951bb992 100644 --- a/apps/client/src/features/ai-chat/utils/count-stream-tokens.ts +++ b/apps/client/src/features/ai-chat/utils/count-stream-tokens.ts @@ -2,18 +2,10 @@ * Rough client-side token estimation for AI-chat UI affordances. * * No provider streams exact per-token usage mid-stream, so any in-flight figure - * is a CLIENT ESTIMATE (chars/≈4 heuristic). Pure + unit-testable: it never runs - * a real BPE tokenizer (that would be O(n²) on the hot path, bloat the bundle, - * and be wrong for Gemini/Ollama anyway). Used by the in-body reasoning counter - * ("Thinking · N tokens"). + * is a CLIENT ESTIMATE. This re-exports the SHARED estimator from + * `@docmost/token-estimate` (chars/2.5) so the in-body counter and the server's + * replay budgeter use the SAME heuristic — two divergent estimators would mean + * "the badge shows 60%" while "the budgeter already trimmed" (#490). Used by the + * in-body reasoning counter ("Thinking · N tokens"). */ - -/** - * Rough token estimate for a piece of text using the standard chars/≈4 heuristic. - * Returns 0 for empty/whitespace-free-of-content input, and ceils so any - * non-empty text counts as at least one token. - */ -export function estimateTokens(text: string): number { - if (!text) return 0; - return Math.ceil(text.length / 4); -} +export { estimateTokens } from "@docmost/token-estimate"; diff --git a/apps/client/src/features/ai-chat/utils/error-message.test.ts b/apps/client/src/features/ai-chat/utils/error-message.test.ts index 04d94225..14efe5ae 100644 --- a/apps/client/src/features/ai-chat/utils/error-message.test.ts +++ b/apps/client/src/features/ai-chat/utils/error-message.test.ts @@ -89,6 +89,23 @@ describe("describeChatError", () => { expect(view.title).not.toBe("AI provider not configured"); }); + it("classifies a token-degeneration abort under the SAME 'Response stopped.' marker the live view shows (#495)", () => { + // The exact reason the server persists in metadata.error on a degeneration + // abort (ai-chat.service OUTPUT_DEGENERATION_ERROR). Live, this event shows + // the neutral "Response stopped." notice; the persisted banner MUST match it + // so live and refetch never disagree. + const view = describeChatError( + "Output degeneration detected (repeated token loop)", + t, + ); + expect(view.title).toBe("Response stopped."); + expect(view.detail).toBe( + "The answer was stopped automatically because the model fell into a repeated output loop.", + ); + // Regression guard: it must NOT fall through to the generic heading. + expect(view.title).not.toBe("Something went wrong"); + }); + it("classifies a dropped connection (ECONNRESET) as a lost-connection error", () => { expect( describeChatError("Cannot connect to API: read ECONNRESET", t).title, diff --git a/apps/client/src/features/ai-chat/utils/error-message.ts b/apps/client/src/features/ai-chat/utils/error-message.ts index 897bb629..13320392 100644 --- a/apps/client/src/features/ai-chat/utils/error-message.ts +++ b/apps/client/src/features/ai-chat/utils/error-message.ts @@ -77,6 +77,22 @@ export function describeChatError( }; } + // Our own token-degeneration abort (#444): the server aborts a runaway + // repetition loop and persists this exact reason in metadata.error. LIVE, the + // same abort surfaces as the neutral "Response stopped." notice (the client + // cannot tell it from a manual Stop mid-stream), so the persisted banner must + // read the SAME "Response stopped." marker — otherwise the live view and a + // later refetch show two different texts for one event. The detail explains the + // loop-guard cause without contradicting the shared heading. + if (/output degeneration detected|repeated token loop/i.test(msg)) { + return { + title: t("Response stopped."), + detail: t( + "The answer was stopped automatically because the model fell into a repeated output loop.", + ), + }; + } + if (/"statusCode"\s*:\s*403\b/.test(msg)) { return { title: t("AI chat is disabled"), diff --git a/apps/client/src/features/ai-chat/utils/resume-helpers.test.ts b/apps/client/src/features/ai-chat/utils/resume-helpers.test.ts index 8a4dc7ef..42c0d1b6 100644 --- a/apps/client/src/features/ai-chat/utils/resume-helpers.test.ts +++ b/apps/client/src/features/ai-chat/utils/resume-helpers.test.ts @@ -4,7 +4,8 @@ import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.t import { isStreamingTail, isSettledAssistantTail, - seedRows, + stepsPersistedOf, + mergeDeltaRowsIntoPages, mergeById, } from "./resume-helpers.ts"; @@ -12,8 +13,18 @@ function row( id: string, role: string, status?: string, + stepsPersisted?: number, ): IAiChatMessageRow { - return { id, role, content: "", status, createdAt: "2026-01-01T00:00:00Z" }; + return { + id, + role, + content: "", + status, + createdAt: "2026-01-01T00:00:00Z", + ...(stepsPersisted !== undefined + ? { metadata: { stepsPersisted } } + : {}), + }; } function makeMsg(id: string, text: string): UIMessage { @@ -65,23 +76,92 @@ describe("isSettledAssistantTail", () => { }); }); -describe("seedRows", () => { - const rows = [row("u1", "user"), row("a1", "assistant", "streaming")]; - - it("returns the rows unchanged when not stripping", () => { - expect(seedRows(rows, false)).toBe(rows); +describe("stepsPersistedOf", () => { + it("reads metadata.stepsPersisted", () => { + expect(stepsPersistedOf(row("a1", "assistant", "streaming", 3))).toBe(3); + expect(stepsPersistedOf(row("a1", "assistant", "streaming", 0))).toBe(0); }); - it("drops the last row when stripping", () => { - const seeded = seedRows(rows, true); - expect(seeded).toHaveLength(1); - expect(seeded[0].id).toBe("u1"); + it("defaults to 0 for a pre-#491 row (absent), null/undefined, or a bad value", () => { + expect(stepsPersistedOf(row("a1", "assistant", "streaming"))).toBe(0); + expect(stepsPersistedOf(null)).toBe(0); + expect(stepsPersistedOf(undefined)).toBe(0); + expect( + stepsPersistedOf({ + id: "a1", + role: "assistant", + content: "", + createdAt: "x", + metadata: { stepsPersisted: -2 }, + }), + ).toBe(0); }); - it("returns an empty list when stripping a single-row list", () => { - expect(seedRows([row("a1", "assistant", "streaming")], true)).toHaveLength( - 0, - ); + it("floors a non-integer count", () => { + expect( + stepsPersistedOf({ + id: "a1", + role: "assistant", + content: "", + createdAt: "x", + metadata: { stepsPersisted: 2.9 }, + }), + ).toBe(2); + }); +}); + +describe("mergeDeltaRowsIntoPages", () => { + const pages = () => [ + { items: [row("u1", "user"), row("a1", "assistant", "streaming", 1)], meta: {} }, + ]; + + it("returns the pages unchanged for an empty delta", () => { + const p = pages(); + expect(mergeDeltaRowsIntoPages(p, [])).toBe(p); + }); + + it("appends a genuinely new row to the last page in chronological order", () => { + const merged = mergeDeltaRowsIntoPages(pages(), [row("a2", "assistant", "streaming", 0)]); + expect(merged[0].items.map((i) => i.id)).toEqual(["u1", "a1", "a2"]); + }); + + it("replaces a grown row in place (per-step growth), never appends a duplicate", () => { + const merged = mergeDeltaRowsIntoPages(pages(), [ + row("a1", "assistant", "streaming", 2), + ]); + expect(merged[0].items.map((i) => i.id)).toEqual(["u1", "a1"]); + // the in-place replacement carries the grown step frontier. + expect(stepsPersistedOf(merged[0].items[1])).toBe(2); + }); + + it("does not mutate the input pages", () => { + const input = pages(); + const before = input[0].items.slice(); + mergeDeltaRowsIntoPages(input, [row("a2", "assistant", "streaming", 0)]); + expect(input[0].items).toEqual(before); // untouched + }); + + // #491 CONTRACT: the delta overlap window re-delivers the same rows, so merging + // MUST be idempotent — applying a delta twice equals applying it once (no growth, + // no reorder). A regression re-introduces duplicate assistant bubbles per poll. + it("is idempotent: applying the same delta twice equals once", () => { + const delta = [ + row("a1", "assistant", "streaming", 2), // grown existing row + row("a2", "assistant", "streaming", 0), // new row + ]; + const once = mergeDeltaRowsIntoPages(pages(), delta); + const twice = mergeDeltaRowsIntoPages(once, delta); + const thrice = mergeDeltaRowsIntoPages(twice, delta); + expect(once[0].items.map((i) => i.id)).toEqual(["u1", "a1", "a2"]); + expect(twice[0].items.map((i) => i.id)).toEqual(["u1", "a1", "a2"]); + expect(twice).toEqual(once); + expect(thrice).toEqual(once); + }); + + it("seeds a first page when the cache is empty", () => { + const merged = mergeDeltaRowsIntoPages([], [row("u1", "user")]); + expect(merged).toHaveLength(1); + expect(merged[0].items.map((i) => i.id)).toEqual(["u1"]); }); }); @@ -109,4 +189,37 @@ describe("mergeById", () => { expect(mergeById(prev, null)).toBe(prev); expect(mergeById(prev, undefined)).toBe(prev); }); + + // #491 CONTRACT: the delta poll's overlap window GUARANTEES the same row is + // re-delivered across close polls, so merging must be IDEMPOTENT by id — merging + // the same row (or an equal-length list of rows) twice must not duplicate or + // reorder. This is the property the whole delta-poll design leans on; a + // regression here would re-introduce duplicate assistant bubbles on every poll. + it("is idempotent by id: re-merging the same row does not duplicate or reorder", () => { + const seed = [makeMsg("u1", "hi"), makeMsg("a1", "step 1")]; + const repeat = makeMsg("a1", "step 1"); // the SAME row the overlap re-delivers + const once = mergeById(seed, repeat); + const twice = mergeById(once, repeat); + const thrice = mergeById(twice, repeat); + // Length is stable (no growth), order is stable (user then assistant). + expect(once.map((m) => m.id)).toEqual(["u1", "a1"]); + expect(twice.map((m) => m.id)).toEqual(["u1", "a1"]); + expect(thrice.map((m) => m.id)).toEqual(["u1", "a1"]); + // The repeated merge converges: the row is replaced in place, never appended. + expect(twice[1]).toBe(repeat); + }); + + it("is idempotent across a batch of repeated + grown rows (delta re-delivery)", () => { + // A delta poll re-delivers a1 (unchanged) and a2 (grown one step). Applying the + // batch twice must equal applying it once — the poll can re-send either. + const start = [makeMsg("u1", "hi"), makeMsg("a1", "done")]; + const batch = [makeMsg("a1", "done"), makeMsg("a2", "grown step 2")]; + const apply = (list: typeof start) => + batch.reduce((acc, row) => mergeById(acc, row), list); + const once = apply(start); + const twice = apply(once); + expect(once.map((m) => m.id)).toEqual(["u1", "a1", "a2"]); + expect(twice.map((m) => m.id)).toEqual(["u1", "a1", "a2"]); + expect(twice).toEqual(once); + }); }); diff --git a/apps/client/src/features/ai-chat/utils/resume-helpers.ts b/apps/client/src/features/ai-chat/utils/resume-helpers.ts index bddb4d01..8c21f42b 100644 --- a/apps/client/src/features/ai-chat/utils/resume-helpers.ts +++ b/apps/client/src/features/ai-chat/utils/resume-helpers.ts @@ -11,9 +11,10 @@ import type { IAiChatMessageRow } from "@/features/ai-chat/types/ai-chat.types.t /** * A STREAMING tail: the last persisted row is an assistant row still marked - * `status === 'streaming'`. Such a tail is stripped from the seed and rebuilt by - * the replay (`expect=live`), since the SDK's `text-start` always pushes a new - * part and replaying over a seeded in-progress row would duplicate its text. + * `status === 'streaming'`. #491 (tail-only): such a tail is seeded UNCHANGED — + * it carries the persisted steps 0..N-1 — and the run-stream registry's tail + * (frames for steps >= N) is APPENDED to it by the SDK's `readUIMessageStream` + * continuation. Only the presence of this tail decides WHETHER to attach. */ export function isStreamingTail(rows: IAiChatMessageRow[]): boolean { const tail = rows[rows.length - 1]; @@ -32,15 +33,61 @@ export function isSettledAssistantTail(rows: IAiChatMessageRow[]): boolean { } /** - * Seed rows for `useChat`: return the rows unchanged, or without the last row when - * `strip` is set (the streaming tail is stripped so the live replay rebuilds it - * without duplicating parts). + * #491 tail-only anchor: the count of FINISHED steps whose parts are persisted in + * THIS assistant row (`metadata.stepsPersisted`), written atomically with `parts` + * server-side. The resume client reads it as its persisted step frontier N — the + * tail-only attach asks the run-stream registry for the frames of step N onward + * (the seed already carries steps 0..N-1). Absent on pre-#491 rows => 0. */ -export function seedRows( +export function stepsPersistedOf( + row: IAiChatMessageRow | null | undefined, +): number { + const n = row?.metadata?.stepsPersisted; + return typeof n === "number" && n >= 0 ? Math.floor(n) : 0; +} + +/** One page of the messages infinite-query cache (`{ items, meta }`). */ +export interface IMessagePage { + items: IAiChatMessageRow[]; + meta: unknown; +} + +/** + * #491 delta-poll merge: upsert the delta poll's `rows` into the messages + * infinite-query page structure IDEMPOTENTLY by id. The delta endpoint's overlap + * window GUARANTEES occasional REPEATS, so this MUST converge: a row already + * present is REPLACED IN PLACE (per-step growth of an in-progress row), a new row + * is APPENDED to the last page in chronological order (the server returns delta + * rows oldest-first). Applying the same delta twice equals applying it once. Never + * mutates the input pages (returns fresh page objects with cloned item arrays). + */ +export function mergeDeltaRowsIntoPages( + pages: IMessagePage[], rows: IAiChatMessageRow[], - strip: boolean, -): IAiChatMessageRow[] { - return strip ? rows.slice(0, -1) : rows; +): IMessagePage[] { + if (rows.length === 0) return pages; + const next: IMessagePage[] = pages.map((p) => ({ + ...p, + items: p.items.slice(), + })); + const locate = (id: string): [number, number] | null => { + for (let pi = 0; pi < next.length; pi++) { + const ii = next[pi].items.findIndex((it) => it.id === id); + if (ii !== -1) return [pi, ii]; + } + return null; + }; + for (const row of rows) { + const at = locate(row.id); + if (at) { + next[at[0]].items[at[1]] = row; // replace in place — idempotent by id + } else if (next.length > 0) { + next[next.length - 1].items.push(row); // append chronologically + } else { + next.push({ items: [row], meta: undefined }); + } + } + return next; } /** diff --git a/apps/client/src/features/ai-chat/utils/sdk-continuation-tripwire.test.ts b/apps/client/src/features/ai-chat/utils/sdk-continuation-tripwire.test.ts new file mode 100644 index 00000000..c9fc19dc --- /dev/null +++ b/apps/client/src/features/ai-chat/utils/sdk-continuation-tripwire.test.ts @@ -0,0 +1,83 @@ +import { describe, it, expect } from "vitest"; +import { readUIMessageStream, type UIMessage } from "ai"; +import pkg from "../../../../package.json"; + +/** + * PIN-SPEC TRIP-WIRE (#491). The tail-only attach continuation relies on THREE + * behaviors of `ai@6.0.207`, verified line-by-line in the issue. Without this + * test, an `ai` bump could silently break attach (the client would append the + * live tail to the wrong message, or duplicate a step): + * + * 1. `readUIMessageStream({ message })` CONTINUES the passed message — it does + * not start a fresh one — so the tail streamed after a re-seed is appended to + * the seeded assistant row (the same DB id). + * 2. A `start` frame does NOT reset the existing message's parts (so the seeded + * steps 0..N-1 survive; the synthetic `start` the registry prepends only + * carries the run-fact metadata). + * 3. Text parts do NOT cross a `finish-step` boundary — a new `text-start` after + * `finish-step` is a NEW part — so the reconstructed steps stay separated and + * the step frontier stays meaningful. + * + * If an `ai` upgrade changes any of these, this test fails LOUD instead of the + * resume path silently corrupting. + */ +describe("ai SDK continuation trip-wire (#491, tail-only attach)", () => { + it("is pinned to the exact ai version the continuation was verified against", () => { + // A caret/range bump is exactly what would silently break attach — require an + // exact pin. Bumping ai MUST re-verify the behavior asserted below, then this. + expect((pkg as { dependencies: Record }).dependencies.ai).toBe( + "6.0.207", + ); + }); + + it("continues the seeded message: start does not reset parts, the tail appends as new parts", async () => { + // A seeded assistant row with ONE finished step already reconstructed. + const seeded: UIMessage = { + id: "assistant-1", + role: "assistant", + parts: [ + { type: "step-start" }, + { type: "text", text: "STEP0", state: "done" }, + ], + } as UIMessage; + + // The tail the registry delivers on re-attach: a synthetic start (run-fact), + // then step 1's frames, then finish. As UI-message chunks (what the SSE frames + // decode to). + const chunks = [ + { type: "start", messageMetadata: { runId: "r1", chatId: "c1" } }, + { type: "start-step" }, + { type: "text-start", id: "t1" }, + { type: "text-delta", id: "t1", delta: "STEP1" }, + { type: "text-end", id: "t1" }, + { type: "finish-step" }, + { type: "finish" }, + ]; + const stream = new ReadableStream({ + start(c) { + for (const ch of chunks) c.enqueue(ch); + c.close(); + }, + }); + + let last: UIMessage | undefined; + for await (const msg of readUIMessageStream({ message: seeded, stream })) { + last = msg; + } + + expect(last).toBeDefined(); + // Same message id (continuation, not a fresh message). + expect(last!.id).toBe("assistant-1"); + // The seeded step-0 parts SURVIVED the `start` frame, and step 1 was appended + // as SEPARATE parts (text did not cross the finish-step boundary). + const shape = last!.parts.map((p) => `${p.type}:${(p as { text?: string }).text ?? ""}`); + expect(shape).toEqual([ + "step-start:", + "text:STEP0", + "step-start:", + "text:STEP1", + ]); + // The run-fact metadata from the synthetic start frame is applied. + expect(last!.metadata).toMatchObject({ runId: "r1", chatId: "c1" }); + }); +}); diff --git a/apps/client/src/features/editor/components/fixed-toolbar/use-toolbar-state.test.ts b/apps/client/src/features/editor/components/fixed-toolbar/use-toolbar-state.test.ts new file mode 100644 index 00000000..396925dc --- /dev/null +++ b/apps/client/src/features/editor/components/fixed-toolbar/use-toolbar-state.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from "vitest"; +import * as Y from "yjs"; +import { yHistoryAvailability } from "./use-toolbar-state.ts"; + +// Undo/redo availability is derived from the Yjs UndoManager's PRIVATE +// `undoStack` / `redoStack` fields (see use-toolbar-state.ts for why we read the +// stack lengths directly instead of the expensive `editor.can().undo()` dry-run). +// These tests lock in the behavior AND pin the library shape so a yjs / y-undo +// upgrade that renames/restructures those internals fails loudly here rather than +// silently enabling/disabling the toolbar buttons in production. +describe("yHistoryAvailability", () => { + it("reports availability from the stack lengths", () => { + expect(yHistoryAvailability({ undoStack: [], redoStack: [] })).toEqual({ + canUndo: false, + canRedo: false, + }); + expect( + yHistoryAvailability({ undoStack: [{}], redoStack: [] }), + ).toEqual({ canUndo: true, canRedo: false }); + expect( + yHistoryAvailability({ undoStack: [{}], redoStack: [{}, {}] }), + ).toEqual({ canUndo: true, canRedo: true }); + }); + + it("returns null when the private stack shape is unrecognized (upgrade guard)", () => { + // Simulates a yjs / y-undo upgrade that renames or restructures the private + // fields: the caller then falls back to the safe prosemirror-history default + // instead of throwing on `.length` of undefined or reading garbage. + expect(yHistoryAvailability(undefined)).toBeNull(); + expect(yHistoryAvailability(null)).toBeNull(); + expect(yHistoryAvailability({})).toBeNull(); + expect(yHistoryAvailability({ undoStack: 5, redoStack: 5 })).toBeNull(); + // Only one stack present (partial rename) is still not trusted. + expect(yHistoryAvailability({ undoStack: [] })).toBeNull(); + }); + + it("pin-test: a real yjs UndoManager still exposes undoStack/redoStack arrays", () => { + const doc = new Y.Doc(); + const text = doc.getText("prosemirror"); + const undoManager = new Y.UndoManager(text); + + // Fresh manager: both stacks empty -> nothing to undo/redo. + expect(yHistoryAvailability(undoManager)).toEqual({ + canUndo: false, + canRedo: false, + }); + + // A tracked edit must push onto the private undoStack. If a future yjs + // renames these fields, yHistoryAvailability(undoManager) returns null and + // the expectation below fails loudly. + text.insert(0, "hello"); + undoManager.stopCapturing(); + expect(yHistoryAvailability(undoManager)).toEqual({ + canUndo: true, + canRedo: false, + }); + + // Undoing moves the item to the redoStack -> redo becomes available. + undoManager.undo(); + expect(yHistoryAvailability(undoManager)).toEqual({ + canUndo: false, + canRedo: true, + }); + }); +}); diff --git a/apps/client/src/features/editor/components/fixed-toolbar/use-toolbar-state.ts b/apps/client/src/features/editor/components/fixed-toolbar/use-toolbar-state.ts index 2f8e3a7f..ec47be69 100644 --- a/apps/client/src/features/editor/components/fixed-toolbar/use-toolbar-state.ts +++ b/apps/client/src/features/editor/components/fixed-toolbar/use-toolbar-state.ts @@ -35,6 +35,30 @@ export interface ToolbarState { // When neither history backend is installed (the pre-sync static editor — // mainExtensions only, undoRedo disabled), both fall through to 0 -> false, // matching the previous `safeCan` behavior. +// Reads the Yjs UndoManager's undo/redo availability from its stack lengths. +// +// `undoStack` / `redoStack` are PRIVATE y-undo / yjs internals, so we touch them +// defensively: a yjs or y-undo upgrade that renames or restructures these fields +// must not silently mis-drive the toolbar buttons (nor throw on `.length` of +// `undefined`). We only trust them when they are actually arrays; otherwise this +// returns null and the caller falls back to a safe default. The pin-test in +// use-toolbar-state.test.ts asserts the current library shape, so an upgrade that +// breaks this contract fails loudly there instead of failing silently in the UI. +export function yHistoryAvailability( + undoManager: unknown, +): { canUndo: boolean; canRedo: boolean } | null { + if (!undoManager || typeof undoManager !== "object") return null; + const { undoStack, redoStack } = undoManager as { + undoStack?: unknown; + redoStack?: unknown; + }; + if (!Array.isArray(undoStack) || !Array.isArray(redoStack)) return null; + return { + canUndo: undoStack.length > 0, + canRedo: redoStack.length > 0, + }; +} + function historyAvailability(editor: Editor): { canUndo: boolean; canRedo: boolean; @@ -43,16 +67,14 @@ function historyAvailability(editor: Editor): { // Collaboration history (Yjs) takes precedence when present. const yState = yUndoPluginKey.getState(state) as - | { undoManager?: { undoStack: unknown[]; redoStack: unknown[] } } + | { undoManager?: unknown } | undefined; - if (yState?.undoManager) { - return { - canUndo: yState.undoManager.undoStack.length > 0, - canRedo: yState.undoManager.redoStack.length > 0, - }; - } + const yAvail = yHistoryAvailability(yState?.undoManager); + if (yAvail) return yAvail; // Plain prosemirror-history (returns 0 when the history plugin is absent). + // This is also the safe default when a Yjs UndoManager is present but its + // private stack shape is no longer recognized (yHistoryAvailability -> null). return { canUndo: undoDepth(state) > 0, canRedo: redoDepth(state) > 0, diff --git a/apps/client/src/features/editor/gitmost/gitmost-recording.test.ts b/apps/client/src/features/editor/gitmost/gitmost-recording.test.ts index 1f5feef8..f357b070 100644 --- a/apps/client/src/features/editor/gitmost/gitmost-recording.test.ts +++ b/apps/client/src/features/editor/gitmost/gitmost-recording.test.ts @@ -9,7 +9,7 @@ import { Italic } from "@tiptap/extension-italic"; import { Link } from "@tiptap/extension-link"; import { gitmostInsertTranscriptIntoEditor } from "./gitmost-recording.ts"; -const ZWSP = "​"; // U+200B, the helper's block-trigger neutralizer +const ZWSP = "​"; // U+200B — asserted ABSENT (the block-escape lives in the serializer now) /** * #377 — the web-side bridge must append the native host's transcript below the @@ -18,8 +18,9 @@ const ZWSP = "​"; // U+200B, the helper's block-trigger neutralizer * regression would be caught), asserting the resulting document rather than * mocking the editor: transcript present -> "Transcript" heading + one paragraph * per non-empty line; content is inserted as LITERAL TEXT (no HTML/markdown - * parsing); col-0 markdown block triggers are neutralized so git-sync keeps them - * paragraphs; absent/empty/non-string -> no-op. + * parsing); col-0 markdown block triggers are stored verbatim (the git-sync + * serializer block-escapes them, so no client-side ZWSP is needed); + * absent/empty/non-string -> no-op. */ describe("gitmostInsertTranscriptIntoEditor", () => { const makeEditor = () => @@ -91,19 +92,22 @@ describe("gitmostInsertTranscriptIntoEditor", () => { editor.destroy(); }); - it("neutralizes col-0 markdown block triggers with a leading ZWSP (git-sync safety)", () => { + it("inserts col-0 markdown block triggers as verbatim paragraph text (no ZWSP workaround)", () => { const editor = makeEditor(); - // Trigger lines (some with a leaked indent) + a normal prefixed line. + // Trigger lines (some with a leaked indent) + a normal prefixed line. The + // git-sync serializer now block-escapes a leading trigger itself, so the + // bridge inserts each line's TEXT byte-exact (only the leaked indent is + // trimmed) — no invisible ZWSP is prepended anymore. const inserted = gitmostInsertTranscriptIntoEditor( editor, [ "- dash", - " > quote", // leading indent must be trimmed then neutralized + " > quote", // leading indent is trimmed, text otherwise verbatim "# hash", "1. one", "> [!info] note", "```js", - "---", // solid thematic break -> horizontalRule (text-losing) if unneutralized + "---", "***", "___", "You: normal line", @@ -116,20 +120,23 @@ describe("gitmostInsertTranscriptIntoEditor", () => { .map((n: any) => n.content?.[0]?.text) .filter((t: any) => typeof t === "string") as string[]; - // Every block-trigger line is prefixed with the invisible ZWSP (indent - // trimmed first); the normal `You:` line is left byte-exact. + // Each trigger line is stored as its own byte-exact text (indent trimmed); + // the git-sync round-trip keeps it a paragraph via the serializer's + // block-escape, so no ZWSP is needed here. expect(texts).toEqual([ - ZWSP + "- dash", - ZWSP + "> quote", - ZWSP + "# hash", - ZWSP + "1. one", - ZWSP + "> [!info] note", - ZWSP + "```js", - ZWSP + "---", - ZWSP + "***", - ZWSP + "___", + "- dash", + "> quote", + "# hash", + "1. one", + "> [!info] note", + "```js", + "---", + "***", + "___", "You: normal line", ]); + // Guard: no invisible ZWSP leaked into any inserted line. + for (const t of texts) expect(t).not.toContain(ZWSP); editor.destroy(); }); diff --git a/apps/client/src/features/editor/gitmost/gitmost-recording.ts b/apps/client/src/features/editor/gitmost/gitmost-recording.ts index 2856ac08..1be0d638 100644 --- a/apps/client/src/features/editor/gitmost/gitmost-recording.ts +++ b/apps/client/src/features/editor/gitmost/gitmost-recording.ts @@ -240,45 +240,22 @@ export async function gitmostUploadFileToEditor( } } -// Zero-width space (U+200B). Prepended to a transcript line that begins with a -// markdown BLOCK trigger: it is invisible in the rendered doc but shifts the -// trigger off column 0, so the git-sync doc->markdown->doc round-trip keeps the -// line a plain paragraph (see GITMOST_MD_BLOCK_TRIGGER_RE). -const GITMOST_ZWSP = "​"; - -// A markdown BLOCK-level construct that, sitting at column 0 of a paragraph -// line, the git-sync markdown serializer (packages/prosemirror-markdown -// markdown-converter.ts, `case "paragraph"`) would re-parse into a NON-paragraph -// block on the doc->markdown->doc cycle. That serializer emits paragraph text -// verbatim with NO block-escape (the pre-existing root cause), so a leading -// `#`/`-`/`*`/`+`/`>`, an ordered-list `N.`/`N)`, a code fence ```/~~~, a table -// `|`, or a `> [!info]` callout opener would silently become a heading / list / -// quote / code block / table / callout. The final alternative matches a WHOLE- -// LINE thematic break — solid `---`/`***`/`___` or spaced `- - -`/`_ _ _` (3+ of -// the same `-`/`*`/`_`) — which round-trips into a `horizontalRule`; because -// that node carries NO text, an un-neutralized separator line would LOSE its -// text entirely (worse than the list/quote case). This matches a TRIMMED line's -// start; the transcript's own `You:` / `Speaker N:` prefix begins with a letter -// and never matches, so prefixed lines are left byte-exact. -const GITMOST_MD_BLOCK_TRIGGER_RE = - /^(?:#{1,6}(?:\s|$)|[-*+](?:\s|$)|>|\d+[.)](?:\s|$)|```|~~~|\||([-*_])(?:\s*\1){2,}\s*$)/; - // Append a transcript block BELOW the recording's audio node in a live editor: // a "Transcript" heading followed by one paragraph per non-empty transcript // line. The transcript is plain text, `\n`-separated, each line already // formatted as `You: ...` / `Speaker N: ...` by the native host — line text is // inserted as a TEXT node (never HTML/markdown), so there is no injection or // mark-parsing surface. Each kept line is trimmed (drops an indent that would -// both leak into the display and, at col 0, form a markdown block trigger) and, -// if it still begins with a col-0 markdown block trigger, gets an invisible -// zero-width space prepended so the git-sync round-trip cannot turn it into a -// list/quote/heading/callout/code/table (defensive boundary against the -// serializer's missing block-escape). This is best-effort and meant to run -// AFTER the audio has already been inserted; the caller must guard against a -// throw so a transcript failure never fails the (already successful) recording. -// Returns true when a block was inserted, false when there was nothing to -// insert (transcript undefined/empty/not-a-string). A non-string value is a -// no-op, not an error. +// leak into the display). A line that begins with a col-0 markdown block +// trigger (`#`/`-`/`>`/`1.`/fence/`---`/…) needs no client-side workaround: the +// git-sync serializer (packages/prosemirror-markdown, `case "paragraph"`) now +// block-escapes such a leading trigger, so the doc->markdown->doc round-trip +// keeps the line a paragraph on its own — the former invisible-ZWSP defense is +// gone. This is best-effort and meant to run AFTER the audio has already been +// inserted; the caller must guard against a throw so a transcript failure never +// fails the (already successful) recording. Returns true when a block was +// inserted, false when there was nothing to insert (transcript +// undefined/empty/not-a-string). A non-string value is a no-op, not an error. export function gitmostInsertTranscriptIntoEditor( editor: Editor, transcript: unknown, @@ -288,13 +265,7 @@ export function gitmostInsertTranscriptIntoEditor( .split("\n") // Trim each line and drop blank (whitespace-only) ones. .map((line) => line.trim()) - .filter((line) => line.length > 0) - // Neutralize a col-0 markdown block trigger with an invisible ZWSP so the - // git-sync round-trip keeps the line a paragraph. Host lines (`You:` / - // `Speaker N:`) never match and stay byte-exact. - .map((line) => - GITMOST_MD_BLOCK_TRIGGER_RE.test(line) ? GITMOST_ZWSP + line : line, - ); + .filter((line) => line.length > 0); if (lines.length === 0) return false; const content = [ diff --git a/apps/client/src/features/share/components/share-alias-section.test.tsx b/apps/client/src/features/share/components/share-alias-section.test.tsx index f83e00c7..deef2426 100644 --- a/apps/client/src/features/share/components/share-alias-section.test.tsx +++ b/apps/client/src/features/share/components/share-alias-section.test.tsx @@ -13,8 +13,7 @@ let currentAlias: IShareAlias | null = null; let availabilityResult: { valid: boolean; available: boolean; - currentPageId: string | null; -} = { valid: true, available: true, currentPageId: null }; +} = { valid: true, available: true }; vi.mock("@/features/share/queries/share-query.ts", () => ({ useShareAliasForPageQuery: () => ({ data: currentAlias }), @@ -56,7 +55,7 @@ describe("ShareAliasSection — taken-name handling is never a dead end", () => beforeEach(() => { setMutateAsync.mockReset(); currentAlias = null; - availabilityResult = { valid: true, available: true, currentPageId: null }; + availabilityResult = { valid: true, available: true }; }); it("shows a 'will move it here' HINT (not a terminal error) when the name belongs to another page, and keeps Save enabled", async () => { @@ -65,7 +64,6 @@ describe("ShareAliasSection — taken-name handling is never a dead end", () => availabilityResult = { valid: true, available: false, - currentPageId: "page-X", }; renderSection("page-Y"); @@ -97,7 +95,6 @@ describe("ShareAliasSection — taken-name handling is never a dead end", () => availabilityResult = { valid: true, available: false, - currentPageId: "page-X", }; // The server rejects the un-confirmed save asking the client to confirm. setMutateAsync.mockRejectedValueOnce({ @@ -106,7 +103,6 @@ describe("ShareAliasSection — taken-name handling is never a dead end", () => status: 409, data: { code: "ALIAS_REASSIGN_REQUIRED", - currentPageId: "page-X", currentPageTitle: "Alias Test Page X", }, }, diff --git a/apps/client/src/features/share/components/share-alias-section.tsx b/apps/client/src/features/share/components/share-alias-section.tsx index a3938548..4e5cc261 100644 --- a/apps/client/src/features/share/components/share-alias-section.tsx +++ b/apps/client/src/features/share/components/share-alias-section.tsx @@ -48,7 +48,6 @@ export default function ShareAliasSection({ const [availability, setAvailability] = useState<{ valid: boolean; available: boolean; - currentPageId: string | null; } | null>(null); const [reassign, setReassign] = useState<{ alias: string; @@ -76,7 +75,6 @@ export default function ShareAliasSection({ setAvailability({ valid: res.valid, available: res.available, - currentPageId: res.currentPageId, }); } catch { setAvailability(null); diff --git a/apps/client/src/features/share/types/share.types.ts b/apps/client/src/features/share/types/share.types.ts index caba0b1b..0752d12e 100644 --- a/apps/client/src/features/share/types/share.types.ts +++ b/apps/client/src/features/share/types/share.types.ts @@ -108,7 +108,6 @@ export interface IShareAliasAvailability { alias: string; valid: boolean; available: boolean; - currentPageId: string | null; } export interface ISharedPageTree { diff --git a/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.tsx b/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.tsx index f3beb39b..aca7e406 100644 --- a/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.tsx +++ b/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.tsx @@ -28,6 +28,7 @@ import { IAiMcpServerCreate, IAiMcpServerUpdate, } from "@/features/workspace/services/ai-mcp-server-service.ts"; +import { resolveToolAllowlist } from "./ai-mcp-server-form.utils.ts"; const formSchema = z.object({ name: z.string().min(1), @@ -121,13 +122,20 @@ export default function AiMcpServerForm({ async function handleSubmit(values: FormValues) { const headers = resolveHeaders(); + // An empty tag field means "no restriction" (sent as null) — since #476 the + // server persists a literal `[]` as deny-all (zero tools). But a server that + // was ALREADY deny-all loads into an empty field too; sending null there + // would silently widen it to allow-all on a routine edit, so preserve `[]`. + // See resolveToolAllowlist for the full rationale. + const toolAllowlist = resolveToolAllowlist(values.toolAllowlist, server); + if (isEdit && server) { const payload: IAiMcpServerUpdate = { id: server.id, name: values.name, transport: values.transport, url: values.url, - toolAllowlist: values.toolAllowlist, + toolAllowlist, // Always sent: a blank value clears the stored guidance (server -> null). instructions: values.instructions, enabled: values.enabled, @@ -140,7 +148,7 @@ export default function AiMcpServerForm({ name: values.name, transport: values.transport, url: values.url, - toolAllowlist: values.toolAllowlist, + toolAllowlist, // Blank => server stores null (no guidance). instructions: values.instructions, enabled: values.enabled, diff --git a/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.utils.test.ts b/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.utils.test.ts new file mode 100644 index 00000000..cb3e124d --- /dev/null +++ b/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.utils.test.ts @@ -0,0 +1,29 @@ +import { describe, it, expect } from "vitest"; +import { resolveToolAllowlist } from "./ai-mcp-server-form.utils.ts"; + +describe("resolveToolAllowlist", () => { + it("sends the typed tools when the field is non-empty", () => { + expect(resolveToolAllowlist(["a", "b"], { toolAllowlist: null })).toEqual([ + "a", + "b", + ]); + }); + + it("creates as null (unrestricted) when empty and there is no server", () => { + expect(resolveToolAllowlist([], undefined)).toBeNull(); + }); + + it("sends null for an empty field on a previously-unrestricted server", () => { + expect(resolveToolAllowlist([], { toolAllowlist: null })).toBeNull(); + }); + + it("preserves deny-all: an empty field on a `[]` server stays `[]`, not null", () => { + // The core #476/#477 guard: editing a deny-all server (rename/toggle) with + // an empty tag field must NOT silently widen it to allow-all. + expect(resolveToolAllowlist([], { toolAllowlist: [] })).toEqual([]); + }); + + it("still sends explicit tools even if the server was deny-all", () => { + expect(resolveToolAllowlist(["x"], { toolAllowlist: [] })).toEqual(["x"]); + }); +}); diff --git a/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.utils.ts b/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.utils.ts new file mode 100644 index 00000000..e40bf820 --- /dev/null +++ b/apps/client/src/features/workspace/components/settings/components/ai-mcp-server-form.utils.ts @@ -0,0 +1,22 @@ +import { IAiMcpServer } from "@/features/workspace/services/ai-mcp-server-service.ts"; + +// Resolve the tool allowlist value to persist from the form field. +// +// An empty tag field normally means "no restriction" and is sent as null so +// the server drops the column (all tools allowed). But a server that was +// ALREADY deny-all (a stored literal `[]`, meaning zero tools — creatable via +// the API) loads into the form as an empty field too. Coercing that empty +// field to null on submit would SILENTLY widen a deny-all server to allow-all +// on any routine edit (rename, toggle) — the exact silent-widen class #476 +// closed on the read side. So when the edited server was deny-all, preserve +// `[]` (deny-all); only a genuinely-unrestricted server (stored null/absent) +// stays null. +export function resolveToolAllowlist( + fieldValue: string[], + server?: Pick, +): string[] | null { + if (fieldValue.length > 0) return fieldValue; + const wasDenyAll = + Array.isArray(server?.toolAllowlist) && server.toolAllowlist.length === 0; + return wasDenyAll ? [] : null; +} diff --git a/apps/client/src/features/workspace/components/settings/components/ai-provider-settings.spec.tsx b/apps/client/src/features/workspace/components/settings/components/ai-provider-settings.spec.tsx index 79f94ab7..5c9fcc19 100644 --- a/apps/client/src/features/workspace/components/settings/components/ai-provider-settings.spec.tsx +++ b/apps/client/src/features/workspace/components/settings/components/ai-provider-settings.spec.tsx @@ -6,6 +6,8 @@ import { nextReindexPollInterval, isReindexComplete, isReindexButtonLoading, + reindexRunKey, + isNewReindexRun, } from './ai-provider-settings'; describe('resolveCardStatus', () => { @@ -221,6 +223,128 @@ describe('isReindexComplete', () => { }); }); +describe('reindexRunKey', () => { + it('is null when the status carries no run identity', () => { + expect(reindexRunKey(undefined)).toBeNull(); + expect( + reindexRunKey({ reindexing: false, indexedPages: 5, totalPages: 5 }), + ).toBeNull(); + }); + + it('is null for a legacy/degraded record with an empty runId', () => { + // The server sends runId='' for a record written before the field existed; + // the client must treat that as "no identity" (fall back to prior behaviour). + expect( + reindexRunKey({ + reindexing: true, + indexedPages: 0, + totalPages: 10, + runId: '', + reindexStartedAt: 1000, + }), + ).toBeNull(); + }); + + it('folds runId and startedAt into one stable key', () => { + expect( + reindexRunKey({ + reindexing: true, + indexedPages: 0, + totalPages: 10, + runId: 'run-a', + reindexStartedAt: 1000, + }), + ).toBe('run-a:1000'); + }); + + it('changes when the runId changes for the same startedAt', () => { + const a = reindexRunKey({ + reindexing: true, + indexedPages: 0, + totalPages: 10, + runId: 'run-a', + reindexStartedAt: 1000, + }); + const b = reindexRunKey({ + reindexing: true, + indexedPages: 0, + totalPages: 10, + runId: 'run-b', + reindexStartedAt: 1000, + }); + expect(a).not.toBe(b); + }); + + it('changes when the same runId restarts at a new startedAt', () => { + const a = reindexRunKey({ + reindexing: true, + indexedPages: 0, + totalPages: 10, + runId: 'run-a', + reindexStartedAt: 1000, + }); + const b = reindexRunKey({ + reindexing: true, + indexedPages: 0, + totalPages: 10, + runId: 'run-a', + reindexStartedAt: 2000, + }); + expect(a).not.toBe(b); + }); +}); + +describe('isNewReindexRun (poll keying on runId)', () => { + // Derive the status shape from the helper itself so the test needs no export + // of the component-internal ReindexStatus type. + type ReindexStatusLike = NonNullable[0]>; + const run = (runId: string, startedAt: number): ReindexStatusLike => ({ + reindexing: true, + indexedPages: 0, + totalPages: 10, + runId, + reindexStartedAt: startedAt, + }); + + it('first identity after none latched is a NEW run', () => { + expect(isNewReindexRun(null, run('run-a', 1000))).toBe(true); + }); + + it('the SAME identity is not a new run (same run being watched)', () => { + const key = reindexRunKey(run('run-a', 1000)); + expect(isNewReindexRun(key, run('run-a', 1000))).toBe(false); + }); + + it('a DIFFERENT runId is a new run (reset per-run poll state)', () => { + const key = reindexRunKey(run('run-a', 1000)); + expect(isNewReindexRun(key, run('run-b', 1000))).toBe(true); + }); + + it('an identity-less poll (no runId / cleared record) is never a new run', () => { + const key = reindexRunKey(run('run-a', 1000)); + expect( + isNewReindexRun(key, { + reindexing: false, + indexedPages: 10, + totalPages: 10, + }), + ).toBe(false); + }); + + it('a legacy empty-runId poll does not spuriously reset a latched run', () => { + const key = reindexRunKey(run('run-a', 1000)); + expect( + isNewReindexRun(key, { + reindexing: true, + indexedPages: 3, + totalPages: 10, + runId: '', + reindexStartedAt: 1000, + }), + ).toBe(false); + }); +}); + describe('isReindexButtonLoading', () => { it('loads while the POST mutation is pending', () => { expect( diff --git a/apps/client/src/features/workspace/components/settings/components/ai-provider-settings.tsx b/apps/client/src/features/workspace/components/settings/components/ai-provider-settings.tsx index 1a0024ac..09ae0dd6 100644 --- a/apps/client/src/features/workspace/components/settings/components/ai-provider-settings.tsx +++ b/apps/client/src/features/workspace/components/settings/components/ai-provider-settings.tsx @@ -173,9 +173,43 @@ export function resolveKeyField( // Subset of the status payload that drives the reindex poll decisions. type ReindexStatus = Pick< IAiSettings, - "reindexing" | "indexedPages" | "totalPages" + "reindexing" | "indexedPages" | "totalPages" | "runId" | "reindexStartedAt" >; +/** + * A stable per-RUN key for the reindex poll: `runId:startedAt`, or `null` when + * the status carries no run identity (no active run, or a legacy/degraded + * server record with an empty runId). Two polls of the SAME run share a key; a + * new run mints a fresh runId and so a different key. + * + * This is the single place the client turns the server's run identity into the + * value it keys on — it removes the "is this the same run I've been watching or + * a brand-new one?" ambiguity that made a class of reindex-status bugs (a stale + * pre-reindex snapshot vs a fresh run) get fixed twice (#262). `startedAt` is + * folded in so a run that somehow reuses a runId but restarted is still new. + */ +export function reindexRunKey(status: ReindexStatus | undefined): string | null { + const runId = status?.runId; + if (!runId) return null; + return `${runId}:${status?.reindexStartedAt ?? ""}`; +} + +/** + * Decide whether the latest poll represents a NEW reindex run relative to the + * run key the client last latched (`prevKey`, `null` if none yet). True only + * when the status carries an identity AND it differs from the latched one — the + * signal to reset any per-run poll state (the "seen active" latch / progress the + * UI held). The same identity (or no identity) is NOT a new run, so an unchanged + * or identity-less poll never resets mid-run. + */ +export function isNewReindexRun( + prevKey: string | null, + status: ReindexStatus | undefined, +): boolean { + const key = reindexRunKey(status); + return key !== null && key !== prevKey; +} + /** * Decide the TanStack Query `refetchInterval` while a reindex may be running. * Returns the poll interval (ms) to keep polling, or `false` to stop. @@ -320,6 +354,13 @@ export default function AiProviderSettings() { // counter at 0 until a manual reload. A ref (not state) because it must not // trigger a render and is only ever read where `reindexing` is already false. const reindexSeenActiveRef = useRef(false); + // The run identity (runId:startedAt) the current poll window is keyed on. When + // a poll reports a DIFFERENT runId the server has started a NEW run, so we + // re-latch to it and reset `reindexSeenActiveRef` — a fresh run must never + // inherit the previous run's "seen active"/completion state (which would stop + // polling immediately or read the old run's counters as this run's). null = + // no run keyed yet (steady state, or a legacy record without a runId). + const reindexRunKeyRef = useRef(null); // Only admins may read the (masked) AI settings; the server enforces this too. const { data: settings, isLoading } = useAiSettingsQuery(isAdmin, (query) => @@ -336,6 +377,14 @@ export default function AiProviderSettings() { // unmount because the deadline state goes away with the component. useEffect(() => { if (reindexDeadline === null) return; + // Key the poll on the run identity: if this poll carries a runId different + // from the one we latched, the server started a NEW run, so adopt it and + // drop the per-run "seen active" latch (a fresh run must not inherit the + // previous run's completion state). Same runId => same run, leave it alone. + if (isNewReindexRun(reindexRunKeyRef.current, settings)) { + reindexRunKeyRef.current = reindexRunKey(settings); + reindexSeenActiveRef.current = false; + } // Latch "we have seen the active run" the moment a poll reports it, so the // completion check below (and the refetchInterval's) only fires once the run // has genuinely started — never on the stale pre-reindex snapshot. @@ -1220,6 +1269,10 @@ export default function AiProviderSettings() { // immediately. onSuccess: () => { reindexSeenActiveRef.current = false; + // Forget the previous run's identity so the first poll of + // this window (carrying the new run's runId) is recognized + // as a new run and keyed afresh. + reindexRunKeyRef.current = null; setReindexDeadline(Date.now() + REINDEX_POLL_CAP_MS); }, }) diff --git a/apps/client/src/features/workspace/services/ai-mcp-server-service.ts b/apps/client/src/features/workspace/services/ai-mcp-server-service.ts index 782e1412..84f27d92 100644 --- a/apps/client/src/features/workspace/services/ai-mcp-server-service.ts +++ b/apps/client/src/features/workspace/services/ai-mcp-server-service.ts @@ -27,7 +27,9 @@ export interface IAiMcpServerCreate { // Auth headers map (e.g. { Authorization: 'Bearer ...' }). Encrypted on save; // never returned. headers?: Record; - toolAllowlist?: string[]; + // Omit/null => no restriction; `[]` is persisted verbatim and means + // deny-all (zero tools) since #476. + toolAllowlist?: string[] | null; // Admin-authored prompt guidance (#180). Blank => stored as null. instructions?: string; enabled?: boolean; @@ -43,7 +45,9 @@ export interface IAiMcpServerUpdate { transport?: McpTransport; url?: string; headers?: Record; - toolAllowlist?: string[]; + // Absent => unchanged; null => no restriction; `[]` is persisted verbatim + // and means deny-all (zero tools) since #476. + toolAllowlist?: string[] | null; // Admin-authored prompt guidance (#180). Absent => unchanged; blank => cleared. instructions?: string; enabled?: boolean; diff --git a/apps/client/src/features/workspace/services/ai-settings-service.ts b/apps/client/src/features/workspace/services/ai-settings-service.ts index e12d1ebb..16b90674 100644 --- a/apps/client/src/features/workspace/services/ai-settings-service.ts +++ b/apps/client/src/features/workspace/services/ai-settings-service.ts @@ -51,6 +51,14 @@ export interface IAiSettings { // True while a full workspace reindex is actively running; the counts above // then reflect the live run progress (done climbs 0 -> total). reindexing?: boolean; + // Identity of the ACTIVE reindex run (present only while `reindexing`). The + // poll keys on `runId`: a changed value means a NEW run (reset the per-run + // poll state the UI latched), the same value is the run already being watched. + // Absent/empty ('') => no identity available; the client keeps prior behaviour. + runId?: string; + // Epoch-ms the active run started; paired with `runId` so a restart with a + // recycled id is still detected as a new run. + reindexStartedAt?: number; } // Update payload. Key semantics (same for `apiKey` and `embeddingApiKey`): diff --git a/apps/client/src/lib/telemetry/route-template.test.ts b/apps/client/src/lib/telemetry/route-template.test.ts index aa798a3d..4b16883e 100644 --- a/apps/client/src/lib/telemetry/route-template.test.ts +++ b/apps/client/src/lib/telemetry/route-template.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { templateRoute } from "./route-template"; +import { templateRoute, KNOWN_ROUTE_TEMPLATES } from "./route-template"; describe("templateRoute", () => { it("templates a space page path (never leaks slugs)", () => { @@ -32,4 +32,30 @@ describe("templateRoute", () => { expect(templateRoute("/weird/unknown/thing")).toBe("other"); expect(templateRoute("/s/team/p/slug/extra/segments")).toBe("other"); }); + + // The server's /api/telemetry/vitals mirror (ALLOWED_ROUTE_TEMPLATES) drops any + // route outside KNOWN_ROUTE_TEMPLATES, so templateRoute must NEVER emit a label + // that is not in that dictionary — otherwise legit client metrics get dropped. + it("only ever emits labels contained in KNOWN_ROUTE_TEMPLATES (#495)", () => { + const samples = [ + "/", + "/home", + "/settings/members", + "/settings/groups/g-1", + "/s/team", + "/s/team/trash", + "/s/team/p/slug", + "/p/slug", + "/share/abc", + "/share/abc/p/slug", + "/share/p/slug", + "/labels/urgent", + "/invites/inv-1", + "/weird/unknown/thing", // -> "other" + "/deep/unmatched/x/y/z", // -> "other" + ]; + for (const path of samples) { + expect(KNOWN_ROUTE_TEMPLATES.has(templateRoute(path))).toBe(true); + } + }); }); diff --git a/apps/client/src/lib/telemetry/route-template.ts b/apps/client/src/lib/telemetry/route-template.ts index 45fe4855..b3a29c9d 100644 --- a/apps/client/src/lib/telemetry/route-template.ts +++ b/apps/client/src/lib/telemetry/route-template.ts @@ -44,6 +44,22 @@ const STATIC_ROUTES = new Set([ '/settings/sharing', ]); +/** + * The COMPLETE, finite vocabulary `templateRoute` can ever emit: the two + * synthetic labels (`/` and `other`), the static routes, and the dynamic + * templates. Exported so the public `/api/telemetry/vitals` endpoint can reject + * any `route` outside this dictionary server-side (the endpoint is anonymous, so + * an un-checked `route` is a free-text write surface). The server keeps a mirror + * (`ALLOWED_ROUTE_TEMPLATES` in client-metrics.constants.ts) — this is the + * canonical source; keep them in lockstep. + */ +export const KNOWN_ROUTE_TEMPLATES: ReadonlySet = new Set([ + '/', + 'other', + ...STATIC_ROUTES, + ...ROUTE_PATTERNS.map((p) => p.template), +]); + export function templateRoute(pathname: string): string { // Normalise a trailing slash (except root). const path = diff --git a/apps/client/src/main.tsx b/apps/client/src/main.tsx index eb94771d..caa4545d 100644 --- a/apps/client/src/main.tsx +++ b/apps/client/src/main.tsx @@ -3,6 +3,7 @@ import "@mantine/spotlight/styles.css"; import "@mantine/notifications/styles.css"; import '@mantine/dates/styles.css'; import "@/styles/a11y-overrides.css"; +import "@/styles/notification-overrides.css"; import ReactDOM from "react-dom/client"; import App from "./App.tsx"; @@ -47,7 +48,15 @@ function renderApp() { - + {/* top-center: toasts sit in the top of the viewport, in the line + of sight, and no longer cover centered content (e.g. "Load + more"). The below-chrome vertical offset is applied via a + position-scoped CSS rule in notification-overrides.css (NOT an + inline `style`): Mantine renders all six position containers at + once and an inline root style would land on every one, giving the + bottom-* containers both top+bottom → full-viewport transparent + overlays that swallow clicks. */} + {/* Root boundary above every lazy route's Suspense: a stale-chunk 404 after a deploy is caught and recovered here instead of diff --git a/apps/client/src/styles/notification-overrides.css b/apps/client/src/styles/notification-overrides.css new file mode 100644 index 00000000..a1dbbe33 --- /dev/null +++ b/apps/client/src/styles/notification-overrides.css @@ -0,0 +1,64 @@ +/* + * Toast (Mantine Notification) visibility overrides. + * Mantine renders colorless toasts on --mantine-color-body (== the page + * background: white in light mode) with a faint shadow, so on white pages the + * card has no visible edge. These rules give every toast a type-tinted + * background, a WCAG-checked border and a stronger shadow so it separates from + * the page. The [data-mantine-color-scheme] + static-class selector (0,2,0) + * beats Mantine's own (0,1,0) rules regardless of stylesheet order (Mantine's + * bg/border rules wrap the scheme attribute in :where(), so they stay (0,1,0)). + * --notification-color is defined on the same element (defaults to primary, + * set per `color` prop), so tint/border follow the toast type. This also covers + * the loading/import toast (no accent bar, since the spinner takes the icon + * slot): its visibility comes from tone + border + shadow + the colored spinner. + */ + +/* + * Push the top-anchored toast containers below the top chrome (fixed 45px + * header + optional 45px format toolbar + ~6px gap) so a toast (z-index 10000) + * neither covers nor intercepts clicks on the header/toolbar (both z-index 99). + * + * Scoped to [data-position^='top'] on purpose. Mantine renders ALL SIX position + * containers simultaneously (`position` only routes toasts into one via the + * store); the root `style` prop would be applied to every one of them by + * getStyles("root"). A blanket `top` would land on the bottom-* containers too + * (which carry `bottom:16px`) → position:fixed + both edges + height:auto makes + * them stretch the full viewport height, and the container root has neither + * pointer-events:none nor a background, so those transparent z-10000 overlays + * would swallow clicks across the whole page. Restricting to top-* leaves the + * bottom containers at height:0. + * + * Specificity: `.mantine-Notifications-root[data-position^='top']` is (0,2,0) + * (class + attribute) and beats Mantine's own top rule + * `.m_b37d9ac7:where([data-position='top-center']){top:16px}` which is (0,1,0) + * (the :where() contributes 0), regardless of stylesheet order. + */ +.mantine-Notifications-root[data-position^='top'] { + top: 96px; +} + +[data-mantine-color-scheme='light'] .mantine-Notification-root { + /* ~10% type color over white: clearly off-white, text contrast preserved */ + background-color: color-mix(in srgb, var(--notification-color) 10%, var(--mantine-color-white)); + /* Border must clear WCAG 3:1 non-text contrast on white. The repo rejects + gray-4 for this (a11y-overrides.css); gray-6 base (~3.32:1) darkened by the + type color stays >= 3:1. */ + border: 1px solid color-mix(in srgb, var(--notification-color) 45%, var(--mantine-color-gray-6)); + box-shadow: var(--mantine-shadow-xl); +} + +[data-mantine-color-scheme='dark'] .mantine-Notification-root { + /* Dark page (dark-7/8) vs toast (dark-6) already separate a little; border + + shadow carry the type cue here (a 7% dark tint was near-invisible). */ + background-color: color-mix(in srgb, var(--notification-color) 14%, var(--mantine-color-dark-6)); + border: 1px solid color-mix(in srgb, var(--notification-color) 45%, var(--mantine-color-dark-3)); + box-shadow: var(--mantine-shadow-xl); +} + +/* Mantine's message-with-title color is gray-6 (#868e96, already only ~3.32:1 + on white — below AA 4.5:1); the new tint pushes it lower. Bump to gray-7 to + keep multi-line colored toasts readable, consistent with the repo's existing + WCAG tuning (theme.ts already bumps this same gray-6 up elsewhere). */ +[data-mantine-color-scheme='light'] .mantine-Notification-description[data-with-title] { + color: var(--mantine-color-gray-7); +} diff --git a/apps/server/package.json b/apps/server/package.json index a6791d80..7b97752a 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -23,7 +23,7 @@ "migration:reset": "tsx src/database/migrate.ts down-to NO_MIGRATIONS", "migration:codegen": "kysely-codegen --dialect=postgres --camel-case --env-file=../../.env --out-file=./src/database/types/db.d.ts", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", - "pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build", + "pretest": "pnpm --filter @docmost/editor-ext build && pnpm --filter @docmost/prosemirror-markdown build && pnpm --filter @docmost/token-estimate build", "test": "jest", "test:int": "jest --config test/jest-integration.json", "test:watch": "jest --watch", @@ -44,6 +44,7 @@ "@docmost/mcp": "workspace:*", "@docmost/pdf-inspector": "1.9.6", "@docmost/prosemirror-markdown": "workspace:*", + "@docmost/token-estimate": "workspace:*", "@fastify/compress": "^9.0.0", "@fastify/cookie": "^11.0.2", "@fastify/multipart": "^10.0.0", @@ -206,6 +207,7 @@ "^@docmost/db/(.*)$": "/database/$1", "^@docmost/transactional/(.*)$": "/integrations/transactional/$1", "^@docmost/ee/(.*)$": "/ee/$1", + "^@docmost/token-estimate$": "/../../../packages/token-estimate/src/index.ts", "^src/(.*)$": "/$1", "^@tiptap/react$": "/../test/stubs/tiptap-react.js" } diff --git a/apps/server/src/app.module.ts b/apps/server/src/app.module.ts index 527035b9..c46310f6 100644 --- a/apps/server/src/app.module.ts +++ b/apps/server/src/app.module.ts @@ -25,7 +25,7 @@ import { CacheModule } from '@nestjs/cache-manager'; import KeyvRedis from '@keyv/redis'; import { LoggerModule } from './common/logger/logger.module'; import { ClsModule } from 'nestjs-cls'; -import { NoopAuditModule } from './integrations/audit/audit.module'; +import { AuditModule } from './integrations/audit/audit.module'; import { ThrottleModule } from './integrations/throttle/throttle.module'; import { McpModule } from './integrations/mcp/mcp.module'; import { SandboxModule } from './integrations/sandbox/sandbox.module'; @@ -55,7 +55,7 @@ try { middleware: { mount: true }, }), LoggerModule, - NoopAuditModule, + AuditModule, CoreModule, DatabaseModule, EnvironmentModule, diff --git a/apps/server/src/collaboration/yjs.util.spec.ts b/apps/server/src/collaboration/yjs.util.spec.ts index 8f6fc2e4..f35e44f4 100644 --- a/apps/server/src/collaboration/yjs.util.spec.ts +++ b/apps/server/src/collaboration/yjs.util.spec.ts @@ -529,4 +529,107 @@ describe('replaceYjsMarkedText', () => { expect(result).toEqual({ applied: false, currentText: 'abcdef' }); expect(text.toDelta()).toEqual(before); }); + + // #496: apply must NOT silently strip the replaced run's inline formatting. + // Build a paragraph and format the marked range with extra marks, then assert + // the replacement carries them. + function buildFormatted( + runs: Array<{ text: string; attrs?: Record }>, + ): { fragment: Y.XmlFragment; text: Y.XmlText } { + const ydoc = new Y.Doc(); + const fragment = ydoc.getXmlFragment('default'); + const para = new Y.XmlElement('paragraph'); + fragment.insert(0, [para]); + const text = new Y.XmlText(); + para.insert(0, [text]); + text.insert(0, runs.map((r) => r.text).join('')); + let offset = 0; + for (const run of runs) { + if (run.attrs) text.format(offset, run.text.length, run.attrs); + offset += run.text.length; + } + return { fragment, text }; + } + + it('preserves the original run formatting (bold + link) on the replacement', () => { + const { fragment, text } = buildFormatted([ + { text: 'see ' }, + { + text: 'old', + attrs: { + comment: { commentId: 'c1', resolved: false }, + bold: true, + link: { href: 'https://x.test' }, + }, + }, + { text: ' end' }, + ]); + + const result = replaceYjsMarkedText(fragment, 'c1', 'old', 'new'); + + expect(result).toEqual({ applied: true, currentText: 'new' }); + // The comment anchor AND the bold/link marks survive the delete+insert. + expect(text.toDelta()).toEqual([ + { insert: 'see ' }, + { + insert: 'new', + attributes: { + comment: { commentId: 'c1', resolved: false }, + bold: true, + link: { href: 'https://x.test' }, + }, + }, + { insert: ' end' }, + ]); + }); + + it('mixed formatting under the mark: replacement takes the DOMINANT (longest) run, NOT the leading one', () => { + // Leading run is SHORT + plain ("x", 1 char); the following run is LONGER + + // bold ("bolded", 6 chars), same commentId. The longest run is deliberately + // NOT first: a "first-wins" pick would carry plain (no bold), so asserting + // bold on the result only holds if the code genuinely selects the LONGEST run. + const { fragment, text } = buildFormatted([ + { text: 'x', attrs: { comment: { commentId: 'c1', resolved: false } } }, + { + text: 'bolded', + attrs: { comment: { commentId: 'c1', resolved: false }, bold: true }, + }, + ]); + + const result = replaceYjsMarkedText(fragment, 'c1', 'xbolded', 'Z'); + + expect(result).toEqual({ applied: true, currentText: 'Z' }); + expect(text.toDelta()).toEqual([ + { + insert: 'Z', + attributes: { comment: { commentId: 'c1', resolved: false }, bold: true }, + }, + ]); + }); + + it('mixed formatting under the mark: on a length tie the FIRST run wins', () => { + // Two equal-length runs (2 chars each) with different formatting, same + // commentId. The reduce keeps the accumulator on a tie, so the FIRST run + // (italic) prevails over the later bold one. + const { fragment, text } = buildFormatted([ + { + text: 'AA', + attrs: { comment: { commentId: 'c1', resolved: false }, italic: true }, + }, + { + text: 'BB', + attrs: { comment: { commentId: 'c1', resolved: false }, bold: true }, + }, + ]); + + const result = replaceYjsMarkedText(fragment, 'c1', 'AABB', 'Z'); + + expect(result).toEqual({ applied: true, currentText: 'Z' }); + expect(text.toDelta()).toEqual([ + { + insert: 'Z', + attributes: { comment: { commentId: 'c1', resolved: false }, italic: true }, + }, + ]); + }); }); diff --git a/apps/server/src/collaboration/yjs.util.ts b/apps/server/src/collaboration/yjs.util.ts index bea28dfc..03e9d687 100644 --- a/apps/server/src/collaboration/yjs.util.ts +++ b/apps/server/src/collaboration/yjs.util.ts @@ -145,6 +145,10 @@ type MarkedSegment = { length: number; text: string; markAttrs: Record; + // The FULL attribute set of this delta run — the `comment` mark plus any + // inline formatting (bold/italic/code/link/…). Captured so apply can carry the + // original run's formatting onto the replacement instead of dropping it. + attributes: Record; }; /** @@ -202,6 +206,7 @@ export function replaceYjsMarkedText( length, text: insert, markAttrs: markAttr, + attributes, }); } offset += length; @@ -251,15 +256,25 @@ export function replaceYjsMarkedText( return { applied: false, currentText: joinedText }; } - // 3. All guards passed: delete the marked run and re-insert newText with the - // same comment attributes at the same offset. Atomic within the caller's - // transaction. + // 3. All guards passed: delete the marked run and re-insert newText at the + // same offset. Atomic within the caller's transaction. const start = segments[0].offset; const len = segments.reduce((sum, s) => sum + s.length, 0); - const markAttrs = segments[0].markAttrs; + + // Carry the ORIGINAL run's formatting onto the replacement (#496): inserting + // with only the `comment` mark silently dropped bold/italic/code/link of the + // replaced text. Yjs applies one flat attribute set to the whole insert, so + // when the marked run mixes formatting we pick the DOMINANT segment (the one + // covering the most characters) and apply its attributes — a v1 that preserves + // the common single-format case exactly and, for a mixed run, keeps the + // prevailing style rather than losing all of it. `attributes` already carries + // the `comment` mark (every collected segment is filtered on it above), so the + // anchor is preserved by copying the run's attribute set verbatim. + const dominant = segments.reduce((a, b) => (b.length > a.length ? b : a)); + const insertAttrs = { ...dominant.attributes }; node.delete(start, len); - node.insert(start, newText, { comment: markAttrs }); + node.insert(start, newText, insertAttrs); return { applied: true, currentText: newText }; } diff --git a/apps/server/src/core/ai-chat/ai-chat-stream-registry.service.ts b/apps/server/src/core/ai-chat/ai-chat-stream-registry.service.ts index 1ef476f5..e338d01a 100644 --- a/apps/server/src/core/ai-chat/ai-chat-stream-registry.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat-stream-registry.service.ts @@ -1,42 +1,122 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; /** - * In-memory run-stream registry (#184 phase 1.5). A durable agent run tees its - * SSE frames here (via `pipeUIMessageStreamToResponse({ consumeSseStream })`) - * so a LATE tab — one that reloaded, or opened after the starter dropped — can - * attach through `GET /ai-chat/runs/:chatId/stream`, replay the frames buffered - * so far, and then follow the live tail as a normal streamer. + * In-memory run-stream registry (#184 phase 1.5, step-aligned retention #491). A + * durable agent run tees its SSE frames here (via + * `pipeUIMessageStreamToResponse({ consumeSseStream })`) so a LATE tab — one that + * reloaded, or opened after the starter dropped — can attach through + * `GET /ai-chat/runs/:chatId/stream`, be handed the TAIL past the step it already + * has persisted, and then follow the live tail as a normal streamer. * * This is deliberately single-process and best-effort: it holds nothing the DB * does not (the run + assistant row are the source of truth), so a process * restart simply drops in-flight entries and the client falls back to its * restore + degraded-poll path. The async `attach` return type is the seam for a * future phase-2 cross-process backend (Redis) — the interface does not change. + * + * ── #491 step-aligned retention (the OOM fix) ──────────────────────────────── + * The old registry buffered up to 32MB of raw SSE frames PER active run (V8 ~2× + * in memory) and, on attach, blasted the WHOLE buffer to the socket synchronously + * with no drain — a handful of marathon runs on a 1GB container OOM'd. #491 caps + * the ring at a few MB (env-tunable, default 4MB) and keeps it there by ROTATING: + * + * - Every buffered frame is STAMPED with a step number at tee (see ingestFrame). + * Convention: the stamp of a frame is the number of `finish-step` parts seen + * BEFORE it (starting at 0). The finish-step frame itself carries the current + * value, THEN the counter increments. So a frame stamped `s` is the content of + * the (s+1)-th step — 0-based step index `s` — and the stamp aligns EXACTLY + * with `metadata.stepsPersisted`: a client whose persisted `stepsPersisted` is + * N has steps 0..N-1 on disk (and in its seed) and needs the tail `stamp >= N`. + * + * - The ring rotates ONLY on a CONFIRMED persist of step N + * (`confirmPersistedStep`), dropping frames with `stamp < N` (those steps are + * now on disk and a fresh client seed carries them). A NON-confirmed step is + * never rotated away, so a persist FAILURE just makes the ring cover MORE + * (auto-safe). This is the anti-inversion rule: a naive "rotate in .then()" + * that rotated after an UNwritten step would drop a step nobody has → silent + * hole. Rotation is gated on a real, successful persist. + * + * - If the ring still exceeds its byte cap after rotation (a single fat step, or + * a lagging persist), the OLDEST frames are evicted to stay bounded. Evicting a + * not-yet-persisted frame opens a GAP: an attach whose N falls at or below an + * evicted step answers 204 and the client degrades to restore+poll. The gap is + * NOT sticky — the coverage floor is recomputed from the ring, so a later + * persist that rotates past the holey steps clears it. + * + * ── attach numbering / coverage (the wire convention) ──────────────────────── + * The step marker N comes ONLY FROM THE CLIENT (a query param). The server never + * reads the row to derive N — a server-side N from a stale seed would open a + * silent one-step hole. N is the client's persisted `stepsPersisted` (a COUNT): + * - the tail it needs = frames with `stamp >= N`; + * - coverage is OK ⟺ `coverageFloor(entry) <= N`, where coverageFloor is the + * smallest step FULLY present in the ring (its smallest retained stamp, bumped + * by one when that leading step was only partially evicted by overflow). If + * `coverageFloor > N` the ring starts AFTER the client's frontier (a hole, or + * the client's seed simply lagged behind a rotation) → 204 → the client + * refetches (a larger N) and re-attaches. + * The N cutoff is applied in ALL branches, INCLUDING the finished-retained replay. + * + * ── same-tick invariants (unchanged, still load-bearing) ───────────────────── + * invariant 1: only the matching run may mutate/observe an entry (runId check). + * invariant 2: retention deletes ONLY its own entry (a replacement may own the key). + * invariant 3: open() over a live entry mirrors the done-path (subscribers released). + * invariant 4: the tail SLICE + subscriber registration happen in ONE synchronous + * tick inside attach() — no await between them — so a concurrently + * ingested frame is EITHER in the snapshot (buffered before the sync + * block, and the just-added subscriber never sees it) OR fanned out to + * the paused subscriber's `pending` (ingested after) — never both and + * never neither: no loss, no duplication. NOTE (#491): the controller + * now AWAITS the drain-respecting tail write BEFORE calling start(), so + * frames ingested during that await accumulate in `pending`; this is + * bounded by the subscriber cap (an overflow degrades start() to an + * end(), a 204-equivalent). It is the SYNCHRONOUS snapshot+registration + * — not a same-tick start() — that makes this correct. + * invariant 5: the controller wires close-cleanup BEFORE any write. + * invariant 6: no cross-run replay — the `anchor` (the client's assistant row id) + * must match this run's assistant id, or a foreign run's transcript + * would be appended to the client's message. */ /** How long a finished entry is retained for late attach (replay + immediate end). */ export const RUN_STREAM_RETAIN_FINISHED_MS = 30_000; /** - * Per-run replay buffer cap. Past this the buffer is dropped (attach -> 204, and - * the client falls back to its restore + degraded-poll path, #430). - * - * Raised from 4MB to 32MB (#430): marathon autonomous runs (11-25 min observed) - * stream far more than 4MB of SSE frames, so a live disconnect mid-run would find - * an already-overflowed buffer and could only degrade-poll instead of re-attaching - * to the live tail. 32MB comfortably covers those runs while staying bounded. - * - * Memory cost: this is the WORST-CASE retained size PER ACTIVE run (the buffer is - * freed on finish + retention, or dropped immediately on overflow). With the small - * number of concurrent autonomous runs a single workspace realistically has, 32MB - * each is an acceptable ceiling; the overflow->204->degraded-poll fallback remains - * the backstop for anything larger, so correctness never depends on this bound. + * DEFAULT per-run replay ring cap (#491, down from 32MB). SSE frames carry + * UNcompacted tool outputs + framing overhead (×1.5–2 vs the persisted parts), so + * a "2–3 large reads + reasoning" step routinely blows past 2MB; 4MB comfortably + * holds a step or two of TAIL, which is all a resuming client needs (steps below + * its persisted frontier come from the seed, not the ring). The ring stays bounded + * because it rotates on every confirmed persist; this cap is only the ceiling for + * the un-persisted tail between rotations. Env-tunable via + * AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES (bytes); a 0/invalid value falls back to this. */ -export const RUN_STREAM_MAX_BUFFER_BYTES = 32 * 1024 * 1024; +export const AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES = 4 * 1024 * 1024; -// 2x the replay cap: a just-written full-replay burst alone can never trip the -// per-subscriber cap (see controller); only a genuinely stalled socket can. -export const SUBSCRIBER_MAX_BUFFERED_BYTES = 2 * RUN_STREAM_MAX_BUFFER_BYTES; +// 2× the ring cap: a just-written full-tail burst alone can never trip the +// per-subscriber cap (see controller); only a genuinely stalled socket can. This +// derivative relationship is preserved even when the ring cap is env-overridden. +export const SUBSCRIBER_MAX_BUFFERED_BYTES = 2 * AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES; + +/** + * A finish-step boundary frame is exactly `data: {"type":"finish-step"...}\n\n` + * (verified empirically against ai@6.0.207 — each UI-message-stream part is a + * single `data: {json}\n\n` event, never split across `data:` lines, and `type` + * is always the first key). A prefix match is cheaper than JSON.parse-per-frame + * and has no false positives: a literal `"type":"finish-step"` inside a text + * delta is JSON-escaped (`\"type\":...`), and the frame would start with + * `data: {"type":"text-delta"` anyway. + */ +const FINISH_STEP_FRAME_PREFIX = 'data: {"type":"finish-step"'; + +/** Resolve the ring cap from the environment, falling back to the default. */ +function resolveMaxBufferBytes(): number { + const raw = process.env.AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES; + if (!raw) return AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 + ? Math.floor(parsed) + : AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES; +} export interface RunStreamCallbacks { onFrame: (frame: string) => void; @@ -44,6 +124,9 @@ export interface RunStreamCallbacks { } export interface RunStreamAttachment { + // The synthetic `start` frame (carrying { runId, chatId }) followed by the + // buffered TAIL filtered to `stamp >= N`. The controller writes these to the + // socket in chunks respecting drain, then calls start(). replay: string[]; finished: boolean; start(): void; // drain pending frames (order preserved) and go live @@ -53,14 +136,19 @@ export interface RunStreamAttachment { interface Subscriber extends RunStreamCallbacks { started: boolean; pending: string[]; - // Byte size of `pending`, capped at SUBSCRIBER_MAX_BUFFERED_BYTES. `start()` is - // called in the SAME tick as `attach()` today (see attach), so `pending` never - // holds more than one microtask of frames — but the async `attach` signature is - // a phase-2 seam: an await between attach and start would let a stalled paused - // subscriber buffer the WHOLE run here. The cap is the structural backstop. + // Byte size of `pending`, capped at the subscriber cap. `start()` is called in + // the SAME tick as `attach()` today, so `pending` never holds more than one + // microtask of frames — but the controller writes the (potentially large) tail + // respecting drain BEFORE start(), so a stalled socket can accumulate here; the + // cap is the structural backstop (an overflow degrades start() to an end()). pendingBytes: number; overflowed: boolean; pendingEnd: boolean; + // The client's step frontier N: this subscriber only receives frames with + // `stamp >= minStamp` (the tail past what it already persisted). Live frames + // always satisfy this (their stamp is the current, highest step), so it only + // filters the rare out-of-order below-frontier frame. + minStamp: number; } interface Entry { @@ -68,8 +156,20 @@ interface Entry { // The persisted assistant row id of this run (set at bind; undefined if the // seed failed). Used by the attach anchor check (invariant 6). assistantMessageId?: string; + // Parallel arrays: frames[i] is the SSE string, stamps[i] its step number. frames: string[]; + stamps: number[]; bytes: number; + // The running step counter used to stamp the NEXT frame (number of finish-step + // frames seen so far). + currentStamp: number; + // The highest confirmed `stepsPersisted`: frames with stamp < persistedFloor are + // on disk (safe to drop, never re-buffered). Monotonic (confirmPersistedStep). + persistedFloor: number; + // The highest stamp EVICTED by an overflow (unsafe) drop, -1 if none. Used to + // detect a partially-evicted leading step when computing the coverage floor. + overflowThroughStamp: number; + // Sticky-for-logging only: at least one unsafe (overflow) eviction happened. overflowed: boolean; finished: boolean; subscribers: Set; @@ -80,6 +180,10 @@ interface Entry { export class AiChatStreamRegistryService implements OnModuleDestroy { private readonly logger = new Logger(AiChatStreamRegistryService.name); private readonly entries = new Map(); // key: chatId + // Env-resolved caps (per instance) so a deployment can tune the ceiling without + // a code change. The subscriber cap keeps the documented 2× relationship. + readonly maxBufferBytes = resolveMaxBufferBytes(); + readonly subscriberMaxBufferedBytes = 2 * this.maxBufferBytes; /** * Register a fresh entry at the START of a run (before any frame), so a tab @@ -105,7 +209,11 @@ export class AiChatStreamRegistryService implements OnModuleDestroy { this.entries.set(chatId, { runId, frames: [], + stamps: [], bytes: 0, + currentStamp: 0, + persistedFloor: 0, + overflowThroughStamp: -1, overflowed: false, finished: false, subscribers: new Set(), @@ -150,6 +258,34 @@ export class AiChatStreamRegistryService implements OnModuleDestroy { void pump(); } + /** + * Confirm that step `stepsPersisted` (a COUNT: steps 0..stepsPersisted-1) is on + * disk for this run, and ROTATE the ring: drop the buffered frames of those + * now-persisted steps (stamp < stepsPersisted). This is the ONLY thing that + * rotates the ring, and it is called ONLY after a genuinely SUCCESSFUL per-step + * persist (see ai-chat.service updateStreaming). A failed persist never calls + * it, so the ring covers more (auto-safe). Identity-checked (invariant 1) and + * monotonic (a stale lower count is ignored). + */ + confirmPersistedStep( + chatId: string, + runId: string, + stepsPersisted: number, + ): void { + const entry = this.entries.get(chatId); + if (!entry || entry.runId !== runId) return; + if (!Number.isFinite(stepsPersisted) || stepsPersisted <= entry.persistedFloor) + return; + entry.persistedFloor = stepsPersisted; + // Clean rotation: drop the persisted steps from the head. These frames are on + // disk + carried by a fresh client seed, so this NEVER opens a gap. + while (entry.frames.length > 0 && entry.stamps[0] < stepsPersisted) { + entry.bytes -= Buffer.byteLength(entry.frames[0]); + entry.frames.shift(); + entry.stamps.shift(); + } + } + /** * Terminate a run's entry from the OUTER catch of the stream method (a failure * before/while wiring the pipe, so `done` will never arrive). Identity-checked @@ -162,36 +298,77 @@ export class AiChatStreamRegistryService implements OnModuleDestroy { } /** - * Attach to a run's stream. Async only for the phase-2 Redis seam — the body - * runs synchronously so the replay snapshot and the subscriber registration - * happen in ONE tick with no await between them (invariant 4): a frame ingested - * concurrently cannot slip into the gap and be lost or duplicated. + * Attach to a run's stream from the client's step frontier `n` (its persisted + * `stepsPersisted`). Async only for the phase-2 Redis seam — the body runs + * synchronously so the tail SLICE and the subscriber registration happen in ONE + * tick with no await between them (invariant 4). * * Returns null (-> the caller answers 204) when: - * - there is no entry, or it overflowed (replay is gone); - * - expect=live with an anchor that does not match this run's assistant id - * (invariant 6: a stripped tab must never replay a FOREIGN run's transcript); - * - the run finished and the caller did not expect a live tail. - * A finished run with expect=live yields a replay-only attachment (no - * subscriber registered). Otherwise a paused subscriber is registered and the - * caller replays `replay`, then calls start() to drain and go live. + * - there is no entry; + * - the `anchor` does not match this run's assistant id (invariant 6); + * - the ring does not cover the client's frontier (coverageFloor > n): a hole + * from overflow, or the client's seed simply lagged behind a rotation. The + * client then refetches (a larger n) and re-attaches. + * + * Otherwise the attachment's `replay` is a synthetic `start` frame (the run-fact + * on re-attach) followed by the buffered tail filtered to `stamp >= n`. For a + * FINISHED run this is replay-only (no subscriber) and ends after the replay — + * with n = N_final that tail is just the run's `finish` frame, so the client + * closes the stream. For a LIVE run a paused subscriber is registered; the + * caller writes the replay (respecting drain) then calls start() to drain the + * pending frames and go live. */ async attach( chatId: string, - expectLive: boolean, anchor: string | undefined, + // The client's persisted step frontier. `null` = a NOT-tail-aware client (no + // `n` query param) — a legacy/parameterless tab that expects the old + // "finished -> 204 -> poll" contract; distinct from `0` (a tail-aware client + // with nothing persisted yet). + n: number | null, cb: RunStreamCallbacks, ): Promise { const entry = this.entries.get(chatId); - if (!entry || entry.overflowed) return null; + if (!entry) return null; // Invariant 6: cross-run replay is forbidden. Before bind, assistantMessageId // is undefined and mismatches any anchor -> 204 -> client restore+poll path. - if (expectLive && anchor && entry.assistantMessageId !== anchor) return null; - if (entry.finished && !expectLive) return null; - if (entry.finished && expectLive) { + if (anchor && entry.assistantMessageId !== anchor) return null; + // #491 regression guard (#137/#161 dup): a NOT-tail-aware client (no `n`) + // resuming a FINISHED run must 204 and poll — the old `finished && !expectLive` + // gate. Without this, a missing `n` collapsing to frontier 0 would serve the + // WHOLE tail of a finished, NON-rotated run (coverageFloor 0), and a + // parameterless client that never stripped its transcript would APPEND that + // full replay onto the steps it already shows -> duplicated text. A tail-aware + // client (n present, incl. n=0) still gets the tail past its frontier. + if (entry.finished && n === null) return null; + // A finished entry with NOTHING in the ring (aborted before the first frame, + // or fully overflowed) has no tail to deliver -> 204 -> the client polls. + if (entry.finished && entry.frames.length === 0) return null; + // A LIVE run with no `n` (legacy parameterless) replays from step 0 (the old + // behavior); a tail-aware client resumes from its frontier. + const frontier = n ?? 0; + const floor = this.coverageFloor(entry); + if (floor > frontier) { + this.logger.warn( + `run-stream attach gap for run=${entry.runId}: coverageFloor=${floor} ` + + `> client frontier=${frontier} -> 204 (client refetches + re-attaches)`, + ); + return null; + } + + const startFrame = this.buildStartFrame(chatId, entry.runId); + const sliceTail = (): string[] => { + const out: string[] = [startFrame]; + for (let i = 0; i < entry.frames.length; i++) { + if (entry.stamps[i] >= frontier) out.push(entry.frames[i]); + } + return out; + }; + + if (entry.finished) { // Replay-only: the run is done, no subscriber is registered. return { - replay: entry.frames.slice(), + replay: sliceTail(), finished: true, start: () => undefined, unsubscribe: () => undefined, @@ -206,15 +383,12 @@ export class AiChatStreamRegistryService implements OnModuleDestroy { pendingBytes: 0, overflowed: false, pendingEnd: false, + minStamp: frontier, }; + // Register + snapshot in the SAME synchronous block (invariant 4). No await + // separates them, so a concurrently ingested frame cannot be lost/duplicated. entry.subscribers.add(sub); - // Snapshot in the SAME synchronous block as the registration (invariant 4). - const replay = entry.frames.slice(); - // CONTRACT: the caller MUST call start() in the SAME tick as this attach() - // returns — no await between them. While a subscriber is paused, every frame - // is buffered in sub.pending; a delayed start() lets a whole run accumulate - // there. The pendingBytes cap (see ingestFrame) is the structural backstop if - // that contract is ever broken (e.g. the phase-2 Redis await seam). + const replay = sliceTail(); return { replay, finished: false, @@ -263,24 +437,83 @@ export class AiChatStreamRegistryService implements OnModuleDestroy { this.entries.clear(); } - /** Buffer + fan-out a single frame. See invariant/overflow semantics inline. */ + /** The synthetic `start` frame the tail is prefixed with — the source of the + * run-fact (runId/chatId) on re-attach. A `start` frame does NOT reset the + * client's message parts (ai@6.0.207 createStreamingUIMessageState), so it is + * safe to prepend even when the sliced tail begins mid-message. */ + private buildStartFrame(chatId: string, runId: string): string { + return `data: ${JSON.stringify({ + type: 'start', + messageMetadata: { runId, chatId }, + })}\n\n`; + } + + /** + * The smallest step FULLY present in the ring: its smallest retained stamp, or + * (when the leading step was only partially evicted by an overflow) one past it. + * When the ring is empty it is the current step (only the live tail is coming). + * An attach at frontier `n` is covered ⟺ coverageFloor <= n. + */ + private coverageFloor(entry: Entry): number { + // Empty ring: only the live tail is coming. The floor is the current step, + // but never below persistedFloor — a confirmed persist can rotate the ring + // empty while currentStamp still lags a beat behind on another connection, so + // max() keeps the invariant STRUCTURAL (a client with n = persistedFloor is + // always covered) rather than timing-dependent. + if (entry.frames.length === 0) + return Math.max(entry.currentStamp, entry.persistedFloor); + const min = entry.stamps[0]; + return entry.overflowThroughStamp >= min ? min + 1 : min; + } + + /** + * Buffer (step-stamped) + fan-out a single frame. The stamp is the number of + * finish-step frames seen BEFORE this one; a finish-step frame carries the + * current value and THEN increments the counter (so its stamp equals the 0-based + * index of the step it closes). Only frames at/above persistedFloor are buffered + * (already-persisted steps are on disk); the ring is then trimmed to the byte + * cap, an unsafe eviction opening a gap. Fan-out is always live (filtered per + * subscriber by its frontier). + */ private ingestFrame(entry: Entry, frame: string): void { - entry.bytes += Buffer.byteLength(frame); - if (!entry.overflowed) { + const size = Buffer.byteLength(frame); + const stamp = entry.currentStamp; + if (frame.startsWith(FINISH_STEP_FRAME_PREFIX)) { + entry.currentStamp = stamp + 1; + } + + // Buffer for replay only if this step is not already persisted+rotated away. + if (stamp >= entry.persistedFloor) { entry.frames.push(frame); - if (entry.bytes > RUN_STREAM_MAX_BUFFER_BYTES) { - // The crossing frame was already counted AND (below) fanned out; only the - // replay buffer is dropped. After overflow no more frames are buffered, - // but live fan-out continues. - entry.overflowed = true; - entry.frames = []; - this.logger.warn( - `run-stream buffer overflow for run=${entry.runId}; ` + - `late attach will 204 until the run ends`, - ); + entry.stamps.push(stamp); + entry.bytes += size; + // Enforce the ring cap. Evicting a not-yet-persisted frame (stamp >= + // persistedFloor) opens a GAP; a leftover persisted frame (< floor) is a + // safe drop. Keep evicting until the ring is back under the cap. + while (entry.bytes > this.maxBufferBytes && entry.frames.length > 0) { + const evStamp = entry.stamps[0]; + entry.bytes -= Buffer.byteLength(entry.frames[0]); + entry.frames.shift(); + entry.stamps.shift(); + if (evStamp >= entry.persistedFloor) { + if (evStamp > entry.overflowThroughStamp) + entry.overflowThroughStamp = evStamp; + if (!entry.overflowed) { + entry.overflowed = true; + this.logger.warn( + `run-stream ring overflow for run=${entry.runId}: an un-persisted ` + + `step was evicted to stay under ${this.maxBufferBytes}B; a late ` + + `attach at an evicted step will 204 until a later persist confirms`, + ); + } + } } } + + // Fan out live, filtered to each subscriber's frontier (a subscriber only + // wants the tail past the step it already persisted). for (const sub of entry.subscribers) { + if (stamp < sub.minStamp) continue; if (sub.started) { try { sub.onFrame(frame); @@ -289,12 +522,12 @@ export class AiChatStreamRegistryService implements OnModuleDestroy { } } else { sub.pending.push(frame); - sub.pendingBytes += Buffer.byteLength(frame); - if (sub.pendingBytes > SUBSCRIBER_MAX_BUFFERED_BYTES) { + sub.pendingBytes += size; + if (sub.pendingBytes > this.subscriberMaxBufferedBytes) { // The paused subscriber's buffer overflowed — only possible if start() - // was delayed past the same-tick contract (the phase-2 await seam). - // Drop it rather than buffer the whole run; on start() it degrades to an - // immediate end (a 204-equivalent) instead of replaying a partial. + // was delayed (the controller's drain-respecting tail write, or the + // phase-2 await seam). Drop it rather than buffer the whole run; on + // start() it degrades to an immediate end (a 204-equivalent). sub.overflowed = true; sub.pending = []; entry.subscribers.delete(sub); diff --git a/apps/server/src/core/ai-chat/ai-chat-stream-registry.spec.ts b/apps/server/src/core/ai-chat/ai-chat-stream-registry.spec.ts index 193b76b1..9388b2b6 100644 --- a/apps/server/src/core/ai-chat/ai-chat-stream-registry.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat-stream-registry.spec.ts @@ -1,19 +1,27 @@ import { AiChatStreamRegistryService, - RUN_STREAM_MAX_BUFFER_BYTES, + AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES, RUN_STREAM_RETAIN_FINISHED_MS, - SUBSCRIBER_MAX_BUFFERED_BYTES, RunStreamCallbacks, } from './ai-chat-stream-registry.service'; /** - * Unit tests for the in-memory run-stream registry (#184 phase 1.5). The registry - * is the whole of the resumable-transport contract: replay ordering, paused -> - * live hand-off, overflow, retention, the anchor check (invariant 6), and the - * mirror-the-done-path replace semantics (invariant 3). Every enumerated case in - * the issue's task 1.5 has a test here. + * Unit tests for the in-memory run-stream registry (#184 phase 1.5, step-aligned + * retention #491). The registry is the whole of the resumable-transport contract: + * step-stamped retention, tail-only attach at the client's frontier N, the + * confirmed-persist ring rotation (and the anti-inversion rule), the memory bound, + * the overflow gap, paused -> live hand-off, retention, the anchor check + * (invariant 6), and the mirror-the-done-path replace semantics (invariant 3). */ +// Real ai@6 UI-message-stream SSE frames are `data: {json}\n\n`, one part each. +const sse = (part: Record): string => + `data: ${JSON.stringify(part)}\n\n`; +const finishStep = (): string => sse({ type: 'finish-step' }); +const textDelta = (id: string, delta: string): string => + sse({ type: 'text-delta', id, delta }); +const finish = (): string => sse({ type: 'finish' }); + // A ReadableStream whose frames the test pushes explicitly, plus close/error. function makePushStream(): { stream: ReadableStream; @@ -58,6 +66,9 @@ function collector(): { }; } +// The tail past the synthetic start frame (replay[0] is always the start frame). +const tail = (replay: string[]): string[] => replay.slice(1); + describe('AiChatStreamRegistryService', () => { const CHAT = 'chat-1'; let registry: AiChatStreamRegistryService; @@ -71,7 +82,21 @@ describe('AiChatStreamRegistryService', () => { registry.onModuleDestroy(); }); - it('replays frames in arrival order (live attach)', async () => { + it('prepends a synthetic start frame carrying { runId, chatId }', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push('a'); + await flush(); + + const c = collector(); + const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!; + const start = JSON.parse(att.replay[0].replace(/^data: /, '').trim()); + expect(start.type).toBe('start'); + expect(start.messageMetadata).toEqual({ runId: 'run-1', chatId: CHAT }); + }); + + it('replays the buffered tail (from frontier 0) in arrival order (live attach)', async () => { registry.open(CHAT, 'run-1'); const src = makePushStream(); registry.bind(CHAT, 'run-1', 'assist-1', src.stream); @@ -81,13 +106,13 @@ describe('AiChatStreamRegistryService', () => { await flush(); const c = collector(); - const att = await registry.attach(CHAT, false, undefined, c.cb); + const att = await registry.attach(CHAT, 'assist-1', 0, c.cb); expect(att).not.toBeNull(); - expect(att!.replay).toEqual(['a', 'b', 'c']); + expect(tail(att!.replay)).toEqual(['a', 'b', 'c']); expect(att!.finished).toBe(false); }); - it('late attach gets the full prefix as replay plus the live tail', async () => { + it('late attach gets the buffered prefix as tail plus the live tail', async () => { registry.open(CHAT, 'run-1'); const src = makePushStream(); registry.bind(CHAT, 'run-1', 'assist-1', src.stream); @@ -96,17 +121,16 @@ describe('AiChatStreamRegistryService', () => { await flush(); const c = collector(); - const att = (await registry.attach(CHAT, false, undefined, c.cb))!; - expect(att.replay).toEqual(['a', 'b']); + const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!; + expect(tail(att.replay)).toEqual(['a', 'b']); att.start(); - // Live tail arrives after start(). src.push('c'); src.push('d'); await flush(); expect(c.frames).toEqual(['c', 'd']); }); - it('a paused subscriber receives frames buffered during pause in order, then live (no loss/reorder)', async () => { + it('a paused subscriber receives frames buffered during pause in order, then live', async () => { registry.open(CHAT, 'run-1'); const src = makePushStream(); registry.bind(CHAT, 'run-1', 'assist-1', src.stream); @@ -114,81 +138,45 @@ describe('AiChatStreamRegistryService', () => { await flush(); const c = collector(); - // Attach (paused). Frames that arrive BEFORE start() must queue, not drop. - const att = (await registry.attach(CHAT, false, undefined, c.cb))!; - expect(att.replay).toEqual(['a']); + const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!; + expect(tail(att.replay)).toEqual(['a']); src.push('b'); // arrives while paused -> pending src.push('c'); await flush(); expect(c.frames).toEqual([]); // nothing delivered yet (paused) - att.start(); // drains pending in order + att.start(); expect(c.frames).toEqual(['b', 'c']); - src.push('d'); // now live + src.push('d'); await flush(); expect(c.frames).toEqual(['b', 'c', 'd']); }); it('a run that finishes while a subscriber is paused ends it on start()', async () => { registry.open(CHAT, 'run-1'); + registry.bind(CHAT, 'run-1', 'assist-1', makePushStream().stream); const c = collector(); - const att = (await registry.attach(CHAT, false, undefined, c.cb))!; - // Terminate the run while the subscriber is still paused. + const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!; registry.abortEntry(CHAT, 'run-1'); expect(c.ended()).toBe(0); // paused: not ended yet att.start(); expect(c.ended()).toBe(1); // start() drains + ends }); - it('finished + expect=live returns a replay WITHOUT registering a subscriber', async () => { - registry.open(CHAT, 'run-1'); - const src = makePushStream(); - registry.bind(CHAT, 'run-1', 'assist-1', src.stream); - src.push('a'); - src.push('b'); - src.close(); - await flush(); - - const c = collector(); - const att = (await registry.attach(CHAT, true, undefined, c.cb))!; - expect(att.finished).toBe(true); - expect(att.replay).toEqual(['a', 'b']); - // No subscriber registered: start()/unsubscribe are no-ops and the entry has - // zero subscribers. - const entry = (registry as any).entries.get(CHAT); - expect(entry.subscribers.size).toBe(0); - att.start(); - expect(c.frames).toEqual([]); - }); - - it('finished WITHOUT expect=live returns null', async () => { - registry.open(CHAT, 'run-1'); - const src = makePushStream(); - registry.bind(CHAT, 'run-1', 'assist-1', src.stream); - src.push('a'); - src.close(); - await flush(); - - const c = collector(); - expect(await registry.attach(CHAT, false, undefined, c.cb)).toBeNull(); - }); - - it('anchor mismatch with expect=live returns null (and null before bind sets assistantMessageId)', async () => { + it('anchor mismatch returns null (and null before bind sets assistantMessageId)', async () => { registry.open(CHAT, 'run-1'); const c = collector(); // Before bind: assistantMessageId is undefined -> mismatches any anchor. - expect( - await registry.attach(CHAT, true, 'assist-1', c.cb), - ).toBeNull(); + expect(await registry.attach(CHAT, 'assist-1', 0, c.cb)).toBeNull(); const src = makePushStream(); registry.bind(CHAT, 'run-1', 'assist-1', src.stream); src.push('a'); await flush(); // Wrong anchor -> null (cross-run replay forbidden, invariant 6). - expect(await registry.attach(CHAT, true, 'other-id', c.cb)).toBeNull(); + expect(await registry.attach(CHAT, 'other-id', 0, c.cb)).toBeNull(); }); - it('matching anchor with expect=live attaches', async () => { + it('matching anchor attaches', async () => { registry.open(CHAT, 'run-1'); const src = makePushStream(); registry.bind(CHAT, 'run-1', 'assist-1', src.stream); @@ -196,97 +184,60 @@ describe('AiChatStreamRegistryService', () => { await flush(); const c = collector(); - const att = await registry.attach(CHAT, true, 'assist-1', c.cb); + const att = await registry.attach(CHAT, 'assist-1', 0, c.cb); expect(att).not.toBeNull(); - expect(att!.replay).toEqual(['a']); + expect(tail(att!.replay)).toEqual(['a']); }); - it('overflow: attach returns null, but the LIVE subscriber keeps receiving (incl. the crossing frame)', async () => { + it('a throwing onFrame ejects only that subscriber; the ingest loop stays alive', async () => { registry.open(CHAT, 'run-1'); const src = makePushStream(); registry.bind(CHAT, 'run-1', 'assist-1', src.stream); - // A live (started) subscriber attached before the flood. + const bad = collector(); + const badAtt = (await registry.attach(CHAT, 'assist-1', 0, { + onFrame: () => { + throw new Error('boom'); + }, + onEnd: bad.cb.onEnd, + }))!; + badAtt.start(); + + const good = collector(); + const goodAtt = (await registry.attach(CHAT, 'assist-1', 0, good.cb))!; + goodAtt.start(); + + src.push('a'); // bad throws on this frame -> ejected + src.push('b'); // good still receives both + await flush(); + + const entry = (registry as any).entries.get(CHAT); + expect(entry.subscribers.size).toBe(1); + expect(good.frames).toEqual(['a', 'b']); + }); + + it('open() over a LIVE entry ends started subscribers once; a late done never touches the new entry (invariant 3)', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push('a'); + await flush(); + const c = collector(); - const att = (await registry.attach(CHAT, false, undefined, c.cb))!; + const att = (await registry.attach(CHAT, 'assist-1', 0, c.cb))!; att.start(); - // Cap-relative so it survives a buffer-cap change (#430): a quarter-cap frame - // means 5 frames comfortably exceed the replay cap; the last one crosses. - const chunk = 'x'.repeat(Math.floor(RUN_STREAM_MAX_BUFFER_BYTES / 4)); - for (let i = 0; i < 5; i++) src.push(chunk + i); - await flush(); - - const entry = (registry as any).entries.get(CHAT); - expect(entry.overflowed).toBe(true); - expect(entry.bytes).toBeGreaterThan(RUN_STREAM_MAX_BUFFER_BYTES); - // The live subscriber received ALL 5 frames, including the crossing one. - expect(c.frames).toHaveLength(5); - expect(c.frames[4]).toBe(chunk + 4); - - // A NEW attach after overflow gets null (replay buffer is gone). - const c2 = collector(); - expect(await registry.attach(CHAT, false, undefined, c2.cb)).toBeNull(); - }); - - it('a paused subscriber whose pending buffer overflows is dropped and ends on start(); other subscribers keep receiving', async () => { - registry.open(CHAT, 'run-1'); - const src = makePushStream(); - registry.bind(CHAT, 'run-1', 'assist-1', src.stream); - - // A: paused (start() deliberately delayed to simulate the phase-2 await seam). - const a = collector(); - const attA = (await registry.attach(CHAT, false, undefined, a.cb))!; - // B: live (started) — its delivery must be unaffected by A's overflow. - const b = collector(); - const attB = (await registry.attach(CHAT, false, undefined, b.cb))!; - attB.start(); - - // Cap-relative so it survives a buffer-cap change (#430): a quarter-of-the- - // per-subscriber-cap frame means 5 frames exceed A's paused-pending cap while - // B streams every frame live. - const chunk = 'x'.repeat(Math.floor(SUBSCRIBER_MAX_BUFFERED_BYTES / 4)); - for (let i = 0; i < 5; i++) src.push(chunk + i); - await flush(); - - const entry = (registry as any).entries.get(CHAT); - // A was dropped from the subscriber set on overflow; B (started) remains. - expect(entry.subscribers.size).toBe(1); - expect(a.frames).toEqual([]); // paused + overflowed: nothing was delivered - // B received every frame live (delivery unaffected by A's overflow). - expect(b.frames).toHaveLength(5); - - // A's start() (arriving late) degrades to an immediate end, not a partial replay. - attA.start(); - expect(a.frames).toEqual([]); - expect(a.ended()).toBe(1); - }); - - it('open() over a LIVE entry ends started subscribers exactly once and a late done does not touch the new entry (invariant 3)', async () => { - registry.open(CHAT, 'run-1'); - const src = makePushStream(); - registry.bind(CHAT, 'run-1', 'assist-1', src.stream); - src.push('a'); - await flush(); - - const c = collector(); - const att = (await registry.attach(CHAT, false, undefined, c.cb))!; - att.start(); // started subscriber on run-1 - - // run-2 starts on the same chat while run-1's tee is still reading. registry.open(CHAT, 'run-2'); - expect(c.ended()).toBe(1); // exactly one onEnd from the replace + expect(c.ended()).toBe(1); const newEntry = (registry as any).entries.get(CHAT); expect(newEntry.runId).toBe('run-2'); expect(newEntry.finished).toBe(false); - // The old tee now completes: its late done must NOT double-end nor delete the - // new entry. src.push('b'); src.close(); await flush(); - expect(c.ended()).toBe(1); // still exactly one + expect(c.ended()).toBe(1); const still = (registry as any).entries.get(CHAT); expect(still).toBe(newEntry); expect(still.runId).toBe('run-2'); @@ -299,7 +250,6 @@ describe('AiChatStreamRegistryService', () => { src.push('a'); await flush(); const entry = (registry as any).entries.get(CHAT); - // Frames were NOT ingested (bind bailed), assistantMessageId untouched. expect(entry.frames).toEqual([]); expect(entry.assistantMessageId).toBeUndefined(); }); @@ -310,32 +260,276 @@ describe('AiChatStreamRegistryService', () => { const entry = (registry as any).entries.get(CHAT); expect(entry.finished).toBe(false); }); +}); - it('a throwing onFrame ejects only that subscriber; the ingest loop stays alive', async () => { +/** + * #491 step-stamped retention: the boundary detector, tail-only slicing at the + * client's frontier N, the confirmed-persist rotation (+ anti-inversion), the + * overflow gap, the memory bound, and the finished-retained tail. All observable + * against the REAL registry driven through open/bind/ingest. + */ +describe('AiChatStreamRegistryService step-aligned retention (#491)', () => { + const CHAT = 'chat-s'; + let registry: AiChatStreamRegistryService; + + beforeEach(() => { + registry = new AiChatStreamRegistryService(); + jest.spyOn((registry as any).logger, 'warn').mockImplementation(() => {}); + }); + afterEach(() => registry.onModuleDestroy()); + + const entryOf = () => (registry as any).entries.get(CHAT); + + it('stamps frames by finish-step count, aligned with stepsPersisted', async () => { registry.open(CHAT, 'run-1'); const src = makePushStream(); registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + // step 0 content, its finish-step, step 1 content, its finish-step, finish. + src.push(textDelta('t0', 'a')); // stamp 0 + src.push(finishStep()); // stamp 0 (the finish-step frame carries the pre value) + src.push(textDelta('t1', 'b')); // stamp 1 + src.push(finishStep()); // stamp 1 + src.push(finish()); // stamp 2 + await flush(); + const e = entryOf(); + expect(e.stamps).toEqual([0, 0, 1, 1, 2]); + expect(e.currentStamp).toBe(2); + }); - const bad = collector(); - const badAtt = (await registry.attach(CHAT, false, undefined, { - onFrame: () => { - throw new Error('boom'); - }, - onEnd: bad.cb.onEnd, - }))!; - badAtt.start(); + it('does NOT treat a text delta that merely quotes "finish-step" as a boundary', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + // A model that literally types "type":"finish-step" — JSON-escaped in the frame. + src.push(textDelta('t0', '"type":"finish-step"')); + await flush(); + expect(entryOf().currentStamp).toBe(0); // no false boundary + }); - const good = collector(); - const goodAtt = (await registry.attach(CHAT, false, undefined, good.cb))!; - goodAtt.start(); - - src.push('a'); // bad throws on this frame -> ejected - src.push('b'); // good still receives both + it('tail-only: attach at N slices frames with stamp >= N', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push(textDelta('t0', 'a')); // 0 + src.push(finishStep()); // 0 + src.push(textDelta('t1', 'b')); // 1 + src.push(finishStep()); // 1 + src.push(textDelta('t2', 'c')); // 2 (in-progress) await flush(); - const entry = (registry as any).entries.get(CHAT); - expect(entry.subscribers.size).toBe(1); // bad ejected, good remains - expect(good.frames).toEqual(['a', 'b']); + const c = collector(); + // Client persisted 2 steps -> wants the tail from step 2. + const att = (await registry.attach(CHAT, 'assist-1', 2, c.cb))!; + expect(tail(att.replay)).toEqual([textDelta('t2', 'c')]); + }); + + it('attach in the MIDDLE of a step (N between finish-steps) slices from that step', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push(textDelta('t0', 'a')); // 0 + src.push(finishStep()); // 0 + src.push(textDelta('t1', 'b1')); // 1 + src.push(textDelta('t1', 'b2')); // 1 (still step 1, no finish-step yet) + await flush(); + + const c = collector(); + const att = (await registry.attach(CHAT, 'assist-1', 1, c.cb))!; + // Step 0's frames are dropped from the tail; the whole in-progress step 1 is kept. + expect(tail(att.replay)).toEqual([textDelta('t1', 'b1'), textDelta('t1', 'b2')]); + }); + + it('rotates the ring ONLY on a confirmed persist (drops stamp < N)', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push(textDelta('t0', 'a')); // 0 + src.push(finishStep()); // 0 + src.push(textDelta('t1', 'b')); // 1 + await flush(); + expect(entryOf().stamps).toEqual([0, 0, 1]); + + // Confirm step 0 persisted (stepsPersisted = 1) -> drop stamp < 1. + registry.confirmPersistedStep(CHAT, 'run-1', 1); + expect(entryOf().stamps).toEqual([1]); + expect(entryOf().persistedFloor).toBe(1); + }); + + it('persist FAILED but the ring still fits -> attach SUCCEEDS and the tail includes step N', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push(textDelta('t0', 'a')); // 0 + src.push(finishStep()); // 0 + src.push(textDelta('t1', 'b')); // 1 (step 1's persist FAILED -> no confirm) + await flush(); + // No confirmPersistedStep for step 1: the ring still holds step 1. + + const c = collector(); + // Client's last successful persist was step 0 -> stepsPersisted = 1. + const att = await registry.attach(CHAT, 'assist-1', 1, c.cb); + expect(att).not.toBeNull(); + expect(tail(att!.replay)).toEqual([textDelta('t1', 'b')]); // includes step 1 + }); + + it('persist failed AND the ring overflowed past N -> 204 (coverage gap)', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + // Step 0: a fat step that blows past the cap with NO persist confirmation. + const big = 'x'.repeat(Math.floor(AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES / 2)); + src.push(textDelta('t0', big)); // 0 + src.push(textDelta('t0', big)); // 0 + src.push(textDelta('t0', big)); // 0 -> overflow evicts stamp-0 frames + await flush(); + const e = entryOf(); + expect(e.overflowed).toBe(true); + expect(e.bytes).toBeLessThanOrEqual(registry.maxBufferBytes); + + // A client at frontier 0 falls at/below an evicted step -> gap -> null. + const c = collector(); + expect(await registry.attach(CHAT, 'assist-1', 0, c.cb)).toBeNull(); + }); + + it('stale N (client seed lagged behind a rotation) -> 204; after a refetch (larger N) -> success', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push(textDelta('t0', 'a')); // 0 + src.push(finishStep()); // 0 + src.push(textDelta('t1', 'b')); // 1 + src.push(finishStep()); // 1 + src.push(textDelta('t2', 'c')); // 2 + await flush(); + // Server confirmed steps 0 and 1 -> rotate away stamp < 2. + registry.confirmPersistedStep(CHAT, 'run-1', 2); + expect(entryOf().stamps).toEqual([2]); + + // A client whose seed still says stepsPersisted = 1 -> below minStamp -> 204. + const stale = collector(); + expect(await registry.attach(CHAT, 'assist-1', 1, stale.cb)).toBeNull(); + + // It refetches (now stepsPersisted = 2) and re-attaches -> success. + const fresh = collector(); + const att = await registry.attach(CHAT, 'assist-1', 2, fresh.cb); + expect(att).not.toBeNull(); + expect(tail(att!.replay)).toEqual([textDelta('t2', 'c')]); + }); + + it('overflow gap CLEARS once a later persist rotates out the holey steps', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + const big = 'x'.repeat(Math.floor(AI_CHAT_RUN_STREAM_MAX_BUFFER_BYTES / 2)); + src.push(textDelta('t0', big)); // 0 + src.push(textDelta('t0', big)); // 0 + src.push(finishStep()); // 0 (still stamp 0) + src.push(textDelta('t1', 'small')); // 1 + src.push(finishStep()); // 1 + src.push(textDelta('t2', 'c')); // 2 + await flush(); + expect(entryOf().overflowed).toBe(true); + + // Late persist confirms steps 0..1 -> rotates out the holey step-0 frames. + registry.confirmPersistedStep(CHAT, 'run-1', 2); + // A client at frontier 2 is now cleanly covered (the hole was below it). + const c = collector(); + const att = await registry.attach(CHAT, 'assist-1', 2, c.cb); + expect(att).not.toBeNull(); + expect(tail(att!.replay)).toEqual([textDelta('t2', 'c')]); + }); + + it('finished-retained + N = N_final -> empty tail plus the finish frame', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push(textDelta('t0', 'a')); // 0 + src.push(finishStep()); // 0 + src.push(finish()); // 1 (N_final = 1) + src.close(); + await flush(); + // The last step's per-step persist confirmed stepsPersisted = 1. + registry.confirmPersistedStep(CHAT, 'run-1', 1); + + const c = collector(); + const att = (await registry.attach(CHAT, 'assist-1', 1, c.cb))!; + expect(att.finished).toBe(true); + // Empty step tail; just the finish frame so the client's SDK closes the stream. + expect(tail(att.replay)).toEqual([finish()]); + // No subscriber registered for a finished run. + expect(entryOf().subscribers.size).toBe(0); + }); + + it('#491 regression (#137/#161 dup): a PARAMETERLESS attach (n=null) to a finished NON-rotated run -> 204, but n=0 still gets the tail', async () => { + // A finished, non-rotated run: frames present, coverageFloor 0. A missing `n` + // (null — a legacy/parameterless tab that never stripped its transcript) must + // 204 -> poll, NOT receive the whole tail it would append (duplicate). A + // tail-aware client (n=0 present) still resumes. + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push(textDelta('t0', 'a')); // 0 + src.push(finishStep()); // 0 + src.push(finish()); // 1 + src.close(); + await flush(); + // NOT rotated (no confirmPersistedStep) -> stamps[0]=0, coverageFloor=0. + // MUTATION-VERIFY: revert the `finished && n === null -> null` gate (default n + // to 0) and the parameterless attach below serves the full tail instead of 204. + expect(await registry.attach(CHAT, 'assist-1', null, collector().cb)).toBeNull(); + // A tail-aware client at frontier 0 IS served (the distinction: null != 0). + const tailAware = await registry.attach(CHAT, 'assist-1', 0, collector().cb); + expect(tailAware).not.toBeNull(); + expect(tailAware!.finished).toBe(true); + }); + + it('confirmPersistedStep is monotonic and identity-checked', async () => { + registry.open(CHAT, 'run-1'); + const src = makePushStream(); + registry.bind(CHAT, 'run-1', 'assist-1', src.stream); + src.push(textDelta('t0', 'a')); + src.push(finishStep()); + src.push(textDelta('t1', 'b')); + await flush(); + registry.confirmPersistedStep(CHAT, 'run-1', 1); + expect(entryOf().persistedFloor).toBe(1); + // A stale lower count is ignored. + registry.confirmPersistedStep(CHAT, 'run-1', 0); + expect(entryOf().persistedFloor).toBe(1); + // A foreign runId is ignored. + registry.confirmPersistedStep(CHAT, 'WRONG', 5); + expect(entryOf().persistedFloor).toBe(1); + }); + + it('MEMORY BOUND: 5 parallel marathon runs each stream well past 32MB; each ring stays <= the cap', async () => { + const cap = registry.maxBufferBytes; + const chats = ['m0', 'm1', 'm2', 'm3', 'm4']; + const srcs = chats.map((chat) => { + registry.open(chat, `run-${chat}`); + const s = makePushStream(); + registry.bind(chat, `run-${chat}`, `assist-${chat}`, s.stream); + return s; + }); + // ~256KB frames; 160 per chat = 40MB streamed each, well past the old 32MB. + // Interleave a finish-step every 8 frames so steps advance realistically. No + // persist confirmation -> the ONLY thing keeping memory bounded is the cap. + const frame = 'y'.repeat(256 * 1024); + for (let batch = 0; batch < 20; batch++) { + for (let i = 0; i < 8; i++) { + for (const s of srcs) s.push(textDelta('t', frame)); + } + for (const s of srcs) s.push(finishStep()); + await flush(); // drain the pump so queues never hold a whole run + } + let total = 0; + for (const chat of chats) { + const e = (registry as any).entries.get(chat); + expect(e.bytes).toBeLessThanOrEqual(cap); + total += e.bytes; + } + // Total retained across all 5 runs is bounded by 5x the per-run cap — the old + // registry would have retained ~5x40MB = 200MB here. + expect(total).toBeLessThanOrEqual(cap * chats.length); }); }); @@ -361,7 +555,7 @@ describe('AiChatStreamRegistryService retention timers', () => { it('a finished entry is removed after the retention window', () => { registry.open(CHAT, 'run-1'); - registry.abortEntry(CHAT, 'run-1'); // finalize -> retention armed + registry.abortEntry(CHAT, 'run-1'); expect((registry as any).entries.get(CHAT)).toBeDefined(); jest.advanceTimersByTime(RUN_STREAM_RETAIN_FINISHED_MS + 1); expect((registry as any).entries.get(CHAT)).toBeUndefined(); @@ -369,20 +563,18 @@ describe('AiChatStreamRegistryService retention timers', () => { it('retention deletes ONLY its own entry (invariant 2)', () => { registry.open(CHAT, 'run-1'); - registry.abortEntry(CHAT, 'run-1'); // arm retention for entry A - // Simulate the race where the key was replaced without clearing A's timer. + registry.abortEntry(CHAT, 'run-1'); const sentinel = { marker: true }; (registry as any).entries.set(CHAT, sentinel); jest.advanceTimersByTime(RUN_STREAM_RETAIN_FINISHED_MS + 1); - // A's timer saw entries.get(CHAT) !== A, so it did NOT delete the successor. expect((registry as any).entries.get(CHAT)).toBe(sentinel); }); it('open() over a retained entry clears its timer and the successor survives', () => { registry.open(CHAT, 'run-1'); - registry.abortEntry(CHAT, 'run-1'); // retained, timer armed + registry.abortEntry(CHAT, 'run-1'); const clearSpy = jest.spyOn(global, 'clearTimeout'); - registry.open(CHAT, 'run-2'); // must clear run-1's retain timer + registry.open(CHAT, 'run-2'); expect(clearSpy).toHaveBeenCalled(); jest.advanceTimersByTime(RUN_STREAM_RETAIN_FINISHED_MS + 1); const entry = (registry as any).entries.get(CHAT); diff --git a/apps/server/src/core/ai-chat/ai-chat.controller.attach.spec.ts b/apps/server/src/core/ai-chat/ai-chat.controller.attach.spec.ts index 67aa7cbe..50666847 100644 --- a/apps/server/src/core/ai-chat/ai-chat.controller.attach.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.controller.attach.spec.ts @@ -8,10 +8,12 @@ import { SUBSCRIBER_MAX_BUFFERED_BYTES } from './ai-chat-stream-registry.service import type { User, Workspace } from '@docmost/db/types/entity.types'; /** - * Wiring spec for the #184 phase 1.5 attach endpoint + * Wiring spec for the #184 phase 1.5 attach endpoint (tail-only #491) * (`GET /ai-chat/runs/:chatId/stream`). Owner-gated via assertOwnedChat; the - * registry is mocked so this exercises ONLY the controller's replay/live/204/ - * cleanup wiring against a fake raw socket. Constructor order is (aiChatService, + * registry is mocked so this exercises ONLY the controller's tail-write/live/204/ + * cleanup wiring against a fake raw socket. The attach signature is now + * `(chatId, anchor, n, cb)` — the client hands its persisted step frontier `n` + * and its assistant row id `anchor`. Constructor order is (aiChatService, * aiChatRunService, aiChatRepo, aiChatMessageRepo, aiTranscription, pageRepo, * streamRegistry, environment). */ @@ -86,8 +88,8 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => { attach: jest.fn( ( _chatId: string, - _live: boolean, _anchor: string | undefined, + _n: number, cb: RunStreamCallbacks, ) => { capturedCb = cb; @@ -156,7 +158,7 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => { expect(res.hijack).not.toHaveBeenCalled(); }); - it('threads expect=live and anchor through to the registry', async () => { + it('threads anchor and the numeric frontier n through to the registry', async () => { const { controller, streamRegistry } = makeController({ chat: owned, attachment: null, @@ -165,8 +167,8 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => { const { req } = makeReq(); await controller.attachRunStream( 'c1', - 'live', 'anchor-1', + '2', req, res, user, @@ -174,13 +176,44 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => { ); expect(streamRegistry.attach).toHaveBeenCalledWith( 'c1', - true, 'anchor-1', + 2, // parsed to a number expect.anything(), ); }); - it('passes expect=false when the query is absent', async () => { + it('#491: an ABSENT/invalid n passes null (not 0) so a finished run 204s (not-tail-aware)', async () => { + // Distinguishing a MISSING `n` from `n=0` is the #137/#161 dup guard: a + // parameterless/legacy tab must be handed null (-> the registry 204s a finished + // run) rather than frontier 0 (which would serve a finished non-rotated run's + // whole tail). MUTATION-VERIFY: revert to `Number(n) || 0` and this asserts 0. + const { controller, streamRegistry } = makeController({ + chat: owned, + attachment: null, + }); + for (const bad of [undefined, '', 'abc']) { + streamRegistry.attach.mockClear(); + const { res } = makeRawRes(); + const { req } = makeReq(); + await controller.attachRunStream( + 'c1', + undefined, + bad, + req, + res, + user, + workspace, + ); + expect(streamRegistry.attach).toHaveBeenCalledWith( + 'c1', + undefined, + null, + expect.anything(), + ); + } + }); + + it('#491: a PRESENT n=0 passes 0 (tail-aware, distinct from absent)', async () => { const { controller, streamRegistry } = makeController({ chat: owned, attachment: null, @@ -190,7 +223,7 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => { await controller.attachRunStream( 'c1', undefined, - undefined, + '0', req, res, user, @@ -198,8 +231,8 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => { ); expect(streamRegistry.attach).toHaveBeenCalledWith( 'c1', - false, undefined, + 0, expect.anything(), ); }); @@ -245,8 +278,8 @@ describe('AiChatController attach endpoint (#184 phase 1.5)', () => { const { req } = makeReq(); await controller.attachRunStream( 'c1', - 'live', 'a1', + '1', req, res, user, diff --git a/apps/server/src/core/ai-chat/ai-chat.controller.delta.spec.ts b/apps/server/src/core/ai-chat/ai-chat.controller.delta.spec.ts new file mode 100644 index 00000000..2d418766 --- /dev/null +++ b/apps/server/src/core/ai-chat/ai-chat.controller.delta.spec.ts @@ -0,0 +1,108 @@ +import { ForbiddenException } from '@nestjs/common'; +import { AiChatController } from './ai-chat.controller'; +import type { User, Workspace } from '@docmost/db/types/entity.types'; + +/** + * Wiring spec for the #491 delta-poll endpoint (`POST /ai-chat/messages/delta`). + * Owner-gated via assertOwnedChat (same gate as the other reads), NOT flag-gated. + * The run fact rides IN the delta response (no separate /run poll). Hand-rolled + * mocks — no Nest graph, no DB. Constructor order: (aiChatService, + * aiChatRunService, aiChatRepo, aiChatMessageRepo, aiTranscription, pageRepo). + */ +describe('AiChatController POST /ai-chat/messages/delta (#491)', () => { + const user = { id: 'u1' } as User; + const workspace = { id: 'ws1' } as Workspace; + + function makeController(opts: { + chat?: unknown; + delta?: { rows: unknown[]; cursor: string }; + run?: unknown; + }) { + const aiChatRunService = { + getLatestForChat: jest.fn().mockResolvedValue(opts.run), + }; + const aiChatRepo = { + findById: jest.fn().mockResolvedValue(opts.chat), + }; + const aiChatMessageRepo = { + findByChatUpdatedAfter: jest + .fn() + .mockResolvedValue(opts.delta ?? { rows: [], cursor: 'C1' }), + }; + const controller = new AiChatController( + {} as never, + aiChatRunService as never, + aiChatRepo as never, + aiChatMessageRepo as never, + {} as never, + {} as never, + ); + return { controller, aiChatRunService, aiChatRepo, aiChatMessageRepo }; + } + + it('owner-gates: a chat the user does not own throws, never reaching the repo', async () => { + const { controller, aiChatMessageRepo, aiChatRunService } = makeController({ + chat: { id: 'c1', creatorId: 'someone-else' }, + }); + await expect( + controller.getMessagesDelta({ chatId: 'c1' }, user, workspace), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(aiChatMessageRepo.findByChatUpdatedAfter).not.toHaveBeenCalled(); + expect(aiChatRunService.getLatestForChat).not.toHaveBeenCalled(); + }); + + it('returns { rows, cursor, run:{id,status} } with the run fact inlined', async () => { + const rows = [{ id: 'm1' }]; + const { controller } = makeController({ + chat: { id: 'c1', creatorId: 'u1' }, + delta: { rows, cursor: 'C2' }, + run: { id: 'r1', status: 'running', error: 'ignored', stepCount: 3 }, + }); + const res = await controller.getMessagesDelta( + { chatId: 'c1', cursor: 'C1' }, + user, + workspace, + ); + expect(res).toEqual({ + rows, + cursor: 'C2', + // ONLY id + status — never the whole run row. + run: { id: 'r1', status: 'running' }, + }); + }); + + it('run is null when the chat has never had a run', async () => { + const { controller } = makeController({ + chat: { id: 'c1', creatorId: 'u1' }, + run: undefined, + }); + const res = await controller.getMessagesDelta( + { chatId: 'c1' }, + user, + workspace, + ); + expect(res.run).toBeNull(); + }); + + it('passes cursor through, defaulting a missing cursor to null (first poll)', async () => { + const { controller, aiChatMessageRepo } = makeController({ + chat: { id: 'c1', creatorId: 'u1' }, + }); + await controller.getMessagesDelta({ chatId: 'c1' }, user, workspace); + expect(aiChatMessageRepo.findByChatUpdatedAfter).toHaveBeenCalledWith( + 'c1', + 'ws1', + null, + ); + await controller.getMessagesDelta( + { chatId: 'c1', cursor: 'CX' }, + user, + workspace, + ); + expect(aiChatMessageRepo.findByChatUpdatedAfter).toHaveBeenLastCalledWith( + 'c1', + 'ws1', + 'CX', + ); + }); +}); diff --git a/apps/server/src/core/ai-chat/ai-chat.controller.ts b/apps/server/src/core/ai-chat/ai-chat.controller.ts index f9b58b98..2b8aa6ab 100644 --- a/apps/server/src/core/ai-chat/ai-chat.controller.ts +++ b/apps/server/src/core/ai-chat/ai-chat.controller.ts @@ -51,6 +51,7 @@ import { ChatIdDto, ExportChatDto, GeneratePageTitleDto, + GetChatDeltaDto, GetChatMessagesDto, GetRunDto, RenameChatDto, @@ -63,6 +64,47 @@ import { SUBSCRIBER_MAX_BUFFERED_BYTES, } from './ai-chat-stream-registry.service'; import { startSseHeartbeat } from './sse-resilience'; + +/** + * Write the attach TAIL to the hijacked socket in chunks that RESPECT drain + * (#491): each `write()` that returns false (the kernel buffer is full) is awaited + * on the next 'drain' before continuing. The old code wrote the whole buffer + * synchronously, which — with the pre-#491 32MB ring — spiked memory (half the + * OOM). Bails immediately if the socket ended/errored mid-write. Frames that the + * paused registry subscriber buffers while this awaits are delivered by start(). + */ +async function writeTailRespectingDrain( + raw: { + write(chunk: string): boolean; + writableEnded?: boolean; + destroyed?: boolean; + once(event: string, cb: () => void): unknown; + removeListener?(event: string, cb: () => void): unknown; + }, + frames: string[], +): Promise { + for (const frame of frames) { + if (raw.writableEnded || raw.destroyed) return; + const ok = raw.write(frame); + if (!ok) { + // Kernel buffer full — wait for drain (or an early close/error) before the + // next chunk, so a slow reader never forces the whole tail into memory. + // Remove ALL three listeners once any fires, so a many-chunk tail with + // repeated backpressure never leaks (MaxListenersExceededWarning). + await new Promise((resolve) => { + const finish = (): void => { + raw.removeListener?.('drain', finish); + raw.removeListener?.('close', finish); + raw.removeListener?.('error', finish); + resolve(); + }; + raw.once('drain', finish); + raw.once('close', finish); + raw.once('error', finish); + }); + } + } +} import { EnvironmentService } from '../../integrations/environment/environment.service'; /** @@ -149,6 +191,46 @@ export class AiChatController { ); } + /** + * Delta poll (#491) — the degraded-poll fallback's payload. Returns the chat's + * message rows changed since `cursor` (a DB-clock timestamp from the previous + * poll), a FRESH cursor, AND the current run fact `{ id, status } | null`. This + * replaces the old degraded poll that refetched ALL infinite-query pages (full + * parts) every 2.5s: the client seeds once and thereafter merges only the + * deltas by id (the overlap window guarantees repeats — the merge is idempotent, + * see mergeById). The run fact rides IN the delta (a separate /run poll would + * double the poll QPS), so the client FSM gets the run's status on the same tick. + * Owner-gated via assertOwnedChat (same gate as the other read endpoints). + */ + @HttpCode(HttpStatus.OK) + @Post('messages/delta') + async getMessagesDelta( + @Body() dto: GetChatDeltaDto, + @AuthUser() user: User, + @AuthWorkspace() workspace: Workspace, + ): Promise<{ + rows: AiChatMessage[]; + cursor: string; + run: { id: string; status: string } | null; + }> { + await this.assertOwnedChat(dto.chatId, user, workspace); + const { rows, cursor } = + await this.aiChatMessageRepo.findByChatUpdatedAfter( + dto.chatId, + workspace.id, + dto.cursor ?? null, + ); + const run = await this.aiChatRunService.getLatestForChat( + dto.chatId, + workspace.id, + ); + return { + rows, + cursor, + run: run ? { id: run.id, status: run.status } : null, + }; + } + /** * Export a chat to Markdown (#183). The DB is the single source of truth: the * whole transcript is loaded (oldest -> newest) and rendered server-side. Now @@ -249,19 +331,25 @@ export class AiChatController { } /** - * Attach to a chat's live run stream (#184 phase 1.5). A late/reloaded tab - * replays the frames buffered so far and then follows the live tail as a normal - * streamer. Owner-gated via assertOwnedChat (same gate as getRun). When there is - * nothing to resume — no entry, a finished run without expect=live, an - * overflowed buffer, or an anchor that pins a DIFFERENT run — the endpoint - * answers 204, the ONLY "nothing to resume" signal the AI SDK's reconnect - * accepts (it maps 204 to a silent no-op). With AI_CHAT_RESUMABLE_STREAM off the - * registry is never populated, so attach always 204s. + * Attach to a chat's live run stream from the client's step frontier (#184 phase + * 1.5, tail-only #491). A late/reloaded tab hands the server the step count it + * has PERSISTED (`n` = the seeded row's `metadata.stepsPersisted`) and its + * assistant row id (`anchor`); the registry answers with the TAIL past step `n` + * (a synthetic `start` frame + the buffered frames stamped >= n) and then the + * live tail. Owner-gated via assertOwnedChat (same gate as getRun). When there + * is nothing to resume — no entry, a ring that does not cover the client's + * frontier (overflow gap, or the client's seed lagged a rotation), or an anchor + * that pins a DIFFERENT run (invariant 6) — the endpoint answers 204, the ONLY + * "nothing to resume" signal the AI SDK's reconnect accepts (it maps 204 to a + * silent no-op); the client then refetches (a larger n) and re-attaches. With + * AI_CHAT_RESUMABLE_STREAM off the registry is never populated, so attach always + * 204s. * - * `expect=live` opts into replaying a finished-but-retained run (safe only when - * the client stripped the streaming tail); `anchor` is the client's assistant - * row id, which must match this run's (invariant 6) or a foreign run's - * transcript would be replayed into the store. + * The step marker `n` comes ONLY from the client — the server never reads the + * row to derive it, because a server-side n from a stale seed would open a + * silent one-step hole. The tail is written to the socket in CHUNKS respecting + * drain (writeTailRespectingDrain): the old code synchronously blasted the whole + * buffer, which — with the old 32MB cap — was half the OOM. */ @SkipTransform() @UseGuards(JwtAuthGuard, UserThrottlerGuard) @@ -269,39 +357,49 @@ export class AiChatController { @Get('runs/:chatId/stream') async attachRunStream( @Param('chatId', new ParseUUIDPipe()) chatId: string, - @Query('expect') expect: string | undefined, @Query('anchor') anchor: string | undefined, + @Query('n') n: string | undefined, @Req() req: FastifyRequest, @Res() res: FastifyReply, @AuthUser() user: User, @AuthWorkspace() workspace: Workspace, ): Promise { await this.assertOwnedChat(chatId, user, workspace); // same gate as getRun + // The client's persisted step frontier. #491: distinguish a MISSING/invalid `n` + // (null — a NOT-tail-aware, legacy/parameterless tab expecting the old + // "finished -> 204 -> poll" contract) from `n=0` (a tail-aware client with + // nothing persisted yet). Passing 0 for a missing `n` would serve a finished, + // non-rotated run's WHOLE tail and a parameterless client would append it onto + // the steps it already shows -> #137/#161 duplicate. null makes the registry + // 204 such a finished run (see attach); a tail-aware n=0 still resumes. + const frontier: number | null = + n === undefined || n === '' || !Number.isFinite(Number(n)) + ? null + : Math.max(0, Number(n)); + // The per-subscriber backpressure cap tracks the (env-tunable) ring cap. + const subscriberCap = + this.streamRegistry?.subscriberMaxBufferedBytes ?? + SUBSCRIBER_MAX_BUFFERED_BYTES; let stopHeartbeat: () => void = () => undefined; - const attachment = await this.streamRegistry?.attach( - chatId, - expect === 'live', - anchor, - { - onFrame: (frame) => { - // Backpressure guard: 2x the replay cap, so the initial replay burst - // alone can never trip it; only a genuinely stalled socket can. - try { - if (res.raw.writableLength > SUBSCRIBER_MAX_BUFFERED_BYTES) { - res.raw.destroy(); // 'close' fires -> unsubscribe below - return; - } - if (!res.raw.writableEnded) res.raw.write(frame); - } catch { - res.raw.destroy(); + const attachment = await this.streamRegistry?.attach(chatId, anchor, frontier, { + onFrame: (frame) => { + // Backpressure guard: 2x the ring cap, so the initial tail burst alone + // can never trip it; only a genuinely stalled socket can. + try { + if (res.raw.writableLength > subscriberCap) { + res.raw.destroy(); // 'close' fires -> unsubscribe below + return; } - }, - onEnd: () => { - stopHeartbeat(); - if (!res.raw.writableEnded) res.raw.end(); - }, + if (!res.raw.writableEnded) res.raw.write(frame); + } catch { + res.raw.destroy(); + } }, - ); + onEnd: () => { + stopHeartbeat(); + if (!res.raw.writableEnded) res.raw.end(); + }, + }); if (!attachment) { res.status(204).send(); // the ONLY "nothing to resume" signal the SDK accepts return; @@ -330,13 +428,16 @@ export class AiChatController { // deliberately NO Connection/Keep-Alive (hop-by-hop; Safari/HTTP2) }); res.raw.flushHeaders?.(); - for (const frame of attachment.replay) res.raw.write(frame); + // Write the tail in chunks respecting drain (not a synchronous blast, which + // was half the OOM). Frames the paused subscriber buffers meanwhile are + // drained by start() below; its cap is the backstop for a stalled socket. + await writeTailRespectingDrain(res.raw, attachment.replay); if (attachment.finished) { - res.raw.end(); + if (!res.raw.writableEnded) res.raw.end(); return; } stopHeartbeat = startSseHeartbeat(res.raw, 15_000); - attachment.start(); // drain pending accumulated during replay, go live + attachment.start(); // drain pending accumulated during the tail write, go live } catch { attachment.unsubscribe(); stopHeartbeat(); diff --git a/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts index 8b170c11..b15c2ed1 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.run-race.spec.ts @@ -181,7 +181,7 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => { {} as never, // pageAccess { isAiChatDeferredToolsEnabled: () => false, isAiChatFinalStepLockdownEnabled: () => false } as never, // environment ); - return { svc }; + return { svc, aiChatMessageRepo }; } const body = { @@ -287,7 +287,7 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => { // Drive stream() to the point streamText is called, capturing the options object // (which carries onStepFinish/onFinish/onError/onAbort) and the run hooks. async function captureStreamCallbacks() { - const { svc } = makeService(); + const { svc, aiChatMessageRepo } = makeService(); let capturedOpts: any; streamTextMock.mockImplementation((opts: any) => { capturedOpts = opts; @@ -314,7 +314,7 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => { runHooks: runHooks as never, }); expect(capturedOpts).toBeDefined(); - return { capturedOpts, runHooks }; + return { capturedOpts, runHooks, aiChatMessageRepo }; } it('F9: onStepFinish bumps the run step count, onFinish settles the run "completed" (the dominant autonomous-run path)', async () => { @@ -369,6 +369,51 @@ describe('AiChatService.stream — abortSignal wiring (#184 F3)', () => { expect.stringContaining('provider exploded'), ); }); + + // #490 reactive branch: a provider CONTEXT-OVERFLOW 400 in onError is classified, + // records a distinguishable cause, and stamps metadata.replayOverflow so the NEXT + // turn's budgeter trims aggressively (the recovery that un-bricks the chat). + it('#490: a context-overflow 400 stamps replayOverflow on the finalized row', async () => { + jest + .spyOn(Logger.prototype, 'error') + .mockImplementation(() => undefined as never); + jest + .spyOn(Logger.prototype, 'warn') + .mockImplementation(() => undefined as never); + const { capturedOpts, aiChatMessageRepo } = await captureStreamCallbacks(); + + const overflow = Object.assign(new Error('too large'), { + statusCode: 400, + message: + "This model's maximum context length is 128000 tokens. However, your messages resulted in 214000 tokens. Please reduce the length.", + }); + await capturedOpts.onError({ error: overflow }); + + // The seed row exists (finalizeOwner is the owner-write path). + expect(aiChatMessageRepo.finalizeOwner).toHaveBeenCalled(); + const calls = aiChatMessageRepo.finalizeOwner.mock.calls as any[][]; + const patch = calls[calls.length - 1][2] as { + status: string; + metadata: Record; + }; + expect(patch.status).toBe('error'); + expect(patch.metadata.replayOverflow).toBe(true); + expect(patch.metadata.error).toContain('контекстное окно'); + }); + + it('#490: a non-overflow error does NOT stamp replayOverflow', async () => { + jest + .spyOn(Logger.prototype, 'error') + .mockImplementation(() => undefined as never); + const { capturedOpts, aiChatMessageRepo } = await captureStreamCallbacks(); + await capturedOpts.onError({ error: new Error('network reset') }); + const calls = aiChatMessageRepo.finalizeOwner.mock.calls as any[][]; + const patch = calls[calls.length - 1][2] as { + status: string; + metadata: Record; + }; + expect('replayOverflow' in patch.metadata).toBe(false); + }); }); /** diff --git a/apps/server/src/core/ai-chat/ai-chat.service.spec.ts b/apps/server/src/core/ai-chat/ai-chat.service.spec.ts index 8dd32123..5d4ff1b6 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.spec.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.spec.ts @@ -13,6 +13,7 @@ import { compactToolOutput, assistantParts, serializeSteps, + type StepPartsCache, rowToUiMessage, prepareAgentStep, stepBudgetWarning, @@ -28,10 +29,14 @@ import { FINAL_STEP_NUDGE, STEP_LIMIT_NO_ANSWER_MARKER, OUTPUT_DEGENERATION_ERROR, + lastAssistantContextTokens, + lastAssistantReplayOverflow, + seedActivatedTools, } from './ai-chat.service'; import type { AiChatMessage, Workspace } from '@docmost/db/types/entity.types'; import { buildSystemPrompt } from './ai-chat.prompt'; import type { McpClientsService } from './external-mcp/mcp-clients.service'; +import { resolveEffectiveReplayThreshold } from './history-budget'; /** * Unit tests for compactToolOutput: the pure helper that shrinks tool outputs @@ -114,6 +119,54 @@ describe('compactToolOutput', () => { describe('assistantParts', () => { type AnyPart = Record; + // #490 memoization: assistantParts builds each step's parts once and caches + // them by the step OBJECT's identity, so a mid-stream flush does not + // re-stringify every prior step's (large) output. Observable property: with a + // shared cache, the second call over the SAME step object returns the cached + // (identical) part array even if the step's underlying output was swapped — + // proving the work was memoized, not redone. + it('memoizes a step by identity (shared cache => one build per step)', () => { + const cache: StepPartsCache = new WeakMap(); + const step = { + text: 'x', + toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: {} }], + toolResults: [{ toolCallId: 'c1', toolName: 'getPage', output: { v: 1 } }], + }; + const first = assistantParts([step], '', cache) as AnyPart[]; + expect((first.find((p) => p.type === 'tool-getPage')!.output as any).v).toBe( + 1, + ); + // Swap the output for a NEW value; a re-build would pick it up, a cache hit + // keeps the first result. + step.toolResults[0] = { + toolCallId: 'c1', + toolName: 'getPage', + output: { v: 2 }, + }; + const second = assistantParts([step], '', cache) as AnyPart[]; + expect((second.find((p) => p.type === 'tool-getPage')!.output as any).v).toBe( + 1, + ); + // Same cached part objects are reused. + expect(second.find((p) => p.type === 'tool-getPage')).toBe( + first.find((p) => p.type === 'tool-getPage'), + ); + }); + + it('without a cache, each call rebuilds (no stale memo)', () => { + const step = { + text: 'x', + toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: {} }], + toolResults: [{ toolCallId: 'c1', toolName: 'getPage', output: { v: 1 } }], + }; + const first = assistantParts([step], '') as AnyPart[]; + step.toolResults[0].output = { v: 2 }; + const second = assistantParts([step], '') as AnyPart[]; + expect((second.find((p) => p.type === 'tool-getPage')!.output as any).v).toBe( + 2, + ); + }); + it('emits output-available for a tool-call WITH a paired result', () => { const steps = [ { @@ -231,61 +284,320 @@ describe('assistantParts', () => { }); }); -describe('serializeSteps', () => { +// #490 trace format v2: per call the trace stores { input } for the call and an +// OUTCOME element — { ok: true } on success, { error, kind: 'thrown' } on a +// thrown tool-error, { error, kind: 'interrupted' } on a mid-step abort. The tool +// OUTPUT is no longer duplicated here (it lives once in metadata.parts). +describe('serializeSteps (trace v2)', () => { it('returns null when there are no calls or results', () => { expect(serializeSteps([])).toBeNull(); }); - it('flattens calls and results into a compact trace', () => { + it('pairs a successful call with an { ok: true } outcome and NO output', () => { const trace = serializeSteps([ { - toolCalls: [{ toolName: 'getPage', input: { id: 'p1' } }], - toolResults: [{ toolName: 'getPage', output: { title: 'T' } }], + toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: { id: 'p1' } }], + toolResults: [{ toolCallId: 'c1', toolName: 'getPage' }], }, ]) as Array>; expect(trace).toHaveLength(2); expect(trace[0]).toEqual({ toolName: 'getPage', input: { id: 'p1' } }); - expect(trace[1]).toEqual({ toolName: 'getPage', output: { title: 'T' } }); + expect(trace[1]).toEqual({ toolName: 'getPage', ok: true }); + // The output is NOT stored in the trace any more (dedup: it lives in parts). + expect(trace.some((e) => 'output' in e)).toBe(false); }); - it('records a THROWN tool failure (tool-error part) with its error message', () => { + it('records a THROWN failure with { error, kind: "thrown" }', () => { const trace = serializeSteps([ { - toolCalls: [{ toolName: 'editPageText', input: { id: 'p1' } }], + toolCalls: [ + { toolCallId: 'c1', toolName: 'editPageText', input: { id: 'p1' } }, + ], toolResults: [], content: [ { type: 'tool-error', + toolCallId: 'c1', toolName: 'editPageText', error: new Error('page is locked'), }, ], }, ]) as Array>; - // The call element is followed by a paired error element (mirroring how a - // successful result is appended), so the failure survives in the trace. expect(trace).toHaveLength(2); expect(trace[0]).toEqual({ toolName: 'editPageText', input: { id: 'p1' } }); expect(trace[1]).toEqual({ toolName: 'editPageText', error: 'page is locked', + kind: 'thrown', }); }); - it('truncates a very long tool-error message to the tool-output limit', () => { + it('marks an interrupted call (no result, no throw) with kind "interrupted"', () => { + const trace = serializeSteps([ + { + toolCalls: [ + { toolCallId: 'c1', toolName: 'createComment', input: { x: 1 } }, + ], + toolResults: [], + content: [], + }, + ]) as Array>; + expect(trace).toHaveLength(2); + expect(trace[1]).toEqual({ + toolName: 'createComment', + error: 'Tool call did not complete.', + kind: 'interrupted', + }); + // Structurally distinct from a thrown hard-fail so it never inflates an + // error-rate scan. + expect((trace[1] as { kind: string }).kind).not.toBe('thrown'); + }); + + it('truncates a very long thrown-error message to the tool-output limit', () => { const long = 'x'.repeat(5000); const trace = serializeSteps([ { - toolCalls: [{ toolName: 'editPageText', input: {} }], + toolCalls: [{ toolCallId: 'c1', toolName: 'editPageText', input: {} }], toolResults: [], - content: [{ type: 'tool-error', toolName: 'editPageText', error: long }], + content: [ + { + type: 'tool-error', + toolCallId: 'c1', + toolName: 'editPageText', + error: long, + }, + ], }, ]) as Array>; const errorText = trace[1].error as string; - // Truncated (not the full 5000 chars) and carries the omission marker. expect(errorText.length).toBeLessThan(long.length); expect(errorText).toContain('chars omitted'); }); + + it('pairs parallel calls in one step with their outcomes by id', () => { + const trace = serializeSteps([ + { + toolCalls: [ + { toolCallId: 'a', toolName: 'getPage', input: {} }, + { toolCallId: 'b', toolName: 'searchPages', input: {} }, + ], + toolResults: [{ toolCallId: 'b', toolName: 'searchPages' }], + content: [ + { type: 'tool-error', toolCallId: 'a', toolName: 'getPage', error: 'nope' }, + ], + }, + ]) as Array>; + // call a, outcome a (thrown), call b, outcome b (ok) + expect(trace).toHaveLength(4); + expect(trace[1]).toEqual({ toolName: 'getPage', error: 'nope', kind: 'thrown' }); + expect(trace[3]).toEqual({ toolName: 'searchPages', ok: true }); + }); +}); + +// #490: every assistant row flushAssistant writes carries the v2 era marker so a +// dual-shape diagnostic query can branch on the trace shape without inspecting it. +describe('toolTraceVersion era marker (#490)', () => { + it('stamps metadata.toolTraceVersion = 2 on every flushed row', () => { + const seed = flushAssistant([], '', 'streaming'); + expect(seed.metadata.toolTraceVersion).toBe(2); + const done = flushAssistant( + [ + { + text: 'ok', + toolCalls: [{ toolCallId: 'c1', toolName: 'getPage', input: {} }], + toolResults: [{ toolCallId: 'c1', toolName: 'getPage' }], + }, + ], + '', + 'completed', + { finishReason: 'stop' }, + ); + expect(done.metadata.toolTraceVersion).toBe(2); + }); +}); + +// #490 replay-budget signal helpers over persisted history. +describe('lastAssistantContextTokens', () => { + const row = ( + role: string, + metadata: Record | null, + ): AiChatMessage => ({ role, metadata }) as unknown as AiChatMessage; + + it('reads the most recent assistant turn contextTokens (provider fact)', () => { + const hist = [ + row('user', null), + row('assistant', { contextTokens: 12000 }), + row('user', null), + row('assistant', { contextTokens: 41000 }), + ]; + expect(lastAssistantContextTokens(hist)).toBe(41000); + }); + + it('returns undefined when the last assistant turn recorded no usage', () => { + const hist = [row('assistant', { error: 'boom' }), row('user', null)]; + expect(lastAssistantContextTokens(hist)).toBeUndefined(); + expect(lastAssistantContextTokens([])).toBeUndefined(); + }); +}); + +// #490 snapshotOpenPage fast-path: skip the full Markdown export + upsert when a +// snapshot already exists at the page's CURRENT version (same updated_at instant). +describe('snapshotOpenPage fast-path (#490)', () => { + function makeSvc(existingSnapshot: unknown, pageUpdatedAt: Date) { + const exportPageMarkdown = jest.fn(async () => '# md'); + const upsert = jest.fn(async () => undefined); + const findByChatPage = jest.fn(async () => existingSnapshot); + const pageRepo = { + findById: jest.fn(async () => ({ + id: 'p1', + workspaceId: 'ws1', + updatedAt: pageUpdatedAt, + })), + }; + const svc = new AiChatService( + {} as never, // ai + {} as never, // aiChatRepo + {} as never, // aiChatMessageRepo + { findByChatPage, upsert } as never, // aiChatPageSnapshotRepo + {} as never, // aiSettings + { exportPageMarkdown } as never, // tools + {} as never, // mcpClients + {} as never, // aiAgentRoleRepo + pageRepo as never, // pageRepo + {} as never, // pageAccess + {} as never, // environment + ); + return { svc, exportPageMarkdown, upsert, findByChatPage }; + } + + const args = () => + [ + 'chat1', + 'p1', + { id: 'ws1' } as never, + { id: 'u1' } as never, + 'sess', + ] as const; + + it('skips export + upsert when the snapshot is already at this page version', async () => { + const t = new Date('2026-07-07T10:00:00Z'); + const { svc, exportPageMarkdown, upsert } = makeSvc( + { pageUpdatedAt: t, contentMd: '# md' }, + t, + ); + await (svc as unknown as { snapshotOpenPage: (...a: unknown[]) => Promise }) + .snapshotOpenPage(...args()); + expect(exportPageMarkdown).not.toHaveBeenCalled(); + expect(upsert).not.toHaveBeenCalled(); + }); + + it('exports + upserts when the page advanced since the snapshot', async () => { + const { svc, exportPageMarkdown, upsert } = makeSvc( + { pageUpdatedAt: new Date('2026-07-07T10:00:00Z'), contentMd: 'old' }, + new Date('2026-07-07T11:00:00Z'), + ); + await (svc as unknown as { snapshotOpenPage: (...a: unknown[]) => Promise }) + .snapshotOpenPage(...args()); + expect(exportPageMarkdown).toHaveBeenCalledTimes(1); + expect(upsert).toHaveBeenCalledTimes(1); + }); + + it('seeds (exports + upserts) on the first turn (no snapshot yet)', async () => { + const { svc, exportPageMarkdown, upsert } = makeSvc( + undefined, + new Date('2026-07-07T10:00:00Z'), + ); + await (svc as unknown as { snapshotOpenPage: (...a: unknown[]) => Promise }) + .snapshotOpenPage(...args()); + expect(exportPageMarkdown).toHaveBeenCalledTimes(1); + expect(upsert).toHaveBeenCalledTimes(1); + }); +}); + +// #490 deferred-tool activation persisted across turns. +describe('seedActivatedTools', () => { + const valid = new Set(['Search_web', 'getPageJson', 'diffPageVersions']); + + it('seeds from persisted metadata, intersected with current valid names', () => { + expect( + seedActivatedTools( + { activatedTools: ['Search_web', 'getPageJson'] }, + valid, + ), + ).toEqual(['Search_web', 'getPageJson']); + }); + + it('drops a stored tool that is no longer valid (allowlist/role changed)', () => { + // 'Habr_publish' was activated before but is not in the current allowlist. + expect( + seedActivatedTools({ activatedTools: ['Search_web', 'Habr_publish'] }, valid), + ).toEqual(['Search_web']); + }); + + it('is empty/robust for missing, non-array, or unknown-shaped metadata', () => { + expect(seedActivatedTools(undefined, valid)).toEqual([]); + expect(seedActivatedTools({}, valid)).toEqual([]); + expect(seedActivatedTools({ activatedTools: 'nope' }, valid)).toEqual([]); + expect( + seedActivatedTools({ activatedTools: [1, 'getPageJson', null] }, valid), + ).toEqual(['getPageJson']); + }); + + it('de-duplicates stored names', () => { + expect( + seedActivatedTools( + { activatedTools: ['getPageJson', 'getPageJson'] }, + valid, + ), + ).toEqual(['getPageJson']); + }); +}); + +describe('lastAssistantReplayOverflow', () => { + const row = ( + role: string, + metadata: Record | null, + ): AiChatMessage => ({ role, metadata }) as unknown as AiChatMessage; + + it('is true only when the LAST assistant turn overflowed', () => { + expect( + lastAssistantReplayOverflow([ + row('assistant', { replayOverflow: true }), + row('user', null), + ]), + ).toBe(true); + // A recovered (later, non-overflow) assistant turn clears it. + expect( + lastAssistantReplayOverflow([ + row('assistant', { replayOverflow: true }), + row('user', null), + row('assistant', { contextTokens: 5 }), + ]), + ).toBe(false); + expect(lastAssistantReplayOverflow([])).toBe(false); + }); + + // #490 reactive recovery: a prior turn stamped `replayOverflow` must make the + // NEXT turn's effective budget the AGGRESSIVE 0.5x cut — that harder trim is + // what un-bricks a chat that just 400'd on the context window. This exercises + // the exact wiring the service uses: read the stamp, then scale the threshold. + it('#490: a prior replayOverflow drives the next turn to the 0.5x aggressive budget', () => { + const history = [ + row('assistant', { replayOverflow: true }), + row('user', null), + ]; + const priorOverflowed = lastAssistantReplayOverflow(history); + expect(priorOverflowed).toBe(true); + // Base budget 100k -> aggressive recovery halves it to 50k this turn. + expect(resolveEffectiveReplayThreshold(100_000, priorOverflowed)).toBe(50_000); + // Odd base floors, not rounds. + expect(resolveEffectiveReplayThreshold(99_999, true)).toBe(49_999); + // No prior overflow -> the base budget is used verbatim (no aggressive cut). + expect(resolveEffectiveReplayThreshold(100_000, false)).toBe(100_000); + // An explicit off-switch (null) is never overridden, even on recovery. + expect(resolveEffectiveReplayThreshold(null, true)).toBeNull(); + }); }); describe('rowToUiMessage', () => { @@ -618,6 +930,23 @@ describe('flushAssistant', () => { expect(flushed.metadata.error).toBe('boom'); }); + // #490 observability: the replay budgeter's decision is stamped on the turn. + it('records replayTrimmedToTokens + replayOverflow when provided', () => { + const f = flushAssistant([], '', 'error', { + error: 'ctx', + replayTrimmedToTokens: 42_000, + replayOverflow: true, + }); + expect(f.metadata.replayTrimmedToTokens).toBe(42_000); + expect(f.metadata.replayOverflow).toBe(true); + }); + + it('omits the replay metadata when not provided', () => { + const f = flushAssistant([], '', 'completed', { finishReason: 'stop' }); + expect('replayTrimmedToTokens' in f.metadata).toBe(false); + expect('replayOverflow' in f.metadata).toBe(false); + }); + // #274 observability: the page-change diff the agent saw this turn is persisted // to metadata.pageChanged when a non-empty diff was injected, and omitted when // the diff is empty/whitespace or the arg is not supplied. diff --git a/apps/server/src/core/ai-chat/ai-chat.service.ts b/apps/server/src/core/ai-chat/ai-chat.service.ts index 66d21473..9922ba5a 100644 --- a/apps/server/src/core/ai-chat/ai-chat.service.ts +++ b/apps/server/src/core/ai-chat/ai-chat.service.ts @@ -55,6 +55,12 @@ import { type SelectionContext, } from './tools/current-page.util'; import { roleModelOverride } from './roles/role-model-config'; +import { + resolveReplayBudget, + resolveEffectiveReplayThreshold, + isContextOverflowError, + trimHistoryForReplay, +} from './history-budget'; import { startSseHeartbeat, stripStreamingHopByHopHeaders, @@ -117,9 +123,14 @@ const FINAL_STEP_NUDGE = // NO text at all (#444, mitigates the "empty turn" the lockdown used to prevent // when the toggle is OFF). Makes the exhausted-without-answer state explicit to // the user and, on replay, to the model on the next turn. +// The persisted content is the app's base locale (en-US) — which is ALSO the +// i18n key the client localizes through `t()` — instead of a hardcoded Russian +// string (it used to render Russian for every locale, and fed Russian back to +// the model on replay). Keep it a plain, model-readable English sentence so the +// next turn's replay reads cleanly; the client resolves the locale. const STEP_LIMIT_NO_ANSWER_MARKER = - '(Достигнут лимит шагов — итоговый ответ не сформулирован; работа могла ' + - 'остаться незавершённой. Напишите «продолжай», чтобы агент продолжил.)'; + '(Step limit reached — no final answer was produced; the work may be ' + + 'unfinished. Reply "continue" to let the agent carry on.)'; // Reason recorded in ai_chat_runs.error / the assistant row when the token- // degeneration detector (#444) aborts a run. Distinct from a user Stop (no error) @@ -127,6 +138,15 @@ const STEP_LIMIT_NO_ANSWER_MARKER = const OUTPUT_DEGENERATION_ERROR = 'Output degeneration detected (repeated token loop)'; +// Prefix recorded on the assistant row when the provider rejected the turn for +// CONTEXT OVERFLOW (#490): the replayed history exceeded the model's window. The +// row is ALSO stamped `metadata.replayOverflow` so the NEXT turn's budgeter trims +// aggressively (the reactive recovery — the overflowing turn had no usage signal +// to trigger preventive trimming, so the classified 400 is what un-bricks it). +export const CONTEXT_OVERFLOW_ERROR_PREFIX = + 'Диалог превысил контекстное окно модели; история будет агрессивно ' + + 'сокращена на следующем ходу.'; + /** * Compute the step-budget warning text (#444), or '' when this step is outside * the warning band. The warning fires on steps @@ -882,6 +902,21 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { const freshPage = await this.pageRepo.findById(pageId); // Page deleted during the turn (or somehow foreign) => don't write. if (!freshPage || freshPage.workspaceId !== workspace.id) return; + // Fast-path (#490): if a snapshot already exists at THIS page version + // (same updated_at instant), its content is already current — skip the full + // Markdown export + upsert entirely. A turn that did NOT touch the open page + // (the common case) thus does no snapshot work. This mirrors the read-side + // fast path in detectPageChange (sameInstant): both trust that a page edit + // bumps updated_at. When the agent (or a human) DID edit the page this turn, + // updated_at advanced, so this does not match and we re-export as before. + const existing = await this.aiChatPageSnapshotRepo.findByChatPage( + chatId, + pageId, + workspace.id, + ); + if (existing && sameInstant(existing.pageUpdatedAt, freshPage.updatedAt)) { + return; + } const currentMd = await this.tools.exportPageMarkdown( user, sessionId, @@ -921,10 +956,17 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // supplied or the supplied one does not belong to this workspace. let isNewChat = false; let chatId = body.chatId; + // Persisted chat-level metadata bag (#490): read once here so the deferred-tool + // activation set can be seeded from the previous turn. Undefined for a new chat. + let chatMetadata: Record | undefined; if (chatId) { const existing = await this.aiChatRepo.findById(chatId, workspace.id); if (!existing) { chatId = undefined; + } else { + chatMetadata = (existing.metadata ?? undefined) as + | Record + | undefined; } } // The open page the client sent is attacker-controllable — BOTH its id and @@ -1086,7 +1128,7 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // per-row conversion and degraded to plain text with a "[tool context // omitted]" marker rather than 500-ing the whole turn (silent loss of tool // context is not acceptable — the model must see the truncation). - const messages = await convertHistoryResilient(uiMessages, (index, err) => + let messages = await convertHistoryResilient(uiMessages, (index, err) => this.logger.warn( `Degraded unconvertible history row ${index} on chat ${chatId} to text: ${ err instanceof Error ? err.message : 'unknown error' @@ -1135,6 +1177,56 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // Here we only need the admin-configured system prompt. const resolved = await this.aiSettings.resolve(workspace.id); + // History-replay token budget (#490). The full conversation is replayed to + // the provider every turn, so a long chat eventually 400s on the context + // window — forever. Bound the REPLAYED history (never the persisted rows). + // PRIMARY signal is the provider's own fact: the last turn's contextTokens. + const replayBudget = resolveReplayBudget(resolved?.chatContextWindowRaw); + if (replayBudget.usedDefault) { + // The default fires precisely for installs with NO configured window — + // the ones that hit terminal overflow. Warn so it is observable. + this.logger.warn( + `AI chat (chat ${chatId}): no chatContextWindow configured; ` + + `applying the default replay budget (${replayBudget.thresholdTokens} tokens).`, + ); + } + // Last turn's provider-reported context size (authoritative when present). + const priorContextTokens = lastAssistantContextTokens(oldHistory); + // Reactive recovery (#490): if the LAST turn was rejected for context + // overflow (stamped by onError), trim AGGRESSIVELY this turn — the + // overflowing turn produced no usage signal, so a normal-threshold trim may + // not shrink enough to fit. This is what un-bricks a chat that just 400'd. + const priorOverflowed = lastAssistantReplayOverflow(oldHistory); + const effectiveThreshold = resolveEffectiveReplayThreshold( + replayBudget.thresholdTokens, + priorOverflowed, + ); + if (priorOverflowed) { + this.logger.warn( + `AI chat (chat ${chatId}): previous turn hit context overflow; ` + + `applying aggressive replay budget (${effectiveThreshold} tokens).`, + ); + } + const preTrim = trimHistoryForReplay( + messages, + effectiveThreshold, + // A prior OVERFLOW means the provider count is stale/absent — force the + // char-estimate path by ignoring priorContextTokens on recovery. + priorOverflowed ? undefined : priorContextTokens, + ); + messages = preTrim.messages; + // Observability (#490): record the budgeter's decision on the turn so the UI + // can surface "replay truncated at N tokens". Threaded into flushAssistant. + let replayTrimmedToTokens: number | undefined = preTrim.trimmed + ? preTrim.estimatedTokens + : undefined; + if (preTrim.trimmed) { + this.logger.log( + `AI chat (chat ${chatId}): replay history trimmed to ~${preTrim.estimatedTokens} ` + + `tokens (budget ${replayBudget.thresholdTokens}).`, + ); + } + // Build the external MCP toolset FIRST so the system prompt can carry each // connected server's admin-authored guidance (#180). Merge in admin- // configured external MCP tools (web search, etc.; §6.8). A down/slow @@ -1326,10 +1418,19 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // tools + ALL external MCP tools), computed from the ACTUAL toolset so an // external tool is loadable by its namespaced name. loadTools rejects any // name outside this set. - const activatedTools = new Set(); const validDeferredNames = new Set( Object.keys(baseTools).filter((k) => !CORE_TOOL_SET.has(k)), ); + // #490: seed the activation set from the chat's PERSISTED set so the model + // does not re-run loadTools every turn to re-activate the same tools. Only + // when deferred loading is enabled, and ALWAYS intersected with the CURRENT + // valid deferred names — an allowlist/role change must never resurrect a tool + // that no longer exists (prepareAgentStep would get a phantom active name). + const activatedTools = new Set( + deferredEnabled + ? seedActivatedTools(chatMetadata, validDeferredNames) + : [], + ); // Add the loadTools meta-tool ONLY when the feature is enabled; when off the // toolset and behavior are exactly as before. const tools = deferredEnabled @@ -1339,6 +1440,39 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { } : baseTools; + // #490: persist the (deterministically ordered) activation set back onto the + // chat metadata at turn end, so the NEXT turn seeds from it. Once-guarded and + // skipped when nothing new was activated (the set equals its seed) so an + // ordinary turn adds no extra write. Preserves other metadata keys. + let activatedToolsPersisted = false; + const persistActivatedTools = async (): Promise => { + if (!deferredEnabled || activatedToolsPersisted || !chatId) return; + activatedToolsPersisted = true; + const current = [...activatedTools].sort(); + const seeded = seedActivatedTools(chatMetadata, validDeferredNames).sort(); + if (current.length === 0 || current.join('') === seeded.join('')) { + return; // nothing new activated -> no write + } + try { + await this.aiChatRepo.update( + chatId, + { + metadata: { + ...(chatMetadata ?? {}), + activatedTools: current, + }, + } as never, + workspace.id, + ); + } catch (err) { + this.logger.warn( + `Failed to persist activated tools (chat ${chatId}): ${ + err instanceof Error ? err.message : 'unknown error' + }`, + ); + } + }; + // Accumulate the turn's streamed output so a provider error / disconnect can // persist the PARTIAL answer the user already saw — the SDK's onError/onAbort // callbacks don't hand us the in-progress text. `capturedSteps` holds finished @@ -1347,6 +1481,11 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { const capturedSteps: StepLike[] = []; let inProgressText = ''; + // Per-turn step->parts memo (#490): shared across every flushAssistant call + // this turn so each finished step's (large) output is JSON-stringified ONCE, + // not re-stringified on every subsequent onStepFinish flush (was O(N²)). + const partsCache: StepPartsCache = new WeakMap(); + // Token-degeneration guard (#444). When the final-step lockdown is OFF, a // runaway repetition loop (the 255KB "loadTools." incident) is aborted via // this internal controller, unioned with the run/socket signal below. The @@ -1404,27 +1543,39 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // Per-step (non-terminal) update: persist the finished steps the moment a // step ends. Tolerant — a failed update is logged and swallowed so it never // throws into the stream. Keeps status 'streaming'. - const updateStreaming = async (): Promise => { - if (!assistantId) return; + // + // #491: it now SIGNALS its outcome — the persisted `stepsPersisted` count on + // a CONFIRMED write, or null when it was skipped/failed. The caller rotates + // the run-stream registry ring ONLY on a non-null return (a confirmed + // persist), so a failed persist never rotates away a step nobody has (the + // classic inversion bug); a failure just makes the ring cover more. + const updateStreaming = async (): Promise => { + if (!assistantId) return null; // Cheap short-circuit once the turn is finalized (see `finalized` below). // The AUTHORITATIVE guard is `onlyIfStreaming` on the UPDATE: a late // fire-and-forget step update could still be in flight on another pool // connection when finalize runs, so the SQL `WHERE status='streaming'` // (not this flag) is what prevents it clobbering the terminal row. - if (finalized) return; + if (finalized) return null; + // Build the flush ONCE so the returned count is EXACTLY the persisted + // `stepsPersisted` (both derive from capturedSteps.length at this instant). + const flushed = flushAssistant(capturedSteps, '', 'streaming', { + pageChanged, + partsCache, + }); + const stepsPersisted = flushed.metadata.stepsPersisted as number; try { - await this.aiChatMessageRepo.update( - assistantId, - workspace.id, - flushAssistant(capturedSteps, '', 'streaming', { pageChanged }), - { onlyIfStreaming: true }, - ); + await this.aiChatMessageRepo.update(assistantId, workspace.id, flushed, { + onlyIfStreaming: true, + }); + return stepsPersisted; } catch (err) { this.logger.warn( `Failed to update streaming assistant row: ${ err instanceof Error ? err.message : 'unknown error' }`, ); + return null; } }; @@ -1514,6 +1665,13 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { system, messages, tools, + // Pin the AI SDK per-request retry budget explicitly instead of relying + // on its default (which is also 2). Connection arithmetic per turn: + // (1 + maxRetries=2) × (1 + AI_STREAM_PRE_RESPONSE_RETRIES) network + // connects worst-case — the two retry layers compose, so making the SDK + // side explicit keeps that ceiling visible and pinned against SDK-default + // drift. + maxRetries: 2, // No maxOutputTokens cap on the agent: tool-call arguments (e.g. a full // page body for the write tools) are emitted as OUTPUT tokens, so a fixed // cap would truncate complex tool calls mid-argument. Let the model use its @@ -1593,7 +1751,24 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // this point still recovers the step. Not awaited here (never block the // stream), but SERIALIZED via stepUpdateChain so the writes commit in // step order; updateStreaming is error-tolerant (logs + swallows). - stepUpdateChain = stepUpdateChain.then(() => updateStreaming()); + // #491: on a CONFIRMED persist, rotate the run-stream registry ring to + // drop the now-on-disk steps (stamp < stepsPersisted). Gated on the + // resumable flag (same as open/bind) and identity-checked in the + // registry; a null return (skipped/failed) rotates NOTHING (auto-safe). + stepUpdateChain = stepUpdateChain.then(async () => { + const persisted = await updateStreaming(); + if ( + persisted != null && + runId && + this.environment?.isAiChatResumableStreamEnabled?.() + ) { + this.streamRegistry?.confirmPersistedStep( + chatId, + runId, + persisted, + ); + } + }); // #184: persist the run's progress (finished-step count). Fire-and- // forget; the hook swallows its own errors. if (runId) runHooks?.onStep?.(runId, capturedSteps.length); @@ -1649,6 +1824,8 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // closure scope here). Omitted/0 = no limit. maxContextTokens: resolved?.chatContextWindow, pageChanged, + partsCache, + replayTrimmedToTokens, }), ); // #184/#487: the RUN is finalized ALWAYS (never gated on the message). @@ -1673,6 +1850,8 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // own edits are baked in — and this also SEEDS the snapshot on the first // turn. Runs once across every terminal path (see snapshotTurnEnd). await snapshotTurnEnd(); + // #490: persist the deferred-tool activation set for the next turn. + await persistActivatedTools(); // Generate the chat title for a freshly created chat AFTER the stream's // provider call has completed — NOT concurrently with it. The z.ai coding @@ -1696,7 +1875,16 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // object, so the actual provider cause is clearly logged. Reuse the // shared formatter so provider error formatting stays unified. const e = error as { stack?: string }; - const errorText = describeProviderError(error, String(error)); + // #490 reactive branch: classify a CONTEXT-OVERFLOW rejection (the + // replayed history exceeded the model window). The overflowing turn had + // no prior usage to trigger preventive trimming, so we record a clear, + // distinguishable cause AND stamp the row so the NEXT turn's budgeter + // trims aggressively — the reactive recovery that un-bricks the chat. + const overflow = isContextOverflowError(error); + const providerError = describeProviderError(error, String(error)); + const errorText = overflow + ? `${CONTEXT_OVERFLOW_ERROR_PREFIX} (${providerError})` + : providerError; this.logger.error(`AI chat stream error: ${errorText}`, e?.stack); // DIAGNOSTIC (Safari stream-drop investigation) — temporary: timing of // an error-terminated stream. @@ -1714,6 +1902,9 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { flushAssistant(capturedSteps, inProgressText, 'error', { error: errorText, pageChanged, + partsCache, + replayTrimmedToTokens, + replayOverflow: overflow || undefined, }), ); // #184: settle the RUN as failed, carrying the provider/transport cause. @@ -1723,6 +1914,8 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // committed before the error must be baked into the snapshot, or the // next turn would mis-report it as a user edit. await snapshotTurnEnd(); + // #490: persist the deferred-tool activation set for the next turn. + await persistActivatedTools(); }, onAbort: async ({ steps }) => { // #444: distinguish a degeneration abort (our internal controller) from @@ -1737,6 +1930,7 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { flushAssistant(capturedSteps, truncated, 'error', { error: OUTPUT_DEGENERATION_ERROR, pageChanged, + partsCache, }), ); if (runId) @@ -1747,6 +1941,8 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { ); await closeExternalClients(); await snapshotTurnEnd(); + // #490: persist the deferred-tool activation set for the next turn. + await persistActivatedTools(); return; } const partialChars = @@ -1771,6 +1967,7 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { await finalizeAssistant( flushAssistant(capturedSteps, inProgressText, 'aborted', { pageChanged, + partsCache, }), ); // #184: settle the RUN as aborted (an explicit user stop reached the @@ -1781,6 +1978,8 @@ export class AiChatService implements OnModuleInit, OnModuleDestroy { // committed before the client disconnect / stop() must be baked into the // snapshot, or the next turn would mis-report it as a user edit. await snapshotTurnEnd(); + // #490: persist the deferred-tool activation set for the next turn. + await persistActivatedTools(); }, }); @@ -2091,6 +2290,70 @@ export function chatStreamMetadata( return undefined; } +/** + * The provider-reported context size of the most recent assistant turn, read from + * its persisted `metadata.contextTokens` (#490 replay budgeter's PRIMARY signal — + * the provider's own fact, not an estimate). Returns undefined for a chat with no + * assistant turn yet, or one whose last turn recorded no usage (e.g. it errored), + * in which case the budgeter falls back to the char-estimate. + */ +export function lastAssistantContextTokens( + history: ReadonlyArray, +): number | undefined { + for (let i = history.length - 1; i >= 0; i--) { + const row = history[i]; + if (row.role !== 'assistant') continue; + const meta = (row.metadata ?? {}) as { contextTokens?: unknown }; + const n = meta.contextTokens; + return typeof n === 'number' && Number.isFinite(n) && n > 0 ? n : undefined; + } + return undefined; +} + +/** + * Seed the per-turn deferred-tool activation set from a chat's persisted metadata + * (#490), INTERSECTED with the current valid deferred names. Persisting the set + * across turns saves the model re-running loadTools every turn to re-activate the + * same tools; intersecting on load means a changed allowlist / role can never + * resurrect a tool that no longer exists (which would hand prepareAgentStep a + * phantom active name). Tolerant of any stored shape — a non-array is ignored. + */ +export function seedActivatedTools( + metadata: Record | undefined, + validDeferredNames: ReadonlySet, +): string[] { + const stored = metadata?.activatedTools; + if (!Array.isArray(stored)) return []; + const seen = new Set(); + const out: string[] = []; + for (const name of stored) { + if (typeof name === 'string' && validDeferredNames.has(name) && !seen.has(name)) { + seen.add(name); + out.push(name); + } + } + return out; +} + +/** + * Whether the most recent assistant turn was rejected for CONTEXT OVERFLOW + * (#490): its row carries `metadata.replayOverflow` (stamped by the stream's + * onError). The next turn's budgeter reads this to trim aggressively — the + * reactive recovery. Only the LAST assistant turn matters (an older overflow was + * already recovered), so we stop at the first assistant row scanning backwards. + */ +export function lastAssistantReplayOverflow( + history: ReadonlyArray, +): boolean { + for (let i = history.length - 1; i >= 0; i--) { + const row = history[i]; + if (row.role !== 'assistant') continue; + const meta = (row.metadata ?? {}) as { replayOverflow?: unknown }; + return meta.replayOverflow === true; + } + return false; +} + /** The last message with role 'user' from a useChat payload, if any. */ function lastUserMessage( messages: UIMessage[] | undefined, @@ -2152,6 +2415,15 @@ export function sanitizeUserParts( /** Marker for a history row whose tool parts could not be replayed (#489). */ export const TOOL_CONTEXT_OMITTED_MARKER = '[tool context omitted]'; +/** + * Synthetic error text for a tool call that neither returned a result nor threw + * a `tool-error` — i.e. it was interrupted mid-step (an abort / server restart). + * Shared by `assistantParts` (the replayed `output-error` part) and + * `serializeSteps` (the `{ kind: 'interrupted' }` trace element) so the replay + * text and the trace stay in lockstep (#490). + */ +export const TOOL_CALL_INCOMPLETE_TEXT = 'Tool call did not complete.'; + /** * Convert persisted UI history to model messages, tolerating a single poisoned * row (#489). `convertToModelMessages` over the WHOLE array throws if ANY row is @@ -2359,71 +2631,97 @@ function normalizeToolError(error: unknown): string { */ // Exported only so the unit tests can import these pure helpers; exporting // them does not change runtime behavior. +/** + * Per-turn memo for {@link assistantParts}: a step's rebuilt parts keyed by the + * step OBJECT's identity (#490). A finished step in `capturedSteps` keeps a stable + * reference across every mid-stream flush, and `compactToolOutput` inside it does a + * `JSON.stringify` of the whole (often 50–200 KB) output — so without a memo each + * `onStepFinish` re-stringifies EVERY prior step's output (O(N²) stringify over a + * turn). Keyed by step identity => one stringify per step per turn. WeakMap so a + * turn's steps are GC'd with the turn. + */ +export type StepPartsCache = WeakMap>>; + +/** Build the parts for ONE step (text + a part per tool call). Pure. */ +function buildStepParts(step: StepLike): Array> { + const parts: Array> = []; + if (step.text) { + parts.push({ type: 'text', text: step.text }); + } + // Index this step's results by tool call id to pair calls with outputs. + const resultsById = new Map(); + for (const r of step.toolResults ?? []) { + if (r.toolCallId) resultsById.set(r.toolCallId, r.output); + } + // Index this step's THROWN tool failures (ai@6 `tool-error` content parts) + // by tool call id, so a call that failed replays with its real error text. + const errorsById = new Map(); + for (const part of step.content ?? []) { + if (part.type === 'tool-error' && part.toolCallId) { + errorsById.set(part.toolCallId, part.error); + } + } + for (const call of step.toolCalls ?? []) { + if (!call.toolName || !call.toolCallId) continue; + const hasResult = resultsById.has(call.toolCallId); + if (hasResult) { + // output-available: the tool returned; the next turn replays its result. + parts.push({ + type: `tool-${call.toolName}`, + toolCallId: call.toolCallId, + state: 'output-available', + input: call.input, + output: compactToolOutput(resultsById.get(call.toolCallId)), + }); + } else if (errorsById.has(call.toolCallId)) { + // The tool THREW: replay the REAL error so the model on the next turn + // knows WHY the call failed (and does not blindly repeat it). An + // output-error round-trips through convertToModelMessages as a balanced + // tool-call + tool-result, keeping the rebuilt history valid. + parts.push({ + type: `tool-${call.toolName}`, + toolCallId: call.toolCallId, + state: 'output-error', + input: call.input, + errorText: normalizeToolError(errorsById.get(call.toolCallId)), + }); + } else { + // No paired result AND no tool-error (e.g. aborted mid-step). Persisting + // a bare tool-call (input-available) would replay as an unpaired call and + // throw MissingToolResultsError on the next turn (convertToModelMessages + // emits no tool-result for it). Emit a SYNTHETIC paired result instead: + // an output-error round-trips through convertToModelMessages as a + // balanced tool-call + tool-result, keeping the rebuilt history valid. + parts.push({ + type: `tool-${call.toolName}`, + toolCallId: call.toolCallId, + state: 'output-error', + input: call.input, + errorText: TOOL_CALL_INCOMPLETE_TEXT, + }); + } + } + return parts; +} + export function assistantParts( steps: ReadonlyArray | undefined, fallbackText: string, + cache?: StepPartsCache, ): UIMessage['parts'] { const parts: Array> = []; - let sawText = false; for (const step of steps ?? []) { - if (step.text) { - parts.push({ type: 'text', text: step.text }); - sawText = true; - } - // Index this step's results by tool call id to pair calls with outputs. - const resultsById = new Map(); - for (const r of step.toolResults ?? []) { - if (r.toolCallId) resultsById.set(r.toolCallId, r.output); - } - // Index this step's THROWN tool failures (ai@6 `tool-error` content parts) - // by tool call id, so a call that failed replays with its real error text. - const errorsById = new Map(); - for (const part of step.content ?? []) { - if (part.type === 'tool-error' && part.toolCallId) { - errorsById.set(part.toolCallId, part.error); - } - } - for (const call of step.toolCalls ?? []) { - if (!call.toolName || !call.toolCallId) continue; - const hasResult = resultsById.has(call.toolCallId); - if (hasResult) { - // output-available: the tool returned; the next turn replays its result. - parts.push({ - type: `tool-${call.toolName}`, - toolCallId: call.toolCallId, - state: 'output-available', - input: call.input, - output: compactToolOutput(resultsById.get(call.toolCallId)), - }); - } else if (errorsById.has(call.toolCallId)) { - // The tool THREW: replay the REAL error so the model on the next turn - // knows WHY the call failed (and does not blindly repeat it). An - // output-error round-trips through convertToModelMessages as a balanced - // tool-call + tool-result, keeping the rebuilt history valid. - parts.push({ - type: `tool-${call.toolName}`, - toolCallId: call.toolCallId, - state: 'output-error', - input: call.input, - errorText: normalizeToolError(errorsById.get(call.toolCallId)), - }); - } else { - // No paired result AND no tool-error (e.g. aborted mid-step). Persisting - // a bare tool-call (input-available) would replay as an unpaired call and - // throw MissingToolResultsError on the next turn (convertToModelMessages - // emits no tool-result for it). Emit a SYNTHETIC paired result instead: - // an output-error round-trips through convertToModelMessages as a - // balanced tool-call + tool-result, keeping the rebuilt history valid. - parts.push({ - type: `tool-${call.toolName}`, - toolCallId: call.toolCallId, - state: 'output-error', - input: call.input, - errorText: 'Tool call did not complete.', - }); - } + // Memoize per step object (#490): a finished step is immutable and keeps its + // reference across flushes, so its parts (and the costly output stringify) are + // built exactly once per turn. A cache miss (or no cache) just rebuilds. + let stepParts = cache?.get(step as object); + if (!stepParts) { + stepParts = buildStepParts(step); + cache?.set(step as object, stepParts); } + parts.push(...stepParts); } + const sawText = parts.some((p) => p.type === 'text'); if (!sawText && fallbackText) { // No per-step text (e.g. a single final block): append the final text after // any tool parts so the natural call -> result -> answer order is preserved. @@ -2586,6 +2884,16 @@ export function flushAssistant( maxContextTokens?: number; error?: string; pageChanged?: { title: string; diff: string } | null; + // Per-turn step->parts memo (#490): pass the SAME cache on every flush of a + // turn so each finished step's output is stringified once, not once per flush. + partsCache?: StepPartsCache; + // #490 observability: when the replay budgeter trimmed this turn's history, + // the (estimated) token size it trimmed to — the UI can show "replay truncated + // at N tokens". Omitted when nothing was trimmed. + replayTrimmedToTokens?: number; + // #490 reactive branch: set when the provider rejected this turn for context + // overflow. Stamped into metadata so the NEXT turn's budgeter trims aggressively. + replayOverflow?: boolean; }, ): AssistantFlush { const finished = capturedSteps ?? []; @@ -2595,13 +2903,32 @@ export function flushAssistant( // in-progress step's text (the partial answer cut off by an error/abort, or // simply not yet flushed mid-stream) as the last text part so the persisted // parts match what streamed to the client. - const parts = assistantParts(finished, '') as unknown as Array< - Record - >; + const parts = assistantParts( + finished, + '', + extra?.partsCache, + ) as unknown as Array>; if (trailing) parts.push({ type: 'text', text: trailing }); const metadata: Record = { parts: parts as unknown as UIMessage['parts'], + // Era marker for the `tool_calls` trace shape (#490): v2 stores outcome flags + // ({ ok } / { error, kind }) and NO tool output (the output lives once in + // `parts`). Old rows have no marker and the legacy { output } shape; a + // dual-shape query branches on this. Old rows are deliberately NOT migrated. + toolTraceVersion: 2, + // #491 STEP MARKER: the number of FINISHED steps whose parts are in THIS row, + // written by the SAME flush that builds `parts` (atomically — they are both + // derived from `finished`, so the marker can NEVER disagree with the persisted + // parts). This is the step-alignment anchor the resume stack builds on: + // - the registry rotates its retention ring only on a CONFIRMED persist of + // step N (commit 3); + // - attach slices the tail at "step > N" from the client's persisted seed. + // It is NOT `run.stepCount`: recordStep is fire-and-forget and NOT atomic with + // the parts write, so stepCount could race ahead of the persisted parts + // (seed↔marker drift). The in-progress trailing text (an error/abort partial, + // or a mid-stream flush) is NOT a finished step and is excluded from the count. + stepsPersisted: finished.length, }; // finishReason: prefer an explicit one; else derive a sensible value from the // terminal status (so onError/onAbort records keep their historical reason). @@ -2617,6 +2944,9 @@ export function flushAssistant( if (extra?.contextTokens) metadata.contextTokens = extra.contextTokens; if (extra?.maxContextTokens) metadata.maxContextTokens = extra.maxContextTokens; + if (extra?.replayTrimmedToTokens) + metadata.replayTrimmedToTokens = extra.replayTrimmedToTokens; + if (extra?.replayOverflow) metadata.replayOverflow = true; if (extra?.error) metadata.error = extra.error; // Persist the page-change diff the agent saw this turn (#274 observability), // so history / the Markdown export can show what the user changed. Only when @@ -2642,42 +2972,85 @@ export function flushAssistant( /** * Reduce SDK step objects to a compact, JSON-serializable trace for the - * `tool_calls` column. Stores only what the UI action-log and history need — - * never raw provider payloads or keys. + * `tool_calls` column — trace format **v2** (#490). + * + * v2 stores, per call, ONLY the metadata a queryable trace needs — never the + * tool OUTPUT. Before #490 each output was persisted TWICE: once here (compacted) + * and once in `metadata.parts` (via `assistantParts`), so a 50-step run with + * 50–200 KB outputs wrote hundreds of MB per turn (each `onStepFinish` rewrote + * the whole row). The parts copy is the one the model replays and the UI/Markdown + * export render, so the trace copy of the output was pure duplication. v2 keeps + * the output ONLY in parts and reduces the trace to outcome flags. + * + * Element shapes (paired per call, in order): + * - `{ toolName, input }` — the call + * - `{ toolName, ok: true }` — it returned a result (success) + * - `{ toolName, error, kind: 'thrown' }` — it threw a `tool-error` + * - `{ toolName, error, kind: 'interrupted' }` — no result and no throw (an + * abort / server restart mid-step). `kind` is MANDATORY: without it a + * synthetic "Tool call did not complete." is indistinguishable from a real + * hard-fail and pollutes any error-rate scan. The distinction is STRUCTURAL + * (an `errorsById` hit vs the synthetic fallback branch), NOT a per-tool + * classifier — soft failures stay OUT of the trace (they live in + * `metadata.parts` outputs; a per-tool mirror would persist its own bugs). + * + * Rows carry `metadata.toolTraceVersion: 2` (set by {@link flushAssistant}) so a + * dual-shape query can branch on the era. Old rows are NOT migrated (rewriting + * giant jsonb is the very WAL churn this removes); see docs/reading-ai-logs.md. */ export function serializeSteps( steps: ReadonlyArray<{ - toolCalls?: ReadonlyArray<{ toolName?: string; input?: unknown }>; - toolResults?: ReadonlyArray<{ toolName?: string; output?: unknown }>; + toolCalls?: ReadonlyArray<{ + toolCallId?: string; + toolName?: string; + input?: unknown; + }>; + toolResults?: ReadonlyArray<{ toolCallId?: string; toolName?: string }>; content?: ReadonlyArray<{ type?: string; + toolCallId?: string; toolName?: string; error?: unknown; }>; }>, ): unknown { - const calls: Array<{ - toolName?: string; - input?: unknown; - output?: unknown; - error?: string; - }> = []; + const calls: Array< + | { toolName?: string; input?: unknown } + | { toolName?: string; ok: true } + | { toolName?: string; error: string; kind: 'thrown' | 'interrupted' } + > = []; for (const step of steps ?? []) { + // Index this step's results + thrown errors by tool call id, so each call is + // paired with its outcome (mirrors assistantParts' pairing exactly). + const resultIds = new Set(); + for (const r of step.toolResults ?? []) { + if (r.toolCallId) resultIds.add(r.toolCallId); + } + const errorsById = new Map(); + for (const part of step.content ?? []) { + if (part.type === 'tool-error' && part.toolCallId) { + errorsById.set(part.toolCallId, part.error); + } + } for (const call of step.toolCalls ?? []) { calls.push({ toolName: call.toolName, input: call.input }); - } - for (const r of step.toolResults ?? []) { - calls.push({ toolName: r.toolName, output: compactToolOutput(r.output) }); - } - // ai@6 surfaces a THROWN tool failure as a `tool-error` content part, NOT as - // a `toolResults` entry. Record it as its own paired element (mirroring how a - // successful result is appended) so the failure and its reason survive in the - // trace instead of leaving an orphaned call with no result. - for (const part of step.content ?? []) { - if (part.type === 'tool-error') { + if (call.toolCallId && resultIds.has(call.toolCallId)) { + // Success: the output itself lives in metadata.parts, not here. + calls.push({ toolName: call.toolName, ok: true }); + } else if (call.toolCallId && errorsById.has(call.toolCallId)) { + // Hard fail: the tool threw. Persist the real (bounded) reason. calls.push({ - toolName: part.toolName, - error: normalizeToolError(part.error), + toolName: call.toolName, + error: normalizeToolError(errorsById.get(call.toolCallId)), + kind: 'thrown', + }); + } else { + // Neither a result nor a throw: interrupted mid-step (abort/restart). + // Marked structurally so it never inflates a thrown-error count. + calls.push({ + toolName: call.toolName, + error: TOOL_CALL_INCOMPLETE_TEXT, + kind: 'interrupted', }); } } diff --git a/apps/server/src/core/ai-chat/ai-chat.step-marker.spec.ts b/apps/server/src/core/ai-chat/ai-chat.step-marker.spec.ts new file mode 100644 index 00000000..a4398a3d --- /dev/null +++ b/apps/server/src/core/ai-chat/ai-chat.step-marker.spec.ts @@ -0,0 +1,65 @@ +import { flushAssistant } from './ai-chat.service'; + +/** + * #491 STEP MARKER — `metadata.stepsPersisted` is written by the SAME flush that + * builds `metadata.parts`, so the marker can never disagree with the persisted + * parts (the step-alignment anchor the resume stack builds on). These are + * PROPERTY tests: they assert the marker tracks the number of FINISHED steps for + * every flush shape. + */ + +// A finished step carrying one line of text and one tool call/result. +function step(i: number) { + return { + text: `step ${i}`, + toolCalls: [ + { toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } }, + ], + toolResults: [ + { toolCallId: `c${i}`, toolName: 'getPage', output: { title: `T${i}` } }, + ], + }; +} + +describe('flushAssistant step marker (#491)', () => { + it('seed (no steps) → stepsPersisted 0', () => { + const f = flushAssistant([], '', 'streaming'); + expect(f.metadata.stepsPersisted).toBe(0); + }); + + it('PROPERTY: stepsPersisted equals the number of FINISHED steps, for any N', () => { + for (let n = 0; n <= 6; n++) { + const steps = Array.from({ length: n }, (_, i) => step(i)); + const f = flushAssistant(steps, '', 'streaming'); + expect(f.metadata.stepsPersisted).toBe(n); + // ...and the parts actually contain those N steps' text (marker agrees with + // the persisted parts — the atomicity the whole design relies on). + const parts = f.metadata.parts as Array>; + const textParts = parts.filter((p) => p.type === 'text'); + expect(textParts).toHaveLength(n); + } + }); + + it('an in-progress trailing partial does NOT increment the marker', () => { + // 2 finished steps + a partial (not-yet-finished) trailing text: the marker + // counts only the CONFIRMED step boundaries, not the partial. + const f = flushAssistant([step(0), step(1)], 'partial third step', 'error', { + error: 'boom', + }); + expect(f.metadata.stepsPersisted).toBe(2); + // The partial text IS persisted in parts (so the user sees it), but it is not a + // counted step. + const parts = f.metadata.parts as Array>; + expect(parts[parts.length - 1]).toEqual({ + type: 'text', + text: 'partial third step', + }); + }); + + it('terminal completed flush counts all finished steps', () => { + const f = flushAssistant([step(0), step(1), step(2)], '', 'completed', { + finishReason: 'stop', + }); + expect(f.metadata.stepsPersisted).toBe(3); + }); +}); diff --git a/apps/server/src/core/ai-chat/ai-chat.write-volume.int-spec.ts b/apps/server/src/core/ai-chat/ai-chat.write-volume.int-spec.ts new file mode 100644 index 00000000..cbd12ad2 --- /dev/null +++ b/apps/server/src/core/ai-chat/ai-chat.write-volume.int-spec.ts @@ -0,0 +1,209 @@ +import { randomBytes } from 'crypto'; +import { Client } from 'pg'; +import { flushAssistant, serializeSteps } from './ai-chat.service'; + +/** + * #490 write-volume regression — an OBSERVABLE-PROPERTY test on a LIVE Postgres, + * not "bytes through a mock repo" (a mock measures exactly the thing that does not + * hurt). It drives a realistic 50-step run where each step returns a ~100 KB tool + * output and, at every `onStepFinish`, UPDATEs the assistant row the way the + * service does — then reads the REAL write volume via the `pg_current_wal_lsn()` + * delta around the run. + * + * The property proven: v2 stores each tool OUTPUT only in `metadata.parts`, no + * longer ALSO in the `tool_calls` trace. So: + * 1. the trace (`tool_calls`) column's write volume is now O(Σ steps) — tiny, + * linear outcome flags — vs the pre-#490 O(N²) that re-persisted every prior + * output on every step; and + * 2. the FULL-row write volume drops sharply (the duplicated output copy is gone). + * + * Connects to the local gitmost test Postgres (docker `gitmost-test-pg` on :5432); + * SKIPS cleanly when that DB is not reachable so it never breaks a DB-less CI. + */ +const CONN = + process.env.WAL_TEST_DATABASE_URL ?? + 'postgresql://docmost:docmost_dev_pw@localhost:5432/docmost'; + +// A step whose tool output is ~100 KB (a page read), in the SDK StepLike shape. +// The body is INCOMPRESSIBLE random text — a `'x'.repeat()` filler would TOAST- +// compress to nothing and hide the real write volume (a page body does not). +function makeStep(i: number, outputBytes = 100_000) { + const body = randomBytes(Math.ceil(outputBytes * 0.75)).toString('base64'); + return { + text: `step ${i} reasoning`, + toolCalls: [{ toolCallId: `c${i}`, toolName: 'getPage', input: { id: `p${i}` } }], + toolResults: [ + { + toolCallId: `c${i}`, + toolName: 'getPage', + output: { id: `p${i}`, title: `Page ${i}`, body }, + }, + ], + }; +} + +// The pre-#490 (v1) trace: outputs stored a SECOND time in `tool_calls` +// (the duplication #490 removed). Mirrors the OLD serializeSteps shape. +function v1Trace(steps: ReturnType[]): unknown { + const calls: unknown[] = []; + for (const s of steps) { + for (const c of s.toolCalls) calls.push({ toolName: c.toolName, input: c.input }); + for (const r of s.toolResults) + calls.push({ toolName: r.toolName, output: r.output }); + } + return calls; +} + +async function walDelta( + client: Client, + fn: () => Promise, +): Promise { + const before = (await client.query('SELECT pg_current_wal_lsn() AS l')).rows[0] + .l as string; + await fn(); + // NOTE: do NOT pg_switch_wal() here — a segment switch pads the LSN to the next + // 16 MB boundary and would swamp the actual write delta. The raw LSN advances by + // the bytes of WAL emitted, which is exactly what we want to measure. + const after = (await client.query('SELECT pg_current_wal_lsn() AS l')).rows[0] + .l as string; + return Number( + (await client.query('SELECT pg_wal_lsn_diff($1,$2) AS d', [after, before])) + .rows[0].d, + ); +} + +describe('#490 write-volume on a live Postgres (pg_current_wal_lsn delta)', () => { + let client: Client | undefined; + let available = false; + + beforeAll(async () => { + try { + client = new Client(CONN); + await client.connect(); + await client.query('SELECT pg_current_wal_lsn()'); + available = true; + } catch { + available = false; + client = undefined; + } + }); + + afterAll(async () => { + await client?.end().catch(() => undefined); + }); + + const STEPS = 50; + + it('v2 trace write volume is O(Σ steps) — a tiny fraction of the v1 duplicate', async () => { + if (!available || !client) { + console.warn('SKIP: gitmost-test-pg not reachable; skipping WAL test.'); + return; + } + const c = client; + // Isolated table so we measure only the tool_calls (trace) column's writes. + await c.query('DROP TABLE IF EXISTS _wal_trace'); + await c.query('CREATE TABLE _wal_trace(id int primary key, tool_calls jsonb)'); + await c.query("INSERT INTO _wal_trace VALUES (1, '[]'::jsonb)"); + + const steps: ReturnType[] = []; + + // v1: each step re-persists ALL prior outputs into the trace (the O(N²) churn). + const v1 = await walDelta(c, async () => { + const acc: ReturnType[] = []; + for (let i = 0; i < STEPS; i++) { + acc.push(makeStep(i)); + await c.query('UPDATE _wal_trace SET tool_calls=$1 WHERE id=1', [ + JSON.stringify(v1Trace(acc)), + ]); + } + steps.push(...acc); + }); + + await c.query("UPDATE _wal_trace SET tool_calls='[]'::jsonb WHERE id=1"); + + // v2: the REAL serializeSteps — outcome flags only, NO outputs. + const v2 = await walDelta(c, async () => { + const acc: ReturnType[] = []; + for (let i = 0; i < STEPS; i++) { + acc.push(makeStep(i)); + await c.query('UPDATE _wal_trace SET tool_calls=$1 WHERE id=1', [ + JSON.stringify(serializeSteps(acc)), + ]); + } + }); + + await c.query('DROP TABLE IF EXISTS _wal_trace'); + + // eslint-disable-next-line no-console + console.log( + `[#490 WAL] trace column over ${STEPS} steps: v1=${(v1 / 1e6).toFixed(1)}MB ` + + `v2=${(v2 / 1e6).toFixed(2)}MB (${(v1 / v2).toFixed(0)}x smaller)`, + ); + + // The trace no longer carries outputs: v2 is a tiny fraction of v1's WAL. + expect(v2).toBeLessThan(v1 * 0.1); + // And v2's trace WAL is small in absolute terms — O(Σ steps) of flags, not + // O(N² × output). 50 steps of ~40-byte flags is well under a few MB of WAL. + expect(v2).toBeLessThan(5_000_000); + // v1's duplicate alone is huge (≈ the 100 KB output re-written N² times). + expect(v1).toBeGreaterThan(50_000_000); + }, 120_000); + + it('the full assistant row write drops sharply once the duplicate is gone', async () => { + if (!available || !client) return; + const c = client; + await c.query('DROP TABLE IF EXISTS _wal_full'); + await c.query( + 'CREATE TABLE _wal_full(id int primary key, content text, tool_calls jsonb, metadata jsonb, status text)', + ); + await c.query("INSERT INTO _wal_full VALUES (1, '', '[]'::jsonb, '{}'::jsonb, 'streaming')"); + + const writeRow = async (patch: { + content: string; + toolCalls: unknown; + metadata: unknown; + status: string; + }) => + c.query( + 'UPDATE _wal_full SET content=$1, tool_calls=$2, metadata=$3, status=$4 WHERE id=1', + [ + patch.content, + JSON.stringify(patch.toolCalls ?? null), + JSON.stringify(patch.metadata), + patch.status, + ], + ); + + // v2 (real flushAssistant): outputs live once, in metadata.parts. + const v2 = await walDelta(c, async () => { + const acc: ReturnType[] = []; + for (let i = 0; i < STEPS; i++) { + acc.push(makeStep(i)); + await writeRow(flushAssistant(acc as never, '', 'streaming')); + } + }); + + await c.query("UPDATE _wal_full SET content='', tool_calls='[]'::jsonb, metadata='{}'::jsonb WHERE id=1"); + + // v1: same row PLUS the duplicated outputs in the trace column. + const v1 = await walDelta(c, async () => { + const acc: ReturnType[] = []; + for (let i = 0; i < STEPS; i++) { + acc.push(makeStep(i)); + const f = flushAssistant(acc as never, '', 'streaming'); + await writeRow({ ...f, toolCalls: v1Trace(acc) }); + } + }); + + await c.query('DROP TABLE IF EXISTS _wal_full'); + + // eslint-disable-next-line no-console + console.log( + `[#490 WAL] full row over ${STEPS} steps: v1=${(v1 / 1e6).toFixed(1)}MB ` + + `v2=${(v2 / 1e6).toFixed(1)}MB (saved ${((1 - v2 / v1) * 100).toFixed(0)}%)`, + ); + + // Removing the duplicated trace copy is a large, real write-volume reduction. + expect(v2).toBeLessThan(v1 * 0.75); + }, 120_000); +}); diff --git a/apps/server/src/core/ai-chat/chat-markdown.util.spec.ts b/apps/server/src/core/ai-chat/chat-markdown.util.spec.ts index e53b5dfa..ea238430 100644 --- a/apps/server/src/core/ai-chat/chat-markdown.util.spec.ts +++ b/apps/server/src/core/ai-chat/chat-markdown.util.spec.ts @@ -1,5 +1,10 @@ -import { buildChatMarkdown, normalizeLang } from './chat-markdown.util'; +import { + buildChatMarkdown, + normalizeLang, + labelledToolNames, +} from './chat-markdown.util'; import type { AiChatMessage } from '@docmost/db/types/entity.types'; +import { SHARED_TOOL_SPECS } from '../../../../../packages/mcp/src/tool-specs'; /** * normalizeLang: the client sends `i18n.language` — a FULL locale tag like @@ -455,3 +460,43 @@ describe('buildChatMarkdown (server) — structure', () => { expect(md).toContain('````'); }); }); + +/** + * #494 — REVERSE drift-guard for the export's friendly tool labels. A label keyed + * by a tool name that no longer exists silently degrades to the generic + * "Ran tool " line; nothing reddened before. This asserts every labelled + * name is a real in-app tool and that both languages label the same set. + */ +describe('tool-label parity (#494)', () => { + // In-app tool names come from the shared registry (inAppKey, excluding + // mcpOnly specs) PLUS the inline in-app-only tools that carry a friendly label. + // The only labelled inline tool is the hybrid semantic search. + const INLINE_INAPP_LABELLED = new Set(['searchPages']); + + function validInAppToolNames(): Set { + const names = new Set(INLINE_INAPP_LABELLED); + for (const spec of Object.values(SHARED_TOOL_SPECS)) { + if ((spec as { mcpOnly?: boolean }).mcpOnly) continue; + names.add((spec as { inAppKey: string }).inAppKey); + } + return names; + } + + it('en and ru label the SAME set of tools', () => { + expect(labelledToolNames('en').sort()).toEqual( + labelledToolNames('ru').sort(), + ); + }); + + it('every labelled tool name is a real in-app tool', () => { + const valid = validInAppToolNames(); + const dead = labelledToolNames('en').filter((n) => !valid.has(n)); + expect(dead).toEqual([]); + }); + + it('the guard REDDENS for an unknown label key (mutation check)', () => { + const valid = validInAppToolNames(); + // A hypothetical renamed-away label must be caught. + expect(valid.has('getPageRenamedAway')).toBe(false); + }); +}); diff --git a/apps/server/src/core/ai-chat/chat-markdown.util.ts b/apps/server/src/core/ai-chat/chat-markdown.util.ts index 00da9da9..6be64b45 100644 --- a/apps/server/src/core/ai-chat/chat-markdown.util.ts +++ b/apps/server/src/core/ai-chat/chat-markdown.util.ts @@ -154,6 +154,17 @@ function toolLabel(name: string, lang: ExportLang): string { return LABELS[lang].tools[name] ?? LABELS[lang].ranTool(name); } +/** + * The tool names that carry a hand-written friendly export label, per language. + * Exported for the drift-guard (#494): a label keyed by a tool name that no + * longer exists is a DEAD entry (the tool was renamed and now silently falls back + * to the generic `ranTool(name)` line). The guard asserts every key here is a + * real in-app tool AND that the two languages label the SAME set of tools. + */ +export function labelledToolNames(lang: ExportLang): string[] { + return Object.keys(LABELS[lang].tools); +} + /** * Stringify an arbitrary tool input/output value for a fenced block. Strings * pass through as-is; everything else is pretty-printed JSON, falling back to diff --git a/apps/server/src/core/ai-chat/dto/ai-chat.dto.ts b/apps/server/src/core/ai-chat/dto/ai-chat.dto.ts index 1fd3c8ff..517f74ed 100644 --- a/apps/server/src/core/ai-chat/dto/ai-chat.dto.ts +++ b/apps/server/src/core/ai-chat/dto/ai-chat.dto.ts @@ -1,4 +1,10 @@ -import { IsOptional, IsString, MaxLength, MinLength } from 'class-validator'; +import { + IsISO8601, + IsOptional, + IsString, + MaxLength, + MinLength, +} from 'class-validator'; /** Identify a chat by id (workspace-scoped on the server). */ export class ChatIdDto { @@ -37,6 +43,24 @@ export class GetChatMessagesDto { cursor?: string; } +/** + * Delta poll (#491): pull the chat's rows changed since `cursor` (a DB-clock + * timestamp from the previous poll) plus the current run fact — the degraded-poll + * fallback's payload, replacing the full infinite-query refetch. Omit `cursor` on + * the first poll (returns just a fresh cursor to start the chain). + */ +export class GetChatDeltaDto { + @IsString() + chatId: string; + + // ISO-8601 timestamp echoed from the previous poll's response. Validated as + // ISO-8601 (not a bare string): a malformed cursor would otherwise reach the + // `::timestamptz` cast in findByChatUpdatedAfter and 500 instead of a clean 400. + @IsOptional() + @IsISO8601() + cursor?: string; +} + /** Resolve the chat bound to a document (the page's most-recent owned chat). */ export class BoundChatDto { @IsString() diff --git a/apps/server/src/core/ai-chat/external-mcp/dto/create-mcp-server.dto.ts b/apps/server/src/core/ai-chat/external-mcp/dto/create-mcp-server.dto.ts index b422fba8..6c993f11 100644 --- a/apps/server/src/core/ai-chat/external-mcp/dto/create-mcp-server.dto.ts +++ b/apps/server/src/core/ai-chat/external-mcp/dto/create-mcp-server.dto.ts @@ -37,10 +37,13 @@ export class CreateMcpServerDto { @IsObject() headers?: Record; + // Omit/null => no restriction; `[]` is persisted verbatim and means deny-all + // (zero tools) since #476. @IsOptional() skips validation for null as well, + // so an explicit null is accepted. @IsOptional() @IsArray() @IsString({ each: true }) - toolAllowlist?: string[]; + toolAllowlist?: string[] | null; // Admin-authored guidance ("how/when to use this server's tools") injected // into the agent system prompt next to the tool descriptions (#180). Trusted, diff --git a/apps/server/src/core/ai-chat/external-mcp/dto/update-mcp-server.dto.ts b/apps/server/src/core/ai-chat/external-mcp/dto/update-mcp-server.dto.ts index aa8063c6..3156b17b 100644 --- a/apps/server/src/core/ai-chat/external-mcp/dto/update-mcp-server.dto.ts +++ b/apps/server/src/core/ai-chat/external-mcp/dto/update-mcp-server.dto.ts @@ -38,10 +38,13 @@ export class UpdateMcpServerDto { @IsObject() headers?: Record; + // Absent => unchanged; null => no restriction; `[]` is persisted verbatim + // and means deny-all (zero tools) since #476. @IsOptional() skips validation + // for null as well, so an explicit null is accepted. @IsOptional() @IsArray() @IsString({ each: true }) - toolAllowlist?: string[]; + toolAllowlist?: string[] | null; // Admin-authored prompt guidance (#180). Absent => unchanged; blank => cleared // (stored as null by the repo). Capped to bound prompt/token size. diff --git a/apps/server/src/core/ai-chat/external-mcp/mcp-allowlist-filter.spec.ts b/apps/server/src/core/ai-chat/external-mcp/mcp-allowlist-filter.spec.ts new file mode 100644 index 00000000..cb4d898e --- /dev/null +++ b/apps/server/src/core/ai-chat/external-mcp/mcp-allowlist-filter.spec.ts @@ -0,0 +1,169 @@ +import { type Tool } from 'ai'; +import { McpClientsService } from './mcp-clients.service'; + +/** + * Tool-allowlist filtering semantics on the merged external toolset (#476). + * + * COVERAGE CHOICE (documented per issue #476): the full corrupt-row chain + * (DB value -> repo normalizeRow -> toolsFor filter) is covered on TWO levels + * instead of one live-stub-MCP-server integration test: + * (a) apps/server/test/integration/ai-mcp-server-repo.int-spec.ts pins the + * repo read/write semantics against a real Postgres — `[]` round-trips + * as jsonb `[]`, a present-but-corrupt value fails CLOSED to `[]` with + * an error log; + * (b) THIS spec pins what the toolset builder does with the repo's output — + * null = unrestricted, `['alpha']` = only alpha, `[]` (including the + * corrupt-row fallback) = ZERO tools. + * Together they prove the end-to-end property "corrupt/empty allowlist can + * never widen to all tools" without a live stub HTTP MCP server. + * + * The drive path mirrors mcp-namespacing.spec.ts: stub the repo's listEnabled, + * spy the private `connect` to return a fake client, inspect the merged keys. + */ + +function fakeTool(): Tool { + return { description: 'x', inputSchema: undefined } as unknown as Tool; +} + +interface FakeServer { + id: string; + name: string; + transport: string; + url: string; + headersEnc: string | null; + toolAllowlist: string[] | null; +} + +function server( + over: Partial & { id: string; name: string }, +): FakeServer { + return { + transport: 'http', + url: 'https://example.com/mcp', + headersEnc: null, + toolAllowlist: null, + ...over, + }; +} + +/** + * Build a service whose repo returns `servers` and whose fake clients expose + * `rawTools` from tools(). Returns the merged tool keys produced by toolsFor. + */ +async function mergedKeysFor( + servers: FakeServer[], + rawTools: Record, +): Promise { + const repoStub = { + listEnabled: jest.fn().mockResolvedValue(servers), + }; + const service = new McpClientsService(repoStub as never, {} as never); + + jest + .spyOn( + service as unknown as { connect: (s: FakeServer) => unknown }, + 'connect', + ) + .mockImplementation(() => + Promise.resolve({ + tools: () => Promise.resolve(rawTools), + close: () => Promise.resolve(), + }), + ); + + const toolset = await service.toolsFor('ws-1'); + // Release the lease so the service does not hold the fake clients open. + await Promise.all(toolset.clients.map((c) => c.close())); + return Object.keys(toolset.tools); +} + +describe('external MCP tool-allowlist filtering (via toolsFor, #476)', () => { + afterEach(() => jest.restoreAllMocks()); + + const RAW = () => ({ + alpha: fakeTool(), + beta: fakeTool(), + gamma: fakeTool(), + }); + + it("['alpha'] lets ONLY alpha through", async () => { + const keys = await mergedKeysFor( + [server({ id: 'id-1', name: 'srv', toolAllowlist: ['alpha'] })], + RAW(), + ); + expect(keys).toEqual(['srv_alpha']); + }); + + it('null (no restriction) lets every tool through', async () => { + const keys = await mergedKeysFor( + [server({ id: 'id-1', name: 'srv', toolAllowlist: null })], + RAW(), + ); + expect(keys.sort()).toEqual(['srv_alpha', 'srv_beta', 'srv_gamma']); + }); + + it('[] (deny-all) yields ZERO tools — an empty array is authoritative, not falsy (#476)', async () => { + // This is the regression the #476 change guards: `[]` used to fall through + // the old `allow.length > 0` check and expose ALL tools. It must expose NONE. + const keys = await mergedKeysFor( + [server({ id: 'id-1', name: 'srv', toolAllowlist: [] })], + RAW(), + ); + expect(keys).toEqual([]); + }); + + it('the corrupt-row fallback ([] from the repo) also yields ZERO tools (#476)', async () => { + // The repo turns a present-but-corrupt tool_allowlist into `[]` (fail-closed, + // see normalizeRow in ai-mcp-server.repo.ts + the int-spec); this pins that + // the toolset builder honours that fallback as deny-all rather than allow-all. + const corruptFallback: string[] = []; + const keys = await mergedKeysFor( + [server({ id: 'id-1', name: 'srv', toolAllowlist: corruptFallback })], + RAW(), + ); + expect(keys).toEqual([]); + }); + + it('allowlisted names not exposed by the server are ignored (no phantom tools)', async () => { + const keys = await mergedKeysFor( + [ + server({ + id: 'id-1', + name: 'srv', + toolAllowlist: ['alpha', 'does-not-exist'], + }), + ], + RAW(), + ); + expect(keys).toEqual(['srv_alpha']); + }); + + it('a deny-all server contributes no prompt instructions (0 tools merged)', async () => { + const repoStub = { + listEnabled: jest.fn().mockResolvedValue([ + { + ...server({ id: 'id-1', name: 'srv', toolAllowlist: [] }), + instructions: 'use the tools wisely', + }, + ]), + }; + const service = new McpClientsService(repoStub as never, {} as never); + jest + .spyOn( + service as unknown as { connect: (s: FakeServer) => unknown }, + 'connect', + ) + .mockImplementation(() => + Promise.resolve({ + tools: () => Promise.resolve(RAW()), + close: () => Promise.resolve(), + }), + ); + + const toolset = await service.toolsFor('ws-1'); + await Promise.all(toolset.clients.map((c) => c.close())); + expect(Object.keys(toolset.tools)).toEqual([]); + // mergeNamespaced reported 0 contributed tools, so no guidance is attached. + expect(toolset.instructions).toEqual([]); + }); +}); diff --git a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts index 20397acc..32842956 100644 --- a/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts +++ b/apps/server/src/core/ai-chat/external-mcp/mcp-clients.service.ts @@ -444,9 +444,13 @@ export class McpClientsService { try { client = await this.connectWithTimeout(server, CONNECT_TIMEOUT_MS); const raw = await withTimeout(client.tools(), CONNECT_TIMEOUT_MS); + // Allowlist semantics (#476): null/absent = no restriction (all tools); + // ANY array — including `[]` — is authoritative, so an EMPTY allowlist + // yields ZERO tools (deny-all). Do NOT add a `.length > 0` escape here: + // that read `[]` as falsy and silently widened deny-all to allow-all + // (the repo also fails corrupt rows closed to `[]` for the same reason). const allow = server.toolAllowlist; - const picked = - Array.isArray(allow) && allow.length > 0 ? pick(raw, allow) : raw; + const picked = Array.isArray(allow) ? pick(raw, allow) : raw; // Bound each tool's execute with a per-call total-timeout guard before // merging, so a single chatty-but-stuck call is aborted after the cap. const guarded = wrapToolsWithCallTimeout(picked, callTimeoutMs); diff --git a/apps/server/src/core/ai-chat/external-mcp/mcp-servers.service.ts b/apps/server/src/core/ai-chat/external-mcp/mcp-servers.service.ts index 6d366a2f..afece156 100644 --- a/apps/server/src/core/ai-chat/external-mcp/mcp-servers.service.ts +++ b/apps/server/src/core/ai-chat/external-mcp/mcp-servers.service.ts @@ -100,7 +100,8 @@ export class McpServersService { transport: dto.transport, url: dto.url, headersEnc, - // undefined => unchanged; [] / value handled by repo (empty => null). + // undefined => unchanged; null => no restriction; `[]` is persisted + // verbatim and means deny-all (#476). toolAllowlist: dto.toolAllowlist, // undefined => unchanged; blank => cleared (null) by the repo. instructions: dto.instructions, diff --git a/apps/server/src/core/ai-chat/history-budget.spec.ts b/apps/server/src/core/ai-chat/history-budget.spec.ts new file mode 100644 index 00000000..5f1fb72b --- /dev/null +++ b/apps/server/src/core/ai-chat/history-budget.spec.ts @@ -0,0 +1,266 @@ +import type { ModelMessage } from 'ai'; +import { + resolveReplayBudget, + isContextOverflowError, + estimateMessagesTokens, + trimHistoryForReplay, + REPLAY_BUDGET_DEFAULT_TOKENS, + REPLAY_TRUNCATION_MARKER, + REPLAY_TURN_COLLAPSED_MARKER, +} from './history-budget'; + +describe('resolveReplayBudget', () => { + it('uses floor(0.7 x window) for a configured window (no cap)', () => { + // 0.7 x 60k = 42k + expect(resolveReplayBudget(60_000)).toEqual({ + thresholdTokens: 42_000, + usedDefault: false, + }); + // 0.7 x 1M = 700k — NOT capped (anti-brick vs the window, not a cost limiter). + expect(resolveReplayBudget(1_000_000)).toEqual({ + thresholdTokens: 700_000, + usedDefault: false, + }); + }); + + it('accepts the raw ::text stored form', () => { + expect(resolveReplayBudget('60000').thresholdTokens).toBe(42_000); + }); + + // The crux (#490): a chat with NO context window configured must STILL be + // budgeted — those are exactly the installs that hit terminal overflow. + it('applies the flat default when the window is unset/empty', () => { + expect(resolveReplayBudget(undefined)).toEqual({ + thresholdTokens: REPLAY_BUDGET_DEFAULT_TOKENS, + usedDefault: true, + }); + expect(resolveReplayBudget('')).toEqual({ + thresholdTokens: REPLAY_BUDGET_DEFAULT_TOKENS, + usedDefault: true, + }); + expect(resolveReplayBudget(' ')).toEqual({ + thresholdTokens: REPLAY_BUDGET_DEFAULT_TOKENS, + usedDefault: true, + }); + }); + + it('treats an explicit 0 as the off-switch (distinct from unset)', () => { + expect(resolveReplayBudget(0)).toEqual({ + thresholdTokens: null, + usedDefault: false, + }); + expect(resolveReplayBudget('0')).toEqual({ + thresholdTokens: null, + usedDefault: false, + }); + }); + + it('falls back to the default on a negative/garbage value', () => { + expect(resolveReplayBudget(-5).usedDefault).toBe(true); + expect(resolveReplayBudget('abc').usedDefault).toBe(true); + }); +}); + +describe('isContextOverflowError', () => { + it('classifies a real provider 400 context-overflow shape', () => { + // OpenAI-compatible shape. + expect( + isContextOverflowError({ + statusCode: 400, + message: + "This model's maximum context length is 128000 tokens. However, your messages resulted in 214000 tokens. Please reduce the length of the messages.", + }), + ).toBe(true); + // Anthropic-style wording. + expect( + isContextOverflowError({ + status: 400, + message: 'prompt is too long: 250000 tokens > 200000 maximum', + }), + ).toBe(true); + // Nested body + string status. + expect( + isContextOverflowError({ + response: { status: '400' }, + message: 'input is too long for the requested model', + }), + ).toBe(true); + // Error instance with the cause carrying the body. + const e = new Error('Bad request'); + (e as any).statusCode = 400; + (e as any).cause = new Error('maximum context window exceeded'); + expect(isContextOverflowError(e)).toBe(true); + }); + + it('does NOT classify unrelated 400s or auth/rate-limit errors', () => { + expect( + isContextOverflowError({ statusCode: 400, message: 'invalid tool schema' }), + ).toBe(false); + expect( + isContextOverflowError({ + statusCode: 429, + message: 'context length exceeded but rate limited', + }), + ).toBe(false); + expect(isContextOverflowError({ statusCode: 500, message: 'server error' })).toBe( + false, + ); + expect(isContextOverflowError(undefined)).toBe(false); + expect(isContextOverflowError('some random string')).toBe(false); + }); +}); + +// Helpers to build ModelMessage fixtures in the ai@6 shape. +const userMsg = (text: string): ModelMessage => + ({ role: 'user', content: [{ type: 'text', text }] }) as ModelMessage; +const assistantMsg = ( + text: string, + toolCallId?: string, + toolName?: string, +): ModelMessage => + ({ + role: 'assistant', + content: [ + { type: 'text', text }, + ...(toolCallId + ? [{ type: 'tool-call', toolCallId, toolName, input: {} }] + : []), + ], + }) as ModelMessage; +const toolMsg = ( + toolCallId: string, + toolName: string, + value: unknown, +): ModelMessage => + ({ + role: 'tool', + content: [ + { type: 'tool-result', toolCallId, toolName, output: { type: 'json', value } }, + ], + }) as ModelMessage; + +describe('trimHistoryForReplay', () => { + it('null budget disables trimming (returns the same reference)', () => { + const msgs = [userMsg('hi'), assistantMsg('yo')]; + const r = trimHistoryForReplay(msgs, null); + expect(r.trimmed).toBe(false); + expect(r.messages).toBe(msgs); + }); + + it('leaves history under budget untouched (same reference)', () => { + const msgs = [userMsg('hi'), assistantMsg('a short answer')]; + const r = trimHistoryForReplay(msgs, 100_000); + expect(r.trimmed).toBe(false); + expect(r.messages).toBe(msgs); + }); + + it('truncates OLD tool outputs but keeps recent turns full', () => { + const big = 'X'.repeat(40_000); // ~16k tokens on its own + const msgs: ModelMessage[] = []; + // 6 OLD turns (indices 0..5), each with a huge tool output. + for (let i = 0; i < 6; i++) { + msgs.push(userMsg(`old q${i}`)); + msgs.push(assistantMsg('looking', `c${i}`, 'getPage')); + msgs.push(toolMsg(`c${i}`, 'getPage', { body: big })); + msgs.push(assistantMsg(`old a${i}`)); + } + // 3 small recent turns, then the CURRENT turn with its own huge tool output. + // With REPLAY_KEEP_RECENT_TURNS=4 the last 4 user-turns stay full, so only + // these small recent turns + the current big one are kept full; the 6 old + // turns above fall in the trim region. + for (let i = 0; i < 3; i++) { + msgs.push(userMsg(`recent q${i}`)); + msgs.push(assistantMsg(`recent a${i}`)); + } + msgs.push(userMsg('current q')); + msgs.push(assistantMsg('looking', 'cR', 'getPage')); + msgs.push(toolMsg('cR', 'getPage', { body: big })); + msgs.push(assistantMsg('current a')); + + // Budget large enough that phase-1 tool truncation alone brings it under. + const r = trimHistoryForReplay(msgs, 30_000); + expect(r.trimmed).toBe(true); + const flat = JSON.stringify(r.messages); + // The CURRENT turn's tool output survives in full. + expect(flat).toContain(big); + // Old outputs were truncated with the marker. + expect(flat).toContain(REPLAY_TRUNCATION_MARKER); + // Phase 1 sufficed: the oldest turns were NOT collapsed. + expect(flat).not.toContain(REPLAY_TURN_COLLAPSED_MARKER); + expect(estimateMessagesTokens(r.messages)).toBeLessThan( + estimateMessagesTokens(msgs), + ); + }); + + it('collapses the oldest turns when tool truncation is not enough', () => { + // Many turns with LARGE assistant TEXT (not tool output) so phase 1 can't help. + const bigText = 'слово '.repeat(8_000); // large Cyrillic text per turn + const msgs: ModelMessage[] = []; + for (let i = 0; i < 12; i++) { + msgs.push(userMsg(`q${i}`)); + msgs.push(assistantMsg(bigText)); + } + const r = trimHistoryForReplay(msgs, 30_000); + expect(r.trimmed).toBe(true); + // Oldest turns collapsed; result fits (best-effort) and is much smaller. + expect(estimateMessagesTokens(r.messages)).toBeLessThan( + estimateMessagesTokens(msgs), + ); + // The LAST turn's text is preserved in full (recent turns stay full). + expect(JSON.stringify(r.messages[r.messages.length - 1])).toContain(bigText); + }); + + it('is deterministic / byte-stable for identical inputs', () => { + const big = 'Y'.repeat(30_000); + const build = (): ModelMessage[] => { + const m: ModelMessage[] = []; + for (let i = 0; i < 10; i++) { + m.push(userMsg(`q${i}`)); + m.push(assistantMsg('t', `c${i}`, 'getPage')); + m.push(toolMsg(`c${i}`, 'getPage', { body: big })); + } + return m; + }; + const a = trimHistoryForReplay(build(), 15_000); + const b = trimHistoryForReplay(build(), 15_000); + expect(JSON.stringify(a.messages)).toBe(JSON.stringify(b.messages)); + }); + + it('never leaves an unpaired tool-call after collapsing (balanced history)', () => { + const big = 'Z'.repeat(40_000); + const msgs: ModelMessage[] = []; + for (let i = 0; i < 10; i++) { + msgs.push(userMsg(`q${i}`)); + msgs.push(assistantMsg('t', `c${i}`, 'getPage')); + msgs.push(toolMsg(`c${i}`, 'getPage', { body: big })); + } + const r = trimHistoryForReplay(msgs, 8_000); + // Count tool-call vs tool-result parts in the trimmed output. + let calls = 0; + let results = 0; + for (const m of r.messages) { + if (!Array.isArray(m.content)) continue; + for (const p of m.content as Array<{ type?: string }>) { + if (p.type === 'tool-call') calls++; + if (p.type === 'tool-result' || p.type === 'tool-error') results++; + } + } + // Every surviving tool-call has a surviving result (collapsing drops BOTH). + expect(calls).toBe(results); + // Collapsed turns carry the marker. + expect(JSON.stringify(r.messages)).toContain(REPLAY_TURN_COLLAPSED_MARKER); + }); + + it('respects the provider fact: under-budget contextTokens skips trimming', () => { + const big = 'W'.repeat(60_000); + const msgs = [ + userMsg('q'), + assistantMsg('t', 'c1', 'getPage'), + toolMsg('c1', 'getPage', { body: big }), + ]; + // char-estimate is high, but the provider says we are well under budget. + const r = trimHistoryForReplay(msgs, 100_000, 5_000); + expect(r.trimmed).toBe(false); + expect(r.messages).toBe(msgs); + }); +}); diff --git a/apps/server/src/core/ai-chat/history-budget.ts b/apps/server/src/core/ai-chat/history-budget.ts new file mode 100644 index 00000000..38cb6465 --- /dev/null +++ b/apps/server/src/core/ai-chat/history-budget.ts @@ -0,0 +1,375 @@ +/** + * History-replay token budget (#490). + * + * The whole persisted conversation is replayed to the provider on EVERY turn, so + * a long chat eventually exceeds the model's context window and the provider 400s + * on every turn — terminally (the chat "bricks"). This module bounds the replayed + * history at REPLAY TIME only: it never mutates what is persisted (the DB stays + * the full record), and its output is a deterministic, byte-stable function of its + * input so the trimmed prefix is identical turn to turn (provider prompt-cache + * friendliness — real money on long chats). + * + * The PRIMARY signal is the provider's own fact: `metadata.contextTokens` from the + * last turn. The chars-based {@link estimateTokens} (shared with the client) is + * used only for the DELTA of not-yet-sent messages, to decide WHAT to trim, and as + * the fallback for chats with no usage yet. + */ +import type { ModelMessage } from 'ai'; +import { estimateTokens } from '@docmost/token-estimate'; + +/** Flat default budget when no context window is configured (tokens). */ +export const REPLAY_BUDGET_DEFAULT_TOKENS = 100_000; +/** Fraction of a configured context window used as the budget. */ +export const REPLAY_BUDGET_WINDOW_FRACTION = 0.7; +/** + * Fraction of the normal budget used for the REACTIVE re-trim after a provider + * context-overflow 400 — the preventive estimate under-counted, so cut harder. + */ +export const REPLAY_AGGRESSIVE_FRACTION = 0.5; +/** + * Turns (a user message + its assistant/tool replies) kept FULL at the tail, + * including the current one — never trimmed. Older turns are compacted first. + */ +export const REPLAY_KEEP_RECENT_TURNS = 4; +/** Leading chars kept from a truncated old tool output. */ +export const REPLAY_TOOL_OUTPUT_HEAD = 800; +/** Trailing chars kept from a truncated old tool output. */ +export const REPLAY_TOOL_OUTPUT_TAIL = 300; +/** Marker inserted where an old tool output was truncated for replay. */ +export const REPLAY_TRUNCATION_MARKER = + '[…truncated for replay; call the tool again to read the full output]'; +/** Marker for a whole old turn collapsed to its text. */ +export const REPLAY_TURN_COLLAPSED_MARKER = + '[earlier tool activity omitted for replay]'; + +export interface ReplayBudget { + /** Token threshold above which replay history is trimmed; `null` = OFF. */ + thresholdTokens: number | null; + /** True when the flat default was used (no context window configured). */ + usedDefault: boolean; +} + +/** + * Resolve the replay budget from the RAW stored `chatContextWindow` (text/number). + * - a positive value -> `floor(fraction × window)` (NO cap — the budgeter is + * anti-brick protection against the window itself, not a cost/economy limiter, + * exactly as the codebase already treats maxOutputTokens; the reactive branch + * still guarantees anti-brick regardless of how high this budget is) + * - explicit `0` -> OFF (admin opt-out; `null` threshold) + * - unset/empty/invalid-> the flat default (still protects — the installations + * that hit terminal overflow are exactly the ones that never set a window) + * + * Note the raw value is needed because the parsed `chatContextWindow` collapses + * both `0` and unset to `undefined`, which would erase the explicit off-switch. + */ +export function resolveReplayBudget(rawContextWindow: unknown): ReplayBudget { + let n: number | undefined; + if (typeof rawContextWindow === 'number') { + n = rawContextWindow; + } else if (typeof rawContextWindow === 'string') { + const t = rawContextWindow.trim(); + n = t === '' ? undefined : Number(t); + } + // Unset / empty / non-numeric / negative -> flat default (the protective case). + if (n === undefined || !Number.isFinite(n) || n < 0) { + return { thresholdTokens: REPLAY_BUDGET_DEFAULT_TOKENS, usedDefault: true }; + } + // Explicit 0 -> off-switch. + if (n === 0) { + return { thresholdTokens: null, usedDefault: false }; + } + return { + thresholdTokens: Math.floor(REPLAY_BUDGET_WINDOW_FRACTION * n), + usedDefault: false, + }; +} + +/** + * The effective replay threshold for THIS turn, given the base budget and whether + * the PREVIOUS turn hit a context-overflow 400 (the reactive-recovery signal, + * `metadata.replayOverflow`). On recovery the base budget is scaled down by + * {@link REPLAY_AGGRESSIVE_FRACTION}: the overflowing turn produced no usage + * signal, so the preventive estimate under-counted and a normal-threshold trim may + * not shrink enough to fit — this harder cut is what un-bricks the chat. + * + * A `null` base budget (trimming OFF) is passed through unchanged: an explicit + * off-switch is never overridden by the recovery path. + */ +export function resolveEffectiveReplayThreshold( + thresholdTokens: number | null, + priorOverflowed: boolean, +): number | null { + if (!priorOverflowed || thresholdTokens == null) return thresholdTokens; + return Math.floor(thresholdTokens * REPLAY_AGGRESSIVE_FRACTION); +} + +/** + * True when a provider error is a CONTEXT-OVERFLOW rejection (the prompt exceeds + * the model's window). Providers surface this as an HTTP 400 with a recognizable + * message; match both the status and the message patterns robustly across + * OpenAI-compatible / Anthropic / Gemini wordings, since the exact shape varies. + */ +export function isContextOverflowError(error: unknown): boolean { + const status = extractStatus(error); + const msg = extractMessage(error).toLowerCase(); + // Message patterns seen across providers for "prompt too long". + const overflowPattern = + /context (?:length|window)|maximum context|too many tokens|too large for|reduce the length|prompt is too long|input (?:is )?too long|exceeds? the (?:maximum )?(?:context|token)|maximum.*tokens|string too long/; + if (!overflowPattern.test(msg)) return false; + // A 400/413 with an overflow-shaped message is an overflow. Some providers + // omit/rewrite the status, so accept the message match when the status is + // unknown, but reject it for auth/rate-limit statuses that never mean overflow. + if (status === 400 || status === 413) return true; + if (status === 401 || status === 403 || status === 429) return false; + return true; +} + +function extractStatus(error: unknown): number | undefined { + if (!error || typeof error !== 'object') return undefined; + const e = error as Record; + for (const k of ['statusCode', 'status']) { + const v = e[k]; + if (typeof v === 'number') return v; + if (typeof v === 'string' && /^\d+$/.test(v)) return Number(v); + } + // Nested (e.g. { response: { status } } / { cause: { statusCode } }). + for (const k of ['response', 'cause', 'data']) { + const nested = e[k]; + if (nested && typeof nested === 'object') { + const s = extractStatus(nested); + if (s !== undefined) return s; + } + } + return undefined; +} + +function extractMessage(error: unknown): string { + if (error == null) return ''; + if (typeof error === 'string') return error; + if (error instanceof Error) { + // Include nested causes (provider libs wrap the real body in `cause`). + const cause = (error as { cause?: unknown }).cause; + return `${error.message} ${cause ? extractMessage(cause) : ''}`; + } + if (typeof error === 'object') { + const e = error as Record; + const parts: string[] = []; + for (const k of ['message', 'error', 'body', 'responseBody', 'data']) { + const v = e[k]; + if (typeof v === 'string') parts.push(v); + else if (v && typeof v === 'object') parts.push(extractMessage(v)); + } + return parts.join(' '); + } + return String(error); +} + +/** Rough token size of a ModelMessage array via the shared chars estimator. */ +export function estimateMessagesTokens( + messages: ReadonlyArray, +): number { + let total = 0; + for (const m of messages) { + total += estimateTokens(serializeContent(m.content)); + } + return total; +} + +function serializeContent(content: unknown): string { + if (typeof content === 'string') return content; + try { + return JSON.stringify(content) ?? ''; + } catch { + return ''; + } +} + +/** Deep JSON string of an arbitrary value, bounded so estimation never throws. */ +function stringifyValue(value: unknown): string { + if (typeof value === 'string') return value; + try { + return JSON.stringify(value) ?? String(value); + } catch { + return String(value); + } +} + +export interface TrimResult { + messages: ModelMessage[]; + /** Whether any trimming was applied. */ + trimmed: boolean; + /** Estimated tokens of the returned messages (chars-based). */ + estimatedTokens: number; +} + +/** + * Bound the replayed history to `budgetTokens`, deterministically. Returns the + * SAME array reference (no copy) when nothing needs trimming, so the common case + * is free and byte-identical. Trimming order (spec #490): + * 1. truncate OLD turns' tool outputs (head+tail + marker) — the bulk of the size + * 2. mechanically collapse the OLDEST turns to their text (concatenation, no LLM) + * 3. the current + last {@link REPLAY_KEEP_RECENT_TURNS} turns stay FULL + * + * `budgetTokens === null` disables trimming. `priorContextTokens` (the provider's + * fact from last turn) short-circuits the decision: when it is known and already + * under budget we skip trimming even if the char-estimate is higher (the provider + * count is authoritative). The char-estimate drives WHAT to cut. + */ +export function trimHistoryForReplay( + messages: ModelMessage[], + budgetTokens: number | null, + priorContextTokens?: number, +): TrimResult { + if (budgetTokens == null) { + return { messages, trimmed: false, estimatedTokens: 0 }; + } + const estimated = estimateMessagesTokens(messages); + // Decision signal: prefer the provider's fact (last turn's contextTokens) plus + // the estimated delta of the messages appended since; fall back to the pure + // char-estimate for a chat with no usage yet. + const projected = + priorContextTokens != null + ? Math.max(priorContextTokens, estimated) + : estimated; + if (projected <= budgetTokens) { + return { messages, trimmed: false, estimatedTokens: estimated }; + } + + // The tail we always keep full: from the Nth-from-last user message onward. + const boundary = recentBoundaryIndex(messages, REPLAY_KEEP_RECENT_TURNS); + const tail = messages.slice(boundary); + let head = messages.slice(0, boundary).map(cloneMessage); + + // Phase 1: truncate old tool outputs. + for (const m of head) { + if (m.role === 'tool') truncateToolMessage(m); + } + let out = [...head, ...tail]; + let est = estimateMessagesTokens(out); + if (est <= budgetTokens) { + return { messages: out, trimmed: true, estimatedTokens: est }; + } + + // Phase 2: collapse the oldest turns (in `head`) to their text, one at a time, + // from the oldest, until we fit or the whole head is collapsed. + const turns = splitTurns(head); + const collapsed: ModelMessage[] = []; + let i = 0; + for (; i < turns.length; i++) { + if (est <= budgetTokens) break; + collapsed.push(...collapseTurn(turns[i])); + // Re-estimate the whole prospective output. + const remaining = turns.slice(i + 1).flat(); + out = [...collapsed, ...remaining, ...tail]; + est = estimateMessagesTokens(out); + } + // Include any turns we didn't need to collapse. + const remaining = turns.slice(i).flat(); + out = [...collapsed, ...remaining, ...tail]; + est = estimateMessagesTokens(out); + return { messages: out, trimmed: true, estimatedTokens: est }; +} + +/** Index of the first message of the Nth-from-last user turn (0 if fewer). */ +function recentBoundaryIndex( + messages: ReadonlyArray, + keepTurns: number, +): number { + const userIdx: number[] = []; + for (let i = 0; i < messages.length; i++) { + if (messages[i].role === 'user') userIdx.push(i); + } + if (userIdx.length <= keepTurns) return 0; + return userIdx[userIdx.length - keepTurns]; +} + +/** Split a message list into turns; each turn starts at a `user` message. */ +function splitTurns(messages: ModelMessage[]): ModelMessage[][] { + const turns: ModelMessage[][] = []; + for (const m of messages) { + if (m.role === 'user' || turns.length === 0) turns.push([m]); + else turns[turns.length - 1].push(m); + } + return turns; +} + +/** + * Collapse a whole turn to its plain text (mechanical concatenation, not an LLM + * summary). Keeps the user message; replaces the assistant/tool messages with a + * single assistant text message = the assistant's concatenated text + a marker + * when tool activity was dropped. Dropping BOTH the tool-call and tool-result + * parts together keeps the rebuilt history balanced (no unpaired calls). + */ +function collapseTurn(turn: ModelMessage[]): ModelMessage[] { + const out: ModelMessage[] = []; + let assistantText = ''; + let hadTools = false; + for (const m of turn) { + if (m.role === 'user') { + out.push(m); + } else if (m.role === 'assistant') { + const { text, tools } = extractAssistantText(m.content); + assistantText += text; + hadTools = hadTools || tools; + } else if (m.role === 'tool') { + hadTools = true; + } else { + out.push(m); + } + } + const note = + (assistantText ? assistantText.trimEnd() : '') + + (hadTools + ? `${assistantText ? '\n\n' : ''}${REPLAY_TURN_COLLAPSED_MARKER}` + : ''); + if (note) out.push({ role: 'assistant', content: note } as ModelMessage); + return out; +} + +function extractAssistantText(content: unknown): { + text: string; + tools: boolean; +} { + if (typeof content === 'string') return { text: content, tools: false }; + if (!Array.isArray(content)) return { text: '', tools: false }; + let text = ''; + let tools = false; + for (const part of content) { + const type = (part as { type?: string })?.type; + if (type === 'text') text += (part as { text?: string }).text ?? ''; + else if (type === 'tool-call') tools = true; + } + return { text, tools }; +} + +/** Truncate every tool-result output in a `tool` message to head+tail+marker. */ +function truncateToolMessage(message: ModelMessage): void { + const content = message.content; + if (!Array.isArray(content)) return; + for (const part of content) { + const p = part as { type?: string; output?: { type?: string; value?: unknown } }; + if (p.type !== 'tool-result' && p.type !== 'tool-error') continue; + if (!p.output) continue; + const raw = stringifyValue(p.output.value); + const budget = REPLAY_TOOL_OUTPUT_HEAD + REPLAY_TOOL_OUTPUT_TAIL; + if (raw.length <= budget + REPLAY_TRUNCATION_MARKER.length) continue; + const truncated = + raw.slice(0, REPLAY_TOOL_OUTPUT_HEAD) + + `\n${REPLAY_TRUNCATION_MARKER}\n` + + raw.slice(raw.length - REPLAY_TOOL_OUTPUT_TAIL); + // Represent the shrunk output as a text output (a valid tool-result output). + p.output = { type: 'text', value: truncated }; + } +} + +/** Shallow-ish clone so trimming never mutates the caller's (persisted-derived) + * message objects — only the OLD region is cloned before it is edited. */ +function cloneMessage(m: ModelMessage): ModelMessage { + if (typeof m.content === 'string') return { ...m }; + return { + ...m, + content: (m.content as unknown[]).map((p) => + p && typeof p === 'object' ? { ...(p as object) } : p, + ), + } as ModelMessage; +} diff --git a/apps/server/src/core/ai-chat/output-degeneration.spec.ts b/apps/server/src/core/ai-chat/output-degeneration.spec.ts index 3595465d..e9f42e62 100644 --- a/apps/server/src/core/ai-chat/output-degeneration.spec.ts +++ b/apps/server/src/core/ai-chat/output-degeneration.spec.ts @@ -9,9 +9,73 @@ import { DEGENERATION_CHECK_STEP, REPEATED_LINES_THRESHOLD, MIN_PERIOD_REPEATS, + degenerationThresholds, } from './output-degeneration'; import { AiChatService } from './ai-chat.service'; +// Part A (#495 iter10): the detector thresholds are env-tunable. These drive the +// resolver against real repeat-count shapes and mutation-verify that the env +// override actually changes the trigger point (not a vacuous read). +describe('degeneration thresholds are env-configurable', () => { + const VARS = [ + 'AI_CHAT_DEGENERATION_REPEATED_LINES', + 'AI_CHAT_DEGENERATION_PERIOD_MAX_LEN', + 'AI_CHAT_DEGENERATION_PERIOD_MIN_REPEATS', + 'AI_CHAT_DEGENERATION_CHECK_STEP', + ]; + const saved: Record = {}; + beforeEach(() => { + for (const v of VARS) saved[v] = process.env[v]; + }); + afterEach(() => { + for (const v of VARS) { + if (saved[v] === undefined) delete process.env[v]; + else process.env[v] = saved[v]; + } + }); + + it('defaults to the compiled constants when unset', () => { + for (const v of VARS) delete process.env[v]; + expect(degenerationThresholds()).toEqual({ + repeatedLines: REPEATED_LINES_THRESHOLD, + maxPeriodLen: 150, + minPeriodRepeats: MIN_PERIOD_REPEATS, + checkStep: DEGENERATION_CHECK_STEP, + }); + }); + + it('falls back to the default on blank / invalid / non-positive values', () => { + for (const bad of ['', ' ', 'abc', '0', '-3', '1.5']) { + process.env.AI_CHAT_DEGENERATION_REPEATED_LINES = bad; + // '1.5' floors to 1 (still ≥1, valid); every other bad value → default. + const expected = bad === '1.5' ? 1 : REPEATED_LINES_THRESHOLD; + expect(degenerationThresholds().repeatedLines).toBe(expected); + } + }); + + it('a RAISED check-step suppresses a burst the default would have flagged', () => { + // A ~3.3KB periodic burst is periodic-degenerate, but shouldCheckDegeneration + // is the throttle gate. Default checkStep=2000 arms on it; raising the step + // above the burst size means the throttle never re-fires for it. + const burstLen = 'loadTools.\n'.repeat(300).length; // ~3300 + delete process.env.AI_CHAT_DEGENERATION_CHECK_STEP; + expect(shouldCheckDegeneration(burstLen, 0)).toBe(true); // default 2000 + process.env.AI_CHAT_DEGENERATION_CHECK_STEP = String(burstLen + 1); + expect(shouldCheckDegeneration(burstLen, 0)).toBe(false); // raised gate + }); + + it('a LOWERED repeated-lines threshold trips on a shorter identical-line run', () => { + // 8 identical lines: below the default 25 (rule 1) and below the periodic + // rule's 20 repeats — so isDegenerateOutput is false by default. + const shortRun = 'x\n'.repeat(8); + delete process.env.AI_CHAT_DEGENERATION_REPEATED_LINES; + expect(isDegenerateOutput(shortRun)).toBe(false); + // Lower rule 1 to 5 → the 8-line run now trips. + process.env.AI_CHAT_DEGENERATION_REPEATED_LINES = '5'; + expect(isDegenerateOutput(shortRun)).toBe(true); + }); +}); + // Mock ONLY streamText so we can capture the onChunk/onStepFinish callbacks the // service registers and drive them by hand; every other `ai` export the service // uses (convertToModelMessages, stepCountIs, …) stays real. diff --git a/apps/server/src/core/ai-chat/output-degeneration.ts b/apps/server/src/core/ai-chat/output-degeneration.ts index eff4c080..d9d3e5ec 100644 --- a/apps/server/src/core/ai-chat/output-degeneration.ts +++ b/apps/server/src/core/ai-chat/output-degeneration.ts @@ -23,6 +23,54 @@ export const MAX_PERIOD_LEN = 150; /** Rule 2: minimum number of consecutive block repeats to trigger. */ export const MIN_PERIOD_REPEATS = 20; +/** + * Read a positive-integer threshold from an env var, falling back to `fallback` + * on unset/blank/invalid/non-positive. Mirrors the `AI_STREAM_PRE_RESPONSE_RETRIES` + * resolver in `ai-streaming-fetch.ts`: read the RAW string first so a blank value + * is treated as "unset" (→ fallback) rather than coercing to 0. Thresholds must + * stay ≥ 1 — a 0/negative would make the detector fire on any text (or never), so + * a bad value degrades to the safe compiled default instead. Env-tunable so an + * operator can retune the anti-babble guard (#444) without a redeploy, following + * the `AI_CHAT_FINAL_STEP_LOCKDOWN` toggle convention. + */ +function envThreshold(name: string, fallback: number): number { + const rawStr = process.env[name]; + if (rawStr === undefined || rawStr.trim() === '') return fallback; + const raw = Number(rawStr); + return Number.isFinite(raw) && raw >= 1 ? Math.floor(raw) : fallback; +} + +/** + * Resolve the degeneration-detector thresholds from the environment, each + * defaulting to the compiled constant above. Read fresh per call (not cached at + * import) so a test — or a runtime env change — takes effect deterministically. + */ +export function degenerationThresholds(): { + repeatedLines: number; + maxPeriodLen: number; + minPeriodRepeats: number; + checkStep: number; +} { + return { + repeatedLines: envThreshold( + 'AI_CHAT_DEGENERATION_REPEATED_LINES', + REPEATED_LINES_THRESHOLD, + ), + maxPeriodLen: envThreshold( + 'AI_CHAT_DEGENERATION_PERIOD_MAX_LEN', + MAX_PERIOD_LEN, + ), + minPeriodRepeats: envThreshold( + 'AI_CHAT_DEGENERATION_PERIOD_MIN_REPEATS', + MIN_PERIOD_REPEATS, + ), + checkStep: envThreshold( + 'AI_CHAT_DEGENERATION_CHECK_STEP', + DEGENERATION_CHECK_STEP, + ), + }; +} + /** * Rule 1 — ≥`REPEATED_LINES_THRESHOLD` consecutive IDENTICAL non-empty lines at * the tail. Catches the classic newline-delimited loop ("loadTools.\n" ×N). @@ -128,7 +176,11 @@ export function hasPeriodicTail( * Pure — the caller owns the abort side effect. */ export function isDegenerateOutput(text: string): boolean { - return hasRepeatedLineRun(text) || hasPeriodicTail(text); + const cfg = degenerationThresholds(); + return ( + hasRepeatedLineRun(text, cfg.repeatedLines) || + hasPeriodicTail(text, cfg.maxPeriodLen, cfg.minPeriodRepeats) + ); } /** @@ -154,7 +206,7 @@ export function shouldCheckDegeneration( textLen: number, lastCheckLen: number, ): boolean { - return textLen - lastCheckLen >= DEGENERATION_CHECK_STEP; + return textLen - lastCheckLen >= degenerationThresholds().checkStep; } /** diff --git a/apps/server/src/core/ai-chat/public-share-chat.service.ts b/apps/server/src/core/ai-chat/public-share-chat.service.ts index 236067eb..0c2d21ac 100644 --- a/apps/server/src/core/ai-chat/public-share-chat.service.ts +++ b/apps/server/src/core/ai-chat/public-share-chat.service.ts @@ -307,6 +307,10 @@ export class PublicShareChatService { system, messages: modelMessages, tools, + // Pin the AI SDK per-request retry budget explicitly (matches the SDK + // default of 2). Connection arithmetic: (1 + maxRetries) × (1 + + // AI_STREAM_PRE_RESPONSE_RETRIES) worst-case connects per turn. + maxRetries: 2, // Bound the agent loop for anonymous callers. stopWhen: stepCountIs(5), // Cap per-request output so one anonymous call cannot run up the provider diff --git a/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts b/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts index 58561ea2..2e8e2fa4 100644 --- a/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts +++ b/apps/server/src/core/ai-chat/tools/ai-chat-tools.service.ts @@ -426,6 +426,7 @@ export class AiChatToolsService { const { sharedToolSpecs, createCommentSignalTracker, + createListCommentsProbe, searchShapes, getGuideSection, } = await loadDocmostMcp(); @@ -718,7 +719,11 @@ export class AiChatToolsService { if (spec.mcpOnly) continue; if (spec.inlineBothHosts) continue; const run = spec.inAppExecute ?? spec.execute; - if (!run) continue; // defensive: a shared spec always carries one of them. + // Guaranteed present by assertEverySpecIsRegisterable() (#494), which runs + // at tool-specs module load and throws if a non-inline spec the in-app host + // registers carries neither inAppExecute nor execute — so this can no longer + // silently drop a mis-declared tool. Kept as a type-narrowing guard. + if (!run) continue; tools[spec.inAppKey] = sharedTool( spec, (async (args) => @@ -764,35 +769,21 @@ export class AiChatToolsService { // wrapper below) so the race governs the whole call. The client carries the // per-call composite signal via setToolAbortSignal. const capMs = inAppToolCallCapMs(); - if (!createCommentSignalTracker) { + // The signal needs BOTH the tracker factory AND the shared count-source probe + // factory (#494). Either being absent (a stale @docmost/mcp build or a mocked + // loader) => signal disabled, tool results byte-identical. + if (!createCommentSignalTracker || !createListCommentsProbe) { return wrapInAppToolsWithCap(tools, client, capMs); } + // Shared probe (#494): the SAME factory the standalone MCP host uses, so the + // in-app probe body is no longer a hand-mirror that could drift (counting the + // full feed newer than the watermark, labelling a hit with the light page + // title). `client` supplies the loopback listComments/getPageRaw reads. const tracker = createCommentSignalTracker({ - probe: async (pageId: string, sinceMs: number) => { - const { items } = await client.listComments(pageId, true); - const count = (items as Array<{ createdAt?: string }>).filter((c) => { - const created = c?.createdAt ? new Date(c.createdAt).getTime() : NaN; - return Number.isFinite(created) && created > sinceMs; - }).length; - let title: string | undefined; - if (count > 0) { - // Title labels the signal; untrusted, defanged by the shared builder. - // Fetched only on a hit so the no-signal path never pays for it. Uses - // the LIGHT raw page info (title only) — mirroring the standalone MCP - // probe's getPageRaw — instead of the heavy getPage (which also renders - // Markdown + subpages) just to read one field. - try { - const res = (await client.getPageRaw(pageId)) as { - title?: string; - } | null; - title = res?.title ?? undefined; - } catch { - // Title is optional — omit it when the page can't be fetched. - } - } - return { count, title }; - }, + probe: createListCommentsProbe( + client as unknown as Parameters[0], + ), }); return wrapInAppToolsWithCap( diff --git a/apps/server/src/core/ai-chat/tools/comment-signal-inapp.spec.ts b/apps/server/src/core/ai-chat/tools/comment-signal-inapp.spec.ts index 8a7734aa..efdb3996 100644 --- a/apps/server/src/core/ai-chat/tools/comment-signal-inapp.spec.ts +++ b/apps/server/src/core/ai-chat/tools/comment-signal-inapp.spec.ts @@ -21,7 +21,10 @@ import { SHARED_TOOL_SPECS } from '../../../../../../packages/mcp/src/tool-specs // The REAL shared tracker factory, imported from source (same cross-boundary // approach the tool-specs spec uses) so the in-app wiring is exercised against // exactly the watermark/debounce/injection-safe logic the package ships. -import { createCommentSignalTracker } from '../../../../../../packages/mcp/src/comment-signal'; +import { + createCommentSignalTracker, + createListCommentsProbe, +} from '../../../../../../packages/mcp/src/comment-signal'; // The REAL client-side citation extractor: proves that the passive signal does // NOT strip a tool's citations (the #417 in-app regression this spec guards). import { toolCitations } from '../../../../../../apps/client/src/features/ai-chat/utils/tool-parts'; @@ -284,9 +287,13 @@ describe('AiChatToolsService forUser + comment signal (real tracker)', () => { return fakeClient as DocmostClientLike; } as unknown as loader.DocmostClientCtor, sharedToolSpecs: SHARED_TOOL_SPECS as unknown as Record, - // Wire the REAL factory so the in-app path is exercised end to end. + // Wire the REAL factories so the in-app path is exercised end to end — + // including the shared count-source probe (#494) the service now builds the + // tracker's `probe` from. createCommentSignalTracker: createCommentSignalTracker as unknown as loader.CommentSignalTrackerFactory, + createListCommentsProbe: + createListCommentsProbe as unknown as loader.CreateListCommentsProbeFn, // Pure no-network draw.io helpers (#424) — required on the loader return; // this comment-signal test doesn't exercise them, so no-op stubs suffice. searchShapes: (() => []) as unknown as loader.SearchShapesFn, diff --git a/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts b/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts index 8029a239..fe9ee327 100644 --- a/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts +++ b/apps/server/src/core/ai-chat/tools/docmost-client.loader.ts @@ -150,6 +150,27 @@ export type CommentSignalTrackerFactory = (options: { debounceMs?: number; }) => CommentSignalTrackerLike; +/** + * Local mirror of `@docmost/mcp`'s `createListCommentsProbe` (#494): the SHARED + * count-source probe both hosts use, so the in-app probe body is no longer a + * hand-copy of the standalone MCP one. Given a client with the light comment feed + * + raw-page-title reads, it returns the tracker's `probe` (count comments newer + * than the watermark, label a hit with the page title). Loosely typed at this + * cross-package boundary, like the rest of this loader. + */ +export type CreateListCommentsProbeFn = (client: { + listComments( + pageId: string, + includeResolved: boolean, + ): Promise<{ items: Array<{ createdAt?: string | null }> }>; + getPageRaw( + pageId: string, + ): Promise<{ title?: string | null } | null | undefined>; +}) => ( + pageId: string, + sinceMs: number, +) => Promise; + // Pure, no-network draw.io helpers (#424). These are plain functions on the // module (NOT DocmostClient methods) — the in-app AI-SDK service calls them // directly to wire drawioShapes / drawioGuide, mirroring the MCP server. @@ -170,6 +191,10 @@ interface DocmostMcpModule { // loader in unit tests. The in-app layer treats an absent factory as "signal // disabled" — a pure no-op that leaves tool results byte-identical. createCommentSignalTracker?: CommentSignalTrackerFactory; + // Optional (#494): the shared count-source probe factory. Absent on a pre-#494 + // build or a mocked loader; the in-app layer only builds a probe when the + // signal factory above is also present. + createListCommentsProbe?: CreateListCommentsProbeFn; // Optional (#447): a deterministic hash of the tool-specs registry content, // generated into build/ by the package's build. Absent on a pre-#447 build (or // the mocked loader in unit tests) — the stale-check below is a NO-OP when it @@ -284,6 +309,7 @@ export async function loadDocmostMcp(): Promise<{ DocmostClient: DocmostClientCtor; sharedToolSpecs: Record; createCommentSignalTracker?: CommentSignalTrackerFactory; + createListCommentsProbe?: CreateListCommentsProbeFn; searchShapes: SearchShapesFn; getGuideSection: GetGuideSectionFn; }> { @@ -331,6 +357,9 @@ export async function loadDocmostMcp(): Promise<{ // Optional: forwarded when present so the in-app layer can build the passive // comment signal (#417); undefined on a stale build => signal disabled. createCommentSignalTracker: mod.createCommentSignalTracker, + // Optional (#494): the shared count-source probe factory; undefined on a + // stale build => the in-app layer falls back to no signal. + createListCommentsProbe: mod.createListCommentsProbe, // Pure no-network draw.io helpers (#424); not client methods. searchShapes: mod.searchShapes, getGuideSection: mod.getGuideSection, diff --git a/apps/server/src/core/comment/comment.controller.ts b/apps/server/src/core/comment/comment.controller.ts index fbd2c190..c04ed622 100644 --- a/apps/server/src/core/comment/comment.controller.ts +++ b/apps/server/src/core/comment/comment.controller.ts @@ -16,6 +16,7 @@ import { UpdateCommentDto } from './dto/update-comment.dto'; import { ResolveCommentDto } from './dto/resolve-comment.dto'; import { ApplySuggestionDto } from './dto/apply-suggestion.dto'; import { DismissSuggestionDto } from './dto/dismiss-suggestion.dto'; +import { ResyncSuggestionAnchorDto } from './dto/resync-suggestion-anchor.dto'; import { PageIdDto, CommentIdDto } from './dto/comments.input'; import { AuthUser } from '../../common/decorators/auth-user.decorator'; import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator'; @@ -235,6 +236,39 @@ export class CommentController { return this.commentService.applySuggestion(comment, user, provenance); } + @HttpCode(HttpStatus.OK) + @Post('resync-suggestion-anchor') + async resyncSuggestionAnchor( + @Body() dto: ResyncSuggestionAnchorDto, + @AuthUser() user: User, + @AuthWorkspace() workspace: Workspace, + ) { + const comment = await this.commentRepo.findById(dto.commentId, { + includeCreator: true, + includeResolvedBy: true, + }); + if (!comment) { + throw new NotFoundException('Comment not found'); + } + + const page = await this.pageRepo.findById(comment.pageId); + if (!page || page.deletedAt) { + throw new NotFoundException('Page not found'); + } + + // Authorize BEFORE revealing structural detail (mirrors apply/dismiss). + // Re-anchoring does NOT change the page text — it only corrects the stored + // selection metadata — so the page-level gate is comment access. The service + // further restricts it to the suggestion's own author. + await this.pageAccessService.validateCanComment(page, user, workspace.id); + + return this.commentService.resyncSuggestionAnchor( + comment, + dto.selection, + user, + ); + } + @HttpCode(HttpStatus.OK) @Post('dismiss-suggestion') async dismissSuggestion( diff --git a/apps/server/src/core/comment/comment.service.apply-suggestion.spec.ts b/apps/server/src/core/comment/comment.service.apply-suggestion.spec.ts index 486f89bb..100dae67 100644 --- a/apps/server/src/core/comment/comment.service.apply-suggestion.spec.ts +++ b/apps/server/src/core/comment/comment.service.apply-suggestion.spec.ts @@ -146,11 +146,19 @@ describe('CommentService — applySuggestion', () => { 'page-1', expect.objectContaining({ operation: 'commentDeleted', commentId: 'c-1' }), ); + // #496: hard-deleted row → the audit payload is the only surviving record. expect(auditService.log).toHaveBeenCalledWith( expect.objectContaining({ event: AuditEvent.COMMENT_SUGGESTION_APPLIED, resourceType: AuditResource.COMMENT, resourceId: 'c-1', + metadata: expect.objectContaining({ + pageId: 'page-1', + suggestedText: 'new text', + selection: 'old text', + commentAuthor: 'user-1', + decidedBy: 'user-1', + }), }), ); expect(result.outcome).toBe('deleted'); @@ -189,17 +197,25 @@ describe('CommentService — applySuggestion', () => { expect(resolvePatch.resolvedAt).toBeInstanceOf(Date); expect(resolvePatch.resolvedById).toBe('user-1'); - // NOT deleted; broadcast an update, not a deletion. + // NOT deleted. expect(commentRepo.deleteComment).not.toHaveBeenCalled(); expect(collaborationGateway.handleYjsEvent).not.toHaveBeenCalledWith( 'deleteCommentMark', expect.anything(), expect.anything(), ); + // #496 dedup: resolveComment broadcasts `commentResolved` with the enriched + // row; finalize must NOT ALSO emit a redundant `commentUpdated`. So the + // thread receives exactly ONE resolve broadcast and no update broadcast. expect(wsService.emitCommentEvent).toHaveBeenCalledWith( 'space-1', 'page-1', - expect.objectContaining({ operation: 'commentUpdated', comment: UPDATED }), + expect.objectContaining({ operation: 'commentResolved', comment: UPDATED }), + ); + expect(wsService.emitCommentEvent).not.toHaveBeenCalledWith( + 'space-1', + 'page-1', + expect.objectContaining({ operation: 'commentUpdated' }), ); expect(auditService.log).toHaveBeenCalledWith( @@ -211,6 +227,36 @@ describe('CommentService — applySuggestion', () => { expect(result.outcome).toBe('resolved'); }); + it('re-entry: already applied+resolved WITH replies → emits commentUpdated (dedup does not over-suppress)', async () => { + // suggestionAppliedAt set → idempotent finalize; resolvedAt set → resolveComment + // is skipped, so there is NO commentResolved broadcast. The applied-stamp state + // must still reach clients via a single commentUpdated. + const { service, wsService } = makeService( + { applied: false, currentText: 'new text' }, + true, + ); + + await service.applySuggestion( + suggestionComment({ + suggestionAppliedAt: new Date(), + resolvedAt: new Date(), + }), + user(), + ); + + expect(wsService.emitCommentEvent).toHaveBeenCalledWith( + 'space-1', + 'page-1', + expect.objectContaining({ operation: 'commentUpdated', comment: UPDATED }), + ); + // Nothing resolved this time (already resolved) → no resolve broadcast. + expect(wsService.emitCommentEvent).not.toHaveBeenCalledWith( + 'space-1', + 'page-1', + expect.objectContaining({ operation: 'commentResolved' }), + ); + }); + // --- error / rejection branches ----------------------------------------- it('applied=false and currentText differs → ConflictException with currentText in payload', async () => { diff --git a/apps/server/src/core/comment/comment.service.dismiss-suggestion.spec.ts b/apps/server/src/core/comment/comment.service.dismiss-suggestion.spec.ts index 819071ff..1899377a 100644 --- a/apps/server/src/core/comment/comment.service.dismiss-suggestion.spec.ts +++ b/apps/server/src/core/comment/comment.service.dismiss-suggestion.spec.ts @@ -107,11 +107,21 @@ describe('CommentService — dismissSuggestion', () => { 'page-1', expect.objectContaining({ operation: 'commentDeleted', commentId: 'c-1' }), ); + // #496: the row is hard-deleted, so the audit payload must carry the + // decision's substance (what was suggested, the anchored text, who authored + // it, who decided) — it is the only surviving record. expect(auditService.log).toHaveBeenCalledWith( expect.objectContaining({ event: AuditEvent.COMMENT_SUGGESTION_DISMISSED, resourceType: AuditResource.COMMENT, resourceId: 'c-1', + metadata: expect.objectContaining({ + pageId: 'page-1', + suggestedText: 'new text', + selection: 'old text', + commentAuthor: 'user-1', + decidedBy: 'user-1', + }), }), ); expect(result.outcome).toBe('deleted'); diff --git a/apps/server/src/core/comment/comment.service.resync-anchor.spec.ts b/apps/server/src/core/comment/comment.service.resync-anchor.spec.ts new file mode 100644 index 00000000..3602a53f --- /dev/null +++ b/apps/server/src/core/comment/comment.service.resync-anchor.spec.ts @@ -0,0 +1,126 @@ +import { BadRequestException, ForbiddenException } from '@nestjs/common'; +import { CommentService } from './comment.service'; + +/** + * Coverage for CommentService.resyncSuggestionAnchor (#496): re-anchoring a + * suggestion's stored selection (== apply-time expectedText) to the live-doc + * substring. The service is built directly with jest-mocked deps (the + * @InjectQueue tokens can't be resolved by Test.createTestingModule — see the + * sibling specs). + */ +describe('CommentService — resyncSuggestionAnchor', () => { + const UPDATED = { id: 'c-1', selection: 'new anchor', __updated: true } as any; + + function makeService() { + const commentRepo: any = { + updateComment: jest.fn(async () => undefined), + findById: jest.fn(async () => UPDATED), + }; + const service = new CommentService( + commentRepo, + {} as any, + { emitCommentEvent: jest.fn() } as any, + {} as any, + { add: jest.fn() } as any, + { add: jest.fn() } as any, + { log: jest.fn() } as any, + ); + return { service, commentRepo }; + } + + const suggestion = (over?: Partial): any => ({ + id: 'c-1', + creatorId: 'user-1', + parentCommentId: null, + selection: 'old anchor', + suggestedText: 'new text', + suggestionAppliedAt: null, + resolvedAt: null, + ...over, + }); + const user = (over?: Partial): any => ({ id: 'user-1', ...over }); + + it('persists the new selection and returns the enriched comment', async () => { + const { service, commentRepo } = makeService(); + + const out = await service.resyncSuggestionAnchor( + suggestion(), + 'new anchor', + user(), + ); + + expect(commentRepo.updateComment).toHaveBeenCalledWith( + { selection: 'new anchor' }, + 'c-1', + ); + expect(out).toBe(UPDATED); + }); + + it('is idempotent: no write when the anchor already matches', async () => { + const { service, commentRepo } = makeService(); + + const out = await service.resyncSuggestionAnchor( + suggestion({ selection: 'same' }), + 'same', + user(), + ); + + expect(commentRepo.updateComment).not.toHaveBeenCalled(); + expect(out).toEqual(suggestion({ selection: 'same' })); + }); + + it('rejects a non-author (only the suggestion owner may re-anchor)', async () => { + const { service, commentRepo } = makeService(); + await expect( + service.resyncSuggestionAnchor(suggestion(), 'new anchor', user({ id: 'other' })), + ).rejects.toBeInstanceOf(ForbiddenException); + expect(commentRepo.updateComment).not.toHaveBeenCalled(); + }); + + it('rejects a reply / a comment with no suggestion', async () => { + const { service } = makeService(); + await expect( + service.resyncSuggestionAnchor( + suggestion({ parentCommentId: 'p-1' }), + 'x', + user(), + ), + ).rejects.toBeInstanceOf(BadRequestException); + await expect( + service.resyncSuggestionAnchor( + suggestion({ suggestedText: null }), + 'x', + user(), + ), + ).rejects.toBeInstanceOf(BadRequestException); + }); + + it('rejects re-anchoring an already applied or resolved suggestion', async () => { + const { service } = makeService(); + await expect( + service.resyncSuggestionAnchor( + suggestion({ suggestionAppliedAt: new Date() }), + 'x', + user(), + ), + ).rejects.toBeInstanceOf(BadRequestException); + await expect( + service.resyncSuggestionAnchor( + suggestion({ resolvedAt: new Date() }), + 'x', + user(), + ), + ).rejects.toBeInstanceOf(BadRequestException); + }); + + it('rejects a no-op selection equal to the suggested text', async () => { + const { service } = makeService(); + await expect( + service.resyncSuggestionAnchor( + suggestion({ suggestedText: 'new text' }), + 'new text', + user(), + ), + ).rejects.toBeInstanceOf(BadRequestException); + }); +}); diff --git a/apps/server/src/core/comment/comment.service.ts b/apps/server/src/core/comment/comment.service.ts index ef97c108..a5911047 100644 --- a/apps/server/src/core/comment/comment.service.ts +++ b/apps/server/src/core/comment/comment.service.ts @@ -370,6 +370,76 @@ export class CommentService { return updatedComment; } + /** + * Re-sync a suggestion's stored `selection` (== apply-time expectedText) to the + * RAW substring the inline mark actually covers in the LIVE document (#496). + * + * The MCP client creates the comment from a DEBOUNCED REST snapshot, then + * anchors the mark in the live collab doc. When the two disagree (the doc moved + * on in the debounce window) the stored selection no longer equals the marked + * text, so EVERY apply 409s ("the commented text changed"). After anchoring the + * client re-reads the exact marked substring and calls this to store it, making + * apply's strict equality hold. + * + * Only meaningful for an un-settled top-level suggestion authored by the + * caller: applying/resolving freezes the anchor, and a reply-carrying thread is + * preserved rather than mutated. The new text must still differ from the + * suggestion (else "apply" would be a no-op), preserving create()'s invariant. + */ + async resyncSuggestionAnchor( + comment: Comment, + selection: string, + user: User, + ): Promise { + if (comment.creatorId !== user.id) { + throw new ForbiddenException( + 'You can only re-anchor your own suggestion', + ); + } + if (comment.parentCommentId) { + throw new BadRequestException( + 'Only a top-level comment can carry a suggested edit', + ); + } + if (!comment.suggestedText) { + throw new BadRequestException('This comment has no suggested edit'); + } + // A settled suggestion's anchor is frozen: re-anchoring an applied/resolved + // thread is meaningless and could resurrect a stale expectedText. + if (comment.suggestionAppliedAt || comment.resolvedAt) { + throw new BadRequestException( + 'Cannot re-anchor a suggestion that was already applied or resolved', + ); + } + const trimmed = selection.trim(); + if (trimmed.length === 0) { + throw new BadRequestException('The re-anchored selection cannot be empty'); + } + // Same no-op guard as create(): the suggestion must differ from the text it + // replaces, or apply becomes indistinguishable from already-applied. + if (trimmed === comment.suggestedText.trim()) { + throw new BadRequestException( + 'A suggested edit must differ from the selected text', + ); + } + + // Idempotent: nothing to persist when the anchor already matches. + if (comment.selection === selection) { + return comment; + } + + await this.commentRepo.updateComment({ selection }, comment.id); + + const updatedComment = await this.commentRepo.findById(comment.id, { + includeCreator: true, + includeResolvedBy: true, + }); + + // Re-anchoring only corrects stored metadata; it does not change the page + // text or the comment body, so no ws broadcast / notification is warranted. + return updatedComment; + } + /** * Apply the suggested edit carried by a top-level inline comment: atomically * replace the text under the comment mark in the collaborative document with @@ -524,7 +594,7 @@ export class CommentService { resourceType: AuditResource.COMMENT, resourceId: comment.id, spaceId: comment.spaceId, - metadata: { pageId: comment.pageId }, + metadata: this.suggestionAuditMetadata(comment, user), }); return { ...updatedComment, outcome: 'resolved' }; } @@ -538,7 +608,7 @@ export class CommentService { resourceType: AuditResource.COMMENT, resourceId: comment.id, spaceId: comment.spaceId, - metadata: { pageId: comment.pageId }, + metadata: this.suggestionAuditMetadata(comment, user), }); return settled; } @@ -577,8 +647,10 @@ export class CommentService { // Auto-resolve the thread. resolveComment handles the resolve mark, its ws // broadcast and the resolve notification. Stay defensive on re-entry. + let didResolveBroadcast = false; if (!comment.resolvedAt) { await this.resolveComment(comment, true, user, provenance); + didResolveBroadcast = true; } const updatedComment = await this.commentRepo.findById(comment.id, { @@ -586,18 +658,27 @@ export class CommentService { includeResolvedBy: true, }); - this.wsService.emitCommentEvent(comment.spaceId, comment.pageId, { - operation: 'commentUpdated', - pageId: comment.pageId, - comment: updatedComment, - }); + // #496 dedup: resolveComment already broadcast `commentResolved` carrying + // the fully-enriched row (the applied stamps were persisted above, before + // that call, so its re-read reflects them). Emitting `commentUpdated` here + // too made the client receive TWO events for one apply. Broadcast the + // update ONLY when we did NOT resolve — i.e. the rare re-entry on an + // already-resolved thread, where the applied-stamp change still needs a + // broadcast and resolveComment did not run. + if (!didResolveBroadcast) { + this.wsService.emitCommentEvent(comment.spaceId, comment.pageId, { + operation: 'commentUpdated', + pageId: comment.pageId, + comment: updatedComment, + }); + } this.auditService.log({ event: AuditEvent.COMMENT_SUGGESTION_APPLIED, resourceType: AuditResource.COMMENT, resourceId: comment.id, spaceId: comment.spaceId, - metadata: { pageId: comment.pageId }, + metadata: this.suggestionAuditMetadata(comment, user), }); return { ...updatedComment, outcome: 'resolved' }; @@ -616,7 +697,7 @@ export class CommentService { resourceType: AuditResource.COMMENT, resourceId: comment.id, spaceId: comment.spaceId, - metadata: { pageId: comment.pageId }, + metadata: this.suggestionAuditMetadata(comment, user), }); return settled; @@ -627,14 +708,17 @@ export class CommentService { * inline `comment` anchor mark, then ATOMICALLY hard-delete the row only if it * is still childless. Shared by the apply/dismiss no-replies branches (#329). * - * ORDER MATTERS: the anchor mark is removed FIRST and FATALLY (mirrors - * applySuggestion, which mutates the doc before writing the DB). The row - * delete is irreversible, so if the mark removal fails — including the - * COLLAB_DISABLE_REDIS "no live instance" hard-error — we must NOT delete the - * row and report success, or the document is left with a permanent orphan - * anchor pointing at a comment that no longer exists (the exact data-integrity - * bug #329 targets). Let the exception propagate (→ 5xx); the operation is - * then repeatable with row + mark still consistent. + * ORDER MATTERS (updated #399 → #496): what runs FIRST and FATALLY here is the + * mark-removal ENQUEUE (a fast, durable Redis add), NOT the mark op itself. + * deleteCommentMark awaits only the enqueue, so a failed add throws BEFORE the + * irreversible row delete — the row + mark stay consistent and the operation is + * repeatable. The actual anchor strip then runs off the HTTP path in the worker + * (idempotent, 3 retries). Only an EXHAUSTED-retries job could leave the doc + * with an orphan anchor pointing at a hard-deleted comment (the data-integrity + * bug #329 targets); that residual divergence is now self-healed by the + * resolve/unresolve mark worker, which strips an orphan mark whenever its + * comment row is gone (#496), and it is meanwhile VISIBLE via BullMQ failed-job + * metrics rather than a silently-swallowed warn. * * RACE (#338 F4): the caller read `hasChildren` BEFORE the (slow) mark * removal, so a reply can land in that window. `comments.parent_comment_id` is @@ -732,6 +816,27 @@ export class CommentService { return this.generalQueue.add(QueueJob.COMMENT_MARK_UPDATE, jobData); } + /** + * Build the audit metadata for a suggestion apply/dismiss decision (#496). + * The subject comment is HARD-DELETED on the childless path, so the audit row + * is the only surviving record — capture the decision's substance (what was + * suggested, the anchored text it replaced, who authored it, who decided) + * before the row can vanish. `decidedBy` is the acting user; `commentAuthor` + * is the suggestion's creator. + */ + private suggestionAuditMetadata( + comment: Comment, + user: User, + ): Record { + return { + pageId: comment.pageId, + suggestedText: comment.suggestedText ?? null, + selection: comment.selection ?? null, + commentAuthor: comment.creatorId ?? null, + decidedBy: user.id, + }; + } + private async queueCommentNotification( content: any, oldMentionIds: string[], diff --git a/apps/server/src/core/comment/dto/resync-suggestion-anchor.dto.ts b/apps/server/src/core/comment/dto/resync-suggestion-anchor.dto.ts new file mode 100644 index 00000000..003edc9d --- /dev/null +++ b/apps/server/src/core/comment/dto/resync-suggestion-anchor.dto.ts @@ -0,0 +1,20 @@ +import { IsString, IsUUID, MaxLength, MinLength } from 'class-validator'; + +/** + * #496: after the MCP client anchors a suggestion in the LIVE collab doc, it + * re-reads the exact substring under the new mark and syncs it here as the + * comment's stored `selection` (== apply-time expectedText). Fixes the perpetual + * 409 where expectedText came from a debounced REST snapshot while the mark sat + * in the live doc. + */ +export class ResyncSuggestionAnchorDto { + @IsUUID() + commentId: string; + + // The raw substring the mark now covers in the live document. Bounded like the + // create-time selection (2000) so a legitimate anchored span is never cut. + @IsString() + @MinLength(1) + @MaxLength(2000) + selection: string; +} diff --git a/apps/server/src/core/page/dto/move-page.dto.spec.ts b/apps/server/src/core/page/dto/move-page.dto.spec.ts index 7b71995a..0f894ca5 100644 --- a/apps/server/src/core/page/dto/move-page.dto.spec.ts +++ b/apps/server/src/core/page/dto/move-page.dto.spec.ts @@ -17,9 +17,10 @@ import { MovePageDto } from './move-page.dto'; // a valid ordering key the server itself generated would be refused on move. // // The tests below assert the CORRECT contract: any key the generator can produce -// must satisfy the DTO. The genuinely-failing case is marked `test.failing` so the -// suite stays green while locking the bug; it flips red (alerting us) once the DTO -// bounds are widened to cover the generator's real range. +// must satisfy the DTO. FIXED (#495 item 9): the DTO now validates `position` by +// CHARSET ([0-9A-Za-z], the generator's base-62 alphabet) instead of the wrong +// @MaxLength(12) length bound, so dense between-inserts are accepted; the former +// `test.failing` bug-lock is now a passing assertion. function constraintErrors(position: unknown) { const dto = plainToInstance(MovePageDto, { @@ -47,24 +48,33 @@ describe('MovePageDto.position vs generateJitteredKeyBetween parity', () => { expect(hasError(errors, 'position')).toBe(false); }); - // BUG LOCK: dense between-inserts produce keys longer than 12 chars, which - // MaxLength(12) rejects even though they are valid ordering keys. This SHOULD - // pass; it currently fails. Flips green when the DTO bound is fixed. - test.failing( - 'accepts dense between-inserted keys (currently rejected by MaxLength(12))', - async () => { - let lo = generateJitteredKeyBetween(null, null); - let hi = generateJitteredKeyBetween(lo, null); - // Repeatedly insert just above `lo`, shrinking the gap so the key grows. - let longest = lo; - for (let i = 0; i < 40; i++) { - const mid = generateJitteredKeyBetween(lo, hi); - if (mid.length > longest.length) longest = mid; - hi = mid; - } - expect(longest.length).toBeGreaterThan(12); // sanity: we produced a long key - const errors = await constraintErrors(longest); - expect(hasError(errors, 'position')).toBe(false); - }, - ); + // FIXED: dense between-inserts produce keys longer than 12 chars, which the old + // MaxLength(12) rejected even though they are valid ordering keys. Now accepted. + it('accepts dense between-inserted keys longer than 12 chars', async () => { + let lo = generateJitteredKeyBetween(null, null); + let hi = generateJitteredKeyBetween(lo, null); + // Repeatedly insert just above `lo`, shrinking the gap so the key grows. The + // generator is JITTERED (random), so use enough iterations that the longest + // key reliably clears the old 12-char bound: measured min-over-50-trials is + // ~11 at 40 iterations (flaky) but ~36 at 200 (robust margin). + let longest = lo; + for (let i = 0; i < 200; i++) { + const mid = generateJitteredKeyBetween(lo, hi); + if (mid.length > longest.length) longest = mid; + hi = mid; + } + expect(longest.length).toBeGreaterThan(12); // sanity: we produced a long key + const errors = await constraintErrors(longest); + expect(hasError(errors, 'position')).toBe(false); + }); + + // The charset guard replaces the length bound: reject anything outside the + // generator's [0-9A-Za-z] alphabet (control chars, separators, injection) and + // the empty string, while still accepting every real key. + it('rejects a position with characters outside the fractional-index alphabet', async () => { + for (const bad of ['a0/b', 'a b', 'a\n0', 'a.b', '', "a';--"]) { + const errors = await constraintErrors(bad); + expect(hasError(errors, 'position')).toBe(true); + } + }); }); diff --git a/apps/server/src/core/page/dto/move-page.dto.ts b/apps/server/src/core/page/dto/move-page.dto.ts index 92dc2c5c..a8baabee 100644 --- a/apps/server/src/core/page/dto/move-page.dto.ts +++ b/apps/server/src/core/page/dto/move-page.dto.ts @@ -1,8 +1,8 @@ import { IsString, IsOptional, - MinLength, MaxLength, + Matches, IsNotEmpty, } from 'class-validator'; @@ -10,9 +10,19 @@ export class MovePageDto { @IsString() pageId: string; + // `position` is a fractional-indexing key from `generateJitteredKeyBetween` + // (the SAME generator page.service uses). Validate by CHARSET, not length: the + // generator's default base-62 alphabet is [0-9A-Za-z], and DENSE between-inserts + // legitimately grow a key well past a dozen chars (measured >40), so the old + // @MinLength(5)/@MaxLength(12) bounds wrongly 400'd valid ordering keys the + // server itself produced (Gitea #139 item 6). The charset regex rejects control + // chars / separators / injection, and a generous MaxLength stays only as a + // DoS guard — far above any realistic key, so it never rejects a real move. @IsString() - @MinLength(5) - @MaxLength(12) + @Matches(/^[0-9A-Za-z]+$/, { + message: 'position must be a fractional-index key ([0-9A-Za-z])', + }) + @MaxLength(256) position: string; @IsOptional() diff --git a/apps/server/src/core/page/dto/page-identity.dto.spec.ts b/apps/server/src/core/page/dto/page-identity.dto.spec.ts new file mode 100644 index 00000000..25604399 --- /dev/null +++ b/apps/server/src/core/page/dto/page-identity.dto.spec.ts @@ -0,0 +1,49 @@ +import 'reflect-metadata'; +import { plainToInstance } from 'class-transformer'; +import { validate } from 'class-validator'; +import { PageIdDto } from './page.dto'; + +// #435: PageIdDto.pageId carries a page's DOUBLE identity (internal UUID OR +// public 10-char slugId), both as bare strings. The DTO must accept exactly +// those two FORMATS and reject a malformed / swapped identity at the boundary. +async function pageIdErrors(pageId: unknown) { + const dto = plainToInstance(PageIdDto, { pageId }); + const errors = await validate(dto as object); + return errors.some((e) => e.property === 'pageId'); +} + +const UUID = '019f499a-9f8c-7d68-b7be-ce100d7c6c56'; +const SLUG = 'aB3xQ7kR2p'; + +describe('PageIdDto pageId format validation', () => { + it('accepts a canonical page UUID', async () => { + expect(await pageIdErrors(UUID)).toBe(false); + }); + + it('accepts a 10-char slugId', async () => { + expect(await pageIdErrors(SLUG)).toBe(false); + }); + + it('rejects a truncated / wrong-length slug', async () => { + expect(await pageIdErrors('aB3xQ7kR2')).toBe(true); // 9 chars + expect(await pageIdErrors('aB3xQ7kR2pX')).toBe(true); // 11 chars + }); + + it('rejects a slug with an illegal character', async () => { + expect(await pageIdErrors('aB3xQ7kR2!')).toBe(true); + }); + + it('rejects a full URL / path-shaped identity (not the bare id)', async () => { + expect(await pageIdErrors(`my-page-title-${SLUG}`)).toBe(true); + expect(await pageIdErrors(`https://x/p/${SLUG}`)).toBe(true); + }); + + it('rejects a malformed UUID', async () => { + expect(await pageIdErrors('019f499a-9f8c-7d68-b7be')).toBe(true); + expect(await pageIdErrors('not-a-uuid-at-all-really')).toBe(true); + }); + + it('rejects an empty string', async () => { + expect(await pageIdErrors('')).toBe(true); + }); +}); diff --git a/apps/server/src/core/page/dto/page-identity.validator.ts b/apps/server/src/core/page/dto/page-identity.validator.ts new file mode 100644 index 00000000..01c7b343 --- /dev/null +++ b/apps/server/src/core/page/dto/page-identity.validator.ts @@ -0,0 +1,39 @@ +import { applyDecorators } from '@nestjs/common'; +import { Matches, ValidationOptions } from 'class-validator'; + +/** + * A page identity at the API boundary is EITHER the internal page UUID or the + * public 10-char slugId (page.repo.findById matches a non-UUID input as a + * slugId). Both arrive as bare strings, which is exactly how the two got swapped + * silently (incident family #435). This regex pins the two accepted FORMATS so a + * malformed / cross-wired identity (a truncated slug, a full URL, an email, an + * id from another entity kind that isn't even shaped like either) is rejected at + * the boundary instead of falling through to a confusing 404. + * + * - UUID: canonical 8-4-4-4-12 hex (version-agnostic — page ids are UUIDv7, + * so only the shape/length is enforced, matching the MCP's UUID_RE + * and the server's isValidUUID acceptance). + * - slugId: exactly 10 chars over [0-9A-Za-z] (nanoid `generateSlugId`). + * + * The two are disjoint (a UUID is 36 chars WITH dashes, a slugId 10 chars + * WITHOUT), so a value can only satisfy one branch. + */ +export const PAGE_ID_OR_SLUG_ID_REGEX = + /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}|[0-9A-Za-z]{10})$/i; + +/** + * Validate that a DTO string field is a well-formed page identity (page UUID OR + * 10-char slugId). Composed decorator so the same format rule is applied + * consistently wherever a DTO accepts a `pageId` that may be either form. + */ +export function IsPageIdOrSlugId( + validationOptions?: ValidationOptions, +): PropertyDecorator { + return applyDecorators( + Matches(PAGE_ID_OR_SLUG_ID_REGEX, { + message: + 'pageId must be a page UUID or a 10-character slugId', + ...validationOptions, + }), + ); +} diff --git a/apps/server/src/core/page/dto/page.dto.ts b/apps/server/src/core/page/dto/page.dto.ts index c53f8ade..c3b64b23 100644 --- a/apps/server/src/core/page/dto/page.dto.ts +++ b/apps/server/src/core/page/dto/page.dto.ts @@ -9,10 +9,16 @@ import { import { Transform } from 'class-transformer'; import { ContentFormat } from './create-page.dto'; +import { IsPageIdOrSlugId } from './page-identity.validator'; export class PageIdDto { @IsString() @IsNotEmpty() + // Format-validate the double identity (#435): accept only a page UUID or a + // 10-char slugId so a malformed / swapped identity is rejected at the boundary + // rather than passed to the repo as a bare string. Base for PageInfoDto, + // DeletePageDto, BacklinksListDto, AddLabelsDto/RemoveLabelDto, etc. + @IsPageIdOrSlugId() pageId: string; } diff --git a/apps/server/src/core/page/services/page.service.ts b/apps/server/src/core/page/services/page.service.ts index 0b4d8d80..421319a4 100644 --- a/apps/server/src/core/page/services/page.service.ts +++ b/apps/server/src/core/page/services/page.service.ts @@ -53,8 +53,10 @@ import { extractPageSlugId, } from '../../../integrations/export/utils'; import { canonicalizeFootnotes } from '@docmost/editor-ext'; -import { markdownToProseMirror } from '@docmost/prosemirror-markdown'; -import { normalizeForeignMarkdown } from '../../../integrations/import/utils/foreign-markdown'; +import { + markdownToProseMirror, + normalizeForeignMarkdown, +} from '@docmost/prosemirror-markdown'; import { WatcherService } from '../../watcher/watcher.service'; import { sql } from 'kysely'; import { TransclusionService } from '../transclusion/transclusion.service'; diff --git a/apps/server/src/core/page/services/spec/temporary-note-cleanup.service.spec.ts b/apps/server/src/core/page/services/spec/temporary-note-cleanup.service.spec.ts index 54fd49aa..ea104f96 100644 --- a/apps/server/src/core/page/services/spec/temporary-note-cleanup.service.spec.ts +++ b/apps/server/src/core/page/services/spec/temporary-note-cleanup.service.spec.ts @@ -1,19 +1,38 @@ import { TemporaryNoteCleanupService } from '../temporary-note-cleanup.service'; /** - * Chainable Kysely stub that records every `.where(...)` call so the test can - * assert the sweep only selects armed, expired, not-yet-trashed notes. The - * terminal `.execute()` resolves the configured expired rows (the batch SELECT); - * `.executeTakeFirst()` resolves the per-row deadline re-read done just before - * each `removePage`. By default the re-read reports the note as still armed and - * still expired (epoch deadline < now), so the sweep proceeds to delete it; - * tests override `reReadFirst` to simulate a concurrent "Make permanent". + * Chainable Kysely stub for the temporary-note sweep. + * + * `this.db` serves the non-locking candidate SELECT (selectFrom/select/where/ + * limit/execute -> the configured expired rows) AND `.transaction().execute(cb)`, + * which runs `cb` with a separate `trx` builder. The `trx` builder serves the + * per-row LOCKED re-check (selectFrom/select/where/forUpdate/skipLocked/ + * executeTakeFirst). `lockedRows` drives what that locked re-check returns per + * candidate — an id/creator/workspace row means "still expired, delete it"; + * `undefined` means the predicate no longer matched (made permanent / re-armed / + * already trashed) or the row was SKIP-LOCKED by another worker, so it is skipped. */ -function makeDbStub(expiredRows: any[]) { +function makeDbStub(expiredRows: any[], lockedRows?: any[]) { const whereCalls: any[][] = []; - const reReadFirst = jest - .fn() - .mockResolvedValue({ temporaryExpiresAt: new Date(0), deletedAt: null }); + const locked = [ + ...(lockedRows ?? + expiredRows.map((r) => ({ + id: r.id, + creatorId: r.creatorId, + workspaceId: r.workspaceId, + }))), + ]; + const lockedTakeFirst = jest.fn(() => Promise.resolve(locked.shift())); + const forUpdate = jest.fn(() => trxBuilder); + const skipLocked = jest.fn(() => trxBuilder); + const trxBuilder: any = { + selectFrom: jest.fn(() => trxBuilder), + select: jest.fn(() => trxBuilder), + where: jest.fn(() => trxBuilder), + forUpdate, + skipLocked, + executeTakeFirst: lockedTakeFirst, + }; const builder: any = { selectFrom: jest.fn(() => builder), select: jest.fn(() => builder), @@ -23,9 +42,11 @@ function makeDbStub(expiredRows: any[]) { }), limit: jest.fn(() => builder), execute: jest.fn().mockResolvedValue(expiredRows), - executeTakeFirst: reReadFirst, + transaction: jest.fn(() => ({ + execute: (cb: (trx: any) => Promise) => Promise.resolve(cb(trxBuilder)), + })), }; - return { builder, whereCalls, reReadFirst }; + return { builder, whereCalls, lockedTakeFirst, forUpdate, skipLocked }; } describe('TemporaryNoteCleanupService.sweepExpiredTemporaryNotes', () => { @@ -52,20 +73,36 @@ describe('TemporaryNoteCleanupService.sweepExpiredTemporaryNotes', () => { expect(builder.limit.mock.calls[0][0]).toBeGreaterThan(0); }); - it('soft-deletes each expired note via removePage, attributed to its creator', async () => { + it('soft-deletes each expired note via removePage under a row lock, attributed to its creator', async () => { const expired = [ { id: 'p1', creatorId: 'u1', workspaceId: 'w1' }, { id: 'p2', creatorId: 'u2', workspaceId: 'w1' }, ]; - const { builder } = makeDbStub(expired); + const { builder, forUpdate, skipLocked } = makeDbStub(expired); const pageRepo = { removePage: jest.fn().mockResolvedValue(undefined) } as any; const service = new TemporaryNoteCleanupService(builder, pageRepo); await service.sweepExpiredTemporaryNotes(); expect(pageRepo.removePage).toHaveBeenCalledTimes(2); - expect(pageRepo.removePage).toHaveBeenNthCalledWith(1, 'p1', 'u1', 'w1'); - expect(pageRepo.removePage).toHaveBeenNthCalledWith(2, 'p2', 'u2', 'w1'); + // The 4th arg is the locking transaction — the delete runs inside it. + expect(pageRepo.removePage).toHaveBeenNthCalledWith( + 1, + 'p1', + 'u1', + 'w1', + expect.anything(), + ); + expect(pageRepo.removePage).toHaveBeenNthCalledWith( + 2, + 'p2', + 'u2', + 'w1', + expect.anything(), + ); + // The re-check acquired a FOR UPDATE SKIP LOCKED lock (once per candidate). + expect(forUpdate).toHaveBeenCalledTimes(2); + expect(skipLocked).toHaveBeenCalledTimes(2); }); it('continues past a failing note (one bad removePage does not abort the sweep)', async () => { @@ -86,60 +123,29 @@ describe('TemporaryNoteCleanupService.sweepExpiredTemporaryNotes', () => { service.sweepExpiredTemporaryNotes(), ).resolves.toBeUndefined(); expect(pageRepo.removePage).toHaveBeenCalledTimes(2); - expect(pageRepo.removePage).toHaveBeenNthCalledWith(2, 'good', 'u2', 'w1'); + expect(pageRepo.removePage).toHaveBeenNthCalledWith( + 2, + 'good', + 'u2', + 'w1', + expect.anything(), + ); }); - it('does NOT trash a note made permanent in the race window', async () => { - // The batch SELECT saw the note as expired, but before its turn in the loop - // the user clicked "Make permanent" (temporary_expires_at -> null). The - // deadline re-read must catch this and skip the delete so the keep wins. + it('does NOT trash a note made permanent / re-armed / already trashed (locked re-check returns nothing)', async () => { + // The batch SELECT saw the note as expired, but by the time the LOCKED + // re-check runs the row no longer matches the still-armed+expired+not-trashed + // predicate (make-permanent, re-arm to a future deadline, or already trashed), + // OR another worker holds the row (SKIP LOCKED). In every case the locked + // SELECT returns nothing and the delete is skipped so the keep/other worker wins. const expired = [{ id: 'p1', creatorId: 'u1', workspaceId: 'w1' }]; - const { builder, reReadFirst } = makeDbStub(expired); - reReadFirst.mockResolvedValueOnce({ - temporaryExpiresAt: null, - deletedAt: null, - }); + const { builder, lockedTakeFirst } = makeDbStub(expired, [undefined]); const pageRepo = { removePage: jest.fn() } as any; const service = new TemporaryNoteCleanupService(builder, pageRepo); await service.sweepExpiredTemporaryNotes(); - expect(reReadFirst).toHaveBeenCalledTimes(1); - expect(pageRepo.removePage).not.toHaveBeenCalled(); - }); - - it('skips a note already trashed since the batch SELECT', async () => { - const expired = [{ id: 'p1', creatorId: 'u1', workspaceId: 'w1' }]; - const { builder, reReadFirst } = makeDbStub(expired); - reReadFirst.mockResolvedValueOnce({ - temporaryExpiresAt: new Date(0), - deletedAt: new Date(), - }); - const pageRepo = { removePage: jest.fn() } as any; - const service = new TemporaryNoteCleanupService(builder, pageRepo); - - await service.sweepExpiredTemporaryNotes(); - - expect(pageRepo.removePage).not.toHaveBeenCalled(); - }); - - it('does NOT trash a note re-armed to a future deadline in the race window', async () => { - // The batch SELECT saw the note as expired, but before its turn in the loop - // the user disarmed it and re-armed it to a fresh, still-future deadline - // (temporary_expires_at -> now + 1h). The deadline re-read must catch that - // the note is no longer expired and skip the delete so the keep wins. - const expired = [{ id: 'p1', creatorId: 'u1', workspaceId: 'w1' }]; - const { builder, reReadFirst } = makeDbStub(expired); - reReadFirst.mockResolvedValueOnce({ - temporaryExpiresAt: new Date(Date.now() + 60 * 60 * 1000), - deletedAt: null, - }); - const pageRepo = { removePage: jest.fn() } as any; - const service = new TemporaryNoteCleanupService(builder, pageRepo); - - await service.sweepExpiredTemporaryNotes(); - - expect(reReadFirst).toHaveBeenCalledTimes(1); + expect(lockedTakeFirst).toHaveBeenCalledTimes(1); expect(pageRepo.removePage).not.toHaveBeenCalled(); }); @@ -151,4 +157,31 @@ describe('TemporaryNoteCleanupService.sweepExpiredTemporaryNotes', () => { await service.sweepExpiredTemporaryNotes(); expect(pageRepo.removePage).not.toHaveBeenCalled(); }); + + it('sweeps once on application bootstrap (catches notes expired during downtime)', async () => { + const expired = [{ id: 'p1', creatorId: 'u1', workspaceId: 'w1' }]; + const { builder } = makeDbStub(expired); + const pageRepo = { removePage: jest.fn().mockResolvedValue(undefined) } as any; + const service = new TemporaryNoteCleanupService(builder, pageRepo); + + await service.onApplicationBootstrap(); + + expect(pageRepo.removePage).toHaveBeenCalledTimes(1); + expect(pageRepo.removePage).toHaveBeenCalledWith( + 'p1', + 'u1', + 'w1', + expect.anything(), + ); + }); + + it('a startup-sweep failure never blocks application boot', async () => { + const { builder } = makeDbStub([]); + // Make the candidate SELECT throw to simulate a boot-time DB hiccup. + builder.execute.mockRejectedValueOnce(new Error('db not ready')); + const pageRepo = { removePage: jest.fn() } as any; + const service = new TemporaryNoteCleanupService(builder, pageRepo); + + await expect(service.onApplicationBootstrap()).resolves.toBeUndefined(); + }); }); diff --git a/apps/server/src/core/page/services/temporary-note-cleanup.service.ts b/apps/server/src/core/page/services/temporary-note-cleanup.service.ts index d08275ed..791dd468 100644 --- a/apps/server/src/core/page/services/temporary-note-cleanup.service.ts +++ b/apps/server/src/core/page/services/temporary-note-cleanup.service.ts @@ -1,8 +1,13 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { + Injectable, + Logger, + OnApplicationBootstrap, +} from '@nestjs/common'; import { Interval } from '@nestjs/schedule'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB } from '@docmost/db/types/kysely.types'; import { PageRepo } from '@docmost/db/repos/page/page.repo'; +import { executeTx } from '@docmost/db/utils'; /** * Background sweeper for temporary notes ("structure or die"). A note whose @@ -11,7 +16,7 @@ import { PageRepo } from '@docmost/db/repos/page/page.repo'; * TrashCleanupService; `@nestjs/schedule` is already enabled globally. */ @Injectable() -export class TemporaryNoteCleanupService { +export class TemporaryNoteCleanupService implements OnApplicationBootstrap { private readonly logger = new Logger(TemporaryNoteCleanupService.name); // Cap a single sweep so a large backlog (e.g. many notes created during @@ -24,6 +29,20 @@ export class TemporaryNoteCleanupService { private readonly pageRepo: PageRepo, ) {} + // Sweep once at startup so notes that expired during downtime are trashed + // right away instead of waiting up to an hour for the first @Interval tick. + // Best-effort: never let a startup-sweep failure block application boot. + async onApplicationBootstrap() { + try { + await this.sweepExpiredTemporaryNotes(); + } catch (error) { + this.logger.error( + 'Temporary-note startup sweep failed', + error instanceof Error ? error.stack : undefined, + ); + } + } + // Hourly granularity: lifetimes are configured in hours, so a sub-hour // overshoot past the deadline is acceptable. @Interval('temporary-note-cleanup', 60 * 60 * 1000) @@ -31,9 +50,11 @@ export class TemporaryNoteCleanupService { try { const now = new Date(); + // Candidate ids (non-locking). The authoritative re-check happens per row + // under a row lock below, so this cheap pass just bounds the batch. const expired = await this.db .selectFrom('pages') - .select(['id', 'creatorId', 'workspaceId']) + .select(['id']) .where('temporaryExpiresAt', 'is not', null) .where('temporaryExpiresAt', '<', now) .where('deletedAt', 'is', null) // not already in trash @@ -41,50 +62,53 @@ export class TemporaryNoteCleanupService { .execute(); let trashed = 0; - for (const page of expired) { + for (const candidate of expired) { try { - // Re-check the deadline at deletion time. The SELECT above is not - // transactional, so a user may click "Make permanent" - // (toggleTemporary sets temporary_expires_at = null) in the window - // between the SELECT and this per-row removePage. removePage deletes - // by id with only a `deletedAt IS NULL` filter and never re-reads the - // deadline, so without this guard a concurrently-kept note would - // still be trashed. Re-read the row and skip it unless it is still - // armed AND still expired, so a concurrent make-permanent wins. - const current = await this.db - .selectFrom('pages') - .select(['temporaryExpiresAt', 'deletedAt']) - .where('id', '=', page.id) - .executeTakeFirst(); + const didTrash = await executeTx(this.db, async (trx) => { + // Re-check the row UNDER A LOCK inside the transaction. `FOR UPDATE + // SKIP LOCKED`: + // - serialises against a concurrent "Make permanent" + // (toggleTemporary UPDATE takes the same row lock): if it commits + // first, the deadline predicate below no longer matches and we + // skip; if we lock first, it waits until this delete commits. + // - SKIP LOCKED lets a second worker/instance skip a row another + // sweeper already claimed instead of blocking on it (no double + // processing, no thundering herd). + // The predicate re-asserts still-armed AND still-expired AND + // not-already-trashed, so a make-permanent / prior sweep drops the row. + const locked = await trx + .selectFrom('pages') + .select(['id', 'creatorId', 'workspaceId']) + .where('id', '=', candidate.id) + .where('temporaryExpiresAt', 'is not', null) + .where('temporaryExpiresAt', '<', now) + .where('deletedAt', 'is', null) + .forUpdate() + .skipLocked() + .executeTakeFirst(); - if ( - !current || - current.deletedAt !== null || - current.temporaryExpiresAt === null || - new Date(current.temporaryExpiresAt) >= now - ) { - // Made permanent, already trashed, or no longer expired since the - // SELECT — leave it alone. - continue; - } + if (!locked) return false; - // Reuse the exact soft-delete path: recursive over children, removes - // shares in a transaction, and emits PAGE_SOFT_DELETED (tree - // invalidation + watcher notifications). Attribute the automatic - // deletion to the note's creator (no schema change). Both the SELECT - // above and removePage filter `deletedAt IS NULL`, so a double sweep - // is idempotent. - await this.pageRepo.removePage( - page.id, - // creatorId is set on every created page; a temporary note always - // has one. Cast to satisfy the non-null deletedById parameter. - page.creatorId as string, - page.workspaceId, - ); - trashed++; + // Reuse the exact soft-delete path (recursive children + share + // removal + PAGE_SOFT_DELETED broadcast), running IN this locked + // transaction so the delete is atomic with the re-check and cannot + // deadlock on a nested independent transaction. The broadcast is + // deferred by removePage to this transaction's commit. Attribute the + // automatic deletion to the note's creator (no schema change). + await this.pageRepo.removePage( + locked.id, + // creatorId is set on every created page; a temporary note always + // has one. Cast to satisfy the non-null deletedById parameter. + locked.creatorId as string, + locked.workspaceId, + trx, + ); + return true; + }); + if (didTrash) trashed++; } catch (error) { this.logger.error( - `Failed to trash expired temporary note ${page.id}`, + `Failed to trash expired temporary note ${candidate.id}`, error instanceof Error ? error.stack : undefined, ); } diff --git a/apps/server/src/core/share/share-alias.controller.spec.ts b/apps/server/src/core/share/share-alias.controller.spec.ts index 0ff1d2fc..eb030eac 100644 --- a/apps/server/src/core/share/share-alias.controller.spec.ts +++ b/apps/server/src/core/share/share-alias.controller.spec.ts @@ -133,6 +133,9 @@ describe('ShareAliasController authz gates', () => { creatorId: 'u-1', alias: 'promo', confirmReassign: true, + // The requesting user is forwarded so setAlias can gate the reassign + // 409 title disclosure on target-page view permission (#495). + user, }); expect(result).toEqual({ id: 'alias-1' }); }); diff --git a/apps/server/src/core/share/share-alias.controller.ts b/apps/server/src/core/share/share-alias.controller.ts index 5c50f8b7..3b8e81db 100644 --- a/apps/server/src/core/share/share-alias.controller.ts +++ b/apps/server/src/core/share/share-alias.controller.ts @@ -79,6 +79,9 @@ export class ShareAliasController { creatorId: user.id, alias: dto.alias, confirmReassign: dto.confirmReassign, + // Gates whether the reassign 409 may reveal the current target's title + // (view-permission check on that page) — see setAlias (#495). + user, }); } diff --git a/apps/server/src/core/share/share-alias.service.spec.ts b/apps/server/src/core/share/share-alias.service.spec.ts index 471ccf8e..23931d52 100644 --- a/apps/server/src/core/share/share-alias.service.spec.ts +++ b/apps/server/src/core/share/share-alias.service.spec.ts @@ -1,4 +1,8 @@ -import { BadRequestException, ConflictException } from '@nestjs/common'; +import { + BadRequestException, + ConflictException, + ForbiddenException, +} from '@nestjs/common'; import { NoResultError } from 'kysely'; import { ShareAliasService } from './share-alias.service'; @@ -7,6 +11,8 @@ import { ShareAliasService } from './share-alias.service'; * 409 reassign guard, uniqueness-race handling, availability probe, and the * request-time readable-target resolution (which re-runs the share boundary). */ +const USER = { id: 'u-1' } as any; + describe('ShareAliasService', () => { // Sentinel handed to repo calls so tests can assert they ran inside the tx. const trx = { __trx: true }; @@ -27,6 +33,10 @@ describe('ShareAliasService', () => { resolveReadableSharePage: jest.fn(), isSharingAllowed: jest.fn(), }; + // Default: the requester CAN view the target page (validateCanView resolves), + // so the reassign 409 may disclose its title. Tests override to reject to + // assert the no-leak path. + const pageAccessService = { validateCanView: jest.fn().mockResolvedValue(undefined) }; // Fake kysely db: only .transaction().execute(cb) is used by setAlias. const db = { transaction: jest.fn(() => ({ @@ -37,9 +47,10 @@ describe('ShareAliasService', () => { shareAliasRepo as any, pageRepo as any, shareService as any, + pageAccessService as any, db as any, ); - return { service, shareAliasRepo, pageRepo, shareService, db }; + return { service, shareAliasRepo, pageRepo, shareService, pageAccessService, db }; } describe('setAlias', () => { @@ -50,6 +61,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'A', // too short + uppercase }), ).rejects.toBeInstanceOf(BadRequestException); @@ -66,6 +78,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: ' My Page ', }); @@ -114,6 +127,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'ted', }); @@ -144,6 +158,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', }); @@ -179,6 +194,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'new', }); @@ -190,30 +206,77 @@ describe('ShareAliasService', () => { ); }); - it('throws 409 with current target when name is taken and not confirmed', async () => { - const { service, shareAliasRepo, pageRepo } = makeService(); + it('throws 409 with the target TITLE (never its id) when the requester CAN view it', async () => { + const { service, shareAliasRepo, pageRepo, pageAccessService } = + makeService(); shareAliasRepo.findByAliasAndWorkspace.mockResolvedValue({ id: 'a-1', alias: 'foo', pageId: 'p-other', }); pageRepo.findById.mockResolvedValue({ id: 'p-other', title: 'Other' }); + pageAccessService.validateCanView.mockResolvedValue(undefined); // can view try { await service.setAlias({ workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', }); fail('expected ConflictException'); } catch (err) { expect(err).toBeInstanceOf(ConflictException); - expect((err as ConflictException).getResponse()).toMatchObject({ + const body = (err as ConflictException).getResponse(); + expect(body).toMatchObject({ code: 'ALIAS_REASSIGN_REQUIRED', - currentPageId: 'p-other', currentPageTitle: 'Other', }); + // SECURITY (#495): the page id is NEVER disclosed, even to a viewer. + expect(body).not.toHaveProperty('currentPageId'); + expect(pageAccessService.validateCanView).toHaveBeenCalledWith( + expect.objectContaining({ id: 'p-other' }), + USER, + ); + } + expect(shareAliasRepo.updatePageId).not.toHaveBeenCalled(); + }); + + it('throws 409 WITHOUT the title or id when the requester CANNOT view the target (#495)', async () => { + const { service, shareAliasRepo, pageRepo, pageAccessService } = + makeService(); + shareAliasRepo.findByAliasAndWorkspace.mockResolvedValue({ + id: 'a-1', + alias: 'foo', + pageId: 'p-secret', + }); + pageRepo.findById.mockResolvedValue({ id: 'p-secret', title: 'Secret' }); + // No view permission on the target page -> validateCanView throws. + pageAccessService.validateCanView.mockRejectedValue( + new ForbiddenException(), + ); + + try { + await service.setAlias({ + workspaceId: 'ws-1', + pageId: 'p-1', + creatorId: 'u-1', + user: USER, + alias: 'foo', + }); + fail('expected ConflictException'); + } catch (err) { + expect(err).toBeInstanceOf(ConflictException); + const body = (err as ConflictException).getResponse() as Record< + string, + unknown + >; + expect(body).toMatchObject({ code: 'ALIAS_REASSIGN_REQUIRED' }); + // The enumeration hole: neither the id nor the title of a page the + // requester cannot see may leak. + expect(body).not.toHaveProperty('currentPageId'); + expect(body.currentPageTitle ?? null).toBeNull(); } expect(shareAliasRepo.updatePageId).not.toHaveBeenCalled(); }); @@ -231,6 +294,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', confirmReassign: true, }); @@ -269,6 +333,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', }); fail('expected ConflictException'); @@ -294,6 +359,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', }); fail('expected ConflictException'); @@ -317,6 +383,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', }); fail('expected ConflictException'); @@ -346,6 +413,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', }); fail('expected ConflictException'); @@ -375,6 +443,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', confirmReassign: true, }); @@ -406,6 +475,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'ted', }); fail('expected ConflictException'); @@ -428,6 +498,7 @@ describe('ShareAliasService', () => { workspaceId: 'ws-1', pageId: 'p-1', creatorId: 'u-1', + user: USER, alias: 'foo', }), ).rejects.toBeInstanceOf(BadRequestException); @@ -450,18 +521,22 @@ describe('ShareAliasService', () => { alias: 'free-name', valid: true, available: true, - currentPageId: null, }); + // SECURITY (#495): the availability probe must NOT leak any page id. + expect(res).not.toHaveProperty('currentPageId'); }); - it('reports taken with the current target page', async () => { + it('reports taken WITHOUT leaking the current target page id (#495)', async () => { const { service, shareAliasRepo } = makeService(); shareAliasRepo.findByAliasAndWorkspace.mockResolvedValue({ id: 'a-1', pageId: 'p-9', }); const res = await service.checkAvailability('taken', 'ws-1'); - expect(res).toMatchObject({ available: false, currentPageId: 'p-9' }); + expect(res).toMatchObject({ available: false }); + // The row exists (available:false) but its pageId is never returned — an + // authenticated member cannot map an alias name to a page id it can't view. + expect(res).not.toHaveProperty('currentPageId'); }); }); diff --git a/apps/server/src/core/share/share-alias.service.ts b/apps/server/src/core/share/share-alias.service.ts index 8b05eabb..bb3fa5cd 100644 --- a/apps/server/src/core/share/share-alias.service.ts +++ b/apps/server/src/core/share/share-alias.service.ts @@ -7,7 +7,8 @@ import { import { ShareAliasRepo } from '@docmost/db/repos/share-alias/share-alias.repo'; import { PageRepo } from '@docmost/db/repos/page/page.repo'; import { ShareService } from './share.service'; -import { Page, ShareAlias } from '@docmost/db/types/entity.types'; +import { PageAccessService } from '../page/page-access/page-access.service'; +import { Page, ShareAlias, User } from '@docmost/db/types/entity.types'; import { isValidShareAlias, normalizeShareAlias } from './share-alias.util'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB } from '@docmost/db/types/kysely.types'; @@ -43,6 +44,7 @@ export class ShareAliasService { private readonly shareAliasRepo: ShareAliasRepo, private readonly pageRepo: PageRepo, private readonly shareService: ShareService, + private readonly pageAccessService: PageAccessService, @InjectKysely() private readonly db: KyselyDB, ) {} @@ -55,9 +57,13 @@ export class ShareAliasService { * `/l/` link survives * - name already points at pageId -> no-op (idempotent) * - name points at ANOTHER page -> the "swap". Without confirmReassign - * we throw 409 carrying the current target so the client can confirm; - * with it we UPDATE the single row's page_id (every /l/ link - * follows the 302 to the new page instantly — no stale cache). + * we throw 409 so the client can confirm. SECURITY (#495): the 409 reveals + * the current target's title ONLY when `user` may VIEW that page, and never + * its id — otherwise any member with one editable+shared page could iterate + * alias names with confirmReassign=false and map them to (id, title) of + * pages they cannot see. With confirmReassign we UPDATE the single row's + * page_id (every /l/ link follows the 302 to the new page instantly + * — no stale cache). * * To keep the invariant self-healing we DELETE every other alias row still * pointing at this page (a legacy duplicate, or the target page's own former @@ -77,8 +83,12 @@ export class ShareAliasService { creatorId: string; alias: string; confirmReassign?: boolean; + // The requesting user — used ONLY to gate whether the reassign 409 may reveal + // the current target page's title (view-permission check). Not an authz gate + // for the write itself (the controller already validated edit on `pageId`). + user: User; }): Promise { - const { workspaceId, pageId, creatorId, confirmReassign } = opts; + const { workspaceId, pageId, creatorId, confirmReassign, user } = opts; const alias = normalizeShareAlias(opts.alias); if (!isValidShareAlias(alias)) { throw new BadRequestException( @@ -97,14 +107,30 @@ export class ShareAliasService { // The name is occupied by a DIFFERENT (or dangling) target page. if (byName && byName.pageId !== pageId) { if (!confirmReassign) { + // SECURITY (#495): only disclose the current target's TITLE, and only + // when the requester may VIEW that page. Never disclose its id (the + // client's confirm-reassign UX doesn't use it, and it is an enumerable + // identity). A member with one editable+shared page must NOT be able to + // iterate alias names and map them to (id, title) of pages they cannot + // see. When view is denied (or the alias is dangling) the 409 is the + // bare "occupied" fact — the client still shows a generic confirm modal. const currentPage = byName.pageId ? await this.pageRepo.findById(byName.pageId) : null; + let currentPageTitle: string | null = null; + if (currentPage) { + try { + await this.pageAccessService.validateCanView(currentPage, user); + currentPageTitle = currentPage.title ?? null; + } catch { + // No view permission on the target -> do not reveal its title. + currentPageTitle = null; + } + } throw new ConflictException({ message: 'Alias already in use', code: 'ALIAS_REASSIGN_REQUIRED', - currentPageId: byName.pageId, - currentPageTitle: currentPage?.title ?? null, + currentPageTitle, }); } // Confirmed swap. ORDER MATTERS: the partial unique index on @@ -223,21 +249,27 @@ export class ShareAliasService { alias: string; valid: boolean; available: boolean; - currentPageId: string | null; }> { const alias = normalizeShareAlias(rawAlias); if (!isValidShareAlias(alias)) { - return { alias, valid: false, available: false, currentPageId: null }; + return { alias, valid: false, available: false }; } const existing = await this.shareAliasRepo.findByAliasAndWorkspace( alias, workspaceId, ); + // SECURITY (#495): return ONLY the boolean availability. The previous shape + // leaked `currentPageId` — the id of whatever page the alias already targets — + // to ANY authenticated workspace member, with no view-permission check on that + // page. An attacker could enumerate alias names and map them to page ids they + // have no access to. The taken/free bit is all the "is this address free" + // probe needs. The reassign flow (setAlias 409) may surface the target's + // TITLE, but only behind a `validateCanView` on that page (see setAlias); it + // never returns the page id. return { alias, valid: true, available: !existing, - currentPageId: existing?.pageId ?? null, }; } diff --git a/apps/server/src/core/telemetry/client-metrics.constants.ts b/apps/server/src/core/telemetry/client-metrics.constants.ts index a23fd06f..a3a0e469 100644 --- a/apps/server/src/core/telemetry/client-metrics.constants.ts +++ b/apps/server/src/core/telemetry/client-metrics.constants.ts @@ -24,6 +24,54 @@ export const ALLOWED_RATINGS = new Set([ 'poor', ]); +// The ONLY route labels accepted. The endpoint is anonymous, so an un-checked +// `route` is a free-text write surface (arbitrary high-cardinality strings / +// injected text into the metrics table). The client only ever sends a label from +// a finite template dictionary (`templateRoute`), so we drop anything not in it. +// +// PARITY: this MUST mirror `KNOWN_ROUTE_TEMPLATES` in the client's +// `apps/client/src/lib/telemetry/route-template.ts` (the canonical source). A +// drift means legit client routes get dropped — keep the two in lockstep; the +// client self-consistency test asserts `templateRoute` only emits these values. +export const ALLOWED_ROUTE_TEMPLATES = new Set([ + '/', + 'other', + // Static routes. + '/home', + '/spaces', + '/favorites', + '/login', + '/forgot-password', + '/password-reset', + '/setup/register', + '/settings/account/profile', + '/settings/account/preferences', + '/settings/workspace', + '/settings/ai', + '/settings/members', + '/settings/groups', + '/settings/spaces', + '/settings/sharing', + // Dynamic templates (slugs/ids are already collapsed to `:param`). + '/share/:shareId/p/:slug', + '/share/p/:slug', + '/share/:shareId', + '/p/:slug', + '/s/:space/p/:slug', + '/s/:space/trash', + '/s/:space', + '/labels/:label', + '/invites/:invitationId', + '/settings/groups/:groupId', +]); + +// `attr` is a web-vitals attribution TARGET: a CSS-selector-ish string (an +// element path like `html>body>div#app>button.cta`), never free prose. Constrain +// it to a conservative CSS-selector charset so the anonymous endpoint cannot be +// used to write arbitrary text / PII / markup into the metrics table. A value +// containing anything outside this set is DROPPED (-> null); the event is kept. +export const ATTR_ALLOWED_CHARSET = /^[A-Za-z0-9#.\-_> :()[\]="'*+~,]+$/; + // Max events accepted per batch; the rest are ignored. export const MAX_EVENTS_PER_BATCH = 50; @@ -77,14 +125,20 @@ export function sanitizeVitalEvent( ? e.rating : null; + // route: accept ONLY a known template label (dictionary check), else drop to + // null. The length cap stays as a cheap pre-guard before the Set lookup. let route: string | null = null; if (typeof e.route === 'string' && e.route.length > 0) { - route = e.route.slice(0, MAX_ROUTE_LENGTH); + const candidate = e.route.slice(0, MAX_ROUTE_LENGTH); + route = ALLOWED_ROUTE_TEMPLATES.has(candidate) ? candidate : null; } + // attr: truncate, then accept ONLY if it is a CSS-selector-shaped string + // (charset whitelist); anything with characters outside the set is dropped. let attr: string | null = null; if (typeof e.attr === 'string' && e.attr.length > 0) { - attr = e.attr.slice(0, MAX_ATTR_LENGTH); + const candidate = e.attr.slice(0, MAX_ATTR_LENGTH); + attr = ATTR_ALLOWED_CHARSET.test(candidate) ? candidate : null; } let docSize: number | null = null; diff --git a/apps/server/src/core/telemetry/vitals.service.spec.ts b/apps/server/src/core/telemetry/vitals.service.spec.ts index 12fb78b9..62da97ee 100644 --- a/apps/server/src/core/telemetry/vitals.service.spec.ts +++ b/apps/server/src/core/telemetry/vitals.service.spec.ts @@ -90,6 +90,44 @@ describe('VitalsService.buildRows', () => { expect(rows[0].attr).toHaveLength(MAX_ATTR_LENGTH); }); + it('keeps a known route template but DROPS an unknown/free-text route (#495)', () => { + const rows = svc.buildRows( + { + events: [ + { name: 'INP', value: 1, route: '/s/:space/p/:slug' }, // known + { name: 'INP', value: 2, route: '/s/acme-corp/p/secret-slug' }, // raw path (slugs) — not a template + { name: 'INP', value: 3, route: 'DROP TABLE client_metrics;--' }, // injected free text + { name: 'INP', value: 4, route: '/home' }, // known static + ], + }, + WS, + ); + expect(rows.map((r) => r.route)).toEqual([ + '/s/:space/p/:slug', + null, // raw path dropped + null, // free text dropped + '/home', + ]); + }); + + it('DROPS an attr that is not a CSS-selector-shaped string (#495)', () => { + const rows = svc.buildRows( + { + events: [ + { name: 'INP', value: 1, attr: 'div#app>button.cta' }, // valid selector + { name: 'INP', value: 2, attr: 'user@example.com wrote a note' }, // free text / PII + { name: 'INP', value: 3, attr: '' }, // markup + ], + }, + WS, + ); + expect(rows.map((r) => r.attr)).toEqual([ + 'div#app>button.cta', + null, + null, + ]); + }); + it('caps the batch at 50 events', () => { const events = Array.from({ length: 200 }, () => ({ name: 'CLS', value: 1 })); const rows = svc.buildRows({ events }, WS); diff --git a/apps/server/src/database/concurrent-indexes.spec.ts b/apps/server/src/database/concurrent-indexes.spec.ts new file mode 100644 index 00000000..cc3247bc --- /dev/null +++ b/apps/server/src/database/concurrent-indexes.spec.ts @@ -0,0 +1,114 @@ +import * as path from 'path'; +import { readFileSync } from 'fs'; + +// Mock ONLY kysely's `sql.raw(...).execute()` so we can observe what +// ensureConcurrentIndexes runs and how it handles failures, without a DB. +const execMock = jest.fn((_db: unknown) => Promise.resolve(undefined)); +const rawMock = jest.fn((_stmt: string) => ({ execute: execMock })); +jest.mock('kysely', () => { + const actual = jest.requireActual('kysely'); + return { ...actual, sql: { ...actual.sql, raw: rawMock } }; +}); + +import { CONCURRENT_INDEXES, ensureConcurrentIndexes } from './concurrent-indexes'; + +describe('ensureConcurrentIndexes', () => { + const fakeDb = { __topLevelKysely: true } as never; + + beforeEach(() => { + execMock.mockReset().mockResolvedValue(undefined); + rawMock.mockClear(); + }); + + it('redefines the inlinable f_unaccent BEFORE building any index, then builds each CONCURRENTLY outside a transaction', async () => { + const onLog = jest.fn(); + await ensureConcurrentIndexes(fakeDb, onLog); + + // One f_unaccent redefine + one statement per index. + expect(rawMock).toHaveBeenCalledTimes(CONCURRENT_INDEXES.length + 1); + const statements = rawMock.mock.calls.map((c) => c[0] as string); + // ORDER: the f_unaccent redefine MUST be first — otherwise an existing + // tenant's old 2-arg f_unaccent makes every CONCURRENTLY build fail. + expect(statements[0]).toContain('CREATE OR REPLACE FUNCTION f_unaccent'); + expect(statements[0]).toContain('SELECT public.unaccent($1)'); + expect(statements[0]).not.toContain('CREATE INDEX'); + // The rest are the CONCURRENTLY index builds. + for (const stmt of statements.slice(1)) { + expect(stmt).toContain('CREATE INDEX CONCURRENTLY'); + expect(stmt).toContain('IF NOT EXISTS'); + } + // Executed against the top-level db (a transaction would forbid CONCURRENTLY). + for (const call of execMock.mock.calls) { + expect(call[0]).toBe(fakeDb); + } + expect(onLog).toHaveBeenCalledTimes(CONCURRENT_INDEXES.length + 1); + }); + + it('still builds the indexes when the f_unaccent redefine fails (fresh DB, best-effort)', async () => { + // Redefine (first execute) throws; the index loop must still run. + execMock + .mockRejectedValueOnce(new Error('extension "unaccent" does not exist')) + .mockResolvedValue(undefined); + const onLog = jest.fn(); + + await expect( + ensureConcurrentIndexes(fakeDb, onLog), + ).resolves.toBeUndefined(); + + // 1 redefine attempt + one per index. + expect(execMock).toHaveBeenCalledTimes(CONCURRENT_INDEXES.length + 1); + const errored = onLog.mock.calls.filter((c) => c[1] !== undefined); + expect(errored).toHaveLength(1); + expect(String(errored[0][1])).toContain('unaccent'); + }); + + it('is best-effort: a failing index does not abort the rest and is reported', async () => { + // Redefine ok; fail the FIRST index; the remaining ones must still be tried. + execMock + .mockResolvedValueOnce(undefined) // f_unaccent redefine + .mockRejectedValueOnce(new Error('relation "pages" does not exist')) + .mockResolvedValue(undefined); + const onLog = jest.fn(); + + await expect( + ensureConcurrentIndexes(fakeDb, onLog), + ).resolves.toBeUndefined(); + + expect(execMock).toHaveBeenCalledTimes(CONCURRENT_INDEXES.length + 1); + const errored = onLog.mock.calls.filter((c) => c[1] !== undefined); + expect(errored).toHaveLength(1); + expect(String(errored[0][1])).toContain('does not exist'); + }); +}); + +// DRIFT GUARD: each CONCURRENT_INDEXES entry pre-builds an index that a plain +// migration ALSO creates with `IF NOT EXISTS`. If the two expressions diverge, +// Postgres would treat them as different indexes and the pre-build would NOT +// make the migration a no-op. Assert the migration files still contain each +// index's functional expression. +describe('CONCURRENT_INDEXES parity with the migrations', () => { + const migrationsDir = path.join(__dirname, 'migrations'); + const files = [ + '20260705T120000-perf-indexes.ts', + '20260706T120000-search-lookup-trgm.ts', + ].map((f) => readFileSync(path.join(migrationsDir, f), 'utf8')); + const allMigrationSrc = files.join('\n'); + + it.each(CONCURRENT_INDEXES.map((i) => [i.name, i]))( + 'migration source still creates %s with the same expression', + (_name, idx) => { + // Extract the `USING gin ((...expr...) gin_trgm_ops)` tail from the + // canonical create and assert the migration source contains it verbatim. + const m = (idx as { create: string }).create.match( + /ON \w+ (USING gin .+)$/, + ); + expect(m).not.toBeNull(); + const expr = (m as RegExpMatchArray)[1]; + expect(allMigrationSrc).toContain(expr); + // And the migration must build it by the SAME index name. + expect(allMigrationSrc).toContain( + `CREATE INDEX IF NOT EXISTS ${(idx as { name: string }).name}`, + ); + }, + ); +}); diff --git a/apps/server/src/database/concurrent-indexes.ts b/apps/server/src/database/concurrent-indexes.ts new file mode 100644 index 00000000..c17e0652 --- /dev/null +++ b/apps/server/src/database/concurrent-indexes.ts @@ -0,0 +1,142 @@ +import { Kysely, sql } from 'kysely'; + +/** + * Indexes that MUST be built with `CREATE INDEX CONCURRENTLY` so an auto-deploy + * migration never takes a `SHARE` lock that blocks writes on a hot table + * (`pages`) for the — potentially minutes-long — GIN trigram build (#495 item 12). + * + * Kysely runs each migration INSIDE a transaction (Postgres has transactional + * DDL), and `CREATE INDEX CONCURRENTLY` cannot run inside a transaction block, so + * these cannot live in an ordinary migration. Instead {@link ensureConcurrentIndexes} + * builds them out-of-band (no transaction) BEFORE the migrator runs; the matching + * migrations keep a plain `CREATE INDEX IF NOT EXISTS` as a backstop, which then + * no-ops because the index already exists. So: + * - existing prod DB, incremental deploy: the pre-build FIRST redefines + * `f_unaccent` to its index-inlinable 1-arg form (see below), THEN builds each + * index CONCURRENTLY (no write lock); the migration's IF NOT EXISTS no-ops → + * the write-blocking build is gone; + * - fresh DB (or a DB whose `pages` / `unaccent` extension does not exist yet): + * the pre-build fails and is swallowed (best-effort), and the migration builds + * the index normally on an empty/small table where the lock is irrelevant. + * + * NOT a strict "worst case = previous behaviour": + * - The f_unaccent form matters. The trigram expressions use `LOWER(f_unaccent(col))`. + * The INDEX-INLINABLE 1-arg `f_unaccent(text)` (`SELECT public.unaccent($1)`) + * is created INSIDE migration 20260705, which runs AFTER this pre-build. On an + * existing tenant the live `f_unaccent` is still the OLD 2-arg form from + * 20250729, which Postgres CANNOT inline into a CONCURRENTLY index expression + * (`function unaccent(unknown, text) does not exist ... during inlining`) — the + * build fails outright. So this pre-build redefines `f_unaccent` to the 1-arg + * form FIRST (idempotent, output-identical, in lockstep with 20260705). Without + * that redefine the whole feature is dead-on-arrival for the very case it + * targets. + * - An INTERRUPTED `CREATE INDEX CONCURRENTLY` (killed pod, cancelled query) with + * a working `f_unaccent` leaves an INVALID index behind. A subsequent name-based + * `IF NOT EXISTS` — in BOTH this pre-build and the migration backstop — sees the + * name and skips, so the invalid index is NEVER repaired automatically and the + * query keeps seq-scanning until an operator `DROP INDEX`es it. The old + * in-transaction build could not leave an invalid index (a failed tx rolled the + * whole index back), so this is a genuinely new failure mode, not "= previous". + * (A future hardening could `DROP` an `indisvalid = false` index before + * rebuilding; not done here.) + * + * The `create` text is the CANONICAL definition — it MUST match the migration's + * `IF NOT EXISTS` create expression exactly (same functional expression + opclass) + * or Postgres would treat them as two different indexes. + */ +export const CONCURRENT_INDEXES: ReadonlyArray<{ + name: string; + create: string; +}> = [ + { + // #348 perf-indexes — pages.title trigram (coalesce-free functional expr). + name: 'idx_pages_title_trgm', + create: + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pages_title_trgm ' + + 'ON pages USING gin ((LOWER(f_unaccent(title))) gin_trgm_ops)', + }, + { + // #348 perf-indexes — users.name trigram (member search-suggest). + name: 'idx_users_name_trgm', + create: + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_users_name_trgm ' + + 'ON users USING gin ((LOWER(f_unaccent(name))) gin_trgm_ops)', + }, + { + // #443 search-lookup-trgm — pages.text_content trigram (the slow, large one). + name: 'idx_pages_text_content_trgm', + create: + 'CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_pages_text_content_trgm ' + + 'ON pages USING gin ((LOWER(f_unaccent(text_content))) gin_trgm_ops)', + }, +]; + +/** + * Idempotent, output-identical redefinition of `f_unaccent` to the INDEX-INLINABLE + * 1-arg form. MUST stay byte-for-byte in lockstep with migration + * `20260705T120000-perf-indexes.ts` (same signature + body): the trigram indexes + * above only build CONCURRENTLY once `f_unaccent(text)` inlines, and on an existing + * tenant it is still the old 2-arg form until that migration runs — which is AFTER + * this pre-build. + */ +const F_UNACCENT_REDEF = ` + CREATE OR REPLACE FUNCTION f_unaccent(text) + RETURNS text + LANGUAGE sql + IMMUTABLE PARALLEL SAFE STRICT + AS $func$ + SELECT public.unaccent($1); + $func$ +`; + +/** + * Best-effort, non-transactional pre-build of {@link CONCURRENT_INDEXES}. Run + * BEFORE the migrator so the blocking `CREATE INDEX` in the corresponding + * migration becomes an `IF NOT EXISTS` no-op. + * + * `db` MUST be the top-level Kysely instance (NOT a transaction): each statement + * then executes on its own connection with no surrounding `BEGIN`, which is + * required for `CONCURRENTLY`. Every statement is independent and swallowed on + * error: the migration backstop still builds the index, so a failure here is + * never fatal. `onLog` reports progress/failures for the caller to route to its + * logger. + * + * ORDER MATTERS: redefine `f_unaccent` to the inlinable form BEFORE the index + * loop — otherwise an existing tenant's old 2-arg `f_unaccent` makes every + * CONCURRENTLY build fail and the feature is a no-op (see the module docstring). + */ +export async function ensureConcurrentIndexes( + db: Kysely, + onLog?: (message: string, error?: unknown) => void, +): Promise { + // Make f_unaccent inlinable FIRST. On a fresh DB (no `unaccent` extension yet) + // this throws harmlessly and is swallowed — the migration builds everything. + try { + await sql.raw(F_UNACCENT_REDEF).execute(db); + onLog?.('f_unaccent redefined to the index-inlinable 1-arg form'); + } catch (error) { + onLog?.( + 'f_unaccent redefine skipped (fresh DB / no unaccent extension) — ' + + 'the migration will define it and build the indexes', + error, + ); + } + + for (const idx of CONCURRENT_INDEXES) { + try { + await sql.raw(idx.create).execute(db); + onLog?.(`Concurrent index ensured: ${idx.name}`); + } catch (error) { + // Non-fatal by design — the migration's IF NOT EXISTS create is the + // backstop. Benign on a fresh DB (the table/extension does not exist yet). + // NOT benign, but still swallowed, on an existing tenant whose f_unaccent + // could not be redefined above (then the migration builds the index NON- + // concurrently under the SHARE lock this feature meant to avoid). + onLog?.( + `Concurrent index pre-build skipped for ${idx.name} ` + + `(will fall back to the in-migration build)`, + error, + ); + } + } +} diff --git a/apps/server/src/database/execute-tx-after-commit.spec.ts b/apps/server/src/database/execute-tx-after-commit.spec.ts new file mode 100644 index 00000000..29495f13 --- /dev/null +++ b/apps/server/src/database/execute-tx-after-commit.spec.ts @@ -0,0 +1,113 @@ +import { executeTx, registerAfterCommit } from './utils'; +import { KyselyDB, KyselyTransaction } from './types/kysely.types'; + +// Post-commit hook contract (#495 item 13): a side effect registered via +// registerAfterCommit must run ONLY AFTER the owning transaction commits, and a +// hook registered against a passed-through existingTrx must fire at the OUTER +// commit boundary — never inside the inner call. We fake the Kysely transaction +// runner so the ordering is observable without a real DB. + +/** + * A minimal db whose `.transaction().execute(cb)` records the commit ORDER: it + * runs `cb(trx)`, pushes 'commit' onto `log` (simulating the real commit that + * happens after the callback resolves), then returns the callback's result. + */ +function fakeDb(log: string[]): { db: KyselyDB; trx: KyselyTransaction } { + const trx = { __fakeTrx: true } as unknown as KyselyTransaction; + const db = { + transaction: () => ({ + execute: async (cb: (t: KyselyTransaction) => Promise) => { + const result = await cb(trx); + log.push('commit'); + return result; + }, + }), + } as unknown as KyselyDB; + return { db, trx }; +} + +describe('executeTx post-commit hooks', () => { + it('runs an afterCommit hook only AFTER the transaction commits', async () => { + const log: string[] = []; + const { db } = fakeDb(log); + + await executeTx(db, async (trx) => { + log.push('body'); + registerAfterCommit(trx, () => { + log.push('hook'); + }); + // The hook must NOT have run yet — the tx is still open. + expect(log).toEqual(['body']); + }); + + // Order proves post-commit: body → commit → hook (never body → hook → commit). + expect(log).toEqual(['body', 'commit', 'hook']); + }); + + it('drains hooks registered against a passed-through existingTrx at the OUTER commit', async () => { + const log: string[] = []; + const { db, trx: outerTrx } = fakeDb(log); + + await executeTx(db, async (outer) => { + // Nested executeTx reuses the outer trx: it must NOT commit or drain now. + await executeTx( + db, + async (inner) => { + registerAfterCommit(inner, () => { + log.push('inner-hook'); + }); + }, + outer, + ); + // Still inside the outer tx — the inner hook has not fired. + expect(log).toEqual([]); + }); + + // The single (outer) commit drains the hook registered on the shared trx. + expect(log).toEqual(['commit', 'inner-hook']); + // Sanity: the trx the hooks were registered against is the outer one. + expect(outerTrx).toBeDefined(); + }); + + it('a hook failure does not reject the already-committed executeTx', async () => { + const log: string[] = []; + const { db } = fakeDb(log); + + await expect( + executeTx(db, async (trx) => { + registerAfterCommit(trx, () => { + throw new Error('cache del blew up'); + }); + registerAfterCommit(trx, () => { + log.push('second-hook-still-runs'); + }); + return 'ok'; + }), + ).resolves.toBe('ok'); + + // The throwing hook is swallowed; a later hook still runs. + expect(log).toEqual(['commit', 'second-hook-still-runs']); + }); + + it('does NOT run afterCommit hooks when the transaction body throws (rollback)', async () => { + // The body rejects -> the fake transaction never pushes 'commit' and + // db.transaction().execute() rejects, mirroring a real rolled-back tx. The + // drain runs only AFTER the awaited (committed) transaction, so a rollback + // must leave every registered hook UN-run — otherwise a cache-bust / event + // would fire for a write that never landed. + const log: string[] = []; + const { db } = fakeDb(log); + const hook = jest.fn(); + + await expect( + executeTx(db, async (trx) => { + registerAfterCommit(trx, hook); + throw new Error('write failed -> rollback'); + }), + ).rejects.toThrow('write failed -> rollback'); + + // No commit happened, and the post-commit hook never ran. + expect(log).toEqual([]); // no 'commit' + expect(hook).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/server/src/database/jsonb-bind.spec.ts b/apps/server/src/database/jsonb-bind.spec.ts index 4e9d3ffa..1bac075b 100644 --- a/apps/server/src/database/jsonb-bind.spec.ts +++ b/apps/server/src/database/jsonb-bind.spec.ts @@ -35,4 +35,25 @@ describe('jsonbBind', () => { expect(out).not.toBeNull(); expect(out).toBeDefined(); }); + + // preserveEmpty (#476): opts a column OUT of the empty-to-null collapse so an + // empty container is persisted verbatim (e.g. `[]` = deny-all for + // tool_allowlist). null stays null regardless of the flag. + describe('preserveEmpty', () => { + it('returns a (non-null) bind for an empty array', () => { + const out = jsonbBind([], { preserveEmpty: true }); + expect(out).not.toBeNull(); + expect(out).toBeDefined(); + }); + + it('returns a (non-null) bind for an empty object', () => { + const out = jsonbBind({}, { preserveEmpty: true }); + expect(out).not.toBeNull(); + expect(out).toBeDefined(); + }); + + it('still returns null for null (null means null, flag or not)', () => { + expect(jsonbBind(null, { preserveEmpty: true })).toBeNull(); + }); + }); }); diff --git a/apps/server/src/database/migrations/20260705T120000-perf-indexes.ts b/apps/server/src/database/migrations/20260705T120000-perf-indexes.ts index b39112d4..d9d97243 100644 --- a/apps/server/src/database/migrations/20260705T120000-perf-indexes.ts +++ b/apps/server/src/database/migrations/20260705T120000-perf-indexes.ts @@ -33,16 +33,18 @@ import { type Kysely, sql } from 'kysely'; * - comments: `findPageComments` does WHERE page_id ORDER BY id ASC, but only * `(page_id)` exists → extra sort. * - * DEPLOY-TIME LOCK WARNING: these are plain (non-CONCURRENT) CREATE INDEX - * statements — CONCURRENTLY is impossible because Kysely runs each migration in a - * transaction. They take a SHARE lock that BLOCKS writes (INSERT/UPDATE/DELETE) on - * pages/users/groups/comments/page_history for the duration of the build. The two - * GIN trigram builds on pages.title / users.name are the slow ones and can take - * minutes on a large tenant → a write-outage window during the deploy migration. - * For large installations, run this migration in a maintenance window, or build - * the trigram indexes out-of-band with CREATE INDEX CONCURRENTLY before deploying - * (then this migration's `IF NOT EXISTS` is a no-op). Small/typical tenants are - * unaffected. + * DEPLOY-TIME LOCK: these are plain (non-CONCURRENT) CREATE INDEX statements — + * CONCURRENTLY is impossible HERE because Kysely runs each migration in a + * transaction. The two GIN trigram builds on pages.title / users.name are the + * slow ones and would take a SHARE lock that BLOCKS writes on pages/users for + * minutes on a large tenant. To avoid that, `ensureConcurrentIndexes` + * (database/concurrent-indexes.ts) now pre-builds BOTH trigram indexes with + * CREATE INDEX CONCURRENTLY (no transaction) BEFORE the migrator runs, so on an + * existing DB the two `IF NOT EXISTS` trigram creates below no-op and no write + * lock is taken. On a fresh DB the pre-build is skipped and they build on an + * empty table. Keep the two trigram creates in lockstep with their CANONICAL + * definitions in CONCURRENT_INDEXES. (The plain b-tree indexes further down are + * fast metadata-only builds; they are not pre-built.) */ export async function up(db: Kysely): Promise { // Index-compatible, output-identical redefinition of f_unaccent (see header). diff --git a/apps/server/src/database/migrations/20260706T120000-search-lookup-trgm.ts b/apps/server/src/database/migrations/20260706T120000-search-lookup-trgm.ts index c28f4cf0..299e1980 100644 --- a/apps/server/src/database/migrations/20260706T120000-search-lookup-trgm.ts +++ b/apps/server/src/database/migrations/20260706T120000-search-lookup-trgm.ts @@ -26,13 +26,16 @@ import { type Kysely, sql } from 'kysely'; * to update than b-trees); on the small instances this fork targets that cost * is acceptable and the read win on agent lookups is the priority. * - * DEPLOY-TIME LOCK WARNING: plain (non-CONCURRENT) CREATE INDEX — Kysely runs - * each migration in a transaction, so CONCURRENTLY is impossible. The build takes - * a SHARE lock that BLOCKS writes on `pages` for its duration. The text_content - * GIN build is the slow one and can take minutes on a large tenant. For large - * installations, run this in a maintenance window or build the index out-of-band - * with CREATE INDEX CONCURRENTLY before deploying (then `IF NOT EXISTS` no-ops - * here). Small/typical tenants are unaffected. + * DEPLOY-TIME LOCK: this is a plain (non-CONCURRENT) CREATE INDEX — Kysely runs + * each migration in a transaction, so CONCURRENTLY is impossible HERE, and the + * build would take a SHARE lock that BLOCKS writes on `pages` for its duration + * (the text_content GIN build can take minutes on a large tenant). To avoid that, + * `ensureConcurrentIndexes` (database/concurrent-indexes.ts) now pre-builds this + * index with CREATE INDEX CONCURRENTLY (no transaction) BEFORE the migrator runs, + * so on an existing DB the `IF NOT EXISTS` below no-ops and no write lock is taken. + * On a fresh DB the pre-build is skipped and this builds it on an empty table + * where the lock is irrelevant. Keep this create in lockstep with the CANONICAL + * definition in CONCURRENT_INDEXES (same expression + opclass). */ export async function up(db: Kysely): Promise { // The title predicate is served by #348's idx_pages_title_trgm — see header. diff --git a/apps/server/src/database/migrations/20260707T120000-ai-chat-metadata.ts b/apps/server/src/database/migrations/20260707T120000-ai-chat-metadata.ts new file mode 100644 index 00000000..429a6bd9 --- /dev/null +++ b/apps/server/src/database/migrations/20260707T120000-ai-chat-metadata.ts @@ -0,0 +1,24 @@ +import { type Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + // Chat-level metadata bag (#490). First use: the deferred-tool ACTIVATION set + // (`activatedTools`) is persisted here so it survives across turns — previously + // the set was reset every turn, forcing the model to re-run loadTools and pay a + // fresh round-trip to re-activate the same tools each turn. On load the stored + // set is intersected with the current valid deferred names, so an allowlist / + // role change can never inject a now-nonexistent tool. + // + // jsonb, defaulted to '{}' so every row (incl. pre-migration ones, backfilled + // by the default) is a readable object — the app never has to null-guard the + // bag itself, only individual keys. + await db.schema + .alterTable('ai_chats') + .addColumn('metadata', 'jsonb', (col) => + col.notNull().defaultTo(sql`'{}'::jsonb`), + ) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('ai_chats').dropColumn('metadata').execute(); +} diff --git a/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.delta.int-spec.ts b/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.delta.int-spec.ts new file mode 100644 index 00000000..063b2d15 --- /dev/null +++ b/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.delta.int-spec.ts @@ -0,0 +1,280 @@ +import { randomUUID } from 'crypto'; +import { CamelCasePlugin, Kysely, sql } from 'kysely'; +import { PostgresJSDialect } from 'kysely-postgres-js'; +// NOT a default import: the project tsconfig is `module: commonjs` with NO +// esModuleInterop, so `import postgres from 'postgres'` compiles to +// `postgres_1.default(...)` and the CJS `postgres` export has no `.default` — +// it threw in beforeAll, was swallowed as "DB unreachable", and SILENTLY voided +// all six tests. Mirror the working integration harness (test/integration/db.ts). +import * as postgres from 'postgres'; +import { AiChatMessageRepo } from './ai-chat-message.repo'; +import { AiChatRunRepo } from './ai-chat-run.repo'; + +/** + * #491 delta-poll — OBSERVABLE-PROPERTY tests against a LIVE Postgres (the local + * gitmost test DB, docker `gitmost-test-pg` on :5432), not "rows through a mock" + * (a mock cannot observe the DB clock nor the overlap-window race — the very + * things that matter here). Drives the REAL repo methods (`findByChatUpdatedAfter`, + * the now()-stamped `update`) and asserts: + * 1. delta-relevant writes stamp `updatedAt` from the DB clock, not the app clock + * (proven by faking the process clock far into the future and observing the + * stamp stays on real DB time); + * 2. the poll returns only rows changed after the cursor, ordered, with a fresh + * DB-clock cursor; + * 3. the "committed late but stamped earlier than the cursor" RACE is caught by + * the overlap window (a naive `updatedAt > cursor` would MISS it); + * 4. the overlap GUARANTEES repeats across close polls — the contract behind the + * client's idempotent merge (mergeById). + * + * INTEGRATION lane (`*.int-spec.ts`): runs under `test:int`, whose global-setup + * DROPS + RE-CREATES + MIGRATES `docmost_test`, so the real `ai_chat_messages` / + * `ai_chat_runs` tables EXIST here. (It was previously a `.spec.ts` defaulting to + * the UNmigrated dev `docmost`; in the CI unit lane — where `WAL_TEST_DATABASE_URL` + * is unset and only `test:int` migrates — that meant 5/6 ERROR + * `relation "ai_chat_messages" does not exist`, silently voiding coverage of the + * risky cursor/overlap logic. Renaming to `.int-spec.ts` + defaulting the DSN to + * `docmost_test` fixes the CI fidelity.) + * + * FK triggers are bypassed (`session_replication_role = replica`) so synthetic + * chat/workspace ids need no parent fixtures; a single pooled connection (max 1) + * keeps that session setting for every query. SKIPS cleanly when the DB is + * unreachable so a DB-less CI never breaks. + */ +const CONN = + process.env.WAL_TEST_DATABASE_URL ?? + process.env.TEST_DATABASE_URL ?? + 'postgresql://docmost:docmost_dev_pw@localhost:5432/docmost_test'; + +let db: Kysely; +let sqlClient: ReturnType; +let msgRepo: AiChatMessageRepo; +let runRepo: AiChatRunRepo; +let reachable = false; + +const workspaceId = randomUUID(); +const chatId = randomUUID(); + +beforeAll(async () => { + try { + sqlClient = postgres(CONN, { max: 1, onnotice: () => {} }); + db = new Kysely({ + dialect: new PostgresJSDialect({ postgres: sqlClient }), + plugins: [new CamelCasePlugin()], + }); + // Single connection keeps this session-scoped bypass for the whole suite. + await sql`set session_replication_role = replica`.execute(db); + await sql`select 1`.execute(db); + reachable = true; + } catch (err) { + reachable = false; + // A genuine connection failure (ECONNREFUSED etc.) is a legitimate skip on a + // DB-less CI. A PROGRAMMING error (bad import, typo, driver misuse) must NOT + // masquerade as "DB unreachable" and silently void the whole suite (that is + // exactly the bug that hid this spec's zero coverage) — rethrow it so the + // suite fails LOUDLY. + const msg = String((err as Error)?.message ?? err); + if ( + !/ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EHOSTUNREACH|connect|terminating|password|authentication|role .* does not exist|database .* does not exist/i.test( + msg, + ) + ) { + throw err; + } + } + msgRepo = new AiChatMessageRepo(db as never); + runRepo = new AiChatRunRepo(db as never); +}); + +afterAll(async () => { + if (db) { + try { + await db + .deleteFrom('aiChatMessages') + .where('workspaceId', '=', workspaceId) + .execute(); + await db + .deleteFrom('aiChatRuns') + .where('workspaceId', '=', workspaceId) + .execute(); + } catch { + /* best-effort cleanup */ + } + await db.destroy(); + } +}); + +afterEach(() => { + jest.useRealTimers(); +}); + +async function seedMessage(overrides: Record = {}) { + return msgRepo.insert({ + chatId, + workspaceId, + userId: null as never, + role: 'assistant', + content: 'x', + status: 'streaming', + ...overrides, + } as never); +} + +async function dbNow(): Promise { + const r = await sql<{ now: Date }>`select now() as now`.execute(db); + return r.rows[0].now.toISOString(); +} + +// Fake ONLY the Date object (so in-process `new Date()`/`Date.now()` jump), while +// leaving every TIMER function real. Faking timers wholesale freezes postgres.js's +// internal connection/query timers, so the awaited DB round-trip would hang the +// test (and the afterAll cleanup) at the jest 5s cap. With Date-only faking the +// query resolves normally, and we still prove the stamp is the DB clock (not the +// skewed process clock). +function fakeDateOnly(iso: string): void { + jest.useFakeTimers({ + doNotFake: [ + 'hrtime', + 'nextTick', + 'performance', + 'queueMicrotask', + 'requestAnimationFrame', + 'cancelAnimationFrame', + 'requestIdleCallback', + 'cancelIdleCallback', + 'setImmediate', + 'clearImmediate', + 'setInterval', + 'clearInterval', + 'setTimeout', + 'clearTimeout', + ], + now: new Date(iso), + }); +} + +const maybe = (name: string, fn: () => Promise) => + it(name, async () => { + if (!reachable) { + console.warn(`SKIP (${name}): test DB unreachable at ${CONN}`); + return; + } + await fn(); + }); + +describe('AiChatMessageRepo.findByChatUpdatedAfter (#491 delta poll)', () => { + maybe('null cursor returns no rows and a fresh DB-clock cursor', async () => { + const before = await dbNow(); + const { rows, cursor } = await msgRepo.findByChatUpdatedAfter( + chatId, + workspaceId, + null, + ); + expect(rows).toEqual([]); + expect(new Date(cursor).getTime()).toBeGreaterThanOrEqual( + new Date(before).getTime(), + ); + }); + + maybe('returns only rows changed after the cursor', async () => { + const { cursor: c0 } = await msgRepo.findByChatUpdatedAfter( + chatId, + workspaceId, + null, + ); + const m = await seedMessage(); + const { rows, cursor: c1 } = await msgRepo.findByChatUpdatedAfter( + chatId, + workspaceId, + c0, + ); + expect(rows.map((r) => r.id)).toContain(m.id); + // Cursor is monotonic (advances). + expect(new Date(c1).getTime()).toBeGreaterThanOrEqual( + new Date(c0).getTime(), + ); + }); + + maybe( + 'RACE: a row stamped BEFORE the cursor but seen after is caught by the overlap', + async () => { + // Cursor taken now; then a row appears whose updatedAt is 2s in the PAST + // (committed late on another connection but stamped earlier). A naive + // `updatedAt > cursor` would MISS it; the 5s overlap window catches it. + const cursor = await dbNow(); + const m = await seedMessage(); + await sql`update ai_chat_messages set updated_at = now() - interval '2 seconds' where id = ${m.id}`.execute( + db, + ); + const { rows } = await msgRepo.findByChatUpdatedAfter( + chatId, + workspaceId, + cursor, + ); + expect(rows.map((r) => r.id)).toContain(m.id); + }, + ); + + maybe( + 'overlap GUARANTEES repeats across close polls (idempotent-merge contract)', + async () => { + const { cursor: c0 } = await msgRepo.findByChatUpdatedAfter( + chatId, + workspaceId, + null, + ); + const m = await seedMessage(); + const first = await msgRepo.findByChatUpdatedAfter( + chatId, + workspaceId, + c0, + ); + expect(first.rows.map((r) => r.id)).toContain(m.id); + // Immediately re-poll with the JUST-returned cursor: the row is still within + // the overlap window, so it is returned AGAIN — the client MUST dedupe by id. + const second = await msgRepo.findByChatUpdatedAfter( + chatId, + workspaceId, + first.cursor, + ); + expect(second.rows.map((r) => r.id)).toContain(m.id); + }, + ); + + maybe( + 'update() stamps updatedAt from the DB clock, not the app clock', + async () => { + const m = await seedMessage(); + // Skew the PROCESS clock ~73 years into the future (Date only). If the stamp + // came from `new Date()` the row would read year 2099; sql now() keeps it on + // DB time. + fakeDateOnly('2099-01-01T00:00:00Z'); + const updated = await msgRepo.update(m.id, workspaceId, { + content: 'y', + }); + jest.useRealTimers(); + expect(updated).toBeDefined(); + expect(new Date(updated!.updatedAt).getFullYear()).toBeLessThan(2099); + }, + ); + + maybe( + 'run update() also stamps updatedAt from the DB clock', + async () => { + const run = await runRepo.insert({ + chatId, + workspaceId, + createdBy: null as never, + trigger: 'user', + status: 'running', + stepCount: 0, + } as never); + fakeDateOnly('2099-01-01T00:00:00Z'); + const updated = await runRepo.update(run.id, workspaceId, { + stepCount: 1, + }); + jest.useRealTimers(); + expect(updated).toBeDefined(); + expect(new Date(updated!.updatedAt).getFullYear()).toBeLessThan(2099); + }, + ); +}); diff --git a/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.ts b/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.ts index 96070e77..57e82baa 100644 --- a/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.ts +++ b/apps/server/src/database/repos/ai-chat/ai-chat-message.repo.ts @@ -25,6 +25,20 @@ const SWEEP_STREAMING_STALE_MS = 10 * 60 * 1000; // 10 minutes // into memory; far above any realistic transcript length. const FIND_ALL_BY_CHAT_LIMIT = 5000; +// Delta-poll overlap (#491): the poll query reaches this far BEHIND the client's +// echoed cursor, so a row that committed with an `updatedAt` marginally before the +// previous cursor was taken (on another autocommit connection) is still caught. +// Sized well above realistic single-row commit skew; the client merge is +// idempotent by id (mergeById), so the guaranteed repeats the overlap produces are +// harmless. +export const DELTA_POLL_OVERLAP_SECONDS = 5; + +// Hard cap on rows one delta poll returns — a safety bound (a poll should carry a +// handful of just-changed rows, never a whole transcript). Ordered by (updatedAt, +// id) asc, so on the pathological overflow the OLDEST changes win and the newest +// are picked up by the next poll (its cursor did not advance past them). +export const DELTA_POLL_MAX_ROWS = 500; + @Injectable() export class AiChatMessageRepo { private readonly logger = new Logger(AiChatMessageRepo.name); @@ -139,6 +153,72 @@ export class AiChatMessageRepo { .executeTakeFirst(); } + /** + * Delta read (#491) for the degraded poll: the chat's messages whose row + * changed AFTER the client's `cursor`, plus a FRESH cursor taken from the DB + * clock. Replaces the old "refetch ALL infinite-query pages every 2.5s with + * full parts" poll — the client seeds once (findByChat) and thereafter pulls + * only the deltas and merges them by id (mergeById). + * + * Cursor: a DB-clock timestamp (now()) the client echoes back each poll. All + * delta-relevant writes stamp `updatedAt` with now() (see `update` / + * `finalizeOwner`), so this is a SINGLE monotonic axis. The query overlaps the + * cursor by DELTA_POLL_OVERLAP_SECONDS to catch a row committed with an + * `updatedAt` marginally BEFORE the previous cursor was taken on another + * connection (single-row autocommit UPDATEs; no long transactions). The overlap + * GUARANTEES occasional REPEATS, so the client merge MUST be idempotent by id. + * + * `cursor === null` (first poll after the full seed) returns NO rows — there is + * nothing "new" relative to a just-loaded seed — only the fresh cursor to start + * the delta chain. The fresh cursor is read AFTER the rows, so it is >= every + * returned row's `updatedAt` (they were read strictly earlier) — a row that + * commits between the rows-read and the cursor-read is at most + * DELTA_POLL_OVERLAP_SECONDS behind the returned cursor, so the next poll's + * overlap window always re-includes it (no miss). + */ + async findByChatUpdatedAfter( + chatId: string, + workspaceId: string, + cursor: string | null, + ): Promise<{ rows: AiChatMessage[]; cursor: string }> { + if (cursor === null) { + const nowRow = await sql<{ now: Date }>`select now() as now`.execute( + this.db, + ); + return { rows: [], cursor: nowRow.rows[0].now.toISOString() }; + } + // Overlap the client cursor by DELTA_POLL_OVERLAP_SECONDS, computed in SQL off + // the echoed cursor so the whole comparison stays on the DB clock. + const rows = await this.db + .selectFrom('aiChatMessages') + .select(this.baseFields) + .where('chatId', '=', chatId) + .where('workspaceId', '=', workspaceId) + .where('deletedAt', 'is', null) + .where( + 'updatedAt', + '>', + sql`${cursor}::timestamptz - make_interval(secs => ${DELTA_POLL_OVERLAP_SECONDS})`, + ) + .orderBy('updatedAt', 'asc') + .orderBy('id', 'asc') + .limit(DELTA_POLL_MAX_ROWS) + .execute(); + // When the page filled (pathological overflow), DO NOT advance the cursor to + // now(): that would skip the changed rows past the cap that this poll did not + // return. Resume from the last returned row's updatedAt instead (the next + // poll's overlap re-includes ties by id). In the normal case the fresh DB-clock + // now() is the cursor. + if (rows.length === DELTA_POLL_MAX_ROWS) { + return { + rows, + cursor: rows[rows.length - 1].updatedAt.toISOString(), + }; + } + const nowRow = await sql<{ now: Date }>`select now() as now`.execute(this.db); + return { rows, cursor: nowRow.rows[0].now.toISOString() }; + } + async insert( insertable: InsertableAiChatMessage, trx?: KyselyTransaction, @@ -172,7 +252,13 @@ export class AiChatMessageRepo { const db = dbOrTx(this.db, opts?.trx); let query = db .updateTable('aiChatMessages') - .set({ ...(patch as Record), updatedAt: new Date() }) + // #491: stamp `updatedAt` from the DB clock (sql now()), NOT the app clock + // (new Date()). The delta-poll cursor (findByChatUpdatedAfter) is a single + // DB-clock axis; a per-step 'streaming' UPDATE stamped with the app clock + // would be a SECOND, skewed clock source and could leave a row's updatedAt + // just under a cursor taken from now() on another connection — an + // independent source of delta MISSES. All delta-relevant writes use now(). + .set({ ...(patch as Record), updatedAt: sql`now()` }) .where('id', '=', id) .where('workspaceId', '=', workspaceId); // Concurrency guard (#183 review): a per-step 'streaming' update must NEVER @@ -214,7 +300,9 @@ export class AiChatMessageRepo { const db = dbOrTx(this.db, trx); return db .updateTable('aiChatMessages') - .set({ ...(patch as Record), updatedAt: new Date() }) + // #491: DB-clock stamp (see `update`) — this terminal write flips the row's + // status, which the delta poll must observe on the shared now() cursor axis. + .set({ ...(patch as Record), updatedAt: sql`now()` }) .where('id', '=', id) .where('workspaceId', '=', workspaceId) .where((eb) => @@ -249,7 +337,9 @@ export class AiChatMessageRepo { .set({ status, metadata: sql`coalesce(metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`, - updatedAt: new Date(), + // #491: DB-clock stamp (see `update`) so a reconcile status flip lands on + // the same now() cursor axis the delta poll reads. + updatedAt: sql`now()`, }) .where('id', '=', id) .where('workspaceId', '=', workspaceId) @@ -307,7 +397,9 @@ export class AiChatMessageRepo { .set({ status: 'aborted', metadata: sql`coalesce(m.metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`, - updatedAt: new Date(), + // #491: DB-clock stamp (see `update`). The staleness WHERE below stays on + // the app clock — a >minutes window makes the ms-scale skew irrelevant. + updatedAt: sql`now()`, }) .where('m.status', '=', 'streaming') .where('m.updatedAt', '<', staleBefore) @@ -351,7 +443,8 @@ export class AiChatMessageRepo { .set({ status: 'aborted', metadata: sql`coalesce(metadata, '{}'::jsonb) || jsonb_build_object('finalizeFailed', true)`, - updatedAt: new Date(), + // #491: DB-clock stamp (see `update`). Staleness WHERE stays app-clock. + updatedAt: sql`now()`, }) .where('status', '=', 'streaming') .where('updatedAt', '<', staleBefore) diff --git a/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.spec.ts b/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.spec.ts index d03a43e3..5d6faa9d 100644 --- a/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.spec.ts +++ b/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.spec.ts @@ -62,10 +62,17 @@ describe('AiChatRunRepo.sweepRunning', () => { // ...but a fresh 'running' run (updatedAt = now) must NOT be skipped: no // updatedAt predicate at all on the boot path. expect(rec.wheres.some(([col]) => col === 'updatedAt')).toBe(false); - // It flips to 'aborted' and stamps finishedAt. - expect(rec.set).toEqual( - expect.objectContaining({ status: 'aborted', finishedAt: expect.any(Date) }), - ); + // It flips to 'aborted' and stamps finishedAt + updatedAt. #491: the stamps + // are now DB-clock `sql now()` expressions (raw builders), NOT app-clock + // `new Date()`, so the run row shares the delta poll's single now() cursor axis + // — assert they are present and are the sql raw-builder objects (not a Date, + // not undefined). + expect(rec.set?.status).toBe('aborted'); + for (const stamp of ['finishedAt', 'updatedAt'] as const) { + expect(rec.set?.[stamp]).toBeDefined(); + expect(rec.set?.[stamp]).not.toBeInstanceOf(Date); + expect(typeof rec.set?.[stamp]).toBe('object'); + } }); it('phase-2 path: an explicit staleMs reintroduces the updatedAt window', async () => { diff --git a/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts b/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts index c27480bf..5758867e 100644 --- a/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts +++ b/apps/server/src/database/repos/ai-chat/ai-chat-run.repo.ts @@ -136,7 +136,11 @@ export class AiChatRunRepo { const db = dbOrTx(this.db, trx); return db .updateTable('aiChatRuns') - .set({ ...(patch as Record), updatedAt: new Date() }) + // #491: DB-clock stamp (sql now()) so the run row shares the delta poll's + // single now() cursor axis with the assistant message rows — a run-status + // change (the run fact the delta carries) must never sit on a skewed app + // clock relative to the message updatedAt cursor. + .set({ ...(patch as Record), updatedAt: sql`now()` }) .where('id', '=', id) .where('workspaceId', '=', workspaceId) .returning(this.baseFields) @@ -162,14 +166,15 @@ export class AiChatRunRepo { trx?: KyselyTransaction, ): Promise { const db = dbOrTx(this.db, trx); - const now = new Date(); return db .updateTable('aiChatRuns') .set({ status: patch.status, error: patch.error, - finishedAt: now, - updatedAt: now, + // #491: DB-clock stamps (finished_at + updated_at) so the terminal run + // fact lands on the delta poll's now() cursor axis. + finishedAt: sql`now()`, + updatedAt: sql`now()`, }) .where('id', '=', id) .where('workspaceId', '=', workspaceId) @@ -192,7 +197,8 @@ export class AiChatRunRepo { const db = dbOrTx(this.db, trx); return db .updateTable('aiChatRuns') - .set({ stopRequestedAt: new Date(), updatedAt: new Date() }) + // #491: DB-clock stamps (see `update`). + .set({ stopRequestedAt: sql`now()`, updatedAt: sql`now()` }) .where('id', '=', id) .where('workspaceId', '=', workspaceId) .where('status', 'in', ACTIVE_RUN_STATUSES as unknown as string[]) @@ -249,13 +255,14 @@ export class AiChatRunRepo { trx?: KyselyTransaction, ): Promise { const db = dbOrTx(this.db, trx); - const now = new Date(); let query = db .updateTable('aiChatRuns') .set({ status: 'aborted', - finishedAt: now, - updatedAt: now, + // #491: DB-clock stamps (see `update`). The staleness WHERE below stays on + // the app clock — a >minutes window makes the ms-scale skew irrelevant. + finishedAt: sql`now()`, + updatedAt: sql`now()`, error: sql`coalesce(error, ${'Run interrupted by a server restart.'})`, }) .where('status', 'in', ACTIVE_RUN_STATUSES as unknown as string[]); @@ -263,7 +270,7 @@ export class AiChatRunRepo { // sibling replica's live run is never aborted. Omitted on the phase-1 boot // sweep -> unconditional. if (typeof opts.staleMs === 'number') { - const staleBefore = new Date(now.getTime() - opts.staleMs); + const staleBefore = new Date(Date.now() - opts.staleMs); query = query.where('updatedAt', '<', staleBefore); } const rows = await query.returning('id').execute(); diff --git a/apps/server/src/database/repos/ai-chat/ai-mcp-server.repo.ts b/apps/server/src/database/repos/ai-chat/ai-mcp-server.repo.ts index 8bcfc661..8ddcbc5b 100644 --- a/apps/server/src/database/repos/ai-chat/ai-mcp-server.repo.ts +++ b/apps/server/src/database/repos/ai-chat/ai-mcp-server.repo.ts @@ -78,7 +78,9 @@ export class AiMcpServerRepo { headersEnc: values.headersEnc ?? null, // jsonb column: the postgres driver would otherwise encode a JS array as // a Postgres array literal. Bind the JSON text and cast it to jsonb. - toolAllowlist: jsonbBind(values.toolAllowlist), + // preserveEmpty (#476): `[]` is a real value here (deny-all), distinct + // from null ("no restriction") — it must round-trip as `[]`, not null. + toolAllowlist: jsonbBind(values.toolAllowlist, { preserveEmpty: true }), // Plain text column: blank/whitespace-only guidance is stored as null. instructions: blankToNull(values.instructions), enabled: values.enabled ?? true, @@ -111,7 +113,10 @@ export class AiMcpServerRepo { if (patch.url !== undefined) set.url = patch.url; if (patch.headersEnc !== undefined) set.headersEnc = patch.headersEnc; if (patch.toolAllowlist !== undefined) { - set.toolAllowlist = jsonbBind(patch.toolAllowlist); + // preserveEmpty (#476): see insert — `[]` (deny-all) must not become null. + set.toolAllowlist = jsonbBind(patch.toolAllowlist, { + preserveEmpty: true, + }); } if (patch.instructions !== undefined) { // Blank/whitespace-only guidance clears the column (stored as null). @@ -158,7 +163,9 @@ export function blankToNull(value: string | null | undefined): string | null { * fix), so the driver hands back a string like `'["a","b"]'` rather than an * array. Be tolerant: normalize a JSON string to its value, then accept it only * if it is an array of strings; null / a non-array / unparseable value / an - * array with a non-string element all become null (unrestricted). + * array with a non-string element all become null. NOTE: null here only means + * "could not parse" — the null-vs-deny-all policy decision lives in + * normalizeRow (#476: present-but-corrupt fails CLOSED to `[]`). */ export function parseToolAllowlist(value: unknown): string[] | null { // Shape guard only; the legacy double-encoding self-heal lives in @@ -173,17 +180,20 @@ export function parseToolAllowlist(value: unknown): string[] | null { /** * Normalize a DB row so `toolAllowlist` is always `string[] | null`. * - * FAIL-OPEN logging: a stored value that is present but cannot be parsed into a - * string[] (corrupt JSON, a non-array, non-string elements) degrades to `null` = - * "no restriction", so the agent silently gets ALL of the server's tools. Log - * one line (server id only, never the contents) so that widening is not silent. + * FAIL-CLOSED (#476): a stored value that is PRESENT but cannot be parsed into + * a string[] (corrupt JSON, a non-array, non-string elements) degrades to `[]` + * = deny-all, so a corrupted allowlist can never silently widen to "the agent + * gets ALL of the server's tools" (the old fail-open null). An error line is + * logged (server id only, never the contents) so the admin can repair the row. + * A column that is truly NULL/absent stays `null` = "no restriction". */ function normalizeRow(row: AiMcpServer): AiMcpServer { const parsed = parseToolAllowlist(row.toolAllowlist); if (parsed === null && row.toolAllowlist != null) { - logger.warn( - `Corrupt tool_allowlist for MCP server ${row.id}; ignoring it (no tool restriction applied)`, + logger.error( + `Corrupt tool_allowlist for MCP server ${row.id}; failing closed (NO tools allowed) — re-save the server's allowlist to repair it`, ); + return { ...row, toolAllowlist: [] }; } return { ...row, toolAllowlist: parsed }; } diff --git a/apps/server/src/database/repos/page/page.repo.ts b/apps/server/src/database/repos/page/page.repo.ts index 7fed922a..561f5dbf 100644 --- a/apps/server/src/database/repos/page/page.repo.ts +++ b/apps/server/src/database/repos/page/page.repo.ts @@ -1,7 +1,7 @@ import { Injectable } from '@nestjs/common'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB, KyselyTransaction } from '../../types/kysely.types'; -import { dbOrTx, executeTx } from '../../utils'; +import { dbOrTx, executeTx, registerAfterCommit } from '../../utils'; import { InsertablePage, Page, @@ -349,14 +349,23 @@ export class PageRepo { pageId: string, deletedById: string, workspaceId: string, + // Optional caller transaction. When passed, the reads + soft-delete run in + // THAT transaction (so a caller holding a `FOR UPDATE` lock on the row — e.g. + // the temporary-note sweeper — can delete under the lock without deadlocking + // on a nested independent transaction) and the PAGE_SOFT_DELETED broadcast is + // deferred to the caller's COMMIT via registerAfterCommit (so a rolled-back + // delete never broadcasts). With no trx the behaviour is unchanged: own + // transaction, broadcast right after it commits. + existingTrx?: KyselyTransaction, ): Promise { const currentDate = new Date(); + const readDb = dbOrTx(this.db, existingTrx); // Read the root snapshot up front so PAGE_SOFT_DELETED can carry it without // a post-commit DB read (variant A). Only the root of the deleted subtree is // needed for the tree broadcast — the client `treeModel.remove` drops all // descendants, so we don't snapshot/broadcast every descendant. - const rootSnapshot = await this.db + const rootSnapshot = await readDb .selectFrom('pages') .select([ 'id', @@ -371,7 +380,7 @@ export class PageRepo { .where('deletedAt', 'is', null) .executeTakeFirst(); - const descendants = await this.db + const descendants = await readDb .withRecursive('page_descendants', (db) => db .selectFrom('pages') @@ -393,39 +402,60 @@ export class PageRepo { const pageIds = descendants.map((d) => d.id); if (pageIds.length > 0) { - await executeTx(this.db, async (trx) => { - await trx - .updateTable('pages') - .set({ - deletedById: deletedById, - deletedAt: currentDate, - }) - .where('id', 'in', pageIds) - .where('deletedAt', 'is', null) - .execute(); + // Reuse the caller's transaction when given (executeTx passes it straight + // through), else own a fresh one. + await executeTx( + this.db, + async (trx) => { + await trx + .updateTable('pages') + .set({ + deletedById: deletedById, + deletedAt: currentDate, + }) + .where('id', 'in', pageIds) + .where('deletedAt', 'is', null) + .execute(); - await trx.deleteFrom('shares').where('pageId', 'in', pageIds).execute(); - }); + await trx + .deleteFrom('shares') + .where('pageId', 'in', pageIds) + .execute(); + }, + existingTrx, + ); - this.eventEmitter.emit(EventName.PAGE_SOFT_DELETED, { - pageIds: pageIds, - workspaceId, - // Root-only snapshot: one `deleteTreeNode` is enough, the client removes - // the whole subtree. Skip if the root vanished between the two reads. - pages: rootSnapshot - ? [ - { - id: rootSnapshot.id, - slugId: rootSnapshot.slugId, - title: rootSnapshot.title, - icon: rootSnapshot.icon, - position: rootSnapshot.position, - spaceId: rootSnapshot.spaceId, - parentPageId: rootSnapshot.parentPageId, - }, - ] - : [], - }); + const emitSoftDeleted = () => { + this.eventEmitter.emit(EventName.PAGE_SOFT_DELETED, { + pageIds: pageIds, + workspaceId, + // Root-only snapshot: one `deleteTreeNode` is enough, the client + // removes the whole subtree. Skip if the root vanished between reads. + pages: rootSnapshot + ? [ + { + id: rootSnapshot.id, + slugId: rootSnapshot.slugId, + title: rootSnapshot.title, + icon: rootSnapshot.icon, + position: rootSnapshot.position, + spaceId: rootSnapshot.spaceId, + parentPageId: rootSnapshot.parentPageId, + }, + ] + : [], + }); + }; + + if (existingTrx) { + // Inside a caller transaction: the delete above is NOT committed yet. + // Defer the tree broadcast to the caller's commit so a rolled-back delete + // never broadcasts a phantom removal. + registerAfterCommit(existingTrx, emitSoftDeleted); + } else { + // Own transaction already committed above — broadcast now. + emitSoftDeleted(); + } } } diff --git a/apps/server/src/database/repos/workspace/workspace.repo.ts b/apps/server/src/database/repos/workspace/workspace.repo.ts index 89996766..b6d0fde7 100644 --- a/apps/server/src/database/repos/workspace/workspace.repo.ts +++ b/apps/server/src/database/repos/workspace/workspace.repo.ts @@ -3,7 +3,7 @@ import { CACHE_MANAGER } from '@nestjs/cache-manager'; import { Cache } from 'cache-manager'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB, KyselyTransaction } from '../../types/kysely.types'; -import { dbOrTx } from '../../utils'; +import { dbOrTx, registerAfterCommit } from '../../utils'; import { InsertableWorkspace, UpdatableWorkspace, @@ -80,16 +80,30 @@ export class WorkspaceRepo { */ private async bustWorkspaceCache( workspace?: Pick | undefined, + trx?: KyselyTransaction, ): Promise { - try { - await this.cacheManager.del(CacheKey.WORKSPACE_SELF_HOSTED); - if (workspace?.hostname) { - await this.cacheManager.del( - CacheKey.WORKSPACE_BY_HOST(workspace.hostname), - ); + const del = async () => { + try { + await this.cacheManager.del(CacheKey.WORKSPACE_SELF_HOSTED); + if (workspace?.hostname) { + await this.cacheManager.del( + CacheKey.WORKSPACE_BY_HOST(workspace.hostname), + ); + } + } catch { + // cache is best-effort; TTL is the backstop } - } catch { - // cache is best-effort; TTL is the backstop + }; + if (trx) { + // Inside a caller transaction the write is NOT yet committed: busting now + // opens a repopulation window (a concurrent reader reloads the cache with + // the pre-commit / stale row, which then survives until TTL). Defer the del + // to the transaction's commit (drained by the owning executeTx) (#495). + registerAfterCommit(trx, del); + } else { + // No transaction: the mutation above already auto-committed, so this del is + // already post-commit. + await del(); } } @@ -180,7 +194,7 @@ export class WorkspaceRepo { .where('id', '=', workspaceId) .returning(this.baseFields) .executeTakeFirst(); - await this.bustWorkspaceCache(workspace); + await this.bustWorkspaceCache(workspace, trx); return workspace; } @@ -195,7 +209,7 @@ export class WorkspaceRepo { .returning(this.baseFields) .executeTakeFirst(); // Bust the cached "not found" so a fresh install / new tenant is seen at once. - await this.bustWorkspaceCache(workspace); + await this.bustWorkspaceCache(workspace, trx); return workspace; } @@ -249,7 +263,7 @@ export class WorkspaceRepo { .where('id', '=', workspaceId) .returning(this.baseFields) .executeTakeFirst(); - await this.bustWorkspaceCache(workspace); + await this.bustWorkspaceCache(workspace, trx); return workspace; } @@ -271,7 +285,7 @@ export class WorkspaceRepo { .where('id', '=', workspaceId) .returning(this.baseFields) .executeTakeFirst(); - await this.bustWorkspaceCache(workspace); + await this.bustWorkspaceCache(workspace, trx); return workspace; } @@ -326,7 +340,7 @@ export class WorkspaceRepo { .where('id', '=', workspaceId) .returning(this.baseFields) .executeTakeFirst(); - await this.bustWorkspaceCache(workspace); + await this.bustWorkspaceCache(workspace, trx); return workspace; } @@ -354,7 +368,7 @@ export class WorkspaceRepo { .where('id', '=', workspaceId) .returning(this.baseFields) .executeTakeFirst(); - await this.bustWorkspaceCache(workspace); + await this.bustWorkspaceCache(workspace, trx); return workspace; } @@ -376,7 +390,7 @@ export class WorkspaceRepo { .where('id', '=', workspaceId) .returning(this.baseFields) .executeTakeFirst(); - await this.bustWorkspaceCache(workspace); + await this.bustWorkspaceCache(workspace, trx); return workspace; } @@ -398,7 +412,7 @@ export class WorkspaceRepo { .where('id', '=', workspaceId) .returning(this.baseFields) .executeTakeFirst(); - await this.bustWorkspaceCache(workspace); + await this.bustWorkspaceCache(workspace, trx); return workspace; } diff --git a/apps/server/src/database/services/migration.service.ts b/apps/server/src/database/services/migration.service.ts index 2b63f728..e70d28c4 100644 --- a/apps/server/src/database/services/migration.service.ts +++ b/apps/server/src/database/services/migration.service.ts @@ -4,6 +4,7 @@ import { promises as fs } from 'fs'; import { Migrator, FileMigrationProvider } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB } from '@docmost/db/types/kysely.types'; +import { ensureConcurrentIndexes } from '@docmost/db/concurrent-indexes'; @Injectable() export class MigrationService { @@ -12,6 +13,16 @@ export class MigrationService { constructor(@InjectKysely() private readonly db: KyselyDB) {} async migrateToLatest(): Promise { + // Build write-blocking trigram indexes CONCURRENTLY (no transaction) BEFORE + // the migrator runs, so the corresponding in-migration `CREATE INDEX IF NOT + // EXISTS` no-ops instead of taking a SHARE lock on `pages` during deploy + // (#495). Best-effort: on a fresh DB (no `pages`/`f_unaccent` yet) this is a + // no-op and the migrations build the index normally. + await ensureConcurrentIndexes(this.db, (message, error) => { + if (error) this.logger.warn(`${message}: ${String(error)}`); + else this.logger.log(message); + }); + const migrator = new Migrator({ db: this.db, provider: new FileMigrationProvider({ diff --git a/apps/server/src/database/types/db.d.ts b/apps/server/src/database/types/db.d.ts index bf5a9663..a17d628c 100644 --- a/apps/server/src/database/types/db.d.ts +++ b/apps/server/src/database/types/db.d.ts @@ -606,6 +606,9 @@ export interface AiChats { // The document the chat was created in (open page at first message). NULL => // started outside any document. ON DELETE SET NULL on the page FK. pageId: string | null; + // Chat-level metadata bag (#490). jsonb, defaulted to '{}'. First key: + // `activatedTools` — the deferred-tool activation set persisted across turns. + metadata: Generated; createdAt: Generated; updatedAt: Generated; deletedAt: Timestamp | null; diff --git a/apps/server/src/database/utils.ts b/apps/server/src/database/utils.ts index 84c3693b..c78596b9 100644 --- a/apps/server/src/database/utils.ts +++ b/apps/server/src/database/utils.ts @@ -6,16 +6,76 @@ import { KyselyDB, KyselyTransaction } from './types/kysely.types'; * If an existing transaction is provided, it directly executes the callback with it. * Otherwise, it starts a new transaction using the provided database instance and executes the callback within that transaction. */ +/** + * Post-commit side-effect hooks, keyed by the transaction they were registered + * against. A WeakMap so an abandoned/never-drained transaction's entry is GC'd + * with the trx object (no leak). Used by {@link registerAfterCommit} / + * {@link executeTx}. + */ +const afterCommitHooks = new WeakMap< + KyselyTransaction, + Array<() => Promise | void> +>(); + +/** + * Register a side effect to run ONLY AFTER the transaction that owns `trx` + * commits. THE fix for "bust the cache inside the open transaction" bugs: a + * cache-invalidation (or any read-your-write-visible side effect) done while the + * writing transaction is still open opens a window where a concurrent reader + * repopulates the cache with the PRE-COMMIT (stale) row, so after commit the + * cache holds the old value until its TTL. Deferring the effect to post-commit + * closes that window. + * + * The hook is drained by the OUTERMOST {@link executeTx} that actually owns + * (created) this transaction — so registering against a passed-through + * `existingTrx` still fires at the real commit boundary, not at the inner call. + * NOTE: a hook registered against a transaction that was NOT created via + * `executeTx` (untracked) will never be drained — always create transactions + * through `executeTx` when you rely on post-commit hooks. + */ +export function registerAfterCommit( + trx: KyselyTransaction, + hook: () => Promise | void, +): void { + const existing = afterCommitHooks.get(trx); + if (existing) existing.push(hook); + else afterCommitHooks.set(trx, [hook]); +} + export async function executeTx( db: KyselyDB, callback: (trx: KyselyTransaction) => Promise, existingTrx?: KyselyTransaction, ): Promise { if (existingTrx) { - return await callback(existingTrx); // Execute callback with existing transaction - } else { - return await db.transaction().execute((trx) => callback(trx)); // Start new transaction and execute callback + // Reuse the caller's transaction. Any post-commit hooks registered here are + // drained by the OUTER executeTx that created `existingTrx`, at the true + // commit boundary — so we must NOT drain them now. + return await callback(existingTrx); } + // We OWN this transaction: run the body, then (only once it has COMMITTED) + // drain the post-commit hooks registered against it during the body. + let ownTrx: KyselyTransaction | undefined; + const result = await db.transaction().execute((trx) => { + ownTrx = trx; + return callback(trx); + }); + if (ownTrx) { + const hooks = afterCommitHooks.get(ownTrx); + if (hooks) { + afterCommitHooks.delete(ownTrx); + for (const hook of hooks) { + // Best-effort: a failed side effect (e.g. a cache del) must not fail the + // already-committed transaction. + try { + await hook(); + } catch { + // swallow — the durable write already committed + } + } + } + } + return result; } /* @@ -78,18 +138,30 @@ export function violatedConstraint(err: unknown): string | undefined { * verbatim); `::jsonb` then parses it into a real array/object. Read-side * parsers repair rows written the old buggy way without a migration. * - * Returns `null` for null/undefined and for "empty" values (an empty array, or - * an object with no own enumerable keys) — callers treat empty as "clear/unset", - * so an empty allowlist/config never round-trips as `[]`/`{}`. + * Returns `null` for null/undefined. By default it ALSO returns `null` for + * "empty" values (an empty array, or an object with no own enumerable keys) — + * most callers treat empty as "clear/unset", so an empty config never + * round-trips as `[]`/`{}`. + * + * `preserveEmpty` (issue #476) opts a column OUT of that empty-to-null + * normalization so `[]`/`{}` are persisted as real jsonb values. Needed where + * empty and null mean DIFFERENT things: an empty `tool_allowlist` is + * deny-all ("zero tools allowed"), while null is "no restriction" — collapsing + * `[]` to null silently widened deny-all to allow-all. Deliberately an opt-in + * flag, NOT a global change: the other jsonb callers (model_config, source) + * keep the empty-means-unset contract. */ export function jsonbBind( value: T | null | undefined, + opts?: { preserveEmpty?: boolean }, ): RawBuilder | null { if (value === null || value === undefined) return null; - if (Array.isArray(value)) { - if (value.length === 0) return null; - } else if (typeof value === 'object') { - if (Object.keys(value as object).length === 0) return null; + if (!opts?.preserveEmpty) { + if (Array.isArray(value)) { + if (value.length === 0) return null; + } else if (typeof value === 'object') { + if (Object.keys(value as object).length === 0) return null; + } } return sql`${JSON.stringify(value)}::text::jsonb`; } diff --git a/apps/server/src/integrations/ai/ai-settings.service.spec.ts b/apps/server/src/integrations/ai/ai-settings.service.spec.ts index ba263a3e..b4d03106 100644 --- a/apps/server/src/integrations/ai/ai-settings.service.spec.ts +++ b/apps/server/src/integrations/ai/ai-settings.service.spec.ts @@ -99,10 +99,12 @@ describe('AiSettingsService.getMasked reindex progress', () => { // actually pins the progress.total branch rather than coincidentally // matching the DB fallback. With fix #1 the two sources agree in practice, // but getMasked must still return progress.total when a record is active. + const startedAt = Date.now(); reindexProgress.get.mockResolvedValue({ total: 500, done: 120, - startedAt: Date.now(), + startedAt, + runId: 'run-abc', }); const masked = await service.getMasked(WORKSPACE_ID); @@ -110,6 +112,10 @@ describe('AiSettingsService.getMasked reindex progress', () => { expect(masked.indexedPages).toBe(120); // progress.done, not DB 478 expect(masked.totalPages).toBe(500); // progress.total, not DB 478 expect(masked.reindexing).toBe(true); + // The status payload must carry the run identity so the client can key its + // poll on it (a changed runId => a NEW run). + expect(masked.runId).toBe('run-abc'); + expect(masked.reindexStartedAt).toBe(startedAt); }); it('falls back to countIndexedPages when no reindex is active', async () => { @@ -121,6 +127,10 @@ describe('AiSettingsService.getMasked reindex progress', () => { expect(masked.indexedPages).toBe(478); expect(masked.totalPages).toBe(478); expect(masked.reindexing).toBe(false); + // No active run -> no run identity surfaced (the client keeps its prior + // steady-state behaviour). + expect(masked.runId).toBeUndefined(); + expect(masked.reindexStartedAt).toBeUndefined(); }); }); diff --git a/apps/server/src/integrations/ai/ai-settings.service.ts b/apps/server/src/integrations/ai/ai-settings.service.ts index 6be0b5be..e9bb0ec7 100644 --- a/apps/server/src/integrations/ai/ai-settings.service.ts +++ b/apps/server/src/integrations/ai/ai-settings.service.ts @@ -245,6 +245,9 @@ export class AiSettingsService { // Max context window for the chat header badge denominator. Stored as // ::text; 0/unset/invalid = no limit (undefined). chatContextWindow: parsePositiveInt(provider.chatContextWindow), + // RAW stored value (#490): the replay budgeter reads this to distinguish an + // explicit `0` (off-switch) from unset, which parsePositiveInt cannot. + chatContextWindowRaw: provider.chatContextWindow, // Plain passthrough; getChatModel defaults unset to 'openai-compatible'. chatApiStyle: provider.chatApiStyle, // Cheap model id for the anonymous public-share assistant; reuses the chat @@ -371,6 +374,12 @@ export class AiSettingsService { totalPages, // Optional hint for the client: a reindex run is currently in progress. reindexing: progress != null, + // Per-run identity so the client can key its poll on a stable run id and + // reset its per-run state when a NEW run starts. Present only while a run + // is active; `runId` may be '' for a legacy/degraded record (the client + // treats that as "no identity"). + runId: progress?.runId, + reindexStartedAt: progress?.startedAt, }; } diff --git a/apps/server/src/integrations/ai/ai.service.provider-fetch.spec.ts b/apps/server/src/integrations/ai/ai.service.provider-fetch.spec.ts new file mode 100644 index 00000000..558aca5b --- /dev/null +++ b/apps/server/src/integrations/ai/ai.service.provider-fetch.spec.ts @@ -0,0 +1,86 @@ +// `.provider` alone cannot prove the gemini/ollama chat factories were built +// with the instrumented streaming fetch — a regression dropping it (which drops +// them back to the global undici fetch: no keep-alive recycle, no reset retries, +// unbounded silence timeout; incident classes #140/#175/#310) would still pass. +// So mock the factories and assert the exact fetch argument. jest.mock is +// module-scoped, hence a dedicated file. + +const mockGeminiModel = { provider: 'google.generative-ai', modelId: 'm' }; +const mockOllamaModel = { provider: 'ollama.chat', modelId: 'm' }; + +// jest allows `mock`-prefixed vars inside a jest.mock factory. +const mockCreateGoogle = jest.fn((_settings: unknown) => () => mockGeminiModel); +const mockCreateOllama = jest.fn((_settings: unknown) => () => mockOllamaModel); + +jest.mock('@ai-sdk/google', () => ({ + createGoogleGenerativeAI: (settings: unknown) => mockCreateGoogle(settings), +})); +jest.mock('ai-sdk-ollama', () => ({ + createOllama: (settings: unknown) => mockCreateOllama(settings), +})); + +import { AiService } from './ai.service'; + +describe('AiService.getChatModel provider transport fetch (gemini/ollama)', () => { + function serviceWith(cfg: Record) { + const aiSettings = { + resolve: jest.fn().mockResolvedValue(cfg), + }; + return new AiService( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + aiSettings as any, + { find: jest.fn() } as never, + { decryptSecret: jest.fn() } as never, + ); + } + + beforeEach(() => { + mockCreateGoogle.mockClear(); + mockCreateOllama.mockClear(); + }); + + it('builds the gemini chat model with the instrumented streaming fetch', async () => { + await serviceWith({ + driver: 'gemini', + chatModel: 'gemini-2.5-pro', + apiKey: 'the-key', + }).getChatModel('ws-1'); + expect(mockCreateGoogle).toHaveBeenCalledTimes(1); + expect(mockCreateGoogle).toHaveBeenCalledWith( + expect.objectContaining({ + apiKey: 'the-key', + fetch: expect.any(Function), + }), + ); + }); + + it('builds the ollama chat model with the instrumented streaming fetch', async () => { + await serviceWith({ + driver: 'ollama', + chatModel: 'llama3', + baseUrl: 'http://localhost:11434/api', + }).getChatModel('ws-1'); + expect(mockCreateOllama).toHaveBeenCalledTimes(1); + expect(mockCreateOllama).toHaveBeenCalledWith( + expect.objectContaining({ + baseURL: 'http://localhost:11434/api', + fetch: expect.any(Function), + }), + ); + }); + + it('reuses ONE service-lifetime fetch instance across both providers', async () => { + const svc = serviceWith({ + driver: 'gemini', + chatModel: 'gemini-2.5-pro', + apiKey: 'k', + }); + await svc.getChatModel('ws-1'); + const geminiFetch = mockCreateGoogle.mock.calls[0][0] as { fetch: unknown }; + // Same instance on a second call — the fetch is held for the service + // lifetime to reuse the streaming dispatcher's connection pool. + await svc.getChatModel('ws-1'); + const geminiFetch2 = mockCreateGoogle.mock.calls[1][0] as { fetch: unknown }; + expect(geminiFetch.fetch).toBe(geminiFetch2.fetch); + }); +}); diff --git a/apps/server/src/integrations/ai/ai.service.ts b/apps/server/src/integrations/ai/ai.service.ts index 16aa6997..2f999e72 100644 --- a/apps/server/src/integrations/ai/ai.service.ts +++ b/apps/server/src/integrations/ai/ai.service.ts @@ -190,10 +190,22 @@ export class AiService { }).chat(chatModel); } case 'gemini': - return createGoogleGenerativeAI({ apiKey })(chatModel); + // Route gemini through the same instrumented streaming fetch as openai + // (finite silence timeouts + keep-alive recycling + pre-response + // connection-reset retry). Without it the provider ran on the global + // undici fetch — no keep-alive recycle, no reset retries, default + // (unbounded silence) timeout — so incident classes #140/#175/#310 were + // reproducible for gemini too. + return createGoogleGenerativeAI({ + apiKey, + fetch: this.aiProviderFetch, + })(chatModel); case 'ollama': - // Ollama needs no API key. - return createOllama({ baseURL: baseUrl })(chatModel); + // Ollama needs no API key. Same transport hardening as above (#140/#175/#310). + return createOllama({ + baseURL: baseUrl, + fetch: this.aiProviderFetch, + })(chatModel); default: throw new AiNotConfiguredException(); } diff --git a/apps/server/src/integrations/ai/ai.types.ts b/apps/server/src/integrations/ai/ai.types.ts index 06bf83e3..50f38312 100644 --- a/apps/server/src/integrations/ai/ai.types.ts +++ b/apps/server/src/integrations/ai/ai.types.ts @@ -105,6 +105,10 @@ export interface ResolvedAiConfig extends Partial { // Max context window in tokens; surfaced to the chat header badge as the // "current / max" denominator. 0/unset = no limit. chatContextWindow?: number; + // RAW stored context window (::text), BEFORE parsePositiveInt collapses `0` and + // unset to `undefined`. The #490 replay budgeter needs the raw value to honor an + // explicit `0` off-switch distinctly from "unset -> flat default". + chatContextWindowRaw?: string | number; // Cheap model id for the public-share assistant; reuses the chat creds. publicShareChatModel?: string; // Agent-role id whose persona the public-share assistant adopts (empty/unset @@ -149,4 +153,14 @@ export interface MaskedAiSettings { // True while a full workspace reindex is actively running (the counts above // then reflect the live run progress rather than the steady-state DB count). reindexing?: boolean; + // Identity of the ACTIVE reindex run (present only while `reindexing`). The + // client keys its poll on `runId`: a changed value means a NEW run (reset the + // per-run poll state it latched), the same value means the run it is already + // watching — removing the "same run or a fresh one?" ambiguity a stale + // pre-reindex snapshot otherwise causes. Absent/empty degrades gracefully. + runId?: string; + // Epoch-ms the active run started (present only while `reindexing`). Paired + // with `runId` so a run that restarts with the same (recycled) id is still + // seen as new. + reindexStartedAt?: number; } diff --git a/apps/server/src/integrations/ai/embedding-reindex-progress.service.spec.ts b/apps/server/src/integrations/ai/embedding-reindex-progress.service.spec.ts index 496d55e8..5be96e69 100644 --- a/apps/server/src/integrations/ai/embedding-reindex-progress.service.spec.ts +++ b/apps/server/src/integrations/ai/embedding-reindex-progress.service.spec.ts @@ -48,19 +48,38 @@ describe('EmbeddingReindexProgressService', () => { } describe('get', () => { - it('maps a valid hash to a ReindexProgress object', async () => { + it('maps a valid hash to a ReindexProgress object (incl. the run identity)', async () => { const { redis, hgetall } = makeRedis(); - hgetall.mockResolvedValue({ total: '478', done: '120', startedAt: '1000' }); + hgetall.mockResolvedValue({ + total: '478', + done: '120', + startedAt: '1000', + runId: 'run-xyz', + }); const service = makeService(redis); await expect(service.get(WORKSPACE_ID)).resolves.toEqual({ total: 478, done: 120, startedAt: 1000, + runId: 'run-xyz', }); expect(hgetall).toHaveBeenCalledWith(KEY); }); + it('degrades a missing runId to an empty string (legacy/partial record)', async () => { + const { redis, hgetall } = makeRedis(); + // A record written before runId existed: get() must still succeed and + // report runId='' so the client treats it as "no identity", never breaks. + hgetall.mockResolvedValue({ total: '10', done: '3', startedAt: '5' }); + await expect(makeService(redis).get(WORKSPACE_ID)).resolves.toEqual({ + total: 10, + done: 3, + startedAt: 5, + runId: '', + }); + }); + it('returns null for an empty hash (no record)', async () => { const { redis, hgetall } = makeRedis(); hgetall.mockResolvedValue({}); @@ -87,11 +106,17 @@ describe('EmbeddingReindexProgressService', () => { it('coerces a non-finite startedAt to 0', async () => { const { redis, hgetall } = makeRedis(); - hgetall.mockResolvedValue({ total: '10', done: '2', startedAt: 'nope' }); + hgetall.mockResolvedValue({ + total: '10', + done: '2', + startedAt: 'nope', + runId: 'run-1', + }); await expect(makeService(redis).get(WORKSPACE_ID)).resolves.toEqual({ total: 10, done: 2, startedAt: 0, + runId: 'run-1', }); }); @@ -115,6 +140,21 @@ describe('EmbeddingReindexProgressService', () => { expect(multiObj.exec).toHaveBeenCalledTimes(1); }); + it('mints a fresh non-empty runId into the record on each start', async () => { + const { redis, multiObj } = makeRedis(); + const service = makeService(redis); + await service.start(WORKSPACE_ID, 1); + await service.start(WORKSPACE_ID, 1); + + const firstRunId = multiObj.hset.mock.calls[0][1].runId; + const secondRunId = multiObj.hset.mock.calls[1][1].runId; + expect(typeof firstRunId).toBe('string'); + expect(firstRunId).not.toBe(''); + // Each run gets its OWN identity so the client can tell a re-trigger apart + // from the run it is already watching. + expect(secondRunId).not.toBe(firstRunId); + }); + it('defaults the expire TTL to the full 1h record TTL', async () => { const { redis, multiObj } = makeRedis(); await makeService(redis).start(WORKSPACE_ID, 478); diff --git a/apps/server/src/integrations/ai/embedding-reindex-progress.service.ts b/apps/server/src/integrations/ai/embedding-reindex-progress.service.ts index 2f551f0d..22bc5424 100644 --- a/apps/server/src/integrations/ai/embedding-reindex-progress.service.ts +++ b/apps/server/src/integrations/ai/embedding-reindex-progress.service.ts @@ -1,17 +1,28 @@ import { Injectable, Logger } from '@nestjs/common'; import { RedisService } from '@nestjs-labs/nestjs-ioredis'; +import { randomUUID } from 'node:crypto'; import type { Redis } from 'ioredis'; /** * Live progress of an in-flight workspace embeddings reindex run. * `total` is the number of pages the run will process, `done` how many it has * already processed (success OR handled failure), `startedAt` the epoch-ms the - * record was created. + * record was created, and `runId` a per-run identity minted at `start()`. + * + * `runId` gives each reindex run a stable identity so a poller can tell "same + * run I've been watching" from "a NEW run started" WITHOUT guessing from the + * progress counters (the ambiguity that a stale pre-reindex snapshot vs a fresh + * run otherwise causes — the bug class fixed twice under #262). It is best- + * effort like the rest of this record: a record written before this field + * existed (or a Redis hiccup) yields an empty `runId`, which the client must + * treat as "no identity available" and degrade to its prior behaviour, never + * break. */ export interface ReindexProgress { total: number; done: number; startedAt: number; + runId: string; } /** Redis key namespace for the per-workspace reindex-progress record. */ @@ -86,12 +97,18 @@ export class EmbeddingReindexProgressService { ): Promise { const key = this.key(workspaceId); try { + // A fresh identity per run so the client poll can key on it: a changed + // runId means a genuinely NEW run (reset any latched per-run poll state), + // the same runId means the run the client is already watching. Best-effort + // like the counters — never surfaced to the user, only used to disambiguate. + const runId = randomUUID(); await this.redis .multi() .hset(key, { total: String(total), done: '0', startedAt: String(Date.now()), + runId, }) .expire(key, ttlSeconds) .exec(); @@ -150,7 +167,15 @@ export class EmbeddingReindexProgressService { const done = Number(data.done); const startedAt = Number(data.startedAt); if (!Number.isFinite(total) || !Number.isFinite(done)) return null; - return { total, done, startedAt: Number.isFinite(startedAt) ? startedAt : 0 }; + // `runId` degrades gracefully: a pre-existing record (written before this + // field) or a stripped value reads as '' — the client treats that as "no + // identity" and keeps its prior behaviour rather than breaking the poll. + return { + total, + done, + startedAt: Number.isFinite(startedAt) ? startedAt : 0, + runId: typeof data.runId === 'string' ? data.runId : '', + }; } catch (err) { this.logger.warn( `reindex-progress read failed for workspace ${workspaceId}; ` + diff --git a/apps/server/src/integrations/audit/audit.module.ts b/apps/server/src/integrations/audit/audit.module.ts index eafffdd7..07ef04d3 100644 --- a/apps/server/src/integrations/audit/audit.module.ts +++ b/apps/server/src/integrations/audit/audit.module.ts @@ -1,14 +1,19 @@ import { Global, Module } from '@nestjs/common'; -import { AUDIT_SERVICE, NoopAuditService } from './audit.service'; +import { AUDIT_SERVICE } from './audit.service'; +import { DatabaseAuditService } from './database-audit.service'; +// #496: bind the audit token to a real DB-backed trail (was NoopAuditService, +// which silently dropped every event). Kysely (@Global DatabaseModule) and +// ClsService (@Global ClsModule) are both globally available, so this module +// needs no extra imports. @Global() @Module({ providers: [ { provide: AUDIT_SERVICE, - useClass: NoopAuditService, + useClass: DatabaseAuditService, }, ], exports: [AUDIT_SERVICE], }) -export class NoopAuditModule {} +export class AuditModule {} diff --git a/apps/server/src/integrations/audit/database-audit.service.spec.ts b/apps/server/src/integrations/audit/database-audit.service.spec.ts new file mode 100644 index 00000000..ba586166 --- /dev/null +++ b/apps/server/src/integrations/audit/database-audit.service.spec.ts @@ -0,0 +1,198 @@ +import { Logger } from '@nestjs/common'; +import { DatabaseAuditService } from './database-audit.service'; +import { AUDIT_CONTEXT_KEY } from '../../common/middlewares/audit-context.middleware'; +import { AuditEvent, AuditResource } from '../../common/events/audit-events'; + +/** + * Observable-property coverage for the DB-backed audit trail (#496): every + * assertion pins what actually reaches the `audit` table (or that nothing does), + * driven through a chainable Kysely mock that captures the inserted rows. + */ +describe('DatabaseAuditService', () => { + function makeService(clsContext: any) { + const inserted: any[] = []; + const updated: any[] = []; + let failNextInsert = false; + + const db: any = { + insertInto: jest.fn(() => ({ + values: jest.fn((rows: any) => ({ + execute: jest.fn(async () => { + if (failNextInsert) { + failNextInsert = false; + throw new Error('boom'); + } + inserted.push(rows); + }), + })), + })), + updateTable: jest.fn(() => ({ + set: jest.fn((patch: any) => ({ + where: jest.fn(() => ({ + execute: jest.fn(async () => { + updated.push(patch); + }), + })), + })), + })), + }; + + const store: Record = { [AUDIT_CONTEXT_KEY]: clsContext }; + const cls: any = { + get: jest.fn((key: string) => store[key]), + set: jest.fn((key: string, val: any) => { + store[key] = val; + }), + }; + + const service = new DatabaseAuditService(db, cls); + return { + service, + inserted, + updated, + cls, + store, + failInsert: () => { + failNextInsert = true; + }, + }; + } + + const applyPayload = () => ({ + event: AuditEvent.COMMENT_SUGGESTION_APPLIED, + resourceType: AuditResource.COMMENT, + resourceId: 'c-1', + spaceId: 'space-1', + metadata: { pageId: 'p-1', suggestedText: 'new', decidedBy: 'u-2' }, + }); + + const ctx = (over?: any) => ({ + workspaceId: 'ws-1', + actorId: 'u-2', + actorType: 'user', + ipAddress: '10.0.0.1', + userAgent: 'jest', + ...over, + }); + + it('log() persists a row with the CLS context merged onto the payload', async () => { + const { service, inserted } = makeService(ctx()); + service.log(applyPayload()); + // log() is fire-and-forget; flush the microtask queue. + await Promise.resolve(); + await Promise.resolve(); + + expect(inserted).toHaveLength(1); + expect(inserted[0]).toMatchObject({ + workspaceId: 'ws-1', + actorId: 'u-2', + actorType: 'user', + event: AuditEvent.COMMENT_SUGGESTION_APPLIED, + resourceType: AuditResource.COMMENT, + resourceId: 'c-1', + spaceId: 'space-1', + ipAddress: '10.0.0.1', + metadata: { pageId: 'p-1', suggestedText: 'new', decidedBy: 'u-2' }, + }); + }); + + it('log() is a no-op when there is no workspace in scope', async () => { + const { service, inserted } = makeService(ctx({ workspaceId: null })); + service.log(applyPayload()); + await Promise.resolve(); + expect(inserted).toHaveLength(0); + }); + + it('log() drops EXCLUDED_AUDIT_EVENTS (e.g. comment.created)', async () => { + const { service, inserted } = makeService(ctx()); + service.log({ + event: AuditEvent.COMMENT_CREATED, + resourceType: AuditResource.COMMENT, + resourceId: 'c-1', + spaceId: 'space-1', + }); + await Promise.resolve(); + await Promise.resolve(); + expect(inserted).toHaveLength(0); + }); + + it('a failed insert is swallowed with a warn and floats no rejection (audit is a side-record)', async () => { + // This pins the load-bearing swallow inside persist(). Because log() is + // fire-and-forget (`void this.persist(...)`), it always returns synchronously + // without throwing — so `not.toThrow()` alone would stay green even if the + // try/catch were removed. We instead observe the two effects the catch is + // responsible for: a warn IS emitted, and NO unhandled rejection floats. + // Removing persist()'s try/catch reddens both assertions (warn count 0 + a + // captured rejection). + const warnSpy = jest + .spyOn(Logger.prototype, 'warn') + .mockImplementation(() => undefined as any); + const rejections: unknown[] = []; + const onRejection = (err: unknown) => rejections.push(err); + process.on('unhandledRejection', onRejection); + try { + const { service, failInsert } = makeService(ctx()); + failInsert(); + expect(() => service.log(applyPayload())).not.toThrow(); + // Flush microtasks so the rejected insert settles, then give any floated + // rejection a macrotask tick to be reported by the runtime. + await Promise.resolve(); + await Promise.resolve(); + await new Promise((r) => setImmediate(r)); + + expect(warnSpy).toHaveBeenCalledTimes(1); + expect(String(warnSpy.mock.calls[0][0])).toContain( + 'Failed to persist audit event', + ); + expect(rejections).toHaveLength(0); + } finally { + process.off('unhandledRejection', onRejection); + warnSpy.mockRestore(); + } + }); + + it('logWithContext() persists with an explicit (non-request) context', async () => { + const { service, inserted } = makeService(undefined); + service.logWithContext(applyPayload(), ctx({ actorType: 'system' }) as any); + await Promise.resolve(); + await Promise.resolve(); + expect(inserted).toHaveLength(1); + expect(inserted[0].actorType).toBe('system'); + expect(inserted[0].workspaceId).toBe('ws-1'); + }); + + it('logBatchWithContext() inserts only non-excluded events', async () => { + const { service, inserted } = makeService(undefined); + service.logBatchWithContext( + [ + applyPayload(), + { + event: AuditEvent.COMMENT_CREATED, + resourceType: AuditResource.COMMENT, + resourceId: 'c-2', + }, + ], + ctx() as any, + ); + await Promise.resolve(); + await Promise.resolve(); + // Batch is a single insert call carrying only the applied event. + expect(inserted).toHaveLength(1); + expect(inserted[0]).toHaveLength(1); + expect(inserted[0][0].event).toBe(AuditEvent.COMMENT_SUGGESTION_APPLIED); + }); + + it('setActorId / setActorType mutate the ambient CLS context', () => { + const { service, store } = makeService(ctx({ actorId: null })); + service.setActorId('u-9'); + service.setActorType('api_key'); + expect(store[AUDIT_CONTEXT_KEY].actorId).toBe('u-9'); + expect(store[AUDIT_CONTEXT_KEY].actorType).toBe('api_key'); + }); + + it('updateRetention() writes the workspace retention window', async () => { + const { service, updated } = makeService(ctx()); + await service.updateRetention('ws-1', 30); + expect(updated).toEqual([{ auditRetentionDays: 30 }]); + }); +}); diff --git a/apps/server/src/integrations/audit/database-audit.service.ts b/apps/server/src/integrations/audit/database-audit.service.ts new file mode 100644 index 00000000..cfa3f321 --- /dev/null +++ b/apps/server/src/integrations/audit/database-audit.service.ts @@ -0,0 +1,160 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectKysely } from 'nestjs-kysely'; +import { ClsService } from 'nestjs-cls'; +import { KyselyDB } from '@docmost/db/types/kysely.types'; +import { + AuditContext, + AUDIT_CONTEXT_KEY, +} from '../../common/middlewares/audit-context.middleware'; +import { + AuditLogPayload, + ActorType, + EXCLUDED_AUDIT_EVENTS, +} from '../../common/events/audit-events'; +import { AuditLogContext, IAuditService } from './audit.service'; + +/** + * Minimal DB-backed audit trail (#496). Replaces NoopAuditService so that + * decision-bearing events — notably comment.suggestion_applied / + * comment.suggestion_dismissed, whose subject comment is HARD-DELETED on the + * childless path — leave a durable record of who decided what. Without this the + * events were emitted (comment.service / *.controller) but swallowed, so an + * applied/dismissed suggestion was unrecoverable once the row was gone. + * + * Rows land in the pre-existing `audit` table (migration 20260228T223532). The + * per-request actor/workspace/ip come from the CLS AuditContext populated by + * AuditContextMiddleware + AuditActorInterceptor; callers that run OUTSIDE a + * request (queue workers, imports) pass an explicit context via + * logWithContext / logBatchWithContext. + * + * Audit is a side-record: a write failure MUST NOT break the originating + * request, so every persistence path swallows its error with a warn. Events in + * EXCLUDED_AUDIT_EVENTS (high-volume/low-signal) are dropped. + */ +@Injectable() +export class DatabaseAuditService implements IAuditService { + private readonly logger = new Logger(DatabaseAuditService.name); + + constructor( + @InjectKysely() private readonly db: KyselyDB, + private readonly cls: ClsService, + ) {} + + /** + * Persist a single event using the ambient request-scoped AuditContext. A + * no-op when there is no workspace in scope (the table's workspace_id is NOT + * NULL) or the event is excluded. Fire-and-forget: the returned promise is not + * awaited by hot callers, and its rejection is swallowed here. + */ + log(payload: AuditLogPayload): void { + const context = this.cls?.get(AUDIT_CONTEXT_KEY); + if (!context?.workspaceId) { + // No workspace in scope — nothing we can attribute the row to. This is + // expected for events emitted outside an HTTP request; those callers must + // use logWithContext instead. + return; + } + void this.persist(payload, { + workspaceId: context.workspaceId, + actorId: context.actorId ?? undefined, + actorType: context.actorType, + ipAddress: context.ipAddress ?? undefined, + userAgent: context.userAgent ?? undefined, + }); + } + + /** Persist a single event with an explicit (non-request) context. */ + logWithContext(payload: AuditLogPayload, context: AuditLogContext): void { + if (!context?.workspaceId) return; + void this.persist(payload, context); + } + + /** Persist a batch of events sharing one explicit context (imports). */ + logBatchWithContext( + payloads: AuditLogPayload[], + context: AuditLogContext, + ): void { + if (!context?.workspaceId || payloads.length === 0) return; + const rows = payloads + .filter((p) => !EXCLUDED_AUDIT_EVENTS.has(p.event)) + .map((p) => this.toRow(p, context)); + if (rows.length === 0) return; + this.db + .insertInto('audit') + .values(rows) + .execute() + .catch((err: any) => + this.logger.warn(`Failed to persist ${rows.length} audit events: ${err?.message}`), + ); + } + + /** Update the ambient request actor (e.g. after login resolves the user). */ + setActorId(actorId: string): void { + const context = this.cls?.get(AUDIT_CONTEXT_KEY); + if (context) { + context.actorId = actorId; + this.cls.set(AUDIT_CONTEXT_KEY, context); + } + } + + /** Update the ambient request actor type (user | system | api_key). */ + setActorType(actorType: ActorType): void { + const context = this.cls?.get(AUDIT_CONTEXT_KEY); + if (context) { + context.actorType = actorType; + this.cls.set(AUDIT_CONTEXT_KEY, context); + } + } + + /** Persist a workspace's audit-log retention window (days). */ + async updateRetention( + workspaceId: string, + retentionDays: number, + ): Promise { + try { + await this.db + .updateTable('workspaces') + .set({ auditRetentionDays: retentionDays }) + .where('id', '=', workspaceId) + .execute(); + } catch (err: any) { + this.logger.warn( + `Failed to update audit retention for workspace ${workspaceId}: ${err?.message}`, + ); + } + } + + private async persist( + payload: AuditLogPayload, + context: AuditLogContext, + ): Promise { + if (EXCLUDED_AUDIT_EVENTS.has(payload.event)) return; + try { + await this.db + .insertInto('audit') + .values(this.toRow(payload, context)) + .execute(); + } catch (err: any) { + // Audit is a side-record; never let a failed write surface to the caller. + this.logger.warn( + `Failed to persist audit event ${payload.event}: ${err?.message}`, + ); + } + } + + private toRow(payload: AuditLogPayload, context: AuditLogContext) { + return { + workspaceId: context.workspaceId, + actorId: context.actorId ?? null, + actorType: context.actorType ?? 'user', + event: payload.event, + resourceType: payload.resourceType, + resourceId: payload.resourceId ?? null, + spaceId: payload.spaceId ?? null, + // jsonb columns: node-postgres serializes plain objects to JSON. + changes: payload.changes ? (payload.changes as any) : null, + metadata: payload.metadata ? (payload.metadata as any) : null, + ipAddress: context.ipAddress ?? null, + }; + } +} diff --git a/apps/server/src/integrations/import/services/file-import-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts index a5115c43..ab1debab 100644 --- a/apps/server/src/integrations/import/services/file-import-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -22,10 +22,12 @@ import { v7 } from 'uuid'; import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; import { FileTask, InsertablePage } from '@docmost/db/types/entity.types'; import { canonicalizeFootnotes } from '@docmost/editor-ext'; -import { markdownToProseMirror } from '@docmost/prosemirror-markdown'; +import { + markdownToProseMirror, + normalizeForeignMarkdown, +} from '@docmost/prosemirror-markdown'; import { getProsemirrorContent } from '../../../common/helpers/prosemirror/utils'; import { formatImportHtml } from '../utils/import-formatter'; -import { normalizeForeignMarkdown } from '../utils/foreign-markdown'; import { buildAttachmentCandidates, collectMarkdownAndHtmlFiles, diff --git a/apps/server/src/integrations/import/services/import.service.ts b/apps/server/src/integrations/import/services/import.service.ts index dd86d71e..7c480ef2 100644 --- a/apps/server/src/integrations/import/services/import.service.ts +++ b/apps/server/src/integrations/import/services/import.service.ts @@ -18,8 +18,10 @@ import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; import { TiptapTransformer } from '@hocuspocus/transformer'; import * as Y from 'yjs'; import { canonicalizeFootnotes } from '@docmost/editor-ext'; -import { markdownToProseMirror } from '@docmost/prosemirror-markdown'; -import { normalizeForeignMarkdown } from '../utils/foreign-markdown'; +import { + markdownToProseMirror, + normalizeForeignMarkdown, +} from '@docmost/prosemirror-markdown'; import { FileTaskStatus, FileTaskType, diff --git a/apps/server/src/integrations/queue/processors/general-queue.processor.comment-mark.spec.ts b/apps/server/src/integrations/queue/processors/general-queue.processor.comment-mark.spec.ts index 7adf30ed..29c958c7 100644 --- a/apps/server/src/integrations/queue/processors/general-queue.processor.comment-mark.spec.ts +++ b/apps/server/src/integrations/queue/processors/general-queue.processor.comment-mark.spec.ts @@ -139,13 +139,19 @@ describe('GeneralQueueProcessor — COMMENT_MARK_UPDATE (#399)', () => { ); }); - it('skips (no throw) when the comment row has vanished', async () => { + it('reconcile (#496): comment row vanished → strips the orphan anchor mark', async () => { const { proc, collaborationGateway, commentRepo } = makeProc(); commentRepo.findById.mockResolvedValue(undefined); await expect( proc.process(job({ ...base, action: 'resolve', ts: 1000 })), ).resolves.toBeUndefined(); - expect(collaborationGateway.handleYjsEvent).not.toHaveBeenCalled(); + // A resolve/unresolve mark job whose comment row is gone leaves a silent + // orphan; the worker self-heals by stripping the anchor instead of returning. + expect(collaborationGateway.handleYjsEvent).toHaveBeenCalledWith( + 'deleteCommentMark', + 'page.page-1', + { commentId: 'c-1', user: { id: 'user-1' } }, + ); }); }); diff --git a/apps/server/src/integrations/queue/processors/general-queue.processor.ts b/apps/server/src/integrations/queue/processors/general-queue.processor.ts index e75cd8c7..d2dee503 100644 --- a/apps/server/src/integrations/queue/processors/general-queue.processor.ts +++ b/apps/server/src/integrations/queue/processors/general-queue.processor.ts @@ -95,7 +95,8 @@ export class GeneralQueueProcessor * #399: apply a comment's inline-mark mirror in the collab Y.Doc, off the HTTP * critical path. Runs the SAME gateway path the synchronous comment.service * code used (byte-identical mark op): - * - resolve / unresolve → resolveCommentMark (flip the `resolved` attribute); + * - resolve / unresolve → resolveCommentMark (flip the `resolved` attribute), + * OR strip an orphan anchor when the comment row has vanished (#496); * - delete → deleteCommentMark (strip the ephemeral-suggestion anchor #329). * The op is idempotent, so a BullMQ retry is safe. Throwing propagates to * WorkerHost → the job is retried and, on exhaustion, surfaces in failed-job @@ -133,7 +134,18 @@ export class GeneralQueueProcessor // of this resolve), skip it rather than flip the mark to a stale state. const comment = await this.commentRepo.findById(commentId); if (!comment) { - // The comment vanished (e.g. hard-deleted) → nothing left to mirror. + // #496 reconcile: the comment row is GONE (e.g. an ephemeral apply/dismiss + // hard-deleted it while this resolve/unresolve mark job sat in the queue), + // but its inline anchor may still live in the doc — a silent orphan mark + // pointing at a comment that no longer exists. Self-heal by stripping it + // instead of just returning: this closes the divergence the fire-and-forget + // resolve/unresolve enqueue (comment.service resolveComment) could leave. + // Idempotent — deleteCommentMark on an already-absent mark is a no-op. + await this.getCollaborationGateway().handleYjsEvent( + 'deleteCommentMark', + documentName, + { commentId, user }, + ); return; } const wantResolved = action === 'resolve'; diff --git a/apps/server/test/integration/ai-chat-attach.int-spec.ts b/apps/server/test/integration/ai-chat-attach.int-spec.ts index 70da3cbd..6e57199e 100644 --- a/apps/server/test/integration/ai-chat-attach.int-spec.ts +++ b/apps/server/test/integration/ai-chat-attach.int-spec.ts @@ -30,11 +30,13 @@ import { * tees the SSE frames into it via `consumeSseStream` while stamping the DB row id * via `generateMessageId` (both gated on runId + the resumable flag). * - * Proven here: a finished run's replay is the full frame sequence incl `[DONE]` - * with `start.messageId` == the seeded DB row id; the anchor check (invariant 6); - * an attach opened BEFORE the first frame follows the live stream from frame 0; an - * explicit stop surfaces `{"type":"abort"}` + `[DONE]` + end to the subscriber; - * and the legacy (non-run) path tees nothing. + * Proven here (tail-only #491): a finished run attached at its persisted frontier + * N_final delivers only the TAIL past N (a synthetic `start` carrying the run-fact + * + the terminal `finish`/`[DONE]`) — the step content below N lives in the seeded + * DB row, NOT the ring; the anchor check (invariant 6); an attach opened BEFORE the + * first frame follows the live stream from frame 0; an explicit stop surfaces + * `{"type":"abort"}` + `[DONE]` + end to the subscriber; and the legacy (non-run) + * path tees nothing. */ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); @@ -133,14 +135,16 @@ function liveSink(): { }; } -// The SSE `start` frame carries the message id; pull it out of a `data: {...}`. -function parseStartMessageId(frames: string[]): string | undefined { +// Parse the first `start` frame's JSON out of a `data: {...}` sequence. +function parseStartFrame( + frames: string[], +): { messageId?: string; messageMetadata?: any } | undefined { for (const f of frames) { const m = /^data: (\{.*\})\s*$/m.exec(f.trim()); if (!m) continue; try { const json = JSON.parse(m[1]); - if (json.type === 'start') return json.messageId; + if (json.type === 'start') return json; } catch { /* not this frame */ } @@ -270,7 +274,7 @@ describe('AiChatService run-stream attach [integration]', () => { await destroyTestDb(); }); - it('run-wrapped: replay is the full frame sequence incl [DONE], start.messageId == the seeded DB row id', async () => { + it('run-wrapped, tail-only: a finished run at N_final delivers the run-fact start + finish/[DONE]; the step content lives in the seeded row', async () => { const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id; const registry = new AiChatStreamRegistryService(); const runService = new AiChatRunService(runRepo, { @@ -300,27 +304,37 @@ describe('AiChatService run-stream attach [integration]', () => { ); }); const rowId = await assistantRowId(chatId); + // The client reads its persisted step frontier N from the seeded row. + const row: any = await msgRepo.findById(rowId, workspaceId); + const nFinal = row.metadata.stepsPersisted as number; + expect(nFinal).toBe(1); // a single finished step + // The step content is in the SEEDED row (parts/content), not the ring. + expect(JSON.stringify(row.metadata.parts)).toContain('Hello'); - // Finished-run replay with expect=live + the correct anchor. + // Attach at N_final with the correct anchor: the tail past step 1 is just + // the terminal frames; step 0's 'Hello' is BELOW the frontier (seeded). const sink = liveSink(); - const att = await registry.attach(chatId, true, rowId, sink.cb); + const att = await registry.attach(chatId, rowId, nFinal, sink.cb); expect(att).not.toBeNull(); expect(att!.finished).toBe(true); - // The tee captured frames (consumeSseStream was wired). - expect(att!.replay.length).toBeGreaterThan(0); - // generateMessageId stamped the DB row id onto the streamed start frame. - expect(parseStartMessageId(att!.replay)).toBe(rowId); - // The full sequence includes the streamed text and the terminal marker. - const joined = att!.replay.join(''); - expect(joined).toContain('Hello'); + // The synthetic start frame carries the run-fact (runId/chatId), the source + // of the run-fact on re-attach. + const start = parseStartFrame(att!.replay); + expect(start?.messageMetadata).toMatchObject({ + runId: box.runId, + chatId, + }); + // The terminal marker is delivered so the client's SDK closes the stream. expect(att!.replay.some((f) => f.includes('[DONE]'))).toBe(true); + // 'Hello' (step 0, below the frontier) is NOT re-streamed — it is seeded. + expect(att!.replay.some((f) => f.includes('Hello'))).toBe(false); } finally { registry.onModuleDestroy(); await cleanup(); } }); - it('anchor mismatch with expect=live returns null (invariant 6)', async () => { + it('anchor mismatch returns null (invariant 6)', async () => { const chatId = (await createChat(db, { workspaceId, creatorId: userId })).id; const registry = new AiChatStreamRegistryService(); const runService = new AiChatRunService(runRepo, { @@ -347,7 +361,7 @@ describe('AiChatService run-stream attach [integration]', () => { const sink = liveSink(); // A foreign anchor must NOT replay this run's transcript. expect( - await registry.attach(chatId, true, 'a-different-run-row', sink.cb), + await registry.attach(chatId, 'a-different-run-row', 1, sink.cb), ).toBeNull(); } finally { registry.onModuleDestroy(); @@ -376,8 +390,11 @@ describe('AiChatService run-stream attach [integration]', () => { try { // Attach while the entry exists (opened at begin) but before any frame. const sink = liveSink(); - const att = (await registry.attach(chatId, false, undefined, sink.cb))!; - expect(att.replay).toEqual([]); // nothing streamed yet -> replay from 0 + const att = (await registry.attach(chatId, undefined, 0, sink.cb))!; + // Nothing streamed yet -> the tail is just the synthetic start frame; the + // whole live stream (start..DONE) follows via onFrame after start(). + expect(att.replay).toHaveLength(1); + expect(att.replay[0]).toContain('"type":"start"'); att.start(); // go live (drains nothing, then follows) // Now emit the whole turn. @@ -448,7 +465,7 @@ describe('AiChatService run-stream attach [integration]', () => { }); try { const sink = liveSink(); - const att = (await registry.attach(chatId, false, undefined, sink.cb))!; + const att = (await registry.attach(chatId, undefined, 0, sink.cb))!; att.start(); // Give streamText a beat to begin consuming the partial output. @@ -526,7 +543,9 @@ describe('AiChatService run-stream attach [integration]', () => { expect(entry).toBeDefined(); expect(entry.finished).toBe(true); const sink = liveSink(); - expect(await registry.attach(chatId, false, undefined, sink.cb)).toBeNull(); + // Finished with an EMPTY ring (aborted before any frame) -> null -> the + // client degrades to poll instead of hanging on an empty stream. + expect(await registry.attach(chatId, undefined, 0, sink.cb)).toBeNull(); } finally { registry.onModuleDestroy(); await cleanup(); @@ -556,8 +575,8 @@ describe('AiChatService run-stream attach [integration]', () => { }); const sink = liveSink(); // No entry was ever opened; attach always yields null. - expect(await registry.attach(chatId, false, undefined, sink.cb)).toBeNull(); - expect(await registry.attach(chatId, true, 'anything', sink.cb)).toBeNull(); + expect(await registry.attach(chatId, undefined, 0, sink.cb)).toBeNull(); + expect(await registry.attach(chatId, 'anything', 1, sink.cb)).toBeNull(); } finally { registry.onModuleDestroy(); await cleanup(); diff --git a/apps/server/test/integration/ai-mcp-server-repo.int-spec.ts b/apps/server/test/integration/ai-mcp-server-repo.int-spec.ts index 2e181791..23354b63 100644 --- a/apps/server/test/integration/ai-mcp-server-repo.int-spec.ts +++ b/apps/server/test/integration/ai-mcp-server-repo.int-spec.ts @@ -1,5 +1,6 @@ import { Kysely, sql } from 'kysely'; import { randomUUID } from 'node:crypto'; +import { Logger } from '@nestjs/common'; import { AiMcpServerRepo } from '@docmost/db/repos/ai-chat/ai-mcp-server.repo'; import { getTestDb, destroyTestDb, createWorkspace } from './db'; @@ -54,7 +55,11 @@ describe('AiMcpServerRepo tool_allowlist jsonb round-trip [integration]', () => expect(Array.isArray(found?.toolAllowlist)).toBe(true); }); - it('an empty allowlist is normalized to null (no restriction), not []', async () => { + // #476 (deliberate behaviour change): an empty allowlist used to be + // normalized to SQL NULL, which downstream means "no restriction" — so an + // admin's deny-all `[]` silently became allow-all. It must now round-trip as + // a real jsonb `[]` (deny-all), distinct from NULL. + it('an empty allowlist round-trips as jsonb [] (deny-all), not null (#476)', async () => { const row = await repo.insert({ workspaceId: ws, name: `srv-${randomUUID()}`, @@ -62,7 +67,27 @@ describe('AiMcpServerRepo tool_allowlist jsonb round-trip [integration]', () => url: 'https://example.com/mcp', toolAllowlist: [], }); - // The column is SQL NULL, so jsonb_typeof returns SQL NULL (JS null). + // The column holds a real (empty) jsonb ARRAY, not SQL NULL. + expect(await jsonbTypeof(row.id)).toBe('array'); + expect((await repo.findById(row.id, ws))?.toolAllowlist).toEqual([]); + }); + + it('update to [] persists jsonb [] and update to null clears to SQL NULL (#476)', async () => { + const row = await repo.insert({ + workspaceId: ws, + name: `srv-${randomUUID()}`, + transport: 'http', + url: 'https://example.com/mcp', + toolAllowlist: ['search'], + }); + + // Deny-all via update: [] must survive as a real jsonb array. + await repo.update(row.id, ws, { toolAllowlist: [] }); + expect(await jsonbTypeof(row.id)).toBe('array'); + expect((await repo.findById(row.id, ws))?.toolAllowlist).toEqual([]); + + // Explicit clear (null) still means "no restriction" = SQL NULL. + await repo.update(row.id, ws, { toolAllowlist: null }); expect(await jsonbTypeof(row.id)).toBeNull(); expect((await repo.findById(row.id, ws))?.toolAllowlist).toBeNull(); }); @@ -92,23 +117,60 @@ describe('AiMcpServerRepo tool_allowlist jsonb round-trip [integration]', () => expect(healed?.toolAllowlist).toEqual(['alpha', 'beta']); }); - it('FAIL-OPEN: a present-but-corrupt tool_allowlist reads back as null (no restriction)', async () => { - // #185 re-review pt 8: normalizeRow's fail-open branch — the column is - // PRESENT but does not parse into a string[] (here a jsonb string scalar - // holding non-array JSON). The read must degrade to `null` ("no restriction"), - // not crash. (A warn is logged with the server id; not asserted here.) - const id = randomUUID(); - await sql` - INSERT INTO ai_mcp_servers (id, workspace_id, name, transport, url, tool_allowlist) - VALUES ( - ${id}, ${ws}, ${`srv-${id}`}, 'http', 'https://example.com/mcp', - to_jsonb(${'{"not":"an array"}'}::text) - ) - `.execute(db); - // Sanity: the column is present (a jsonb string scalar), not SQL NULL. - expect(await jsonbTypeof(id)).toBe('string'); - // ...yet the read degrades to null (fail-open). - expect((await repo.findById(id, ws))?.toolAllowlist).toBeNull(); + // #476 (deliberate behaviour change, replaces the old FAIL-OPEN pin): a + // present-but-corrupt tool_allowlist used to degrade to null ("no + // restriction"), silently handing the agent ALL of the server's tools. It + // must now FAIL CLOSED to `[]` (deny-all) and log an error. + it('FAIL-CLOSED: a present-but-corrupt tool_allowlist reads back as [] (deny-all) + error log (#476)', async () => { + const errorSpy = jest + .spyOn(Logger.prototype, 'error') + .mockImplementation(() => undefined); + try { + // The column is PRESENT but does not parse into a string[] — a jsonb + // string scalar holding unparseable text (a truncated legacy write). + const id = randomUUID(); + await sql` + INSERT INTO ai_mcp_servers (id, workspace_id, name, transport, url, tool_allowlist) + VALUES ( + ${id}, ${ws}, ${`srv-${id}`}, 'http', 'https://example.com/mcp', + to_jsonb(${'{oops'}::text) + ) + `.execute(db); + // Sanity: the column is present (a jsonb string scalar), not SQL NULL. + expect(await jsonbTypeof(id)).toBe('string'); + // ...and the read degrades to [] (fail-closed deny-all), not null. + expect((await repo.findById(id, ws))?.toolAllowlist).toEqual([]); + // The narrowing is not silent: an error names the server id (never the + // corrupt contents). + expect(errorSpy).toHaveBeenCalledWith( + expect.stringContaining(`Corrupt tool_allowlist for MCP server ${id}`), + ); + expect( + errorSpy.mock.calls.some((c) => String(c[0]).includes('{oops')), + ).toBe(false); + } finally { + errorSpy.mockRestore(); + } + }); + + it('FAIL-CLOSED: corrupt non-array JSON (an object) also reads back as [] (#476)', async () => { + const errorSpy = jest + .spyOn(Logger.prototype, 'error') + .mockImplementation(() => undefined); + try { + const id = randomUUID(); + await sql` + INSERT INTO ai_mcp_servers (id, workspace_id, name, transport, url, tool_allowlist) + VALUES ( + ${id}, ${ws}, ${`srv-${id}`}, 'http', 'https://example.com/mcp', + to_jsonb(${'{"not":"an array"}'}::text) + ) + `.execute(db); + expect(await jsonbTypeof(id)).toBe('string'); + expect((await repo.findById(id, ws))?.toolAllowlist).toEqual([]); + } finally { + errorSpy.mockRestore(); + } }); }); diff --git a/apps/server/test/integration/share-alias-one-per-page.int-spec.ts b/apps/server/test/integration/share-alias-one-per-page.int-spec.ts index b372a996..440f5a88 100644 --- a/apps/server/test/integration/share-alias-one-per-page.int-spec.ts +++ b/apps/server/test/integration/share-alias-one-per-page.int-spec.ts @@ -33,6 +33,11 @@ describe('share_aliases one-per-page invariant [integration]', () => { const pageRepo = { findById: async (id: string) => ({ id, title: `title-${id}` }), }; + // The requester can view the target page (permissive), so the reassign 409 may + // include its title — these tests exercise the one-per-page invariant, not the + // #495 disclosure gate (that is unit-tested in share-alias.service.spec.ts). + const pageAccessService = { validateCanView: async () => {} }; + const USER = { id: 'u-int' } as any; beforeAll(async () => { db = getTestDb(); @@ -41,6 +46,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { repo as any, pageRepo as any, {} as any, // shareService — unused by setAlias + pageAccessService as any, db as any, ); wsId = (await createWorkspace(db)).id; @@ -188,6 +194,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId, creatorId, + user: USER, alias: 'te', }); expect(first.alias).toBe('te'); @@ -196,6 +203,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId, creatorId, + user: USER, alias: 'ted', }); // Same row id — a RENAME, not a new insert. @@ -217,12 +225,14 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId, creatorId: null as any, + user: USER, alias: 'hello', }); const again = await service.setAlias({ workspaceId: wsId, pageId, creatorId: null as any, + user: USER, alias: 'hello', }); expect(again.id).toBe(inserted.id); @@ -244,6 +254,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { flakyRepo as any, pageRepo as any, {} as any, + pageAccessService as any, db as any, ); @@ -252,6 +263,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId, creatorId: null as any, + user: USER, alias: 'rollback-me', }), ).rejects.toBeInstanceOf(BadRequestException); @@ -275,6 +287,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId: pageA, creatorId: null as any, + user: USER, alias: 'shared', }); @@ -283,6 +296,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId: pageB, creatorId: null as any, + user: USER, alias: 'shared', }), ).rejects.toBeInstanceOf(ConflictException); @@ -291,6 +305,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId: pageB, creatorId: null as any, + user: USER, alias: 'shared', confirmReassign: true, }); @@ -317,12 +332,14 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId: pageA, creatorId: null as any, + user: USER, alias: 'shared-target', }); await service.setAlias({ workspaceId: wsId, pageId: pageB, creatorId: null as any, + user: USER, alias: 'bee', }); @@ -330,6 +347,7 @@ describe('share_aliases one-per-page invariant [integration]', () => { workspaceId: wsId, pageId: pageB, creatorId: null as any, + user: USER, alias: 'shared-target', confirmReassign: true, }); diff --git a/apps/server/test/jest-integration.json b/apps/server/test/jest-integration.json index 30f62397..cd3633c5 100644 --- a/apps/server/test/jest-integration.json +++ b/apps/server/test/jest-integration.json @@ -22,6 +22,7 @@ "^@docmost/db/(.*)$": "/src/database/$1", "^@docmost/transactional/(.*)$": "/src/integrations/transactional/$1", "^@docmost/ee/(.*)$": "/src/ee/$1", + "^@docmost/token-estimate$": "/../../packages/token-estimate/src/index.ts", "^src/(.*)$": "/src/$1" } } diff --git a/docs/reading-ai-logs.md b/docs/reading-ai-logs.md index fbf2e211..135085dc 100644 --- a/docs/reading-ai-logs.md +++ b/docs/reading-ai-logs.md @@ -8,19 +8,35 @@ real pain (a "which tools fail most?" analysis that confidently answered Read the **Gotchas** section before you trust any error count. +> **TWO ERAS — check the marker first.** The `tool_calls` shape changed in **#490 +> (trace v2)**. A row written by v2 carries `metadata.toolTraceVersion = 2`; older +> rows have no such key. The two shapes store DIFFERENT things (v2 dropped the tool +> OUTPUT from the trace), so **every query below is dual-shape** — branch on the +> marker. **Never compare an aggregate or trend across the era boundary**: a metric +> jump on the cut-over week is an artifact of the shape change, not a behavior +> change. + ## TL;DR - Agent chats live in Postgres, DB `docmost`, tables `ai_chat_*`. -- Each tool invocation is stored as **two** array elements (a `tool-call` part and - a `tool-result` part), so naive counting double-counts. -- **A tool that *throws* writes no result part.** Since the #407 fix its error is - persisted as a dedicated `{toolName, error}` element in `tool_calls` (queryable + - replayed to the model). **Rows written before #407 still drop it** — the error is - nowhere in the DB and shows only in the live UI. So `isError` / `success=false` - scans under-report by design, and pre-#407 thrown errors are invisible. -- To find where agents fail: (1) soft-failure markers in `tool_calls`, (2) the new - `error` field for thrown errors (new rows) / the orphan-gap proxy (old rows), - (3) server logs / the live UI for full stack traces beyond the truncated message. +- **Era marker:** `metadata.toolTraceVersion = 2` ⇒ v2 (#490) row; absent ⇒ legacy row. +- Each tool invocation is stored as **two** consecutive array elements — a + `tool-call` part then an OUTCOME part — so naive counting double-counts. + - **v2 (#490):** outcome is `{toolName, ok: true}` on success, or + `{toolName, error, kind: 'thrown'|'interrupted'}` on failure. The tool **OUTPUT + is NOT in `tool_calls`** any more — it lives once in `metadata.parts` (this + removed a hundreds-of-MB-per-run write duplication). Soft-failure analysis + therefore reads `metadata.parts`, not `tool_calls`. + - **legacy:** outcome is `{toolName, output}` on success; a **thrown** failure is + a `{toolName, error}` element **only on rows after #407**, and is dropped + entirely (silent orphan) on pre-#407 rows. +- **A tool that *throws* writes no result part.** In v2 it is a + `{error, kind:'thrown'}` element; an interrupted/aborted call is a distinct + `{error, kind:'interrupted'}`. `isError`/`success=false` scans read the *output* + and so under-report thrown failures in every era. +- To find where agents fail: (1) soft-failure markers in `metadata.parts` outputs + (v2) / `tool_calls` outputs (legacy), (2) the `error`/`kind` fields for thrown + failures (v2 + post-#407), (3) server logs / the live UI for full stack traces. ## Where the data lives @@ -53,33 +69,67 @@ are rows in `workspaces`, not separate deployments. separate `tool` role), `content` (text), `tool_calls` (jsonb array), `metadata` (jsonb, holds run `error` + rendered `parts`), `status`, `tsv` (full-text index). +## Era marker — check this before every query + +```sql +-- how many rows are in each era? +SELECT COALESCE((metadata->>'toolTraceVersion'), 'legacy') AS era, count(*) +FROM ai_chat_messages +WHERE role = 'assistant' AND jsonb_typeof(tool_calls) = 'array' +GROUP BY 1 ORDER BY 2 DESC; +``` + +- `toolTraceVersion = '2'` → **v2** (#490): outcome flags, **no output in the trace**. +- `NULL` (`'legacy'`) → pre-#490: outcome carries the tool `output` inline. + +**Do not trend a metric across the cut-over.** The shape change alone shifts counts +(e.g. "elements with `output`" collapses to zero for v2), so a week that straddles +the boundary shows an artifact, not a behavior change. Segment by era, or restrict to +one era, before comparing. + ## How tool calls are stored — READ THIS Tool calls are **not** one-object-per-call. Each logical invocation is split into -two consecutive elements of the `tool_calls` array: +two consecutive elements of the `tool_calls` array — a **call** then an **outcome**. +The outcome shape is era-dependent: ```text -index 0: { "toolName": "getPage", "input": { "pageId": "…" } } ← tool-call (has input, NO output) -index 1: { "toolName": "getPage", "output": { … } } ← tool-result (has output, NO input) +# v2 (#490) — metadata.toolTraceVersion = 2 +index 0: { "toolName":"getPage", "input":{...} } ← call (has input) +index 1: { "toolName":"getPage", "ok":true } ← success (NO output here) + or : { "toolName":"getPage", "error":"…", "kind":"thrown" } ← threw + or : { "toolName":"getPage", "error":"…", "kind":"interrupted" } ← aborted mid-step + +# legacy — no toolTraceVersion +index 0: { "toolName":"getPage", "input":{...} } ← call (has input, NO output) +index 1: { "toolName":"getPage", "output":{...} } ← success (has output) + or : { "toolName":"getPage", "error":"…" } ← threw (post-#407 only) ``` -The keys that appear on an element are `toolName`, `input`, `output`, and — for a -**thrown** failure on rows written after the #407 fix — `error` (the tool's error -message; see the "Hard failures" section below). There is no `state`, no `errorText`, -no `type`. On pre-#407 rows a thrown failure has NO paired result element at all -(silent orphan). Consequences: +The keys that can appear: `toolName`, `input` (call), and on the outcome — **v2:** +`ok` **or** `error`+`kind`; **legacy:** `output` **or** (post-#407) `error`. There is +no `state`, no `errorText`, no `type` in `tool_calls` (those live on `metadata.parts`). +Consequences: -1. **Real invocation count = elements that have `output` or `error`.** Counting every - element double-counts (you get ~2× and a spurious "~50% of every tool has no output"). -2. **Pairing:** a call = a `tool-call` part followed by its result part. A success - carries `output`; a thrown failure (post-#407) carries `error` instead. Both carry - `toolName`, so you can group by tool on either. +1. **Real invocation count** — count the OUTCOME elements, not every element (else you + double-count): **v2** = elements with `ok` or `error`; **legacy** = elements with + `output` or `error`. +2. **Pairing:** a call (`input`) is followed by its outcome. `toolName` is on both, so + you can group by tool on either. In v2 the `kind` field separates a real hard-fail + (`thrown`) from an aborted call (`interrupted`) — a distinction legacy rows cannot + make (both are orphans; see below). +3. **The tool OUTPUT is only in `metadata.parts` on v2 rows.** To inspect what a tool + returned (soft-error markers, page bodies) on a v2 row, read the parts + (`part->>'type' LIKE 'tool-%'`, `part->>'state' = 'output-available'`, `part->'output'`), + not `tool_calls`. ## The two classes of failure (and which the DB can see) ### 1. Soft failures — tool RAN and returned an error-shaped result → PERSISTED ✅ -These are visible in the `tool-result` `output`. The marker differs per tool: +These are visible in the tool `output` — **on v2 rows in `metadata.parts`** (the +`output-available` part's `output`), on **legacy rows in the `tool_calls` outcome +element's `output`**. The marker differs per tool: | Tool(s) | Error marker in `output` | | --- | --- | @@ -91,37 +141,32 @@ These are visible in the `tool-result` `output`. The marker differs per tool: Note `editPageText` returns `failed: []` on success — filtering on the *presence* of the key gives false positives; filter on **non-empty**. -### 2. Hard failures — tool THREW → NOW PERSISTED ✅ (since the #407 fix) +### 2. Hard failures — tool THREW → PERSISTED ✅ When a tool throws (the classic one is `patchNode` / `insertNode` / `tableUpdateCell` → `Failed to encode document to Yjs (fromJSON): Unknown node type: undefined`), the -runtime still writes **no `tool-result` part** — the failure is an ai@6 `tool-error` -content part instead. **Since the #407 fix, that error is persisted**: `serializeSteps` -appends a dedicated element `{toolName, error: ""}` right after the failed -call, mirroring how a successful `{toolName, output}` element is appended. So a thrown -error now leaves a queryable `error` field carrying its (truncated) reason, and the -same real text is replayed to the model on the next turn (an `output-error` part with -the real `errorText`, no longer the `'Tool call did not complete.'` placeholder). +runtime writes **no `tool-result` part** — the failure is an ai@6 `tool-error` content +part. How that lands in `tool_calls` depends on the era: -**Cutover caveat — old rows keep the old blind shape.** Rows written **before** this -change have the two-part shape (`call` + `output` only) and simply **drop** thrown -errors, leaving a silent **orphan** (a `call` with no `output` *and* no `error`). Rows -written **after** the fix additionally carry the `error` element. So: +- **v2 (#490):** a `{toolName, error, kind:'thrown'}` outcome element. An interrupted / + aborted mid-step call is a **distinct** `{toolName, error:'Tool call did not + complete.', kind:'interrupted'}` element — so you can tell a real hard-fail from an + abort **directly, without the orphan heuristic**. Query `kind = 'thrown'`. +- **post-#407 legacy:** a `{toolName, error}` element (no `kind`) right after the call. +- **pre-#407 legacy:** the error is **dropped** — a silent **orphan** (a `call` with no + `output` *and* no `error`). -- **New rows:** query the `error` field directly (see the hard-error query below) — no - orphan heuristic needed for thrown failures. -- **Old rows (pre-#407):** the only DB-side proxy is still an **orphan**: a `tool-call` - part with no matching `tool-result` *and* no `error`. Orphans also appear when a run - is **aborted** mid-flight (server restart), so a high-volume tool (`createComment`, - `searchInPage`, `Search_web_search`) shows orphans from aborts, not real errors on - old rows. Treat the orphan gap as an *upper bound*, and cross-check the tool: a gap on - a structural editor (`patchNode`, `insertNode`, `updatePageJson`, `transformPage`) is - almost certainly a thrown Yjs-encode error; a gap on `createComment` is mostly aborts. +The same real error text is replayed to the model on the next turn (an `output-error` +part with the real `errorText`, from `metadata.parts`), in every era. -A note on the aborted-call fallback: a call with **neither** a result **nor** a -`tool-error` (genuinely interrupted mid-step) still replays with the -`'Tool call did not complete.'` placeholder and persists as an orphan — that path is -unchanged, and is distinct from a real thrown error, which now carries `error`. +**Cutover caveat.** Only pre-#407 legacy rows need the orphan proxy: an orphan is a +`tool-call` with no matching outcome. Orphans there also appear when a run is **aborted** +mid-flight (server restart), so a high-volume tool (`createComment`, `searchInPage`, +`Search_web_search`) shows orphans from aborts, not real errors. Treat the orphan gap as +an *upper bound* and cross-check the tool: a gap on a structural editor (`patchNode`, +`insertNode`, `updatePageJson`, `transformPage`) is almost certainly a thrown Yjs-encode +error; a gap on `createComment` is mostly aborts. **On v2 rows this ambiguity is gone** +— `kind` labels each outcome. ### 3. Run-level failures → `ai_chat_runs` @@ -134,22 +179,34 @@ the wild: `Run interrupted by a server restart.` (aborts) and Run all of these via `docker exec gitmost-postgresql psql -U docmost -d docmost -P pager=off -c "…"`. -**Real invocation count per tool** (result parts only — the correct denominator): +**Real invocation count per tool** (outcome parts only — the correct denominator). +Dual-shape: a v2 outcome has `ok` or `error`; a legacy outcome has `output` or `error`: ```sql SELECT elem->>'toolName' AS tool, count(*) AS calls FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem -WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'output' +WHERE jsonb_typeof(m.tool_calls) = 'array' + AND (elem ? 'ok' OR elem ? 'output' OR elem ? 'error') GROUP BY 1 ORDER BY 2 DESC; ``` -**Soft errors per tool** (everything the DB can honestly see): +**Soft errors per tool.** The soft-error marker lives in the tool OUTPUT — which on +**v2 rows is in `metadata.parts`**, on **legacy rows is in the `tool_calls` outcome +element**. This query UNIONs both eras, projecting each output as `o`: ```sql WITH res AS ( + -- v2 (#490): output is in metadata.parts (output-available tool parts) + SELECT part->>'type' AS tool, part->'output' AS o + FROM ai_chat_messages m, jsonb_array_elements(m.metadata->'parts') part + WHERE (m.metadata->>'toolTraceVersion') = '2' + AND part->>'type' LIKE 'tool-%' AND part->>'state' = 'output-available' + UNION ALL + -- legacy: output is inline in the tool_calls outcome element SELECT elem->>'toolName' AS tool, elem->'output' AS o FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem - WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'output' + WHERE (m.metadata->>'toolTraceVersion') IS NULL + AND jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'output' ) SELECT tool, count(*) AS calls, sum(COALESCE( @@ -167,13 +224,23 @@ FROM res GROUP BY tool HAVING sum(COALESCE( ORDER BY soft_errors DESC; ``` -**`editPageText` failure reasons** (the most common real agent mistake — bad `find`): +Note the v2 `tool` label is the part type (`tool-editPageText`); strip the `tool-` +prefix if you join it against the legacy `toolName`. + +**`editPageText` failure reasons** (the most common real agent mistake — bad `find`). +Same dual-shape output source: ```sql WITH res AS ( + SELECT part->'output' AS o + FROM ai_chat_messages m, jsonb_array_elements(m.metadata->'parts') part + WHERE (m.metadata->>'toolTraceVersion') = '2' + AND part->>'type' = 'tool-editPageText' AND part->>'state' = 'output-available' + UNION ALL SELECT elem->'output' AS o FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem - WHERE jsonb_typeof(m.tool_calls) = 'array' + WHERE (m.metadata->>'toolTraceVersion') IS NULL + AND jsonb_typeof(m.tool_calls) = 'array' AND elem->>'toolName' = 'editPageText' AND elem ? 'output' ) SELECT f->>'reason' AS reason, count(*) @@ -182,30 +249,43 @@ WHERE jsonb_typeof(o->'failed') = 'array' GROUP BY 1 ORDER BY 2 DESC; ``` -**Hard errors — persisted `error` field per tool (NEW rows, since #407)** — thrown -tool failures now carry their real reason, so query them directly: +**Hard errors — persisted `error` field per tool (v2 + post-#407 rows)** — thrown tool +failures carry their real reason, so query them directly. On **v2** rows exclude the +`interrupted` kind so an aborted call is not counted as a hard-fail: ```sql SELECT elem->>'toolName' AS tool, count(*) AS thrown_errors, min(elem->>'error') AS sample_error FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem ? 'error' + -- v2 rows label the kind; a legacy error element has no kind (count it). + AND COALESCE(elem->>'kind', 'thrown') = 'thrown' +GROUP BY 1 ORDER BY 2 DESC; +``` + +Aborted mid-step calls on v2 rows are a distinct, directly countable population: + +```sql +SELECT elem->>'toolName' AS tool, count(*) AS interrupted +FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem +WHERE jsonb_typeof(m.tool_calls) = 'array' AND elem->>'kind' = 'interrupted' GROUP BY 1 ORDER BY 2 DESC; ``` **Hard-error proxy for OLD rows (pre-#407) — orphan gap per tool, WITH a spread column** -(call parts minus result parts, plus how many distinct chats the gap is spread across). -This covers rows written before thrown errors were persisted; on new rows a thrown -failure now has its own `error` element (use the query above) and an orphan means only -a genuinely aborted mid-step call: +(call parts minus outcome parts, plus how many distinct chats the gap is spread across). +This is needed ONLY for pre-#407 legacy rows (v2 and post-#407 rows carry the error / +`kind` directly — use the queries above). The `WHERE` restricts to the legacy era so v2 +rows (where an `ok` outcome is not an `output`) never produce phantom orphans: ```sql WITH parts AS ( SELECT m.chat_id, elem->>'toolName' AS tool, - (elem ? 'input' AND NOT (elem ? 'output')) AS is_call, - (elem ? 'output' OR elem ? 'error') AS is_result + (elem ? 'input' AND NOT (elem ? 'output') AND NOT (elem ? 'ok')) AS is_call, + (elem ? 'output' OR elem ? 'error' OR elem ? 'ok') AS is_result FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem WHERE jsonb_typeof(m.tool_calls) = 'array' AND m.role = 'assistant' + AND (m.metadata->>'toolTraceVersion') IS NULL ), per_chat AS ( SELECT tool, chat_id, sum(is_call::int) - sum(is_result::int) AS gap @@ -261,11 +341,21 @@ WHERE tsv @@ websearch_to_tsquery('english', 'some phrase') LIMIT 20; ## Don't blow up your context -A single `tool_calls` row can be **300–400 KB** (results embed full page content and -search payloads). Never `SELECT tool_calls` (or `jsonb_pretty(tool_calls)`) raw. -Always project just the keys you need and truncate: +Tool outputs embed full page content and search payloads (hundreds of KB per row). +On **legacy** rows they are in `tool_calls`; on **v2** rows they moved to +`metadata->'parts'` (the `tool_calls` trace itself is now small). Never `SELECT +tool_calls` / `metadata` (or `jsonb_pretty(...)`) raw — project just the keys you need +and truncate: ```sql +-- v2: outputs live in metadata.parts +SELECT part->>'type', + left(regexp_replace((part->'output')::text, '\s+', ' ', 'g'), 200) +FROM ai_chat_messages m, jsonb_array_elements(m.metadata->'parts') part +WHERE (m.metadata->>'toolTraceVersion') = '2' + AND part->>'state' = 'output-available' LIMIT 5; + +-- legacy: outputs live in tool_calls SELECT elem->>'toolName', left(regexp_replace((elem->'output')::text, '\s+', ' ', 'g'), 200) FROM ai_chat_messages m, jsonb_array_elements(m.tool_calls) elem @@ -280,26 +370,32 @@ docker compose -p gitmost logs -f --tail=100 # whole stack ``` Logging is `json-file`, `max-size=10m max-file=5` → ~50 MB retained, then rotated, -and **wiped on container recreate**. Since the #407 fix, thrown-tool error text is -**persisted in the `error` field** of `tool_calls` (see the hard-error query above), so -you no longer depend on live logs for it. Logs/live UI remain useful for **pre-#407 -rows** (whose thrown errors were dropped) and for full stack traces beyond the -truncated stored message. A per-tool `tool_calls_total{tool,status}` metric to -VictoriaMetrics is still a possible future add for aggregate dashboards. +and **wiped on container recreate**. Thrown-tool error text is **persisted** — in the +`error` field of `tool_calls` (v2 `kind:'thrown'` / post-#407 legacy) — so you no longer +depend on live logs for it. Logs/live UI remain useful for **pre-#407 rows** (whose +thrown errors were dropped) and for full stack traces beyond the truncated stored +message. A per-tool `tool_calls_total{tool,status}` metric to VictoriaMetrics is still a +possible future add for aggregate dashboards. ## Gotchas checklist -- [ ] Counting every `tool_calls` element → **overcount**. Count `output` elements; add `error` elements for thrown failures (new rows), but don't count both as invocations. -- [ ] `isError` / `success=false` ≈ 0 does **not** mean "no errors" — thrown errors are a separate `error` element (new rows) or dropped entirely (pre-#407 rows). -- [ ] Thrown errors persist only on rows written **after the #407 fix** — pre-#407 rows still drop them (orphan only). Mind the cutover when trending over time. +- [ ] **Check `metadata.toolTraceVersion` first.** v2 (`= 2`) has no output in `tool_calls`; legacy has it inline. Never trend a metric across the era boundary. +- [ ] Counting every `tool_calls` element → **overcount**. Count OUTCOME elements — v2: `ok` or `error`; legacy: `output` or `error` — never both call+outcome as invocations. +- [ ] `isError` / `success=false` ≈ 0 does **not** mean "no errors" — thrown errors are an `error` element (v2 `kind:'thrown'` / post-#407), not in the output. +- [ ] **v2:** soft-error markers (the tool output) are in `metadata.parts`, NOT `tool_calls`. Legacy: they are in the `tool_calls` outcome `output`. +- [ ] **v2:** `kind` splits a real hard-fail (`thrown`) from an aborted call (`interrupted`) directly — no orphan heuristic needed. The orphan gap is a pre-#407-legacy-only proxy. - [ ] `editPageText.failed` is `[]` on success — test for **non-empty**, not presence. -- [ ] Orphan gap on OLD rows mixes thrown errors **and** aborted runs — split by tool. On NEW rows a thrown error is its own `error` element, so a gap ≈ aborted call. - [ ] `aborted` runs = server restarts, `failed` runs = provider overload — not agent mistakes. -- [ ] Never dump a raw `tool_calls` cell — it can be hundreds of KB. -- [ ] Logs are ephemeral (≤50 MB, wiped on recreate) — grab hard-error text live. +- [ ] Never dump a raw `tool_calls` **or** `metadata.parts` cell — outputs are hundreds of KB. +- [ ] Logs are ephemeral (≤50 MB, wiped on recreate) — grab pre-#407 hard-error text live. ## Snapshot (2026-07-07, illustrative — rerun the queries for current numbers) +> All rows in this snapshot predate #490, so they are **legacy-era** (outputs inline in +> `tool_calls`, orphan proxy for thrown errors). Do not trend these numbers against v2 +> rows — segment by `toolTraceVersion` first. + + - 226 chats, 732 messages, 46 runs; ~4 400 real tool invocations. - Soft errors (persisted): `editPageText` 4/79 (bad/non-unique `find`) + 9 markdown-in-`find` warnings; `semanticSearch` 3/4 (`unavailable`); `Habr_update_draft_from_docmost` 1/2 (`doc` sent as object, not string). - Missing-result proxy, read WITH the spread column: diff --git a/packages/git-sync/src/engine/stabilize.ts b/packages/git-sync/src/engine/stabilize.ts index ce1acdcf..e6769cd0 100644 --- a/packages/git-sync/src/engine/stabilize.ts +++ b/packages/git-sync/src/engine/stabilize.ts @@ -72,7 +72,13 @@ export async function stabilizePageFile( * keeps re-pulls of an unchanged page byte-identical (no churn, loop-guard). */ export async function stabilizePageBody(content: unknown): Promise { - const md1 = convertProseMirrorToMarkdown(content); + // git-sync is the LOSSLESS mirror path, so run the serializer in `strict` + // mode: a node/mark type the converter has no case for (e.g. one added to the + // schema without a matching serializer arm) throws a ConverterLossError here + // rather than silently degrading — surfacing the loss loudly at write time + // instead of committing a lossy file. Valid content (every current schema type + // has a case) is unaffected. + const md1 = convertProseMirrorToMarkdown(content, { strict: true }); const doc2 = await markdownToProseMirror(md1); - return convertProseMirrorToMarkdown(doc2); + return convertProseMirrorToMarkdown(doc2, { strict: true }); } diff --git a/packages/git-sync/test/stabilize.test.ts b/packages/git-sync/test/stabilize.test.ts index c781546e..66191aa0 100644 --- a/packages/git-sync/test/stabilize.test.ts +++ b/packages/git-sync/test/stabilize.test.ts @@ -4,6 +4,7 @@ import { stabilizePageFile, type PageMeta } from '../src/engine/stabilize.js'; // global DOM via jsdom at module load time (required for @tiptap/html under Node). import { markdownToProseMirror } from '@docmost/prosemirror-markdown'; import { parseDocmostMarkdown } from '@docmost/prosemirror-markdown'; +import { ConverterLossError } from '@docmost/prosemirror-markdown'; // stabilize.ts (SPEC §11 normalize-on-write) was 0% covered (only the gated e2e // touched it). stabilizePageFile is import-testable: build a small ProseMirror @@ -66,6 +67,23 @@ describe('stabilizePageFile — normalize-on-write fixpoint (SPEC §11)', () => expect(body1).toContain('data-src="/d.drawio"'); }); + it('runs the serializer in STRICT mode — an unmappable node throws, not a lossy write (#493)', async () => { + // git-sync is the lossless mirror path: a node type the converter has no + // case for (here a fabricated one, standing in for a schema type added + // without a matching serializer arm) must surface loudly at write time + // rather than being silently flattened into a lossy .md file. + const content = { + type: 'doc', + content: [ + { type: 'paragraph', content: [{ type: 'text', text: 'ok' }] }, + { type: 'quantumWidget', content: [{ type: 'text', text: 'lost?' }] }, + ], + }; + await expect(stabilizePageFile(content, meta)).rejects.toBeInstanceOf( + ConverterLossError, + ); + }); + it('already-stable content is unchanged by the pass (idempotent)', async () => { // Plain prose is already a fixpoint; stabilizing it once and twice agree. const content = { diff --git a/packages/mcp/src/client.ts b/packages/mcp/src/client.ts index 75c62871..e00f96c3 100644 --- a/packages/mcp/src/client.ts +++ b/packages/mcp/src/client.ts @@ -33,6 +33,12 @@ import { TransformsMixin, type ITransformsMixin } from "./client/transforms.js"; export type { DocmostMcpConfig, SandboxPut } from "./client/context.js"; export { formatDocmostAxiosError, assertFullUuid } from "./client/errors.js"; +// Branded canonical page-identity type (#435): the internal page UUID is a +// distinct nominal type so an unresolved raw/slug string can't be swapped into +// the seams that require the canonical id (see lib/page-id.ts). Re-exported on +// the package surface for hosts that type against the resolved id. +export type { PageId } from "./lib/page-id.js"; + // The full public + shared instance surface of the assembled client. Built by // INTERSECTING each domain mixin's public interface (each DERIVED from its class // and enforced by that class's `implements` clause — issue #446, no hand-mirror) diff --git a/packages/mcp/src/client/comments.ts b/packages/mcp/src/client/comments.ts index c1fd557d..a2872f16 100644 --- a/packages/mcp/src/client/comments.ts +++ b/packages/mcp/src/client/comments.ts @@ -59,6 +59,39 @@ import { mergeFootnoteDefinitions, } from "../lib/transforms.js"; +// Max concurrent per-page comment fetches in checkNewComments (#490). The scan is +// O(N) independent REST reads over the working set; running them one-at-a-time made +// a large space linear in round-trips. A small cap parallelizes without hammering +// the server (or exhausting sockets). 6 is a conservative middle of the 5–8 band. +const COMMENT_SCAN_CONCURRENCY = 6; + +/** + * Map `items` through `fn` with at most `limit` in flight, preserving INPUT ORDER + * in the returned array. A tiny bounded pool (no p-limit dependency): `limit` + * workers pull the next index off a shared cursor until the list is drained. + */ +async function mapWithConcurrency( + items: readonly T[], + limit: number, + fn: (item: T, index: number) => Promise, +): Promise { + const results = new Array(items.length); + let cursor = 0; + const worker = async (): Promise => { + for (;;) { + const i = cursor++; + if (i >= items.length) return; + results[i] = await fn(items[i], i); + } + }; + const workers = Array.from( + { length: Math.max(1, Math.min(limit, items.length)) }, + () => worker(), + ); + await Promise.all(workers); + return results; +} + // Public method surface of CommentsMixin (issue #450) — a NAMED type so the factory // return type is expressible in the emitted .d.ts (the anonymous mixin class // carries the base's protected shared state, which would otherwise trip TS4094). @@ -450,6 +483,12 @@ export function CommentsMixin>( // can surface the closest-block / spans-multiple-blocks hint built from the // LIVE document (the pre-check page is not in scope there). let liveNotFoundError: Error | null = null; + // #496: the RAW substring the mark actually covers in the LIVE doc. The + // stored selection (payload.selection) came from a DEBOUNCED REST snapshot, + // which can differ from the live doc — and apply compares the marked live + // text to the stored selection strictly, so a stale snapshot 409s on EVERY + // apply. Captured here (same doc version the mark is set in) and synced below. + let liveAnchoredSelection: string | null = null; try { const collabToken = await this.getCollabTokenWithReauth(); // Open the collab doc by the canonical UUID, never the slugId (#260). The @@ -489,6 +528,12 @@ export function CommentsMixin>( } if (applyAnchorInDoc(doc, selection as string, newCommentId)) { anchored = true; + // For a suggestion, re-read the exact substring now under the mark + // (the mark is an attribute, so it does not change the raw text) to + // sync as the stored expectedText after the mutation resolves. + if (hasSuggestion) { + liveAnchoredSelection = getAnchoredText(doc, selection as string); + } return doc; } // Selection text not found in the LIVE document: abort the write. The @@ -527,6 +572,36 @@ export function CommentsMixin>( ); } + // #496: sync the stored selection (== apply-time expectedText) to the RAW + // substring the mark actually covers in the LIVE doc when it diverged from + // the debounced REST snapshot we stored at create time. Without this, apply + // strictly compares the marked live text to a stale stored selection and + // 409s every time. Best-effort: the comment is already correctly anchored, so + // a resync failure must NOT roll it back — it only risks a later apply 409, + // which we surface as a soft warning. + if ( + hasSuggestion && + liveAnchoredSelection != null && + liveAnchoredSelection !== payload.selection + ) { + try { + await this.client.post("/comments/resync-suggestion-anchor", { + commentId: newCommentId, + selection: liveAnchoredSelection, + }); + // Reflect the corrected anchor in the returned comment. + if (result.data) result.data.selection = liveAnchoredSelection; + } catch (e) { + if (process.env.DEBUG) { + console.error("Failed to resync suggestion anchor:", e); + } + result.warning = + "The suggestion was anchored, but its stored selection could not be " + + "synced to the live document; applying it may report a conflict if the " + + "text changed. Re-create the suggestion if Apply fails."; + } + } + // Soft warning (like editPageText): the selection only matched after // stripping markdown, so the caller likely quoted a styled fragment. if (anchorNormalized) { @@ -655,27 +730,32 @@ export function CommentsMixin>( parentPageId, ); - // 2. Fetch comments for each page, keep ones created after since - const results: any[] = []; - for (const page of pagesInScope) { - try { - // Full feed (incl. resolved): a "new comments since" scan reports all - // recent activity; the active-only filter is scoped to listComments. - const comments = (await this.listComments(page.id, true)).items; - const newComments = comments.filter( - (c: any) => new Date(c.createdAt) > sinceDate, - ); - if (newComments.length > 0) { - results.push({ - pageId: page.id, - pageTitle: page.title, - comments: newComments, - }); + // 2. Fetch comments for each page, keep ones created after since. Runs with + // bounded concurrency (#490) instead of one-at-a-time — the per-page reads are + // independent, so a large working set no longer costs O(N) serial round-trips. + // Order is preserved (mapWithConcurrency keeps input order), so the output is + // deterministic regardless of which fetch finishes first. + const perPage = await mapWithConcurrency( + pagesInScope, + COMMENT_SCAN_CONCURRENCY, + async (page: any) => { + try { + // Full feed (incl. resolved): a "new comments since" scan reports all + // recent activity; the active-only filter is scoped to listComments. + const comments = (await this.listComments(page.id, true)).items; + const newComments = comments.filter( + (c: any) => new Date(c.createdAt) > sinceDate, + ); + return newComments.length > 0 + ? { pageId: page.id, pageTitle: page.title, comments: newComments } + : null; + } catch (e: any) { + // Skip pages with errors (e.g. deleted between calls) + return null; } - } catch (e: any) { - // Skip pages with errors (e.g. deleted between calls) - } - } + }, + ); + const results: any[] = perPage.filter((r): r is any => r !== null); const totalNewComments = results.reduce( (sum, r) => sum + r.comments.length, diff --git a/packages/mcp/src/client/context.ts b/packages/mcp/src/client/context.ts index 8c784f53..effa6197 100644 --- a/packages/mcp/src/client/context.ts +++ b/packages/mcp/src/client/context.ts @@ -24,6 +24,7 @@ import { isCollabAuthFailedError, } from "../lib/collab-session.js"; import { withPageLock, isUuid } from "../lib/page-lock.js"; +import type { PageId } from "../lib/page-id.js"; import { getCollabToken, performLogin } from "../lib/auth-utils.js"; import { formatDocmostAxiosError } from "./errors.js"; import { GetPageConversionCache } from "./getpage-cache.js"; @@ -553,6 +554,13 @@ export abstract class DocmostClientContext { try { return await write(collabToken); } catch (e) { + // INVARIANT (#494/#435): this auto-retry MUST stay auth-only. A collab + // write can fail INDETERMINATE — its update may already have reached and + // persisted on the server (e.g. an LRU eviction of a busy session, tagged + // via isCollabIndeterminateError). Blindly retrying such a write duplicates + // it (the #435 double-apply class). If this gate is ever widened to retry a + // broader error class, FIRST check `isCollabIndeterminateError(e)` and do + // NOT retry an indeterminate write — re-read and verify before any retry. if (!isCollabAuthFailedError(e)) throw e; // The WS handshake rejected our token: drop it from the cache so it can't // be reused for the rest of the TTL, mint a fresh one (forceRefresh bypasses @@ -715,10 +723,18 @@ export abstract class DocmostClientContext { * once via getPageRaw and cached (both slugId->uuid and uuid->uuid), so * repeated edits on the same page add no extra request. */ - protected async resolvePageId(pageId: string): Promise { - if (isUuid(pageId)) return pageId; + protected async resolvePageId(pageId: string): Promise { + // This is the ONE canonicalization seam, so it is where the `PageId` brand + // is minted (#435). The value is validated here — a UUID input by isUuid, a + // resolved id as the server's own page.id — so the downstream write path + // (withPageLock / mutatePageContent) can require the brand and reject any + // unresolved raw id at compile time. The brand is a pure compile-time marker + // applied by cast (no runtime guard): the guarantee is that this seam is the + // only place a `PageId` is produced, so every branded value went through the + // UUID/resolve check above. + if (isUuid(pageId)) return pageId as PageId; const cached = this.pageIdCache.get(pageId); - if (cached) return cached; + if (cached) return cached as PageId; const data = await this.getPageRaw(pageId); const uuid = data?.id; if (typeof uuid !== "string" || !uuid) { @@ -727,7 +743,7 @@ export abstract class DocmostClientContext { ); } this.pageIdCache.set(pageId, uuid); - return uuid; + return uuid as PageId; } diff --git a/packages/mcp/src/client/drawio.ts b/packages/mcp/src/client/drawio.ts index c3c4530f..b0e5ad9d 100644 --- a/packages/mcp/src/client/drawio.ts +++ b/packages/mcp/src/client/drawio.ts @@ -57,11 +57,11 @@ import { // Derived from the class below; `implements IDrawioMixin` fails to compile on drift. export interface IDrawioMixin { drawioGet(pageId: string, node: string, format?: "xml" | "svg"): Promise<{ pageId: string; nodeId: string; format: "xml" | "svg"; content: string; meta: { attachmentId: string | null; title: string | null; width: number | null; height: number | null; cellCount: number; hash: string; }; }>; - drawioCreate(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, xml: string, title?: string, layout?: "elk"): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; verify?: any; }>; + drawioCreate(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, xml: string, title?: string, layout?: "elk"): Promise<{ success: boolean; nodeId: string | null; attachmentId: string; warnings: string[]; verify?: any; }>; drawioUpdate(pageId: string, node: string, xml: string, baseHash: string, layout?: "elk"): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; verify?: any; }>; drawioEditCells(pageId: string, node: string, operations: CellOp[], baseHash: string): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; verify?: any; }>; - drawioFromGraph(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, graph: Graph, direction?: "LR" | "RL" | "TB" | "BT", preset?: string, layout?: GraphLayoutMode, node?: string): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; iconsResolved: number; iconsMissing: string[]; verify?: any; }>; - drawioFromMermaid(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, mermaid: string, preset?: string): Promise<{ success: boolean; nodeId: string; attachmentId: string; warnings: string[]; iconsResolved: number; iconsMissing: string[]; verify?: any; }>; + drawioFromGraph(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, graph: Graph, direction?: "LR" | "RL" | "TB" | "BT", preset?: string, layout?: GraphLayoutMode, node?: string): Promise<{ success: boolean; nodeId: string | null; attachmentId: string; warnings: string[]; iconsResolved: number; iconsMissing: string[]; verify?: any; }>; + drawioFromMermaid(pageId: string, where: { position: "before" | "after" | "append"; anchorNodeId?: string; anchorText?: string; }, mermaid: string, preset?: string): Promise<{ success: boolean; nodeId: string | null; attachmentId: string; warnings: string[]; iconsResolved: number; iconsMissing: string[]; verify?: any; }>; } export function DrawioMixin>(Base: TBase): GConstructor & TBase { @@ -164,7 +164,9 @@ export function DrawioMixin>(Ba layout?: "elk", ): Promise<{ success: boolean; - nodeId: string; + // `null` when the diagram was written but nested (no addressable "#" + // handle) — see the nested-insert branch below (#494). + nodeId: string | null; attachmentId: string; warnings: string[]; verify?: any; @@ -276,11 +278,28 @@ export function DrawioMixin>(Ba // anchor), where "#" — which addresses only top-level blocks — // cannot reference it. drawio nodes carry no persisted id, so there is no // stable handle for a nested diagram. - throw new Error( - `drawioCreate: the diagram was inserted on page ${pageId} but not as a ` + - `top-level block, so it has no addressable "#" handle. Anchor ` + - `on a top-level block (or append) so the diagram can be re-read.`, - ); + // + // CRITICAL (#494): the diagram is ALREADY WRITTEN and committed at this + // point (the mutation above succeeded). Throwing here reported a FAILURE for + // a write that in fact landed, so a retry-prone agent re-ran drawioCreate + // and inserted a DUPLICATE diagram (the #435 double-apply class). Return + // SUCCESS with nodeId:null and a warning instead: the write is + // acknowledged, and the agent is told there is no addressable handle and how + // to re-read the diagram — so it never blind-retries a landed write. + return { + success: true, + nodeId: null, + attachmentId: att.id, + warnings: [ + ...prepared.warnings, + `The diagram was written on page ${pageId} but as a NESTED block (not ` + + `top-level), so it has no addressable "#" handle. It is saved ` + + `— do NOT re-create it. To read or edit it, locate it via getOutline ` + + `/ getPageJson (attachmentId ${att.id}). To get a stable "#" ` + + `handle, anchor on a top-level block (or append).`, + ], + verify: mutation.verify, + }; } // The returned handle is POSITIONAL ("#"): valid for the immediate @@ -597,7 +616,9 @@ export function DrawioMixin>(Ba node?: string, ): Promise<{ success: boolean; - nodeId: string; + // `null` when written nested (no addressable handle) — inherited from + // drawioCreate (#494). + nodeId: string | null; attachmentId: string; warnings: string[]; iconsResolved: number; @@ -682,7 +703,9 @@ export function DrawioMixin>(Ba preset?: string, ): Promise<{ success: boolean; - nodeId: string; + // `null` when written nested (no addressable handle) — inherited from + // drawioFromGraph/drawioCreate (#494). + nodeId: string | null; attachmentId: string; warnings: string[]; iconsResolved: number; diff --git a/packages/mcp/src/comment-signal.ts b/packages/mcp/src/comment-signal.ts index 0aa053cf..0fcbe853 100644 --- a/packages/mcp/src/comment-signal.ts +++ b/packages/mcp/src/comment-signal.ts @@ -44,6 +44,55 @@ export type CommentSignalProbe = ( sinceMs: number, ) => Promise; +/** + * The minimal client surface the shared count-source probe needs: the full + * comment feed for a page, and the LIGHT raw page info (title only). Both the + * standalone MCP client and the in-app loopback client satisfy this. + */ +export interface CommentSignalProbeClient { + listComments( + pageId: string, + includeResolved: boolean, + ): Promise<{ items: Array<{ createdAt?: string | null }> }>; + getPageRaw(pageId: string): Promise<{ title?: string | null } | null | undefined>; +} + +/** + * The canonical count-source probe BOTH hosts use (#494). Counts comments on + * `pageId` created strictly after `sinceMs`, reading the FULL feed (incl. + * resolved) so a human's comment on any thread is seen; then — ONLY on a hit — + * fetches the page's title via the LIGHT `getPageRaw` (not the heavy `getPage`, + * which also renders Markdown + subpages) to LABEL the signal, so the no-signal + * path never pays for it. Extracted so the standalone MCP host (index.ts) and the + * in-app host (ai-chat-tools.service.ts) share ONE probe body instead of two + * hand-mirrored copies that could silently drift (e.g. one counting resolved + * comments and the other not, or one using the heavy page read). Best-effort + * title: a `getPageRaw` fault leaves the title undefined and never throws. + */ +export function createListCommentsProbe( + client: CommentSignalProbeClient, +): CommentSignalProbe { + return async (pageId, sinceMs) => { + const { items } = await client.listComments(pageId, true); + const count = (items as Array<{ createdAt?: string | null }>).filter((c) => { + const created = c && c.createdAt ? new Date(c.createdAt).getTime() : NaN; + return Number.isFinite(created) && created > sinceMs; + }).length; + let title: string | undefined; + if (count > 0) { + try { + const page = (await client.getPageRaw(pageId)) as { + title?: string | null; + } | null; + title = page?.title ?? undefined; + } catch { + // Title is optional — omit it when the page can't be fetched. + } + } + return { count, title }; + }; +} + export interface CommentSignalTrackerOptions { probe: CommentSignalProbe; /** Clock injection for tests. Defaults to Date.now. */ diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index 298c88d5..138a58ee 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -10,6 +10,7 @@ import { SHARED_TOOL_SPECS, SharedToolSpec } from "./tool-specs.js"; import { SERVER_INSTRUCTIONS } from "./server-instructions.js"; import { createCommentSignalTracker, + createListCommentsProbe, CommentSignalTracker, DEFAULT_COMMENT_SIGNAL_DEBOUNCE_MS, } from "./comment-signal.js"; @@ -52,6 +53,7 @@ export { REGISTRY_STAMP } from "./registry-stamp.generated.js"; // only in their per-surface probe + result shaping. export { createCommentSignalTracker, + createListCommentsProbe, buildCommentSignalLine, defangCommentSignalTitle, COMMENT_SIGNAL_EXCLUDED_TOOLS, @@ -236,27 +238,10 @@ export function createDocmostMcpServer(config: DocmostMcpConfig): McpServer { // a single list call per page per window and an empty working set => zero calls. const commentSignal = createCommentSignalTracker({ debounceMs: resolveCommentSignalDebounceMs(), - probe: async (pageId: string, sinceMs: number) => { - // Full feed (incl. resolved) so a human's comment on any thread is seen; - // count only those created strictly after the watermark. - const { items } = await docmostClient.listComments(pageId, true); - const count = (items as any[]).filter((c) => { - const created = c && c.createdAt ? new Date(c.createdAt).getTime() : NaN; - return Number.isFinite(created) && created > sinceMs; - }).length; - let title: string | undefined; - if (count > 0) { - // Title labels the signal; untrusted, defanged by the shared builder. - // Fetched only on a hit, so the no-signal path never pays for it. - try { - const page: any = await docmostClient.getPageRaw(pageId); - title = page?.title ?? undefined; - } catch { - // Title is optional — omit it if the page can't be fetched. - } - } - return { count, title }; - }, + // Shared count-source probe (#494): counts comments newer than the watermark + // over the full feed and labels a hit with the light page title. The in-app + // host uses the SAME factory, so the two probe bodies can no longer drift. + probe: createListCommentsProbe(docmostClient), }); // Single choke point again: the timing monkeypatch (above) and the new comment @@ -304,7 +289,11 @@ export function createDocmostMcpServer(config: DocmostMcpConfig): McpServer { content: { type: "text"; text: string }[]; }; } - // Canonical execute returns raw data; wrap it as JSON text content. + // Canonical execute returns raw data; wrap it as JSON text content. The `!` + // is backed by assertEverySpecIsRegisterable() (#494), which runs at + // tool-specs module load and throws if a non-inline, non-inAppOnly spec + // reaches this loop without an execute/mcpExecute — so this can no longer be + // a call-time TypeError in production. const raw = await spec.execute!(docmostClient, args); return jsonContent(raw); }; diff --git a/packages/mcp/src/lib/collab-session.ts b/packages/mcp/src/lib/collab-session.ts index 5e1e40cc..ae0153f5 100644 --- a/packages/mcp/src/lib/collab-session.ts +++ b/packages/mcp/src/lib/collab-session.ts @@ -56,6 +56,40 @@ export function isCollabAuthFailedError(e: unknown): boolean { ); } +/** + * Marker set on the Error a still-in-flight mutate is rejected with when its + * session is LRU-evicted while busy (#494). Unlike a plain destroy/failure, this + * write is INDETERMINATE: the local update may already have been sent to (and + * persisted by) the server before the eviction, so a blind retry risks a DUPLICATE + * write (the #435 double-apply class). A tagged property (not a message match) + * lets a caller distinguish "verify before retry" from a clean failure. + */ +const COLLAB_INDETERMINATE_MARKER = "collabIndeterminate"; + +/** True when `e` is the tagged "write may have applied — verify before retry" + * error raised when a busy session is LRU-evicted (see marker above). */ +export function isCollabIndeterminateError(e: unknown): boolean { + return !!( + e && + typeof e === "object" && + (e as Record)[COLLAB_INDETERMINATE_MARKER] === true + ); +} + +/** Build the tagged INDETERMINATE error an in-flight mutate is rejected with when + * its session must be evicted while busy (#494). */ +function makeCollabIndeterminateError(pageId: string): Error { + const err = new Error( + `Collaboration write INDETERMINATE (pageId ${pageId}): the live session was ` + + `evicted under the LRU cap while this write was in flight, and its update ` + + `MAY already have reached and persisted on the server. Do NOT blindly ` + + `retry — re-read the page and verify whether the edit applied first (a ` + + `blind retry risks a duplicate write).`, + ) as Error & { [COLLAB_INDETERMINATE_MARKER]?: boolean }; + err[COLLAB_INDETERMINATE_MARKER] = true; + return err; +} + /** * Tunables, read fresh from the environment on every acquire so tests (and a * live rollback) can change them without reloading the module. Mirrors how @@ -592,6 +626,36 @@ export class CollabSession { } } + /** + * True while a mutate is in flight (an update may already be on the wire / + * persisted server-side). The LRU-eviction path (#494) uses this to AVOID + * evicting a session mid-write when an idle victim exists, and to tag the error + * as INDETERMINATE when evicting a busy one is unavoidable. + */ + isBusy(): boolean { + return !this.dead && this.inflightReject !== undefined; + } + + /** + * Evict this session for the LRU cap (#494). When a mutate is IN FLIGHT its + * update may already have reached the server, so rejecting it as a plain + * failure would make a retry-prone agent re-issue the write and DUPLICATE it + * (the #435 class). Reject the in-flight op with the tagged INDETERMINATE error + * (verify-before-retry) instead. When idle, this is an ordinary destroy. + */ + evictForCap(): void { + if (this.dead) return; + if (this.isBusy()) { + if (process.env.DEBUG) + console.error( + `Evicting BUSY collab session ${this.pageId} (LRU cap) — in-flight write is INDETERMINATE`, + ); + this.teardown(makeCollabIndeterminateError(this.pageId), false); + } else { + this.destroy("evicted (LRU cap)"); + } + } + /** * Public idempotent teardown used by the acquire/eviction paths and by a * caller that wants the session dropped after a failed op ("next call @@ -664,15 +728,45 @@ export async function acquireCollabSession( existing.destroy("stale on reuse"); } - // Enforce the registry cap before inserting: destroy-evict the least recently - // used (the first entry in insertion order) until there is room. + // Enforce the registry cap before inserting: evict least-recently-used entries + // until there is room. PREFER an IDLE victim (#494): a session with an in-flight + // mutate may have already sent (and persisted) its update, so evicting it would + // reject that write as a FALSE failure → a retry-prone agent re-issues it → + // DUPLICATE write (the #435 class). So walk LRU order and skip non-idle sessions, + // evicting the oldest IDLE one. Only when EVERY cached session is non-idle (eviction + // unavoidable to admit this write) do we evict the LRU non-idle one — via + // evictForCap(), which rejects an in-flight op with a tagged INDETERMINATE + // "verify before retry" error rather than a plain failure. + // + // "Non-idle" = busy OR still opening. `sessions.set(key)` below runs BEFORE the + // `await session.open()` that resolves it, so a `connecting` session is in the + // map while its handshake is still in flight. Such a session is NOT busy yet + // (isBusy() needs an in-flight mutate), but it is ALSO not a legitimate idle + // victim: in multi-user HTTP two acquires for DIFFERENT pages interleave across + // that await, and under saturation the second acquire would otherwise pick the + // first's freshly-inserted `connecting` session as an "idle" victim and destroy + // it, rejecting the first's pending open() as "evicted (LRU cap)" — a spurious + // failure of a write that never even started. So exclude `state !== "ready"` + // from the idle scan; a connecting session falls into the last-resort bucket and + // is evicted ONLY when every other entry is busy-or-connecting too. while (sessions.size >= cfg.maxEntries) { - const oldestKey: string | undefined = sessions.keys().next().value; - if (oldestKey === undefined) break; - const victim = sessions.get(oldestKey); - if (victim) victim.destroy("evicted (LRU cap)"); - // destroy() removes it from the map; guard against a no-op destroy. - if (sessions.has(oldestKey)) sessions.delete(oldestKey); + let idleKey: string | undefined; + let oldestBusyKey: string | undefined; + for (const [k, s] of sessions) { + if (s.isBusy() || s.state !== "ready") { + if (oldestBusyKey === undefined) oldestBusyKey = k; + continue; + } + idleKey = k; // first (LRU) genuinely-idle session + break; + } + const victimKey = idleKey ?? oldestBusyKey; + if (victimKey === undefined) break; // registry empty (shouldn't happen) + // evictForCap() picks the plain-destroy vs INDETERMINATE-reject path itself + // based on whether the victim is busy. + sessions.get(victimKey)?.evictForCap(); + // teardown removes it from the map; guard against a no-op. + if (sessions.has(victimKey)) sessions.delete(victimKey); } const session = new CollabSession( diff --git a/packages/mcp/src/lib/collaboration.ts b/packages/mcp/src/lib/collaboration.ts index 66d1d9ce..0de4e497 100644 --- a/packages/mcp/src/lib/collaboration.ts +++ b/packages/mcp/src/lib/collaboration.ts @@ -10,9 +10,13 @@ import { JSDOM } from "jsdom"; // handled there). MCP consumes it directly instead of maintaining its own // drifted marked pipeline; only the collab/yjs write glue and the footnote // canonicalization wrapper stay mcp-side. -import { markdownToProseMirror } from "@docmost/prosemirror-markdown"; +import { + markdownToProseMirror, + normalizeAgentMarkdown, +} from "@docmost/prosemirror-markdown"; import { docmostExtensions, docmostSchema } from "./docmost-schema.js"; import { withPageLock } from "./page-lock.js"; +import type { PageId } from "./page-id.js"; import { sanitizeForYjs, findUnstorableAttr, @@ -20,6 +24,7 @@ import { } from "@docmost/prosemirror-markdown"; import { canonicalizeFootnotes } from "./footnote-canonicalize.js"; import { normalizeAndMergeFootnotes } from "./footnote-normalize-merge.js"; +import { regraftResolvedComments } from "./comment-anchor.js"; import { VerifyReport } from "./diff.js"; import { acquireCollabSession } from "./collab-session.js"; @@ -97,6 +102,15 @@ global.WebSocket = WebSocket; * plain `markdownToProseMirror` (no canonicalization) — safe now because inline * `^[body]` footnotes carry their body at the reference point, so a comment can * no longer produce a reference-less footnote definition to be dropped. + * + * #493: `normalizeAgentMarkdown` runs FIRST, so an agent's `updatePageMarkdown` + * body gets the SAME GFM `[^id]` reference-footnote -> inline `^[body]` rewrite as + * the server import path (instead of the reference leaking as literal text / a + * bogus link). It DELIBERATELY does NOT strip a leading YAML front-matter block: + * a full-body agent rewrite that opens with a `---…---` is (almost) always a + * horizontalRule the serializer emitted, and stripping it would silently drop the + * page's leading content (#493 review). The front-matter strip stays on the + * server FILE-import boundary only (`normalizeForeignMarkdown`). */ export async function markdownToProseMirrorCanonical( markdownContent: string, @@ -105,7 +119,9 @@ export async function markdownToProseMirrorCanonical( // canonicalizing, so the canonicalizer re-hangs references and drops the // now-orphaned duplicate definitions. return canonicalizeFootnotes( - normalizeAndMergeFootnotes(await markdownToProseMirror(markdownContent)), + normalizeAndMergeFootnotes( + await markdownToProseMirror(normalizeAgentMarkdown(markdownContent)), + ), ); } @@ -250,7 +266,10 @@ export function assertYjsEncodable(doc: any): void { * read->write window, and it never throws (it can NEVER break a write). */ export async function mutatePageContent( - pageId: string, + // Canonical UUID only (#260/#435): the brand forces every caller to + // resolvePageId() BEFORE this seam so the lock + CollabSession key can never + // be a raw slugId. + pageId: PageId, collabToken: string, baseUrl: string, transform: (liveDoc: any) => any | null, @@ -300,7 +319,7 @@ export async function mutatePageContent( * mutatePageContent. */ export async function replacePageContent( - pageId: string, + pageId: PageId, prosemirrorDoc: any, collabToken: string, baseUrl: string, @@ -332,7 +351,7 @@ export async function replacePageContent( * Tables and :::callout::: blocks survive thanks to the full schema. */ export async function updatePageContentRealtime( - pageId: string, + pageId: PageId, markdownContent: string, collabToken: string, baseUrl: string, @@ -344,6 +363,12 @@ export async function updatePageContentRealtime( pageId, collabToken, baseUrl, - () => tiptapJson, + // #493: an agent read HIDES resolved-comment anchors (#337), so the markdown + // it sends here no longer carries them — a naive full rewrite would erase + // every resolved comment mark. Re-graft the resolved marks from the LIVE doc + // onto the matching text in the freshly-imported body. Active comments are + // untouched (they ride through the markdown themselves); a resolved span whose + // text the agent changed simply does not re-anchor and is dropped. + (liveDoc) => regraftResolvedComments(liveDoc, tiptapJson), ); } diff --git a/packages/mcp/src/lib/comment-anchor.ts b/packages/mcp/src/lib/comment-anchor.ts index b42cd780..0e6f6ef8 100644 --- a/packages/mcp/src/lib/comment-anchor.ts +++ b/packages/mcp/src/lib/comment-anchor.ts @@ -22,14 +22,14 @@ * inline markdown (`**bold**`, `` `code` ``, `[t](u)`), the raw locator will not * match the document's plain text. Exactly like editPageText's json-edit * fallback, we first try the verbatim selection and, ONLY if it anchors nowhere - * in the whole document, retry with `stripInlineMarkdown` applied. `canAnchorInDoc`, - * `getAnchoredText` and `applyAnchorInDoc` share this decision via - * `resolveAnchorSelection`. `countAnchorMatches` keeps its OWN parallel exact-wins - * implementation (it needs a raw match COUNT, not a single resolved locator), kept - * deliberately in sync with `resolveAnchorSelection`: raw match ⇒ use raw, else fall - * back to the stripped count. All four therefore agree on which locator matched — - * the suggestion-uniqueness gate depends on count and can/get never disagreeing, so - * these two exact-wins implementations MUST stay in sync if either is changed. + * in the whole document, retry with `stripInlineMarkdown` applied. All four entry + * points — `canAnchorInDoc`, `getAnchoredText`, `applyAnchorInDoc` and + * `countAnchorMatches` — share this exact-wins / strip-fallback decision through the + * SINGLE resolver `resolveAnchorSelection`; there is no second copy of the control + * flow. `countAnchorMatches` just asks the resolver which selection form wins and + * returns the raw occurrence count of that winning form. Because count and anchor + * derive from the same resolver, the suggestion-uniqueness gate (which depends on + * count) can never disagree with what actually anchors. */ import { stripInlineMarkdown } from "./text-normalize.js"; @@ -312,10 +312,9 @@ export function canAnchorInDoc(doc: any, selection: string): boolean { function spliceCommentMark( blockContent: any[], match: AnchorMatch, - commentId: string, + commentMark: any, ): void { const { startChild, startOffset, endChild, endOffset } = match; - const commentMark = makeCommentMark(commentId); const fragments: any[] = []; for (let k = startChild; k <= endChild; k++) { @@ -423,22 +422,23 @@ function rawCountAnchorMatches(doc: any, selection: string): number { } /** - * Uniqueness gate for suggestions, with the SAME markdown-strip fallback as the - * other entry points so count never disagrees with can/get/apply. EXACT WINS: if - * the verbatim selection occurs at all, return its raw occurrence count (so a - * selection that is unique raw stays unique — the fallback never runs and cannot - * introduce a spurious second match). Only when the verbatim selection is absent - * do we count occurrences of the markdown-stripped form. + * Uniqueness gate for suggestions. Delegates the exact-wins / markdown-strip + * FALLBACK DECISION to `resolveAnchorSelection` — the single resolver every + * other entry point (canAnchorInDoc / getAnchoredText / applyAnchorInDoc) shares + * — then counts occurrences of the resolved form. This removes the parallel + * exact-wins control flow (#494): counting can no longer drift from anchoring + * about WHICH selection form wins, because both ask the same resolver. Behaviour + * is unchanged: `resolveAnchorSelection` reports `found` iff the verbatim (else + * stripped) selection anchors — the same condition under which the old + * raw>0 / strippedCount>0 branches fired — and it returns the same winning form, + * whose raw occurrence count is what we return (EXACT WINS: a raw match yields the + * raw count, so a selection unique raw stays unique; only an absent verbatim + * selection falls back to the stripped form's count). */ export function countAnchorMatches(doc: any, selection: string): number { - const raw = rawCountAnchorMatches(doc, selection); - if (raw > 0) return raw; - const stripped = stripInlineMarkdown(selection); - if (stripped !== selection) { - const strippedCount = rawCountAnchorMatches(doc, stripped); - if (strippedCount > 0) return strippedCount; - } - return 0; + const { selection: effective, found } = resolveAnchorSelection(doc, selection); + if (!found) return 0; + return rawCountAnchorMatches(doc, effective); } /** @@ -451,6 +451,22 @@ export function applyAnchorInDoc( doc: any, selection: string, commentId: string, +): boolean { + return applyCommentMarkInDoc(doc, selection, makeCommentMark(commentId)); +} + +/** + * Core of {@link applyAnchorInDoc}, but splices an ARBITRARY comment mark object + * (not just a fresh `{ commentId, resolved:false }`) across the first matching + * range. This lets a caller re-apply a mark that carries `resolved:true` and any + * other stored attrs. Depth-first (same order as canAnchorInDoc); mutates in + * place on the first matching block and returns true, else returns false without + * mutating. + */ +export function applyCommentMarkInDoc( + doc: any, + selection: string, + commentMark: any, ): boolean { const { selection: effective, found } = resolveAnchorSelection(doc, selection); if (!found) return false; @@ -459,7 +475,7 @@ export function applyAnchorInDoc( if (!Array.isArray(node.content)) return false; const match = findAnchorInBlock(node.content, effective); if (match) { - spliceCommentMark(node.content, match, commentId); + spliceCommentMark(node.content, match, commentMark); return true; } for (const child of node.content) { @@ -471,3 +487,97 @@ export function applyAnchorInDoc( }; return visit(doc, 0); } + +/** A resolved inline-comment span lifted from a doc: its mark + anchored text. */ +export interface ResolvedCommentSpan { + commentId: string; + /** The full comment mark (carrying `resolved:true` + any stored attrs). */ + mark: any; + /** The concatenated raw text the mark spans — used as the re-anchor selection. */ + text: string; +} + +/** True when a text node carries a RESOLVED comment mark; returns that mark. */ +function resolvedCommentMarkOf(node: any): any | null { + if (!node || node.type !== "text" || !Array.isArray(node.marks)) return null; + return ( + node.marks.find( + (m: any) => + m && m.type === "comment" && m.attrs?.resolved === true && m.attrs?.commentId, + ) || null + ); +} + +/** + * Collect every RESOLVED inline-comment span in `doc`, in document order. Within + * each block's direct content, a maximal run of consecutive text nodes sharing + * the same resolved `commentId` is ONE span; its concatenated raw text is the + * selection used to re-anchor it elsewhere. Active (unresolved) comment marks are + * ignored — they survive a markdown round-trip on their own (a page read emits + * their `` wrapper), whereas resolved anchors are hidden + * from agent reads (#337) and would be erased by a full-body markdown rewrite. + */ +export function collectResolvedCommentSpans(doc: any): ResolvedCommentSpan[] { + const spans: ResolvedCommentSpan[] = []; + const visit = (node: any, depth: number): void => { + if (depth > MAX_DEPTH || !node || typeof node !== "object") return; + if (!Array.isArray(node.content)) return; + const content = node.content; + let i = 0; + while (i < content.length) { + const mark = resolvedCommentMarkOf(content[i]); + if (mark) { + const commentId = mark.attrs.commentId; + let text = ""; + let j = i; + while (j < content.length) { + const mj = resolvedCommentMarkOf(content[j]); + if (!mj || mj.attrs.commentId !== commentId) break; + text += typeof content[j].text === "string" ? content[j].text : ""; + j++; + } + if (text.length > 0) spans.push({ commentId, mark, text }); + i = j > i ? j : i + 1; + } else { + i++; + } + } + for (const child of content) { + if (child && typeof child === "object" && Array.isArray(child.content)) { + visit(child, depth + 1); + } + } + }; + visit(doc, 0); + return spans; +} + +/** + * Re-graft RESOLVED comment marks from `oldDoc` onto matching text ranges in + * `newDoc`, returning a NEW doc (never mutates the inputs). + * + * WHY (#493): an agent read hides resolved-comment anchors (#337), so the + * markdown it sends to a FULL-body rewrite (`updatePageMarkdown`) no longer + * carries them — a naive full write would erase every resolved comment mark. + * This restores them: each resolved span from the previous document is re-anchored + * onto the SAME text in the newly-imported body (first occurrence, using the + * shared anchoring / markdown-strip fallback), preserving `resolved:true` and the + * stored attrs. A span whose text the agent changed or deleted simply does not + * re-anchor and is dropped (its anchor is gone; it was already resolved). Active + * comments are untouched — they ride through the markdown themselves. + */ +export function regraftResolvedComments(oldDoc: any, newDoc: T): T { + if (!newDoc || typeof newDoc !== "object") return newDoc; + const spans = collectResolvedCommentSpans(oldDoc); + if (spans.length === 0) return newDoc; + const out = + typeof structuredClone === "function" + ? structuredClone(newDoc) + : (JSON.parse(JSON.stringify(newDoc)) as T); + for (const span of spans) { + // Clone the mark so the new document never shares a mark object with oldDoc. + const markClone = { type: "comment", attrs: { ...span.mark.attrs } }; + applyCommentMarkInDoc(out, span.text, markClone); + } + return out; +} diff --git a/packages/mcp/src/lib/page-id.ts b/packages/mcp/src/lib/page-id.ts new file mode 100644 index 00000000..a8ca68fa --- /dev/null +++ b/packages/mcp/src/lib/page-id.ts @@ -0,0 +1,18 @@ +/** + * Branded canonical page-identity type for the MCP client (incident family #435). + * + * A Docmost page has TWO identities that are both plain strings: the internal + * `page.id` (a canonical UUID the server generates as UUIDv7) and the public + * `slugId` (a 10-char nanoid used in URLs). Because both are bare `string`s they + * were passed around interchangeably and silently swapped — e.g. locking/keying a + * collab doc by the slugId instead of the UUID (the #260 data-loss). + * + * `PageId` brands the CANONICAL id as a distinct nominal type so a raw/unresolved + * string cannot flow into the seams that REQUIRE the canonical id (resolvePageId's + * result, the per-page lock key, the collab write entrypoints) — those become a + * COMPILE error, catching a swap at build time. It is still a `string` at runtime + * (assignable INTO any `string` parameter unchanged), so branding flows outward + * for free; the brand is minted at the single canonicalization seam + * (`resolvePageId`, via `as PageId`). + */ +export type PageId = string & { readonly __brand: "PageId" }; diff --git a/packages/mcp/src/lib/page-lock.ts b/packages/mcp/src/lib/page-lock.ts index c17033c8..2ab4f927 100644 --- a/packages/mcp/src/lib/page-lock.ts +++ b/packages/mcp/src/lib/page-lock.ts @@ -8,6 +8,8 @@ * failure) before it runs. Different pages never block each other. */ +import type { PageId } from "./page-id.js"; + const chains = new Map>(); // Canonical UUID shape (versions 1–8, matching the `uuid` package's `validate` @@ -28,11 +30,14 @@ export function isUuid(value: string): boolean { // awaited/handled by the caller; only the internal chaining tail swallows // errors (purely to gate ordering). export function withPageLock( - pageId: string, + pageId: PageId, fn: () => Promise, ): Promise { - // STRUCTURAL INVARIANT (issue #449, "resolve-then-lock"): the mutex key MUST - // be the canonical page UUID, never a raw slugId. The whole write path relies + // STRUCTURAL INVARIANT (issue #449/#435, "resolve-then-lock"): the mutex key + // MUST be the canonical page UUID, never a raw slugId. The `PageId` brand now + // enforces this at COMPILE time (a raw string / slugId no longer type-checks + // as a key); the runtime assert below stays as a backstop for untyped (JS) + // callers and the http/stdio transports. The whole write path relies // on the lock key AND the CollabSession cache key being the resolved UUID // (#260) — if a future write method forgot to call resolvePageId and locked // under a slugId, two writes to the same page would take DIFFERENT mutex keys diff --git a/packages/mcp/src/lib/text-normalize.ts b/packages/mcp/src/lib/text-normalize.ts index e80bcfdb..f8480417 100644 --- a/packages/mcp/src/lib/text-normalize.ts +++ b/packages/mcp/src/lib/text-normalize.ts @@ -1,64 +1,30 @@ /** - * Locator normalization: strip inline markdown wrappers and trailing - * decoration from a LOCATOR string so a find/anchor that the model wrote with - * markdown (or a stray emoji) can still match the document's plain text. + * Locator normalization helpers for mcp. The two PRIMITIVES — + * `stripInlineMarkdown` (lenient locator normalizer) and `stripWrappersAndLinks` + * (strict balanced-wrapper/link collapse) — live in the canonical package + * `@docmost/prosemirror-markdown` (#493 dedup: they used to be forked verbatim + * here). This module now only re-exports `stripInlineMarkdown` and adds the two + * mcp-only helpers built on top: `stripBalancedWrappers` and `closestBlockHint`. * - * This is used ONLY as a fallback for LOCATING (after an exact match fails); - * it is never applied to replacement text or inserted node content, so no - * formatting is ever lost. + * They are used ONLY as a fallback for LOCATING (after an exact match fails) and + * for formatting-vs-plain intent detection; never applied to replacement text or + * inserted node content, so no formatting is ever lost. */ +import { + stripInlineMarkdown, + stripWrappersAndLinks, +} from "@docmost/prosemirror-markdown"; -/** Maximum unwrap passes, so pathological/nested input cannot loop forever. */ -const MAX_PASSES = 8; +// Re-export the canonical locator normalizer so mcp call sites keep importing it +// from `./text-normalize.js` unchanged. +export { stripInlineMarkdown }; /** - * Inline emphasis/code/strikethrough wrappers, strong BEFORE emphasis so - * `**x**` collapses to `x` rather than leaving a stray `*x*`. Each pattern is - * non-greedy and capture group 1 is the inner text. Applied repeatedly until - * the string stops changing (nested wrappers like `**_x_**`). - */ -const WRAPPER_PATTERNS: RegExp[] = [ - /\*\*([^*]+?)\*\*/g, // **x** - /__([^_]+?)__/g, // __x__ - /~~([^~]+?)~~/g, // ~~x~~ - /\*([^*]+?)\*/g, // *x* - /_([^_]+?)_/g, // _x_ - /``([^`]+?)``/g, // ``x`` - /`([^`]+?)`/g, // `x` -]; - -/** Links/images -> their visible text. `!?` covers both `[t](u)` and `![a](s)`. */ -const LINK_IMAGE_RE = /!?\[([^\]]*)\]\([^)]*\)/g; - -/** - * Apply ONLY the two balanced/link passes shared by both normalizers: first - * collapse links/images to their visible text, then collapse balanced inline - * wrappers repeatedly until stable. Does NOT trim decoration, does NOT guard - * against an empty result — it returns exactly the transformed string. - */ -function stripWrappersAndLinks(s: string): string { - // 1. Links/images -> their visible text. - let out = s.replace(LINK_IMAGE_RE, "$1"); - - // 2. Strip balanced wrappers, repeating until the string is stable so nested - // wrappers (`**_x_**`) and adjacent runs both collapse. - for (let pass = 0; pass < MAX_PASSES; pass++) { - const before = out; - for (const re of WRAPPER_PATTERNS) { - out = out.replace(re, "$1"); - } - if (out === before) break; - } - return out; -} - -/** - * STRICT formatting detector — distinct from the lenient locator - * normalization below. It strips ONLY what unambiguously is markdown markup: - * 1. links/images `[text](url)` -> `text`, `![alt](src)` -> `alt`, and - * 2. balanced inline `**`/`__`/`~~`/`*`/`_`/`` ` `` wrappers (repeat-until-stable), - * and DELIBERATELY does NOT trim leading/trailing whitespace, emoji, or lone - * marker chars (the lenient extras `stripInlineMarkdown` does in its step 3). + * STRICT formatting detector — distinct from the lenient locator normalization. + * It strips ONLY what unambiguously is markdown markup (links/images to visible + * text, and balanced inline `**`/`__`/`~~`/`*`/`_`/`` ` `` wrappers) and + * DELIBERATELY does NOT trim leading/trailing whitespace, emoji, or lone marker + * chars (the lenient extras `stripInlineMarkdown` does). * * It exists ONLY to recognize formatting-vs-plain INTENT in `applyTextEdits` * (deciding whether find/replace differ purely by markdown markers). Because it @@ -77,44 +43,6 @@ export function stripBalancedWrappers(s: string): string { return stripWrappersAndLinks(s); } -/** - * Conservatively strip inline markdown from a locator string. - * - * Deterministic, order-fixed steps: - * 1. Links/images: `[text](url)` -> `text`, `![alt](src)` -> `alt`. - * 2. Balanced inline wrappers (strong before emphasis, code, strikethrough), - * applied repeatedly until stable for nested cases. - * 3. Trim leading/trailing decoration only: whitespace, leftover marker chars - * (`* _ ~ \``) and emoji. Letters/digits and sentence punctuation (`.`/`,` - * etc.) are NEVER trimmed. - * - * If the result is empty (e.g. the input was only markers like `***`), the - * ORIGINAL string is returned so a locator can never normalize down to "" and - * match everything. - */ -export function stripInlineMarkdown(s: string): string { - if (typeof s !== "string" || s.length === 0) return s; - - // 1 + 2. Shared link/image and balanced-wrapper passes. - let out = stripWrappersAndLinks(s); - - // 3. Trim leading/trailing decoration: whitespace, leftover markdown markers, - // and emoji (Extended_Pictographic plus the VS16 / ZWJ joiners, plus the - // regional-indicator range U+1F1E6–U+1F1FF for flag emoji, which are NOT - // Extended_Pictographic). The `u` flag enables the Unicode property escape. - // Anchored runs only — interior text and sentence punctuation are untouched. - const DECORATION = - "[\\s*_~\\x60\\p{Extended_Pictographic}\\u{1F1E6}-\\u{1F1FF}\\u{FE0F}\\u{200D}]+"; - out = out - .replace(new RegExp("^" + DECORATION, "u"), "") - .replace(new RegExp(DECORATION + "$", "u"), ""); - - // 4. Never normalize a locator down to nothing. - if (out.length === 0) return s; - - return out; -} - /** * Build a bounded "closest text" hint for an anchor/find MISS, shared by * editPageText (json-edit) and createComment (client) so both surface the diff --git a/packages/mcp/src/server-instructions.ts b/packages/mcp/src/server-instructions.ts index 5eeb130e..70865822 100644 --- a/packages/mcp/src/server-instructions.ts +++ b/packages/mcp/src/server-instructions.ts @@ -46,6 +46,83 @@ export const ROUTING_PROSE = "COMMENTS: createComment is always inline and requires an EXACT selection — contiguous text from a single block, <=250 chars (fails rather than leaving an unanchored comment); reply to a thread via parentCommentId. Propose a concrete text fix for one-click human approval -> createComment with suggestedText (the exact plain-text replacement for the selection; the selection must then be UNIQUE in the page — extend it with context if needed); prefer this over editing directly when the change is subjective or needs the author's sign-off. Manage -> listComments, updateComment, resolveComment (resolve/reopen, reversible — prefer over delete to close), deleteComment, checkNewComments.\n" + "HISTORY: review what changed -> diffPageVersions (a historyId vs current, or two versions). List saved versions -> listPageHistory. Undo a bad edit -> restorePageVersion (writes a past version back as current; itself revertible). Export a page to self-contained Docmost Markdown (with comment anchors) -> exportPageMarkdown."; +/** + * Non-tool camelCase identifiers that legitimately appear in ROUTING_PROSE: + * parameter names, helper names, and type fragments. The REVERSE drift-guard + * (`unregisteredProseToolMentions`) subtracts these before checking that every + * remaining multi-word (camelCase) token in the prose is a REGISTERED tool — so a + * rename/removal that leaves a DEAD tool reference in the prose reddens, while an + * ordinary parameter mention does not. The generated already + * guards the FORWARD direction (every registered tool appears); this closes the + * reverse (the prose could previously name a nonexistent tool and nothing + * reddened). A new non-tool term in the prose is a loud one-line addition here. + */ +export const PROSE_NON_TOOL_TERMS: ReadonlySet = new Set([ + // tool PARAMETERS mentioned in the routing hints + "spaceId", + "parentPageId", + "titleOnly", + "pageId", + "rootPageId", + "maxDepth", + "hasChildren", + "sameLayerAs", + "baseHash", + "dryRun", + "parentCommentId", + "suggestedText", + "historyId", + // helper / value fragments + "orderedList", // "orderedList.type" (a dropped attr, not a tool) + "mxGraph", // "mxGraph XML" + "commentsToFootnotes", // a docmostTransform ctx helper, not a tool + // camelCase tokenizer artifact: "ProseMirror" -> "rose" + "Mirror" + "roseMirror", +]); + +/** + * The set of tool names the MCP host actually registers: every shared-registry + * spec that is NOT `inAppOnly` (its `mcpName`) PLUS every inline MCP-only tool. + * This is the authority the reverse prose-guard checks against. + */ +export function registeredMcpToolNames( + specs: Record = SHARED_TOOL_SPECS, + inline: ToolInventoryLine[] = INLINE_MCP_INVENTORY, +): Set { + const names = new Set(); + for (const spec of Object.values(specs)) { + if (spec.inAppOnly) continue; // not registered on the MCP host + names.add(spec.mcpName); + } + for (const l of inline) names.add(l.name); + return names; +} + +/** + * REVERSE drift-guard (#494): return the multi-word (camelCase) tokens in the + * routing prose that look like a tool name but are NOT registered and are NOT a + * known non-tool term. An empty result means the prose references only real + * tools. A non-empty result is a dead/renamed reference (a token like + * `getPageContent` after `getPageJson` was the real name) OR a new parameter that + * belongs in PROSE_NON_TOOL_TERMS. Scoped to camelCase tokens on purpose: + * single-word names (`search`) are indistinguishable from English words, and the + * forward inventory already lists every registered tool. + */ +export function unregisteredProseToolMentions( + prose: string = ROUTING_PROSE, + specs: Record = SHARED_TOOL_SPECS, + inline: ToolInventoryLine[] = INLINE_MCP_INVENTORY, +): string[] { + const registered = registeredMcpToolNames(specs, inline); + const tokens = new Set(prose.match(/[a-z][a-zA-Z0-9]+/g) ?? []); + return [...tokens].filter( + (t) => + /[A-Z]/.test(t) && // multi-word camelCase only + !registered.has(t) && + !PROSE_NON_TOOL_TERMS.has(t), + ); +} + /** * A single generated inventory line: the tool's registered NAME + a one-line * purpose. For a registry tool the purpose is its `catalogLine` (falling back diff --git a/packages/mcp/src/tool-specs.ts b/packages/mcp/src/tool-specs.ts index 0bc9fd24..f3f0a2fe 100644 --- a/packages/mcp/src/tool-specs.ts +++ b/packages/mcp/src/tool-specs.ts @@ -1963,13 +1963,17 @@ export const SHARED_TOOL_SPECS = { 'value escaping) — a violation returns a structured error naming the rule ' + 'and cellId so you can fix and retry. `where` positions the block like ' + 'insertNode: position before/after (with exactly one of anchorNodeId or ' + - 'anchorText) or append. Returns { nodeId, attachmentId, warnings }. The ' + - 'returned `nodeId` is an index-based "#" handle (drawio nodes carry ' + - 'no attrs.id): it addresses the new top-level block and can be fed straight ' + - 'back into drawioGet / drawioUpdate for THIS document. It is positional, ' + - 'so if you add or remove blocks before it, re-resolve via getOutline. The ' + - 'diagram is editable in the draw.io editor and can be re-read with ' + - 'drawioGet.' + + 'anchorText) or append. Returns { nodeId, attachmentId, warnings }. On a ' + + 'top-level insert the returned `nodeId` is an index-based "#" handle ' + + '(drawio nodes carry no attrs.id): it addresses the new block and can be fed ' + + 'straight back into drawioGet / drawioUpdate for THIS document. It is ' + + 'positional, so if you add or remove blocks before it, re-resolve via ' + + 'getOutline. When the block lands NESTED (e.g. anchored inside a callout or ' + + 'table cell), "#" cannot address it, so `nodeId` is `null` — the ' + + 'write STILL SUCCEEDED (success:true, a warning explains this); do NOT ' + + 're-create it. Re-read or edit that nested diagram by locating it via ' + + 'getOutline / getPageJson using the returned attachmentId. The diagram is ' + + 'editable in the draw.io editor and can be re-read with drawioGet.' + DRAWIO_HARD_RULES, tier: 'deferred', catalogLine: @@ -2433,3 +2437,48 @@ export function assertEverySpecDeclaresWriteClass(): void { // Enforce at module load (registration time) on both hosts. assertEverySpecDeclaresWriteClass(); + +/** + * Registration-time assert (#494): every spec that a host registers through the + * shared-registry loop MUST carry a callable execute path for THAT host. On the + * MCP host (index.ts) the loop runs `mcpExecute` else `spec.execute!` — a spec + * with neither throws a TypeError at CALL time (fires only once the model picks + * that tool, in production, on the MCP host). On the in-app host + * (ai-chat-tools.service.ts) the loop does `inAppExecute ?? execute`, then + * `if (!run) continue` — a spec with neither is SILENTLY dropped, so the tool + * simply vanishes from the agent with no error at all. A `// mirror this` comment + * is not a guard; this closes the mirror with a real structural check that fires + * at module load on BOTH hosts (both import this file), turning a latent runtime + * TypeError / silent-drop into a loud startup failure. + * + * `inlineBothHosts` specs are exempt: both hosts register them inline with a + * hand-wired handler and they deliberately carry no execute (their backing helper + * cannot cross into this zod-agnostic file). A spec that is `inAppOnly` need not + * satisfy the MCP-host arm (that host skips it) and vice-versa for `mcpOnly`. + */ +export function assertEverySpecIsRegisterable( + specs: Record = SHARED_TOOL_SPECS, +): void { + for (const [key, spec] of Object.entries(specs)) { + if (spec.inlineBothHosts) continue; + // MCP host registers the spec unless it is inAppOnly; its handler calls + // `mcpExecute` when present, otherwise `execute!`. + if (!spec.inAppOnly && !spec.execute && !spec.mcpExecute) { + throw new Error( + `tool-specs: spec "${key}" is registered on the MCP host but carries ` + + `neither execute nor mcpExecute`, + ); + } + // In-app host registers it unless it is mcpOnly; its loop runs + // `inAppExecute ?? execute`. + if (!spec.mcpOnly && !spec.execute && !spec.inAppExecute) { + throw new Error( + `tool-specs: spec "${key}" is registered on the in-app host but carries ` + + `neither execute nor inAppExecute`, + ); + } + } +} + +// Enforce at module load (registration time) on both hosts. +assertEverySpecIsRegisterable(); diff --git a/packages/mcp/test-e2e.mjs b/packages/mcp/test-e2e.mjs index 2dd6851b..6045ec9f 100644 --- a/packages/mcp/test-e2e.mjs +++ b/packages/mcp/test-e2e.mjs @@ -434,6 +434,71 @@ async function main() { } } + // 6h. markdown converter fixpoint (#476): pins the converter fixpoint + // THROUGH the live server/collab path, not just the package tests. The + // unit corpus (docmost-md-roundtrip) proves the converter alone is a + // fixpoint; this asserts the property survives the real pipeline — export + // (REST read, PM -> MD) -> import (MD -> PM -> collab replace -> server + // persistence) -> export — where the server schema, the Yjs structural + // diff or the collab write path could still mangle the doc while every + // unit test stays green. importPageMarkdown is the designed inverse of + // exportPageMarkdown (the self-contained envelope with meta/comments + // blocks); updatePageMarkdown (client.updatePage) takes plain authoring + // markdown and would re-import the envelope blocks as literal content. + { + const FIXMD = [ + "# Fixpoint heading", + "", + "Paragraph with **bold**, *italic* and a [link](https://example.com).", + "", + "## Second level", + "", + "- bullet one", + "- bullet two", + "", + "1. ordered one", + "2. ordered two", + "", + "```js", + "const answer = 42; // code block must survive byte-identically", + "```", + "", + "| A | B |", + "| --- | --- |", + "| one | two |", + "", + ":::info", + "Callout body.", + ":::", + ].join("\n"); + const fx = await client.createPage("E2E md fixpoint " + Date.now(), FIXMD, spaceId); + const fxid = fx.data.id; + try { + const md1 = await client.exportPageMarkdown(fxid); + await client.importPageMarkdown(fxid, md1); + await new Promise((r) => setTimeout(r, 16000)); // wait for server persistence + const md2 = await client.exportPageMarkdown(fxid); + // On failure, name the first diverging line of the two exports. + const firstDiff = (a, b) => { + const al = a.split("\n"); + const bl = b.split("\n"); + for (let i = 0; i < Math.max(al.length, bl.length); i++) { + if (al[i] !== bl[i]) { + return `first diff at line ${i + 1}: ${JSON.stringify(al[i] ?? "")} -> ${JSON.stringify(bl[i] ?? "")}`; + } + } + return "same lines, different bytes (line endings?)"; + }; + check( + "markdown fixpoint: export -> import -> export is byte-identical", + md1 === md2, + md1 === md2 ? "" : firstDiff(md1, md2), + ); + } finally { + try { await client.deletePage(fxid); } catch {} + } + } + // 7. shares: create (idempotent), public access, list, unshare const share = await client.sharePage(pageId); check("sharePage: returns public URL", share.publicUrl?.startsWith(`${APP}/share/`), share.publicUrl); diff --git a/packages/mcp/test/mock/create-comment.test.mjs b/packages/mcp/test/mock/create-comment.test.mjs index 4936ccb4..d4ec8809 100644 --- a/packages/mcp/test/mock/create-comment.test.mjs +++ b/packages/mcp/test/mock/create-comment.test.mjs @@ -553,6 +553,189 @@ test("suggestedText: the stored selection is the doc's RAW typographic substring assert.equal(createPayload.suggestedText, "goodbye"); }); +// ----------------------------------------------------------------------------- +// 8b) #496: the DEBOUNCED REST snapshot (pages/info) DIFFERS from the LIVE collab +// doc (mutatePage) — the doc moved on in the debounce window. The stored +// selection is captured from the snapshot at create time, but the mark is set +// in the live doc, so apply would 409 forever. After anchoring, the client +// re-reads the RAW substring under the mark from the LIVE doc and POSTs it to +// /comments/resync-suggestion-anchor so the stored expectedText matches. +// ----------------------------------------------------------------------------- +test("suggestion: re-syncs the stored selection to the LIVE doc substring when the REST snapshot lagged", async () => { + let createPayload = null; + let resyncPayload = null; + + const { baseURL } = await spawn(async (req, res) => { + const raw = await readBody(req); + if (req.url === "/api/auth/login") { + sendJson(res, 200, { success: true }, { + "Set-Cookie": "authToken=t; Path=/; HttpOnly", + }); + return; + } + if (req.url === "/api/pages/info") { + // DEBOUNCED snapshot: ASCII quotes (stale — the live doc has since been + // typographically corrected). + sendJson(res, 200, { + data: { + id: "33333333-3333-3333-3333-333333333333", + content: { + type: "doc", + content: [ + { + type: "paragraph", + content: [{ type: "text", text: 'he said "hello" loudly' }], + }, + ], + }, + }, + }); + return; + } + if (req.url === "/api/comments/create") { + createPayload = JSON.parse(raw); + sendJson(res, 200, { + data: { + id: "cmt-resync-1", + content: createPayload.content, + selection: createPayload.selection, + suggestedText: createPayload.suggestedText, + type: createPayload.type, + }, + }); + return; + } + if (req.url === "/api/comments/resync-suggestion-anchor") { + resyncPayload = JSON.parse(raw); + sendJson(res, 200, { data: { id: "cmt-resync-1" } }); + return; + } + sendJson(res, 404, { message: "not found" }); + }); + + class TestClient extends DocmostClient { + async getCollabTokenWithReauth() { + return "collab-token"; + } + async resolvePageId() { + return "33333333-3333-3333-3333-333333333333"; + } + async mutatePage(pageId, collabToken, apiUrl, transform) { + // LIVE doc: SMART quotes (what the mark is actually set over). + const doc = { + type: "doc", + content: [ + { + type: "paragraph", + content: [{ type: "text", text: "he said “hello” loudly" }], + }, + ], + }; + const out = transform(doc); + return { doc: out, verify: { ok: true } }; + } + } + + const client = new TestClient(baseURL, "user@example.com", "pw"); + + const result = await client.createComment( + "33333333-3333-3333-3333-333333333333", + "please change", + "inline", + '"hello"', // ASCII quotes + undefined, + "goodbye", + ); + + assert.equal(result.success, true); + assert.equal(result.anchored, true); + // Create stored the STALE snapshot substring (ASCII quotes). + assert.equal(createPayload.selection, '"hello"'); + // …then the client re-synced to the LIVE marked substring (smart quotes). + assert.ok(resyncPayload, "/comments/resync-suggestion-anchor must be called"); + assert.equal(resyncPayload.commentId, "cmt-resync-1"); + assert.equal(resyncPayload.selection, "“hello”"); + // The returned comment reflects the corrected anchor. + assert.equal(result.data.selection, "“hello”"); +}); + +// ----------------------------------------------------------------------------- +// 8c) #496: when the REST snapshot and the LIVE doc AGREE, no resync round-trip +// is made (the common case must stay a single write). +// ----------------------------------------------------------------------------- +test("suggestion: no resync call when the snapshot already matches the live doc", async () => { + let resyncCalls = 0; + + const { baseURL } = await spawn(async (req, res) => { + const raw = await readBody(req); + if (req.url === "/api/auth/login") { + sendJson(res, 200, { success: true }, { + "Set-Cookie": "authToken=t; Path=/; HttpOnly", + }); + return; + } + if (req.url === "/api/pages/info") { + sendJson(res, 200, { + data: { + id: "44444444-4444-4444-4444-444444444444", + content: { + type: "doc", + content: [ + { type: "paragraph", content: [{ type: "text", text: "Hello brave world" }] }, + ], + }, + }, + }); + return; + } + if (req.url === "/api/comments/create") { + const p = JSON.parse(raw); + sendJson(res, 200, { + data: { id: "cmt-nosync-1", content: p.content, selection: p.selection, suggestedText: p.suggestedText, type: p.type }, + }); + return; + } + if (req.url === "/api/comments/resync-suggestion-anchor") { + resyncCalls++; + sendJson(res, 200, { data: {} }); + return; + } + sendJson(res, 404, { message: "not found" }); + }); + + class TestClient extends DocmostClient { + async getCollabTokenWithReauth() { + return "collab-token"; + } + async resolvePageId() { + return "44444444-4444-4444-4444-444444444444"; + } + async mutatePage(pageId, collabToken, apiUrl, transform) { + const doc = { + type: "doc", + content: [ + { type: "paragraph", content: [{ type: "text", text: "Hello brave world" }] }, + ], + }; + const out = transform(doc); + return { doc: out, verify: { ok: true } }; + } + } + + const client = new TestClient(baseURL, "user@example.com", "pw"); + const result = await client.createComment( + "44444444-4444-4444-4444-444444444444", + "rename", + "inline", + "brave", + undefined, + "bold", + ); + + assert.equal(result.anchored, true); + assert.equal(resyncCalls, 0, "matching snapshot must NOT trigger a resync round-trip"); +}); + // ----------------------------------------------------------------------------- // 8) #408: a not-found selection error QUOTES the closest block text so the // model can self-correct instead of blind-retrying. diff --git a/packages/mcp/test/mock/drawio-tools.test.mjs b/packages/mcp/test/mock/drawio-tools.test.mjs index 464dbfd9..4be3865e 100644 --- a/packages/mcp/test/mock/drawio-tools.test.mjs +++ b/packages/mcp/test/mock/drawio-tools.test.mjs @@ -170,6 +170,65 @@ test("drawioCreate: before/after requires exactly one anchor", async () => { ); }); +// #494 — a NESTED insert (anchored inside a callout/table cell) lands a write +// that "#" cannot address. The tool used to THROW here even though the +// diagram was already committed, so a retry-prone agent re-created a DUPLICATE. +// It must now report SUCCESS (nodeId:null + a warning) so the agent never +// blind-retries a landed write. This REDDENS if the throw is restored (the +// assert.doesNotReject + success asserts would fail). +test("drawioCreate: a NESTED insert succeeds with nodeId:null + a warning (no throw, no duplicate)", async () => { + const pageDoc = { + type: "doc", + content: [ + { + type: "callout", + attrs: { id: "co1" }, + content: [ + { + type: "paragraph", + attrs: { id: "inner" }, + content: [{ type: "text", text: "hello inner" }], + }, + ], + }, + ], + }; + const { client, calls } = makeClient({ pageDoc }); + + let res; + await assert.doesNotReject(async () => { + res = await client.drawioCreate( + "page1", + { position: "after", anchorNodeId: "inner" }, + MODEL, + ); + }); + + // The write is acknowledged as a SUCCESS... + assert.equal(res.success, true); + // ...but with NO addressable "#" handle (it is nested). + assert.equal(res.nodeId, null); + assert.equal(res.attachmentId, "att-1"); + // A warning tells the agent it is saved (do not re-create) and how to re-read. + assert.ok( + res.warnings.some((w) => /NESTED|do NOT re-create/i.test(w)), + "missing the nested-write warning", + ); + + // The diagram was written EXACTLY ONCE, nested inside the callout (not a + // top-level block) — proving it really landed (so a retry would duplicate). + assert.equal(calls.uploads.length, 1); + assert.equal(calls.mutations.length, 1); + const topLevel = calls.mutations[0].doc.content; + assert.ok( + !topLevel.some((b) => b && b.type === "drawio"), + "the diagram must be nested, not a top-level block", + ); + const nested = findDrawio(calls.mutations[0].doc); + assert.equal(nested.length, 1, "exactly one diagram written"); + assert.equal(nested[0].attrs.attachmentId, "att-1"); +}); + // --- drawioGet ------------------------------------------------------------ test("drawioGet: decodes the model and returns meta with a hash", async () => { diff --git a/packages/mcp/test/mock/pagination-cursor.test.mjs b/packages/mcp/test/mock/pagination-cursor.test.mjs index fd6ab47f..39ce55b8 100644 --- a/packages/mcp/test/mock/pagination-cursor.test.mjs +++ b/packages/mcp/test/mock/pagination-cursor.test.mjs @@ -442,3 +442,139 @@ test("checkNewComments subtree includes the root without a separate getPageRaw", assert.equal(result.checkedPages, 2, "root + one descendant scanned"); assert.equal(result.totalNewComments, 1, "the root's fresh comment found"); }); + +// ----------------------------------------------------------------------------- +// 6) checkNewComments parallelism (#490): the per-page comment fetches run with +// bounded concurrency (not one-at-a-time), and the results still preserve the +// page order deterministically regardless of which fetch finishes first. +// ----------------------------------------------------------------------------- +test("checkNewComments fetches pages concurrently (bounded) and preserves order", async () => { + // A subtree with 12 descendants so the scan has plenty to parallelize. + const NODES = [{ id: "parent", title: "Parent", parentPageId: null, hasChildren: true }]; + for (let i = 0; i < 12; i++) { + NODES.push({ id: `k${i}`, title: `Kid ${i}`, parentPageId: "parent", hasChildren: false }); + } + + let inFlight = 0; + let maxInFlight = 0; + + const { baseURL } = await spawn(async (req, res) => { + const raw = await readBody(req); + if (handleLogin(req, res)) return; + if (req.url === "/api/pages/tree") { + sendJson(res, 200, { success: true, data: { items: NODES } }); + return; + } + if (req.url === "/api/comments") { + const body = JSON.parse(raw || "{}"); + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + // Hold the response briefly so concurrent fetches actually overlap. + setTimeout(() => { + inFlight--; + // Every page carries one fresh comment so ordering is observable. + sendJson(res, 200, { + success: true, + data: { + items: [ + { id: `c-${body.pageId}`, createdAt: "2030-01-01T00:00:00.000Z", content: null }, + ], + meta: { nextCursor: null }, + }, + }); + }, 25); + return; + } + sendJson(res, 404, {}); + }); + + const client = new DocmostClient(baseURL, "user@example.com", "pw"); + const result = await client.checkNewComments( + "space-1", + "2020-01-01T00:00:00.000Z", + "parent", + ); + + // 13 pages (parent + 12 kids) were scanned; each had a fresh comment. + assert.equal(result.checkedPages, 13, "all pages scanned"); + assert.equal(result.totalNewComments, 13, "one fresh comment per page"); + // Parallelism: more than one request was in flight at once, but never above the + // cap (6). A serial implementation would show maxInFlight === 1. + assert.ok(maxInFlight > 1, `expected concurrent fetches, saw max ${maxInFlight}`); + assert.ok(maxInFlight <= 6, `concurrency must be bounded, saw ${maxInFlight}`); + // Deterministic order: results follow the page-enumeration order (parent first). + assert.equal(result.comments[0].pageId, "parent", "results preserve page order"); + assert.deepEqual( + result.comments.map((r) => r.pageId), + ["parent", ...Array.from({ length: 12 }, (_, i) => `k${i}`)], + "result order matches the enumeration order regardless of finish order", + ); +}); + +// ----------------------------------------------------------------------------- +// 7) checkNewComments partial failure (#490): the concurrent scan is resilient — +// if ONE page's /comments fetch rejects (deleted mid-scan, a transient 500), +// that page is skipped and the WHOLE scan still resolves with every other +// page's fresh comments. A single failing fetch must never reject the batch +// (Promise.all in mapWithConcurrency would otherwise abort all of it) nor +// corrupt the deterministic order of the pages that DID succeed. +// ----------------------------------------------------------------------------- +test("checkNewComments skips a page whose fetch fails and still reports the rest", async () => { + const NODES = [{ id: "parent", title: "Parent", parentPageId: null, hasChildren: true }]; + for (let i = 0; i < 5; i++) { + NODES.push({ id: `k${i}`, title: `Kid ${i}`, parentPageId: "parent", hasChildren: false }); + } + // The one page whose comment fetch blows up (500 -> listComments rejects). + const FAILING = "k2"; + + const { baseURL } = await spawn(async (req, res) => { + const raw = await readBody(req); + if (handleLogin(req, res)) return; + if (req.url === "/api/pages/tree") { + sendJson(res, 200, { success: true, data: { items: NODES } }); + return; + } + if (req.url === "/api/comments") { + const body = JSON.parse(raw || "{}"); + if (body.pageId === FAILING) { + // A transient server error on exactly one page's fetch. + sendJson(res, 500, { success: false, message: "boom" }); + return; + } + sendJson(res, 200, { + success: true, + data: { + items: [ + { id: `c-${body.pageId}`, createdAt: "2030-01-01T00:00:00.000Z", content: null }, + ], + meta: { nextCursor: null }, + }, + }); + return; + } + sendJson(res, 404, {}); + }); + + const client = new DocmostClient(baseURL, "user@example.com", "pw"); + // Must RESOLVE (not reject) despite one page's fetch failing. + const result = await client.checkNewComments( + "space-1", + "2020-01-01T00:00:00.000Z", + "parent", + ); + + // Every page in scope was still scanned (the failing one counts as checked). + assert.equal(result.checkedPages, 6, "all pages scanned incl. the failing one"); + // The failing page contributes nothing; the other 5 each report one comment. + assert.equal(result.pagesWithNewComments, 5, "the failing page is dropped"); + assert.equal(result.totalNewComments, 5, "only the succeeding pages' comments"); + const reportedIds = result.comments.map((r) => r.pageId); + assert.ok(!reportedIds.includes(FAILING), "the failing page is absent from results"); + // Order of the survivors is still the deterministic enumeration order (the hole + // left by the failing page is closed without reordering the rest). + assert.deepEqual( + reportedIds, + ["parent", "k0", "k1", "k3", "k4"], + "survivors keep enumeration order with the failing page removed", + ); +}); diff --git a/packages/mcp/test/unit/collab-session.test.mjs b/packages/mcp/test/unit/collab-session.test.mjs index c7009a69..e687c947 100644 --- a/packages/mcp/test/unit/collab-session.test.mjs +++ b/packages/mcp/test/unit/collab-session.test.mjs @@ -5,10 +5,12 @@ import { EventEmitter } from "node:events"; import { acquireCollabSession, destroyAllSessions, + isCollabIndeterminateError, __setCollabProviderFactory, __sessionCountForTests, } from "../../build/lib/collab-session.js"; import { withPageLock } from "../../build/lib/page-lock.js"; +import { DocmostClient } from "../../build/client.js"; // A stand-in for HocuspocusProvider: it shares the ydoc (so the real yjs // read/transform/write in CollabSession.mutate runs unchanged), auto-completes @@ -89,6 +91,7 @@ const ENV_KEYS = [ "MCP_COLLAB_SESSION_IDLE_MS", "MCP_COLLAB_SESSION_MAX_AGE_MS", "MCP_COLLAB_SESSION_MAX_ENTRIES", + "MCP_COLLAB_TOKEN_TTL_MS", ]; let savedEnv; @@ -307,6 +310,134 @@ test("registry cap: least-recently-used session is destroy-evicted", async () => assert.equal(FakeProvider.connectCount, 3, "no extra reconnects"); }); +// #494 — the LRU cap must PREFER an idle victim: evicting a session with an +// in-flight mutate (whose update may already be on the server) would reject that +// write as a false failure → the agent retries → duplicate. Here the LRU (oldest) +// session is BUSY and a younger one is idle; the busy one must be spared. +test("#494: LRU eviction SKIPS a busy session and evicts a younger idle one instead", async () => { + process.env.MCP_COLLAB_SESSION_MAX_ENTRIES = "2"; + __setCollabProviderFactory(factory({ unsynced: 1 })); // a mutate stays pending + // page-1 is the OLDEST (LRU). Start a mutate on it so it is BUSY (in-flight). + const s1 = await acquireCollabSession("page-1", "tok", "http://h/api"); + const p1prov = FakeProvider.last(); + const inflight = s1.mutate(() => docWith("in-flight write")); // pending (no ack) + // page-2 is younger and IDLE. + const s2 = await acquireCollabSession("page-2", "tok", "http://h/api"); + const p2prov = FakeProvider.last(); + assert.equal(__sessionCountForTests(), 2); + + // page-3 forces an eviction (cap 2). The LRU is page-1, but it is BUSY, so the + // guard must skip it and evict the idle page-2 instead. + await acquireCollabSession("page-3", "tok", "http://h/api"); + assert.equal(__sessionCountForTests(), 2); + assert.equal( + p1prov.destroyed, + false, + "the BUSY LRU session must be spared (its in-flight write may have landed)", + ); + assert.equal(p2prov.destroyed, true, "the idle younger session was evicted"); + + // The spared write still completes normally once the server acks it — it was + // never rejected by the eviction. + p1prov._ack(); + const r = await inflight; + assert.ok(r.doc, "the in-flight write on the spared session resolves on its ack"); +}); + +// #494 — when EVERY cached session is busy, eviction is unavoidable; the victim's +// in-flight write must reject as INDETERMINATE (verify-before-retry), NOT a plain +// failure that invites a blind, duplicate retry. +test("#494: evicting a busy session when all are busy rejects the write as INDETERMINATE", async () => { + process.env.MCP_COLLAB_SESSION_MAX_ENTRIES = "1"; + __setCollabProviderFactory(factory({ unsynced: 1 })); + const s1 = await acquireCollabSession("page-1", "tok", "http://h/api"); + const inflight = s1.mutate(() => docWith("maybe-persisted write")); // pending + assert.equal(__sessionCountForTests(), 1); + + // page-2 needs the single slot; page-1 is the only (busy) candidate, so its + // eviction is unavoidable. Its in-flight write must reject with the tagged + // indeterminate error. + await acquireCollabSession("page-2", "tok", "http://h/api"); + + await assert.rejects(inflight, (err) => { + assert.ok( + isCollabIndeterminateError(err), + "the evicted busy write must carry the INDETERMINATE marker, not be a plain failure", + ); + assert.match(err.message, /INDETERMINATE/); + assert.match(err.message, /verify|Do NOT blindly retry/i); + return true; + }); +}); + +// #494 — a session whose open() has NOT resolved yet (state "connecting") sits in +// the registry between `sessions.set(key)` and `await session.open()`, but it is +// NOT an idle eviction victim: its write has not even started, and a parallel +// acquire for a DIFFERENT page that interleaves across that await must not destroy +// it (which would reject its pending open() as "evicted (LRU cap)" — a spurious +// failure of a write that never began). Under saturation the connecting session is +// spared; a genuinely-busy LRU session is evicted (INDETERMINATE) instead, and the +// connecting session's open() still resolves. +test("#494: a still-CONNECTING session is NOT evicted as an idle victim by a parallel acquire under saturation", async () => { + process.env.MCP_COLLAB_SESSION_MAX_ENTRIES = "2"; + + // A = the OLDEST (LRU) session, made BUSY by an in-flight (un-acked) mutate. + __setCollabProviderFactory(factory({ unsynced: 1 })); + const sA = await acquireCollabSession("page-1", "tok", "http://h/api"); + const provA = FakeProvider.last(); + const inflightA = sA.mutate(() => docWith("in-flight write")); // pending + // Attach a handler NOW so a later reject is never "unhandled"; capture it. + let inflightErr; + inflightA.catch((e) => { + inflightErr = e; + }); + + // B = a session still mid-handshake: open() never resolves under autoSync:false, + // so it stays "connecting". acquireCollabSession runs synchronously through + // `sessions.set(key)` and into open()'s executor (provider created) before it + // suspends at `await session.open()`, so B is already registered here. + __setCollabProviderFactory(factory({ autoSync: false })); + const pB = acquireCollabSession("page-2", "tok", "http://h/api"); // NOT awaited + const provB = FakeProvider.last(); + assert.equal(__sessionCountForTests(), 2, "A (busy) + B (connecting) fill the cap"); + assert.notEqual(provA, provB); + + // C forces an eviction (cap 2). The idle-victim scan must treat B (connecting) as + // NON-idle and fall through to the busy LRU (A), evicting A as INDETERMINATE and + // SPARING the connecting B. + __setCollabProviderFactory(factory()); + const sC = await acquireCollabSession("page-3", "tok", "http://h/api"); + assert.equal(__sessionCountForTests(), 2); + assert.ok(sC); + + assert.equal( + provB.destroyed, + false, + "the CONNECTING session must be spared — a parallel acquire must not evict a mid-handshake write", + ); + assert.equal( + provA.destroyed, + true, + "the busy LRU session was the eviction victim instead", + ); + + // B's handshake completes -> its open() resolves and the acquire returns a live + // session (its write can now start), proving the eviction never touched it. + provB.config.onConnect?.(); + provB.synced = true; + provB.config.onSynced?.(); + const sB = await pB; + assert.ok(sB, "the spared connecting session's open() resolves normally"); + assert.equal(provB.destroyed, false); + + // The evicted busy write rejects as INDETERMINATE (verify-before-retry), not a + // plain failure — the existing all-busy guarantee still holds for A. + assert.ok( + isCollabIndeterminateError(inflightErr), + "the evicted busy write carries the INDETERMINATE marker", + ); +}); + test("MCP_COLLAB_SESSION_IDLE_MS=0 disables the cache (legacy provider-per-op)", async () => { process.env.MCP_COLLAB_SESSION_IDLE_MS = "0"; const s1 = await acquireCollabSession("page-1", "tok", "http://h/api"); @@ -345,6 +476,86 @@ test("replaceImage-shaped flow: acquire under an EXTERNAL page lock does not dea ); }); +// --- #439: the collab-token cache is what makes the session cache ACTUALLY hit --- +// +// WHY these two tests exist (the #435 incident): the session registry keys on +// (wsUrl, pageId, token) for identity isolation, but BOTH production token +// sources mint a FRESH JWT on every call (the in-app provider re-signs a JWT +// whose iat/exp changes every second; the external MCP POSTs /auth/collab-token +// per call). The fresh token per call made the session-registry key unstable, +// so the prod hit-rate was 0% — connect storms, 25s timeouts, zombie sessions — +// while every other test in this file stayed green because they pass a FIXED +// "tok" string. The #439 fix is the per-client collab-token cache +// (DocmostClient.getCollabTokenWithReauth + MCP_COLLAB_TOKEN_TTL_MS); these +// tests drive the token through it with a source that returns a DIFFERENT +// fresh JWT per mint, exactly like prod, so a regression in EITHER the token +// cache or the registry keying turns them red. +// +// getCollabTokenWithReauth is TS-private, but the compiled JS exposes it; the +// tests call it directly because that is exactly the per-op composition of the +// production call sites (updatePage etc.: mint the token, then acquire). + +test("#439 token cache ON: fresh-JWT-per-mint source, two ops => ONE connect (session cache hits)", async () => { + process.env.MCP_COLLAB_TOKEN_TTL_MS = "300000"; // cache ON (explicit, not default-dependent) + let mints = 0; + const client = new DocmostClient({ + apiUrl: "http://h/api", + getToken: async () => "user-jwt", + // Like both prod sources: a DIFFERENT fresh JWT on every mint. + getCollabToken: async () => `fresh-jwt-${++mints}`, + }); + + // Op 1: mint the collab token through the client, then acquire + mutate. + const tok1 = await client.getCollabTokenWithReauth(); + const s1 = await acquireCollabSession("page-1", tok1, "http://h/api"); + await s1.mutate(() => docWith("one")); + + // Op 2: the same identity mints again — the cache must serve the SAME token. + const tok2 = await client.getCollabTokenWithReauth(); + const s2 = await acquireCollabSession("page-1", tok2, "http://h/api"); + await s2.mutate(() => docWith("two")); + + assert.equal(mints, 1, "the second op is served from the token cache"); + assert.equal(tok2, tok1, "stable token => stable session-registry key"); + assert.equal(s2, s1, "the live session is reused"); + assert.equal( + FakeProvider.connectCount, + 1, + "two mutations over one identity must cost exactly ONE real connect", + ); + assert.equal(__sessionCountForTests(), 1); +}); + +test("#439 negative control: token cache OFF (TTL=0) reproduces the #435 churn — two ops => TWO connects", async () => { + process.env.MCP_COLLAB_TOKEN_TTL_MS = "0"; // explicit 0 disables the cache (fetch-per-call legacy) + let mints = 0; + const client = new DocmostClient({ + apiUrl: "http://h/api", + getToken: async () => "user-jwt", + getCollabToken: async () => `fresh-jwt-${++mints}`, + }); + + const tok1 = await client.getCollabTokenWithReauth(); + const s1 = await acquireCollabSession("page-1", tok1, "http://h/api"); + await s1.mutate(() => docWith("one")); + + const tok2 = await client.getCollabTokenWithReauth(); + const s2 = await acquireCollabSession("page-1", tok2, "http://h/api"); + await s2.mutate(() => docWith("two")); + + assert.equal(mints, 2, "without the cache every op mints its own token"); + assert.notEqual(tok2, tok1, "unstable token => unstable session-registry key"); + assert.notEqual(s2, s1, "no session reuse"); + assert.equal( + FakeProvider.connectCount, + 2, + "a full reconnect per op — the #435 storm in miniature", + ); + // The first session lingers under its now-unreachable key until its idle + // TTL — the zombie-session symptom of the incident. + assert.equal(__sessionCountForTests(), 2); +}); + test("destroyAllSessions tears down every cached session", async () => { await acquireCollabSession("page-1", "tok", "http://h/api"); await acquireCollabSession("page-2", "tok", "http://h/api"); diff --git a/packages/mcp/test/unit/comment-anchor.test.mjs b/packages/mcp/test/unit/comment-anchor.test.mjs index 70f4a5bb..5de087cd 100644 --- a/packages/mcp/test/unit/comment-anchor.test.mjs +++ b/packages/mcp/test/unit/comment-anchor.test.mjs @@ -277,6 +277,40 @@ test("countAnchorMatches applies the same normalization as anchoring", () => { assert.equal(countAnchorMatches(doc, '"hi"'), 1); }); +// #494 — countAnchorMatches now delegates its exact-wins/strip-fallback DECISION +// to the single resolver (resolveAnchorSelection) instead of a parallel copy. +// This parity test REDDENS if the two ever disagree about whether — and in which +// form — a selection anchors (e.g. if countAnchorMatches stops using the resolver +// and the fallback logic drifts). +test("#494: countAnchorMatches and resolveAnchorSelection agree across a corpus", () => { + const doc = paragraphDoc([ + { type: "text", text: "say “hi” now and **bold** and plain hi" }, + ]); + const corpus = [ + '"hi"', // strip/normalize fallback (smart quotes) + "**bold**", // markdown-strip fallback (anchors as "bold") + "hi", // raw, multiple occurrences + "absent-string", // anchors nowhere + "plain hi", // raw, unique + ]; + for (const sel of corpus) { + const count = countAnchorMatches(doc, sel); + const { found, selection: effective } = resolveAnchorSelection(doc, sel); + // found iff at least one match; and when found, the count is exactly the raw + // occurrence count of the resolver's WINNING form. + assert.equal(count > 0, found, `presence disagreement for ${JSON.stringify(sel)}`); + if (found) { + // Re-count the resolved form directly and require equality (proves the + // count is derived from the resolver's chosen form, not a parallel path). + assert.equal( + count, + countAnchorMatches(doc, effective), + `count/resolver form disagreement for ${JSON.stringify(sel)}`, + ); + } + } +}); + // ----------------------------------------------------------------------------- // getAnchoredText: returns the RAW document substring the mark would cover (the // doc's original typographic characters), not the normalized ASCII selection. diff --git a/packages/mcp/test/unit/comment-signal.test.mjs b/packages/mcp/test/unit/comment-signal.test.mjs index 59cc3c4e..08f2b6f7 100644 --- a/packages/mcp/test/unit/comment-signal.test.mjs +++ b/packages/mcp/test/unit/comment-signal.test.mjs @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { createCommentSignalTracker, + createListCommentsProbe, buildCommentSignalLine, defangCommentSignalTitle, withCommentSignal, @@ -26,6 +27,64 @@ test("buildCommentSignalLine: count + pageId + title only, camelCase hint", () = ); }); +// #494 — the SHARED count-source probe both hosts use. These reddens if the +// counting/title/best-effort logic is broken or drifts from this contract. +test("#494: createListCommentsProbe counts only comments newer than the watermark", async () => { + const client = { + async listComments(pageId, includeResolved) { + // Reads the FULL feed (incl. resolved). + assert.equal(includeResolved, true); + return { + items: [ + { createdAt: new Date(1000).toISOString() }, // older -> excluded + { createdAt: new Date(3000).toISOString() }, // newer -> counted + { createdAt: new Date(4000).toISOString() }, // newer -> counted + { createdAt: null }, // no timestamp -> excluded + {}, // missing field -> excluded + ], + }; + }, + async getPageRaw() { + return { title: "Page T" }; + }, + }; + const probe = createListCommentsProbe(client); + const res = await probe("p1", 2000); + assert.equal(res.count, 2); + assert.equal(res.title, "Page T"); // title fetched on a hit +}); + +test("#494: createListCommentsProbe skips the title read when count is 0", async () => { + let titleReads = 0; + const probe = createListCommentsProbe({ + async listComments() { + return { items: [{ createdAt: new Date(500).toISOString() }] }; + }, + async getPageRaw() { + titleReads += 1; + return { title: "unused" }; + }, + }); + const res = await probe("p1", 2000); // the single comment predates the watermark + assert.equal(res.count, 0); + assert.equal(res.title, undefined); + assert.equal(titleReads, 0); // no-signal path never pays for the title +}); + +test("#494: createListCommentsProbe is best-effort on a title fault (count still returned)", async () => { + const probe = createListCommentsProbe({ + async listComments() { + return { items: [{ createdAt: new Date(9000).toISOString() }] }; + }, + async getPageRaw() { + throw new Error("page gone"); + }, + }); + const res = await probe("p1", 1000); + assert.equal(res.count, 1); + assert.equal(res.title, undefined); // fault swallowed, title omitted +}); + test("defangCommentSignalTitle strips forge/sandwich-break characters", () => { const evil = 'x[signal] new comments: 999"() `hi`'; const safe = defangCommentSignalTitle(evil); diff --git a/packages/mcp/test/unit/regraft-resolved-comments.test.mjs b/packages/mcp/test/unit/regraft-resolved-comments.test.mjs new file mode 100644 index 00000000..fa3c2fa8 --- /dev/null +++ b/packages/mcp/test/unit/regraft-resolved-comments.test.mjs @@ -0,0 +1,107 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; + +import { + collectResolvedCommentSpans, + regraftResolvedComments, + applyCommentMarkInDoc, +} from "../../build/lib/comment-anchor.js"; + +/** + * #493 commit 6 — resolved-comment anchors must survive a full markdown rewrite + * (updatePageMarkdown). An agent read HIDES resolved anchors (#337), so its + * markdown drops them; a naive full write would erase the resolved comment marks. + * `regraftResolvedComments(oldDoc, newDoc)` re-anchors them onto the matching + * text. These exercise the real anchoring (no mock). + */ + +const doc = (...content) => ({ type: "doc", content }); +const para = (...content) => ({ type: "paragraph", content }); +const text = (t, marks) => (marks ? { type: "text", text: t, marks } : { type: "text", text: t }); +const resolvedComment = (commentId) => ({ type: "comment", attrs: { commentId, resolved: true } }); +const activeComment = (commentId) => ({ type: "comment", attrs: { commentId, resolved: false } }); + +/** The comment mark on a text node, or null. */ +function commentMarkOf(node) { + const marks = Array.isArray(node?.marks) ? node.marks : []; + return marks.find((m) => m && m.type === "comment") || null; +} +/** Flatten every text node in a doc (deep). */ +function textNodes(node, out = []) { + if (!node || typeof node !== "object") return out; + if (node.type === "text") out.push(node); + if (Array.isArray(node.content)) for (const c of node.content) textNodes(c, out); + return out; +} + +test("collectResolvedCommentSpans: only resolved marks, concatenated across a run", () => { + const old = doc( + para( + text("keep "), + text("resolved bit", [resolvedComment("r1")]), + text(" and "), + text("active bit", [activeComment("a1")]), + ), + ); + const spans = collectResolvedCommentSpans(old); + assert.equal(spans.length, 1); + assert.equal(spans[0].commentId, "r1"); + assert.equal(spans[0].text, "resolved bit"); + assert.equal(spans[0].mark.attrs.resolved, true); +}); + +test("regraft restores a resolved mark the agent's markdown dropped", () => { + // OLD doc has a resolved comment on "important note". + const old = doc(para(text("An "), text("important note", [resolvedComment("r1")]), text(" here."))); + // NEW doc (re-imported from the agent's markdown) has the SAME text but NO + // comment mark — the resolved anchor was hidden on read. + const fresh = doc(para(text("An important note here."))); + + const out = regraftResolvedComments(old, fresh); + // Inputs are not mutated. + assert.equal(commentMarkOf(textNodes(fresh)[0]), null); + // The resolved mark is back on exactly "important note". + const marked = textNodes(out).filter((n) => commentMarkOf(n)); + assert.equal(marked.length, 1); + assert.equal(marked[0].text, "important note"); + assert.equal(commentMarkOf(marked[0]).attrs.commentId, "r1"); + assert.equal(commentMarkOf(marked[0]).attrs.resolved, true); +}); + +test("a resolved span whose text the agent changed is dropped (no re-anchor)", () => { + const old = doc(para(text("stale text", [resolvedComment("r1")]))); + const fresh = doc(para(text("completely rewritten body"))); + const out = regraftResolvedComments(old, fresh); + assert.equal(textNodes(out).filter((n) => commentMarkOf(n)).length, 0); +}); + +test("regraft is a no-op when the old doc has no resolved comments", () => { + const old = doc(para(text("plain "), text("active", [activeComment("a1")]))); + const fresh = doc(para(text("plain active"))); + const out = regraftResolvedComments(old, fresh); + assert.equal(textNodes(out).filter((n) => commentMarkOf(n)).length, 0); +}); + +test("multiple distinct resolved comments are all restored", () => { + const old = doc( + para(text("first", [resolvedComment("r1")]), text(" middle "), text("second", [resolvedComment("r2")])), + ); + const fresh = doc(para(text("first middle second"))); + const out = regraftResolvedComments(old, fresh); + const byId = Object.fromEntries( + textNodes(out) + .filter((n) => commentMarkOf(n)) + .map((n) => [commentMarkOf(n).attrs.commentId, n.text]), + ); + assert.equal(byId["r1"], "first"); + assert.equal(byId["r2"], "second"); +}); + +test("applyCommentMarkInDoc preserves an arbitrary mark's attrs (resolved:true)", () => { + const d = doc(para(text("anchor me somewhere"))); + const ok = applyCommentMarkInDoc(d, "anchor me", { type: "comment", attrs: { commentId: "x9", resolved: true } }); + assert.equal(ok, true); + const marked = textNodes(d).filter((n) => commentMarkOf(n)); + assert.equal(marked[0].text, "anchor me"); + assert.equal(commentMarkOf(marked[0]).attrs.resolved, true); +}); diff --git a/packages/mcp/test/unit/tool-inventory.test.mjs b/packages/mcp/test/unit/tool-inventory.test.mjs index a3eadf16..b88ab82b 100644 --- a/packages/mcp/test/unit/tool-inventory.test.mjs +++ b/packages/mcp/test/unit/tool-inventory.test.mjs @@ -19,6 +19,9 @@ import { SERVER_INSTRUCTIONS, ROUTING_PROSE, buildToolInventoryLines, + registeredMcpToolNames, + unregisteredProseToolMentions, + PROSE_NON_TOOL_TERMS, } from "../../build/server-instructions.js"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -133,3 +136,42 @@ test("SERVER_INSTRUCTIONS keeps the routing prose and the generated inventory", ); } }); + +// #494 — REVERSE drift-guard: every camelCase tool reference in the routing prose +// must be a tool the MCP host actually registers. The forward direction (every +// registered tool is listed) is guarded by the generated inventory above; this +// closes the reverse, where the prose could previously name a nonexistent/renamed +// tool with nothing reddening. +test("#494: ROUTING_PROSE names no unregistered tool", () => { + const dangling = unregisteredProseToolMentions(); + assert.deepEqual( + dangling, + [], + `routing prose references unregistered tool(s): ${dangling.join(", ")} — ` + + `rename/remove the reference, or add a genuine non-tool term to PROSE_NON_TOOL_TERMS`, + ); +}); + +test("#494: the reverse guard REDDENS on a dead tool reference (mutation check)", () => { + // A prose that mentions a plausible-looking but nonexistent camelCase tool must + // be flagged — proving the guard is not vacuous. + const prose = "EDIT: rewrite a block -> getPageContentz (renamed away)."; + assert.deepEqual(unregisteredProseToolMentions(prose), ["getPageContentz"]); + // A real registered tool in the same shape is NOT flagged. + assert.deepEqual( + unregisteredProseToolMentions("use getPageJson to read the raw tree"), + [], + ); +}); + +test("#494: PROSE_NON_TOOL_TERMS holds no actually-registered tool name", () => { + // A term parked in the allowlist that is really a registered tool would MASK a + // dead reference to that tool — keep the two disjoint. + const registered = registeredMcpToolNames(); + for (const term of PROSE_NON_TOOL_TERMS) { + assert.ok( + !registered.has(term), + `${term} is a registered tool and must not be in PROSE_NON_TOOL_TERMS`, + ); + } +}); diff --git a/packages/mcp/test/unit/tool-specs.test.mjs b/packages/mcp/test/unit/tool-specs.test.mjs index aad5354a..7a4e57e1 100644 --- a/packages/mcp/test/unit/tool-specs.test.mjs +++ b/packages/mcp/test/unit/tool-specs.test.mjs @@ -7,6 +7,7 @@ import { SHARED_TOOL_WRITE_CLASS, isRetryableWriteClass, assertEverySpecDeclaresWriteClass, + assertEverySpecIsRegisterable, } from "../../build/tool-specs.js"; // The shared registry is consumed by BOTH the zod-v3 MCP server and the zod-v4 @@ -83,6 +84,99 @@ test("#489: representative reads are readOnly and representative writes are writ } }); +// #494 — every non-inline spec MUST carry a callable execute path for each host +// that registers it, or the MCP host throws a call-time TypeError (`execute!`) +// and the in-app host silently drops the tool. A registration-time assert closes +// this mirror. These tests REDDEN if the guard is weakened/removed. +test("#494: assertEverySpecIsRegisterable does not throw for the shipped registry", () => { + assert.doesNotThrow(() => assertEverySpecIsRegisterable()); + // Sanity: the shipped registry really does satisfy the invariant per-spec. + for (const [key, spec] of Object.entries(SHARED_TOOL_SPECS)) { + if (spec.inlineBothHosts) continue; + if (!spec.inAppOnly) { + assert.ok( + spec.execute || spec.mcpExecute, + `${key}: MCP-host spec missing execute/mcpExecute`, + ); + } + if (!spec.mcpOnly) { + assert.ok( + spec.execute || spec.inAppExecute, + `${key}: in-app-host spec missing execute/inAppExecute`, + ); + } + } +}); + +test("#494: a non-inline spec with no execute/mcpExecute is rejected (MCP-host arm)", () => { + // A shared spec (registered on BOTH hosts) with no execute at all. + const bad = { + lonely: { + mcpName: "lonely", + inAppKey: "lonely", + writeClass: "readOnly", + description: "no execute anywhere", + tier: "core", + catalogLine: "lonely — nothing", + }, + }; + assert.throws( + () => assertEverySpecIsRegisterable(bad), + /lonely.*neither execute nor mcpExecute/, + ); +}); + +test("#494: an inAppOnly spec with no execute/inAppExecute is rejected (in-app-host arm)", () => { + const bad = { + lonely: { + mcpName: "lonely", + inAppKey: "lonely", + writeClass: "readOnly", + description: "in-app only, but no runner", + tier: "core", + catalogLine: "lonely — nothing", + inAppOnly: true, + }, + }; + assert.throws( + () => assertEverySpecIsRegisterable(bad), + /lonely.*neither execute nor inAppExecute/, + ); +}); + +test("#494: an mcpExecute-only spec passes the MCP arm; an inAppExecute-only spec passes the in-app arm", () => { + // mcpOnly spec with only mcpExecute — the MCP arm is satisfied, the in-app arm + // is skipped (mcpOnly), so it must NOT throw. + const mcpOnly = { + t: { + mcpName: "t", inAppKey: "t", writeClass: "readOnly", + description: "x", tier: "core", catalogLine: "t — x", + mcpOnly: true, mcpExecute: async () => ({}), + }, + }; + assert.doesNotThrow(() => assertEverySpecIsRegisterable(mcpOnly)); + // inAppOnly spec with only inAppExecute — symmetric. + const inAppOnly = { + t: { + mcpName: "t", inAppKey: "t", writeClass: "readOnly", + description: "x", tier: "core", catalogLine: "t — x", + inAppOnly: true, inAppExecute: async () => ({}), + }, + }; + assert.doesNotThrow(() => assertEverySpecIsRegisterable(inAppOnly)); +}); + +test("#494: an inlineBothHosts spec is exempt from the execute requirement", () => { + const inline = { + t: { + mcpName: "t", inAppKey: "t", writeClass: "readOnly", + description: "x", tier: "core", catalogLine: "t — x", + inlineBothHosts: true, // no execute — registered inline by both hosts + }, + }; + assert.doesNotThrow(() => assertEverySpecIsRegisterable(inline)); +}); + test("buildShape (when present) returns a usable ZodRawShape with a real zod", () => { for (const [key, spec] of Object.entries(SHARED_TOOL_SPECS)) { if (!spec.buildShape) continue; diff --git a/apps/server/src/integrations/import/utils/foreign-markdown.ts b/packages/prosemirror-markdown/src/lib/foreign-markdown.ts similarity index 81% rename from apps/server/src/integrations/import/utils/foreign-markdown.ts rename to packages/prosemirror-markdown/src/lib/foreign-markdown.ts index dd7b012a..a49bc4e3 100644 --- a/apps/server/src/integrations/import/utils/foreign-markdown.ts +++ b/packages/prosemirror-markdown/src/lib/foreign-markdown.ts @@ -1,7 +1,14 @@ /** * Foreign-markdown normalizer — an input-liberal / output-canonical adapter that * runs at the IMPORT boundary, BEFORE the canonical parser - * (`markdownToProseMirror` from `@docmost/prosemirror-markdown`). + * (`markdownToProseMirror`, this package). + * + * OWNED BY THIS PACKAGE (#493): the normalizer used to live only in + * apps/server's import path, so the MCP page-write path (`updatePageMarkdown` -> + * `markdownToProseMirrorCanonical`) handled the SAME foreign input differently + * (no front-matter strip, no `[^id]` reference-footnote rewrite) than the server + * importer. Moving it here — and calling it from `markdownToProseMirrorCanonical` + * — makes every canonical import boundary treat foreign markdown identically. * * The canonical parser is deliberately STRICT: it only understands Docmost's * canonical markdown surface (Obsidian-style `> [!type]` callouts, Pandoc/Obsidian @@ -247,11 +254,18 @@ function convertReferenceFootnotes(markdown: string): string { const YAML_FRONT_MATTER_RE = /^\uFEFF?---\n[\s\S]*?\n---\n?/; /** - * Normalize a foreign markdown string into Docmost's canonical markdown surface - * so the strict canonical parser accepts it losslessly: normalize line endings, - * strip a leading YAML front-matter block, then rewrite GFM reference footnotes - * into inline footnotes. Add further fixture-driven foreign-surface cases here as - * they are found. + * Normalize a foreign markdown string from a FILE IMPORT into Docmost's canonical + * markdown surface so the strict canonical parser accepts it losslessly: normalize + * line endings, strip a leading YAML front-matter block, then rewrite GFM reference + * footnotes into inline footnotes. Add further fixture-driven foreign-surface cases + * here as they are found. + * + * FRONT-MATTER STRIP IS IMPORT-ONLY (#493 review): use this ONLY at the server + * file-import boundary, where a `.md` file really can open with an Obsidian/Hugo + * YAML header. Do NOT use it on the canonical AGENT-WRITE path — see + * {@link normalizeAgentMarkdown} for why a full-body agent rewrite must NOT strip + * a leading `---…---` (it is normally a horizontalRule the serializer emitted, and + * stripping it would silently drop the page's leading content). */ export function normalizeForeignMarkdown(markdown: string): string { if (!markdown) return markdown; @@ -264,3 +278,26 @@ export function normalizeForeignMarkdown(markdown: string): string { const withoutFrontMatter = src.replace(YAML_FRONT_MATTER_RE, '').trimStart(); return convertReferenceFootnotes(withoutFrontMatter); } + +/** + * Canonical AGENT-WRITE normalization: normalize line endings and rewrite GFM + * `[^id]` reference footnotes to inline `^[body]` — but DELIBERATELY NOT strip a + * leading YAML front-matter block. + * + * WHY the split (#493 review): the reference-footnote rewrite is the drift the + * MCP page-write path (`updatePageMarkdown` -> `markdownToProseMirrorCanonical`) + * needed unified with the server import (an agent may paste GFM footnotes). The + * front-matter strip, however, is a FILE-import concern: on a full-body agent + * rewrite a leading `---…---` is (almost) always a `horizontalRule` the + * serializer emitted plus a later rule/heading — NOT a foreign YAML header — so + * `YAML_FRONT_MATTER_RE` would match it and SILENTLY DELETE the page's leading + * content (a page that starts with a horizontal rule and contains a second `---` + * lost everything up to it). Agent writes must never lose already-stored content, + * so this variant skips the strip. It IS a no-op on canonical serialized content + * (which never emits `[^id]:` reference-definition lines). + */ +export function normalizeAgentMarkdown(markdown: string): string { + if (!markdown) return markdown; + const src = markdown.replace(/\r\n/g, '\n'); + return convertReferenceFootnotes(src); +} diff --git a/packages/prosemirror-markdown/src/lib/index.ts b/packages/prosemirror-markdown/src/lib/index.ts index cbf8f462..8bdfeb14 100644 --- a/packages/prosemirror-markdown/src/lib/index.ts +++ b/packages/prosemirror-markdown/src/lib/index.ts @@ -15,7 +15,10 @@ export { } from "./markdown-document.js"; export type { DocmostMdMeta } from "./markdown-document.js"; -export { convertProseMirrorToMarkdown } from "./markdown-converter.js"; +export { + convertProseMirrorToMarkdown, + ConverterLossError, +} from "./markdown-converter.js"; export type { ConvertProseMirrorToMarkdownOptions } from "./markdown-converter.js"; export { @@ -23,6 +26,19 @@ export { markdownToProseMirrorSync, } from "./markdown-to-prosemirror.js"; +// Foreign-markdown normalizer (#493): the input-liberal pre-pass that rewrites +// GFM `[^id]` reference footnotes to canonical inline `^[body]`. Two variants: +// `normalizeForeignMarkdown` (server FILE-import boundary) ALSO strips a leading +// YAML front-matter block; `normalizeAgentMarkdown` (canonical AGENT-WRITE path, +// mcp `markdownToProseMirrorCanonical`) does NOT — a full-body agent rewrite must +// not lose a leading `---…---` horizontalRule to the front-matter strip (#493 +// review). The reference-footnote rewrite is shared so agent + import stay unified +// where it matters, without the content-losing strip on the write path. +export { + normalizeForeignMarkdown, + normalizeAgentMarkdown, +} from "./foreign-markdown.js"; + // The Docmost tiptap schema mirror. Exposed so consumers (and the sync // engine's schema-validity regression tests) can build the exact ProseMirror // schema the converter targets. @@ -76,6 +92,17 @@ export type { OutlineEntry } from "./node-ops.js"; // string (#414: single copy shared by mcp and the CommonJS server app). export { parseNodeArg } from "./parse-node-arg.js"; +// Locator markdown-stripping (#493 dedup): the single canonical copy of the +// markdown-tolerant anchor-normalization primitives, imported by mcp's +// text-normalize.ts instead of a forked duplicate. `stripInlineMarkdown` is the +// lenient locator normalizer (trims stray decoration); `stripWrappersAndLinks` +// is the strict balanced-wrapper/link primitive mcp builds `stripBalancedWrappers` +// on top of. +export { + stripInlineMarkdown, + stripWrappersAndLinks, +} from "./text-normalize.js"; + // Inline-footnote authoring convention (#414: single copy, formerly the mcp // `footnote-authoring.ts` fork), shared with the importer's `assembleFootnotes`. export { diff --git a/packages/prosemirror-markdown/src/lib/markdown-converter.ts b/packages/prosemirror-markdown/src/lib/markdown-converter.ts index 8389d2b0..3f647a75 100644 --- a/packages/prosemirror-markdown/src/lib/markdown-converter.ts +++ b/packages/prosemirror-markdown/src/lib/markdown-converter.ts @@ -33,6 +33,26 @@ import { */ const MAX_NODE_DEPTH = 400; +/** + * Thrown by {@link convertProseMirrorToMarkdown} in `strict` mode when it hits a + * node or mark type it has no lossless markdown form for (the serializer would + * otherwise silently degrade it — drop an unknown mark, flatten an unknown node + * to its children). Carries the offending kind/name so a caller (git-sync) can + * surface exactly what would have been lost. + */ +export class ConverterLossError extends Error { + readonly kind: "node" | "mark"; + readonly typeName: string; + constructor(kind: "node" | "mark", typeName: string) { + super( + `convertProseMirrorToMarkdown: unknown ${kind} type "${typeName}" has no lossless markdown representation (strict mode)`, + ); + this.name = "ConverterLossError"; + this.kind = kind; + this.typeName = typeName; + } +} + /** * Options for {@link convertProseMirrorToMarkdown}. */ @@ -46,6 +66,23 @@ export interface ConvertProseMirrorToMarkdownOptions { * path where resolved anchors MUST be preserved for round-tripping. */ dropResolvedCommentAnchors?: boolean; + /** + * Optional sink for LOSS warnings. When the serializer reaches a node or mark + * type it has no dedicated case for, it degrades gracefully (flattens an + * unknown node to its children, drops an unknown mark) — historically a SILENT + * data loss. When this array is provided, one human-readable message per such + * event is pushed here so the caller can observe (and log) what was degraded. + * Not provided by default -> behavior is byte-identical to before for existing + * callers. + */ + warnings?: string[]; + /** + * When true, THROW a {@link ConverterLossError} on the FIRST unknown node/mark + * instead of degrading silently — a warning becomes a hard error. Used by the + * lossless git-sync export path and the converter tests, where an unmapped + * type is a bug to surface, not data to quietly drop. + */ + strict?: boolean; } /** @@ -63,6 +100,70 @@ export interface ConvertProseMirrorToMarkdownOptions { * separator is emitted for any other join, so non-list output is unchanged. */ const LIST_MARKER_SEPARATOR = ""; + +/** + * Backslash-escape a leading markdown BLOCK trigger so a serialized paragraph + * line re-parses as a PARAGRAPH, not another block. Without this, a paragraph + * whose text begins at column 0 with an ATX heading `#`, a blockquote/callout + * `>`, a bullet marker `-`/`*`/`+`, an ordered marker `N.`/`N)`, a code fence + * (```` ``` ````/`~~~`), a table `|`, or a thematic break (`---`/`***`/`___`, + * solid or spaced) silently becomes a heading/list/quote/code block/table/rule + * on the next markdown -> ProseMirror import — a known data-loss class (the + * thematic-break case drops the text entirely, since a horizontalRule carries + * none). CommonMark's escape tokenizer decodes the inserted `\` back to the + * literal character on import AND stops the block interpretation, so the line + * round-trips byte-exact as paragraph text. Only the FIRST offending character + * is escaped (the minimum needed to break block recognition); a line that does + * NOT open a block — emphasis `**x**`, an inline code span, ordinary prose — is + * returned verbatim, so there is no backslash churn for the common case. + * + * Applied ONLY to paragraph text, once per `\n`-separated LINE (the paragraph + * case splits on `\n` — each hardBreak emits ` \n` — so a trigger on a + * continuation line is escaped too): headings/lists/blockquotes legitimately + * open with these markers and render them from their own cases. This is the + * single, canonical fix for the class the client bridge worked around with a + * ZWSP (`gitmost-recording.ts`) and the generative suite self-censored around + * (`text-arbitraries.ts`) — both now removed. + */ +function escapeLeadingBlockTrigger(line: string): string { + // ATX heading: 1..6 `#` then whitespace/EOL. + if (/^#{1,6}(?:\s|$)/.test(line)) return "\\" + line; + // Blockquote / Docmost callout opener (`>` or `> [!info]`). + if (line.startsWith(">")) return "\\" + line; + // Bullet list marker then whitespace/EOL. Emphasis (`*x*`, `**x**`) has no + // space after the leading marker and is intentionally left verbatim. + if (/^[-*+](?:\s|$)/.test(line)) return "\\" + line; + // Ordered list marker `N.` / `N)`: escape the DELIMITER so the digits stay + // literal (`1. x` -> `1\. x`, which imports back as the text `1. x`). + const ordered = line.match(/^(\d+)[.)](?:\s|$)/); + if (ordered) { + const digits = ordered[1].length; + return line.slice(0, digits) + "\\" + line.slice(digits); + } + // Fenced code block: 3+ backticks or tildes. A single/double backtick is an + // inline code span and is left verbatim. + if (/^(?:`{3,}|~{3,})/.test(line)) return "\\" + line; + // Thematic break: a WHOLE line of 3+ identical `-`/`*`/`_`, optionally spaced. + if (/^([-*_])(?:\s*\1){2,}\s*$/.test(line)) return "\\" + line; + // Setext underline: a continuation line (after a hardBreak) that is ONLY `-` + // or ONLY `=` (any count, trailing spaces allowed). Under a paragraph line + // such a line re-parses as a SETEXT HEADING and SILENTLY DROPS its own text + // (`a\n--` -> heading "a", the `--` is LOST; `a\n=` -> heading "a", `=` LOST). + // The bullet arm above catches a lone `-` (via its `$`) and the thematic arm + // catches 3+ dashes, but exactly TWO dashes (`--`) fall through both; and no + // arm covers a lone `=` at all (a `==` pair is neutralized earlier by the + // inline `==`->`\=\=` escape, so only a single `=` line reaches here). Escaping + // the leading char (`\--`, `\=`) breaks the setext interpretation so the line + // round-trips as paragraph text. The WHOLE line must be the marker (anchored + // `^-+`/`^=+` to EOL), so a mid-content `-`/`=` is never spuriously escaped; + // and a `---`/`----` already handled by the thematic arm never reaches here, + // so there is no double-escape. + if (/^-+[ \t]*$/.test(line) || /^=+[ \t]*$/.test(line)) return "\\" + line; + // GFM table row opener. + if (line.startsWith("|")) return "\\" + line; + return line; +} + function listMarkerFamily(type: string | undefined): "ul" | "ol" | null { if (type === "bulletList" || type === "taskList") return "ul"; if (type === "orderedList") return "ol"; @@ -109,6 +210,26 @@ export function convertProseMirrorToMarkdown( // callers (mcp getPage / in-app AI chat) pass it true. const dropResolvedCommentAnchors = options.dropResolvedCommentAnchors === true; + // Loss reporting for node/mark types with no dedicated serializer case. In + // `strict` mode the FIRST such type throws (git-sync, tests); otherwise the + // serializer degrades gracefully (as it always has) but records one warning + // per unmapped type into the optional sink so the loss is observable, not + // silent. Deduped per type so a document with many unknown nodes of one type + // produces one message. + const strict = options.strict === true; + const warningsSink = options.warnings; + const seenLossTypes = new Set(); + const warnLoss = (kind: "node" | "mark", typeName: string): void => { + if (strict) throw new ConverterLossError(kind, typeName); + if (!warningsSink) return; + const key = `${kind}:${typeName}`; + if (seenLossTypes.has(key)) return; + seenLossTypes.add(key); + warningsSink.push( + `Unknown ${kind} type "${typeName}" has no lossless markdown form; it was degraded on export.`, + ); + }; + // Escape a value interpolated into an HTML double-quoted attribute value // (textAlign, colors, image src, math `text`, all data-* attrs, etc.). In the // ATTRIBUTE context only the quote that delimits the value and the ampersand @@ -412,7 +533,17 @@ export function convertProseMirrorToMarkdown( } case "paragraph": { - const text = renderInlineChildren(nodeContent); + // Escape a leading block trigger on EVERY line of the paragraph, not + // just the first: a hardBreak serializes as ` \n`, so a `#`/`-`/`>`/ + // `1.`/`|`/fence/`---` at the start of a CONTINUATION line would also + // re-parse into another block on the next import (a heading/list/table/ + // setext-`---`), and for the text-less thematic/setext case would LOSE + // that line's text entirely. Escaping each `\n`-separated line closes + // the class for multi-line paragraphs too. + const text = renderInlineChildren(nodeContent) + .split("\n") + .map(escapeLeadingBlockTrigger) + .join("\n"); const align = node.attrs?.textAlign; // Non-default alignment round-trips as an ATTACHED HTML comment at the // END of the block line (#293 canon #9): @@ -595,6 +726,12 @@ export function convertProseMirrorToMarkdown( } break; } + default: + // Unknown mark: no dedicated case, so it has no markdown form and + // is dropped from the run. Report the loss (throws in strict + // mode) then leave the text unwrapped — the historical behavior. + warnLoss("mark", String(mark.type)); + break; } } } @@ -1173,7 +1310,11 @@ export function convertProseMirrorToMarkdown( } default: - // Fallback: process children + // Unknown node type: no dedicated case, so the node's identity + attrs + // have no lossless markdown form. Report the loss (throws in strict + // mode) then degrade by flattening to its children — the historical + // graceful fallback. + warnLoss("node", String(type)); return nodeContent.map(processNode).join(""); } }; @@ -1297,6 +1438,12 @@ export function convertProseMirrorToMarkdown( t = `${t}`; } break; + default: + // Unknown mark on the raw-HTML path: dropped (no HTML form). Report + // the loss (throws in strict mode) — same policy as the markdown + // path's marks loop above. + warnLoss("mark", String(mark.type)); + break; } } return t; diff --git a/packages/prosemirror-markdown/src/lib/text-normalize.ts b/packages/prosemirror-markdown/src/lib/text-normalize.ts index ea84d648..5e5ad26c 100644 --- a/packages/prosemirror-markdown/src/lib/text-normalize.ts +++ b/packages/prosemirror-markdown/src/lib/text-normalize.ts @@ -7,13 +7,12 @@ * it is never applied to replacement text or inserted node content, so no * formatting is ever lost. * - * Scope note (#414): this package-local copy exists so `node-ops.ts` — which - * lives here now (the single canonical copy) — can resolve its markdown-tolerant - * anchor fallback without a circular dependency back on `@docmost/mcp`. It - * intentionally carries ONLY `stripInlineMarkdown` (the primitive `node-ops` - * needs); the mcp-side `text-normalize.ts` (which additionally serves - * `json-edit.ts` via `stripBalancedWrappers`) is the subject of a separate - * dedup task and is left untouched here. + * CANONICAL HOME (#414/#493): this is the single source of truth for locator + * markdown-stripping. `node-ops.ts` (which lives here) uses it directly, and the + * mcp-side `text-normalize.ts` now IMPORTS `stripInlineMarkdown` and the shared + * `stripWrappersAndLinks` primitive from here (via `@docmost/prosemirror-markdown`) + * instead of keeping a drifting copy — mcp only adds its own thin + * `stripBalancedWrappers`/`closestBlockHint` on top. */ /** Maximum unwrap passes, so pathological/nested input cannot loop forever. */ @@ -44,7 +43,7 @@ const LINK_IMAGE_RE = /!?\[([^\]]*)\]\([^)]*\)/g; * Does NOT trim decoration, does NOT guard against an empty result — it returns * exactly the transformed string. */ -function stripWrappersAndLinks(s: string): string { +export function stripWrappersAndLinks(s: string): string { // 1. Links/images -> their visible text. let out = s.replace(LINK_IMAGE_RE, "$1"); diff --git a/packages/prosemirror-markdown/test/converter-loss-warnings.test.ts b/packages/prosemirror-markdown/test/converter-loss-warnings.test.ts new file mode 100644 index 00000000..fc6e22b8 --- /dev/null +++ b/packages/prosemirror-markdown/test/converter-loss-warnings.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest"; +import { + convertProseMirrorToMarkdown, + ConverterLossError, +} from "../src/lib/markdown-converter.js"; + +/** + * #493 commit 3 — a node/mark type the serializer has no dedicated case for used + * to be degraded SILENTLY (an unknown node flattened to its children, an unknown + * mark dropped from the run). The serializer now REPORTS the loss: + * - default (non-strict): unchanged graceful degradation, but one warning per + * unmapped type is pushed into an optional `warnings` sink so callers can + * observe it; + * - strict: the FIRST unmapped type throws a ConverterLossError (git-sync + + * tests), turning a silent loss into a hard, surfaced error. + * + * Exercised through the REAL converter (no mock): the observable properties are + * the emitted markdown, the warnings collected, and the thrown error. + */ + +const doc = (...nodes: any[]) => ({ type: "doc", content: nodes }); + +describe("converter loss reporting — unknown node types", () => { + const unknownNode = doc({ + type: "quantumWidget", + content: [{ type: "text", text: "inner text" }], + }); + + it("degrades to children AND records a warning (non-strict, sink provided)", () => { + const warnings: string[] = []; + const md = convertProseMirrorToMarkdown(unknownNode, { warnings }); + // Graceful degrade: the child text still survives (historical behavior). + expect(md).toContain("inner text"); + // The loss is now observable. + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("quantumWidget"); + expect(warnings[0]).toContain("node"); + }); + + it("stays byte-identical for callers that pass no sink (zero behavior change)", () => { + const withSink: string[] = []; + const a = convertProseMirrorToMarkdown(unknownNode, { warnings: withSink }); + const b = convertProseMirrorToMarkdown(unknownNode); + expect(b).toBe(a); // the sink does not alter the produced markdown + }); + + it("throws ConverterLossError in strict mode", () => { + try { + convertProseMirrorToMarkdown(unknownNode, { strict: true }); + expect.unreachable("strict mode must throw on an unknown node"); + } catch (e) { + expect(e).toBeInstanceOf(ConverterLossError); + expect((e as ConverterLossError).kind).toBe("node"); + expect((e as ConverterLossError).typeName).toBe("quantumWidget"); + } + }); + + it("dedupes the warning per type (many unknown nodes -> one message)", () => { + const warnings: string[] = []; + convertProseMirrorToMarkdown( + doc( + { type: "quantumWidget", content: [{ type: "text", text: "a" }] }, + { type: "quantumWidget", content: [{ type: "text", text: "b" }] }, + ), + { warnings }, + ); + expect(warnings).toHaveLength(1); + }); +}); + +describe("converter loss reporting — unknown mark types", () => { + const unknownMark = doc({ + type: "paragraph", + content: [{ type: "text", text: "glowing", marks: [{ type: "glow" }] }], + }); + + it("drops the mark but keeps the text AND records a warning (non-strict)", () => { + const warnings: string[] = []; + const md = convertProseMirrorToMarkdown(unknownMark, { warnings }); + expect(md).toBe("glowing"); // text survives, mark silently had no form + expect(warnings).toHaveLength(1); + expect(warnings[0]).toContain("glow"); + expect(warnings[0]).toContain("mark"); + }); + + it("throws ConverterLossError in strict mode", () => { + expect(() => + convertProseMirrorToMarkdown(unknownMark, { strict: true }), + ).toThrow(ConverterLossError); + }); +}); + +describe("converter loss reporting — known content is never flagged", () => { + it("a fully-mapped document produces no warnings and does not throw in strict mode", () => { + const d = doc( + { type: "heading", attrs: { level: 2 }, content: [{ type: "text", text: "Title" }] }, + { + type: "paragraph", + content: [ + { type: "text", text: "bold", marks: [{ type: "bold" }] }, + { type: "text", text: " and " }, + { type: "text", text: "link", marks: [{ type: "link", attrs: { href: "https://x.y" } }] }, + ], + }, + { type: "bulletList", content: [{ type: "listItem", content: [{ type: "paragraph", content: [{ type: "text", text: "item" }] }] }] }, + ); + const warnings: string[] = []; + const md = convertProseMirrorToMarkdown(d, { warnings, strict: true }); + expect(warnings).toEqual([]); + expect(md).toContain("## Title"); + }); +}); diff --git a/apps/server/src/integrations/import/utils/foreign-markdown.spec.ts b/packages/prosemirror-markdown/test/foreign-markdown.test.ts similarity index 75% rename from apps/server/src/integrations/import/utils/foreign-markdown.spec.ts rename to packages/prosemirror-markdown/test/foreign-markdown.test.ts index 8a43fa20..135233fb 100644 --- a/apps/server/src/integrations/import/utils/foreign-markdown.spec.ts +++ b/packages/prosemirror-markdown/test/foreign-markdown.test.ts @@ -1,12 +1,15 @@ +import { describe, it, expect } from 'vitest'; +import { convertProseMirrorToMarkdown } from '../src/lib/markdown-converter.js'; +import { markdownToProseMirror } from '../src/lib/markdown-to-prosemirror.js'; import { - convertProseMirrorToMarkdown, - markdownToProseMirror, -} from '@docmost/prosemirror-markdown'; -import { normalizeForeignMarkdown } from './foreign-markdown'; + normalizeForeignMarkdown, + normalizeAgentMarkdown, +} from '../src/lib/foreign-markdown.js'; /** - * STEP 2 goldens for issue #345: the foreign-markdown normalizer that runs at the - * import boundary BEFORE the strict canonical parser (`markdownToProseMirror`). + * STEP 2 goldens for issue #345 (moved into the package with the normalizer in + * #493): the foreign-markdown normalizer that runs at the import boundary BEFORE + * the strict canonical parser (`markdownToProseMirror`). * * Two layers: * 1. PURE string→string cases pinning the normalizer's own behavior (GFM @@ -216,3 +219,53 @@ describe('foreign markdown import acceptance (normalizer + canonical parser)', ( ).toHaveLength(1); }); }); + +describe('normalizeAgentMarkdown vs normalizeForeignMarkdown — front-matter strip is IMPORT-only (#493 review)', () => { + // A page that OPENS with a horizontalRule and contains a later `---` serializes + // to a `---…---`-shaped body. On a full-body AGENT rewrite this must NOT be + // mistaken for YAML front-matter and stripped — that silently dropped the + // page's leading content. + const rulePage = '---\n\nIntro\n\nMore\n\n---\n\nRest'; + + it('normalizeAgentMarkdown does NOT strip a leading ---…--- (no content loss)', () => { + expect(normalizeAgentMarkdown(rulePage)).toBe(rulePage); + }); + + it('normalizeForeignMarkdown (file import) STILL strips a real leading YAML front-matter block', () => { + const withYaml = '---\ntitle: My Page\ntags: [a, b]\n---\n\nBody here.'; + const out = normalizeForeignMarkdown(withYaml); + expect(out).toBe('Body here.'); + // And the horizontalRule-shaped body IS stripped on the import path (its + // documented file-import behavior) — the two variants differ ONLY here. + expect(normalizeForeignMarkdown(rulePage)).not.toContain('Intro'); + }); + + it('agent-write round-trip keeps a horizontalRule-led doc with a second rule intact', async () => { + // Simulate the serializer output for [horizontalRule, para, para, horizontalRule, para]. + const doc = { + type: 'doc', + content: [ + { type: 'horizontalRule' }, + { type: 'paragraph', content: [{ type: 'text', text: 'Intro' }] }, + { type: 'paragraph', content: [{ type: 'text', text: 'More' }] }, + { type: 'horizontalRule' }, + { type: 'paragraph', content: [{ type: 'text', text: 'Rest' }] }, + ], + }; + const body = convertProseMirrorToMarkdown(doc); + // The agent-write normalization must NOT eat the head; re-import keeps every + // paragraph's text. + const back = await markdownToProseMirror(normalizeAgentMarkdown(body)); + const texts = JSON.stringify(back); + for (const t of ['Intro', 'More', 'Rest']) expect(texts).toContain(t); + // Both horizontal rules survive. + expect(back.content.filter((n: any) => n.type === 'horizontalRule')).toHaveLength(2); + }); + + it('agent-write STILL rewrites GFM reference footnotes (the shared drift-fix)', () => { + const gfm = 'See[^1].\n\n[^1]: the note.'; + const out = normalizeAgentMarkdown(gfm); + expect(out).toContain('^[the note.]'); + expect(out).not.toMatch(/\[\^1\]:/); + }); +}); diff --git a/packages/prosemirror-markdown/test/generative/text-arbitraries.ts b/packages/prosemirror-markdown/test/generative/text-arbitraries.ts index 76353b76..63067427 100644 --- a/packages/prosemirror-markdown/test/generative/text-arbitraries.ts +++ b/packages/prosemirror-markdown/test/generative/text-arbitraries.ts @@ -212,25 +212,93 @@ export function normalizeInline(nodes: any[]): any[] { return out; } +/** + * #493 commit 1: a plain-text run whose text DELIBERATELY OPENS with a markdown + * BLOCK trigger — ATX heading `#`, bullet `-`/`*`/`+`, blockquote `>`, ordered + * `N.`/`N)`, or a table `|` — followed by safe text. Pre-#493 the corpus + * self-censored these away (safeTextArb's leading-word guarantee); the paragraph + * serializer now BLOCK-ESCAPES a leading trigger, so the generative round-trip + * itself proves the data-loss class is closed rather than avoiding it. + * + * DELIBERATELY excludes the code-fence (backtick) trigger — the backtick is a + * code-span delimiter that re-pairs globally (see specialCharArb's note), an + * instability UNRELATED to block-escape — and the whole-line thematic break + * (`---`), which only triggers when the line is ONLY dashes; both are covered by + * the deterministic pin (gitmost-transcript-neutralization.test.ts). Each still + * ENDS in a word (safeTextArb) so adjacent-run concatenation stays safe. + */ +export const blockTriggerLeadRunArb: fc.Arbitrary = fc + .tuple( + fc.constantFrom('# ', '## ', '- ', '* ', '+ ', '> ', '1. ', '1) ', '| '), + safeTextArb, + ) + .map(([trigger, rest]) => ({ type: 'text', text: trigger + rest })); + +/** + * A hardBreak IMMEDIATELY followed by a block-trigger-leading run — a two-node + * segment. Because a hardBreak serializes as ` \n`, the trigger then sits at + * the START of a CONTINUATION line, exercising the serializer's PER-LINE block + * escape (not just the first line). #493 review: without this the fuzzer never + * placed a trigger after a hardBreak, so a single-line-only escape passed P1–P3. + */ +export const hardBreakThenTriggerArb: fc.Arbitrary = fc + .tuple(hardBreakArb, blockTriggerLeadRunArb) + .map(([hb, trigger]) => [hb, trigger]); + +/** + * #493 (setext data-loss): a WHOLE-LINE setext underline landing on a + * continuation line. A setext underline is a line of ONLY `-` (any count) or + * ONLY `=` (any count) that FOLLOWS a paragraph line; on re-parse it turns the + * preceding line into a heading and DROPS its own text. The block-escape must + * neutralize it. Unlike blockTriggerLeadRunArb, the underline must occupy the + * whole line, so we sandwich it between two hardBreaks (underline on its own + * line, preceded by earlier paragraph content, followed by a trailing word so + * the closing hardBreak is not dropped by normalizeInline). Covers underlines + * of every length: `--` (the two-dash case the bullet/thematic arms miss), a + * lone `=`, `==`/`====` (neutralized by the inline `==` escape), and `---`/ + * `----` (regression for the existing thematic case). + */ +export const hardBreakThenSetextArb: fc.Arbitrary = fc + .tuple( + fc.constantFrom('--', '=', '==', '====', '---', '----'), + safeTextArb, + ) + .map(([underline, rest]) => [ + { type: 'hardBreak' }, + { type: 'text', text: underline }, + { type: 'hardBreak' }, + { type: 'text', text: rest }, + ]); + /** * Inline content for a paragraph: at least one marked text run, optionally with - * inline atoms (math/mention) and hard breaks interspersed. Always starts with a - * text run so the paragraph never opens with a block trigger. (Ported.) + * inline atoms (math/mention) and hard breaks interspersed. The FIRST run is + * usually an ordinary marked run, but sometimes a block-trigger-leading run + * (blockTriggerLeadRunArb) so the paragraph OPENS with a markdown block trigger; + * and a `hardBreak + trigger` segment can appear anywhere in the rest, so a + * trigger also lands at the start of a CONTINUATION line — both exercising the + * serializer's per-line block-escape end-to-end. (Ported, with the #493 + * leading-trigger + post-hardBreak dimensions added.) */ export const inlineContentArb: fc.Arbitrary = fc .tuple( - markedTextRunArb, + fc.oneof( + { weight: 5, arbitrary: markedTextRunArb }, + { weight: 1, arbitrary: blockTriggerLeadRunArb }, + ), fc.array( fc.oneof( - { weight: 5, arbitrary: markedTextRunArb }, - { weight: 1, arbitrary: mathInlineArb }, - { weight: 1, arbitrary: mentionArb }, - { weight: 1, arbitrary: hardBreakArb }, + { weight: 5, arbitrary: markedTextRunArb.map((n) => [n]) }, + { weight: 1, arbitrary: mathInlineArb.map((n) => [n]) }, + { weight: 1, arbitrary: mentionArb.map((n) => [n]) }, + { weight: 1, arbitrary: hardBreakArb.map((n) => [n]) }, + { weight: 2, arbitrary: hardBreakThenTriggerArb }, + { weight: 2, arbitrary: hardBreakThenSetextArb }, ), { minLength: 0, maxLength: 4 }, ), ) - .map(([first, rest]) => normalizeInline([first, ...rest])); + .map(([first, rest]) => normalizeInline([first, ...rest.flat()])); /** * Inline content for a HEADING — identical to a paragraph's, but WITHOUT hard diff --git a/packages/prosemirror-markdown/test/gitmost-transcript-neutralization.test.ts b/packages/prosemirror-markdown/test/gitmost-transcript-neutralization.test.ts index 6fb35f1f..20b9f092 100644 --- a/packages/prosemirror-markdown/test/gitmost-transcript-neutralization.test.ts +++ b/packages/prosemirror-markdown/test/gitmost-transcript-neutralization.test.ts @@ -5,32 +5,21 @@ import { convertProseMirrorToMarkdown } from "../src/lib/markdown-converter.js"; import { markdownToProseMirror } from "../src/lib/markdown-to-prosemirror.js"; /** - * gitmost #377 (round-1 review, finding #1) — proof, against the REAL - * converter, that the transcript-insert boundary defense survives git-sync. + * #493 commit 1 — the paragraph serializer's leading-block-escape closes the + * data-loss class where a paragraph whose text opens at column 0 with a markdown + * block trigger (`#`/`-`/`*`/`+`/`>`, an ordered `N.`/`N)`, a code fence, a + * table `|`, a callout opener, or a thematic break) silently re-parsed into a + * heading / list / quote / code block / table / horizontalRule on the git-sync + * doc -> markdown -> doc cycle. The thematic-break case was the worst: a + * horizontalRule carries NO text, so the line's text was lost entirely. * - * The web bridge (apps/client .../gitmost/gitmost-recording.ts, - * `gitmostInsertTranscriptIntoEditor`) appends each transcript line as a - * PARAGRAPH text node. The paragraph serializer here (`case "paragraph"`) emits - * that text VERBATIM with no block-escape, so a line whose text begins with a - * col-0 markdown block trigger would, on the doc -> markdown -> doc git-sync - * cycle, silently re-parse into a heading / list / quote / callout / code block. - * That missing block-escape is the pre-existing root cause; the bridge's - * boundary defense prepends an invisible zero-width space (U+200B) to a line - * that begins with such a trigger, shifting it off column 0. - * - * This test keeps a COPY of the bridge's trigger regex (the bridge is in a - * different package and can't be imported here) and asserts: - * 1. bare trigger lines DO corrupt (documents the root cause), and - * 2. the ZWSP-neutralized form round-trips as a single PARAGRAPH with the - * text byte-preserved. + * This is the deterministic PIN, one assertion per trigger, exercised through + * the REAL converter round-trip (not a mock): each bare trigger line now + * round-trips as a SINGLE paragraph with its text byte-preserved — proving the + * class is closed WITHOUT the former client-side ZWSP workaround (removed) or + * the generative suite's leading-word self-censorship (removed). */ -const ZWSP = "​"; // U+200B - -// MUST stay in sync with GITMOST_MD_BLOCK_TRIGGER_RE in the client bridge. -const MD_BLOCK_TRIGGER_RE = - /^(?:#{1,6}(?:\s|$)|[-*+](?:\s|$)|>|\d+[.)](?:\s|$)|```|~~~|\||([-*_])(?:\s*\1){2,}\s*$)/; - const doc = (...nodes: any[]) => ({ type: "doc", content: nodes }); const para = (t: string) => ({ type: "paragraph", @@ -43,78 +32,117 @@ const roundtrip = async (text: string) => { return back.content as any[]; }; -describe("gitmost transcript neutralization (git-sync round-trip)", () => { - // Lines that, at column 0, the serializer's missing block-escape would let - // git-sync re-parse into a non-paragraph block. +describe("paragraph block-escape (git-sync round-trip)", () => { + // Every line here, at column 0, WOULD (pre-fix) re-parse into a non-paragraph + // block. Each is now block-escaped by the serializer and round-trips clean. const triggerLines = [ "- dash", "* star", "+ plus", "> quote", "# hash", + "## two hash", + "###### six hash", "1. one", "1) one", "> [!info] note", "```js", "~~~", - // Solid + spaced thematic breaks — these re-parse into a `horizontalRule`, - // which carries NO text, so a bare separator line LOSES its text entirely - // (round-2 finding). `_` also only forms a block via this construct. + "| a | b |", + // Solid + spaced thematic breaks — the text-LOSING case pre-fix. "---", "***", "___", - "- - -", // spaced dash break (solid form is caught by [-*+]\s too, but this is the break) + "- - -", "_ _ _", ]; - it("BARE trigger lines corrupt into non-paragraph blocks (root cause)", async () => { + it("every bare trigger line round-trips as a single paragraph, text byte-preserved", async () => { for (const line of triggerLines) { const blocks = await roundtrip(line); - // At least one produced block is NOT a paragraph — i.e. corruption. - const allParagraphs = blocks.every((b) => b.type === "paragraph"); - expect( - allParagraphs, - `expected "${line}" to corrupt when inserted bare`, - ).toBe(false); - } - }); - - it("BARE solid thematic breaks corrupt into a text-LOSING horizontalRule", async () => { - // The severe case: no text node survives. Documents why neutralization - // matters more here than for list/quote (where the text survived). - for (const line of ["---", "***", "___"]) { - const blocks = await roundtrip(line); - expect(blocks.map((b) => b.type)).toContain("horizontalRule"); - // No block carries the original text anywhere. - const flat = JSON.stringify(blocks); - expect(flat).not.toContain(line); - } - }); - - it("ZWSP-neutralized trigger lines round-trip as a single paragraph, text preserved", async () => { - for (const line of triggerLines) { - // The regex must actually classify each as a trigger. - expect(MD_BLOCK_TRIGGER_RE.test(line), `regex missed "${line}"`).toBe( - true, + expect(blocks, `"${line}" should be one block`).toHaveLength(1); + expect(blocks[0].type, `"${line}" should stay a paragraph`).toBe( + "paragraph", ); - const neutralized = ZWSP + line; - const blocks = await roundtrip(neutralized); - - expect(blocks).toHaveLength(1); - expect(blocks[0].type).toBe("paragraph"); - // Text is byte-preserved (ZWSP + original line), so the display is the - // original line with only an invisible leading character. - expect(blocks[0].content[0].text).toBe(neutralized); + expect( + blocks[0].content?.[0]?.text, + `"${line}" text should survive byte-exact`, + ).toBe(line); } }); - it("normal host-prefixed lines never match the trigger regex and round-trip byte-exact", async () => { + it("emphasis / inline-code paragraphs are NOT escaped (no backslash churn)", async () => { + // These open with `*`/`` ` `` but are NOT block triggers; the serialized + // markdown must not gain a stray leading backslash, and they round-trip. + for (const [text, mark] of [ + ["bold", "bold"], + ["italic", "italic"], + ["code", "code"], + ] as const) { + const node = doc({ + type: "paragraph", + content: [{ type: "text", text, marks: [{ type: mark }] }], + }); + const md = convertProseMirrorToMarkdown(node); + expect(md.startsWith("\\"), `${mark} must not be block-escaped`).toBe( + false, + ); + const back = await markdownToProseMirror(md); + expect(back.content[0].type).toBe("paragraph"); + expect(back.content[0].content[0].text).toBe(text); + expect(back.content[0].content[0].marks?.[0]?.type).toBe(mark); + } + }); + + it("a block trigger on a CONTINUATION line (after a hardBreak) is escaped too", async () => { + // A hardBreak serializes as ` \n`, so a trigger on the second line would, + // without a per-line escape, re-parse into another block. The worst case is + // `---`: a setext underline would turn the first line into a heading and LOSE + // the `---` text entirely. Each pair round-trips as ONE paragraph with the + // hardBreak and both texts preserved. + for (const [first, second] of [ + ["a", "# b"], + ["a", "- b"], + ["a", "> b"], + ["a", "1. b"], + ["a", "| b |"], + ["a", "---"], // setext / thematic (3 dashes) — the text-losing case + ["a", "--"], // setext underline, EXACTLY two dashes (bullet/thematic miss it) + ["a", "----"], // setext / thematic (4 dashes) + ["a", "="], // setext H1 underline, a lone `=` (no other arm covers it) + ["a", "===="], // setext H1 underline, run of `=` + ]) { + const d = doc({ + type: "paragraph", + content: [ + { type: "text", text: first }, + { type: "hardBreak" }, + { type: "text", text: second }, + ], + }); + const back = await markdownToProseMirror(convertProseMirrorToMarkdown(d)); + expect(back.content, `"${first}⏎${second}" should be one block`).toHaveLength(1); + expect(back.content[0].type).toBe("paragraph"); + const texts = (back.content[0].content as any[]) + .filter((n) => n.type === "text") + .map((n) => n.text); + const hasBreak = (back.content[0].content as any[]).some( + (n) => n.type === "hardBreak", + ); + expect(hasBreak, `"${first}⏎${second}" should keep the hardBreak`).toBe(true); + expect(texts, `"${first}⏎${second}" should preserve both line texts`).toEqual([ + first, + second, + ]); + } + }); + + it("normal host-prefixed lines round-trip byte-exact (unaffected)", async () => { for (const line of [ "You: hello there", "Speaker 1: - and then a dash mid-line", "Speaker 2: 1. not a list", ]) { - expect(MD_BLOCK_TRIGGER_RE.test(line)).toBe(false); const blocks = await roundtrip(line); expect(blocks).toHaveLength(1); expect(blocks[0].type).toBe("paragraph"); diff --git a/packages/prosemirror-markdown/test/markdown-converter-gaps.test.ts b/packages/prosemirror-markdown/test/markdown-converter-gaps.test.ts index c3bb1e2b..9008c34b 100644 --- a/packages/prosemirror-markdown/test/markdown-converter-gaps.test.ts +++ b/packages/prosemirror-markdown/test/markdown-converter-gaps.test.ts @@ -430,7 +430,7 @@ describe('converter gap coverage — emission branches (specs 1–11)', () => { }); }); -describe('converter gap coverage — documented round-trip data loss (specs 12–14)', () => { +describe('converter gap coverage — formerly-lossy round-trips, now closed (specs 12–14)', () => { // 12. A 3-backtick fence inside a codeBlock body is now lengthened: the outer // fence widens to (longest inner run + 1) backticks per CommonMark, so the // inner ``` is treated as content and the block survives as ONE node. @@ -460,25 +460,24 @@ describe('converter gap coverage — documented round-trip data loss (specs 12 expect(docsCanonicallyEqual(d, doc2)).toBe(false); }); - // 13. A leading ordered-list marker in paragraph text is NOT escaped, so a - // plain paragraph silently becomes an orderedList on re-import. - it('a paragraph starting with "1. " is promoted to an orderedList on re-import', async () => { + // 13. #493 commit 1: a leading ordered-list marker in paragraph text is now + // BLOCK-ESCAPED, so the paragraph round-trips as a paragraph instead of + // silently becoming an orderedList (was documented data loss, now closed). + it('a paragraph starting with "1. " is block-escaped and stays a paragraph', async () => { const d = doc({ type: 'paragraph', content: [{ type: 'text', text: '1. not a list' }], }); const md1 = convertProseMirrorToMarkdown(d); - expect(md1).toBe('1. not a list'); // no backslash escape + expect(md1).toBe('1\\. not a list'); // the ordered-list delimiter is escaped const doc2 = await markdownToProseMirror(md1); - expect(doc2.content?.[0]?.type).toBe('orderedList'); - const li = doc2.content[0].content?.[0]; - expect(li?.type).toBe('listItem'); - expect(li.content?.[0]?.content?.[0]).toMatchObject({ + expect(doc2.content?.[0]?.type).toBe('paragraph'); + expect(doc2.content[0].content?.[0]).toMatchObject({ type: 'text', - text: 'not a list', // the "1. " was consumed as a list marker + text: '1. not a list', // the escape decodes back to the literal text }); - expect(docsCanonicallyEqual(d, doc2)).toBe(false); + expect(docsCanonicallyEqual(d, doc2)).toBe(true); }); // 14. #293 canon #4: the image title now round-trips via the attached diff --git a/packages/prosemirror-markdown/test/schema-editor-ext-contract.test.ts b/packages/prosemirror-markdown/test/schema-editor-ext-contract.test.ts index 4778ea5c..7570ff11 100644 --- a/packages/prosemirror-markdown/test/schema-editor-ext-contract.test.ts +++ b/packages/prosemirror-markdown/test/schema-editor-ext-contract.test.ts @@ -16,14 +16,16 @@ import * as editorExt from "@docmost/editor-ext"; // or mark added upstream that the mirror forgets to vendor fails CI loudly // (otherwise it is silently dropped on the markdown <-> ProseMirror round-trip). // -// LIMITATION (intentional, see schema-surface-snapshot.test.ts): this is a -// NAME-LEVEL contract only, not a full attribute-level structural compare. -// editor-ext's Tiptap representation (node views, commands, suggestion plugins, -// addGlobalAttributes spread across separate extensions) differs from this -// minimal mirror, so a mechanical attribute-by-attribute equality would be -// fragile and produce false drift. Attribute parity is guarded by the inline -// surface snapshot (reviewed in every diff); this test guards that no canonical -// node/mark TYPE goes unmirrored. StarterKit-provided types (paragraph, bold, +// This file now holds TWO contracts (see the two describe blocks): the original +// NAME-LEVEL type contract (no canonical node/mark TYPE goes unmirrored) AND, as +// of #493, an ATTRIBUTE-LEVEL contract that compares each editor-ext node/mark's +// OWN declared attributes (names + defaults) against the mirror's built schema. +// A full mechanical attribute-by-attribute EQUALITY would be fragile (the mirror +// is a deliberate superset: it injects the global id/textAlign/indent attrs and +// normalizes some editor-ext defaults to null), so the attribute contract is +// asymmetric — editor-ext -> mirror — with a small, reasoned, stale-guarded +// allowlist for the two blessed divergence kinds (non-round-trippable omissions +// and null-normalized defaults). StarterKit-provided types (paragraph, bold, // heading, …) are contributed by @tiptap/starter-kit in the mirror rather than // by editor-ext, so they are naturally covered by the mirror's superset. // @@ -85,3 +87,191 @@ describe("docmost schema vs @docmost/editor-ext (name-level contract)", () => { expect(missing).toEqual([]); }); }); + +// ── ATTRIBUTE-LEVEL CONTRACT (#493 commit 2) ──────────────────────────────── +// +// The name-level contract above catches a WHOLE node/mark type going unmirrored, +// but not ATTRIBUTE drift within a vendored type — the exact class that silently +// dropped `subpages.recursive`: editor-ext grew an attribute the hand-synced +// mirror forgot, so documents using it lost that attribute on a git-sync +// round-trip while CI stayed green. This closes that gap by comparing each +// editor-ext node/mark's OWN declared attributes (names + defaults) against the +// mirror's built ProseMirror schema `spec.attrs`. +// +// DIRECTION: editor-ext -> mirror. The mirror is deliberately a SUPERSET (it +// injects the global `id`/`textAlign`/`indent` attributes and normalizes some +// editor-ext "required" attrs to a `null` default), so a reverse compare would +// be pure false drift; the meaningful failure is an editor-ext attribute the +// mirror DROPS (name) or whose DEFAULT it silently changes. Both directions of +// staleness are guarded so the allowlists cannot rot. + +/** + * The attributes an editor-ext Tiptap Node/Mark DECLARES itself, read from its + * `config.addAttributes()`. Global attributes injected by separate extensions + * (unique-id, indent, textAlign) are NOT included here — they are the mirror's + * superset and are not part of a per-type declaration — so this isolates each + * type's own contribution. A declared attribute with no explicit `default` is a + * required attr (Tiptap default `undefined`); we surface that as-is so the + * default compare can skip it (the mirror makes such attrs optional/`null`). + */ +function editorExtOwnAttrs(): Map< + string, + { kind: "node" | "mark"; attrs: Record } +> { + const out = new Map< + string, + { kind: "node" | "mark"; attrs: Record } + >(); + for (const value of Object.values(editorExt)) { + if (!isTiptapNodeOrMark(value)) continue; + const ext = value as unknown as { + name: string; + type: "node" | "mark"; + options?: unknown; + storage?: unknown; + config?: { addAttributes?: () => Record }; + }; + const fn = ext.config?.addAttributes; + // addAttributes reads `this.options`/`this.name`; bind a minimal context + // (verified sufficient for every editor-ext extension — none reach for + // `this.editor` here). A type with no addAttributes contributes no attrs. + const declared = + typeof fn === "function" + ? fn.call({ + options: ext.options ?? {}, + name: ext.name, + parent: undefined, + storage: ext.storage ?? {}, + } as never) + : {}; + const attrs: Record = {}; + for (const [attr, spec] of Object.entries(declared || {})) { + // `undefined` marks a required (no-default) attr; keep it so the default + // compare can distinguish "no default declared" from "default is null". + attrs[attr] = (spec as { default?: unknown })?.default; + } + out.set(ext.name, { kind: ext.type, attrs }); + } + return out; +} + +/** The mirror's built-schema `spec.attrs` for a type: attr name -> default. */ +function mirrorAttrs( + name: string, + kind: "node" | "mark", +): Record | null { + const schema = getSchema(docmostExtensions as never); + const spec = kind === "node" ? schema.nodes[name]?.spec : schema.marks[name]?.spec; + if (!spec) return null; + const out: Record = {}; + for (const [attr, def] of Object.entries(spec.attrs || {})) { + out[attr] = (def as { default?: unknown }).default; + } + return out; +} + +// An editor-ext attribute the mirror deliberately does NOT vendor because it has +// NO markdown round-trip representation — dropping it loses nothing on the +// git-sync cycle (the same rationale the flat-roundtrip property suite uses to +// allowlist e.g. `tableCell.backgroundColorName`). Blessed by the hand-curated +// surface snapshot (schema-surface-snapshot.test.ts), reviewed in every diff. +const ACCEPTED_ATTR_OMISSIONS = new Set([ + "highlight.colorName", // only `highlight.color` round-trips (==text==); the + // secondary palette-name is presentational and has no markdown form. +]); + +// An editor-ext attribute the mirror vendors but with a DIFFERENT default: the +// mirror normalizes an "absent" value to `null` (its uniform optional-attr +// convention) rather than editor-ext's UI-oriented default. None of these attrs +// is emitted on the markdown surface (the converter round-trips only the +// serializable ones), so the default never round-trips and the divergence is +// inert — but pinned here so a NEW default change on either side forces review. +const ACCEPTED_DEFAULT_DIVERGENCE = new Set([ + "image.src", // mirror null vs editor "" (an image is never emitted src-less) + "link.internal", // mirror null vs editor false (routing attr, not in md link) + "pdf.width", // mirror null vs editor 800 (presentational sizing, not in md) + "pdf.height", // mirror null vs editor 600 (presentational sizing, not in md) +]); + +describe("docmost schema vs @docmost/editor-ext (attribute-level contract)", () => { + it("vendors every editor-ext attribute (name) of every shared type — no silently-dropped attrs", () => { + const dropped: string[] = []; + for (const [name, { kind, attrs }] of editorExtOwnAttrs()) { + const mirror = mirrorAttrs(name, kind); + if (!mirror) continue; // whole-type omission is the name-level test's job + for (const attr of Object.keys(attrs)) { + const key = `${name}.${attr}`; + if (!(attr in mirror) && !ACCEPTED_ATTR_OMISSIONS.has(key)) { + dropped.push(key); + } + } + } + // Any entry here exists on the editor-ext node/mark but NOT in the mirror + // (and is not a blessed non-round-trippable omission): documents using it + // lose that attribute on a git-sync round-trip — the subpages.recursive + // class. Re-sync src/lib/docmost-schema.ts (and the surface snapshot) or add + // a reasoned ACCEPTED_ATTR_OMISSIONS entry before clearing. + expect(dropped.sort()).toEqual([]); + }); + + it("keeps every editor-ext attribute DEFAULT in sync — no silent default drift", () => { + const drift: string[] = []; + for (const [name, { kind, attrs }] of editorExtOwnAttrs()) { + const mirror = mirrorAttrs(name, kind); + if (!mirror) continue; + for (const [attr, extDefault] of Object.entries(attrs)) { + const key = `${name}.${attr}`; + // Skip attrs editor-ext declares WITHOUT a default (required attrs): + // the mirror deliberately makes them optional (`null`), a safe superset. + if (extDefault === undefined) continue; + if (!(attr in mirror)) continue; // a drop, reported by the name test + if ( + JSON.stringify(mirror[attr]) !== JSON.stringify(extDefault) && + !ACCEPTED_DEFAULT_DIVERGENCE.has(key) + ) { + drift.push( + `${key}: mirror=${JSON.stringify(mirror[attr])} editor-ext=${JSON.stringify(extDefault)}`, + ); + } + } + } + expect(drift.sort()).toEqual([]); + }); + + it("the attribute allowlists have no stale rows (each is really omitted / divergent)", () => { + const ext = editorExtOwnAttrs(); + const staleOmission: string[] = []; + for (const key of ACCEPTED_ATTR_OMISSIONS) { + const [name, attr] = key.split("."); + const entry = ext.get(name); + const mirror = entry ? mirrorAttrs(name, entry.kind) : null; + // Stale if editor-ext no longer declares it, or the mirror now DOES vendor + // it (so it should be removed from the omission allowlist). + if (!entry || !(attr in entry.attrs) || (mirror && attr in mirror)) { + staleOmission.push(key); + } + } + expect(staleOmission, "stale ACCEPTED_ATTR_OMISSIONS rows").toEqual([]); + + const staleDivergence: string[] = []; + for (const key of ACCEPTED_DEFAULT_DIVERGENCE) { + const [name, attr] = key.split("."); + const entry = ext.get(name); + const mirror = entry ? mirrorAttrs(name, entry.kind) : null; + const extDefault = entry?.attrs[attr]; + // Stale if the divergence no longer exists (attr gone, or defaults now + // agree) — the row should be dropped so the allowlist stays honest. + if ( + !entry || + !mirror || + !(attr in entry.attrs) || + !(attr in mirror) || + extDefault === undefined || + JSON.stringify(mirror[attr]) === JSON.stringify(extDefault) + ) { + staleDivergence.push(key); + } + } + expect(staleDivergence, "stale ACCEPTED_DEFAULT_DIVERGENCE rows").toEqual([]); + }); +}); diff --git a/packages/token-estimate/package.json b/packages/token-estimate/package.json new file mode 100644 index 00000000..eef030a6 --- /dev/null +++ b/packages/token-estimate/package.json @@ -0,0 +1,19 @@ +{ + "name": "@docmost/token-estimate", + "version": "0.1.0", + "description": "Shared, provider-agnostic token estimator (chars/2.5) used by the AI-chat client counter and the server history-replay budgeter, so the two never diverge.", + "private": true, + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "scripts": { + "build": "tsc", + "watch": "tsc --watch", + "test": "vitest run", + "test:watch": "vitest" + }, + "license": "MIT", + "devDependencies": { + "typescript": "^5.0.0", + "vitest": "4.1.6" + } +} diff --git a/packages/token-estimate/src/index.test.ts b/packages/token-estimate/src/index.test.ts new file mode 100644 index 00000000..2854efd0 --- /dev/null +++ b/packages/token-estimate/src/index.test.ts @@ -0,0 +1,31 @@ +import { describe, it, expect } from 'vitest'; +import { estimateTokens, CHARS_PER_TOKEN } from './index'; + +describe('estimateTokens (shared chars/2.5)', () => { + it('returns 0 for empty / nullish input', () => { + expect(estimateTokens('')).toBe(0); + expect(estimateTokens(null)).toBe(0); + expect(estimateTokens(undefined)).toBe(0); + }); + + it('uses the chars/2.5 ratio, ceiled', () => { + expect(CHARS_PER_TOKEN).toBe(2.5); + // 5 chars / 2.5 = 2 + expect(estimateTokens('abcde')).toBe(2); + // any non-empty string is at least 1 token (ceil) + expect(estimateTokens('a')).toBe(1); + // 100 chars / 2.5 = 40 + expect(estimateTokens('x'.repeat(100))).toBe(40); + }); + + it('counts Cyrillic ~2x higher than the old chars/4 rule (no undercount)', () => { + const cyr = 'привет мир как дела'; // 19 chars + expect(estimateTokens(cyr)).toBe(Math.ceil(19 / 2.5)); // 8 + expect(estimateTokens(cyr)).toBeGreaterThan(Math.ceil(19 / 4)); // > 5 + }); + + it('is deterministic / byte-stable (same input => same output)', () => { + const s = 'the quick brown fox'; + expect(estimateTokens(s)).toBe(estimateTokens(s)); + }); +}); diff --git a/packages/token-estimate/src/index.ts b/packages/token-estimate/src/index.ts new file mode 100644 index 00000000..ec8d2c04 --- /dev/null +++ b/packages/token-estimate/src/index.ts @@ -0,0 +1,35 @@ +/** + * Shared, provider-agnostic token estimator (#490). + * + * No provider exposes an exact tokenizer we can afford to run on the hot path (a + * real BPE pass is O(n²)-ish, bloats the client bundle, and is wrong for + * Gemini/Ollama anyway), so both the client's in-body counter AND the server's + * history-replay budgeter use this ONE cheap chars-based heuristic. Keeping it in + * a single shared module is deliberate: two independent estimators drift, and then + * "the badge shows 60%" while "the budgeter already trimmed" — the exact confusion + * this package prevents. + * + * Ratio: **chars / 2.5**. Most content here is Cyrillic, where a token is ~2.5 + * characters; the common English `chars/4` rule of thumb UNDER-counts Cyrillic by + * ~2×, which for a budget check is the dangerous direction (it lets the context + * overflow). 2.5 slightly over-estimates pure English/code, which is the SAFE + * direction for a budget. This is an estimate, never an exact count — the + * authoritative figure is always the provider's reported usage; the estimate is + * for UI affordances, the delta of not-yet-sent messages, and deciding what to + * trim. + */ + +/** Characters per token for the shared estimate. See the module comment. */ +export const CHARS_PER_TOKEN = 2.5; + +/** + * Rough token estimate for a piece of text (chars / {@link CHARS_PER_TOKEN}). + * Returns 0 for empty/nullish input, and ceils so any non-empty text counts as at + * least one token. Pure and deterministic (byte-stable), so the same text always + * yields the same estimate — which the server budgeter relies on to keep replay + * trimming stable turn to turn (provider prompt-cache friendliness). + */ +export function estimateTokens(text: string | null | undefined): number { + if (!text) return 0; + return Math.ceil(text.length / CHARS_PER_TOKEN); +} diff --git a/packages/token-estimate/tsconfig.json b/packages/token-estimate/tsconfig.json new file mode 100644 index 00000000..dec9218d --- /dev/null +++ b/packages/token-estimate/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "CommonJS", + "moduleResolution": "Node", + "outDir": "./dist", + "rootDir": "./src", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "declaration": true + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bdf58b2b..c876808e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -287,6 +287,9 @@ importers: '@docmost/prosemirror-markdown': specifier: workspace:* version: link:../../packages/prosemirror-markdown + '@docmost/token-estimate': + specifier: workspace:* + version: link:../../packages/token-estimate '@excalidraw/excalidraw': specifier: 0.18.0-3a5ef40 version: 0.18.0-3a5ef40(@types/react-dom@18.3.1)(@types/react@18.3.12)(react-dom@18.3.1(react@18.3.1))(react@18.3.1) @@ -561,6 +564,9 @@ importers: '@docmost/prosemirror-markdown': specifier: workspace:* version: link:../../packages/prosemirror-markdown + '@docmost/token-estimate': + specifier: workspace:* + version: link:../../packages/token-estimate '@fastify/compress': specifier: ^9.0.0 version: 9.0.0 @@ -1148,6 +1154,15 @@ importers: specifier: 4.1.6 version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@20.19.43)(@vitest/coverage-v8@4.1.6)(happy-dom@20.8.9)(jsdom@25.0.0)(vite@8.0.5(@types/node@20.19.43)(esbuild@0.28.0)(jiti@2.4.2)(less@4.2.0)(sugarss@5.0.1(postcss@8.5.14))(terser@5.39.0)(tsx@4.21.0)(yaml@2.8.3)) + packages/token-estimate: + devDependencies: + typescript: + specifier: ^5.0.0 + version: 5.9.3 + vitest: + specifier: 4.1.6 + version: 4.1.6(@opentelemetry/api@1.9.0)(@types/node@25.5.0)(@vitest/coverage-v8@4.1.6)(happy-dom@20.8.9)(jsdom@27.4.0(@noble/hashes@2.0.1))(vite@8.0.5(@types/node@25.5.0)(esbuild@0.28.0)(jiti@2.4.2)(less@4.2.0)(sugarss@5.0.1(postcss@8.5.14))(terser@5.39.0)(tsx@4.21.0)(yaml@2.8.3)) + packages: '@aashutoshrathi/word-wrap@1.2.6': diff --git a/scripts/ci/image-smoke.sh b/scripts/ci/image-smoke.sh new file mode 100755 index 00000000..d974e17b --- /dev/null +++ b/scripts/ci/image-smoke.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Boot-smoke for the exact Docker image that is about to be pushed (issue #476). +# +# Retrospective class "local logic is right, the integration property was never +# checked" (#353/#452/#361): every other CI job builds and tests code from the +# working tree, but the IMAGE watchtower pulls was never actually started +# anywhere before this gate. This script boots the built image against the +# publish job's postgres/redis services and asserts four integration +# properties end-to-end: +# S1 the app boots and /api/health answers (startup migrator + boot) +# S2 the first-run workspace setup endpoint works (API + DB writes) +# S3 the client dist is inside the image and served +# S4 hashed assets are served immutable (#452) with the precompressed +# brotli copy shipped in the image +set -euo pipefail + +IMAGE="${1:?usage: image-smoke.sh }" + +fail() { echo "FAIL: $*"; exit 1; } + +# Boot the exact image that will be pushed, wired to the job services via host +# network (postgres on localhost:5432, redis on localhost:6379). The container +# is deliberately NOT removed on failure so the workflow's dump-on-failure step +# can read `docker logs gitmost-smoke`. +docker run -d --name gitmost-smoke --network host \ + -e DATABASE_URL=postgresql://docmost:docmost@localhost:5432/docmost \ + -e REDIS_URL=redis://localhost:6379 \ + -e APP_SECRET=ci-smoke-secret-change-me-min-32-characters \ + -e APP_URL=http://localhost:3000 \ + "$IMAGE" + +# S1: wait for /api/health — covers the startup migrator + boot inside the +# shipped image (#361-boot, #353 runtime class): a migration the Kysely startup +# migrator rejects, or a runtime module missing from the image, dies right here. +healthy=0 +for _ in $(seq 1 60); do + if curl -fsS http://localhost:3000/api/health > /dev/null 2>&1; then + healthy=1 + break + fi + sleep 2 +done +[ "$healthy" -eq 1 ] || fail "S1: /api/health did not answer within 120s (boot or startup migration failed)" +echo "OK S1: image booted and /api/health answers" + +# S2: the first-run workspace setup works end-to-end (controller -> service -> +# DB write chain inside the shipped image, not just a static health probe). +curl -fsS -X POST http://localhost:3000/api/auth/setup \ + -H "Content-Type: application/json" \ + -d '{"name":"Smoke","email":"smoke@example.com","password":"SmokePassword123","workspaceName":"Smoke"}' \ + > /dev/null || fail "S2: POST /api/auth/setup failed" +echo "OK S2: workspace setup succeeded" + +# S3: the client dist is actually inside the image and served — the SPA HTML +# must reference hashed /assets/ bundles (a broken client COPY in the +# Dockerfile would serve an empty shell that every other job stays green on). +HTML=$(curl -fsS http://localhost:3000/) || fail "S3: fetching / failed" +grep -q '/assets/' <<<"$HTML" || fail "S3: served HTML references no /assets/ bundle (client dist missing from the image?)" +echo "OK S3: client dist served (HTML references /assets/)" + +# S4: hashed /assets/ files must be served with an immutable cache-control +# (#452 class: static.module.ts resolveStaticAssetHeaders owns the header) AND +# with the precompressed brotli neighbour. Both checks are mandatory — verified +# against the code: resolveStaticAssetHeaders marks every /assets/ path +# immutable, and the client build (vite-plugin-compression2, include covers +# .js) emits a .br copy next to every bundle that the Dockerfile ships and +# @fastify/static serves via preCompressed:true. +ASSET=$(grep -oE '/assets/[A-Za-z0-9._@/-]+\.js' <<<"$HTML" | head -1 || true) +[ -n "$ASSET" ] || fail "S4: no /assets/*.js path found in the served HTML" +HDRS=$(curl -fsSI -H 'Accept-Encoding: br' "http://localhost:3000$ASSET") || fail "S4: HEAD $ASSET failed" +grep -qi '^cache-control:.*immutable' <<<"$HDRS" || fail "S4: $ASSET served without an immutable cache-control (#452)" +echo "OK S4: hashed asset served with immutable cache-control" +grep -qi '^content-encoding:.*br' <<<"$HDRS" || fail "S4: $ASSET not served brotli-precompressed (content-encoding: br missing)" +echo "OK S4: hashed asset served with the precompressed brotli copy" + +# Remove the container ONLY on success, so the failure path keeps it around for +# the workflow's "Dump smoke container log on failure" step. +docker rm -f gitmost-smoke > /dev/null +echo "OK image smoke passed"