Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| eee01f9621 | |||
| 3ebdbfec81 | |||
| 1e61042bf6 | |||
| 3085ec1b50 | |||
| 05ec9feaf9 | |||
| 8e12579925 | |||
| 20703d06c2 | |||
| dab2660999 | |||
| 751d55e9db | |||
| 02308012a6 | |||
| 0665fcb630 | |||
| 97bd554cb5 | |||
| f77a6b42de | |||
| 43b11d92ab | |||
| f759084f41 |
@@ -151,6 +151,12 @@ jobs:
|
||||
- name: Build editor-ext
|
||||
run: pnpm --filter @docmost/editor-ext build
|
||||
|
||||
# @docmost/prosemirror-markdown is an ESM workspace package the server
|
||||
# imports at runtime; its build/ is gitignored and test:e2e has no pretest
|
||||
# hook, so build it before the e2e run (mirrors the test.yml job).
|
||||
- name: Build prosemirror-markdown
|
||||
run: pnpm --filter @docmost/prosemirror-markdown build
|
||||
|
||||
- name: Run migrations
|
||||
run: pnpm --filter ./apps/server migration:latest
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
GitmostListPagesResult,
|
||||
GitmostListSpacesResult,
|
||||
gitmostDecodePayloadToFile,
|
||||
gitmostInsertTranscriptIntoEditor,
|
||||
gitmostUploadFileToEditor,
|
||||
} from "@/features/editor/gitmost/gitmost-recording.ts";
|
||||
|
||||
@@ -281,6 +282,18 @@ export default function GitmostGlobalBridge() {
|
||||
pageId: page.id,
|
||||
};
|
||||
}
|
||||
|
||||
// Best-effort: append the transcript (heading + one paragraph per line)
|
||||
// below the just-inserted audio node. The audio insert already
|
||||
// succeeded, so a transcript failure must NOT turn this into an error —
|
||||
// wrap it and, on any throw, log and still return ok. A missing/empty/
|
||||
// non-string transcript is a no-op inside the helper (audio only).
|
||||
try {
|
||||
gitmostInsertTranscriptIntoEditor(editor, payload?.transcript);
|
||||
} catch (err) {
|
||||
console.error("[gitmost] transcript insert failed", err);
|
||||
}
|
||||
|
||||
return { ok: true, pageId: page.id };
|
||||
} catch (err: any) {
|
||||
console.error("[gitmost] createPageWithRecording failed", err);
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { Editor } from "@tiptap/core";
|
||||
import { Document } from "@tiptap/extension-document";
|
||||
import { Paragraph } from "@tiptap/extension-paragraph";
|
||||
import { Text } from "@tiptap/extension-text";
|
||||
import { Heading } from "@tiptap/extension-heading";
|
||||
import { Bold } from "@tiptap/extension-bold";
|
||||
import { Italic } from "@tiptap/extension-italic";
|
||||
import { Link } from "@tiptap/extension-link";
|
||||
import { gitmostInsertTranscriptIntoEditor } from "./gitmost-recording.ts";
|
||||
|
||||
const ZWSP = ""; // U+200B, the helper's block-trigger neutralizer
|
||||
|
||||
/**
|
||||
* #377 — the web-side bridge must append the native host's transcript below the
|
||||
* recording. These exercise the pure insert helper through a REAL Tiptap editor
|
||||
* (Document/Paragraph/Text/Heading + Bold/Italic/Link marks so an HTML-parsing
|
||||
* regression would be caught), asserting the resulting document rather than
|
||||
* mocking the editor: transcript present -> "Transcript" heading + one paragraph
|
||||
* per non-empty line; content is inserted as LITERAL TEXT (no HTML/markdown
|
||||
* parsing); col-0 markdown block triggers are neutralized so git-sync keeps them
|
||||
* paragraphs; absent/empty/non-string -> no-op.
|
||||
*/
|
||||
describe("gitmostInsertTranscriptIntoEditor", () => {
|
||||
const makeEditor = () =>
|
||||
new Editor({
|
||||
// Bold/Italic/Link are registered specifically so that IF the helper ever
|
||||
// regressed to inserting an HTML/markdown string (instead of a text node),
|
||||
// TipTap would parse `<b>`/`*..*`/`[..](..)` into marks and the literal-
|
||||
// text assertions below would fail.
|
||||
extensions: [Document, Paragraph, Text, Heading, Bold, Italic, Link],
|
||||
// Start from a single empty paragraph (a fresh page's baseline). The
|
||||
// helper appends at the end of the doc, i.e. below existing content.
|
||||
content: { type: "doc", content: [{ type: "paragraph" }] },
|
||||
});
|
||||
|
||||
it("inserts a Transcript heading + one paragraph per non-empty line, verbatim", () => {
|
||||
const editor = makeEditor();
|
||||
|
||||
const inserted = gitmostInsertTranscriptIntoEditor(
|
||||
editor,
|
||||
"You: hello there\nSpeaker 1: hi\n\nYou: bye",
|
||||
);
|
||||
|
||||
expect(inserted).toBe(true);
|
||||
|
||||
const nodes = (editor.getJSON().content ?? []) as any[];
|
||||
// A level-2 "Transcript" heading is present.
|
||||
const heading = nodes.find((n) => n.type === "heading");
|
||||
expect(heading?.attrs?.level).toBe(2);
|
||||
expect(heading?.content?.[0]?.text).toBe("Transcript");
|
||||
|
||||
// Every non-empty transcript line becomes a paragraph, in order, verbatim;
|
||||
// the blank line between them is dropped.
|
||||
const texts = nodes
|
||||
.filter((n) => n.type === "paragraph")
|
||||
.map((n) => n.content?.[0]?.text)
|
||||
.filter((t) => typeof t === "string");
|
||||
expect(texts).toEqual(["You: hello there", "Speaker 1: hi", "You: bye"]);
|
||||
|
||||
editor.destroy();
|
||||
});
|
||||
|
||||
it("inserts HTML + markdown metacharacters as LITERAL text (no injection / no mark parsing)", () => {
|
||||
const editor = makeEditor();
|
||||
const line =
|
||||
"You: <b>bold</b> <script>alert(1)</script> and *stars* and [link](x)";
|
||||
|
||||
const inserted = gitmostInsertTranscriptIntoEditor(editor, line);
|
||||
expect(inserted).toBe(true);
|
||||
|
||||
const paras = (editor.getJSON().content ?? []).filter(
|
||||
(n: any) => n.type === "paragraph",
|
||||
) as any[];
|
||||
// The transcript line is exactly ONE paragraph holding a SINGLE text node
|
||||
// whose text is the verbatim string — not split into bold/link/other nodes,
|
||||
// not carrying any marks, not raw HTML. This FAILS if the helper switched to
|
||||
// insertContent(htmlString): TipTap would then parse <b>/[link](x)/*stars*.
|
||||
const content = paras[paras.length - 1].content;
|
||||
expect(content).toHaveLength(1);
|
||||
expect(content[0].type).toBe("text");
|
||||
expect(content[0].marks ?? []).toEqual([]);
|
||||
expect(content[0].text).toBe(line);
|
||||
// And no bold/italic/link mark exists anywhere in the document.
|
||||
const html = editor.getHTML();
|
||||
expect(html).not.toMatch(/<(strong|b|em|i|a)\b/);
|
||||
// The angle brackets survived as escaped entities (literal text), not a live
|
||||
// <script>/<b> element.
|
||||
expect(html).not.toMatch(/<script/i);
|
||||
|
||||
editor.destroy();
|
||||
});
|
||||
|
||||
it("neutralizes col-0 markdown block triggers with a leading ZWSP (git-sync safety)", () => {
|
||||
const editor = makeEditor();
|
||||
// Trigger lines (some with a leaked indent) + a normal prefixed line.
|
||||
const inserted = gitmostInsertTranscriptIntoEditor(
|
||||
editor,
|
||||
[
|
||||
"- dash",
|
||||
" > quote", // leading indent must be trimmed then neutralized
|
||||
"# hash",
|
||||
"1. one",
|
||||
"> [!info] note",
|
||||
"```js",
|
||||
"---", // solid thematic break -> horizontalRule (text-losing) if unneutralized
|
||||
"***",
|
||||
"___",
|
||||
"You: normal line",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(inserted).toBe(true);
|
||||
|
||||
const texts = (editor.getJSON().content ?? [])
|
||||
.filter((n: any) => n.type === "paragraph")
|
||||
.map((n: any) => n.content?.[0]?.text)
|
||||
.filter((t: any) => typeof t === "string") as string[];
|
||||
|
||||
// Every block-trigger line is prefixed with the invisible ZWSP (indent
|
||||
// trimmed first); the normal `You:` line is left byte-exact.
|
||||
expect(texts).toEqual([
|
||||
ZWSP + "- dash",
|
||||
ZWSP + "> quote",
|
||||
ZWSP + "# hash",
|
||||
ZWSP + "1. one",
|
||||
ZWSP + "> [!info] note",
|
||||
ZWSP + "```js",
|
||||
ZWSP + "---",
|
||||
ZWSP + "***",
|
||||
ZWSP + "___",
|
||||
"You: normal line",
|
||||
]);
|
||||
|
||||
editor.destroy();
|
||||
});
|
||||
|
||||
it("is a no-op for undefined / empty / whitespace-only / non-string transcripts", () => {
|
||||
for (const value of [undefined, "", " \n \n", 42, {}, null]) {
|
||||
const editor = makeEditor();
|
||||
const before = JSON.stringify(editor.getJSON());
|
||||
|
||||
const inserted = gitmostInsertTranscriptIntoEditor(editor, value as any);
|
||||
|
||||
expect(inserted).toBe(false);
|
||||
// Document is untouched (audio-only behavior preserved).
|
||||
expect(JSON.stringify(editor.getJSON())).toBe(before);
|
||||
editor.destroy();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -65,6 +65,11 @@ export interface GitmostCreatePagePayload {
|
||||
base64: string;
|
||||
filename: string;
|
||||
mimeType: string;
|
||||
// Optional transcript for the recording: plain text, `\n`-separated, each
|
||||
// line already formatted as `You: ...` / `Speaker N: ...` by the native host
|
||||
// (ready to insert, no parsing needed). Omitted (no speech / no models) ->
|
||||
// audio only.
|
||||
transcript?: string;
|
||||
}
|
||||
|
||||
export interface GitmostCreatePageResult {
|
||||
@@ -235,6 +240,83 @@ export async function gitmostUploadFileToEditor(
|
||||
}
|
||||
}
|
||||
|
||||
// Zero-width space (U+200B). Prepended to a transcript line that begins with a
|
||||
// markdown BLOCK trigger: it is invisible in the rendered doc but shifts the
|
||||
// trigger off column 0, so the git-sync doc->markdown->doc round-trip keeps the
|
||||
// line a plain paragraph (see GITMOST_MD_BLOCK_TRIGGER_RE).
|
||||
const GITMOST_ZWSP = "";
|
||||
|
||||
// A markdown BLOCK-level construct that, sitting at column 0 of a paragraph
|
||||
// line, the git-sync markdown serializer (packages/prosemirror-markdown
|
||||
// markdown-converter.ts, `case "paragraph"`) would re-parse into a NON-paragraph
|
||||
// block on the doc->markdown->doc cycle. That serializer emits paragraph text
|
||||
// verbatim with NO block-escape (the pre-existing root cause), so a leading
|
||||
// `#`/`-`/`*`/`+`/`>`, an ordered-list `N.`/`N)`, a code fence ```/~~~, a table
|
||||
// `|`, or a `> [!info]` callout opener would silently become a heading / list /
|
||||
// quote / code block / table / callout. The final alternative matches a WHOLE-
|
||||
// LINE thematic break — solid `---`/`***`/`___` or spaced `- - -`/`_ _ _` (3+ of
|
||||
// the same `-`/`*`/`_`) — which round-trips into a `horizontalRule`; because
|
||||
// that node carries NO text, an un-neutralized separator line would LOSE its
|
||||
// text entirely (worse than the list/quote case). This matches a TRIMMED line's
|
||||
// start; the transcript's own `You:` / `Speaker N:` prefix begins with a letter
|
||||
// and never matches, so prefixed lines are left byte-exact.
|
||||
const GITMOST_MD_BLOCK_TRIGGER_RE =
|
||||
/^(?:#{1,6}(?:\s|$)|[-*+](?:\s|$)|>|\d+[.)](?:\s|$)|```|~~~|\||([-*_])(?:\s*\1){2,}\s*$)/;
|
||||
|
||||
// Append a transcript block BELOW the recording's audio node in a live editor:
|
||||
// a "Transcript" heading followed by one paragraph per non-empty transcript
|
||||
// line. The transcript is plain text, `\n`-separated, each line already
|
||||
// formatted as `You: ...` / `Speaker N: ...` by the native host — line text is
|
||||
// inserted as a TEXT node (never HTML/markdown), so there is no injection or
|
||||
// mark-parsing surface. Each kept line is trimmed (drops an indent that would
|
||||
// both leak into the display and, at col 0, form a markdown block trigger) and,
|
||||
// if it still begins with a col-0 markdown block trigger, gets an invisible
|
||||
// zero-width space prepended so the git-sync round-trip cannot turn it into a
|
||||
// list/quote/heading/callout/code/table (defensive boundary against the
|
||||
// serializer's missing block-escape). This is best-effort and meant to run
|
||||
// AFTER the audio has already been inserted; the caller must guard against a
|
||||
// throw so a transcript failure never fails the (already successful) recording.
|
||||
// Returns true when a block was inserted, false when there was nothing to
|
||||
// insert (transcript undefined/empty/not-a-string). A non-string value is a
|
||||
// no-op, not an error.
|
||||
export function gitmostInsertTranscriptIntoEditor(
|
||||
editor: Editor,
|
||||
transcript: unknown,
|
||||
): boolean {
|
||||
if (typeof transcript !== "string") return false;
|
||||
const lines = transcript
|
||||
.split("\n")
|
||||
// Trim each line and drop blank (whitespace-only) ones.
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0)
|
||||
// Neutralize a col-0 markdown block trigger with an invisible ZWSP so the
|
||||
// git-sync round-trip keeps the line a paragraph. Host lines (`You:` /
|
||||
// `Speaker N:`) never match and stay byte-exact.
|
||||
.map((line) =>
|
||||
GITMOST_MD_BLOCK_TRIGGER_RE.test(line) ? GITMOST_ZWSP + line : line,
|
||||
);
|
||||
if (lines.length === 0) return false;
|
||||
|
||||
const content = [
|
||||
{
|
||||
type: "heading",
|
||||
attrs: { level: 2 },
|
||||
content: [{ type: "text", text: "Transcript" }],
|
||||
},
|
||||
...lines.map((line) => ({
|
||||
type: "paragraph",
|
||||
content: [{ type: "text", text: line }],
|
||||
})),
|
||||
];
|
||||
|
||||
// Append at the end of the document. On a freshly-created recording page the
|
||||
// audio node is the last block, so the end position places the transcript
|
||||
// directly below it.
|
||||
const endPos = editor.state.doc.content.size;
|
||||
editor.chain().focus().insertContentAt(endPos, content).run();
|
||||
return true;
|
||||
}
|
||||
|
||||
// Full insert path used by the open-page bridge (insertRecording): guard the
|
||||
// editor, validate/decode the payload, then upload. Never throws — resolves to
|
||||
// a result code.
|
||||
|
||||
@@ -1,3 +1,10 @@
|
||||
export const HISTORY_INTERVAL = 5 * 60 * 1000;
|
||||
export const HISTORY_FAST_INTERVAL = 60 * 1000;
|
||||
export const HISTORY_FAST_THRESHOLD = 5 * 60 * 1000;
|
||||
|
||||
// #348 — debounce window for the per-page RAG re-embed job. Repeated saves
|
||||
// within this window collapse to a single delayed job (coalesced by a stable
|
||||
// jobId), so active editing does not pile up expensive re-embeds (external API
|
||||
// + page_embeddings rewrite, concurrency 1). The worker reads the CURRENT page
|
||||
// state at run time, so the last content within the window wins.
|
||||
export const EMBED_DEBOUNCE_MS = 30 * 1000;
|
||||
|
||||
@@ -431,7 +431,17 @@ describe('PersistenceExtension.onStoreDocument — Approach-A boundary snapshot'
|
||||
it('uses the canonical page.id (not the slugId doc name) for post-store side effects (#260)', async () => {
|
||||
const SLUG = 'slug-1'; // persistedHumanPage.slugId; findById resolves it
|
||||
const document = ydocFor(doc('NEW AGENT CONTENT'));
|
||||
pageRepo.findById.mockResolvedValue(persistedHumanPage('NEW AGENT CONTENT'));
|
||||
// #348 — the transclusion sync now runs only when the new OR the previously
|
||||
// persisted content carries a transclusion-family node. Give the persisted
|
||||
// (old) content a pageEmbed so the sync path is exercised and the #260
|
||||
// UUID-vs-slugId contract asserted below is still verified.
|
||||
pageRepo.findById.mockResolvedValue({
|
||||
...persistedHumanPage('NEW AGENT CONTENT'),
|
||||
content: {
|
||||
type: 'doc',
|
||||
content: [{ type: 'pageEmbed', attrs: { sourcePageId: 'src-1' } }],
|
||||
},
|
||||
});
|
||||
pageHistoryRepo.findPageLastHistory.mockResolvedValue(null);
|
||||
|
||||
// A `page.<slugId>` document name (the bug's smoking gun), agent store over
|
||||
|
||||
@@ -36,11 +36,13 @@ import {
|
||||
import { Page } from '@docmost/db/types/entity.types';
|
||||
import { CollabHistoryService } from '../services/collab-history.service';
|
||||
import {
|
||||
EMBED_DEBOUNCE_MS,
|
||||
HISTORY_FAST_INTERVAL,
|
||||
HISTORY_FAST_THRESHOLD,
|
||||
HISTORY_INTERVAL,
|
||||
} from '../constants';
|
||||
import { TransclusionService } from '../../core/page/transclusion/transclusion.service';
|
||||
import { hasTransclusionFamilyNodes } from '../../core/page/transclusion/utils/transclusion-prosemirror.util';
|
||||
import { observeCollabStore } from '../../integrations/metrics/metrics.registry';
|
||||
|
||||
/**
|
||||
@@ -415,7 +417,18 @@ export class PersistenceExtension implements Extension {
|
||||
// Use the canonical page UUID (page.id), not the doc-name id, which may be
|
||||
// a slugId for a `page.<slugId>` doc (#260). The transclusion/reference
|
||||
// syncs write uuid-typed columns, so a slugId here threw Postgres 22P02.
|
||||
await this.syncTransclusion(page.id, page.workspaceId, tiptapJson);
|
||||
//
|
||||
// #348 — skip the three sync SELECTs when neither the new content nor the
|
||||
// previously-persisted content has any transclusion/reference/pageEmbed
|
||||
// node: nothing to insert, and (the DB mirrors the old content) nothing to
|
||||
// delete. Whenever either side has one, run the idempotent sync exactly as
|
||||
// before so removals are still reconciled.
|
||||
if (
|
||||
hasTransclusionFamilyNodes(tiptapJson) ||
|
||||
hasTransclusionFamilyNodes(page.content)
|
||||
) {
|
||||
await this.syncTransclusion(page.id, page.workspaceId, tiptapJson);
|
||||
}
|
||||
}
|
||||
|
||||
if (page) {
|
||||
@@ -431,7 +444,17 @@ export class PersistenceExtension implements Extension {
|
||||
(m) => m.entityId,
|
||||
);
|
||||
|
||||
if (userMentions.length > 0) {
|
||||
// #348 — only enqueue when the mentioned-user set actually GAINED a member.
|
||||
// The processor (processPageMention) already no-ops when every current
|
||||
// mention was present before (newMentions.length === 0), so skipping the
|
||||
// enqueue in that case is behavior-identical and avoids piling up no-op jobs
|
||||
// on every save of a page that merely CONTAINS (unchanged) mentions.
|
||||
const oldMentionedUserIdSet = new Set(oldMentionedUserIds);
|
||||
const hasNewMentionedUser = userMentions.some(
|
||||
(m) => !oldMentionedUserIdSet.has(m.entityId),
|
||||
);
|
||||
|
||||
if (hasNewMentionedUser) {
|
||||
await this.notificationQueue.add(QueueJob.PAGE_MENTION_NOTIFICATION, {
|
||||
userMentions: userMentions.map((m) => ({
|
||||
userId: m.entityId,
|
||||
@@ -446,12 +469,23 @@ export class PersistenceExtension implements Extension {
|
||||
} as IPageMentionNotificationJob);
|
||||
}
|
||||
|
||||
await this.aiQueue.add(QueueJob.PAGE_CONTENT_UPDATED, {
|
||||
// Canonical UUID: the embedding reindex resolves pages by uuid, so a
|
||||
// slugId here threw Postgres 22P02 invalid-uuid (#260).
|
||||
pageIds: [page.id],
|
||||
workspaceId: page.workspaceId,
|
||||
});
|
||||
await this.aiQueue.add(
|
||||
QueueJob.PAGE_CONTENT_UPDATED,
|
||||
{
|
||||
// Canonical UUID: the embedding reindex resolves pages by uuid, so a
|
||||
// slugId here threw Postgres 22P02 invalid-uuid (#260).
|
||||
pageIds: [page.id],
|
||||
workspaceId: page.workspaceId,
|
||||
},
|
||||
// #348 — coalesce re-embeds during active editing. A stable per-page
|
||||
// jobId + delay means repeated saves within EMBED_DEBOUNCE_MS collapse
|
||||
// to one delayed job instead of one expensive re-embed per save. The
|
||||
// worker reads the current page state at run time, so last content wins.
|
||||
// BullMQ forbids ':' in custom job ids (Redis key separator), so '-' is
|
||||
// used; page.id is a UUID, so the id is unique per page. removeOnComplete
|
||||
// (queue.module) frees the id after each run so the next window re-arms.
|
||||
{ jobId: `embed-${page.id}`, delay: EMBED_DEBOUNCE_MS },
|
||||
);
|
||||
|
||||
await this.enqueuePageHistory(page, lastUpdatedSource);
|
||||
}
|
||||
|
||||
@@ -220,6 +220,13 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
};
|
||||
|
||||
async maintainLock(documentName: string) {
|
||||
// #348 — clear any existing timer for this document before installing a new
|
||||
// one. Without this, a second maintainLock for the same document (a
|
||||
// reload-without-unload) overwrites this.locks[documentName] and leaks the
|
||||
// previous interval, which keeps firing SET forever with no way to clear it.
|
||||
if (this.locks[documentName]) {
|
||||
clearInterval(this.locks[documentName]);
|
||||
}
|
||||
this.locks[documentName] = setInterval(() => {
|
||||
this.pub.set(
|
||||
this.getKey(documentName),
|
||||
|
||||
@@ -4,8 +4,21 @@ export const CacheKey = {
|
||||
`perm:space-roles:${userId}:${spaceId}`,
|
||||
PAGE_CAN_EDIT: (userId: string, pageId: string) =>
|
||||
`perm:can-edit:${userId}:${pageId}`,
|
||||
// #348 — DomainMiddleware workspace resolution. Self-hosted resolves the single
|
||||
// workspace (constant key); cloud resolves by the request subdomain (lowercased
|
||||
// to match the case-insensitive `LOWER(hostname)` lookup). Every WorkspaceRepo
|
||||
// mutator busts these, so staleness is bounded by both explicit invalidation and
|
||||
// the short TTL below.
|
||||
WORKSPACE_SELF_HOSTED: 'workspace:self-hosted',
|
||||
WORKSPACE_BY_HOST: (subdomain: string) =>
|
||||
`workspace:byhost:${subdomain.toLowerCase()}`,
|
||||
};
|
||||
|
||||
// Permission caches dedupe repeated checks within and across short request bursts.
|
||||
// 5s keeps staleness on revocations bounded.
|
||||
export const PERMISSION_CACHE_TTL_MS = 5_000;
|
||||
|
||||
// #348 — workspace row changes rarely; a short TTL bounds staleness of
|
||||
// security-relevant fields (enforceSso/enforceMfa/status) even if an explicit
|
||||
// bust is ever missed, while still removing the per-request workspace query.
|
||||
export const WORKSPACE_CACHE_TTL_MS = 15_000;
|
||||
|
||||
@@ -1,13 +1,42 @@
|
||||
import { Injectable, NestMiddleware, NotFoundException } from '@nestjs/common';
|
||||
import { Inject, Injectable, NestMiddleware } from '@nestjs/common';
|
||||
import { FastifyRequest, FastifyReply } from 'fastify';
|
||||
import { CACHE_MANAGER } from '@nestjs/cache-manager';
|
||||
import { Cache } from 'cache-manager';
|
||||
import { EnvironmentService } from '../../integrations/environment/environment.service';
|
||||
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
|
||||
import { Workspace } from '@docmost/db/types/entity.types';
|
||||
import { withCache } from '../helpers/with-cache';
|
||||
import { CacheKey, WORKSPACE_CACHE_TTL_MS } from '../helpers/cache-keys';
|
||||
|
||||
// #348 — timestamptz columns on the workspace row. The cache store (Keyv/Redis)
|
||||
// JSON-serializes values, so a cached workspace comes back with these fields as
|
||||
// ISO strings. Reviving them to Date keeps the cached path byte-identical to the
|
||||
// direct DB path (postgres.js returns Date), so nothing downstream can observe a
|
||||
// cache hit vs miss. Idempotent: `new Date(date)` on an already-Date value is a
|
||||
// no-op-equivalent. Keep in sync with the workspace timestamptz columns.
|
||||
const WORKSPACE_DATE_FIELDS: Array<keyof Workspace> = [
|
||||
'createdAt',
|
||||
'updatedAt',
|
||||
'deletedAt',
|
||||
'trialEndAt',
|
||||
];
|
||||
|
||||
function reviveWorkspaceDates(workspace: Workspace): Workspace {
|
||||
for (const field of WORKSPACE_DATE_FIELDS) {
|
||||
const value = workspace[field];
|
||||
if (value != null) {
|
||||
(workspace as any)[field] = new Date(value as any);
|
||||
}
|
||||
}
|
||||
return workspace;
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class DomainMiddleware implements NestMiddleware {
|
||||
constructor(
|
||||
private workspaceRepo: WorkspaceRepo,
|
||||
private environmentService: EnvironmentService,
|
||||
@Inject(CACHE_MANAGER) private readonly cacheManager: Cache,
|
||||
) {}
|
||||
async use(
|
||||
req: FastifyRequest['raw'],
|
||||
@@ -15,13 +44,21 @@ export class DomainMiddleware implements NestMiddleware {
|
||||
next: () => void,
|
||||
) {
|
||||
if (this.environmentService.isSelfHosted()) {
|
||||
const workspace = await this.workspaceRepo.findFirst();
|
||||
// #348 — cache the single-workspace lookup that runs on every request.
|
||||
// Invalidated by every WorkspaceRepo mutator (see bustWorkspaceCache).
|
||||
const workspace = await withCache(
|
||||
this.cacheManager,
|
||||
CacheKey.WORKSPACE_SELF_HOSTED,
|
||||
WORKSPACE_CACHE_TTL_MS,
|
||||
() => this.workspaceRepo.findFirst(),
|
||||
);
|
||||
if (!workspace) {
|
||||
//throw new NotFoundException('Workspace not found');
|
||||
(req as any).workspaceId = null;
|
||||
return next();
|
||||
}
|
||||
|
||||
reviveWorkspaceDates(workspace);
|
||||
// TODO: unify
|
||||
(req as any).workspaceId = workspace.id;
|
||||
(req as any).workspace = workspace;
|
||||
@@ -29,13 +66,21 @@ export class DomainMiddleware implements NestMiddleware {
|
||||
const header = req.headers.host;
|
||||
const subdomain = header.split('.')[0];
|
||||
|
||||
const workspace = await this.workspaceRepo.findByHostname(subdomain);
|
||||
// #348 — cache per-subdomain workspace resolution. Keyed by subdomain (the
|
||||
// hostname column); busted per hostname by every WorkspaceRepo mutator.
|
||||
const workspace = await withCache(
|
||||
this.cacheManager,
|
||||
CacheKey.WORKSPACE_BY_HOST(subdomain),
|
||||
WORKSPACE_CACHE_TTL_MS,
|
||||
() => this.workspaceRepo.findByHostname(subdomain),
|
||||
);
|
||||
|
||||
if (!workspace) {
|
||||
(req as any).workspaceId = null;
|
||||
return next();
|
||||
}
|
||||
|
||||
reviveWorkspaceDates(workspace);
|
||||
(req as any).workspaceId = workspace.id;
|
||||
(req as any).workspace = workspace;
|
||||
}
|
||||
|
||||
@@ -51,7 +51,21 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
|
||||
throw new UnauthorizedException();
|
||||
}
|
||||
|
||||
const workspace = await this.workspaceRepo.findById(payload.workspaceId);
|
||||
// #348 — reuse the workspace DomainMiddleware already loaded for this request
|
||||
// instead of re-querying it. `validate()` above has confirmed
|
||||
// `req.raw.workspaceId === payload.workspaceId` (or that it is unset), and the
|
||||
// middleware sets `req.raw.workspace` alongside `req.raw.workspaceId` from the
|
||||
// SAME workspace row, so when the ids match this is that row. NOTE it is the
|
||||
// middleware's `selectAll` object (a superset of the fallback `findById` base
|
||||
// fields — it also carries licenseKey/auditRetentionDays); that is harmless
|
||||
// here because every consumer reads this workspace via the AuthWorkspace
|
||||
// decorator, which already preferred `req.raw.workspace` (the selectAll object)
|
||||
// over `req.user.workspace` before this change. Fall back to the query if the
|
||||
// middleware did not populate it (a path that bypasses DomainMiddleware).
|
||||
const workspace =
|
||||
req.raw.workspace && req.raw.workspaceId === payload.workspaceId
|
||||
? req.raw.workspace
|
||||
: await this.workspaceRepo.findById(payload.workspaceId);
|
||||
|
||||
if (!workspace) {
|
||||
throw new UnauthorizedException();
|
||||
|
||||
@@ -38,6 +38,8 @@ export class FavoriteService {
|
||||
await this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds: result.items,
|
||||
userId,
|
||||
// #348 — favorites load at app-start; enable the workspace short-circuit.
|
||||
workspaceId,
|
||||
});
|
||||
const accessibleSet = new Set(accessibleIds);
|
||||
result.items = result.items.filter((id) => accessibleSet.has(id));
|
||||
@@ -125,6 +127,8 @@ export class FavoriteService {
|
||||
await this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds,
|
||||
userId,
|
||||
// #348 — workspace-level short-circuit for the favorites list.
|
||||
workspaceId,
|
||||
});
|
||||
accessiblePageSet = new Set(accessibleIds);
|
||||
}
|
||||
|
||||
@@ -23,7 +23,12 @@ export class NotificationController {
|
||||
@Body() dto: ListNotificationsDto,
|
||||
@AuthUser() user: User,
|
||||
) {
|
||||
return this.notificationService.findByUserId(user.id, dto, dto.type);
|
||||
return this.notificationService.findByUserId(
|
||||
user.id,
|
||||
dto,
|
||||
dto.type,
|
||||
user.workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
|
||||
@@ -45,6 +45,7 @@ export class NotificationService {
|
||||
userId: string,
|
||||
pagination: PaginationOptions,
|
||||
type: NotificationTab = 'all',
|
||||
workspaceId?: string | null,
|
||||
) {
|
||||
const result = await this.notificationRepo.findByUserId(
|
||||
userId,
|
||||
@@ -61,6 +62,8 @@ export class NotificationService {
|
||||
await this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds,
|
||||
userId,
|
||||
// #348 — notifications list; enable the workspace short-circuit.
|
||||
workspaceId,
|
||||
});
|
||||
const accessibleSet = new Set(accessiblePageIds);
|
||||
|
||||
|
||||
@@ -446,7 +446,11 @@ export class PageController {
|
||||
);
|
||||
}
|
||||
|
||||
return this.pageService.getRecentPages(user.id, pagination);
|
||||
return this.pageService.getRecentPages(
|
||||
user.id,
|
||||
pagination,
|
||||
user.workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@@ -469,7 +473,13 @@ export class PageController {
|
||||
}
|
||||
}
|
||||
|
||||
return this.pageService.getCreatedByPages(targetUserId, user.id, pagination, dto.spaceId);
|
||||
return this.pageService.getCreatedByPages(
|
||||
targetUserId,
|
||||
user.id,
|
||||
pagination,
|
||||
dto.spaceId,
|
||||
user.workspaceId,
|
||||
);
|
||||
}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
|
||||
@@ -1165,6 +1165,7 @@ export class PageService {
|
||||
async getRecentPages(
|
||||
userId: string,
|
||||
pagination: PaginationOptions,
|
||||
workspaceId?: string | null,
|
||||
): Promise<CursorPaginationResult<Page>> {
|
||||
const result = await this.pageRepo.getRecentPages(userId, pagination);
|
||||
|
||||
@@ -1174,6 +1175,8 @@ export class PageService {
|
||||
await this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds,
|
||||
userId,
|
||||
// #348 — cross-space "recent"; enable the workspace short-circuit.
|
||||
workspaceId,
|
||||
});
|
||||
const accessibleSet = new Set(accessibleIds);
|
||||
result.items = result.items.filter((p) => accessibleSet.has(p.id));
|
||||
@@ -1187,6 +1190,7 @@ export class PageService {
|
||||
requestingUserId: string,
|
||||
pagination: PaginationOptions,
|
||||
spaceId?: string,
|
||||
workspaceId?: string | null,
|
||||
): Promise<CursorPaginationResult<Page>> {
|
||||
const result = await this.pageRepo.getCreatedByPages(
|
||||
creatorId,
|
||||
@@ -1201,6 +1205,9 @@ export class PageService {
|
||||
await this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds,
|
||||
userId: requestingUserId,
|
||||
spaceId,
|
||||
// #348 — enable the workspace short-circuit when not space-scoped.
|
||||
workspaceId,
|
||||
});
|
||||
const accessibleSet = new Set(accessibleIds);
|
||||
result.items = result.items.filter((p) => accessibleSet.has(p.id));
|
||||
|
||||
@@ -93,6 +93,41 @@ function collectNodes<T>(
|
||||
return Array.from(byKey.values());
|
||||
}
|
||||
|
||||
/**
|
||||
* #348 — cheap early-exit probe: does this doc contain ANY node the transclusion
|
||||
* syncs care about (`transclusionSource` / `transclusionReference` / `pageEmbed`)?
|
||||
* Lets the collab store skip the three sync SELECTs when neither the previous nor
|
||||
* the new content has any such node — there is nothing to insert, and (since the
|
||||
* DB mirrors the previously-persisted content) nothing to delete. Walks once and
|
||||
* short-circuits on the first match; uses the same depth ceiling as the
|
||||
* collectors. Deliberately does NOT skip `transclusionSource` subtrees: it only
|
||||
* answers "any node present?", so descending everywhere is strictly conservative
|
||||
* (it can never wrongly report "none").
|
||||
*/
|
||||
export function hasTransclusionFamilyNodes(doc: unknown): boolean {
|
||||
const visit = (node: any, depth: number): boolean => {
|
||||
if (!node || typeof node !== 'object') return false;
|
||||
if (depth > MAX_PM_WALK_DEPTH) return false;
|
||||
|
||||
if (
|
||||
node.type === TRANSCLUSION_TYPE ||
|
||||
node.type === REFERENCE_TYPE ||
|
||||
node.type === PAGE_EMBED_TYPE
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (Array.isArray(node.content)) {
|
||||
for (const child of node.content) {
|
||||
if (visit(child, depth + 1)) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
return visit(doc, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks a ProseMirror JSON document and returns one snapshot per top-level
|
||||
* `transclusion` node. Does not recurse into transclusions (schema disallows
|
||||
|
||||
@@ -155,6 +155,8 @@ export class SearchService {
|
||||
pageIds,
|
||||
userId: opts.userId,
|
||||
spaceId: searchParams.spaceId,
|
||||
// #348 — enables the workspace-level short-circuit when not space-scoped.
|
||||
workspaceId: opts.workspaceId,
|
||||
});
|
||||
const accessibleSet = new Set(accessibleIds);
|
||||
results = results.filter((r: any) => accessibleSet.has(r.id));
|
||||
@@ -266,6 +268,8 @@ export class SearchService {
|
||||
await this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds,
|
||||
userId,
|
||||
// #348 — workspace-level short-circuit for the suggest path.
|
||||
workspaceId,
|
||||
});
|
||||
const accessibleSet = new Set(accessibleIds);
|
||||
pages = pages.filter((p) => accessibleSet.has(p.id));
|
||||
|
||||
@@ -4,7 +4,6 @@ import { EventName } from '../../common/events/event.contants';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { QueueJob, QueueName } from '../../integrations/queue/constants';
|
||||
import { Queue } from 'bullmq';
|
||||
import { EnvironmentService } from '../../integrations/environment/environment.service';
|
||||
|
||||
/**
|
||||
* Thin snapshot of a page node carried inside domain events so the WebSocket
|
||||
@@ -112,48 +111,24 @@ export class PageListener {
|
||||
private readonly logger = new Logger(PageListener.name);
|
||||
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
@InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
|
||||
@InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
|
||||
) {}
|
||||
|
||||
@OnEvent(EventName.PAGE_CREATED)
|
||||
async handlePageCreated(event: PageEvent) {
|
||||
const { pageIds, workspaceId } = event;
|
||||
if (this.isTypesense()) {
|
||||
await this.searchQueue.add(QueueJob.PAGE_CREATED, {
|
||||
pageIds,
|
||||
});
|
||||
}
|
||||
|
||||
await this.aiQueue.add(QueueJob.PAGE_CREATED, { pageIds, workspaceId });
|
||||
}
|
||||
|
||||
@OnEvent(EventName.PAGE_UPDATED)
|
||||
async handlePageUpdated(event: PageEvent) {
|
||||
const { pageIds } = event;
|
||||
|
||||
await this.searchQueue.add(QueueJob.PAGE_UPDATED, { pageIds });
|
||||
}
|
||||
|
||||
@OnEvent(EventName.PAGE_DELETED)
|
||||
async handlePageDeleted(event: PageEvent) {
|
||||
const { pageIds, workspaceId } = event;
|
||||
if (this.isTypesense()) {
|
||||
await this.searchQueue.add(QueueJob.PAGE_DELETED, { pageIds });
|
||||
}
|
||||
|
||||
await this.aiQueue.add(QueueJob.PAGE_DELETED, { pageIds, workspaceId });
|
||||
}
|
||||
|
||||
@OnEvent(EventName.PAGE_SOFT_DELETED)
|
||||
async handlePageSoftDeleted(event: PageEvent) {
|
||||
const { pageIds, workspaceId } = event;
|
||||
|
||||
if (this.isTypesense()) {
|
||||
await this.searchQueue.add(QueueJob.PAGE_SOFT_DELETED, { pageIds });
|
||||
}
|
||||
|
||||
await this.aiQueue.add(QueueJob.PAGE_SOFT_DELETED, {
|
||||
pageIds,
|
||||
workspaceId,
|
||||
@@ -163,14 +138,6 @@ export class PageListener {
|
||||
@OnEvent(EventName.PAGE_RESTORED)
|
||||
async handlePageRestored(event: PageEvent) {
|
||||
const { pageIds, workspaceId } = event;
|
||||
if (this.isTypesense()) {
|
||||
await this.searchQueue.add(QueueJob.PAGE_RESTORED, { pageIds });
|
||||
}
|
||||
|
||||
await this.aiQueue.add(QueueJob.PAGE_RESTORED, { pageIds, workspaceId });
|
||||
}
|
||||
|
||||
isTypesense(): boolean {
|
||||
return this.environmentService.getSearchDriver() === 'typesense';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import { EventName } from '../../common/events/event.contants';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { QueueJob, QueueName } from '../../integrations/queue/constants';
|
||||
import { Queue } from 'bullmq';
|
||||
import { EnvironmentService } from '../../integrations/environment/environment.service';
|
||||
|
||||
export class SpaceEvent {
|
||||
spaceId: string;
|
||||
@@ -15,22 +14,12 @@ export class SpaceListener {
|
||||
private readonly logger = new Logger(SpaceListener.name);
|
||||
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
@InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
|
||||
@InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
|
||||
) {}
|
||||
|
||||
@OnEvent(EventName.SPACE_DELETED)
|
||||
async handleSpaceDeleted(event: SpaceEvent) {
|
||||
const { spaceId } = event;
|
||||
if (this.isTypesense()) {
|
||||
await this.searchQueue.add(QueueJob.SPACE_DELETED, { spaceId });
|
||||
}
|
||||
|
||||
await this.aiQueue.add(QueueJob.SPACE_DELETED, { spaceId });
|
||||
}
|
||||
|
||||
isTypesense(): boolean {
|
||||
return this.environmentService.getSearchDriver() === 'typesense';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,6 @@ import { EventName } from '../../common/events/event.contants';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { QueueJob, QueueName } from '../../integrations/queue/constants';
|
||||
import { Queue } from 'bullmq';
|
||||
import { EnvironmentService } from '../../integrations/environment/environment.service';
|
||||
|
||||
export class WorkspaceEvent {
|
||||
workspaceId: string;
|
||||
@@ -15,22 +14,12 @@ export class WorkspaceListener {
|
||||
private readonly logger = new Logger(WorkspaceListener.name);
|
||||
|
||||
constructor(
|
||||
private readonly environmentService: EnvironmentService,
|
||||
@InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
|
||||
@InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue,
|
||||
) {}
|
||||
|
||||
@OnEvent(EventName.WORKSPACE_DELETED)
|
||||
async handlePageDeleted(event: WorkspaceEvent) {
|
||||
const { workspaceId } = event;
|
||||
if (this.isTypesense()) {
|
||||
await this.searchQueue.add(QueueJob.WORKSPACE_DELETED, { workspaceId });
|
||||
}
|
||||
|
||||
await this.aiQueue.add(QueueJob.WORKSPACE_DELETED, { workspaceId });
|
||||
}
|
||||
|
||||
isTypesense(): boolean {
|
||||
return this.environmentService.getSearchDriver() === 'typesense';
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
import { type Kysely, sql } from 'kysely';
|
||||
|
||||
/**
|
||||
* #348 — targeted hot-path indexes.
|
||||
*
|
||||
* 1. GIN trigram indexes for `/search/suggest`. That endpoint runs a
|
||||
* leading-wildcard `LOWER(f_unaccent(col)) LIKE '%q%'` per keystroke, which
|
||||
* is a sequential scan without a trigram index. The index EXPRESSIONS below
|
||||
* are `LOWER(f_unaccent(title|name))`, matching the predicates in
|
||||
* search.service.ts exactly so the planner uses them (verified with EXPLAIN:
|
||||
* the suggest predicate resolves to a Bitmap Index Scan on these indexes).
|
||||
*
|
||||
* IMMUTABLE-wrapper fix (required for the index to build): `f_unaccent` was
|
||||
* defined as `SELECT unaccent('unaccent', $1)` (the two-arg, dictionary-named
|
||||
* unaccent). That body CANNOT be used in an index expression: when Postgres
|
||||
* inlines the IMMUTABLE SQL wrapper while building the index it fails to
|
||||
* resolve the two-arg call (`function unaccent(unknown, text) does not exist`,
|
||||
* the `'unaccent'` literal loses its regdictionary coercion). The single-arg
|
||||
* `unaccent($1)` is the same operation (the default text-search dictionary IS
|
||||
* `unaccent`; verified byte-equal on accented samples), and — crucially —
|
||||
* SCHEMA-QUALIFIED as `public.unaccent($1)` it inlines cleanly, so the index
|
||||
* builds. We therefore `CREATE OR REPLACE` `f_unaccent` to the qualified
|
||||
* single-arg body. This is output-identical for every existing caller (the
|
||||
* tsvector trigger, the main `tsv @@` search, and the suggest LIKE), so no
|
||||
* reindex/backfill is needed; `down()` restores the original two-arg body.
|
||||
* (The `unaccent` extension is installed in `public` in this codebase, which
|
||||
* is why `public.unaccent` is the correct qualification.)
|
||||
*
|
||||
* 2. Composite indexes for two ORDER-BY-only-on-id queries that currently sort
|
||||
* on top of a created_at index:
|
||||
* - page_history: `findPageHistoryByPageId` does WHERE page_id ORDER BY id
|
||||
* DESC, but only `(page_id, created_at DESC)` exists → extra sort.
|
||||
* - comments: `findPageComments` does WHERE page_id ORDER BY id ASC, but only
|
||||
* `(page_id)` exists → extra sort.
|
||||
*
|
||||
* DEPLOY-TIME LOCK WARNING: these are plain (non-CONCURRENT) CREATE INDEX
|
||||
* statements — CONCURRENTLY is impossible because Kysely runs each migration in a
|
||||
* transaction. They take a SHARE lock that BLOCKS writes (INSERT/UPDATE/DELETE) on
|
||||
* pages/users/groups/comments/page_history for the duration of the build. The two
|
||||
* GIN trigram builds on pages.title / users.name are the slow ones and can take
|
||||
* minutes on a large tenant → a write-outage window during the deploy migration.
|
||||
* For large installations, run this migration in a maintenance window, or build
|
||||
* the trigram indexes out-of-band with CREATE INDEX CONCURRENTLY before deploying
|
||||
* (then this migration's `IF NOT EXISTS` is a no-op). Small/typical tenants are
|
||||
* unaffected.
|
||||
*/
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
// Index-compatible, output-identical redefinition of f_unaccent (see header).
|
||||
await sql`
|
||||
CREATE OR REPLACE FUNCTION f_unaccent(text)
|
||||
RETURNS text
|
||||
LANGUAGE sql
|
||||
IMMUTABLE PARALLEL SAFE STRICT
|
||||
AS $func$
|
||||
SELECT public.unaccent($1);
|
||||
$func$
|
||||
`.execute(db);
|
||||
|
||||
// Search-suggest trigram indexes. Expressions match search.service.ts.
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_title_trgm
|
||||
ON pages USING gin ((LOWER(f_unaccent(title))) gin_trgm_ops)
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_users_name_trgm
|
||||
ON users USING gin ((LOWER(f_unaccent(name))) gin_trgm_ops)
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_groups_name_trgm
|
||||
ON groups USING gin ((LOWER(f_unaccent(name))) gin_trgm_ops)
|
||||
`.execute(db);
|
||||
|
||||
// page_history: WHERE page_id ORDER BY id DESC (findPageHistoryByPageId).
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_page_history_page_id
|
||||
ON page_history (page_id, id DESC)
|
||||
`.execute(db);
|
||||
|
||||
// comments: WHERE page_id ORDER BY id ASC (findPageComments).
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_comments_page_id_id
|
||||
ON comments (page_id, id)
|
||||
`.execute(db);
|
||||
|
||||
// page_access(workspace_id): #348 made hasRestrictedPagesInWorkspace uncached
|
||||
// (F1 fix), so `EXISTS(SELECT 1 FROM page_access WHERE workspace_id=?)` now runs
|
||||
// per-request on every whole-workspace list endpoint (global search + suggest,
|
||||
// favorites, notifications, recent, created-by). page_access only had a
|
||||
// space_id index → that EXISTS was a seq scan in the common zero-restriction
|
||||
// case. This index makes it an index-only existence probe.
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_page_access_workspace_id
|
||||
ON page_access (workspace_id)
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
// Drop the expression indexes before restoring the function body.
|
||||
await sql`DROP INDEX IF EXISTS idx_pages_title_trgm`.execute(db);
|
||||
await sql`DROP INDEX IF EXISTS idx_users_name_trgm`.execute(db);
|
||||
await sql`DROP INDEX IF EXISTS idx_groups_name_trgm`.execute(db);
|
||||
await sql`DROP INDEX IF EXISTS idx_page_history_page_id`.execute(db);
|
||||
await sql`DROP INDEX IF EXISTS idx_comments_page_id_id`.execute(db);
|
||||
await sql`DROP INDEX IF EXISTS idx_page_access_workspace_id`.execute(db);
|
||||
|
||||
// Restore the original two-arg (dictionary-named) f_unaccent body.
|
||||
await sql`
|
||||
CREATE OR REPLACE FUNCTION f_unaccent(text)
|
||||
RETURNS text
|
||||
LANGUAGE sql
|
||||
IMMUTABLE PARALLEL SAFE STRICT
|
||||
AS $func$
|
||||
SELECT unaccent('unaccent', $1);
|
||||
$func$
|
||||
`.execute(db);
|
||||
}
|
||||
@@ -657,8 +657,9 @@ export class PagePermissionRepo {
|
||||
pageIds: string[];
|
||||
userId: string;
|
||||
spaceId?: string;
|
||||
workspaceId?: string | null;
|
||||
}): Promise<string[]> {
|
||||
const { pageIds, userId, spaceId } = opts;
|
||||
const { pageIds, userId, spaceId, workspaceId } = opts;
|
||||
if (pageIds.length === 0) return [];
|
||||
|
||||
if (spaceId) {
|
||||
@@ -666,6 +667,17 @@ export class PagePermissionRepo {
|
||||
if (!hasRestrictions) {
|
||||
return pageIds;
|
||||
}
|
||||
} else if (workspaceId) {
|
||||
// #348 — whole-workspace callers (no spaceId: favorites, notifications,
|
||||
// recent, created-by, global search) skip the recursive-ancestor CTE + anti
|
||||
// -join entirely when the workspace has ZERO restricted pages. When any
|
||||
// restriction DOES exist, fall through to the identical CTE below, so
|
||||
// behavior is unchanged whenever restrictions are present.
|
||||
const hasRestrictions =
|
||||
await this.hasRestrictedPagesInWorkspace(workspaceId);
|
||||
if (!hasRestrictions) {
|
||||
return pageIds;
|
||||
}
|
||||
}
|
||||
|
||||
const results = await this.db
|
||||
@@ -903,6 +915,39 @@ export class PagePermissionRepo {
|
||||
return Boolean(result?.exists);
|
||||
}
|
||||
|
||||
/**
|
||||
* Workspace-level analogue of hasRestrictedPagesInSpace: does ANY page in the
|
||||
* whole workspace carry a restriction? Lets whole-workspace access filters
|
||||
* short-circuit the recursive-ancestor CTE when nothing is restricted at all.
|
||||
*
|
||||
* UNCACHED (like the sibling hasRestrictedPagesInSpace) — a single cheap
|
||||
* `EXISTS(pageAccess WHERE workspaceId=?)` per call. This is an ACCESS-CONTROL
|
||||
* gate on whole-workspace list endpoints, so it must never go stale: caching it
|
||||
* (even 5s) reintroduced a leak the space-path never had — a concurrent
|
||||
* whole-workspace read in the insert->commit window of the FIRST restricted page
|
||||
* could re-populate `false` under withCache (read-then-set, no del-during-read
|
||||
* guard) and override the insert bust, leaking that page to unauthorized users
|
||||
* for up to the TTL (#348 review F1). An uncached EXISTS removes both the
|
||||
* cache/DB asymmetry with hasRestrictedPagesInSpace and that race; the space
|
||||
* path already accepts this exact per-call cost.
|
||||
*/
|
||||
async hasRestrictedPagesInWorkspace(workspaceId: string): Promise<boolean> {
|
||||
const result = await this.db
|
||||
.selectNoFrom((eb) =>
|
||||
eb
|
||||
.exists(
|
||||
eb
|
||||
.selectFrom('pageAccess')
|
||||
.select(sql`1`.as('one'))
|
||||
.where('pageAccess.workspaceId', '=', workspaceId),
|
||||
)
|
||||
.as('exists'),
|
||||
)
|
||||
.executeTakeFirst();
|
||||
|
||||
return Boolean(result?.exists);
|
||||
}
|
||||
|
||||
/**
|
||||
* Given a list of parent page IDs, return which ones have at least one accessible child.
|
||||
* Efficient batch query for sidebar hasChildren calculation.
|
||||
|
||||
@@ -581,6 +581,9 @@ export class PageRepo {
|
||||
const query = this.db
|
||||
.selectFrom('pages')
|
||||
.select(this.baseFields)
|
||||
// NOTE: `content` IS needed here — the trash UI reads page.content to render
|
||||
// the deleted-page preview modal (trash.tsx handlePageClick ->
|
||||
// TrashPageContentModal pageContent). Do NOT drop it (see #348 review F3).
|
||||
.select('content')
|
||||
.select((eb) => this.withSpace(eb))
|
||||
.select((eb) => this.withDeletedBy(eb))
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Inject, Injectable } from '@nestjs/common';
|
||||
import { CACHE_MANAGER } from '@nestjs/cache-manager';
|
||||
import { Cache } from 'cache-manager';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '../../types/kysely.types';
|
||||
import { dbOrTx } from '../../utils';
|
||||
@@ -9,6 +11,7 @@ import {
|
||||
} from '@docmost/db/types/entity.types';
|
||||
import { ExpressionBuilder, sql } from 'kysely';
|
||||
import { DB, Workspaces } from '@docmost/db/types/db';
|
||||
import { CacheKey } from '../../../common/helpers/cache-keys';
|
||||
|
||||
/**
|
||||
* Writable `settings.ai.provider` keys, enforced at this generic SQL layer. This
|
||||
@@ -61,7 +64,34 @@ export class WorkspaceRepo {
|
||||
'temporaryNoteHours',
|
||||
'isScimEnabled',
|
||||
];
|
||||
constructor(@InjectKysely() private readonly db: KyselyDB) {}
|
||||
constructor(
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
@Inject(CACHE_MANAGER) private readonly cacheManager: Cache,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* #348 — bust the DomainMiddleware workspace caches after any workspace write.
|
||||
* Deletes BOTH the self-hosted (constant) key and the cloud per-hostname key so
|
||||
* a single implementation covers either deployment mode (the irrelevant key is a
|
||||
* harmless no-op). Best-effort: a cache error must never fail the write, and a
|
||||
* missed bust is bounded by WORKSPACE_CACHE_TTL_MS. Note: a hostname RENAME only
|
||||
* busts the NEW hostname's key (the row returned here carries the new hostname);
|
||||
* the old key expires via TTL.
|
||||
*/
|
||||
private async bustWorkspaceCache(
|
||||
workspace?: Pick<Workspace, 'hostname'> | undefined,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await this.cacheManager.del(CacheKey.WORKSPACE_SELF_HOSTED);
|
||||
if (workspace?.hostname) {
|
||||
await this.cacheManager.del(
|
||||
CacheKey.WORKSPACE_BY_HOST(workspace.hostname),
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
// cache is best-effort; TTL is the backstop
|
||||
}
|
||||
}
|
||||
|
||||
async findById(
|
||||
workspaceId: string,
|
||||
@@ -144,12 +174,14 @@ export class WorkspaceRepo {
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<Workspace> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
const workspace = await db
|
||||
.updateTable('workspaces')
|
||||
.set({ ...updatableWorkspace, updatedAt: new Date() })
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
await this.bustWorkspaceCache(workspace);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
async insertWorkspace(
|
||||
@@ -157,11 +189,14 @@ export class WorkspaceRepo {
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<Workspace> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
const workspace = await db
|
||||
.insertInto('workspaces')
|
||||
.values(insertableWorkspace)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
// Bust the cached "not found" so a fresh install / new tenant is seen at once.
|
||||
await this.bustWorkspaceCache(workspace);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
async count(): Promise<number> {
|
||||
@@ -203,7 +238,7 @@ export class WorkspaceRepo {
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
const workspace = await db
|
||||
.updateTable('workspaces')
|
||||
.set({
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb)
|
||||
@@ -214,6 +249,8 @@ export class WorkspaceRepo {
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
await this.bustWorkspaceCache(workspace);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
async updateAiSettings(
|
||||
@@ -223,7 +260,7 @@ export class WorkspaceRepo {
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
const workspace = await db
|
||||
.updateTable('workspaces')
|
||||
.set({
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb)
|
||||
@@ -234,6 +271,8 @@ export class WorkspaceRepo {
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
await this.bustWorkspaceCache(workspace);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -272,7 +311,7 @@ export class WorkspaceRepo {
|
||||
entries.flatMap(([k, v]) => [sql.lit(k), sql`${v}::text`]),
|
||||
)})`
|
||||
: sql`'{}'::jsonb`;
|
||||
return db
|
||||
const workspace = await db
|
||||
.updateTable('workspaces')
|
||||
.set({
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb) || jsonb_build_object(
|
||||
@@ -287,6 +326,8 @@ export class WorkspaceRepo {
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
await this.bustWorkspaceCache(workspace);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -303,7 +344,7 @@ export class WorkspaceRepo {
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
const workspace = await db
|
||||
.updateTable('workspaces')
|
||||
.set({
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb)
|
||||
@@ -313,6 +354,8 @@ export class WorkspaceRepo {
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
await this.bustWorkspaceCache(workspace);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
async updateSharingSettings(
|
||||
@@ -322,7 +365,7 @@ export class WorkspaceRepo {
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
const workspace = await db
|
||||
.updateTable('workspaces')
|
||||
.set({
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb)
|
||||
@@ -333,6 +376,8 @@ export class WorkspaceRepo {
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
await this.bustWorkspaceCache(workspace);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
async updateTemplateSettings(
|
||||
@@ -342,7 +387,7 @@ export class WorkspaceRepo {
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
const workspace = await db
|
||||
.updateTable('workspaces')
|
||||
.set({
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb)
|
||||
@@ -353,6 +398,8 @@ export class WorkspaceRepo {
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
await this.bustWorkspaceCache(workspace);
|
||||
return workspace;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -2,15 +2,36 @@ import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
import { isStreamingResponse } from './metrics.constants';
|
||||
import { observeHttp } from './metrics.registry';
|
||||
|
||||
// URL path prefixes served by @fastify/static (client build output under
|
||||
// client/dist). `/assets/` holds the content-hashed bundle (index-*.js,
|
||||
// chunk-*.js) — a NEW set of names every deploy, i.e. an UNBOUNDED label set
|
||||
// (#362); the others (/vad/, /brand/, /locales/, /icons/ — copied verbatim from
|
||||
// public/) have stable names, so they are merely repetitive per-file labels
|
||||
// rather than unbounded. Either way none of these belong in the API-route
|
||||
// histogram: collapse them all to one bounded `static` label. (Edge latency for
|
||||
// static is already measured by Traefik's traefik_router_request_duration_*.)
|
||||
const STATIC_PATH_PREFIXES = [
|
||||
'/assets/',
|
||||
'/vad/',
|
||||
'/brand/',
|
||||
'/locales/',
|
||||
'/icons/',
|
||||
];
|
||||
|
||||
/**
|
||||
* Resolve the BOUNDED route label for an HTTP response.
|
||||
*
|
||||
* HARD REQUIREMENT (#355): use the ROUTE TEMPLATE (`/pages/:id`), NEVER the raw
|
||||
* URL (`/pages/abc-123`), so label cardinality stays finite. Fastify exposes the
|
||||
* matched template on `req.routeOptions.url`. On 404s (no route matched) that is
|
||||
* missing → collapse to the literal `unknown`.
|
||||
* HARD REQUIREMENT (#355): use the ROUTE TEMPLATE (`/pages/:id`), NEVER a raw
|
||||
* URL (`/pages/abc-123` or `/assets/index-CAbxDtto.js`), so label cardinality
|
||||
* stays finite. Fastify exposes the matched template on `req.routeOptions.url`,
|
||||
* BUT @fastify/static serves each file through a route whose matched url is the
|
||||
* raw (hashed) file path — so for static assets that value is itself unbounded.
|
||||
* Detect static requests by their path prefix FIRST and collapse to `static`;
|
||||
* otherwise use the route template; on a 404 (no route matched) → `unknown`.
|
||||
*/
|
||||
export function resolveRouteLabel(req: FastifyRequest): string {
|
||||
const path = (req.url ?? '').split('?', 1)[0];
|
||||
if (STATIC_PATH_PREFIXES.some((p) => path.startsWith(p))) return 'static';
|
||||
const url = req.routeOptions?.url;
|
||||
return typeof url === 'string' && url.length > 0 ? url : 'unknown';
|
||||
}
|
||||
|
||||
@@ -46,7 +46,6 @@ export class MetricsBullService implements OnModuleInit, OnModuleDestroy {
|
||||
@InjectQueue(QueueName.GENERAL_QUEUE) generalQueue: Queue,
|
||||
@InjectQueue(QueueName.BILLING_QUEUE) billingQueue: Queue,
|
||||
@InjectQueue(QueueName.FILE_TASK_QUEUE) fileTaskQueue: Queue,
|
||||
@InjectQueue(QueueName.SEARCH_QUEUE) searchQueue: Queue,
|
||||
@InjectQueue(QueueName.AI_QUEUE) aiQueue: Queue,
|
||||
@InjectQueue(QueueName.HISTORY_QUEUE) historyQueue: Queue,
|
||||
@InjectQueue(QueueName.NOTIFICATION_QUEUE) notificationQueue: Queue,
|
||||
@@ -58,7 +57,6 @@ export class MetricsBullService implements OnModuleInit, OnModuleDestroy {
|
||||
{ label: 'general', queue: generalQueue },
|
||||
{ label: 'billing', queue: billingQueue },
|
||||
{ label: 'file-task', queue: fileTaskQueue },
|
||||
{ label: 'search', queue: searchQueue },
|
||||
{ label: 'ai', queue: aiQueue },
|
||||
{ label: 'history', queue: historyQueue },
|
||||
{ label: 'notification', queue: notificationQueue },
|
||||
|
||||
@@ -25,6 +25,61 @@ describe('resolveRouteLabel (histogram route label)', () => {
|
||||
const req = { url: '/x' } as unknown as FastifyRequest;
|
||||
expect(resolveRouteLabel(req)).toBe('unknown');
|
||||
});
|
||||
|
||||
it.each([
|
||||
'/assets/index-CAbxDtto.js',
|
||||
'/assets/chunk-3OPIFGDE-CJOt9nr5.js',
|
||||
'/assets/excalidraw-menu-DpsI0kFW.js',
|
||||
'/vad/silero_vad_v5.onnx',
|
||||
'/brand/logo.svg',
|
||||
'/locales/en.json',
|
||||
'/icons/app-icon-192x192.png',
|
||||
])('collapses hashed/static asset %p to "static" (#362 cardinality)', (url) => {
|
||||
// @fastify/static serves each file through a route whose matched url is the
|
||||
// raw (hashed) file path, so routeOptions.url is itself unbounded here.
|
||||
const req = {
|
||||
url,
|
||||
routeOptions: { url },
|
||||
} as unknown as FastifyRequest;
|
||||
const label = resolveRouteLabel(req);
|
||||
expect(label).toBe('static');
|
||||
expect(label).not.toContain('.js');
|
||||
expect(label).not.toContain('index-');
|
||||
});
|
||||
|
||||
it('strips the query string before the static-prefix check', () => {
|
||||
const req = {
|
||||
url: '/assets/index-CAbxDtto.js?v=2',
|
||||
routeOptions: { url: '/assets/index-CAbxDtto.js' },
|
||||
} as unknown as FastifyRequest;
|
||||
expect(resolveRouteLabel(req)).toBe('static');
|
||||
});
|
||||
|
||||
it('does NOT collapse a real API route that merely mentions assets', () => {
|
||||
// A templated API route is kept as-is; only the static path PREFIXES collapse.
|
||||
const req = {
|
||||
url: '/api/pages/assets-guide',
|
||||
routeOptions: { url: '/api/pages/:id' },
|
||||
} as unknown as FastifyRequest;
|
||||
expect(resolveRouteLabel(req)).toBe('/api/pages/:id');
|
||||
});
|
||||
|
||||
it.each([
|
||||
// The TRAILING SLASH on the prefix is the anti-false-collapse guard: a path
|
||||
// that is the prefix WITHOUT its slash, or merely shares the prefix as a
|
||||
// substring of a longer segment, must NOT collapse. These would collapse
|
||||
// under a buggy `includes('/assets/')` / slashless-prefix impl.
|
||||
'/assets',
|
||||
'/assetsx/foo.js',
|
||||
'/iconset/x.png',
|
||||
])('does NOT collapse the prefix-boundary case %p', (url) => {
|
||||
const req = {
|
||||
url,
|
||||
routeOptions: { url: '/some/:route' },
|
||||
} as unknown as FastifyRequest;
|
||||
expect(resolveRouteLabel(req)).not.toBe('static');
|
||||
expect(resolveRouteLabel(req)).toBe('/some/:route');
|
||||
});
|
||||
});
|
||||
|
||||
describe('isStreamingResponse (SSE exclusion)', () => {
|
||||
|
||||
@@ -4,7 +4,6 @@ export enum QueueName {
|
||||
GENERAL_QUEUE = '{general-queue}',
|
||||
BILLING_QUEUE = '{billing-queue}',
|
||||
FILE_TASK_QUEUE = '{file-task-queue}',
|
||||
SEARCH_QUEUE = '{search-queue}',
|
||||
AI_QUEUE = '{ai-queue}',
|
||||
HISTORY_QUEUE = '{history-queue}',
|
||||
NOTIFICATION_QUEUE = '{notification-queue}',
|
||||
@@ -32,12 +31,6 @@ export enum QueueJob {
|
||||
IMPORT_TASK = 'import-task',
|
||||
EXPORT_TASK = 'export-task',
|
||||
|
||||
SEARCH_INDEX_PAGE = 'search-index-page',
|
||||
SEARCH_INDEX_PAGES = 'search-index-pages',
|
||||
SEARCH_INDEX_COMMENT = 'search-index-comment',
|
||||
SEARCH_INDEX_COMMENTS = 'search-index-comments',
|
||||
SEARCH_INDEX_ATTACHMENT = 'search-index-attachment',
|
||||
SEARCH_INDEX_ATTACHMENTS = 'search-index-attachments',
|
||||
SEARCH_REMOVE_PAGE = 'search-remove-page',
|
||||
SEARCH_REMOVE_ASSET = 'search-remove-attachment',
|
||||
SEARCH_REMOVE_FACE = 'search-remove-comment',
|
||||
|
||||
@@ -57,14 +57,6 @@ import { GeneralQueueProcessor } from './processors/general-queue.processor';
|
||||
attempts: 1,
|
||||
},
|
||||
}),
|
||||
BullModule.registerQueue({
|
||||
name: QueueName.SEARCH_QUEUE,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
attempts: 2,
|
||||
},
|
||||
}),
|
||||
BullModule.registerQueue({
|
||||
name: QueueName.AI_QUEUE,
|
||||
defaultJobOptions: {
|
||||
|
||||
@@ -38,6 +38,24 @@ export const TEST_DATABASE_URL =
|
||||
process.env.TEST_DATABASE_URL ??
|
||||
'postgresql://docmost:docmost_dev_pw@localhost:5432/docmost_test';
|
||||
|
||||
// Build the raw postgres.js client (mirrors database.module.ts: max pool,
|
||||
// silenced notices, bigint-as-number parsing). Kept separate so the singleton
|
||||
// can hold a reference to bound its shutdown in destroyTestDb.
|
||||
function buildTestSql(url: string = TEST_DATABASE_URL) {
|
||||
return postgres(url, {
|
||||
max: 5,
|
||||
onnotice: () => {},
|
||||
types: {
|
||||
bigint: {
|
||||
to: 20,
|
||||
from: [20, 1700],
|
||||
serialize: (value: number) => value.toString(),
|
||||
parse: (value: string) => Number.parseInt(value),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a Kysely instance that MIRRORS the app's setup in database.module.ts:
|
||||
* PostgresJSDialect over postgres(), CamelCasePlugin, and the bigint type
|
||||
@@ -47,38 +65,40 @@ export const TEST_DATABASE_URL =
|
||||
*/
|
||||
export function buildTestDb(url: string = TEST_DATABASE_URL): Kysely<any> {
|
||||
return new Kysely<any>({
|
||||
dialect: new PostgresJSDialect({
|
||||
postgres: postgres(url, {
|
||||
max: 5,
|
||||
onnotice: () => {},
|
||||
types: {
|
||||
bigint: {
|
||||
to: 20,
|
||||
from: [20, 1700],
|
||||
serialize: (value: number) => value.toString(),
|
||||
parse: (value: string) => Number.parseInt(value),
|
||||
},
|
||||
},
|
||||
}),
|
||||
}),
|
||||
dialect: new PostgresJSDialect({ postgres: buildTestSql(url) }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
}
|
||||
|
||||
let singleton: Kysely<any> | undefined;
|
||||
let singletonSql: ReturnType<typeof buildTestSql> | undefined;
|
||||
|
||||
/** Lazily-built shared Kysely for the test suite (one per worker; maxWorkers=1). */
|
||||
export function getTestDb(): Kysely<any> {
|
||||
if (!singleton) {
|
||||
singleton = buildTestDb();
|
||||
singletonSql = buildTestSql();
|
||||
singleton = new Kysely<any>({
|
||||
dialect: new PostgresJSDialect({ postgres: singletonSql }),
|
||||
plugins: [new CamelCasePlugin()],
|
||||
});
|
||||
}
|
||||
return singleton;
|
||||
}
|
||||
|
||||
export async function destroyTestDb(): Promise<void> {
|
||||
if (singleton) {
|
||||
await singleton.destroy();
|
||||
singleton = undefined;
|
||||
if (!singleton) return;
|
||||
const sql = singletonSql;
|
||||
// Clear the refs first so a hung end() cannot leave a half-closed singleton.
|
||||
singleton = undefined;
|
||||
singletonSql = undefined;
|
||||
// postgres.js .end() waits indefinitely for in-flight queries by default; a
|
||||
// leaked/stuck pooled connection would hang the afterAll hook (a 60s hook
|
||||
// timeout in CI). Bound the shutdown: the { timeout } grace period lets
|
||||
// active queries drain, then force-closes lingering sockets so teardown
|
||||
// always completes. We close the pool directly instead of Kysely.destroy()
|
||||
// (which would call sql.end() again with no timeout).
|
||||
if (sql) {
|
||||
await sql.end({ timeout: 5 });
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission.repo';
|
||||
import { GroupRepo } from '@docmost/db/repos/group/group.repo';
|
||||
import {
|
||||
getTestDb,
|
||||
destroyTestDb,
|
||||
createWorkspace,
|
||||
createSpace,
|
||||
createUser,
|
||||
createPage,
|
||||
} from './db';
|
||||
|
||||
/**
|
||||
* #348 — the whole-workspace access-filter short-circuit is an ACCESS-CONTROL
|
||||
* path, so it must produce the SAME result as the full recursive-ancestor CTE.
|
||||
*
|
||||
* filterAccessiblePageIds({ workspaceId }) (no spaceId — the favorites /
|
||||
* notifications / recent / created-by / global-search callers) skips the CTE only
|
||||
* when the workspace has ZERO restricted pages. A page is "restricted &
|
||||
* inaccessible" when it (or an ancestor) has a `pageAccess` row and the user has
|
||||
* no matching `pagePermissions`. Driven against real Postgres, asserts:
|
||||
* 1. zero restrictions -> short-circuit returns the full input set;
|
||||
* 2. a restriction present -> the CTE runs and drops the page the user can't
|
||||
* reach while keeping the reachable ones (behavior unchanged);
|
||||
* 3. inserting the FIRST pageAccess flips hasRestrictedPagesInWorkspace
|
||||
* false -> true immediately (the 0->1 transition — now uncached, no stale
|
||||
* window, review F1); it is scoped per workspace.
|
||||
*/
|
||||
describe('#348 filterAccessiblePageIds workspace short-circuit (real PG)', () => {
|
||||
let db: Kysely<any>;
|
||||
let repo: PagePermissionRepo;
|
||||
let workspaceId: string;
|
||||
let otherWorkspaceId: string;
|
||||
let userId: string;
|
||||
let spaceId: string;
|
||||
|
||||
beforeAll(async () => {
|
||||
db = getTestDb();
|
||||
// hasRestrictedPagesInWorkspace is now uncached, and no other cached
|
||||
// permission path is exercised here, so a no-op cache stub suffices.
|
||||
const cacheStub = {
|
||||
get: async () => undefined,
|
||||
set: async () => undefined,
|
||||
del: async () => undefined,
|
||||
} as never;
|
||||
repo = new PagePermissionRepo(db, new GroupRepo(db), cacheStub);
|
||||
|
||||
const ws = await createWorkspace(db);
|
||||
workspaceId = ws.id;
|
||||
const other = await createWorkspace(db);
|
||||
otherWorkspaceId = other.id;
|
||||
const user = await createUser(db, workspaceId);
|
||||
userId = user.id;
|
||||
const space = await createSpace(db, workspaceId);
|
||||
spaceId = space.id;
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('zero restrictions: short-circuit returns the full input set', async () => {
|
||||
const p1 = await createPage(db, { workspaceId, spaceId });
|
||||
const p2 = await createPage(db, { workspaceId, spaceId });
|
||||
|
||||
expect(await repo.hasRestrictedPagesInWorkspace(workspaceId)).toBe(false);
|
||||
|
||||
const ids = [p1.id, p2.id];
|
||||
const filtered = await repo.filterAccessiblePageIds({
|
||||
pageIds: ids,
|
||||
userId,
|
||||
workspaceId,
|
||||
});
|
||||
expect(new Set(filtered)).toEqual(new Set(ids));
|
||||
});
|
||||
|
||||
it('a restriction present: filters out the page the user cannot reach', async () => {
|
||||
const openPage = await createPage(db, { workspaceId, spaceId });
|
||||
const restrictedPage = await createPage(db, { workspaceId, spaceId });
|
||||
|
||||
// Add a pageAccess row on restrictedPage with NO matching pagePermissions for
|
||||
// `userId` → the CTE anti-join marks it inaccessible for this user.
|
||||
await db
|
||||
.insertInto('pageAccess')
|
||||
.values({
|
||||
pageId: restrictedPage.id,
|
||||
workspaceId,
|
||||
spaceId,
|
||||
accessLevel: 'read',
|
||||
creatorId: userId,
|
||||
})
|
||||
.execute();
|
||||
|
||||
// 0->1 transition is reflected immediately (uncached).
|
||||
expect(await repo.hasRestrictedPagesInWorkspace(workspaceId)).toBe(true);
|
||||
|
||||
const filtered = await repo.filterAccessiblePageIds({
|
||||
pageIds: [openPage.id, restrictedPage.id],
|
||||
userId,
|
||||
workspaceId,
|
||||
});
|
||||
expect(filtered).toContain(openPage.id);
|
||||
expect(filtered).not.toContain(restrictedPage.id);
|
||||
});
|
||||
|
||||
it('hasRestrictedPagesInWorkspace is scoped per workspace', async () => {
|
||||
// The other workspace has no pageAccess rows → still false, unaffected by the
|
||||
// restriction added above in `workspaceId`.
|
||||
expect(await repo.hasRestrictedPagesInWorkspace(otherWorkspaceId)).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,25 @@
|
||||
import { Kysely } from 'kysely';
|
||||
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
|
||||
import { CacheKey } from 'src/common/helpers/cache-keys';
|
||||
import { getTestDb, destroyTestDb, createWorkspace } from './db';
|
||||
|
||||
// A minimal Map-backed cache double with a working `del` (the previous `{}` stub
|
||||
// made bustWorkspaceCache's `del` throw into its own try/catch, so the #348
|
||||
// invalidation was never actually exercised — review F6).
|
||||
function makeCacheDouble() {
|
||||
const store = new Map<string, unknown>();
|
||||
return {
|
||||
store,
|
||||
get: async (k: string) => store.get(k),
|
||||
set: async (k: string, v: unknown) => {
|
||||
store.set(k, v);
|
||||
},
|
||||
del: async (k: string) => {
|
||||
store.delete(k);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* A — WorkspaceRepo.updateSetting jsonb-MERGE (the html-embed kill-switch
|
||||
* write-half). Setting a single top-level key must NOT clobber sibling
|
||||
@@ -15,7 +33,9 @@ describe('WorkspaceRepo.updateSetting (jsonb merge) [integration]', () => {
|
||||
beforeAll(() => {
|
||||
db = getTestDb();
|
||||
// Repos are plain classes taking @InjectKysely() db — instantiate directly.
|
||||
repo = new WorkspaceRepo(db as any);
|
||||
// 2nd arg is CACHE_MANAGER (used only to bust the #348 workspace cache); a
|
||||
// stub is fine here since bustWorkspaceCache is best-effort (try/catch).
|
||||
repo = new WorkspaceRepo(db as any, {} as any);
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
@@ -58,3 +78,62 @@ describe('WorkspaceRepo.updateSetting (jsonb merge) [integration]', () => {
|
||||
expect(updated.settings).toEqual({ htmlEmbed: false });
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* #348 F6 — the DomainMiddleware workspace cache (WORKSPACE_SELF_HOSTED /
|
||||
* WORKSPACE_BY_HOST, 15s TTL) caches security-relevant fields (enforceSso/
|
||||
* enforceMfa/status). Its correctness rests entirely on bustWorkspaceCache being
|
||||
* called from every mutator. This exercises the real invalidation with a working
|
||||
* cache double (not the {} stub, whose del throws-and-swallows): warm the cache
|
||||
* like DomainMiddleware, mutate, and assert the busted key is gone so a stale
|
||||
* workspace row can't outlive the mutation.
|
||||
*/
|
||||
describe('WorkspaceRepo bustWorkspaceCache invalidation [integration]', () => {
|
||||
let db: Kysely<any>;
|
||||
|
||||
beforeAll(() => {
|
||||
db = getTestDb();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await destroyTestDb();
|
||||
});
|
||||
|
||||
it('updateSetting busts the self-hosted workspace cache key', async () => {
|
||||
const cache = makeCacheDouble();
|
||||
const repo = new WorkspaceRepo(db as any, cache as any);
|
||||
const ws = await createWorkspace(db, { settings: {} });
|
||||
|
||||
// Warm the cache as DomainMiddleware would (self-hosted key).
|
||||
cache.store.set(CacheKey.WORKSPACE_SELF_HOSTED, ws);
|
||||
expect(cache.store.has(CacheKey.WORKSPACE_SELF_HOSTED)).toBe(true);
|
||||
|
||||
await repo.updateSetting(ws.id, 'htmlEmbed', true);
|
||||
|
||||
// The mutation must have invalidated the cached row.
|
||||
expect(cache.store.has(CacheKey.WORKSPACE_SELF_HOSTED)).toBe(false);
|
||||
});
|
||||
|
||||
it('updateSharingSettings busts the by-host workspace cache key too', async () => {
|
||||
const cache = makeCacheDouble();
|
||||
const repo = new WorkspaceRepo(db as any, cache as any);
|
||||
const ws = await createWorkspace(db, { settings: {} });
|
||||
// createWorkspace assigns a unique hostname; read it back for the by-host key.
|
||||
const { hostname } = await db
|
||||
.selectFrom('workspaces')
|
||||
.select(['hostname'])
|
||||
.where('id', '=', ws.id)
|
||||
.executeTakeFirstOrThrow();
|
||||
|
||||
// Warm BOTH keys (self-hosted + by-host); the by-host bust needs the row's
|
||||
// hostname, which the mutator returns from the DB.
|
||||
cache.store.set(CacheKey.WORKSPACE_SELF_HOSTED, ws);
|
||||
cache.store.set(CacheKey.WORKSPACE_BY_HOST(hostname as string), ws);
|
||||
|
||||
await repo.updateSharingSettings(ws.id, 'allowInvite', true);
|
||||
|
||||
expect(cache.store.has(CacheKey.WORKSPACE_SELF_HOSTED)).toBe(false);
|
||||
expect(cache.store.has(CacheKey.WORKSPACE_BY_HOST(hostname as string))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,10 +4,14 @@
|
||||
"testEnvironment": "node",
|
||||
"testRegex": ".e2e-spec.ts$",
|
||||
"transform": {
|
||||
"prosemirror-markdown/build/.+\\.js$": [
|
||||
"babel-jest",
|
||||
{ "presets": [["@babel/preset-env", { "targets": { "node": "current" } }]] }
|
||||
],
|
||||
"^.+\\.(t|j)sx?$": ["ts-jest", { "tsconfig": { "allowJs": true } }]
|
||||
},
|
||||
"transformIgnorePatterns": [
|
||||
"/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom|lib0|@sindresorhus[+/][a-z0-9-]+|escape-string-regexp|p-limit|yocto-queue)(@|/))"
|
||||
"/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom|lib0|@sindresorhus[+/][a-z0-9-]+|escape-string-regexp|p-limit|yocto-queue|@docmost/prosemirror-markdown)(@|/))"
|
||||
],
|
||||
"moduleNameMapper": {
|
||||
"^@docmost/db/(.*)$": "<rootDir>/../src/database/$1",
|
||||
|
||||
@@ -4,14 +4,19 @@
|
||||
"testRegex": ".*\\.int-spec\\.ts$",
|
||||
"testPathIgnorePatterns": ["/node_modules/"],
|
||||
"transform": {
|
||||
"prosemirror-markdown/build/.+\\.js$": [
|
||||
"babel-jest",
|
||||
{ "presets": [["@babel/preset-env", { "targets": { "node": "current" } }]] }
|
||||
],
|
||||
"^.+\\.(t|j)sx?$": "ts-jest"
|
||||
},
|
||||
"transformIgnorePatterns": [
|
||||
"/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom|lib0)(@|/))"
|
||||
"/node_modules/(?!(\\.pnpm/)?(nanoid|uuid|image-dimensions|marked|happy-dom|lib0|@docmost/prosemirror-markdown)(@|/))"
|
||||
],
|
||||
"testEnvironment": "node",
|
||||
"testTimeout": 60000,
|
||||
"maxWorkers": 1,
|
||||
"forceExit": true,
|
||||
"globalSetup": "<rootDir>/test/integration/global-setup.ts",
|
||||
"globalTeardown": "<rootDir>/test/integration/global-teardown.ts",
|
||||
"moduleNameMapper": {
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
// Import DIRECTLY from src (NOT the docmost-client barrel, which pulls in
|
||||
// collaboration.ts and mutates global DOM at import time).
|
||||
import { convertProseMirrorToMarkdown } from "../src/lib/markdown-converter.js";
|
||||
import { markdownToProseMirror } from "../src/lib/markdown-to-prosemirror.js";
|
||||
|
||||
/**
|
||||
* gitmost #377 (round-1 review, finding #1) — proof, against the REAL
|
||||
* converter, that the transcript-insert boundary defense survives git-sync.
|
||||
*
|
||||
* The web bridge (apps/client .../gitmost/gitmost-recording.ts,
|
||||
* `gitmostInsertTranscriptIntoEditor`) appends each transcript line as a
|
||||
* PARAGRAPH text node. The paragraph serializer here (`case "paragraph"`) emits
|
||||
* that text VERBATIM with no block-escape, so a line whose text begins with a
|
||||
* col-0 markdown block trigger would, on the doc -> markdown -> doc git-sync
|
||||
* cycle, silently re-parse into a heading / list / quote / callout / code block.
|
||||
* That missing block-escape is the pre-existing root cause; the bridge's
|
||||
* boundary defense prepends an invisible zero-width space (U+200B) to a line
|
||||
* that begins with such a trigger, shifting it off column 0.
|
||||
*
|
||||
* This test keeps a COPY of the bridge's trigger regex (the bridge is in a
|
||||
* different package and can't be imported here) and asserts:
|
||||
* 1. bare trigger lines DO corrupt (documents the root cause), and
|
||||
* 2. the ZWSP-neutralized form round-trips as a single PARAGRAPH with the
|
||||
* text byte-preserved.
|
||||
*/
|
||||
|
||||
const ZWSP = ""; // U+200B
|
||||
|
||||
// MUST stay in sync with GITMOST_MD_BLOCK_TRIGGER_RE in the client bridge.
|
||||
const MD_BLOCK_TRIGGER_RE =
|
||||
/^(?:#{1,6}(?:\s|$)|[-*+](?:\s|$)|>|\d+[.)](?:\s|$)|```|~~~|\||([-*_])(?:\s*\1){2,}\s*$)/;
|
||||
|
||||
const doc = (...nodes: any[]) => ({ type: "doc", content: nodes });
|
||||
const para = (t: string) => ({
|
||||
type: "paragraph",
|
||||
content: [{ type: "text", text: t }],
|
||||
});
|
||||
|
||||
const roundtrip = async (text: string) => {
|
||||
const md = convertProseMirrorToMarkdown(doc(para(text)));
|
||||
const back = await markdownToProseMirror(md);
|
||||
return back.content as any[];
|
||||
};
|
||||
|
||||
describe("gitmost transcript neutralization (git-sync round-trip)", () => {
|
||||
// Lines that, at column 0, the serializer's missing block-escape would let
|
||||
// git-sync re-parse into a non-paragraph block.
|
||||
const triggerLines = [
|
||||
"- dash",
|
||||
"* star",
|
||||
"+ plus",
|
||||
"> quote",
|
||||
"# hash",
|
||||
"1. one",
|
||||
"1) one",
|
||||
"> [!info] note",
|
||||
"```js",
|
||||
"~~~",
|
||||
// Solid + spaced thematic breaks — these re-parse into a `horizontalRule`,
|
||||
// which carries NO text, so a bare separator line LOSES its text entirely
|
||||
// (round-2 finding). `_` also only forms a block via this construct.
|
||||
"---",
|
||||
"***",
|
||||
"___",
|
||||
"- - -", // spaced dash break (solid form is caught by [-*+]\s too, but this is the break)
|
||||
"_ _ _",
|
||||
];
|
||||
|
||||
it("BARE trigger lines corrupt into non-paragraph blocks (root cause)", async () => {
|
||||
for (const line of triggerLines) {
|
||||
const blocks = await roundtrip(line);
|
||||
// At least one produced block is NOT a paragraph — i.e. corruption.
|
||||
const allParagraphs = blocks.every((b) => b.type === "paragraph");
|
||||
expect(
|
||||
allParagraphs,
|
||||
`expected "${line}" to corrupt when inserted bare`,
|
||||
).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("BARE solid thematic breaks corrupt into a text-LOSING horizontalRule", async () => {
|
||||
// The severe case: no text node survives. Documents why neutralization
|
||||
// matters more here than for list/quote (where the text survived).
|
||||
for (const line of ["---", "***", "___"]) {
|
||||
const blocks = await roundtrip(line);
|
||||
expect(blocks.map((b) => b.type)).toContain("horizontalRule");
|
||||
// No block carries the original text anywhere.
|
||||
const flat = JSON.stringify(blocks);
|
||||
expect(flat).not.toContain(line);
|
||||
}
|
||||
});
|
||||
|
||||
it("ZWSP-neutralized trigger lines round-trip as a single paragraph, text preserved", async () => {
|
||||
for (const line of triggerLines) {
|
||||
// The regex must actually classify each as a trigger.
|
||||
expect(MD_BLOCK_TRIGGER_RE.test(line), `regex missed "${line}"`).toBe(
|
||||
true,
|
||||
);
|
||||
const neutralized = ZWSP + line;
|
||||
const blocks = await roundtrip(neutralized);
|
||||
|
||||
expect(blocks).toHaveLength(1);
|
||||
expect(blocks[0].type).toBe("paragraph");
|
||||
// Text is byte-preserved (ZWSP + original line), so the display is the
|
||||
// original line with only an invisible leading character.
|
||||
expect(blocks[0].content[0].text).toBe(neutralized);
|
||||
}
|
||||
});
|
||||
|
||||
it("normal host-prefixed lines never match the trigger regex and round-trip byte-exact", async () => {
|
||||
for (const line of [
|
||||
"You: hello there",
|
||||
"Speaker 1: - and then a dash mid-line",
|
||||
"Speaker 2: 1. not a list",
|
||||
]) {
|
||||
expect(MD_BLOCK_TRIGGER_RE.test(line)).toBe(false);
|
||||
const blocks = await roundtrip(line);
|
||||
expect(blocks).toHaveLength(1);
|
||||
expect(blocks[0].type).toBe("paragraph");
|
||||
expect(blocks[0].content[0].text).toBe(line);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user