feat(ai): wire up workspace RAG bulk reindex + manual "Reindex now"

The WORKSPACE_CREATE_EMBEDDINGS / WORKSPACE_DELETE_EMBEDDINGS jobs were
enqueued (on AI Search enable/disable) but had no AI_QUEUE handler, so
existing pages were never indexed ("Indexed 0 of N pages") and disabling
never purged embeddings.

- EmbeddingProcessor: handle WORKSPACE_CREATE_EMBEDDINGS (bulk reindex all
  live pages) and WORKSPACE_DELETE_EMBEDDINGS (purge workspace embeddings)
- EmbeddingIndexerService: add reindexWorkspace() (skips when embeddings
  unconfigured; per-page error isolation) and removeWorkspace()
- PageRepo.getIdsByWorkspace(), PageEmbeddingRepo.deleteByWorkspace()
- AiSettingsService.reindex() + admin-only POST /workspace/ai-settings/reindex
- Frontend: "Reindex now" button, service call and mutation
- Stable per-workspace jobId with remove-before-add so a stale job can't
  block future reindexes; cancel the delayed purge on enable/reindex so it
  can't wipe freshly-built embeddings
This commit is contained in:
vvzvlad
2026-06-18 02:15:18 +03:00
parent 4cf6b73d3e
commit 52e19fe678
12 changed files with 253 additions and 16 deletions
@@ -151,6 +151,61 @@ export class EmbeddingIndexerService {
);
}
/**
* (Re)build embeddings for EVERY non-deleted page in a workspace. Used by the
* bulk reindex (WORKSPACE_CREATE_EMBEDDINGS, fired when AI Search is enabled
* and by the manual "Reindex now" action).
*
* Resolves the embeddings model once up front: if the workspace has no
* embeddings provider configured, the whole batch is skipped (otherwise each
* page would no-op individually after a wasted read). Pages are processed
* sequentially and each is isolated in try/catch so one failure never aborts
* the batch.
*/
async reindexWorkspace(workspaceId: string): Promise<void> {
try {
await this.aiService.getEmbeddingModel(workspaceId);
} catch (err) {
if (err instanceof AiEmbeddingNotConfiguredException) {
this.logger.debug(
`reindexWorkspace: embeddings not configured for workspace ${workspaceId}, skipping`,
);
return;
}
throw err;
}
const pageIds = await this.pageRepo.getIdsByWorkspace(workspaceId);
this.logger.debug(
`reindexWorkspace: reindexing ${pageIds.length} page(s) for workspace ${workspaceId}`,
);
let failed = 0;
for (const pageId of pageIds) {
try {
await this.reindexPage(pageId);
} catch (err) {
// Per-page isolation: one failure must not abort the whole batch.
failed++;
this.logger.error(
`reindexWorkspace: failed to reindex page ${pageId}: ${
err instanceof Error ? err.message : 'Unknown error'
}`,
);
}
}
this.logger.debug(
`reindexWorkspace: done for workspace ${workspaceId} (${
pageIds.length - failed
}/${pageIds.length} pages)`,
);
}
/** Purge ALL embeddings for a workspace (WORKSPACE_DELETE_EMBEDDINGS). */
async removeWorkspace(workspaceId: string): Promise<void> {
await this.pageEmbeddingRepo.deleteByWorkspace(workspaceId);
}
/** Remove all embeddings for a deleted page (used by the delete path). */
async removePage(pageId: string, workspaceId: string): Promise<void> {
await this.pageEmbeddingRepo.deleteByPage(pageId, workspaceId);
@@ -2,7 +2,10 @@ import { Logger, OnModuleDestroy } from '@nestjs/common';
import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
import { Job } from 'bullmq';
import { QueueJob, QueueName } from '../../../integrations/queue/constants';
import { IPageContentUpdatedJob } from '../../../integrations/queue/constants/queue.interface';
import {
IPageContentUpdatedJob,
IWorkspaceEmbeddingsJob,
} from '../../../integrations/queue/constants/queue.interface';
import { EmbeddingIndexerService } from './embedding-indexer.service';
/**
@@ -30,11 +33,15 @@ export class EmbeddingProcessor extends WorkerHost implements OnModuleDestroy {
super();
}
async process(job: Job<IPageContentUpdatedJob, void>): Promise<void> {
const { pageIds, workspaceId } = job.data ?? {
pageIds: [],
workspaceId: '',
};
async process(
job: Job<IPageContentUpdatedJob | IWorkspaceEmbeddingsJob, void>,
): Promise<void> {
// The workspace-wide jobs carry `{ workspaceId }` only (no `pageIds`), so
// read `pageIds` defensively — it is absent on the workspace payload.
const data: Partial<IPageContentUpdatedJob & IWorkspaceEmbeddingsJob> =
job.data ?? {};
const pageIds = data.pageIds ?? [];
const workspaceId = data.workspaceId ?? '';
const ids = Array.isArray(pageIds) ? pageIds : [];
switch (job.name) {
@@ -70,6 +77,28 @@ export class EmbeddingProcessor extends WorkerHost implements OnModuleDestroy {
break;
}
case QueueJob.WORKSPACE_CREATE_EMBEDDINGS: {
try {
await this.indexer.reindexWorkspace(workspaceId);
} catch (err) {
this.logger.error(
`Failed to reindex workspace ${workspaceId}: ${this.errMessage(err)}`,
);
}
break;
}
case QueueJob.WORKSPACE_DELETE_EMBEDDINGS: {
try {
await this.indexer.removeWorkspace(workspaceId);
} catch (err) {
this.logger.error(
`Failed to remove embeddings for workspace ${workspaceId}: ${this.errMessage(err)}`,
);
}
break;
}
default:
// Other AI_QUEUE job names are not handled here (e.g. future jobs).
this.logger.debug(`Ignoring AI_QUEUE job: ${job.name}`);
@@ -513,9 +513,24 @@ export class WorkspaceService {
});
if (after.aiSearch === true) {
await this.aiQueue.add(QueueJob.WORKSPACE_CREATE_EMBEDDINGS, {
workspaceId,
});
// Cancel any pending delayed purge from a previous disable so it can't
// wipe the embeddings we are about to (re)build. The purge is a delayed
// job, so remove() simply deletes it (returning 0 if it is absent, without
// throwing). The .catch only guards against transport/Redis errors.
await this.aiQueue
.remove(`ai-search-disabled-${workspaceId}`)
.catch(() => undefined);
// Stable jobId de-duplicates with the manual "Reindex now" path and with
// repeated enable toggles (one full reindex at a time).
await this.aiQueue.add(
QueueJob.WORKSPACE_CREATE_EMBEDDINGS,
{ workspaceId },
{
jobId: `ai-reindex-${workspaceId}`,
removeOnComplete: true,
removeOnFail: true,
},
);
} else if (after.aiSearch === false) {
const deleteJobId = `ai-search-disabled-${workspaceId}`;
await this.aiQueue.add(