feat(ai): wire up workspace RAG bulk reindex + manual "Reindex now"
The WORKSPACE_CREATE_EMBEDDINGS / WORKSPACE_DELETE_EMBEDDINGS jobs were
enqueued (on AI Search enable/disable) but had no AI_QUEUE handler, so
existing pages were never indexed ("Indexed 0 of N pages") and disabling
never purged embeddings.
- EmbeddingProcessor: handle WORKSPACE_CREATE_EMBEDDINGS (bulk reindex all
live pages) and WORKSPACE_DELETE_EMBEDDINGS (purge workspace embeddings)
- EmbeddingIndexerService: add reindexWorkspace() (skips when embeddings
unconfigured; per-page error isolation) and removeWorkspace()
- PageRepo.getIdsByWorkspace(), PageEmbeddingRepo.deleteByWorkspace()
- AiSettingsService.reindex() + admin-only POST /workspace/ai-settings/reindex
- Frontend: "Reindex now" button, service call and mutation
- Stable per-workspace jobId with remove-before-add so a stale job can't
block future reindexes; cancel the delayed purge on enable/reindex so it
can't wipe freshly-built embeddings
This commit is contained in:
@@ -151,6 +151,61 @@ export class EmbeddingIndexerService {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* (Re)build embeddings for EVERY non-deleted page in a workspace. Used by the
|
||||
* bulk reindex (WORKSPACE_CREATE_EMBEDDINGS, fired when AI Search is enabled
|
||||
* and by the manual "Reindex now" action).
|
||||
*
|
||||
* Resolves the embeddings model once up front: if the workspace has no
|
||||
* embeddings provider configured, the whole batch is skipped (otherwise each
|
||||
* page would no-op individually after a wasted read). Pages are processed
|
||||
* sequentially and each is isolated in try/catch so one failure never aborts
|
||||
* the batch.
|
||||
*/
|
||||
async reindexWorkspace(workspaceId: string): Promise<void> {
|
||||
try {
|
||||
await this.aiService.getEmbeddingModel(workspaceId);
|
||||
} catch (err) {
|
||||
if (err instanceof AiEmbeddingNotConfiguredException) {
|
||||
this.logger.debug(
|
||||
`reindexWorkspace: embeddings not configured for workspace ${workspaceId}, skipping`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
|
||||
const pageIds = await this.pageRepo.getIdsByWorkspace(workspaceId);
|
||||
this.logger.debug(
|
||||
`reindexWorkspace: reindexing ${pageIds.length} page(s) for workspace ${workspaceId}`,
|
||||
);
|
||||
|
||||
let failed = 0;
|
||||
for (const pageId of pageIds) {
|
||||
try {
|
||||
await this.reindexPage(pageId);
|
||||
} catch (err) {
|
||||
// Per-page isolation: one failure must not abort the whole batch.
|
||||
failed++;
|
||||
this.logger.error(
|
||||
`reindexWorkspace: failed to reindex page ${pageId}: ${
|
||||
err instanceof Error ? err.message : 'Unknown error'
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
this.logger.debug(
|
||||
`reindexWorkspace: done for workspace ${workspaceId} (${
|
||||
pageIds.length - failed
|
||||
}/${pageIds.length} pages)`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Purge ALL embeddings for a workspace (WORKSPACE_DELETE_EMBEDDINGS). */
|
||||
async removeWorkspace(workspaceId: string): Promise<void> {
|
||||
await this.pageEmbeddingRepo.deleteByWorkspace(workspaceId);
|
||||
}
|
||||
|
||||
/** Remove all embeddings for a deleted page (used by the delete path). */
|
||||
async removePage(pageId: string, workspaceId: string): Promise<void> {
|
||||
await this.pageEmbeddingRepo.deleteByPage(pageId, workspaceId);
|
||||
|
||||
@@ -2,7 +2,10 @@ import { Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
|
||||
import { Job } from 'bullmq';
|
||||
import { QueueJob, QueueName } from '../../../integrations/queue/constants';
|
||||
import { IPageContentUpdatedJob } from '../../../integrations/queue/constants/queue.interface';
|
||||
import {
|
||||
IPageContentUpdatedJob,
|
||||
IWorkspaceEmbeddingsJob,
|
||||
} from '../../../integrations/queue/constants/queue.interface';
|
||||
import { EmbeddingIndexerService } from './embedding-indexer.service';
|
||||
|
||||
/**
|
||||
@@ -30,11 +33,15 @@ export class EmbeddingProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
super();
|
||||
}
|
||||
|
||||
async process(job: Job<IPageContentUpdatedJob, void>): Promise<void> {
|
||||
const { pageIds, workspaceId } = job.data ?? {
|
||||
pageIds: [],
|
||||
workspaceId: '',
|
||||
};
|
||||
async process(
|
||||
job: Job<IPageContentUpdatedJob | IWorkspaceEmbeddingsJob, void>,
|
||||
): Promise<void> {
|
||||
// The workspace-wide jobs carry `{ workspaceId }` only (no `pageIds`), so
|
||||
// read `pageIds` defensively — it is absent on the workspace payload.
|
||||
const data: Partial<IPageContentUpdatedJob & IWorkspaceEmbeddingsJob> =
|
||||
job.data ?? {};
|
||||
const pageIds = data.pageIds ?? [];
|
||||
const workspaceId = data.workspaceId ?? '';
|
||||
const ids = Array.isArray(pageIds) ? pageIds : [];
|
||||
|
||||
switch (job.name) {
|
||||
@@ -70,6 +77,28 @@ export class EmbeddingProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
break;
|
||||
}
|
||||
|
||||
case QueueJob.WORKSPACE_CREATE_EMBEDDINGS: {
|
||||
try {
|
||||
await this.indexer.reindexWorkspace(workspaceId);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to reindex workspace ${workspaceId}: ${this.errMessage(err)}`,
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case QueueJob.WORKSPACE_DELETE_EMBEDDINGS: {
|
||||
try {
|
||||
await this.indexer.removeWorkspace(workspaceId);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to remove embeddings for workspace ${workspaceId}: ${this.errMessage(err)}`,
|
||||
);
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
// Other AI_QUEUE job names are not handled here (e.g. future jobs).
|
||||
this.logger.debug(`Ignoring AI_QUEUE job: ${job.name}`);
|
||||
|
||||
@@ -513,9 +513,24 @@ export class WorkspaceService {
|
||||
});
|
||||
|
||||
if (after.aiSearch === true) {
|
||||
await this.aiQueue.add(QueueJob.WORKSPACE_CREATE_EMBEDDINGS, {
|
||||
workspaceId,
|
||||
});
|
||||
// Cancel any pending delayed purge from a previous disable so it can't
|
||||
// wipe the embeddings we are about to (re)build. The purge is a delayed
|
||||
// job, so remove() simply deletes it (returning 0 if it is absent, without
|
||||
// throwing). The .catch only guards against transport/Redis errors.
|
||||
await this.aiQueue
|
||||
.remove(`ai-search-disabled-${workspaceId}`)
|
||||
.catch(() => undefined);
|
||||
// Stable jobId de-duplicates with the manual "Reindex now" path and with
|
||||
// repeated enable toggles (one full reindex at a time).
|
||||
await this.aiQueue.add(
|
||||
QueueJob.WORKSPACE_CREATE_EMBEDDINGS,
|
||||
{ workspaceId },
|
||||
{
|
||||
jobId: `ai-reindex-${workspaceId}`,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
},
|
||||
);
|
||||
} else if (after.aiSearch === false) {
|
||||
const deleteJobId = `ai-search-disabled-${workspaceId}`;
|
||||
await this.aiQueue.add(
|
||||
|
||||
Reference in New Issue
Block a user