feat(ai): wire up workspace RAG bulk reindex + manual "Reindex now"
The WORKSPACE_CREATE_EMBEDDINGS / WORKSPACE_DELETE_EMBEDDINGS jobs were
enqueued (on AI Search enable/disable) but had no AI_QUEUE handler, so
existing pages were never indexed ("Indexed 0 of N pages") and disabling
never purged embeddings.
- EmbeddingProcessor: handle WORKSPACE_CREATE_EMBEDDINGS (bulk reindex all
live pages) and WORKSPACE_DELETE_EMBEDDINGS (purge workspace embeddings)
- EmbeddingIndexerService: add reindexWorkspace() (skips when embeddings
unconfigured; per-page error isolation) and removeWorkspace()
- PageRepo.getIdsByWorkspace(), PageEmbeddingRepo.deleteByWorkspace()
- AiSettingsService.reindex() + admin-only POST /workspace/ai-settings/reindex
- Frontend: "Reindex now" button, service call and mutation
- Stable per-workspace jobId with remove-before-add so a stale job can't
block future reindexes; cancel the delayed purge on enable/reindex so it
can't wipe freshly-built embeddings
This commit is contained in:
@@ -75,4 +75,16 @@ export class AiSettingsController {
|
||||
this.assertAdmin(user, workspace);
|
||||
return this.aiService.testConnection(workspace.id);
|
||||
}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('reindex')
|
||||
async reindex(
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
) {
|
||||
this.assertAdmin(user, workspace);
|
||||
await this.aiSettingsService.reindex(workspace.id);
|
||||
// Return refreshed masked settings so the client can update the counter.
|
||||
return this.aiSettingsService.getMasked(workspace.id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { BadRequestException, Injectable } from '@nestjs/common';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { Queue } from 'bullmq';
|
||||
import { QueueName, QueueJob } from '../queue/constants';
|
||||
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
|
||||
import { AiProviderCredentialsRepo } from '@docmost/db/repos/ai-chat/ai-provider-credentials.repo';
|
||||
import { PageEmbeddingRepo } from '@docmost/db/repos/ai-chat/page-embedding.repo';
|
||||
@@ -43,8 +46,49 @@ export class AiSettingsService {
|
||||
private readonly pageEmbeddingRepo: PageEmbeddingRepo,
|
||||
private readonly pageRepo: PageRepo,
|
||||
private readonly secretBox: SecretBoxService,
|
||||
@InjectQueue(QueueName.AI_QUEUE) private readonly aiQueue: Queue,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Enqueue a full workspace RAG reindex (manual "Reindex now").
|
||||
*
|
||||
* Uses a stable per-workspace jobId so rapid re-triggers de-duplicate instead
|
||||
* of stacking multiple full reindex passes. A prior non-active job with that
|
||||
* id is removed first so a lingering completed/failed/waiting entry can never
|
||||
* block a fresh reindex (BullMQ ignores add() when the jobId already exists).
|
||||
* If a reindex is already running, remove() is a no-op (it leaves a
|
||||
* locked/active job in place, returning 0 without throwing), and the add()
|
||||
* below then de-duplicates against that still-present jobId — so the running
|
||||
* pass is kept and no duplicate is started. The .catch only guards against
|
||||
* transport/Redis errors.
|
||||
*
|
||||
* Also cancels any pending delayed WORKSPACE_DELETE_EMBEDDINGS job (scheduled
|
||||
* when AI Search was disabled) so it cannot wipe the embeddings we are about
|
||||
* to rebuild. The job no-ops if embeddings are unconfigured.
|
||||
*/
|
||||
async reindex(workspaceId: string): Promise<void> {
|
||||
// A reindex means embeddings must persist: drop the delayed purge, if any.
|
||||
await this.aiQueue
|
||||
.remove(`ai-search-disabled-${workspaceId}`)
|
||||
.catch(() => undefined);
|
||||
|
||||
const jobId = `ai-reindex-${workspaceId}`;
|
||||
// Clear a prior non-active entry so a stale job can't block this reindex.
|
||||
// A locked/active job is left in place (remove() no-ops) and the add() below
|
||||
// de-duplicates against it, keeping the in-progress pass.
|
||||
await this.aiQueue.remove(jobId).catch(() => undefined);
|
||||
|
||||
await this.aiQueue.add(
|
||||
QueueJob.WORKSPACE_CREATE_EMBEDDINGS,
|
||||
{ workspaceId },
|
||||
{
|
||||
jobId,
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Read the stored non-secret provider settings for a workspace. */
|
||||
private async readProvider(
|
||||
workspaceId: string,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { BullModule } from '@nestjs/bullmq';
|
||||
import { CryptoModule } from '../crypto/crypto.module';
|
||||
import { QueueName } from '../queue/constants';
|
||||
import { AiService } from './ai.service';
|
||||
import { AiSettingsService } from './ai-settings.service';
|
||||
import { AiSettingsController } from './ai-settings.controller';
|
||||
@@ -12,7 +14,10 @@ import { AiSettingsController } from './ai-settings.controller';
|
||||
* (CaslModule, global) are resolved without explicit imports.
|
||||
*/
|
||||
@Module({
|
||||
imports: [CryptoModule],
|
||||
imports: [
|
||||
CryptoModule,
|
||||
BullModule.registerQueue({ name: QueueName.AI_QUEUE }),
|
||||
],
|
||||
controllers: [AiSettingsController],
|
||||
providers: [AiService, AiSettingsService],
|
||||
exports: [AiService, AiSettingsService],
|
||||
|
||||
@@ -33,6 +33,14 @@ export interface IPageContentUpdatedJob {
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* AI_QUEUE payload for workspace-wide RAG embedding jobs
|
||||
* (WORKSPACE_CREATE_EMBEDDINGS / WORKSPACE_DELETE_EMBEDDINGS).
|
||||
*/
|
||||
export interface IWorkspaceEmbeddingsJob {
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface INotificationCreateJob {
|
||||
userId: string;
|
||||
workspaceId: string;
|
||||
|
||||
Reference in New Issue
Block a user