feat(ai): wire up workspace RAG bulk reindex + manual "Reindex now"

The WORKSPACE_CREATE_EMBEDDINGS / WORKSPACE_DELETE_EMBEDDINGS jobs were
enqueued (on AI Search enable/disable) but had no AI_QUEUE handler, so
existing pages were never indexed ("Indexed 0 of N pages") and disabling
never purged embeddings.

- EmbeddingProcessor: handle WORKSPACE_CREATE_EMBEDDINGS (bulk reindex all
  live pages) and WORKSPACE_DELETE_EMBEDDINGS (purge workspace embeddings)
- EmbeddingIndexerService: add reindexWorkspace() (skips when embeddings
  unconfigured; per-page error isolation) and removeWorkspace()
- PageRepo.getIdsByWorkspace(), PageEmbeddingRepo.deleteByWorkspace()
- AiSettingsService.reindex() + admin-only POST /workspace/ai-settings/reindex
- Frontend: "Reindex now" button, service call and mutation
- Stable per-workspace jobId with remove-before-add so a stale job can't
  block future reindexes; cancel the delayed purge on enable/reindex so it
  can't wipe freshly-built embeddings
This commit is contained in:
vvzvlad
2026-06-18 02:15:18 +03:00
parent 4cf6b73d3e
commit 52e19fe678
12 changed files with 253 additions and 16 deletions

View File

@@ -75,4 +75,16 @@ export class AiSettingsController {
this.assertAdmin(user, workspace);
return this.aiService.testConnection(workspace.id);
}
@HttpCode(HttpStatus.OK)
@Post('reindex')
async reindex(
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
) {
this.assertAdmin(user, workspace);
await this.aiSettingsService.reindex(workspace.id);
// Return refreshed masked settings so the client can update the counter.
return this.aiSettingsService.getMasked(workspace.id);
}
}

View File

@@ -1,4 +1,7 @@
import { BadRequestException, Injectable } from '@nestjs/common';
import { InjectQueue } from '@nestjs/bullmq';
import { Queue } from 'bullmq';
import { QueueName, QueueJob } from '../queue/constants';
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
import { AiProviderCredentialsRepo } from '@docmost/db/repos/ai-chat/ai-provider-credentials.repo';
import { PageEmbeddingRepo } from '@docmost/db/repos/ai-chat/page-embedding.repo';
@@ -43,8 +46,49 @@ export class AiSettingsService {
private readonly pageEmbeddingRepo: PageEmbeddingRepo,
private readonly pageRepo: PageRepo,
private readonly secretBox: SecretBoxService,
@InjectQueue(QueueName.AI_QUEUE) private readonly aiQueue: Queue,
) {}
/**
* Enqueue a full workspace RAG reindex (manual "Reindex now").
*
* Uses a stable per-workspace jobId so rapid re-triggers de-duplicate instead
* of stacking multiple full reindex passes. A prior non-active job with that
* id is removed first so a lingering completed/failed/waiting entry can never
* block a fresh reindex (BullMQ ignores add() when the jobId already exists).
* If a reindex is already running, remove() is a no-op (it leaves a
* locked/active job in place, returning 0 without throwing), and the add()
* below then de-duplicates against that still-present jobId — so the running
* pass is kept and no duplicate is started. The .catch only guards against
* transport/Redis errors.
*
* Also cancels any pending delayed WORKSPACE_DELETE_EMBEDDINGS job (scheduled
* when AI Search was disabled) so it cannot wipe the embeddings we are about
* to rebuild. The job no-ops if embeddings are unconfigured.
*/
async reindex(workspaceId: string): Promise<void> {
// A reindex means embeddings must persist: drop the delayed purge, if any.
await this.aiQueue
.remove(`ai-search-disabled-${workspaceId}`)
.catch(() => undefined);
const jobId = `ai-reindex-${workspaceId}`;
// Clear a prior non-active entry so a stale job can't block this reindex.
// A locked/active job is left in place (remove() no-ops) and the add() below
// de-duplicates against it, keeping the in-progress pass.
await this.aiQueue.remove(jobId).catch(() => undefined);
await this.aiQueue.add(
QueueJob.WORKSPACE_CREATE_EMBEDDINGS,
{ workspaceId },
{
jobId,
removeOnComplete: true,
removeOnFail: true,
},
);
}
/** Read the stored non-secret provider settings for a workspace. */
private async readProvider(
workspaceId: string,

View File

@@ -1,5 +1,7 @@
import { Module } from '@nestjs/common';
import { BullModule } from '@nestjs/bullmq';
import { CryptoModule } from '../crypto/crypto.module';
import { QueueName } from '../queue/constants';
import { AiService } from './ai.service';
import { AiSettingsService } from './ai-settings.service';
import { AiSettingsController } from './ai-settings.controller';
@@ -12,7 +14,10 @@ import { AiSettingsController } from './ai-settings.controller';
* (CaslModule, global) are resolved without explicit imports.
*/
@Module({
imports: [CryptoModule],
imports: [
CryptoModule,
BullModule.registerQueue({ name: QueueName.AI_QUEUE }),
],
controllers: [AiSettingsController],
providers: [AiService, AiSettingsService],
exports: [AiService, AiSettingsService],