fix(ai): include content-bearing pages in reindex coverage; correct progress race & hot path (F6-F10)

F6: extend embeddablePredicate to pages with body content but null text_content,
    keyed on the text-node marker "type":"text" (not a bare "text": key, which
    also matched math nodes' attrs.text and would leave math-only pages stuck
    below 100%). Numerator and denominator share the predicate; tests assert the
    compiled WHERE is byte-identical and a math-only doc is excluded.
F7: correct the start() JSDoc (both totals are the real page count).
F8: nextReindexPollInterval reuses isReindexComplete.
F9: getMasked reads progress first and skips the two COUNTs while a reindex is active.
F10: pre-seed the progress entry with a short 45s TTL so a deduped enqueue's
     phantom "0 of N" expires quickly instead of sticking for the 1h TTL.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

apps/server/src/integrations/ai/ai-settings.service.spec.ts

@@ -172,7 +172,17 @@ describe('AiSettingsService.reindex progress seed', () => {
 
     await service.reindex(WORKSPACE_ID);
 
-    expect(reindexProgress.start).toHaveBeenCalledWith(WORKSPACE_ID, 478);
+    // The pre-seed carries the real page count AND a SHORT ttl (3rd arg) so a
+    // de-duplicated enqueue against a just-finishing job can't leave a phantom
+    // "reindexing: 0 of N" stuck for the full record TTL (F10).
+    expect(reindexProgress.start).toHaveBeenCalledWith(
+      WORKSPACE_ID,
+      478,
+      expect.any(Number),
+    );
+    const ttl = reindexProgress.start.mock.calls[0][2];
+    expect(ttl).toBeGreaterThan(0);
+    expect(ttl).toBeLessThanOrEqual(60); // short, not the full 1h record TTL
     expect(aiQueue.add).toHaveBeenCalledTimes(1);
     // Seed must precede the enqueue so the first poll already reports done=0.
     expect(order).toEqual(['start', 'add']);
@@ -204,7 +214,11 @@ describe('AiSettingsService.reindex progress seed', () => {
 
     await expect(service.reindex(WORKSPACE_ID)).rejects.toBe(boom);
 
-    expect(reindexProgress.start).toHaveBeenCalledWith(WORKSPACE_ID, 478);
+    expect(reindexProgress.start).toHaveBeenCalledWith(
+      WORKSPACE_ID,
+      478,
+      expect.any(Number),
+    );
     expect(reindexProgress.clear).toHaveBeenCalledWith(WORKSPACE_ID);
   });
 

apps/server/src/integrations/ai/ai-settings.service.ts

@@ -31,6 +31,17 @@ export function parsePositiveInt(raw: unknown): number | undefined {
   return Number.isFinite(n) && n > 0 ? Math.floor(n) : undefined;
 }
 
+/**
+ * TTL (seconds) for the enqueue-time progress PRE-SEED written by `reindex()`
+ * before the worker starts. Deliberately SHORT: if `aiQueue.add()` de-duplicates
+ * against a job that is just finishing (the worker's finally already ran
+ * `clear()` but removeOnComplete hasn't yet removed the job), no new worker runs
+ * to overwrite/clear this seed — so a short TTL lets the phantom "reindexing:
+ * 0 of N" expire in seconds instead of sticking for the full 1h record TTL. A
+ * worker that DOES start re-seeds with the full TTL, so a real run is unaffected.
+ */
+const PRE_SEED_TTL_SECONDS = 45;
+
 /**
  * Shape of the partial update accepted by `update`. Mirrors the validated
  * controller DTO. `apiKey` / `embeddingApiKey` are write-only: undefined =
@@ -117,7 +128,15 @@ export class AiSettingsService {
     let seeded = false;
     if ((await this.reindexProgress.get(workspaceId)) === null) {
       const totalPages = await this.pageRepo.countEmbeddablePages(workspaceId);
-      await this.reindexProgress.start(workspaceId, totalPages);
+      // Short TTL: if add() below de-duplicates against a just-finishing job
+      // whose worker already clear()ed but isn't removed yet, no worker runs to
+      // clear this seed — the short TTL expires the phantom record in seconds
+      // rather than leaving a stuck "reindexing: 0 of N" for the full record TTL.
+      await this.reindexProgress.start(
+        workspaceId,
+        totalPages,
+        PRE_SEED_TTL_SECONDS,
+      );
       seeded = true;
     }
 
@@ -286,22 +305,33 @@ export class AiSettingsService {
       hasSttApiKey = !!creds?.sttApiKeyEnc;
     }
 
-    // totalPages now counts only pages with embeddable content (non-empty text
-    // or already-stored embeddings), so empty/text-less pages don't keep the
-    // "Indexed N of M pages" bar below 100% forever.
-    const [indexedPages, totalPages] = await Promise.all([
-      this.pageEmbeddingRepo.countIndexedPages(workspaceId),
-      this.pageRepo.countEmbeddablePages(workspaceId),
-    ]);
-
     // While a reindex run is active, report its LIVE progress (done climbs 0 ->
-    // total) so the settings UI can watch it advance. Without this the counter
-    // never drops: the per-page reindex hard-replaces rows in its own small
-    // transaction, so countIndexedPages stays ~= total for the whole run. With
-    // no active record we fall back to the steady-state DB coverage count, which
+    // total) so the settings UI can watch it advance. Read progress FIRST and
+    // short-circuit: this endpoint is polled every ~5s for the whole run, so when
+    // a record is active we skip the two coverage COUNTs entirely (their results
+    // would be discarded anyway). Without the live progress the counter never
+    // drops: the per-page reindex hard-replaces rows in its own small
+    // transaction, so countIndexedPages stays ~= total for the whole run. With no
+    // active record we fall back to the steady-state DB coverage count, which
     // preserves the existing display and the client's "done == total -> stop
     // polling" condition (the run ends -> record cleared -> DB count == total).
+    //
+    // The fallback `totalPages` counts only pages with embeddable content
+    // (non-empty text, content-borne text, or already-stored embeddings), so
+    // empty/text-less pages don't keep the "Indexed N of M pages" bar below 100%
+    // forever.
     const progress = await this.reindexProgress.get(workspaceId);
+    let indexedPages: number;
+    let totalPages: number;
+    if (progress) {
+      indexedPages = progress.done;
+      totalPages = progress.total;
+    } else {
+      [indexedPages, totalPages] = await Promise.all([
+        this.pageEmbeddingRepo.countIndexedPages(workspaceId),
+        this.pageRepo.countEmbeddablePages(workspaceId),
+      ]);
+    }
 
     return {
       driver: provider.driver,
@@ -321,8 +351,8 @@ export class AiSettingsService {
       hasApiKey,
       hasEmbeddingApiKey,
       hasSttApiKey,
-      indexedPages: progress ? progress.done : indexedPages,
-      totalPages: progress ? progress.total : totalPages,
+      indexedPages,
+      totalPages,
       // Optional hint for the client: a reindex run is currently in progress.
       reindexing: progress != null,
     };

apps/server/src/integrations/ai/embedding-reindex-progress.service.spec.ts

@@ -115,6 +115,22 @@ describe('EmbeddingReindexProgressService', () => {
       expect(multiObj.exec).toHaveBeenCalledTimes(1);
     });
 
+    it('defaults the expire TTL to the full 1h record TTL', async () => {
+      const { redis, multiObj } = makeRedis();
+      await makeService(redis).start(WORKSPACE_ID, 478);
+      // Default ttl = full record TTL (60 * 60) so a real run never expires
+      // mid-flight before the worker refreshes it on each increment.
+      expect(multiObj.expire).toHaveBeenCalledWith(KEY, 60 * 60);
+    });
+
+    it('honours an explicit short ttlSeconds for the enqueue-time pre-seed (F10)', async () => {
+      const { redis, multiObj } = makeRedis();
+      // The reindex() pre-seed passes a short ttl so a phantom record left by a
+      // de-duplicated enqueue expires in seconds, not after the full 1h TTL.
+      await makeService(redis).start(WORKSPACE_ID, 478, 45);
+      expect(multiObj.expire).toHaveBeenCalledWith(KEY, 45);
+    });
+
     it('swallows a thrown Redis error (best-effort)', async () => {
       const { redis } = makeRedis({
         execImpl: () => Promise.reject(new Error('redis down')),

apps/server/src/integrations/ai/embedding-reindex-progress.service.ts

@@ -65,12 +65,25 @@ export class EmbeddingReindexProgressService {
 
   /**
    * Begin (or reset) the progress record for a workspace: `total` pages, `done`
-   * back to 0, `startedAt` now. Called at reindex enqueue time (placeholder
-   * total, so the very first status poll already reports done=0) and again at
-   * the worker start (overwriting `total` with the real page count). Resets
-   * `done` to 0 so a re-trigger never inherits a stale count.
+   * back to 0, `startedAt` now. Called twice for a run, BOTH with the real page
+   * count (countEmbeddablePages) so the two totals coincide: once at reindex
+   * enqueue time (so the very first status poll already reports done=0) and again
+   * at the worker start (which re-asserts the same total and resets `done`).
+   * Resets `done` to 0 so a re-trigger never inherits a stale count.
+   *
+   * `ttlSeconds` lets the caller pick the record's lifetime. The enqueue-time
+   * pre-seed passes a SHORT ttl: if `aiQueue.add()` de-duplicates against a job
+   * that is just finishing (its worker hasn't yet removed the job but already
+   * ran its `clear()`), no new worker starts to clear this phantom seed, so a
+   * short ttl lets it expire in seconds instead of sticking for the full TTL.
+   * The worker's own `start()` at the begin of a real run overwrites this entry
+   * and raises the ttl back to the default full TTL.
    */
-  async start(workspaceId: string, total: number): Promise<void> {
+  async start(
+    workspaceId: string,
+    total: number,
+    ttlSeconds: number = TTL_SECONDS,
+  ): Promise<void> {
     const key = this.key(workspaceId);
     try {
       await this.redis
@@ -80,7 +93,7 @@ export class EmbeddingReindexProgressService {
           done: '0',
           startedAt: String(Date.now()),
         })
-        .expire(key, TTL_SECONDS)
+        .expire(key, ttlSeconds)
         .exec();
     } catch (err) {
       this.logger.warn(