feat(ai): server-side voice dictation (STT) with mic in chat and editor

Add push-to-talk voice dictation that transcribes recorded audio on the
server via the workspace's OpenAI-compatible AI provider (Whisper /
gpt-4o-transcribe / self-hosted whisper), then inserts the text.

Backend:
- New `stt_api_key_enc` column + migration; STT creds parity with chat/
  embeddings (sttModel/sttBaseUrl/sttApiKey, write-only key, fallbacks to
  chat baseUrl/key). Both provider whitelists updated (service + repo).
- AiService.getTranscriptionModel + AiTranscriptionService.
- Gated POST /ai-chat/transcribe (dictation flag → 403, JWT + workspace
  scope + throttle, 25MB cap, MIME whitelist, never logs audio/key).
- New `settings.ai.dictation` workspace flag (DTO + service + audit).

Frontend:
- Wire up the Voice/STT settings card (model/base URL/key) and the
  Voice-dictation toggle.
- New `features/dictation`: useDictation (MediaRecorder state machine),
  MicButton, transcribe service; integrated into the chat composer and a
  new editor-toolbar dictation group, both gated by ai.dictation.
This commit is contained in:
vvzvlad
2026-06-18 18:45:33 +03:00
parent 49eba22201
commit 874bdd021c
24 changed files with 845 additions and 39 deletions
@@ -1,4 +1,5 @@
import {
BadRequestException,
Body,
Controller,
ForbiddenException,
@@ -9,6 +10,7 @@ import {
Req,
Res,
UseGuards,
UseInterceptors,
} from '@nestjs/common';
import { Throttle } from '@nestjs/throttler';
import { FastifyReply, FastifyRequest } from 'fastify';
@@ -22,7 +24,9 @@ import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
import { UserThrottlerGuard } from '../../integrations/throttle/user-throttler.guard';
import { AI_CHAT_THROTTLER } from '../../integrations/throttle/throttler-names';
import { FileInterceptor } from '../../common/interceptors/file.interceptor';
import { AiChatService, AiChatStreamBody } from './ai-chat.service';
import { AiTranscriptionService } from './ai-transcription.service';
import {
ChatIdDto,
GetChatMessagesDto,
@@ -43,6 +47,7 @@ export class AiChatController {
private readonly aiChatService: AiChatService,
private readonly aiChatRepo: AiChatRepo,
private readonly aiChatMessageRepo: AiChatMessageRepo,
private readonly aiTranscription: AiTranscriptionService,
) {}
/** List the requesting user's chats in this workspace (paginated). */
@@ -180,6 +185,74 @@ export class AiChatController {
}
}
/**
* Transcribe an uploaded audio clip to text using the workspace STT model.
* Gated by settings.ai.dictation (403 when disabled). Returns { text }.
*/
@HttpCode(HttpStatus.OK)
@UseGuards(JwtAuthGuard, UserThrottlerGuard)
@Throttle({ [AI_CHAT_THROTTLER]: { limit: 20, ttl: 60000 } })
@Post('transcribe')
@UseInterceptors(FileInterceptor)
async transcribe(
@Req() req: any,
@AuthWorkspace() workspace: Workspace,
): Promise<{ text: string }> {
// Gate: dictation must be explicitly enabled for the workspace.
const settings = (workspace.settings ?? {}) as {
ai?: { dictation?: boolean };
};
if (settings.ai?.dictation !== true) {
throw new ForbiddenException('Dictation is disabled');
}
let file = null;
try {
// Whisper hard-caps uploads at 25MB; allow a single file.
file = await req.file({ limits: { fileSize: 25 * 1024 * 1024, files: 1 } });
} catch (err: any) {
if (err?.statusCode === 413) {
throw new BadRequestException('Audio file too large (max 25MB)');
}
throw err;
}
if (!file) throw new BadRequestException('No audio uploaded');
// Whitelist audio container types produced by browser MediaRecorder
// (Chrome/FF: webm/opus, Safari: mp4) plus common STT-accepted formats.
const allowedMime = new Set([
'audio/webm',
'audio/ogg',
'audio/mp4',
'audio/mpeg',
'audio/wav',
'audio/x-wav',
'audio/wave',
'audio/m4a',
'audio/x-m4a',
]);
// MediaRecorder mimetypes carry parameters (e.g. "audio/webm;codecs=opus");
// compare only the base type.
const baseMime = file.mimetype.split(';')[0].trim().toLowerCase();
if (!allowedMime.has(baseMime)) {
throw new BadRequestException('Unsupported audio format');
}
let buf: Buffer;
try {
buf = await file.toBuffer();
} catch (err: any) {
// With @fastify/multipart throwFileSizeLimit:true, the 25MB cap is enforced
// when the stream is consumed (here), not at req.file().
if (err?.statusCode === 413) {
throw new BadRequestException('Audio file too large (max 25MB)');
}
throw err;
}
const text = await this.aiTranscription.transcribe(workspace.id, buf);
return { text };
}
/**
* Ensure the chat exists, belongs to this workspace, AND was created by the
* requesting user (per-user isolation). Throws ForbiddenException otherwise.
@@ -3,6 +3,7 @@ import { AiModule } from '../../integrations/ai/ai.module';
import { TokenModule } from '../auth/token.module';
import { AiChatController } from './ai-chat.controller';
import { AiChatService } from './ai-chat.service';
import { AiTranscriptionService } from './ai-transcription.service';
import { AiChatToolsService } from './tools/ai-chat-tools.service';
import { EmbeddingModule } from './embedding/embedding.module';
import { ExternalMcpModule } from './external-mcp/external-mcp.module';
@@ -21,6 +22,6 @@ import { ExternalMcpModule } from './external-mcp/external-mcp.module';
@Module({
imports: [AiModule, TokenModule, EmbeddingModule, ExternalMcpModule],
controllers: [AiChatController],
providers: [AiChatService, AiChatToolsService],
providers: [AiChatService, AiTranscriptionService, AiChatToolsService],
})
export class AiChatModule {}
@@ -0,0 +1,20 @@
import { Injectable } from '@nestjs/common';
import { experimental_transcribe as transcribe } from 'ai';
import { AiService } from '../../integrations/ai/ai.service';
/**
* Transcribes uploaded audio to text using the per-workspace STT model.
* Thin wrapper over the AI SDK's experimental_transcribe; never logs the
* audio or the key.
*/
@Injectable()
export class AiTranscriptionService {
constructor(private readonly ai: AiService) {}
// Transcribe an uploaded audio buffer using the workspace STT model.
async transcribe(workspaceId: string, audio: Uint8Array): Promise<string> {
const model = await this.ai.getTranscriptionModel(workspaceId);
const { text } = await transcribe({ model, audio });
return text.trim();
}
}
@@ -49,6 +49,10 @@ export class UpdateWorkspaceDto extends PartialType(CreateWorkspaceDto) {
@IsBoolean()
aiChat: boolean;
@IsOptional()
@IsBoolean()
aiDictation: boolean;
@IsOptional()
@IsInt()
@Min(1)
@@ -497,6 +497,20 @@ export class WorkspaceService {
);
}
if (typeof updateWorkspaceDto.aiDictation !== 'undefined') {
const prev = settingsBefore?.ai?.dictation ?? false;
if (prev !== updateWorkspaceDto.aiDictation) {
before.aiDictation = prev;
after.aiDictation = updateWorkspaceDto.aiDictation;
}
await this.workspaceRepo.updateAiSettings(
workspaceId,
'dictation',
updateWorkspaceDto.aiDictation,
trx,
);
}
delete updateWorkspaceDto.restrictApiToAdmins;
delete updateWorkspaceDto.aiSearch;
delete updateWorkspaceDto.generativeAi;
@@ -504,6 +518,7 @@ export class WorkspaceService {
delete updateWorkspaceDto.mcpEnabled;
delete updateWorkspaceDto.allowMemberTemplates;
delete updateWorkspaceDto.aiChat;
delete updateWorkspaceDto.aiDictation;
await this.workspaceRepo.updateWorkspace(
updateWorkspaceDto,