feat(ai): server-side voice dictation (STT) with mic in chat and editor
Add push-to-talk voice dictation that transcribes recorded audio on the server via the workspace's OpenAI-compatible AI provider (Whisper / gpt-4o-transcribe / self-hosted whisper), then inserts the text. Backend: - New `stt_api_key_enc` column + migration; STT creds parity with chat/ embeddings (sttModel/sttBaseUrl/sttApiKey, write-only key, fallbacks to chat baseUrl/key). Both provider whitelists updated (service + repo). - AiService.getTranscriptionModel + AiTranscriptionService. - Gated POST /ai-chat/transcribe (dictation flag → 403, JWT + workspace scope + throttle, 25MB cap, MIME whitelist, never logs audio/key). - New `settings.ai.dictation` workspace flag (DTO + service + audit). Frontend: - Wire up the Voice/STT settings card (model/base URL/key) and the Voice-dictation toggle. - New `features/dictation`: useDictation (MediaRecorder state machine), MicButton, transcribe service; integrated into the chat composer and a new editor-toolbar dictation group, both gated by ai.dictation.
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
Body,
|
||||
Controller,
|
||||
ForbiddenException,
|
||||
@@ -9,6 +10,7 @@ import {
|
||||
Req,
|
||||
Res,
|
||||
UseGuards,
|
||||
UseInterceptors,
|
||||
} from '@nestjs/common';
|
||||
import { Throttle } from '@nestjs/throttler';
|
||||
import { FastifyReply, FastifyRequest } from 'fastify';
|
||||
@@ -22,7 +24,9 @@ import { AiChatRepo } from '@docmost/db/repos/ai-chat/ai-chat.repo';
|
||||
import { AiChatMessageRepo } from '@docmost/db/repos/ai-chat/ai-chat-message.repo';
|
||||
import { UserThrottlerGuard } from '../../integrations/throttle/user-throttler.guard';
|
||||
import { AI_CHAT_THROTTLER } from '../../integrations/throttle/throttler-names';
|
||||
import { FileInterceptor } from '../../common/interceptors/file.interceptor';
|
||||
import { AiChatService, AiChatStreamBody } from './ai-chat.service';
|
||||
import { AiTranscriptionService } from './ai-transcription.service';
|
||||
import {
|
||||
ChatIdDto,
|
||||
GetChatMessagesDto,
|
||||
@@ -43,6 +47,7 @@ export class AiChatController {
|
||||
private readonly aiChatService: AiChatService,
|
||||
private readonly aiChatRepo: AiChatRepo,
|
||||
private readonly aiChatMessageRepo: AiChatMessageRepo,
|
||||
private readonly aiTranscription: AiTranscriptionService,
|
||||
) {}
|
||||
|
||||
/** List the requesting user's chats in this workspace (paginated). */
|
||||
@@ -180,6 +185,74 @@ export class AiChatController {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Transcribe an uploaded audio clip to text using the workspace STT model.
|
||||
* Gated by settings.ai.dictation (403 when disabled). Returns { text }.
|
||||
*/
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@UseGuards(JwtAuthGuard, UserThrottlerGuard)
|
||||
@Throttle({ [AI_CHAT_THROTTLER]: { limit: 20, ttl: 60000 } })
|
||||
@Post('transcribe')
|
||||
@UseInterceptors(FileInterceptor)
|
||||
async transcribe(
|
||||
@Req() req: any,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
): Promise<{ text: string }> {
|
||||
// Gate: dictation must be explicitly enabled for the workspace.
|
||||
const settings = (workspace.settings ?? {}) as {
|
||||
ai?: { dictation?: boolean };
|
||||
};
|
||||
if (settings.ai?.dictation !== true) {
|
||||
throw new ForbiddenException('Dictation is disabled');
|
||||
}
|
||||
|
||||
let file = null;
|
||||
try {
|
||||
// Whisper hard-caps uploads at 25MB; allow a single file.
|
||||
file = await req.file({ limits: { fileSize: 25 * 1024 * 1024, files: 1 } });
|
||||
} catch (err: any) {
|
||||
if (err?.statusCode === 413) {
|
||||
throw new BadRequestException('Audio file too large (max 25MB)');
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (!file) throw new BadRequestException('No audio uploaded');
|
||||
|
||||
// Whitelist audio container types produced by browser MediaRecorder
|
||||
// (Chrome/FF: webm/opus, Safari: mp4) plus common STT-accepted formats.
|
||||
const allowedMime = new Set([
|
||||
'audio/webm',
|
||||
'audio/ogg',
|
||||
'audio/mp4',
|
||||
'audio/mpeg',
|
||||
'audio/wav',
|
||||
'audio/x-wav',
|
||||
'audio/wave',
|
||||
'audio/m4a',
|
||||
'audio/x-m4a',
|
||||
]);
|
||||
// MediaRecorder mimetypes carry parameters (e.g. "audio/webm;codecs=opus");
|
||||
// compare only the base type.
|
||||
const baseMime = file.mimetype.split(';')[0].trim().toLowerCase();
|
||||
if (!allowedMime.has(baseMime)) {
|
||||
throw new BadRequestException('Unsupported audio format');
|
||||
}
|
||||
|
||||
let buf: Buffer;
|
||||
try {
|
||||
buf = await file.toBuffer();
|
||||
} catch (err: any) {
|
||||
// With @fastify/multipart throwFileSizeLimit:true, the 25MB cap is enforced
|
||||
// when the stream is consumed (here), not at req.file().
|
||||
if (err?.statusCode === 413) {
|
||||
throw new BadRequestException('Audio file too large (max 25MB)');
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
const text = await this.aiTranscription.transcribe(workspace.id, buf);
|
||||
return { text };
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure the chat exists, belongs to this workspace, AND was created by the
|
||||
* requesting user (per-user isolation). Throws ForbiddenException otherwise.
|
||||
|
||||
@@ -3,6 +3,7 @@ import { AiModule } from '../../integrations/ai/ai.module';
|
||||
import { TokenModule } from '../auth/token.module';
|
||||
import { AiChatController } from './ai-chat.controller';
|
||||
import { AiChatService } from './ai-chat.service';
|
||||
import { AiTranscriptionService } from './ai-transcription.service';
|
||||
import { AiChatToolsService } from './tools/ai-chat-tools.service';
|
||||
import { EmbeddingModule } from './embedding/embedding.module';
|
||||
import { ExternalMcpModule } from './external-mcp/external-mcp.module';
|
||||
@@ -21,6 +22,6 @@ import { ExternalMcpModule } from './external-mcp/external-mcp.module';
|
||||
@Module({
|
||||
imports: [AiModule, TokenModule, EmbeddingModule, ExternalMcpModule],
|
||||
controllers: [AiChatController],
|
||||
providers: [AiChatService, AiChatToolsService],
|
||||
providers: [AiChatService, AiTranscriptionService, AiChatToolsService],
|
||||
})
|
||||
export class AiChatModule {}
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { experimental_transcribe as transcribe } from 'ai';
|
||||
import { AiService } from '../../integrations/ai/ai.service';
|
||||
|
||||
/**
|
||||
* Transcribes uploaded audio to text using the per-workspace STT model.
|
||||
* Thin wrapper over the AI SDK's experimental_transcribe; never logs the
|
||||
* audio or the key.
|
||||
*/
|
||||
@Injectable()
|
||||
export class AiTranscriptionService {
|
||||
constructor(private readonly ai: AiService) {}
|
||||
|
||||
// Transcribe an uploaded audio buffer using the workspace STT model.
|
||||
async transcribe(workspaceId: string, audio: Uint8Array): Promise<string> {
|
||||
const model = await this.ai.getTranscriptionModel(workspaceId);
|
||||
const { text } = await transcribe({ model, audio });
|
||||
return text.trim();
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,10 @@ export class UpdateWorkspaceDto extends PartialType(CreateWorkspaceDto) {
|
||||
@IsBoolean()
|
||||
aiChat: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
aiDictation: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsInt()
|
||||
@Min(1)
|
||||
|
||||
@@ -497,6 +497,20 @@ export class WorkspaceService {
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof updateWorkspaceDto.aiDictation !== 'undefined') {
|
||||
const prev = settingsBefore?.ai?.dictation ?? false;
|
||||
if (prev !== updateWorkspaceDto.aiDictation) {
|
||||
before.aiDictation = prev;
|
||||
after.aiDictation = updateWorkspaceDto.aiDictation;
|
||||
}
|
||||
await this.workspaceRepo.updateAiSettings(
|
||||
workspaceId,
|
||||
'dictation',
|
||||
updateWorkspaceDto.aiDictation,
|
||||
trx,
|
||||
);
|
||||
}
|
||||
|
||||
delete updateWorkspaceDto.restrictApiToAdmins;
|
||||
delete updateWorkspaceDto.aiSearch;
|
||||
delete updateWorkspaceDto.generativeAi;
|
||||
@@ -504,6 +518,7 @@ export class WorkspaceService {
|
||||
delete updateWorkspaceDto.mcpEnabled;
|
||||
delete updateWorkspaceDto.allowMemberTemplates;
|
||||
delete updateWorkspaceDto.aiChat;
|
||||
delete updateWorkspaceDto.aiDictation;
|
||||
|
||||
await this.workspaceRepo.updateWorkspace(
|
||||
updateWorkspaceDto,
|
||||
|
||||
Reference in New Issue
Block a user