gitmost/apps/server/src/integrations/import/services/import.service.ts

import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import { PageRepo } from '@docmost/db/repos/page/page.repo';
import { MultipartFile } from '@fastify/multipart';
import * as path from 'path';
import {
  htmlToJson,
  jsonToText,
  tiptapExtensions,
} from '../../../collaboration/collaboration.util';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import {
  generateSlugId,
  sanitizeFileName,
  createByteCountingStream,
} from '../../../common/helpers';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { TiptapTransformer } from '@hocuspocus/transformer';
import * as Y from 'yjs';
import { canonicalizeFootnotes } from '@docmost/editor-ext';
import {
  markdownToProseMirror,
  normalizeForeignMarkdown,
} from '@docmost/prosemirror-markdown';
import {
  FileTaskStatus,
  FileTaskType,
  getFileTaskFolderPath,
} from '../utils/file.utils';
import { v7 as uuid7 } from 'uuid';
import { StorageService } from '../../storage/storage.service';
import { InjectQueue } from '@nestjs/bullmq';
import { Queue } from 'bullmq';
import { QueueJob, QueueName } from '../../queue/constants';
import { load } from 'cheerio';
import { normalizeImportHtml } from '../utils/import-formatter';

@Injectable()
export class ImportService {
  private readonly logger = new Logger(ImportService.name);

  constructor(
    private readonly pageRepo: PageRepo,
    private readonly storageService: StorageService,
    @InjectKysely() private readonly db: KyselyDB,
    @InjectQueue(QueueName.FILE_TASK_QUEUE)
    private readonly fileTaskQueue: Queue,
  ) {}

  async importPage(
    filePromise: Promise<MultipartFile>,
    userId: string,
    spaceId: string,
    workspaceId: string,
    // #502: when true, the markdown importer runs with the two layered
    // extensions OFF (a `$…$` span stays literal text; a schemeless `www.host` is
    // NOT autolinked). ONLY the MCP agent `createPage` path sets this; a HUMAN
    // file upload never passes it, so it defaults false and math/autolink stay ON
    // for human imports (their `$x^2$` still becomes a formula).
    disableMarkdownExtensions = false,
  ) {
    const file = await filePromise;
    const fileBuffer = await file.toBuffer();
    const fileExtension = path.extname(file.filename).toLowerCase();
    const fileName = sanitizeFileName(
      path.basename(file.filename, fileExtension),
    );
    const fileContent = fileBuffer.toString();

    let prosemirrorState = null;
    let createdPage = null;

    try {
      if (fileExtension.endsWith('.md')) {
        prosemirrorState = await this.processMarkdown(
          fileContent,
          disableMarkdownExtensions,
        );
      } else if (fileExtension.endsWith('.html')) {
        prosemirrorState = await this.processHTML(fileContent);
      }
    } catch (err) {
      // Surface the real cause instead of a generic mask, so the failure is
      // diagnosable from the HTTP response (project convention: never swallow).
      const reason =
        err instanceof Error ? `${err.name}: ${err.message}` : String(err);
      this.logger.error(`Error processing file content: ${reason}`, err);
      throw new BadRequestException(
        `Error processing file content: ${reason}`,
      );
    }

    if (!prosemirrorState) {
      const message = 'Failed to create ProseMirror state';
      this.logger.error(message);
      throw new BadRequestException(message);
    }

    const extracted = this.extractTitleAndRemoveHeading(prosemirrorState);
    const title = extracted.title;
    // The markdown path now canonicalizes footnotes itself (the package parser),
    // but the HTML path (processHTML -> htmlToJson) does NOT run the editor's
    // footnoteSyncPlugin, so an imported HTML doc can keep its source's PHYSICAL
    // definition order (out of order vs. references), retain orphan definitions,
    // and not be deduped. Canonicalize before persisting so the stored page
    // matches the editor's invariant (issue #228); it is an idempotent no-op on
    // the already-canonical markdown output.
    // Pure + idempotent + shape-safe: a doc with no footnotes is unchanged.
    // (Future consolidation, architecture B: this import path persists directly
    // via pageRepo.insertPage rather than through PageService.createPage, so the
    // canonicalize call lives here; folding both into one "prepare JSON for
    // persist" helper is a sensible follow-up.)
    const prosemirrorJson = canonicalizeFootnotes(extracted.prosemirrorJson);

    const pageTitle = title || fileName;

    if (prosemirrorJson) {
      try {
        const pagePosition = await this.getNewPagePosition(spaceId);

        createdPage = await this.pageRepo.insertPage({
          slugId: generateSlugId(),
          title: pageTitle,
          content: prosemirrorJson,
          textContent: jsonToText(prosemirrorJson),
          ydoc: await this.createYdoc(prosemirrorJson),
          position: pagePosition,
          spaceId: spaceId,
          creatorId: userId,
          workspaceId: workspaceId,
          lastUpdatedById: userId,
        });

        this.logger.debug(
          `Successfully imported "${title}${fileExtension}. ID: ${createdPage.id} - SlugId: ${createdPage.slugId}"`,
        );
      } catch (err) {
        const reason =
          err instanceof Error ? `${err.name}: ${err.message}` : String(err);
        this.logger.error(`Failed to create imported page: ${reason}`, err);
        throw new BadRequestException(
          `Failed to create imported page: ${reason}`,
        );
      }
    }

    return createdPage;
  }

  async processMarkdown(
    markdownInput: string,
    // #502: forwarded to the importer. DEFAULT false keeps math + fuzzy autolink
    // ON (human uploads unaffected); the MCP agent `createPage` path passes true.
    disableMarkdownExtensions = false,
  ): Promise<any> {
    // Canonical markdown -> ProseMirror JSON directly via
    // `@docmost/prosemirror-markdown` (issue #345) — no HTML intermediate and no
    // second editor-ext markdown layer. Foreign markdown surfaces the strict
    // canonical parser does not accept (GFM `[^id]` reference footnotes) are
    // rewritten to the canonical inline form by `normalizeForeignMarkdown` first.
    // The HTML-cleanup pass (`normalizeImportHtml`) is intentionally skipped here:
    // it targets foreign *HTML* (Notion/XWiki), which only ever arrives on the
    // `.html` path (`processHTML`), never as canonical markdown.
    return markdownToProseMirror(
      normalizeForeignMarkdown(markdownInput),
      disableMarkdownExtensions
        ? { parseMath: false, fuzzyLinkify: false }
        : undefined,
    );
  }

  async processHTML(htmlInput: string): Promise<any> {
    try {
      const $ = load(htmlInput);
      normalizeImportHtml($, $.root());
      return htmlToJson($.html() || '');
    } catch (err) {
      throw err;
    }
  }

  async createYdoc(prosemirrorJson: any): Promise<Buffer | null> {
    if (prosemirrorJson) {
      // this.logger.debug(`Converting prosemirror json state to ydoc`);

      const ydoc = TiptapTransformer.toYdoc(
        prosemirrorJson,
        'default',
        tiptapExtensions,
      );

      Y.encodeStateAsUpdate(ydoc);

      return Buffer.from(Y.encodeStateAsUpdate(ydoc));
    }
    return null;
  }

  extractTitleAndRemoveHeading(prosemirrorState: any) {
    let title: string | null = null;

    const content = prosemirrorState.content ?? [];

    if (
      content.length > 0 &&
      content[0].type === 'heading' &&
      content[0].attrs?.level === 1
    ) {
      title = content[0].content?.[0]?.text ?? null;
      content.shift();
    }

    // ensure at least one paragraph
    if (content.length === 0) {
      content.push({
        type: 'paragraph',
        content: [],
      });
    }

    return {
      title,
      prosemirrorJson: {
        ...prosemirrorState,
        content,
      },
    };
  }

  async getNewPagePosition(
    spaceId: string,
    parentPageId?: string,
  ): Promise<string> {
    let query = this.db
      .selectFrom('pages')
      .select(['id', 'position'])
      .where('spaceId', '=', spaceId)
      .orderBy('position', (ob) => ob.collate('C').desc())
      .limit(1);

    if (parentPageId) {
      query = query.where('parentPageId', '=', parentPageId);
    } else {
      query = query.where('parentPageId', 'is', null);
    }

    const lastPage = await query.executeTakeFirst();

    if (lastPage) {
      return generateJitteredKeyBetween(lastPage.position, null);
    } else {
      return generateJitteredKeyBetween(null, null);
    }
  }

  async importZip(
    filePromise: Promise<MultipartFile>,
    source: string,
    userId: string,
    spaceId: string,
    workspaceId: string,
  ) {
    const file = await filePromise;
    const fileExtension = path.extname(file.filename).toLowerCase();
    const fileName = sanitizeFileName(
      path.basename(file.filename, fileExtension),
    );
    const fileNameWithExt = fileName + fileExtension;

    const fileTaskId = uuid7();
    const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileNameWithExt}`;

    // upload file
    const { stream, getBytesRead } = createByteCountingStream(file.file);

    await this.storageService.upload(filePath, stream);

    const fileSize = getBytesRead();

    const fileTask = await this.db
      .insertInto('fileTasks')
      .values({
        id: fileTaskId,
        type: FileTaskType.Import,
        source: source,
        status: FileTaskStatus.Processing,
        fileName: fileNameWithExt,
        filePath: filePath,
        fileSize: fileSize,
        fileExt: 'zip',
        creatorId: userId,
        spaceId: spaceId,
        workspaceId: workspaceId,
      })
      .returningAll()
      .executeTakeFirst();

    await this.fileTaskQueue.add(QueueJob.IMPORT_TASK, {
      fileTaskId: fileTaskId,
    });

    return fileTask;
  }
}