gitmost/apps/server/src/core/search/search-query-parser.ts

// #529 Phase A — server-side query parser (the SINGLE source of query semantics).
//
// Clients (web-UI, MCP agent, public share) send a RAW query string plus flags
// (`match`, `mode`); ALL query interpretation happens HERE, so every consumer
// gets identical operator/phrase/morphology behaviour. The parser is PURE (no
// SQL, no DB) so it is exhaustively unit-testable; SearchService turns the parsed
// AST into a parameterized tsquery/predicate tree (never string-concatenated SQL).
//
// Grammar (A2):
//   - Whitespace splits tokens, but a double-quoted run is ONE token ("a b" is a
//     phrase). An unbalanced quote is dropped.
//   - A token is an OPERATOR token only when it STARTS with `+` or `-`, the
//     remainder is non-empty, and the remainder is not itself only operators.
//     `+`/`-` inside a token (`WB-MGE-30D86B`, `10.0.12.5`, `a:b`) is a LITERAL —
//     the token is a single term. A bare `-`/`+` is dropped.
//   - `"phrase"` → phrase term; `+"phrase"`/`-"phrase"` apply the operator to it.
//   - Positive terms (bare + phrase, no operator) form the OR recall set.
//     `+term` is a REQUIRED predicate, `-term` an EXCLUDED predicate (A2): both
//     are applied in SQL WHERE against the whole candidate set, not folded into
//     the positive tsquery.
//   - Only-negation (no positive term) short-circuits to an empty result with
//     reason `only-negation` (never runs a costly NOT-scan).

export type SearchMatchMode = 'auto' | 'word' | 'prefix' | 'substring';
export type SearchBooleanMode = 'or' | 'and';

// How a single term is matched against the index.
//   - 'fts'       : full-text lexeme, exact (no trailing prefix).
//   - 'ftsPrefix' : full-text lexeme with a `:*` prefix match.
//   - 'phrase'    : an adjacency phrase (phraseto_tsquery).
//   - 'substring' : a literal LOWER(f_unaccent(col)) LIKE '%needle%' branch
//                   (identifiers the tokenizer mangles: IPs, hostnames, IDs).
export type SearchTermBranch = 'fts' | 'ftsPrefix' | 'phrase' | 'substring';

export interface ParsedTerm {
  // The user-visible term text, operator stripped, quotes removed. This is what
  // `matchedTerms` echoes back per hit.
  text: string;
  branch: SearchTermBranch;
}

export interface ParsedQuery {
  raw: string;
  // OR-recall set (bare + phrase terms with no operator).
  positive: ParsedTerm[];
  // AND predicates (`+term`) — the candidate MUST match each of these.
  required: ParsedTerm[];
  // NOT predicates (`-term`) — the candidate must match NONE of these.
  excluded: ParsedTerm[];
  mode: SearchBooleanMode;
  // Set only when the query yields no positive recall: 'empty' (nothing usable)
  // or 'only-negation' (there were exclusions but no positive term).
  reason?: 'empty' | 'only-negation';
}

interface RawToken {
  op: '' | '+' | '-';
  kind: 'word' | 'phrase';
  text: string;
}

// tsquery metacharacters that must never reach to_tsquery from a bare term — they
// are what turned adversarial input into a 500 before (#139). Stripped for the FTS
// branch; the substring branch keeps them (they are literal there).
const TSQUERY_META = /[:&|!()*<>\\]+/g;

// A term is "identifier-like" when it carries a digit or one of . _ : / - AND is
// not purely alphabetic (letters only). Such tokens (10.31.41, esp32,
// WB-MGE-30D86B) are mangled by the FTS tokenizer, so `match: auto` routes them
// to the substring branch. A purely-alphabetic word (печат, ресторан) stays FTS.
const IDENTIFIER_SIGNAL = /[0-9._:/\\-]/;
const PURELY_ALPHA = /^\p{L}+$/u;

function isIdentifierLike(text: string): boolean {
  return IDENTIFIER_SIGNAL.test(text) && !PURELY_ALPHA.test(text);
}

// Clean a bare term for the FTS branch: NFC-normalize, drop tsquery metacharacters
// and collapse whitespace. Returns '' when nothing usable remains.
export function cleanFtsLexeme(raw: string): string {
  return (raw ?? '')
    .normalize('NFC')
    .replace(TSQUERY_META, ' ')
    .replace(/\s+/g, ' ')
    .trim();
}

// Split a raw query into tokens, honouring double quotes and a single leading
// +/- operator. Unbalanced quotes and bare operators are dropped.
function tokenize(raw: string): RawToken[] {
  const tokens: RawToken[] = [];
  const s = raw ?? '';
  let i = 0;
  const n = s.length;

  const isSpace = (c: string) => /\s/.test(c);

  while (i < n) {
    // Skip leading whitespace.
    while (i < n && isSpace(s[i])) i++;
    if (i >= n) break;

    let op: '' | '+' | '-' = '';
    // A single leading +/- is a tentative operator. Only ONE leading operator is
    // consumed; a second (`--x`) leaves `-x` as the remainder (a literal dash).
    if (s[i] === '+' || s[i] === '-') {
      op = s[i] as '+' | '-';
      i++;
    }

    if (i < n && s[i] === '"') {
      // Quoted phrase: read until the closing quote.
      const close = s.indexOf('"', i + 1);
      if (close === -1) {
        // Unbalanced quote → drop this token and everything the open quote would
        // have consumed (the rest of the string).
        break;
      }
      const phrase = s.slice(i + 1, close);
      i = close + 1;
      if (phrase.trim().length > 0) {
        tokens.push({ op, kind: 'phrase', text: phrase.trim() });
      }
      continue;
    }

    // Bare word: read until the next whitespace.
    let j = i;
    while (j < n && !isSpace(s[j])) j++;
    const word = s.slice(i, j);
    i = j;

    // A bare operator (`-`/`+` with no remainder) or an all-operator remainder is
    // dropped.
    if (word.length === 0) continue;
    if (op && /^[+-]+$/.test(word)) continue;

    tokens.push({ op, kind: 'word', text: word });
  }

  return tokens;
}

// Resolve the match branch for a single term given the global match mode.
function branchForTerm(
  text: string,
  kind: 'word' | 'phrase',
  mode: SearchMatchMode,
): SearchTermBranch {
  if (kind === 'phrase') return 'phrase';
  switch (mode) {
    case 'word':
      return 'fts';
    case 'prefix':
      return 'ftsPrefix';
    case 'substring':
      return 'substring';
    case 'auto':
    default:
      // Identifiers the tokenizer mangles go to substring; words get a prefix
      // FTS match (so `печат` still finds `печатать`, but `печат` no longer drags
      // in `впечатления` because the russian stemmer anchors the stem).
      return isIdentifierLike(text) ? 'substring' : 'ftsPrefix';
  }
}

/**
 * Parse a raw user query + flags into a structured, SQL-agnostic AST.
 * Pure and total: never throws, always returns a ParsedQuery.
 */
export function parseSearchQuery(
  raw: string,
  opts: { match?: SearchMatchMode; mode?: SearchBooleanMode } = {},
): ParsedQuery {
  const match: SearchMatchMode = opts.match ?? 'auto';
  const mode: SearchBooleanMode = opts.mode ?? 'or';

  const positive: ParsedTerm[] = [];
  const required: ParsedTerm[] = [];
  const excluded: ParsedTerm[] = [];

  for (const tok of tokenize(raw)) {
    // For an FTS branch, the token must survive metacharacter cleaning; for the
    // substring/phrase branch the literal text is used. A term that cleans to
    // nothing AND is not usable as a substring is dropped.
    const branch = branchForTerm(tok.text, tok.kind, match);

    let usableText: string;
    if (branch === 'fts' || branch === 'ftsPrefix') {
      usableText = cleanFtsLexeme(tok.text);
    } else {
      // phrase / substring keep the literal (trimmed) text.
      usableText = tok.text.trim();
    }
    if (!usableText) continue;

    const term: ParsedTerm = { text: usableText, branch };

    if (tok.op === '+') required.push(term);
    else if (tok.op === '-') excluded.push(term);
    else positive.push(term);
  }

  const parsed: ParsedQuery = { raw: raw ?? '', positive, required, excluded, mode };

  if (positive.length === 0) {
    // Required terms with no positive recall still form a valid positive set (the
    // required predicates ARE the recall). Only when there is neither a positive
    // nor a required term is the query empty / only-negation.
    if (required.length === 0) {
      parsed.reason = excluded.length > 0 ? 'only-negation' : 'empty';
    }
  }

  return parsed;
}

/**
 * Does this parsed query have any positive recall to run? False means we must
 * short-circuit to an empty result (with `reason`), never a costly NOT-only scan.
 */
export function hasPositiveRecall(parsed: ParsedQuery): boolean {
  return parsed.positive.length > 0 || parsed.required.length > 0;
}