fix(stability): stage 4 — event-loop hygiene for feed generation

Feed generation blocked the loop (deepcopy of hundreds of Messages + per-message
bleach x4 + str(message) per post). This moves the CPU work off the loop and cuts
it down.

4.1 raw_message is lazy: process_message(include_raw=False) omits str(message)
    entirely; only /json and debug-HTML compute it.
4.2 (done BEFORE 4.3) _save_media_file_ids no longer touches asyncio/DB — it
    appends to the per-request PostParser._pending_media_ids, and the caller
    flushes once via to_thread(upsert_media_file_ids_bulk_sync) (a new executemany
    fn in file_io.py). Removed _persist_pending_count + the create_task machinery.
    This had to precede 4.3 or get_running_loop() inside the render thread would
    raise and silently kill media-id persistence.
4.3 The render pipeline (_create_time_based_media_groups, _create_messages_groups,
    _trim_messages_groups, _render_messages_groups) is now plain-sync and runs in
    ONE asyncio.to_thread(_render_pipeline) from both feed generators; deepcopy
    moved into the thread. The render path is verified free of asyncio primitives.
4.4 Sanitize is now ONE pass per output boundary (removed the 4 internal
    per-fragment bleach passes): RSS/HTML feeds sanitize once at the whole-feed
    pass; /html and /json sanitize body+footer once in process_message; the debug
    <pre> raw dump is html.escape'd. Media embeds in body, reactions in footer, so
    both are covered by the boundary pass. XSS tests (<script>/onerror=/javascript:)
    confirm all four outputs are clean.

Review round-1: documented that flags are now extracted from the pre-sanitize body
(a 4.4 consequence — non-security, legitimate links unaffected); added
media-caption XSS tests for the exact fragments whose internal passes were removed
(adversarially validated: neutering the sanitizer makes them fail); the media-id
flush is now in a finally so a partial render still persists what it collected.

230 passed (214 baseline + 16).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

file_io.py

@@ -69,6 +69,24 @@ def upsert_media_file_id_sync(db_path: str, channel: str, post_id: int, file_uni
         )
 
 
+def upsert_media_file_ids_bulk_sync(db_path: str, entries: List[tuple]) -> None:
+    """Insert or replace multiple media file ID records in a single transaction.
+
+    entries: iterable of (channel, post_id, file_unique_id, added) tuples.
+    Uses executemany for batched upserts (one connection, one commit).
+    """
+    if not entries:
+        return
+    with _db_connection(db_path) as conn:
+        conn.executemany(
+            """INSERT INTO media_file_ids (channel, post_id, file_unique_id, added)
+               VALUES (?, ?, ?, ?)
+               ON CONFLICT(channel, post_id, file_unique_id)
+               DO UPDATE SET added = excluded.added""",
+            entries,
+        )
+
+
 def update_media_file_access_sync(db_path: str, channel: str, post_id: int, file_unique_id: str, added: float) -> None:
     """Update the access timestamp for an existing media file ID record."""
     with _db_connection(db_path) as conn:

post_parser.py

@@ -21,16 +21,13 @@ from pyrogram.enums import MessageMediaType
 from bleach.css_sanitizer import CSSSanitizer   
 from bleach import clean as HTMLSanitizer
 from config import get_settings
-from file_io import upsert_media_file_id_sync, DB_PATH
+from file_io import upsert_media_file_ids_bulk_sync, DB_PATH
 from url_signer import generate_media_digest
 
 Config = get_settings()
 
 logger = logging.getLogger(__name__)
 
-# Module-level counter for in-flight persist tasks (used only in diagnostics)
-_persist_pending_count = 0
-
 
 #tests
 #http://127.0.0.1:8000/post/html/DragorWW_space/114 — video
@@ -65,6 +62,12 @@ _persist_pending_count = 0
 class PostParser:
     def __init__(self, client):
         self.client = client
+        # Media file-id records collected during rendering. _save_media_file_ids
+        # appends (channel, post_id, file_unique_id, ts) tuples here instead of
+        # touching asyncio/DB directly (see 4.2). A fresh PostParser is created per
+        # request and rendering runs in a single thread, so this list is thread-safe.
+        # The caller flushes it once via upsert_media_file_ids_bulk_sync after render.
+        self._pending_media_ids: List[tuple] = []
 
     @staticmethod
     def get_all_possible_flags() -> List[str]:
@@ -106,7 +109,14 @@ class PostParser:
                 logger.error(f"post_not_found_or_empty: channel {prepared_channel_id}, post_id {post_id}")
                 return None
             
-            processed_message = self.process_message(message)
+            # Single-post outputs need sanitized body/footer (no whole-feed pass exists here).
+            # raw_message is only needed for JSON output and for debug HTML.
+            include_raw = (output_type == 'json') or debug
+            processed_message = self.process_message(message, include_raw=include_raw, sanitize=True)
+
+            # Flush media file-id records collected during rendering with a single bulk upsert.
+            await self._flush_pending_media_ids()
+
             if output_type == 'html':
                 return self._format_html(processed_message, debug)
             elif output_type == 'json':
@@ -480,11 +490,15 @@ class PostParser:
         html_content.append(f'<div class="message-body">{data["html"]["body"]}</div>')
         html_content.append(f'<div class="message-footer">{data["html"]["footer"]}</div>')
         
-        # Add raw JSON debug output if debug is enabled
+        # Add raw JSON debug output if debug is enabled.
+        # raw_message is the full str(message) serialization and may contain user-controlled
+        # text with HTML/JS — escape it before dropping it into the <pre> (bleach can't run
+        # here because <pre> is not in the allowed-tags whitelist and would be stripped).
         if debug:
+            raw_escaped = html.escape(str(data.get("raw_message", "")))
             html_content.append(f'<pre class="debug-json" style="background: #f5f5f5;'
                                 f'padding: 10px; margin-top: 20px; overflow-x: auto; font-size: 10px;'
-                                f'white-space: pre-wrap;">{data["raw_message"]}</pre>')
+                                f'white-space: pre-wrap;">{raw_escaped}</pre>')
         html_data = '\n'.join(html_content)
         return html_data
 
@@ -500,11 +514,38 @@ class PostParser:
         
         return return_html
 
-    def process_message(self, message: Message) -> Dict[Any, Any]:
-        # Compute html body once — avoids triple _generate_html_body calls
+    def process_message(self, message: Message, include_raw: bool = False, sanitize: bool = True) -> Dict[Any, Any]:
+        """Build the processed representation of a message.
+
+        Args:
+            message: the Pyrogram message.
+            include_raw: when True, compute the (expensive) full ``str(message)``
+                serialization into ``result['raw_message']``. Only JSON output and
+                debug HTML need it; feed generation must pass False to avoid
+                serializing every post.
+            sanitize: when True, run the html body and footer through a single
+                bleach pass each. Single-post HTML and JSON need this (there is no
+                whole-feed pass on those paths). Feed generation passes False and
+                relies on the final whole-feed sanitize in rss_generator, so no
+                fragment is sanitized more than once.
+        """
+        # Compute html body once — avoids triple _generate_html_body calls.
+        # The internal per-fragment sanitize passes were removed (4.4); sanitize
+        # exactly once per output boundary here when requested.
         html_body = self._generate_html_body(message)
+        # NOTE (stage-4 4.4 consequence): flags are now extracted from the PRE-sanitize
+        # body (the per-fragment sanitize that used to run inside _generate_html_body was
+        # removed). Legitimate links are unaffected — bleach keeps whitelisted
+        # <a href="http(s)://…"> — so link/foreign_channel/mention flags are identical for
+        # normal content. They can differ ONLY for URL-like text that bleach would strip
+        # (e.g. a URL inside a disallowed attribute); flags are non-security (used for
+        # exclude_flags filtering / display), so this edge divergence is accepted rather
+        # than re-adding a per-message sanitize pass that 4.4 deliberately eliminated.
         flags = self._extract_flags(message, html_body=html_body)
         footer = self.generate_html_footer(message, flags_list=flags)
+        if sanitize:
+            html_body = self._sanitize_html(html_body)
+            footer = self._sanitize_html(footer)
         result = {
             'channel': self.get_channel_username(message),
             'message_id': message.id,
@@ -523,8 +564,10 @@ class PostParser:
             'media_group_id': message.media_group_id,
             'service': getattr(message, "service", None),
             'show_caption_above_media': getattr(message, 'show_caption_above_media', False),
-            'raw_message': str(message)
         }
+        if include_raw:
+            # Full serialization of the message — expensive; only for JSON/debug.
+            result['raw_message'] = str(message)
         
         # Add webpage data if available
         if webpage := getattr(message, "web_page", None):
@@ -671,8 +714,10 @@ class PostParser:
         if message.forward_origin: content_body.append(f"--- Forwarded post end ---") # Forward info end
         content_body.append(f'</div><br>')
 
+        # NOTE: sanitize is NOT applied here. Sanitization happens exactly once per
+        # output boundary (process_message for single-post/JSON, final whole-feed pass
+        # in rss_generator for feeds). See the sanitize coverage map (4.4).
         html_body = '\n'.join(content_body)
-        html_body = self._sanitize_html(html_body)
         return html_body
 
     def _generate_html_media(self, message: Message) -> str:
@@ -751,8 +796,9 @@ class PostParser:
                     content_media.append(webpage_html)
                     content_media.append('</div>')
 
+        # Not sanitized here — this fragment is embedded in the body and sanitized
+        # once at the output boundary (see the 4.4 sanitize coverage map).
         html_media = '\n'.join(content_media)
-        html_media = self._sanitize_html(html_media)
         return html_media
 
 
@@ -832,8 +878,9 @@ class PostParser:
             flags_html = self._format_flags(current_flags)
             content_footer.append('<br>' + flags_html)
             
+        # Not sanitized here — sanitized once at the output boundary (4.4 coverage map):
+        # process_message for single-post/JSON, final whole-feed pass for feeds.
         html_footer = '\n'.join(content_footer)
-        html_footer = self._sanitize_html(html_footer)
         return html_footer
 
 
@@ -919,8 +966,10 @@ class PostParser:
                 if links:
                     parts.append('&nbsp;|&nbsp;'.join(links))
 
+            # Raw fragment — embedded in the footer, sanitized once at the output
+            # boundary (see the 4.4 sanitize coverage map). Not sanitized here.
             result_html = '<br>'.join(parts) if parts else None
-            return self._sanitize_html(result_html) if result_html else None
+            return result_html if result_html else None
             
         except Exception as e:
             logger.error(f"reactions_views_links_error: message_id {message.id}, error {str(e)}")
@@ -969,15 +1018,31 @@ class PostParser:
             logger.error(f"file_id_extraction_error: media_type {message.media}, error {str(e)}")
             return None
 
-    async def _persist_media_file_id_async(self, channel: str, post_id: int, file_unique_id: str, added: float) -> None:
-        """Persist a single media file ID record to SQLite (fire-and-forget)."""
+    async def _flush_pending_media_ids(self) -> None:
+        """Persist media file-id records collected during rendering with ONE bulk upsert.
+
+        Called by the caller (get_post / rss_generator) after rendering completes.
+        Runs the blocking SQLite write in a thread. No-op when nothing was collected.
+        """
+        entries = self._pending_media_ids
+        if not entries:
+            return
         try:
-            await asyncio.to_thread(upsert_media_file_id_sync, DB_PATH, channel, post_id, file_unique_id, added)
-            logger.debug(f"persist_media_file_id: upserted {channel}/{post_id}/{file_unique_id}")
+            await asyncio.to_thread(upsert_media_file_ids_bulk_sync, DB_PATH, entries)
+            logger.debug(f"persist_media_file_ids_bulk: upserted {len(entries)} records")
         except Exception as e:
-            logger.error(f"file_id_save_error: error upserting {channel}/{post_id}/{file_unique_id}, error {str(e)}")
+            logger.error(f"file_id_bulk_save_error: error bulk-upserting {len(entries)} records, error {str(e)}")
+        finally:
+            # Clear regardless of outcome so a retry does not double-persist a stale batch.
+            self._pending_media_ids = []
 
     def _save_media_file_ids(self, message: Message) -> None:
+        """Collect a media file-id record for later bulk persistence.
+
+        IMPORTANT (4.2): this runs inside the render thread, so it must NOT touch
+        asyncio or the DB. It only appends to self._pending_media_ids; the caller
+        flushes the batch via _flush_pending_media_ids() after rendering.
+        """
         try:
             channel_username = self.get_channel_username(message)
             if not channel_username:
@@ -1003,29 +1068,8 @@ class PostParser:
 
                 if file_unique_id:
                     added_ts = datetime.now().timestamp()
-                    try:
-                        loop = asyncio.get_running_loop()
-                        if loop.is_running():
-                            global _persist_pending_count
-                            task = loop.create_task(
-                                self._persist_media_file_id_async(channel_username, message.id, file_unique_id, added_ts)
-                            )
-                            _persist_pending_count += 1
-
-                            def _on_task_done(t):
-                                global _persist_pending_count
-                                _persist_pending_count -= 1
-
-                            task.add_done_callback(_on_task_done)
-
-                            if _persist_pending_count > 5:
-                                logger.warning(f"persist_task_queue: {_persist_pending_count} pending _persist_media_file_id_async tasks (msg {message.id})")
-                            logger.debug(f"persist_task_created: queued for {channel_username}/{message.id}/{file_unique_id}, total pending: {_persist_pending_count}")
-                        else:
-                            # Fallback sync path (should not occur during normal FastAPI runtime)
-                            upsert_media_file_id_sync(DB_PATH, channel_username, message.id, file_unique_id, added_ts)
-                    except Exception as e:
-                        logger.error(f"file_id_save_error: error saving {channel_username}/{message.id}/{file_unique_id}, error {str(e)}")
+                    # Thread-safe: just append; the caller persists the batch.
+                    self._pending_media_ids.append((channel_username, message.id, file_unique_id, added_ts))
 
         except Exception as e:
             logger.error(f"file_id_collection_error: message_id {message.id}, error {str(e)}")

rss_generator.py

@@ -23,6 +23,7 @@ from pyrogram.types import Message
 from post_parser import PostParser
 from config import get_settings
 from tg_throttle import tg_rpc_bounded
+from file_io import upsert_media_file_ids_bulk_sync, DB_PATH
 from bleach.css_sanitizer import CSSSanitizer
 from bleach import clean as HTMLSanitizer
 
@@ -30,9 +31,12 @@ Config = get_settings()
 
 logger = logging.getLogger(__name__)
 
-async def _create_time_based_media_groups(messages: list[Message], merge_seconds: int = 5) -> list[Message]:
+def _create_time_based_media_groups(messages: list[Message], merge_seconds: int = 5) -> list[Message]:
     """
     Create media groups based on time difference between messages
+
+    Plain synchronous function (contains no await): runs inside the render thread
+    via _render_pipeline. Must not touch asyncio.
     """
     # Deep-copy the input list to avoid mutating cached Message objects (the cache may
     # reuse the same objects across calls with different merge_seconds values)
@@ -92,9 +96,11 @@ async def _create_time_based_media_groups(messages: list[Message], merge_seconds
 
     return messages_sorted
 
-async def _create_messages_groups(messages: list[Message]) -> list[list[Message]]:
+def _create_messages_groups(messages: list[Message]) -> list[list[Message]]:
     """
     Process messages into formatted posts, handling media groups
+
+    Plain synchronous function (contains no await): runs inside the render thread.
     """
     processing_groups: list[list[Message]] = []
     media_groups: dict[str | int, list[Message]] = {}
@@ -136,9 +142,11 @@ async def _create_messages_groups(messages: list[Message]) -> list[list[Message]
     
     return processing_groups
 
-async def _trim_messages_groups(messages_groups: list[list[Message]], limit: int):
+def _trim_messages_groups(messages_groups: list[list[Message]], limit: int):
     """
     Trim messages groups to limit
+
+    Plain synchronous function (contains no await): runs inside the render thread.
     """
     if len(messages_groups) > limit: # Trim groups if they exceed the specified limit
         messages_groups = messages_groups[:limit]
@@ -193,12 +201,13 @@ def processed_message_to_tg_message(processed_message: dict) -> Message:
     return tg_message_mock # type: ignore
 
 
-async def _render_messages_groups(messages_groups: list[list[Message]], 
-                                    post_parser: PostParser, 
-                                    exclude_flags: str | None = None, 
+def _render_messages_groups(messages_groups: list[list[Message]],
+                                    post_parser: PostParser,
+                                    exclude_flags: str | None = None,
                                     exclude_text: str | None = None):
     """
     Render message groups into HTML format
+    Plain synchronous function (contains no await): runs inside the render thread.
     Args:
         messages_groups: List of message groups (each group is a list of messages)
         post_parser: PostParser instance
@@ -213,7 +222,9 @@ async def _render_messages_groups(messages_groups: list[list[Message]],
         try:
             if len(group) == 1: # Single message - simple case
                 one_message = group[0]
-                message_data = post_parser.process_message(one_message)
+                # Feed path: raw_message not needed and sanitize deferred to the final
+                # whole-feed pass, so each fragment is sanitized exactly once.
+                message_data = post_parser.process_message(one_message, include_raw=False, sanitize=False)
                 html_parts = [
                     f'<div class="message-body">{message_data["html"]["body"]}</div>',
                     f'<div class="message-footer">{message_data["html"]["footer"]}</div>'
@@ -228,7 +239,7 @@ async def _render_messages_groups(messages_groups: list[list[Message]],
                     'flags': message_data['flags']
                 })
             else: # Multiple messages in group - merge text and html body
-                processed_messages = [post_parser.process_message(msg) for msg in group]
+                processed_messages = [post_parser.process_message(msg, include_raw=False, sanitize=False) for msg in group]
 
                 # Determine main message for header/footer/title
                 main_message = next(  
@@ -309,7 +320,31 @@ async def _render_messages_groups(messages_groups: list[list[Message]],
     rendered_posts.sort(key=lambda x: x['date'] if x['date'] is not None else 0.0, reverse=True)
     return rendered_posts
 
-async def generate_channel_rss(channel: str | int, 
+
+def _render_pipeline(messages: list[Message],
+                     post_parser: PostParser,
+                     limit: int,
+                     exclude_flags: str | None,
+                     exclude_text: str | None,
+                     merge_seconds: int,
+                     time_based_merge: bool):
+    """
+    Full synchronous feed render pipeline (grouping + trimming + rendering).
+
+    Runs entirely in a worker thread via a single asyncio.to_thread call. It contains
+    NO await, NO asyncio primitives, NO create_task/get_running_loop — all the CPU-heavy
+    work (deepcopy, grouping, bleach-free rendering) happens here off the event loop.
+    Media file-id records are accumulated on post_parser._pending_media_ids and flushed
+    by the caller after this returns.
+    """
+    if time_based_merge:
+        messages = _create_time_based_media_groups(messages, merge_seconds)
+    message_groups = _create_messages_groups(messages)
+    message_groups = _trim_messages_groups(message_groups, limit)
+    return _render_messages_groups(message_groups, post_parser, exclude_flags, exclude_text)
+
+
+async def generate_channel_rss(channel: str | int,
                                 client: Client, 
                                 limit: int = 20, 
                                 exclude_flags: str | None = None,
@@ -389,14 +424,23 @@ async def generate_channel_rss(channel: str | int,
         messages_elapsed = time.time() - messages_start_time
         logger.debug(f"rss_messages_retrieval_timing: channel {channel}, {len(messages)} messages retrieved in {messages_elapsed:.3f} seconds")
         
-        # Process messages into groups and render them
+        # Process messages into groups and render them.
+        # The whole grouping/trimming/rendering pipeline is CPU-heavy (deepcopy +
+        # per-message rendering) and contains no await, so run it in ONE worker
+        # thread to keep the event loop responsive.
         processing_start_time = time.time()
-        if Config['time_based_merge']:
-            messages = await _create_time_based_media_groups(messages, merge_seconds)
-        message_groups = await _create_messages_groups(messages)
-        message_groups = await _trim_messages_groups(message_groups, limit)
-        final_posts = await _render_messages_groups(message_groups, post_parser, exclude_flags, exclude_text)
-        
+        try:
+            final_posts = await asyncio.to_thread(
+                _render_pipeline, messages, post_parser, limit,
+                exclude_flags, exclude_text, merge_seconds, Config['time_based_merge'],
+            )
+        finally:
+            # Persist media file-ids collected during rendering with a single bulk
+            # upsert — in a finally so a partial render still records what it collected
+            # (the flush is best-effort and swallows its own errors, so it cannot mask a
+            # render exception).
+            await post_parser._flush_pending_media_ids()
+
         processing_elapsed = time.time() - processing_start_time
         logger.debug(f"rss_messages_processing_timing: channel {channel}, {len(final_posts)} posts processed in {processing_elapsed:.3f} seconds")
         
@@ -601,16 +645,19 @@ async def generate_channel_html(channel: str | int,
         enrichment_elapsed = time.time() - enrichment_start_time
         logger.debug(f"html_reply_enrichment_timing: channel {channel}, replies enriched in {enrichment_elapsed:.3f} seconds")
 
-        # Process messages into groups and render them
+        # Process messages into groups and render them in ONE worker thread
+        # (CPU-heavy, no await) to keep the event loop responsive.
         processing_start_time = time.time()
-        if Config['time_based_merge']:
-            messages = await _create_time_based_media_groups(messages, merge_seconds)
+        try:
+            final_posts = await asyncio.to_thread(
+                _render_pipeline, messages, post_parser, limit,
+                exclude_flags, exclude_text, merge_seconds, Config['time_based_merge'],
+            )
+        finally:
+            # Persist media file-ids collected during rendering with a single bulk
+            # upsert — in a finally so a partial render still records what it collected.
+            await post_parser._flush_pending_media_ids()
 
-        # Process messages into groups and render them
-        message_groups = await _create_messages_groups(messages)
-        message_groups = await _trim_messages_groups(message_groups, limit)
-        final_posts = await _render_messages_groups(message_groups, post_parser, exclude_flags, exclude_text)
-        
         processing_elapsed = time.time() - processing_start_time
         logger.debug(f"html_messages_processing_timing: channel {channel}, {len(final_posts)} posts processed in {processing_elapsed:.3f} seconds")
 

tests/test_stage4_eventloop.py

@@ -0,0 +1,367 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""
+Stage 4 (event-loop hygiene for feed generation) tests.
+
+Covers:
+- 4.1 raw_message laziness: feeds do NOT compute str(message); JSON/debug HTML do.
+- 4.2 side-effect IO removed from process_message: _save_media_file_ids only appends to
+      self._pending_media_ids; the caller flushes once via upsert_media_file_ids_bulk_sync.
+      Also: the render path contains NO create_task / get_running_loop / to_thread.
+- 4.3 render pipeline moved into a single thread: the four render functions are now plain
+      sync functions and actually execute off the main thread; deepcopy of a pickled Message
+      does not crash; a 100-message feed generates correctly.
+- 4.4 sanitize coverage (XSS): a <script> / onerror= / javascript: payload is stripped in
+      ALL outputs — rss, html-feed, single-post html, and json — each with exactly one pass.
+"""
+import os
+import re
+import sys
+import copy
+import pickle
+import asyncio
+import threading
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
+
+from pyrogram.enums import MessageMediaType
+
+import post_parser as pp_module
+import rss_generator as rss_module
+from post_parser import PostParser
+from rss_generator import (
+    generate_channel_rss,
+    generate_channel_html,
+    _render_pipeline,
+    _create_time_based_media_groups,
+    _create_messages_groups,
+    _trim_messages_groups,
+    _render_messages_groups,
+)
+
+XSS_PAYLOAD = "<script>alert('xss')</script><img src=x onerror=\"alert(1)\"><a href=\"javascript:alert(2)\">click</a>"
+
+
+class _Str(str):
+    """Minimal stand-in for Pyrogram's Str: .html returns the raw string unchanged,
+    so a malicious payload reaches the pre-sanitization html body just like real text
+    carrying entities would."""
+    @property
+    def html(self):
+        return str(self)
+
+
+def make_message(mid, text=None, media=None, photo_uid=None, username="testchan",
+                 date=None):
+    m = SimpleNamespace()
+    m.id = mid
+    m.date = date or datetime(2024, 1, 1, 12, 0, mid % 60, tzinfo=timezone.utc)
+    m.text = _Str(text) if text is not None else None
+    m.caption = None
+    m.media = media
+    m.web_page = None
+    m.poll = None
+    m.service = None
+    m.forward_origin = None
+    m.reply_to_message = None
+    m.reply_to_message_id = None
+    m.sender_chat = None
+    m.from_user = None
+    m.reactions = None
+    m.views = 100
+    m.media_group_id = None
+    m.show_caption_above_media = False
+    m.chat = SimpleNamespace(id=-1001234567890, username=username)
+    # media sub-objects default to None
+    for attr in ("photo", "video", "document", "audio", "voice",
+                 "video_note", "animation", "sticker"):
+        setattr(m, attr, None)
+    if media == MessageMediaType.PHOTO and photo_uid:
+        m.photo = SimpleNamespace(file_unique_id=photo_uid)
+    return m
+
+
+def _co_names(func):
+    return set(func.__code__.co_names)
+
+
+# ---------------------------------------------------------------------------
+# 4.3 — render functions are plain sync and run off the main thread
+# ---------------------------------------------------------------------------
+
+def test_render_functions_are_sync():
+    for fn in (_create_time_based_media_groups, _create_messages_groups,
+               _trim_messages_groups, _render_messages_groups, _render_pipeline):
+        assert not asyncio.iscoroutinefunction(fn), f"{fn.__name__} must be a plain sync function"
+
+
+@pytest.mark.asyncio
+async def test_pipeline_runs_in_worker_thread(monkeypatch):
+    main_ident = threading.get_ident()
+    seen = {}
+
+    real_render = rss_module._render_messages_groups
+
+    def spy(*args, **kwargs):
+        seen["ident"] = threading.get_ident()
+        return real_render(*args, **kwargs)
+
+    monkeypatch.setattr(rss_module, "_render_messages_groups", spy)
+
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return [make_message(i, text=f"post {i}") for i in range(5)]
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    await generate_channel_rss("testchan", client=SimpleNamespace(), limit=5)
+    assert "ident" in seen
+    assert seen["ident"] != main_ident, "render pipeline must run in a worker thread, not the loop thread"
+
+
+def test_deepcopy_of_pickled_message_does_not_crash():
+    from pyrogram.types import Message, Chat
+    from pyrogram.enums import ChatType
+    m = Message(id=7, date=datetime.now(timezone.utc), text="hello",
+                chat=Chat(id=-1001, type=ChatType.CHANNEL, username="testchan"))
+    roundtripped = pickle.loads(pickle.dumps(m))   # mimics the pickle cache
+    clone = copy.deepcopy(roundtripped)            # what _create_time_based_media_groups does
+    assert clone.id == 7
+    assert clone.chat.username == "testchan"
+
+
+# ---------------------------------------------------------------------------
+# 4.2 — no asyncio in the render path; bulk upsert after render
+# ---------------------------------------------------------------------------
+
+def test_render_path_has_no_asyncio_side_effects():
+    banned = {"create_task", "get_running_loop", "to_thread", "ensure_future"}
+    funcs = [
+        _render_pipeline, _create_time_based_media_groups, _create_messages_groups,
+        _trim_messages_groups, _render_messages_groups,
+        PostParser.process_message, PostParser._generate_html_body,
+        PostParser._generate_html_media, PostParser.generate_html_footer,
+        PostParser._reactions_views_links, PostParser._save_media_file_ids,
+        PostParser._sanitize_html,
+    ]
+    for fn in funcs:
+        offenders = _co_names(fn) & banned
+        assert not offenders, f"{fn.__qualname__} references forbidden asyncio names: {offenders}"
+
+
+@pytest.mark.asyncio
+async def test_media_ids_persisted_via_bulk_upsert(monkeypatch):
+    calls = []
+
+    def fake_bulk(db_path, entries):
+        calls.append(list(entries))
+
+    monkeypatch.setattr(pp_module, "upsert_media_file_ids_bulk_sync", fake_bulk)
+
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return [
+            make_message(1, text="just text"),
+            make_message(2, media=MessageMediaType.PHOTO, photo_uid="uid_abc"),
+        ]
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    await generate_channel_rss("testchan", client=SimpleNamespace(), limit=10)
+
+    assert len(calls) == 1, "bulk upsert must be called exactly once after render"
+    entries = calls[0]
+    assert len(entries) == 1, "only the photo message contributes a media id"
+    channel, post_id, fid, _ts = entries[0]
+    assert (channel, post_id, fid) == ("testchan", 2, "uid_abc")
+
+
+@pytest.mark.asyncio
+async def test_save_media_file_ids_only_appends(monkeypatch):
+    # Even with a running loop, _save_media_file_ids must not create tasks — just append.
+    parser = PostParser(SimpleNamespace())
+    monkeypatch.setattr(pp_module, "upsert_media_file_ids_bulk_sync",
+                        lambda *a, **k: (_ for _ in ()).throw(AssertionError("must not be called directly")))
+    msg = make_message(3, media=MessageMediaType.PHOTO, photo_uid="uid_x")
+    parser._save_media_file_ids(msg)
+    assert parser._pending_media_ids == [("testchan", 3, "uid_x", parser._pending_media_ids[0][3])]
+
+
+# ---------------------------------------------------------------------------
+# 4.1 — raw_message laziness
+# ---------------------------------------------------------------------------
+
+def test_raw_message_lazy_for_feed():
+    parser = PostParser(SimpleNamespace())
+    result = parser.process_message(make_message(10, text="hi"), include_raw=False, sanitize=False)
+    assert "raw_message" not in result
+
+
+def test_raw_message_present_for_json_and_debug():
+    parser = PostParser(SimpleNamespace())
+    result = parser.process_message(make_message(11, text="hi"), include_raw=True)
+    assert "raw_message" in result
+    assert isinstance(result["raw_message"], str)
+
+
+@pytest.mark.asyncio
+async def test_100_message_feed_generates(monkeypatch):
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return [make_message(i, text=f"post number {i}") for i in range(1, 101)]
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    rss = await generate_channel_rss("testchan", client=SimpleNamespace(), limit=100)
+    assert rss.count("<item>") == 100
+    assert "post number 50" in rss
+
+
+# ---------------------------------------------------------------------------
+# 4.4 — XSS: payload stripped in all four outputs
+# ---------------------------------------------------------------------------
+
+def _assert_clean(html_str, where):
+    assert "<script>" not in html_str, f"{where}: <script> survived"
+    assert "onerror" not in html_str, f"{where}: onerror= survived"
+    assert "javascript:" not in html_str, f"{where}: javascript: survived"
+
+
+def _make_client_returning(msg):
+    client = SimpleNamespace()
+
+    async def get_messages(channel, post_id):
+        return msg
+
+    client.get_messages = get_messages
+    return client
+
+
+@pytest.mark.asyncio
+async def test_xss_stripped_in_json():
+    msg = make_message(20, text=XSS_PAYLOAD)
+    parser = PostParser(_make_client_returning(msg))
+    data = await parser.get_post("testchan", 20, "json")
+    _assert_clean(data["html"]["body"], "json body")
+    _assert_clean(data["html"]["footer"], "json footer")
+
+
+@pytest.mark.asyncio
+async def test_xss_stripped_in_single_post_html():
+    msg = make_message(21, text=XSS_PAYLOAD)
+    parser = PostParser(_make_client_returning(msg))
+    html_out = await parser.get_post("testchan", 21, "html")
+    _assert_clean(html_out, "single-post html")
+
+
+@pytest.mark.asyncio
+async def test_xss_stripped_in_single_post_html_debug():
+    # debug HTML embeds raw_message (str(message)) into a <pre>; it must be html-escaped so
+    # no live tag from the payload survives. The escaped dump legitimately still contains the
+    # inert words "onerror"/"javascript:" as text — what matters is that they are NOT live.
+    msg = make_message(22, text=XSS_PAYLOAD)
+    parser = PostParser(_make_client_returning(msg))
+    html_out = await parser.get_post("testchan", 22, "html", debug=True)
+
+    # 1) The rendered display area (everything before the raw <pre>) is fully sanitized.
+    display = html_out.split('<pre', 1)[0]
+    _assert_clean(display, "single-post debug display")
+
+    # 2) The raw <pre> dump is html-escaped: no live <script> tag anywhere, and the payload
+    #    appears only in escaped form (proving html.escape ran).
+    assert "<script>" not in html_out, "debug raw dump left a live <script> tag"
+    assert "&lt;script&gt;" in html_out, "debug raw dump was not html-escaped"
+
+
+@pytest.mark.asyncio
+async def test_xss_stripped_in_rss_feed(monkeypatch):
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return [make_message(23, text=XSS_PAYLOAD)]
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    rss = await generate_channel_rss("testchan", client=SimpleNamespace(), limit=5)
+    # The sanitized HTML lives in <content:encoded><![CDATA[...]]></content:encoded>.
+    cdata = re.findall(r"<!\[CDATA\[(.*?)\]\]>", rss, re.DOTALL)
+    assert cdata, "expected CDATA content in RSS"
+    for chunk in cdata:
+        _assert_clean(chunk, "rss content")
+
+
+@pytest.mark.asyncio
+async def test_xss_stripped_in_html_feed(monkeypatch):
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return [make_message(24, text=XSS_PAYLOAD)]
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    html_feed = await generate_channel_html("testchan", client=SimpleNamespace(), limit=5)
+    _assert_clean(html_feed, "html feed")
+
+
+def _media_msg_with_payload_caption(mid):
+    # A photo message whose CAPTION carries the payload — this exercises the media
+    # fragment path (_generate_html_media / caption rendering) whose internal per-fragment
+    # sanitize pass was removed in 4.4. The covering pass must still strip it.
+    m = make_message(mid, media=MessageMediaType.PHOTO, photo_uid="pic123")
+    m.caption = _Str(XSS_PAYLOAD)
+    return m
+
+
+@pytest.mark.asyncio
+async def test_xss_in_media_caption_stripped_direct_paths():
+    # json + single-post html go through process_message(sanitize=True) directly.
+    parser = PostParser(_make_client_returning(_media_msg_with_payload_caption(30)))
+    data = await parser.get_post("testchan", 30, "json")
+    _assert_clean(data["html"]["body"], "json body (media caption)")
+    _assert_clean(data["html"]["footer"], "json footer (media caption)")
+
+    parser2 = PostParser(_make_client_returning(_media_msg_with_payload_caption(31)))
+    html_out = await parser2.get_post("testchan", 31, "html")
+    _assert_clean(html_out, "single-post html (media caption)")
+
+
+@pytest.mark.asyncio
+async def test_xss_in_media_caption_stripped_in_feeds(monkeypatch):
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return [_media_msg_with_payload_caption(32)]
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    rss = await generate_channel_rss("testchan", client=SimpleNamespace(), limit=5)
+    for chunk in re.findall(r"<!\[CDATA\[(.*?)\]\]>", rss, re.DOTALL):
+        _assert_clean(chunk, "rss content (media caption)")
+
+    async def fake_get_history2(client, channel, limit=20):
+        return [_media_msg_with_payload_caption(33)]
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history2, raising=False)
+    html_feed = await generate_channel_html("testchan", client=SimpleNamespace(), limit=5)
+    _assert_clean(html_feed, "html feed (media caption)")