test(media): покрыть backoff-кап + best-effort restart-колбэк (ревью #48)

- test_backoff_capped_at_max: 8 фейлов подряд (сырая экспонента 7680с) ->
  _download_backoff_remaining <= _DOWNLOAD_BACKOFF_MAX (600с). Мутация: убрать
  min()-кап -> тест краснеет (7680>600).
- test_restart_callback_error_does_not_abort: бросающий restart-колбэк на
  verified-рестарте -> recovery завершается (watchdog пере-армлен, _restarting
  сброшен, SIGTERM-fallback не сработал), ошибка залогирована. Мутация: убрать
  try/except вокруг колбэка -> исключение уходит наружу -> тест краснеет.

Только тесты; прод-код байт-в-байт не тронут.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

api_server.py

@@ -17,6 +17,7 @@ import mimetypes
 from typing import List, Union, Any
 
 import json
+from collections import OrderedDict
 from datetime import datetime
 import time
 from contextlib import asynccontextmanager
@@ -43,10 +44,35 @@ from file_io import (DB_PATH, init_db_sync, get_all_media_file_ids_sync,
                      update_media_file_access_sync, update_media_file_access_bulk_sync,
                      remove_media_file_ids_sync,
                      get_mime_type_sync, set_mime_type_sync)
+from tg_cache import cleanup_legacy_cache_files, sweep_tgcache
+from channel_key import canonical_channel_key
+from migrate_channel_keys import migrate_channel_keys_sync
 
 # Global python-magic instance for MIME type detection
 magic_mime = magic.Magic(mime=True)
 
+# Media cache on-disk layout. Resolved once at import so every producer/consumer of the
+# cache agrees on the exact same absolute path format instead of re-deriving it inline.
+MEDIA_CACHE_DIR = os.path.abspath(os.path.join("data", "cache"))  # resolved once at import
+
+
+def media_cache_path(channel: str, post_id: int, file_unique_id: str | None = None) -> str:
+    """Single source of truth for the on-disk layout: <root>/<channel>/<post_id>[/<fid>].
+
+    Callers that need only the per-post directory omit ``file_unique_id``; passing it yields
+    the full per-file path. ``channel`` is stringified by the caller where it may be an int.
+    """
+    path = os.path.join(MEDIA_CACHE_DIR, str(channel), str(post_id))
+    if file_unique_id is not None:
+        path = os.path.join(path, file_unique_id)
+    return path
+
+
+# MIME types are immutable per file_unique_id, so a process-lifetime dict in front of
+# SQLite removes a to_thread + connect from every cache-hit response.
+_mime_types: dict[tuple[str, int, str], str] = {}
+_MIME_CACHE_MAX = 50_000  # crude bound; clear-all on overflow is fine at this size
+
 # Define custom exception for zero-size files
 class ZeroSizeFileError(Exception):
     """Custom exception for zero-size files found or downloaded."""
@@ -92,6 +118,75 @@ Config = get_settings()
 HTTP_DOWNLOAD_SEMAPHORE = asyncio.Semaphore(3)  # semaphore for live HTTP media requests
 BACKGROUND_DOWNLOAD_SEMAPHORE = asyncio.Semaphore(2)  # semaphore for background cache worker
 download_queue = asyncio.Queue(maxsize=100)
+# Keys currently enqueued for (or being processed by) the background download worker.
+# download_new_files re-scans the whole media table every sweep and, under a FloodWait,
+# the worker drains slowly — without this guard the SAME not-yet-downloaded file gets
+# re-enqueued on every pass, flooding the queue with duplicates. The event loop is
+# single-threaded and every check/mutation below runs with no await in between, so a plain
+# set is race-free: download_new_files adds the key before put_nowait; the worker discards
+# it in its finally alongside task_done().
+_queued_media: set[tuple[str, int, str]] = set()
+# --- Failing-download backoff (negative cache) --------------------------------
+# A media file whose download keeps failing (hang/timeout/not-found) must not be
+# retried on every 60s cache sweep, nor keep occupying a scarce Pyrogram
+# transmission slot. We remember recent failures per (channel, post_id,
+# file_unique_id) and skip / fast-reject re-attempts until an exponentially
+# growing backoff elapses. Cleared on the first successful download. All access is
+# from the single event-loop thread, so a plain dict needs no lock.
+_DOWNLOAD_BACKOFF_BASE = 60.0     # seconds; backoff after the first failure
+# Cap on the backoff. Kept deliberately short (10 min, not 1h): the media self-heal
+# restart recovers in minutes, and a verified restart clears this cache outright (see
+# _clear_all_download_failures), so a system that just healed must not hold a stale,
+# long backoff that keeps fast-503'ing a now-downloadable file.
+_DOWNLOAD_BACKOFF_MAX = 600.0     # seconds; cap on the backoff
+# LRU cap on the negative cache. Permanently-404 / deleted files would otherwise leave
+# eternal entries and slowly leak memory on a long-uptime process. We keep at most this
+# many most-recently-failed keys and evict the oldest. Eviction is harmless: a dropped
+# key is simply treated as "never failed" again — at worst one extra retry attempt, which
+# re-arms its backoff on failure. 10k keys is a tiny footprint yet far above any realistic
+# concurrent-failure working set.
+_DOWNLOAD_FAILURES_MAX = 10000
+# key -> (consecutive_failures, retry_not_before_monotonic). OrderedDict so we can evict
+# in least-recently-updated order once the LRU cap is exceeded.
+_download_failures: "OrderedDict[tuple[str, int, str], tuple[int, float]]" = OrderedDict()
+
+def _download_backoff_remaining(key: tuple[str, int, str]) -> float:
+    """Seconds until `key` may be retried; 0.0 if allowed now (or never failed)."""
+    entry = _download_failures.get(key)
+    if entry is None:
+        return 0.0
+    return max(0.0, entry[1] - time.monotonic())
+
+def _record_download_failure(key: tuple[str, int, str]) -> None:
+    """Register a failed download and (re)arm an exponential backoff for `key`."""
+    fails = _download_failures.get(key, (0, 0.0))[0] + 1
+    backoff = min(_DOWNLOAD_BACKOFF_MAX, _DOWNLOAD_BACKOFF_BASE * (2 ** (fails - 1)))
+    _download_failures[key] = (fails, time.monotonic() + backoff)
+    _download_failures.move_to_end(key)  # mark as most-recently-updated for LRU eviction
+    # Bound memory (see _DOWNLOAD_FAILURES_MAX): evict the oldest entries beyond the cap.
+    while len(_download_failures) > _DOWNLOAD_FAILURES_MAX:
+        _download_failures.popitem(last=False)
+    logger.warning(f"download_backoff_armed: {key[0]}/{key[1]}/{key[2]} failed {fails}x, next retry in {backoff:.0f}s")
+
+def _clear_download_failure(key: tuple[str, int, str]) -> None:
+    """Forget any recorded failure for `key` after a successful download."""
+    if _download_failures.pop(key, None) is not None:
+        logger.info(f"download_backoff_cleared: {key[0]}/{key[1]}/{key[2]} recovered")
+
+def _clear_all_download_failures() -> None:
+    """Drop the entire download negative cache. Registered with telegram_client and invoked
+    ONLY after a VERIFIED self-heal restart (verify_get_me OK). _restart_client() rebuilds
+    the WHOLE client — all DCs, including the media DC — so every per-file backoff is stale
+    and a previously-failing file may now download. Without this a file that reached the
+    backoff cap would keep fast-503'ing for up to _DOWNLOAD_BACKOFF_MAX after recovery, and
+    nothing would retry it (the sweeper skips backed-off keys, get_media fast-503s them), so
+    it could not self-heal until the backoff expired — defeating the self-heal's purpose.
+    The retry-storm risk if the restart did not actually help is bounded: each re-download is
+    timeout-bounded and simply re-arms its backoff."""
+    n = len(_download_failures)
+    _download_failures.clear()
+    if n:
+        logger.warning(f"download_backoff_cleared_all: dropped {n} entries after verified self-heal restart")
 # How stale a temp_* file's mtime must be before a serve refreshes it (keeps the 1h
 # sweeper from deleting an actively-viewed video). Well below 1h so the file stays alive,
 # but large enough that the mtime — and thus FileResponse's ETag — is stable within any
@@ -204,12 +299,36 @@ async def lifespan(_: FastAPI):
     io_executor = ThreadPoolExecutor(max_workers=Config["io_thread_pool_size"], thread_name_prefix="io")
     loop.set_default_executor(io_executor)
 
-    base_cache_dir = os.path.abspath("./data/cache")
-    os.makedirs(base_cache_dir, exist_ok=True) # Create cache directory
+    os.makedirs(MEDIA_CACHE_DIR, exist_ok=True) # Create cache directory
 
     # Initialize SQLite database (creates table if not present)
     await asyncio.to_thread(init_db_sync, DB_PATH)
 
+    # One-shot migration of existing cache dirs + DB rows to the canonical channel key
+    # (case-insensitive username collapse). Runs AFTER init_db_sync and BEFORE client.start()
+    # and the background tasks. Wrapped in to_thread because it is a blocking FS-rename +
+    # SQLite routine that would otherwise stall the event loop. Idempotent: a re-run is a no-op.
+    await asyncio.to_thread(migrate_channel_keys_sync, DB_PATH, MEDIA_CACHE_DIR)
+
+    # One-shot startup maintenance of the history/chatinfo cache: drop legacy pickle files
+    # (which are now always a miss), age-sweep stale tgcache entries, and remove the
+    # pre-SQLite data/media_file_ids.json legacy dump (no longer read by any code).
+    await asyncio.to_thread(cleanup_legacy_cache_files)
+    await asyncio.to_thread(sweep_tgcache)
+    legacy_media_ids = os.path.join("data", "media_file_ids.json")
+    if os.path.exists(legacy_media_ids):
+        try:
+            await asyncio.to_thread(os.remove, legacy_media_ids)
+            logger.info(f"legacy_media_file_ids_removed: {legacy_media_ids}")
+        except OSError as e:
+            logger.warning(f"legacy_media_file_ids_remove_error: {e}")
+
+    # Wire the self-heal restart -> negative-cache clear hook. telegram_client owns the
+    # restart; the negative cache lives here. Registering a plain callback (rather than
+    # importing api_server from telegram_client) keeps the dependency one-way and avoids a
+    # circular import. The hook fires ONLY on a verified restart — see _restart_client.
+    client.set_restart_callback(_clear_all_download_failures)
+
     await client.start()
     # Supervise the background tasks: if either dies (not via cancellation) it is logged
     # CRITICAL and restarted, so a crash can no longer silently stop cache sweeping or downloads.
@@ -287,9 +406,26 @@ if __name__ == "__main__":
 async def find_file_id_in_message(message: Message, file_unique_id: str) -> Union[str, None]:
     """Find file_id by checking all possible media types in message"""
     if message.media == MessageMediaType.POLL:
-        logger.debug(f"Message {message.id} is a poll, skipping media search")
+        # Kurigram 2.2.23: polls may carry media in description_media and
+        # explanation_media (MessageContent objects). The bridge renders only
+        # description_media, but explanation_media is searched too: if a signed URL
+        # for it ever exists, the download must still work. getattr-only access —
+        # older Poll objects/mocks do not define these fields.
+        poll = getattr(message, 'poll', None)
+        for container_name in ('description_media', 'explanation_media'):
+            content = getattr(poll, container_name, None) if poll else None
+            if content is None:
+                continue
+            for media_attr in ('photo', 'video', 'animation', 'sticker',
+                               'document', 'audio', 'voice', 'video_note'):
+                media_obj = getattr(content, media_attr, None)
+                if media_obj is None:
+                    continue
+                if getattr(media_obj, 'file_unique_id', None) == file_unique_id:
+                    return getattr(media_obj, 'file_id', None)
+        logger.debug(f"Message {message.id} is a poll, media '{file_unique_id}' not found in poll content")
         return None
-        
+
     media_found = []
     if message.photo:
         media_found.append(f"photo ({message.photo.file_unique_id})")
@@ -327,7 +463,21 @@ async def find_file_id_in_message(message: Message, file_unique_id: str) -> Unio
         media_found.append(f"document ({message.document.file_unique_id})")
         if message.document.file_unique_id == file_unique_id:
             return message.document.file_id
-            
+    # New media types (Kurigram 2.2.23): getattr-only access, the attributes do not
+    # exist on older Message objects/mocks.
+    if live_photo := getattr(message, 'live_photo', None):
+        media_found.append(f"live_photo ({getattr(live_photo, 'file_unique_id', None)})")
+        if getattr(live_photo, 'file_unique_id', None) == file_unique_id:
+            return getattr(live_photo, 'file_id', None)
+    if story := getattr(message, 'story', None):
+        for story_attr in ('photo', 'video'):
+            story_media = getattr(story, story_attr, None)
+            if story_media is None:
+                continue
+            media_found.append(f"story.{story_attr} ({getattr(story_media, 'file_unique_id', None)})")
+            if getattr(story_media, 'file_unique_id', None) == file_unique_id:
+                return getattr(story_media, 'file_id', None)
+
     # If we reached here, the file_unique_id was not found
     channel_id_log = message.chat.id if message.chat else 'unknown_chat'
     logger.warning(f"Could not find media with file_unique_id '{file_unique_id}' in message {message.id} (channel: {channel_id_log}). Found media: {', '.join(media_found) or 'None'}")
@@ -387,9 +537,18 @@ async def prepare_file_response(file_path: str, request: Request, delete_after:
     media_type: str | None = None
 
     if media_key is not None:
-        # Try to load the cached MIME type from the database (avoids repeated python-magic I/O)
+        # Fast path: the in-memory MIME cache. MIME is immutable per file_unique_id, so a
+        # hit here serves the response with no to_thread hop and no SQLite connection at all.
         channel_key, post_id_key, file_unique_id_key = media_key
-        media_type = await asyncio.to_thread(get_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key)
+        media_type = _mime_types.get(media_key)
+        if not media_type:
+            # Dict miss — consult the SQLite type cache (still avoids python-magic I/O).
+            media_type = await asyncio.to_thread(get_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key)
+            if media_type:
+                # Populate the dict so the next request skips the to_thread + connect.
+                if len(_mime_types) >= _MIME_CACHE_MAX:
+                    _mime_types.clear()
+                _mime_types[media_key] = media_type
 
     if not media_type:
         # Cache miss or no media_key — detect with python-magic in a thread to avoid blocking the event loop
@@ -399,9 +558,13 @@ async def prepare_file_response(file_path: str, request: Request, delete_after:
             logger.warning(f"Failed to determine MIME type using python-magic: {str(e)}")
             media_type = None
 
-        # Persist the detected MIME type so the next request can skip python-magic
+        # Persist the detected MIME type so the next request can skip python-magic — into
+        # BOTH the SQLite type cache and the in-memory dict.
         if media_type and media_key is not None:
             await asyncio.to_thread(set_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key, media_type)
+            if len(_mime_types) >= _MIME_CACHE_MAX:
+                _mime_types.clear()
+            _mime_types[media_key] = media_type
 
     if not media_type: media_type, _ = mimetypes.guess_type(file_path)  # Fallback to mimetypes if python-magic failed
     if not media_type: media_type = "application/octet-stream"  # Final fallback to octet-stream
@@ -499,9 +662,13 @@ async def _download_deduped(channel: Union[str, int], post_id: int, file_unique_
         async def _runner():
             try:
                 result = await download_media_file(channel, post_id, file_unique_id)
+                _clear_download_failure(key)
                 if not fut.done():
                     fut.set_result(result)
             except BaseException as e:  # noqa: BLE001 — must forward ANY failure to waiters
+                # Arm backoff for real failures only, never for a shutdown-time cancel.
+                if isinstance(e, Exception):
+                    _record_download_failure(key)
                 if not fut.done():
                     fut.set_exception(e)
             finally:
@@ -529,11 +696,8 @@ async def download_media_file(channel: Union[str, int], post_id: int, file_uniqu
     """
     import time as _time
     _fn_start = _time.monotonic()
-    base_cache_dir = os.path.abspath("./data/cache")
-    
     # Create nested cache structure
-    channel_dir = os.path.join(base_cache_dir, str(channel))
-    post_dir = os.path.join(channel_dir, str(post_id))
+    post_dir = media_cache_path(str(channel), post_id)
     os.makedirs(post_dir, exist_ok=True)
     
     # Convert numeric channel ID to int if needed
@@ -753,7 +917,13 @@ async def download_new_files(media_files: list, cache_dir: str) -> None:
             if not all([channel, post_id, file_unique_id]):
                 logger.error(f"Invalid file data: {file_data}")
                 continue
-                
+
+            # Skip files that recently kept failing — do not re-queue them on every 60s
+            # sweep (that retry-storm is what kept the download slots jammed). The backoff
+            # expires on its own; a successful download elsewhere clears it.
+            if _download_backoff_remaining((str(channel), int(post_id), file_unique_id)) > 0:
+                continue
+
             channel_dir = os.path.join(cache_dir, str(channel))
             post_dir = os.path.join(channel_dir, str(post_id))
             os.makedirs(post_dir, exist_ok=True)
@@ -765,14 +935,22 @@ async def download_new_files(media_files: list, cache_dir: str) -> None:
                 
             cache_path = os.path.join(post_dir, file_unique_id)
             if not os.path.exists(cache_path):
+                # Skip files already queued/in-flight so a slow-draining queue (FloodWait)
+                # isn't refilled with duplicates on every sweep. Same key shape as the
+                # worker's bg_key. No await between this check and the add -> race-free.
+                key = (str(channel), int(post_id), file_unique_id)
+                if key in _queued_media:
+                    continue
                 try:
                     # put_nowait so a full queue raises QueueFull instead of blocking
                     # cache_media_files (and thus the sweeper) forever. `await put()`
                     # never raises QueueFull, which made the except below dead code.
+                    _queued_media.add(key)
                     download_queue.put_nowait((channel, post_id, file_unique_id))
                     files_queued += 1
                     logger.debug(f"Queued for background download: {channel}/{post_id}/{file_unique_id}")
                 except asyncio.QueueFull:
+                    _queued_media.discard(key)
                     logger.warning(f"Download queue is full, skipping {channel}/{post_id}/{file_unique_id}")
                     break
         
@@ -791,51 +969,70 @@ async def background_download_worker():
         # successful get(). Cancellation propagates cleanly here (nothing to unbalance).
         item = await download_queue.get()
         channel, post_id, file_unique_id = item
+        bg_key = (str(channel), int(post_id), file_unique_id)
         logger.info(f"Background download: {channel}/{post_id}/{file_unique_id}")
         try:
             async with BACKGROUND_DOWNLOAD_SEMAPHORE:  # limit concurrent background downloads
                 await download_media_file(channel, post_id, file_unique_id)
+            _clear_download_failure(bg_key)
             await asyncio.sleep(2)
         except errors.FloodWait as e:
             # Must be caught BEFORE the generic Exception (FloodWait subclasses RPCError),
-            # otherwise the worker would hammer Telegram while under a flood wait.
+            # otherwise the worker would hammer Telegram while under a flood wait. A flood
+            # wait is a global throttle, not a per-file fault, so it does NOT arm backoff.
             logger.warning(f"bg_download_floodwait: {channel}/{post_id}/{file_unique_id} sleeping {e.value}s")
             await asyncio.sleep(min(int(e.value) + 5, 900))
         except Exception as e:
+            _record_download_failure(bg_key)
             logger.error(f"Background download error for {channel}/{post_id}/{file_unique_id}: {e}")
         finally:
+            # Free the dedup slot so this file can be re-enqueued by a later sweep if it is
+            # still missing (e.g. the download failed or was throttled). No await here, so the
+            # discard and task_done() happen atomically w.r.t. download_new_files.
+            _queued_media.discard(bg_key)
             download_queue.task_done()
 
 async def cache_media_files() -> None:
     """Background task for cache management: removes old files and downloads new ones"""
-    delay = 60
+    delay = Config["cache_sweep_interval"]
     while True:
         try:
             # Load all media file ID records from SQLite
             media_files = await asyncio.to_thread(get_all_media_file_ids_sync, DB_PATH)
 
-            cache_dir = os.path.abspath("./data/cache")
+            cache_dir = MEDIA_CACHE_DIR
             updated_media_files, files_removed = await asyncio.to_thread(remove_old_cached_files_sync, media_files, cache_dir)
 
-            if files_removed > 0:
-                try:
-                    # Determine which entries were removed and delete them from SQLite
-                    updated_set = {
-                        (r['channel'], r['post_id'], r['file_unique_id'])
-                        for r in updated_media_files
-                    }
-                    removed_entries = [
-                        (r['channel'], r['post_id'], r['file_unique_id'])
-                        for r in media_files
-                        if (r['channel'], r['post_id'], r['file_unique_id']) not in updated_set
-                    ]
-                    if removed_entries:
-                        await asyncio.to_thread(remove_media_file_ids_sync, DB_PATH, removed_entries)
-                    logger.info(f"Removed {files_removed} old files from cache")
-                except Exception as e:
-                    logger.error(f"Failed to remove old entries from SQLite: {str(e)}")
+            try:
+                # Compute the DB diff UNCONDITIONALLY (not only when files_removed > 0):
+                # remove_old_cached_files_sync drops an entry older than 20 days from the
+                # surviving list even when its file was already gone from disk (that branch
+                # does NOT bump files_removed). If we only purged when a real file was removed,
+                # such a fileless-but-expired row would stay in SQLite forever. The surviving
+                # set reflects what is actually left after the sweep; removed_entries are the
+                # rows present in the OLD set but not in the surviving set.
+                updated_set = {
+                    (r['channel'], r['post_id'], r['file_unique_id'])
+                    for r in updated_media_files
+                }
+                removed_entries = [
+                    (r['channel'], r['post_id'], r['file_unique_id'])
+                    for r in media_files
+                    if (r['channel'], r['post_id'], r['file_unique_id']) not in updated_set
+                ]
+                if removed_entries:
+                    await asyncio.to_thread(remove_media_file_ids_sync, DB_PATH, removed_entries)
+                    logger.info(f"cache_sweep: purged {len(removed_entries)} entries "
+                                f"({files_removed} files removed from disk)")
+            except Exception as e:
+                logger.error(f"Failed to remove old entries from SQLite: {str(e)}")
 
             await download_new_files(updated_media_files, cache_dir)
+
+            # Age-sweep the history/chatinfo cache once per pass (dead channels, orphaned
+            # uuid tmp files). MUST run off-loop via to_thread (blocking filesystem walk).
+            await asyncio.to_thread(sweep_tgcache)
+
             await asyncio.sleep(delay)  # Check every delay seconds
 
         except Exception as e:
@@ -848,7 +1045,7 @@ def calculate_cache_stats() -> dict[str, Any]:
     Calculate cache statistics including file count, total size in MB, and time difference in days.
     Returns a dictionary with keys: 'cache_files_count', 'cache_total_size_mb', 'cache_time_diff_days', 'channels'.
     """
-    base_cache_dir = os.path.abspath("./data/cache")
+    base_cache_dir = MEDIA_CACHE_DIR
     cache_files_count = 0
     cache_total_size_bytes = 0
     channels_stats = {}
@@ -1160,27 +1357,36 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, request: Re
         #else:
         #    logger.info(f"Valid digest for media {url}: {digest}")   
             
-        # Convert numeric channel ID to int if needed
-        channel_id: Union[str, int] = channel
-        if isinstance(channel, str) and channel.startswith('-100'):
-            channel_id = int(channel)
-            
+        # Canonical filesystem/DB/API key. Computed AFTER the digest check (which must run
+        # against the ORIGINAL url string) so 'Durov', 'durov' and '@durov' collapse to one
+        # on-disk tree, one _access_updates/media_key identity and one download identity. The
+        # canonical form is API-safe: usernames are case-insensitive on Telegram's side and
+        # numeric '-100...' ids are preserved verbatim (download_media_file re-ints them).
+        fs_channel = canonical_channel_key(channel)
+
         try: # Wrap the download and prepare call
             import time as _time
 
             # Pre-semaphore cache check: serve already-cached files without acquiring the semaphore
-            base_cache_dir = os.path.abspath("./data/cache")
-            channel_dir = os.path.join(base_cache_dir, str(channel))
-            post_dir = os.path.join(channel_dir, str(post_id))
-            cache_path = os.path.join(post_dir, file_unique_id)
+            cache_path = media_cache_path(fs_channel, post_id, file_unique_id)
             if os.path.exists(cache_path) and os.path.getsize(cache_path) > 0:
                 # File is already in cache — skip semaphore and serve directly
-                logger.info(f"pre_semaphore_cache_hit: {channel}/{post_id}/{file_unique_id}")
+                logger.info(f"pre_semaphore_cache_hit: {fs_channel}/{post_id}/{file_unique_id}")
                 # Record the access time into the accumulator instead of firing a per-hit
-                # SQLite write. Key channel as str(channel) — see _access_updates.
-                _access_updates[(str(channel), post_id, file_unique_id)] = datetime.now().timestamp()
+                # SQLite write. Key channel as the canonical fs_channel — see _access_updates.
+                _access_updates[(fs_channel, post_id, file_unique_id)] = datetime.now().timestamp()
                 return await prepare_file_response(cache_path, request=request,
-                                                   media_key=(str(channel), post_id, file_unique_id))
+                                                   media_key=(fs_channel, post_id, file_unique_id))
+
+            # A file that recently kept failing is in backoff: fast-reject instead of
+            # occupying a scarce download slot (and a Pyrogram transmission permit) on a
+            # request that will very likely hang again. Cached files already returned above,
+            # so this only guards the live-download path.
+            backoff_remaining = _download_backoff_remaining((fs_channel, post_id, file_unique_id))
+            if backoff_remaining > 0:
+                logger.info(f"media_backoff_skip: {channel}/{post_id}/{file_unique_id} in backoff {backoff_remaining:.0f}s")
+                return Response(status_code=503, content="Media temporarily unavailable, retry later",
+                                headers={"Retry-After": str(int(backoff_remaining) + 1)})
 
             _sem_wait_start = _time.monotonic()
             # Bound the wait for a live-download permit: a saturated semaphore must not
@@ -1211,7 +1417,7 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, request: Re
                 if _sem_wait > 0.5:
                     logger.warning(f"diag_semaphore_wait: {channel}/{post_id}/{file_unique_id} waited {_sem_wait:.3f}s for HTTP_DOWNLOAD_SEMAPHORE")
                 _dl_start = _time.monotonic()
-                file_path, delete_after = await _download_deduped(channel_id, post_id, file_unique_id)
+                file_path, delete_after = await _download_deduped(fs_channel, post_id, file_unique_id)
                 _dl_elapsed = _time.monotonic() - _dl_start
                 logger.info(f"diag_download_timing: {channel}/{post_id}/{file_unique_id} download_media_file took {_dl_elapsed:.3f}s (semaphore_wait={_sem_wait:.3f}s)")
             finally:
@@ -1220,7 +1426,7 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, request: Re
                 raise HTTPException(status_code=404, detail="File not found")
             if file_path:
                 return await prepare_file_response(file_path, request=request, delete_after=delete_after,
-                                                   media_key=(str(channel), post_id, file_unique_id))
+                                                   media_key=(fs_channel, post_id, file_unique_id))
         except ZeroSizeFileError as e: # Catch zero-size file errors
             logger.warning(f"zero_size_file_encountered: {str(e)}. Instructing client to retry.")
             return Response(

channel_key.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# flake8: noqa
+# pylint: disable=missing-function-docstring
+
+"""Canonical channel key.
+
+Dependency-free helper shared by tg_cache, post_parser and api_server. Kept free of
+any project imports so it can be imported by all three layers without introducing a
+circular import.
+"""
+
+from typing import Union
+
+
+def canonical_channel_key(channel: Union[str, int]) -> str:
+    """Canonical cache/DB key for a channel.
+
+    Telegram usernames are case-insensitive -> lowercase them.
+    Numeric '-100...' ids keep their exact string form. The '@' prefix is stripped.
+    The canonical form is also SAFE for Telegram API calls (usernames case-insensitive
+    on the API side; numeric unchanged) -- callers may thread one value through both
+    filesystem paths and API calls.
+    """
+    s = str(channel).strip().lstrip('@')
+    if s.startswith('-100') and s[4:].isdigit():
+        return s
+    return s.lower()

config.py

@@ -140,8 +140,25 @@ def get_settings() -> dict[str, Any]:
         "media_download_timeout_min": _parse_int_env("MEDIA_DOWNLOAD_TIMEOUT_MIN", 120),
         "media_download_timeout_max": _parse_int_env("MEDIA_DOWNLOAD_TIMEOUT_MAX", 1800),
         "media_download_min_speed": _parse_int_env("MEDIA_DOWNLOAD_MIN_SPEED", 256 * 1024),
+        # Interval (seconds) between cache-sweep passes in cache_media_files. Each pass is a
+        # full media_file_ids table scan plus an os.walk of the whole cache tree, so running
+        # it every 60s under a "20 days"/"1 hour" retention policy is wasteful. A value below
+        # the 60s floor is rejected at startup (_parse_int_env exits) so a misconfiguration
+        # can never turn the sweeper into a hot loop.
+        "cache_sweep_interval": _parse_int_env("CACHE_SWEEP_INTERVAL", 900, minimum=60),
         # Size of the asyncio default ThreadPoolExecutor. SQLite/python-magic/pickle/os.walk
         # all run via asyncio.to_thread; the interpreter default (min(32, cpu+4)) is only 5-6
         # on a 1-2 CPU container, which starves those under load. 32 gives ample headroom.
         "io_thread_pool_size": _parse_int_env("IO_THREAD_POOL_SIZE", 32),
+        # Max concurrent Telegram file transmissions (Pyrogram get_file/save_file
+        # semaphores). Kurigram's default is 1, which lets a single hung download block
+        # ALL media downloads process-wide; 3 aligns with HTTP_DOWNLOAD_SEMAPHORE. This is
+        # only a blast-radius limiter — the real cure for a zombie media connection is the
+        # download-timeout-triggered restart below.
+        "tg_max_concurrent_transmissions": _parse_int_env("TG_MAX_CONCURRENT_TRANSMISSIONS", 3),
+        # After this many CONSECUTIVE media-download timeouts, force an in-process client
+        # restart to rebuild the (zombie) media-DC connection. Any successful download
+        # resets the streak, so this only fires on a genuine death loop, not on the odd
+        # slow large-video timeout. The watchdog cannot catch this — it probes the main DC.
+        "media_timeout_restart_threshold": _parse_int_env("MEDIA_TIMEOUT_RESTART_THRESHOLD", 5),
     }

dockercompose.yml

@@ -29,6 +29,8 @@ services:
       # MEDIA_DOWNLOAD_TIMEOUT_MAX: 1800  # Max per-download timeout, seconds — cap for the largest videos (default: 1800)
       # MEDIA_DOWNLOAD_MIN_SPEED: 262144  # Assumed floor download speed, bytes/s — large-video timeout ≈ file_size / this, clamped to [MIN,MAX] (default: 262144 = 256 KB/s)
       # IO_THREAD_POOL_SIZE: 32           # Size of the asyncio default threadpool for blocking I/O (SQLite/python-magic/pickle/os.walk); raise on a busy 1-2 CPU box (default: 32)
+      # TG_MAX_CONCURRENT_TRANSMISSIONS: 3   # Max concurrent Telegram file transmissions (Pyrogram get_file semaphore). Kurigram default is 1, so one hung download blocks ALL media (default: 3)
+      # MEDIA_TIMEOUT_RESTART_THRESHOLD: 5   # Consecutive media-download timeouts before an in-process restart rebuilds the zombie media-DC connection (the main-DC watchdog can't see this) (default: 5)
       PYROGRAM_BRIDGE_URL: https://pgbridge.example.com
       API_PORT: 80
       TOKEN: ХХХ

message_snapshot.py

@@ -0,0 +1,585 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# flake8: noqa
+# pylint: disable=broad-exception-raised, raise-missing-from, too-many-arguments, redefined-outer-name
+# pylint: disable=multiple-statements, logging-fstring-interpolation, trailing-whitespace, line-too-long
+# pylint: disable=broad-exception-caught, missing-function-docstring, missing-class-docstring
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+
+"""JSON snapshot / restore for pyrogram Message objects (issue #23).
+
+The history cache no longer pickles live pyrogram objects. On write a plain JSON
+snapshot is extracted from a Message via an explicit allowlist (schema v1); on read
+a duck-typed CachedMessage is restored that is indistinguishable from a real Message
+to the render pipeline (post_parser.py / rss_generator.py are NOT changed).
+"""
+
+import json
+import logging
+from datetime import datetime
+from types import SimpleNamespace
+from typing import Any, List, Optional
+
+from pyrogram.enums import MessageMediaType
+
+logger = logging.getLogger(__name__)
+
+# Bump when the snapshot schema changes in a backwards-incompatible way; a mismatch
+# makes _load_entry treat the cache file as a miss (old files are simply re-fetched).
+# v2: added the special-media info-block types (story, contact, location, venue, dice,
+# game, giveaway, giveaway_winners, checklist, paid_media) plus live_photo. A v1 file lacks
+# these keys, so a cached special-media / live-photo message would restore with an empty
+# block — invalidate v1 files.
+SNAPSHOT_VERSION = 2
+
+
+class CachedStr(str):
+    """A ``str`` subclass carrying a precomputed ``.html`` rendering.
+
+    Mirrors pyrogram's ``Str`` (which exposes ``.html`` computed from entities). Here the
+    HTML is computed once at snapshot time and stored, so the render pipeline can read
+    ``message.text.html`` on a restored message exactly as on a live one. The instance
+    ``__dict__`` (holding ``.html``) is preserved across deepcopy/pickle via ``__reduce__``.
+    """
+
+    html: str
+
+    @classmethod
+    def build(cls, plain: str, html: str) -> "CachedStr":
+        obj = cls(plain)
+        obj.html = html
+        return obj
+
+    def __reduce__(self):
+        # Preserve the precomputed .html across copy.deepcopy() and pickle. Without this a
+        # str subclass would be reconstructed as a bare str and lose the instance __dict__.
+        return (CachedStr.build, (str(self), self.html))
+
+
+def _unwrap_text(value: Any) -> Optional[str]:
+    """Unwrap a styled-text container to a plain string.
+
+    In kurigram 2.2.23 Poll.question and every PollOption.text are always FormattedText
+    objects (``.text`` holds the string). The rule ``v.text if hasattr(v, 'text') else
+    str(v)`` is correct for FormattedText, for a bare Str and for a plain string. Storing
+    a FormattedText as-is would make json.dump raise TypeError, silently killing the cache
+    for any channel that has polls.
+    """
+    if value is None:
+        return None
+    inner = value.text if hasattr(value, "text") else value
+    return str(inner)
+
+
+def _snapshot_str(value: Any) -> Optional[dict]:
+    """Snapshot a text/caption value: {'plain', 'html'} or None if absent.
+
+    ``.html`` is computed at write time (live pyrogram Str exposes it); if unavailable the
+    html falls back to the plain text.
+    """
+    if value is None:
+        return None
+    plain = str(value)
+    html = getattr(value, "html", None)
+    if html is None:
+        html = plain
+    return {"plain": plain, "html": str(html)}
+
+
+def _enum_name(value: Any) -> Any:
+    """Return an enum member's ``.name`` (JSON-safe) or the value unchanged."""
+    if value is None:
+        return None
+    return value.name if hasattr(value, "name") else value
+
+
+def _snapshot_obj(obj: Any, keys: List[str]) -> Optional[dict]:
+    """Snapshot the allowlisted ``keys`` off ``obj`` (getattr-only), or None if obj is None."""
+    if obj is None:
+        return None
+    return {k: getattr(obj, k, None) for k in keys}
+
+
+def _snapshot_chat(chat: Any) -> Optional[dict]:
+    if chat is None:
+        return None
+    usernames = getattr(chat, "usernames", None)
+    snap_usernames = None
+    if usernames:
+        snap_usernames = [
+            {"username": getattr(u, "username", None), "active": getattr(u, "active", None)}
+            for u in usernames
+        ]
+    return {
+        "id": getattr(chat, "id", None),
+        "username": getattr(chat, "username", None),
+        "title": getattr(chat, "title", None),
+        "usernames": snap_usernames,
+    }
+
+
+# forward_origin is the ONE object whose key SET mirrors the live object: the
+# MessageOrigin* classes each carry a DIFFERENT set of attributes and
+# _format_forward_info branches by hasattr (Case 1-5). Presence of a key in the
+# snapshot must mirror presence of the attribute on the source object.
+_FORWARD_ORIGIN_KEYS = ["type", "chat", "sender_user_name", "sender_user", "chat_id", "title"]
+
+
+def _snapshot_forward_origin(fo: Any) -> Optional[dict]:
+    if fo is None:
+        return None
+    result: dict = {}
+    for key in _FORWARD_ORIGIN_KEYS:
+        if not hasattr(fo, key):
+            continue
+        value = getattr(fo, key)
+        if key == "type":
+            result[key] = _enum_name(value)
+        elif key == "chat":
+            result[key] = _snapshot_obj(value, ["id", "title", "username"])
+        elif key == "sender_user":
+            result[key] = _snapshot_obj(value, ["first_name", "last_name", "username"])
+        else:
+            result[key] = value
+    return result
+
+
+def _snapshot_reactions(reactions: Any) -> Optional[list]:
+    if reactions is None:
+        return None
+    rlist = getattr(reactions, "reactions", None)
+    if rlist is None:
+        return None
+    out = []
+    for r in rlist:
+        cid = getattr(r, "custom_emoji_id", None)
+        out.append({
+            # Custom-emoji reactions carry no unicode emoji: emoji is null and the
+            # custom_emoji_id is stored as a STRING = str(document_id).
+            "emoji": None if cid is not None else getattr(r, "emoji", None),
+            "custom_emoji_id": str(cid) if cid is not None else None,
+            "count": getattr(r, "count", None),
+            "is_paid": getattr(r, "is_paid", None),
+        })
+    return out
+
+
+def _snapshot_poll(poll: Any) -> Optional[dict]:
+    if poll is None:
+        return None
+    options = getattr(poll, "options", None)
+    snap_options = None
+    if options:
+        snap_options = [{"text": _unwrap_text(getattr(o, "text", None))} for o in options]
+    return {
+        "question": _unwrap_text(getattr(poll, "question", None)),
+        "options": snap_options,
+    }
+
+
+def _snapshot_web_page(wp: Any) -> Optional[dict]:
+    if wp is None:
+        return None
+    return {
+        "type": getattr(wp, "type", None),
+        "url": getattr(wp, "url", None),
+        "display_url": getattr(wp, "display_url", None),
+        "site_name": getattr(wp, "site_name", None),
+        "title": getattr(wp, "title", None),
+        "description": getattr(wp, "description", None),
+        "has_large_media": getattr(wp, "has_large_media", None),
+        "photo": _snapshot_obj(getattr(wp, "photo", None), ["file_unique_id"]),
+    }
+
+
+# --------------------------------------------------------------------------- #
+# Special-media snapshots (issue #23 review fix).
+#
+# _format_special_media (post_parser.py) and find_file_id_in_message
+# (api_server.py) read a handful of type-specific attributes off the Message that
+# were NOT in schema v1. On a cache hit a restored Message carried media=<enum> but
+# message.<attr>=None, so the special info block silently vanished (render
+# divergence vs. a live Message). Each helper below snapshots EXACTLY the sub-fields
+# the renderer + find_file_id read for that type — nothing more, nothing less.
+# --------------------------------------------------------------------------- #
+def _snapshot_story(story: Any) -> Optional[dict]:
+    """STORY: _story_media_object / find_file_id_in_message read story.video and
+    story.photo, each by .file_unique_id (video wins over photo). The chosen object is
+    also the one _save_media_file_ids reads .file_size off for the >100MB skip, so
+    file_size is snapshotted too (else a >100MB story media would be collected on a
+    cache hit — F1). The render URL is still built from file_unique_id."""
+    if story is None:
+        return None
+    return {
+        "video": _snapshot_obj(getattr(story, "video", None), ["file_unique_id", "file_size"]),
+        "photo": _snapshot_obj(getattr(story, "photo", None), ["file_unique_id", "file_size"]),
+    }
+
+
+def _snapshot_venue(venue: Any) -> Optional[dict]:
+    """VENUE: reads .title, .address and its nested .location (.latitude/.longitude via
+    _format_osm_link)."""
+    if venue is None:
+        return None
+    return {
+        "title": getattr(venue, "title", None),
+        "address": getattr(venue, "address", None),
+        "location": _snapshot_obj(getattr(venue, "location", None), ["latitude", "longitude"]),
+    }
+
+
+def _snapshot_giveaway(giveaway: Any) -> Optional[dict]:
+    """GIVEAWAY: reads .quantity, .months, .stars, .until_date (a datetime rendered via
+    strftime) and .description. until_date is stored as isoformat and restored to a
+    datetime so the strftime branch reproduces byte-for-byte."""
+    if giveaway is None:
+        return None
+    until = getattr(giveaway, "until_date", None)
+    return {
+        "quantity": getattr(giveaway, "quantity", None),
+        "months": getattr(giveaway, "months", None),
+        "stars": getattr(giveaway, "stars", None),
+        # Only a real datetime renders (renderer gates on hasattr(.,'strftime')); anything
+        # else is dropped to None so the same "no until date" branch is taken on restore.
+        "until_date": until.isoformat() if hasattr(until, "isoformat") else None,
+        "description": getattr(giveaway, "description", None),
+    }
+
+
+def _snapshot_checklist(checklist: Any) -> Optional[dict]:
+    """CHECKLIST: reads .title (used ONLY when it isinstance str) and .tasks; each task
+    reads .text and the truthiness of .completed_by / .completion_date (→ ☑/☐).
+
+    title is stored only when it is a str (matching the renderer's isinstance gate, which
+    otherwise renders an empty title). completed_by / completion_date are stored as bools
+    (the renderer only tests their truthiness) so a live User/datetime stays JSON-safe."""
+    if checklist is None:
+        return None
+    title = getattr(checklist, "title", None)
+    snap_tasks = []
+    for task in (getattr(checklist, "tasks", None) or []):
+        text = getattr(task, "text", "")
+        # Mirror the renderer: a non-str text is str()-ified. Storing the resolved string
+        # keeps the restored value isinstance-str, reproducing the same rendered bytes.
+        text_str = text if isinstance(text, str) else str(text)
+        snap_tasks.append({
+            "text": text_str,
+            "completed_by": bool(getattr(task, "completed_by", None)),
+            "completion_date": bool(getattr(task, "completion_date", None)),
+        })
+    return {
+        "title": title if isinstance(title, str) else None,
+        "tasks": snap_tasks,
+    }
+
+
+def _snapshot_paid_media(paid_media: Any) -> Optional[dict]:
+    """PAID_MEDIA: _generate_html_media reads .stars_amount (default 0) and the LENGTH of
+    .media. Snapshot the renderer's effective stars value and the item count; the media
+    list is restored as N placeholders so len() reproduces."""
+    if paid_media is None:
+        return None
+    media = getattr(paid_media, "media", None)
+    count = len(media) if isinstance(media, (list, tuple)) else 0
+    return {
+        # Snapshot getattr(...,0) so a missing attribute restores to 0 exactly as the
+        # renderer would have defaulted it (a present None stays None on both sides).
+        "stars_amount": getattr(paid_media, "stars_amount", 0),
+        "media_count": count,
+    }
+
+
+def snapshot_message(message: Any) -> dict:
+    """Extract a JSON-serializable snapshot (schema v1) from a pyrogram Message.
+
+    Every field is read via getattr(..., None). See the module docstring / issue #23 for
+    the field contract.
+    """
+    date = getattr(message, "date", None)
+    media = getattr(message, "media", None)
+    service = getattr(message, "service", None)
+    return {
+        "id": getattr(message, "id", None),
+        # isoformat()/fromisoformat() preserve naive/aware exactly as pyrogram stores it.
+        "date": date.isoformat() if date is not None else None,
+        "text": _snapshot_str(getattr(message, "text", None)),
+        "caption": _snapshot_str(getattr(message, "caption", None)),
+        "media": media.name if media is not None else None,
+        "service": service.name if service is not None else None,
+        "media_group_id": getattr(message, "media_group_id", None),
+        "views": getattr(message, "views", None),
+        "show_caption_above_media": getattr(message, "show_caption_above_media", None),
+        "reply_to_message_id": getattr(message, "reply_to_message_id", None),
+        "empty": getattr(message, "empty", None),
+        "chat": _snapshot_chat(getattr(message, "chat", None)),
+        "sender_chat": _snapshot_obj(getattr(message, "sender_chat", None), ["id", "title", "username"]),
+        "from_user": _snapshot_obj(getattr(message, "from_user", None), ["first_name", "last_name", "username"]),
+        "forward_origin": _snapshot_forward_origin(getattr(message, "forward_origin", None)),
+        "reactions": _snapshot_reactions(getattr(message, "reactions", None)),
+        "poll": _snapshot_poll(getattr(message, "poll", None)),
+        "web_page": _snapshot_web_page(getattr(message, "web_page", None)),
+        "photo": _snapshot_obj(getattr(message, "photo", None), ["file_unique_id"]),
+        "video": _snapshot_obj(getattr(message, "video", None), ["file_unique_id", "file_size"]),
+        # file_size is snapshotted for every type whose selected object flows into
+        # _save_media_file_ids' >100MB skip (MEDIA_SOURCES: document/audio/animation/
+        # video_note select this exact object). Without it a restored >100MB media
+        # would have file_size=None and be wrongly collected on a cache hit (F1).
+        "document": _snapshot_obj(getattr(message, "document", None), ["file_unique_id", "mime_type", "file_size"]),
+        "audio": _snapshot_obj(getattr(message, "audio", None), ["file_unique_id", "mime_type", "file_size"]),
+        "voice": _snapshot_obj(getattr(message, "voice", None), ["file_unique_id", "mime_type"]),
+        "video_note": _snapshot_obj(getattr(message, "video_note", None), ["file_unique_id", "file_size"]),
+        "animation": _snapshot_obj(getattr(message, "animation", None), ["file_unique_id", "file_size"]),
+        "sticker": _snapshot_obj(getattr(message, "sticker", None), ["file_unique_id", "emoji", "is_video"]),
+        # LIVE_PHOTO (Kurigram 2.2.23) renders as a video element via the video_loop_400
+        # kind. MEDIA_SOURCES/_get_file_unique_id/_save_media_file_ids read live_photo's
+        # file_unique_id + file_size (the >100MB skip); find_file_id also reads file_unique_id.
+        # Mirror the `video` allowlist so a cached live-photo message keeps its media block.
+        "live_photo": _snapshot_obj(getattr(message, "live_photo", None), ["file_unique_id", "file_size"]),
+        # Special-media info-block types (issue #23 review fix). Each reads a fixed set of
+        # sub-fields in _format_special_media / find_file_id_in_message; snapshot exactly those.
+        "story": _snapshot_story(getattr(message, "story", None)),
+        "contact": _snapshot_obj(getattr(message, "contact", None), ["first_name", "last_name", "phone_number"]),
+        "location": _snapshot_obj(getattr(message, "location", None), ["latitude", "longitude"]),
+        "venue": _snapshot_venue(getattr(message, "venue", None)),
+        "dice": _snapshot_obj(getattr(message, "dice", None), ["emoji", "value"]),
+        "game": _snapshot_obj(getattr(message, "game", None), ["title"]),
+        "giveaway": _snapshot_giveaway(getattr(message, "giveaway", None)),
+        "giveaway_winners": _snapshot_obj(
+            getattr(message, "giveaway_winners", None), ["winner_count", "quantity", "prize_description"]),
+        "checklist": _snapshot_checklist(getattr(message, "checklist", None)),
+        "paid_media": _snapshot_paid_media(getattr(message, "paid_media", None)),
+    }
+
+
+def _ns(d: Optional[dict], keys: List[str]) -> Optional[SimpleNamespace]:
+    """Restore a nested object with the FULL schema key set (None-defaults).
+
+    Live pyrogram objects always set every attribute in __init__, and the pipeline reads
+    them directly (e.g. rss_generator ~131 message.chat.username in an except; post_parser
+    ~1087 u.username / u.active without getattr). So restored objects must carry all keys.
+    Returns None if ``d`` is None (the attribute was absent on the live object).
+    """
+    if d is None:
+        return None
+    ns = SimpleNamespace()
+    for k in keys:
+        setattr(ns, k, d.get(k))
+    return ns
+
+
+def _restore_chat(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    usernames = d.get("usernames")
+    restored_usernames = None
+    if usernames is not None:
+        restored_usernames = [_ns(u, ["username", "active"]) for u in usernames]
+    return SimpleNamespace(
+        id=d.get("id"),
+        username=d.get("username"),
+        title=d.get("title"),
+        usernames=restored_usernames,
+    )
+
+
+def _restore_forward_origin(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    """Restore forward_origin mirroring ONLY the recorded keys (presence-semantics)."""
+    if d is None:
+        return None
+    ns = SimpleNamespace()
+    for key, value in d.items():
+        if key == "chat":
+            setattr(ns, key, _ns(value, ["id", "title", "username"]))
+        elif key == "sender_user":
+            setattr(ns, key, _ns(value, ["first_name", "last_name", "username"]))
+        else:
+            setattr(ns, key, value)
+    return ns
+
+
+def _restore_reactions(items: Optional[list]) -> Optional[SimpleNamespace]:
+    if items is None:
+        return None
+    reactions = [_ns(r, ["emoji", "custom_emoji_id", "count", "is_paid"]) for r in items]
+    # Mirror pyrogram: message.reactions is an object exposing a .reactions list.
+    return SimpleNamespace(reactions=reactions)
+
+
+def _restore_poll(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    options = d.get("options") or []
+    # options must be namespace objects with a .text string; a bare string would make
+    # getattr(option, 'text', '') return '' and render empty options.
+    restored_options = [SimpleNamespace(text=o.get("text")) for o in options]
+    return SimpleNamespace(question=d.get("question"), options=restored_options)
+
+
+def _restore_web_page(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    return SimpleNamespace(
+        type=d.get("type"),
+        url=d.get("url"),
+        display_url=d.get("display_url"),
+        site_name=d.get("site_name"),
+        title=d.get("title"),
+        description=d.get("description"),
+        has_large_media=d.get("has_large_media"),
+        photo=_ns(d.get("photo"), ["file_unique_id"]),
+    )
+
+
+def _restore_media(name: Optional[str]) -> Optional[MessageMediaType]:
+    if name is None:
+        return None
+    try:
+        return MessageMediaType[name]
+    except KeyError:
+        # A media type unknown to this kurigram build: warn and drop to None so the 31
+        # post_parser sites that compare against / index by the enum keep working.
+        logger.warning(f"snapshot_unknown_media_type: {name}")
+        return None
+
+
+def _restore_str(d: Optional[dict]) -> Optional[CachedStr]:
+    if d is None:
+        return None
+    return CachedStr.build(d.get("plain", ""), d.get("html", d.get("plain", "")))
+
+
+# --------------------------------------------------------------------------- #
+# Special-media restores (issue #23 review fix). Mirror the snapshot helpers above:
+# rebuild each object so the exact attributes _format_special_media /
+# find_file_id_in_message read exist, restoring the special block on a cache hit.
+# --------------------------------------------------------------------------- #
+def _restore_story(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    return SimpleNamespace(
+        video=_ns(d.get("video"), ["file_unique_id", "file_size"]),
+        photo=_ns(d.get("photo"), ["file_unique_id", "file_size"]),
+    )
+
+
+def _restore_venue(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    return SimpleNamespace(
+        title=d.get("title"),
+        address=d.get("address"),
+        location=_ns(d.get("location"), ["latitude", "longitude"]),
+    )
+
+
+def _restore_giveaway(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    until = d.get("until_date")
+    return SimpleNamespace(
+        quantity=d.get("quantity"),
+        months=d.get("months"),
+        stars=d.get("stars"),
+        # Restore a datetime so hasattr(until_date, 'strftime') holds exactly as on a live
+        # object; None (no/invalid date) restores as None and the strftime branch is skipped.
+        until_date=datetime.fromisoformat(until) if until is not None else None,
+        description=d.get("description"),
+    )
+
+
+def _restore_checklist(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    restored_tasks = [
+        SimpleNamespace(
+            text=t.get("text"),
+            completed_by=t.get("completed_by"),
+            completion_date=t.get("completion_date"),
+        )
+        for t in (d.get("tasks") or [])
+    ]
+    return SimpleNamespace(title=d.get("title"), tasks=restored_tasks)
+
+
+def _restore_paid_media(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    count = d.get("media_count") or 0
+    # media is only ever len()'d by the renderer, so N placeholder items reproduce the count.
+    return SimpleNamespace(stars_amount=d.get("stars_amount"), media=[None] * count)
+
+
+class CachedMessage:
+    """Duck-typed stand-in for a pyrogram Message, restored from a snapshot dict.
+
+    Mutable (reply enrichment assigns ``.reply_to_message``). Every top-level attribute the
+    render pipeline reads exists with a None/False default so getattr never raises.
+    """
+
+    def __init__(self, data: dict):
+        self._snapshot = data
+        self.id = data.get("id")
+        date = data.get("date")
+        self.date = datetime.fromisoformat(date) if date is not None else None
+        self.text = _restore_str(data.get("text"))
+        self.caption = _restore_str(data.get("caption"))
+        self.media = _restore_media(data.get("media"))
+        # service restored as a plain string; all consumers use truthiness and
+        # `'X' in str(service)`.
+        self.service = data.get("service")
+        self.media_group_id = data.get("media_group_id")
+        self.views = data.get("views")
+        self.show_caption_above_media = data.get("show_caption_above_media")
+        self.reply_to_message_id = data.get("reply_to_message_id")
+        self.reply_to_message = None
+        self.empty = bool(data.get("empty"))
+        self.chat = _restore_chat(data.get("chat"))
+        self.sender_chat = _ns(data.get("sender_chat"), ["id", "title", "username"])
+        self.from_user = _ns(data.get("from_user"), ["first_name", "last_name", "username"])
+        self.forward_origin = _restore_forward_origin(data.get("forward_origin"))
+        self.reactions = _restore_reactions(data.get("reactions"))
+        self.poll = _restore_poll(data.get("poll"))
+        self.web_page = _restore_web_page(data.get("web_page"))
+        self.photo = _ns(data.get("photo"), ["file_unique_id"])
+        self.video = _ns(data.get("video"), ["file_unique_id", "file_size"])
+        self.document = _ns(data.get("document"), ["file_unique_id", "mime_type", "file_size"])
+        self.audio = _ns(data.get("audio"), ["file_unique_id", "mime_type", "file_size"])
+        self.voice = _ns(data.get("voice"), ["file_unique_id", "mime_type"])
+        self.video_note = _ns(data.get("video_note"), ["file_unique_id", "file_size"])
+        self.animation = _ns(data.get("animation"), ["file_unique_id", "file_size"])
+        self.sticker = _ns(data.get("sticker"), ["file_unique_id", "emoji", "is_video"])
+        # LIVE_PHOTO: restored like `video` so the video_loop_400 media block renders on a
+        # cache hit (file_unique_id → URL; file_size → the >100MB collection skip).
+        self.live_photo = _ns(data.get("live_photo"), ["file_unique_id", "file_size"])
+        # Special-media info-block types (issue #23 review fix): restored so the type's
+        # info block renders on a cache hit exactly as for a live Message.
+        self.story = _restore_story(data.get("story"))
+        self.contact = _ns(data.get("contact"), ["first_name", "last_name", "phone_number"])
+        self.location = _ns(data.get("location"), ["latitude", "longitude"])
+        self.venue = _restore_venue(data.get("venue"))
+        self.dice = _ns(data.get("dice"), ["emoji", "value"])
+        self.game = _ns(data.get("game"), ["title"])
+        self.giveaway = _restore_giveaway(data.get("giveaway"))
+        self.giveaway_winners = _ns(
+            data.get("giveaway_winners"), ["winner_count", "quantity", "prize_description"])
+        self.checklist = _restore_checklist(data.get("checklist"))
+        self.paid_media = _restore_paid_media(data.get("paid_media"))
+
+    def __str__(self) -> str:
+        return json.dumps(self._snapshot, default=str)
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+
+def restore_message(data: dict) -> CachedMessage:
+    return CachedMessage(data)
+
+
+def snapshot_messages(messages: List[Any]) -> List[dict]:
+    return [snapshot_message(m) for m in messages]
+
+
+def restore_messages(items: List[dict]) -> List[CachedMessage]:
+    return [restore_message(d) for d in items]

migrate_channel_keys.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# flake8: noqa
+# pylint: disable=missing-function-docstring, broad-exception-caught, logging-fstring-interpolation
+
+"""One-shot migration of existing cache data to the canonical channel key.
+
+Collapses non-canonical (mixed-case) channel names in both the on-disk media cache
+(data/cache/<channel>/...) and the SQLite media_file_ids table onto their canonical,
+lowercase form. Numeric '-100...' ids are already canonical and are never touched.
+
+Unit of work = a WHOLE channel, processed FS FIRST then SQL. The order is essential:
+if the FS rename/merge fails, the channel's SQL rows are left old-cased so the old tree
+is still reachable by the media sweeper via its DB rows (no eternal orphan directory).
+
+The migration is idempotent: a second run finds nothing to do and is a no-op.
+"""
+
+import os
+import shutil
+import sqlite3
+import logging
+
+from channel_key import canonical_channel_key
+
+logger = logging.getLogger(__name__)
+
+
+def _is_safe_channel_segment(name: str) -> bool:
+    """True iff ``name`` is a plain single path segment safe to join under cache_dir.
+
+    Defends the destructive FS ops below (os.rename / _merge_dir_tree / shutil.rmtree)
+    against a dirty channel string that reached the DB or a crafted dir on disk,
+    regardless of any upstream route-traversal. A dirty name (containing '/', '\\' or a
+    '..' component, or that is not a bare basename, or empty/'.') would let a rename or
+    rmtree escape cache_dir, so it is rejected and the channel is left un-migrated.
+    """
+    if not name or name == '.':
+        return False
+    if '/' in name or '\\' in name:
+        return False
+    if '..' in name.replace('\\', '/').split('/'):
+        return False
+    if os.path.basename(name) != name:
+        return False
+    return True
+
+
+def _merge_dir_tree(src: str, dst: str) -> None:
+    """Merge every file under ``src`` into ``dst`` where an existing ``dst`` file WINS.
+
+    Both trees share the same filesystem (siblings under the media cache root), so files
+    are moved with os.rename. Any file that already exists in ``dst`` is kept and the
+    ``src`` copy is discarded. Emptied ``src`` remnants are removed at the end.
+    """
+    for root, _dirs, files in os.walk(src):
+        rel = os.path.relpath(root, src)
+        target_root = dst if rel == '.' else os.path.join(dst, rel)
+        os.makedirs(target_root, exist_ok=True)
+        for name in files:
+            s = os.path.join(root, name)
+            t = os.path.join(target_root, name)
+            if os.path.exists(t):
+                os.remove(s)  # existing file in dst wins
+            else:
+                os.rename(s, t)
+    shutil.rmtree(src, ignore_errors=True)
+
+
+def migrate_channel_keys_sync(db_path: str, cache_dir: str) -> None:
+    """Migrate mixed-case channel cache dirs and DB rows onto the canonical key.
+
+    Blocking (FS renames + SQLite). Call via asyncio.to_thread from an async context.
+    """
+    rows_merged = 0
+    dirs_renamed = 0
+    samefile_noops = 0
+    failures = 0
+
+    conn = sqlite3.connect(db_path)
+    conn.execute("PRAGMA busy_timeout = 5000")
+    try:
+        # ------------------------------------------------------------------ #
+        # Candidate channels: mixed-case first-level cache dirs UNION mixed-case DB rows.
+        # ------------------------------------------------------------------ #
+        candidates: set[str] = set()
+        try:
+            for entry in os.listdir(cache_dir):
+                full = os.path.join(cache_dir, entry)
+                if not os.path.isdir(full):
+                    continue
+                if entry.startswith('-100'):
+                    continue
+                if entry != entry.lower():
+                    candidates.add(entry)
+        except FileNotFoundError:
+            pass  # No cache dir yet — nothing to migrate on the FS side.
+
+        try:
+            cur = conn.execute(
+                "SELECT DISTINCT channel FROM media_file_ids "
+                "WHERE channel != lower(channel) AND channel NOT LIKE '-100%'"
+            )
+            for (ch,) in cur.fetchall():
+                candidates.add(ch)
+        except sqlite3.Error as e:
+            logger.error(f"migrate_channel_keys: DB candidate query failed: {e}")
+
+        for name in sorted(candidates):
+            # Traversal guard: reject any dirty channel name BEFORE it reaches the
+            # destructive FS ops (rename/merge/rmtree). Applies to both candidate sources
+            # (DB rows and the dir listing). A rejected channel is left as-is (its old-cased
+            # rows/dirs survive until the 20-day sweep, same as any un-migrated channel).
+            if not _is_safe_channel_segment(name):
+                logger.warning(
+                    f"migrate_channel_keys: unsafe channel name {name!r} rejected "
+                    f"(would escape cache_dir), skipping"
+                )
+                failures += 1
+                continue
+            canonical = canonical_channel_key(name)
+            if canonical == name:
+                # Nothing to change (already canonical) — defensive, candidates shouldn't hit this.
+                continue
+            try:
+                # ---------------------------- FS step ---------------------------- #
+                src = os.path.join(cache_dir, name)
+                dst = os.path.join(cache_dir, canonical)
+                if os.path.isdir(src):
+                    try:
+                        if os.path.exists(dst) and os.path.samefile(src, dst):
+                            # Case-insensitive FS (macOS/APFS, Docker Desktop for Mac):
+                            # src and dst are ONE directory. Renaming/merging would destroy
+                            # the channel's entire cache — treat as a pure no-op.
+                            samefile_noops += 1
+                        elif not os.path.exists(dst):
+                            os.rename(src, dst)
+                            dirs_renamed += 1
+                        else:
+                            # Genuinely different dirs (case-sensitive FS, both forms present):
+                            # per-file merge with the existing dst file winning.
+                            _merge_dir_tree(src, dst)
+                            dirs_renamed += 1
+                    except OSError as e:
+                        # FS step failed (EACCES etc.): mark orphan candidate and SKIP the SQL
+                        # step so the old-cased DB rows keep the old tree visible to the sweeper.
+                        logger.error(
+                            f"migrate_channel_keys: FS step failed for {name!r} -> {canonical!r} "
+                            f"(orphan candidate, SQL skipped): {e}"
+                        )
+                        failures += 1
+                        continue
+
+                # ---------------------------- SQL step --------------------------- #
+                old_rows = conn.execute(
+                    "SELECT post_id, file_unique_id, added, mime_type "
+                    "FROM media_file_ids WHERE channel = ?",
+                    (name,),
+                ).fetchall()
+                for (post_id, file_unique_id, added, mime_type) in old_rows:
+                    twin = conn.execute(
+                        "SELECT added, mime_type FROM media_file_ids "
+                        "WHERE channel = ? AND post_id = ? AND file_unique_id = ?",
+                        (canonical, post_id, file_unique_id),
+                    ).fetchone()
+                    if twin is None:
+                        # No lowercase twin: a plain re-key is safe (no PK conflict).
+                        conn.execute(
+                            "UPDATE media_file_ids SET channel = ? "
+                            "WHERE channel = ? AND post_id = ? AND file_unique_id = ?",
+                            (canonical, name, post_id, file_unique_id),
+                        )
+                    else:
+                        # Twin exists: merge (added = max, prefer non-NULL mime_type) into the
+                        # canonical row, then delete the old-cased row. A bare
+                        # UPDATE ... SET channel=lower(channel) is FORBIDDEN (PK conflict).
+                        t_added, t_mime = twin
+                        new_added = max(added, t_added)
+                        new_mime = t_mime if t_mime is not None else mime_type
+                        conn.execute(
+                            "UPDATE media_file_ids SET added = ?, mime_type = ? "
+                            "WHERE channel = ? AND post_id = ? AND file_unique_id = ?",
+                            (new_added, new_mime, canonical, post_id, file_unique_id),
+                        )
+                        conn.execute(
+                            "DELETE FROM media_file_ids "
+                            "WHERE channel = ? AND post_id = ? AND file_unique_id = ?",
+                            (name, post_id, file_unique_id),
+                        )
+                        rows_merged += 1
+                conn.commit()
+            except Exception as e:
+                # Never crash startup: log and move on. A re-run is an idempotent no-op.
+                try:
+                    conn.rollback()
+                except Exception:
+                    pass
+                logger.error(f"migrate_channel_keys: channel {name!r} migration failed: {e}")
+                failures += 1
+    finally:
+        conn.close()
+
+    logger.info(
+        f"migration_summary: rows merged {rows_merged}, dirs renamed {dirs_renamed}, "
+        f"samefile no-ops {samefile_noops}, failures {failures}"
+    )

plans/2026-07-05-render-pipeline-refactor.md

@@ -0,0 +1,792 @@
+# Рефакторинг пайплайна рендера: спецификация (v6)
+
+Статус: пять раундов адверсариального ревью — четыре у постоянного критика +
+независимый холодный проход (журнал в §8), блокеров нет. Холодный проход
+независимо подтвердил все line-refs и эмпирические утверждения спеки.
+v6: golden-корпус переведён с синтетики на записанные реальные сообщения
+(решение владельца, обоснование в этапе 0 и §8).
+База (обновлено 2026-07-05, после мёржа PR #21): ветка
+`refactor/render-pipeline` режется от **актуального `main`** (merge-коммит
+`88ac436`+), НЕ от голого f9550d8 — main теперь содержит kurigram-код
+(f9550d8) + review-fix `6388f2f` + саму эту спеку и корпус фикстур
+(коммиты docs). Все line-refs спеки по-прежнему действительны: `6388f2f` —
+чисто тестовый (только `tests/test_new_media_types.py`), source
+post_parser.py / rss_generator.py / api_server.py не сдвинулся ни на строку
+относительно f9550d8. Предусловие «kurigram в main» — ВЫПОЛНЕНО.
+Ишью: эпик [#34](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/34),
+этапы 0–6 → [#27](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/27),
+[#28](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/28),
+[#29](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/29),
+[#30](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/30),
+[#31](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/31),
+[#32](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/32),
+[#33](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/33).
+Устаревшее [#22](https://gitea.vvzvlad.xyz/vvzvlad/pyrogram-bridge/issues/22)
+(этап 1 по v1) закрыто как superseded.
+
+## 1. Проблема
+
+Рендер фидов (RSS/HTML) — не однонаправленный пайплайн, а набор циклов с
+обратными ходами и дублированием:
+
+- `generate_channel_rss` и `generate_channel_html` — близнецы на ~80%
+  (rss_generator.py:347–527 и 578–717).
+- Обратный ход данных: `processed_message_to_tg_message`
+  (rss_generator.py:156–201) конвертирует отрендеренный dict обратно в
+  мок-Message ради повторного рендера футера — при живых `Message` в `group`.
+- Санитайз-конфиг скопирован трижды (post_parser.py:702, rss_generator.py:468,
+  679) и разъехался: `s`/`del` разрешены только в post_parser.
+- RSS-путь: отдельный `asyncio.to_thread` + новый `CSSSanitizer` + вложенная
+  функция НА КАЖДЫЙ пост — при том, что `_render_pipeline` уже в worker-треде.
+- `_create_time_based_media_groups`: `copy.deepcopy` всех сообщений на каждый
+  запрос (rss_generator.py:43) ради защиты кэша от мутации `media_group_id`.
+- post_parser: три параллельные лестницы выбора media-объекта
+  (`_get_file_unique_id`, `_save_media_file_ids`, `_generate_html_media`),
+  синхронизируемые вручную.
+
+Латентные баги, найденные при анализе и ревью:
+
+- `<hr class="post-divider">` добавляется ДО финального санитайза, `hr` нет в
+  whitelist → bleach вырезает все разделители HTML-фида.
+- `_sanitize_html` при исключении bleach возвращает сырой HTML (fail-open) —
+  потенциальный stored-XSS; фидовые копии fail-closed.
+- `_generate_html_media`: при `file_unique_id is None` (в т.ч. массовый случай
+  WEB_PAGE без фото) открытый `<div class="message-media">` не закрывается;
+  при whole-feed санитайзе html5lib «заглатывает» в него все последующие посты.
+- **Naive/aware краш в дефолтном пути**: kurigram-даты naive-local
+  (`datetime.fromtimestamp`), фолбэк в sort-ключе групп — aware UTC
+  (rss_generator.py:141). Один None-date пост в фиде с реальными датами даёт
+  TypeError → 500 при ЛЮБОМ `time_based_merge` (проверено эмпирически);
+  при `time_based_merge=True` тот же класс краша дополнительно в
+  тайм-кластеризации (строки 45–46, 61–62). Тесты слепы: мокают aware-даты.
+- FloodWait из `cached_get_chat_history` перехватывается `except Exception` и
+  превращается в ValueError → HTTP 400 вместо 429 (rss_generator.py:416–422,
+  636–642; маппинг ValueError→400 — api_server.py:1334–1337).
+- `_reactions_views_links`: reactions-объект с пустым списком реакций даёт
+  `first_line_parts.append("")` → ведущий разделитель `…|…` в футере
+  (post_parser.py:1081–1104, branch).
+
+## 2. Целевая архитектура
+
+```text
+                    ┌───────────── async-зона (event loop) ─────────────┐
+api_server ──► generate_channel_rss ─┐
+                                     ├─► _prepare_feed_posts(...) ──► PreparedFeed
+api_server ──► generate_channel_html ┘        │
+                                              │  fetch (cached_get_chat / history)
+                                              │  enrich (_reply_enrichment, опция)
+                                              ▼
+                    ┌───────── to_thread: _render_pipeline (sync) ──────┐
+                    │ сортировка date-ASC (naive-safe) при time_merge    │
+                    │ _compute_time_based_group_ids (чистая, без мутаций)│
+                    │ _create_messages_groups(msgs, group_ids)           │
+                    │ [:limit] → _render_messages_groups                 │
+                    │            (рендер → фильтры → sort — внутри неё)  │
+                    │ затем sanitize_html на каждый ОТФИЛЬТРОВАННЫЙ пост │
+                    └────────────────────────────────────── санитайз ───┘
+                                              │
+                     RSS: feedgen из готовых постов     HTML: '\n<hr>\n'.join(...)
+```
+
+Решения:
+
+1. Санитайз внутри `_render_pipeline`, per-post, ПОСЛЕ фильтров (не тратить
+   bleach на отфильтровываемые посты).
+2. Один модуль `sanitizer.py` — единственный источник конфига bleach, включая
+   `protocols=['http','https','tg']` и `strip=True` (оба отличаются от
+   дефолтов bleach!).
+3. Футер merged-групп из настоящего main-сообщения; конвертер удаляется.
+4. Группировка — чистая функция: маппинг `msg.id → effective_group_id`;
+   кэшированные Message пайплайном не мутируются.
+5. Общий препроцессинг `_prepare_feed_posts`; различия путей — явные параметры.
+6. Единая таблица медиа-типов в post_parser (селектор + renderer на kind).
+
+Одиночный пост (`get_post` → `process_message(sanitize=True)` → `_format_html`)
+не меняется, за исключением реестра §3 пп. 2, 13, 14, 15 (fail-closed,
+guard >100 МБ, закрытие div, пустые реакции — эти механизмы общие с фидовым
+путём).
+
+## 3. Реестр намеренных изменений поведения
+
+Всё, что не перечислено здесь, обязано остаться бит-в-бит идентичным.
+Golden-эталоны (§5, этап 0) обновляются только коммитом со ссылкой на пункт
+этого реестра.
+
+| № | Изменение | Тип | Этап |
+| --- | --------- | --- | ---- |
+| 1 | `s`, `del` разрешены и в фидах | багфикс | 1 |
+| 2 | `sanitize_html`: fail-open → fail-closed (`html.escape`); затрагивает и single-post/JSON путь | security | 1 |
+| 3 | `<hr class="post-divider">` реально виден в HTML-фиде | багфикс | 1 |
+| 4 | Несбалансированный HTML-фрагмент нормализуется в границах СВОЕГО поста и не искажает DOM последующих (только HTML-фид; RSS уже per-post и не меняется) | багфикс | 1 |
+| 5 | Fail-closed гранулярность HTML-фида: эскейпится упавший пост, а не весь фид | улучшение | 1 |
+| 6 | Merged-футер: custom-emoji реакции больше не агрегируются в один «❓ N» — отдельный span на каждую, как у одиночных постов | унификация | 2 |
+| 7 | Merged-футер: дата печатается из naive-local даты реального Message вместо UTC-мока — на серверах с TZ≠UTC видимый сдвиг. Golden (TZ=UTC) эту дельту НЕ видит — верифицируется выделенным тестом с не-UTC TZ | унификация | 2 |
+| 8 | Порядок флагов merged-поста детерминирован (first-seen order) | детерминизм | 2 |
+| 9 | FloodWait при получении истории пробрасывается → HTTP 429 (было: ValueError → 400) | фикс HTTP | 3 |
+| 10 | Тексты `detail` в 400-ответах унифицируются (исчезает суффикс «in HTML generation») | минорное | 3 |
+| 11 | None-date сообщения исключаются из ТАЙМ-кластеризации (не получают entry в маппинге; их собственный `media_group_id` продолжает действовать). Было: смешанный naive+None вход (прод-даты kurigram) ронял фид TypeError'ом; полностью-None и aware+None входы (возникают только в тестах с aware-моками) выживали и кластеризовали None-date хвост по порядку вставки, включая adoption truthy id. ВСЕ эти поведения заменяются; новое поведение закрепляется отдельными тестами как сознательное | багфикс | 4 |
+| 12 | Sort-ключи naive-безопасны (timestamp-based): None-date больше не роняет сортировку групп (краш жил в ДЕФОЛТНОМ пути, см. §1). Позиция None-date групп: фолбэк `+inf` в сортировке ГРУПП — детерминированно переживают `[:limit]`-срез как новейшие (теоретическая дельта: при постах с датой в будущем старый aware-путь ставил None-date «на сейчас», т.е. ниже них; прод-naive путь всё равно падал); позиция в итоговом выводе не меняется — существующий фолбэк `0.0` финальной сортировки постов ставит их в конец фида | багфикс | 4 |
+| 13 | Правило «>100 МБ не кэшируем» применяется к любому медиа-объекту, не только `message.video` | унификация | 5b |
+| 14 | Незакрытый `<div class="message-media">` закрывается во всех ветках | багфикс | 5b |
+| 15 | Пустой reactions-объект больше не даёт ведущий разделитель в футере (`_reactions_views_links` не добавляет пустую строку); затрагивает и одиночные посты | багфикс | 2 |
+| 16 | Логи ошибок санитайза объединяются под ОДНИМ именем `html_sanitization_error` + log_context (было три grep-имени: `html_sanitization_error` / `rss_html_sanitization_error` / `html_final_sanitization_error`) — правила grep-мониторинга обновить при деплое | наблюдаемость | 1 |
+
+## 4. Общие правила всех этапов
+
+- База: `refactor/render-pipeline` от актуального main (88ac436+, содержит
+  f9550d8 + спеку + корпус). Один этап = один коммит
+  (этап 5 — два: 5a/5b); после каждого — `pytest` полностью зелёный.
+- Правки существующих тестов допустимы в двух случаях: (а) тест закрепляет
+  старое поведение из реестра §3 — правка с комментарием-ссылкой на номер
+  пункта; (б) чистое перемещение API (импорты, monkeypatch-цели) — правка с
+  пометкой «API relocation, no behavior change».
+- Два слоя эталонов, не смешивать: **feed-level** golden-снапшоты (этап 0,
+  санитайженный выход RSS/HTML) и **fragment-level** снапшоты
+  `_generate_html_media` (этап 5a, до санитайза). Этап 5b не «чинит» то,
+  что уже изменил этап 1 на feed-уровне.
+- Комментарии в коде — только на английском. Точечные правки. Устаревшие
+  комментарии («4.4 coverage map» и т.п.) обновлять по ходу.
+- Наблюдаемость: имена существующих лог-строк и их контекст (счётчики
+  сообщений, `rss_date_range`, channel/message_id в ошибках санитайза)
+  сохраняются — grep-мониторинг не должен ослепнуть. Единственное
+  зарегистрированное исключение — объединение трёх имён санитайз-ошибок
+  (§3.16).
+- Новые тесты используют **naive**-даты (как отдаёт kurigram на проде), а не
+  aware-UTC.
+
+## 5. Этапы
+
+### Этап 0 — golden-эталоны фидов (до любых правок кода)
+
+**Корпус — записанные РЕАЛЬНЫЕ сообщения с прод-сервера, не синтетика (v6).**
+Источник: рабочий кэш бриджа `data/tgcache/` на сервере — там уже лежат
+пиклы ровно нужного формата: `{channel}.cache` =
+`{'timestamp', 'limit', 'messages': List[Message]}`
+(`tg_cache._save_history_to_cache`) и `{channel}.chatinfo` (для
+`cached_get_chat`). Отобранные ПАРЫ файлов лежат в
+`tests/test_data/recorded/` и коммитятся как замороженный корпус (контент
+этих публичных каналов попадает в репо).
+
+Реплей: тестовый загрузчик распикливает файлы напрямую (`timestamp`/`limit`
+игнорируются — никакой проверки свежести) и monkeypatch'ит
+`tg_cache.cached_get_chat_history` / `cached_get_chat` на возврат записанных
+объектов. Это буквально прод-путь cache-hit: сериализация и структура
+объектов — те же, что видит рендер в бою.
+
+Почему реальные, а не моки: (а) настоящие kurigram-объекты закрывают класс
+ложной зелени «мок и код согласованно ждут атрибут, которого нет у реального
+объекта» — холодный проход (§8, раунд 5) пометил его как неловимый на моках;
+(б) naive-даты, реальные entities/reactions/webpage-структуры и комбинации
+форматирования достаются бесплатно.
+
+**Первое действие этапа — проверка совместимости пиклов**: кэш сервера
+записан прод-версией kurigram, корпус распикливается под 2.2.23 (база
+рефакторинга f9550d8). ВЫПОЛНЕНО — см. «Статус» ниже: 90/90 + 89/89 без
+ошибок, фолбэк не понадобился.
+
+**Инвентаризация покрытия**: мини-скрипт проходит по корпусу и печатает
+media-типы, наличие media_groups / time-кластеров / forward / reply /
+реакций (обычные, custom, paid, пустой объект) / webpage с фото и без /
+зачёркивания (`<s>`/`<del>` в entities). Дыры покрытия закрываются
+СИНТЕТИЧЕСКОЙ добавкой — только для недостижимого в записанном.
+
+**Статус (2026-07-05): снапшот, проверка, инвентарь И ОТБОР уже выполнены.**
+Снапшот прод-кэша: `data/tgcache_prod_2026-07-05/` (90 `.cache` +
+89 `.chatinfo`, 53 МБ, вне git — `data/` в .gitignore). Отбор сделан жадно
+по матрице признаков, 4 канала (2.7 МБ) уже лежат в
+`tests/test_data/recorded/` (untracked; коммитятся первым коммитом этапа 0):
+
+- `bladerunnerblues` — единственный источник GIVEAWAY + GIVEAWAY_WINNERS;
+  плюс POLL, pdf, DOCUMENT, ANIMATION, r_paid (71), wp_nophoto;
+- `theyforcedme` — вся редкая медиа-палитра разом: STICKER, AUDIO, VOICE,
+  VIDEO_NOTE, POLL, ANIMATION; богат forward (13) и reply (10);
+- `embedoka` — r_empty + r_custom (35) + strike (7) + POLL + pdf +
+  wp_nophoto; forward 20, reply 14;
+- `meow_design` — 51 time-кластерная пара (ядро для time_based_merge),
+  media_groups 22, pdf, POLL, strike.
+
+Суммарно закрыта ВСЯ матрица: PHOTO 246, VIDEO 40, media_groups 66,
+forward 35, reply 27, wp_photo 11 + все редкие признаки выше.
+Совместимость подтверждена: ВСЕ файлы распикливаются под kurigram
+2.2.23 без ошибок. Инвентарь (8582 сообщения): PHOTO 5397, text-only 1609,
+WEB_PAGE 764 (с фото 581 / без 183), VIDEO 653, DOCUMENT 51, ANIMATION 33,
+POLL 31, VIDEO_NOTE 15, STICKER 13, AUDIO 8, VOICE 6, GIVEAWAY и
+GIVEAWAY_WINNERS по 1; media_group-членов 3159, forward 670, reply 357,
+time-кластерных пар (gap≤5s) ~294; реакции: обычных 17981, custom 1018,
+paid 593, ПУСТЫХ reactions-объектов 17 (§3.15-edge покрыт реальными
+данными!); зачёркиваний 150 (§3.1 покрыт); все 8582 даты naive (посылка
+спеки подтверждена); None-date — 0. Синтетика нужна ТОЛЬКО для: None-date
+(этап 4) и отсутствующих в корпусе типов — PAID_MEDIA, STORY, LIVE_PHOTO,
+CHECKLIST, CONTACT/LOCATION/VENUE/DICE/GAME/INVOICE/UNSUPPORTED
+(fragment-уровень этапа 5a).
+
+Сценарии exclude_flags/exclude_text в golden НЕ входят: фильтры не меняют
+байты выживших постов; их семантика (membership / regex) закрепляется парой
+unit-тестов — дешевле и меньше площадь флаки-рисков.
+
+**None-date пост в этап 0 НЕ входит** (в записанных данных его и не бывает):
+текущий код падает на нём в ДЕФОЛТНОМ пути (naive/aware TypeError в
+sort-ключе групп, см. §1). Синтетическая None-date фикстура добавляется в
+HTML-golden коммитом этапа 4 со ссылкой на §3.11–12.
+
+Снимаются и кладутся в `tests/test_data/golden/`:
+
+- RSS XML — полный выход `generate_channel_rss` (tg_cache замокан);
+- HTML-фид — полный выход `generate_channel_html`.
+
+**Детерминизм снапшотов (обязательные меры):**
+
+- TZ раннера пинится: `os.environ['TZ'] = 'UTC'; time.tzset()` в conftest
+  (naive-даты фикстур интерпретируются `.timestamp()`-ом в локальной TZ —
+  без пина pubDate плавает между машинами);
+- волатильные строки RSS XML нормализуются перед сравнением:
+  `<lastBuildDate>` (feedgen 1.0.0 ставит now() ОДИН раз в конструкторе
+  `FeedGenerator` — байты стабильны внутри процесса, но меняются между
+  прогонами) вырезается regex'ом и в golden, и в актуальном выводе;
+  `<generator>` в feedgen 1.0.0 версии НЕ содержит — его нормализация не
+  обязательна, оставлена как дешёвая страховка от апгрейда библиотеки;
+- корпус не содержит постов, где `pubDate` берётся из `datetime.now()`
+  (это только None-date случай — он исключён, см. выше);
+- порядок merged-флагов ДО этапа 2 недетерминирован: `list(set(...))`
+  (rss_generator.py:260–265) зависит от PYTHONHASHSEED процесса, который из
+  conftest не запинить — содержимое `<div class="message-flags">`
+  нормализуется сортировкой при сравнении golden; нормализация снимается
+  коммитом этапа 2 со ссылкой на §3.8;
+- ключ подписи media-URL пинится autouse-фикстурой
+  `monkeypatch.setattr(KeyManager, "signing_key", ...)` (образец —
+  tests/test_new_media_types.py): иначе golden, снятый на dev-машине,
+  краснеет на CI — свежий checkout генерирует новый `secrets.token_hex`
+  в `data/media_digest.key`, и все digest'ы в URL меняются;
+- корпус подаётся monkeypatch'ем `tg_cache.cached_get_chat` /
+  `cached_get_chat_history` (возврат распикленных записанных объектов) —
+  lazy-import внутри фид-функций разрешает имена поздно, поэтому патч модуля
+  tg_cache работает (образец — test_stage4_eventloop.py);
+- запись media-id пинится: monkeypatch `upsert_media_file_ids_bulk_sync`
+  (или `DB_PATH` → tmp_path) — иначе golden-тесты с медиа-фикстурами пишут
+  в реальную `./data/media_file_ids.db` (`DB_PATH` cwd-относителен,
+  file_io.py:13); байты фида это не меняет (flush глотает ошибки), но
+  side-effect вне tests/ недопустим; образец — test_stage4_eventloop.py:164;
+- фикстура time-based кластера требует `time_based_merge=True`: ключ читает
+  только `rss_generator.Config` — его патча достаточно; `post_parser.Config` —
+  независимый dict из того же `get_settings()`, патчить оба — дешёвая
+  страховка от будущего дрейфа, но не необходимость.
+
+Следствие пина TZ=UTC: дельту §3.7 (сдвиг TZ даты merged-футера) golden
+физически не видит — она верифицируется выделенным тестом этапа 2 с не-UTC TZ.
+
+Оракул эквивалентности всех этапов. Обновление golden — только коммитом со
+ссылкой на пункт §3. Ожидаемые изменения: этап 1 (пп.1, 3, 4; пп.2 и 5 —
+fail-closed ветки, golden их НЕ видит — верифицируются юнит-тестами
+sanitizer), этап 2 (пп.6, 8, 15 + снятие нормализации флагов), этап 4
+(добавление None-date фикстуры в HTML-golden, пп.11–12), этап 5b (п.14 — фикстура
+webpage-без-фото меняет и feed-level байты: до 5b незакрытый div
+дозакрывался bleach'ем в конце фрагмента).
+
+DoD: записанный корпус + отчёт инвентаризации + снапшоты в репо; тест
+сравнения зелёный и детерминированный (два прогона подряд — идентичные
+байты); любое изменение рендера валит его с внятным диффом. Пиклы корпуса
+сцеплены с версией kurigram — при апгрейде библиотеки корпус перезаписывается
+с сервера (зафиксировать в README тестов).
+
+### Этап 1 — `sanitizer.py` + санитайз в пайплайне (ишью #22, обновить)
+
+```python
+# sanitizer.py — the ONLY bleach configuration in the project.
+ALLOWED_TAGS = ['p', 'a', 'b', 'i', 'strong', 'em', 's', 'del',
+                'ul', 'ol', 'li', 'br', 'div', 'span',
+                'img', 'video', 'audio', 'source']          # union of the 3 old copies
+ALLOWED_ATTRIBUTES = { ... }        # identical in all 3 copies — move as-is
+ALLOWED_CSS_PROPERTIES = ["max-width", "max-height", "object-fit", "width", "height"]
+ALLOWED_PROTOCOLS = ['http', 'https', 'tg']   # non-default! tg:// links in footers
+
+# Shared instance is safe: CSSSanitizer only holds the allowed-properties list
+# (stateless config); bleach builds a fresh Cleaner per clean() call anyway.
+_CSS_SANITIZER = CSSSanitizer(allowed_css_properties=ALLOWED_CSS_PROPERTIES)
+
+def sanitize_html(html_raw: str, log_context: str = "") -> str:
+    """Sanitize one HTML fragment. FAIL-CLOSED: on any bleach error the
+    fragment is html.escape()d, never returned raw (stored-XSS guard).
+    log_context (e.g. "channel X, message_id Y") is included in error/slow
+    logs to keep operational grep-ability."""
+    # The FULL call — both non-default params are load-bearing:
+    #   clean(html_raw, tags=ALLOWED_TAGS, attributes=ALLOWED_ATTRIBUTES,
+    #         protocols=ALLOWED_PROTOCOLS, css_sanitizer=_CSS_SANITIZER,
+    #         strip=True)
+    # strip=True: disallowed tags are REMOVED (bleach default False would
+    # escape them into visible text). strip_comments stays default (True),
+    # matching all current call sites. Keep the >0.05s diag_sanitize_slow
+    # warning with input_len here.
+    # Error log name: html_sanitization_error (SINGLE name replacing the
+    # three per-path names — registry §3.16); log_context distinguishes
+    # the call sites, tests assert the name.
+```
+
+Правки:
+
+1. `post_parser._sanitize_html` (branch:702–737) → делегат в
+   `sanitizer.sanitize_html`. Fail-open ветка исчезает (§3.2).
+2. `_render_pipeline` получает параметр `channel` (только для логов) и после
+   `_render_messages_groups` (т.е. после фильтров) выполняет
+   `p['html'] = sanitize_html(p['html'], log_context=f"channel {channel}, message_id {p['message_id']}")`.
+   Комментарий: the pipeline already runs in a worker thread.
+   ВАЖНО: в этапах 1–2 близнецы ещё не слиты — аргумент `channel` добавляется
+   в ОБА вызова `to_thread(_render_pipeline, …)` (rss_generator.py:433 и 657).
+3. RSS-цикл (branch:458–497): удалить вложенный `_sanitize_sync` + `to_thread`;
+   `fe.content(content=post['html'], type='CDATA')` напрямую.
+4. HTML-путь (branch:671–705): удалить `_concat_html`/`_sanitize_sync` +
+   оба `to_thread`; `html = '\n<hr class="post-divider">\n'.join(...)` —
+   join ПОСЛЕ санитайза (§3.3, §3.4).
+5. Импорты bleach/CSSSanitizer из rss_generator удалить. Тест
+   test_stage4_eventloop.py:474 monkeypatch'ит `rss_module.HTMLSanitizer` —
+   перенацелить на `sanitizer` (API relocation).
+6. Обновить golden (§3 пп.1–5) и комментарии о границах санитайза.
+
+Тесты: `tests/test_sanitizer.py` — s/del выживают; script/onerror ВЫРЕЗАЮТСЯ
+(не эскейпятся в текст — проверка strip=True); `tg://` href выживает;
+fail-closed при исключении; hr-разделитель в HTML-фиде.
+
+DoD: одно определение allowed_tags в репо; в rss_generator нет bleach;
+pytest + golden зелёные.
+
+### Этап 2 — выпил `processed_message_to_tg_message`
+
+```python
+# Select the main RAW message with the same criterion the processed dicts used:
+# first message that has text or caption, else the first of the group.
+main_idx = next((i for i, m in enumerate(group) if (m.text or m.caption)), 0)
+main_raw = group[main_idx]
+main_message = processed_messages[main_idx]
+
+# Deterministic merged flags: first-seen order, then 'merged'.
+merged_flags = list(dict.fromkeys(f for msg in processed_messages for f in msg['flags']))
+merged_flags.append("merged")
+
+footer_html = post_parser.generate_html_footer(main_raw, flags_list=merged_flags)
+```
+
+Дополнительно (§3.15): в `_reactions_views_links` пустая `reactions_html` НЕ
+добавляется в `first_line_parts` (иначе после перехода на реальный Message
+merged-посты с пустым reactions-объектом получили бы ведущий разделитель —
+артефакт, который сегодня есть и у одиночных постов; чиним для всех).
+
+Удалить конвертер целиком и импорт `SimpleNamespace`. Обновить golden
+(§3 пп.6, 8, 15) и СНЯТЬ сорт-нормализацию `message-flags` в
+golden-сравнении, введённую этапом 0: порядок флагов теперь детерминирован
+(§3.8), нормализация больше не нужна и лишь маскировала бы регрессии.
+
+Тесты: футер merged-группы — реальные ссылки главного сообщения; несколько
+custom-emoji → отдельные «❓ N» span'ы; paid → «⭐ N»; пустой reactions-объект →
+нет ведущего разделителя (и для одиночного поста); порядок флагов стабилен;
+**выделенный тест §3.7 с не-UTC TZ** (например `TZ=Europe/Moscow` + `tzset`):
+дата merged-футера совпадает с датой футера одиночного поста того же
+сообщения. TZ ОБЯЗАТЕЛЬНО восстанавливается в teardown (fixture/try-finally),
+иначе не-UTC протечёт в golden-тесты, которым conftest запинил UTC.
+
+DoD: функция удалена, pytest + golden зелёные.
+
+### Этап 3 — слияние близнецов вокруг `_prepare_feed_posts`
+
+```python
+@dataclass
+class PreparedFeed:
+    channel_username: str
+    channel_title: str            # used by RSS metadata only
+    posts: list[dict]             # rendered, filtered, sorted, SANITIZED
+
+class ChannelNotFound(Exception):
+    def __init__(self, channel_identifier):   # prepared id, for create_error_feed
+        self.channel_identifier = channel_identifier
+
+# NO timed() abstraction: it could not carry the paired rss_/html_ log-line
+# names nor the extra context (message counts) anyway. Keep today's explicit
+# logger.debug lines, prefixed via log_prefix.
+
+async def _prepare_feed_posts(channel, client, *,
+                              limit, exclude_flags, exclude_text, merge_seconds,
+                              history_limit, enrich_replies: bool,
+                              log_prefix: str) -> PreparedFeed:
+    # log_prefix: 'rss' | 'html' — preserves today's paired log-line names
+    # (f"{log_prefix}_channel_info_timing", f"{log_prefix}_messages_retrieval_timing", ...)
+    # 1) validate limit (1..200)
+    # 2) channel_name_prepare + cached_get_chat:
+    #    UsernameInvalid/UsernameNotOccupied/no-username -> ChannelNotFound(prepared)
+    #    FloodWait -> re-raise (api_server maps to 429)
+    #    other errors -> ValueError chain (unified text, §3.10)
+    # 3) cached_get_chat_history(limit=history_limit):
+    #    FloodWait -> re-raise BEFORE the ValueError wrap (§3.9)
+    # NOTE: keep `from tg_cache import ...` INSIDE this function — feed tests
+    # monkeypatch tg_cache and rely on late name resolution.
+    # 4) if enrich_replies: messages = await _reply_enrichment(client, messages)
+    # 5) try: to_thread(_render_pipeline, ...) finally: flush_pending_media_ids()
+```
+
+Форматтеры:
+
+- `generate_channel_rss`: `history_limit=limit*2, enrich_replies=False,
+  log_prefix="rss"` (RSS over-fetches so merging still yields ~limit posts) →
+  fg-метаданные + entries (`rss_date_range`-лог сохраняется) →
+  `to_thread(fg.rss_str)`.
+- `generate_channel_html`: `history_limit=limit, enrich_replies=True,
+  log_prefix="html"` (enrichment is HTML-only to keep RSS polling cheap —
+  deliberate) → join с `<hr>`.
+- Оба: `except ChannelNotFound as e: return create_error_feed(str(e.channel_identifier), base_url)`
+  (байт-эквивалентно текущему выводу: prepared-значение уже используется
+  сегодня, int/str интерполируются одинаково — проверено в ревью).
+- Внешние catch-логи ОСТАЮТСЯ в форматтерах со своими сегодняшними именами
+  (`generate_channel_rss: …` — rss_generator.py:526, `html_generation_error:
+  …` — 716): каждый форматтер сохраняет собственный внешний try/except.
+
+Тесты: golden без изменений (главный критерий); ChannelNotFound → error feed
+в обоих путях; FloodWait из get_chat → 429; FloodWait из истории → 429
+(новый, §3.9); ValueError из истории → 400.
+
+DoD: один блок get_chat/get_history/render; diffstat отрицательный;
+pytest + golden зелёные.
+
+### Этап 4 — группировка без deepcopy и мутаций
+
+```python
+def _compute_time_based_group_ids(messages, merge_seconds) -> dict[int, str | int]:
+    """Return {message.id: effective_media_group_id} WITHOUT mutating messages.
+
+    Contract: all messages belong to ONE chat (message.id is unique only
+    per chat); callers must not mix chats in a single call.
+
+    Reproduces the old algorithm for every input the old code survived on
+    PRODUCTION data (naive kurigram dates). Inputs only aware-date test mocks
+    could produce (fully-None and aware+None mixes, where the old code
+    clustered the None-date tail by insertion order, incl. truthy-id
+    adoption) are deliberately replaced — registry §3.11:
+    - messages WITHOUT a date do not participate in time clustering and get
+      NO mapping entry (their own media_group_id still applies downstream) —
+      registry §3.11;
+    - sort by date ascending, timestamp-based key (naive-safe);
+    - a message joins the current cluster if the gap to the PREVIOUS message
+      is <= merge_seconds; the gap is computed by NAIVE datetime subtraction
+      (msg.date - prev.date).total_seconds(), exactly as the old code — NOT
+      via timestamps (they diverge during a DST fold, and the old behavior
+      is the contract);
+    - effective id of a cluster = the FIRST TRUTHY media_group_id seen in
+      cluster order (old code used truthiness, not `is not None`; it also
+      overwrote members' own differing ids — keep that);
+    - if no member has a truthy id and len(cluster) >= 2:
+      synthetic id f"time_{min(dates)}" (keep the exact format);
+    - singleton clusters and clusters with no effective id produce NO
+      entries. Every member of a cluster with an effective id gets an entry.
+    """
+```
+
+Правки:
+
+- `_create_messages_groups(messages, group_ids=None)`:
+  `effective = (group_ids or {}).get(message.id, message.media_group_id)`;
+  sort-ключ групп — timestamp-based (naive-безопасный), фолбэк для None-date
+  групп `float('inf')` → детерминированно переживают `[:limit]`-срез как
+  новейшие; финальная сортировка постов в `_render_messages_groups`
+  (фолбэк `0.0`) НЕ меняется — None-date посты выводятся в конце фида (§3.12).
+- `_render_pipeline` при `time_based_merge`: маппинг + пред-сортировка входа
+  date-ASC тем же timestamp-ключом (фолбэк `+inf` — None-date в конце);
+  стабильный sorted на fetch-order входе воспроизводит порядок старого
+  `messages_sorted`, включая ties (старый sorted работал на deepcopy того же
+  fetch-order списка, мутации происходили после сортировки — проверено).
+  `sorted()` не мутирует вход — deepcopy не нужен.
+- Удалить `_create_time_based_media_groups` и импорт `copy`; поправить
+  модульный импорт в test_stage4_eventloop.py:34–42 (API relocation).
+- Добавить None-date фикстуру ТОЛЬКО в HTML-golden (§3.11–12): в RSS
+  None-date пост получает `pubDate = datetime.now()` (rss_generator.py:
+  503–507) — недетерминированно, и новую нормализацию для этого не вводим;
+  RSS-семантика None-date закрепляется unit-тестом (pubDate присутствует,
+  entry сгенерирован), вне golden.
+
+Тесты (`tests/test_group_ids.py`, naive-даты): time-кластер без id →
+синтетический; усыновление truthy id с backfill и перезаписью чужого id;
+falsy id (`0`, `""`) игнорируется как в старом коде; одиночка без entry;
+None-date не получает entry в маппинге, но его собственный `media_group_id`
+продолжает действовать (медиагруппа с None-датами собирается); смешанный
+None-date вход не падает (§3.11); полностью-None вход: новое поведение
+закреплено явно; None-date группы переживают `[:limit]`-срез (ключ групп
+`+inf`), в итоговом выводе — в конце фида (финальный фолбэк `0.0`, §3.12);
+вход не мутирован;
+ties при равных датах → порядок как у старого кода; кластеризация через
+DST-fold — как у старого кода (naive-вычитание).
+
+DoD: `copy.deepcopy` отсутствует; pytest + golden зелёные (golden-дифф — только
+добавленная None-date фикстура, §3.11–12).
+
+### Этап 5 — таблица медиа-типов в post_parser (два коммита)
+
+**5a — рефакторинг байт-в-байт.**
+
+Шаг 0: fragment-level снапшоты `_generate_html_media` (ДО санитайза) для всех
+типов: PHOTO, VIDEO, ANIMATION, VIDEO_NOTE, AUDIO, VOICE, STICKER img/video,
+DOCUMENT pdf/обычный, LIVE_PHOTO, STORY video/photo, POLL с description_media,
+PAID_MEDIA, WEB_PAGE с фото (пустой media-div!), **и edge-ветки**: WEB_PAGE
+без фото (незакрытый div — воспроизводится в 5a буквально), file_unique_id
+is None, channel_username is None, гейт webpage-превью «text ≤ 10».
+
+Источник сообщений для fragment-снапшотов (v6): ЗАПИСАННЫЙ КОРПУС этапа 0 —
+для всех типов, которые в нём есть (по инвентарю: PHOTO, VIDEO, ANIMATION,
+VIDEO_NOTE, AUDIO, VOICE, STICKER, DOCUMENT, POLL, WEB_PAGE, GIVEAWAY,
+GIVEAWAY_WINNERS). Моки — ТОЛЬКО для отсутствующих в корпусе типов
+(PAID_MEDIA, STORY, LIVE_PHOTO, CHECKLIST и прочая экзотика) — ограничение
+инвариант-теста «ложная зелень на моках» сужается до этих типов.
+
+```python
+# Single source of truth: media type -> (object selector, render kind).
+# kind=None means "selector-only entry": the object participates in file-id
+# extraction/collection, but rendering happens outside the dispatcher.
+# The ONLY kind=None entry is WEB_PAGE (rendered by _format_webpage).
+# PAID_MEDIA has NO entry at all: its info block stays a separate branch in
+# _generate_html_media, and it is deliberately not collected/downloadable.
+MEDIA_SOURCES: dict[MessageMediaType, Callable[[Message], tuple[Any, str | None]]] = {
+    MessageMediaType.PHOTO:    lambda m: (m.photo, 'img_400'),
+    MessageMediaType.DOCUMENT: _select_document,   # returns kind 'pdf' OR 'img_400' by mime_type
+    MessageMediaType.STICKER:  _select_sticker,    # video_loop_200 vs img_200_sticker
+    MessageMediaType.STORY:    _select_story,      # maps helper kinds video->'video_400', img->'img_400'
+    MessageMediaType.POLL:     _select_poll_media, # same helper-kind mapping
+    MessageMediaType.WEB_PAGE: lambda m: (getattr(m.web_page, 'photo', None), None),
+    # VIDEO/ANIMATION/VIDEO_NOTE -> video_400; AUDIO/VOICE -> audio;
+    # LIVE_PHOTO -> video_loop_400
+}
+
+@dataclass
+class RenderCtx:
+    url: str                      # signed /media URL — assembled by _generate_html_media,
+                                  # which KEEPS the digest call and the channel_username
+                                  # guard inline (they are not the renderer's business)
+    tg_link: str | None = None    # t.me deep link — only the 'pdf' renderer uses it
+    emoji: str = ''               # sticker alt text
+    mime: str | None = None       # audio/voice source type; the DEFAULT is chosen
+                                  # by the ctx builder in _generate_html_media BY
+                                  # MEDIA TYPE (AUDIO -> audio/mpeg, VOICE ->
+                                  # audio/ogg, branch:907/912) — the renderer
+                                  # receives a ready value and never guesses
+
+# Renderers, NOT naked format strings: a renderer returns list[str] so the
+# byte structure of '\n'.join is preserved (audio emits TWO items: tag + <br>;
+# pdf emits its two-append div block). Renderer bodies are lifted VERBATIM
+# from the existing if/elif branches — including the `src="{url}"style=`
+# concatenation artifacts — byte-for-byte fidelity is the 5a contract.
+RENDERERS: dict[str, Callable[[RenderCtx], list[str]]] = { ... }
+```
+
+PDF идёт ЧЕРЕЗ таблицу: `_select_document` возвращает kind `'pdf'` для
+`application/pdf`, `RENDERERS['pdf']` использует `ctx.tg_link` (сборка ссылки
+`t.me/c/...` vs `t.me/...` — в `_generate_html_media`, как сейчас).
+
+Потребители: `_get_file_unique_id`, `_save_media_file_ids`,
+`_generate_html_media` — все через селектор. Ограничение: `flags.append(...)`
+НЕ выносить из `_extract_flags` (эндпоинт `/flags` парсит его исходник через
+`inspect.getsource`); тест: `get_all_possible_flags()` непуст и содержит
+известные флаги.
+
+Межмодульный инвариант-тест (границу с api_server закрепить машинно):
+для каждого типа из MEDIA_SOURCES объект, выбранный селектором, находится
+`api_server.find_file_id_in_message` по своему `file_unique_id` — новые entry
+таблицы не могут дать URL, который download-путь не разрешит (иначе /media
+404). Ограничение теста: обе стороны работают на моках — класс багов «обе
+функции ждут атрибут, которого нет у реального kurigram-объекта» он не ловит;
+опциональная best-effort митигация — сверка атрибутов моков с
+`pyrogram.types` (сама интроспекция хрупка при апгрейдах kurigram; ядро
+теста ценно и без неё, обязательной её не делать).
+Сама `find_file_id_in_message` НЕ сливается с таблицей: её контракт шире —
+поиск любого скачиваемого объекта независимо от `message.media`, включая
+`explanation_media`, который рендер сознательно игнорирует (§7).
+
+**5b — зарегистрированные фиксы** (отдельный коммит): закрыть
+`</div>` во всех ветках (§3.14); guard «>100 МБ» на любой медиа-объект
+(§3.13); обновить fragment-снапшоты с diff-комментарием. Feed-level golden
+5b не трогает сверх этих пунктов.
+
+DoD: прямые атрибутные ЦЕПОЧКИ вида `m.photo.file_unique_id` — только в
+селекторах таблицы, хелперах `_poll_media_object`/`_story_media_object`
+(это и есть реализация селекторов) и `_format_webpage`. Потребители
+извлекают uid единообразно — `getattr(selected_obj, 'file_unique_id', None)`
+от объекта, ВОЗВРАЩЁННОГО селектором; санкционированные точки извлечения:
+`_get_file_unique_id`, `_save_media_file_ids`, инвариант-тест. Три лестницы
+удалены; снапшоты обоих уровней + pytest зелёные; инвариант-тест с
+api_server зелёный.
+
+### Этап 6 — косметика
+
+1. `_wrap_post_html(body, footer)` — единая обёртка (сейчас три копии:
+   `_format_html` и обе ветки `_render_messages_groups`).
+2. Инлайн-стили 400px/200px → константы (если не закрыто этапом 5).
+3. `_trim_messages_groups` → инлайн-срез `[:limit]`; поправить импорт в
+   test_stage4_eventloop.py (API relocation).
+4. Комментарии «stage-4»/«4.4» — переписать под новую границу санитайза.
+5. Фильтр exclude_flags → comprehension; фильтр exclude_text — оставить
+   циклом или обернуть предикатом, но `excluded_post`-debug-лог СОХРАНИТЬ
+   (единственный след, почему пост выпал из фида).
+
+## 6. Порядок и зависимости
+
+`0 → 1 → 2 → 3` строго последовательно. `4` и `5` независимы, после 3.
+`6` — последним. Каждый этап мержибелен сам по себе.
+
+## 7. Сознательно НЕ делаем (отложено)
+
+- Унификация глубины истории RSS (`limit*2`) vs HTML (`limit`).
+  Симптом: кэш истории — ОДИН файл на канал, limit хранится внутри и при
+  несовпадении — miss (tg_cache.py:43–52, 106–109); канал с обоими
+  потребителями живёт в вечном miss-цикле со взаимным вытеснением и
+  удвоенным RPC. НЕ отложено — чинится в отдельном эпике кешей (ишью #23,
+  Package B: префиксная выдача истории, `cached_limit >= limit` → hit).
+  Здесь ничего делать не нужно, но при мёрже учитывать пересечение (см.
+  комментарий к эпику #34).
+- `_reply_enrichment` в RSS-пути (RPC-нагрузка от частых опросов ридеров).
+- HTML-страница ошибки для `generate_channel_html` (сейчас RSS-XML).
+- Кэширование/демутация `_reply_enrichment` (мутирует кэшированные Message).
+- `api_server.find_file_id_in_message`: остаётся отдельной (контракт шире
+  рендера — см. этап 5a), граница закрыта инвариант-тестом.
+- Валидация `merge_seconds` в api_server (сейчас принимает `<=0` из query).
+- Валидация `exclude_text` (невалидный regex → `re.error` → 500 и сейчас,
+  и после этапа 3 — api_server пробрасывает параметр как есть).
+- CDATA-риск feedgen: html5lib не эскейпит `>` в атрибутах, `]]>` внутри href
+  теоретически рвёт CDATA. Пре-существующий, вне скоупа.
+- `/flags` через `inspect.getsource` — хрупкая механика, замена отложена.
+
+## 8. Журнал адверсариального ревью
+
+### Раунд 1 (по v1): 26 претензий (2 blocker, 8 major, 12 minor, 4 nit)
+
+- Приняты полностью и внесены: №3 (protocols), №4 (правки тестов при API
+  relocation), №5 (golden-оракул → этап 0), №6 (TZ merged-футера, §3.7),
+  №7 (FloodWait истории → 429, §3.9), №11 (§3.10), №12 (lazy-import пин),
+  №13 (наблюдаемость, log_context), №14 (реальная дельта реакций, §3.6),
+  №16+№15 (naive/aware краш → §3.11–12, truthiness), №18 (edge-матрица
+  снапшотов), №19 (payload ChannelNotFound), №20 (§3.5), №21 (обоснование
+  шареного CSSSanitizer), №22 (/flags-мина), №23 (excluded_post-лог),
+  №24 (контракт one-chat), №25 (§2), №26 (CDATA в §7).
+- №1 (blocker): снято разделением этапа 5 на 5a/5b и двумя слоями эталонов.
+- №2 (blocker): понижено до пункта реестра §3.4 по согласованной формулировке.
+- №8: контракты `find_file_id_in_message` и MEDIA_SOURCES признаны разными;
+  граница закрыта инвариант-тестом по предложению критика.
+- №9: закрыт пред-сортировкой входа naive-безопасным ключом.
+- №10: база запинена на f9550d8.
+- №17: `TAG_TEMPLATES: dict[str, str]` заменён на renderer-функции.
+
+### Раунд 2 (по v2): 13 претензий (1 blocker, 4 major, 5 minor, 3 nit)
+
+- №1 (blocker): краш naive/aware живёт в ДЕФОЛТНОМ пути сортировки групп, а
+  не только в тайм-кластеризации (подтверждено эмпирически) → §1 исправлен,
+  None-date фикстура перенесена из этапа 0 в этап 4 (вариант «а» критика),
+  §3.12 уточнён.
+- №2: направление дельты пустых реакций было инвертировано → выбран
+  fix-вариант: §3.15 (чинится для всех постов), формулировка §3.6 очищена.
+- №3: `strip=True` внесён в спеку полным вызовом `clean()` (тот же класс
+  дыры, что protocols в раунде 1).
+- №4: детерминизм golden обеспечен (TZ-пин, нормализация lastBuildDate/
+  generator); зафиксировано, что §3.7 golden не верифицирует → выделенный
+  не-UTC тест в этапе 2.
+- №5: противоречие DOCUMENT/PDF разрешено в пользу «PDF через таблицу»,
+  ctx расширен полем tg_link.
+- №6: RenderCtx специфицирован (поля, владелец сборки URL, verbatim-правило).
+- №7: базис gap зафиксирован — naive-вычитание как в старом коде (timestamp
+  расходится в DST-fold).
+- №8: позиция None-date групп зарегистрирована (§3.12: новейшие, `+inf`).
+- №9: формулировка теста исправлена («нет entry в маппинге», а не «синглтон»).
+- №10: ограничение инвариант-теста (ложная зелень на моках) проговорено +
+  митигация.
+- №11: `_render_pipeline` получает `channel` для log_context (этап 1 п.2).
+- №12: диаграмма §2 исправлена (фильтры/sort внутри `_render_messages_groups`,
+  санитайз после фильтров).
+- №13: DoD 5a включает хелперы `_poll_media_object`/`_story_media_object`.
+- Чистыми признаны: эквивалентность error feed (B5), §3.9 против tg_cache и
+  429/Retry-After (B6), эквивалентность пред-сортировки (B3), внесение всех
+  исходов раунда 1.
+
+### Раунд 3 (верификация дельты v3)
+
+- Пять из шести выборов подтверждены (None-date → этап 4; fix пустых реакций;
+  PDF через таблицу; naive-gap; полный clean()).
+- Возражение по №5 принято: `+inf` определяет только выживание группы при
+  `[:limit]`-срезе, итоговую позицию задаёт финальная сортировка с фолбэком
+  `0.0` (None-date посты — в конце фида, как и раньше) — §3.12, этап 4 и его
+  тест переформулированы.
+- Криво внесённое исправлено: список ожидаемых golden-изменений дополнен
+  этапом 5b (п.14); исключения single-post пути в §2 расширены до
+  пп. 2, 13, 14, 15.
+- Практические заметки внесены: teardown TZ в не-UTC тесте §3.7; два
+  независимых Config-дикта при monkeypatch `time_based_merge`.
+
+### Раунд 4 (по v3): 17 пунктов (2 major, 4 minor, остальное nit/экономика)
+
+- Детерминизм golden добит двумя major-находками: порядок merged-флагов до
+  этапа 2 зависит от PYTHONHASHSEED (`list(set)`) → сорт-нормализация
+  `message-flags` до этапа 2; ключ подписи media-URL (`secrets.token_hex` в
+  `data/media_digest.key`) → пин autouse-фикстурой.
+- Согласован состав фикстур: добавлен strikethrough-пост (иначе §3.1
+  невидим), пп.2/5 помечены как невидимые для golden; exclude-сценарии
+  вынесены из golden в unit-тесты (экономическая критика).
+- `timed()` выкинут: не мог сохранить парные `rss_/html_` имена логов и не
+  окупался — заменён параметром `log_prefix` у `_prepare_feed_posts`.
+- Кодер-готовность: оба call-site'а `_render_pipeline` в этапах 1–2
+  проговорены; владелец mime-дефолта — ctx-билдер по media-типу; указатель
+  на паттерн monkeypatch tg_cache в этапе 0; обоснование двойного
+  Config-патча исправлено (достаточно rss_generator.Config).
+- Митигация инвариант-теста через интроспекцию `pyrogram.types` понижена до
+  опциональной (сама хрупка при апгрейдах).
+- §7 пополнен: вечный miss-цикл кэша истории у dual-consumer каналов (цена
+  отложенной унификации глубины), валидация exclude_text.
+- C-остатки признаны чистыми: частичный flush и дубли upsert закреплены
+  существующими тестами; лог одиночного санитайза контекста и сегодня не
+  имеет; api_server пробрасывает exclude_* без валидации — поведение не
+  меняется.
+- Экономический вердикт критика: аппарат тяжёл для ~700 строк рефакторинга,
+  но каждый элемент отвечает конкретному найденному багу; срезаны только
+  `timed()`, exclude-фикстуры и обязательность интроспекции. Этапы не
+  сливать: раздельная мержибельность дешевле.
+
+### Раунд 5 (по v4): независимый холодный проход, 8 пунктов (1 blocker, 1 major)
+
+Второй критик — без контекста предыдущих раундов, с запретом доверять
+журналу и реестру на слово. Итог валидации: ВСЕ line-refs §1 и ключевые
+эмпирические утверждения спеки подтверждены независимо (включая вырезание
+hr, заглатывание постов незакрытым div, strip/protocols-семантику, TypeError
+в дефолтном пути, miss-цикл кэша, соответствие MEDIA_SOURCES реальной
+лестнице вплоть до артефактов, находимость всех селекторных объектов в
+find_file_id_in_message); архитектурные решения признаны чистыми. Найдены
+дефекты исполнимости:
+
+- Blocker: None-date фикстура в RSS-golden недетерминированна
+  (`pubDate = now()` для None-date entry) → фикстура включается только в
+  HTML-golden, RSS-семантика — unit-тестом (этап 4 переписан).
+- Major: три имени санитайз-логов физически сливаются в одно при делегате —
+  противоречие с §4 → зарегистрировано как §3.16 (единое
+  `html_sanitization_error` + log_context); внешние catch-логи форматтеров
+  явно оставлены в этапе 3.
+- Механизм волатильности feedgen уточнён (lastBuildDate — один раз в
+  конструкторе, не при сериализации; generator без версии — нормализация
+  опциональна).
+- §3.11/контракт этапа 4: «Было» дополнено выжившими aware+None и
+  fully-None входами (кластеризация None-хвоста с adoption), заголовок
+  docstring сужен до «survived on production data».
+- DoD 5a переписан: единообразное извлечение uid через
+  `getattr(selected_obj, ...)` с санкционированными точками (сигнатура
+  селектора uid не возвращает — старая формулировка была невыполнима).
+- Этап 0: добавлен пин записи media-id (cwd-относительный DB_PATH писал бы
+  в реальную data/ из golden-тестов); поправлен line-ref api_server;
+  каветт §3.12 про будущие даты.
+
+Отклонённых претензий нет; по всем спорным пунктам достигнут консенсус.
+
+### v6 — golden-корпус: записанные реальные сообщения (решение владельца)
+
+Не раунд ревью — изменение дизайна по решению владельца проекта, с
+немедленной эмпирической проверкой:
+
+- Корпус этапа 0 переведён с синтетики на записанные кэши прод-сервера
+  (`data/tgcache/*.cache|.chatinfo` — пиклы того же формата, что читает
+  прод-путь cache-hit). Мотив: холодный проход (раунд 5) пометил класс
+  «мок и код согласованно ждут несуществующий атрибут» как неловимый на
+  моках — реальные объекты закрывают его целиком.
+- Снапшот кэша снят (90 каналов), совместимость пиклов со старым kurigram
+  проверена под 2.2.23: 179/179 файлов без ошибок.
+- Инвентарь покрытия: 8582 сообщения, все даты naive (посылка спеки
+  подтверждена данными); покрыты в т.ч. edge-кейсы §3.1 (зачёркивания: 150)
+  и §3.15 (пустые reactions-объекты: 17), webpage с/без фото, ~294
+  time-кластерные пары. Синтетика сузилась до None-date и отсутствующих
+  типов (PAID_MEDIA, STORY, LIVE_PHOTO, CHECKLIST, прочая экзотика).
+- Fragment-снапшоты этапа 5a — тоже на корпусе, где тип доступен;
+  ограничение инвариант-теста сузилось до mock-only типов.

plans/cache-optimization-spec.md

@@ -0,0 +1,619 @@
+# Спецификация: рефакторинг и оптимизация кешей pyrogram-bridge
+
+Статус: **принята владельцем 2026-07-05. Адверсарное ревью — 4 раунда, возражений не осталось.**
+История ревью — раздел 12. Реализация разбита на ишью в gitea (по PR-юнитам: A+B+F, C, D, E).
+
+## 0. Контекст и цели
+
+В проекте три слоя кеша:
+
+1. **История сообщений и инфо о канале** — pickle-файлы в `data/tgcache/` (`tg_cache.py`).
+2. **Медиафайлы** — файлы в `data/cache/<channel>/<post_id>/<file_unique_id>` + метаданные в SQLite `data/media_file_ids.db` (`api_server.py`, `file_io.py`).
+3. **Runtime-структуры** — `_access_updates`, `_inflight`, `download_queue` в `api_server.py`.
+
+Проблемы, которые закрывает спецификация (по результатам аудита):
+
+| # | Проблема | Пакет |
+|---|---|---|
+| 1 | Pickle полных `Message` — формат привязан к версии pyrogram/kurigram, тихий дрейф схемы, opaque-формат | A |
+| 2 | Строгое `cached_limit == limit` + разные limit у RSS (`limit*2`) и HTML (`limit`) → кеш истории фактически не работает | B |
+| 3 | Один канал живёт под ключами разного регистра/формы записи (`Durov`/`durov`, `@name`/`name`) в трёх слоях → дубли кеша. Унификация ID↔username — сознательная НЕ-цель (см. раздел 8) | C |
+| 4 | Два почти идентичных pickle-кеша в `tg_cache.py` (история/chatinfo) — дублирование кода | A |
+| 5 | Legacy-мусор (`*_history.cache`, двойной pickle, `data/media_file_ids.json`); чистка `data/tgcache` вводится как стартовая + периодический age-sweep (мгновенной GC по событию не будет — файлы мёртвых каналов живут до 7 суток) | A, F |
+| 6 | Свипер медиа-кеша: полный проход таблицы + `os.walk` каждые 60 с при политиках «20 дней»/«1 час» | D |
+| 7 | Баг: чистка «мёртвых» строк SQLite выполняется только при `files_removed > 0` | D |
+| 8 | Чтение MIME из SQLite (новое соединение + threadpool-hop) на каждом хите `/media` | E |
+| 9 | Джиттер TTL перебрасывается на каждом чтении → недетерминированное поведение у границы TTL (усложняет рассуждения и тесты) | F |
+| 10 | Очередь фоновых загрузок без дедупа — повторная постановка одного файла каждым проходом свипа | D |
+| 11 | Путь медиа-кеша (`./data/cache` + join) собирается вручную в 7 местах | E |
+
+## 1. Состав пакетов и порядок
+
+| Пакет | Задания | Что | Зависимости | Файлы |
+|---|---|---|---|---|
+| **A** | 1–5 | Снапшот-словари вместо pickle + generic JSON-store + чистка legacy tgcache | — | `message_snapshot.py` (новый), `tg_cache.py`, `api_server.py` (1 строка), тесты |
+| **B** | 6–7 | Префиксная выдача истории вместо строгого `limit` | после A | `tg_cache.py`, тесты |
+| **C** | 8–11 | Единая канонизация ключа канала + one-shot миграция | **после A** (задание 9.1 канонизирует ключ внутри `_cache_file_path`, который создаётся заданием 2) | `channel_key.py` (новый), `tg_cache.py`, `post_parser.py`, `api_server.py` |
+| **D** | 12–14 | Фиксы свипера медиа-кеша + дедуп очереди | независим | `api_server.py`, `config.py` |
+| **E** | 15–16 | Хелпер путей медиа-кеша + in-memory MIME-кеш | независим | `api_server.py` |
+| **F** | 17–18 | Джиттер TTL при записи + age-sweep tgcache + добить legacy | после A | `tg_cache.py`, `api_server.py` |
+
+Рекомендуемая сборка: A → B → F одним PR (все правят `tg_cache.py`); C — отдельный PR (меняет генерацию URL и содержит миграцию); D и E — параллельно с любым.
+
+---
+
+## 2. Пакет A — кеш истории на извлечённых словарях
+
+### 2.1. Принцип
+
+Кеш истории перестаёт хранить pickle живых объектов pyrogram. При записи из `Message`
+извлекается JSON-словарь по явному allowlist полей (снапшот); при чтении из него
+восстанавливается лёгкий duck-typed объект `CachedMessage`, неотличимый для конвейера
+рендеринга (`rss_generator.py`, `post_parser.py`) от настоящего `Message`.
+**Конвейер не меняется вообще** — главный инвариант дизайна.
+
+Выгоды: формат на диске отвязан от версии pyrogram/kurigram; версионирование схемы
+(несовпадение → честный miss вместо тихого дрейфа); JSON читаем при отладке;
+явный документированный контракт «от каких полей Message зависит рендер»
+(сейчас он размазан по ~2000 строк); файл меньше и парсится быстрее полного графа объектов.
+Отклонённая альтернатива — раздел 11.
+
+### 2.2. Ключевые проектные решения
+
+| # | Решение | Обоснование |
+|---|---|---|
+| Р1 | Новый модуль `message_snapshot.py`: `snapshot_message()` / `restore_message()` | Схема сериализации отделена от механики кеша; меняются независимо |
+| Р2 | `text`/`caption` хранятся парой `{plain, html}`; `.html` вычисляется **при записи** через pyrogram `Str.html` | Конвертация entities→HTML требует машинерии pyrogram, доступной на живом объекте (`post_parser.py:674-675`). Проверено: `Str.html` работает офлайн, без клиента |
+| Р3 | При восстановлении `text`/`caption` — `CachedStr(str)` с атрибутом `.html` | Потребители используют и строковые операции (`len`, `strip`, regex, `or`), и `.text.html`; str-подкласс покрывает всё. Это паттерн pyrogram `Str` |
+| Р4 | `media` восстанавливается в настоящий `MessageMediaType[name]`; `KeyError` → `None` + warning | 31 место сравнивает с enum и использует его ключом dict (`post_parser.py:1004-1017`) |
+| Р5 | `service` хранится и восстанавливается **строкой-именем** (`"PINNED_MESSAGE"`) | Все потребители — truthiness и `'X' in str(service)` (`rss_generator.py:112-121`, `post_parser.py:305`). Строка версионно-устойчива |
+| Р6 | `date` — `isoformat()` туда, `fromisoformat()` обратно | Сохраняет naive/aware ровно как у pyrogram; `strftime`, `timestamp()`, сортировки работают |
+| Р7 | Вложенные объекты восстанавливаются с **полным набором ключей схемы, None-дефолты** — «как живые». **Единственное исключение — `forward_origin`**: у него присутствие ключа зеркалит присутствие атрибута на исходном объекте | Живые pyrogram-объекты всегда имеют все атрибуты (ставятся в `__init__`, напр. `Reaction`: `emoji=None, custom_emoji_id=None, is_paid=None`) — код обращается к вложенным полям и напрямую (`rss_generator.py:131` — `message.chat.username` в except-хендлере; `post_parser.py:1087-1090` — `u.username`/`u.active` без getattr), omit-None дал бы там `AttributeError` → 500 на фид. А вот живые `MessageOrigin*`-классы имеют РАЗНЫЕ наборы атрибутов, и `_format_forward_info` ветвится по `hasattr` (Case 1–5, `post_parser.py:629-669`) — для forward_origin присутствие ключей семантично |
+| Р8 | `CachedMessage` — мутабельный, полный набор top-level атрибутов с дефолтами `None`/`False` | `_reply_enrichment` присваивает `message.reply_to_message` (`rss_generator.py:574`); доступ к атрибутам не должен кидать `AttributeError`; `copy.deepcopy` (`rss_generator.py:43`) должен работать |
+| Р9 | Файл: JSON `{"version", "timestamp", "limit", "messages"}`, имена `<key>.history.json` / `<key>.chatinfo.json`. Атомарная запись: **уникальный tmp `<path>.tmp.<uuid4().hex>`** → `os.replace`, в `finally` — удаление своего tmp (образец: `_download_atomic`, `api_server.py:460-479`) | Версия отсекает старые схемы; новые расширения игнорируют старые pickle; уникальный tmp исключает перемешивание байтов при конкурентных miss'ах одного канала (RSS+HTML параллельно пишут через `asyncio.to_thread`); rename чинит порчу при падении посреди записи |
+| Р10 | TTL / джиттер / строгий `limit` в пакете A — **без изменений** | Behavior-neutral рефакторинг; limit меняет пакет B, джиттер — пакет F |
+| Р11 | Миграции старых pickle нет: старые файлы = miss; стартовая чистка удаляет legacy | Кеш самовосстанавливается за ≤ TTL (история 8 ч; chatinfo 12 ч по умолчанию, настраивается `TG_CHAT_CACHE_TTL_HOURS`) |
+
+### 2.3. Схема снапшота v1 (контракт)
+
+Выведена из фактического потребления полей конвейером (инвентарь по `rss_generator.py`
+и `post_parser.py`, включая `hasattr`-семантику и enum-сравнения) и сверена с типами
+kurigram 2.2.23 (`.venv`).
+
+```jsonc
+{
+  "id": 123,                          // message.id
+  "date": "2026-07-05T12:00:00",      // isoformat or null
+  "text":    {"plain": "...", "html": "..."},   // null if absent
+  "caption": {"plain": "...", "html": "..."},   // null if absent
+  "media": "PHOTO",                   // MessageMediaType.name or null
+  "service": "PINNED_MESSAGE",        // MessageServiceType.name or null
+  "media_group_id": 456,              // or null
+  "views": 789,                       // or null
+  "show_caption_above_media": false,
+  "reply_to_message_id": 42,          // needed by _reply_enrichment
+  "empty": false,
+  "chat": {"id": -100123, "username": "durov", "title": "...",
+           "usernames": [{"username": "x", "active": true}]},
+  "sender_chat": {"id": 1, "title": "...", "username": "..."},
+  "from_user": {"first_name": "...", "last_name": "...", "username": "..."},
+  "forward_origin": {                 // ONLY keys present on the live object (hasattr semantics!)
+      "type": "channel",
+      "chat": {"id": 1, "title": "...", "username": "..."},
+      "sender_user_name": "...",
+      "sender_user": {"first_name": "...", "last_name": "...", "username": "..."},
+      "chat_id": 1, "title": "..."
+  },
+  "reactions": [                      // from message.reactions.reactions; null if none
+      {"emoji": "👍", "count": 5, "is_paid": null, "custom_emoji_id": null},
+      // custom-emoji reaction: emoji is null, custom_emoji_id is a STRING (kurigram
+      // builds it as str(document_id)) — restored with the FULL key set, like live objects
+      // (live kurigram sets is_paid=None unless it is a paid reaction):
+      {"emoji": null, "count": 2, "is_paid": null, "custom_emoji_id": "987..."}
+  ],
+  "poll": {"question": "...",         // plain string — FormattedText unwrapped at snapshot time
+           "options": [{"text": "..."}]},   // text unwrapped to plain string; see 2.4.1
+  "web_page": {"type": "...", "url": "...", "display_url": "...", "site_name": "...",
+               "title": "...", "description": "...", "has_large_media": false,
+               "photo": {"file_unique_id": "..."}},
+  // media payloads — per-type allowlist:
+  "photo":      {"file_unique_id": "..."},
+  "video":      {"file_unique_id": "...", "file_size": 123},   // file_size: large-video rule
+  "document":   {"file_unique_id": "...", "mime_type": "..."},  // mime_type: PDF branch
+  "audio":      {"file_unique_id": "...", "mime_type": "..."},
+  "voice":      {"file_unique_id": "...", "mime_type": "..."},
+  "video_note": {"file_unique_id": "..."},
+  "animation":  {"file_unique_id": "..."},
+  "sticker":    {"file_unique_id": "...", "emoji": "...", "is_video": false}
+}
+```
+
+### 2.4. Критические тонкости (обязательны к учёту)
+
+1. **`poll.question` и КАЖДЫЙ `poll.options[].text` в kurigram 2.2.23 — всегда объекты
+   `FormattedText`** (проверено: `poll.py:105,220`, `poll_option.py:72,84` в `.venv`), не строки.
+   Снапшот обязан разворачивать оба в plain-строку правилом
+   `v.text if hasattr(v, 'text') else str(v)` (правило корректно и для `Str`, и для голой
+   строки). Если положить `FormattedText` в словарь как есть — `json.dump` кинет `TypeError`,
+   `_save_history_to_cache` проглотит её, и **кеш молча перестанет сохраняться для любого
+   канала с опросом в выборке**. Восстановление: `question` → str,
+   `options` → namespace с `.text` (голая строка в options дала бы
+   `getattr(option, 'text', '') == ''` → пустые опции, `post_parser.py:992`).
+2. **Реакции восстанавливаются с полным набором ключей** (`emoji`, `custom_emoji_id`,
+   `count`, `is_paid`; отсутствующие значения — `None`) — ровно как живой `Reaction`,
+   у которого `__init__` всегда ставит все атрибуты. Ветвление в
+   `post_parser.py:399-410` и `:928-934` на живых объектах работает через truthiness,
+   а не через отсутствие атрибута — восстановленный объект обязан вести себя так же.
+   `custom_emoji_id` — **строка** (kurigram: `str(reaction.document_id)`).
+3. **`forward_origin`** — единственный объект с presence-семантикой: Case 4 различается по
+   `hasattr(forward_origin, "chat_id") and hasattr(..., "title")`; при снапшоте пишутся
+   только реально присутствующие (non-None) поля-кандидаты, при восстановлении — только
+   записанные ключи.
+4. **`video.file_size` обязателен** — правило «не кешировать видео >100 МБ» в
+   `_save_media_file_ids` (`post_parser.py:1058`).
+5. **`chat.usernames`** — список объектов с `.username`/`.active`
+   (`post_parser.py:1087-1090`, прямой доступ без getattr — оба ключа всегда присутствуют).
+6. **`CachedMessage.__str__`** — читаемый JSON-дамп: попадает в error-логи
+   (`post_parser.py:980`).
+
+### 2.5. Задание 1 — новый модуль `message_snapshot.py`
+
+```python
+SNAPSHOT_VERSION = 1
+
+class CachedStr(str):
+    """str subclass carrying a precomputed .html rendering, mirroring pyrogram's Str.
+    deepcopy/pickle preserve the instance __dict__, so .html survives copying."""
+    # factory: CachedStr.build(plain, html)
+
+class CachedMessage:
+    """Duck-typed stand-in for pyrogram Message, restored from a snapshot dict.
+    Mutable (reply enrichment assigns .reply_to_message). All pipeline-consumed
+    top-level attributes exist with None/False defaults, so getattr never raises."""
+    # __str__/__repr__: json.dumps of the snapshot dict, default=str
+
+def snapshot_message(message) -> dict: ...
+def restore_message(data: dict) -> CachedMessage: ...
+def snapshot_messages(messages) -> list[dict]: ...
+def restore_messages(items: list[dict]) -> list[CachedMessage]: ...
+```
+
+Требования:
+- `snapshot_message` извлекает строго по схеме 2.3; каждое поле через `getattr(..., None)`.
+- Styled-text (`FormattedText`/`Str`/str) разворачивается правилом
+  `v.text if hasattr(v, 'text') else str(v)` — применяется к `poll.question` и каждому
+  `poll.options[].text` (см. 2.4.1).
+- `text`/`caption`: `{"plain": str(value), "html": value.html}`; если `.html` недоступен —
+  `html = plain`.
+- `media`: `message.media.name`; `service`: `message.service.name`
+  (оба через безопасный `getattr(x, 'name', str(x))`).
+- Восстановление: помощник `_ns(d: dict, keys: tuple) -> SimpleNamespace` — ставит ВСЕ
+  перечисленные ключи схемы (None-дефолт для отсутствующих); для `forward_origin` — режим
+  «только записанные ключи». Спец-обработка: `text/caption → CachedStr`,
+  `media → MessageMediaType[name]` (при `KeyError` — warning + `None`),
+  `date → datetime.fromisoformat`, `reactions → SimpleNamespace(reactions=[...])`
+  (обёртка-контейнер, как у pyrogram).
+- Дефолты `CachedMessage`: `id, date, text, caption, media, service, media_group_id, views,
+  show_caption_above_media, reply_to_message_id, reply_to_message=None, empty=False, chat,
+  sender_chat, from_user, forward_origin, reactions, poll, web_page, photo, video, document,
+  audio, voice, video_note, animation, sticker`.
+- Комментарии в коде — только английские.
+
+### 2.6. Задание 2 — переписать `tg_cache.py` на generic JSON-store
+
+1. Убрать `import pickle` полностью. Удалить ветку double-pickle (`tg_cache.py:111-116`).
+2. Generic-пара (заменяет обе дублирующиеся тройки функций):
+
+```python
+def _store_entry(path: str, payload: dict) -> None:
+    """Atomically write {'version', 'timestamp', **payload} as JSON.
+    Writes to a unique '<path>.tmp.<uuid4hex>' and os.replace()s it into place;
+    the finally block always removes this writer's own tmp file."""
+
+def _load_entry(path: str, max_age_hours: float) -> Optional[dict]:
+    """Return payload dict, or None on: missing file, version mismatch,
+    expired TTL (keep the existing up-to-20% read-time jitter as-is in package A;
+    package F moves it to write time), JSON error."""
+```
+
+3. Пути: `<safe_key>.history.json` и `<safe_key>.chatinfo.json`
+   (общий помощник `_cache_file_path(key, suffix)` вместо двух копий).
+4. `_save_history_to_cache`: `payload = {'limit': limit, 'messages': snapshot_messages(messages)}`.
+   `_get_history_from_cache`: проверка `limit` как сейчас (пакет A не меняет), возврат
+   `restore_messages(...)`.
+5. `_save_chat_to_cache` / `_get_chat_from_cache`: тот же store, `payload = {'data': {...}}`.
+6. Новая функция `cleanup_legacy_cache_files() -> int` — удаляет из `CACHE_DIR` файлы
+   `*.cache` и `*.chatinfo` (старые pickle-форматы, включая `*_history.cache`), возвращает
+   число удалённых. Ошибки удаления — warning, не исключение.
+7. Сигнатуры и поведение `cached_get_chat_history` / `cached_get_chat` не меняются:
+   на miss возвращаются живые `Message`, на hit — `CachedMessage`; обе формы duck-совместимы
+   (как сегодня pickle-копия vs живой объект).
+
+### 2.7. Задание 3 — стартовая чистка в `api_server.py`
+
+В `lifespan` после `init_db_sync`, **до** запуска фоновых задач:
+
+```python
+# One-shot cleanup of legacy pickle cache files (pre-JSON formats)
+await asyncio.to_thread(cleanup_legacy_cache_files)
+```
+
+Больше в `api_server.py` в рамках пакета A ничего не трогать.
+
+### 2.8. Задание 4 — тесты `tests/test_message_snapshot.py`
+
+Фейковые `Message` — на `SimpleNamespace` (паттерн существующих тестов); `text` — мини-класс
+`str` с атрибутом `.html`. Обязательные кейсы:
+
+1. Round-trip основных полей: id, date (naive и aware — оба сохраняют tz-ность), text.html,
+   caption.html, media enum (`is MessageMediaType.PHOTO`), views, media_group_id.
+2. **Poll с FormattedText-подобными объектами**: фейк `question = SimpleNamespace(text="Q?",
+   entities=[])`, каждый option — `SimpleNamespace(text=SimpleNamespace(text="Opt",
+   entities=[]))`. Проверить: снапшот JSON-сериализуем (`json.dumps` не падает);
+   восстановленный `poll.question` — строка; `getattr(option, 'text', '') == "Opt"`.
+   Тест обязан ПАДАТЬ, если снапшот кладёт FormattedText как есть.
+3. Реакции: обычная / paid / custom-emoji. У восстановленной custom-реакции:
+   `hasattr(r, 'emoji') is True`, `r.emoji is None`, `r.custom_emoji_id` — строка;
+   ветка в `_reactions_views_links`-подобной логике выбирается по truthiness, как у живой.
+4. forward_origin Case 1–5: `hasattr`-ветвление выбирает правильный case
+   (проверять именно наличие/отсутствие атрибутов — здесь presence-семантика сохраняется).
+5. service: `'PINNED_MESSAGE' in str(restored.service)`.
+6. Мутабельность + deepcopy: присвоение `msg.reply_to_message = X`;
+   `copy.deepcopy(restored)` сохраняет `.text.html`.
+7. Неизвестный media name (`"FUTURE_TYPE"`) → `media is None`, без исключения.
+8. Store: `_store_entry`/`_load_entry` — round-trip, TTL-протухание, version mismatch → None,
+   битый JSON → None; уникальность tmp-путей: перехватить `os.replace` (или замокать
+   `uuid.uuid4`) и проверить, что два вызова `_store_entry` в один path использовали
+   РАЗНЫЕ tmp-имена, а итоговый файл валиден — одиночный «конкурентный» прогон окно
+   гонки не ловит и сломанную реализацию с фиксированным tmp пропустит;
+   `cleanup_legacy_cache_files` удаляет `*.cache`/`*.chatinfo`, не трогает
+   `*.history.json` (использовать `tmp_path`).
+9. `_save_media_file_ids` на восстановленном сообщении с `video.file_size` > 100 МБ ничего
+   не добавляет в `_pending_media_ids`.
+10. Восстановленный `chat` для канала без username: `message.chat.username` — `None`
+    (не `AttributeError`) — регресс-тест на правило Р7 (сценарий `rss_generator.py:131`).
+
+### 2.9. Задание 5 — верификация пакета A
+
+- `pytest` — весь существующий набор зелёный.
+- `grep pickle tg_cache.py` — пусто; `post_parser.py` и `rss_generator.py` не изменены.
+
+---
+
+## 3. Пакет B — история: префикс вместо строгого равенства `limit`
+
+**Проблема:** `tg_cache.py:106-109` требует `cached_limit == limit`; RSS просит `limit*2`
+(`rss_generator.py:419`), HTML — `limit` (`rss_generator.py:639`), оба пишут в один файл.
+Чередование запросов превращает кеш в решето.
+
+### 3.1. Задание 6 — логика префикса в `_get_history_from_cache`
+
+Сообщения в кеше лежат от новых к старым (порядок `get_chat_history`), поэтому запрос
+с меньшим `limit` — это префикс:
+
+```python
+cached_limit = payload.get('limit', 0)
+raw_messages = payload['messages']
+# Serve from cache when it was fetched with an equal-or-larger limit, OR when the
+# channel is exhausted (fewer messages exist than were asked for at fetch time —
+# the cache then holds the channel's entire recent history and satisfies ANY limit).
+if cached_limit < limit and len(raw_messages) >= cached_limit:
+    log("history_cache_limit_insufficient: cached %s < requested %s", cached_limit, limit)
+    return None
+return restore_messages(raw_messages[:limit])
+```
+
+- `_save_history_to_cache` не меняется: хранит `limit`, с которым делался fetch.
+- Лог хита дополнить: `served {limit} of cached {cached_limit}`.
+- Сходимость: первый запрос с большим `limit` после протухания перезапишет кеш «широкой»
+  версией, дальше меньшие запросы — хиты.
+
+### 3.2. Задание 7 — тесты
+
+1. Кеш `limit=100` (100 сообщений) → запрос `limit=50` — хит, ровно 50 первых, порядок сохранён.
+2. Кеш `limit=50` → запрос `limit=100` — промах.
+3. Исчерпанный канал: кеш `limit=100`, 37 сообщений → запрос `limit=200` — хит, 37 сообщений.
+4. TTL работает независимо от limit.
+
+---
+
+## 4. Пакет C — единая канонизация ключа канала
+
+**Проблема:** один канал живёт под ключами `Durov` / `durov` / `@name` в tgcache-файлах,
+каталогах `data/cache/` и строках SQLite → дубли кеша, двойные походы в Telegram,
+осиротевшие деревья при смене регистра/формы. (Унификация ID↔username — не-цель, раздел 8.)
+
+### 4.1. Задание 8 — новый модуль `channel_key.py`
+
+Отдельный модуль без зависимостей (его импортируют `tg_cache`, `post_parser`, `api_server` —
+отдельность исключает циклические импорты):
+
+```python
+def canonical_channel_key(channel: str | int) -> str:
+    """Canonical cache/DB key for a channel.
+
+    Telegram usernames are case-insensitive -> lowercase them.
+    Numeric '-100...' ids keep their exact string form.
+    The '@' prefix is stripped.
+
+    The canonical form is also SAFE to pass to Telegram API calls (usernames are
+    case-insensitive on the API side; the numeric form is unchanged) — callers that
+    thread one value through both filesystem paths and API calls may use it for both.
+    """
+    s = str(channel).strip().lstrip('@')
+    if s.startswith('-100') and s[4:].isdigit():
+        return s
+    return s.lower()
+```
+
+### 4.2. Задание 9 — применить ключ во всех трёх слоях
+
+1. **`tg_cache.py`**: в `_cache_file_path(key, suffix)` прогонять ключ через
+   `canonical_channel_key`.
+2. **`post_parser.get_channel_username`** (`post_parser.py:1081-1097`): возвращать username
+   в нижнем регистре (обе ветки — `usernames`-список и одиночный `username`). Это
+   канонизирует записи SQLite (`_save_media_file_ids`), генерируемые media-URL и `t.me`-ссылки
+   (t.me к регистру нечувствителен).
+3. **`api_server.get_media`** (`api_server.py:1153-1183`): после проверки digest вычислить
+   `fs_channel = canonical_channel_key(channel)` и использовать вместо `str(channel)` везде
+   ниже: pre-semaphore путь, ключ `_access_updates`, `media_key`, аргумент `_download_deduped`
+   (протекает в `download_media_file` → каталоги, API-вызов и удаление из SQLite — для API
+   канонический идентификатор безопасен, см. docstring 4.1).
+   **Digest проверять по исходной строке URL** — иначе сломаются все выданные ссылки.
+
+### 4.3. Задание 10 — one-shot миграция существующих данных
+
+Без миграции каждый ранее закешированный медиафайл канала с не-lowercase username даёт
+после деплоя cache-miss → повторную загрузку из Telegram (шторм ограничен такими каналами
+и фактически запрашиваемыми файлами, но страховка дешёвая — делаем).
+
+Новая функция `migrate_channel_keys_sync(db_path: str, cache_dir: str) -> None`. Вызов —
+в `lifespan` **после `init_db_sync`, ДО `client.start()` и до запуска фоновых задач**,
+строго через `await asyncio.to_thread(migrate_channel_keys_sync, ...)` — функция
+синхронная (FS-rename + SQLite), голый вызов заблокировал бы event loop на время
+миграции; размещение до `client.start()` дополнительно исключает влияние на сетевые
+таски pyrogram и гонки со свипером/access-flush.
+
+Единица работы — **канал целиком, сначала FS, потом SQL** (порядок принципиален:
+при провале FS-части SQL-строки канала остаются старорегистровыми, и старое дерево
+по-прежнему видно свиперу через строки БД — вечных orphan-каталогов не возникает):
+
+1. Найти каналы-кандидаты: имена каталогов первого уровня в `cache_dir` с не-lowercase
+   именем (кроме `-100...`) ∪ `SELECT DISTINCT channel ... WHERE channel != lower(channel)
+   AND channel NOT LIKE '-100%'`.
+2. Для каждого канала — **FS-шаг**:
+   - **ОБЯЗАТЕЛЬНЫЙ guard до любых действий**: если `os.path.exists(dst)` и
+     `os.path.samefile(src, dst)` — **no-op FS-шага, перейти к SQL-шагу**. На
+     case-insensitive FS (macOS/APFS, Docker Desktop for Mac — том `./data` наследует
+     семантику хоста) `Durov/` и `durov/` — один и тот же каталог; merge без guard'а
+     удалил бы весь кеш канала.
+   - `dst` не существует → `os.rename(src, dst)`.
+   - `dst` существует и это другой каталог (case-sensitive FS, обе формы реально есть) →
+     пофайловый merge: существующий файл в `dst` выигрывает; затем удалить пустые
+     остатки `src`.
+   - FS-шаг упал (EACCES и т.п.) → log error с пометкой orphan-кандидата, **SQL-шаг
+     канала пропустить**, перейти к следующему каналу.
+3. **SQL-шаг** (только после успешного FS-шага): для каждой строки канала:
+   - lowercase-строки с тем же `(post_id, file_unique_id)` нет — просто `UPDATE` канала;
+   - есть — merge: `added = max(added)` из двух, `mime_type` — предпочесть non-NULL;
+     затем `DELETE` старорегистровой строки. Голый `UPDATE ... SET channel=lower(channel)`
+     запрещён — ловит PK-конфликт при наличии обеих форм.
+4. Ошибки миграции — log error + продолжить работу (миграция не должна уронить старт);
+   повторный запуск функции — идемпотентный no-op.
+5. **Итоговая сводка одной строкой лога** (сигнал оператору, что деплой C прошёл штатно):
+   `migration_summary: rows merged N, dirs renamed M, samefile no-ops K, failures F`.
+
+### 4.4. Задание 11 — тесты + заметка о миграции
+
+- Юнит: `'Durov'→'durov'`, `'@durov'→'durov'`, `-1001…` (int и str) → `'-1001…'`.
+- Интеграция: путь tgcache-файла одинаков для `Durov`/`durov`; `get_channel_username`
+  на фейковом chat с `username='MixedCase'` → `'mixedcase'`.
+- Миграция: (а) SQL-merge обеих форм — остаётся одна строка с `max(added)` и non-NULL
+  `mime_type`; (б) guard: вызов с `src`/`dst`, резолвящимися в один каталог (samefile), —
+  no-op, данные целы; (в) merge на двух реально разных каталогах — файлы объединены,
+  target выигрывает; (г) повторный запуск — no-op.
+- В PR: (а) остаточные старорегистровые строки/каталоги, не покрытые миграцией (например,
+  появившиеся между бэкапом и деплоем), доживают до 20-дневного вытеснения сами;
+  (б) **миграция односторонняя** — reverse-миграции нет; откат пакета C после её
+  выполнения возвращает код со старорегистровыми ключами на lowercase-данные и повторяет
+  re-download-сценарий зеркально (ограниченный, самоизлечивается за 20 дней) — это
+  ожидаемый признак отката, не авария.
+
+---
+
+## 5. Пакет D — свипер медиа-кеша
+
+### 5.1. Задание 12 — фикс чистки БД (баг)
+
+`api_server.py:820-836`: diff и удаление из SQLite выполняются только `if files_removed > 0`.
+Запись старше 20 дней **без файла на диске** выпадает из списка без инкремента счётчика
+(`api_server.py:663-687`) — и остаётся в БД навсегда, если в том же проходе не удалился
+ни один реальный файл. Исправление:
+
+```python
+# Compute the DB diff unconditionally: entries dropped from the list because the
+# file was already gone must still be purged from SQLite.
+updated_set = {...}
+removed_entries = [...]
+if removed_entries:
+    await asyncio.to_thread(remove_media_file_ids_sync, DB_PATH, removed_entries)
+    logger.info(f"cache_sweep: purged {len(removed_entries)} entries "
+                f"({files_removed} files removed from disk)")
+```
+
+### 5.2. Задание 13 — интервал свипа в конфиг
+
+Сейчас `delay = 60` (`api_server.py:811`). Добавить в `config.py` по существующему паттерну
+ключ `cache_sweep_interval` (env `CACHE_SWEEP_INTERVAL`, дефолт `900`, минимум `60`),
+использовать в `cache_media_files`.
+
+### 5.3. Задание 14 — дедуп очереди фоновых загрузок
+
+Module-level `_queued_media: set[tuple[str, int, str]]`:
+- в `download_new_files`: ключ в set'е → skip; иначе `add` + `put_nowait`
+  (при `QueueFull` — `discard` и `break` как сейчас);
+- в `background_download_worker`: в `finally` рядом с `task_done()` — `discard(key)`.
+
+Event loop однопоточный, обе точки — корутины без await между проверкой и мутацией → гонок нет.
+Тест: два подряд вызова `download_new_files` с одним списком кладут в очередь один элемент.
+
+---
+
+## 6. Пакет E — горячий путь `/media`
+
+### 6.1. Задание 15 — константа и хелпер путей
+
+`os.path.abspath("./data/cache")` + ручной `join` собираются в **семи** местах
+(`api_server.py:207, 532-536, 667, 757-766, 817, 851, 1172-1175`). Ввести на уровне модуля:
+
+```python
+MEDIA_CACHE_DIR = os.path.abspath(os.path.join("data", "cache"))  # resolved once at import
+
+def media_cache_path(channel: str, post_id: int, file_unique_id: str | None = None) -> str:
+    """Single source of truth for the on-disk layout: <root>/<channel>/<post_id>[/<fid>]."""
+```
+
+Заменить все семь мест. `remove_old_cached_files_sync`/`download_new_files` сохраняют
+параметр `cache_dir` (тестируемость), вызыватели передают константу.
+
+### 6.2. Задание 16 — in-memory кеш MIME
+
+Запись access-time уже вынесена с горячего пути в аккумулятор, а чтение MIME до сих пор
+делает threadpool-hop + новое SQLite-соединение на каждый хит (`api_server.py:389-404`).
+MIME по ключу `file_unique_id` иммутабелен.
+
+```python
+# MIME types are immutable per file_unique_id, so a process-lifetime dict in front of
+# SQLite removes a to_thread + connect from every cache-hit response.
+_mime_types: dict[tuple[str, int, str], str] = {}
+_MIME_CACHE_MAX = 50_000  # crude bound; clear-all on overflow is fine at this size
+```
+
+Порядок в `prepare_file_response`: dict → (miss) SQLite → (miss) python-magic → записать
+и в SQLite, и в dict. При успешном чтении из SQLite — заполнить dict. При переполнении —
+`clear()`. Тест: замокать `get_mime_type_sync`, два запроса подряд — второй его не вызывает.
+
+---
+
+## 7. Пакет F — довесок к новому store
+
+### 7.1. Задание 17 — джиттер TTL при записи
+
+Сейчас `random_factor` перебрасывается на каждом чтении (`tg_cache.py:98, 192`) — срок жизни
+записи недетерминирован, что усложняет рассуждения о поведении и тесты (одна и та же запись
+у границы TTL может дать разный результат в соседних чтениях; устойчивого «мигания» нет —
+первый же miss ведёт к перезаписи со свежим timestamp). В generic-store пакета A:
+- `_store_entry` дополнительно пишет `'jitter': random.uniform(0.8, 1.0)`;
+- `_load_entry` считает `adjusted_max_age = max_age_hours * 3600 * entry.get('jitter', 1.0)`
+  и не дёргает `random` при чтении.
+
+Срок жизни записи детерминирован после записи, разброс между инстансами/каналами сохраняется.
+Тест: одна и та же запись около границы TTL даёт стабильный результат при повторных чтениях.
+
+### 7.2. Задание 18 — age-sweep tgcache + добить legacy-данные
+
+1. Новая функция `sweep_tgcache(max_age_days: int = 7) -> int` в `tg_cache.py`: удаляет из
+   `CACHE_DIR` любые файлы с mtime старше порога. Живые файлы перезаписываются каждые ≤12 ч,
+   поэтому mtime > 7 суток — гарантированно мёртвый файл (умерший/переименованный канал,
+   неканоничный ключ, осиротевший uuid-tmp от упавшего писателя). Гонка с писателем
+   ВОЗМОЖНА (stat→unlink не атомарен против `os.replace`: sweep может удалить файл,
+   обновлённый между stat и unlink), но безвредна — худший исход один лишний miss-refetch;
+   гонка с читателем так же самоизлечивается как miss. Не выдавать это за инвариант
+   атомарности в комментариях кода.
+2. Вызовы: (а) один раз при старте — рядом с `cleanup_legacy_cache_files` (задание 3);
+   (б) из цикла `cache_media_files` раз в проход, **обязательно через `asyncio.to_thread`**
+   (listdir+stat+unlink — синхронный FS I/O; инвариант кодовой базы — ничего блокирующего
+   на event loop). Каталог плоский, стоимость — миллисекунды.
+3. Стартовая legacy-чистка (задание 3) остаётся: sweep покрывает legacy-файлы сам, но
+   с задержкой до 7 суток; чистка по расширениям делает это мгновенно.
+4. Расширить стартовую чистку: удалить `data/media_file_ids.json` (наследие до-SQLite
+   хранилища, кодом не читается — проверено grep). Лог — одной строкой с перечнем удалённого.
+
+---
+
+## 8. Границы (сознательно НЕ входит)
+
+- **Унификация ID↔username** (`/rss/-100…` и `/rss/durov` одного канала остаются разными
+  ключами во всех слоях): потребовала бы резолва через `get_chat` (лишний RPC, свои отказы
+  и кеш-инвалидация). Канонизация пакета C устраняет только дубли регистра/формы записи.
+- Переезд метаданных медиа с SQLite куда-либо ещё — SQLite остаётся.
+- Кеширование результатов рендеринга (processed dicts / готовый HTML фида) — отдельная
+  большая тема с инвалидацией по параметрам запроса.
+- Оптимизация `calculate_cache_stats` (полный walk на `/health`) — низкий приоритет.
+- `SimpleNamespace` → dataclass в `cached_get_chat` — косметика.
+
+## 9. Сквозные критерии приёмки
+
+1. `pytest` зелёный; `post_parser.py`/`rss_generator.py` изменены только в объёме
+   задания 9.2 (lowercase в `get_channel_username`).
+2. `grep pickle tg_cache.py` — пусто.
+3. Сценарий «RSS(limit=100) → HTML(limit=50) → RSS(100)» порождает **один** поход в Telegram
+   (сейчас — три).
+4. `/rss/Durov` и `/rss/durov` используют один tgcache-файл; `/media/Durov/...` и
+   `/media/durov/...` — один файл на диске.
+5. Повторный `/media`-хит не создаёт SQLite-соединений (ни на чтение MIME, ни на запись
+   access-time). Рецепт проверки: monkeypatch `file_io._open_db` счётчиком соединений;
+   два последовательных запроса TestClient к одному файлу — на втором счётчик не растёт.
+6. Строка SQLite с `added` старше 20 дней и отсутствующим файлом исчезает из БД за один
+   проход свипа.
+7. Кеш истории сохраняется и для каналов с опросами в выборке (снапшот JSON-сериализуем).
+8. Миграция ключей на case-insensitive FS — no-op без потери данных (samefile-guard).
+
+## 10. Риски
+
+| Риск | Закрытие / статус |
+|---|---|
+| Пропущенное поле allowlist → тихая деградация рендера | **Живой риск, не закрыт полностью.** Инвентарь снят по всем потребителям (2.3–2.4) и сверен с kurigram 2.2.23; ревью нашло и закрыло два пропуска (FormattedText в poll, omit-None семантика) — но гарантии полноты ручного инвентаря нет. Митигция: тесты 2.8, деградация проявляется как выпадение конкретного элемента рендера, не крэш; TTL 8 ч ограничивает окно |
+| Смешение живых `Message` (miss) и `CachedMessage` (hit) | Так работает и текущий pickle-кеш; живой объект — надмножество контракта |
+| Порча файла при падении посреди записи / конкурентная запись | Уникальный tmp + `os.replace` + finally-очистка (Р9) |
+| Старые pickle после деплоя | Игнорируются новыми именами + `cleanup_legacy_cache_files()` |
+| Канонизация: существующие данные со старым регистром | One-shot миграция (задание 10) с samefile-guard; остатки доживают до 20-дневного LRU |
+| Миграция на case-insensitive FS | Обязательный `os.path.samefile`-guard → no-op (критерий 9.8) |
+
+## 11. Рассмотренные альтернативы
+
+**Version-stamped pickle** (минимальный вариант): оставить pickle, дописать в конверт
+`pyrogram.__version__` + версию схемы, несовпадение считать miss'ом (~5 строк). Закрывает
+«тихий дрейф» и стоимость апгрейда библиотеки (один TTL-эквивалентный miss на канал —
+пренебрежимо при TTL 8 ч). **Отклонена решением владельца проекта** (альтернатива
+предлагалась явно на этапе аудита как «рекомендация-минимум»; выбран стратегический
+вариант) по причинам: (а) исходная цель — «проще и поддерживаемее»: JSON читаем глазами
+при отладке, pickle — нет; (б) снапшот делает зависимость рендера от полей `Message`
+явным документированным контрактом (схема 2.3) вместо размазанной по ~2000 строк;
+(в) меньший файл и дешёвый parse вместо unpickle графа объектов в threadpool.
+Цена: ручной инвентарь полей с остаточным риском пропуска (раздел 10, первый пункт).
+
+## 12. История ревью
+
+Спецификация прошла два раунда адверсарного ревью (субагент-критик с доступом к кодовой
+базе и установленному kurigram 2.2.23). Итоги:
+
+- **C1 (BLOCKER, принято)**: исходная схема не разворачивала `FormattedText` в
+  `poll.options[].text` → `json.dump` падал бы, кеш молча переставал сохраняться для
+  каналов с опросами; исходные тест-фикстуры дефект не ловили. Исправлено: 2.4.1, тест 2.8.2.
+- **C2 (MAJOR, принято с усилением)**: blanket omit-None ломал эквивалентность с живыми
+  объектами (у которых все атрибуты всегда присутствуют) и давал крэш-пути
+  (`rss_generator.py:131`, `post_parser.py:1087`). Итог: правило Р7 «полные ключи, кроме
+  forward_origin».
+- **C3 (MAJOR, принято)**: канонизация без миграции → повторные загрузки медиа после деплоя.
+  Добавлено задание 10. Встречное возражение критика к первой версии миграции
+  (**потеря кеша на case-insensitive FS**) принято: обязательный samefile-guard.
+- **C4 (MAJOR → MINOR)**: критик требовал обосновать отказ от version-stamped pickle.
+  Закрыто разделом 11; довод «холодный кеш на каждый бамп библиотеки» в обоснование
+  сознательно НЕ включён (стоимость — один miss на канал, пренебрежимо). Остаточный MINOR:
+  риск ручного инвентаря остаётся живым (раздел 10).
+- **C5–C12 (MINOR, приняты)**: уникальный tmp; docstring `canonical_channel_key`;
+  сужение проблемы #3; седьмое место сборки пути (`:817`); TTL 8–12 ч; `custom_emoji_id: str`;
+  честная мотивация джиттера; age-sweep tgcache (+`to_thread` из фонового цикла).
+- **Раунд 4 (новые векторы: атака на собственные правки критика, межпакетные
+  взаимодействия, реализуемость, рантайм-семантика, операционка): 6 MINOR, все приняты.**
+  D1 — `to_thread` + размещение миграции до `client.start()`; D2 — порядок «FS → SQL
+  per-channel», чтобы частичный отказ FS не рождал orphan-деревья, невидимые свиперу;
+  D3 — гонка sweep/писатель честно названа возможной-но-безвредной; D4 — зависимость
+  «C после A» зафиксирована в таблице (задание 9.1 опирается на `_cache_file_path` из
+  задания 2); D5 — тест 2.8.8 переписан на дискриминирующий (уникальность tmp-путей),
+  критерий 9.5 получил рецепт проверки; D6 — итоговая сводка миграции в лог + пометка
+  об односторонности миграции при откате. По направлениям «откаты A/B/F», «смешение
+  живых и восстановленных объектов», «json.dumps(CachedStr)», «deepcopy», «exclude_*»
+  критик существенных находок не нашёл (с доказательствами по коду: hit/miss атомарен
+  на весь список, `/json`-эндпоинт идёт мимо кеша истории, tz-гомогенность гарантирована Р6).

post_parser.py

@@ -14,12 +14,12 @@ import re
 import os
 import html
 import inspect
+from dataclasses import dataclass
 from datetime import datetime
-from typing import Union, Dict, Any, List, Optional
+from typing import Union, Dict, Any, List, Optional, Callable, Tuple
 from pyrogram.types import Message
 from pyrogram.enums import MessageMediaType
-from bleach.css_sanitizer import CSSSanitizer   
-from bleach import clean as HTMLSanitizer
+from sanitizer import sanitize_html
 from config import get_settings
 from file_io import upsert_media_file_ids_bulk_sync, DB_PATH
 from url_signer import generate_media_digest
@@ -28,6 +28,234 @@ Config = get_settings()
 
 logger = logging.getLogger(__name__)
 
+# Media types that never yield a renderable image/video in the feed: they are
+# represented by info blocks (or nothing), so posts carrying them get "no_image".
+NO_IMAGE_MEDIA_TYPES = {
+    MessageMediaType.GIVEAWAY,
+    MessageMediaType.GIVEAWAY_WINNERS,
+    MessageMediaType.CHECKLIST,
+    MessageMediaType.CONTACT,
+    MessageMediaType.LOCATION,
+    MessageMediaType.VENUE,
+    MessageMediaType.DICE,
+    MessageMediaType.GAME,
+    MessageMediaType.INVOICE,
+    MessageMediaType.UNSUPPORTED,
+    MessageMediaType.PAID_MEDIA,
+}
+
+
+def _poll_media_object(message):
+    """Return (media_obj, kind) attached to a poll's description_media, or (None, None).
+
+    Kurigram 2.2.23 polls may carry media in description_media / explanation_media
+    (MessageContent objects). Only description_media is considered for rendering:
+    explanation_media is the quiz-answer explanation attachment and does not belong
+    in a channel feed, so it is deliberately NOT rendered (api_server's download
+    lookup still covers it in case a URL for it exists).
+
+    kind is the render hint: 'img' for photo/sticker, 'video' for video/animation.
+
+    All attribute access goes through getattr: older Poll objects and test mocks do
+    not define these fields. A candidate is accepted only when its file_unique_id is
+    a non-empty str — without it nothing can be served through the /media pipeline,
+    and this also keeps loose MagicMock-based poll mocks from producing false
+    positives via auto-created attributes.
+    """
+    poll = getattr(message, 'poll', None)
+    if poll is None:
+        return None, None
+    description_media = getattr(poll, 'description_media', None)
+    if description_media is None:
+        return None, None
+    candidates = (
+        ('photo', 'img'),
+        ('video', 'video'),
+        ('animation', 'video'),
+        ('sticker', 'img'),
+    )
+    for attr, kind in candidates:
+        media_obj = getattr(description_media, attr, None)
+        if media_obj is None:
+            continue
+        file_unique_id = getattr(media_obj, 'file_unique_id', None)
+        if isinstance(file_unique_id, str) and file_unique_id:
+            return media_obj, kind
+    return None, None
+
+
+def _story_media_object(message):
+    """Return (media_obj, kind) for message.story media (video wins over photo), or (None, None).
+
+    kind is the render hint ('video' or 'img'). Rendering, URL generation and file-id
+    collection all go through this helper, so they always agree on WHICH story object
+    is used (e.g. when a story video lacks a usable file_unique_id and the helper
+    falls back to the photo, the tag type follows the fallback too).
+
+    Same defensive rules as _poll_media_object: getattr-only access and a non-empty
+    str file_unique_id requirement.
+    """
+    story = getattr(message, 'story', None)
+    if story is None:
+        return None, None
+    for attr, kind in (('video', 'video'), ('photo', 'img')):
+        media_obj = getattr(story, attr, None)
+        if media_obj is None:
+            continue
+        file_unique_id = getattr(media_obj, 'file_unique_id', None)
+        if isinstance(file_unique_id, str) and file_unique_id:
+            return media_obj, kind
+    return None, None
+
+
+# --------------------------------------------------------------------------- #
+# Single source of truth for media handling (render-pipeline refactor, stage 5).
+#
+# MEDIA_SOURCES maps a media type to a selector returning (media object, render
+# kind). It replaces the three hand-synced ladders that used to live in
+# _get_file_unique_id (dict map), _save_media_file_ids (if/elif) and
+# _generate_html_media (elif cascade). All three now go through this table.
+#
+# kind=None means "selector-only entry": the object participates in file-id
+# extraction/collection but is NOT rendered by the dispatcher. The ONLY kind=None
+# entry is WEB_PAGE (its preview is rendered by _format_webpage). PAID_MEDIA has
+# NO entry at all: its info block is a dedicated branch in _generate_html_media
+# and it is deliberately never collected/downloaded (paid content).
+# --------------------------------------------------------------------------- #
+def _select_document(message):
+    """DOCUMENT selector: PDF documents render as a t.me link ('pdf'), everything
+    else as an image ('img_400'). The selected object is always message.document."""
+    document = message.document
+    if (document is not None and hasattr(document, 'mime_type')
+            and document.mime_type == 'application/pdf'):
+        return document, 'pdf'
+    return document, 'img_400'
+
+
+def _select_sticker(message):
+    """STICKER selector: video stickers loop as <video> ('video_loop_200'), image
+    stickers render as <img> ('img_200_sticker')."""
+    sticker = message.sticker
+    if getattr(sticker, 'is_video', False):
+        return sticker, 'video_loop_200'
+    return sticker, 'img_200_sticker'
+
+
+# Helper kinds returned by _story_media_object / _poll_media_object map to render kinds.
+_HELPER_KIND_TO_RENDER = {'video': 'video_400', 'img': 'img_400'}
+
+
+def _select_story(message):
+    """STORY selector: object choice (video over photo) is delegated to
+    _story_media_object; its helper kind is mapped to a render kind."""
+    media_obj, helper_kind = _story_media_object(message)
+    return media_obj, _HELPER_KIND_TO_RENDER.get(helper_kind)
+
+
+def _select_poll_media(message):
+    """POLL selector: object comes from _poll_media_object (description_media); its
+    helper kind is mapped to a render kind."""
+    media_obj, helper_kind = _poll_media_object(message)
+    return media_obj, _HELPER_KIND_TO_RENDER.get(helper_kind)
+
+
+MEDIA_SOURCES: Dict[Any, Callable[[Message], Tuple[Any, Optional[str]]]] = {
+    MessageMediaType.PHOTO:      lambda m: (m.photo, 'img_400'),
+    MessageMediaType.VIDEO:      lambda m: (m.video, 'video_400'),
+    MessageMediaType.ANIMATION:  lambda m: (m.animation, 'video_400'),
+    MessageMediaType.VIDEO_NOTE: lambda m: (m.video_note, 'video_400'),
+    MessageMediaType.AUDIO:      lambda m: (m.audio, 'audio'),
+    MessageMediaType.VOICE:      lambda m: (m.voice, 'audio'),
+    MessageMediaType.DOCUMENT:   _select_document,
+    MessageMediaType.STICKER:    _select_sticker,
+    MessageMediaType.LIVE_PHOTO: lambda m: (getattr(m, 'live_photo', None), 'video_loop_400'),
+    MessageMediaType.STORY:      _select_story,
+    MessageMediaType.POLL:       _select_poll_media,
+    MessageMediaType.WEB_PAGE:   lambda m: (getattr(m.web_page, 'photo', None), None),
+}
+
+
+# Inline media-sizing style literals, named so the renderers below read as intent
+# rather than magic numbers. Values are substituted verbatim into the style strings,
+# so the emitted bytes are unchanged.
+MEDIA_MAX_HEIGHT_PX = "400px"        # standard image/video max-height
+MEDIA_MAX_HEIGHT_SMALL_PX = "200px"  # looping/sticker media max-height
+MEDIA_MAX_WIDTH_PX = "400px"         # audio player max-width
+
+
+def _wrap_post_html(body: str, footer: str) -> str:
+    """Shared post-HTML shell: the message-body + message-footer div pair joined by a
+    single newline. Used by PostParser._format_html (single/debug post) and by both
+    branches of rss_generator._render_messages_groups (single message and merged
+    group), so the emitted byte structure stays identical across all three call sites."""
+    return (f'<div class="message-body">{body}</div>\n'
+            f'<div class="message-footer">{footer}</div>')
+
+
+@dataclass
+class RenderCtx:
+    """Everything a renderer needs. _generate_html_media assembles it (URL signing,
+    channel_username guard, and the mime/emoji/tg_link values chosen BY MEDIA TYPE);
+    a renderer only formats the string and never inspects the Message."""
+    url: str
+    tg_link: Optional[str] = None   # t.me deep link — only the 'pdf' renderer uses it
+    emoji: str = ''                 # sticker alt text
+    mime: Optional[str] = None      # audio/voice <source> type; default chosen by media type
+
+
+# Renderers return list[str] so the byte structure of the '\n'.join in
+# _generate_html_media is preserved (audio emits TWO items: <audio> tag + <br>;
+# pdf emits its two-append div block). Bodies are lifted VERBATIM from the old
+# if/elif branches, including the `src="{url}"style=` concatenation artifacts —
+# byte-for-byte fidelity is the stage-5a contract.
+def _render_img_400(ctx: 'RenderCtx') -> List[str]:
+    return [f'<img src="{ctx.url}" style="max-width:100%; width:auto; height:auto;'
+            f'max-height:{MEDIA_MAX_HEIGHT_PX}; object-fit:contain;">']
+
+
+def _render_video_400(ctx: 'RenderCtx') -> List[str]:
+    return [f'<video controls src="{ctx.url}" style="max-width:100%; width:auto;'
+            f'height:auto; max-height:{MEDIA_MAX_HEIGHT_PX};"></video>']
+
+
+def _render_audio(ctx: 'RenderCtx') -> List[str]:
+    return [f'<audio controls style="width:100%; max-width:{MEDIA_MAX_WIDTH_PX};">'
+            f'<source src="{ctx.url}" type="{ctx.mime}"></audio>',
+            '<br>']
+
+
+def _render_video_loop_200(ctx: 'RenderCtx') -> List[str]:
+    return [f'<video controls autoplay loop muted src="{ctx.url}"'
+            f'style="max-width:100%; width:auto; height:auto; max-height:{MEDIA_MAX_HEIGHT_SMALL_PX};'
+            f'object-fit:contain;"></video>']
+
+
+def _render_img_200_sticker(ctx: 'RenderCtx') -> List[str]:
+    return [f'<img src="{ctx.url}" alt="Sticker {ctx.emoji}" style="max-width:100%;'
+            f'width:auto; height:auto; max-height:{MEDIA_MAX_HEIGHT_SMALL_PX}; object-fit:contain;">']
+
+
+def _render_video_loop_400(ctx: 'RenderCtx') -> List[str]:
+    return [f'<video controls autoplay loop muted src="{ctx.url}"'
+            f'style="max-width:100%; width:auto; height:auto; max-height:{MEDIA_MAX_HEIGHT_PX};'
+            f'object-fit:contain;"></video>']
+
+
+def _render_pdf(ctx: 'RenderCtx') -> List[str]:
+    return [f'<div class="document-pdf" style="padding: 10px;">',
+            f'<a href="{ctx.tg_link}" target="_blank">[PDF-файл]</a></div>']
+
+
+RENDERERS: Dict[str, Callable[['RenderCtx'], List[str]]] = {
+    'img_400':         _render_img_400,
+    'video_400':       _render_video_400,
+    'audio':           _render_audio,
+    'video_loop_200':  _render_video_loop_200,
+    'img_200_sticker': _render_img_200_sticker,
+    'video_loop_400':  _render_video_loop_400,
+    'pdf':             _render_pdf,
+}
+
 
 #tests
 #http://127.0.0.1:8000/post/html/DragorWW_space/114 — video
@@ -221,6 +449,31 @@ class PostParser:
             if message.media == MessageMediaType.VOICE:       return "🎤 Voice"
             if message.media == MessageMediaType.VIDEO_NOTE:  return "📱 Video circle"
             if message.media == MessageMediaType.STICKER:     return "🎯 Sticker"
+            # New media types (Kurigram 2.2.23). New Message attributes are accessed
+            # via getattr only — older objects/mocks may not define them.
+            if message.media == MessageMediaType.LIVE_PHOTO:  return "📸 Live Photo"
+            if message.media == MessageMediaType.STORY:       return "📖 Story"
+            if message.media == MessageMediaType.GIVEAWAY:    return "🎁 Giveaway"
+            if message.media == MessageMediaType.GIVEAWAY_WINNERS: return "🏆 Giveaway winners"
+            if message.media == MessageMediaType.PAID_MEDIA:  return "⭐ Paid media"
+            if message.media == MessageMediaType.CHECKLIST:
+                checklist = getattr(message, 'checklist', None)
+                title = getattr(checklist, 'title', None) if checklist else None
+                if isinstance(title, str) and title.strip():
+                    return f"📝 Checklist: {title.strip()[:50]}"
+                return "📝 Checklist"
+            if message.media == MessageMediaType.CONTACT:     return "👤 Contact"
+            if message.media == MessageMediaType.LOCATION:    return "📍 Location"
+            if message.media == MessageMediaType.VENUE:
+                venue = getattr(message, 'venue', None)
+                venue_title = getattr(venue, 'title', None) if venue else None
+                if isinstance(venue_title, str) and venue_title.strip():
+                    return f"📍 {venue_title.strip()}"
+                return "📍 Venue"
+            if message.media == MessageMediaType.DICE:        return "🎲 Dice"
+            if message.media == MessageMediaType.GAME:        return "🎮 Game"
+            if message.media == MessageMediaType.INVOICE:     return "🧾 Invoice"
+            if message.media == MessageMediaType.UNSUPPORTED: return "⚠️ Unsupported content"
 
         # Web pages (if no text or media title)
         if message.web_page:
@@ -357,7 +610,9 @@ class PostParser:
             flags.append("fwd")
 
         # Add flag "video" if the message media is VIDEO or ANIMATION and the body text is up to 200 characters.
-        if (message.media in [MessageMediaType.VIDEO, MessageMediaType.ANIMATION, MessageMediaType.VIDEO_NOTE] and 
+        # LIVE_PHOTO (Kurigram 2.2.23) renders as a video element, so it counts too.
+        if (message.media in [MessageMediaType.VIDEO, MessageMediaType.ANIMATION, MessageMediaType.VIDEO_NOTE,
+                              MessageMediaType.LIVE_PHOTO] and
             len((message.text or message.caption or '').strip()) <= 200):
             flags.append("video")
 
@@ -366,8 +621,12 @@ class PostParser:
             len((message.text or message.caption or '').strip()) <= 200):
             flags.append("audio")
         
-        # Add flag for posts without images
-        if not message.media or message.media == MessageMediaType.POLL:
+        # Add flag for posts without images: no media at all, an info-block-only media
+        # type (see NO_IMAGE_MEDIA_TYPES), or a poll without renderable description_media.
+        # A poll WITH description_media renders an image/video, so it is NOT flagged.
+        if (not message.media
+                or message.media in NO_IMAGE_MEDIA_TYPES
+                or (message.media == MessageMediaType.POLL and _poll_media_object(message)[0] is None)):
             flags.append("no_image")
 
         # Add flag for sticker messages
@@ -486,9 +745,11 @@ class PostParser:
         html_content = []
 
         if debug:
-            html_content.append(f'<div class="title">Title: {data["html"]["title"]}</div><br>')
-        html_content.append(f'<div class="message-body">{data["html"]["body"]}</div>')
-        html_content.append(f'<div class="message-footer">{data["html"]["footer"]}</div>')
+            # title comes from _generate_title (user-controlled post text) and never goes
+            # through bleach — escape it before embedding, same as raw_message below.
+            title_escaped = html.escape(str(data["html"]["title"]))
+            html_content.append(f'<div class="title">Title: {title_escaped}</div><br>')
+        html_content.append(_wrap_post_html(data["html"]["body"], data["html"]["footer"]))
         
         # Add raw JSON debug output if debug is enabled.
         # raw_message is the full str(message) serialization and may contain user-controlled
@@ -525,23 +786,23 @@ class PostParser:
                 serializing every post.
             sanitize: when True, run the html body and footer through a single
                 bleach pass each. Single-post HTML and JSON need this (there is no
-                whole-feed pass on those paths). Feed generation passes False and
-                relies on the final sanitize in rss_generator (per-post for RSS,
-                whole-feed for HTML), so no
-                fragment is sanitized more than once.
+                feed-level pass on those paths). Feed generation passes False and
+                relies on the per-post sanitize in rss_generator._render_pipeline
+                (per-post for BOTH RSS and HTML), so no fragment is sanitized more
+                than once.
         """
         # Compute html body once — avoids triple _generate_html_body calls.
-        # The internal per-fragment sanitize passes were removed (4.4); sanitize
-        # exactly once per output boundary here when requested.
+        # The internal per-fragment sanitize passes are gone; sanitize runs exactly
+        # once at the output boundary here when requested, never inside body/footer.
         html_body = self._generate_html_body(message)
-        # NOTE (stage-4 4.4 consequence): flags are now extracted from the PRE-sanitize
-        # body (the per-fragment sanitize that used to run inside _generate_html_body was
-        # removed). Legitimate links are unaffected — bleach keeps whitelisted
-        # <a href="http(s)://…"> — so link/foreign_channel/mention flags are identical for
-        # normal content. They can differ ONLY for URL-like text that bleach would strip
-        # (e.g. a URL inside a disallowed attribute); flags are non-security (used for
-        # exclude_flags filtering / display), so this edge divergence is accepted rather
-        # than re-adding a per-message sanitize pass that 4.4 deliberately eliminated.
+        # NOTE: flags are extracted from the PRE-sanitize body (there is no longer a
+        # per-fragment sanitize inside _generate_html_body). Legitimate links are
+        # unaffected — bleach keeps whitelisted <a href="http(s)://…"> — so
+        # link/foreign_channel/mention flags are identical for normal content. They can
+        # differ ONLY for URL-like text that bleach would strip (e.g. a URL inside a
+        # disallowed attribute); flags are non-security (used for exclude_flags filtering
+        # / display), so this edge divergence is accepted rather than re-adding the
+        # per-message sanitize pass that was deliberately eliminated.
         flags = self._extract_flags(message, html_body=html_body)
         footer = self.generate_html_footer(message, flags_list=flags)
         if sanitize:
@@ -587,41 +848,10 @@ class PostParser:
     
 
     def _sanitize_html(self, html_raw: str) -> str:
-        import time as _time
-        sanitize_start = _time.monotonic()
-        allowed_tags = ['p', 'a', 'b', 'i', 'strong',
-                        'em', 's', 'del', 'ul', 'ol', 'li', 'br',
-                        'div', 'span', 'img', 'video', 'audio',
-                        'source']
-        allowed_attributes = {
-            'a': ['href', 'title', 'target'],
-            'img': ['src', 'alt', 'style'],
-            'video': ['controls', 'src', 'style'],
-            'audio': ['controls', 'style'],
-            'source': ['src', 'type'],
-            'div': ['class', 'style'],
-            'span': ['class']
-        }
-
-        try:
-            css_sanitizer = CSSSanitizer(
-                allowed_css_properties=["max-width", "max-height", "object-fit", "width", "height"]
-            )
-            sanitized_html = HTMLSanitizer(
-                html_raw,
-                tags=allowed_tags,
-                attributes=allowed_attributes,
-                protocols=['http', 'https', 'tg'],
-                css_sanitizer=css_sanitizer,
-                strip=True,
-            )
-            elapsed = _time.monotonic() - sanitize_start
-            if elapsed > 0.05:
-                logger.warning(f"diag_sanitize_slow: bleach.clean() took {elapsed:.3f}s, input_len={len(html_raw)}")
-            return sanitized_html
-        except Exception as e:
-            logger.error(f"html_sanitization_error: error {str(e)}")
-            return html_raw
+        # Delegate to the single project-wide bleach config (sanitizer.sanitize_html).
+        # This drops the former fail-open branch: sanitize_html is fail-closed and
+        # html.escape()s on any bleach error (registry §3.2).
+        return sanitize_html(html_raw)
 
     def _format_forward_info(self, message: Message) -> Union[str, None]:
         if forward_origin := getattr(message, "forward_origin", None):
@@ -712,12 +942,13 @@ class PostParser:
 
 
         if poll_html: content_body.append(poll_html) # Poll
+        if special_html := self._format_special_media(message): content_body.append(special_html) # Special media info blocks
         if message.forward_origin: content_body.append(f"--- Forwarded post end ---") # Forward info end
         content_body.append(f'</div><br>')
 
-        # NOTE: sanitize is NOT applied here. Sanitization happens exactly once per
+        # NOTE: sanitize is NOT applied here. Sanitization happens exactly once at the
         # output boundary (process_message for single-post/JSON; in rss_generator the
-        # per-post pass for RSS and the whole-feed pass for HTML). See the map (4.4).
+        # per-post pass in _render_pipeline for BOTH RSS and HTML).
         html_body = '\n'.join(content_body)
         return html_body
 
@@ -727,7 +958,27 @@ class PostParser:
 
         content_media = []
         base_url = Config['pyrogram_bridge_url']
-        if message.media and message.media != MessageMediaType.POLL:
+        # Poll media (Kurigram 2.2.23): a poll may carry description_media which IS
+        # renderable through the regular /media pipeline.
+        poll_media_obj, poll_media_kind = _poll_media_object(message)
+        if message.media == MessageMediaType.PAID_MEDIA:
+            # Paid media cannot be downloaded (it is paid content) — render an info
+            # block instead of a media element. Attributes via getattr only.
+            paid_media = getattr(message, 'paid_media', None)
+            stars_amount = getattr(paid_media, 'stars_amount', 0) if paid_media else 0
+            paid_items = getattr(paid_media, 'media', None) if paid_media else None
+            items_count = len(paid_items) if isinstance(paid_items, (list, tuple)) else 0
+            content_media.append(f'<div class="message-media">')
+            content_media.append(f'<div class="paid-media">⭐ Paid media ({stars_amount} stars, '
+                                f'{items_count} item(s)) — available in Telegram</div>')
+            content_media.append('</div>')
+        # Info-block-only media types (NO_IMAGE_MEDIA_TYPES) are rendered by
+        # _format_special_media — do not open an empty message-media container for
+        # them. PAID_MEDIA (also in that set) is handled by the branch above; POLL is
+        # not in the set and is let in only when it carries renderable poll media.
+        elif (message.media
+                and message.media not in NO_IMAGE_MEDIA_TYPES
+                and (message.media != MessageMediaType.POLL or poll_media_obj is not None)):
             content_media.append(f'<div class="message-media">')
 
             file_unique_id = self._get_file_unique_id(message)
@@ -738,7 +989,6 @@ class PostParser:
                 # Guard: channel_username may be None for private chats without a username
                 if not channel_username:
                     logger.warning(f"Could not generate media URL for message {message.id}: channel username is missing.")
-                    content_media.append('</div>')
                 else:
                     file = f"{channel_username}/{message.id}/{file_unique_id}"
                     digest = generate_media_digest(file)
@@ -746,50 +996,35 @@ class PostParser:
 
                     logger.debug(f"Collected media file: {channel_username}/{message.id}/{file_unique_id}")
 
-                    # Check if document is a PDF file
-                    if (message.media == MessageMediaType.DOCUMENT and
-                        message.document is not None and hasattr(message.document, 'mime_type') and
-                        message.document.mime_type == 'application/pdf'):
-                        # Only attempt to create link if channel_username is available
-                        if channel_username.startswith('-100'):
-                            tg_link = f"https://t.me/c/{channel_username[4:]}/{message.id}"
-                        else:
-                            tg_link = f"https://t.me/{channel_username}/{message.id}"
-                        content_media.append(f'<div class="document-pdf" style="padding: 10px;">')
-                        content_media.append(f'<a href="{tg_link}" target="_blank">[PDF-файл]</a></div>')
-                    elif message.media in [MessageMediaType.PHOTO, MessageMediaType.DOCUMENT]:
-                        content_media.append(f'<img src="{url}" style="max-width:100%; width:auto; height:auto;'
-                                            f'max-height:400px; object-fit:contain;">')
-                    elif message.media == MessageMediaType.VIDEO:
-                        content_media.append(f'<video controls src="{url}" style="max-width:100%; width:auto;'
-                                            f'height:auto; max-height:400px;"></video>')
-                    elif message.media == MessageMediaType.ANIMATION:
-                        content_media.append(f'<video controls src="{url}" style="max-width:100%; width:auto;'
-                                            f'height:auto; max-height:400px;"></video>')
-                    elif message.media == MessageMediaType.VIDEO_NOTE:
-                        content_media.append(f'<video controls src="{url}" style="max-width:100%; width:auto;'
-                                            f'height:auto; max-height:400px;"></video>')
-                    elif message.media == MessageMediaType.AUDIO:
-                        mime_type = getattr(message.audio, 'mime_type', 'audio/mpeg')
-                        content_media.append(f'<audio controls style="width:100%; max-width:400px;">'
-                                            f'<source src="{url}" type="{mime_type}"></audio>')
-                        content_media.append('<br>')
-                    elif message.media == MessageMediaType.VOICE:
-                        mime_type = getattr(message.voice, 'mime_type', 'audio/ogg')
-                        content_media.append(f'<audio controls style="width:100%; max-width:400px;">'
-                                            f'<source src="{url}" type="{mime_type}"></audio>')
-                        content_media.append('<br>')
-                    elif message.media == MessageMediaType.STICKER:
-                        emoji = getattr(message.sticker, 'emoji', '')
-                        if getattr(message.sticker, 'is_video', False):
-                            content_media.append(f'<video controls autoplay loop muted src="{url}"'
-                                                f'style="max-width:100%; width:auto; height:auto; max-height:200px;'
-                                                f'object-fit:contain;"></video>')
-                        else:
-                            content_media.append(f'<img src="{url}" alt="Sticker {emoji}" style="max-width:100%;'
-                                                f'width:auto; height:auto; max-height:200px; object-fit:contain;">')
-                    content_media.append('</div>')
-        
+                    # Dispatch through the media table: the selector gives the render
+                    # kind, the RENDERERS map gives the fragment. mime/emoji/tg_link
+                    # defaults are chosen here BY MEDIA TYPE (the renderer never guesses);
+                    # kind None (WEB_PAGE) emits no element — the empty container matches
+                    # the old cascade where no branch matched.
+                    selector = MEDIA_SOURCES.get(message.media)
+                    kind = selector(message)[1] if selector is not None else None
+                    renderer = RENDERERS.get(kind) if kind else None
+                    if renderer is not None:
+                        ctx = RenderCtx(url=url)
+                        if kind == 'pdf':
+                            if channel_username.startswith('-100'):
+                                ctx.tg_link = f"https://t.me/c/{channel_username[4:]}/{message.id}"
+                            else:
+                                ctx.tg_link = f"https://t.me/{channel_username}/{message.id}"
+                        elif kind == 'audio':
+                            if message.media == MessageMediaType.VOICE:
+                                ctx.mime = getattr(message.voice, 'mime_type', 'audio/ogg')
+                            else:
+                                ctx.mime = getattr(message.audio, 'mime_type', 'audio/mpeg')
+                        elif kind == 'img_200_sticker':
+                            ctx.emoji = getattr(message.sticker, 'emoji', '')
+                        content_media.extend(renderer(ctx))
+            # Registry §3.14: close the message-media container in EVERY branch. It used
+            # to be closed only when a URL was built (or the username guard fired), so a
+            # None file_unique_id (e.g. WEB_PAGE without photo) left an open <div> that
+            # html5lib later swallowed the following posts into.
+            content_media.append('</div>')
+
         if webpage := getattr(message, "web_page", None): # Web page preview
             if webpage_html := self._format_webpage(webpage, message):
                 if len((message.text or message.caption or '').strip()) <= 10:
@@ -798,7 +1033,7 @@ class PostParser:
                     content_media.append('</div>')
 
         # Not sanitized here — this fragment is embedded in the body and sanitized
-        # once at the output boundary (see the 4.4 sanitize coverage map).
+        # once at the output boundary (single sanitize pass, see process_message).
         html_media = '\n'.join(content_media)
         return html_media
 
@@ -879,8 +1114,9 @@ class PostParser:
             flags_html = self._format_flags(current_flags)
             content_footer.append('<br>' + flags_html)
             
-        # Not sanitized here — sanitized once at the output boundary (4.4 coverage map):
-        # process_message for single-post/JSON; per-post (RSS) / whole-feed (HTML) for feeds.
+        # Not sanitized here — sanitized once at the output boundary:
+        # process_message for single-post/JSON; per-post in _render_pipeline for feeds
+        # (both RSS and HTML).
         html_footer = '\n'.join(content_footer)
         return html_footer
 
@@ -930,7 +1166,12 @@ class PostParser:
                     else: emoji = "❓" # Default for unknown cases
                     reactions_html += f'<span class="reaction">{emoji} {reaction.count}&nbsp;&nbsp;</span>'
                 reactions_html = reactions_html.rstrip()
-                first_line_parts.append(reactions_html)
+                # An empty reactions object (reactions.reactions == []) produces no
+                # spans; appending the empty string would emit a leading
+                # '&nbsp;&nbsp;|&nbsp;&nbsp;' separator in the footer. Skip it
+                # (registry §3.15). Affects single posts too.
+                if reactions_html:
+                    first_line_parts.append(reactions_html)
 
             # Add views
             if views := getattr(message, "views", None):
@@ -968,7 +1209,7 @@ class PostParser:
                     parts.append('&nbsp;|&nbsp;'.join(links))
 
             # Raw fragment — embedded in the footer, sanitized once at the output
-            # boundary (see the 4.4 sanitize coverage map). Not sanitized here.
+            # boundary (single sanitize pass, see process_message). Not sanitized here.
             result_html = '<br>'.join(parts) if parts else None
             return result_html if result_html else None
             
@@ -996,25 +1237,117 @@ class PostParser:
             logger.error(f"poll_parsing_error: {str(e)}")
             return '<div class="message-poll">[Error displaying poll]</div>'
 
+    @staticmethod
+    def _format_osm_link(location) -> Union[str, None]:
+        """Build an OpenStreetMap link for a location object, or None if coords are unusable."""
+        latitude = getattr(location, 'latitude', None) if location else None
+        longitude = getattr(location, 'longitude', None) if location else None
+        if not isinstance(latitude, (int, float)) or not isinstance(longitude, (int, float)):
+            return None
+        osm_url = f"https://www.openstreetmap.org/?mlat={latitude}&mlon={longitude}#map=16/{latitude}/{longitude}"
+        return f'<a href="{osm_url}">{latitude:.5f}, {longitude:.5f}</a>'
+
+    def _format_special_media(self, message: Message) -> Union[str, None]:
+        """Render an info block for media types that carry no downloadable file.
+
+        Covers giveaways, giveaway winners, checklists, contacts, locations, venues,
+        dice, games, invoices and UNSUPPORTED content (Kurigram 2.2.23). Each block is
+        gated on message.media so unrelated messages never render these. All new
+        Message attributes are accessed via getattr only (older objects/mocks do not
+        define them) and ALL user-controlled strings go through html.escape.
+        """
+        try:
+            media = getattr(message, 'media', None)
+            block = None
+
+            if media == MessageMediaType.GIVEAWAY and (giveaway := getattr(message, 'giveaway', None)):
+                quantity = getattr(giveaway, 'quantity', None)
+                months = getattr(giveaway, 'months', None)
+                stars = getattr(giveaway, 'stars', None)
+                until_date = getattr(giveaway, 'until_date', None)
+                description = getattr(giveaway, 'description', None)
+                block = f"🎁 Giveaway: {quantity} prize(s)"
+                if months: block += f" × {months} months Premium"
+                elif stars: block += f" × {stars} Stars"
+                if until_date is not None and hasattr(until_date, 'strftime'):
+                    block += f" — until {until_date.strftime('%d/%m/%Y')}"
+                if isinstance(description, str) and description.strip():
+                    block += f"<br>{html.escape(description.strip())}"
+
+            elif media == MessageMediaType.GIVEAWAY_WINNERS and (winners := getattr(message, 'giveaway_winners', None)):
+                winner_count = getattr(winners, 'winner_count', None)
+                quantity = getattr(winners, 'quantity', None)
+                prize_description = getattr(winners, 'prize_description', None)
+                block = f"🏆 Giveaway winners: {winner_count} of {quantity}"
+                if isinstance(prize_description, str) and prize_description.strip():
+                    block += f"<br>{html.escape(prize_description.strip())}"
+
+            elif media == MessageMediaType.CHECKLIST and (checklist := getattr(message, 'checklist', None)):
+                title = getattr(checklist, 'title', None)
+                title_str = title if isinstance(title, str) else ''
+                lines = [f"📝 {html.escape(title_str)}" if title_str else "📝 Checklist"]
+                for task in (getattr(checklist, 'tasks', None) or []):
+                    completed = bool(getattr(task, 'completed_by', None) or getattr(task, 'completion_date', None))
+                    mark = "☑" if completed else "☐"
+                    task_text = getattr(task, 'text', '')
+                    task_str = task_text if isinstance(task_text, str) else str(task_text)
+                    lines.append(f"{mark} {html.escape(task_str)}")
+                block = '<br>'.join(lines)
+
+            elif media == MessageMediaType.CONTACT and (contact := getattr(message, 'contact', None)):
+                first_name = getattr(contact, 'first_name', None) or ''
+                last_name = getattr(contact, 'last_name', None) or ''
+                phone_number = getattr(contact, 'phone_number', None) or ''
+                full_name = ' '.join(part for part in [str(first_name), str(last_name)] if part)
+                block = f"👤 {html.escape(full_name)}"
+                if phone_number:
+                    block += f" — {html.escape(str(phone_number))}"
+
+            elif media == MessageMediaType.LOCATION and (location := getattr(message, 'location', None)):
+                osm_link = self._format_osm_link(location)
+                block = f"📍 Location: {osm_link}" if osm_link else "📍 Location"
+
+            elif media == MessageMediaType.VENUE and (venue := getattr(message, 'venue', None)):
+                venue_title = getattr(venue, 'title', None) or ''
+                venue_address = getattr(venue, 'address', None) or ''
+                venue_label = ', '.join(part for part in [str(venue_title), str(venue_address)] if part)
+                block = f"📍 {html.escape(venue_label)}" if venue_label else "📍 Venue"
+                if osm_link := self._format_osm_link(getattr(venue, 'location', None)):
+                    block += f" — {osm_link}"
+
+            elif media == MessageMediaType.DICE and (dice := getattr(message, 'dice', None)):
+                dice_emoji = getattr(dice, 'emoji', None) or '🎲'
+                dice_value = getattr(dice, 'value', None)
+                block = f"🎲 {html.escape(str(dice_emoji))}: {dice_value}"
+
+            elif media == MessageMediaType.GAME:
+                game_title = getattr(getattr(message, 'game', None), 'title', None)
+                if isinstance(game_title, str) and game_title.strip():
+                    block = f"🎮 Game: {html.escape(game_title.strip())}"
+                else:
+                    block = "🎮 Game"
+
+            elif media == MessageMediaType.INVOICE:
+                block = "🧾 Invoice"
+
+            elif media == MessageMediaType.UNSUPPORTED:
+                block = "⚠️ This post contains content not supported by the bridge — open it in Telegram."
+
+            if block is None:
+                return None
+            return f'<div class="message-special">{block}</div>'
+        except Exception as e:
+            logger.error(f"special_media_parsing_error: message_id {getattr(message, 'id', 'unknown')}, error {str(e)}")
+            return None
+
     def _get_file_unique_id(self, message: Message) -> Union[str, None]:
         try:
-            media_mapping = {
-                MessageMediaType.PHOTO:         lambda m: m.photo.file_unique_id,
-                MessageMediaType.VIDEO:         lambda m: m.video.file_unique_id,
-                MessageMediaType.DOCUMENT:      lambda m: m.document.file_unique_id,
-                MessageMediaType.AUDIO:         lambda m: m.audio.file_unique_id,
-                MessageMediaType.VOICE:         lambda m: m.voice.file_unique_id,
-                MessageMediaType.VIDEO_NOTE:    lambda m: m.video_note.file_unique_id,
-                MessageMediaType.ANIMATION:     lambda m: m.animation.file_unique_id,
-                MessageMediaType.STICKER:       lambda m: m.sticker.file_unique_id,
-                MessageMediaType.WEB_PAGE:      lambda m: m.web_page.photo.file_unique_id if m.web_page and m.web_page.photo else None
-            }
-            
-            if message.media in media_mapping:
-                return media_mapping[message.media](message)
-            
-            return None
-            
+            selector = MEDIA_SOURCES.get(message.media)
+            if selector is None:
+                return None
+            selected_obj, _kind = selector(message)
+            return getattr(selected_obj, 'file_unique_id', None)
+
         except Exception as e:
             logger.error(f"file_id_extraction_error: media_type {message.media}, error {str(e)}")
             return None
@@ -1051,21 +1384,20 @@ class PostParser:
                 return
 
             if message.media:
-                # Skip large videos - they shouldn't be cached permanently
-                if message.video and message.video.file_size and message.video.file_size > 100 * 1024 * 1024:
+                # Object selection goes through the single MEDIA_SOURCES table instead
+                # of the old truthy-attribute if/elif ladder. paid_media has no table
+                # entry and is deliberately NOT collected (paid content).
+                selector = MEDIA_SOURCES.get(message.media)
+                selected_obj = selector(message)[0] if selector is not None else None
+
+                # Registry §3.13: the ">100MB don't cache" rule applies to ANY selected
+                # media object, not just message.video (the old code guarded only the
+                # video type here, plus the newer live_photo/story/poll sources).
+                file_size = getattr(selected_obj, 'file_size', None)
+                if isinstance(file_size, int) and file_size > 100 * 1024 * 1024:
                     return
 
-                file_unique_id = ''
-                if   message.photo:      file_unique_id = message.photo.file_unique_id
-                elif message.video:      file_unique_id = message.video.file_unique_id
-                elif message.document:   file_unique_id = message.document.file_unique_id
-                elif message.audio:      file_unique_id = message.audio.file_unique_id
-                elif message.voice:      file_unique_id = message.voice.file_unique_id
-                elif message.video_note: file_unique_id = message.video_note.file_unique_id
-                elif message.animation:  file_unique_id = message.animation.file_unique_id
-                elif message.sticker:    file_unique_id = message.sticker.file_unique_id
-                elif message.web_page and message.web_page.photo:
-                    file_unique_id = message.web_page.photo.file_unique_id
+                file_unique_id = getattr(selected_obj, 'file_unique_id', '') or ''
 
                 if file_unique_id:
                     added_ts = datetime.now().timestamp()
@@ -1084,8 +1416,8 @@ class PostParser:
         if hasattr(chat, 'usernames') and chat.usernames: # Check many usernames
             active_usernames = [u.username for u in chat.usernames if u.active]
             if active_usernames:
-                return active_usernames[0]
-        if hasattr(chat, 'username') and chat.username: return chat.username # Check single username
+                return active_usernames[0].lower() # Canonicalize: usernames are case-insensitive
+        if hasattr(chat, 'username') and chat.username: return chat.username.lower() # Check single username (canonicalized)
         
         # Return numeric ID only if no username found
         if isinstance(chat.id, int) and str(chat.id).startswith('-100'): return str(chat.id)

pyrogram-bridge.code-workspace

@@ -6,24 +6,24 @@
 	],
 	"settings": {
 		"workbench.colorCustomizations": {
-			"activityBar.activeBackground": "#ebec63",
-			"activityBar.background": "#ebec63",
-			"activityBar.foreground": "#15202b",
-			"activityBar.inactiveForeground": "#15202b99",
-			"activityBarBadge.background": "#15a9aa",
-			"activityBarBadge.foreground": "#e7e7e7",
-			"commandCenter.border": "#15202b99",
-			"sash.hoverBorder": "#ebec63",
-			"titleBar.activeBackground": "#e5e636",
-			"titleBar.activeForeground": "#15202b",
-			"titleBar.inactiveBackground": "#e5e63699",
-			"titleBar.inactiveForeground": "#15202b99",
-			"statusBar.background": "#e5e636",
-			"statusBar.foreground": "#15202b",
-			"statusBarItem.hoverBackground": "#cecf1a",
-			"statusBarItem.remoteBackground": "#e5e636",
-			"statusBarItem.remoteForeground": "#15202b"
+			"activityBar.activeBackground": "#ad46b7",
+			"activityBar.background": "#ad46b7",
+			"activityBar.foreground": "#e7e7e7",
+			"activityBar.inactiveForeground": "#e7e7e799",
+			"activityBarBadge.background": "#b8ae47",
+			"activityBarBadge.foreground": "#15202b",
+			"commandCenter.border": "#e7e7e799",
+			"sash.hoverBorder": "#ad46b7",
+			"titleBar.activeBackground": "#8a3892",
+			"titleBar.activeForeground": "#e7e7e7",
+			"titleBar.inactiveBackground": "#8a389299",
+			"titleBar.inactiveForeground": "#e7e7e799",
+			"statusBar.background": "#8a3892",
+			"statusBar.foreground": "#e7e7e7",
+			"statusBarItem.hoverBackground": "#ad46b7",
+			"statusBarItem.remoteBackground": "#8a3892",
+			"statusBarItem.remoteForeground": "#e7e7e7"
 		},
-		"peacock.color": "#e5e636"
+		"peacock.color": "#8a3892"
 	}
 }

pytest.ini

@@ -1,8 +1,9 @@
 [pytest]
+testpaths = tests
 addopts = -ra
 # The stage-1 anti-hang tests are async; run them without per-test event loops
 # being set up by hand. (The tests also carry @pytest.mark.asyncio explicitly, so
 # they work in strict mode too — but auto keeps the plugin's requirement obvious.)
 asyncio_mode = auto
 filterwarnings =
-    ignore::DeprecationWarning 
+    ignore::DeprecationWarning

requirements.txt

@@ -4,7 +4,7 @@ fastapi==0.115.8
 starlette==0.45.3
 uvicorn==0.34.0
 python-multipart==0.0.20
-Kurigram==2.2.22
+Kurigram==2.2.23
 #git+https://github.com/KurimuzonAkuma/pyrogram.git@dev#egg=Kurigram
 TgCrypto
 uvloop

rss_generator.py

@@ -9,94 +9,116 @@
 # pylance: disable=reportMissingImports, reportMissingModuleSource
 # mypy: disable-error-code="import-untyped"
 
-import copy
 import logging
 import asyncio
 import re
 import time
-from html import escape as html_escape
+from dataclasses import dataclass
 from datetime import datetime, timezone
-from types import SimpleNamespace
 from typing import Optional
 from feedgen.feed import FeedGenerator
 from pyrogram import errors, Client
 from pyrogram.types import Message
-from post_parser import PostParser
+from post_parser import PostParser, _wrap_post_html
 from config import get_settings
 from tg_throttle import tg_rpc_bounded
-from bleach.css_sanitizer import CSSSanitizer
-from bleach import clean as HTMLSanitizer
+from sanitizer import sanitize_html
 
 Config = get_settings()
 
 logger = logging.getLogger(__name__)
 
-def _create_time_based_media_groups(messages: list[Message], merge_seconds: int = 5) -> list[Message]:
-    """
-    Create media groups based on time difference between messages
+
+@dataclass
+class PreparedFeed:
+    """Output of _prepare_feed_posts — everything both formatters need after
+    fetch -> enrich -> render -> filter -> sanitize."""
+    channel_username: str
+    channel_title: str            # used by RSS metadata only
+    posts: list[dict]             # rendered, filtered, sorted, SANITIZED
+
+
+class ChannelNotFound(Exception):
+    """Channel could not be resolved to a username; formatter -> create_error_feed."""
+    def __init__(self, channel_identifier):
+        # The prepared identifier (str or int), rendered into the error feed.
+        self.channel_identifier = channel_identifier
+        super().__init__(str(channel_identifier))
+
+def _compute_time_based_group_ids(messages: list[Message], merge_seconds: int = 5) -> dict[int, str | int]:
+    """Return {message.id: effective_media_group_id} WITHOUT mutating messages.
 
     Plain synchronous function (contains no await): runs inside the render thread
     via _render_pipeline. Must not touch asyncio.
+
+    Contract: all messages belong to ONE chat (message.id is unique only per
+    chat); callers must not mix chats in a single call.
+
+    Pure re-implementation of the old _create_time_based_media_groups mutation,
+    reproducing its result for every input the old code survived on PRODUCTION
+    data (naive kurigram dates). Inputs only aware-date test mocks could produce
+    (fully-None and aware+None mixes, where the old code clustered the None-date
+    tail by insertion order incl. truthy-id adoption) are deliberately replaced
+    (registry §3.11):
+      - messages WITHOUT a date do not participate in time clustering and get
+        NO mapping entry (their own media_group_id still applies downstream);
+      - dated messages are ordered ascending by date via a timestamp key
+        (naive-safe: no aware/naive mix once None-date are excluded);
+      - a message joins the current cluster if the gap to the PREVIOUS message
+        is <= merge_seconds; the gap is a NAIVE datetime subtraction
+        (msg.date - prev.date).total_seconds(), exactly as the old code — NOT a
+        timestamp diff (they diverge across a DST fold, and the old behavior is
+        the contract);
+      - the effective id of a cluster is the FIRST TRUTHY media_group_id in
+        cluster order (old code used truthiness, not `is not None`, and
+        overwrote members' own differing ids — kept);
+      - a cluster of >= 2 members with no truthy id gets a synthetic
+        f"time_{min(dates)}" id (exact old format);
+      - singleton clusters and clusters with no effective id produce NO entries;
+        every member of a cluster with an effective id gets one.
     """
-    # Deep-copy the input list to avoid mutating cached Message objects (the cache may
-    # reuse the same objects across calls with different merge_seconds values)
-    messages = copy.deepcopy(messages)
-    # Compute fallback once so all None-date messages get the same sort key (deterministic order)
-    _sort_fallback = datetime.now(timezone.utc)
-    messages_sorted = sorted(messages, key=lambda msg: msg.date or _sort_fallback) # type: ignore
+    dated = sorted((m for m in messages if m.date is not None),
+                   key=lambda m: m.date.timestamp())  # type: ignore
+    group_ids: dict[int, str | int] = {}
+
+    def _flush(cluster: list[Message], effective: str | int | None) -> None:
+        if len(cluster) < 2:
+            return
+        if not effective:
+            # All members are dated here (None-date excluded above), so min() is safe.
+            effective = f"time_{min(m.date for m in cluster)}"  # type: ignore
+        for m in cluster:
+            group_ids[m.id] = effective
+
     cluster: list[Message] = []
-    last_msg_date: datetime = datetime.now(timezone.utc)
-    current_media_group_id: Optional[str] = None
+    effective: str | int | None = None
+    prev_date: Optional[datetime] = None
 
-    for msg in messages_sorted:
-        
+    for msg in dated:
+        mgid = getattr(msg, "media_group_id", None)
         if not cluster:
-            cluster.append(msg)
-            # Use current time as fallback when date is None
-            last_msg_date = msg.date or datetime.now(timezone.utc) # type: ignore
-            current_media_group_id = getattr(msg, "media_group_id", None)
-            continue
-        
-        # Use current time as fallback when date is None to avoid TypeError in subtraction
-        msg_date = msg.date or datetime.now(timezone.utc)
-        time_diff = (msg_date - last_msg_date).total_seconds()
-        
-        msg_media_group_id = getattr(msg, "media_group_id", None)
-        
-        if time_diff <= merge_seconds:
-            if current_media_group_id:
-                msg.media_group_id = current_media_group_id  # type: ignore
-            elif msg_media_group_id:
-                current_media_group_id = msg_media_group_id
-                for m in cluster:
-                    m.media_group_id = current_media_group_id  # type: ignore
-            cluster.append(msg)
-            # Use current time as fallback when date is None
-            last_msg_date = msg.date or datetime.now(timezone.utc) # type: ignore
-        else:
-            if len(cluster) >= 2 and not current_media_group_id:
-                dates = [m.date for m in cluster if m.date is not None]
-                if dates:
-                    min_date = min(dates)
-                    new_group_id = f"time_{min_date}"
-                    for m in cluster:
-                        m.media_group_id = new_group_id  # type: ignore
             cluster = [msg]
-            # Use current time as fallback when date is None
-            last_msg_date = msg.date or datetime.now(timezone.utc) # type: ignore
-            current_media_group_id = msg_media_group_id
-    
-    if len(cluster) >= 2 and not current_media_group_id:
-        dates = [m.date for m in cluster if m.date is not None]
-        if dates:
-            min_date = min(dates)
-            new_group_id = f"time_{min_date}"
-            for m in cluster:
-                m.media_group_id = new_group_id  # type: ignore
+            effective = mgid or None
+            prev_date = msg.date
+            continue
+        time_diff = (msg.date - prev_date).total_seconds()  # type: ignore
+        if time_diff <= merge_seconds:
+            cluster.append(msg)
+            # First truthy id in cluster order wins; keep it even if this member
+            # carries a different truthy id (the old code overwrote it).
+            if not effective and mgid:
+                effective = mgid
+            prev_date = msg.date
+        else:
+            _flush(cluster, effective)
+            cluster = [msg]
+            effective = mgid or None
+            prev_date = msg.date
 
-    return messages_sorted
+    _flush(cluster, effective)
+    return group_ids
 
-def _create_messages_groups(messages: list[Message]) -> list[list[Message]]:
+def _create_messages_groups(messages: list[Message], group_ids: dict[int, str | int] | None = None) -> list[list[Message]]:
     """
     Process messages into formatted posts, handling media groups
 
@@ -104,7 +126,11 @@ def _create_messages_groups(messages: list[Message]) -> list[list[Message]]:
     """
     processing_groups: list[list[Message]] = []
     media_groups: dict[str | int, list[Message]] = {}
-    
+    # Time-clustering supplies effective ids as a PURE mapping (no message mutation,
+    # no deepcopy). Absent an entry, the message's own media_group_id applies
+    # (registry §3.11 / spec Этап 4).
+    group_ids = group_ids or {}
+
     # First pass - collect messages and organize into processing groups
     for message in messages:
         try:
@@ -120,10 +146,11 @@ def _create_messages_groups(messages: list[Message]) -> list[list[Message]]:
                 if 'CHANNEL_CHAT_CREATED'   in str(message.service): continue
                 if 'DELETE_CHAT_PHOTO'      in str(message.service): continue
 
-            if message.media_group_id:
-                if message.media_group_id not in media_groups:
-                    media_groups[message.media_group_id] = []
-                media_groups[message.media_group_id].append(message)
+            effective_group_id = group_ids.get(message.id, message.media_group_id)
+            if effective_group_id:
+                if effective_group_id not in media_groups:
+                    media_groups[effective_group_id] = []
+                media_groups[effective_group_id].append(message)
             else:
                 processing_groups.append([message]) # Single message becomes its own processing group
                 
@@ -137,70 +164,17 @@ def _create_messages_groups(messages: list[Message]) -> list[list[Message]]:
         media_group.sort(key=lambda x: x.id, reverse=False)
         processing_groups.append(media_group)
     
-    # Sort processing groups by date of first message in each group
-    processing_groups.sort(key=lambda group: group[0].date if group[0].date else datetime.now(timezone.utc), reverse=True)
-    
+    # Sort processing groups by date of first message in each group. Timestamp-based key
+    # is naive-safe: kurigram dates are naive-local, and a None-date group used to fall
+    # back to an AWARE datetime.now(timezone.utc) here, raising TypeError on the first
+    # naive-vs-aware comparison — a 500 on ANY feed carrying a None-date post, in the
+    # DEFAULT path (registry §3.12). None-date groups now sort as newest (float('inf'))
+    # and deterministically survive the later [:limit] slice; the final post sort in
+    # _render_messages_groups (0.0 fallback) still places them at the tail of the feed.
+    processing_groups.sort(key=lambda group: group[0].date.timestamp() if group[0].date else float('inf'), reverse=True)
+
     return processing_groups
 
-def _trim_messages_groups(messages_groups: list[list[Message]], limit: int):
-    """
-    Trim messages groups to limit
-
-    Plain synchronous function (contains no await): runs inside the render thread.
-    """
-    if len(messages_groups) > limit: # Trim groups if they exceed the specified limit
-        messages_groups = messages_groups[:limit]
-    
-    return messages_groups
-
-def processed_message_to_tg_message(processed_message: dict) -> Message:
-    """
-    Convert processed message dictionary into a Message-like object
-    containing only the attributes needed by generate_html_footer.
-    """
-    # Create a simple chat object
-    chat_info = SimpleNamespace()
-    channel_identifier = processed_message.get('channel')
-    if isinstance(channel_identifier, str) and channel_identifier.startswith('-100'):
-        setattr(chat_info, 'id', int(channel_identifier))
-        setattr(chat_info, 'username', None)
-    else:
-        setattr(chat_info, 'id', None) # Or some placeholder if needed
-        setattr(chat_info, 'username', channel_identifier)
-
-
-    # Convert reactions dict to list of objects
-    reactions_list = []
-    if reactions_dict := processed_message.get('reactions'):
-        for emoji, count in reactions_dict.items():
-            # Assuming no custom/paid reactions in this simplified structure
-            reactions_list.append(SimpleNamespace(emoji=emoji, count=count, is_paid=False, custom_emoji_id=None))
-    
-    # Recreate reactions structure expected by Pyrogram's reaction handling
-    reactions_obj = SimpleNamespace(reactions=reactions_list) if reactions_list else None
-
-    # Create the message-like object
-    tg_message_mock = SimpleNamespace(
-        id=processed_message.get('message_id'),
-        date=datetime.fromtimestamp(processed_message['date'], tz=timezone.utc) if processed_message.get('date') else None,
-        views=processed_message.get('views'),
-        reactions=reactions_obj,
-        chat=chat_info,
-        # Add other attributes if generate_html_footer or its dependencies need them
-        # For now, these seem sufficient based on the analysis of generate_html_footer
-        # and _reactions_views_links.
-        text=processed_message.get('text'), # Add text just in case
-        caption=None, # Assume caption is merged into text by process_message
-        forward_origin=None, # Not directly needed by footer generation logic itself
-        reply_to_message=None, # Not directly needed by footer generation logic itself
-        media=None, # Not needed by footer
-        service=processed_message.get('service') # Potentially needed? Added just in case.
-    )
-
-    # Cast to Message type hint for static analysis, although it's a mock object
-    return tg_message_mock # type: ignore
-
-
 def _render_messages_groups(messages_groups: list[list[Message]],
                                     post_parser: PostParser,
                                     exclude_flags: str | None = None,
@@ -222,15 +196,12 @@ def _render_messages_groups(messages_groups: list[list[Message]],
         try:
             if len(group) == 1: # Single message - simple case
                 one_message = group[0]
-                # Feed path: raw_message not needed and sanitize deferred to the final
-                # whole-feed pass, so each fragment is sanitized exactly once.
+                # Feed path: raw_message not needed and sanitize deferred to the per-post
+                # pass in _render_pipeline (both RSS and HTML), so each fragment is
+                # sanitized exactly once.
                 message_data = post_parser.process_message(one_message, include_raw=False, sanitize=False)
-                html_parts = [
-                    f'<div class="message-body">{message_data["html"]["body"]}</div>',
-                    f'<div class="message-footer">{message_data["html"]["footer"]}</div>'
-                ]
                 rendered_posts.append({
-                    'html': '\n'.join(html_parts),
+                    'html': _wrap_post_html(message_data["html"]["body"], message_data["html"]["footer"]),
                     'date': message_data['date'],
                     'message_id': message_data['message_id'],
                     'title': message_data['html']['title'],
@@ -241,12 +212,14 @@ def _render_messages_groups(messages_groups: list[list[Message]],
             else: # Multiple messages in group - merge text and html body
                 processed_messages = [post_parser.process_message(msg, include_raw=False, sanitize=False) for msg in group]
 
-                # Determine main message for header/footer/title
-                main_message = next(  
-                    (msg for msg in processed_messages if msg['text']),
-                    processed_messages[0]  # fallback if no message contains text
-                )
-            
+                # Determine the main message with the SAME criterion the processed dicts
+                # used: first message that has text or caption, else the first of the
+                # group. processed_messages[i] corresponds to group[i], so main_message
+                # (dict, for title/date/author) and main_raw (the real Message, for the
+                # footer) point at the same index.
+                main_idx = next((i for i, m in enumerate(group) if (m.text or m.caption)), 0)
+                main_raw = group[main_idx]
+                main_message = processed_messages[main_idx]
 
                 # Merge text fields from all messages
                 all_texts = [msg['text'] for msg in processed_messages if msg['text']]
@@ -256,27 +229,19 @@ def _render_messages_groups(messages_groups: list[list[Message]],
                 all_html_bodies = [msg['html']['body'] for msg in processed_messages if msg['html']['body']]
                 combined_html_body = '\n<br><br>\n'.join(all_html_bodies)
 
-                # Collect all unique flags from all messages in the group
-                all_flags = set()
-                for msg in processed_messages:
-                    if msg.get('flags'): # Check if flags exist and are not empty
-                        all_flags.update(msg['flags'])
-                all_flags.add("merged")
-                merged_flags = list(all_flags) # Convert back to list if needed, or keep as set
+                # Deterministic merged flags: first-seen order across the group, then
+                # 'merged' (registry §3.8 — replaces the hash-ordered list(set(...))).
+                merged_flags = list(dict.fromkeys(f for msg in processed_messages for f in msg['flags']))
+                merged_flags.append("merged")
 
-                # generate tg-message from processed message
-                tg_message = processed_message_to_tg_message(main_message)
-
-
-                footer_html = post_parser.generate_html_footer(tg_message, flags_list=merged_flags)
-
-                html_parts = [
-                    f'<div class="message-body">{combined_html_body}</div>',
-                    f'<div class="message-footer">{footer_html}</div>'
-                ]
+                # Render the merged footer DIRECTLY from the real main Message — no
+                # dict->mock round-trip. The raw Message carries its real reactions
+                # (custom emojis get their own span, registry §3.6) and its naive-local
+                # date (registry §3.7), matching a single post of the same message.
+                footer_html = post_parser.generate_html_footer(main_raw, flags_list=merged_flags)
 
                 rendered_posts.append({
-                    'html': '\n'.join(html_parts),
+                    'html': _wrap_post_html(combined_html_body, footer_html),
                     'date': main_message['date'],
                     'message_id': main_message['message_id'],
                     'title': main_message['html']['title'],
@@ -292,16 +257,13 @@ def _render_messages_groups(messages_groups: list[list[Message]],
     # Filter posts by exclude_flags
     if exclude_flags:
         exclude_flag_list = [flag.strip() for flag in exclude_flags.split(',')] # Split comma-separated flags into list
-        filtered_posts = []
-        for post in rendered_posts:
-            # If "all" is specified and the post has any flags, exclude the post.
-            if "all" in exclude_flag_list and post['flags']:
-                continue
-            # Exclude post if any flag in the exclude list is present in the post's flags.
-            if any(flag in post['flags'] for flag in exclude_flag_list):
-                continue
-            filtered_posts.append(post)
-        rendered_posts = filtered_posts
+        rendered_posts = [
+            post for post in rendered_posts
+            # Keep a post unless "all" is requested and it carries any flag, or any of
+            # its flags appears in the exclude list.
+            if not ("all" in exclude_flag_list and post['flags'])
+            and not any(flag in post['flags'] for flag in exclude_flag_list)
+        ]
     
     # Filter posts by exclude_text
     if exclude_text:
@@ -327,35 +289,164 @@ def _render_pipeline(messages: list[Message],
                      exclude_flags: str | None,
                      exclude_text: str | None,
                      merge_seconds: int,
-                     time_based_merge: bool):
+                     time_based_merge: bool,
+                     channel: str | int):
     """
-    Full synchronous feed render pipeline (grouping + trimming + rendering).
+    Full synchronous feed render pipeline (grouping + trimming + rendering + sanitize).
 
     Runs entirely in a worker thread via a single asyncio.to_thread call. It contains
     NO await, NO asyncio primitives, NO create_task/get_running_loop — all the CPU-heavy
-    work (deepcopy, grouping, bleach-free rendering) happens here off the event loop.
+    work (grouping, rendering, bleach) happens here off the event loop.
     Media file-id records are accumulated on post_parser._pending_media_ids and flushed
     by the caller after this returns.
+
+    `channel` is used only for the sanitize log_context (grep-ability).
     """
     if time_based_merge:
-        messages = _create_time_based_media_groups(messages, merge_seconds)
-    message_groups = _create_messages_groups(messages)
-    message_groups = _trim_messages_groups(message_groups, limit)
-    return _render_messages_groups(message_groups, post_parser, exclude_flags, exclude_text)
+        # Pure mapping msg.id -> effective_group_id (no message mutation, no deepcopy),
+        # plus a date-ASC pre-sort with the same naive-safe timestamp key (None-date last
+        # via +inf). A stable sorted() on the fetch-order input reproduces the old
+        # `messages_sorted` order — including ties — and does NOT mutate the input, so the
+        # cache-protecting deepcopy is gone (spec Этап 4).
+        group_ids = _compute_time_based_group_ids(messages, merge_seconds)
+        messages = sorted(messages, key=lambda m: m.date.timestamp() if m.date else float('inf'))
+        message_groups = _create_messages_groups(messages, group_ids)
+    else:
+        message_groups = _create_messages_groups(messages)
+    # Trim groups if they exceed the requested limit (slice is a no-op when shorter).
+    message_groups = message_groups[:limit]
+    posts = _render_messages_groups(message_groups, post_parser, exclude_flags, exclude_text)
+    # Sanitize each surviving (post-filter) post exactly once, here in the worker
+    # thread — no per-post thread hop / per-post CSSSanitizer anymore. Per-post
+    # granularity means a failed post is html.escape()d in isolation instead of the
+    # whole feed (registry §3.5), and an unbalanced fragment is normalized within its
+    # OWN post rather than swallowing the following ones (registry §3.4). sanitize_html
+    # is fail-closed (registry §3.2). For the HTML path the <hr> divider is joined in
+    # the formatter AFTER this, so it survives sanitize (registry §3.3).
+    for post in posts:
+        post['html'] = sanitize_html(
+            post['html'],
+            log_context=f"channel {channel}, message_id {post['message_id']}",
+        )
+    return posts
+
+
+async def _prepare_feed_posts(channel: str | int,
+                              client: Client,
+                              *,
+                              limit: int,
+                              exclude_flags: str | None,
+                              exclude_text: str | None,
+                              merge_seconds: int,
+                              history_limit: int,
+                              enrich_replies: bool,
+                              log_prefix: str) -> PreparedFeed:
+    """Single code path feeding both feeds: validate -> resolve chat -> fetch history ->
+    (optionally) enrich replies -> render/filter/sort/sanitize in one worker thread. The
+    RSS/HTML formatters used to duplicate this ~80% verbatim; path differences are now
+    EXPLICIT parameters (history_limit: RSS over-fetches limit*2; enrich_replies: HTML-only;
+    log_prefix 'rss'|'html' keeps today's paired log-line names).
+
+    Raises ChannelNotFound (formatter -> create_error_feed), re-raises errors.FloodWait
+    for both the get_chat and the get_history path (§3.9; api_server -> HTTP 429), and
+    wraps any other resolution/history error in ValueError with unified text (§3.10).
+    """
+    # 1) Validate limit.
+    if limit < 1:
+        raise ValueError(f"limit must be positive, got {limit}")
+    if limit > 200:
+        raise ValueError(f"limit cannot exceed 200, got {limit}")
+
+    post_parser = PostParser(client=client)
+
+    # 2) Resolve the chat. NOTE: keep `from tg_cache import ...` INSIDE this function —
+    # feed tests monkeypatch tg_cache and rely on late name resolution.
+    channel_info_start_time = time.time()
+    try:
+        channel = post_parser.channel_name_prepare(channel)
+        from tg_cache import cached_get_chat
+        channel_info = await cached_get_chat(post_parser.client, channel)
+        channel_title = channel_info.title or f"Telegram: {channel}"
+        channel_username = channel_info.username or (str(channel_info.id) if channel_info.id and str(channel_info.id).startswith('-100') else None)
+        if not channel_username:
+            # Prepared channel (which could be int) is carried into the error feed.
+            logger.warning(f"Could not get username for channel {channel}, using identifier for error feed.")
+            raise ChannelNotFound(channel)
+    except (errors.UsernameInvalid, errors.UsernameNotOccupied) as e:
+        logger.warning(f"Channel not found error for {channel}: {str(e)}")
+        raise ChannelNotFound(channel) from e
+    except errors.FloodWait:
+        # Let FloodWait bubble up to api_server.py, which returns HTTP 429 with Retry-After.
+        raise
+    except ChannelNotFound:
+        # The no-username branch above — not a resolution failure to wrap in ValueError.
+        raise
+    except Exception as e:
+        logger.error(f"Error during get_chat for channel '{channel}' (type: {type(channel)}): {str(e)}", exc_info=True)
+        raise ValueError(f"Failed to get chat info for {channel}: {str(e)}") from e
+
+    channel_info_elapsed = time.time() - channel_info_start_time
+    logger.debug(f"{log_prefix}_channel_info_timing: channel {channel}, retrieved in {channel_info_elapsed:.3f} seconds")
+
+    # 3) Fetch history.
+    messages_start_time = time.time()
+    try:
+        from tg_cache import cached_get_chat_history
+        messages = await cached_get_chat_history(post_parser.client, channel, limit=history_limit)
+    except errors.FloodWait:
+        # §3.9: FloodWait from history propagates -> HTTP 429 (previously wrapped -> 400).
+        raise
+    except Exception as e:
+        logger.error(f"Error during get_chat_history for channel '{channel}' (type: {type(channel)}): {str(e)}", exc_info=True)
+        raise ValueError(f"Failed to get chat history for {channel}: {str(e)}") from e
+
+    messages_elapsed = time.time() - messages_start_time
+    logger.debug(f"{log_prefix}_messages_retrieval_timing: channel {channel}, {len(messages)} messages retrieved in {messages_elapsed:.3f} seconds")
+
+    # 4) Optional reply enrichment (HTML-only today — deliberate, keeps RSS polling cheap).
+    if enrich_replies:
+        enrichment_start_time = time.time()
+        messages = await _reply_enrichment(client, messages)
+        enrichment_elapsed = time.time() - enrichment_start_time
+        logger.debug(f"{log_prefix}_reply_enrichment_timing: channel {channel}, replies enriched in {enrichment_elapsed:.3f} seconds")
+
+    # 5) Process messages into groups and render them. The whole grouping/trimming/
+    # rendering pipeline is CPU-heavy (per-message rendering) and contains no
+    # await, so run it in ONE worker thread to keep the event loop responsive.
+    processing_start_time = time.time()
+    try:
+        posts = await asyncio.to_thread(
+            _render_pipeline, messages, post_parser, limit,
+            exclude_flags, exclude_text, merge_seconds, Config['time_based_merge'],
+            channel,
+        )
+    finally:
+        # Persist media file-ids collected during rendering with a single bulk upsert —
+        # in a finally so a partial render still records what it collected (the flush is
+        # best-effort and swallows its own errors, so it cannot mask a render exception).
+        await post_parser._flush_pending_media_ids()
+
+    processing_elapsed = time.time() - processing_start_time
+    logger.debug(f"{log_prefix}_messages_processing_timing: channel {channel}, {len(posts)} posts processed in {processing_elapsed:.3f} seconds")
+
+    return PreparedFeed(channel_username=channel_username, channel_title=channel_title, posts=posts)
 
 
 async def generate_channel_rss(channel: str | int,
-                                client: Client, 
-                                limit: int = 20, 
+                                client: Client,
+                                limit: int = 20,
                                 exclude_flags: str | None = None,
                                 exclude_text: str | None = None,
                                 merge_seconds: int = 5
                                 ) -> str:
     """
-    Generate RSS feed for channel using actual messages
+    Generate RSS feed for channel using actual messages.
+
+    Thin formatter over the shared _prepare_feed_posts: it only turns the prepared,
+    sanitized posts into a feedgen RSS document. RSS over-fetches (history_limit=limit*2)
+    and skips reply enrichment (enrich_replies=False).
     Args:
         channel: Telegram channel name
-        post_parser: Optional PostParser instance. If not provided, will create new one
         client: Telegram client instance
         limit: Maximum number of posts to include in the RSS feed
         exclude_flags: Flags to exclude from the RSS feed
@@ -364,44 +455,22 @@ async def generate_channel_rss(channel: str | int,
         RSS feed as string in XML format
     """
     total_start_time = time.time()
-    
-    if limit < 1:
-        raise ValueError(f"limit must be positive, got {limit}")
-    if limit > 200:
-        raise ValueError(f"limit cannot exceed 200, got {limit}")
+    base_url = Config['pyrogram_bridge_url']
 
     try:
-        post_parser = PostParser(client=client)
-            
+        prepared = await _prepare_feed_posts(
+            channel, client,
+            limit=limit, exclude_flags=exclude_flags, exclude_text=exclude_text,
+            merge_seconds=merge_seconds, history_limit=limit * 2,
+            enrich_replies=False, log_prefix="rss",
+        )
+
+        channel_username = prepared.channel_username
+        channel_title = prepared.channel_title
+        final_posts = prepared.posts
+
         fg = FeedGenerator()
         fg.load_extension('dc')
-        base_url = Config['pyrogram_bridge_url']
-        
-        channel_info_start_time = time.time()
-        try:
-            channel  = post_parser.channel_name_prepare(channel)
-            from tg_cache import cached_get_chat
-            channel_info = await cached_get_chat(post_parser.client, channel)
-            channel_title = channel_info.title or f"Telegram: {channel}"
-            channel_username = channel_info.username or (str(channel_info.id) if channel_info.id and str(channel_info.id).startswith('-100') else None)
-            if not channel_username:
-                # Use prepared channel (which could be int) for error feed if username fails
-                logger.warning(f"Could not get username for channel {channel}, using identifier for error feed.")
-                return create_error_feed(str(channel), base_url) # Ensure channel is string for error feed
-        except (errors.UsernameInvalid, errors.UsernameNotOccupied) as e:
-            logger.warning(f"Channel not found error for {channel}: {str(e)}")
-            return create_error_feed(channel, base_url)
-        except errors.FloodWait:
-            # Let FloodWait bubble up to api_server.py, which returns HTTP 429 with Retry-After header
-            raise
-        except Exception as e:
-            logger.error(f"Error during get_chat for channel '{channel}' (type: {type(channel)}): {str(e)}", exc_info=True) # Log error specifically for get_chat
-            # Re-raise the original exception to be caught by the outer handler if needed,
-            # but add specific logging here.
-            raise ValueError(f"Failed to get chat info for {channel}: {str(e)}") from e # Raise a more specific error perhaps
-        
-        channel_info_elapsed = time.time() - channel_info_start_time
-        logger.debug(f"rss_channel_info_timing: channel {channel}, retrieved in {channel_info_elapsed:.3f} seconds")
 
         # Set feed metadata
         main_name = f"{channel_title} (@{channel_username})"
@@ -410,40 +479,7 @@ async def generate_channel_rss(channel: str | int,
         fg.description(f'Telegram channel {channel_username} RSS feed')
         fg.language('ru')
         fg.id(f"{base_url}/rss/{channel_username}") # Use username for feed ID consistency
-        
-        # Collect messages
-        messages_start_time = time.time()
-        try:
-            from tg_cache import cached_get_chat_history
-            
-            messages = await cached_get_chat_history(post_parser.client, channel, limit=limit*2)
-        except Exception as e:
-            logger.error(f"Error during get_chat_history for channel '{channel}' (type: {type(channel)}): {str(e)}", exc_info=True) # Log error specifically for get_chat_history
-            raise ValueError(f"Failed to get chat history for {channel}: {str(e)}") from e # Raise a more specific error
-            
-        messages_elapsed = time.time() - messages_start_time
-        logger.debug(f"rss_messages_retrieval_timing: channel {channel}, {len(messages)} messages retrieved in {messages_elapsed:.3f} seconds")
-        
-        # Process messages into groups and render them.
-        # The whole grouping/trimming/rendering pipeline is CPU-heavy (deepcopy +
-        # per-message rendering) and contains no await, so run it in ONE worker
-        # thread to keep the event loop responsive.
-        processing_start_time = time.time()
-        try:
-            final_posts = await asyncio.to_thread(
-                _render_pipeline, messages, post_parser, limit,
-                exclude_flags, exclude_text, merge_seconds, Config['time_based_merge'],
-            )
-        finally:
-            # Persist media file-ids collected during rendering with a single bulk
-            # upsert — in a finally so a partial render still records what it collected
-            # (the flush is best-effort and swallows its own errors, so it cannot mask a
-            # render exception).
-            await post_parser._flush_pending_media_ids()
 
-        processing_elapsed = time.time() - processing_start_time
-        logger.debug(f"rss_messages_processing_timing: channel {channel}, {len(final_posts)} posts processed in {processing_elapsed:.3f} seconds")
-        
         # Generate feed entries
         feed_gen_start_time = time.time()
         
@@ -463,38 +499,10 @@ async def generate_channel_rss(channel: str | int,
             fe.link(href=post_link)
             
             fe.description(post['text'].replace('\n', ' '))
-            # Sanitize heavy HTML in thread to avoid blocking the loop
-            try:
-                def _sanitize_sync(html_raw: str) -> str:
-                    css_sanitizer = CSSSanitizer(
-                        allowed_css_properties=["max-width", "max-height", "object-fit", "width", "height"]
-                    )
-                    return HTMLSanitizer(
-                        html_raw,
-                        tags=['p', 'a', 'b', 'i', 'strong', 'em', 'ul', 'ol', 'li', 'br', 'div', 'span', 'img', 'video', 'audio', 'source'],
-                        attributes={
-                            'a': ['href', 'title', 'target'],
-                            'img': ['src', 'alt', 'style'],
-                            'video': ['controls', 'src', 'style'],
-                            'audio': ['controls', 'style'],
-                            'source': ['src', 'type'],
-                            'div': ['class', 'style'],
-                            'span': ['class']
-                        },
-                        protocols=['http', 'https', 'tg'],
-                        css_sanitizer=css_sanitizer,
-                        strip=True,
-                    )
-                sanitized_html = await asyncio.to_thread(_sanitize_sync, post['html'])
-            except Exception as e:
-                # FAIL CLOSED: since 4.4 removed the per-fragment passes, post['html'] is
-                # raw channel-controlled HTML, and this is its ONLY sanitize. If bleach
-                # itself throws (e.g. RecursionError on pathological nested HTML), do NOT
-                # emit the raw payload (that would be stored XSS) — html.escape it so the
-                # content survives as inert text.
-                logger.error(f"rss_html_sanitization_error: channel {channel}, message_id {post['message_id']}, error {str(e)}")
-                sanitized_html = html_escape(post['html'])
-            fe.content(content=sanitized_html, type='CDATA')
+            # post['html'] is already sanitized per-post inside _render_pipeline
+            # (single project-wide bleach config, fail-closed). No per-post thread
+            # hop / CSSSanitizer here anymore.
+            fe.content(content=post['html'], type='CDATA')
             
             if post['date'] is not None:
                 pub_date = datetime.fromtimestamp(post['date'], tz=timezone.utc)
@@ -521,7 +529,9 @@ async def generate_channel_rss(channel: str | int,
         total_elapsed = time.time() - total_start_time
         logger.debug(f"rss_total_generation_timing: channel {channel}, total time {total_elapsed:.3f} seconds")
         return rss_feed
-        
+
+    except ChannelNotFound as e:
+        return create_error_feed(str(e.channel_identifier), base_url)
     except Exception as e:
         logger.error(f"generate_channel_rss: channel {channel}, error {str(e)}")
         raise
@@ -583,127 +593,41 @@ async def generate_channel_html(channel: str | int,
                                 merge_seconds: int = 5
                                 ) -> str:
     """
-    Generate HTML feed for channel using actual messages
+    Generate HTML feed for channel using actual messages.
+
+    Thin formatter over the shared _prepare_feed_posts: it only joins the prepared,
+    sanitized posts with the <hr> divider. HTML fetches exactly `limit` messages and
+    enables reply enrichment (enrich_replies=True).
     Args:
         channel: Telegram channel name
-        post_parser: Optional PostParser instance. If not provided, will create new one
         client: Telegram client instance
-        limit: Maximum number of posts to include in the RSS feed
-        exclude_flags: Flags to exclude from the RSS feed
+        limit: Maximum number of posts to include in the HTML feed
+        exclude_flags: Flags to exclude from the HTML feed
         exclude_text: Text to exclude from posts
     Returns:
         HTML feed as string
     """
     total_start_time = time.time()
-    
-    if limit < 1:
-        raise ValueError(f"limit must be positive, got {limit}")
-    if limit > 200:
-        raise ValueError(f"limit cannot exceed 200, got {limit}")
+    base_url = Config['pyrogram_bridge_url']
 
     try:
-        post_parser = PostParser(client=client)
-            
-        base_url = Config['pyrogram_bridge_url']
-        
-        channel_info_start_time = time.time()
-        try:
-            channel = post_parser.channel_name_prepare(channel)
-            logger.debug(f"Prepared channel identifier for HTML: {channel} (type: {type(channel)})") # Log prepared channel
-            from tg_cache import cached_get_chat
-            channel_info = await cached_get_chat(post_parser.client, channel)
-            channel_username = channel_info.username or (str(channel_info.id) if channel_info.id and str(channel_info.id).startswith('-100') else None)
-            if not channel_username:
-                logger.warning(f"Could not get username for channel {channel} in HTML generation, returning error feed structure (as string). NOTE: This should ideally return HTML error page.")
-                # For HTML, returning an error feed string might not be ideal. Consider returning a dedicated HTML error page.
-                return create_error_feed(str(channel), base_url) # Ensure channel is string for error feed
-        except (errors.UsernameInvalid, errors.UsernameNotOccupied) as e:
-            logger.warning(f"Channel not found error for {channel} in HTML generation: {str(e)}")
-            # Consider returning a dedicated HTML error page.
-            return create_error_feed(channel, base_url)
-        except errors.FloodWait:
-            # Let FloodWait bubble up to api_server.py, which returns HTTP 429 with Retry-After header
-            raise
-        except Exception as e:
-            logger.error(f"Error during get_chat for channel '{channel}' (type: {type(channel)}) in HTML generation: {str(e)}", exc_info=True)
-            raise ValueError(f"Failed to get chat info for {channel} in HTML generation: {str(e)}") from e
+        prepared = await _prepare_feed_posts(
+            channel, client,
+            limit=limit, exclude_flags=exclude_flags, exclude_text=exclude_text,
+            merge_seconds=merge_seconds, history_limit=limit,
+            enrich_replies=True, log_prefix="html",
+        )
 
-        channel_info_elapsed = time.time() - channel_info_start_time
-        logger.debug(f"html_channel_info_timing: channel {channel}, retrieved in {channel_info_elapsed:.3f} seconds")
+        final_posts = prepared.posts
 
-        # Collect messages
-        messages_start_time = time.time()
-        try:
-            from tg_cache import cached_get_chat_history
-            
-            messages = await cached_get_chat_history(post_parser.client, channel, limit=limit)
-        except Exception as e:
-            logger.error(f"Error during get_chat_history for channel '{channel}' (type: {type(channel)}) in HTML generation: {str(e)}", exc_info=True)
-            raise ValueError(f"Failed to get chat history for {channel} in HTML generation: {str(e)}") from e
-
-        messages_elapsed = time.time() - messages_start_time
-        logger.debug(f"html_messages_retrieval_timing: channel {channel}, {len(messages)} messages retrieved in {messages_elapsed:.3f} seconds")
-
-        # Enrich messages with reply to messages
-        enrichment_start_time = time.time()
-        messages = await _reply_enrichment(client, messages)
-        enrichment_elapsed = time.time() - enrichment_start_time
-        logger.debug(f"html_reply_enrichment_timing: channel {channel}, replies enriched in {enrichment_elapsed:.3f} seconds")
-
-        # Process messages into groups and render them in ONE worker thread
-        # (CPU-heavy, no await) to keep the event loop responsive.
-        processing_start_time = time.time()
-        try:
-            final_posts = await asyncio.to_thread(
-                _render_pipeline, messages, post_parser, limit,
-                exclude_flags, exclude_text, merge_seconds, Config['time_based_merge'],
-            )
-        finally:
-            # Persist media file-ids collected during rendering with a single bulk
-            # upsert — in a finally so a partial render still records what it collected.
-            await post_parser._flush_pending_media_ids()
-
-        processing_elapsed = time.time() - processing_start_time
-        logger.debug(f"html_messages_processing_timing: channel {channel}, {len(final_posts)} posts processed in {processing_elapsed:.3f} seconds")
-
-        # Generate HTML content
+        # Generate HTML content.
         html_gen_start_time = time.time()
-        # Concatenate HTML in thread to avoid blocking the loop on large payloads
-        html_posts = [post['html'] for post in final_posts]
-        def _concat_html(parts: list[str]) -> str:
-            return '\n<hr class="post-divider">\n'.join(parts)
-        html = await asyncio.to_thread(_concat_html, html_posts)
-        
-        # Optionally re-sanitize the final big HTML to ensure safety without blocking the loop
-        try:
-            def _sanitize_sync(html_raw: str) -> str:
-                css_sanitizer = CSSSanitizer(
-                    allowed_css_properties=["max-width", "max-height", "object-fit", "width", "height"]
-                )
-                return HTMLSanitizer(
-                    html_raw,
-                    tags=['p', 'a', 'b', 'i', 'strong', 'em', 'ul', 'ol', 'li', 'br', 'div', 'span', 'img', 'video', 'audio', 'source'],
-                    attributes={
-                        'a': ['href', 'title', 'target'],
-                        'img': ['src', 'alt', 'style'],
-                        'video': ['controls', 'src', 'style'],
-                        'audio': ['controls', 'style'],
-                        'source': ['src', 'type'],
-                        'div': ['class', 'style'],
-                        'span': ['class']
-                    },
-                    protocols=['http', 'https', 'tg'],
-                    css_sanitizer=css_sanitizer,
-                    strip=True,
-                )
-            html = await asyncio.to_thread(_sanitize_sync, html)
-        except Exception as e:
-            # FAIL CLOSED (see the RSS path): `html` is now the raw concatenated feed
-            # (4.4 removed the per-fragment passes). If bleach throws, html.escape the
-            # whole feed rather than returning raw channel HTML/JS to the client.
-            logger.error(f"html_final_sanitization_error: channel {channel}, error {str(e)}")
-            html = html_escape(html)
-        
+        # Each post is already sanitized per-post inside _render_pipeline. Join with the
+        # <hr> divider AFTER sanitize so the divider survives (registry §3.3), and each
+        # post's DOM was normalized within its own fragment (registry §3.4). The join is
+        # a trivial string op — no worker thread needed.
+        html = '\n<hr class="post-divider">\n'.join(post['html'] for post in final_posts)
+
         html_gen_elapsed = time.time() - html_gen_start_time
         logger.debug(f"html_generation_timing: channel {channel}, HTML generated in {html_gen_elapsed:.3f} seconds")
         
@@ -711,7 +635,9 @@ async def generate_channel_html(channel: str | int,
         logger.debug(f"html_total_generation_timing: channel {channel}, total time {total_elapsed:.3f} seconds")
         
         return html
-        
+
+    except ChannelNotFound as e:
+        return create_error_feed(str(e.channel_identifier), base_url)
     except Exception as e:
         logger.error(f"html_generation_error: channel {channel}, error {str(e)}")
         raise

sanitizer.py

@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# flake8: noqa
+# pylint: disable=broad-exception-caught, logging-fstring-interpolation, line-too-long
+
+"""The ONLY bleach configuration in the project.
+
+Before this module the sanitize config was copy-pasted three times (the single-post
+path in post_parser plus the RSS and HTML feed paths in rss_generator) and had
+drifted: `s`/`del` survived only in the single-post copy, and the three error-log
+names diverged. Here the config lives exactly once; every render path routes
+through sanitize_html().
+"""
+
+import logging
+import time as _time
+from html import escape as _html_escape
+
+# Imported under this name so tests can monkeypatch `sanitizer.HTMLSanitizer`
+# (API relocation from rss_generator — no behaviour change).
+from bleach import clean as HTMLSanitizer
+from bleach.css_sanitizer import CSSSanitizer
+
+logger = logging.getLogger(__name__)
+
+# Union of the three former copies. `s`/`del` (strikethrough) were previously
+# allowed only in the single-post path; registry §3.1 makes them survive in feeds too.
+ALLOWED_TAGS = ['p', 'a', 'b', 'i', 'strong', 'em', 's', 'del',
+                'ul', 'ol', 'li', 'br', 'div', 'span',
+                'img', 'video', 'audio', 'source']
+
+# Identical in all three former copies — moved here as-is.
+ALLOWED_ATTRIBUTES = {
+    'a': ['href', 'title', 'target'],
+    'img': ['src', 'alt', 'style'],
+    'video': ['controls', 'src', 'style'],
+    'audio': ['controls', 'style'],
+    'source': ['src', 'type'],
+    'div': ['class', 'style'],
+    'span': ['class'],
+}
+
+ALLOWED_CSS_PROPERTIES = ["max-width", "max-height", "object-fit", "width", "height"]
+
+# Non-default! bleach's default protocol list would strip tg:// links (channel
+# footers / service links use them). Load-bearing.
+ALLOWED_PROTOCOLS = ['http', 'https', 'tg']
+
+# Shared instance is safe: CSSSanitizer only holds the allowed-properties list
+# (stateless config); bleach builds a fresh Cleaner per clean() call anyway.
+_CSS_SANITIZER = CSSSanitizer(allowed_css_properties=ALLOWED_CSS_PROPERTIES)
+
+
+def sanitize_html(html_raw: str, log_context: str = "") -> str:
+    """Sanitize one HTML fragment.
+
+    FAIL-CLOSED: on any bleach error the fragment is html.escape()d, never returned
+    raw (stored-XSS guard — registry §3.2). log_context (e.g. "channel X, message_id
+    Y") is included in the error/slow logs to keep operational grep-ability across
+    call sites.
+    """
+    sanitize_start = _time.monotonic()
+    try:
+        # Both non-default params are load-bearing:
+        #   protocols=ALLOWED_PROTOCOLS keeps tg:// links alive;
+        #   strip=True REMOVES disallowed tags (bleach's default False would escape
+        #   them into visible text). strip_comments stays default (True), matching
+        #   all former call sites.
+        sanitized_html = HTMLSanitizer(
+            html_raw,
+            tags=ALLOWED_TAGS,
+            attributes=ALLOWED_ATTRIBUTES,
+            protocols=ALLOWED_PROTOCOLS,
+            css_sanitizer=_CSS_SANITIZER,
+            strip=True,
+        )
+    except Exception as e:
+        # Single error-log name across all call sites (registry §3.16); log_context
+        # distinguishes them. Fail-closed: escape rather than emit the raw payload.
+        _ctx = f"{log_context}, " if log_context else ""
+        logger.error(f"html_sanitization_error: {_ctx}error {str(e)}")
+        return _html_escape(html_raw)
+    elapsed = _time.monotonic() - sanitize_start
+    if elapsed > 0.05:
+        _ctx = f", {log_context}" if log_context else ""
+        logger.warning(f"diag_sanitize_slow: bleach.clean() took {elapsed:.3f}s, input_len={len(html_raw)}{_ctx}")
+    return sanitized_html

telegram_client.py

@@ -33,10 +33,19 @@ class TelegramClient:
             api_hash=settings["tg_api_hash"],
             workdir=settings["session_path"],
             proxy=settings["proxy"],  # MTProto proxy config, None if not set
+            max_concurrent_transmissions=settings["tg_max_concurrent_transmissions"],
         )
         self.max_disconnects = settings["tg_disconnect_flap_limit"]      # Max disconnects within the flap window before restart
         self._shutting_down = False  # Guard to prevent re-triggering restart during shutdown
         self._restarting = False            # Guard: an intentional in-process restart is in progress
+        # Consecutive media-download timeouts. A zombie media-DC connection makes every
+        # download time out while the main-DC watchdog stays green; when this streak
+        # reaches the threshold we reuse _restart_client() to rebuild the connection. Any
+        # successful download resets it (see note_download_ok / note_download_timeout).
+        self._download_timeout_streak = 0
+        self.media_timeout_restart_threshold = settings["media_timeout_restart_threshold"]
+        self._media_recovery_task = None   # strong ref to a scheduled recovery restart task
+        self._on_restart_verified = None   # optional callback fired after a VERIFIED restart (see set_restart_callback)
         self._disconnect_times = []         # Monotonic timestamps of recent disconnects (sliding window)
         self.disconnect_window = settings["tg_disconnect_flap_window"]   # Seconds; window for flap detection
         self._watchdog_task = None
@@ -180,6 +189,17 @@ class TelegramClient:
         except Exception as e:
             logger.critical(f"watchdog: loop crashed unexpectedly ({type(e).__name__}: {e}); liveness protection is now DISABLED until next start")
 
+    def set_restart_callback(self, callback) -> None:
+        """Register a callback invoked after a VERIFIED in-process restart (verify_get_me OK).
+
+        Used by api_server to clear its download negative cache: a completed restart rebuilds
+        the whole client (all DCs, including the media DC), so any per-file download backoff is
+        stale and a previously-failing file may now download. Kept as a plain hook so
+        telegram_client never has to import api_server (which would be a circular import).
+        The callback is synchronous, best-effort, and never fires on a failed/aborted restart.
+        """
+        self._on_restart_verified = callback
+
     async def _restart_client(self, reason: str = "unspecified"):
         """Recover the client without killing the process when possible.
 
@@ -206,10 +226,14 @@ class TelegramClient:
                 await asyncio.wait_for(self.client.start(), timeout=self.watchdog_restart_timeout)
             duration = time.monotonic() - restart_started
             self._wd_restart_count += 1
-            # Verification probe to prove the network layer is actually back (diagnostic only).
+            # Verification probe to prove the network layer is actually back. It also gates the
+            # verified-restart callback below: we only treat the restart as a real recovery
+            # (and clear the download negative cache) when this probe actually succeeds.
+            verify_ok = False
             try:
                 me = await asyncio.wait_for(self.client.get_me(), timeout=self.watchdog_timeout)
                 self._wd_last_ok_monotonic = time.monotonic()
+                verify_ok = True
                 verify = f", verify_get_me ok (me_id={getattr(me, 'id', None)})"
             except Exception as ve:
                 verify = f", verify_get_me FAILED ({type(ve).__name__}: {ve})"
@@ -218,6 +242,16 @@ class TelegramClient:
                 f"(is_connected={self.client.is_connected}{verify}, total in-process restarts={self._wd_restart_count})"
             )
             self._disconnect_times.clear()
+            # On a VERIFIED restart the whole client (all DCs, incl. the media DC) is
+            # re-established, so any per-file download backoff is stale — fire the registered
+            # hook (api_server clears its negative cache) so recovered media load immediately
+            # instead of fast-503'ing until the backoff expires. Only on verify success, never
+            # on a failed/aborted restart. Best-effort: a callback error must not abort recovery.
+            if verify_ok and self._on_restart_verified is not None:
+                try:
+                    self._on_restart_verified()
+                except Exception as cbe:
+                    logger.warning(f"recovery: restart-verified callback raised {type(cbe).__name__}: {cbe}")
             # Re-arm the watchdog in case it had previously crashed (self-healing).
             self._start_watchdog()
         except asyncio.CancelledError:
@@ -270,6 +304,39 @@ class TelegramClient:
             return None
         return time.monotonic() - self._wd_last_ok_monotonic
 
+    def note_download_ok(self) -> None:
+        """Reset the media-download timeout streak after any successful download."""
+        if self._download_timeout_streak:
+            logger.info(f"media_download: recovered, resetting timeout streak (was {self._download_timeout_streak})")
+        self._download_timeout_streak = 0
+
+    def note_download_timeout(self) -> None:
+        """Count a media-download timeout; force a connection-rebuilding restart on a streak.
+
+        A zombie media-DC connection makes EVERY download time out while the main-DC
+        watchdog probe (get_me) stays green, so this is the only signal that can trigger
+        recovery. The restart runs as a detached task so it never blocks the download path;
+        the streak is reset immediately so we schedule at most one restart per streak.
+        """
+        self._download_timeout_streak += 1
+        logger.warning(
+            f"media_download_timeout: streak {self._download_timeout_streak}/{self.media_timeout_restart_threshold}"
+        )
+        if self._download_timeout_streak < self.media_timeout_restart_threshold:
+            return
+        self._download_timeout_streak = 0
+        if self._restarting or self._shutting_down:
+            return
+        if self._media_recovery_task is not None and not self._media_recovery_task.done():
+            return  # a recovery restart is already scheduled/running
+        logger.critical(
+            f"media_download: {self.media_timeout_restart_threshold} consecutive download timeouts — "
+            f"scheduling media-connection restart (main-DC watchdog cannot see this)"
+        )
+        self._media_recovery_task = asyncio.create_task(
+            self._restart_client(reason="media download timeout streak")
+        )
+
     async def safe_get_messages(self, channel_id, post_id, max_retries=2):
         """Wrapper with retry logic for auth errors"""
         for attempt in range(max_retries):
@@ -289,14 +356,29 @@ class TelegramClient:
         """Wrapper with retry logic for download errors.
 
         `timeout` bounds each download attempt; for large videos the caller scales it
-        with file size (see api_server._media_download_timeout)."""
+        with file size (see api_server._media_download_timeout). A timeout cancels the
+        underlying download (freeing the Pyrogram transmission slot); a streak of timeouts
+        escalates to a connection-rebuilding restart via note_download_timeout()."""
         for attempt in range(max_retries):
             try:
-                return await asyncio.wait_for(
+                result = await asyncio.wait_for(
                     self.client.download_media(file_id, file_name=file_name),
                     timeout=timeout
                 )
+                self.note_download_ok()
+                return result
+            except asyncio.TimeoutError:
+                # Hung download: the wait_for above already cancelled it and released the
+                # get_file semaphore. Count it toward the media-connection restart streak.
+                self.note_download_timeout()
+                raise
             except Exception as e:
+                # INTENTIONAL: the restart streak counts CONSECUTIVE TIMEOUTS only. Non-timeout
+                # download errors (RPCError, FILE_REFERENCE_EXPIRED, etc.) reach here and call
+                # neither note_download_ok nor note_download_timeout, so they are streak-neutral:
+                # they neither advance nor reset it. This is deliberate — a zombie media-DC
+                # manifests as timeouts, not RPC errors, so only timeouts should escalate to a
+                # connection-rebuilding restart; a burst of unrelated RPC errors must not.
                 if isinstance(e, KeyError) and attempt < max_retries - 1:
                     logger.warning(f"Download auth error on attempt {attempt + 1}, retrying...")
                     await asyncio.sleep(5)

tests/conftest.py

@@ -0,0 +1,41 @@
+"""Shared test bootstrap (issue #17).
+
+Ensure the repo root is importable and install the config mock BEFORE any test
+module imports application code. Pytest imports conftest.py before collecting
+test modules, so doing this here (instead of a per-module preamble) makes the
+suite order-independent regardless of collection order or invocation directory.
+"""
+import os
+import sys
+import time
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+if ROOT not in sys.path:
+    sys.path.insert(0, ROOT)
+
+import tests.mock_config as _mock_config
+
+sys.modules['config'] = _mock_config
+
+# Pin the runner timezone to UTC (stage-0 golden determinism, issue #27).
+# Naive kurigram dates are interpreted by datetime.timestamp() in the LOCAL tz, so
+# without this pin RSS <pubDate> values drift between machines (this sandbox is MSK).
+os.environ['TZ'] = 'UTC'
+time.tzset()
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _reset_media_mime_cache():
+    """Clear the process-lifetime MIME dict before each test (issue #26).
+
+    ``api_server._mime_types`` persists for the whole process, so an entry populated by one
+    test would otherwise leak into another and mask a get/magic call the next test asserts on.
+    """
+    try:
+        import api_server
+        api_server._mime_types.clear()
+    except Exception:
+        pass
+    yield

tests/golden_replay.py

@@ -0,0 +1,181 @@
+# flake8: noqa
+# pylint: disable=import-outside-toplevel, missing-function-docstring, line-too-long
+"""Stage-0 golden-baseline replay helpers (render-pipeline refactor epic, issue #27/#34).
+
+Replays the frozen recorded corpus (tests/test_data/recorded/) through the REAL
+cache-hit path and captures the full generate_channel_rss / generate_channel_html
+output as an equivalence oracle for every later refactor stage. NO production render
+code is touched here — only a test loader + determinism pins.
+
+The recorded corpus was written by the prod bridge cache (tg_cache._save_*_to_cache):
+  {channel}.cache    -> {'timestamp', 'limit', 'messages': List[Message]}
+  {channel}.chatinfo -> {'timestamp', 'data': {'id', 'title', 'username'}}
+`timestamp` / `limit` are ignored (no freshness check) — this is literally the prod
+cache-hit payload the renderer sees in production.
+
+Run `python -m tests.golden_replay` from the repo root to (re)generate the goldens.
+"""
+import os
+import re
+import time
+import pickle
+from types import SimpleNamespace
+
+TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.dirname(TESTS_DIR)
+RECORDED_DIR = os.path.join(TESTS_DIR, "test_data", "recorded")
+GOLDEN_DIR = os.path.join(TESTS_DIR, "test_data", "golden")
+
+# Corpus channels frozen on `main` (spec "Этап 0"). exclude_flags / exclude_text
+# scenarios are intentionally NOT in golden: filters do not change the bytes of the
+# surviving posts; their membership/regex semantics are covered by dedicated unit tests.
+CORPUS_CHANNELS = ["bladerunnerblues", "embedoka", "meow_design", "theyforcedme"]
+
+# Recorded corpus is 100 messages/channel; the feed cap is 200. limit=100 exercises the
+# whole corpus (grouping only reduces the post count below the message count).
+GOLDEN_LIMIT = 100
+
+# Fixed media-URL signing key so a golden captured on any machine/checkout reproduces on
+# regeneration (a fresh checkout otherwise mints a new secrets.token_hex and every URL
+# digest changes). Used identically by the generator and the comparison test.
+GOLDEN_SIGNING_KEY = "stage0-golden-fixed-signing-key-0000000000000000"
+
+
+# --------------------------------------------------------------------------- #
+# Replay loader — the literal prod cache-hit path.
+# --------------------------------------------------------------------------- #
+def load_recorded(channel):
+    """Unpickle a recorded {channel}.cache / {channel}.chatinfo pair.
+
+    Returns (messages: List[Message], chatinfo_data: dict). timestamp/limit ignored."""
+    with open(os.path.join(RECORDED_DIR, f"{channel}.cache"), "rb") as f:
+        cache = pickle.load(f)
+    with open(os.path.join(RECORDED_DIR, f"{channel}.chatinfo"), "rb") as f:
+        chatinfo = pickle.load(f)
+    return cache["messages"], chatinfo["data"]
+
+
+def patch_tg_cache(monkeypatch, channel):
+    """Monkeypatch tg_cache.cached_get_chat_history / cached_get_chat to return the
+    recorded objects for `channel`. The feed functions lazy-import tg_cache, so patching
+    the module resolves late and works (mirrors test_stage4_eventloop.py)."""
+    messages, chatinfo = load_recorded(channel)
+
+    async def fake_get_chat_history(client, channel_id, limit=20):
+        return messages
+
+    async def fake_get_chat(client, channel_id):
+        # cached_get_chat returns SimpleNamespace(**data) with .id/.title/.username.
+        return SimpleNamespace(**chatinfo)
+
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_chat_history, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+
+
+def pin_environment(monkeypatch):
+    """Apply the spec's non-TZ determinism pins (TZ=UTC is pinned globally in conftest /
+    the __main__ bootstrap so both the test runner and the generator agree)."""
+    import post_parser
+    import rss_generator
+    from url_signer import KeyManager
+
+    # Pin the media-URL signing key (see GOLDEN_SIGNING_KEY).
+    monkeypatch.setattr(KeyManager, "signing_key", GOLDEN_SIGNING_KEY)
+
+    # time_based_merge=True so the meow_design time-cluster core is actually exercised.
+    # The flag is read from rss_generator.Config at call time; post_parser.Config is a
+    # sibling dict from the same get_settings() — pin both (cheap insurance vs. drift).
+    monkeypatch.setitem(rss_generator.Config, "time_based_merge", True)
+    monkeypatch.setitem(post_parser.Config, "time_based_merge", True)
+
+    # No media-id DB side effect outside tests/ (byte-neutral for the feed, but the write
+    # to ./data/media_file_ids.db is a forbidden side effect). upsert is imported INTO the
+    # post_parser namespace, so patch it there.
+    monkeypatch.setattr(post_parser, "upsert_media_file_ids_bulk_sync", lambda *a, **k: None)
+
+
+# --------------------------------------------------------------------------- #
+# Capture.
+# --------------------------------------------------------------------------- #
+def capture_rss(channel):
+    import asyncio
+    from rss_generator import generate_channel_rss
+    return asyncio.run(generate_channel_rss(channel, client=SimpleNamespace(), limit=GOLDEN_LIMIT))
+
+
+def capture_html(channel):
+    import asyncio
+    from rss_generator import generate_channel_html
+    return asyncio.run(generate_channel_html(channel, client=SimpleNamespace(), limit=GOLDEN_LIMIT))
+
+
+# --------------------------------------------------------------------------- #
+# Normalizations (applied SYMMETRICALLY to golden and actual before comparison).
+# Only the spec-sanctioned set — over-normalizing is exactly how a regression hides.
+# --------------------------------------------------------------------------- #
+# feedgen sets <lastBuildDate> to now() once in the FeedGenerator constructor: stable
+# within a process, but changes between capture runs — regex it out on both sides.
+_LASTBUILDDATE_RE = re.compile(r"<lastBuildDate>.*?</lastBuildDate>", re.DOTALL)
+# feedgen 1.0.0 emits no <generator>; normalized anyway as cheap insurance vs. a lib upgrade.
+_GENERATOR_RE = re.compile(r"<generator>.*?</generator>", re.DOTALL)
+# NOTE: the stage-0 flag-sort normalization (_FLAGS_DIV_RE / _sort_flags_div) was
+# removed in stage 2 (§3.8). Merged-post flags are now built in deterministic
+# first-seen order (rss_generator._render_messages_groups: dict.fromkeys(...)),
+# so the golden stores the real order and no normalization is needed — keeping it
+# would only mask a real flag-order regression.
+
+
+def normalize_rss(xml):
+    xml = _LASTBUILDDATE_RE.sub("<lastBuildDate/>", xml)
+    xml = _GENERATOR_RE.sub("<generator/>", xml)
+    return xml
+
+
+def normalize_html(html):
+    return html
+
+
+def golden_path(channel, kind):
+    ext = {"rss": "rss.xml", "html": "feed.html"}[kind]
+    return os.path.join(GOLDEN_DIR, f"{channel}.{ext}")
+
+
+# --------------------------------------------------------------------------- #
+# Generator entry point: `python -m tests.golden_replay`
+# --------------------------------------------------------------------------- #
+def _bootstrap_standalone():
+    """Reproduce the conftest bootstrap for the standalone generator: repo root on the
+    path, mocked config, UTC timezone."""
+    import sys
+    if ROOT not in sys.path:
+        sys.path.insert(0, ROOT)
+    import tests.mock_config as _mock_config
+    sys.modules["config"] = _mock_config
+    os.environ["TZ"] = "UTC"
+    time.tzset()
+
+
+def generate_all():
+    from _pytest.monkeypatch import MonkeyPatch
+
+    os.makedirs(GOLDEN_DIR, exist_ok=True)
+    for channel in CORPUS_CHANNELS:
+        mp = MonkeyPatch()
+        try:
+            pin_environment(mp)
+            patch_tg_cache(mp, channel)
+            rss = capture_rss(channel)
+            html = capture_html(channel)
+        finally:
+            mp.undo()
+        with open(golden_path(channel, "rss"), "w", encoding="utf-8") as f:
+            f.write(rss)
+        with open(golden_path(channel, "html"), "w", encoding="utf-8") as f:
+            f.write(html)
+        print(f"{channel}: rss={len(rss)}B items={rss.count('<item>')} "
+              f"html={len(html)}B posts={html.count('message-body') if html else 0}")
+
+
+if __name__ == "__main__":
+    _bootstrap_standalone()
+    generate_all()

tests/media_fragment_replay.py

@@ -0,0 +1,211 @@
+# flake8: noqa
+# pylint: disable=import-outside-toplevel, missing-function-docstring, line-too-long
+"""Stage-5 fragment-level snapshot helpers (render-pipeline refactor epic, issue #32/#34).
+
+Captures the raw `_generate_html_media` output (BEFORE sanitize) for every media
+render kind and edge branch, as a SEPARATE oracle layer from the stage-0 feed goldens
+(spec §4: "два слоя эталонов, не смешивать").
+
+`media_fragments.json` is the PRE-REFACTOR BASE reference: it was captured by running
+THIS harness against the BASE `post_parser.py` checkout (the pre-refactor code), NOT
+against stage-5 code — so it is a genuine base-anchored golden layer, not a circular
+self-snapshot of the refactor. The 5a refactor (MEDIA_SOURCES table + renderers) must
+reproduce these base fragments byte-for-byte. The ONLY 5b-registered changes vs the base
+are the two entries in REGISTERED_DELTAS (§3.14 unclosed-div close); the §3.13 large-file
+guard is collection-only and changes no fragment bytes.
+
+Cases whose type exists in the recorded corpus could be pulled from it, but the media
+FRAGMENT is a pure function of a single Message, so deterministic hand-built mocks give
+the same bytes with far less machinery and also reach the mock-only exotic types
+(PAID_MEDIA, LIVE_PHOTO, STORY) — exactly the set the spec allows mocks for.
+
+Run `python -m tests.media_fragment_replay` from the repo root to (re)generate the snapshot.
+"""
+import os
+import json
+from types import SimpleNamespace
+
+TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT = os.path.dirname(TESTS_DIR)
+SNAPSHOT_PATH = os.path.join(TESTS_DIR, "test_data", "media_fragments.json")
+
+# Fixed signing key so digests in the snapshot reproduce on any checkout (mirrors the
+# stage-0 golden pin). Applied by the generator and by the comparison test.
+FRAGMENT_SIGNING_KEY = "stage5-fragment-fixed-signing-key-000000000000"
+
+
+class _Str(str):
+    """Minimal stand-in for Pyrogram's Str: .html returns the raw string unchanged."""
+    @property
+    def html(self):
+        return str(self)
+
+
+def _msg(mid, media, *, text=None, username="testchan", chat_id=-1001234567890, **extra):
+    """Build a Message-like mock with the attributes _generate_html_media touches.
+
+    Top-level media attributes default to None (truthy checks in _save_media_file_ids);
+    new Kurigram 2.2.23 attributes (live_photo/story/giveaway/...) are intentionally
+    absent unless passed, so getattr-only production code is exercised as on real objects.
+    """
+    m = SimpleNamespace()
+    m.id = mid
+    m.media = media
+    m.text = _Str(text) if text is not None else None
+    m.caption = None
+    m.web_page = None
+    m.poll = None
+    m.paid_media = None
+    m.forward_origin = None
+    m.show_caption_above_media = False
+    if chat_id is None and username is None:
+        m.chat = None
+    else:
+        m.chat = SimpleNamespace(id=chat_id, username=username)
+    for attr in ("photo", "video", "document", "audio", "voice",
+                 "video_note", "animation", "sticker"):
+        setattr(m, attr, None)
+    for key, value in extra.items():
+        setattr(m, key, value)
+    return m
+
+
+def _obj(**kw):
+    return SimpleNamespace(**kw)
+
+
+# --------------------------------------------------------------------------- #
+# Fragment cases. Each entry: name -> Message factory.
+# Covers the full spec §5a "Шаг 0" list + edge branches.
+# --------------------------------------------------------------------------- #
+def _webpage(photo=None, url="https://example.com", title="Example",
+             description=None, site_name=None, wp_type=""):
+    return _obj(photo=photo, url=url, title=title, description=description,
+                site_name=site_name, type=wp_type, display_url=None)
+
+
+def build_cases():
+    from pyrogram.enums import MessageMediaType as T
+    cases = {}
+    cases["photo"] = lambda: _msg(1, T.PHOTO, photo=_obj(file_unique_id="ph_uid", file_id="ph_fid"))
+    cases["video"] = lambda: _msg(2, T.VIDEO, video=_obj(file_unique_id="vid_uid", file_id="vid_fid", file_size=1024))
+    cases["animation"] = lambda: _msg(3, T.ANIMATION, animation=_obj(file_unique_id="ani_uid", file_id="ani_fid"))
+    cases["video_note"] = lambda: _msg(4, T.VIDEO_NOTE, video_note=_obj(file_unique_id="vn_uid", file_id="vn_fid"))
+    cases["audio_default_mime"] = lambda: _msg(5, T.AUDIO, audio=_obj(file_unique_id="au_uid", file_id="au_fid"))
+    cases["audio_explicit_mime"] = lambda: _msg(6, T.AUDIO, audio=_obj(file_unique_id="au2_uid", file_id="au2_fid", mime_type="audio/flac"))
+    cases["voice_default_mime"] = lambda: _msg(7, T.VOICE, voice=_obj(file_unique_id="vo_uid", file_id="vo_fid"))
+    cases["voice_explicit_mime"] = lambda: _msg(8, T.VOICE, voice=_obj(file_unique_id="vo2_uid", file_id="vo2_fid", mime_type="audio/wav"))
+    cases["sticker_img"] = lambda: _msg(9, T.STICKER, sticker=_obj(file_unique_id="st_uid", file_id="st_fid", emoji="😀", is_video=False))
+    cases["sticker_video"] = lambda: _msg(10, T.STICKER, sticker=_obj(file_unique_id="stv_uid", file_id="stv_fid", emoji="🎬", is_video=True))
+    cases["document_pdf_public"] = lambda: _msg(11, T.DOCUMENT, username="pubchan", chat_id=None,
+                                                document=_obj(file_unique_id="pdf_uid", file_id="pdf_fid", mime_type="application/pdf"))
+    cases["document_pdf_private"] = lambda: _msg(12, T.DOCUMENT, username=None, chat_id=-1009876543210,
+                                                 document=_obj(file_unique_id="pdf2_uid", file_id="pdf2_fid", mime_type="application/pdf"))
+    cases["document_normal"] = lambda: _msg(13, T.DOCUMENT, document=_obj(file_unique_id="doc_uid", file_id="doc_fid", mime_type="image/png"))
+    cases["live_photo"] = lambda: _msg(14, T.LIVE_PHOTO, live_photo=_obj(file_unique_id="lp_uid", file_id="lp_fid"))
+    cases["story_video"] = lambda: _msg(15, T.STORY, story=_obj(video=_obj(file_unique_id="sv_uid", file_id="sv_fid"), photo=None))
+    cases["story_photo"] = lambda: _msg(16, T.STORY, story=_obj(video=None, photo=_obj(file_unique_id="sp_uid", file_id="sp_fid")))
+    cases["poll_media_img"] = lambda: _msg(17, T.POLL, poll=_obj(question=_Str("Q?"), options=[],
+                                           description_media=_obj(photo=_obj(file_unique_id="pl_uid", file_id="pl_fid"))))
+    cases["poll_media_video"] = lambda: _msg(18, T.POLL, poll=_obj(question=_Str("Q?"), options=[],
+                                             description_media=_obj(video=_obj(file_unique_id="plv_uid", file_id="plv_fid"))))
+    cases["paid_media"] = lambda: _msg(19, T.PAID_MEDIA,
+                                       paid_media=_obj(stars_amount=50, media=[_obj(), _obj()]))
+    # WEB_PAGE with photo: opens an EMPTY message-media div (no elif matches WEB_PAGE),
+    # plus the webpage-preview block (short text gate <=10).
+    cases["webpage_with_photo"] = lambda: _msg(20, T.WEB_PAGE, text="hi",
+                                               web_page=_webpage(photo=_obj(file_unique_id="wp_uid", file_id="wp_fid")))
+    # WEB_PAGE without photo: file_unique_id is None -> message-media div opened and
+    # left UNCLOSED (§3.14 target). Short text so the preview block renders.
+    cases["webpage_without_photo"] = lambda: _msg(21, T.WEB_PAGE, text="hi",
+                                                  web_page=_webpage(photo=None))
+    # WEB_PAGE with photo but long text (>10): preview gate closed, only the empty media div.
+    cases["webpage_photo_long_text"] = lambda: _msg(22, T.WEB_PAGE,
+                                                    text="this text is definitely longer than ten characters",
+                                                    web_page=_webpage(photo=_obj(file_unique_id="wpl_uid", file_id="wpl_fid")))
+    # file_unique_id is None on a normal media type -> unclosed div (§3.14 target).
+    cases["file_unique_id_none"] = lambda: _msg(23, T.PHOTO, photo=_obj(file_unique_id=None, file_id="x_fid"))
+    # channel_username is None but uid present -> the div IS closed on the guard branch.
+    cases["channel_username_none"] = lambda: _msg(24, T.PHOTO, username=None, chat_id=555,
+                                                  photo=_obj(file_unique_id="cu_uid", file_id="cu_fid"))
+    return cases
+
+
+# --------------------------------------------------------------------------- #
+# Registered 5b deltas vs the pre-refactor base snapshot.
+#
+# Spec §3.14: the empty <div class="message-media"> container is now CLOSED in every
+# render branch. The base (pre-refactor) code left it OPEN whenever the selected media
+# object had no usable file_unique_id, so the base snapshot captured an unbalanced div
+# for exactly these two cases. `collected` is byte-identical to the base; only `html`
+# differs by the added `</div>`. These are the ONLY fragments the 5b registered fixes
+# change relative to the pre-refactor base — every other case reproduces base bytes.
+REGISTERED_DELTAS = {
+    "file_unique_id_none": {
+        "collected": [],
+        "html": "<div class=\"message-media\">\n</div>",
+    },
+    "webpage_without_photo": {
+        "collected": [],
+        "html": "<div class=\"message-media\">\n</div>\n<div class=\"webpage-preview\">\n<div class=\"webpage-preview\" style=\"border-left: 3px solid #ccc; padding-left: 10px; margin: 10px 0;\">\n<div class=\"webpage-title\" style=\"font-weight:bold; margin:5px 0;\">\n<a href=\"https://example.com\" target=\"_blank\">Example</a></div>\n<div class=\"webpage-url\" style=\"color:#666; font-size:0.9em;margin-bottom:5px;\">https://example.com</div>\n</div>\n</div>",
+    },
+}
+
+
+# --------------------------------------------------------------------------- #
+# Capture / compare.
+# --------------------------------------------------------------------------- #
+def capture_fragments():
+    """Return {case_name: {"html": fragment, "collected": [[chan, id, fuid], ...]}}.
+
+    Covers all three ladders in one shot: _generate_html_media renders the fragment
+    (render ladder + _get_file_unique_id ladder) and calls _save_media_file_ids
+    (collection ladder), whose result is read off _pending_media_ids."""
+    from post_parser import PostParser
+    parser = PostParser(SimpleNamespace())
+    out = {}
+    for name, factory in build_cases().items():
+        parser._pending_media_ids = []
+        html = parser._generate_html_media(factory())
+        collected = [[c, p, f] for c, p, f, _ in parser._pending_media_ids]
+        out[name] = {"html": html, "collected": collected}
+    return out
+
+
+def pin_signing_key(monkeypatch):
+    from url_signer import KeyManager
+    monkeypatch.setattr(KeyManager, "signing_key", FRAGMENT_SIGNING_KEY)
+
+
+def load_snapshot():
+    with open(SNAPSHOT_PATH, encoding="utf-8") as f:
+        return json.load(f)
+
+
+def _bootstrap_standalone():
+    import sys
+    import time
+    if ROOT not in sys.path:
+        sys.path.insert(0, ROOT)
+    import tests.mock_config as _mock_config
+    sys.modules["config"] = _mock_config
+    os.environ["TZ"] = "UTC"
+    time.tzset()
+
+
+def generate_snapshot():
+    # WARNING: this MUST be run against the BASE (pre-refactor) `post_parser.py`, never
+    # against stage-5 code. Regenerating from stage-5 output would make the oracle a
+    # circular self-snapshot instead of a base-anchored golden layer (spec §4).
+    from url_signer import KeyManager
+    KeyManager.signing_key = FRAGMENT_SIGNING_KEY
+    data = capture_fragments()
+    os.makedirs(os.path.dirname(SNAPSHOT_PATH), exist_ok=True)
+    with open(SNAPSHOT_PATH, "w", encoding="utf-8") as f:
+        json.dump(data, f, ensure_ascii=False, indent=2, sort_keys=True)
+    print(f"wrote {len(data)} fragment snapshots to {SNAPSHOT_PATH}")
+
+
+if __name__ == "__main__":
+    _bootstrap_standalone()
+    generate_snapshot()

tests/mock_config.py

@@ -38,4 +38,7 @@ def get_settings():
         "media_download_timeout_max": 1800,
         "media_download_min_speed": 256 * 1024,
         "io_thread_pool_size": 32,
+        "tg_max_concurrent_transmissions": 3,
+        "media_timeout_restart_threshold": 5,
+        "cache_sweep_interval": 900,
     }

tests/test_channel_key.py

@@ -0,0 +1,350 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""Tests for the canonical channel key: helper, the three wiring layers, and the
+one-shot migration (issue #24, tasks 8-11)."""
+
+import logging
+import os
+import shutil
+import sqlite3
+from types import SimpleNamespace
+
+import pytest
+
+import tg_cache
+from channel_key import canonical_channel_key
+from migrate_channel_keys import migrate_channel_keys_sync
+from post_parser import PostParser
+
+
+# --------------------------------------------------------------------------- #
+# Task 11 — unit: canonical_channel_key.
+# --------------------------------------------------------------------------- #
+def test_canonical_lowercases_username():
+    assert canonical_channel_key('Durov') == 'durov'
+
+
+def test_canonical_strips_at_prefix():
+    assert canonical_channel_key('@durov') == 'durov'
+    assert canonical_channel_key('@Durov') == 'durov'
+
+
+def test_canonical_numeric_id_str_preserved():
+    assert canonical_channel_key('-1001234567890') == '-1001234567890'
+
+
+def test_canonical_numeric_id_int_preserved():
+    assert canonical_channel_key(-1001234567890) == '-1001234567890'
+
+
+def test_canonical_already_lowercase_idempotent():
+    assert canonical_channel_key('durov') == 'durov'
+    assert canonical_channel_key(canonical_channel_key('Durov')) == 'durov'
+
+
+# --------------------------------------------------------------------------- #
+# Task 11 — tg_cache path is identical for the two casings.
+# --------------------------------------------------------------------------- #
+def test_tgcache_path_identical_for_casings(tmp_path, monkeypatch):
+    monkeypatch.setattr(tg_cache, "CACHE_DIR", str(tmp_path))
+    p_upper = tg_cache._cache_file_path('Durov', 'history.json')
+    p_lower = tg_cache._cache_file_path('durov', 'history.json')
+    p_at = tg_cache._cache_file_path('@DUROV', 'history.json')
+    assert p_upper == p_lower == p_at
+    assert os.path.basename(p_upper) == 'durov.history.json'
+
+
+def test_tgcache_path_numeric_preserved(tmp_path, monkeypatch):
+    monkeypatch.setattr(tg_cache, "CACHE_DIR", str(tmp_path))
+    p = tg_cache._cache_file_path('-1001234567890', 'chatinfo.json')
+    assert os.path.basename(p) == '-1001234567890.chatinfo.json'
+
+
+# --------------------------------------------------------------------------- #
+# Task 11 — get_channel_username lowercases both branches.
+# --------------------------------------------------------------------------- #
+def _parser():
+    return PostParser(SimpleNamespace())
+
+
+def test_get_channel_username_single_lowercased():
+    parser = _parser()
+    msg = SimpleNamespace(chat=SimpleNamespace(username='MixedCase', id=1))
+    assert parser.get_channel_username(msg) == 'mixedcase'
+
+
+def test_get_channel_username_usernames_list_lowercased():
+    parser = _parser()
+    chat = SimpleNamespace(
+        usernames=[SimpleNamespace(username='MixedCase', active=True)],
+        username=None,
+        id=1,
+    )
+    assert parser.get_channel_username(SimpleNamespace(chat=chat)) == 'mixedcase'
+
+
+def test_get_channel_username_numeric_id_unchanged():
+    parser = _parser()
+    chat = SimpleNamespace(usernames=None, username=None, id=-1001234567890)
+    assert parser.get_channel_username(SimpleNamespace(chat=chat)) == '-1001234567890'
+
+
+# --------------------------------------------------------------------------- #
+# Migration helpers.
+# --------------------------------------------------------------------------- #
+def _make_db(path):
+    conn = sqlite3.connect(path)
+    conn.execute(
+        """CREATE TABLE media_file_ids (
+               channel        TEXT    NOT NULL,
+               post_id        INTEGER NOT NULL,
+               file_unique_id TEXT    NOT NULL,
+               added          REAL    NOT NULL,
+               mime_type      TEXT,
+               PRIMARY KEY (channel, post_id, file_unique_id)
+           )"""
+    )
+    conn.commit()
+    conn.close()
+
+
+def _rows(path):
+    conn = sqlite3.connect(path)
+    conn.row_factory = sqlite3.Row
+    rows = [dict(r) for r in conn.execute(
+        "SELECT channel, post_id, file_unique_id, added, mime_type FROM media_file_ids "
+        "ORDER BY channel, post_id, file_unique_id")]
+    conn.close()
+    return rows
+
+
+# --------------------------------------------------------------------------- #
+# Task 11 (a) — SQL merge of both forms → one row, max(added), non-NULL mime_type.
+# --------------------------------------------------------------------------- #
+def test_migration_sql_merge(tmp_path):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+    conn = sqlite3.connect(db)
+    # Old-cased row (higher added, NULL mime) and lowercase twin (lower added, has mime).
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('Durov', 5, 'fid', 200.0, None))
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('durov', 5, 'fid', 100.0, 'image/jpeg'))
+    conn.commit()
+    conn.close()
+
+    migrate_channel_keys_sync(db, str(cache))
+
+    rows = _rows(db)
+    assert len(rows) == 1
+    r = rows[0]
+    assert r['channel'] == 'durov'
+    assert r['added'] == 200.0            # max(added)
+    assert r['mime_type'] == 'image/jpeg'  # non-NULL preferred
+
+
+def test_migration_sql_rekey_no_twin(tmp_path):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+    conn = sqlite3.connect(db)
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('Durov', 7, 'fid', 50.0, 'video/mp4'))
+    conn.commit()
+    conn.close()
+
+    migrate_channel_keys_sync(db, str(cache))
+
+    rows = _rows(db)
+    assert len(rows) == 1
+    assert rows[0]['channel'] == 'durov'
+    assert rows[0]['mime_type'] == 'video/mp4'
+
+
+# --------------------------------------------------------------------------- #
+# Task 11 (b) — samefile guard → no-op, data intact (case-insensitive FS simulated).
+# --------------------------------------------------------------------------- #
+def test_migration_samefile_guard_no_data_loss(tmp_path, monkeypatch):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+    # Single dir 'Durov' with a file; on a case-insensitive FS 'durov' would be the same dir.
+    src = cache / "Durov" / "5"
+    src.mkdir(parents=True)
+    (src / "fid").write_bytes(b"payload")
+
+    # Simulate a case-insensitive FS: any existence check on the lowercase twin resolves,
+    # and samefile reports src == dst so the FS step must be a pure no-op.
+    real_exists = os.path.exists
+
+    def fake_exists(p):
+        if os.path.basename(p) == 'durov':
+            return True
+        return real_exists(p)
+
+    monkeypatch.setattr(os.path, "exists", fake_exists)
+    monkeypatch.setattr(os.path, "samefile", lambda a, b: True)
+
+    migrate_channel_keys_sync(db, str(cache))
+
+    # Data intact: the original file must still be present and unchanged.
+    assert (cache / "Durov" / "5" / "fid").read_bytes() == b"payload"
+
+
+# --------------------------------------------------------------------------- #
+# Task 11 (c) — merge of two genuinely different dirs → combined, target wins.
+# --------------------------------------------------------------------------- #
+def test_migration_fs_merge_different_dirs(tmp_path):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+
+    # Old-cased tree.
+    (cache / "Durov" / "5").mkdir(parents=True)
+    (cache / "Durov" / "5" / "shared").write_bytes(b"OLD")   # collides -> dst wins
+    (cache / "Durov" / "5" / "only_old").write_bytes(b"OLD_ONLY")
+    # Canonical tree.
+    (cache / "durov" / "5").mkdir(parents=True)
+    (cache / "durov" / "5" / "shared").write_bytes(b"NEW")   # existing dst wins
+    (cache / "durov" / "6").mkdir(parents=True)
+    (cache / "durov" / "6" / "only_new").write_bytes(b"NEW_ONLY")
+
+    migrate_channel_keys_sync(db, str(cache))
+
+    # Old dir gone, everything merged under 'durov'.
+    assert not (cache / "Durov").exists()
+    assert (cache / "durov" / "5" / "shared").read_bytes() == b"NEW"        # target wins
+    assert (cache / "durov" / "5" / "only_old").read_bytes() == b"OLD_ONLY"  # brought over
+    assert (cache / "durov" / "6" / "only_new").read_bytes() == b"NEW_ONLY"  # untouched
+
+
+def test_migration_fs_rename_when_dst_missing(tmp_path):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+    (cache / "Durov" / "5").mkdir(parents=True)
+    (cache / "Durov" / "5" / "fid").write_bytes(b"data")
+
+    migrate_channel_keys_sync(db, str(cache))
+
+    assert not (cache / "Durov").exists()
+    assert (cache / "durov" / "5" / "fid").read_bytes() == b"data"
+
+
+# --------------------------------------------------------------------------- #
+# Task 11 (d) — re-run is a no-op.
+# --------------------------------------------------------------------------- #
+def test_migration_rerun_noop(tmp_path):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+    conn = sqlite3.connect(db)
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('Durov', 5, 'fid', 200.0, None))
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('durov', 5, 'fid', 100.0, 'image/jpeg'))
+    conn.commit()
+    conn.close()
+    (cache / "Durov" / "5").mkdir(parents=True)
+    (cache / "Durov" / "5" / "fid").write_bytes(b"data")
+
+    migrate_channel_keys_sync(db, str(cache))
+    first = _rows(db)
+    # Second run: nothing left to migrate.
+    migrate_channel_keys_sync(db, str(cache))
+    second = _rows(db)
+
+    assert first == second
+    assert all(r['channel'] == r['channel'].lower() for r in second)
+    assert (cache / "durov" / "5" / "fid").read_bytes() == b"data"
+
+
+# --------------------------------------------------------------------------- #
+# Data-loss guard — an FS-step failure must SKIP the SQL step (rows stay old-cased).
+# Has teeth: fails if the `continue` after the FS OSError is removed.
+# --------------------------------------------------------------------------- #
+def test_migration_fs_failure_skips_sql(tmp_path, monkeypatch, caplog):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+    conn = sqlite3.connect(db)
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('Durov', 5, 'fid', 100.0, 'image/jpeg'))
+    conn.commit()
+    conn.close()
+    # Old-cased dir with no lowercase twin → the migration takes the os.rename path.
+    (cache / "Durov" / "5").mkdir(parents=True)
+    (cache / "Durov" / "5" / "fid").write_bytes(b"data")
+
+    def boom(*_a, **_k):
+        raise OSError("disk on fire")
+
+    monkeypatch.setattr(os, "rename", boom)
+
+    with caplog.at_level(logging.INFO):
+        migrate_channel_keys_sync(db, str(cache))  # (a) must NOT crash startup
+
+    # (b) SQL step skipped: the row is still OLD-cased.
+    rows = _rows(db)
+    assert len(rows) == 1
+    assert rows[0]['channel'] == 'Durov'
+    # (c) failures counter increased (surfaced in the summary log).
+    assert "failures 1" in caplog.text
+
+
+# --------------------------------------------------------------------------- #
+# Reverse-direction max(added): lowercase twin has the LARGER `added` → it wins.
+# Proves max() is not one-directional ("old always wins").
+# --------------------------------------------------------------------------- #
+def test_migration_sql_merge_reverse_max(tmp_path):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+    conn = sqlite3.connect(db)
+    # OLD-cased added=100 (SMALLER), lowercase twin added=300 (LARGER).
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('Durov', 5, 'fid', 100.0, 'image/jpeg'))
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('durov', 5, 'fid', 300.0, None))
+    conn.commit()
+    conn.close()
+
+    migrate_channel_keys_sync(db, str(cache))
+
+    rows = _rows(db)
+    assert len(rows) == 1
+    assert rows[0]['channel'] == 'durov'
+    assert rows[0]['added'] == 300.0          # max(added): the twin's LARGER value survives
+    assert rows[0]['mime_type'] == 'image/jpeg'  # non-NULL preferred (from the old row)
+
+
+# --------------------------------------------------------------------------- #
+# Traversal guard — a DB channel name with '/' or '..' is SKIPPED before any FS op.
+# Has teeth: fails if the _is_safe_channel_segment guard is removed.
+# --------------------------------------------------------------------------- #
+def test_migration_traversal_guard_skips_dirty_channel(tmp_path, monkeypatch):
+    db = str(tmp_path / "m.db")
+    cache = tmp_path / "cache"
+    cache.mkdir()
+    _make_db(db)
+    conn = sqlite3.connect(db)
+    # Dirty mixed-case channel names → become migration candidates via the DB query.
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('../Evil', 5, 'fid', 100.0, None))
+    conn.execute("INSERT INTO media_file_ids VALUES (?,?,?,?,?)", ('a/B', 6, 'fid', 100.0, None))
+    conn.commit()
+    conn.close()
+
+    # Spy: no destructive FS op may run (all dirty candidates rejected before the FS step).
+    called = []
+    monkeypatch.setattr(os, "rename", lambda *a, **k: called.append(('rename', a)))
+    monkeypatch.setattr(shutil, "rmtree", lambda *a, **k: called.append(('rmtree', a)))
+
+    migrate_channel_keys_sync(db, str(cache))  # must NOT crash
+
+    assert called == []  # nothing renamed/removed → no op escaped cache_dir
+    # Rejected channels are left un-migrated (rows keep their original dirty names).
+    channels = {r['channel'] for r in _rows(db)}
+    assert channels == {'../Evil', 'a/B'}

tests/test_data/media_fragments.json

@@ -0,0 +1,218 @@
+{
+  "animation": {
+    "collected": [
+      [
+        "testchan",
+        3,
+        "ani_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<video controls src=\"http://test.example.com/media/testchan/3/ani_uid/ef4319c9\" style=\"max-width:100%; width:auto;height:auto; max-height:400px;\"></video>\n</div>"
+  },
+  "audio_default_mime": {
+    "collected": [
+      [
+        "testchan",
+        5,
+        "au_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<audio controls style=\"width:100%; max-width:400px;\"><source src=\"http://test.example.com/media/testchan/5/au_uid/be14c993\" type=\"audio/mpeg\"></audio>\n<br>\n</div>"
+  },
+  "audio_explicit_mime": {
+    "collected": [
+      [
+        "testchan",
+        6,
+        "au2_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<audio controls style=\"width:100%; max-width:400px;\"><source src=\"http://test.example.com/media/testchan/6/au2_uid/015e2737\" type=\"audio/flac\"></audio>\n<br>\n</div>"
+  },
+  "channel_username_none": {
+    "collected": [],
+    "html": "<div class=\"message-media\">\n</div>"
+  },
+  "document_normal": {
+    "collected": [
+      [
+        "testchan",
+        13,
+        "doc_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<img src=\"http://test.example.com/media/testchan/13/doc_uid/4c9075c7\" style=\"max-width:100%; width:auto; height:auto;max-height:400px; object-fit:contain;\">\n</div>"
+  },
+  "document_pdf_private": {
+    "collected": [
+      [
+        "-1009876543210",
+        12,
+        "pdf2_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<div class=\"document-pdf\" style=\"padding: 10px;\">\n<a href=\"https://t.me/c/9876543210/12\" target=\"_blank\">[PDF-файл]</a></div>\n</div>"
+  },
+  "document_pdf_public": {
+    "collected": [
+      [
+        "pubchan",
+        11,
+        "pdf_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<div class=\"document-pdf\" style=\"padding: 10px;\">\n<a href=\"https://t.me/pubchan/11\" target=\"_blank\">[PDF-файл]</a></div>\n</div>"
+  },
+  "file_unique_id_none": {
+    "collected": [],
+    "html": "<div class=\"message-media\">"
+  },
+  "live_photo": {
+    "collected": [
+      [
+        "testchan",
+        14,
+        "lp_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<video controls autoplay loop muted src=\"http://test.example.com/media/testchan/14/lp_uid/7cbfcf53\"style=\"max-width:100%; width:auto; height:auto; max-height:400px;object-fit:contain;\"></video>\n</div>"
+  },
+  "paid_media": {
+    "collected": [],
+    "html": "<div class=\"message-media\">\n<div class=\"paid-media\">⭐ Paid media (50 stars, 2 item(s)) — available in Telegram</div>\n</div>"
+  },
+  "photo": {
+    "collected": [
+      [
+        "testchan",
+        1,
+        "ph_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<img src=\"http://test.example.com/media/testchan/1/ph_uid/08e6b2ef\" style=\"max-width:100%; width:auto; height:auto;max-height:400px; object-fit:contain;\">\n</div>"
+  },
+  "poll_media_img": {
+    "collected": [
+      [
+        "testchan",
+        17,
+        "pl_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<img src=\"http://test.example.com/media/testchan/17/pl_uid/492d238c\" style=\"max-width:100%; width:auto; height:auto;max-height:400px; object-fit:contain;\">\n</div>"
+  },
+  "poll_media_video": {
+    "collected": [
+      [
+        "testchan",
+        18,
+        "plv_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<video controls src=\"http://test.example.com/media/testchan/18/plv_uid/abb609a3\" style=\"max-width:100%; width:auto;height:auto; max-height:400px;\"></video>\n</div>"
+  },
+  "sticker_img": {
+    "collected": [
+      [
+        "testchan",
+        9,
+        "st_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<img src=\"http://test.example.com/media/testchan/9/st_uid/03dc35d2\" alt=\"Sticker 😀\" style=\"max-width:100%;width:auto; height:auto; max-height:200px; object-fit:contain;\">\n</div>"
+  },
+  "sticker_video": {
+    "collected": [
+      [
+        "testchan",
+        10,
+        "stv_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<video controls autoplay loop muted src=\"http://test.example.com/media/testchan/10/stv_uid/cedd7494\"style=\"max-width:100%; width:auto; height:auto; max-height:200px;object-fit:contain;\"></video>\n</div>"
+  },
+  "story_photo": {
+    "collected": [
+      [
+        "testchan",
+        16,
+        "sp_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<img src=\"http://test.example.com/media/testchan/16/sp_uid/d8a5d9c9\" style=\"max-width:100%; width:auto; height:auto;max-height:400px; object-fit:contain;\">\n</div>"
+  },
+  "story_video": {
+    "collected": [
+      [
+        "testchan",
+        15,
+        "sv_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<video controls src=\"http://test.example.com/media/testchan/15/sv_uid/1aba6362\" style=\"max-width:100%; width:auto;height:auto; max-height:400px;\"></video>\n</div>"
+  },
+  "video": {
+    "collected": [
+      [
+        "testchan",
+        2,
+        "vid_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<video controls src=\"http://test.example.com/media/testchan/2/vid_uid/e0c6ba2e\" style=\"max-width:100%; width:auto;height:auto; max-height:400px;\"></video>\n</div>"
+  },
+  "video_note": {
+    "collected": [
+      [
+        "testchan",
+        4,
+        "vn_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<video controls src=\"http://test.example.com/media/testchan/4/vn_uid/a5ee9944\" style=\"max-width:100%; width:auto;height:auto; max-height:400px;\"></video>\n</div>"
+  },
+  "voice_default_mime": {
+    "collected": [
+      [
+        "testchan",
+        7,
+        "vo_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<audio controls style=\"width:100%; max-width:400px;\"><source src=\"http://test.example.com/media/testchan/7/vo_uid/f16627e0\" type=\"audio/ogg\"></audio>\n<br>\n</div>"
+  },
+  "voice_explicit_mime": {
+    "collected": [
+      [
+        "testchan",
+        8,
+        "vo2_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n<audio controls style=\"width:100%; max-width:400px;\"><source src=\"http://test.example.com/media/testchan/8/vo2_uid/30fed86a\" type=\"audio/wav\"></audio>\n<br>\n</div>"
+  },
+  "webpage_photo_long_text": {
+    "collected": [
+      [
+        "testchan",
+        22,
+        "wpl_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n</div>"
+  },
+  "webpage_with_photo": {
+    "collected": [
+      [
+        "testchan",
+        20,
+        "wp_uid"
+      ]
+    ],
+    "html": "<div class=\"message-media\">\n</div>\n<div class=\"webpage-preview\">\n<div class=\"webpage-preview\" style=\"border-left: 3px solid #ccc; padding-left: 10px; margin: 10px 0;\">\n<div class=\"webpage-title\" style=\"font-weight:bold; margin:5px 0;\">\n<a href=\"https://example.com\" target=\"_blank\">Example</a></div>\n<div class=\"webpage-url\" style=\"color:#666; font-size:0.9em;margin-bottom:5px;\">https://example.com</div>\n<div class=\"webpage-photo\" style=\"margin-top:10px;\">\n<a href=\"https://example.com\" target=\"_blank\">\n<img src=\"http://test.example.com/media/testchan/20/wp_uid/a3dc7111\" style=\"max-width:100%; width:auto;height:auto; max-height:200px; object-fit:contain;\"></a></div>\n</div>\n</div>"
+  },
+  "webpage_without_photo": {
+    "collected": [],
+    "html": "<div class=\"message-media\">\n<div class=\"webpage-preview\">\n<div class=\"webpage-preview\" style=\"border-left: 3px solid #ccc; padding-left: 10px; margin: 10px 0;\">\n<div class=\"webpage-title\" style=\"font-weight:bold; margin:5px 0;\">\n<a href=\"https://example.com\" target=\"_blank\">Example</a></div>\n<div class=\"webpage-url\" style=\"color:#666; font-size:0.9em;margin-bottom:5px;\">https://example.com</div>\n</div>\n</div>"
+  }
+}

tests/test_group_ids.py

@@ -0,0 +1,272 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""Stage 4 (render-pipeline refactor, issue #31 / epic #34) — pure time-clustering.
+
+`_create_time_based_media_groups` deep-copied every cached message per feed request and
+MUTATED media_group_id. It is replaced by `_compute_time_based_group_ids`, a PURE function
+returning {message.id: effective_media_group_id}; `_create_messages_groups` reads that
+mapping instead of a mutated attribute. All tests use NAIVE dates (as kurigram emits on
+prod), not aware-UTC mocks.
+
+Behavior registry items exercised here: §3.11 (None-date excluded from clustering, no
+mapping entry; own media_group_id still applies) and §3.12 (naive-safe sort keys; None-date
+groups survive [:limit] as newest and land at the tail of the feed via the 0.0 final sort).
+"""
+import os
+import time as _time
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+import rss_generator as rss_module
+from rss_generator import (
+    _compute_time_based_group_ids,
+    _create_messages_groups,
+    generate_channel_html,
+)
+
+
+D = datetime  # naive datetimes throughout
+
+
+class _Str(str):
+    """Str stand-in: .html returns the raw string (mirrors kurigram's Str)."""
+    @property
+    def html(self):
+        return str(self)
+
+
+class Msg:
+    """Minimal message stand-in for the pure clustering function (needs id/date/mgid)."""
+    def __init__(self, mid, date, media_group_id=None):
+        self.id = mid
+        self.date = date
+        self.media_group_id = media_group_id
+        self.service = None
+
+
+def at(sec, minute=0):
+    # Naive local datetime, same shape kurigram's datetime.fromtimestamp() produces.
+    return D(2024, 1, 1, 12, minute, sec)
+
+
+# --------------------------------------------------------------------------- #
+# Core clustering equivalence with the old mutation.
+# --------------------------------------------------------------------------- #
+def test_time_cluster_without_id_gets_synthetic():
+    a, b = Msg(1, at(0)), Msg(2, at(2))
+    mapping = _compute_time_based_group_ids([a, b], merge_seconds=5)
+    synthetic = f"time_{at(0)}"
+    assert mapping == {1: synthetic, 2: synthetic}
+
+
+def test_adoption_backfill_and_overwrite():
+    # First member has no id, second brings "B" (first truthy in cluster order -> wins and
+    # is BACKFILLED onto member 1), third brings "C" which is OVERWRITTEN to "B".
+    a, b, c = Msg(1, at(0)), Msg(2, at(2), "B"), Msg(3, at(4), "C")
+    mapping = _compute_time_based_group_ids([a, b, c], merge_seconds=5)
+    assert mapping == {1: "B", 2: "B", 3: "B"}
+
+
+def test_falsy_id_ignored_like_old_code():
+    # 0 and "" are falsy: they are NOT adopted as the cluster id, exactly as the old
+    # truthiness check. A 2-member cluster with only falsy ids gets a synthetic id.
+    a, b = Msg(1, at(0), 0), Msg(2, at(2), "")
+    mapping = _compute_time_based_group_ids([a, b], merge_seconds=5)
+    synthetic = f"time_{at(0)}"
+    assert mapping == {1: synthetic, 2: synthetic}
+
+
+def test_singleton_gets_no_entry():
+    # A lone message (even one carrying a truthy media_group_id) forms a singleton cluster
+    # and gets NO entry; downstream it falls back to its own media_group_id.
+    lonely = Msg(1, at(0), "MG")
+    far = Msg(2, at(30, minute=1), "OTHER")  # >5s gap -> separate singleton
+    mapping = _compute_time_based_group_ids([lonely, far], merge_seconds=5)
+    assert mapping == {}
+
+
+def test_input_is_not_mutated():
+    a, b, c = Msg(1, at(0)), Msg(2, at(2), "B"), Msg(3, at(4), "C")
+    before = [(m.id, m.date, m.media_group_id) for m in (a, b, c)]
+    _compute_time_based_group_ids([a, b, c], merge_seconds=5)
+    after = [(m.id, m.date, m.media_group_id) for m in (a, b, c)]
+    assert before == after
+    assert a.media_group_id is None and b.media_group_id == "B" and c.media_group_id == "C"
+
+
+def test_ties_equal_dates_cluster_in_fetch_order():
+    # Equal dates -> stable sort keeps fetch order; they cluster (gap 0 <= merge) and the
+    # first truthy id in fetch order wins.
+    a, b = Msg(1, at(5), "FIRST"), Msg(2, at(5), "SECOND")
+    mapping = _compute_time_based_group_ids([a, b], merge_seconds=5)
+    assert mapping == {1: "FIRST", 2: "FIRST"}
+
+
+def test_gap_uses_naive_datetime_subtraction_not_timestamps():
+    # The gap is a NAIVE wall-clock subtraction, exactly as the old code — NOT a timestamp
+    # diff (they DIVERGE across a DST fold, and the old behavior is the contract). Pin a
+    # DST zone and straddle the ambiguous "fall back" hour with the two folds:
+    #   a = 01:30:00 fold=0 -> the FIRST (EDT, UTC-4) occurrence of that wall-clock time;
+    #   b = 01:30:02 fold=1 -> the SECOND (EST, UTC-5) occurrence, ~1h later in REAL time.
+    # Naive subtraction (b.date - a.date) = 2s -> they cluster. A timestamp diff would be
+    # ~3602s -> they would NOT cluster. So mutating the gap to a `.timestamp()` diff turns
+    # this red, locking the naive-subtraction contract.
+    os.environ["TZ"] = "America/New_York"
+    _time.tzset()
+    try:
+        a = Msg(1, D(2024, 11, 3, 1, 30, 0, fold=0))
+        b = Msg(2, D(2024, 11, 3, 1, 30, 2, fold=1))
+        # Sanity-check the fixture itself: naive gap 2s, real (timestamp) gap ~1h.
+        assert (b.date - a.date).total_seconds() == 2
+        assert b.date.timestamp() - a.date.timestamp() > 3600
+        mapping = _compute_time_based_group_ids([a, b], merge_seconds=5)
+        synthetic = f"time_{D(2024, 11, 3, 1, 30, 0)}"
+        assert mapping == {1: synthetic, 2: synthetic}
+    finally:
+        os.environ["TZ"] = "UTC"
+        _time.tzset()
+
+
+# --------------------------------------------------------------------------- #
+# §3.11 — None-date posts: excluded from clustering, own media_group_id still applies.
+# --------------------------------------------------------------------------- #
+def test_none_date_gets_no_mapping_entry():
+    dated, nodate = Msg(1, at(0), "MG"), Msg(2, None, "MG")
+    mapping = _compute_time_based_group_ids([dated, nodate], merge_seconds=5)
+    assert 2 not in mapping  # None-date never participates
+
+
+def test_mixed_none_date_does_not_crash():
+    # The historical prod bug: a single None-date post amid naive-dated posts raised
+    # TypeError. The pure function must simply skip it.
+    msgs = [Msg(1, at(0)), Msg(2, None), Msg(3, at(2))]
+    mapping = _compute_time_based_group_ids(msgs, merge_seconds=5)  # must not raise
+    assert 2 not in mapping
+    # The two dated posts still cluster (2s apart) under a synthetic id.
+    synthetic = f"time_{at(0)}"
+    assert mapping == {1: synthetic, 3: synthetic}
+
+
+def test_fully_none_input_yields_empty_mapping():
+    # Registry §3.11: the old code clustered a fully-None tail by insertion order (only
+    # reachable via aware-date test mocks). New behavior: no clustering at all.
+    msgs = [Msg(1, None, "A"), Msg(2, None), Msg(3, None)]
+    assert _compute_time_based_group_ids(msgs, merge_seconds=5) == {}
+
+
+def test_none_date_media_group_still_assembled_downstream():
+    # None-date posts get no mapping entry but keep their own media_group_id, so a media
+    # group made of None-date members is still assembled by _create_messages_groups.
+    m1, m2 = Msg(10, None, "SHARED"), Msg(11, None, "SHARED")
+    solo = Msg(12, at(0))
+    mapping = _compute_time_based_group_ids([m1, m2, solo], merge_seconds=5)
+    groups = _create_messages_groups([m1, m2, solo], mapping)
+    shared = [g for g in groups if len(g) == 2]
+    assert len(shared) == 1
+    assert {m.id for m in shared[0]} == {10, 11}
+
+
+# --------------------------------------------------------------------------- #
+# §3.12 — naive-safe group sort: None-date groups survive [:limit] and land at the tail.
+# --------------------------------------------------------------------------- #
+def test_create_messages_groups_none_date_does_not_crash_default_path():
+    # This is the LIVE default-path 500: a None-date group used to fall back to an AWARE
+    # now() in the group sort key and blow up against the naive real dates. No mapping /
+    # time_based_merge needed — plain grouping must survive.
+    dated = Msg(1, at(0))
+    nodate = Msg(2, None)
+    groups = _create_messages_groups([dated, nodate])  # must not raise
+    # None-date group sorts as newest (float('inf')) -> first here (reverse=True).
+    assert groups[0][0].id == 2
+
+
+def _make_message(mid, text, date, media_group_id=None):
+    m = SimpleNamespace()
+    m.id = mid
+    m.date = date
+    m.text = _Str(text) if text is not None else None
+    m.caption = None
+    m.media = None
+    m.web_page = None
+    m.poll = None
+    m.service = None
+    m.forward_origin = None
+    m.reply_to_message = None
+    m.reply_to_message_id = None
+    m.sender_chat = None
+    m.from_user = None
+    m.reactions = None
+    m.views = 100
+    m.media_group_id = media_group_id
+    m.show_caption_above_media = False
+    m.chat = SimpleNamespace(id=-1001234567890, username="testchan")
+    for attr in ("photo", "video", "document", "audio", "voice",
+                 "video_note", "animation", "sticker"):
+        setattr(m, attr, None)
+    return m
+
+
+@pytest.mark.asyncio
+async def test_none_date_post_renders_and_lands_at_feed_end(monkeypatch):
+    # END-TO-END regression for the live prod 500: a None-date post in a feed of real-dated
+    # posts, WITH time_based_merge. Old code raised TypeError (naive vs aware) in BOTH the
+    # group sort key (default path) and the time-cluster sort -> HTTP 500 on the default
+    # feed path. New code renders it and, per §3.12, places it at the tail of the feed.
+    posts = [
+        _make_message(1, "OLDEST_REAL", at(0)),
+        _make_message(2, "NEWEST_REAL", at(30)),
+        _make_message(3, "NONE_DATE_POST", None),
+    ]
+
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return posts
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+    monkeypatch.setitem(rss_module.Config, "time_based_merge", True)
+
+    html = await generate_channel_html("testchan", client=SimpleNamespace(), limit=10)
+
+    assert "NONE_DATE_POST" in html
+    assert "NEWEST_REAL" in html and "OLDEST_REAL" in html
+    # None-date post renders LAST (final post sort fallback 0.0 -> tail of feed, §3.12).
+    assert html.index("NONE_DATE_POST") > html.index("OLDEST_REAL")
+    assert html.index("NONE_DATE_POST") > html.index("NEWEST_REAL")
+
+
+@pytest.mark.asyncio
+async def test_none_date_group_survives_limit_slice(monkeypatch):
+    # §3.12 retention: the group sort key gives None-date groups float('inf') so they sort
+    # as NEWEST and deterministically survive the [:limit] slice in _render_pipeline.
+    # This test applies REAL limit pressure (limit < number of groups) so the slice actually
+    # drops groups — otherwise reverting 'inf' to 0.0 (or any small key) would go unnoticed.
+    # With 5 dated posts + 1 None-date post and limit=3, the surviving 3 groups must be the
+    # None-date group + the 2 newest dated; the 3 oldest dated are dropped. If 'inf' becomes
+    # 0.0 the None-date group sorts OLDEST and is dropped instead -> this assertion fails.
+    posts = [_make_message(mid, f"DATED_{mid}", at(0, minute=mid)) for mid in range(1, 6)]
+    posts.append(_make_message(99, "NONE_DATE_POST", None))
+
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return posts
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    html = await generate_channel_html("testchan", client=SimpleNamespace(), limit=3)
+
+    # Exactly 3 posts survive the slice; the None-date group is one of them.
+    assert html.count('class="message-body"') == 3
+    assert "NONE_DATE_POST" in html, "None-date group must survive [:limit] via the inf key"
+    # The 2 newest dated posts survive; the 3 oldest are dropped by the slice.
+    assert "DATED_5" in html and "DATED_4" in html
+    assert "DATED_1" not in html and "DATED_2" not in html and "DATED_3" not in html

tests/test_media_selfheal.py

@@ -0,0 +1,339 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, redefined-outer-name, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""
+Regression tests for the media self-healing fix (post 'static refactor' outage):
+
+Root cause recap: Kurigram serializes downloads through a single get_file slot; a
+zombie media-DC connection makes every download time out while the main-DC watchdog
+stays green, so downloads jam forever. The fix adds (a) a negative-cache backoff so a
+repeatedly-failing file is not hammered, and (b) a consecutive-timeout streak that
+reuses the existing in-process restart to rebuild the media connection.
+"""
+import asyncio
+
+import pytest
+
+import api_server
+from telegram_client import TelegramClient
+
+
+# --------------------------------------------------------------------------- #
+# Negative-cache backoff helpers
+# --------------------------------------------------------------------------- #
+def test_backoff_arms_and_clears():
+    key = ("selfheal_chan", 1, "fid_a")
+    api_server._download_failures.pop(key, None)
+    assert api_server._download_backoff_remaining(key) == 0.0  # never failed -> allowed
+    api_server._record_download_failure(key)
+    assert api_server._download_backoff_remaining(key) > 0.0   # armed -> blocked
+    api_server._clear_download_failure(key)
+    assert api_server._download_backoff_remaining(key) == 0.0  # recovered -> allowed
+
+
+def test_backoff_failure_counter_increments():
+    key = ("selfheal_chan", 2, "fid_b")
+    api_server._download_failures.pop(key, None)
+    api_server._record_download_failure(key)
+    first_fails = api_server._download_failures[key][0]
+    api_server._record_download_failure(key)
+    second_fails = api_server._download_failures[key][0]
+    assert first_fails == 1
+    assert second_fails == 2  # consecutive-failure counter grows -> longer backoff
+    api_server._clear_download_failure(key)
+
+
+# --------------------------------------------------------------------------- #
+# Download-timeout streak -> single media-connection restart
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_timeout_streak_triggers_single_restart(monkeypatch):
+    c = TelegramClient()
+    calls = []
+
+    async def fake_restart(reason: str = "unspecified"):
+        calls.append(reason)
+
+    monkeypatch.setattr(c, "_restart_client", fake_restart)
+    threshold = c.media_timeout_restart_threshold
+
+    # One short of the threshold: no restart scheduled yet.
+    for _ in range(threshold - 1):
+        c.note_download_timeout()
+    assert c._media_recovery_task is None
+    assert c._download_timeout_streak == threshold - 1
+
+    # The threshold-th timeout schedules exactly one restart and resets the streak.
+    c.note_download_timeout()
+    assert c._download_timeout_streak == 0
+    assert c._media_recovery_task is not None
+    await c._media_recovery_task
+    assert calls == ["media download timeout streak"]
+
+
+@pytest.mark.asyncio
+async def test_success_resets_streak():
+    c = TelegramClient()
+    c.note_download_timeout()
+    c.note_download_timeout()
+    assert c._download_timeout_streak == 2
+    c.note_download_ok()
+    assert c._download_timeout_streak == 0
+
+
+@pytest.mark.asyncio
+async def test_no_restart_while_already_restarting(monkeypatch):
+    c = TelegramClient()
+    calls = []
+
+    async def fake_restart(reason: str = "unspecified"):
+        calls.append(reason)
+
+    monkeypatch.setattr(c, "_restart_client", fake_restart)
+    c._restarting = True  # a restart is already underway
+
+    for _ in range(c.media_timeout_restart_threshold):
+        c.note_download_timeout()
+
+    # Streak reached the threshold but no new restart is scheduled during a restart.
+    assert c._media_recovery_task is None
+    assert calls == []
+
+
+# --------------------------------------------------------------------------- #
+# Integration: jam -> auto-recovery -> negative cache cleared (Fix 1)
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_hung_download_times_out_and_frees_slot(monkeypatch):
+    """A hung download must be cancelled by asyncio.wait_for, freeing its transmission slot
+    for the next download (and counting toward the restart streak)."""
+    c = TelegramClient()
+
+    # A single-permit semaphore stands in for Pyrogram's get_file transmission slot: the mock
+    # holds it for the whole (never-completing) download and releases it in the async-with
+    # __aexit__, which runs when wait_for cancels the coroutine.
+    slot = asyncio.Semaphore(1)
+    entered = asyncio.Event()
+
+    async def hung_download(file_id, file_name=None):
+        async with slot:
+            entered.set()
+            await asyncio.sleep(3600)  # never completes within the test's timeout
+
+    monkeypatch.setattr(c.client, "download_media", hung_download)
+
+    with pytest.raises(asyncio.TimeoutError):
+        await c.safe_download_media("fid", "/tmp/x", max_retries=1, timeout=0.05)
+
+    # (a) the download actually started and its slot was released by the cancellation, so a
+    # fresh acquire for the "next" download succeeds immediately.
+    assert entered.is_set()
+    await asyncio.wait_for(slot.acquire(), timeout=1.0)
+    slot.release()
+    # The timeout was counted toward the restart streak.
+    assert c._download_timeout_streak == 1
+
+
+@pytest.mark.asyncio
+async def test_streak_escalates_once_and_verified_restart_clears_cache(monkeypatch):
+    """Full self-heal chain: a timeout streak reaching the threshold escalates to EXACTLY ONE
+    restart, and the verified restart CLEARS the download negative cache (Fix 1)."""
+    c = TelegramClient()
+
+    # Pre-arm the negative cache with a persistently-failing file.
+    key = ("selfheal_chan", 7, "fid_jam")
+    api_server._download_failures.pop(key, None)
+    api_server._record_download_failure(key)
+    assert api_server._download_backoff_remaining(key) > 0.0
+
+    # Wire the REAL self-heal hook and drive a REAL _restart_client with only the network
+    # layer mocked, so the whole chain (streak -> one restart -> verify ok -> cache clear) runs.
+    c.set_restart_callback(api_server._clear_all_download_failures)
+    monkeypatch.setattr(c, "_start_watchdog", lambda: None)
+    monkeypatch.setattr(c.client, "is_connected", True)
+
+    restart_calls = []
+
+    async def fake_client_restart():
+        restart_calls.append(True)
+
+    async def fake_get_me():
+        return type("Me", (), {"id": 42})()
+
+    monkeypatch.setattr(c.client, "restart", fake_client_restart)
+    monkeypatch.setattr(c.client, "get_me", fake_get_me)
+
+    threshold = c.media_timeout_restart_threshold
+    for _ in range(threshold - 1):
+        c.note_download_timeout()
+    assert c._media_recovery_task is None  # one short of threshold: nothing scheduled yet
+
+    c.note_download_timeout()  # threshold-th timeout schedules exactly one restart
+    assert c._media_recovery_task is not None
+    await c._media_recovery_task
+
+    # (b) escalated to EXACTLY ONE underlying client.restart().
+    assert restart_calls == [True]
+    # (c) the verified restart cleared the negative cache -> the file loads again immediately.
+    assert api_server._download_backoff_remaining(key) == 0.0
+    assert len(api_server._download_failures) == 0
+
+
+@pytest.mark.asyncio
+async def test_failed_verify_does_not_clear_cache(monkeypatch):
+    """The cache-clear hook must fire ONLY on a verified restart: if verify_get_me fails, the
+    backoff must survive (a still-broken media DC should not drop the protective backoff)."""
+    c = TelegramClient()
+    key = ("selfheal_chan", 8, "fid_still_broken")
+    api_server._download_failures.pop(key, None)
+    api_server._record_download_failure(key)
+
+    c.set_restart_callback(api_server._clear_all_download_failures)
+    monkeypatch.setattr(c, "_start_watchdog", lambda: None)
+    monkeypatch.setattr(c.client, "is_connected", True)
+
+    async def fake_client_restart():
+        return None
+
+    async def failing_get_me():
+        raise RuntimeError("media DC still dead")
+
+    monkeypatch.setattr(c.client, "restart", fake_client_restart)
+    monkeypatch.setattr(c.client, "get_me", failing_get_me)
+
+    await c._restart_client(reason="test failed verify")
+
+    # verify_get_me failed -> callback NOT fired -> backoff still armed.
+    assert api_server._download_backoff_remaining(key) > 0.0
+    api_server._clear_download_failure(key)
+
+
+# --------------------------------------------------------------------------- #
+# Cross-path negative-cache key consistency
+# --------------------------------------------------------------------------- #
+def test_download_key_consistent_across_paths():
+    """The negative-cache key must be byte-identical across every producer/consumer path so a
+    failure recorded on one path is seen by the guard on another. Reproduce each path's key
+    expression for the same logical file and assert they coincide, then verify end-to-end that
+    a worker-recorded failure is visible to the get_media backoff check."""
+    channel_from_db = "durov"   # background_download_worker / download_new_files: str(channel)
+    post_id_from_db = 123       # ...                                              int(post_id)
+    fid = "AgADfid"
+
+    # get_media receives the raw (possibly differently-cased) URL channel, canonicalizes it,
+    # and uses the int path param post_id.
+    raw_url_channel = "Durov"
+    fs_channel = api_server.canonical_channel_key(raw_url_channel)
+    get_media_key = (fs_channel, 123, fid)
+
+    worker_key = (str(channel_from_db), int(post_id_from_db), fid)      # background_download_worker
+    new_files_key = (str(channel_from_db), int(post_id_from_db), fid)   # download_new_files
+    deduped_key = (str(fs_channel), 123, fid)                          # _download_deduped(fs_channel, post_id, fid)
+
+    assert worker_key == new_files_key == get_media_key == deduped_key
+
+    # End-to-end: a failure recorded by the worker path is seen by the get_media guard.
+    api_server._download_failures.pop(worker_key, None)
+    api_server._record_download_failure(worker_key)
+    try:
+        assert api_server._download_backoff_remaining(get_media_key) > 0.0
+    finally:
+        api_server._clear_download_failure(worker_key)
+
+
+# --------------------------------------------------------------------------- #
+# Fix 2: negative cache is LRU-bounded
+# --------------------------------------------------------------------------- #
+def test_download_failures_lru_bounded(monkeypatch):
+    """The negative cache must not grow unbounded (permanently-404 files leak entries).
+    Once over the cap the oldest entry is evicted."""
+    monkeypatch.setattr(api_server, "_DOWNLOAD_FAILURES_MAX", 3)
+    api_server._download_failures.clear()
+    try:
+        for i in range(5):
+            api_server._record_download_failure(("chan", i, "fid"))
+        assert len(api_server._download_failures) == 3
+        # Oldest two (i=0,1) evicted; newest three retained.
+        remaining = list(api_server._download_failures.keys())
+        assert remaining == [("chan", 2, "fid"), ("chan", 3, "fid"), ("chan", 4, "fid")]
+    finally:
+        api_server._download_failures.clear()
+
+
+# --------------------------------------------------------------------------- #
+# Backoff is capped at _DOWNLOAD_BACKOFF_MAX (min() cap at api_server.py:163)
+# --------------------------------------------------------------------------- #
+def test_backoff_capped_at_max():
+    """The exponential backoff grows as BASE * 2**(fails-1), but must never exceed
+    _DOWNLOAD_BACKOFF_MAX. With BASE=60s and MAX=600s the raw exponential overtakes the cap
+    at the 5th failure (60*2**4 = 960s > 600s), so after that many consecutive failures the
+    effective backoff must stay pinned at the cap. This test FAILS if the min(MAX, ...) cap
+    is removed (the raw exponential would then blow past 600s)."""
+    key = ("selfheal_chan", 99, "fid_cap")
+    api_server._download_failures.pop(key, None)
+
+    # Record well past the point where the raw exponential exceeds the cap: 8 failures ->
+    # raw 60*2**7 = 7680s, an order of magnitude above the 600s cap.
+    for _ in range(8):
+        api_server._record_download_failure(key)
+
+    assert api_server._download_failures[key][0] == 8  # counter really climbed that high
+    remaining = api_server._download_backoff_remaining(key)
+    # The cap holds: remaining is bounded by MAX (with a tiny slack for monotonic drift since
+    # retry_not_before was stamped). Without the min() cap this would be ~7680s.
+    assert remaining <= api_server._DOWNLOAD_BACKOFF_MAX
+    assert remaining > api_server._DOWNLOAD_BACKOFF_MAX - 5  # and it IS pinned near the cap
+
+    api_server._clear_download_failure(key)
+
+
+# --------------------------------------------------------------------------- #
+# Restart-verified callback is best-effort (try/except at telegram_client.py:250-254)
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_restart_callback_error_does_not_abort(monkeypatch, caplog):
+    """A registered restart-verified callback that RAISES must not abort recovery: the error is
+    swallowed + logged, and the restart still completes normally (watchdog re-armed, no SIGTERM
+    fallback, _restarting reset). This test FAILS if the try/except around the callback is
+    removed — the exception would then propagate to the outer handler, skip the watchdog re-arm
+    and trigger the process-restart fallback instead."""
+    c = TelegramClient()
+
+    def exploding_callback():
+        raise RuntimeError("boom in verified-restart callback")
+
+    c.set_restart_callback(exploding_callback)
+
+    # Drive a VERIFIED restart: connected client, restart + get_me both succeed so verify_ok is
+    # True and the callback fires.
+    monkeypatch.setattr(c.client, "is_connected", True)
+
+    async def fake_client_restart():
+        return None
+
+    async def fake_get_me():
+        return type("Me", (), {"id": 42})()
+
+    monkeypatch.setattr(c.client, "restart", fake_client_restart)
+    monkeypatch.setattr(c.client, "get_me", fake_get_me)
+
+    # Observe the success path (watchdog re-arm) vs the failure path (SIGTERM fallback) without
+    # actually killing the test process.
+    watchdog_rearmed = []
+    monkeypatch.setattr(c, "_start_watchdog", lambda: watchdog_rearmed.append(True))
+    sigterm_calls = []
+    monkeypatch.setattr(c, "_restart_app", lambda: sigterm_calls.append(True))
+
+    with caplog.at_level("WARNING"):
+        await c._restart_client(reason="test callback raises")
+
+    # (a) Recovery completed on the SUCCESS path despite the callback raising: watchdog re-armed,
+    # no process-restart fallback, restart flag cleared.
+    assert watchdog_rearmed == [True]
+    assert sigterm_calls == []
+    assert c._restarting is False
+    # (b) The callback error was logged (not silently dropped).
+    assert any(
+        "callback raised" in r.getMessage() and "RuntimeError" in r.getMessage()
+        for r in caplog.records
+    )

tests/test_media_sweeper.py

@@ -0,0 +1,120 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, redefined-outer-name, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""
+Issue #25 (package D) regressions:
+
+  * Задание 12 — the cache sweep must purge a >20-day-old SQLite row whose file is already
+    gone from disk EVEN when nothing was removed from disk in that pass (files_removed == 0).
+    Previously the DB diff ran only `if files_removed > 0`, so such a fileless-but-expired
+    row stayed in the table forever.
+  * Задание 14 — download_new_files must not re-enqueue an already-queued/in-flight file on
+    every sweep pass (dedup via the module-level _queued_media set), and the worker must free
+    the dedup slot in its finally so the file can be re-enqueued later.
+"""
+import sqlite3
+from datetime import datetime
+
+import pytest
+
+import api_server
+from file_io import init_db_sync, get_all_media_file_ids_sync
+
+
+class _StopLoop(Exception):
+    """Sentinel used to break the otherwise-infinite background loops after one iteration."""
+
+
+# --------------------------------------------------------------------------- #
+# Задание 12 — expired row with a missing file is purged when files_removed == 0
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_expired_row_missing_file_purged_when_no_disk_removal(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    db = str(tmp_path / "sweep.db")
+    init_db_sync(db)
+    monkeypatch.setattr(api_server, "DB_PATH", db)
+
+    # A row 21 days old whose cache file does NOT exist on disk. remove_old_cached_files_sync
+    # drops it from the surviving list without incrementing files_removed -> files_removed == 0.
+    old_ts = datetime.now().timestamp() - 21 * 24 * 3600
+    conn = sqlite3.connect(db)
+    conn.execute(
+        "INSERT INTO media_file_ids (channel, post_id, file_unique_id, added) VALUES (?,?,?,?)",
+        ("ghostchan", 42, "ghostfid", old_ts),
+    )
+    conn.commit()
+    conn.close()
+
+    # Run exactly one sweep iteration, then break out of the while-True loop.
+    async def fake_sleep(_delay):
+        raise _StopLoop
+    monkeypatch.setattr(api_server.asyncio, "sleep", fake_sleep)
+
+    with pytest.raises(_StopLoop):
+        await api_server.cache_media_files()
+
+    # The expired, fileless row is gone despite no file having been removed from disk.
+    assert get_all_media_file_ids_sync(db) == []
+
+
+# --------------------------------------------------------------------------- #
+# Задание 14 — dedup: two identical passes enqueue exactly one item
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_download_new_files_dedups_across_passes(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    cache_dir = str(tmp_path / "cache")
+
+    api_server._queued_media.clear()
+    while not api_server.download_queue.empty():
+        api_server.download_queue.get_nowait()
+
+    media_files = [{"channel": "dupchan", "post_id": 7, "file_unique_id": "dupfid"}]
+
+    await api_server.download_new_files(media_files, cache_dir)
+    await api_server.download_new_files(media_files, cache_dir)
+
+    assert api_server.download_queue.qsize() == 1
+    assert ("dupchan", 7, "dupfid") in api_server._queued_media
+
+    # Cleanup shared module state.
+    api_server._queued_media.clear()
+    while not api_server.download_queue.empty():
+        api_server.download_queue.get_nowait()
+
+
+# --------------------------------------------------------------------------- #
+# Задание 14 — the worker frees the dedup slot so the file can be re-enqueued later
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_worker_clears_queued_key_after_processing(monkeypatch):
+    key = ("wchan", 3, "wfid")
+    api_server._queued_media.clear()
+    api_server._queued_media.add(key)
+    while not api_server.download_queue.empty():
+        api_server.download_queue.get_nowait()
+    api_server.download_queue.put_nowait(("wchan", 3, "wfid"))
+
+    async def fake_download(channel, post_id, file_unique_id):
+        return ("/x", False)
+    monkeypatch.setattr(api_server, "download_media_file", fake_download)
+
+    async def fake_sleep(_delay):
+        return None
+    monkeypatch.setattr(api_server.asyncio, "sleep", fake_sleep)
+
+    # Break the while-True worker loop after it has run its finally exactly once.
+    real_task_done = api_server.download_queue.task_done
+    def stop_task_done():
+        real_task_done()
+        raise _StopLoop
+    monkeypatch.setattr(api_server.download_queue, "task_done", stop_task_done)
+
+    with pytest.raises(_StopLoop):
+        await api_server.background_download_worker()
+
+    # Slot freed -> a later sweep may re-enqueue the (still missing) file.
+    assert key not in api_server._queued_media
+
+    api_server._queued_media.clear()

tests/test_message_snapshot.py

@@ -0,0 +1,510 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""Tests for message_snapshot.py (issue #23, Задания 4-5,7).
+
+These verify that a pyrogram Message survives snapshot -> JSON -> restore as a
+CachedMessage that the render pipeline (post_parser.py) consumes identically.
+"""
+import copy
+import json
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from pyrogram.enums import MessageMediaType, MessageServiceType
+
+from message_snapshot import (
+    snapshot_message,
+    restore_message,
+    snapshot_messages,
+    restore_messages,
+    CachedStr,
+    CachedMessage,
+)
+from post_parser import PostParser
+
+
+class FakeStr(str):
+    """Stand-in for a live pyrogram Str: a str carrying a .html rendering."""
+    def __new__(cls, plain, html):
+        obj = str.__new__(cls, plain)
+        obj._html = html
+        return obj
+
+    @property
+    def html(self):
+        return self._html
+
+
+def _roundtrip(message):
+    snap = snapshot_message(message)
+    # The snapshot MUST be JSON-serializable (this is the whole point of the rewrite).
+    json.dumps(snap)
+    return restore_message(snap), snap
+
+
+# --------------------------------------------------------------------------- #
+# Test 1 — top-level round trip.
+# --------------------------------------------------------------------------- #
+def test_roundtrip_basic_fields():
+    msg = SimpleNamespace(
+        id=42,
+        date=datetime(2020, 1, 2, 3, 4, 5),
+        text=FakeStr("hello", "<b>hello</b>"),
+        caption=FakeStr("cap", "<i>cap</i>"),
+        media=MessageMediaType.PHOTO,
+        views=123,
+        media_group_id="mg-1",
+    )
+    restored, _ = _roundtrip(msg)
+    assert restored.id == 42
+    assert restored.date == datetime(2020, 1, 2, 3, 4, 5)
+    assert restored.text.html == "<b>hello</b>"
+    assert str(restored.text) == "hello"
+    assert restored.caption.html == "<i>cap</i>"
+    assert restored.media is MessageMediaType.PHOTO
+    assert restored.views == 123
+    assert restored.media_group_id == "mg-1"
+
+
+def test_roundtrip_date_naive_and_aware():
+    naive = SimpleNamespace(date=datetime(2021, 5, 5, 10, 0, 0))
+    aware = SimpleNamespace(date=datetime(2021, 5, 5, 10, 0, 0, tzinfo=timezone.utc))
+
+    r_naive, _ = _roundtrip(naive)
+    r_aware, _ = _roundtrip(aware)
+
+    assert r_naive.date.tzinfo is None
+    assert r_aware.date.tzinfo is not None
+    assert r_aware.date == datetime(2021, 5, 5, 10, 0, 0, tzinfo=timezone.utc)
+
+
+def test_text_falls_back_to_plain_when_no_html():
+    # A bare str (no .html) -> html defaults to the plain text.
+    msg = SimpleNamespace(text="plain only")
+    restored, _ = _roundtrip(msg)
+    assert restored.text.html == "plain only"
+    assert isinstance(restored.text, CachedStr)
+
+
+# --------------------------------------------------------------------------- #
+# Test 2 — polls with FormattedText fakes (must be JSON-serializable).
+# --------------------------------------------------------------------------- #
+def test_poll_formatted_text_unwrapped():
+    option = SimpleNamespace(text=SimpleNamespace(text="Opt", entities=[]))
+    poll = SimpleNamespace(
+        question=SimpleNamespace(text="Q?", entities=[]),
+        options=[option],
+    )
+    msg = SimpleNamespace(poll=poll)
+
+    snap = snapshot_message(msg)
+    # This MUST NOT raise. If the snapshot stored the FormattedText objects as-is, the
+    # SimpleNamespace values would make json.dumps raise TypeError (regression guard).
+    json.dumps(snap)
+
+    restored = restore_message(snap)
+    assert restored.poll.question == "Q?"
+    assert isinstance(restored.poll.question, str)
+    opt = restored.poll.options[0]
+    assert getattr(opt, "text", "") == "Opt"
+
+
+def test_poll_snapshot_would_fail_if_formattedtext_kept_raw():
+    # Guard proving the assertion above actually catches raw FormattedText: a snapshot dict
+    # that carried the FormattedText object would not be JSON-serializable.
+    bad_snap = {"poll": {"question": SimpleNamespace(text="Q?"), "options": []}}
+    with pytest.raises(TypeError):
+        json.dumps(bad_snap)
+
+
+# --------------------------------------------------------------------------- #
+# Test 3 — reactions (normal / paid / custom).
+# --------------------------------------------------------------------------- #
+def test_reactions_normal_paid_custom():
+    normal = SimpleNamespace(emoji="👍", count=5, is_paid=False)
+    paid = SimpleNamespace(count=3, is_paid=True)
+    custom = SimpleNamespace(custom_emoji_id=1234567890, count=2)
+    msg = SimpleNamespace(reactions=SimpleNamespace(reactions=[normal, paid, custom]))
+
+    restored, _ = _roundtrip(msg)
+    r_normal, r_paid, r_custom = restored.reactions.reactions
+
+    assert r_normal.emoji == "👍"
+    assert r_normal.count == 5
+    assert r_normal.is_paid is False
+    assert r_normal.custom_emoji_id is None
+
+    assert r_paid.is_paid is True
+    assert r_paid.count == 3
+
+    # Custom emoji: emoji is null, custom_emoji_id is a STRING; key still present.
+    assert hasattr(r_custom, "emoji") is True
+    assert r_custom.emoji is None
+    assert isinstance(r_custom.custom_emoji_id, str)
+    assert r_custom.custom_emoji_id == "1234567890"
+
+
+# --------------------------------------------------------------------------- #
+# Test 4 — forward_origin Case 1-5 (presence-semantics drives _format_forward_info).
+# --------------------------------------------------------------------------- #
+def _forward_html(forward_origin):
+    msg = SimpleNamespace(forward_origin=forward_origin)
+    restored, _ = _roundtrip(msg)
+    return PostParser(None)._format_forward_info(restored)
+
+
+def test_forward_origin_cases():
+    # Case 1: channel/supergroup (has .chat)
+    html = _forward_html(SimpleNamespace(
+        type="channel",
+        chat=SimpleNamespace(id=-100, title="Chan", username="chanu"),
+    ))
+    assert "Chan" in html and "@chanu" in html
+
+    # Case 2: hidden user (sender_user_name)
+    html = _forward_html(SimpleNamespace(type="hidden_user", sender_user_name="Hidden Guy"))
+    assert "Hidden Guy" in html
+
+    # Case 3: regular user (sender_user)
+    html = _forward_html(SimpleNamespace(
+        type="user",
+        sender_user=SimpleNamespace(first_name="Ann", last_name="Bee", username="annbee"),
+    ))
+    assert "Ann Bee" in html and "@annbee" in html
+
+    # Case 4: channel without username (chat_id + title, no chat attr)
+    html = _forward_html(SimpleNamespace(type="channel", chat_id=-1002, title="NoUserChan"))
+    assert "NoUserChan" in html
+
+    # Case 5: anything else
+    html = _forward_html(SimpleNamespace(type="something_else"))
+    assert html == '<div class="message-forward">--- Forwarded message ---</div>'
+
+
+def test_forward_origin_presence_semantics():
+    # Only keys present on the live object are recorded (hasattr semantics).
+    restored, snap = _roundtrip(SimpleNamespace(
+        forward_origin=SimpleNamespace(type="hidden_user", sender_user_name="X")))
+    assert "chat" not in snap["forward_origin"]
+    assert "sender_user" not in snap["forward_origin"]
+    assert not hasattr(restored.forward_origin, "chat")
+    assert hasattr(restored.forward_origin, "sender_user_name")
+
+
+# --------------------------------------------------------------------------- #
+# Test 5 — service restored as string usable in `'X' in str(service)`.
+# --------------------------------------------------------------------------- #
+def test_service_pinned_message():
+    msg = SimpleNamespace(service=MessageServiceType.PINNED_MESSAGE)
+    restored, _ = _roundtrip(msg)
+    assert "PINNED_MESSAGE" in str(restored.service)
+
+
+# --------------------------------------------------------------------------- #
+# Test 6 — mutability + deepcopy (.text.html survives).
+# --------------------------------------------------------------------------- #
+def test_mutable_and_deepcopy():
+    msg = SimpleNamespace(id=7, text=FakeStr("body", "<u>body</u>"))
+    restored, _ = _roundtrip(msg)
+
+    # Mutable: reply enrichment assigns reply_to_message.
+    sentinel = object()
+    restored.reply_to_message = sentinel
+    assert restored.reply_to_message is sentinel
+
+    clone = copy.deepcopy(restored)
+    assert clone.text.html == "<u>body</u>"
+    assert str(clone.text) == "body"
+    assert clone.id == 7
+
+
+def test_cachedstr_survives_pickle():
+    import pickle as _pkl
+    s = CachedStr.build("plain", "<b>plain</b>")
+    back = _pkl.loads(_pkl.dumps(s))
+    assert str(back) == "plain"
+    assert back.html == "<b>plain</b>"
+
+
+# --------------------------------------------------------------------------- #
+# Test 7 — unknown media name -> None, no exception.
+# --------------------------------------------------------------------------- #
+def test_unknown_media_type():
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = "FUTURE_TYPE"
+    restored = restore_message(snap)  # must not raise
+    assert restored.media is None
+
+
+# --------------------------------------------------------------------------- #
+# Test 9 — >100MB video is not collected by _save_media_file_ids.
+# --------------------------------------------------------------------------- #
+def test_large_video_not_collected():
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = "VIDEO"
+    snap["video"] = {"file_unique_id": "vid1", "file_size": 200 * 1024 * 1024}
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert pp._pending_media_ids == []
+
+
+def test_normal_video_is_collected():
+    snap = snapshot_message(SimpleNamespace())
+    snap["id"] = 10
+    snap["media"] = "VIDEO"
+    snap["video"] = {"file_unique_id": "vid2", "file_size": 1024}
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert len(pp._pending_media_ids) == 1
+    assert pp._pending_media_ids[0][2] == "vid2"
+
+
+# --------------------------------------------------------------------------- #
+# Test 9b — the >100MB skip must fire for every media type whose selected object
+# flows into _save_media_file_ids' size check, not just video/live_photo. Before
+# file_size was added to these snapshots a restored >100MB doc/audio/animation/
+# video_note lacked file_size and was wrongly collected on a cache hit (F1).
+# --------------------------------------------------------------------------- #
+_BIG = 200 * 1024 * 1024
+
+
+@pytest.mark.parametrize("media,attr,snap_obj", [
+    ("DOCUMENT",   "document",   {"file_unique_id": "doc_big",  "mime_type": "application/zip", "file_size": _BIG}),
+    ("AUDIO",      "audio",      {"file_unique_id": "aud_big",  "mime_type": "audio/mpeg",      "file_size": _BIG}),
+    ("ANIMATION",  "animation",  {"file_unique_id": "anim_big", "file_size": _BIG}),
+    ("VIDEO_NOTE", "video_note", {"file_unique_id": "vn_big",   "file_size": _BIG}),
+])
+def test_large_media_not_collected(media, attr, snap_obj):
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = media
+    snap[attr] = snap_obj
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+    # file_size survives the snapshot/restore round-trip (the round-trip is the fix).
+    assert getattr(restored, attr).file_size == _BIG
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert pp._pending_media_ids == []
+
+
+@pytest.mark.parametrize("media,attr,fuid,snap_obj", [
+    ("DOCUMENT",   "document",   "doc_ok",  {"file_unique_id": "doc_ok",  "mime_type": "application/zip", "file_size": 1024}),
+    ("AUDIO",      "audio",      "aud_ok",  {"file_unique_id": "aud_ok",  "mime_type": "audio/mpeg",      "file_size": 1024}),
+    ("ANIMATION",  "animation",  "anim_ok", {"file_unique_id": "anim_ok", "file_size": 1024}),
+    ("VIDEO_NOTE", "video_note", "vn_ok",   {"file_unique_id": "vn_ok",   "file_size": 1024}),
+])
+def test_normal_media_is_collected(media, attr, fuid, snap_obj):
+    snap = snapshot_message(SimpleNamespace())
+    snap["id"] = 11
+    snap["media"] = media
+    snap[attr] = snap_obj
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert len(pp._pending_media_ids) == 1
+    assert pp._pending_media_ids[0][2] == fuid
+
+
+def test_large_story_media_not_collected():
+    # A >100MB story video (selected over the photo) must be skipped on a cache hit.
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = "STORY"
+    snap["story"] = {
+        "video": {"file_unique_id": "story_vid_big", "file_size": _BIG},
+        "photo": None,
+    }
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+    assert restored.story.video.file_size == _BIG
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert pp._pending_media_ids == []
+
+
+# --------------------------------------------------------------------------- #
+# Test 10 — restored chat without username -> None, not AttributeError.
+# --------------------------------------------------------------------------- #
+def test_chat_without_username():
+    snap = snapshot_message(SimpleNamespace())
+    snap["chat"] = {"id": -100, "username": None, "title": "T", "usernames": None}
+    restored = restore_message(snap)
+    assert restored.chat.username is None
+
+
+def test_chat_usernames_restored_as_objects():
+    chat = SimpleNamespace(
+        id=-100, username=None, title="T",
+        usernames=[SimpleNamespace(username="alt", active=True),
+                   SimpleNamespace(username="old", active=False)],
+    )
+    restored, _ = _roundtrip(SimpleNamespace(chat=chat))
+    active = [u.username for u in restored.chat.usernames if u.active]
+    assert active == ["alt"]
+    # get_channel_username uses exactly this path.
+    assert PostParser(None).get_channel_username(restored) == "alt"
+
+
+# --------------------------------------------------------------------------- #
+# Defaults: every pipeline-consumed attribute exists (getattr never raises).
+# --------------------------------------------------------------------------- #
+def test_empty_message_defaults():
+    restored = restore_message(snapshot_message(SimpleNamespace()))
+    for attr in ["id", "date", "text", "caption", "media", "service", "media_group_id",
+                 "views", "show_caption_above_media", "reply_to_message_id",
+                 "reply_to_message", "chat", "sender_chat", "from_user", "forward_origin",
+                 "reactions", "poll", "web_page", "photo", "video", "document", "audio",
+                 "voice", "video_note", "animation", "sticker"]:
+        assert hasattr(restored, attr)
+    assert restored.empty is False
+
+
+def test_snapshot_messages_list_roundtrip():
+    msgs = [SimpleNamespace(id=i) for i in range(3)]
+    restored = restore_messages(snapshot_messages(msgs))
+    assert [m.id for m in restored] == [0, 1, 2]
+    assert all(isinstance(m, CachedMessage) for m in restored)
+
+
+# --------------------------------------------------------------------------- #
+# Special-media round trips (issue #23 review fix). One per type: assert the exact
+# sub-fields _format_special_media / find_file_id_in_message read survive the trip.
+# These are what silently vanished on a cache hit before the allowlist was completed.
+# --------------------------------------------------------------------------- #
+def test_story_roundtrip_file_unique_id_survives():
+    story = SimpleNamespace(
+        video=SimpleNamespace(file_unique_id="story-vid"),
+        photo=SimpleNamespace(file_unique_id="story-pic"),
+    )
+    restored, _ = _roundtrip(SimpleNamespace(story=story))
+    # find_file_id_in_message / _story_media_object read story.video/photo.file_unique_id.
+    assert restored.story.video.file_unique_id == "story-vid"
+    assert restored.story.photo.file_unique_id == "story-pic"
+
+
+def test_contact_roundtrip():
+    contact = SimpleNamespace(first_name="Ann", last_name="Bee", phone_number="+15551234")
+    restored, _ = _roundtrip(SimpleNamespace(contact=contact))
+    assert restored.contact.first_name == "Ann"
+    assert restored.contact.last_name == "Bee"
+    assert restored.contact.phone_number == "+15551234"
+
+
+def test_location_roundtrip():
+    location = SimpleNamespace(latitude=51.5, longitude=-0.12)
+    restored, _ = _roundtrip(SimpleNamespace(location=location))
+    assert restored.location.latitude == 51.5
+    assert restored.location.longitude == -0.12
+
+
+def test_venue_roundtrip_with_nested_location():
+    venue = SimpleNamespace(
+        title="Big Ben", address="Westminster",
+        location=SimpleNamespace(latitude=51.5, longitude=-0.12),
+    )
+    restored, _ = _roundtrip(SimpleNamespace(venue=venue))
+    assert restored.venue.title == "Big Ben"
+    assert restored.venue.address == "Westminster"
+    assert restored.venue.location.latitude == 51.5
+    assert restored.venue.location.longitude == -0.12
+
+
+def test_dice_roundtrip():
+    restored, _ = _roundtrip(SimpleNamespace(dice=SimpleNamespace(emoji="🎯", value=6)))
+    assert restored.dice.emoji == "🎯"
+    assert restored.dice.value == 6
+
+
+def test_game_roundtrip():
+    restored, _ = _roundtrip(SimpleNamespace(game=SimpleNamespace(title="Chess")))
+    assert restored.game.title == "Chess"
+
+
+def test_giveaway_roundtrip_until_date_datetime():
+    giveaway = SimpleNamespace(
+        quantity=3, months=6, stars=None,
+        until_date=datetime(2030, 12, 31, 12, 0, 0),
+        description="Prizes!",
+    )
+    restored, _ = _roundtrip(SimpleNamespace(giveaway=giveaway))
+    assert restored.giveaway.quantity == 3
+    assert restored.giveaway.months == 6
+    # until_date must restore as a datetime so the renderer's strftime branch fires.
+    assert restored.giveaway.until_date.strftime("%d/%m/%Y") == "31/12/2030"
+    assert restored.giveaway.description == "Prizes!"
+
+
+def test_giveaway_winners_roundtrip():
+    winners = SimpleNamespace(winner_count=5, quantity=10, prize_description="Premium")
+    restored, _ = _roundtrip(SimpleNamespace(giveaway_winners=winners))
+    assert restored.giveaway_winners.winner_count == 5
+    assert restored.giveaway_winners.quantity == 10
+    assert restored.giveaway_winners.prize_description == "Premium"
+
+
+def test_checklist_roundtrip_tasks_and_completion():
+    checklist = SimpleNamespace(
+        title="Todo",
+        tasks=[
+            SimpleNamespace(text="done task", completed_by=SimpleNamespace(id=1), completion_date=None),
+            SimpleNamespace(text="open task", completed_by=None, completion_date=None),
+        ],
+    )
+    restored, _ = _roundtrip(SimpleNamespace(checklist=checklist))
+    assert restored.checklist.title == "Todo"
+    t_done, t_open = restored.checklist.tasks
+    assert t_done.text == "done task"
+    # Renderer marks ☑ when bool(completed_by or completion_date) is True.
+    assert bool(t_done.completed_by or t_done.completion_date) is True
+    assert t_open.text == "open task"
+    assert bool(t_open.completed_by or t_open.completion_date) is False
+
+
+def test_paid_media_roundtrip_stars_and_item_count():
+    paid = SimpleNamespace(stars_amount=50, media=[object(), object(), object()])
+    restored, _ = _roundtrip(SimpleNamespace(paid_media=paid))
+    assert restored.paid_media.stars_amount == 50
+    # Renderer only len()'s .media — the count must survive.
+    assert len(restored.paid_media.media) == 3
+
+
+def test_live_photo_roundtrip_file_unique_id_and_size():
+    live_photo = SimpleNamespace(file_unique_id="lp1", file_size=4096)
+    restored, _ = _roundtrip(SimpleNamespace(live_photo=live_photo))
+    # MEDIA_SOURCES/_get_file_unique_id + find_file_id read live_photo.file_unique_id;
+    # _save_media_file_ids reads file_size (the >100MB skip).
+    assert restored.live_photo.file_unique_id == "lp1"
+    assert restored.live_photo.file_size == 4096
+
+
+def test_large_live_photo_not_collected():
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = "LIVE_PHOTO"
+    snap["live_photo"] = {"file_unique_id": "lpbig", "file_size": 200 * 1024 * 1024}
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert pp._pending_media_ids == []
+
+
+def test_special_media_defaults_present_on_empty_message():
+    restored = restore_message(snapshot_message(SimpleNamespace()))
+    for attr in ["story", "contact", "location", "venue", "dice", "game",
+                 "giveaway", "giveaway_winners", "checklist", "paid_media", "live_photo"]:
+        assert hasattr(restored, attr)
+        assert getattr(restored, attr) is None

tests/test_new_media_types.py

@@ -0,0 +1,580 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""
+Kurigram 2.2.23 — new/uncovered media types (LIVE_PHOTO, STORY, poll media,
+GIVEAWAY, GIVEAWAY_WINNERS, PAID_MEDIA, CHECKLIST, CONTACT, LOCATION, VENUE,
+DICE, GAME, INVOICE, UNSUPPORTED).
+
+Covers:
+- titles for every new media type (_media_message_title via _generate_title);
+- HTML rendering: live photo <video>, story <video>/<img>, poll description_media
+  <img>, paid media info block (no download);
+- info blocks for non-downloadable types (_format_special_media) incl. XSS escaping;
+- flags: no_image semantics for the new types and polls with/without media,
+  "video" flag for live photos;
+- api_server.find_file_id_in_message: live_photo, story, poll description_media /
+  explanation_media lookups.
+"""
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from pyrogram.enums import MessageMediaType
+
+from post_parser import PostParser
+from api_server import find_file_id_in_message
+from url_signer import KeyManager, generate_media_digest
+
+
+class _Str(str):
+    """Minimal stand-in for Pyrogram's Str: .html returns the raw string unchanged."""
+    @property
+    def html(self):
+        return str(self)
+
+
+@pytest.fixture(autouse=True)
+def _pinned_signing_key(monkeypatch):
+    # generate_media_digest reads/creates data/media_digest.key relative to cwd; pin
+    # the in-memory key so digests are deterministic and no file IO happens
+    # regardless of the invocation directory (repo root or tests/).
+    monkeypatch.setattr(KeyManager, "signing_key", "test-signing-key-new-media")
+
+
+@pytest.fixture
+def parser():
+    return PostParser(SimpleNamespace())
+
+
+def make_message(mid=1, media=None, text=None, username="testchan", **extra):
+    m = SimpleNamespace()
+    m.id = mid
+    m.date = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc)
+    m.text = _Str(text) if text is not None else None
+    m.caption = None
+    m.media = media
+    m.web_page = None
+    m.poll = None
+    m.service = None
+    m.forward_origin = None
+    m.reply_to_message = None
+    m.sender_chat = None
+    m.from_user = None
+    m.reactions = None
+    m.views = 100
+    m.media_group_id = None
+    m.show_caption_above_media = False
+    m.chat = SimpleNamespace(id=-1001234567890, username=username)
+    for attr in ("photo", "video", "document", "audio", "voice",
+                 "video_note", "animation", "sticker"):
+        setattr(m, attr, None)
+    # New Kurigram 2.2.23 attributes (live_photo, story, giveaway, checklist, ...)
+    # are deliberately NOT set by default: production code must survive their
+    # absence via getattr (that is exactly what old mocks look like).
+    for key, value in extra.items():
+        setattr(m, key, value)
+    return m
+
+
+def media_url(mid, fuid, username="testchan"):
+    file = f"{username}/{mid}/{fuid}"
+    return f"http://test.example.com/media/{file}/{generate_media_digest(file)}"
+
+
+# ---------------------------------------------------------------------------
+# 1. Titles for every new media type
+# ---------------------------------------------------------------------------
+
+def test_title_live_photo(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.LIVE_PHOTO)) == "📸 Live Photo"
+
+def test_title_story(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.STORY)) == "📖 Story"
+
+def test_title_giveaway(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.GIVEAWAY)) == "🎁 Giveaway"
+
+def test_title_giveaway_winners(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.GIVEAWAY_WINNERS)) == "🏆 Giveaway winners"
+
+def test_title_paid_media(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.PAID_MEDIA)) == "⭐ Paid media"
+
+def test_title_checklist_with_title(parser):
+    msg = make_message(media=MessageMediaType.CHECKLIST,
+                       checklist=SimpleNamespace(title="Shopping list", tasks=[]))
+    assert parser._generate_title(msg) == "📝 Checklist: Shopping list"
+
+def test_title_checklist_long_title_truncated(parser):
+    long_title = "x" * 80
+    msg = make_message(media=MessageMediaType.CHECKLIST,
+                       checklist=SimpleNamespace(title=long_title, tasks=[]))
+    assert parser._generate_title(msg) == f"📝 Checklist: {'x' * 50}"
+
+def test_title_checklist_without_object(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.CHECKLIST)) == "📝 Checklist"
+
+def test_title_contact(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.CONTACT)) == "👤 Contact"
+
+def test_title_location(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.LOCATION)) == "📍 Location"
+
+def test_title_venue_with_title(parser):
+    msg = make_message(media=MessageMediaType.VENUE,
+                       venue=SimpleNamespace(title="Blue Bottle Cafe", address="1 Main St"))
+    assert parser._generate_title(msg) == "📍 Blue Bottle Cafe"
+
+def test_title_venue_fallback(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.VENUE)) == "📍 Venue"
+
+def test_title_dice(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.DICE)) == "🎲 Dice"
+
+def test_title_game(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.GAME)) == "🎮 Game"
+
+def test_title_invoice(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.INVOICE)) == "🧾 Invoice"
+
+def test_title_unsupported(parser):
+    assert parser._generate_title(make_message(media=MessageMediaType.UNSUPPORTED)) == "⚠️ Unsupported content"
+
+
+# ---------------------------------------------------------------------------
+# 2. LIVE_PHOTO rendering
+# ---------------------------------------------------------------------------
+
+def test_live_photo_renders_video_with_signed_url(parser):
+    msg = make_message(101, media=MessageMediaType.LIVE_PHOTO,
+                       live_photo=SimpleNamespace(file_unique_id="lp_uid_1", file_id="lp_fid_1"))
+    html_media = parser._generate_html_media(msg)
+    assert "<video" in html_media
+    assert media_url(101, "lp_uid_1") in html_media
+
+
+def test_live_photo_collected_for_media_ids(parser):
+    msg = make_message(102, media=MessageMediaType.LIVE_PHOTO,
+                       live_photo=SimpleNamespace(file_unique_id="lp_uid_2", file_id="lp_fid_2"))
+    parser._generate_html_media(msg)
+    assert [(c, p, f) for c, p, f, _ in parser._pending_media_ids] == [("testchan", 102, "lp_uid_2")]
+
+
+def test_live_photo_gets_video_flag_not_no_image(parser):
+    msg = make_message(103, media=MessageMediaType.LIVE_PHOTO,
+                       live_photo=SimpleNamespace(file_unique_id="lp_uid_3", file_id="lp_fid_3"))
+    flags = parser._extract_flags(msg)
+    assert "video" in flags
+    assert "no_image" not in flags
+
+
+# ---------------------------------------------------------------------------
+# 3. STORY rendering
+# ---------------------------------------------------------------------------
+
+def test_story_with_video_renders_video(parser):
+    msg = make_message(111, media=MessageMediaType.STORY,
+                       story=SimpleNamespace(video=SimpleNamespace(file_unique_id="st_vid"),
+                                             photo=None))
+    html_media = parser._generate_html_media(msg)
+    assert "<video" in html_media
+    assert media_url(111, "st_vid") in html_media
+
+
+def test_story_with_photo_renders_img(parser):
+    msg = make_message(112, media=MessageMediaType.STORY,
+                       story=SimpleNamespace(video=None,
+                                             photo=SimpleNamespace(file_unique_id="st_pic")))
+    html_media = parser._generate_html_media(msg)
+    assert "<img" in html_media
+    assert "<video" not in html_media
+    assert media_url(112, "st_pic") in html_media
+
+
+def test_story_video_wins_over_photo(parser):
+    msg = make_message(113, media=MessageMediaType.STORY,
+                       story=SimpleNamespace(video=SimpleNamespace(file_unique_id="st_vid2"),
+                                             photo=SimpleNamespace(file_unique_id="st_pic2")))
+    html_media = parser._generate_html_media(msg)
+    assert media_url(113, "st_vid2") in html_media
+    assert "st_pic2" not in html_media
+
+
+def test_story_video_without_uid_falls_back_to_photo_as_img(parser):
+    # Review fix 3: the tag choice must follow the SAME object selection as the URL.
+    # A story video with an unusable file_unique_id is skipped by the helper in
+    # favour of the photo — the render must emit <img>, not <video>.
+    msg = make_message(114, media=MessageMediaType.STORY,
+                       story=SimpleNamespace(video=SimpleNamespace(file_unique_id=None),
+                                             photo=SimpleNamespace(file_unique_id="st_pic3")))
+    html_media = parser._generate_html_media(msg)
+    assert "<img" in html_media
+    assert "<video" not in html_media
+    assert media_url(114, "st_pic3") in html_media
+
+
+def test_story_large_video_not_collected_for_media_ids(parser):
+    # Review fix 2: the >100MB "don't cache" rule applies to story videos too.
+    msg = make_message(115, media=MessageMediaType.STORY,
+                       story=SimpleNamespace(video=SimpleNamespace(file_unique_id="st_big",
+                                                                   file_size=200 * 1024 * 1024),
+                                             photo=None))
+    parser._save_media_file_ids(msg)
+    assert parser._pending_media_ids == []
+
+
+def test_story_small_video_collected_for_media_ids(parser):
+    msg = make_message(116, media=MessageMediaType.STORY,
+                       story=SimpleNamespace(video=SimpleNamespace(file_unique_id="st_small",
+                                                                   file_size=5 * 1024 * 1024),
+                                             photo=None))
+    parser._save_media_file_ids(msg)
+    assert [(c, p, f) for c, p, f, _ in parser._pending_media_ids] == [("testchan", 116, "st_small")]
+
+
+# ---------------------------------------------------------------------------
+# 4. POLL with/without description_media
+# ---------------------------------------------------------------------------
+
+def _poll_with_photo(fuid="poll_pic", fid="poll_pic_fid"):
+    return SimpleNamespace(
+        question=_Str("Pick one?"),
+        options=[],
+        description_media=SimpleNamespace(photo=SimpleNamespace(file_unique_id=fuid, file_id=fid)),
+    )
+
+
+def test_poll_with_description_photo_renders_img(parser):
+    msg = make_message(121, media=MessageMediaType.POLL, poll=_poll_with_photo())
+    html_media = parser._generate_html_media(msg)
+    assert "<img" in html_media
+    assert media_url(121, "poll_pic") in html_media
+
+
+def test_poll_with_description_photo_has_no_no_image_flag(parser):
+    msg = make_message(122, media=MessageMediaType.POLL, poll=_poll_with_photo())
+    flags = parser._extract_flags(msg)
+    assert "no_image" not in flags
+    assert "poll" in flags
+
+
+def test_poll_without_media_keeps_no_image_flag(parser):
+    msg = make_message(123, media=MessageMediaType.POLL,
+                       poll=SimpleNamespace(question=_Str("Plain poll?"), options=[]))
+    flags = parser._extract_flags(msg)
+    assert "no_image" in flags
+    assert "poll" in flags
+
+
+def test_poll_media_collected_for_media_ids(parser):
+    msg = make_message(124, media=MessageMediaType.POLL, poll=_poll_with_photo("poll_pic4"))
+    parser._generate_html_media(msg)
+    assert [(c, p, f) for c, p, f, _ in parser._pending_media_ids] == [("testchan", 124, "poll_pic4")]
+
+
+def test_poll_with_video_description_renders_video(parser):
+    poll = SimpleNamespace(
+        question=_Str("Video poll?"),
+        options=[],
+        description_media=SimpleNamespace(video=SimpleNamespace(file_unique_id="poll_vid", file_id="poll_vid_fid")),
+    )
+    msg = make_message(125, media=MessageMediaType.POLL, poll=poll)
+    html_media = parser._generate_html_media(msg)
+    assert "<video" in html_media
+    assert media_url(125, "poll_vid") in html_media
+
+
+# ---------------------------------------------------------------------------
+# 5. api_server.find_file_id_in_message (async)
+# ---------------------------------------------------------------------------
+
+async def test_find_file_id_live_photo():
+    msg = make_message(201, media=MessageMediaType.LIVE_PHOTO,
+                       live_photo=SimpleNamespace(file_unique_id="lp_uid", file_id="lp_fid"))
+    assert await find_file_id_in_message(msg, "lp_uid") == "lp_fid"
+
+
+async def test_find_file_id_story_video():
+    msg = make_message(202, media=MessageMediaType.STORY,
+                       story=SimpleNamespace(photo=None,
+                                             video=SimpleNamespace(file_unique_id="sv_uid", file_id="sv_fid")))
+    assert await find_file_id_in_message(msg, "sv_uid") == "sv_fid"
+
+
+async def test_find_file_id_story_photo():
+    msg = make_message(203, media=MessageMediaType.STORY,
+                       story=SimpleNamespace(photo=SimpleNamespace(file_unique_id="sp_uid", file_id="sp_fid"),
+                                             video=None))
+    assert await find_file_id_in_message(msg, "sp_uid") == "sp_fid"
+
+
+async def test_find_file_id_poll_description_photo():
+    msg = make_message(204, media=MessageMediaType.POLL, poll=_poll_with_photo("pd_uid", "pd_fid"))
+    assert await find_file_id_in_message(msg, "pd_uid") == "pd_fid"
+
+
+async def test_find_file_id_poll_explanation_video():
+    poll = SimpleNamespace(
+        question=_Str("Quiz?"),
+        options=[],
+        explanation_media=SimpleNamespace(video=SimpleNamespace(file_unique_id="ex_uid", file_id="ex_fid")),
+    )
+    msg = make_message(205, media=MessageMediaType.POLL, poll=poll)
+    assert await find_file_id_in_message(msg, "ex_uid") == "ex_fid"
+
+
+async def test_find_file_id_poll_without_media_returns_none():
+    msg = make_message(206, media=MessageMediaType.POLL,
+                       poll=SimpleNamespace(question=_Str("Plain?"), options=[]))
+    assert await find_file_id_in_message(msg, "whatever") is None
+
+
+async def test_find_file_id_poll_none_object_returns_none():
+    msg = make_message(207, media=MessageMediaType.POLL)  # message.poll stays None
+    assert await find_file_id_in_message(msg, "whatever") is None
+
+
+# ---------------------------------------------------------------------------
+# 6. XSS: user-controlled strings in info blocks are escaped
+# ---------------------------------------------------------------------------
+
+XSS = "<script>alert(1)</script>"
+XSS_ESCAPED = "&lt;script&gt;alert(1)&lt;/script&gt;"
+
+
+def test_contact_name_is_escaped(parser):
+    msg = make_message(301, media=MessageMediaType.CONTACT,
+                       contact=SimpleNamespace(first_name=XSS, last_name="Doe",
+                                               phone_number="+1234567890", user_id=None, vcard=None))
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert XSS_ESCAPED in body
+    assert "+1234567890" in body
+
+
+def test_checklist_task_text_is_escaped(parser):
+    checklist = SimpleNamespace(
+        title="My list",
+        tasks=[
+            SimpleNamespace(id=1, text=XSS, completed_by=None, completion_date=None),
+            SimpleNamespace(id=2, text="done task", completed_by=SimpleNamespace(id=7), completion_date=None),
+        ],
+    )
+    msg = make_message(302, media=MessageMediaType.CHECKLIST, checklist=checklist)
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert f"☐ {XSS_ESCAPED}" in body
+    assert "☑ done task" in body
+    assert "📝 My list" in body
+
+
+def test_venue_title_is_escaped(parser):
+    # VENUE title feeds venue_label -> html.escape(venue_label)
+    venue = SimpleNamespace(title=XSS, address="1 Main St",
+                            location=SimpleNamespace(latitude=1.5, longitude=2.5))
+    msg = make_message(303, media=MessageMediaType.VENUE, venue=venue)
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert XSS_ESCAPED in body
+
+
+def test_venue_address_label_is_escaped(parser):
+    # VENUE address also feeds venue_label -> html.escape(venue_label)
+    venue = SimpleNamespace(title="Blue Bottle", address=XSS,
+                            location=SimpleNamespace(latitude=1.5, longitude=2.5))
+    msg = make_message(304, media=MessageMediaType.VENUE, venue=venue)
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert XSS_ESCAPED in body
+
+
+def test_dice_emoji_is_escaped(parser):
+    # DICE emoji -> html.escape(str(dice_emoji))
+    msg = make_message(305, media=MessageMediaType.DICE,
+                       dice=SimpleNamespace(emoji=XSS, value=6))
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert XSS_ESCAPED in body
+
+
+def test_game_title_is_escaped(parser):
+    # GAME title -> html.escape(game_title.strip())
+    msg = make_message(306, media=MessageMediaType.GAME,
+                       game=SimpleNamespace(title=XSS))
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert XSS_ESCAPED in body
+
+
+def test_checklist_title_is_escaped(parser):
+    # CHECKLIST title -> html.escape(title_str)
+    checklist = SimpleNamespace(title=XSS, tasks=[])
+    msg = make_message(307, media=MessageMediaType.CHECKLIST, checklist=checklist)
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert XSS_ESCAPED in body
+
+
+def test_giveaway_description_is_escaped(parser):
+    # GIVEAWAY description -> html.escape(description.strip())
+    giveaway = SimpleNamespace(quantity=5, months=None, stars=None,
+                               until_date=None, description=XSS)
+    msg = make_message(308, media=MessageMediaType.GIVEAWAY, giveaway=giveaway)
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert XSS_ESCAPED in body
+
+
+def test_giveaway_winners_prize_description_is_escaped(parser):
+    # GIVEAWAY_WINNERS prize_description -> html.escape(prize_description.strip())
+    winners = SimpleNamespace(winner_count=2, quantity=5, prize_description=XSS,
+                              unclaimed_prize_count=0, winners=[])
+    msg = make_message(309, media=MessageMediaType.GIVEAWAY_WINNERS, giveaway_winners=winners)
+    body = parser._generate_html_body(msg)
+    assert XSS not in body
+    assert XSS_ESCAPED in body
+
+
+# ---------------------------------------------------------------------------
+# 7. Giveaway / giveaway winners info blocks
+# ---------------------------------------------------------------------------
+
+def test_giveaway_block_quantity_and_months(parser):
+    giveaway = SimpleNamespace(quantity=10, months=3, stars=None,
+                               until_date=datetime(2026, 2, 1, tzinfo=timezone.utc),
+                               description="Win big!")
+    msg = make_message(311, media=MessageMediaType.GIVEAWAY, giveaway=giveaway)
+    body = parser._generate_html_body(msg)
+    assert "🎁 Giveaway: 10 prize(s)" in body
+    assert "3 months Premium" in body
+    assert "until 01/02/2026" in body
+    assert "Win big!" in body
+
+
+def test_giveaway_block_with_stars(parser):
+    giveaway = SimpleNamespace(quantity=5, months=None, stars=500,
+                               until_date=None, description=None)
+    msg = make_message(312, media=MessageMediaType.GIVEAWAY, giveaway=giveaway)
+    body = parser._generate_html_body(msg)
+    assert "🎁 Giveaway: 5 prize(s)" in body
+    assert "500 Stars" in body
+
+
+def test_giveaway_winners_block(parser):
+    winners = SimpleNamespace(winner_count=2, quantity=5, prize_description="Cool prize",
+                              unclaimed_prize_count=3, winners=[])
+    msg = make_message(313, media=MessageMediaType.GIVEAWAY_WINNERS, giveaway_winners=winners)
+    body = parser._generate_html_body(msg)
+    assert "🏆 Giveaway winners: 2 of 5" in body
+    assert "Cool prize" in body
+
+
+# ---------------------------------------------------------------------------
+# Other info blocks and paid media
+# ---------------------------------------------------------------------------
+
+def test_paid_media_renders_info_block_without_download(parser):
+    paid = SimpleNamespace(stars_amount=50,
+                           media=[SimpleNamespace(width=100, height=100, duration=None, thumbnail=None),
+                                  SimpleNamespace(width=200, height=200, duration=5, thumbnail=None)])
+    msg = make_message(321, media=MessageMediaType.PAID_MEDIA, paid_media=paid)
+    html_media = parser._generate_html_media(msg)
+    assert "⭐ Paid media (50 stars, 2 item(s)) — available in Telegram" in html_media
+    assert "/media/" not in html_media  # nothing downloadable
+    assert parser._pending_media_ids == []  # nothing collected for the download cache
+    assert "no_image" in parser._extract_flags(msg)
+
+
+def test_location_block_has_osm_link(parser):
+    msg = make_message(322, media=MessageMediaType.LOCATION,
+                       location=SimpleNamespace(latitude=55.75123456, longitude=37.61761111))
+    body = parser._generate_html_body(msg)
+    assert "📍 Location:" in body
+    assert "https://www.openstreetmap.org/?mlat=55.75123456&mlon=37.61761111" in body
+    assert "55.75123, 37.61761" in body
+    assert "no_image" in parser._extract_flags(msg)
+
+
+def test_venue_block_with_osm_link(parser):
+    venue = SimpleNamespace(title="Blue Bottle", address="1 Main St",
+                            location=SimpleNamespace(latitude=1.5, longitude=2.5))
+    msg = make_message(323, media=MessageMediaType.VENUE, venue=venue)
+    body = parser._generate_html_body(msg)
+    assert "📍 Blue Bottle, 1 Main St" in body
+    assert "openstreetmap.org/?mlat=1.5&mlon=2.5" in body
+
+
+# Review fix 1: info-block-only media types must not open an empty
+# <div class="message-media"> container — only the special block is rendered.
+
+def test_venue_has_no_empty_media_div(parser):
+    venue = SimpleNamespace(title="Cafe", address="2 Side St",
+                            location=SimpleNamespace(latitude=1.0, longitude=2.0))
+    msg = make_message(328, media=MessageMediaType.VENUE, venue=venue)
+    body = parser._generate_html_body(msg)
+    assert 'class="message-media"' not in body
+    assert 'class="message-special"' in body
+
+
+def test_contact_has_no_empty_media_div(parser):
+    msg = make_message(329, media=MessageMediaType.CONTACT,
+                       contact=SimpleNamespace(first_name="Jane", last_name="Doe",
+                                               phone_number="+1999", user_id=None, vcard=None))
+    body = parser._generate_html_body(msg)
+    assert 'class="message-media"' not in body
+    assert 'class="message-special"' in body
+
+
+def test_paid_media_block_survives_media_div_gate(parser):
+    # PAID_MEDIA is in NO_IMAGE_MEDIA_TYPES but has its own render branch — the
+    # info block (inside its message-media container) must keep rendering.
+    paid = SimpleNamespace(stars_amount=10, media=[SimpleNamespace(width=1, height=1,
+                                                                   duration=None, thumbnail=None)])
+    msg = make_message(330, media=MessageMediaType.PAID_MEDIA, paid_media=paid)
+    body = parser._generate_html_body(msg)
+    assert 'class="message-media"' in body
+    assert "⭐ Paid media (10 stars, 1 item(s)) — available in Telegram" in body
+
+
+def test_dice_block(parser):
+    msg = make_message(324, media=MessageMediaType.DICE,
+                       dice=SimpleNamespace(emoji="🎯", value=6))
+    body = parser._generate_html_body(msg)
+    assert "🎲 🎯: 6" in body
+
+
+def test_game_block_with_title(parser):
+    msg = make_message(325, media=MessageMediaType.GAME,
+                       game=SimpleNamespace(title="Tetris"))
+    body = parser._generate_html_body(msg)
+    assert "🎮 Game: Tetris" in body
+
+
+def test_invoice_block(parser):
+    msg = make_message(326, media=MessageMediaType.INVOICE)
+    body = parser._generate_html_body(msg)
+    assert "🧾 Invoice" in body
+
+
+def test_unsupported_block(parser):
+    msg = make_message(327, media=MessageMediaType.UNSUPPORTED)
+    body = parser._generate_html_body(msg)
+    assert "⚠️ This post contains content not supported by the bridge" in body
+    assert "no_image" in parser._extract_flags(msg)
+
+
+# ---------------------------------------------------------------------------
+# Regression: messages without any of the new attributes still render
+# ---------------------------------------------------------------------------
+
+def test_plain_text_message_unaffected(parser):
+    msg = make_message(331, text="just a plain post with enough text")
+    body = parser._generate_html_body(msg)
+    assert "just a plain post with enough text" in body
+    assert "message-special" not in body
+    assert "paid-media" not in body
+    flags = parser._extract_flags(msg)
+    assert "no_image" in flags

tests/test_postparser_author_info.py

@@ -7,13 +7,6 @@
 
 import unittest
 from unittest.mock import MagicMock
-import sys
-import os
-# Add project root to sys.path to find post_parser
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-
-# Mock the config module
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
 
 from pyrogram.types import Message
 from post_parser import PostParser

tests/test_postparser_extract_flags.py

@@ -7,14 +7,6 @@
 
 import unittest
 from unittest.mock import MagicMock, PropertyMock
-import sys
-import os
-
-# Add project root to sys.path to find post_parser
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-
-# Mock the config module before importing PostParser
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
 
 from pyrogram.types import Message, Chat, Reaction, MessageReactions
 from pyrogram.enums import MessageMediaType

tests/test_postparser_gen_title.py

@@ -7,13 +7,6 @@
 # pylance: disable=reportMissingImports, reportMissingModuleSource
 import unittest
 from unittest.mock import MagicMock, PropertyMock
-import sys
-import os
-# Add project root to sys.path to find post_parser
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-
-# Mock the config module
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
 
 from pyrogram.types import Message, Document
 from pyrogram.enums import MessageMediaType

tests/test_sanitizer.py

@@ -0,0 +1,141 @@
+# flake8: noqa
+# pylint: disable=missing-function-docstring, redefined-outer-name, line-too-long
+"""Unit tests for the single project-wide bleach config (sanitizer.py, issue #28 / §3).
+
+Covers the stage-1 registry items that this module owns:
+  §3.1  s / del (strikethrough) survive.
+  §3.2  fail-closed: on a bleach error the fragment is html.escape()d, never returned raw.
+  §3.16 the single error-log name `html_sanitization_error` + log_context.
+Plus the two non-default bleach params the maintainer flagged as load-bearing:
+  strip=True (disallowed tags are REMOVED, not escaped into visible text) and
+  protocols including 'tg' (tg:// links survive).
+And the one-config invariant: only one ALLOWED_TAGS list exists in the repo.
+"""
+import logging
+
+import pytest
+
+import sanitizer
+from sanitizer import sanitize_html, ALLOWED_TAGS, ALLOWED_PROTOCOLS
+
+
+# --------------------------------------------------------------------------- #
+# §3.1 — strikethrough survives (the drift the single config fixes).
+# --------------------------------------------------------------------------- #
+def test_strikethrough_s_survives():
+    assert sanitize_html("<s>gone</s>") == "<s>gone</s>"
+
+
+def test_strikethrough_del_survives():
+    assert sanitize_html("<del>removed</del>") == "<del>removed</del>"
+
+
+def test_s_and_del_in_allowed_tags():
+    assert "s" in ALLOWED_TAGS
+    assert "del" in ALLOWED_TAGS
+
+
+# --------------------------------------------------------------------------- #
+# strip=True — disallowed tags are REMOVED (bleach's default False would escape
+# them into visible text). Both the tag and its dangerous attrs must vanish.
+# --------------------------------------------------------------------------- #
+def test_script_is_stripped_not_escaped():
+    out = sanitize_html("<b>hi</b><script>alert(1)</script>")
+    assert out == "<b>hi</b>alert(1)"
+    # strip=True removed the tag entirely — it was NOT escaped into visible &lt;script&gt;.
+    assert "&lt;script&gt;" not in out
+    assert "<script" not in out
+
+
+def test_onerror_attribute_is_stripped():
+    out = sanitize_html('<img src="http://e/x.png" onerror="alert(1)">')
+    assert "onerror" not in out
+    assert "alert(1)" not in out
+
+
+def test_javascript_protocol_href_dropped():
+    out = sanitize_html('<a href="javascript:alert(1)">x</a>')
+    # The <a> tag survives (whitelisted) but the dangerous href is dropped by the protocol filter.
+    assert "javascript:" not in out
+
+
+# --------------------------------------------------------------------------- #
+# protocols — non-default 'tg' keeps tg:// links (channel footers use them).
+# --------------------------------------------------------------------------- #
+def test_tg_protocol_href_survives():
+    out = sanitize_html('<a href="tg://resolve?domain=x">x</a>')
+    assert 'href="tg://resolve?domain=x"' in out
+
+
+def test_http_and_https_survive():
+    assert 'href="https://e/x"' in sanitize_html('<a href="https://e/x">x</a>')
+    assert 'href="http://e/x"' in sanitize_html('<a href="http://e/x">x</a>')
+
+
+def test_tg_in_allowed_protocols():
+    assert ALLOWED_PROTOCOLS == ['http', 'https', 'tg']
+
+
+# --------------------------------------------------------------------------- #
+# CSS sanitizer — only the whitelisted properties survive inside style="".
+# --------------------------------------------------------------------------- #
+def test_css_allowed_property_survives():
+    out = sanitize_html('<img src="http://e/x" style="max-width: 100%">')
+    assert "max-width" in out
+
+
+def test_css_disallowed_property_dropped():
+    out = sanitize_html('<div style="position: fixed; max-height: 50px">x</div>')
+    assert "position" not in out
+    assert "max-height" in out
+
+
+# --------------------------------------------------------------------------- #
+# §3.2 — FAIL-CLOSED: any bleach error escapes the raw input, never returns it raw.
+# §3.16 — the single error-log name `html_sanitization_error` + log_context.
+# --------------------------------------------------------------------------- #
+def test_fail_closed_escapes_on_bleach_error(monkeypatch, caplog):
+    def boom(*a, **k):
+        raise RecursionError("bleach exploded")
+    # sanitize_html resolves HTMLSanitizer as a module global at call time.
+    monkeypatch.setattr(sanitizer, "HTMLSanitizer", boom, raising=True)
+
+    payload = '<script>alert(1)</script>'
+    with caplog.at_level(logging.ERROR):
+        out = sanitize_html(payload, log_context="channel test, message_id 7")
+
+    # Fail-closed: the raw payload was html.escape()d, NOT returned raw.
+    assert out == "&lt;script&gt;alert(1)&lt;/script&gt;"
+    assert "<script" not in out
+    # §3.16: single error-log name, with the log_context included for grep-ability.
+    assert "html_sanitization_error" in caplog.text
+    assert "channel test, message_id 7" in caplog.text
+
+
+def test_fail_closed_log_name_without_context(monkeypatch, caplog):
+    def boom(*a, **k):
+        raise ValueError("nope")
+    monkeypatch.setattr(sanitizer, "HTMLSanitizer", boom, raising=True)
+    with caplog.at_level(logging.ERROR):
+        sanitize_html("<b>x</b>")
+    # Same single name even when no context is supplied (single-post path).
+    assert "html_sanitization_error" in caplog.text
+    # The legacy per-path names must be gone.
+    assert "rss_html_sanitization_error" not in caplog.text
+    assert "html_final_sanitization_error" not in caplog.text
+
+
+# --------------------------------------------------------------------------- #
+# One-config invariant: exactly one ALLOWED_TAGS list in the tree, in sanitizer.py.
+# post_parser / rss_generator must route through the module, not re-declare bleach.
+# --------------------------------------------------------------------------- #
+def test_single_bleach_config_no_reimport_in_render_modules():
+    import inspect
+    import post_parser
+    import rss_generator
+    for mod in (post_parser, rss_generator):
+        src = inspect.getsource(mod)
+        assert "allowed_tags" not in src.lower() or "sanitizer" in src, \
+            f"{mod.__name__} appears to re-declare a bleach tag list instead of using sanitizer.py"
+        assert "from bleach" not in src and "import bleach" not in src, \
+            f"{mod.__name__} still imports bleach directly; the only config lives in sanitizer.py"

tests/test_snapshot_render_parity.py

@@ -0,0 +1,302 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""Render-parity oracle for the snapshot cache (issue #23 review fix).
+
+The core ask of the review: prove that snapshot -> restore preserves EVERYTHING the
+render pipeline reads. The old golden test replays a pickle corpus and never renders a
+live Message against its snapshot, so a field dropped from the allowlist (the exact
+bug the reviewer mutation-proved for web_page/sender_chat/... and for the special-media
+types) passes silently.
+
+This test builds a CORPUS of representative live fake Message objects (plain text, every
+media type, every special-media type, web_page, poll, forwards Case 1-5, reactions,
+caption, a multi-message media group) and for EACH case:
+
+  1. renders the LIVE objects through the REAL public feed entry points
+     (rss_generator.generate_channel_html / generate_channel_rss — the same path the
+     golden test and production use, driven by monkeypatching tg_cache),
+  2. snapshot -> restore every message, renders the RESTORED objects the same way,
+  3. asserts the two renders are BYTE-IDENTICAL (HTML and the RSS feed).
+
+Any field the snapshot fails to preserve makes the two renders diverge and this test go
+red. It is a self-contained live-vs-restored parity oracle; it does NOT depend on the
+frozen pickle goldens.
+"""
+import asyncio
+from datetime import datetime
+from types import SimpleNamespace
+
+import pytest
+
+from pyrogram.enums import MessageMediaType
+
+from message_snapshot import snapshot_messages, restore_messages
+from tests import golden_replay as gr
+
+
+# --------------------------------------------------------------------------- #
+# Live-object fakes.
+# --------------------------------------------------------------------------- #
+class FakeStr(str):
+    """Stand-in for a live pyrogram Str: a str carrying a .html rendering."""
+    def __new__(cls, plain, html=None):
+        obj = str.__new__(cls, plain)
+        obj._html = html if html is not None else plain
+        return obj
+
+    @property
+    def html(self):
+        return self._html
+
+
+_CHANNEL = "parity_ch"
+_CHAT = SimpleNamespace(id=-1001234567890, username=_CHANNEL, title="Parity Channel", usernames=None)
+
+# The full top-level attribute surface the render pipeline reads (mirrors CachedMessage).
+# A live fake sets every one so it renders WITHOUT relying on getattr fallbacks that a
+# restored CachedMessage would provide — otherwise the live side, not the snapshot, would
+# be the thing under-specified.
+_DEFAULTS = dict(
+    id=0, date=None, text=None, caption=None, media=None, service=None,
+    media_group_id=None, views=100, show_caption_above_media=False,
+    reply_to_message_id=None, reply_to_message=None, empty=False,
+    chat=_CHAT, sender_chat=None, from_user=None, forward_origin=None,
+    reactions=None, poll=None, web_page=None, photo=None, video=None,
+    document=None, audio=None, voice=None, video_note=None, animation=None,
+    sticker=None, story=None, contact=None, location=None, venue=None,
+    dice=None, game=None, giveaway=None, giveaway_winners=None,
+    checklist=None, paid_media=None, live_photo=None,
+)
+
+_id_counter = [1000]
+
+
+def make_msg(**overrides):
+    _id_counter[0] += 1
+    fields = dict(_DEFAULTS)
+    fields["id"] = _id_counter[0]
+    fields["date"] = datetime(2024, 3, 1, 12, 0, _id_counter[0] % 60)
+    fields.update(overrides)
+    return SimpleNamespace(**fields)
+
+
+def _reactions(*reacts):
+    return SimpleNamespace(reactions=list(reacts))
+
+
+# --------------------------------------------------------------------------- #
+# Corpus: each case is (name, [live messages]) rendered as its own mini feed.
+# --------------------------------------------------------------------------- #
+def build_corpus():
+    corpus = []
+
+    # Plain text.
+    corpus.append(("plain_text", [make_msg(text=FakeStr("Hello world", "Hello <b>world</b>"))]))
+
+    # Caption (photo + caption + show_caption_above_media).
+    corpus.append(("caption_photo", [make_msg(
+        media=MessageMediaType.PHOTO,
+        photo=SimpleNamespace(file_unique_id="pcap"),
+        caption=FakeStr("A caption", "A <i>caption</i>"),
+        show_caption_above_media=True,
+    )]))
+
+    # Regular media types.
+    corpus.append(("photo", [make_msg(media=MessageMediaType.PHOTO,
+                                      photo=SimpleNamespace(file_unique_id="ph1"))]))
+    corpus.append(("video", [make_msg(media=MessageMediaType.VIDEO,
+                                      video=SimpleNamespace(file_unique_id="vd1", file_size=2048))]))
+    corpus.append(("document_pdf", [make_msg(media=MessageMediaType.DOCUMENT,
+                                             document=SimpleNamespace(file_unique_id="doc1", mime_type="application/pdf"))]))
+    corpus.append(("document_other", [make_msg(media=MessageMediaType.DOCUMENT,
+                                               document=SimpleNamespace(file_unique_id="doc2", mime_type="image/png"))]))
+    corpus.append(("audio", [make_msg(media=MessageMediaType.AUDIO,
+                                      audio=SimpleNamespace(file_unique_id="au1", mime_type="audio/mpeg"))]))
+    corpus.append(("voice", [make_msg(media=MessageMediaType.VOICE,
+                                      voice=SimpleNamespace(file_unique_id="vo1", mime_type="audio/ogg"))]))
+    corpus.append(("video_note", [make_msg(media=MessageMediaType.VIDEO_NOTE,
+                                           video_note=SimpleNamespace(file_unique_id="vn1"))]))
+    corpus.append(("animation", [make_msg(media=MessageMediaType.ANIMATION,
+                                          animation=SimpleNamespace(file_unique_id="an1"))]))
+    corpus.append(("sticker_image", [make_msg(media=MessageMediaType.STICKER,
+                                              sticker=SimpleNamespace(file_unique_id="st1", emoji="😀", is_video=False))]))
+    corpus.append(("sticker_video", [make_msg(media=MessageMediaType.STICKER,
+                                              sticker=SimpleNamespace(file_unique_id="st2", emoji="🎥", is_video=True))]))
+    corpus.append(("live_photo", [make_msg(media=MessageMediaType.LIVE_PHOTO,
+                                           live_photo=SimpleNamespace(file_unique_id="lp1", file_size=4096))]))
+
+    # --- Special-media types (Fix 1) ------------------------------------- #
+    corpus.append(("story_photo", [make_msg(
+        media=MessageMediaType.STORY,
+        story=SimpleNamespace(video=None, photo=SimpleNamespace(file_unique_id="sto_p")),
+    )]))
+    corpus.append(("story_video", [make_msg(
+        media=MessageMediaType.STORY,
+        story=SimpleNamespace(video=SimpleNamespace(file_unique_id="sto_v"), photo=None),
+    )]))
+    corpus.append(("contact", [make_msg(
+        media=MessageMediaType.CONTACT,
+        contact=SimpleNamespace(first_name="Ann", last_name="Bee", phone_number="+15551234"),
+    )]))
+    corpus.append(("location", [make_msg(
+        media=MessageMediaType.LOCATION,
+        location=SimpleNamespace(latitude=51.50111, longitude=-0.14222),
+    )]))
+    corpus.append(("venue", [make_msg(
+        media=MessageMediaType.VENUE,
+        venue=SimpleNamespace(title="Big Ben", address="Westminster",
+                              location=SimpleNamespace(latitude=51.50055, longitude=-0.12461)),
+    )]))
+    corpus.append(("dice", [make_msg(
+        media=MessageMediaType.DICE, dice=SimpleNamespace(emoji="🎯", value=6))]))
+    corpus.append(("game", [make_msg(
+        media=MessageMediaType.GAME, game=SimpleNamespace(title="Chess Master"))]))
+    corpus.append(("giveaway", [make_msg(
+        media=MessageMediaType.GIVEAWAY,
+        giveaway=SimpleNamespace(quantity=3, months=6, stars=None,
+                                 until_date=datetime(2030, 12, 31, 12, 0, 0),
+                                 description="Great prizes"),
+    )]))
+    corpus.append(("giveaway_stars", [make_msg(
+        media=MessageMediaType.GIVEAWAY,
+        giveaway=SimpleNamespace(quantity=2, months=None, stars=500,
+                                 until_date=None, description=None),
+    )]))
+    corpus.append(("giveaway_winners", [make_msg(
+        media=MessageMediaType.GIVEAWAY_WINNERS,
+        giveaway_winners=SimpleNamespace(winner_count=5, quantity=10, prize_description="Premium"),
+    )]))
+    corpus.append(("checklist", [make_msg(
+        media=MessageMediaType.CHECKLIST,
+        checklist=SimpleNamespace(title="Todo list", tasks=[
+            SimpleNamespace(text="done task", completed_by=SimpleNamespace(id=1), completion_date=None),
+            SimpleNamespace(text="open task", completed_by=None, completion_date=None),
+        ]),
+    )]))
+    corpus.append(("paid_media", [make_msg(
+        media=MessageMediaType.PAID_MEDIA,
+        paid_media=SimpleNamespace(stars_amount=50, media=[object(), object()]),
+    )]))
+
+    # --- web_page + poll -------------------------------------------------- #
+    corpus.append(("web_page", [make_msg(
+        text=FakeStr("check this", "check this"),
+        media=MessageMediaType.WEB_PAGE,
+        web_page=SimpleNamespace(type="article", url="https://example.com/a",
+                                 display_url="example.com/a", site_name="Example",
+                                 title="An Article", description="Desc here",
+                                 has_large_media=False,
+                                 photo=SimpleNamespace(file_unique_id="wp1")),
+    )]))
+    corpus.append(("poll", [make_msg(
+        media=MessageMediaType.POLL,
+        poll=SimpleNamespace(
+            question=SimpleNamespace(text="Favourite?", entities=[]),
+            options=[SimpleNamespace(text=SimpleNamespace(text="Red", entities=[])),
+                     SimpleNamespace(text=SimpleNamespace(text="Blue", entities=[]))],
+            description_media=None, explanation_media=None,
+        ),
+    )]))
+
+    # --- Forwards Case 1-5 (drive _format_forward_info) ------------------- #
+    corpus.append(("forward_channel", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="channel",
+                                       chat=SimpleNamespace(id=-100, title="Src Chan", username="srcchan")),
+    )]))
+    corpus.append(("forward_hidden", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="hidden_user", sender_user_name="Hidden Guy"),
+    )]))
+    corpus.append(("forward_user", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="user",
+                                       sender_user=SimpleNamespace(first_name="Ann", last_name="Bee", username="annbee")),
+    )]))
+    corpus.append(("forward_channel_nouser", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="channel", chat_id=-1002, title="NoUserChan"),
+    )]))
+    corpus.append(("forward_other", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="something_else"),
+    )]))
+
+    # --- Reactions: normal / paid / custom -------------------------------- #
+    corpus.append(("reactions", [make_msg(
+        text=FakeStr("react", "react"),
+        reactions=_reactions(
+            SimpleNamespace(emoji="👍", count=5, is_paid=False),
+            SimpleNamespace(count=3, is_paid=True),
+            SimpleNamespace(custom_emoji_id=1234567890, count=2),
+        ),
+    )]))
+
+    # --- sender_chat / from_user author paths ----------------------------- #
+    corpus.append(("sender_chat_author", [make_msg(
+        text=FakeStr("a", "a"),
+        sender_chat=SimpleNamespace(id=-100, title="Sender Chan", username="senderchan"),
+    )]))
+    corpus.append(("from_user_author", [make_msg(
+        text=FakeStr("a", "a"),
+        from_user=SimpleNamespace(first_name="Joe", last_name="Doe", username="joedoe"),
+    )]))
+
+    # --- Multi-message media group (grouping merge) ----------------------- #
+    corpus.append(("media_group", [
+        make_msg(media=MessageMediaType.PHOTO, media_group_id="grp1",
+                 photo=SimpleNamespace(file_unique_id="grp_a"),
+                 caption=FakeStr("group cap", "group cap")),
+        make_msg(media=MessageMediaType.PHOTO, media_group_id="grp1",
+                 photo=SimpleNamespace(file_unique_id="grp_b")),
+    ]))
+
+    return corpus
+
+
+# --------------------------------------------------------------------------- #
+# Rendering harness — patches tg_cache to feed a message list into the real
+# generate_channel_* entry points (the golden-replay path).
+# --------------------------------------------------------------------------- #
+def _render(messages, monkeypatch):
+    async def fake_get_chat_history(client, channel_id, limit=20):
+        return messages
+
+    async def fake_get_chat(client, channel_id):
+        return SimpleNamespace(id=_CHAT.id, title=_CHAT.title, username=_CHAT.username)
+
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_chat_history, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+
+    from rss_generator import generate_channel_html, generate_channel_rss
+    html = asyncio.run(generate_channel_html(_CHANNEL, client=SimpleNamespace(), limit=50))
+    rss = asyncio.run(generate_channel_rss(_CHANNEL, client=SimpleNamespace(), limit=50))
+    return html, gr.normalize_rss(rss)
+
+
+@pytest.mark.parametrize("name,messages", build_corpus(), ids=lambda v: v if isinstance(v, str) else "")
+def test_snapshot_render_parity(name, messages, monkeypatch):
+    gr.pin_environment(monkeypatch)
+
+    # 1) Render the LIVE objects through the real pipeline.
+    live_html, live_rss = _render(messages, monkeypatch)
+
+    # 2) snapshot -> restore, then render the RESTORED objects the same way.
+    restored = restore_messages(snapshot_messages(messages))
+    restored_html, restored_rss = _render(restored, monkeypatch)
+
+    # 3) Byte-identical HTML and RSS prove the snapshot preserved everything the
+    #    renderer read for this case. A dropped allowlist field diverges here.
+    assert restored_html == live_html, f"HTML render diverged for case '{name}'"
+    assert restored_rss == live_rss, f"RSS render diverged for case '{name}'"
+
+
+def test_corpus_covers_all_special_media_types():
+    """Guard: every special-media type from Fix 1 is exercised by the parity corpus."""
+    names = {n for n, _ in build_corpus()}
+    required = {"story_photo", "story_video", "contact", "location", "venue",
+                "dice", "game", "giveaway", "giveaway_winners", "checklist", "paid_media",
+                "live_photo"}
+    assert required <= names

tests/test_stage0_golden.py

@@ -0,0 +1,78 @@
+# flake8: noqa
+# pylint: disable=missing-function-docstring, redefined-outer-name, line-too-long
+"""Stage-0 golden oracle (render-pipeline refactor epic, issue #27/#34).
+
+Regenerates the RSS + HTML feeds for the frozen recorded corpus and asserts
+byte-equality against the committed goldens after the spec's declared normalization
+(strip volatile <lastBuildDate>). Any other byte change = a render regression, and
+this test must catch it. (The stage-0 merged-flags sort normalization was removed in
+stage 2: flag order is now deterministic first-seen order — §3.8.)
+
+Guardrails: this stage only ADDS a loader + goldens + this test. No render/pipeline
+production code is touched; the goldens freeze CURRENT behavior including known bugs
+(fixed in later stages, each referencing a §3 registry item).
+
+Regenerate goldens with:  python -m tests.golden_replay
+"""
+import pytest
+
+from tests import golden_replay as gr
+
+
+@pytest.fixture
+def golden_env(monkeypatch):
+    """Apply the determinism pins (signing key, time_based_merge, DB no-op). TZ=UTC is
+    pinned process-wide in conftest."""
+    gr.pin_environment(monkeypatch)
+    return monkeypatch
+
+
+def _read_golden(channel, kind):
+    with open(gr.golden_path(channel, kind), encoding="utf-8") as f:
+        return f.read()
+
+
+@pytest.mark.parametrize("channel", gr.CORPUS_CHANNELS)
+def test_rss_golden(channel, golden_env):
+    gr.patch_tg_cache(golden_env, channel)
+    actual = gr.capture_rss(channel)
+    expected = _read_golden(channel, "rss")
+    assert gr.normalize_rss(actual) == gr.normalize_rss(expected), \
+        f"RSS feed for {channel} diverged from the golden (render regression)"
+
+
+@pytest.mark.parametrize("channel", gr.CORPUS_CHANNELS)
+def test_html_golden(channel, golden_env):
+    gr.patch_tg_cache(golden_env, channel)
+    actual = gr.capture_html(channel)
+    expected = _read_golden(channel, "html")
+    assert gr.normalize_html(actual) == gr.normalize_html(expected), \
+        f"HTML feed for {channel} diverged from the golden (render regression)"
+
+
+def test_all_goldens_present_and_nonempty():
+    """The corpus and its goldens must stay in lockstep — a missing/empty golden would
+    silently pass the parametrized tests only if a channel were also dropped."""
+    import os
+    for channel in gr.CORPUS_CHANNELS:
+        for kind in ("rss", "html"):
+            path = gr.golden_path(channel, kind)
+            assert os.path.exists(path), f"missing golden: {path}"
+            assert os.path.getsize(path) > 0, f"empty golden: {path}"
+
+
+def test_normalization_preserves_flag_order():
+    """Stage 2 (§3.8): merged-post flags are now emitted in deterministic first-seen
+    order, so normalize_html no longer reorders the flags div — a real flag reordering
+    must reach the byte comparison instead of being masked."""
+    a = '<div class="message-flags"> 🏷 video 🏷 fwd 🏷 link </div>'
+    b = '<div class="message-flags"> 🏷 link 🏷 fwd 🏷 video </div>'
+    assert gr.normalize_html(a) != gr.normalize_html(b)
+    assert gr.normalize_html(a) == a
+
+
+def test_normalization_strips_lastbuilddate():
+    """<lastBuildDate> is the one volatile RSS field (feedgen now() in the constructor)."""
+    a = "<x><lastBuildDate>Mon, 06 Jul 2026 07:32:04 +0000</lastBuildDate><y/></x>"
+    b = "<x><lastBuildDate>Mon, 06 Jul 2026 09:15:59 +0000</lastBuildDate><y/></x>"
+    assert gr.normalize_rss(a) == gr.normalize_rss(b)

tests/test_stage1_hangs.py

@@ -10,18 +10,12 @@ Stage 1 (anti-hang) regression tests:
   worker, and task_done stays balanced so queue.join() completes.
 - Gate cancellation during the spacing wait does not lose the permit.
 """
-import os
-import sys
 import time
 import asyncio
 from types import SimpleNamespace
 
 import pytest
 
-# Add project root to sys.path and mock the config module (same pattern as the other tests).
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
-
 import tg_throttle
 import tg_cache
 from pyrogram import errors

tests/test_stage2_footer.py

@@ -0,0 +1,188 @@
+# flake8: noqa
+# pylint: disable=missing-function-docstring, redefined-outer-name, line-too-long
+# pylint: disable=protected-access, wrong-import-position, import-outside-toplevel
+"""Stage-2 footer tests (render-pipeline refactor epic, issue #29/#34).
+
+Stage 2 removed rss_generator.processed_message_to_tg_message and renders the merged
+footer DIRECTLY from the real main Message. These tests lock the registry §3 items that
+become visible as a consequence:
+
+  §3.6  custom-emoji reactions in the merged footer get a separate "❓ N" span each
+        (no more aggregation into one "❓ N"), matching single posts.
+  §3.7  the merged footer date comes from the naive-local date of the real Message,
+        not a UTC mock — verified with a NON-UTC TZ where the delta is visible (the
+        TZ=UTC golden cannot see it).
+  §3.15 an empty reactions object no longer emits a leading footer separator (single
+        posts too).
+
+Messages are built with the shared SimpleNamespace helper (naive dates where relevant,
+as kurigram emits on prod).
+"""
+import os
+import time
+from datetime import datetime
+from types import SimpleNamespace
+
+import pytest
+
+from pyrogram.enums import MessageMediaType
+from post_parser import PostParser
+from rss_generator import _render_messages_groups
+from tests.test_stage4_eventloop import make_message
+
+
+def _custom_reaction(count, custom_emoji_id):
+    # No `.emoji` attribute -> _reactions_views_links falls to the custom-emoji "❓" branch.
+    return SimpleNamespace(count=count, custom_emoji_id=custom_emoji_id)
+
+
+def _normal_reaction(count, emoji):
+    return SimpleNamespace(count=count, emoji=emoji)
+
+
+def _reactions(*items):
+    return SimpleNamespace(reactions=list(items))
+
+
+def _footer_of(post):
+    """Extract the <div class="message-footer">…</div> inner html from a rendered post."""
+    html = post["html"]
+    marker = '<div class="message-footer">'
+    start = html.index(marker) + len(marker)
+    return html[start:]
+
+
+# --------------------------------------------------------------------------- #
+# §3.6 — custom-emoji reactions render as one span each, in the merged footer.
+# --------------------------------------------------------------------------- #
+def test_3_6_two_custom_emoji_render_as_separate_spans():
+    """Single-post baseline: _reactions_views_links (the function the merged footer now
+    uses on the real Message) emits one '❓ N' span per custom emoji, never aggregated."""
+    parser = PostParser(SimpleNamespace())
+    msg = make_message(70, text="hi")
+    msg.reactions = _reactions(_custom_reaction(4, 111), _custom_reaction(2, 222))
+    html = parser._reactions_views_links(msg)
+    assert html.count('<span class="reaction">❓ 4') == 1
+    assert html.count('<span class="reaction">❓ 2') == 1
+    # NOT aggregated into a single "❓ 6".
+    assert "❓ 6" not in html
+
+
+def test_3_6_merged_footer_keeps_custom_spans_separate():
+    """Merged footer, rendered from the real main Message, keeps a span per custom emoji.
+    The old dict round-trip (_extract_reactions) collapsed both customs into one '❓ 6'."""
+    parser = PostParser(SimpleNamespace())
+    main = make_message(80, text="main text")
+    main.media_group_id = "mg_36"
+    main.reactions = _reactions(
+        _normal_reaction(13, "🔥"),
+        _custom_reaction(4, 111),
+        _custom_reaction(2, 222),
+    )
+    other = make_message(81, text="second part")
+    other.media_group_id = "mg_36"
+
+    posts = _render_messages_groups([[main, other]], parser)
+    assert len(posts) == 1
+    footer = _footer_of(posts[0])
+    assert "merged" in posts[0]["flags"]
+    assert footer.count('<span class="reaction">❓ 4') == 1
+    assert footer.count('<span class="reaction">❓ 2') == 1
+    assert "❓ 6" not in footer  # would appear if the customs were still aggregated
+
+
+# --------------------------------------------------------------------------- #
+# §3.15 — an empty reactions object emits no leading footer separator.
+# --------------------------------------------------------------------------- #
+_SEP = "&nbsp;&nbsp;|&nbsp;&nbsp;"
+
+
+def test_3_15_empty_reactions_no_leading_separator():
+    """A reactions object with an empty list must not produce a leading '…|…' separator.
+    Affects single posts (this call site) as well as merged ones."""
+    parser = PostParser(SimpleNamespace())
+    msg = make_message(90, text="hi")
+    msg.reactions = _reactions()  # object present, no reactions inside
+    html = parser._reactions_views_links(msg)
+    # First visible element must be the views span, not the separator.
+    assert html.startswith('<span class="views">')
+    assert not html.startswith(_SEP)
+
+
+def test_3_15_empty_reactions_single_post_render():
+    """Same via the real render path: the footer's first line starts with views, no
+    leading separator injected by the empty reactions object."""
+    parser = PostParser(SimpleNamespace())
+    msg = make_message(91, text="hi")
+    msg.reactions = _reactions()
+    posts = _render_messages_groups([[msg]], parser)
+    assert len(posts) == 1
+    footer = _footer_of(posts[0])
+    assert f'<br>\n{_SEP}<span class="views">' not in footer
+    assert '<br>\n<span class="views">' in footer
+
+
+def test_3_15_present_reactions_still_render():
+    """Control: a non-empty reactions object still renders its spans (the §3.15 guard is
+    not a blanket drop of the reactions line)."""
+    parser = PostParser(SimpleNamespace())
+    msg = make_message(92, text="hi")
+    msg.reactions = _reactions(_normal_reaction(5, "🔥"))
+    html = parser._reactions_views_links(msg)
+    assert html.startswith('<span class="reaction">🔥 5')
+
+
+# --------------------------------------------------------------------------- #
+# §3.7 — merged footer date == single-post date under a NON-UTC TZ.
+# --------------------------------------------------------------------------- #
+@pytest.fixture
+def moscow_tz():
+    """Pin a non-UTC TZ for the duration of the test, then restore the conftest UTC pin.
+    Without restoring, a leaked non-UTC TZ would corrupt the UTC-pinned golden tests."""
+    os.environ["TZ"] = "Europe/Moscow"
+    time.tzset()
+    try:
+        yield
+    finally:
+        os.environ["TZ"] = "UTC"
+        time.tzset()
+
+
+def _date_span(footer):
+    marker = '<span class="date">'
+    start = footer.index(marker) + len(marker)
+    end = footer.index("</span>", start)
+    return footer[start:end]
+
+
+def test_3_7_merged_footer_date_matches_single_post_non_utc(moscow_tz):
+    """On a TZ≠UTC server the merged footer date must equal the single-post date for the
+    same message. Before stage 2 the merged path built a UTC mock date (from the naive
+    date's timestamp) → a visible offset; now both render the real naive-local date."""
+    parser = PostParser(SimpleNamespace())
+    # NAIVE date, as kurigram emits on prod. Under Europe/Moscow the old UTC mock would
+    # have shifted this by the TZ offset.
+    naive_date = datetime(2026, 5, 5, 17, 21, 14)
+
+    def _fresh_main():
+        m = make_message(100, text="main text", date=naive_date)
+        return m
+
+    # Single post.
+    single_posts = _render_messages_groups([[_fresh_main()]], parser)
+    assert len(single_posts) == 1
+    single_date = _date_span(_footer_of(single_posts[0]))
+
+    # Merged group with the same main message + one extra member.
+    main = _fresh_main()
+    main.media_group_id = "mg_37"
+    other = make_message(101, text="second part", date=naive_date)
+    other.media_group_id = "mg_37"
+    merged_posts = _render_messages_groups([[main, other]], parser)
+    assert len(merged_posts) == 1
+    assert "merged" in merged_posts[0]["flags"]
+    merged_date = _date_span(_footer_of(merged_posts[0]))
+
+    assert merged_date == single_date
+    # And it is the naive wall-clock time, not a UTC-shifted one.
+    assert merged_date == "05/05/26, 17:21:14"

tests/test_stage2_static.py

@@ -18,17 +18,12 @@ Covers the four scenarios from the plan plus the subtle in-flight-dedup lifecycl
   fresh partials and non-temp files.
 """
 import os
-import sys
 import time
 import asyncio
 from types import SimpleNamespace
 
 import pytest
 
-# Add project root to sys.path and mock the config module (same pattern as the other tests).
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
-
 import api_server
 from pyrogram import errors
 from pyrogram.enums import MessageMediaType
@@ -116,6 +111,7 @@ async def test_download_atomic_race_loser_keeps_existing_final(monkeypatch, tmp_
 # --------------------------------------------------------------------------- #
 async def test_concurrent_large_video_no_partial_served(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)  # download_media_file writes under ./data/cache
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
 
     msg = SimpleNamespace(
         media=MessageMediaType.VIDEO,

tests/test_stage3_enrich.py

@@ -0,0 +1,289 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, line-too-long
+"""
+Stage 3 (render-pipeline refactor epic, issue #30/#34) — reply-enrichment lock.
+
+Stage 3 merges generate_channel_rss/generate_channel_html around the shared
+_prepare_feed_posts and moves _reply_enrichment (the live client.get_messages
+reply-target fetch) INTO that shared path (enrich_replies=True for HTML,
+False for RSS).
+
+The stage-0 golden oracle CANNOT see this move: its harness replays the corpus
+through monkeypatched tg_cache but leaves client.get_messages a no-op (the client
+is a bare SimpleNamespace, so get_messages raises AttributeError and enrichment
+silently no-ops). So enrichment is locked here instead, exactly as mandated on #30:
+
+  (a) get_messages is BATCHED by chat_id (one call per chat, not one per reply);
+  (b) the fetched reply target is set onto the right source message by
+      (chat_id, message.id);
+  (c) the enriched reply block actually renders in the HTML feed.
+
+Plus: RSS deliberately does NOT enrich (keeps polling cheap) — locked too.
+"""
+from types import SimpleNamespace
+from datetime import datetime, timezone
+
+import pytest
+
+from pyrogram import errors
+
+import rss_generator as rss_module
+from rss_generator import _reply_enrichment, generate_channel_html, generate_channel_rss
+
+
+class _Str(str):
+    """Stand-in for Pyrogram's Str: .html returns the raw string unchanged, so a
+    reply-target text reaches the pre-sanitize body like real entity text would."""
+    @property
+    def html(self):
+        return str(self)
+
+
+def _reply_target(mid, text):
+    """A resolved reply-to-message as _format_reply_info consumes it."""
+    return SimpleNamespace(id=mid, text=text, caption=None, sender_chat=None)
+
+
+def make_message(mid, chat_id=-1001234567890, username="testchan", text="post",
+                 reply_to_message_id=None, reply_to_message=None, date=None):
+    m = SimpleNamespace()
+    m.id = mid
+    m.date = date or datetime(2024, 1, 1, 12, 0, mid % 60, tzinfo=timezone.utc)
+    m.text = _Str(text) if text is not None else None
+    m.caption = None
+    m.media = None
+    m.web_page = None
+    m.poll = None
+    m.service = None
+    m.forward_origin = None
+    m.reply_to_message = reply_to_message
+    m.reply_to_message_id = reply_to_message_id
+    m.sender_chat = None
+    m.from_user = None
+    m.reactions = None
+    m.views = 100
+    m.media_group_id = None
+    m.show_caption_above_media = False
+    m.chat = SimpleNamespace(id=chat_id, username=username)
+    for attr in ("photo", "video", "document", "audio", "voice",
+                 "video_note", "animation", "sticker"):
+        setattr(m, attr, None)
+    return m
+
+
+class RecordingClient:
+    """Fake Telegram client: records each get_messages(chat_id, ids) call and returns,
+    for every requested id, a 'full' message carrying a resolved .reply_to_message."""
+    def __init__(self, target_text_for=lambda chat_id, mid: f"TARGET_{chat_id}_{mid}"):
+        self.calls = []                 # list[(chat_id, list[ids])]
+        self._target_text_for = target_text_for
+
+    async def get_messages(self, chat_id, ids):
+        self.calls.append((chat_id, list(ids)))
+        out = []
+        for mid in ids:
+            full = SimpleNamespace(
+                id=mid,
+                empty=False,
+                reply_to_message=_reply_target(mid + 5000, self._target_text_for(chat_id, mid)),
+            )
+            out.append(full)
+        return out
+
+
+# --------------------------------------------------------------------------- #
+# (a) batching by chat_id  +  (b) target set onto the right source message.
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_reply_enrichment_batches_by_chat_and_sets_target():
+    CHAT_A, CHAT_B = -100111, -100222
+    messages = [
+        make_message(10, chat_id=CHAT_A, reply_to_message_id=1),
+        make_message(11, chat_id=CHAT_A, reply_to_message_id=2),
+        make_message(12, chat_id=CHAT_A, reply_to_message_id=3),
+        make_message(20, chat_id=CHAT_B, reply_to_message_id=4),
+        make_message(21, chat_id=CHAT_B, reply_to_message_id=5),
+        make_message(30, chat_id=CHAT_A, reply_to_message_id=None),  # no reply -> not fetched
+    ]
+    client = RecordingClient()
+
+    result = await _reply_enrichment(client, messages)
+
+    # (a) BATCHED: exactly one call per chat_id (2), NOT one per reply (5).
+    assert len(client.calls) == 2, f"expected 2 batched calls, got {client.calls}"
+    calls_by_chat = dict(client.calls)
+    assert calls_by_chat[CHAT_A] == [10, 11, 12], "chat A ids not batched into one call"
+    assert calls_by_chat[CHAT_B] == [20, 21], "chat B ids not batched into one call"
+
+    # (b) each fetched target is set onto the SOURCE message keyed by (chat_id, id).
+    by_id = {m.id: m for m in result}
+    for mid, chat in [(10, CHAT_A), (11, CHAT_A), (12, CHAT_A), (20, CHAT_B), (21, CHAT_B)]:
+        assert by_id[mid].reply_to_message is not None, f"message {mid} not enriched"
+        assert by_id[mid].reply_to_message.text == f"TARGET_{chat}_{mid}", \
+            f"message {mid} got the wrong reply target"
+    # The reply-less message is untouched.
+    assert by_id[30].reply_to_message is None
+
+
+@pytest.mark.asyncio
+async def test_reply_enrichment_no_replies_makes_no_calls():
+    messages = [make_message(1), make_message(2)]  # no reply_to_message_id
+    client = RecordingClient()
+    await _reply_enrichment(client, messages)
+    assert client.calls == [], "get_messages must not be called when nothing needs enrichment"
+
+
+# --------------------------------------------------------------------------- #
+# (c) the enriched reply block renders in the HTML feed — AND enrichment now runs
+#     inside the shared _prepare_feed_posts path (enrich_replies=True for HTML).
+# --------------------------------------------------------------------------- #
+def _patch_feed_source(monkeypatch, messages):
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return messages
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+
+@pytest.mark.asyncio
+async def test_html_feed_renders_enriched_reply_block(monkeypatch):
+    MARKER = "ENRICHED_REPLY_MARKER_XYZ"
+    # A shallow message: it has a reply_to_message_id but NO resolved reply_to_message
+    # yet — enrichment must fetch and fill it before render.
+    msg = make_message(77, chat_id=-1001234567890, reply_to_message_id=70,
+                       reply_to_message=None, text="body text")
+    _patch_feed_source(monkeypatch, [msg])
+
+    client = RecordingClient(target_text_for=lambda chat_id, mid: MARKER)
+    html = await generate_channel_html("testchan", client=client, limit=5)
+
+    # Enrichment ran inside _prepare_feed_posts and was BATCHED (one call for the chat).
+    assert client.calls == [(-1001234567890, [77])], f"enrichment not run/batched: {client.calls}"
+    # The resolved reply target renders as a reply block carrying the marker text.
+    assert '<div class="message-reply">' in html, "reply block not rendered"
+    assert MARKER in html, "resolved reply-target text missing from the rendered feed"
+
+
+@pytest.mark.asyncio
+async def test_rss_feed_does_not_enrich_replies(monkeypatch):
+    # RSS deliberately skips enrichment (enrich_replies=False) to keep polling cheap.
+    msg = make_message(88, chat_id=-1001234567890, reply_to_message_id=80,
+                       reply_to_message=None, text="body text")
+    _patch_feed_source(monkeypatch, [msg])
+
+    client = RecordingClient()
+    await generate_channel_rss("testchan", client=client, limit=5)
+    assert client.calls == [], "RSS must NOT call get_messages (enrich_replies=False)"
+
+
+# --------------------------------------------------------------------------- #
+# History over-fetch fork: RSS over-fetches limit*2 (headroom for grouping/trim),
+# HTML fetches exactly limit. This is a REAL behavioral fork that the golden oracle
+# CANNOT see (its fake_get_chat_history ignores the limit kwarg and returns the whole
+# corpus, so both paths trim to GOLDEN_LIMIT identically). A refactor that accidentally
+# unified the two would pass every golden test — so lock the forwarded limit here.
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_rss_overfetches_history_html_does_not(monkeypatch):
+    N = 7
+    seen = {}
+
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def spy_get_history(client, channel, limit=20):
+        seen.setdefault("limits", []).append(limit)
+        return []
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", spy_get_history, raising=False)
+
+    await generate_channel_rss("testchan", client=SimpleNamespace(), limit=N)
+    assert seen["limits"] == [2 * N], f"RSS must over-fetch history limit*2, got {seen['limits']}"
+
+    seen["limits"] = []
+    await generate_channel_html("testchan", client=SimpleNamespace(), limit=N)
+    assert seen["limits"] == [N], f"HTML must fetch history limit (no over-fetch), got {seen['limits']}"
+
+
+# --------------------------------------------------------------------------- #
+# Shared error handling in _prepare_feed_posts, verified through BOTH formatters.
+# The golden oracle cannot see these paths (§3.9/§3.10 are error branches), so they
+# are locked here. FEED_FUNCS lets each case assert RSS and HTML behave identically.
+# --------------------------------------------------------------------------- #
+FEED_FUNCS = [generate_channel_rss, generate_channel_html]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("feed_func", FEED_FUNCS)
+async def test_channel_not_found_returns_error_feed(monkeypatch, feed_func):
+    async def raise_not_found(client, channel):
+        raise errors.UsernameNotOccupied("no such user")
+
+    async def fake_get_history(client, channel, limit=20):
+        return []
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", raise_not_found, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    out = await feed_func("ghostchan", client=SimpleNamespace(), limit=5)
+    # ChannelNotFound -> create_error_feed (RSS-XML error feed) in both paths.
+    assert "does not exist" in out, f"{feed_func.__name__} did not return the error feed"
+    assert "ghostchan" in out
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("feed_func", FEED_FUNCS)
+async def test_floodwait_from_get_chat_propagates(monkeypatch, feed_func):
+    async def raise_flood(client, channel):
+        raise errors.FloodWait(value=11)
+
+    async def fake_get_history(client, channel, limit=20):
+        return []
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", raise_flood, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    # FloodWait must reach api_server unwrapped (mapped there to HTTP 429), NOT ValueError.
+    with pytest.raises(errors.FloodWait):
+        await feed_func("floodchan", client=SimpleNamespace(), limit=5)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("feed_func", FEED_FUNCS)
+async def test_floodwait_from_history_propagates(monkeypatch, feed_func):
+    # Registry §3.9 (the fix this stage lands): FloodWait raised while fetching HISTORY
+    # must propagate -> HTTP 429. Before stage 3 it fell into `except Exception` and was
+    # wrapped in ValueError -> HTTP 400. The golden cannot see this; locked here.
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def raise_flood_history(client, channel, limit=20):
+        raise errors.FloodWait(value=13)
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", raise_flood_history, raising=False)
+
+    with pytest.raises(errors.FloodWait):
+        await feed_func("testchan", client=SimpleNamespace(), limit=5)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("feed_func", FEED_FUNCS)
+async def test_other_history_error_becomes_valueerror(monkeypatch, feed_func):
+    # Any NON-FloodWait history failure is still wrapped in ValueError (api_server -> 400).
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def raise_runtime(client, channel, limit=20):
+        raise RuntimeError("history backend exploded")
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", raise_runtime, raising=False)
+
+    with pytest.raises(ValueError):
+        await feed_func("testchan", client=SimpleNamespace(), limit=5)

tests/test_stage3_fileresponse.py

@@ -36,16 +36,11 @@ All 206 responses that carry data return byte-for-byte identical slices old vs n
 real regression is hidden behind an "accepted difference".
 """
 import os
-import sys
 import time
 import logging
 
 import pytest
 
-# Add project root to sys.path and mock the config module (same pattern as the other tests).
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
-
 from fastapi import FastAPI, Request
 from fastapi.testclient import TestClient
 

tests/test_stage4_eventloop.py

@@ -16,10 +16,7 @@ Covers:
 - 4.4 sanitize coverage (XSS): a <script> / onerror= / javascript: payload is stripped in
       ALL outputs — rss, html-feed, single-post html, and json — each with exactly one pass.
 """
-import os
 import re
-import sys
-import copy
 import pickle
 import asyncio
 import threading
@@ -28,9 +25,6 @@ from types import SimpleNamespace
 
 import pytest
 
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
-
 from pyrogram.enums import MessageMediaType
 
 import post_parser as pp_module
@@ -40,9 +34,8 @@ from rss_generator import (
     generate_channel_rss,
     generate_channel_html,
     _render_pipeline,
-    _create_time_based_media_groups,
+    _compute_time_based_group_ids,
     _create_messages_groups,
-    _trim_messages_groups,
     _render_messages_groups,
 )
 
@@ -97,8 +90,10 @@ def _co_names(func):
 # ---------------------------------------------------------------------------
 
 def test_render_functions_are_sync():
-    for fn in (_create_time_based_media_groups, _create_messages_groups,
-               _trim_messages_groups, _render_messages_groups, _render_pipeline):
+    # _trim_messages_groups was inlined into _render_pipeline as a `[:limit]` slice
+    # (render-pipeline cosmetics stage); the trimming path is now covered via _render_pipeline.
+    for fn in (_compute_time_based_group_ids, _create_messages_groups,
+               _render_messages_groups, _render_pipeline):
         assert not asyncio.iscoroutinefunction(fn), f"{fn.__name__} must be a plain sync function"
 
 
@@ -129,15 +124,27 @@ async def test_pipeline_runs_in_worker_thread(monkeypatch):
     assert seen["ident"] != main_ident, "render pipeline must run in a worker thread, not the loop thread"
 
 
-def test_deepcopy_of_pickled_message_does_not_crash():
+def test_time_clustering_does_not_mutate_pickled_message():
+    # Stage 4: _create_time_based_media_groups (which deep-copied the cached list and
+    # MUTATED media_group_id) is gone. Time-clustering is now a PURE mapping function; a
+    # pickled Message straight from the cache must come out untouched.
     from pyrogram.types import Message, Chat
     from pyrogram.enums import ChatType
-    m = Message(id=7, date=datetime.now(timezone.utc), text="hello",
-                chat=Chat(id=-1001, type=ChatType.CHANNEL, username="testchan"))
-    roundtripped = pickle.loads(pickle.dumps(m))   # mimics the pickle cache
-    clone = copy.deepcopy(roundtripped)            # what _create_time_based_media_groups does
-    assert clone.id == 7
-    assert clone.chat.username == "testchan"
+    base = datetime(2024, 1, 1, 12, 0, 0)  # naive, as kurigram emits
+    a = pickle.loads(pickle.dumps(Message(
+        id=7, date=base, text="hello", media_group_id="orig_A",
+        chat=Chat(id=-1001, type=ChatType.CHANNEL, username="testchan"))))
+    b = pickle.loads(pickle.dumps(Message(
+        id=8, date=base.replace(second=2), text="world", media_group_id=None,
+        chat=Chat(id=-1001, type=ChatType.CHANNEL, username="testchan"))))
+
+    mapping = _compute_time_based_group_ids([a, b], merge_seconds=5)
+
+    # The two adjacent posts are clustered under the first truthy id, but only via the
+    # RETURNED mapping — the input objects keep their original media_group_id.
+    assert mapping == {7: "orig_A", 8: "orig_A"}
+    assert a.media_group_id == "orig_A"
+    assert b.media_group_id is None
 
 
 # ---------------------------------------------------------------------------
@@ -147,8 +154,8 @@ def test_deepcopy_of_pickled_message_does_not_crash():
 def test_render_path_has_no_asyncio_side_effects():
     banned = {"create_task", "get_running_loop", "to_thread", "ensure_future"}
     funcs = [
-        _render_pipeline, _create_time_based_media_groups, _create_messages_groups,
-        _trim_messages_groups, _render_messages_groups,
+        _render_pipeline, _compute_time_based_group_ids, _create_messages_groups,
+        _render_messages_groups,
         PostParser.process_message, PostParser._generate_html_body,
         PostParser._generate_html_media, PostParser.generate_html_footer,
         PostParser._reactions_views_links, PostParser._save_media_file_ids,
@@ -289,6 +296,27 @@ async def test_xss_stripped_in_single_post_html_debug():
     assert "&lt;script&gt;" in html_out, "debug raw dump was not html-escaped"
 
 
+@pytest.mark.asyncio
+async def test_xss_in_title_escaped_in_debug_html():
+    # Issue #13: the debug branch of _format_html embeds data["html"]["title"], which is
+    # generated from user-controlled content and never passes through bleach. A poll whose
+    # question carries a <script> tag yields the title "📊 Poll: <script>alert(1)</script>"
+    # verbatim — it must be html-escaped before being embedded in the debug output.
+    msg = make_message(25, media=MessageMediaType.POLL)
+    msg.poll = SimpleNamespace(question="<script>alert(1)</script>")
+    parser = PostParser(_make_client_returning(msg))
+    html_out = await parser.get_post("testchan", 25, "html", debug=True)
+
+    # No live <script> tag anywhere in the output (title div, body, footer, raw <pre>).
+    assert "<script>" not in html_out, "debug html left a live <script> tag"
+    # The title div itself carries the payload only in escaped form (proving html.escape
+    # ran on the title, not just on the raw_message <pre> dump).
+    title_lines = [line for line in html_out.split("\n") if 'class="title"' in line]
+    assert title_lines, "debug output must contain the title div"
+    assert "&lt;script&gt;alert(1)&lt;/script&gt;" in title_lines[0], \
+        "title was not html-escaped in debug output"
+
+
 @pytest.mark.asyncio
 async def test_xss_stripped_in_rss_feed(monkeypatch):
     async def fake_get_chat(client, channel):
@@ -367,6 +395,56 @@ async def test_xss_in_media_caption_stripped_in_feeds(monkeypatch):
     _assert_clean(html_feed, "html feed (media caption)")
 
 
+# --------------------------------------------------------------------------- #
+# §3.4 — per-post sanitize ISOLATION (stage-1 load-bearing invariant, PR #36).
+# A dangling/unbalanced tag in post A must be normalized WITHIN A's own fragment and
+# cannot swallow post B (cross-post DOM/XSS bleed). This holds because _render_pipeline
+# runs a SEPARATE bleach.clean per post and the formatter joins the <hr> divider AFTER
+# sanitize. A FUTURE stage that rejoins BEFORE sanitizing would reintroduce the bleed
+# and pass every single-message XSS test — so this 2-post case MUST turn it red.
+# --------------------------------------------------------------------------- #
+@pytest.mark.asyncio
+async def test_unbalanced_post_does_not_swallow_next_post_html_feed(monkeypatch):
+    # Post A carries a dangling <div><b> with NO closers; post B is well-formed and
+    # carries a unique marker. _Str.html feeds the raw tags into the pre-sanitize body
+    # exactly as a real message with entities would.
+    msg_a = make_message(50, text="<div><b>DANGLING_A no closers here",
+                         date=datetime(2024, 1, 1, 12, 5, 0, tzinfo=timezone.utc))
+    msg_b = make_message(51, text="INTACT_B_MARKER",
+                         date=datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc))
+
+    async def fake_get_chat(client, channel):
+        return SimpleNamespace(title="Test", username="testchan", id=-1001234567890)
+
+    async def fake_get_history(client, channel, limit=20):
+        return [msg_a, msg_b]  # A is newer than B -> A rendered first
+
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
+
+    html = await generate_channel_html("testchan", client=SimpleNamespace(), limit=5)
+
+    # The divider survives (joined AFTER per-post sanitize) and splits the feed into
+    # exactly two TOP-LEVEL fragments. A join-before-sanitize regression strips the
+    # non-whitelisted <hr> entirely (strip=True) -> this alone already goes red.
+    parts = html.split('<hr class="post-divider">')
+    assert len(parts) == 2, "expected exactly one top-level post-divider between the two posts"
+    frag_a, frag_b = parts
+
+    # A's dangling tags were balanced WITHIN A's own fragment, NOT deferred past the
+    # divider. A rejoin-before-sanitize regression closes them only at the very end of
+    # the whole feed (after B), leaving frag_a with more opens than closes.
+    assert "DANGLING_A" in frag_a
+    assert frag_a.count("<div") == frag_a.count("</div>"), "post A's <div> not closed within its own fragment"
+    assert frag_a.count("<b>") == frag_a.count("</b>"), "post A's <b> not closed within its own fragment"
+
+    # B is intact, lives at top level, and is itself balanced — never nested inside A
+    # (its marker must not have been trapped before A's divider).
+    assert "INTACT_B_MARKER" in frag_b, "post B content missing/swallowed by post A"
+    assert "INTACT_B_MARKER" not in frag_a, "post B content bled into post A's fragment"
+    assert frag_b.count("<div") == frag_b.count("</div>"), "post B fragment not self-contained"
+
+
 # --------------------------------------------------------------------------- #
 # Review round-1: the new bulk-upsert SQL executed for real (not mocked).
 # --------------------------------------------------------------------------- #
@@ -452,10 +530,13 @@ async def test_rss_fails_closed_when_sanitizer_raises(monkeypatch):
     monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_history, raising=False)
 
     # Force bleach to blow up (e.g. the RecursionError class already seen in prod).
-    # rss_generator imports it as `from bleach import clean as HTMLSanitizer`.
+    # API relocation: the single bleach config now lives in sanitizer.py, which imports
+    # it as `from bleach import clean as HTMLSanitizer`; sanitize_html resolves the name
+    # at call time, so patching sanitizer.HTMLSanitizer triggers the fail-closed path.
+    import sanitizer as sanitizer_module
     def boom(*a, **k):
         raise RecursionError("bleach exploded")
-    monkeypatch.setattr(rss_module, "HTMLSanitizer", boom, raising=True)
+    monkeypatch.setattr(sanitizer_module, "HTMLSanitizer", boom, raising=True)
 
     rss = await generate_channel_rss("testchan", client=SimpleNamespace(), limit=5)
     chunks = re.findall(r"<!\[CDATA\[(.*?)\]\]>", rss, re.DOTALL)

tests/test_stage5_media_table.py

@@ -0,0 +1,238 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, line-too-long
+"""Stage-5 media table tests (render-pipeline refactor epic, issue #32/#34).
+
+Two layers:
+  * 5a — the MEDIA_SOURCES table + renderers reproduce the three old ladders
+    (_get_file_unique_id, _save_media_file_ids, _generate_html_media). The fragment
+    snapshot (tests/test_data/media_fragments.json) is the PRE-REFACTOR BASE reference,
+    captured against the base post_parser.py; the 5a code reproduces it byte-for-byte
+    (two fragments carry registered §3.14 deltas — see fr.REGISTERED_DELTAS). Plus
+    table/selector/invariant unit tests.
+  * 5b — the registered fixes (§3.13 large-file guard for every object, §3.14 the
+    message-media div is closed in every branch) live in test_stage5b_media_fixes.py.
+
+The cross-module invariant test pins the boundary with api_server.find_file_id_in_message:
+every object a selector returns must be resolvable there by its file_unique_id, or a new
+table entry would mint a /media URL the download path answers with 404.
+"""
+from types import SimpleNamespace
+
+import pytest
+
+from pyrogram.enums import MessageMediaType
+
+from post_parser import (
+    PostParser, MEDIA_SOURCES, RENDERERS, RenderCtx,
+    _select_document, _select_sticker, _select_story, _select_poll_media,
+)
+from api_server import find_file_id_in_message
+from url_signer import KeyManager
+
+from tests import media_fragment_replay as fr
+
+
+@pytest.fixture(autouse=True)
+def _pin_signing_key(monkeypatch):
+    # Deterministic media-URL digests regardless of checkout / cwd.
+    monkeypatch.setattr(KeyManager, "signing_key", fr.FRAGMENT_SIGNING_KEY)
+
+
+@pytest.fixture
+def parser():
+    return PostParser(SimpleNamespace())
+
+
+# --------------------------------------------------------------------------- #
+# 1. Fragment-level snapshot oracle — the 5a byte-for-byte contract.
+# --------------------------------------------------------------------------- #
+@pytest.mark.parametrize("case_name", sorted(fr.build_cases().keys()))
+def test_media_fragment_matches_snapshot(parser, case_name):
+    """_generate_html_media (render ladder + _get_file_unique_id ladder) and the
+    _save_media_file_ids collection ladder reproduce the frozen pre-refactor bytes."""
+    snapshot = fr.load_snapshot()
+    factory = fr.build_cases()[case_name]
+    parser._pending_media_ids = []
+    html = parser._generate_html_media(factory())
+    collected = [[c, p, f] for c, p, f, _ in parser._pending_media_ids]
+    # The snapshot is the pre-refactor BASE reference; the two §3.14 fragments carry a
+    # registered 5b delta (the now-closed message-media div), so expect their 5b bytes.
+    expected = fr.REGISTERED_DELTAS.get(case_name, snapshot[case_name])
+    assert html == expected["html"], f"fragment HTML diverged for {case_name}"
+    assert collected == expected["collected"], f"collection diverged for {case_name}"
+
+
+def test_snapshot_covers_every_spec_case():
+    """Guard: the snapshot and the case builders stay in lockstep (a dropped case
+    would otherwise silently reduce coverage)."""
+    snapshot = fr.load_snapshot()
+    assert set(snapshot.keys()) == set(fr.build_cases().keys())
+    # The spec §5a "Шаг 0" edge branches must all be present.
+    for required in ("photo", "video", "animation", "video_note", "audio_default_mime",
+                     "voice_default_mime", "sticker_img", "sticker_video",
+                     "document_pdf_public", "document_pdf_private", "document_normal",
+                     "live_photo", "story_video", "story_photo", "poll_media_img",
+                     "poll_media_video", "paid_media", "webpage_with_photo",
+                     "webpage_without_photo", "webpage_photo_long_text",
+                     "file_unique_id_none", "channel_username_none"):
+        assert required in snapshot, f"missing spec fragment case: {required}"
+
+
+# --------------------------------------------------------------------------- #
+# 2. Table structure: every render kind has a renderer; the only kind=None entry
+#    is WEB_PAGE; PAID_MEDIA has no entry.
+# --------------------------------------------------------------------------- #
+def test_every_render_kind_has_a_renderer():
+    # Fixed-kind lambda entries (branchy selectors are covered by the selector unit
+    # tests below and the fragment oracle).
+    static = {
+        MessageMediaType.PHOTO: 'img_400',
+        MessageMediaType.VIDEO: 'video_400',
+        MessageMediaType.ANIMATION: 'video_400',
+        MessageMediaType.VIDEO_NOTE: 'video_400',
+        MessageMediaType.AUDIO: 'audio',
+        MessageMediaType.VOICE: 'audio',
+        MessageMediaType.LIVE_PHOTO: 'video_loop_400',
+    }
+    for mt, expected_kind in static.items():
+        assert expected_kind in RENDERERS, f"{mt} kind {expected_kind} has no renderer"
+    # Selector-produced kinds (document/sticker/story/poll) all resolve to renderers
+    # or, for WEB_PAGE, to None.
+    for kind in ('img_400', 'video_400', 'audio', 'pdf', 'video_loop_200',
+                 'img_200_sticker', 'video_loop_400'):
+        assert kind in RENDERERS
+
+
+def test_web_page_is_the_only_kind_none_entry():
+    assert MEDIA_SOURCES[MessageMediaType.WEB_PAGE](SimpleNamespace(web_page=None))[1] is None
+
+
+def test_paid_media_has_no_table_entry():
+    assert MessageMediaType.PAID_MEDIA not in MEDIA_SOURCES
+
+
+# --------------------------------------------------------------------------- #
+# 3. Selector unit tests (the branchy selectors).
+# --------------------------------------------------------------------------- #
+def test_select_document_pdf_vs_image():
+    pdf = SimpleNamespace(document=SimpleNamespace(mime_type="application/pdf", file_unique_id="d1"))
+    png = SimpleNamespace(document=SimpleNamespace(mime_type="image/png", file_unique_id="d2"))
+    assert _select_document(pdf)[1] == 'pdf'
+    assert _select_document(png)[1] == 'img_400'
+    assert _select_document(pdf)[0] is pdf.document
+
+
+def test_select_sticker_video_vs_image():
+    vid = SimpleNamespace(sticker=SimpleNamespace(is_video=True, file_unique_id="s1"))
+    img = SimpleNamespace(sticker=SimpleNamespace(is_video=False, file_unique_id="s2"))
+    assert _select_sticker(vid)[1] == 'video_loop_200'
+    assert _select_sticker(img)[1] == 'img_200_sticker'
+
+
+def test_select_story_maps_helper_kind():
+    vid = SimpleNamespace(story=SimpleNamespace(video=SimpleNamespace(file_unique_id="v"), photo=None))
+    pic = SimpleNamespace(story=SimpleNamespace(video=None, photo=SimpleNamespace(file_unique_id="p")))
+    none = SimpleNamespace(story=None)
+    assert _select_story(vid)[1] == 'video_400'
+    assert _select_story(pic)[1] == 'img_400'
+    assert _select_story(none) == (None, None)
+
+
+def test_select_poll_media_maps_helper_kind():
+    img = SimpleNamespace(poll=SimpleNamespace(description_media=SimpleNamespace(photo=SimpleNamespace(file_unique_id="p"))))
+    vid = SimpleNamespace(poll=SimpleNamespace(description_media=SimpleNamespace(video=SimpleNamespace(file_unique_id="v"))))
+    none = SimpleNamespace(poll=None)
+    assert _select_poll_media(img)[1] == 'img_400'
+    assert _select_poll_media(vid)[1] == 'video_400'
+    assert _select_poll_media(none) == (None, None)
+
+
+# --------------------------------------------------------------------------- #
+# 4. Renderer byte structure (audio emits two items; pdf emits its two-append block).
+# --------------------------------------------------------------------------- #
+def test_audio_renderer_emits_tag_and_br():
+    out = RENDERERS['audio'](RenderCtx(url="U", mime="audio/mpeg"))
+    assert len(out) == 2 and out[1] == '<br>'
+    assert 'type="audio/mpeg"' in out[0]
+
+
+def test_pdf_renderer_emits_two_item_block():
+    out = RENDERERS['pdf'](RenderCtx(url="U", tg_link="https://t.me/x/1"))
+    assert len(out) == 2
+    assert out[0] == '<div class="document-pdf" style="padding: 10px;">'
+    assert out[1] == '<a href="https://t.me/x/1" target="_blank">[PDF-файл]</a></div>'
+
+
+# --------------------------------------------------------------------------- #
+# 5. Cross-module invariant: every selector object is resolvable in api_server.
+# --------------------------------------------------------------------------- #
+def _invariant_messages():
+    """One mock per MEDIA_SOURCES type whose selected object carries a real
+    file_unique_id/file_id, so find_file_id_in_message can resolve it."""
+    def base(**extra):
+        m = SimpleNamespace(id=1, chat=SimpleNamespace(id=-100, username="c"), web_page=None, poll=None)
+        for a in ("photo", "video", "document", "audio", "voice", "video_note",
+                  "animation", "sticker"):
+            setattr(m, a, None)
+        for k, v in extra.items():
+            setattr(m, k, v)
+        return m
+
+    media = SimpleNamespace(file_unique_id="UID", file_id="FID")
+    cases = {
+        MessageMediaType.PHOTO: base(media=MessageMediaType.PHOTO, photo=media),
+        MessageMediaType.VIDEO: base(media=MessageMediaType.VIDEO, video=media),
+        MessageMediaType.ANIMATION: base(media=MessageMediaType.ANIMATION, animation=media),
+        MessageMediaType.VIDEO_NOTE: base(media=MessageMediaType.VIDEO_NOTE, video_note=media),
+        MessageMediaType.AUDIO: base(media=MessageMediaType.AUDIO, audio=media),
+        MessageMediaType.VOICE: base(media=MessageMediaType.VOICE, voice=media),
+        MessageMediaType.DOCUMENT: base(media=MessageMediaType.DOCUMENT,
+                                        document=SimpleNamespace(mime_type="image/png", file_unique_id="UID", file_id="FID")),
+        MessageMediaType.STICKER: base(media=MessageMediaType.STICKER,
+                                       sticker=SimpleNamespace(is_video=False, file_unique_id="UID", file_id="FID")),
+        MessageMediaType.LIVE_PHOTO: base(media=MessageMediaType.LIVE_PHOTO, live_photo=media),
+        MessageMediaType.STORY: base(media=MessageMediaType.STORY,
+                                     story=SimpleNamespace(video=media, photo=None)),
+        MessageMediaType.POLL: base(media=MessageMediaType.POLL,
+                                    poll=SimpleNamespace(description_media=SimpleNamespace(photo=media))),
+        MessageMediaType.WEB_PAGE: base(media=MessageMediaType.WEB_PAGE,
+                                        web_page=SimpleNamespace(photo=media)),
+    }
+    return cases
+
+
+@pytest.mark.parametrize("media_type", list(MEDIA_SOURCES.keys()))
+async def test_selector_object_is_resolvable_in_api_server(media_type):
+    """The object MEDIA_SOURCES selects is found by api_server.find_file_id_in_message
+    via its file_unique_id — the /media download path can always resolve a URL the
+    table produced (spec §5a inter-module invariant)."""
+    message = _invariant_messages()[media_type]
+    selected_obj, _kind = MEDIA_SOURCES[media_type](message)
+    file_unique_id = getattr(selected_obj, "file_unique_id", None)
+    assert file_unique_id, f"{media_type} selector returned no usable file_unique_id"
+    resolved = await find_file_id_in_message(message, file_unique_id)
+    assert resolved == getattr(selected_obj, "file_id", None), \
+        f"{media_type}: selected object not resolvable in find_file_id_in_message"
+
+
+def test_media_sources_covers_all_old_ladder_types():
+    """Every type the pre-refactor _get_file_unique_id dict handled is in the table."""
+    old_types = {
+        MessageMediaType.PHOTO, MessageMediaType.VIDEO, MessageMediaType.DOCUMENT,
+        MessageMediaType.AUDIO, MessageMediaType.VOICE, MessageMediaType.VIDEO_NOTE,
+        MessageMediaType.ANIMATION, MessageMediaType.STICKER, MessageMediaType.WEB_PAGE,
+        MessageMediaType.LIVE_PHOTO, MessageMediaType.STORY, MessageMediaType.POLL,
+    }
+    assert old_types <= set(MEDIA_SOURCES.keys())
+
+
+# --------------------------------------------------------------------------- #
+# 6. /flags endpoint constraint: flags.append(...) stays inside _extract_flags so
+#    inspect.getsource still discovers them.
+# --------------------------------------------------------------------------- #
+def test_get_all_possible_flags_nonempty_and_known():
+    flags = PostParser.get_all_possible_flags()
+    assert flags, "flag introspection returned nothing"
+    for known in ("video", "audio", "no_image", "sticker", "poll", "fwd"):
+        assert known in flags, f"known flag {known} not discovered by /flags introspection"

tests/test_stage5_sqlite.py

@@ -17,17 +17,12 @@ Covers:
 - gotcha: str(channel) key discipline — an int-ish channel on the hot path keys the
       accumulator (and thus the UPDATE) by the string form.
 """
-import os
-import sys
 import sqlite3
 import asyncio
 from types import SimpleNamespace
 
 import pytest
 
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
-
 import api_server
 from file_io import init_db_sync, update_media_file_access_bulk_sync
 
@@ -51,6 +46,7 @@ def _clean_accumulator():
 @pytest.mark.asyncio
 async def test_cache_hit_records_accumulator_no_sqlite(tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     channel, post_id, fid = "testchan", 5, "fidHIT"
     cache_dir = tmp_path / "data" / "cache" / channel / str(post_id)
     cache_dir.mkdir(parents=True)
@@ -82,6 +78,7 @@ async def test_cache_hit_records_accumulator_no_sqlite(tmp_path, monkeypatch):
 @pytest.mark.asyncio
 async def test_get_media_pre_semaphore_cache_hit_no_sqlite(tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     channel, post_id, fid = "gmchan", 11, "fidGM"
     cache_dir = tmp_path / "data" / "cache" / channel / str(post_id)
     cache_dir.mkdir(parents=True)
@@ -113,6 +110,7 @@ async def test_get_media_pre_semaphore_cache_hit_no_sqlite(tmp_path, monkeypatch
 @pytest.mark.asyncio
 async def test_cache_hit_keys_channel_as_str(tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     channel_int, post_id, fid = 12345, 7, "fidINT"
     cache_dir = tmp_path / "data" / "cache" / str(channel_int) / str(post_id)
     cache_dir.mkdir(parents=True)

tests/test_stage5b_media_fixes.py

@@ -0,0 +1,148 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, redefined-outer-name, line-too-long
+"""Stage-5b registered media fixes (render-pipeline refactor epic, issue #32/#34).
+
+Two registry items, both routed through the single MEDIA_SOURCES table:
+  * §3.13 — the ">100MB don't cache" rule now applies to ANY selected media object,
+    not just message.video (previously only video, plus live_photo/story/poll).
+  * §3.14 — the <div class="message-media"> container is CLOSED in every branch;
+    a None file_unique_id used to leave it open (html5lib then swallowed following
+    posts / webpage previews into it).
+
+The fragment snapshot (tests/test_data/media_fragments.json) was regenerated in the 5b
+changeset: exactly two cases moved — `file_unique_id_none` and `webpage_without_photo`
+each gained the now-balanced `</div>` (see test_stage5_media_table for the oracle).
+The feed goldens moved only by the same `</div>` relocation on webpage-without-photo posts.
+"""
+from types import SimpleNamespace
+
+import pytest
+
+from pyrogram.enums import MessageMediaType
+
+from post_parser import PostParser
+from url_signer import KeyManager
+
+from tests import media_fragment_replay as fr
+
+
+@pytest.fixture(autouse=True)
+def _pin_signing_key(monkeypatch):
+    monkeypatch.setattr(KeyManager, "signing_key", fr.FRAGMENT_SIGNING_KEY)
+
+
+@pytest.fixture
+def parser():
+    return PostParser(SimpleNamespace())
+
+
+def _msg(mid, media, **extra):
+    return fr._msg(mid, media, **extra)
+
+
+# --------------------------------------------------------------------------- #
+# §3.14 — the message-media div is closed in every branch.
+# --------------------------------------------------------------------------- #
+def test_none_file_unique_id_closes_media_div(parser):
+    # A media type whose object has no usable file_unique_id: the container used to
+    # be left open. Now it must be closed.
+    msg = _msg(1, MessageMediaType.PHOTO, photo=SimpleNamespace(file_unique_id=None, file_id="x"))
+    html = parser._generate_html_media(msg)
+    assert html == '<div class="message-media">\n</div>'
+
+
+def test_webpage_without_photo_closes_media_div_before_preview(parser):
+    # WEB_PAGE without photo: the empty media div must close BEFORE the webpage-preview
+    # block (previously the open div swallowed the preview and, at feed level, following
+    # posts). Every <div class="message-media"> is now paired with a </div>.
+    msg = _msg(2, MessageMediaType.WEB_PAGE, text="hi",
+               web_page=SimpleNamespace(photo=None, url="https://e.com", title="E",
+                                        description=None, site_name=None, type="", display_url=None))
+    html = parser._generate_html_media(msg)
+    assert html.startswith('<div class="message-media">\n</div>\n<div class="webpage-preview">')
+    assert html.count('<div class="message-media">') == 1
+    # The media container is now empty and closed, not wrapping the preview.
+    assert '<div class="message-media">\n</div>' in html
+
+
+def test_channel_username_missing_still_closes_div(parser):
+    # The username-guard branch also closes the div (unchanged behavior, asserted so a
+    # future refactor cannot regress it).
+    msg = _msg(3, MessageMediaType.PHOTO, username=None, chat_id=555,
+               photo=SimpleNamespace(file_unique_id="u", file_id="f"))
+    html = parser._generate_html_media(msg)
+    assert html == '<div class="message-media">\n</div>'
+
+
+@pytest.mark.parametrize("case_name", ["file_unique_id_none", "webpage_without_photo"])
+def test_5b_fragment_cases_are_now_balanced(parser, case_name):
+    """The two fragments that 5b intentionally changed must have a balanced media div."""
+    factory = fr.build_cases()[case_name]
+    html = parser._generate_html_media(factory())
+    assert html.count('<div class="message-media">') == 1
+    # There is at least one closing tag for the media container now.
+    assert '</div>' in html
+
+
+# --------------------------------------------------------------------------- #
+# §3.13 — the >100MB guard applies to ANY selected object, not only video.
+# --------------------------------------------------------------------------- #
+BIG = 200 * 1024 * 1024
+SMALL = 5 * 1024 * 1024
+
+
+@pytest.mark.parametrize("media_type,attr", [
+    (MessageMediaType.PHOTO, "photo"),
+    (MessageMediaType.DOCUMENT, "document"),
+    (MessageMediaType.AUDIO, "audio"),
+    (MessageMediaType.ANIMATION, "animation"),
+])
+def test_large_non_video_object_not_collected(parser, media_type, attr):
+    """§3.13: a >100MB photo/document/audio/animation is no longer collected for the
+    download cache (before 5b only large VIDEO objects were skipped)."""
+    obj = SimpleNamespace(file_unique_id="big", file_id="f", file_size=BIG, mime_type="image/png")
+    msg = _msg(10, media_type, **{attr: obj})
+    parser._pending_media_ids = []
+    parser._save_media_file_ids(msg)
+    assert parser._pending_media_ids == []
+
+
+@pytest.mark.parametrize("media_type,attr", [
+    (MessageMediaType.PHOTO, "photo"),
+    (MessageMediaType.DOCUMENT, "document"),
+    (MessageMediaType.AUDIO, "audio"),
+])
+def test_small_non_video_object_still_collected(parser, media_type, attr):
+    obj = SimpleNamespace(file_unique_id="small", file_id="f", file_size=SMALL, mime_type="image/png")
+    msg = _msg(11, media_type, **{attr: obj})
+    parser._pending_media_ids = []
+    parser._save_media_file_ids(msg)
+    assert [(c, p, f) for c, p, f, _ in parser._pending_media_ids] == [("testchan", 11, "small")]
+
+
+def test_exactly_100mb_object_is_collected(parser):
+    # §3.13 boundary: the guard is strictly `>` 100MB, so an object of exactly 100MB is
+    # still collected. Pins the operator — a mutation to `>=` would drop this and fail.
+    obj = SimpleNamespace(file_unique_id="edge", file_id="f",
+                          file_size=100 * 1024 * 1024, mime_type="image/png")
+    msg = _msg(14, MessageMediaType.PHOTO, photo=obj)
+    parser._pending_media_ids = []
+    parser._save_media_file_ids(msg)
+    assert [(c, p, f) for c, p, f, _ in parser._pending_media_ids] == [("testchan", 14, "edge")]
+
+
+def test_large_video_still_not_collected(parser):
+    # Regression: the video case (the only one guarded before 5b) still works.
+    msg = _msg(12, MessageMediaType.VIDEO,
+               video=SimpleNamespace(file_unique_id="v", file_id="f", file_size=BIG))
+    parser._pending_media_ids = []
+    parser._save_media_file_ids(msg)
+    assert parser._pending_media_ids == []
+
+
+def test_object_without_file_size_is_collected(parser):
+    # No file_size attribute -> not guarded (the common case for photos).
+    msg = _msg(13, MessageMediaType.PHOTO, photo=SimpleNamespace(file_unique_id="nofs", file_id="f"))
+    parser._pending_media_ids = []
+    parser._save_media_file_ids(msg)
+    assert [(c, p, f) for c, p, f, _ in parser._pending_media_ids] == [("testchan", 13, "nofs")]

tests/test_stage6_exclude_flags.py

@@ -0,0 +1,86 @@
+# flake8: noqa
+# pylint: disable=missing-function-docstring, redefined-outer-name
+"""Stage-6: unit lock for the `exclude_flags` feed filter (issue #33, epic #34).
+
+The filter (rss_generator._render_messages_groups) is a user-facing query-param
+that drops posts from the feed by flag. It is NOT exercised by the stage-0 golden
+oracle (golden_replay runs without exclude_flags), and stage 6 rewrote it from a
+`for/continue` loop into a list-comprehension. These tests pin the membership
+semantics so the rewrite (and any future one) stays honest — this is the
+"dedicated unit tests" that golden_replay.py's header comment refers to.
+
+Flag shapes under the test config: a plain text post carries ['no_image']; a
+merged group additionally carries 'merged'. (Every rendered post carries at least
+one flag, so the `"all"` filter drops every real post; the guard's flagless-keep
+branch — `and post['flags']` — is a media-path concern proven separately by the
+equivalence check in the PR, not reproducible with plain fixtures here.)
+"""
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from rss_generator import _render_messages_groups
+from post_parser import PostParser
+from tests.test_stage4_eventloop import make_message
+
+
+@pytest.fixture
+def parser():
+    return PostParser(SimpleNamespace())
+
+
+def _plain(mid, date=None):
+    # Plain text post -> flags == ['no_image'].
+    return make_message(mid, text="plain", date=date)
+
+
+def _merged_group(mid, date=None):
+    # Merged group -> flags == ['no_image', 'merged']; message_id is the main id.
+    main = make_message(mid, text="m", date=date)
+    main.media_group_id = f"g{mid}"
+    other = make_message(mid + 1, text="o", date=date)
+    other.media_group_id = f"g{mid}"
+    return [main, other]
+
+
+def _ids(posts):
+    return [p["message_id"] for p in posts]
+
+
+def test_exclude_flags_all_drops_flagged_posts(parser):
+    # `"all"` special case: any post carrying >=1 flag is dropped.
+    posts = _render_messages_groups(
+        [[_plain(10)], [_plain(11)]], parser, exclude_flags="all"
+    )
+    assert _ids(posts) == []  # both ['no_image'] -> dropped
+
+
+def test_exclude_flags_specific_drops_matching_keeps_nonmatching(parser):
+    # Exclude a concrete flag: the post carrying it is dropped, one without it kept.
+    posts = _render_messages_groups(
+        [_merged_group(20), [_plain(22)]], parser, exclude_flags="merged"
+    )
+    ids = _ids(posts)
+    assert 22 in ids  # ['no_image'] has no 'merged' -> kept
+    assert 20 not in ids  # ['no_image', 'merged'] -> dropped
+
+
+def test_exclude_flags_none_or_nonmatching_keeps_all(parser):
+    base = [[_plain(30)], [_plain(31)]]
+    assert len(_render_messages_groups(base, parser)) == 2  # no filter
+    assert (
+        len(_render_messages_groups(base, parser, exclude_flags="nonexistent")) == 2
+    )  # flag matches nothing -> all kept
+
+
+def test_exclude_flags_preserves_survivor_order(parser):
+    # Distinct descending dates so the trailing date-sort is deterministic;
+    # dropping the middle (merged) post must not reorder the survivors.
+    a = make_message(40, text="a", date=datetime(2024, 1, 3, tzinfo=timezone.utc))
+    mid = _merged_group(41, date=datetime(2024, 1, 2, tzinfo=timezone.utc))
+    c = make_message(43, text="c", date=datetime(2024, 1, 1, tzinfo=timezone.utc))
+    posts = _render_messages_groups(
+        [[a], mid, [c]], parser, exclude_flags="merged"
+    )
+    assert _ids(posts) == [40, 43]  # merged(41) dropped; a before c by date-desc

tests/test_stage6_healthcheck.py

@@ -15,16 +15,10 @@ Covers:
   fake client's get_me / safe_get_messages proves neither is ever called.
 - TelegramClient.watchdog_last_ok_age(): None when never probed; a positive float afterwards.
 """
-import os
-import sys
 import time
 
 import pytest
 
-# Add project root to sys.path and mock the config module (same pattern as the other tests).
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
-
 from fastapi.testclient import TestClient
 
 import api_server

tests/test_stage7_integration.py

@@ -27,8 +27,6 @@ Automated here:
   an int-ish channel records the str-keyed timestamp, and the flush UPDATE matches the
   str-keyed DB row (hit -> flush -> DB), the exact affinity gotcha the plan warns about.
 """
-import os
-import sys
 import time
 import sqlite3
 import asyncio
@@ -36,10 +34,6 @@ from types import SimpleNamespace
 
 import pytest
 
-# Add project root to sys.path and mock the config module (same pattern as the other tests).
-sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
-sys.modules['config'] = __import__('tests.mock_config', fromlist=['get_settings'])
-
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
 
@@ -79,6 +73,7 @@ def _seed_cache(tmp_path, channel, post_id, fid, body=BODY):
 # --------------------------------------------------------------------------- #
 def test_media_route_range_prefix_0_99(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     _seed_cache(tmp_path, "chan", 3, "fidR")
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     # Keep the MIME path DB-free; the FileResponse/Range machinery is what we exercise.
@@ -96,6 +91,7 @@ def test_media_route_range_prefix_0_99(monkeypatch, tmp_path):
 
 def test_media_route_range_suffix_last_100(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     _seed_cache(tmp_path, "chan", 3, "fidS")
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     monkeypatch.setattr(api_server, "get_mime_type_sync", lambda *a, **k: None)
@@ -111,6 +107,7 @@ def test_media_route_range_suffix_last_100(monkeypatch, tmp_path):
 
 def test_media_route_range_unsatisfiable_416(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     _seed_cache(tmp_path, "chan", 3, "fidU")
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     monkeypatch.setattr(api_server, "get_mime_type_sync", lambda *a, **k: None)
@@ -221,6 +218,7 @@ async def test_ping_reports_degraded_promptly_while_slow_op_pending(monkeypatch)
 # --------------------------------------------------------------------------- #
 async def test_get_media_concurrent_shares_one_download_and_drains_registry(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     api_server._inflight.clear()
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     # Serve step is not under test here; keep it to a sentinel so we assert on dedup + registry.
@@ -252,6 +250,7 @@ async def test_get_media_concurrent_shares_one_download_and_drains_registry(monk
 
 async def test_get_media_request_cancel_does_not_stick_registry_or_hang_sibling(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     api_server._inflight.clear()
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     sentinel = object()
@@ -304,6 +303,7 @@ async def test_get_media_request_cancel_does_not_stick_registry_or_hang_sibling(
 # --------------------------------------------------------------------------- #
 async def test_media_cache_hit_flush_updates_str_keyed_db_row(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     api_server._access_updates = {}
 
     channel, post_id, fid = "12345", 7, "fidINT"  # int-ish channel, passed as the route str

tests/test_stage_e_media_hotpath.py

@@ -0,0 +1,204 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, line-too-long
+"""
+Issue #26 package E — media-cache path helper + in-memory MIME cache on the hot /media path.
+
+Covers:
+- media_cache_path builds EXACTLY the pre-existing on-disk layout (path-format regression).
+- The in-memory _mime_types dict short-circuits the SQLite MIME read on a repeat hit
+  (get_mime_type_sync not called a second time; no new SQLite connection opened).
+- _mime_types overflow (>= _MIME_CACHE_MAX) clears the dict without raising and repopulates.
+"""
+import os
+
+import pytest
+
+from fastapi import FastAPI, Request
+from fastapi.testclient import TestClient
+
+import file_io
+import api_server
+
+
+BODY = bytes(range(256)) * 8  # 2048 deterministic bytes
+
+
+def _make_client(file_path, media_key=None):
+    """Drive prepare_file_response through real ASGI (as the stage-3 tests do)."""
+    app = FastAPI()
+
+    @app.get("/f")
+    async def _serve(request: Request):
+        return await api_server.prepare_file_response(file_path, request=request, media_key=media_key)
+
+    return TestClient(app)
+
+
+@pytest.fixture
+def sample_file(tmp_path):
+    fp = tmp_path / "myfile.bin"
+    fp.write_bytes(BODY)
+    return str(fp)
+
+
+# --------------------------------------------------------------------------- #
+# Task 15 — media_cache_path path-format regression.
+# --------------------------------------------------------------------------- #
+def test_media_cache_path_layout():
+    root = api_server.MEDIA_CACHE_DIR
+    # The constant must resolve to the exact absolute string every legacy site built.
+    assert root == os.path.abspath("./data/cache")
+
+    # Without file_unique_id -> the per-post directory <root>/<channel>/<post_id>.
+    assert api_server.media_cache_path("mychan", 42) == os.path.join(root, "mychan", "42")
+
+    # With file_unique_id -> the full per-file path <root>/<channel>/<post_id>/<fid>.
+    assert api_server.media_cache_path("mychan", 42, "fidABC") == os.path.join(root, "mychan", "42", "fidABC")
+
+    # Faithful drop-in: identical to the old manual assembly (incl. int channel stringified).
+    assert api_server.media_cache_path(-100123, 7, "fidABC") == os.path.join(
+        os.path.abspath("./data/cache"), "-100123", "7", "fidABC"
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Task 16 — in-memory MIME cache short-circuits the SQLite read on a repeat hit.
+# --------------------------------------------------------------------------- #
+def test_second_request_served_from_dict_not_sqlite(sample_file, monkeypatch):
+    calls = {"get": 0, "magic": 0}
+
+    def fake_get(db, ch, pid, fid):
+        calls["get"] += 1
+        return "video/mp4"  # SQLite HIT on the first (dict-miss) request
+
+    def fake_magic(_path):
+        calls["magic"] += 1
+        return "application/octet-stream"
+
+    monkeypatch.setattr(api_server, "get_mime_type_sync", fake_get)
+    monkeypatch.setattr(api_server.magic_mime, "from_file", fake_magic)
+
+    key = ("chanE", 100, "fidE")
+    c = _make_client(sample_file, media_key=key)
+
+    r1 = c.get("/f")
+    r2 = c.get("/f")
+
+    assert r1.status_code == 200 and r2.status_code == 200
+    assert r1.headers["content-type"].startswith("video/mp4")
+    assert r2.headers["content-type"].startswith("video/mp4")
+    # First request populated the dict from SQLite; the second is served from the dict.
+    assert calls["get"] == 1
+    assert calls["magic"] == 0
+    assert api_server._mime_types[key] == "video/mp4"
+
+
+def test_repeat_hit_opens_no_new_db_connection(tmp_path, sample_file, monkeypatch):
+    """Acceptance recipe: monkeypatch file_io._open_db with a counter; two consecutive
+    requests for the same file — the counter must NOT increase on the second."""
+    db_path = str(tmp_path / "media.db")
+    file_io.init_db_sync(db_path)
+    # Seed a row whose MIME is already stored, so the first request reads it from SQLite
+    # (no python-magic, no set-write) and caches it in the dict.
+    file_io.upsert_media_file_id_sync(db_path, "chanDB", 55, "fidDB", 0.0)
+    file_io.set_mime_type_sync(db_path, "chanDB", 55, "fidDB", "image/png")
+
+    monkeypatch.setattr(api_server, "DB_PATH", db_path)
+
+    count = {"n": 0}
+    orig_open = file_io._open_db
+
+    def counting_open(p):
+        count["n"] += 1
+        return orig_open(p)
+
+    monkeypatch.setattr(file_io, "_open_db", counting_open)
+
+    c = _make_client(sample_file, media_key=("chanDB", 55, "fidDB"))
+
+    r1 = c.get("/f")
+    after_first = count["n"]
+    r2 = c.get("/f")
+    after_second = count["n"]
+
+    assert r1.status_code == 200 and r2.status_code == 200
+    assert r1.headers["content-type"].startswith("image/png")
+    assert r2.headers["content-type"].startswith("image/png")
+    assert after_first >= 1  # the first (dict-miss) hit did open a connection for the read
+    # The repeat hit opened NO new SQLite connection.
+    assert after_second == after_first
+
+
+# --------------------------------------------------------------------------- #
+# Task 16 — overflow clears the dict without raising, then repopulates.
+# --------------------------------------------------------------------------- #
+def test_mime_cache_overflow_clears_and_repopulates(sample_file, monkeypatch):
+    def fake_get(db, ch, pid, fid):
+        return "text/plain"
+
+    monkeypatch.setattr(api_server, "get_mime_type_sync", fake_get)
+    monkeypatch.setattr(api_server.magic_mime, "from_file", lambda _p: "text/plain")
+
+    # Fill the dict up to the bound with dummy entries.
+    api_server._mime_types.clear()
+    for i in range(api_server._MIME_CACHE_MAX):
+        api_server._mime_types[("dummy", i, "f")] = "x/y"
+    assert len(api_server._mime_types) == api_server._MIME_CACHE_MAX
+
+    key = ("chanOverflow", 1, "fidOverflow")
+    c = _make_client(sample_file, media_key=key)
+
+    r = c.get("/f")  # must clear-all on overflow, then insert the new key — no exception
+    assert r.status_code == 200
+    assert r.headers["content-type"].startswith("text/plain")
+    # Cleared then repopulated with just the one fresh key.
+    assert len(api_server._mime_types) == 1
+    assert api_server._mime_types[key] == "text/plain"
+
+    # The next request is now a clean dict hit.
+    r2 = c.get("/f")
+    assert r2.status_code == 200
+
+
+# --------------------------------------------------------------------------- #
+# Task 16 — magic cold-start path: dict-miss + SQLite-miss -> python-magic ->
+# write-through to BOTH the SQLite type cache AND the in-memory dict.
+# The other stage-E tests all make SQLite HIT (get_mime_type_sync returns a value),
+# so the magic branch's dict-write (api_server.py:499) is never exercised there.
+# --------------------------------------------------------------------------- #
+def test_magic_cold_start_writes_through_to_sqlite_and_dict(sample_file, monkeypatch):
+    key = ("chanMagic", 7, "fidMagic")
+    # Dict must be empty for this key (dict-miss).
+    api_server._mime_types.pop(key, None)
+
+    calls = {"get": 0, "set": 0, "magic": 0}
+
+    def fake_get(db, ch, pid, fid):
+        calls["get"] += 1
+        return None  # SQLite MISS -> falls through to python-magic
+
+    def fake_set(db, ch, pid, fid, mime):
+        calls["set"] += 1
+        assert (ch, pid, fid) == key
+        assert mime == "image/jpeg"
+
+    def fake_magic(_path):
+        calls["magic"] += 1
+        return "image/jpeg"
+
+    monkeypatch.setattr(api_server, "get_mime_type_sync", fake_get)
+    monkeypatch.setattr(api_server, "set_mime_type_sync", fake_set)
+    monkeypatch.setattr(api_server.magic_mime, "from_file", fake_magic)
+
+    c = _make_client(sample_file, media_key=key)
+    r = c.get("/f")
+
+    assert r.status_code == 200
+    assert r.headers["content-type"].startswith("image/jpeg")
+    # Cold start: dict-miss -> SQLite-miss -> python-magic detected the type.
+    assert calls["get"] == 1
+    assert calls["magic"] == 1
+    # Write-through to BOTH caches.
+    assert calls["set"] == 1                             # SQLite write-through
+    assert api_server._mime_types[key] == "image/jpeg"  # dict write-through

tests/test_tg_cache_store.py

@@ -0,0 +1,206 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""Tests for the tg_cache generic JSON store, prefix-limit, jitter and sweep (issue #23).
+
+Задания 2 / 6 / 17 / 18, verification item 8, 11, 12.
+"""
+import json
+import os
+import time
+from types import SimpleNamespace
+
+import pytest
+
+import tg_cache
+from message_snapshot import SNAPSHOT_VERSION
+
+
+@pytest.fixture
+def cache_dir(tmp_path, monkeypatch):
+    d = tmp_path / "tgcache"
+    d.mkdir()
+    monkeypatch.setattr(tg_cache, "CACHE_DIR", str(d))
+    return d
+
+
+# --------------------------------------------------------------------------- #
+# _store_entry / _load_entry.
+# --------------------------------------------------------------------------- #
+def test_store_load_roundtrip(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    tg_cache._store_entry(path, {"limit": 5, "messages": [{"id": 1}]})
+    loaded = tg_cache._load_entry(path, max_age_hours=8)
+    assert loaded is not None
+    assert loaded["limit"] == 5
+    assert loaded["messages"] == [{"id": 1}]
+    assert loaded["version"] == SNAPSHOT_VERSION
+    assert 0.8 <= loaded["jitter"] <= 1.0
+
+
+def test_load_missing_file(cache_dir):
+    assert tg_cache._load_entry(str(cache_dir / "nope.json"), 8) is None
+
+
+def test_load_expired_ttl(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    tg_cache._store_entry(path, {"data": 1})
+    # max_age_hours=0 -> adjusted max age 0 -> any positive age is expired.
+    assert tg_cache._load_entry(path, max_age_hours=0) is None
+    # Still fresh with a real TTL.
+    assert tg_cache._load_entry(path, max_age_hours=8) is not None
+
+
+def test_load_version_mismatch(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump({"version": SNAPSHOT_VERSION + 999, "timestamp": time.time(), "data": 1}, f)
+    assert tg_cache._load_entry(path, 8) is None
+
+
+def test_load_corrupt_json(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    with open(path, "w", encoding="utf-8") as f:
+        f.write("{not valid json")
+    assert tg_cache._load_entry(path, 8) is None
+
+
+def test_store_uses_unique_tmp_names(cache_dir, monkeypatch):
+    path = str(cache_dir / "x.history.json")
+    seen = []
+
+    real_replace = os.replace
+
+    def spy_replace(src, dst):
+        seen.append(src)
+        return real_replace(src, dst)
+
+    monkeypatch.setattr(tg_cache.os, "replace", spy_replace)
+    tg_cache._store_entry(path, {"data": 1})
+    tg_cache._store_entry(path, {"data": 2})
+
+    assert len(seen) == 2
+    assert seen[0] != seen[1]  # each writer used a distinct tmp path
+    for src in seen:
+        assert src.startswith(path + ".tmp.")
+    # No tmp leftovers.
+    assert [n for n in os.listdir(cache_dir) if ".tmp." in n] == []
+
+
+# --------------------------------------------------------------------------- #
+# cleanup_legacy_cache_files / sweep_tgcache.
+# --------------------------------------------------------------------------- #
+def test_cleanup_legacy_keeps_new_format(cache_dir):
+    (cache_dir / "chan.cache").write_text("x")
+    (cache_dir / "chan_history.cache").write_text("x")
+    (cache_dir / "chan.chatinfo").write_text("x")
+    (cache_dir / "chan.history.json").write_text("x")
+    (cache_dir / "chan.chatinfo.json").write_text("x")
+
+    removed = tg_cache.cleanup_legacy_cache_files()
+    assert removed == 3
+    remaining = set(os.listdir(cache_dir))
+    assert remaining == {"chan.history.json", "chan.chatinfo.json"}
+
+
+def test_sweep_removes_only_old(cache_dir):
+    old = cache_dir / "old.history.json"
+    new = cache_dir / "new.history.json"
+    old.write_text("x")
+    new.write_text("x")
+    old_time = time.time() - 8 * 86400  # 8 days old, threshold is 7
+    os.utime(old, (old_time, old_time))
+
+    removed = tg_cache.sweep_tgcache(max_age_days=7)
+    assert removed == 1
+    remaining = set(os.listdir(cache_dir))
+    assert remaining == {"new.history.json"}
+
+
+def test_sweep_removes_orphan_tmp(cache_dir):
+    orphan = cache_dir / "x.history.json.tmp.deadbeef"
+    orphan.write_text("x")
+    old_time = time.time() - 10 * 86400
+    os.utime(orphan, (old_time, old_time))
+    assert tg_cache.sweep_tgcache(max_age_days=7) == 1
+
+
+# --------------------------------------------------------------------------- #
+# History prefix-limit (Задание 6).
+# --------------------------------------------------------------------------- #
+def _fake_messages(n):
+    return [SimpleNamespace(id=i) for i in range(n)]
+
+
+def test_prefix_larger_cache_serves_smaller_request(cache_dir):
+    tg_cache._save_history_to_cache("chan", _fake_messages(100), limit=100)
+    served = tg_cache._get_history_from_cache("chan", limit=50)
+    assert served is not None
+    assert len(served) == 50
+    assert [m.id for m in served] == list(range(50))  # order preserved (newest-first slice)
+
+
+def test_prefix_smaller_cache_is_miss(cache_dir):
+    tg_cache._save_history_to_cache("chan", _fake_messages(50), limit=50)
+    assert tg_cache._get_history_from_cache("chan", limit=100) is None
+
+
+def test_prefix_exhausted_channel_serves_larger_request(cache_dir):
+    # Fetched with limit 100 but the channel only has 37 messages -> entire history cached.
+    tg_cache._save_history_to_cache("chan", _fake_messages(37), limit=100)
+    served = tg_cache._get_history_from_cache("chan", limit=200)
+    assert served is not None
+    assert len(served) == 37
+
+
+def test_save_stores_fetch_limit_not_len(cache_dir):
+    tg_cache._save_history_to_cache("chan", _fake_messages(37), limit=100)
+    path = tg_cache._cache_file_path("chan", "history.json")
+    payload = tg_cache._load_entry(path, max_age_hours=8)
+    assert payload["limit"] == 100
+    assert len(payload["messages"]) == 37
+
+
+def test_history_ttl_independent_of_prefix(cache_dir):
+    tg_cache._save_history_to_cache("chan", _fake_messages(100), limit=100)
+    # Expired regardless of prefix match.
+    assert tg_cache._get_history_from_cache("chan", limit=50, max_age_hours=0) is None
+
+
+# --------------------------------------------------------------------------- #
+# Jitter stability (Задание 17 / item 12).
+# --------------------------------------------------------------------------- #
+def test_jitter_read_is_stable(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    tg_cache._store_entry(path, {"data": 1})
+    # Repeated reads near a TTL boundary must give a stable result: the jitter is fixed at
+    # write time and _load_entry never calls random().
+    results = [tg_cache._load_entry(path, max_age_hours=8) is not None for _ in range(10)]
+    assert all(results)
+
+
+def test_load_does_not_call_random(cache_dir, monkeypatch):
+    path = str(cache_dir / "x.history.json")
+    tg_cache._store_entry(path, {"data": 1})
+
+    def boom(*a, **k):
+        raise AssertionError("random.uniform must not be called at read time")
+
+    monkeypatch.setattr(tg_cache.random, "uniform", boom)
+    assert tg_cache._load_entry(path, max_age_hours=8) is not None
+
+
+# --------------------------------------------------------------------------- #
+# chatinfo store round trip.
+# --------------------------------------------------------------------------- #
+def test_chatinfo_roundtrip(cache_dir):
+    tg_cache._save_chat_to_cache("chan", {"id": -100, "title": "T", "username": "u"})
+    data = tg_cache._get_chat_from_cache("chan")
+    assert data == {"id": -100, "title": "T", "username": "u"}
+
+
+def test_no_pickle_import():
+    import pathlib
+    src = pathlib.Path(tg_cache.__file__).read_text()
+    assert "pickle" not in src

tg_cache.py

@@ -11,18 +11,23 @@
 
 import os
 import json
-import pickle
+import uuid
 import logging
 import random
 import asyncio
 import time
-from datetime import datetime, timedelta
 from types import SimpleNamespace
 from typing import Any, Optional, Union, List
 from pyrogram import Client
-from pyrogram.types import Chat, Message
+from pyrogram.types import Message
 from tg_throttle import tg_rpc_bounded
 from config import get_settings
+from message_snapshot import (
+    SNAPSHOT_VERSION,
+    snapshot_messages,
+    restore_messages,
+)
+from channel_key import canonical_channel_key
 
 logger = logging.getLogger(__name__)
 
@@ -40,105 +45,152 @@ try:
 except ValueError:
     CHAT_CACHE_TTL_HOURS = 12
 
-def _get_history_cache_file_path(channel_id: Union[str, int]) -> str:
-    """Returns path to the message history cache file for the channel"""
+
+def _safe_key(key: Union[str, int]) -> str:
+    """Sanitize a channel id/username into a filesystem-safe basename component."""
+    return str(key).replace('/', '_').replace('\\', '_')
+
+
+def _cache_file_path(key: Union[str, int], suffix: str) -> str:
+    """Return the cache file path <safe_key>.<suffix> (e.g. 'history.json' / 'chatinfo.json').
+
+    The key is first canonicalized so that 'Durov', 'durov' and '@durov' collapse to a
+    single cache file (Telegram usernames are case-insensitive).
+    """
     if not os.path.exists(CACHE_DIR):
         os.makedirs(CACHE_DIR, exist_ok=True)
         logger.info(f"cache_dir_created: path {CACHE_DIR}")
-    # Convert to string for uniformity
-    channel_id_str = str(channel_id)
-    # Replace potentially problematic characters
-    safe_filename = channel_id_str.replace('/', '_').replace('\\', '_')
-    return os.path.join(CACHE_DIR, f"{safe_filename}.cache")
+    return os.path.join(CACHE_DIR, f"{_safe_key(canonical_channel_key(key))}.{suffix}")
 
-def _save_history_to_cache(channel_id: Union[str, int], messages: List[Message], limit: int) -> None:
-    """Saves message history to cache"""
+
+# --------------------------------------------------------------------------- #
+# Generic JSON entry store.
+# --------------------------------------------------------------------------- #
+def _store_entry(path: str, payload: dict) -> None:
+    """Atomically write {version, timestamp, jitter, **payload} as JSON to ``path``.
+
+    The document is written to a unique '<path>.tmp.<uuid4>' and os.replace()d into place
+    so a concurrent reader never observes a half-written file. The unique per-writer tmp
+    name means two concurrent writers to the same path do not clobber each other's temp
+    file. This writer's own tmp file is always removed in the finally block.
+    """
+    tmp_path = f"{path}.tmp.{uuid.uuid4().hex}"
+    entry = {
+        'version': SNAPSHOT_VERSION,
+        'timestamp': time.time(),
+        # Per-write TTL jitter (§17): decided ONCE at write time, so repeated reads of the
+        # same file are stable (the reader never calls random()).
+        'jitter': random.uniform(0.8, 1.0),
+    }
+    entry.update(payload)
     try:
-        cache_file = _get_history_cache_file_path(channel_id)
-        
-        # Create cache metadata — store messages directly (no inner pickle.dumps)
-        cache_data = {
-            'timestamp': time.time(),
-            'limit': limit,
-            'messages': messages
-        }
-        
-        with open(cache_file, 'wb') as f:
-            pickle.dump(cache_data, f)
-        
+        with open(tmp_path, 'w', encoding='utf-8') as f:
+            json.dump(entry, f)
+        os.replace(tmp_path, path)
+    finally:
+        # Remove our own leftover tmp file if os.replace didn't consume it (e.g. it raised
+        # or json.dump failed). Never touches another writer's uniquely-named tmp file.
+        if os.path.exists(tmp_path):
+            try:
+                os.remove(tmp_path)
+            except OSError:
+                pass
+
+
+def _load_entry(path: str, max_age_hours: float) -> Optional[dict]:
+    """Return the stored payload dict, or None on missing / version mismatch / expired / bad JSON.
+
+    TTL uses the jitter written into the entry (no random() at read time) so repeated
+    reads near the boundary give a stable result.
+    """
+    if not os.path.exists(path):
+        return None
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            entry = json.load(f)
+    except (json.JSONDecodeError, OSError, ValueError) as e:
+        logger.warning(f"cache_entry_read_error: path {path}, error {str(e)}")
+        return None
+
+    if not isinstance(entry, dict) or entry.get('version') != SNAPSHOT_VERSION:
+        logger.info(f"cache_entry_version_mismatch: path {path}")
+        return None
+
+    timestamp = entry.get('timestamp')
+    if not isinstance(timestamp, (int, float)):
+        return None
+    age = time.time() - timestamp
+    adjusted_max_age = max_age_hours * 3600 * entry.get('jitter', 1.0)
+    if age > adjusted_max_age:
+        logger.info(f"cache_entry_expired: path {path}, age {age:.1f}s > adjusted max {adjusted_max_age:.1f}s")
+        return None
+    return entry
+
+
+# --------------------------------------------------------------------------- #
+# History cache.
+# --------------------------------------------------------------------------- #
+def _save_history_to_cache(channel_id: Union[str, int], messages: List[Message], limit: int) -> None:
+    """Save message history (as JSON snapshots) to cache. Stores the fetch limit, not len()."""
+    try:
+        cache_file = _cache_file_path(channel_id, 'history.json')
+        payload = {'limit': limit, 'messages': snapshot_messages(messages)}
+        _store_entry(cache_file, payload)
         logger.info(f"history_cache_saved: channel {channel_id}, limit {limit}, messages {len(messages)}, file {cache_file}")
     except Exception as e:
         logger.error(f"history_cache_save_error: channel {channel_id}, limit {limit}, error {str(e)}")
 
+
 def _get_history_from_cache(channel_id: Union[str, int], limit: int, max_age_hours: int = 8) -> Optional[List[Message]]:
     """
-    Retrieves message history from cache if not older than specified age and matches the limit
-    
-    Args:
-        channel_id: Channel ID or username
-        limit: Required message limit
-        max_age_hours: Maximum cache age in hours (default 8 hours)
-        
-    Returns:
-        List of messages or None if cache not found, expired or limit doesn't match
+    Retrieve message history from cache if fresh and the cached fetch covers ``limit``.
+
+    Messages are stored newest-first. A cached entry fetched with an equal-or-larger limit
+    serves a smaller request by slicing (prefix). A smaller cached fetch is a miss UNLESS
+    the channel is exhausted (fewer messages exist than were asked for), in which case the
+    cache already holds the entire recent history and can serve any larger request.
     """
     try:
-        cache_file = _get_history_cache_file_path(channel_id)
-        
-        if not os.path.exists(cache_file):
-            logger.info(f"history_cache_miss: channel {channel_id}, limit {limit}, cache file not found")
+        cache_file = _cache_file_path(channel_id, 'history.json')
+        payload = _load_entry(cache_file, max_age_hours)
+        if payload is None:
+            logger.info(f"history_cache_miss: channel {channel_id}, limit {limit}")
             return None
-        
-        with open(cache_file, 'rb') as f:
-            cache_data = pickle.load(f)
-        
-        # Check cache age with randomization
-        cache_age = time.time() - cache_data['timestamp']
-        # Add randomness up to 20% of max_age_seconds
-        random_factor = 1 - random.uniform(0, 0.2)
-        adjusted_max_age = max_age_hours * 3600 * random_factor
-        
-        if cache_age > adjusted_max_age:
-            logger.info(f"history_cache_expired: channel {channel_id}, limit {limit}, age {cache_age:.1f}s > adjusted max {adjusted_max_age:.1f}s (random factor: {random_factor:.2f})")
+
+        cached_limit = payload.get('limit', 0)
+        raw_messages = payload['messages']
+        # Serve when fetched with an equal-or-larger limit, OR when the channel is exhausted
+        # (fewer messages exist than asked -> cache holds entire recent history).
+        if cached_limit < limit and len(raw_messages) >= cached_limit:
+            logger.info(f"history_cache_limit_short: channel {channel_id}, cached limit {cached_limit}, requested {limit}")
             return None
-        
-        # Check if limit matches
-        cached_limit = cache_data.get('limit', 0)
-        if cached_limit != limit:
-            logger.info(f"history_cache_limit_mismatch: channel {channel_id}, cached limit {cached_limit}, requested limit {limit}")
-            return None
-        
-        # Restore message list; handle old cache files that used double-pickle (bytes = old format)
-        raw = cache_data['messages']
-        if isinstance(raw, bytes):
-            messages = pickle.loads(raw)
-        else:
-            messages = raw
-        logger.info(f"history_cache_hit: channel {channel_id}, limit {limit}, messages {len(messages)}, age {cache_age:.1f}s")
+
+        messages = restore_messages(raw_messages[:limit])
+        logger.info(f"history_cache_hit: channel {channel_id}, served {limit} of cached {cached_limit}, messages {len(messages)}")
         return messages
-    
     except Exception as e:
         logger.error(f"history_cache_read_error: channel {channel_id}, limit {limit}, error {str(e)}")
-        # In case of cache read error, better return None and request fresh data
         return None
 
+
 async def cached_get_chat_history(client: Client, channel_id: Union[str, int], limit: int = 20) -> List[Message]:
     """
     Gets chat message history with caching.
-    
+
     Args:
         client: Pyrogram client
         channel_id: Channel ID or username
         limit: Maximum number of messages to retrieve
-        
+
     Returns:
-        List of messages, same as original client.get_chat_history()
+        List of messages, same as original client.get_chat_history(). On a cache miss the
+        live pyrogram Messages are returned; on a hit, restored CachedMessage objects.
     """
     cached_messages = await asyncio.to_thread(_get_history_from_cache, channel_id, limit)
-    
+
     if cached_messages is not None:
         return cached_messages
-    
+
     try:
         logger.info(f"history_cache_request: fetching fresh history for channel {channel_id}, limit {limit}")
         # Hold the global RPC gate for the live fetch and bound the RPC body with the
@@ -148,57 +200,39 @@ async def cached_get_chat_history(client: Client, channel_id: Union[str, int], l
         async with tg_rpc_bounded(Config["tg_rpc_timeout"]):
             messages = [m async for m in client.get_chat_history(channel_id, limit=limit)]
         await asyncio.to_thread(_save_history_to_cache, channel_id, messages, limit)
-        
+
         return messages
     except Exception as e:
         logger.error(f"history_cache_request_error: channel {channel_id}, limit {limit}, error {str(e)}")
         raise
 
 
-def _get_chat_cache_file_path(channel_id: Union[str, int]) -> str:
-    """Returns path to the channel-info cache file for the channel."""
-    if not os.path.exists(CACHE_DIR):
-        os.makedirs(CACHE_DIR, exist_ok=True)
-        logger.info(f"cache_dir_created: path {CACHE_DIR}")
-    channel_id_str = str(channel_id)
-    safe_filename = channel_id_str.replace('/', '_').replace('\\', '_')
-    # Distinct suffix so channel-info files never collide with history (.cache) files.
-    return os.path.join(CACHE_DIR, f"{safe_filename}.chatinfo")
-
-
+# --------------------------------------------------------------------------- #
+# Channel-info cache.
+# --------------------------------------------------------------------------- #
 def _save_chat_to_cache(channel_id: Union[str, int], data: dict) -> None:
     """Saves channel-info (id/title/username) to cache."""
     try:
-        cache_file = _get_chat_cache_file_path(channel_id)
-        cache_data = {'timestamp': time.time(), 'data': data}
-        with open(cache_file, 'wb') as f:
-            pickle.dump(cache_data, f)
+        cache_file = _cache_file_path(channel_id, 'chatinfo.json')
+        _store_entry(cache_file, {'data': data})
         logger.info(f"chatinfo_cache_saved: channel {channel_id}, file {cache_file}")
     except Exception as e:
         logger.error(f"chatinfo_cache_save_error: channel {channel_id}, error {str(e)}")
 
 
 def _get_chat_from_cache(channel_id: Union[str, int], max_age_hours: int = CHAT_CACHE_TTL_HOURS) -> Optional[dict]:
-    """Retrieves channel-info from cache if not older than max_age_hours (with up-to-20% randomization)."""
+    """Retrieves channel-info from cache if fresh (jittered TTL)."""
     try:
-        cache_file = _get_chat_cache_file_path(channel_id)
-        if not os.path.exists(cache_file):
-            logger.info(f"chatinfo_cache_miss: channel {channel_id}, cache file not found")
+        cache_file = _cache_file_path(channel_id, 'chatinfo.json')
+        payload = _load_entry(cache_file, max_age_hours)
+        if payload is None:
+            logger.info(f"chatinfo_cache_miss: channel {channel_id}")
             return None
-        with open(cache_file, 'rb') as f:
-            cache_data = pickle.load(f)
-        cache_age = time.time() - cache_data['timestamp']
-        # Add randomness up to 20% of max age, same approach as the history cache.
-        random_factor = 1 - random.uniform(0, 0.2)
-        adjusted_max_age = max_age_hours * 3600 * random_factor
-        if cache_age > adjusted_max_age:
-            logger.info(f"chatinfo_cache_expired: channel {channel_id}, age {cache_age:.1f}s > adjusted max {adjusted_max_age:.1f}s (random factor: {random_factor:.2f})")
-            return None
-        data = cache_data.get('data')
+        data = payload.get('data')
         if not isinstance(data, dict):
             logger.warning(f"chatinfo_cache_invalid: channel {channel_id}, unexpected payload type {type(data).__name__}")
             return None
-        logger.info(f"chatinfo_cache_hit: channel {channel_id}, age {cache_age:.1f}s")
+        logger.info(f"chatinfo_cache_hit: channel {channel_id}")
         return data
     except Exception as e:
         logger.error(f"chatinfo_cache_read_error: channel {channel_id}, error {str(e)}")
@@ -228,3 +262,64 @@ async def cached_get_chat(client: Client, channel_id: Union[str, int]) -> Simple
     }
     await asyncio.to_thread(_save_chat_to_cache, channel_id, data)
     return SimpleNamespace(**data)
+
+
+# --------------------------------------------------------------------------- #
+# Maintenance: legacy cleanup + age sweep.
+# --------------------------------------------------------------------------- #
+def cleanup_legacy_cache_files() -> int:
+    """Delete legacy binary cache files (*.cache incl. *_history.cache, and *.chatinfo).
+
+    The new store uses *.history.json / *.chatinfo.json, so these old-format files are
+    dead weight and would otherwise never be reclaimed. Returns the number removed.
+    """
+    removed = 0
+    if not os.path.isdir(CACHE_DIR):
+        return 0
+    try:
+        names = os.listdir(CACHE_DIR)
+    except OSError as e:
+        logger.warning(f"cleanup_legacy_list_error: dir {CACHE_DIR}, error {str(e)}")
+        return 0
+    for name in names:
+        if name.endswith('.cache') or name.endswith('.chatinfo'):
+            try:
+                os.remove(os.path.join(CACHE_DIR, name))
+                removed += 1
+            except OSError as e:
+                logger.warning(f"cleanup_legacy_remove_error: file {name}, error {str(e)}")
+    if removed:
+        logger.info(f"cleanup_legacy_cache_files: removed {removed} legacy files from {CACHE_DIR}")
+    return removed
+
+
+def sweep_tgcache(max_age_days: int = 7) -> int:
+    """Delete files in CACHE_DIR whose mtime is older than ``max_age_days``.
+
+    Reclaims cache for dead channels and orphaned uuid tmp files. A race with an in-flight
+    writer is POSSIBLE (stat -> unlink is not atomic against os.replace) but harmless: the
+    worst outcome is one extra cache miss. This is NOT an atomicity guarantee. Returns the
+    number of files removed.
+    """
+    removed = 0
+    if not os.path.isdir(CACHE_DIR):
+        return 0
+    cutoff = time.time() - max_age_days * 86400
+    try:
+        names = os.listdir(CACHE_DIR)
+    except OSError as e:
+        logger.warning(f"sweep_tgcache_list_error: dir {CACHE_DIR}, error {str(e)}")
+        return 0
+    for name in names:
+        path = os.path.join(CACHE_DIR, name)
+        try:
+            if not os.path.isfile(path):
+                continue
+            if os.path.getmtime(path) < cutoff:
+                os.remove(path)
+                removed += 1
+        except OSError as e:
+            logger.warning(f"sweep_tgcache_remove_error: file {name}, error {str(e)}")
+    if removed:
+        logger.info(f"sweep_tgcache: removed {removed} stale files (> {max_age_days}d) from {CACHE_DIR}")
+    return removed