Merge pull request 'feat(cache): снапшот-кеш истории вместо pickle + префиксный limit + джиттер + age-sweep' (#44) from feat/23-snapshot-cache into main

api_server.py

@@ -43,6 +43,7 @@ from file_io import (DB_PATH, init_db_sync, get_all_media_file_ids_sync,
                      update_media_file_access_sync, update_media_file_access_bulk_sync,
                      remove_media_file_ids_sync,
                      get_mime_type_sync, set_mime_type_sync)
+from tg_cache import cleanup_legacy_cache_files, sweep_tgcache
 
 # Global python-magic instance for MIME type detection
 magic_mime = magic.Magic(mime=True)
@@ -240,6 +241,19 @@ async def lifespan(_: FastAPI):
     # Initialize SQLite database (creates table if not present)
     await asyncio.to_thread(init_db_sync, DB_PATH)
 
+    # One-shot startup maintenance of the history/chatinfo cache: drop legacy pickle files
+    # (which are now always a miss), age-sweep stale tgcache entries, and remove the
+    # pre-SQLite data/media_file_ids.json legacy dump (no longer read by any code).
+    await asyncio.to_thread(cleanup_legacy_cache_files)
+    await asyncio.to_thread(sweep_tgcache)
+    legacy_media_ids = os.path.join("data", "media_file_ids.json")
+    if os.path.exists(legacy_media_ids):
+        try:
+            await asyncio.to_thread(os.remove, legacy_media_ids)
+            logger.info(f"legacy_media_file_ids_removed: {legacy_media_ids}")
+        except OSError as e:
+            logger.warning(f"legacy_media_file_ids_remove_error: {e}")
+
     await client.start()
     # Supervise the background tasks: if either dies (not via cancellation) it is logged
     # CRITICAL and restarted, so a crash can no longer silently stop cache sweeping or downloads.
@@ -911,6 +925,11 @@ async def cache_media_files() -> None:
                     logger.error(f"Failed to remove old entries from SQLite: {str(e)}")
 
             await download_new_files(updated_media_files, cache_dir)
+
+            # Age-sweep the history/chatinfo cache once per pass (dead channels, orphaned
+            # uuid tmp files). MUST run off-loop via to_thread (blocking filesystem walk).
+            await asyncio.to_thread(sweep_tgcache)
+
             await asyncio.sleep(delay)  # Check every delay seconds
 
         except Exception as e:

message_snapshot.py

@@ -0,0 +1,585 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+# flake8: noqa
+# pylint: disable=broad-exception-raised, raise-missing-from, too-many-arguments, redefined-outer-name
+# pylint: disable=multiple-statements, logging-fstring-interpolation, trailing-whitespace, line-too-long
+# pylint: disable=broad-exception-caught, missing-function-docstring, missing-class-docstring
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+
+"""JSON snapshot / restore for pyrogram Message objects (issue #23).
+
+The history cache no longer pickles live pyrogram objects. On write a plain JSON
+snapshot is extracted from a Message via an explicit allowlist (schema v1); on read
+a duck-typed CachedMessage is restored that is indistinguishable from a real Message
+to the render pipeline (post_parser.py / rss_generator.py are NOT changed).
+"""
+
+import json
+import logging
+from datetime import datetime
+from types import SimpleNamespace
+from typing import Any, List, Optional
+
+from pyrogram.enums import MessageMediaType
+
+logger = logging.getLogger(__name__)
+
+# Bump when the snapshot schema changes in a backwards-incompatible way; a mismatch
+# makes _load_entry treat the cache file as a miss (old files are simply re-fetched).
+# v2: added the special-media info-block types (story, contact, location, venue, dice,
+# game, giveaway, giveaway_winners, checklist, paid_media) plus live_photo. A v1 file lacks
+# these keys, so a cached special-media / live-photo message would restore with an empty
+# block — invalidate v1 files.
+SNAPSHOT_VERSION = 2
+
+
+class CachedStr(str):
+    """A ``str`` subclass carrying a precomputed ``.html`` rendering.
+
+    Mirrors pyrogram's ``Str`` (which exposes ``.html`` computed from entities). Here the
+    HTML is computed once at snapshot time and stored, so the render pipeline can read
+    ``message.text.html`` on a restored message exactly as on a live one. The instance
+    ``__dict__`` (holding ``.html``) is preserved across deepcopy/pickle via ``__reduce__``.
+    """
+
+    html: str
+
+    @classmethod
+    def build(cls, plain: str, html: str) -> "CachedStr":
+        obj = cls(plain)
+        obj.html = html
+        return obj
+
+    def __reduce__(self):
+        # Preserve the precomputed .html across copy.deepcopy() and pickle. Without this a
+        # str subclass would be reconstructed as a bare str and lose the instance __dict__.
+        return (CachedStr.build, (str(self), self.html))
+
+
+def _unwrap_text(value: Any) -> Optional[str]:
+    """Unwrap a styled-text container to a plain string.
+
+    In kurigram 2.2.23 Poll.question and every PollOption.text are always FormattedText
+    objects (``.text`` holds the string). The rule ``v.text if hasattr(v, 'text') else
+    str(v)`` is correct for FormattedText, for a bare Str and for a plain string. Storing
+    a FormattedText as-is would make json.dump raise TypeError, silently killing the cache
+    for any channel that has polls.
+    """
+    if value is None:
+        return None
+    inner = value.text if hasattr(value, "text") else value
+    return str(inner)
+
+
+def _snapshot_str(value: Any) -> Optional[dict]:
+    """Snapshot a text/caption value: {'plain', 'html'} or None if absent.
+
+    ``.html`` is computed at write time (live pyrogram Str exposes it); if unavailable the
+    html falls back to the plain text.
+    """
+    if value is None:
+        return None
+    plain = str(value)
+    html = getattr(value, "html", None)
+    if html is None:
+        html = plain
+    return {"plain": plain, "html": str(html)}
+
+
+def _enum_name(value: Any) -> Any:
+    """Return an enum member's ``.name`` (JSON-safe) or the value unchanged."""
+    if value is None:
+        return None
+    return value.name if hasattr(value, "name") else value
+
+
+def _snapshot_obj(obj: Any, keys: List[str]) -> Optional[dict]:
+    """Snapshot the allowlisted ``keys`` off ``obj`` (getattr-only), or None if obj is None."""
+    if obj is None:
+        return None
+    return {k: getattr(obj, k, None) for k in keys}
+
+
+def _snapshot_chat(chat: Any) -> Optional[dict]:
+    if chat is None:
+        return None
+    usernames = getattr(chat, "usernames", None)
+    snap_usernames = None
+    if usernames:
+        snap_usernames = [
+            {"username": getattr(u, "username", None), "active": getattr(u, "active", None)}
+            for u in usernames
+        ]
+    return {
+        "id": getattr(chat, "id", None),
+        "username": getattr(chat, "username", None),
+        "title": getattr(chat, "title", None),
+        "usernames": snap_usernames,
+    }
+
+
+# forward_origin is the ONE object whose key SET mirrors the live object: the
+# MessageOrigin* classes each carry a DIFFERENT set of attributes and
+# _format_forward_info branches by hasattr (Case 1-5). Presence of a key in the
+# snapshot must mirror presence of the attribute on the source object.
+_FORWARD_ORIGIN_KEYS = ["type", "chat", "sender_user_name", "sender_user", "chat_id", "title"]
+
+
+def _snapshot_forward_origin(fo: Any) -> Optional[dict]:
+    if fo is None:
+        return None
+    result: dict = {}
+    for key in _FORWARD_ORIGIN_KEYS:
+        if not hasattr(fo, key):
+            continue
+        value = getattr(fo, key)
+        if key == "type":
+            result[key] = _enum_name(value)
+        elif key == "chat":
+            result[key] = _snapshot_obj(value, ["id", "title", "username"])
+        elif key == "sender_user":
+            result[key] = _snapshot_obj(value, ["first_name", "last_name", "username"])
+        else:
+            result[key] = value
+    return result
+
+
+def _snapshot_reactions(reactions: Any) -> Optional[list]:
+    if reactions is None:
+        return None
+    rlist = getattr(reactions, "reactions", None)
+    if rlist is None:
+        return None
+    out = []
+    for r in rlist:
+        cid = getattr(r, "custom_emoji_id", None)
+        out.append({
+            # Custom-emoji reactions carry no unicode emoji: emoji is null and the
+            # custom_emoji_id is stored as a STRING = str(document_id).
+            "emoji": None if cid is not None else getattr(r, "emoji", None),
+            "custom_emoji_id": str(cid) if cid is not None else None,
+            "count": getattr(r, "count", None),
+            "is_paid": getattr(r, "is_paid", None),
+        })
+    return out
+
+
+def _snapshot_poll(poll: Any) -> Optional[dict]:
+    if poll is None:
+        return None
+    options = getattr(poll, "options", None)
+    snap_options = None
+    if options:
+        snap_options = [{"text": _unwrap_text(getattr(o, "text", None))} for o in options]
+    return {
+        "question": _unwrap_text(getattr(poll, "question", None)),
+        "options": snap_options,
+    }
+
+
+def _snapshot_web_page(wp: Any) -> Optional[dict]:
+    if wp is None:
+        return None
+    return {
+        "type": getattr(wp, "type", None),
+        "url": getattr(wp, "url", None),
+        "display_url": getattr(wp, "display_url", None),
+        "site_name": getattr(wp, "site_name", None),
+        "title": getattr(wp, "title", None),
+        "description": getattr(wp, "description", None),
+        "has_large_media": getattr(wp, "has_large_media", None),
+        "photo": _snapshot_obj(getattr(wp, "photo", None), ["file_unique_id"]),
+    }
+
+
+# --------------------------------------------------------------------------- #
+# Special-media snapshots (issue #23 review fix).
+#
+# _format_special_media (post_parser.py) and find_file_id_in_message
+# (api_server.py) read a handful of type-specific attributes off the Message that
+# were NOT in schema v1. On a cache hit a restored Message carried media=<enum> but
+# message.<attr>=None, so the special info block silently vanished (render
+# divergence vs. a live Message). Each helper below snapshots EXACTLY the sub-fields
+# the renderer + find_file_id read for that type — nothing more, nothing less.
+# --------------------------------------------------------------------------- #
+def _snapshot_story(story: Any) -> Optional[dict]:
+    """STORY: _story_media_object / find_file_id_in_message read story.video and
+    story.photo, each by .file_unique_id (video wins over photo). The chosen object is
+    also the one _save_media_file_ids reads .file_size off for the >100MB skip, so
+    file_size is snapshotted too (else a >100MB story media would be collected on a
+    cache hit — F1). The render URL is still built from file_unique_id."""
+    if story is None:
+        return None
+    return {
+        "video": _snapshot_obj(getattr(story, "video", None), ["file_unique_id", "file_size"]),
+        "photo": _snapshot_obj(getattr(story, "photo", None), ["file_unique_id", "file_size"]),
+    }
+
+
+def _snapshot_venue(venue: Any) -> Optional[dict]:
+    """VENUE: reads .title, .address and its nested .location (.latitude/.longitude via
+    _format_osm_link)."""
+    if venue is None:
+        return None
+    return {
+        "title": getattr(venue, "title", None),
+        "address": getattr(venue, "address", None),
+        "location": _snapshot_obj(getattr(venue, "location", None), ["latitude", "longitude"]),
+    }
+
+
+def _snapshot_giveaway(giveaway: Any) -> Optional[dict]:
+    """GIVEAWAY: reads .quantity, .months, .stars, .until_date (a datetime rendered via
+    strftime) and .description. until_date is stored as isoformat and restored to a
+    datetime so the strftime branch reproduces byte-for-byte."""
+    if giveaway is None:
+        return None
+    until = getattr(giveaway, "until_date", None)
+    return {
+        "quantity": getattr(giveaway, "quantity", None),
+        "months": getattr(giveaway, "months", None),
+        "stars": getattr(giveaway, "stars", None),
+        # Only a real datetime renders (renderer gates on hasattr(.,'strftime')); anything
+        # else is dropped to None so the same "no until date" branch is taken on restore.
+        "until_date": until.isoformat() if hasattr(until, "isoformat") else None,
+        "description": getattr(giveaway, "description", None),
+    }
+
+
+def _snapshot_checklist(checklist: Any) -> Optional[dict]:
+    """CHECKLIST: reads .title (used ONLY when it isinstance str) and .tasks; each task
+    reads .text and the truthiness of .completed_by / .completion_date (→ ☑/☐).
+
+    title is stored only when it is a str (matching the renderer's isinstance gate, which
+    otherwise renders an empty title). completed_by / completion_date are stored as bools
+    (the renderer only tests their truthiness) so a live User/datetime stays JSON-safe."""
+    if checklist is None:
+        return None
+    title = getattr(checklist, "title", None)
+    snap_tasks = []
+    for task in (getattr(checklist, "tasks", None) or []):
+        text = getattr(task, "text", "")
+        # Mirror the renderer: a non-str text is str()-ified. Storing the resolved string
+        # keeps the restored value isinstance-str, reproducing the same rendered bytes.
+        text_str = text if isinstance(text, str) else str(text)
+        snap_tasks.append({
+            "text": text_str,
+            "completed_by": bool(getattr(task, "completed_by", None)),
+            "completion_date": bool(getattr(task, "completion_date", None)),
+        })
+    return {
+        "title": title if isinstance(title, str) else None,
+        "tasks": snap_tasks,
+    }
+
+
+def _snapshot_paid_media(paid_media: Any) -> Optional[dict]:
+    """PAID_MEDIA: _generate_html_media reads .stars_amount (default 0) and the LENGTH of
+    .media. Snapshot the renderer's effective stars value and the item count; the media
+    list is restored as N placeholders so len() reproduces."""
+    if paid_media is None:
+        return None
+    media = getattr(paid_media, "media", None)
+    count = len(media) if isinstance(media, (list, tuple)) else 0
+    return {
+        # Snapshot getattr(...,0) so a missing attribute restores to 0 exactly as the
+        # renderer would have defaulted it (a present None stays None on both sides).
+        "stars_amount": getattr(paid_media, "stars_amount", 0),
+        "media_count": count,
+    }
+
+
+def snapshot_message(message: Any) -> dict:
+    """Extract a JSON-serializable snapshot (schema v1) from a pyrogram Message.
+
+    Every field is read via getattr(..., None). See the module docstring / issue #23 for
+    the field contract.
+    """
+    date = getattr(message, "date", None)
+    media = getattr(message, "media", None)
+    service = getattr(message, "service", None)
+    return {
+        "id": getattr(message, "id", None),
+        # isoformat()/fromisoformat() preserve naive/aware exactly as pyrogram stores it.
+        "date": date.isoformat() if date is not None else None,
+        "text": _snapshot_str(getattr(message, "text", None)),
+        "caption": _snapshot_str(getattr(message, "caption", None)),
+        "media": media.name if media is not None else None,
+        "service": service.name if service is not None else None,
+        "media_group_id": getattr(message, "media_group_id", None),
+        "views": getattr(message, "views", None),
+        "show_caption_above_media": getattr(message, "show_caption_above_media", None),
+        "reply_to_message_id": getattr(message, "reply_to_message_id", None),
+        "empty": getattr(message, "empty", None),
+        "chat": _snapshot_chat(getattr(message, "chat", None)),
+        "sender_chat": _snapshot_obj(getattr(message, "sender_chat", None), ["id", "title", "username"]),
+        "from_user": _snapshot_obj(getattr(message, "from_user", None), ["first_name", "last_name", "username"]),
+        "forward_origin": _snapshot_forward_origin(getattr(message, "forward_origin", None)),
+        "reactions": _snapshot_reactions(getattr(message, "reactions", None)),
+        "poll": _snapshot_poll(getattr(message, "poll", None)),
+        "web_page": _snapshot_web_page(getattr(message, "web_page", None)),
+        "photo": _snapshot_obj(getattr(message, "photo", None), ["file_unique_id"]),
+        "video": _snapshot_obj(getattr(message, "video", None), ["file_unique_id", "file_size"]),
+        # file_size is snapshotted for every type whose selected object flows into
+        # _save_media_file_ids' >100MB skip (MEDIA_SOURCES: document/audio/animation/
+        # video_note select this exact object). Without it a restored >100MB media
+        # would have file_size=None and be wrongly collected on a cache hit (F1).
+        "document": _snapshot_obj(getattr(message, "document", None), ["file_unique_id", "mime_type", "file_size"]),
+        "audio": _snapshot_obj(getattr(message, "audio", None), ["file_unique_id", "mime_type", "file_size"]),
+        "voice": _snapshot_obj(getattr(message, "voice", None), ["file_unique_id", "mime_type"]),
+        "video_note": _snapshot_obj(getattr(message, "video_note", None), ["file_unique_id", "file_size"]),
+        "animation": _snapshot_obj(getattr(message, "animation", None), ["file_unique_id", "file_size"]),
+        "sticker": _snapshot_obj(getattr(message, "sticker", None), ["file_unique_id", "emoji", "is_video"]),
+        # LIVE_PHOTO (Kurigram 2.2.23) renders as a video element via the video_loop_400
+        # kind. MEDIA_SOURCES/_get_file_unique_id/_save_media_file_ids read live_photo's
+        # file_unique_id + file_size (the >100MB skip); find_file_id also reads file_unique_id.
+        # Mirror the `video` allowlist so a cached live-photo message keeps its media block.
+        "live_photo": _snapshot_obj(getattr(message, "live_photo", None), ["file_unique_id", "file_size"]),
+        # Special-media info-block types (issue #23 review fix). Each reads a fixed set of
+        # sub-fields in _format_special_media / find_file_id_in_message; snapshot exactly those.
+        "story": _snapshot_story(getattr(message, "story", None)),
+        "contact": _snapshot_obj(getattr(message, "contact", None), ["first_name", "last_name", "phone_number"]),
+        "location": _snapshot_obj(getattr(message, "location", None), ["latitude", "longitude"]),
+        "venue": _snapshot_venue(getattr(message, "venue", None)),
+        "dice": _snapshot_obj(getattr(message, "dice", None), ["emoji", "value"]),
+        "game": _snapshot_obj(getattr(message, "game", None), ["title"]),
+        "giveaway": _snapshot_giveaway(getattr(message, "giveaway", None)),
+        "giveaway_winners": _snapshot_obj(
+            getattr(message, "giveaway_winners", None), ["winner_count", "quantity", "prize_description"]),
+        "checklist": _snapshot_checklist(getattr(message, "checklist", None)),
+        "paid_media": _snapshot_paid_media(getattr(message, "paid_media", None)),
+    }
+
+
+def _ns(d: Optional[dict], keys: List[str]) -> Optional[SimpleNamespace]:
+    """Restore a nested object with the FULL schema key set (None-defaults).
+
+    Live pyrogram objects always set every attribute in __init__, and the pipeline reads
+    them directly (e.g. rss_generator ~131 message.chat.username in an except; post_parser
+    ~1087 u.username / u.active without getattr). So restored objects must carry all keys.
+    Returns None if ``d`` is None (the attribute was absent on the live object).
+    """
+    if d is None:
+        return None
+    ns = SimpleNamespace()
+    for k in keys:
+        setattr(ns, k, d.get(k))
+    return ns
+
+
+def _restore_chat(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    usernames = d.get("usernames")
+    restored_usernames = None
+    if usernames is not None:
+        restored_usernames = [_ns(u, ["username", "active"]) for u in usernames]
+    return SimpleNamespace(
+        id=d.get("id"),
+        username=d.get("username"),
+        title=d.get("title"),
+        usernames=restored_usernames,
+    )
+
+
+def _restore_forward_origin(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    """Restore forward_origin mirroring ONLY the recorded keys (presence-semantics)."""
+    if d is None:
+        return None
+    ns = SimpleNamespace()
+    for key, value in d.items():
+        if key == "chat":
+            setattr(ns, key, _ns(value, ["id", "title", "username"]))
+        elif key == "sender_user":
+            setattr(ns, key, _ns(value, ["first_name", "last_name", "username"]))
+        else:
+            setattr(ns, key, value)
+    return ns
+
+
+def _restore_reactions(items: Optional[list]) -> Optional[SimpleNamespace]:
+    if items is None:
+        return None
+    reactions = [_ns(r, ["emoji", "custom_emoji_id", "count", "is_paid"]) for r in items]
+    # Mirror pyrogram: message.reactions is an object exposing a .reactions list.
+    return SimpleNamespace(reactions=reactions)
+
+
+def _restore_poll(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    options = d.get("options") or []
+    # options must be namespace objects with a .text string; a bare string would make
+    # getattr(option, 'text', '') return '' and render empty options.
+    restored_options = [SimpleNamespace(text=o.get("text")) for o in options]
+    return SimpleNamespace(question=d.get("question"), options=restored_options)
+
+
+def _restore_web_page(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    return SimpleNamespace(
+        type=d.get("type"),
+        url=d.get("url"),
+        display_url=d.get("display_url"),
+        site_name=d.get("site_name"),
+        title=d.get("title"),
+        description=d.get("description"),
+        has_large_media=d.get("has_large_media"),
+        photo=_ns(d.get("photo"), ["file_unique_id"]),
+    )
+
+
+def _restore_media(name: Optional[str]) -> Optional[MessageMediaType]:
+    if name is None:
+        return None
+    try:
+        return MessageMediaType[name]
+    except KeyError:
+        # A media type unknown to this kurigram build: warn and drop to None so the 31
+        # post_parser sites that compare against / index by the enum keep working.
+        logger.warning(f"snapshot_unknown_media_type: {name}")
+        return None
+
+
+def _restore_str(d: Optional[dict]) -> Optional[CachedStr]:
+    if d is None:
+        return None
+    return CachedStr.build(d.get("plain", ""), d.get("html", d.get("plain", "")))
+
+
+# --------------------------------------------------------------------------- #
+# Special-media restores (issue #23 review fix). Mirror the snapshot helpers above:
+# rebuild each object so the exact attributes _format_special_media /
+# find_file_id_in_message read exist, restoring the special block on a cache hit.
+# --------------------------------------------------------------------------- #
+def _restore_story(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    return SimpleNamespace(
+        video=_ns(d.get("video"), ["file_unique_id", "file_size"]),
+        photo=_ns(d.get("photo"), ["file_unique_id", "file_size"]),
+    )
+
+
+def _restore_venue(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    return SimpleNamespace(
+        title=d.get("title"),
+        address=d.get("address"),
+        location=_ns(d.get("location"), ["latitude", "longitude"]),
+    )
+
+
+def _restore_giveaway(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    until = d.get("until_date")
+    return SimpleNamespace(
+        quantity=d.get("quantity"),
+        months=d.get("months"),
+        stars=d.get("stars"),
+        # Restore a datetime so hasattr(until_date, 'strftime') holds exactly as on a live
+        # object; None (no/invalid date) restores as None and the strftime branch is skipped.
+        until_date=datetime.fromisoformat(until) if until is not None else None,
+        description=d.get("description"),
+    )
+
+
+def _restore_checklist(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    restored_tasks = [
+        SimpleNamespace(
+            text=t.get("text"),
+            completed_by=t.get("completed_by"),
+            completion_date=t.get("completion_date"),
+        )
+        for t in (d.get("tasks") or [])
+    ]
+    return SimpleNamespace(title=d.get("title"), tasks=restored_tasks)
+
+
+def _restore_paid_media(d: Optional[dict]) -> Optional[SimpleNamespace]:
+    if d is None:
+        return None
+    count = d.get("media_count") or 0
+    # media is only ever len()'d by the renderer, so N placeholder items reproduce the count.
+    return SimpleNamespace(stars_amount=d.get("stars_amount"), media=[None] * count)
+
+
+class CachedMessage:
+    """Duck-typed stand-in for a pyrogram Message, restored from a snapshot dict.
+
+    Mutable (reply enrichment assigns ``.reply_to_message``). Every top-level attribute the
+    render pipeline reads exists with a None/False default so getattr never raises.
+    """
+
+    def __init__(self, data: dict):
+        self._snapshot = data
+        self.id = data.get("id")
+        date = data.get("date")
+        self.date = datetime.fromisoformat(date) if date is not None else None
+        self.text = _restore_str(data.get("text"))
+        self.caption = _restore_str(data.get("caption"))
+        self.media = _restore_media(data.get("media"))
+        # service restored as a plain string; all consumers use truthiness and
+        # `'X' in str(service)`.
+        self.service = data.get("service")
+        self.media_group_id = data.get("media_group_id")
+        self.views = data.get("views")
+        self.show_caption_above_media = data.get("show_caption_above_media")
+        self.reply_to_message_id = data.get("reply_to_message_id")
+        self.reply_to_message = None
+        self.empty = bool(data.get("empty"))
+        self.chat = _restore_chat(data.get("chat"))
+        self.sender_chat = _ns(data.get("sender_chat"), ["id", "title", "username"])
+        self.from_user = _ns(data.get("from_user"), ["first_name", "last_name", "username"])
+        self.forward_origin = _restore_forward_origin(data.get("forward_origin"))
+        self.reactions = _restore_reactions(data.get("reactions"))
+        self.poll = _restore_poll(data.get("poll"))
+        self.web_page = _restore_web_page(data.get("web_page"))
+        self.photo = _ns(data.get("photo"), ["file_unique_id"])
+        self.video = _ns(data.get("video"), ["file_unique_id", "file_size"])
+        self.document = _ns(data.get("document"), ["file_unique_id", "mime_type", "file_size"])
+        self.audio = _ns(data.get("audio"), ["file_unique_id", "mime_type", "file_size"])
+        self.voice = _ns(data.get("voice"), ["file_unique_id", "mime_type"])
+        self.video_note = _ns(data.get("video_note"), ["file_unique_id", "file_size"])
+        self.animation = _ns(data.get("animation"), ["file_unique_id", "file_size"])
+        self.sticker = _ns(data.get("sticker"), ["file_unique_id", "emoji", "is_video"])
+        # LIVE_PHOTO: restored like `video` so the video_loop_400 media block renders on a
+        # cache hit (file_unique_id → URL; file_size → the >100MB collection skip).
+        self.live_photo = _ns(data.get("live_photo"), ["file_unique_id", "file_size"])
+        # Special-media info-block types (issue #23 review fix): restored so the type's
+        # info block renders on a cache hit exactly as for a live Message.
+        self.story = _restore_story(data.get("story"))
+        self.contact = _ns(data.get("contact"), ["first_name", "last_name", "phone_number"])
+        self.location = _ns(data.get("location"), ["latitude", "longitude"])
+        self.venue = _restore_venue(data.get("venue"))
+        self.dice = _ns(data.get("dice"), ["emoji", "value"])
+        self.game = _ns(data.get("game"), ["title"])
+        self.giveaway = _restore_giveaway(data.get("giveaway"))
+        self.giveaway_winners = _ns(
+            data.get("giveaway_winners"), ["winner_count", "quantity", "prize_description"])
+        self.checklist = _restore_checklist(data.get("checklist"))
+        self.paid_media = _restore_paid_media(data.get("paid_media"))
+
+    def __str__(self) -> str:
+        return json.dumps(self._snapshot, default=str)
+
+    def __repr__(self) -> str:
+        return self.__str__()
+
+
+def restore_message(data: dict) -> CachedMessage:
+    return CachedMessage(data)
+
+
+def snapshot_messages(messages: List[Any]) -> List[dict]:
+    return [snapshot_message(m) for m in messages]
+
+
+def restore_messages(items: List[dict]) -> List[CachedMessage]:
+    return [restore_message(d) for d in items]

tests/test_message_snapshot.py

@@ -0,0 +1,510 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""Tests for message_snapshot.py (issue #23, Задания 4-5,7).
+
+These verify that a pyrogram Message survives snapshot -> JSON -> restore as a
+CachedMessage that the render pipeline (post_parser.py) consumes identically.
+"""
+import copy
+import json
+from datetime import datetime, timezone
+from types import SimpleNamespace
+
+import pytest
+
+from pyrogram.enums import MessageMediaType, MessageServiceType
+
+from message_snapshot import (
+    snapshot_message,
+    restore_message,
+    snapshot_messages,
+    restore_messages,
+    CachedStr,
+    CachedMessage,
+)
+from post_parser import PostParser
+
+
+class FakeStr(str):
+    """Stand-in for a live pyrogram Str: a str carrying a .html rendering."""
+    def __new__(cls, plain, html):
+        obj = str.__new__(cls, plain)
+        obj._html = html
+        return obj
+
+    @property
+    def html(self):
+        return self._html
+
+
+def _roundtrip(message):
+    snap = snapshot_message(message)
+    # The snapshot MUST be JSON-serializable (this is the whole point of the rewrite).
+    json.dumps(snap)
+    return restore_message(snap), snap
+
+
+# --------------------------------------------------------------------------- #
+# Test 1 — top-level round trip.
+# --------------------------------------------------------------------------- #
+def test_roundtrip_basic_fields():
+    msg = SimpleNamespace(
+        id=42,
+        date=datetime(2020, 1, 2, 3, 4, 5),
+        text=FakeStr("hello", "<b>hello</b>"),
+        caption=FakeStr("cap", "<i>cap</i>"),
+        media=MessageMediaType.PHOTO,
+        views=123,
+        media_group_id="mg-1",
+    )
+    restored, _ = _roundtrip(msg)
+    assert restored.id == 42
+    assert restored.date == datetime(2020, 1, 2, 3, 4, 5)
+    assert restored.text.html == "<b>hello</b>"
+    assert str(restored.text) == "hello"
+    assert restored.caption.html == "<i>cap</i>"
+    assert restored.media is MessageMediaType.PHOTO
+    assert restored.views == 123
+    assert restored.media_group_id == "mg-1"
+
+
+def test_roundtrip_date_naive_and_aware():
+    naive = SimpleNamespace(date=datetime(2021, 5, 5, 10, 0, 0))
+    aware = SimpleNamespace(date=datetime(2021, 5, 5, 10, 0, 0, tzinfo=timezone.utc))
+
+    r_naive, _ = _roundtrip(naive)
+    r_aware, _ = _roundtrip(aware)
+
+    assert r_naive.date.tzinfo is None
+    assert r_aware.date.tzinfo is not None
+    assert r_aware.date == datetime(2021, 5, 5, 10, 0, 0, tzinfo=timezone.utc)
+
+
+def test_text_falls_back_to_plain_when_no_html():
+    # A bare str (no .html) -> html defaults to the plain text.
+    msg = SimpleNamespace(text="plain only")
+    restored, _ = _roundtrip(msg)
+    assert restored.text.html == "plain only"
+    assert isinstance(restored.text, CachedStr)
+
+
+# --------------------------------------------------------------------------- #
+# Test 2 — polls with FormattedText fakes (must be JSON-serializable).
+# --------------------------------------------------------------------------- #
+def test_poll_formatted_text_unwrapped():
+    option = SimpleNamespace(text=SimpleNamespace(text="Opt", entities=[]))
+    poll = SimpleNamespace(
+        question=SimpleNamespace(text="Q?", entities=[]),
+        options=[option],
+    )
+    msg = SimpleNamespace(poll=poll)
+
+    snap = snapshot_message(msg)
+    # This MUST NOT raise. If the snapshot stored the FormattedText objects as-is, the
+    # SimpleNamespace values would make json.dumps raise TypeError (regression guard).
+    json.dumps(snap)
+
+    restored = restore_message(snap)
+    assert restored.poll.question == "Q?"
+    assert isinstance(restored.poll.question, str)
+    opt = restored.poll.options[0]
+    assert getattr(opt, "text", "") == "Opt"
+
+
+def test_poll_snapshot_would_fail_if_formattedtext_kept_raw():
+    # Guard proving the assertion above actually catches raw FormattedText: a snapshot dict
+    # that carried the FormattedText object would not be JSON-serializable.
+    bad_snap = {"poll": {"question": SimpleNamespace(text="Q?"), "options": []}}
+    with pytest.raises(TypeError):
+        json.dumps(bad_snap)
+
+
+# --------------------------------------------------------------------------- #
+# Test 3 — reactions (normal / paid / custom).
+# --------------------------------------------------------------------------- #
+def test_reactions_normal_paid_custom():
+    normal = SimpleNamespace(emoji="👍", count=5, is_paid=False)
+    paid = SimpleNamespace(count=3, is_paid=True)
+    custom = SimpleNamespace(custom_emoji_id=1234567890, count=2)
+    msg = SimpleNamespace(reactions=SimpleNamespace(reactions=[normal, paid, custom]))
+
+    restored, _ = _roundtrip(msg)
+    r_normal, r_paid, r_custom = restored.reactions.reactions
+
+    assert r_normal.emoji == "👍"
+    assert r_normal.count == 5
+    assert r_normal.is_paid is False
+    assert r_normal.custom_emoji_id is None
+
+    assert r_paid.is_paid is True
+    assert r_paid.count == 3
+
+    # Custom emoji: emoji is null, custom_emoji_id is a STRING; key still present.
+    assert hasattr(r_custom, "emoji") is True
+    assert r_custom.emoji is None
+    assert isinstance(r_custom.custom_emoji_id, str)
+    assert r_custom.custom_emoji_id == "1234567890"
+
+
+# --------------------------------------------------------------------------- #
+# Test 4 — forward_origin Case 1-5 (presence-semantics drives _format_forward_info).
+# --------------------------------------------------------------------------- #
+def _forward_html(forward_origin):
+    msg = SimpleNamespace(forward_origin=forward_origin)
+    restored, _ = _roundtrip(msg)
+    return PostParser(None)._format_forward_info(restored)
+
+
+def test_forward_origin_cases():
+    # Case 1: channel/supergroup (has .chat)
+    html = _forward_html(SimpleNamespace(
+        type="channel",
+        chat=SimpleNamespace(id=-100, title="Chan", username="chanu"),
+    ))
+    assert "Chan" in html and "@chanu" in html
+
+    # Case 2: hidden user (sender_user_name)
+    html = _forward_html(SimpleNamespace(type="hidden_user", sender_user_name="Hidden Guy"))
+    assert "Hidden Guy" in html
+
+    # Case 3: regular user (sender_user)
+    html = _forward_html(SimpleNamespace(
+        type="user",
+        sender_user=SimpleNamespace(first_name="Ann", last_name="Bee", username="annbee"),
+    ))
+    assert "Ann Bee" in html and "@annbee" in html
+
+    # Case 4: channel without username (chat_id + title, no chat attr)
+    html = _forward_html(SimpleNamespace(type="channel", chat_id=-1002, title="NoUserChan"))
+    assert "NoUserChan" in html
+
+    # Case 5: anything else
+    html = _forward_html(SimpleNamespace(type="something_else"))
+    assert html == '<div class="message-forward">--- Forwarded message ---</div>'
+
+
+def test_forward_origin_presence_semantics():
+    # Only keys present on the live object are recorded (hasattr semantics).
+    restored, snap = _roundtrip(SimpleNamespace(
+        forward_origin=SimpleNamespace(type="hidden_user", sender_user_name="X")))
+    assert "chat" not in snap["forward_origin"]
+    assert "sender_user" not in snap["forward_origin"]
+    assert not hasattr(restored.forward_origin, "chat")
+    assert hasattr(restored.forward_origin, "sender_user_name")
+
+
+# --------------------------------------------------------------------------- #
+# Test 5 — service restored as string usable in `'X' in str(service)`.
+# --------------------------------------------------------------------------- #
+def test_service_pinned_message():
+    msg = SimpleNamespace(service=MessageServiceType.PINNED_MESSAGE)
+    restored, _ = _roundtrip(msg)
+    assert "PINNED_MESSAGE" in str(restored.service)
+
+
+# --------------------------------------------------------------------------- #
+# Test 6 — mutability + deepcopy (.text.html survives).
+# --------------------------------------------------------------------------- #
+def test_mutable_and_deepcopy():
+    msg = SimpleNamespace(id=7, text=FakeStr("body", "<u>body</u>"))
+    restored, _ = _roundtrip(msg)
+
+    # Mutable: reply enrichment assigns reply_to_message.
+    sentinel = object()
+    restored.reply_to_message = sentinel
+    assert restored.reply_to_message is sentinel
+
+    clone = copy.deepcopy(restored)
+    assert clone.text.html == "<u>body</u>"
+    assert str(clone.text) == "body"
+    assert clone.id == 7
+
+
+def test_cachedstr_survives_pickle():
+    import pickle as _pkl
+    s = CachedStr.build("plain", "<b>plain</b>")
+    back = _pkl.loads(_pkl.dumps(s))
+    assert str(back) == "plain"
+    assert back.html == "<b>plain</b>"
+
+
+# --------------------------------------------------------------------------- #
+# Test 7 — unknown media name -> None, no exception.
+# --------------------------------------------------------------------------- #
+def test_unknown_media_type():
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = "FUTURE_TYPE"
+    restored = restore_message(snap)  # must not raise
+    assert restored.media is None
+
+
+# --------------------------------------------------------------------------- #
+# Test 9 — >100MB video is not collected by _save_media_file_ids.
+# --------------------------------------------------------------------------- #
+def test_large_video_not_collected():
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = "VIDEO"
+    snap["video"] = {"file_unique_id": "vid1", "file_size": 200 * 1024 * 1024}
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert pp._pending_media_ids == []
+
+
+def test_normal_video_is_collected():
+    snap = snapshot_message(SimpleNamespace())
+    snap["id"] = 10
+    snap["media"] = "VIDEO"
+    snap["video"] = {"file_unique_id": "vid2", "file_size": 1024}
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert len(pp._pending_media_ids) == 1
+    assert pp._pending_media_ids[0][2] == "vid2"
+
+
+# --------------------------------------------------------------------------- #
+# Test 9b — the >100MB skip must fire for every media type whose selected object
+# flows into _save_media_file_ids' size check, not just video/live_photo. Before
+# file_size was added to these snapshots a restored >100MB doc/audio/animation/
+# video_note lacked file_size and was wrongly collected on a cache hit (F1).
+# --------------------------------------------------------------------------- #
+_BIG = 200 * 1024 * 1024
+
+
+@pytest.mark.parametrize("media,attr,snap_obj", [
+    ("DOCUMENT",   "document",   {"file_unique_id": "doc_big",  "mime_type": "application/zip", "file_size": _BIG}),
+    ("AUDIO",      "audio",      {"file_unique_id": "aud_big",  "mime_type": "audio/mpeg",      "file_size": _BIG}),
+    ("ANIMATION",  "animation",  {"file_unique_id": "anim_big", "file_size": _BIG}),
+    ("VIDEO_NOTE", "video_note", {"file_unique_id": "vn_big",   "file_size": _BIG}),
+])
+def test_large_media_not_collected(media, attr, snap_obj):
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = media
+    snap[attr] = snap_obj
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+    # file_size survives the snapshot/restore round-trip (the round-trip is the fix).
+    assert getattr(restored, attr).file_size == _BIG
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert pp._pending_media_ids == []
+
+
+@pytest.mark.parametrize("media,attr,fuid,snap_obj", [
+    ("DOCUMENT",   "document",   "doc_ok",  {"file_unique_id": "doc_ok",  "mime_type": "application/zip", "file_size": 1024}),
+    ("AUDIO",      "audio",      "aud_ok",  {"file_unique_id": "aud_ok",  "mime_type": "audio/mpeg",      "file_size": 1024}),
+    ("ANIMATION",  "animation",  "anim_ok", {"file_unique_id": "anim_ok", "file_size": 1024}),
+    ("VIDEO_NOTE", "video_note", "vn_ok",   {"file_unique_id": "vn_ok",   "file_size": 1024}),
+])
+def test_normal_media_is_collected(media, attr, fuid, snap_obj):
+    snap = snapshot_message(SimpleNamespace())
+    snap["id"] = 11
+    snap["media"] = media
+    snap[attr] = snap_obj
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert len(pp._pending_media_ids) == 1
+    assert pp._pending_media_ids[0][2] == fuid
+
+
+def test_large_story_media_not_collected():
+    # A >100MB story video (selected over the photo) must be skipped on a cache hit.
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = "STORY"
+    snap["story"] = {
+        "video": {"file_unique_id": "story_vid_big", "file_size": _BIG},
+        "photo": None,
+    }
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+    assert restored.story.video.file_size == _BIG
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert pp._pending_media_ids == []
+
+
+# --------------------------------------------------------------------------- #
+# Test 10 — restored chat without username -> None, not AttributeError.
+# --------------------------------------------------------------------------- #
+def test_chat_without_username():
+    snap = snapshot_message(SimpleNamespace())
+    snap["chat"] = {"id": -100, "username": None, "title": "T", "usernames": None}
+    restored = restore_message(snap)
+    assert restored.chat.username is None
+
+
+def test_chat_usernames_restored_as_objects():
+    chat = SimpleNamespace(
+        id=-100, username=None, title="T",
+        usernames=[SimpleNamespace(username="alt", active=True),
+                   SimpleNamespace(username="old", active=False)],
+    )
+    restored, _ = _roundtrip(SimpleNamespace(chat=chat))
+    active = [u.username for u in restored.chat.usernames if u.active]
+    assert active == ["alt"]
+    # get_channel_username uses exactly this path.
+    assert PostParser(None).get_channel_username(restored) == "alt"
+
+
+# --------------------------------------------------------------------------- #
+# Defaults: every pipeline-consumed attribute exists (getattr never raises).
+# --------------------------------------------------------------------------- #
+def test_empty_message_defaults():
+    restored = restore_message(snapshot_message(SimpleNamespace()))
+    for attr in ["id", "date", "text", "caption", "media", "service", "media_group_id",
+                 "views", "show_caption_above_media", "reply_to_message_id",
+                 "reply_to_message", "chat", "sender_chat", "from_user", "forward_origin",
+                 "reactions", "poll", "web_page", "photo", "video", "document", "audio",
+                 "voice", "video_note", "animation", "sticker"]:
+        assert hasattr(restored, attr)
+    assert restored.empty is False
+
+
+def test_snapshot_messages_list_roundtrip():
+    msgs = [SimpleNamespace(id=i) for i in range(3)]
+    restored = restore_messages(snapshot_messages(msgs))
+    assert [m.id for m in restored] == [0, 1, 2]
+    assert all(isinstance(m, CachedMessage) for m in restored)
+
+
+# --------------------------------------------------------------------------- #
+# Special-media round trips (issue #23 review fix). One per type: assert the exact
+# sub-fields _format_special_media / find_file_id_in_message read survive the trip.
+# These are what silently vanished on a cache hit before the allowlist was completed.
+# --------------------------------------------------------------------------- #
+def test_story_roundtrip_file_unique_id_survives():
+    story = SimpleNamespace(
+        video=SimpleNamespace(file_unique_id="story-vid"),
+        photo=SimpleNamespace(file_unique_id="story-pic"),
+    )
+    restored, _ = _roundtrip(SimpleNamespace(story=story))
+    # find_file_id_in_message / _story_media_object read story.video/photo.file_unique_id.
+    assert restored.story.video.file_unique_id == "story-vid"
+    assert restored.story.photo.file_unique_id == "story-pic"
+
+
+def test_contact_roundtrip():
+    contact = SimpleNamespace(first_name="Ann", last_name="Bee", phone_number="+15551234")
+    restored, _ = _roundtrip(SimpleNamespace(contact=contact))
+    assert restored.contact.first_name == "Ann"
+    assert restored.contact.last_name == "Bee"
+    assert restored.contact.phone_number == "+15551234"
+
+
+def test_location_roundtrip():
+    location = SimpleNamespace(latitude=51.5, longitude=-0.12)
+    restored, _ = _roundtrip(SimpleNamespace(location=location))
+    assert restored.location.latitude == 51.5
+    assert restored.location.longitude == -0.12
+
+
+def test_venue_roundtrip_with_nested_location():
+    venue = SimpleNamespace(
+        title="Big Ben", address="Westminster",
+        location=SimpleNamespace(latitude=51.5, longitude=-0.12),
+    )
+    restored, _ = _roundtrip(SimpleNamespace(venue=venue))
+    assert restored.venue.title == "Big Ben"
+    assert restored.venue.address == "Westminster"
+    assert restored.venue.location.latitude == 51.5
+    assert restored.venue.location.longitude == -0.12
+
+
+def test_dice_roundtrip():
+    restored, _ = _roundtrip(SimpleNamespace(dice=SimpleNamespace(emoji="🎯", value=6)))
+    assert restored.dice.emoji == "🎯"
+    assert restored.dice.value == 6
+
+
+def test_game_roundtrip():
+    restored, _ = _roundtrip(SimpleNamespace(game=SimpleNamespace(title="Chess")))
+    assert restored.game.title == "Chess"
+
+
+def test_giveaway_roundtrip_until_date_datetime():
+    giveaway = SimpleNamespace(
+        quantity=3, months=6, stars=None,
+        until_date=datetime(2030, 12, 31, 12, 0, 0),
+        description="Prizes!",
+    )
+    restored, _ = _roundtrip(SimpleNamespace(giveaway=giveaway))
+    assert restored.giveaway.quantity == 3
+    assert restored.giveaway.months == 6
+    # until_date must restore as a datetime so the renderer's strftime branch fires.
+    assert restored.giveaway.until_date.strftime("%d/%m/%Y") == "31/12/2030"
+    assert restored.giveaway.description == "Prizes!"
+
+
+def test_giveaway_winners_roundtrip():
+    winners = SimpleNamespace(winner_count=5, quantity=10, prize_description="Premium")
+    restored, _ = _roundtrip(SimpleNamespace(giveaway_winners=winners))
+    assert restored.giveaway_winners.winner_count == 5
+    assert restored.giveaway_winners.quantity == 10
+    assert restored.giveaway_winners.prize_description == "Premium"
+
+
+def test_checklist_roundtrip_tasks_and_completion():
+    checklist = SimpleNamespace(
+        title="Todo",
+        tasks=[
+            SimpleNamespace(text="done task", completed_by=SimpleNamespace(id=1), completion_date=None),
+            SimpleNamespace(text="open task", completed_by=None, completion_date=None),
+        ],
+    )
+    restored, _ = _roundtrip(SimpleNamespace(checklist=checklist))
+    assert restored.checklist.title == "Todo"
+    t_done, t_open = restored.checklist.tasks
+    assert t_done.text == "done task"
+    # Renderer marks ☑ when bool(completed_by or completion_date) is True.
+    assert bool(t_done.completed_by or t_done.completion_date) is True
+    assert t_open.text == "open task"
+    assert bool(t_open.completed_by or t_open.completion_date) is False
+
+
+def test_paid_media_roundtrip_stars_and_item_count():
+    paid = SimpleNamespace(stars_amount=50, media=[object(), object(), object()])
+    restored, _ = _roundtrip(SimpleNamespace(paid_media=paid))
+    assert restored.paid_media.stars_amount == 50
+    # Renderer only len()'s .media — the count must survive.
+    assert len(restored.paid_media.media) == 3
+
+
+def test_live_photo_roundtrip_file_unique_id_and_size():
+    live_photo = SimpleNamespace(file_unique_id="lp1", file_size=4096)
+    restored, _ = _roundtrip(SimpleNamespace(live_photo=live_photo))
+    # MEDIA_SOURCES/_get_file_unique_id + find_file_id read live_photo.file_unique_id;
+    # _save_media_file_ids reads file_size (the >100MB skip).
+    assert restored.live_photo.file_unique_id == "lp1"
+    assert restored.live_photo.file_size == 4096
+
+
+def test_large_live_photo_not_collected():
+    snap = snapshot_message(SimpleNamespace())
+    snap["media"] = "LIVE_PHOTO"
+    snap["live_photo"] = {"file_unique_id": "lpbig", "file_size": 200 * 1024 * 1024}
+    snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None}
+    restored = restore_message(snap)
+
+    pp = PostParser(None)
+    pp._save_media_file_ids(restored)
+    assert pp._pending_media_ids == []
+
+
+def test_special_media_defaults_present_on_empty_message():
+    restored = restore_message(snapshot_message(SimpleNamespace()))
+    for attr in ["story", "contact", "location", "venue", "dice", "game",
+                 "giveaway", "giveaway_winners", "checklist", "paid_media", "live_photo"]:
+        assert hasattr(restored, attr)
+        assert getattr(restored, attr) is None

tests/test_snapshot_render_parity.py

@@ -0,0 +1,302 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""Render-parity oracle for the snapshot cache (issue #23 review fix).
+
+The core ask of the review: prove that snapshot -> restore preserves EVERYTHING the
+render pipeline reads. The old golden test replays a pickle corpus and never renders a
+live Message against its snapshot, so a field dropped from the allowlist (the exact
+bug the reviewer mutation-proved for web_page/sender_chat/... and for the special-media
+types) passes silently.
+
+This test builds a CORPUS of representative live fake Message objects (plain text, every
+media type, every special-media type, web_page, poll, forwards Case 1-5, reactions,
+caption, a multi-message media group) and for EACH case:
+
+  1. renders the LIVE objects through the REAL public feed entry points
+     (rss_generator.generate_channel_html / generate_channel_rss — the same path the
+     golden test and production use, driven by monkeypatching tg_cache),
+  2. snapshot -> restore every message, renders the RESTORED objects the same way,
+  3. asserts the two renders are BYTE-IDENTICAL (HTML and the RSS feed).
+
+Any field the snapshot fails to preserve makes the two renders diverge and this test go
+red. It is a self-contained live-vs-restored parity oracle; it does NOT depend on the
+frozen pickle goldens.
+"""
+import asyncio
+from datetime import datetime
+from types import SimpleNamespace
+
+import pytest
+
+from pyrogram.enums import MessageMediaType
+
+from message_snapshot import snapshot_messages, restore_messages
+from tests import golden_replay as gr
+
+
+# --------------------------------------------------------------------------- #
+# Live-object fakes.
+# --------------------------------------------------------------------------- #
+class FakeStr(str):
+    """Stand-in for a live pyrogram Str: a str carrying a .html rendering."""
+    def __new__(cls, plain, html=None):
+        obj = str.__new__(cls, plain)
+        obj._html = html if html is not None else plain
+        return obj
+
+    @property
+    def html(self):
+        return self._html
+
+
+_CHANNEL = "parity_ch"
+_CHAT = SimpleNamespace(id=-1001234567890, username=_CHANNEL, title="Parity Channel", usernames=None)
+
+# The full top-level attribute surface the render pipeline reads (mirrors CachedMessage).
+# A live fake sets every one so it renders WITHOUT relying on getattr fallbacks that a
+# restored CachedMessage would provide — otherwise the live side, not the snapshot, would
+# be the thing under-specified.
+_DEFAULTS = dict(
+    id=0, date=None, text=None, caption=None, media=None, service=None,
+    media_group_id=None, views=100, show_caption_above_media=False,
+    reply_to_message_id=None, reply_to_message=None, empty=False,
+    chat=_CHAT, sender_chat=None, from_user=None, forward_origin=None,
+    reactions=None, poll=None, web_page=None, photo=None, video=None,
+    document=None, audio=None, voice=None, video_note=None, animation=None,
+    sticker=None, story=None, contact=None, location=None, venue=None,
+    dice=None, game=None, giveaway=None, giveaway_winners=None,
+    checklist=None, paid_media=None, live_photo=None,
+)
+
+_id_counter = [1000]
+
+
+def make_msg(**overrides):
+    _id_counter[0] += 1
+    fields = dict(_DEFAULTS)
+    fields["id"] = _id_counter[0]
+    fields["date"] = datetime(2024, 3, 1, 12, 0, _id_counter[0] % 60)
+    fields.update(overrides)
+    return SimpleNamespace(**fields)
+
+
+def _reactions(*reacts):
+    return SimpleNamespace(reactions=list(reacts))
+
+
+# --------------------------------------------------------------------------- #
+# Corpus: each case is (name, [live messages]) rendered as its own mini feed.
+# --------------------------------------------------------------------------- #
+def build_corpus():
+    corpus = []
+
+    # Plain text.
+    corpus.append(("plain_text", [make_msg(text=FakeStr("Hello world", "Hello <b>world</b>"))]))
+
+    # Caption (photo + caption + show_caption_above_media).
+    corpus.append(("caption_photo", [make_msg(
+        media=MessageMediaType.PHOTO,
+        photo=SimpleNamespace(file_unique_id="pcap"),
+        caption=FakeStr("A caption", "A <i>caption</i>"),
+        show_caption_above_media=True,
+    )]))
+
+    # Regular media types.
+    corpus.append(("photo", [make_msg(media=MessageMediaType.PHOTO,
+                                      photo=SimpleNamespace(file_unique_id="ph1"))]))
+    corpus.append(("video", [make_msg(media=MessageMediaType.VIDEO,
+                                      video=SimpleNamespace(file_unique_id="vd1", file_size=2048))]))
+    corpus.append(("document_pdf", [make_msg(media=MessageMediaType.DOCUMENT,
+                                             document=SimpleNamespace(file_unique_id="doc1", mime_type="application/pdf"))]))
+    corpus.append(("document_other", [make_msg(media=MessageMediaType.DOCUMENT,
+                                               document=SimpleNamespace(file_unique_id="doc2", mime_type="image/png"))]))
+    corpus.append(("audio", [make_msg(media=MessageMediaType.AUDIO,
+                                      audio=SimpleNamespace(file_unique_id="au1", mime_type="audio/mpeg"))]))
+    corpus.append(("voice", [make_msg(media=MessageMediaType.VOICE,
+                                      voice=SimpleNamespace(file_unique_id="vo1", mime_type="audio/ogg"))]))
+    corpus.append(("video_note", [make_msg(media=MessageMediaType.VIDEO_NOTE,
+                                           video_note=SimpleNamespace(file_unique_id="vn1"))]))
+    corpus.append(("animation", [make_msg(media=MessageMediaType.ANIMATION,
+                                          animation=SimpleNamespace(file_unique_id="an1"))]))
+    corpus.append(("sticker_image", [make_msg(media=MessageMediaType.STICKER,
+                                              sticker=SimpleNamespace(file_unique_id="st1", emoji="😀", is_video=False))]))
+    corpus.append(("sticker_video", [make_msg(media=MessageMediaType.STICKER,
+                                              sticker=SimpleNamespace(file_unique_id="st2", emoji="🎥", is_video=True))]))
+    corpus.append(("live_photo", [make_msg(media=MessageMediaType.LIVE_PHOTO,
+                                           live_photo=SimpleNamespace(file_unique_id="lp1", file_size=4096))]))
+
+    # --- Special-media types (Fix 1) ------------------------------------- #
+    corpus.append(("story_photo", [make_msg(
+        media=MessageMediaType.STORY,
+        story=SimpleNamespace(video=None, photo=SimpleNamespace(file_unique_id="sto_p")),
+    )]))
+    corpus.append(("story_video", [make_msg(
+        media=MessageMediaType.STORY,
+        story=SimpleNamespace(video=SimpleNamespace(file_unique_id="sto_v"), photo=None),
+    )]))
+    corpus.append(("contact", [make_msg(
+        media=MessageMediaType.CONTACT,
+        contact=SimpleNamespace(first_name="Ann", last_name="Bee", phone_number="+15551234"),
+    )]))
+    corpus.append(("location", [make_msg(
+        media=MessageMediaType.LOCATION,
+        location=SimpleNamespace(latitude=51.50111, longitude=-0.14222),
+    )]))
+    corpus.append(("venue", [make_msg(
+        media=MessageMediaType.VENUE,
+        venue=SimpleNamespace(title="Big Ben", address="Westminster",
+                              location=SimpleNamespace(latitude=51.50055, longitude=-0.12461)),
+    )]))
+    corpus.append(("dice", [make_msg(
+        media=MessageMediaType.DICE, dice=SimpleNamespace(emoji="🎯", value=6))]))
+    corpus.append(("game", [make_msg(
+        media=MessageMediaType.GAME, game=SimpleNamespace(title="Chess Master"))]))
+    corpus.append(("giveaway", [make_msg(
+        media=MessageMediaType.GIVEAWAY,
+        giveaway=SimpleNamespace(quantity=3, months=6, stars=None,
+                                 until_date=datetime(2030, 12, 31, 12, 0, 0),
+                                 description="Great prizes"),
+    )]))
+    corpus.append(("giveaway_stars", [make_msg(
+        media=MessageMediaType.GIVEAWAY,
+        giveaway=SimpleNamespace(quantity=2, months=None, stars=500,
+                                 until_date=None, description=None),
+    )]))
+    corpus.append(("giveaway_winners", [make_msg(
+        media=MessageMediaType.GIVEAWAY_WINNERS,
+        giveaway_winners=SimpleNamespace(winner_count=5, quantity=10, prize_description="Premium"),
+    )]))
+    corpus.append(("checklist", [make_msg(
+        media=MessageMediaType.CHECKLIST,
+        checklist=SimpleNamespace(title="Todo list", tasks=[
+            SimpleNamespace(text="done task", completed_by=SimpleNamespace(id=1), completion_date=None),
+            SimpleNamespace(text="open task", completed_by=None, completion_date=None),
+        ]),
+    )]))
+    corpus.append(("paid_media", [make_msg(
+        media=MessageMediaType.PAID_MEDIA,
+        paid_media=SimpleNamespace(stars_amount=50, media=[object(), object()]),
+    )]))
+
+    # --- web_page + poll -------------------------------------------------- #
+    corpus.append(("web_page", [make_msg(
+        text=FakeStr("check this", "check this"),
+        media=MessageMediaType.WEB_PAGE,
+        web_page=SimpleNamespace(type="article", url="https://example.com/a",
+                                 display_url="example.com/a", site_name="Example",
+                                 title="An Article", description="Desc here",
+                                 has_large_media=False,
+                                 photo=SimpleNamespace(file_unique_id="wp1")),
+    )]))
+    corpus.append(("poll", [make_msg(
+        media=MessageMediaType.POLL,
+        poll=SimpleNamespace(
+            question=SimpleNamespace(text="Favourite?", entities=[]),
+            options=[SimpleNamespace(text=SimpleNamespace(text="Red", entities=[])),
+                     SimpleNamespace(text=SimpleNamespace(text="Blue", entities=[]))],
+            description_media=None, explanation_media=None,
+        ),
+    )]))
+
+    # --- Forwards Case 1-5 (drive _format_forward_info) ------------------- #
+    corpus.append(("forward_channel", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="channel",
+                                       chat=SimpleNamespace(id=-100, title="Src Chan", username="srcchan")),
+    )]))
+    corpus.append(("forward_hidden", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="hidden_user", sender_user_name="Hidden Guy"),
+    )]))
+    corpus.append(("forward_user", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="user",
+                                       sender_user=SimpleNamespace(first_name="Ann", last_name="Bee", username="annbee")),
+    )]))
+    corpus.append(("forward_channel_nouser", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="channel", chat_id=-1002, title="NoUserChan"),
+    )]))
+    corpus.append(("forward_other", [make_msg(
+        text=FakeStr("fwd", "fwd"),
+        forward_origin=SimpleNamespace(type="something_else"),
+    )]))
+
+    # --- Reactions: normal / paid / custom -------------------------------- #
+    corpus.append(("reactions", [make_msg(
+        text=FakeStr("react", "react"),
+        reactions=_reactions(
+            SimpleNamespace(emoji="👍", count=5, is_paid=False),
+            SimpleNamespace(count=3, is_paid=True),
+            SimpleNamespace(custom_emoji_id=1234567890, count=2),
+        ),
+    )]))
+
+    # --- sender_chat / from_user author paths ----------------------------- #
+    corpus.append(("sender_chat_author", [make_msg(
+        text=FakeStr("a", "a"),
+        sender_chat=SimpleNamespace(id=-100, title="Sender Chan", username="senderchan"),
+    )]))
+    corpus.append(("from_user_author", [make_msg(
+        text=FakeStr("a", "a"),
+        from_user=SimpleNamespace(first_name="Joe", last_name="Doe", username="joedoe"),
+    )]))
+
+    # --- Multi-message media group (grouping merge) ----------------------- #
+    corpus.append(("media_group", [
+        make_msg(media=MessageMediaType.PHOTO, media_group_id="grp1",
+                 photo=SimpleNamespace(file_unique_id="grp_a"),
+                 caption=FakeStr("group cap", "group cap")),
+        make_msg(media=MessageMediaType.PHOTO, media_group_id="grp1",
+                 photo=SimpleNamespace(file_unique_id="grp_b")),
+    ]))
+
+    return corpus
+
+
+# --------------------------------------------------------------------------- #
+# Rendering harness — patches tg_cache to feed a message list into the real
+# generate_channel_* entry points (the golden-replay path).
+# --------------------------------------------------------------------------- #
+def _render(messages, monkeypatch):
+    async def fake_get_chat_history(client, channel_id, limit=20):
+        return messages
+
+    async def fake_get_chat(client, channel_id):
+        return SimpleNamespace(id=_CHAT.id, title=_CHAT.title, username=_CHAT.username)
+
+    monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_chat_history, raising=False)
+    monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False)
+
+    from rss_generator import generate_channel_html, generate_channel_rss
+    html = asyncio.run(generate_channel_html(_CHANNEL, client=SimpleNamespace(), limit=50))
+    rss = asyncio.run(generate_channel_rss(_CHANNEL, client=SimpleNamespace(), limit=50))
+    return html, gr.normalize_rss(rss)
+
+
+@pytest.mark.parametrize("name,messages", build_corpus(), ids=lambda v: v if isinstance(v, str) else "")
+def test_snapshot_render_parity(name, messages, monkeypatch):
+    gr.pin_environment(monkeypatch)
+
+    # 1) Render the LIVE objects through the real pipeline.
+    live_html, live_rss = _render(messages, monkeypatch)
+
+    # 2) snapshot -> restore, then render the RESTORED objects the same way.
+    restored = restore_messages(snapshot_messages(messages))
+    restored_html, restored_rss = _render(restored, monkeypatch)
+
+    # 3) Byte-identical HTML and RSS prove the snapshot preserved everything the
+    #    renderer read for this case. A dropped allowlist field diverges here.
+    assert restored_html == live_html, f"HTML render diverged for case '{name}'"
+    assert restored_rss == live_rss, f"RSS render diverged for case '{name}'"
+
+
+def test_corpus_covers_all_special_media_types():
+    """Guard: every special-media type from Fix 1 is exercised by the parity corpus."""
+    names = {n for n, _ in build_corpus()}
+    required = {"story_photo", "story_video", "contact", "location", "venue",
+                "dice", "game", "giveaway", "giveaway_winners", "checklist", "paid_media",
+                "live_photo"}
+    assert required <= names

tests/test_tg_cache_store.py

@@ -0,0 +1,206 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long
+# pylance: disable=reportMissingImports, reportMissingModuleSource
+"""Tests for the tg_cache generic JSON store, prefix-limit, jitter and sweep (issue #23).
+
+Задания 2 / 6 / 17 / 18, verification item 8, 11, 12.
+"""
+import json
+import os
+import time
+from types import SimpleNamespace
+
+import pytest
+
+import tg_cache
+from message_snapshot import SNAPSHOT_VERSION
+
+
+@pytest.fixture
+def cache_dir(tmp_path, monkeypatch):
+    d = tmp_path / "tgcache"
+    d.mkdir()
+    monkeypatch.setattr(tg_cache, "CACHE_DIR", str(d))
+    return d
+
+
+# --------------------------------------------------------------------------- #
+# _store_entry / _load_entry.
+# --------------------------------------------------------------------------- #
+def test_store_load_roundtrip(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    tg_cache._store_entry(path, {"limit": 5, "messages": [{"id": 1}]})
+    loaded = tg_cache._load_entry(path, max_age_hours=8)
+    assert loaded is not None
+    assert loaded["limit"] == 5
+    assert loaded["messages"] == [{"id": 1}]
+    assert loaded["version"] == SNAPSHOT_VERSION
+    assert 0.8 <= loaded["jitter"] <= 1.0
+
+
+def test_load_missing_file(cache_dir):
+    assert tg_cache._load_entry(str(cache_dir / "nope.json"), 8) is None
+
+
+def test_load_expired_ttl(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    tg_cache._store_entry(path, {"data": 1})
+    # max_age_hours=0 -> adjusted max age 0 -> any positive age is expired.
+    assert tg_cache._load_entry(path, max_age_hours=0) is None
+    # Still fresh with a real TTL.
+    assert tg_cache._load_entry(path, max_age_hours=8) is not None
+
+
+def test_load_version_mismatch(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump({"version": SNAPSHOT_VERSION + 999, "timestamp": time.time(), "data": 1}, f)
+    assert tg_cache._load_entry(path, 8) is None
+
+
+def test_load_corrupt_json(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    with open(path, "w", encoding="utf-8") as f:
+        f.write("{not valid json")
+    assert tg_cache._load_entry(path, 8) is None
+
+
+def test_store_uses_unique_tmp_names(cache_dir, monkeypatch):
+    path = str(cache_dir / "x.history.json")
+    seen = []
+
+    real_replace = os.replace
+
+    def spy_replace(src, dst):
+        seen.append(src)
+        return real_replace(src, dst)
+
+    monkeypatch.setattr(tg_cache.os, "replace", spy_replace)
+    tg_cache._store_entry(path, {"data": 1})
+    tg_cache._store_entry(path, {"data": 2})
+
+    assert len(seen) == 2
+    assert seen[0] != seen[1]  # each writer used a distinct tmp path
+    for src in seen:
+        assert src.startswith(path + ".tmp.")
+    # No tmp leftovers.
+    assert [n for n in os.listdir(cache_dir) if ".tmp." in n] == []
+
+
+# --------------------------------------------------------------------------- #
+# cleanup_legacy_cache_files / sweep_tgcache.
+# --------------------------------------------------------------------------- #
+def test_cleanup_legacy_keeps_new_format(cache_dir):
+    (cache_dir / "chan.cache").write_text("x")
+    (cache_dir / "chan_history.cache").write_text("x")
+    (cache_dir / "chan.chatinfo").write_text("x")
+    (cache_dir / "chan.history.json").write_text("x")
+    (cache_dir / "chan.chatinfo.json").write_text("x")
+
+    removed = tg_cache.cleanup_legacy_cache_files()
+    assert removed == 3
+    remaining = set(os.listdir(cache_dir))
+    assert remaining == {"chan.history.json", "chan.chatinfo.json"}
+
+
+def test_sweep_removes_only_old(cache_dir):
+    old = cache_dir / "old.history.json"
+    new = cache_dir / "new.history.json"
+    old.write_text("x")
+    new.write_text("x")
+    old_time = time.time() - 8 * 86400  # 8 days old, threshold is 7
+    os.utime(old, (old_time, old_time))
+
+    removed = tg_cache.sweep_tgcache(max_age_days=7)
+    assert removed == 1
+    remaining = set(os.listdir(cache_dir))
+    assert remaining == {"new.history.json"}
+
+
+def test_sweep_removes_orphan_tmp(cache_dir):
+    orphan = cache_dir / "x.history.json.tmp.deadbeef"
+    orphan.write_text("x")
+    old_time = time.time() - 10 * 86400
+    os.utime(orphan, (old_time, old_time))
+    assert tg_cache.sweep_tgcache(max_age_days=7) == 1
+
+
+# --------------------------------------------------------------------------- #
+# History prefix-limit (Задание 6).
+# --------------------------------------------------------------------------- #
+def _fake_messages(n):
+    return [SimpleNamespace(id=i) for i in range(n)]
+
+
+def test_prefix_larger_cache_serves_smaller_request(cache_dir):
+    tg_cache._save_history_to_cache("chan", _fake_messages(100), limit=100)
+    served = tg_cache._get_history_from_cache("chan", limit=50)
+    assert served is not None
+    assert len(served) == 50
+    assert [m.id for m in served] == list(range(50))  # order preserved (newest-first slice)
+
+
+def test_prefix_smaller_cache_is_miss(cache_dir):
+    tg_cache._save_history_to_cache("chan", _fake_messages(50), limit=50)
+    assert tg_cache._get_history_from_cache("chan", limit=100) is None
+
+
+def test_prefix_exhausted_channel_serves_larger_request(cache_dir):
+    # Fetched with limit 100 but the channel only has 37 messages -> entire history cached.
+    tg_cache._save_history_to_cache("chan", _fake_messages(37), limit=100)
+    served = tg_cache._get_history_from_cache("chan", limit=200)
+    assert served is not None
+    assert len(served) == 37
+
+
+def test_save_stores_fetch_limit_not_len(cache_dir):
+    tg_cache._save_history_to_cache("chan", _fake_messages(37), limit=100)
+    path = tg_cache._cache_file_path("chan", "history.json")
+    payload = tg_cache._load_entry(path, max_age_hours=8)
+    assert payload["limit"] == 100
+    assert len(payload["messages"]) == 37
+
+
+def test_history_ttl_independent_of_prefix(cache_dir):
+    tg_cache._save_history_to_cache("chan", _fake_messages(100), limit=100)
+    # Expired regardless of prefix match.
+    assert tg_cache._get_history_from_cache("chan", limit=50, max_age_hours=0) is None
+
+
+# --------------------------------------------------------------------------- #
+# Jitter stability (Задание 17 / item 12).
+# --------------------------------------------------------------------------- #
+def test_jitter_read_is_stable(cache_dir):
+    path = str(cache_dir / "x.history.json")
+    tg_cache._store_entry(path, {"data": 1})
+    # Repeated reads near a TTL boundary must give a stable result: the jitter is fixed at
+    # write time and _load_entry never calls random().
+    results = [tg_cache._load_entry(path, max_age_hours=8) is not None for _ in range(10)]
+    assert all(results)
+
+
+def test_load_does_not_call_random(cache_dir, monkeypatch):
+    path = str(cache_dir / "x.history.json")
+    tg_cache._store_entry(path, {"data": 1})
+
+    def boom(*a, **k):
+        raise AssertionError("random.uniform must not be called at read time")
+
+    monkeypatch.setattr(tg_cache.random, "uniform", boom)
+    assert tg_cache._load_entry(path, max_age_hours=8) is not None
+
+
+# --------------------------------------------------------------------------- #
+# chatinfo store round trip.
+# --------------------------------------------------------------------------- #
+def test_chatinfo_roundtrip(cache_dir):
+    tg_cache._save_chat_to_cache("chan", {"id": -100, "title": "T", "username": "u"})
+    data = tg_cache._get_chat_from_cache("chan")
+    assert data == {"id": -100, "title": "T", "username": "u"}
+
+
+def test_no_pickle_import():
+    import pathlib
+    src = pathlib.Path(tg_cache.__file__).read_text()
+    assert "pickle" not in src

tg_cache.py

@@ -11,18 +11,22 @@
 
 import os
 import json
-import pickle
+import uuid
 import logging
 import random
 import asyncio
 import time
-from datetime import datetime, timedelta
 from types import SimpleNamespace
 from typing import Any, Optional, Union, List
 from pyrogram import Client
-from pyrogram.types import Chat, Message
+from pyrogram.types import Message
 from tg_throttle import tg_rpc_bounded
 from config import get_settings
+from message_snapshot import (
+    SNAPSHOT_VERSION,
+    snapshot_messages,
+    restore_messages,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -40,105 +44,148 @@ try:
 except ValueError:
     CHAT_CACHE_TTL_HOURS = 12
 
-def _get_history_cache_file_path(channel_id: Union[str, int]) -> str:
-    """Returns path to the message history cache file for the channel"""
+
+def _safe_key(key: Union[str, int]) -> str:
+    """Sanitize a channel id/username into a filesystem-safe basename component."""
+    return str(key).replace('/', '_').replace('\\', '_')
+
+
+def _cache_file_path(key: Union[str, int], suffix: str) -> str:
+    """Return the cache file path <safe_key>.<suffix> (e.g. 'history.json' / 'chatinfo.json')."""
     if not os.path.exists(CACHE_DIR):
         os.makedirs(CACHE_DIR, exist_ok=True)
         logger.info(f"cache_dir_created: path {CACHE_DIR}")
-    # Convert to string for uniformity
-    channel_id_str = str(channel_id)
-    # Replace potentially problematic characters
-    safe_filename = channel_id_str.replace('/', '_').replace('\\', '_')
-    return os.path.join(CACHE_DIR, f"{safe_filename}.cache")
+    return os.path.join(CACHE_DIR, f"{_safe_key(key)}.{suffix}")
 
-def _save_history_to_cache(channel_id: Union[str, int], messages: List[Message], limit: int) -> None:
-    """Saves message history to cache"""
+
+# --------------------------------------------------------------------------- #
+# Generic JSON entry store.
+# --------------------------------------------------------------------------- #
+def _store_entry(path: str, payload: dict) -> None:
+    """Atomically write {version, timestamp, jitter, **payload} as JSON to ``path``.
+
+    The document is written to a unique '<path>.tmp.<uuid4>' and os.replace()d into place
+    so a concurrent reader never observes a half-written file. The unique per-writer tmp
+    name means two concurrent writers to the same path do not clobber each other's temp
+    file. This writer's own tmp file is always removed in the finally block.
+    """
+    tmp_path = f"{path}.tmp.{uuid.uuid4().hex}"
+    entry = {
+        'version': SNAPSHOT_VERSION,
+        'timestamp': time.time(),
+        # Per-write TTL jitter (§17): decided ONCE at write time, so repeated reads of the
+        # same file are stable (the reader never calls random()).
+        'jitter': random.uniform(0.8, 1.0),
+    }
+    entry.update(payload)
     try:
-        cache_file = _get_history_cache_file_path(channel_id)
-        
-        # Create cache metadata — store messages directly (no inner pickle.dumps)
-        cache_data = {
-            'timestamp': time.time(),
-            'limit': limit,
-            'messages': messages
-        }
-        
-        with open(cache_file, 'wb') as f:
-            pickle.dump(cache_data, f)
-        
+        with open(tmp_path, 'w', encoding='utf-8') as f:
+            json.dump(entry, f)
+        os.replace(tmp_path, path)
+    finally:
+        # Remove our own leftover tmp file if os.replace didn't consume it (e.g. it raised
+        # or json.dump failed). Never touches another writer's uniquely-named tmp file.
+        if os.path.exists(tmp_path):
+            try:
+                os.remove(tmp_path)
+            except OSError:
+                pass
+
+
+def _load_entry(path: str, max_age_hours: float) -> Optional[dict]:
+    """Return the stored payload dict, or None on missing / version mismatch / expired / bad JSON.
+
+    TTL uses the jitter written into the entry (no random() at read time) so repeated
+    reads near the boundary give a stable result.
+    """
+    if not os.path.exists(path):
+        return None
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            entry = json.load(f)
+    except (json.JSONDecodeError, OSError, ValueError) as e:
+        logger.warning(f"cache_entry_read_error: path {path}, error {str(e)}")
+        return None
+
+    if not isinstance(entry, dict) or entry.get('version') != SNAPSHOT_VERSION:
+        logger.info(f"cache_entry_version_mismatch: path {path}")
+        return None
+
+    timestamp = entry.get('timestamp')
+    if not isinstance(timestamp, (int, float)):
+        return None
+    age = time.time() - timestamp
+    adjusted_max_age = max_age_hours * 3600 * entry.get('jitter', 1.0)
+    if age > adjusted_max_age:
+        logger.info(f"cache_entry_expired: path {path}, age {age:.1f}s > adjusted max {adjusted_max_age:.1f}s")
+        return None
+    return entry
+
+
+# --------------------------------------------------------------------------- #
+# History cache.
+# --------------------------------------------------------------------------- #
+def _save_history_to_cache(channel_id: Union[str, int], messages: List[Message], limit: int) -> None:
+    """Save message history (as JSON snapshots) to cache. Stores the fetch limit, not len()."""
+    try:
+        cache_file = _cache_file_path(channel_id, 'history.json')
+        payload = {'limit': limit, 'messages': snapshot_messages(messages)}
+        _store_entry(cache_file, payload)
         logger.info(f"history_cache_saved: channel {channel_id}, limit {limit}, messages {len(messages)}, file {cache_file}")
     except Exception as e:
         logger.error(f"history_cache_save_error: channel {channel_id}, limit {limit}, error {str(e)}")
 
+
 def _get_history_from_cache(channel_id: Union[str, int], limit: int, max_age_hours: int = 8) -> Optional[List[Message]]:
     """
-    Retrieves message history from cache if not older than specified age and matches the limit
-    
-    Args:
-        channel_id: Channel ID or username
-        limit: Required message limit
-        max_age_hours: Maximum cache age in hours (default 8 hours)
-        
-    Returns:
-        List of messages or None if cache not found, expired or limit doesn't match
+    Retrieve message history from cache if fresh and the cached fetch covers ``limit``.
+
+    Messages are stored newest-first. A cached entry fetched with an equal-or-larger limit
+    serves a smaller request by slicing (prefix). A smaller cached fetch is a miss UNLESS
+    the channel is exhausted (fewer messages exist than were asked for), in which case the
+    cache already holds the entire recent history and can serve any larger request.
     """
     try:
-        cache_file = _get_history_cache_file_path(channel_id)
-        
-        if not os.path.exists(cache_file):
-            logger.info(f"history_cache_miss: channel {channel_id}, limit {limit}, cache file not found")
+        cache_file = _cache_file_path(channel_id, 'history.json')
+        payload = _load_entry(cache_file, max_age_hours)
+        if payload is None:
+            logger.info(f"history_cache_miss: channel {channel_id}, limit {limit}")
             return None
-        
-        with open(cache_file, 'rb') as f:
-            cache_data = pickle.load(f)
-        
-        # Check cache age with randomization
-        cache_age = time.time() - cache_data['timestamp']
-        # Add randomness up to 20% of max_age_seconds
-        random_factor = 1 - random.uniform(0, 0.2)
-        adjusted_max_age = max_age_hours * 3600 * random_factor
-        
-        if cache_age > adjusted_max_age:
-            logger.info(f"history_cache_expired: channel {channel_id}, limit {limit}, age {cache_age:.1f}s > adjusted max {adjusted_max_age:.1f}s (random factor: {random_factor:.2f})")
+
+        cached_limit = payload.get('limit', 0)
+        raw_messages = payload['messages']
+        # Serve when fetched with an equal-or-larger limit, OR when the channel is exhausted
+        # (fewer messages exist than asked -> cache holds entire recent history).
+        if cached_limit < limit and len(raw_messages) >= cached_limit:
+            logger.info(f"history_cache_limit_short: channel {channel_id}, cached limit {cached_limit}, requested {limit}")
             return None
-        
-        # Check if limit matches
-        cached_limit = cache_data.get('limit', 0)
-        if cached_limit != limit:
-            logger.info(f"history_cache_limit_mismatch: channel {channel_id}, cached limit {cached_limit}, requested limit {limit}")
-            return None
-        
-        # Restore message list; handle old cache files that used double-pickle (bytes = old format)
-        raw = cache_data['messages']
-        if isinstance(raw, bytes):
-            messages = pickle.loads(raw)
-        else:
-            messages = raw
-        logger.info(f"history_cache_hit: channel {channel_id}, limit {limit}, messages {len(messages)}, age {cache_age:.1f}s")
+
+        messages = restore_messages(raw_messages[:limit])
+        logger.info(f"history_cache_hit: channel {channel_id}, served {limit} of cached {cached_limit}, messages {len(messages)}")
         return messages
-    
     except Exception as e:
         logger.error(f"history_cache_read_error: channel {channel_id}, limit {limit}, error {str(e)}")
-        # In case of cache read error, better return None and request fresh data
         return None
 
+
 async def cached_get_chat_history(client: Client, channel_id: Union[str, int], limit: int = 20) -> List[Message]:
     """
     Gets chat message history with caching.
-    
+
     Args:
         client: Pyrogram client
         channel_id: Channel ID or username
         limit: Maximum number of messages to retrieve
-        
+
     Returns:
-        List of messages, same as original client.get_chat_history()
+        List of messages, same as original client.get_chat_history(). On a cache miss the
+        live pyrogram Messages are returned; on a hit, restored CachedMessage objects.
     """
     cached_messages = await asyncio.to_thread(_get_history_from_cache, channel_id, limit)
-    
+
     if cached_messages is not None:
         return cached_messages
-    
+
     try:
         logger.info(f"history_cache_request: fetching fresh history for channel {channel_id}, limit {limit}")
         # Hold the global RPC gate for the live fetch and bound the RPC body with the
@@ -148,57 +195,39 @@ async def cached_get_chat_history(client: Client, channel_id: Union[str, int], l
         async with tg_rpc_bounded(Config["tg_rpc_timeout"]):
             messages = [m async for m in client.get_chat_history(channel_id, limit=limit)]
         await asyncio.to_thread(_save_history_to_cache, channel_id, messages, limit)
-        
+
         return messages
     except Exception as e:
         logger.error(f"history_cache_request_error: channel {channel_id}, limit {limit}, error {str(e)}")
         raise
 
 
-def _get_chat_cache_file_path(channel_id: Union[str, int]) -> str:
-    """Returns path to the channel-info cache file for the channel."""
-    if not os.path.exists(CACHE_DIR):
-        os.makedirs(CACHE_DIR, exist_ok=True)
-        logger.info(f"cache_dir_created: path {CACHE_DIR}")
-    channel_id_str = str(channel_id)
-    safe_filename = channel_id_str.replace('/', '_').replace('\\', '_')
-    # Distinct suffix so channel-info files never collide with history (.cache) files.
-    return os.path.join(CACHE_DIR, f"{safe_filename}.chatinfo")
-
-
+# --------------------------------------------------------------------------- #
+# Channel-info cache.
+# --------------------------------------------------------------------------- #
 def _save_chat_to_cache(channel_id: Union[str, int], data: dict) -> None:
     """Saves channel-info (id/title/username) to cache."""
     try:
-        cache_file = _get_chat_cache_file_path(channel_id)
-        cache_data = {'timestamp': time.time(), 'data': data}
-        with open(cache_file, 'wb') as f:
-            pickle.dump(cache_data, f)
+        cache_file = _cache_file_path(channel_id, 'chatinfo.json')
+        _store_entry(cache_file, {'data': data})
         logger.info(f"chatinfo_cache_saved: channel {channel_id}, file {cache_file}")
     except Exception as e:
         logger.error(f"chatinfo_cache_save_error: channel {channel_id}, error {str(e)}")
 
 
 def _get_chat_from_cache(channel_id: Union[str, int], max_age_hours: int = CHAT_CACHE_TTL_HOURS) -> Optional[dict]:
-    """Retrieves channel-info from cache if not older than max_age_hours (with up-to-20% randomization)."""
+    """Retrieves channel-info from cache if fresh (jittered TTL)."""
     try:
-        cache_file = _get_chat_cache_file_path(channel_id)
-        if not os.path.exists(cache_file):
-            logger.info(f"chatinfo_cache_miss: channel {channel_id}, cache file not found")
+        cache_file = _cache_file_path(channel_id, 'chatinfo.json')
+        payload = _load_entry(cache_file, max_age_hours)
+        if payload is None:
+            logger.info(f"chatinfo_cache_miss: channel {channel_id}")
             return None
-        with open(cache_file, 'rb') as f:
-            cache_data = pickle.load(f)
-        cache_age = time.time() - cache_data['timestamp']
-        # Add randomness up to 20% of max age, same approach as the history cache.
-        random_factor = 1 - random.uniform(0, 0.2)
-        adjusted_max_age = max_age_hours * 3600 * random_factor
-        if cache_age > adjusted_max_age:
-            logger.info(f"chatinfo_cache_expired: channel {channel_id}, age {cache_age:.1f}s > adjusted max {adjusted_max_age:.1f}s (random factor: {random_factor:.2f})")
-            return None
-        data = cache_data.get('data')
+        data = payload.get('data')
         if not isinstance(data, dict):
             logger.warning(f"chatinfo_cache_invalid: channel {channel_id}, unexpected payload type {type(data).__name__}")
             return None
-        logger.info(f"chatinfo_cache_hit: channel {channel_id}, age {cache_age:.1f}s")
+        logger.info(f"chatinfo_cache_hit: channel {channel_id}")
         return data
     except Exception as e:
         logger.error(f"chatinfo_cache_read_error: channel {channel_id}, error {str(e)}")
@@ -228,3 +257,64 @@ async def cached_get_chat(client: Client, channel_id: Union[str, int]) -> Simple
     }
     await asyncio.to_thread(_save_chat_to_cache, channel_id, data)
     return SimpleNamespace(**data)
+
+
+# --------------------------------------------------------------------------- #
+# Maintenance: legacy cleanup + age sweep.
+# --------------------------------------------------------------------------- #
+def cleanup_legacy_cache_files() -> int:
+    """Delete legacy binary cache files (*.cache incl. *_history.cache, and *.chatinfo).
+
+    The new store uses *.history.json / *.chatinfo.json, so these old-format files are
+    dead weight and would otherwise never be reclaimed. Returns the number removed.
+    """
+    removed = 0
+    if not os.path.isdir(CACHE_DIR):
+        return 0
+    try:
+        names = os.listdir(CACHE_DIR)
+    except OSError as e:
+        logger.warning(f"cleanup_legacy_list_error: dir {CACHE_DIR}, error {str(e)}")
+        return 0
+    for name in names:
+        if name.endswith('.cache') or name.endswith('.chatinfo'):
+            try:
+                os.remove(os.path.join(CACHE_DIR, name))
+                removed += 1
+            except OSError as e:
+                logger.warning(f"cleanup_legacy_remove_error: file {name}, error {str(e)}")
+    if removed:
+        logger.info(f"cleanup_legacy_cache_files: removed {removed} legacy files from {CACHE_DIR}")
+    return removed
+
+
+def sweep_tgcache(max_age_days: int = 7) -> int:
+    """Delete files in CACHE_DIR whose mtime is older than ``max_age_days``.
+
+    Reclaims cache for dead channels and orphaned uuid tmp files. A race with an in-flight
+    writer is POSSIBLE (stat -> unlink is not atomic against os.replace) but harmless: the
+    worst outcome is one extra cache miss. This is NOT an atomicity guarantee. Returns the
+    number of files removed.
+    """
+    removed = 0
+    if not os.path.isdir(CACHE_DIR):
+        return 0
+    cutoff = time.time() - max_age_days * 86400
+    try:
+        names = os.listdir(CACHE_DIR)
+    except OSError as e:
+        logger.warning(f"sweep_tgcache_list_error: dir {CACHE_DIR}, error {str(e)}")
+        return 0
+    for name in names:
+        path = os.path.join(CACHE_DIR, name)
+        try:
+            if not os.path.isfile(path):
+                continue
+            if os.path.getmtime(path) < cutoff:
+                os.remove(path)
+                removed += 1
+        except OSError as e:
+            logger.warning(f"sweep_tgcache_remove_error: file {name}, error {str(e)}")
+    if removed:
+        logger.info(f"sweep_tgcache: removed {removed} stale files (> {max_age_days}d) from {CACHE_DIR}")
+    return removed