feat(cache): снапшот-кеш истории вместо pickle + префиксный limit + джиттер TTL + age-sweep

Кеш истории (data/tgcache/) больше не хранит pickle живых объектов pyrogram.
Новый модуль message_snapshot.py: при записи из Message извлекается JSON-словарь
по явному allowlist (снапшот), при чтении восстанавливается duck-typed
CachedMessage (CachedStr с .html, mutable, deepcopy-safe), неотличимый для
пайплайна рендера от Message. post_parser.py и rss_generator.py НЕ изменены.

tg_cache.py переписан на generic JSON-store (pickle удалён полностью):
- _store_entry/_load_entry: атомарная запись через уникальный .tmp.<uuid4> +
  os.replace, finally чистит свой tmp; версия/TTL/битый JSON -> None;
- пути <safe_key>.history.json / .chatinfo.json;
- пакет B: префиксный limit (miss только когда cached_limit<limit и есть ещё);
- пакет F: джиттер TTL при записи (random.uniform(0.8,1.0), на чтении random не
  зовётся) + sweep_tgcache(age) + cleanup_legacy_cache_files;
- api_server.py: старт-cleanup+sweep в lifespan, per-pass sweep в cache_media_files,
  удаление legacy data/media_file_ids.json.

closes #23

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tg_cache.py

@@ -11,18 +11,22 @@
 
 import os
 import json
-import pickle
+import uuid
 import logging
 import random
 import asyncio
 import time
-from datetime import datetime, timedelta
 from types import SimpleNamespace
 from typing import Any, Optional, Union, List
 from pyrogram import Client
-from pyrogram.types import Chat, Message
+from pyrogram.types import Message
 from tg_throttle import tg_rpc_bounded
 from config import get_settings
+from message_snapshot import (
+    SNAPSHOT_VERSION,
+    snapshot_messages,
+    restore_messages,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -40,105 +44,148 @@ try:
 except ValueError:
     CHAT_CACHE_TTL_HOURS = 12
 
-def _get_history_cache_file_path(channel_id: Union[str, int]) -> str:
-    """Returns path to the message history cache file for the channel"""
+
+def _safe_key(key: Union[str, int]) -> str:
+    """Sanitize a channel id/username into a filesystem-safe basename component."""
+    return str(key).replace('/', '_').replace('\\', '_')
+
+
+def _cache_file_path(key: Union[str, int], suffix: str) -> str:
+    """Return the cache file path <safe_key>.<suffix> (e.g. 'history.json' / 'chatinfo.json')."""
     if not os.path.exists(CACHE_DIR):
         os.makedirs(CACHE_DIR, exist_ok=True)
         logger.info(f"cache_dir_created: path {CACHE_DIR}")
-    # Convert to string for uniformity
-    channel_id_str = str(channel_id)
-    # Replace potentially problematic characters
-    safe_filename = channel_id_str.replace('/', '_').replace('\\', '_')
-    return os.path.join(CACHE_DIR, f"{safe_filename}.cache")
+    return os.path.join(CACHE_DIR, f"{_safe_key(key)}.{suffix}")
 
-def _save_history_to_cache(channel_id: Union[str, int], messages: List[Message], limit: int) -> None:
-    """Saves message history to cache"""
+
+# --------------------------------------------------------------------------- #
+# Generic JSON entry store.
+# --------------------------------------------------------------------------- #
+def _store_entry(path: str, payload: dict) -> None:
+    """Atomically write {version, timestamp, jitter, **payload} as JSON to ``path``.
+
+    The document is written to a unique '<path>.tmp.<uuid4>' and os.replace()d into place
+    so a concurrent reader never observes a half-written file. The unique per-writer tmp
+    name means two concurrent writers to the same path do not clobber each other's temp
+    file. This writer's own tmp file is always removed in the finally block.
+    """
+    tmp_path = f"{path}.tmp.{uuid.uuid4().hex}"
+    entry = {
+        'version': SNAPSHOT_VERSION,
+        'timestamp': time.time(),
+        # Per-write TTL jitter (§17): decided ONCE at write time, so repeated reads of the
+        # same file are stable (the reader never calls random()).
+        'jitter': random.uniform(0.8, 1.0),
+    }
+    entry.update(payload)
     try:
-        cache_file = _get_history_cache_file_path(channel_id)
-        
-        # Create cache metadata — store messages directly (no inner pickle.dumps)
-        cache_data = {
-            'timestamp': time.time(),
-            'limit': limit,
-            'messages': messages
-        }
-        
-        with open(cache_file, 'wb') as f:
-            pickle.dump(cache_data, f)
-        
+        with open(tmp_path, 'w', encoding='utf-8') as f:
+            json.dump(entry, f)
+        os.replace(tmp_path, path)
+    finally:
+        # Remove our own leftover tmp file if os.replace didn't consume it (e.g. it raised
+        # or json.dump failed). Never touches another writer's uniquely-named tmp file.
+        if os.path.exists(tmp_path):
+            try:
+                os.remove(tmp_path)
+            except OSError:
+                pass
+
+
+def _load_entry(path: str, max_age_hours: float) -> Optional[dict]:
+    """Return the stored payload dict, or None on missing / version mismatch / expired / bad JSON.
+
+    TTL uses the jitter written into the entry (no random() at read time) so repeated
+    reads near the boundary give a stable result.
+    """
+    if not os.path.exists(path):
+        return None
+    try:
+        with open(path, 'r', encoding='utf-8') as f:
+            entry = json.load(f)
+    except (json.JSONDecodeError, OSError, ValueError) as e:
+        logger.warning(f"cache_entry_read_error: path {path}, error {str(e)}")
+        return None
+
+    if not isinstance(entry, dict) or entry.get('version') != SNAPSHOT_VERSION:
+        logger.info(f"cache_entry_version_mismatch: path {path}")
+        return None
+
+    timestamp = entry.get('timestamp')
+    if not isinstance(timestamp, (int, float)):
+        return None
+    age = time.time() - timestamp
+    adjusted_max_age = max_age_hours * 3600 * entry.get('jitter', 1.0)
+    if age > adjusted_max_age:
+        logger.info(f"cache_entry_expired: path {path}, age {age:.1f}s > adjusted max {adjusted_max_age:.1f}s")
+        return None
+    return entry
+
+
+# --------------------------------------------------------------------------- #
+# History cache.
+# --------------------------------------------------------------------------- #
+def _save_history_to_cache(channel_id: Union[str, int], messages: List[Message], limit: int) -> None:
+    """Save message history (as JSON snapshots) to cache. Stores the fetch limit, not len()."""
+    try:
+        cache_file = _cache_file_path(channel_id, 'history.json')
+        payload = {'limit': limit, 'messages': snapshot_messages(messages)}
+        _store_entry(cache_file, payload)
         logger.info(f"history_cache_saved: channel {channel_id}, limit {limit}, messages {len(messages)}, file {cache_file}")
     except Exception as e:
         logger.error(f"history_cache_save_error: channel {channel_id}, limit {limit}, error {str(e)}")
 
+
 def _get_history_from_cache(channel_id: Union[str, int], limit: int, max_age_hours: int = 8) -> Optional[List[Message]]:
     """
-    Retrieves message history from cache if not older than specified age and matches the limit
-    
-    Args:
-        channel_id: Channel ID or username
-        limit: Required message limit
-        max_age_hours: Maximum cache age in hours (default 8 hours)
-        
-    Returns:
-        List of messages or None if cache not found, expired or limit doesn't match
+    Retrieve message history from cache if fresh and the cached fetch covers ``limit``.
+
+    Messages are stored newest-first. A cached entry fetched with an equal-or-larger limit
+    serves a smaller request by slicing (prefix). A smaller cached fetch is a miss UNLESS
+    the channel is exhausted (fewer messages exist than were asked for), in which case the
+    cache already holds the entire recent history and can serve any larger request.
     """
     try:
-        cache_file = _get_history_cache_file_path(channel_id)
-        
-        if not os.path.exists(cache_file):
-            logger.info(f"history_cache_miss: channel {channel_id}, limit {limit}, cache file not found")
+        cache_file = _cache_file_path(channel_id, 'history.json')
+        payload = _load_entry(cache_file, max_age_hours)
+        if payload is None:
+            logger.info(f"history_cache_miss: channel {channel_id}, limit {limit}")
             return None
-        
-        with open(cache_file, 'rb') as f:
-            cache_data = pickle.load(f)
-        
-        # Check cache age with randomization
-        cache_age = time.time() - cache_data['timestamp']
-        # Add randomness up to 20% of max_age_seconds
-        random_factor = 1 - random.uniform(0, 0.2)
-        adjusted_max_age = max_age_hours * 3600 * random_factor
-        
-        if cache_age > adjusted_max_age:
-            logger.info(f"history_cache_expired: channel {channel_id}, limit {limit}, age {cache_age:.1f}s > adjusted max {adjusted_max_age:.1f}s (random factor: {random_factor:.2f})")
+
+        cached_limit = payload.get('limit', 0)
+        raw_messages = payload['messages']
+        # Serve when fetched with an equal-or-larger limit, OR when the channel is exhausted
+        # (fewer messages exist than asked -> cache holds entire recent history).
+        if cached_limit < limit and len(raw_messages) >= cached_limit:
+            logger.info(f"history_cache_limit_short: channel {channel_id}, cached limit {cached_limit}, requested {limit}")
             return None
-        
-        # Check if limit matches
-        cached_limit = cache_data.get('limit', 0)
-        if cached_limit != limit:
-            logger.info(f"history_cache_limit_mismatch: channel {channel_id}, cached limit {cached_limit}, requested limit {limit}")
-            return None
-        
-        # Restore message list; handle old cache files that used double-pickle (bytes = old format)
-        raw = cache_data['messages']
-        if isinstance(raw, bytes):
-            messages = pickle.loads(raw)
-        else:
-            messages = raw
-        logger.info(f"history_cache_hit: channel {channel_id}, limit {limit}, messages {len(messages)}, age {cache_age:.1f}s")
+
+        messages = restore_messages(raw_messages[:limit])
+        logger.info(f"history_cache_hit: channel {channel_id}, served {limit} of cached {cached_limit}, messages {len(messages)}")
         return messages
-    
     except Exception as e:
         logger.error(f"history_cache_read_error: channel {channel_id}, limit {limit}, error {str(e)}")
-        # In case of cache read error, better return None and request fresh data
         return None
 
+
 async def cached_get_chat_history(client: Client, channel_id: Union[str, int], limit: int = 20) -> List[Message]:
     """
     Gets chat message history with caching.
-    
+
     Args:
         client: Pyrogram client
         channel_id: Channel ID or username
         limit: Maximum number of messages to retrieve
-        
+
     Returns:
-        List of messages, same as original client.get_chat_history()
+        List of messages, same as original client.get_chat_history(). On a cache miss the
+        live pyrogram Messages are returned; on a hit, restored CachedMessage objects.
     """
     cached_messages = await asyncio.to_thread(_get_history_from_cache, channel_id, limit)
-    
+
     if cached_messages is not None:
         return cached_messages
-    
+
     try:
         logger.info(f"history_cache_request: fetching fresh history for channel {channel_id}, limit {limit}")
         # Hold the global RPC gate for the live fetch and bound the RPC body with the
@@ -148,57 +195,39 @@ async def cached_get_chat_history(client: Client, channel_id: Union[str, int], l
         async with tg_rpc_bounded(Config["tg_rpc_timeout"]):
             messages = [m async for m in client.get_chat_history(channel_id, limit=limit)]
         await asyncio.to_thread(_save_history_to_cache, channel_id, messages, limit)
-        
+
         return messages
     except Exception as e:
         logger.error(f"history_cache_request_error: channel {channel_id}, limit {limit}, error {str(e)}")
         raise
 
 
-def _get_chat_cache_file_path(channel_id: Union[str, int]) -> str:
-    """Returns path to the channel-info cache file for the channel."""
-    if not os.path.exists(CACHE_DIR):
-        os.makedirs(CACHE_DIR, exist_ok=True)
-        logger.info(f"cache_dir_created: path {CACHE_DIR}")
-    channel_id_str = str(channel_id)
-    safe_filename = channel_id_str.replace('/', '_').replace('\\', '_')
-    # Distinct suffix so channel-info files never collide with history (.cache) files.
-    return os.path.join(CACHE_DIR, f"{safe_filename}.chatinfo")
-
-
+# --------------------------------------------------------------------------- #
+# Channel-info cache.
+# --------------------------------------------------------------------------- #
 def _save_chat_to_cache(channel_id: Union[str, int], data: dict) -> None:
     """Saves channel-info (id/title/username) to cache."""
     try:
-        cache_file = _get_chat_cache_file_path(channel_id)
-        cache_data = {'timestamp': time.time(), 'data': data}
-        with open(cache_file, 'wb') as f:
-            pickle.dump(cache_data, f)
+        cache_file = _cache_file_path(channel_id, 'chatinfo.json')
+        _store_entry(cache_file, {'data': data})
         logger.info(f"chatinfo_cache_saved: channel {channel_id}, file {cache_file}")
     except Exception as e:
         logger.error(f"chatinfo_cache_save_error: channel {channel_id}, error {str(e)}")
 
 
 def _get_chat_from_cache(channel_id: Union[str, int], max_age_hours: int = CHAT_CACHE_TTL_HOURS) -> Optional[dict]:
-    """Retrieves channel-info from cache if not older than max_age_hours (with up-to-20% randomization)."""
+    """Retrieves channel-info from cache if fresh (jittered TTL)."""
     try:
-        cache_file = _get_chat_cache_file_path(channel_id)
-        if not os.path.exists(cache_file):
-            logger.info(f"chatinfo_cache_miss: channel {channel_id}, cache file not found")
+        cache_file = _cache_file_path(channel_id, 'chatinfo.json')
+        payload = _load_entry(cache_file, max_age_hours)
+        if payload is None:
+            logger.info(f"chatinfo_cache_miss: channel {channel_id}")
             return None
-        with open(cache_file, 'rb') as f:
-            cache_data = pickle.load(f)
-        cache_age = time.time() - cache_data['timestamp']
-        # Add randomness up to 20% of max age, same approach as the history cache.
-        random_factor = 1 - random.uniform(0, 0.2)
-        adjusted_max_age = max_age_hours * 3600 * random_factor
-        if cache_age > adjusted_max_age:
-            logger.info(f"chatinfo_cache_expired: channel {channel_id}, age {cache_age:.1f}s > adjusted max {adjusted_max_age:.1f}s (random factor: {random_factor:.2f})")
-            return None
-        data = cache_data.get('data')
+        data = payload.get('data')
         if not isinstance(data, dict):
             logger.warning(f"chatinfo_cache_invalid: channel {channel_id}, unexpected payload type {type(data).__name__}")
             return None
-        logger.info(f"chatinfo_cache_hit: channel {channel_id}, age {cache_age:.1f}s")
+        logger.info(f"chatinfo_cache_hit: channel {channel_id}")
         return data
     except Exception as e:
         logger.error(f"chatinfo_cache_read_error: channel {channel_id}, error {str(e)}")
@@ -228,3 +257,64 @@ async def cached_get_chat(client: Client, channel_id: Union[str, int]) -> Simple
     }
     await asyncio.to_thread(_save_chat_to_cache, channel_id, data)
     return SimpleNamespace(**data)
+
+
+# --------------------------------------------------------------------------- #
+# Maintenance: legacy cleanup + age sweep.
+# --------------------------------------------------------------------------- #
+def cleanup_legacy_cache_files() -> int:
+    """Delete legacy binary cache files (*.cache incl. *_history.cache, and *.chatinfo).
+
+    The new store uses *.history.json / *.chatinfo.json, so these old-format files are
+    dead weight and would otherwise never be reclaimed. Returns the number removed.
+    """
+    removed = 0
+    if not os.path.isdir(CACHE_DIR):
+        return 0
+    try:
+        names = os.listdir(CACHE_DIR)
+    except OSError as e:
+        logger.warning(f"cleanup_legacy_list_error: dir {CACHE_DIR}, error {str(e)}")
+        return 0
+    for name in names:
+        if name.endswith('.cache') or name.endswith('.chatinfo'):
+            try:
+                os.remove(os.path.join(CACHE_DIR, name))
+                removed += 1
+            except OSError as e:
+                logger.warning(f"cleanup_legacy_remove_error: file {name}, error {str(e)}")
+    if removed:
+        logger.info(f"cleanup_legacy_cache_files: removed {removed} legacy files from {CACHE_DIR}")
+    return removed
+
+
+def sweep_tgcache(max_age_days: int = 7) -> int:
+    """Delete files in CACHE_DIR whose mtime is older than ``max_age_days``.
+
+    Reclaims cache for dead channels and orphaned uuid tmp files. A race with an in-flight
+    writer is POSSIBLE (stat -> unlink is not atomic against os.replace) but harmless: the
+    worst outcome is one extra cache miss. This is NOT an atomicity guarantee. Returns the
+    number of files removed.
+    """
+    removed = 0
+    if not os.path.isdir(CACHE_DIR):
+        return 0
+    cutoff = time.time() - max_age_days * 86400
+    try:
+        names = os.listdir(CACHE_DIR)
+    except OSError as e:
+        logger.warning(f"sweep_tgcache_list_error: dir {CACHE_DIR}, error {str(e)}")
+        return 0
+    for name in names:
+        path = os.path.join(CACHE_DIR, name)
+        try:
+            if not os.path.isfile(path):
+                continue
+            if os.path.getmtime(path) < cutoff:
+                os.remove(path)
+                removed += 1
+        except OSError as e:
+            logger.warning(f"sweep_tgcache_remove_error: file {name}, error {str(e)}")
+    if removed:
+        logger.info(f"sweep_tgcache: removed {removed} stale files (> {max_age_days}d) from {CACHE_DIR}")
+    return removed