feat(cache): снапшот-кеш истории вместо pickle + префиксный limit + джиттер TTL + age-sweep
Docker Image CI / build (pull_request) Waiting to run

Кеш истории (data/tgcache/) больше не хранит pickle живых объектов pyrogram.
Новый модуль message_snapshot.py: при записи из Message извлекается JSON-словарь
по явному allowlist (снапшот), при чтении восстанавливается duck-typed
CachedMessage (CachedStr с .html, mutable, deepcopy-safe), неотличимый для
пайплайна рендера от Message. post_parser.py и rss_generator.py НЕ изменены.

tg_cache.py переписан на generic JSON-store (pickle удалён полностью):
- _store_entry/_load_entry: атомарная запись через уникальный .tmp.<uuid4> +
  os.replace, finally чистит свой tmp; версия/TTL/битый JSON -> None;
- пути <safe_key>.history.json / .chatinfo.json;
- пакет B: префиксный limit (miss только когда cached_limit<limit и есть ещё);
- пакет F: джиттер TTL при записи (random.uniform(0.8,1.0), на чтении random не
  зовётся) + sweep_tgcache(age) + cleanup_legacy_cache_files;
- api_server.py: старт-cleanup+sweep в lifespan, per-pass sweep в cache_media_files,
  удаление legacy data/media_file_ids.json.

closes #23

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-10 00:09:08 +03:00
parent 3c9ce72b51
commit 8d317e36c8
5 changed files with 1106 additions and 101 deletions
+202
View File
@@ -0,0 +1,202 @@
"""Tests for the tg_cache generic JSON store, prefix-limit, jitter and sweep (issue #23).
Задания 2 / 6 / 17 / 18, verification item 8, 11, 12.
"""
import json
import os
import time
from types import SimpleNamespace
import pytest
import tg_cache
from message_snapshot import SNAPSHOT_VERSION
@pytest.fixture
def cache_dir(tmp_path, monkeypatch):
d = tmp_path / "tgcache"
d.mkdir()
monkeypatch.setattr(tg_cache, "CACHE_DIR", str(d))
return d
# --------------------------------------------------------------------------- #
# _store_entry / _load_entry.
# --------------------------------------------------------------------------- #
def test_store_load_roundtrip(cache_dir):
path = str(cache_dir / "x.history.json")
tg_cache._store_entry(path, {"limit": 5, "messages": [{"id": 1}]})
loaded = tg_cache._load_entry(path, max_age_hours=8)
assert loaded is not None
assert loaded["limit"] == 5
assert loaded["messages"] == [{"id": 1}]
assert loaded["version"] == SNAPSHOT_VERSION
assert 0.8 <= loaded["jitter"] <= 1.0
def test_load_missing_file(cache_dir):
assert tg_cache._load_entry(str(cache_dir / "nope.json"), 8) is None
def test_load_expired_ttl(cache_dir):
path = str(cache_dir / "x.history.json")
tg_cache._store_entry(path, {"data": 1})
# max_age_hours=0 -> adjusted max age 0 -> any positive age is expired.
assert tg_cache._load_entry(path, max_age_hours=0) is None
# Still fresh with a real TTL.
assert tg_cache._load_entry(path, max_age_hours=8) is not None
def test_load_version_mismatch(cache_dir):
path = str(cache_dir / "x.history.json")
with open(path, "w", encoding="utf-8") as f:
json.dump({"version": SNAPSHOT_VERSION + 999, "timestamp": time.time(), "data": 1}, f)
assert tg_cache._load_entry(path, 8) is None
def test_load_corrupt_json(cache_dir):
path = str(cache_dir / "x.history.json")
with open(path, "w", encoding="utf-8") as f:
f.write("{not valid json")
assert tg_cache._load_entry(path, 8) is None
def test_store_uses_unique_tmp_names(cache_dir, monkeypatch):
path = str(cache_dir / "x.history.json")
seen = []
real_replace = os.replace
def spy_replace(src, dst):
seen.append(src)
return real_replace(src, dst)
monkeypatch.setattr(tg_cache.os, "replace", spy_replace)
tg_cache._store_entry(path, {"data": 1})
tg_cache._store_entry(path, {"data": 2})
assert len(seen) == 2
assert seen[0] != seen[1] # each writer used a distinct tmp path
for src in seen:
assert src.startswith(path + ".tmp.")
# No tmp leftovers.
assert [n for n in os.listdir(cache_dir) if ".tmp." in n] == []
# --------------------------------------------------------------------------- #
# cleanup_legacy_cache_files / sweep_tgcache.
# --------------------------------------------------------------------------- #
def test_cleanup_legacy_keeps_new_format(cache_dir):
(cache_dir / "chan.cache").write_text("x")
(cache_dir / "chan_history.cache").write_text("x")
(cache_dir / "chan.chatinfo").write_text("x")
(cache_dir / "chan.history.json").write_text("x")
(cache_dir / "chan.chatinfo.json").write_text("x")
removed = tg_cache.cleanup_legacy_cache_files()
assert removed == 3
remaining = set(os.listdir(cache_dir))
assert remaining == {"chan.history.json", "chan.chatinfo.json"}
def test_sweep_removes_only_old(cache_dir):
old = cache_dir / "old.history.json"
new = cache_dir / "new.history.json"
old.write_text("x")
new.write_text("x")
old_time = time.time() - 8 * 86400 # 8 days old, threshold is 7
os.utime(old, (old_time, old_time))
removed = tg_cache.sweep_tgcache(max_age_days=7)
assert removed == 1
remaining = set(os.listdir(cache_dir))
assert remaining == {"new.history.json"}
def test_sweep_removes_orphan_tmp(cache_dir):
orphan = cache_dir / "x.history.json.tmp.deadbeef"
orphan.write_text("x")
old_time = time.time() - 10 * 86400
os.utime(orphan, (old_time, old_time))
assert tg_cache.sweep_tgcache(max_age_days=7) == 1
# --------------------------------------------------------------------------- #
# History prefix-limit (Задание 6).
# --------------------------------------------------------------------------- #
def _fake_messages(n):
return [SimpleNamespace(id=i) for i in range(n)]
def test_prefix_larger_cache_serves_smaller_request(cache_dir):
tg_cache._save_history_to_cache("chan", _fake_messages(100), limit=100)
served = tg_cache._get_history_from_cache("chan", limit=50)
assert served is not None
assert len(served) == 50
assert [m.id for m in served] == list(range(50)) # order preserved (newest-first slice)
def test_prefix_smaller_cache_is_miss(cache_dir):
tg_cache._save_history_to_cache("chan", _fake_messages(50), limit=50)
assert tg_cache._get_history_from_cache("chan", limit=100) is None
def test_prefix_exhausted_channel_serves_larger_request(cache_dir):
# Fetched with limit 100 but the channel only has 37 messages -> entire history cached.
tg_cache._save_history_to_cache("chan", _fake_messages(37), limit=100)
served = tg_cache._get_history_from_cache("chan", limit=200)
assert served is not None
assert len(served) == 37
def test_save_stores_fetch_limit_not_len(cache_dir):
tg_cache._save_history_to_cache("chan", _fake_messages(37), limit=100)
path = tg_cache._cache_file_path("chan", "history.json")
payload = tg_cache._load_entry(path, max_age_hours=8)
assert payload["limit"] == 100
assert len(payload["messages"]) == 37
def test_history_ttl_independent_of_prefix(cache_dir):
tg_cache._save_history_to_cache("chan", _fake_messages(100), limit=100)
# Expired regardless of prefix match.
assert tg_cache._get_history_from_cache("chan", limit=50, max_age_hours=0) is None
# --------------------------------------------------------------------------- #
# Jitter stability (Задание 17 / item 12).
# --------------------------------------------------------------------------- #
def test_jitter_read_is_stable(cache_dir):
path = str(cache_dir / "x.history.json")
tg_cache._store_entry(path, {"data": 1})
# Repeated reads near a TTL boundary must give a stable result: the jitter is fixed at
# write time and _load_entry never calls random().
results = [tg_cache._load_entry(path, max_age_hours=8) is not None for _ in range(10)]
assert all(results)
def test_load_does_not_call_random(cache_dir, monkeypatch):
path = str(cache_dir / "x.history.json")
tg_cache._store_entry(path, {"data": 1})
def boom(*a, **k):
raise AssertionError("random.uniform must not be called at read time")
monkeypatch.setattr(tg_cache.random, "uniform", boom)
assert tg_cache._load_entry(path, max_age_hours=8) is not None
# --------------------------------------------------------------------------- #
# chatinfo store round trip.
# --------------------------------------------------------------------------- #
def test_chatinfo_roundtrip(cache_dir):
tg_cache._save_chat_to_cache("chan", {"id": -100, "title": "T", "username": "u"})
data = tg_cache._get_chat_from_cache("chan")
assert data == {"id": -100, "title": "T", "username": "u"}
def test_no_pickle_import():
import pathlib
src = pathlib.Path(tg_cache.__file__).read_text()
assert "pickle" not in src