From 432715200fd2e6ba61a35249c8fa6de67de4e51f Mon Sep 17 00:00:00 2001 From: agent_coder Date: Fri, 10 Jul 2026 01:25:41 +0300 Subject: [PATCH] =?UTF-8?q?fix(cache):=20=D1=81=D0=BD=D0=B0=D0=BF=D1=88?= =?UTF-8?q?=D0=BE=D1=82=20=D0=B2=D1=81=D0=B5=D1=85=20=D1=81=D0=BF=D0=B5?= =?UTF-8?q?=D1=86-=D0=BC=D0=B5=D0=B4=D0=B8=D0=B0=20=D1=82=D0=B8=D0=BF?= =?UTF-8?q?=D0=BE=D0=B2=20+=20render-parity=20=D1=82=D0=B5=D1=81=D1=82=20+?= =?UTF-8?q?=20file=5Fsize=20=D0=B4=D0=BB=D1=8F=20=D0=B2=D1=81=D0=B5=D1=85?= =?UTF-8?q?=20=D0=BC=D0=B5=D0=B4=D0=B8=D0=B0=20(=D1=80=D0=B5=D0=B2=D1=8C?= =?UTF-8?q?=D1=8E=20#44)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ревью #44: allowlist снапшота был неполон -> спец-медиа блоки тихо выпадали на cache-hit (нарушение golden-render инварианта). Исправлено: - Снапшот+восстановление 11 типов (story, contact, location, venue, dice, game, giveaway, giveaway_winners, checklist, paid_media + live_photo) — ровно те подполя, что читает _format_special_media/find_file_id_in_message. SNAPSHOT_VERSION 1->2 (v1-файлы без новых ключей инвалидируются). - Render-parity тест (tests/test_snapshot_render_parity.py): корпус из 35+ кейсов (все медиа/спец-медиа/web_page/poll/forward Case 1-5/reactions/media group), live-рендер == restore(snapshot(live))-рендер байт-в-байт через реальный пайплайн; мутационно проверено (выброс поля -> тест краснеет). - F1: file_size досниман для document/audio/animation/video_note/story-media (правило '>100MB не кэшировать' в _save_media_file_ids читает file_size у любого выбранного медиа-объекта; раньше был только у video/live_photo -> >100MB документ ошибочно кэшировался на cache-hit). - flake8/pylint-хедеры на 3 новых тест-файла. post_parser.py/rss_generator.py не тронуты; pickle отсутствует. Co-Authored-By: Claude Opus 4.8 (1M context) --- message_snapshot.py | 218 ++++++++++++++++++- tests/test_message_snapshot.py | 201 ++++++++++++++++++ tests/test_snapshot_render_parity.py | 302 +++++++++++++++++++++++++++ tests/test_tg_cache_store.py | 4 + 4 files changed, 716 insertions(+), 9 deletions(-) create mode 100644 tests/test_snapshot_render_parity.py diff --git a/message_snapshot.py b/message_snapshot.py index 862281b..8e21ae2 100644 --- a/message_snapshot.py +++ b/message_snapshot.py @@ -27,7 +27,11 @@ logger = logging.getLogger(__name__) # Bump when the snapshot schema changes in a backwards-incompatible way; a mismatch # makes _load_entry treat the cache file as a miss (old files are simply re-fetched). -SNAPSHOT_VERSION = 1 +# v2: added the special-media info-block types (story, contact, location, venue, dice, +# game, giveaway, giveaway_winners, checklist, paid_media) plus live_photo. A v1 file lacks +# these keys, so a cached special-media / live-photo message would restore with an empty +# block — invalidate v1 files. +SNAPSHOT_VERSION = 2 class CachedStr(str): @@ -189,6 +193,103 @@ def _snapshot_web_page(wp: Any) -> Optional[dict]: } +# --------------------------------------------------------------------------- # +# Special-media snapshots (issue #23 review fix). +# +# _format_special_media (post_parser.py) and find_file_id_in_message +# (api_server.py) read a handful of type-specific attributes off the Message that +# were NOT in schema v1. On a cache hit a restored Message carried media= but +# message.=None, so the special info block silently vanished (render +# divergence vs. a live Message). Each helper below snapshots EXACTLY the sub-fields +# the renderer + find_file_id read for that type — nothing more, nothing less. +# --------------------------------------------------------------------------- # +def _snapshot_story(story: Any) -> Optional[dict]: + """STORY: _story_media_object / find_file_id_in_message read story.video and + story.photo, each by .file_unique_id (video wins over photo). The chosen object is + also the one _save_media_file_ids reads .file_size off for the >100MB skip, so + file_size is snapshotted too (else a >100MB story media would be collected on a + cache hit — F1). The render URL is still built from file_unique_id.""" + if story is None: + return None + return { + "video": _snapshot_obj(getattr(story, "video", None), ["file_unique_id", "file_size"]), + "photo": _snapshot_obj(getattr(story, "photo", None), ["file_unique_id", "file_size"]), + } + + +def _snapshot_venue(venue: Any) -> Optional[dict]: + """VENUE: reads .title, .address and its nested .location (.latitude/.longitude via + _format_osm_link).""" + if venue is None: + return None + return { + "title": getattr(venue, "title", None), + "address": getattr(venue, "address", None), + "location": _snapshot_obj(getattr(venue, "location", None), ["latitude", "longitude"]), + } + + +def _snapshot_giveaway(giveaway: Any) -> Optional[dict]: + """GIVEAWAY: reads .quantity, .months, .stars, .until_date (a datetime rendered via + strftime) and .description. until_date is stored as isoformat and restored to a + datetime so the strftime branch reproduces byte-for-byte.""" + if giveaway is None: + return None + until = getattr(giveaway, "until_date", None) + return { + "quantity": getattr(giveaway, "quantity", None), + "months": getattr(giveaway, "months", None), + "stars": getattr(giveaway, "stars", None), + # Only a real datetime renders (renderer gates on hasattr(.,'strftime')); anything + # else is dropped to None so the same "no until date" branch is taken on restore. + "until_date": until.isoformat() if hasattr(until, "isoformat") else None, + "description": getattr(giveaway, "description", None), + } + + +def _snapshot_checklist(checklist: Any) -> Optional[dict]: + """CHECKLIST: reads .title (used ONLY when it isinstance str) and .tasks; each task + reads .text and the truthiness of .completed_by / .completion_date (→ ☑/☐). + + title is stored only when it is a str (matching the renderer's isinstance gate, which + otherwise renders an empty title). completed_by / completion_date are stored as bools + (the renderer only tests their truthiness) so a live User/datetime stays JSON-safe.""" + if checklist is None: + return None + title = getattr(checklist, "title", None) + snap_tasks = [] + for task in (getattr(checklist, "tasks", None) or []): + text = getattr(task, "text", "") + # Mirror the renderer: a non-str text is str()-ified. Storing the resolved string + # keeps the restored value isinstance-str, reproducing the same rendered bytes. + text_str = text if isinstance(text, str) else str(text) + snap_tasks.append({ + "text": text_str, + "completed_by": bool(getattr(task, "completed_by", None)), + "completion_date": bool(getattr(task, "completion_date", None)), + }) + return { + "title": title if isinstance(title, str) else None, + "tasks": snap_tasks, + } + + +def _snapshot_paid_media(paid_media: Any) -> Optional[dict]: + """PAID_MEDIA: _generate_html_media reads .stars_amount (default 0) and the LENGTH of + .media. Snapshot the renderer's effective stars value and the item count; the media + list is restored as N placeholders so len() reproduces.""" + if paid_media is None: + return None + media = getattr(paid_media, "media", None) + count = len(media) if isinstance(media, (list, tuple)) else 0 + return { + # Snapshot getattr(...,0) so a missing attribute restores to 0 exactly as the + # renderer would have defaulted it (a present None stays None on both sides). + "stars_amount": getattr(paid_media, "stars_amount", 0), + "media_count": count, + } + + def snapshot_message(message: Any) -> dict: """Extract a JSON-serializable snapshot (schema v1) from a pyrogram Message. @@ -220,12 +321,34 @@ def snapshot_message(message: Any) -> dict: "web_page": _snapshot_web_page(getattr(message, "web_page", None)), "photo": _snapshot_obj(getattr(message, "photo", None), ["file_unique_id"]), "video": _snapshot_obj(getattr(message, "video", None), ["file_unique_id", "file_size"]), - "document": _snapshot_obj(getattr(message, "document", None), ["file_unique_id", "mime_type"]), - "audio": _snapshot_obj(getattr(message, "audio", None), ["file_unique_id", "mime_type"]), + # file_size is snapshotted for every type whose selected object flows into + # _save_media_file_ids' >100MB skip (MEDIA_SOURCES: document/audio/animation/ + # video_note select this exact object). Without it a restored >100MB media + # would have file_size=None and be wrongly collected on a cache hit (F1). + "document": _snapshot_obj(getattr(message, "document", None), ["file_unique_id", "mime_type", "file_size"]), + "audio": _snapshot_obj(getattr(message, "audio", None), ["file_unique_id", "mime_type", "file_size"]), "voice": _snapshot_obj(getattr(message, "voice", None), ["file_unique_id", "mime_type"]), - "video_note": _snapshot_obj(getattr(message, "video_note", None), ["file_unique_id"]), - "animation": _snapshot_obj(getattr(message, "animation", None), ["file_unique_id"]), + "video_note": _snapshot_obj(getattr(message, "video_note", None), ["file_unique_id", "file_size"]), + "animation": _snapshot_obj(getattr(message, "animation", None), ["file_unique_id", "file_size"]), "sticker": _snapshot_obj(getattr(message, "sticker", None), ["file_unique_id", "emoji", "is_video"]), + # LIVE_PHOTO (Kurigram 2.2.23) renders as a video element via the video_loop_400 + # kind. MEDIA_SOURCES/_get_file_unique_id/_save_media_file_ids read live_photo's + # file_unique_id + file_size (the >100MB skip); find_file_id also reads file_unique_id. + # Mirror the `video` allowlist so a cached live-photo message keeps its media block. + "live_photo": _snapshot_obj(getattr(message, "live_photo", None), ["file_unique_id", "file_size"]), + # Special-media info-block types (issue #23 review fix). Each reads a fixed set of + # sub-fields in _format_special_media / find_file_id_in_message; snapshot exactly those. + "story": _snapshot_story(getattr(message, "story", None)), + "contact": _snapshot_obj(getattr(message, "contact", None), ["first_name", "last_name", "phone_number"]), + "location": _snapshot_obj(getattr(message, "location", None), ["latitude", "longitude"]), + "venue": _snapshot_venue(getattr(message, "venue", None)), + "dice": _snapshot_obj(getattr(message, "dice", None), ["emoji", "value"]), + "game": _snapshot_obj(getattr(message, "game", None), ["title"]), + "giveaway": _snapshot_giveaway(getattr(message, "giveaway", None)), + "giveaway_winners": _snapshot_obj( + getattr(message, "giveaway_winners", None), ["winner_count", "quantity", "prize_description"]), + "checklist": _snapshot_checklist(getattr(message, "checklist", None)), + "paid_media": _snapshot_paid_media(getattr(message, "paid_media", None)), } @@ -326,6 +449,67 @@ def _restore_str(d: Optional[dict]) -> Optional[CachedStr]: return CachedStr.build(d.get("plain", ""), d.get("html", d.get("plain", ""))) +# --------------------------------------------------------------------------- # +# Special-media restores (issue #23 review fix). Mirror the snapshot helpers above: +# rebuild each object so the exact attributes _format_special_media / +# find_file_id_in_message read exist, restoring the special block on a cache hit. +# --------------------------------------------------------------------------- # +def _restore_story(d: Optional[dict]) -> Optional[SimpleNamespace]: + if d is None: + return None + return SimpleNamespace( + video=_ns(d.get("video"), ["file_unique_id", "file_size"]), + photo=_ns(d.get("photo"), ["file_unique_id", "file_size"]), + ) + + +def _restore_venue(d: Optional[dict]) -> Optional[SimpleNamespace]: + if d is None: + return None + return SimpleNamespace( + title=d.get("title"), + address=d.get("address"), + location=_ns(d.get("location"), ["latitude", "longitude"]), + ) + + +def _restore_giveaway(d: Optional[dict]) -> Optional[SimpleNamespace]: + if d is None: + return None + until = d.get("until_date") + return SimpleNamespace( + quantity=d.get("quantity"), + months=d.get("months"), + stars=d.get("stars"), + # Restore a datetime so hasattr(until_date, 'strftime') holds exactly as on a live + # object; None (no/invalid date) restores as None and the strftime branch is skipped. + until_date=datetime.fromisoformat(until) if until is not None else None, + description=d.get("description"), + ) + + +def _restore_checklist(d: Optional[dict]) -> Optional[SimpleNamespace]: + if d is None: + return None + restored_tasks = [ + SimpleNamespace( + text=t.get("text"), + completed_by=t.get("completed_by"), + completion_date=t.get("completion_date"), + ) + for t in (d.get("tasks") or []) + ] + return SimpleNamespace(title=d.get("title"), tasks=restored_tasks) + + +def _restore_paid_media(d: Optional[dict]) -> Optional[SimpleNamespace]: + if d is None: + return None + count = d.get("media_count") or 0 + # media is only ever len()'d by the renderer, so N placeholder items reproduce the count. + return SimpleNamespace(stars_amount=d.get("stars_amount"), media=[None] * count) + + class CachedMessage: """Duck-typed stand-in for a pyrogram Message, restored from a snapshot dict. @@ -359,12 +543,28 @@ class CachedMessage: self.web_page = _restore_web_page(data.get("web_page")) self.photo = _ns(data.get("photo"), ["file_unique_id"]) self.video = _ns(data.get("video"), ["file_unique_id", "file_size"]) - self.document = _ns(data.get("document"), ["file_unique_id", "mime_type"]) - self.audio = _ns(data.get("audio"), ["file_unique_id", "mime_type"]) + self.document = _ns(data.get("document"), ["file_unique_id", "mime_type", "file_size"]) + self.audio = _ns(data.get("audio"), ["file_unique_id", "mime_type", "file_size"]) self.voice = _ns(data.get("voice"), ["file_unique_id", "mime_type"]) - self.video_note = _ns(data.get("video_note"), ["file_unique_id"]) - self.animation = _ns(data.get("animation"), ["file_unique_id"]) + self.video_note = _ns(data.get("video_note"), ["file_unique_id", "file_size"]) + self.animation = _ns(data.get("animation"), ["file_unique_id", "file_size"]) self.sticker = _ns(data.get("sticker"), ["file_unique_id", "emoji", "is_video"]) + # LIVE_PHOTO: restored like `video` so the video_loop_400 media block renders on a + # cache hit (file_unique_id → URL; file_size → the >100MB collection skip). + self.live_photo = _ns(data.get("live_photo"), ["file_unique_id", "file_size"]) + # Special-media info-block types (issue #23 review fix): restored so the type's + # info block renders on a cache hit exactly as for a live Message. + self.story = _restore_story(data.get("story")) + self.contact = _ns(data.get("contact"), ["first_name", "last_name", "phone_number"]) + self.location = _ns(data.get("location"), ["latitude", "longitude"]) + self.venue = _restore_venue(data.get("venue")) + self.dice = _ns(data.get("dice"), ["emoji", "value"]) + self.game = _ns(data.get("game"), ["title"]) + self.giveaway = _restore_giveaway(data.get("giveaway")) + self.giveaway_winners = _ns( + data.get("giveaway_winners"), ["winner_count", "quantity", "prize_description"]) + self.checklist = _restore_checklist(data.get("checklist")) + self.paid_media = _restore_paid_media(data.get("paid_media")) def __str__(self) -> str: return json.dumps(self._snapshot, default=str) diff --git a/tests/test_message_snapshot.py b/tests/test_message_snapshot.py index e1a43b5..548d29b 100644 --- a/tests/test_message_snapshot.py +++ b/tests/test_message_snapshot.py @@ -1,3 +1,7 @@ +# flake8: noqa +# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring +# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long +# pylance: disable=reportMissingImports, reportMissingModuleSource """Tests for message_snapshot.py (issue #23, Задания 4-5,7). These verify that a pyrogram Message survives snapshot -> JSON -> restore as a @@ -265,6 +269,72 @@ def test_normal_video_is_collected(): assert pp._pending_media_ids[0][2] == "vid2" +# --------------------------------------------------------------------------- # +# Test 9b — the >100MB skip must fire for every media type whose selected object +# flows into _save_media_file_ids' size check, not just video/live_photo. Before +# file_size was added to these snapshots a restored >100MB doc/audio/animation/ +# video_note lacked file_size and was wrongly collected on a cache hit (F1). +# --------------------------------------------------------------------------- # +_BIG = 200 * 1024 * 1024 + + +@pytest.mark.parametrize("media,attr,snap_obj", [ + ("DOCUMENT", "document", {"file_unique_id": "doc_big", "mime_type": "application/zip", "file_size": _BIG}), + ("AUDIO", "audio", {"file_unique_id": "aud_big", "mime_type": "audio/mpeg", "file_size": _BIG}), + ("ANIMATION", "animation", {"file_unique_id": "anim_big", "file_size": _BIG}), + ("VIDEO_NOTE", "video_note", {"file_unique_id": "vn_big", "file_size": _BIG}), +]) +def test_large_media_not_collected(media, attr, snap_obj): + snap = snapshot_message(SimpleNamespace()) + snap["media"] = media + snap[attr] = snap_obj + snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None} + restored = restore_message(snap) + # file_size survives the snapshot/restore round-trip (the round-trip is the fix). + assert getattr(restored, attr).file_size == _BIG + + pp = PostParser(None) + pp._save_media_file_ids(restored) + assert pp._pending_media_ids == [] + + +@pytest.mark.parametrize("media,attr,fuid,snap_obj", [ + ("DOCUMENT", "document", "doc_ok", {"file_unique_id": "doc_ok", "mime_type": "application/zip", "file_size": 1024}), + ("AUDIO", "audio", "aud_ok", {"file_unique_id": "aud_ok", "mime_type": "audio/mpeg", "file_size": 1024}), + ("ANIMATION", "animation", "anim_ok", {"file_unique_id": "anim_ok", "file_size": 1024}), + ("VIDEO_NOTE", "video_note", "vn_ok", {"file_unique_id": "vn_ok", "file_size": 1024}), +]) +def test_normal_media_is_collected(media, attr, fuid, snap_obj): + snap = snapshot_message(SimpleNamespace()) + snap["id"] = 11 + snap["media"] = media + snap[attr] = snap_obj + snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None} + restored = restore_message(snap) + + pp = PostParser(None) + pp._save_media_file_ids(restored) + assert len(pp._pending_media_ids) == 1 + assert pp._pending_media_ids[0][2] == fuid + + +def test_large_story_media_not_collected(): + # A >100MB story video (selected over the photo) must be skipped on a cache hit. + snap = snapshot_message(SimpleNamespace()) + snap["media"] = "STORY" + snap["story"] = { + "video": {"file_unique_id": "story_vid_big", "file_size": _BIG}, + "photo": None, + } + snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None} + restored = restore_message(snap) + assert restored.story.video.file_size == _BIG + + pp = PostParser(None) + pp._save_media_file_ids(restored) + assert pp._pending_media_ids == [] + + # --------------------------------------------------------------------------- # # Test 10 — restored chat without username -> None, not AttributeError. # --------------------------------------------------------------------------- # @@ -307,3 +377,134 @@ def test_snapshot_messages_list_roundtrip(): restored = restore_messages(snapshot_messages(msgs)) assert [m.id for m in restored] == [0, 1, 2] assert all(isinstance(m, CachedMessage) for m in restored) + + +# --------------------------------------------------------------------------- # +# Special-media round trips (issue #23 review fix). One per type: assert the exact +# sub-fields _format_special_media / find_file_id_in_message read survive the trip. +# These are what silently vanished on a cache hit before the allowlist was completed. +# --------------------------------------------------------------------------- # +def test_story_roundtrip_file_unique_id_survives(): + story = SimpleNamespace( + video=SimpleNamespace(file_unique_id="story-vid"), + photo=SimpleNamespace(file_unique_id="story-pic"), + ) + restored, _ = _roundtrip(SimpleNamespace(story=story)) + # find_file_id_in_message / _story_media_object read story.video/photo.file_unique_id. + assert restored.story.video.file_unique_id == "story-vid" + assert restored.story.photo.file_unique_id == "story-pic" + + +def test_contact_roundtrip(): + contact = SimpleNamespace(first_name="Ann", last_name="Bee", phone_number="+15551234") + restored, _ = _roundtrip(SimpleNamespace(contact=contact)) + assert restored.contact.first_name == "Ann" + assert restored.contact.last_name == "Bee" + assert restored.contact.phone_number == "+15551234" + + +def test_location_roundtrip(): + location = SimpleNamespace(latitude=51.5, longitude=-0.12) + restored, _ = _roundtrip(SimpleNamespace(location=location)) + assert restored.location.latitude == 51.5 + assert restored.location.longitude == -0.12 + + +def test_venue_roundtrip_with_nested_location(): + venue = SimpleNamespace( + title="Big Ben", address="Westminster", + location=SimpleNamespace(latitude=51.5, longitude=-0.12), + ) + restored, _ = _roundtrip(SimpleNamespace(venue=venue)) + assert restored.venue.title == "Big Ben" + assert restored.venue.address == "Westminster" + assert restored.venue.location.latitude == 51.5 + assert restored.venue.location.longitude == -0.12 + + +def test_dice_roundtrip(): + restored, _ = _roundtrip(SimpleNamespace(dice=SimpleNamespace(emoji="🎯", value=6))) + assert restored.dice.emoji == "🎯" + assert restored.dice.value == 6 + + +def test_game_roundtrip(): + restored, _ = _roundtrip(SimpleNamespace(game=SimpleNamespace(title="Chess"))) + assert restored.game.title == "Chess" + + +def test_giveaway_roundtrip_until_date_datetime(): + giveaway = SimpleNamespace( + quantity=3, months=6, stars=None, + until_date=datetime(2030, 12, 31, 12, 0, 0), + description="Prizes!", + ) + restored, _ = _roundtrip(SimpleNamespace(giveaway=giveaway)) + assert restored.giveaway.quantity == 3 + assert restored.giveaway.months == 6 + # until_date must restore as a datetime so the renderer's strftime branch fires. + assert restored.giveaway.until_date.strftime("%d/%m/%Y") == "31/12/2030" + assert restored.giveaway.description == "Prizes!" + + +def test_giveaway_winners_roundtrip(): + winners = SimpleNamespace(winner_count=5, quantity=10, prize_description="Premium") + restored, _ = _roundtrip(SimpleNamespace(giveaway_winners=winners)) + assert restored.giveaway_winners.winner_count == 5 + assert restored.giveaway_winners.quantity == 10 + assert restored.giveaway_winners.prize_description == "Premium" + + +def test_checklist_roundtrip_tasks_and_completion(): + checklist = SimpleNamespace( + title="Todo", + tasks=[ + SimpleNamespace(text="done task", completed_by=SimpleNamespace(id=1), completion_date=None), + SimpleNamespace(text="open task", completed_by=None, completion_date=None), + ], + ) + restored, _ = _roundtrip(SimpleNamespace(checklist=checklist)) + assert restored.checklist.title == "Todo" + t_done, t_open = restored.checklist.tasks + assert t_done.text == "done task" + # Renderer marks ☑ when bool(completed_by or completion_date) is True. + assert bool(t_done.completed_by or t_done.completion_date) is True + assert t_open.text == "open task" + assert bool(t_open.completed_by or t_open.completion_date) is False + + +def test_paid_media_roundtrip_stars_and_item_count(): + paid = SimpleNamespace(stars_amount=50, media=[object(), object(), object()]) + restored, _ = _roundtrip(SimpleNamespace(paid_media=paid)) + assert restored.paid_media.stars_amount == 50 + # Renderer only len()'s .media — the count must survive. + assert len(restored.paid_media.media) == 3 + + +def test_live_photo_roundtrip_file_unique_id_and_size(): + live_photo = SimpleNamespace(file_unique_id="lp1", file_size=4096) + restored, _ = _roundtrip(SimpleNamespace(live_photo=live_photo)) + # MEDIA_SOURCES/_get_file_unique_id + find_file_id read live_photo.file_unique_id; + # _save_media_file_ids reads file_size (the >100MB skip). + assert restored.live_photo.file_unique_id == "lp1" + assert restored.live_photo.file_size == 4096 + + +def test_large_live_photo_not_collected(): + snap = snapshot_message(SimpleNamespace()) + snap["media"] = "LIVE_PHOTO" + snap["live_photo"] = {"file_unique_id": "lpbig", "file_size": 200 * 1024 * 1024} + snap["chat"] = {"id": -1001, "username": "chan", "title": "t", "usernames": None} + restored = restore_message(snap) + + pp = PostParser(None) + pp._save_media_file_ids(restored) + assert pp._pending_media_ids == [] + + +def test_special_media_defaults_present_on_empty_message(): + restored = restore_message(snapshot_message(SimpleNamespace())) + for attr in ["story", "contact", "location", "venue", "dice", "game", + "giveaway", "giveaway_winners", "checklist", "paid_media", "live_photo"]: + assert hasattr(restored, attr) + assert getattr(restored, attr) is None diff --git a/tests/test_snapshot_render_parity.py b/tests/test_snapshot_render_parity.py new file mode 100644 index 0000000..a155d58 --- /dev/null +++ b/tests/test_snapshot_render_parity.py @@ -0,0 +1,302 @@ +# flake8: noqa +# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring +# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long +# pylance: disable=reportMissingImports, reportMissingModuleSource +"""Render-parity oracle for the snapshot cache (issue #23 review fix). + +The core ask of the review: prove that snapshot -> restore preserves EVERYTHING the +render pipeline reads. The old golden test replays a pickle corpus and never renders a +live Message against its snapshot, so a field dropped from the allowlist (the exact +bug the reviewer mutation-proved for web_page/sender_chat/... and for the special-media +types) passes silently. + +This test builds a CORPUS of representative live fake Message objects (plain text, every +media type, every special-media type, web_page, poll, forwards Case 1-5, reactions, +caption, a multi-message media group) and for EACH case: + + 1. renders the LIVE objects through the REAL public feed entry points + (rss_generator.generate_channel_html / generate_channel_rss — the same path the + golden test and production use, driven by monkeypatching tg_cache), + 2. snapshot -> restore every message, renders the RESTORED objects the same way, + 3. asserts the two renders are BYTE-IDENTICAL (HTML and the RSS feed). + +Any field the snapshot fails to preserve makes the two renders diverge and this test go +red. It is a self-contained live-vs-restored parity oracle; it does NOT depend on the +frozen pickle goldens. +""" +import asyncio +from datetime import datetime +from types import SimpleNamespace + +import pytest + +from pyrogram.enums import MessageMediaType + +from message_snapshot import snapshot_messages, restore_messages +from tests import golden_replay as gr + + +# --------------------------------------------------------------------------- # +# Live-object fakes. +# --------------------------------------------------------------------------- # +class FakeStr(str): + """Stand-in for a live pyrogram Str: a str carrying a .html rendering.""" + def __new__(cls, plain, html=None): + obj = str.__new__(cls, plain) + obj._html = html if html is not None else plain + return obj + + @property + def html(self): + return self._html + + +_CHANNEL = "parity_ch" +_CHAT = SimpleNamespace(id=-1001234567890, username=_CHANNEL, title="Parity Channel", usernames=None) + +# The full top-level attribute surface the render pipeline reads (mirrors CachedMessage). +# A live fake sets every one so it renders WITHOUT relying on getattr fallbacks that a +# restored CachedMessage would provide — otherwise the live side, not the snapshot, would +# be the thing under-specified. +_DEFAULTS = dict( + id=0, date=None, text=None, caption=None, media=None, service=None, + media_group_id=None, views=100, show_caption_above_media=False, + reply_to_message_id=None, reply_to_message=None, empty=False, + chat=_CHAT, sender_chat=None, from_user=None, forward_origin=None, + reactions=None, poll=None, web_page=None, photo=None, video=None, + document=None, audio=None, voice=None, video_note=None, animation=None, + sticker=None, story=None, contact=None, location=None, venue=None, + dice=None, game=None, giveaway=None, giveaway_winners=None, + checklist=None, paid_media=None, live_photo=None, +) + +_id_counter = [1000] + + +def make_msg(**overrides): + _id_counter[0] += 1 + fields = dict(_DEFAULTS) + fields["id"] = _id_counter[0] + fields["date"] = datetime(2024, 3, 1, 12, 0, _id_counter[0] % 60) + fields.update(overrides) + return SimpleNamespace(**fields) + + +def _reactions(*reacts): + return SimpleNamespace(reactions=list(reacts)) + + +# --------------------------------------------------------------------------- # +# Corpus: each case is (name, [live messages]) rendered as its own mini feed. +# --------------------------------------------------------------------------- # +def build_corpus(): + corpus = [] + + # Plain text. + corpus.append(("plain_text", [make_msg(text=FakeStr("Hello world", "Hello world"))])) + + # Caption (photo + caption + show_caption_above_media). + corpus.append(("caption_photo", [make_msg( + media=MessageMediaType.PHOTO, + photo=SimpleNamespace(file_unique_id="pcap"), + caption=FakeStr("A caption", "A caption"), + show_caption_above_media=True, + )])) + + # Regular media types. + corpus.append(("photo", [make_msg(media=MessageMediaType.PHOTO, + photo=SimpleNamespace(file_unique_id="ph1"))])) + corpus.append(("video", [make_msg(media=MessageMediaType.VIDEO, + video=SimpleNamespace(file_unique_id="vd1", file_size=2048))])) + corpus.append(("document_pdf", [make_msg(media=MessageMediaType.DOCUMENT, + document=SimpleNamespace(file_unique_id="doc1", mime_type="application/pdf"))])) + corpus.append(("document_other", [make_msg(media=MessageMediaType.DOCUMENT, + document=SimpleNamespace(file_unique_id="doc2", mime_type="image/png"))])) + corpus.append(("audio", [make_msg(media=MessageMediaType.AUDIO, + audio=SimpleNamespace(file_unique_id="au1", mime_type="audio/mpeg"))])) + corpus.append(("voice", [make_msg(media=MessageMediaType.VOICE, + voice=SimpleNamespace(file_unique_id="vo1", mime_type="audio/ogg"))])) + corpus.append(("video_note", [make_msg(media=MessageMediaType.VIDEO_NOTE, + video_note=SimpleNamespace(file_unique_id="vn1"))])) + corpus.append(("animation", [make_msg(media=MessageMediaType.ANIMATION, + animation=SimpleNamespace(file_unique_id="an1"))])) + corpus.append(("sticker_image", [make_msg(media=MessageMediaType.STICKER, + sticker=SimpleNamespace(file_unique_id="st1", emoji="😀", is_video=False))])) + corpus.append(("sticker_video", [make_msg(media=MessageMediaType.STICKER, + sticker=SimpleNamespace(file_unique_id="st2", emoji="🎥", is_video=True))])) + corpus.append(("live_photo", [make_msg(media=MessageMediaType.LIVE_PHOTO, + live_photo=SimpleNamespace(file_unique_id="lp1", file_size=4096))])) + + # --- Special-media types (Fix 1) ------------------------------------- # + corpus.append(("story_photo", [make_msg( + media=MessageMediaType.STORY, + story=SimpleNamespace(video=None, photo=SimpleNamespace(file_unique_id="sto_p")), + )])) + corpus.append(("story_video", [make_msg( + media=MessageMediaType.STORY, + story=SimpleNamespace(video=SimpleNamespace(file_unique_id="sto_v"), photo=None), + )])) + corpus.append(("contact", [make_msg( + media=MessageMediaType.CONTACT, + contact=SimpleNamespace(first_name="Ann", last_name="Bee", phone_number="+15551234"), + )])) + corpus.append(("location", [make_msg( + media=MessageMediaType.LOCATION, + location=SimpleNamespace(latitude=51.50111, longitude=-0.14222), + )])) + corpus.append(("venue", [make_msg( + media=MessageMediaType.VENUE, + venue=SimpleNamespace(title="Big Ben", address="Westminster", + location=SimpleNamespace(latitude=51.50055, longitude=-0.12461)), + )])) + corpus.append(("dice", [make_msg( + media=MessageMediaType.DICE, dice=SimpleNamespace(emoji="🎯", value=6))])) + corpus.append(("game", [make_msg( + media=MessageMediaType.GAME, game=SimpleNamespace(title="Chess Master"))])) + corpus.append(("giveaway", [make_msg( + media=MessageMediaType.GIVEAWAY, + giveaway=SimpleNamespace(quantity=3, months=6, stars=None, + until_date=datetime(2030, 12, 31, 12, 0, 0), + description="Great prizes"), + )])) + corpus.append(("giveaway_stars", [make_msg( + media=MessageMediaType.GIVEAWAY, + giveaway=SimpleNamespace(quantity=2, months=None, stars=500, + until_date=None, description=None), + )])) + corpus.append(("giveaway_winners", [make_msg( + media=MessageMediaType.GIVEAWAY_WINNERS, + giveaway_winners=SimpleNamespace(winner_count=5, quantity=10, prize_description="Premium"), + )])) + corpus.append(("checklist", [make_msg( + media=MessageMediaType.CHECKLIST, + checklist=SimpleNamespace(title="Todo list", tasks=[ + SimpleNamespace(text="done task", completed_by=SimpleNamespace(id=1), completion_date=None), + SimpleNamespace(text="open task", completed_by=None, completion_date=None), + ]), + )])) + corpus.append(("paid_media", [make_msg( + media=MessageMediaType.PAID_MEDIA, + paid_media=SimpleNamespace(stars_amount=50, media=[object(), object()]), + )])) + + # --- web_page + poll -------------------------------------------------- # + corpus.append(("web_page", [make_msg( + text=FakeStr("check this", "check this"), + media=MessageMediaType.WEB_PAGE, + web_page=SimpleNamespace(type="article", url="https://example.com/a", + display_url="example.com/a", site_name="Example", + title="An Article", description="Desc here", + has_large_media=False, + photo=SimpleNamespace(file_unique_id="wp1")), + )])) + corpus.append(("poll", [make_msg( + media=MessageMediaType.POLL, + poll=SimpleNamespace( + question=SimpleNamespace(text="Favourite?", entities=[]), + options=[SimpleNamespace(text=SimpleNamespace(text="Red", entities=[])), + SimpleNamespace(text=SimpleNamespace(text="Blue", entities=[]))], + description_media=None, explanation_media=None, + ), + )])) + + # --- Forwards Case 1-5 (drive _format_forward_info) ------------------- # + corpus.append(("forward_channel", [make_msg( + text=FakeStr("fwd", "fwd"), + forward_origin=SimpleNamespace(type="channel", + chat=SimpleNamespace(id=-100, title="Src Chan", username="srcchan")), + )])) + corpus.append(("forward_hidden", [make_msg( + text=FakeStr("fwd", "fwd"), + forward_origin=SimpleNamespace(type="hidden_user", sender_user_name="Hidden Guy"), + )])) + corpus.append(("forward_user", [make_msg( + text=FakeStr("fwd", "fwd"), + forward_origin=SimpleNamespace(type="user", + sender_user=SimpleNamespace(first_name="Ann", last_name="Bee", username="annbee")), + )])) + corpus.append(("forward_channel_nouser", [make_msg( + text=FakeStr("fwd", "fwd"), + forward_origin=SimpleNamespace(type="channel", chat_id=-1002, title="NoUserChan"), + )])) + corpus.append(("forward_other", [make_msg( + text=FakeStr("fwd", "fwd"), + forward_origin=SimpleNamespace(type="something_else"), + )])) + + # --- Reactions: normal / paid / custom -------------------------------- # + corpus.append(("reactions", [make_msg( + text=FakeStr("react", "react"), + reactions=_reactions( + SimpleNamespace(emoji="👍", count=5, is_paid=False), + SimpleNamespace(count=3, is_paid=True), + SimpleNamespace(custom_emoji_id=1234567890, count=2), + ), + )])) + + # --- sender_chat / from_user author paths ----------------------------- # + corpus.append(("sender_chat_author", [make_msg( + text=FakeStr("a", "a"), + sender_chat=SimpleNamespace(id=-100, title="Sender Chan", username="senderchan"), + )])) + corpus.append(("from_user_author", [make_msg( + text=FakeStr("a", "a"), + from_user=SimpleNamespace(first_name="Joe", last_name="Doe", username="joedoe"), + )])) + + # --- Multi-message media group (grouping merge) ----------------------- # + corpus.append(("media_group", [ + make_msg(media=MessageMediaType.PHOTO, media_group_id="grp1", + photo=SimpleNamespace(file_unique_id="grp_a"), + caption=FakeStr("group cap", "group cap")), + make_msg(media=MessageMediaType.PHOTO, media_group_id="grp1", + photo=SimpleNamespace(file_unique_id="grp_b")), + ])) + + return corpus + + +# --------------------------------------------------------------------------- # +# Rendering harness — patches tg_cache to feed a message list into the real +# generate_channel_* entry points (the golden-replay path). +# --------------------------------------------------------------------------- # +def _render(messages, monkeypatch): + async def fake_get_chat_history(client, channel_id, limit=20): + return messages + + async def fake_get_chat(client, channel_id): + return SimpleNamespace(id=_CHAT.id, title=_CHAT.title, username=_CHAT.username) + + monkeypatch.setattr("tg_cache.cached_get_chat_history", fake_get_chat_history, raising=False) + monkeypatch.setattr("tg_cache.cached_get_chat", fake_get_chat, raising=False) + + from rss_generator import generate_channel_html, generate_channel_rss + html = asyncio.run(generate_channel_html(_CHANNEL, client=SimpleNamespace(), limit=50)) + rss = asyncio.run(generate_channel_rss(_CHANNEL, client=SimpleNamespace(), limit=50)) + return html, gr.normalize_rss(rss) + + +@pytest.mark.parametrize("name,messages", build_corpus(), ids=lambda v: v if isinstance(v, str) else "") +def test_snapshot_render_parity(name, messages, monkeypatch): + gr.pin_environment(monkeypatch) + + # 1) Render the LIVE objects through the real pipeline. + live_html, live_rss = _render(messages, monkeypatch) + + # 2) snapshot -> restore, then render the RESTORED objects the same way. + restored = restore_messages(snapshot_messages(messages)) + restored_html, restored_rss = _render(restored, monkeypatch) + + # 3) Byte-identical HTML and RSS prove the snapshot preserved everything the + # renderer read for this case. A dropped allowlist field diverges here. + assert restored_html == live_html, f"HTML render diverged for case '{name}'" + assert restored_rss == live_rss, f"RSS render diverged for case '{name}'" + + +def test_corpus_covers_all_special_media_types(): + """Guard: every special-media type from Fix 1 is exercised by the parity corpus.""" + names = {n for n, _ in build_corpus()} + required = {"story_photo", "story_video", "contact", "location", "venue", + "dice", "game", "giveaway", "giveaway_winners", "checklist", "paid_media", + "live_photo"} + assert required <= names diff --git a/tests/test_tg_cache_store.py b/tests/test_tg_cache_store.py index 0e26390..3c69fc8 100644 --- a/tests/test_tg_cache_store.py +++ b/tests/test_tg_cache_store.py @@ -1,3 +1,7 @@ +# flake8: noqa +# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring +# pylint: disable=redefined-outer-name, logging-fstring-interpolation, line-too-long +# pylance: disable=reportMissingImports, reportMissingModuleSource """Tests for the tg_cache generic JSON store, prefix-limit, jitter and sweep (issue #23). Задания 2 / 6 / 17 / 18, verification item 8, 11, 12.