From d766220025a0543319e6f6a7371c6e3c0acb3a4e Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Sun, 27 Apr 2025 20:32:31 +0400 Subject: [PATCH] implement caching for RSS --- rss_generator.py | 121 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 115 insertions(+), 6 deletions(-) diff --git a/rss_generator.py b/rss_generator.py index c84e00d..340e68a 100644 --- a/rss_generator.py +++ b/rss_generator.py @@ -12,7 +12,10 @@ import logging import re import time -from datetime import datetime, timezone +import os +import hashlib +from pathlib import Path +from datetime import datetime, timezone, timedelta from types import SimpleNamespace from typing import Optional from feedgen.feed import FeedGenerator @@ -298,12 +301,55 @@ async def _render_messages_groups(messages_groups: list[list[Message]], rendered_posts.sort(key=lambda x: x['date'], reverse=True) return rendered_posts +def _get_cache_path(channel: str | int) -> Path: + """ + Get the path to the cache file for a given channel + Args: + channel: Telegram channel name or id + Returns: + Path to the cache file + """ + cache_dir = Path(Config.get('cache_dir', 'cache/rss')) + cache_dir.mkdir(parents=True, exist_ok=True) + + # Create a unique filename based on the channel identifier + if isinstance(channel, int): + filename = f"channel_{channel}.xml" + else: + # Use hash for channel names with special characters + if any(c in channel for c in '/<>:"|?*\\'): + channel_hash = hashlib.md5(channel.encode()).hexdigest() + filename = f"channel_{channel_hash}.xml" + else: + filename = f"channel_{channel}.xml" + + return cache_dir / filename + +def _is_cache_valid(cache_path: Path, max_age_hours: int = 2) -> bool: + """ + Check if the cache file exists and is recent enough + Args: + cache_path: Path to the cache file + max_age_hours: Maximum age of the cache in hours + Returns: + True if cache exists and is recent enough, False otherwise + """ + if not cache_path.exists(): + return False + + file_mod_time = datetime.fromtimestamp(cache_path.stat().st_mtime) + max_age = timedelta(hours=max_age_hours) + + return datetime.now() - file_mod_time < max_age + async def generate_channel_rss(channel: str | int, client: Client, limit: int = 20, exclude_flags: str | None = None, exclude_text: str | None = None, - merge_seconds: int = 5 + merge_seconds: int = 5, + use_cache: bool = True, + cache_max_age_hours: int = 2 ) -> str: """ Generate RSS feed for channel using actual messages @@ -314,9 +360,23 @@ async def generate_channel_rss(channel: str | int, limit: Maximum number of posts to include in the RSS feed exclude_flags: Flags to exclude from the RSS feed exclude_text: Text to exclude from posts + use_cache: Whether to use cache + cache_max_age_hours: Maximum age of the cache in hours Returns: RSS feed as string in XML format """ + cache_path = _get_cache_path(channel) + + # Check if we can use cached version + if use_cache and _is_cache_valid(cache_path, cache_max_age_hours): + try: + logger.debug(f"using_cached_rss: channel {channel}, cache_file {cache_path}") + with open(cache_path, 'r', encoding='utf-8') as f: + return f.read() + except Exception as e: + logger.error(f"cache_read_error: channel {channel}, error {str(e)}") + # If reading cache fails, continue to generate new content + total_start_time = time.time() if limit < 1: @@ -409,7 +469,16 @@ async def generate_channel_rss(channel: str | int, rss_feed = fg.rss_str(pretty=True) if isinstance(rss_feed, bytes): - return rss_feed.decode('utf-8') + rss_feed = rss_feed.decode('utf-8') + + # Save to cache + if use_cache: + try: + with open(cache_path, 'w', encoding='utf-8') as f: + f.write(rss_feed) + logger.debug(f"rss_cache_saved: channel {channel}, cache_file {cache_path}") + except Exception as e: + logger.error(f"cache_write_error: channel {channel}, error {str(e)}") total_elapsed = time.time() - total_start_time logger.debug(f"rss_total_generation_timing: channel {channel}, total time {total_elapsed:.3f} seconds") @@ -440,7 +509,9 @@ async def generate_channel_html(channel: str | int, limit: int = 20, exclude_flags: str | None = None, exclude_text: str | None = None, - merge_seconds: int = 5 + merge_seconds: int = 5, + use_cache: bool = True, + cache_max_age_hours: int = 2 ) -> str: """ Generate HTML feed for channel using actual messages @@ -451,9 +522,38 @@ async def generate_channel_html(channel: str | int, limit: Maximum number of posts to include in the RSS feed exclude_flags: Flags to exclude from the RSS feed exclude_text: Text to exclude from posts + use_cache: Whether to use cache + cache_max_age_hours: Maximum age of the cache in hours Returns: HTML feed as string """ + # Use different directory for html cache + cache_dir = Path(Config.get('cache_dir', 'cache/html')) + cache_dir.mkdir(parents=True, exist_ok=True) + + # Create a unique filename for the html cache + if isinstance(channel, int): + filename = f"channel_{channel}.html" + else: + # Use hash for channel names with special characters + if any(c in channel for c in '/<>:"|?*\\'): + channel_hash = hashlib.md5(channel.encode()).hexdigest() + filename = f"channel_{channel_hash}.html" + else: + filename = f"channel_{channel}.html" + + cache_path = cache_dir / filename + + # Check if we can use cached version + if use_cache and _is_cache_valid(cache_path, cache_max_age_hours): + try: + logger.debug(f"using_cached_html: channel {channel}, cache_file {cache_path}") + with open(cache_path, 'r', encoding='utf-8') as f: + return f.read() + except Exception as e: + logger.error(f"html_cache_read_error: channel {channel}, error {str(e)}") + # If reading cache fails, continue to generate new content + total_start_time = time.time() if limit < 1: @@ -522,15 +622,24 @@ async def generate_channel_html(channel: str | int, # Generate HTML content html_gen_start_time = time.time() html_posts = [post['html'] for post in final_posts] - html = '\n
\n'.join(html_posts) + html_content = '\n
\n'.join(html_posts) html_gen_elapsed = time.time() - html_gen_start_time logger.debug(f"html_generation_timing: channel {channel}, HTML generated in {html_gen_elapsed:.3f} seconds") + # Save to cache + if use_cache: + try: + with open(cache_path, 'w', encoding='utf-8') as f: + f.write(html_content) + logger.debug(f"html_cache_saved: channel {channel}, cache_file {cache_path}") + except Exception as e: + logger.error(f"html_cache_write_error: channel {channel}, error {str(e)}") + total_elapsed = time.time() - total_start_time logger.debug(f"html_total_generation_timing: channel {channel}, total time {total_elapsed:.3f} seconds") - return html + return html_content except Exception as e: logger.error(f"html_generation_error: channel {channel}, error {str(e)}")