implement RSS cache update mechanism
This commit is contained in:
vvzvlad
+296
@@ -0,0 +1,296 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
# flake8: noqa
|
||||
# pylint: disable=broad-exception-raised, raise-missing-from, too-many-arguments, redefined-outer-name
|
||||
# pylint: disable=multiple-statements, logging-fstring-interpolation, trailing-whitespace, line-too-long
|
||||
# pylint: disable=broad-exception-caught, missing-function-docstring, missing-class-docstring
|
||||
# pylint: disable=f-string-without-interpolation
|
||||
# pylance: disable=reportMissingImports, reportMissingModuleSource
|
||||
|
||||
import logging
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import asyncio
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Union, Optional
|
||||
from config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
Config = get_settings()
|
||||
|
||||
def get_cache_path(channel: Union[str, int], content_type: str = 'rss') -> Path:
|
||||
"""
|
||||
Get the path to the cache file for a given channel
|
||||
Args:
|
||||
channel: Telegram channel name or id
|
||||
content_type: Type of content (rss or html)
|
||||
Returns:
|
||||
Path to the cache file
|
||||
"""
|
||||
if content_type == 'html':
|
||||
base_dir = './data/html-cache'
|
||||
file_ext = 'html'
|
||||
else:
|
||||
base_dir = './data/rss-cache'
|
||||
file_ext = 'xml'
|
||||
|
||||
cache_dir = Path(Config.get('cache_dir', base_dir))
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Create a unique filename based on the channel identifier
|
||||
if isinstance(channel, int):
|
||||
filename = f"channel_{channel}.{file_ext}"
|
||||
else:
|
||||
# Use hash for channel names with special characters
|
||||
if any(c in channel for c in '/<>:"|?*\\'):
|
||||
channel_hash = hashlib.md5(channel.encode()).hexdigest()
|
||||
filename = f"channel_{channel_hash}.{file_ext}"
|
||||
else:
|
||||
filename = f"channel_{channel}.{file_ext}"
|
||||
|
||||
return cache_dir / filename
|
||||
|
||||
def is_cache_valid(cache_path: Path, max_age_hours: int = 2) -> bool:
|
||||
"""
|
||||
Check if the cache file exists and is recent enough
|
||||
Args:
|
||||
cache_path: Path to the cache file
|
||||
max_age_hours: Maximum age of the cache in hours
|
||||
Returns:
|
||||
True if cache exists and is recent enough, False otherwise
|
||||
"""
|
||||
if not cache_path.exists():
|
||||
return False
|
||||
|
||||
file_mod_time = datetime.fromtimestamp(cache_path.stat().st_mtime)
|
||||
max_age = timedelta(hours=max_age_hours)
|
||||
|
||||
return datetime.now() - file_mod_time < max_age
|
||||
|
||||
def read_cache(channel: Union[str, int],
|
||||
content_type: str = 'rss',
|
||||
max_age_hours: int = 2) -> Union[str, None]:
|
||||
"""
|
||||
Read content from cache if it exists and is valid
|
||||
Args:
|
||||
channel: Channel name or ID
|
||||
content_type: Type of content (rss or html)
|
||||
max_age_hours: Maximum age of the cache in hours
|
||||
Returns:
|
||||
Cached content as string or None if cache is invalid/missing
|
||||
"""
|
||||
cache_path = get_cache_path(channel, content_type)
|
||||
|
||||
if not is_cache_valid(cache_path, max_age_hours):
|
||||
return None
|
||||
|
||||
try:
|
||||
logger.debug(f"using_cached_{content_type}: channel {channel}, cache_file {cache_path}")
|
||||
with open(cache_path, 'r', encoding='utf-8') as f:
|
||||
return f.read()
|
||||
except Exception as e:
|
||||
logger.error(f"cache_read_error: channel {channel}, error {str(e)}")
|
||||
return None
|
||||
|
||||
def write_cache(channel: Union[str, int],
|
||||
content: str,
|
||||
content_type: str = 'rss') -> bool:
|
||||
"""
|
||||
Write content to cache
|
||||
Args:
|
||||
channel: Channel name or ID
|
||||
content: Content to cache
|
||||
content_type: Type of content (rss or html)
|
||||
Returns:
|
||||
True if cache was written successfully, False otherwise
|
||||
"""
|
||||
cache_path = get_cache_path(channel, content_type)
|
||||
|
||||
try:
|
||||
with open(cache_path, 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
logger.debug(f"{content_type}_cache_saved: channel {channel}, cache_file {cache_path}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"cache_write_error: channel {channel}, error {str(e)}")
|
||||
return False
|
||||
|
||||
def get_access_file_path() -> str:
|
||||
"""
|
||||
Get path to the cache access file
|
||||
Returns:
|
||||
Path to the cache access file
|
||||
"""
|
||||
return os.path.join(os.path.abspath("./data"), 'cache_access.json')
|
||||
|
||||
def update_channel_access_time(channel: Union[str, int]) -> bool:
|
||||
"""
|
||||
Update access time for a channel in cache_access.json
|
||||
Args:
|
||||
channel: Channel name or ID
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
access_file_path = get_access_file_path()
|
||||
cache_access_data = {}
|
||||
|
||||
# Load existing data if file exists
|
||||
if os.path.exists(access_file_path):
|
||||
try:
|
||||
with open(access_file_path, 'r', encoding='utf-8') as f:
|
||||
cache_access_data = json.load(f)
|
||||
except json.JSONDecodeError:
|
||||
logger.error(f"Error decoding cache_access.json, creating new file")
|
||||
cache_access_data = {}
|
||||
|
||||
# Update channel access time
|
||||
channel_key = str(channel)
|
||||
cache_access_data[channel_key] = time.time()
|
||||
|
||||
# Save updated data
|
||||
with open(access_file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(cache_access_data, f, indent=2)
|
||||
|
||||
logger.debug(f"Updated access time for channel {channel} in cache_access.json")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to update cache_access.json: {str(e)}")
|
||||
return False
|
||||
|
||||
async def clean_old_channels(max_age_days: int = 30) -> int:
|
||||
"""
|
||||
Remove channels that haven't been accessed for more than specified days
|
||||
|
||||
Args:
|
||||
max_age_days: Maximum age of channels in days
|
||||
|
||||
Returns:
|
||||
Number of removed channels
|
||||
"""
|
||||
access_file_path = get_access_file_path()
|
||||
|
||||
if not os.path.exists(access_file_path):
|
||||
logger.info("access_file_not_found: Creating new file")
|
||||
with open(access_file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump({}, f)
|
||||
return 0
|
||||
|
||||
try:
|
||||
with open(access_file_path, 'r', encoding='utf-8') as f:
|
||||
cache_access = json.load(f)
|
||||
|
||||
now = time.time()
|
||||
max_age_seconds = max_age_days * 86400 # days to seconds
|
||||
|
||||
old_channels = []
|
||||
for channel, timestamp in list(cache_access.items()):
|
||||
age = now - timestamp
|
||||
if age > max_age_seconds:
|
||||
old_channels.append(channel)
|
||||
del cache_access[channel]
|
||||
|
||||
# Save updated data
|
||||
if old_channels:
|
||||
with open(access_file_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(cache_access, f, indent=2)
|
||||
logger.info(f"removed_old_channels: {len(old_channels)} channels removed (older than {max_age_days} days)")
|
||||
|
||||
return len(old_channels)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"clean_old_channels_error: {str(e)}")
|
||||
return 0
|
||||
|
||||
async def update_rss_cache(client, channel: str, limit: int = 50) -> bool:
|
||||
"""
|
||||
Update RSS cache for specified channel
|
||||
|
||||
Args:
|
||||
client: Telegram client
|
||||
channel: Channel identifier
|
||||
limit: Maximum number of posts
|
||||
|
||||
Returns:
|
||||
True if cache was successfully updated, False otherwise
|
||||
"""
|
||||
try:
|
||||
logger.info(f"updating_rss_cache: channel {channel}")
|
||||
|
||||
# Import here to avoid circular imports
|
||||
from rss_generator import generate_channel_rss
|
||||
|
||||
# Force generate new RSS without using cache
|
||||
await generate_channel_rss(
|
||||
channel=channel,
|
||||
client=client,
|
||||
use_cache=False, # Force generate new RSS
|
||||
limit=limit
|
||||
)
|
||||
|
||||
logger.info(f"updated_rss_cache: channel {channel}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"rss_cache_update_error: channel {channel}, error {str(e)}")
|
||||
return False
|
||||
|
||||
async def start_cache_updater(client) -> None:
|
||||
"""
|
||||
Background task for updating RSS cache periodically
|
||||
|
||||
Args:
|
||||
client: Telegram client instance
|
||||
"""
|
||||
# Settings from configuration
|
||||
update_interval = Config.get("rss_cache_update_interval", 3600) # seconds
|
||||
update_delay = Config.get("rss_cache_update_delay", 60) # seconds
|
||||
max_age_days = Config.get("rss_cache_max_age_days", 30)
|
||||
|
||||
logger.info(f"starting_rss_cache_updater: interval={update_interval}s, delay={update_delay}s, max_age={max_age_days}d")
|
||||
|
||||
try:
|
||||
while True:
|
||||
# Clean old channels
|
||||
cleaned = await clean_old_channels(max_age_days)
|
||||
|
||||
# Load channels for update
|
||||
access_file_path = get_access_file_path()
|
||||
try:
|
||||
if os.path.exists(access_file_path):
|
||||
with open(access_file_path, 'r', encoding='utf-8') as f:
|
||||
cache_access = json.load(f)
|
||||
channels = list(cache_access.keys())
|
||||
|
||||
if channels:
|
||||
logger.info(f"starting_cache_update: updating {len(channels)} channels")
|
||||
|
||||
# Sort channels by last access time (newest first)
|
||||
channels_sorted = sorted(channels, key=lambda c: cache_access[c], reverse=True)
|
||||
|
||||
# Update cache for each channel with delay
|
||||
for channel in channels_sorted:
|
||||
success = await update_rss_cache(client, channel)
|
||||
# Sleep between updates to avoid rate limits
|
||||
await asyncio.sleep(update_delay)
|
||||
|
||||
logger.info(f"finished_cache_update: updated {len(channels)} channels")
|
||||
else:
|
||||
logger.info("no_channels_to_update: Waiting for next interval")
|
||||
else:
|
||||
logger.info("no_access_file: Waiting for next interval")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"rss_cache_update_cycle_error: {str(e)}")
|
||||
|
||||
# Wait until next update cycle
|
||||
logger.info(f"waiting_for_next_update: next update in {update_interval} seconds")
|
||||
await asyncio.sleep(update_interval)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.info("rss_cache_updater_cancelled")
|
||||
except Exception as e:
|
||||
logger.error(f"rss_cache_updater_error: {str(e)}")
|
||||
Reference in New Issue
Block a user