From 7c9fd859d7040681e70107900321a7cb90eadb5e Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Sun, 9 Feb 2025 06:04:01 +0300 Subject: [PATCH] Refactor PostParser: Improve code style and method visibility --- api_server.py | 8 +++++--- post_parser.py | 51 ++++++++++++++---------------------------------- rss_generator.py | 18 +++++------------ 3 files changed, 25 insertions(+), 52 deletions(-) diff --git a/api_server.py b/api_server.py index fede967..5d8193d 100644 --- a/api_server.py +++ b/api_server.py @@ -541,11 +541,13 @@ async def get_rss_feed(channel: str, token: str | None = None, limit: int = 50, raise HTTPException(status_code=400, detail=error_message) from e except errors.FloodWait as e: wait_time = e.value - random_additional_wait = random.uniform(0, wait_time * 1.5) + random_additional_wait = random.uniform(0, wait_time * 2.5) total_wait_time = wait_time + random_additional_wait - logger.warning(f"FloodWait detected for channel {channel}, waiting {total_wait_time:.1f} seconds (base: {wait_time}s, random: {random_additional_wait:.1f}s)") + if total_wait_time > 190: total_wait_time = 190 + + logger.warning(f"TG FloodWait for channel {channel}, waiting {total_wait_time:.1f} seconds (base: {wait_time}s, random: {random_additional_wait:.1f}s)") await asyncio.sleep(total_wait_time) - logger.info(f"FloodWait finished for channel {channel}, retrying RSS feed generation") + logger.info(f"TG FloodWait finished for channel {channel}, retrying RSS feed generation") continue except Exception as e: error_message = f"Failed to generate RSS feed for channel {channel}: {str(e)}" diff --git a/post_parser.py b/post_parser.py index 8f1018a..d4ad9f8 100644 --- a/post_parser.py +++ b/post_parser.py @@ -68,7 +68,7 @@ class PostParser: print(debug_message) return - def channel_name_prepare(self, channel: str): + def _channel_name_prepare(self, channel: str): if isinstance(channel, str) and channel.startswith('-100'): # Convert numeric channel ID to int channel_id = int(channel) return channel_id @@ -78,7 +78,7 @@ class PostParser: async def get_post(self, channel: str, post_id: int, output_type: str = 'json') -> Union[str, Dict[Any, Any]]: print(f"Getting post {channel}, {post_id}") try: - channel = self.channel_name_prepare(channel) + channel = self._channel_name_prepare(channel) message = await self.client.get_messages(channel, post_id) self._debug_message(message) @@ -123,14 +123,14 @@ class PostParser: return "✨ Channel created" text = message.text or message.caption or '' if not text: - if message.media == MessageMediaType.PHOTO: return "πŸ“· Photo" - elif message.media == MessageMediaType.VIDEO: return "πŸŽ₯ Video" - elif message.media == MessageMediaType.ANIMATION: return "🎞 GIF" - elif message.media == MessageMediaType.AUDIO: return "🎡 Audio" - elif message.media == MessageMediaType.VOICE: return "🎀 Voice message" - elif message.media == MessageMediaType.VIDEO_NOTE: return "πŸŽ₯ Video message" - elif message.media == MessageMediaType.STICKER: return "🎯 Sticker" - elif message.media == MessageMediaType.POLL: return "πŸ“Š Poll" + if message.media == MessageMediaType.PHOTO: return "πŸ“· Photo" + elif message.media == MessageMediaType.VIDEO: return "πŸŽ₯ Video" + elif message.media == MessageMediaType.ANIMATION: return "🎞 GIF" + elif message.media == MessageMediaType.AUDIO: return "🎡 Audio" + elif message.media == MessageMediaType.VOICE: return "🎀 Voice message" + elif message.media == MessageMediaType.VIDEO_NOTE: return "πŸŽ₯ Video message" + elif message.media == MessageMediaType.STICKER: return "🎯 Sticker" + elif message.media == MessageMediaType.POLL: return "πŸ“Š Poll" return "πŸ“· Media post" # Remove URLs @@ -213,14 +213,14 @@ class PostParser: if re.search(r'(?i)\bΠ΄ΠΎΠ½Π°Ρ‚\w*\b', message_text): flags.append("donat") - # Check if the post's reactions contain more than 5 clown emojis (🀑). + # Check if the post's reactions contain more clown emojis (🀑). if getattr(message, "reactions", None): for reaction in message.reactions.reactions: if reaction.emoji == "🀑" and reaction.count >= 30: flags.append("clown") break - # Check if the post's reactions contain more than 5 poo emojis (πŸ’©). + # Check if the post's reactions contain more poo emojis (πŸ’©). if getattr(message, "reactions", None): for reaction in message.reactions.reactions: if reaction.emoji == "πŸ’©" and reaction.count >= 30: @@ -323,10 +323,8 @@ class PostParser: def _generate_html_header(self, message: Message) -> str: content_header = [] # Add forwarded from or reply info if present - if forward_html := self._format_forward_info(message): - content_header.append(forward_html) - elif reply_html := self._format_reply_info(message): - content_header.append(reply_html) + if forward_html := self._format_forward_info(message): content_header.append(forward_html) + elif reply_html := self._format_reply_info(message): content_header.append(reply_html) html_header = '\n'.join(content_header) html_header = self._sanitize_html(html_header) return html_header @@ -533,24 +531,6 @@ class PostParser: logger.error(f"file_id_extraction_error: media_type {message.media}, error {str(e)}") return None - async def get_recent_posts(self, channel: str, limit: int = 20) -> List[Dict[Any, Any]]: - try: - messages = [] - async for message in self.client.get_chat_history(channel, limit=limit): - try: - post = await self.get_post(channel, message.id, output_type='json') - if post: - messages.append(post) - except Exception as e: - logger.error(f"message_processing_error: channel {channel}, message_id {message.id}, error {str(e)}") - continue - - return messages - - except Exception as e: - logger.error(f"recent_posts_error: channel {channel}, error {str(e)}") - raise - def _save_media_file_ids(self, message: Message) -> None: try: file_data = { @@ -567,8 +547,7 @@ class PostParser: if message.media: # Skip large videos - they shouldn't be cached permanently - if message.video and message.video.file_size > 100 * 1024 * 1024: - return + if message.video and message.video.file_size > 100 * 1024 * 1024: return if message.photo: file_data['file_unique_id'] = message.photo.file_unique_id elif message.video: file_data['file_unique_id'] = message.video.file_unique_id diff --git a/rss_generator.py b/rss_generator.py index afa50ed..0be6926 100644 --- a/rss_generator.py +++ b/rss_generator.py @@ -4,7 +4,6 @@ from typing import Optional from feedgen.feed import FeedGenerator from post_parser import PostParser from config import get_settings -from bs4 import BeautifulSoup Config = get_settings() @@ -69,8 +68,7 @@ async def render_messages_groups(messages_groups, post_parser, exclude_flags: st for group in messages_groups: try: - if len(group) == 1: - # Single message - simple case + if len(group) == 1: # Single message - simple case message_data = post_parser.process_message(group[0]) html_parts = [ f'
{message_data["html"]["header"]}
', @@ -87,12 +85,9 @@ async def render_messages_groups(messages_groups, post_parser, exclude_flags: st 'author': message_data['author'], 'flags': message_data['flags'] }) - else: - # Multiple messages in group - need to merge + else: # Multiple messages in group - need to merge processed_messages = [post_parser.process_message(msg) for msg in group] - - # Find main message (one with text) - main_message = next( + main_message = next( # Find main message (one with text) (msg for msg in processed_messages if msg['text']), processed_messages[0] # fallback to first if none has text ) @@ -247,11 +242,8 @@ async def generate_channel_html(channel: str, post_parser: Optional[PostParser] base_url = Config['pyrogram_bridge_url'] try: - channel_id = channel - if isinstance(channel, str) and channel.startswith('-100'): - channel_id = int(channel) - - channel_info = await post_parser.client.get_chat(channel_id) + channel = post_parser.channel_name_prepare(channel) + channel_info = await post_parser.client.get_chat(channel) channel_username = post_parser.get_channel_username(channel_info) if not channel_username: return create_error_feed(channel, base_url)