From 716fa7ee42ea383051470db8afc2a66a5b62b636 Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Tue, 4 Mar 2025 03:15:26 +1000 Subject: [PATCH] Add text exclusion filter --- api_server.py | 15 ++++++++------ rss_generator.py | 54 +++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 60 insertions(+), 9 deletions(-) diff --git a/api_server.py b/api_server.py index 3f7ac8d..2dfbc62 100644 --- a/api_server.py +++ b/api_server.py @@ -545,14 +545,15 @@ async def get_rss_feed(channel: str, limit: int = 50, output_type: str = 'rss', exclude_flags: str | None = None, + exclude_text: str | None = None, merge_seconds: int = 5 ): if Config["token"]: if token != Config["token"]: - logger.error(f"Invalid token for RSS feed: {token}, expected: {Config['token']}") + logger.error(f"invalid_token_error: token {token}, expected {Config['token']}") raise HTTPException(status_code=403, detail="Invalid token") else: - logger.info(f"Valid token for RSS feed: {token}") + logger.info(f"valid_token: token {token}") while True: try: if output_type == 'rss': @@ -560,6 +561,7 @@ async def get_rss_feed(channel: str, client=client.client, limit=limit, exclude_flags=exclude_flags, + exclude_text=exclude_text, merge_seconds=merge_seconds) return Response(content=rss_content, media_type="application/xml") elif output_type == 'html': @@ -567,10 +569,11 @@ async def get_rss_feed(channel: str, client=client.client, limit=limit, exclude_flags=exclude_flags, + exclude_text=exclude_text, merge_seconds=merge_seconds) return Response(content=rss_content, media_type="text/html") except ValueError as e: - error_message = f"Invalid parameters for RSS feed generation: {str(e)}" + error_message = f"invalid_parameters_error: {str(e)}" logger.error(error_message) raise HTTPException(status_code=400, detail=error_message) from e except errors.FloodWait as e: @@ -579,11 +582,11 @@ async def get_rss_feed(channel: str, total_wait_time = wait_time + random_additional_wait if total_wait_time > 190: total_wait_time = 190 - logger.warning(f"TG FloodWait for channel {channel}, waiting {total_wait_time:.1f} seconds (base: {wait_time}s, random: {random_additional_wait:.1f}s)") + logger.warning(f"flood_wait_error: channel {channel}, waiting {total_wait_time:.1f} seconds (base: {wait_time}s, random: {random_additional_wait:.1f}s)") await asyncio.sleep(total_wait_time) - logger.info(f"TG FloodWait finished for channel {channel}, retrying RSS feed generation") + logger.info(f"flood_wait_retry: channel {channel}") continue except Exception as e: - error_message = f"Failed to generate RSS feed for channel {channel}: {str(e)}" + error_message = f"rss_generation_error: channel {channel}, error {str(e)}" logger.error(error_message) raise HTTPException(status_code=500, detail=error_message) from e diff --git a/rss_generator.py b/rss_generator.py index 4ef281a..b856ed5 100644 --- a/rss_generator.py +++ b/rss_generator.py @@ -108,12 +108,14 @@ async def _trim_messages_groups(messages_groups, limit): return messages_groups -async def _render_messages_groups(messages_groups, post_parser, exclude_flags: str | None = None): +async def _render_messages_groups(messages_groups, post_parser, exclude_flags: str | None = None, exclude_text: str | None = None): """ Render message groups into HTML format Args: messages_groups: List of message groups (each group is a list of messages) post_parser: PostParser instance + exclude_flags: Comma-separated list of flags to exclude + exclude_text: Text to exclude from posts (comma-separated phrases) Returns: List of rendered posts """ @@ -178,6 +180,7 @@ async def _render_messages_groups(messages_groups, post_parser, exclude_flags: s logger.error(f"message_group_rendering_error: error {str(e)}") continue + # Filter posts by exclude_flags if exclude_flags: # Split comma-separated exclude_flags into a list. exclude_flag_list = [flag.strip() for flag in exclude_flags.split(',')] @@ -192,6 +195,46 @@ async def _render_messages_groups(messages_groups, post_parser, exclude_flags: s filtered_posts.append(post) rendered_posts = filtered_posts + # Filter posts by exclude_text + if exclude_text: + # Split by comma, but preserve commas inside quotes + exclude_text_list = [] + current_phrase = "" + in_quotes = False + + for char in exclude_text: + if char == '"' and (not current_phrase.endswith('\\') or current_phrase.endswith('\\\\')): + in_quotes = not in_quotes + current_phrase += char + elif char == ',' and not in_quotes: + if current_phrase: + # Remove surrounding quotes if present + if current_phrase.startswith('"') and current_phrase.endswith('"'): + current_phrase = current_phrase[1:-1] + exclude_text_list.append(current_phrase.strip().lower()) + current_phrase = "" + else: + current_phrase += char + + # Add the last phrase if any + if current_phrase: + if current_phrase.startswith('"') and current_phrase.endswith('"'): + current_phrase = current_phrase[1:-1] + exclude_text_list.append(current_phrase.strip().lower()) + + logger.debug(f"Parsed exclude_text into phrases: {exclude_text_list}") + + filtered_posts = [] + for post in rendered_posts: + post_text = post['text'].lower() + # Check if any of the exclude_text items is in the post text (case-insensitive) + if not any(text in post_text for text in exclude_text_list): + filtered_posts.append(post) + else: + matching_phrases = [text for text in exclude_text_list if text in post_text] + logger.debug(f"Excluded post {post['message_id']} containing phrases: {matching_phrases}") + rendered_posts = filtered_posts + # Sort by date rendered_posts.sort(key=lambda x: x['date'], reverse=True) return rendered_posts @@ -201,6 +244,7 @@ async def generate_channel_rss(channel: str, client = None, limit: int = 20, exclude_flags: str | None = None, + exclude_text: str | None = None, merge_seconds: int = 5 ) -> str: """ @@ -211,6 +255,7 @@ async def generate_channel_rss(channel: str, client: Telegram client instance limit: Maximum number of posts to include in the RSS feed exclude_flags: Flags to exclude from the RSS feed + exclude_text: Text to exclude from posts Returns: RSS feed as string in XML format """ @@ -258,7 +303,7 @@ async def generate_channel_rss(channel: str, messages = await _create_time_based_media_groups(messages, merge_seconds) message_groups = await _create_messages_groups(messages) message_groups = await _trim_messages_groups(message_groups, limit) - final_posts = await _render_messages_groups(message_groups, post_parser, exclude_flags) + final_posts = await _render_messages_groups(message_groups, post_parser, exclude_flags, exclude_text) # Generate feed entries for post in final_posts: @@ -293,6 +338,7 @@ async def generate_channel_html(channel: str, client = None, limit: int = 20, exclude_flags: str | None = None, + exclude_text: str | None = None, merge_seconds: int = 5 ) -> str: """ @@ -302,6 +348,8 @@ async def generate_channel_html(channel: str, post_parser: Optional PostParser instance. If not provided, will create new one client: Telegram client instance limit: Maximum number of posts to include in the RSS feed + exclude_flags: Flags to exclude from the RSS feed + exclude_text: Text to exclude from posts Returns: HTML feed as string """ @@ -339,7 +387,7 @@ async def generate_channel_html(channel: str, messages = await _create_time_based_media_groups(messages, merge_seconds) message_groups = await _create_messages_groups(messages) message_groups = await _trim_messages_groups(message_groups, limit) - final_posts = await _render_messages_groups(message_groups, post_parser, exclude_flags) + final_posts = await _render_messages_groups(message_groups, post_parser, exclude_flags, exclude_text) # Generate HTML content html_posts = [post['html'] for post in final_posts]