diff --git a/README.md b/README.md index 0363cb2..2fad663 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,23 @@ Or, use my https://github.com/vvzvlad/miniflux-tg-add-bot to add channel to mini ``` curl https://pgbridge.example.com/html/DragorWW_space/87 ``` ``` curl https://pgbridge.example.com/json/DragorWW_space/87 ``` + +## Exclude flags + +Exclusion flags are a way to filter channel content based on pre-defined (by me) criteria. It's not a universal regexp-based filtering engine, for example, but it does 99% of my tasks of filtering the content of some toxic tg channels (mostly with fresh memes). + +There are several flags: +video - presence of video and small text in the post +stream, donat - words like "стрим", "донат" +clown, poo - emoticons in post reactions (>30) +advert - tag #реклама +hid_channel, foreign_channel - links to closed tg channels (https://t.me/+S0OfKyMDRi) and to open channels (https://t.me/suprechannel) that do not equal the name of the current channel. + +You can use exclude_flags parameter in rss/html/json urls to exclude posts with certain flags. For example, to exclude all posts with the "video" flag, you can use: + +``` curl https://pgbridge.example.com/rss/DragorWW_space?exclude_flags=video,stream,donat,clown ``` + +Or use meta-flag "all" to exclude all posts: + +``` curl https://pgbridge.example.com/rss/DragorWW_space?exclude_flags=all ``` + diff --git a/api_server.py b/api_server.py index 80010f2..fede967 100644 --- a/api_server.py +++ b/api_server.py @@ -520,21 +520,20 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, digest: str @app.get("/rss/{channel}", response_class=Response) @app.get("/rss/{channel}/{token}", response_class=Response) -async def get_rss_feed(channel: str, token: str | None = None, limit: int = 50, output_type: str = 'rss'): +async def get_rss_feed(channel: str, token: str | None = None, limit: int = 50, output_type: str = 'rss', exclude_flags: str | None = None): if Config["token"]: if token != Config["token"]: logger.error(f"Invalid token for RSS feed: {token}, expected: {Config['token']}") raise HTTPException(status_code=403, detail="Invalid token") else: logger.info(f"Valid token for RSS feed: {token}") - while True: try: if output_type == 'rss': - rss_content = await generate_channel_rss(channel, client=client.client, limit=limit) + rss_content = await generate_channel_rss(channel, client=client.client, limit=limit, exclude_flags=exclude_flags) return Response(content=rss_content, media_type="application/xml") elif output_type == 'html': - rss_content = await generate_channel_html(channel, client=client.client, limit=limit) + rss_content = await generate_channel_html(channel, client=client.client, limit=limit, exclude_flags=exclude_flags) return Response(content=rss_content, media_type="text/html") except ValueError as e: error_message = f"Invalid parameters for RSS feed generation: {str(e)}" diff --git a/post_parser.py b/post_parser.py index 548636c..8f1018a 100644 --- a/post_parser.py +++ b/post_parser.py @@ -196,6 +196,60 @@ class PostParser: return {r.emoji: r.count for r in reactions.reactions} return None + def _extract_flags(self, message: Message) -> List[str]: + # Check if there is no text or caption and add all applicable flags. + message_text = self._generate_html_body(message) + flags = [] + # Add flag "video" if the message media is VIDEO and the body text is up to 100 characters. + if message.media == MessageMediaType.VIDEO and len(message_text.strip()) <= 100: + flags.append("video") + + # Check if the message text contains variations of the word "стрим" + # (e.g., "стрим", "стримы", etc.) or "livestream" in a case-insensitive manner. + if re.search(r'(?i)\b(стрим\w*|livestream)\b', message_text): + flags.append("stream") + + # Check if the message text contains the word "донат" in a case-insensitive manner. + if re.search(r'(?i)\bдонат\w*\b', message_text): + flags.append("donat") + + # Check if the post's reactions contain more than 5 clown emojis (🤡). + if getattr(message, "reactions", None): + for reaction in message.reactions.reactions: + if reaction.emoji == "🤡" and reaction.count >= 30: + flags.append("clown") + break + + # Check if the post's reactions contain more than 5 poo emojis (💩). + if getattr(message, "reactions", None): + for reaction in message.reactions.reactions: + if reaction.emoji == "💩" and reaction.count >= 30: + flags.append("poo") + break + + # Check if the message text contains "#реклама" (advertisement tag) in a case-insensitive manner. + if re.search(r'(?i)#реклама', message_text): + flags.append("advert") + + # New: Check for t.me links: hidden channel and open (foreign) channel. + try: + # Find links with a '+' after t.me/ indicating a hidden channel link. + hidden_links = re.findall(r'https?://(?:www\.)?t\.me/\+([A-Za-z0-9]+)', message_text) + # Find links without a '+' after t.me/ indicating an open (foreign) channel link. + open_links = re.findall(r'https?://(?:www\.)?t\.me/(?!\+)([A-Za-z0-9_]+)', message_text) + if hidden_links: + flags.append("hid_channel") + + current_channel = self.get_channel_username(message) + # Add the flag "foreign_channel" if there's an open link that differs from the current channel. + for open_link in open_links: + if current_channel is None or open_link.lower() != current_channel.lower(): + flags.append("foreign_channel") + break + except Exception as e: + logger.error(f"tme_link_extraction_error: message_id {message.id}, error {str(e)}") + + return flags def _format_html(self, message: Message) -> str: html_content = [] @@ -208,6 +262,11 @@ class PostParser: html_content = '\n'.join(html_content) return html_content + def _format_flags(self, message: Message) -> str: + flags = self._extract_flags(message) + if flags: + return f'
Flags: {", ".join(flags)}
' + return '' def process_message(self, message: Message) -> Dict[Any, Any]: result = { @@ -223,6 +282,7 @@ class PostParser: 'media': self._generate_html_media(message), 'footer': self._generate_html_footer(message) }, + 'flags': self._extract_flags(message), 'author': self._get_author_info(message), 'views': message.views, 'reactions': self._extract_reactions(message), @@ -342,6 +402,8 @@ class PostParser: content_footer = [] if reactions_views_html := self._reactions_views_links(message): # Add reactions, views and links content_footer.append(reactions_views_html) + if flags_html := self._format_flags(message): # Add flags + content_footer.append(flags_html) html_footer = '\n'.join(content_footer) html_footer = self._sanitize_html(html_footer) return html_footer @@ -399,14 +461,22 @@ class PostParser: try: parts = [] - if reactions := getattr(message, "reactions"): - reactions_html = '' + reactions_html = "" + views_html = "" + if reactions := getattr(message, "reactions", None): for reaction in reactions.reactions: reactions_html += f'{reaction.emoji} {reaction.count}  ' - parts.append(reactions_html.rstrip()) + reactions_html = reactions_html.rstrip() if views := getattr(message, "views", None): views_html = f'{views} views' + + # If both reactions and views exist, insert a line break between them. + if reactions_html and views_html: + parts.append(reactions_html + "
" + views_html) + elif reactions_html: + parts.append(reactions_html) + elif views_html: parts.append(views_html) channel_identifier = self.get_channel_username(message) diff --git a/rss_generator.py b/rss_generator.py index 1dbe918..afa50ed 100644 --- a/rss_generator.py +++ b/rss_generator.py @@ -16,50 +16,6 @@ if not logger.handlers: handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')) logger.addHandler(handler) -def reorganize_post_content(html): - """ - Reorganize post content to move text after all media - Args: - html: Post HTML content - Returns: - Reorganized HTML content - """ - soup = BeautifulSoup(html, 'html.parser') - - # Find all media and text blocks - media_blocks = soup.find_all('div', class_='message-media') - text_blocks = soup.find_all('div', class_='message-text') - - if not text_blocks: # No text to move - return html - - # Create new structure - new_content = [] - - # Add all media first - for media in media_blocks: - media.extract() - new_content.append(str(media)) - new_content.append('
') - - # Add text last - for text in text_blocks: - text.extract() - new_content.append(str(text)) - - # Add remaining elements - remaining_elements = 0 - for element in soup.contents: - if isinstance(element, str): - new_content.append(element) - remaining_elements += 1 - elif element.name != 'br' and 'message-media' not in element.get('class', []) and 'message-text' not in element.get('class', []): - new_content.append(str(element)) - remaining_elements += 1 - - logger.debug(f"Content reorganized with {remaining_elements} additional elements") - return ''.join(new_content) - async def create_messages_groups(messages): """ @@ -100,7 +56,7 @@ async def create_messages_groups(messages): return processing_groups -async def render_messages_groups(messages_groups, post_parser): +async def render_messages_groups(messages_groups, post_parser, exclude_flags: str | None = None): """ Render message groups into HTML format Args: @@ -128,7 +84,8 @@ async def render_messages_groups(messages_groups, post_parser): 'message_id': message_data['message_id'], 'title': message_data['html']['title'], 'text': message_data['text'], - 'author': message_data['author'] + 'author': message_data['author'], + 'flags': message_data['flags'] }) else: # Multiple messages in group - need to merge @@ -158,18 +115,33 @@ async def render_messages_groups(messages_groups, post_parser): 'message_id': main_message['message_id'], 'title': main_message['html']['title'], 'text': main_message['text'], - 'author': main_message['author'] + 'author': main_message['author'], + 'flags': main_message['flags'] }) except Exception as e: logger.error(f"message_group_rendering_error: error {str(e)}") continue + + if exclude_flags: + # Split comma-separated exclude_flags into a list. + exclude_flag_list = [flag.strip() for flag in exclude_flags.split(',')] + filtered_posts = [] + for post in rendered_posts: + # If "all" is specified and the post has any flags, exclude the post. + if "all" in exclude_flag_list and post['flags']: + continue + # Exclude post if any flag in the exclude list is present in the post's flags. + if any(flag in post['flags'] for flag in exclude_flag_list): + continue + filtered_posts.append(post) + rendered_posts = filtered_posts # Sort by date rendered_posts.sort(key=lambda x: x['date'], reverse=True) return rendered_posts -async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20) -> str: +async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20, exclude_flags: str | None = None ) -> str: """ Generate RSS feed for channel using actual messages Args: @@ -177,7 +149,7 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = post_parser: Optional PostParser instance. If not provided, will create new one client: Telegram client instance limit: Maximum number of posts to include in the RSS feed - output_type: 'rss' or 'html' + exclude_flags: Flags to exclude from the RSS feed Returns: RSS feed as string in XML format """ @@ -222,7 +194,7 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = # Process messages into groups and render them message_groups = await create_messages_groups(messages) - final_posts = await render_messages_groups(message_groups, post_parser) + final_posts = await render_messages_groups(message_groups, post_parser, exclude_flags) # Generate feed entries for post in final_posts: @@ -252,7 +224,7 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = raise -async def generate_channel_html(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20) -> str: +async def generate_channel_html(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20, exclude_flags: str | None = None) -> str: """ Generate HTML feed for channel using actual messages Args: @@ -280,7 +252,6 @@ async def generate_channel_html(channel: str, post_parser: Optional[PostParser] channel_id = int(channel) channel_info = await post_parser.client.get_chat(channel_id) - channel_title = channel_info.title or f"Telegram: {channel}" channel_username = post_parser.get_channel_username(channel_info) if not channel_username: return create_error_feed(channel, base_url) @@ -290,9 +261,6 @@ async def generate_channel_html(channel: str, post_parser: Optional[PostParser] else: raise - # Set feed metadata - main_name = f"{channel_title} (@{channel_username})" - html_content = f'

{main_name}

' # Collect messages messages = [] @@ -301,7 +269,7 @@ async def generate_channel_html(channel: str, post_parser: Optional[PostParser] # Process messages into groups and render them message_groups = await create_messages_groups(messages) - final_posts = await render_messages_groups(message_groups, post_parser) + final_posts = await render_messages_groups(message_groups, post_parser, exclude_flags) # Generate HTML content html_posts = [post['html'] for post in final_posts]