EPIC: Add content filtering via message flags

README.md

@@ -81,3 +81,23 @@ Or, use my https://github.com/vvzvlad/miniflux-tg-add-bot to add channel to mini
 
 ``` curl https://pgbridge.example.com/html/DragorWW_space/87 ```
 ``` curl https://pgbridge.example.com/json/DragorWW_space/87 ```
+
+## Exclude flags
+
+Exclusion flags are a way to filter channel content based on pre-defined (by me) criteria. It's not a universal regexp-based filtering engine, for example, but it does 99% of my tasks of filtering the content of some toxic tg channels (mostly with fresh memes).  
+
+There are several flags:  
+video - presence of video and small text in the post  
+stream, donat - words like "стрим", "донат"  
+clown, poo - emoticons in post reactions (>30) 
+advert - tag #реклама  
+hid_channel, foreign_channel - links to closed tg channels (https://t.me/+S0OfKyMDRi) and to open channels (https://t.me/suprechannel) that do not equal the name of the current channel. 
+
+You can use exclude_flags parameter in rss/html/json urls to exclude posts with certain flags. For example, to exclude all posts with the "video" flag, you can use:
+
+``` curl https://pgbridge.example.com/rss/DragorWW_space?exclude_flags=video,stream,donat,clown ```
+
+Or use meta-flag "all" to exclude all posts:
+
+``` curl https://pgbridge.example.com/rss/DragorWW_space?exclude_flags=all ```
+

api_server.py

@@ -520,21 +520,20 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, digest: str
 
 @app.get("/rss/{channel}", response_class=Response)
 @app.get("/rss/{channel}/{token}", response_class=Response)
-async def get_rss_feed(channel: str, token: str | None = None, limit: int = 50, output_type: str = 'rss'):
+async def get_rss_feed(channel: str, token: str | None = None, limit: int = 50, output_type: str = 'rss', exclude_flags: str | None = None):
     if Config["token"]:
         if token != Config["token"]:
             logger.error(f"Invalid token for RSS feed: {token}, expected: {Config['token']}")
             raise HTTPException(status_code=403, detail="Invalid token")
         else:
             logger.info(f"Valid token for RSS feed: {token}")
-    
     while True:
         try:
             if output_type == 'rss':
-                rss_content = await generate_channel_rss(channel, client=client.client, limit=limit)
+                rss_content = await generate_channel_rss(channel, client=client.client, limit=limit, exclude_flags=exclude_flags)
                 return Response(content=rss_content, media_type="application/xml")
             elif output_type == 'html':
-                rss_content = await generate_channel_html(channel, client=client.client, limit=limit)
+                rss_content = await generate_channel_html(channel, client=client.client, limit=limit, exclude_flags=exclude_flags)
                 return Response(content=rss_content, media_type="text/html")
         except ValueError as e:
             error_message = f"Invalid parameters for RSS feed generation: {str(e)}"

post_parser.py

@@ -196,6 +196,60 @@ class PostParser:
             return {r.emoji: r.count for r in reactions.reactions}
         return None
 
+    def _extract_flags(self, message: Message) -> List[str]:
+        # Check if there is no text or caption and add all applicable flags.
+        message_text = self._generate_html_body(message)
+        flags = []
+        # Add flag "video" if the message media is VIDEO and the body text is up to 100 characters.
+        if message.media == MessageMediaType.VIDEO and len(message_text.strip()) <= 100:
+            flags.append("video")
+
+        # Check if the message text contains variations of the word "стрим"
+        # (e.g., "стрим", "стримы", etc.) or "livestream" in a case-insensitive manner.
+        if re.search(r'(?i)\b(стрим\w*|livestream)\b', message_text):
+            flags.append("stream")
+
+        # Check if the message text contains the word "донат" in a case-insensitive manner.
+        if re.search(r'(?i)\bдонат\w*\b', message_text):
+            flags.append("donat")
+
+        # Check if the post's reactions contain more than 5 clown emojis (🤡).
+        if getattr(message, "reactions", None):
+            for reaction in message.reactions.reactions:
+                if reaction.emoji == "🤡" and reaction.count >= 30:
+                    flags.append("clown")
+                    break
+
+        # Check if the post's reactions contain more than 5 poo emojis (💩).
+        if getattr(message, "reactions", None):
+            for reaction in message.reactions.reactions:
+                if reaction.emoji == "💩" and reaction.count >= 30:
+                    flags.append("poo")
+                    break
+
+        # Check if the message text contains "#реклама" (advertisement tag) in a case-insensitive manner.
+        if re.search(r'(?i)#реклама', message_text):
+            flags.append("advert")
+
+        # New: Check for t.me links: hidden channel and open (foreign) channel.
+        try:
+            # Find links with a '+' after t.me/ indicating a hidden channel link.
+            hidden_links = re.findall(r'https?://(?:www\.)?t\.me/\+([A-Za-z0-9]+)', message_text)
+            # Find links without a '+' after t.me/ indicating an open (foreign) channel link.
+            open_links = re.findall(r'https?://(?:www\.)?t\.me/(?!\+)([A-Za-z0-9_]+)', message_text)
+            if hidden_links:
+                flags.append("hid_channel")
+
+            current_channel = self.get_channel_username(message)
+            # Add the flag "foreign_channel" if there's an open link that differs from the current channel.
+            for open_link in open_links:
+                if current_channel is None or open_link.lower() != current_channel.lower():
+                    flags.append("foreign_channel")
+                    break
+        except Exception as e:
+            logger.error(f"tme_link_extraction_error: message_id {message.id}, error {str(e)}")
+
+        return flags
 
     def _format_html(self, message: Message) -> str:
         html_content = []
@@ -208,6 +262,11 @@ class PostParser:
         html_content = '\n'.join(html_content)
         return html_content
 
+    def _format_flags(self, message: Message) -> str:
+        flags = self._extract_flags(message)
+        if flags:
+            return f'<div class="message-flags">Flags: {", ".join(flags)}</div>'
+        return ''
 
     def process_message(self, message: Message) -> Dict[Any, Any]:
         result = {
@@ -223,6 +282,7 @@ class PostParser:
                 'media': self._generate_html_media(message),
                 'footer': self._generate_html_footer(message)
             },
+            'flags': self._extract_flags(message),
             'author': self._get_author_info(message),
             'views': message.views,
             'reactions': self._extract_reactions(message),
@@ -342,6 +402,8 @@ class PostParser:
         content_footer = []
         if reactions_views_html := self._reactions_views_links(message):  # Add reactions, views and links
             content_footer.append(reactions_views_html)
+        if flags_html := self._format_flags(message): # Add flags
+            content_footer.append(flags_html)
         html_footer = '\n'.join(content_footer)
         html_footer = self._sanitize_html(html_footer)
         return html_footer
@@ -399,14 +461,22 @@ class PostParser:
         try:
             parts = []
             
-            if reactions := getattr(message, "reactions"):
-                reactions_html = ''
+            reactions_html = ""
+            views_html = ""
+            if reactions := getattr(message, "reactions", None):
                 for reaction in reactions.reactions:
                     reactions_html += f'<span class="reaction">{reaction.emoji} {reaction.count}&nbsp;&nbsp;</span>'
-                parts.append(reactions_html.rstrip())
+                reactions_html = reactions_html.rstrip()
 
             if views := getattr(message, "views", None):
                 views_html = f'<span class="views">{views} views</span>'
+
+            # If both reactions and views exist, insert a line break between them.
+            if reactions_html and views_html:
+                parts.append(reactions_html + "<br>" + views_html)
+            elif reactions_html:
+                parts.append(reactions_html)
+            elif views_html:
                 parts.append(views_html)
 
             channel_identifier = self.get_channel_username(message)

rss_generator.py

@@ -16,50 +16,6 @@ if not logger.handlers:
     handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
     logger.addHandler(handler)
 
-def reorganize_post_content(html):
-    """
-    Reorganize post content to move text after all media
-    Args:
-        html: Post HTML content
-    Returns:
-        Reorganized HTML content
-    """
-    soup = BeautifulSoup(html, 'html.parser')
-    
-    # Find all media and text blocks
-    media_blocks = soup.find_all('div', class_='message-media')
-    text_blocks = soup.find_all('div', class_='message-text')
-    
-    if not text_blocks:  # No text to move
-        return html
-        
-    # Create new structure
-    new_content = []
-    
-    # Add all media first
-    for media in media_blocks:
-        media.extract()
-        new_content.append(str(media))
-        new_content.append('<br>')
-        
-    # Add text last
-    for text in text_blocks:
-        text.extract()
-        new_content.append(str(text))
-    
-    # Add remaining elements
-    remaining_elements = 0
-    for element in soup.contents:
-        if isinstance(element, str):
-            new_content.append(element)
-            remaining_elements += 1
-        elif element.name != 'br' and 'message-media' not in element.get('class', []) and 'message-text' not in element.get('class', []):
-            new_content.append(str(element))
-            remaining_elements += 1
-    
-    logger.debug(f"Content reorganized with {remaining_elements} additional elements")
-    return ''.join(new_content)
-
 
 async def create_messages_groups(messages):
     """
@@ -100,7 +56,7 @@ async def create_messages_groups(messages):
     
     return processing_groups
 
-async def render_messages_groups(messages_groups, post_parser):
+async def render_messages_groups(messages_groups, post_parser, exclude_flags: str | None = None):
     """
     Render message groups into HTML format
     Args:
@@ -128,7 +84,8 @@ async def render_messages_groups(messages_groups, post_parser):
                     'message_id': message_data['message_id'],
                     'title': message_data['html']['title'],
                     'text': message_data['text'],
-                    'author': message_data['author']
+                    'author': message_data['author'],
+                    'flags': message_data['flags']
                 })
             else:
                 # Multiple messages in group - need to merge
@@ -158,18 +115,33 @@ async def render_messages_groups(messages_groups, post_parser):
                     'message_id': main_message['message_id'],
                     'title': main_message['html']['title'],
                     'text': main_message['text'],
-                    'author': main_message['author']
+                    'author': main_message['author'],
+                    'flags': main_message['flags']
                 })
                 
         except Exception as e:
             logger.error(f"message_group_rendering_error: error {str(e)}")
             continue
+
+    if exclude_flags:
+        # Split comma-separated exclude_flags into a list.
+        exclude_flag_list = [flag.strip() for flag in exclude_flags.split(',')]
+        filtered_posts = []
+        for post in rendered_posts:
+            # If "all" is specified and the post has any flags, exclude the post.
+            if "all" in exclude_flag_list and post['flags']:
+                continue
+            # Exclude post if any flag in the exclude list is present in the post's flags.
+            if any(flag in post['flags'] for flag in exclude_flag_list):
+                continue
+            filtered_posts.append(post)
+        rendered_posts = filtered_posts
     
     # Sort by date
     rendered_posts.sort(key=lambda x: x['date'], reverse=True)
     return rendered_posts
 
-async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20) -> str:
+async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20, exclude_flags: str | None = None ) -> str:
     """
     Generate RSS feed for channel using actual messages
     Args:
@@ -177,7 +149,7 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] =
         post_parser: Optional PostParser instance. If not provided, will create new one
         client: Telegram client instance
         limit: Maximum number of posts to include in the RSS feed
-        output_type: 'rss' or 'html'
+        exclude_flags: Flags to exclude from the RSS feed
     Returns:
         RSS feed as string in XML format
     """
@@ -222,7 +194,7 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] =
             
         # Process messages into groups and render them
         message_groups = await create_messages_groups(messages)
-        final_posts = await render_messages_groups(message_groups, post_parser)
+        final_posts = await render_messages_groups(message_groups, post_parser, exclude_flags)
         
         # Generate feed entries
         for post in final_posts:
@@ -252,7 +224,7 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] =
         raise
 
 
-async def generate_channel_html(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20) -> str:
+async def generate_channel_html(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20, exclude_flags: str | None = None) -> str:
     """
     Generate HTML feed for channel using actual messages
     Args:
@@ -280,7 +252,6 @@ async def generate_channel_html(channel: str, post_parser: Optional[PostParser]
                 channel_id = int(channel)
                 
             channel_info = await post_parser.client.get_chat(channel_id)
-            channel_title = channel_info.title or f"Telegram: {channel}"
             channel_username = post_parser.get_channel_username(channel_info)
             if not channel_username:
                 return create_error_feed(channel, base_url)
@@ -290,9 +261,6 @@ async def generate_channel_html(channel: str, post_parser: Optional[PostParser]
             else:
                 raise
 
-        # Set feed metadata
-        main_name = f"{channel_title} (@{channel_username})"
-        html_content = f'<h1>{main_name}</h1>'
         
         # Collect messages
         messages = []
@@ -301,7 +269,7 @@ async def generate_channel_html(channel: str, post_parser: Optional[PostParser]
             
         # Process messages into groups and render them
         message_groups = await create_messages_groups(messages)
-        final_posts = await render_messages_groups(message_groups, post_parser)
+        final_posts = await render_messages_groups(message_groups, post_parser, exclude_flags)
 
         # Generate HTML content
         html_posts = [post['html'] for post in final_posts]