From 67d52b507a33d45ede93fed39348d096dd02e387 Mon Sep 17 00:00:00 2001 From: vvzvlad Date: Sun, 9 Feb 2025 21:44:55 +0300 Subject: [PATCH] Add time-based media group merging for RSS and HTML generation --- api_server.py | 20 +++++++++-- config.py | 3 +- post_parser.py | 32 +++++++++++------- rss_generator.py | 88 ++++++++++++++++++++++++++++++++++++++++++------ 4 files changed, 116 insertions(+), 27 deletions(-) diff --git a/api_server.py b/api_server.py index 5d8193d..97fae56 100644 --- a/api_server.py +++ b/api_server.py @@ -520,7 +520,13 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, digest: str @app.get("/rss/{channel}", response_class=Response) @app.get("/rss/{channel}/{token}", response_class=Response) -async def get_rss_feed(channel: str, token: str | None = None, limit: int = 50, output_type: str = 'rss', exclude_flags: str | None = None): +async def get_rss_feed(channel: str, + token: str | None = None, + limit: int = 50, + output_type: str = 'rss', + exclude_flags: str | None = None, + merge_seconds: int = 5 + ): if Config["token"]: if token != Config["token"]: logger.error(f"Invalid token for RSS feed: {token}, expected: {Config['token']}") @@ -530,10 +536,18 @@ async def get_rss_feed(channel: str, token: str | None = None, limit: int = 50, while True: try: if output_type == 'rss': - rss_content = await generate_channel_rss(channel, client=client.client, limit=limit, exclude_flags=exclude_flags) + rss_content = await generate_channel_rss(channel, + client=client.client, + limit=limit, + exclude_flags=exclude_flags, + merge_seconds=merge_seconds) return Response(content=rss_content, media_type="application/xml") elif output_type == 'html': - rss_content = await generate_channel_html(channel, client=client.client, limit=limit, exclude_flags=exclude_flags) + rss_content = await generate_channel_html(channel, + client=client.client, + limit=limit, + exclude_flags=exclude_flags, + merge_seconds=merge_seconds) return Response(content=rss_content, media_type="text/html") except ValueError as e: error_message = f"Invalid parameters for RSS feed generation: {str(e)}" diff --git a/config.py b/config.py index 2b14ee5..b6168c0 100644 --- a/config.py +++ b/config.py @@ -11,5 +11,6 @@ def get_settings(): "pyrogram_bridge_url": os.getenv("PYROGRAM_BRIDGE_URL", ""), "log_level": os.getenv("LOG_LEVEL", "INFO"), "debug": os.getenv("DEBUG", "False") == "True", - "token": os.getenv("TOKEN", "") + "token": os.getenv("TOKEN", ""), + "time_based_merge": os.getenv("TIME_BASED_MERGE", "False") == "True", } diff --git a/post_parser.py b/post_parser.py index b2fccc7..66a2378 100644 --- a/post_parser.py +++ b/post_parser.py @@ -404,9 +404,9 @@ class PostParser: def _generate_html_footer(self, message: Message) -> str: content_footer = [] - if reactions_views_html := self._reactions_views_links(message): # Add reactions, views and links + if reactions_views_html := self._reactions_views_links(message): # Add reactions, views, date and links content_footer.append(reactions_views_html) - if flags_html := self._format_flags(message): # Add flags + if flags_html := self._format_flags(message): # Add flags content_footer.append(flags_html) html_footer = '\n'.join(content_footer) html_footer = self._sanitize_html(html_footer) @@ -475,14 +475,19 @@ class PostParser: if views := getattr(message, "views", None): views_html = f'{views} views' - # If both reactions and views exist, insert a line break between them. - if reactions_html and views_html: - parts.append(reactions_html + "
" + views_html) - elif reactions_html: - parts.append(reactions_html) - elif views_html: - parts.append(views_html) + # Combine reactions and views on the same line + if reactions_html or views_html: + parts.append(f'{reactions_html}  {views_html}') + # Add date and links on the same line + second_line_parts = [] + + # Format date as DD/MM/YY, HH:MM:SS + if message.date: + formatted_date = message.date.strftime("%d/%m/%y, %H:%M:%S") + second_line_parts.append(formatted_date) + + # Add links channel_identifier = self.get_channel_username(message) if channel_identifier: links = [] @@ -490,12 +495,15 @@ class PostParser: channel_id = channel_identifier[4:] # Remove '-100' prefix for web links links.append(f'Open in Telegram') links.append(f'Open in Web') - else: # For channels with username + else: # For channels with username links.append(f'Open in Telegram') links.append(f'Open in Web') - parts.append('     '.join(links)) + second_line_parts.extend(links) - html = '     '.join(parts) if parts else None + if second_line_parts: + parts.append('    '.join(second_line_parts)) + + html = '
'.join(parts) if parts else None return f"
{html}" if html else None except Exception as e: diff --git a/rss_generator.py b/rss_generator.py index 9b319bb..be5b533 100644 --- a/rss_generator.py +++ b/rss_generator.py @@ -16,6 +16,49 @@ if not logger.handlers: logger.addHandler(handler) +async def create_time_based_media_groups(messages, merge_seconds: int = 5): + messages_sorted = sorted(messages, key=lambda x: x.date) + cluster = [] + last_msg_date = None + current_media_group_id = None + + for msg in messages_sorted: + + if not cluster: + cluster.append(msg) + last_msg_date = msg.date + current_media_group_id = getattr(msg, "media_group_id", None) + continue + + time_diff = (msg.date - last_msg_date).total_seconds() + + msg_media_group_id = getattr(msg, "media_group_id", None) + + if time_diff <= merge_seconds: + if current_media_group_id: + msg.media_group_id = current_media_group_id + elif msg_media_group_id: + current_media_group_id = msg_media_group_id + for m in cluster: + m.media_group_id = current_media_group_id + cluster.append(msg) + last_msg_date = msg.date + else: + if len(cluster) >= 2 and not current_media_group_id: + new_group_id = f"time_{min(m.date for m in cluster)}" + for m in cluster: + m.media_group_id = new_group_id + cluster = [msg] + last_msg_date = msg.date + current_media_group_id = msg_media_group_id + + if len(cluster) >= 2 and not current_media_group_id: + new_group_id = f"time_{min(m.date for m in cluster)}" + for m in cluster: + m.media_group_id = new_group_id + + return messages + async def create_messages_groups(messages): """ Process messages into formatted posts, handling media groups @@ -96,31 +139,38 @@ async def render_messages_groups(messages_groups, post_parser, exclude_flags: st 'author': message_data['author'], 'flags': message_data['flags'] }) - else: # Multiple messages in group - need to merge + else: # Multiple messages in group - merge media, text and html body processed_messages = [post_parser.process_message(msg) for msg in group] - main_message = next( # Find main message (one with text) + main_message = next( # Determine main message for header/footer/title (msg for msg in processed_messages if msg['text']), - processed_messages[0] # fallback to first if none has text + processed_messages[0] # fallback if no message contains text ) - # Collect all media sections + # Collect all media sections from all messages in the group all_media = [msg['html']['media'] for msg in processed_messages if msg['html']['media']] combined_media = '\n'.join(all_media) - - # Combine parts + + # Merge text fields from all messages + all_texts = [msg['text'] for msg in processed_messages if msg['text']] + combined_text = '\n'.join(all_texts) + + # Merge html body sections from all messages + all_html_bodies = [msg['html']['body'] for msg in processed_messages if msg['html']['body']] + combined_html_body = '\n'.join(all_html_bodies) + html_parts = [ f'
{main_message["html"]["header"]}
', f'
{combined_media}
', - f'
{main_message["html"]["body"]}
', + f'
{combined_html_body}
', f'
{main_message["html"]["footer"]}
' ] - + rendered_posts.append({ 'html': '\n'.join(html_parts), 'date': main_message['date'], 'message_id': main_message['message_id'], 'title': main_message['html']['title'], - 'text': main_message['text'], + 'text': combined_text, 'author': main_message['author'], 'flags': main_message['flags'] }) @@ -147,7 +197,13 @@ async def render_messages_groups(messages_groups, post_parser, exclude_flags: st rendered_posts.sort(key=lambda x: x['date'], reverse=True) return rendered_posts -async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20, exclude_flags: str | None = None ) -> str: +async def generate_channel_rss(channel: str, + post_parser: Optional[PostParser] = None, + client = None, + limit: int = 20, + exclude_flags: str | None = None, + merge_seconds: int = 5 + ) -> str: """ Generate RSS feed for channel using actual messages Args: @@ -199,6 +255,8 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = messages.append(message) # Process messages into groups and render them + if Config['time_based_merge']: + messages = await create_time_based_media_groups(messages, merge_seconds) message_groups = await create_messages_groups(messages) message_groups = await trim_messages_groups(message_groups, limit) final_posts = await render_messages_groups(message_groups, post_parser, exclude_flags) @@ -231,7 +289,13 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = raise -async def generate_channel_html(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20, exclude_flags: str | None = None) -> str: +async def generate_channel_html(channel: str, + post_parser: Optional[PostParser] = None, + client = None, + limit: int = 20, + exclude_flags: str | None = None, + merge_seconds: int = 5 + ) -> str: """ Generate HTML feed for channel using actual messages Args: @@ -272,6 +336,8 @@ async def generate_channel_html(channel: str, post_parser: Optional[PostParser] messages.append(message) # Process messages into groups and render them + if Config['time_based_merge']: + messages = await create_time_based_media_groups(messages, merge_seconds) message_groups = await create_messages_groups(messages) message_groups = await trim_messages_groups(message_groups, limit) final_posts = await render_messages_groups(message_groups, post_parser, exclude_flags)