diff --git a/api_server.py b/api_server.py
index be32d65..614041a 100644
--- a/api_server.py
+++ b/api_server.py
@@ -15,7 +15,7 @@ from fastapi import FastAPI, HTTPException, Response
from fastapi.responses import HTMLResponse, FileResponse
from telegram_client import TelegramClient
from config import get_settings
-from rss_generator import generate_channel_rss
+from rss_generator import generate_channel_rss, generate_channel_html
from post_parser import PostParser
from url_signer import verify_media_digest
@@ -419,7 +419,7 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, digest: str
@app.get("/rss/{channel}", response_class=Response)
@app.get("/rss/{channel}/{token}", response_class=Response)
-async def get_rss_feed(channel: str, token: str | None = None, limit: int = 20):
+async def get_rss_feed(channel: str, token: str | None = None, limit: int = 20, output_type: str = 'rss'):
if Config["token"]:
if token != Config["token"]:
logger.error(f"Invalid token for RSS feed: {token}, expected: {Config['token']}")
@@ -429,8 +429,12 @@ async def get_rss_feed(channel: str, token: str | None = None, limit: int = 20):
while True:
try:
- rss_content = await generate_channel_rss(channel, client=client.client, limit=limit)
- return Response(content=rss_content, media_type="application/xml")
+ if output_type == 'rss':
+ rss_content = await generate_channel_rss(channel, client=client.client, limit=limit)
+ return Response(content=rss_content, media_type="application/xml")
+ elif output_type == 'html':
+ rss_content = await generate_channel_html(channel, client=client.client, limit=limit)
+ return Response(content=rss_content, media_type="text/html")
except ValueError as e:
error_message = f"Invalid parameters for RSS feed generation: {str(e)}"
logger.error(error_message)
diff --git a/post_parser.py b/post_parser.py
index 9c7dbb6..4129b18 100644
--- a/post_parser.py
+++ b/post_parser.py
@@ -44,11 +44,13 @@ if not logger.handlers:
#http://127.0.0.1:8000/html/ufjqk/1070 - reply to
#http://127.0.0.1:8000/html/tetstststststststffd/4 - forwarded from channel without name
#http://127.0.0.1:8000/html/tetstststststststffd/14 - forwarded from hidden user
-#https://t.me/smallpharm/4802
-# https://t.me/webstrangler/3987
+#https://t.me/smallpharm/4802 many pics and text
+# https://t.me/webstrangler/3987 many pics without text
# https://t.me/teslacoilpro/7117
# https://t.me/red_spades/1222 many media + text
+#https://t.me/ni404head/1283 file
+
class PostParser:
def __init__(self, client):
self.client = client
@@ -85,24 +87,6 @@ class PostParser:
logger.error(f"post_parsing_error: channel {channel}, post_id {post_id}, error {str(e)}")
raise
- def _format_json(self, message: Message, naked: bool = False) -> Dict[Any, Any]:
- html_content = self._format_html(message, naked=naked)
- result = {
- 'channel': self.get_channel_username(message),
- 'message_id': message.id,
- 'date': datetime.timestamp(message.date),
- 'text': message.text or message.caption or '',
- 'html': html_content,
- 'title': self._generate_title(message),
- 'author': self._get_author_info(message),
- 'views': message.views,
- }
-
- if message.media_group_id:
- result['media_group_id'] = message.media_group_id
-
- return result
-
def _get_author_info(self, message: Message) -> str:
if message.sender_chat:
title = getattr(message.sender_chat, 'title', None)
@@ -194,31 +178,60 @@ class PostParser:
return f'Reply to #{reply_to.id}: {reply_text}
'
return None
+
- def _format_html(self, message: Message, naked: bool = False) -> str:
+ def format_message_for_feed(self, message: Message, top_info: bool = True, bottom_info: bool = True) -> Dict[Any, Any]:
+ return self._format_json(message, top_info=top_info, bottom_info=bottom_info)
+
+ def _format_json(self, message: Message, top_info: bool = True, bottom_info: bool = True) -> Dict[Any, Any]:
+ html_content = self._format_html(message, top_info=top_info, bottom_info=bottom_info)
+ result = {
+ 'channel': self.get_channel_username(message),
+ 'message_id': message.id,
+ 'date': datetime.timestamp(message.date),
+ 'date_formatted': message.date.strftime('%Y-%m-%d %H:%M:%S'),
+ 'text': message.text or message.caption or '',
+ 'html': html_content,
+ 'title': self._generate_title(message),
+ 'author': self._get_author_info(message),
+ 'views': message.views,
+ }
+
+ if message.media_group_id:
+ result['media_group_id'] = message.media_group_id
+
+ return result
+
+
+ def _format_html(self, message: Message, top_info: bool = True, bottom_info: bool = True) -> str:
html_content = []
- # Add forwarded from or reply info if present
- if forward_html := self._format_forward_info(message):
- html_content.append(forward_html)
- elif reply_html := self._format_reply_info(message):
- html_content.append(reply_html)
-
- # Check for "channel created" service message
+ # Create service message for "channel created"
if getattr(message, "channel_chat_created", False):
- service_html = 'Channel created
'
- if not naked:
- return self._wrap_html([service_html])
- return service_html
+ html_content.append('Channel created
')
+ html = '\n'.join(html_content)
+ return html
- text = message.text.html if message.text else message.caption.html if message.caption else ''
- text = text.replace('\n', '
')
+ # Add forwarded from or reply info if present
+ if top_info:
+ if forward_html := self._format_forward_info(message):
+ html_content.append(forward_html)
+ elif reply_html := self._format_reply_info(message):
+ html_content.append(reply_html)
+
+
+
+ if message.text:
+ text = message.text.html
+ elif message.caption:
+ text = message.caption.html
+ else:
+ text = ''
+
+ text = text.replace('\n', '
') # Replace newlines with
+ text = self._add_hyperlinks_to_raw_urls(text) # Add hyperlinks to raw URLs
- # Add hyperlinks to raw URLs
- text = self._add_hyperlinks_to_raw_urls(text)
-
- # Save media file_ids
- self._save_media_file_ids(message)
+ self._save_media_file_ids(message) # Save media file_ids for caching
if poll := getattr(message, "poll", None): # Poll formatting
if poll_html := self._format_poll(poll):
@@ -264,14 +277,15 @@ class PostParser:
if text: # Message text
html_content.append(f'{text}
')
- if not naked:
+ if bottom_info:
if reactions_views_html := self._reactions_views_links(message): # Add reactions, views and links
html_content.append(reactions_views_html)
- if not naked:
- html = self._wrap_html(html_content)
- else:
- html = '\n'.join(html_content)
+ #if not naked:
+ # html = self._wrap_html(html_content)
+ #else:
+
+ html = '\n'.join(html_content)
return html
@@ -421,9 +435,6 @@ class PostParser:
logger.error(f"recent_posts_error: channel {channel}, error {str(e)}")
raise
- def format_message_for_feed(self, message: Message, naked: bool = False) -> Dict[Any, Any]:
- return self._format_json(message, naked=naked)
-
def _save_media_file_ids(self, message: Message) -> None:
try:
file_data = {
diff --git a/rss_generator.py b/rss_generator.py
index 37398b8..dedcc90 100644
--- a/rss_generator.py
+++ b/rss_generator.py
@@ -4,6 +4,7 @@ from typing import Optional
from feedgen.feed import FeedGenerator
from post_parser import PostParser
from config import get_settings
+from bs4 import BeautifulSoup
Config = get_settings()
@@ -15,8 +16,134 @@ if not logger.handlers:
handler.setFormatter(logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s'))
logger.addHandler(handler)
+def reorganize_post_content(html):
+ """
+ Reorganize post content to move text after all media
+ Args:
+ html: Post HTML content
+ Returns:
+ Reorganized HTML content
+ """
+ soup = BeautifulSoup(html, 'html.parser')
+
+ # Find all media and text blocks
+ media_blocks = soup.find_all('div', class_='message-media')
+ text_blocks = soup.find_all('div', class_='message-text')
+
+ if not text_blocks: # No text to move
+ return html
+
+ # Create new structure
+ new_content = []
+
+ # Add all media first
+ for media in media_blocks:
+ media.extract()
+ new_content.append(str(media))
+ new_content.append('
')
+
+ # Add text last
+ for text in text_blocks:
+ text.extract()
+ new_content.append(str(text))
+
+ # Add remaining elements
+ for element in soup.contents:
+ if isinstance(element, str):
+ new_content.append(element)
+ elif element.name != 'br' and 'message-media' not in element.get('class', []) and 'message-text' not in element.get('class', []):
+ new_content.append(str(element))
+
+ return ''.join(new_content)
+
+def merge_posts(posts):
+ """Helper function to merge multiple posts into one"""
+ if not posts:
+ return None
+
+ # Find post with most meaningful title
+ main_post = posts[0]
+ for post in posts:
+ current_title = post.get('title', '')
+ if current_title and current_title not in ['📷 Photo', '📹 Video', '📄 Document']:
+ main_post = post
+ break
+
+ merged_post = main_post.copy()
+ merged_html = []
+ for post in posts:
+ merged_html.append(post['html'])
+
+ merged_post['html'] = '\n
\n'.join(merged_html)
+
+ # Reorganize content after merging
+ merged_post['html'] = reorganize_post_content(merged_post['html'])
+ return merged_post
+
+async def process_messages(messages, post_parser, channel):
+ """
+ Process messages into formatted posts, handling media groups
+ Args:
+ messages: List of raw messages
+ post_parser: PostParser instance
+ channel: Channel name for logging
+ Returns:
+ List of formatted and merged posts
+ """
+ posts = []
+ media_groups = {}
+
+ # First pass - collect messages and media groups
+ for message in messages:
+ try:
+ # Skip service messages about pinned posts
+ if message.service and 'PINNED_MESSAGE' in str(message.service):
+ logger.debug(f"Skipping pinned service message {message.id} in channel {channel}")
+ continue
+
+ if message.media_group_id:
+ if message.media_group_id not in media_groups:
+ media_groups[message.media_group_id] = []
+ media_groups[message.media_group_id].append(message)
+ else:
+ formatted = post_parser.format_message_for_feed(
+ message,
+ top_info=True,
+ bottom_info=True
+ )
+ if formatted:
+ posts.append(formatted)
+
+ except Exception as e:
+ logger.error(f"feed_entry_error: channel {channel}, message_id {message.id}, error {str(e)}")
+ continue
+
+ # Process media groups
+ for _group_id, group_messages in media_groups.items():
+ if not group_messages:
+ continue
+
+ group_messages.sort(key=lambda x: x.id)
+ formatted_posts = []
+
+ for i, msg in enumerate(group_messages):
+ formatted = post_parser.format_message_for_feed(
+ msg,
+ top_info=(i == 0),
+ bottom_info=(i == len(group_messages) - 1)
+ )
+ if formatted:
+ formatted_posts.append(formatted)
+
+ if formatted_posts:
+ posts.append(merge_posts(formatted_posts))
+
+ # Sort all posts by date
+ posts.sort(key=lambda x: x['date'], reverse=True)
+ return posts
+
async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20) -> str:
"""
Generate RSS feed for channel using actual messages
@@ -25,6 +152,7 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] =
post_parser: Optional PostParser instance. If not provided, will create new one
client: Telegram client instance
limit: Maximum number of posts to include in the RSS feed
+ output_type: 'rss' or 'html'
Returns:
RSS feed as string in XML format
"""
@@ -47,7 +175,7 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] =
channel_username = post_parser.get_channel_username(channel_info)
if not channel_username:
return create_error_feed(channel, base_url)
- except Exception as e: # raise error if channel not found
+ except Exception as e:
if "USERNAME_INVALID" in str(e) or "USERNAME_NOT_OCCUPIED" in str(e):
return create_error_feed(channel, base_url)
else:
@@ -59,66 +187,18 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] =
fg.link(href=f"https://t.me/{channel_username}", rel='alternate')
fg.description(f'Telegram channel {channel_username} RSS feed')
fg.language('ru')
-
fg.id(f"{base_url}/rss/{channel}")
- # First collect all posts
- posts = []
- media_groups = {}
-
+ # Collect messages
+ messages = []
async for message in post_parser.client.get_chat_history(channel, limit=limit):
- try:
- # Skip service messages about pinned posts
- if message.service and 'PINNED_MESSAGE' in str(message.service):
- logger.debug(f"Skipping pinned service message {message.id} in channel {channel}")
- continue
-
- naked_html = True if message.media_group_id else False
- post = post_parser.format_message_for_feed(message, naked=naked_html)
- if not post:
- continue
-
- if post.get('media_group_id'):
- if post['media_group_id'] not in media_groups:
- media_groups[post['media_group_id']] = []
- media_groups[post['media_group_id']].append(post)
- else:
- posts.append(post)
-
- except Exception as e:
- logger.error(f"feed_entry_error: channel {channel}, message_id {message.id}, error {str(e)}")
- continue
-
- # Merge media groups
- for _group_id, group_posts in media_groups.items():
- if not group_posts:
- continue
-
- # Sort posts within media group by message_id
- group_posts.sort(key=lambda x: x['message_id'])
+ messages.append(message)
- # Find post with most meaningful title
- main_post = group_posts[0]
- for post in group_posts:
- current_title = post.get('title', '')
- if current_title and current_title not in ['📷 Photo', '📹 Video', '📄 Document']:
- main_post = post
- break
-
- merged_post = main_post.copy()
- merged_html = []
-
- for post in group_posts:
- merged_html.append(post['html'])
-
- merged_post['html'] = '\n'.join(merged_html)
- posts.append(merged_post)
-
- # Sort posts by date
- posts.sort(key=lambda x: x['date'], reverse=True)
+ # Process messages into formatted posts
+ final_posts = await process_messages(messages, post_parser, channel)
# Generate feed entries
- for post in posts:
+ for post in final_posts:
fe = fg.add_entry()
fe.title(post.get('title', 'Untitled post'))
@@ -144,9 +224,66 @@ async def generate_channel_rss(channel: str, post_parser: Optional[PostParser] =
except Exception as e:
logger.error(f"rss_generation_error: channel {channel}, error {str(e)}")
- raise
+ raise
+async def generate_channel_html(channel: str, post_parser: Optional[PostParser] = None, client = None, limit: int = 20) -> str:
+ """
+ Generate HTML feed for channel using actual messages
+ Args:
+ channel: Telegram channel name
+ post_parser: Optional PostParser instance. If not provided, will create new one
+ client: Telegram client instance
+ limit: Maximum number of posts to include in the RSS feed
+ Returns:
+ HTML feed as string
+ """
+ if limit < 1:
+ raise ValueError(f"limit must be positive, got {limit}")
+ if limit > 100:
+ raise ValueError(f"limit cannot exceed 100, got {limit}")
+
+ try:
+ if post_parser is None:
+ post_parser = PostParser(client=client)
+
+ base_url = Config['pyrogram_bridge_url']
+
+ try:
+ channel_info = await post_parser.client.get_chat(channel)
+ channel_title = channel_info.title or f"Telegram: {channel}"
+ channel_username = post_parser.get_channel_username(channel_info)
+ if not channel_username:
+ return create_error_feed(channel, base_url)
+ except Exception as e:
+ if "USERNAME_INVALID" in str(e) or "USERNAME_NOT_OCCUPIED" in str(e):
+ return create_error_feed(channel, base_url)
+ else:
+ raise
+
+ # Set feed metadata
+ main_name = f"{channel_title} (@{channel_username})"
+ html_content = f'{main_name}
'
+
+ # Collect messages
+ messages = []
+ async for message in post_parser.client.get_chat_history(channel, limit=limit):
+ messages.append(message)
+
+ # Process messages into formatted posts
+ final_posts = await process_messages(messages, post_parser, channel)
+
+ html_posts = []
+ for post in final_posts:
+ html_content = post.get('html', '')
+ if html_content:
+ html_posts.append(f'{html_content}
')
+ return '\n
\n'.join(html_posts)
+
+ except Exception as e:
+ logger.error(f"rss_generation_error: channel {channel}, error {str(e)}")
+ raise
+
def create_error_feed(channel: str, base_url: str) -> str:
"""
Create an empty RSS feed with metadata indicating an error when the channel is not found.