#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# flake8: noqa
# pylint: disable=broad-exception-raised, raise-missing-from, too-many-arguments, redefined-outer-name
# pylint: disable=multiple-statements, logging-fstring-interpolation, trailing-whitespace, line-too-long
# pylint: disable=broad-exception-caught, missing-function-docstring, missing-class-docstring
# pylint: disable=f-string-without-interpolation
# pylance: disable=reportMissingImports, reportMissingModuleSource
import logging
import re
import os
import json
import html
import inspect
from datetime import datetime
from typing import Union, Dict, Any, List, Optional
from pyrogram.types import Message
from pyrogram.enums import MessageMediaType
from bleach.css_sanitizer import CSSSanitizer
from bleach import clean as HTMLSanitizer
from config import get_settings
from url_signer import generate_media_digest
Config = get_settings()
logger = logging.getLogger(__name__)
#tests
#http://127.0.0.1:8000/post/html/DragorWW_space/114 — video
#http://127.0.0.1:8000/post/html/DragorWW_space/20 - many photos
#http://127.0.0.1:8000/post/html/DragorWW_space/58 - photos+video
#http://127.0.0.1:8000/post/html/DragorWW_space/44 - poll
#http://127.0.0.1:8000/post/html/DragorWW_space/46 - photo
#http://127.0.0.1:8000/post/html/DragorWW_space/49, http://127.0.0.1:8000/post/html/DragorWW_space/63 — webpage
#http://127.0.0.1:8000/post/html/deckru/826 - animation
#http://127.0.0.1:8000/post/html/DragorWW_space/61 — links
#http://127.0.0.1:8000/post/html/theyforcedme/3577 - video note
#http://127.0.0.1:8000/post/html/theyforcedme/3572 - audio
#http://127.0.0.1:8000/post/html/theyforcedme/3558 - audio-note
#http://127.0.0.1:8000/html/vvzvlad_lytdybr/426 - sticker
#http://127.0.0.1:8000/html/wrkshprn/634
# http://127.0.0.1:8000/html/ni404head/1278
# http://127.0.0.1:8000/html/fieryfiles/4840 — links without
#http://127.0.0.1:8000/html/ru2ch_ban/26586 - large video
#http://127.0.0.1:8000/html/smallpharm/4828 - forwarded from channel
#http://127.0.0.1:8000/html/vvzvlad_lytdybr/659 - forwarded from user
#http://127.0.0.1:8000/html/ufjqk/1070 - reply to
#http://127.0.0.1:8000/html/tetstststststststffd/4 - forwarded from channel without name
#http://127.0.0.1:8000/html/tetstststststststffd/14 - forwarded from hidden user
#https://t.me/smallpharm/4802 many pics and text
# https://t.me/webstrangler/3987 many pics without text
# https://t.me/teslacoilpro/7117
# https://t.me/red_spades/1222 many media + text
#https://t.me/ni404head/1283 file
#http://127.0.0.1:8000/rss/-1002069358234 channel with numeric ID
class PostParser:
def __init__(self, client):
self.client = client
@staticmethod
def get_all_possible_flags() -> List[str]:
"""Dynamically extracts all possible flag names from the _extract_flags method."""
try:
source_code = inspect.getsource(PostParser._extract_flags)
# Find all occurrences of flags.append("flag_name")
flags = re.findall(r'flags\.append\("([^"]+)"\)', source_code)
# Return unique flags
return sorted(list(set(flags)))
except Exception as e:
logger.error(f"flag_extraction_error: Could not extract flags dynamically, error {str(e)}")
# Fallback to a manually defined list might be needed here in case of error,
# but for now, we return an empty list.
return []
def channel_name_prepare(self, channel: str | int) -> Union[str, int]:
if isinstance(channel, str) and channel.startswith('-100'): # Convert numeric channel ID to int
channel_id = int(channel)
return channel_id
else:
return channel
async def get_post(self, channel: str,
post_id: int,
output_type: str = 'json',
debug: bool = False) -> Union[str, Dict[Any, Any], None]:
print(f"Getting post {channel}, {post_id}")
try:
prepared_channel_id: Union[str, int] = self.channel_name_prepare(channel)
message = await self.client.get_messages(prepared_channel_id, post_id)
if Config["debug"]: print(message)
if not message or getattr(message, 'empty', False):
logger.error(f"post_not_found_or_empty: channel {prepared_channel_id}, post_id {post_id}")
return None
processed_message = self.process_message(message)
if output_type == 'html':
return self._format_html(processed_message, debug)
elif output_type == 'json':
return processed_message
else:
logger.error(f"Invalid output type: {output_type}")
return None
except Exception as e:
# Log the specific exception type and message
logger.error(f"post_parsing_error: channel {prepared_channel_id}, post_id {post_id}, error_type {type(e).__name__}, error_message {str(e)}")
# Optional: include traceback for more detail
# import traceback
# logger.error(traceback.format_exc())
raise
def _get_author_info(self, message: Message) -> str: #Tests: tests/postparser_author_info.py
if message.sender_chat:
title = getattr(message.sender_chat, 'title', None)
username = getattr(message.sender_chat, 'username', None)
title = title.strip() if title else ''
username = username.strip() if username else ''
if username:
if title:
return f"{title} (@{username})"
else:
return f"@{username}"
return title
elif message.from_user:
first = getattr(message.from_user, 'first_name', None)
last = getattr(message.from_user, 'last_name', None)
username = getattr(message.from_user, 'username', None)
first = first.strip() if first else ''
last = last.strip() if last else ''
username = username.strip() if username else ''
name = ' '.join(filter(None, [first, last]))
if not name:
return "Unknown author"
if username:
return f"{name} (@{username})"
else:
return name
return "Unknown author"
def _truncate_title(self, first_line: str) -> str:
"""Truncate the title """
# Step 1: Cut at the first period followed by a space, if present
period_match = re.search(r'\.(?=\s)', first_line)
if period_match: first_line = first_line[:period_match.start()].rstrip()
# Step 2: Apply the main cut logic
cut_at = 37
max_extra_chars = 15
limit_index = cut_at + max_extra_chars # 52
if len(first_line) > cut_at:
cut_limit = min(len(first_line), limit_index)
last_space_index = first_line.rfind(' ', 0, cut_limit)
if last_space_index != -1 and last_space_index >= cut_at: cut_index = last_space_index
else: cut_index = cut_limit
title_segment = first_line[:cut_index]
title_segment = re.sub(r'[\s.,;:]+$', '', title_segment).strip()
if len(first_line[:cut_index]) < len(first_line): return f"{title_segment}..."
else: return title_segment
else: return first_line
def _service_message_title(self, message: Message) -> Union[str, None]:
if service := getattr(message, "service", None):
if 'PINNED_MESSAGE' in str(service): return "📌 Pinned message"
elif 'NEW_CHAT_PHOTO' in str(service): return "🖼 New chat photo"
elif 'NEW_CHAT_TITLE' in str(service): return "✏️ New chat title"
elif 'VIDEO_CHAT_STARTED' in str(service): return "▶️ Video chat started"
elif 'VIDEO_CHAT_ENDED' in str(service): return "⏹ Video chat ended"
elif 'VIDEO_CHAT_SCHEDULED' in str(service): return "⏰ Video chat scheduled"
elif 'GROUP_CHAT_CREATED' in str(service): return "✨ Group chat created"
elif 'CHANNEL_CHAT_CREATED' in str(service): return "✨ Chat created"
elif 'DELETE_CHAT_PHOTO' in str(service): return "🗑️ Chat photo deleted"
return None
def _media_message_title(self, message: Message) -> Union[str, None]:
if message.media:
if message.media == MessageMediaType.POLL:
if message.poll is not None and hasattr(message.poll, 'question'):
poll_question = message.poll.question.strip()
if poll_question:
return f"📊 Poll: {poll_question}"
return "📊 Poll"
if message.media == MessageMediaType.DOCUMENT:
is_mime = (message.document is not None and hasattr(message.document, 'mime_type'))
if is_mime and message.document is not None and message.document.mime_type == 'application/pdf':
return "📄 PDF Document"
return "📎 Document"
if message.media == MessageMediaType.PHOTO: return "📷 Photo"
if message.media == MessageMediaType.VIDEO: return "🎥 Video"
if message.media == MessageMediaType.ANIMATION: return "🎞 GIF"
if message.media == MessageMediaType.AUDIO: return "🎵 Audio"
if message.media == MessageMediaType.VOICE: return "🎤 Voice"
if message.media == MessageMediaType.VIDEO_NOTE: return "📱 Video circle"
if message.media == MessageMediaType.STICKER: return "🎯 Sticker"
if message.media == MessageMediaType.STORY: return "📱 Story"
# Web pages (if no text or media title)
if message.web_page:
if message.web_page.title:
return f"🔗 {message.web_page.title}"
return "🔗 Web link"
return None
def _generate_base_title(self, message: Message) -> Union[str, None]:
"""Generates the base title without the FWD prefix."""
# --- Text Processing --- (Phase 1: Process text if available)
text = message.text or message.caption or ''
text_stripped = text.strip()
processed_title = None
text_was_processed = False # Flag to indicate if text processing block was entered
min_title_length = 10 # Minimum length for text to be preferred over media/webpage
if text_stripped:
text_was_processed = True
text_has_urls = bool(re.search(r'https?://[^\\s<>\"\\\']+', text_stripped))
clean_text = html.unescape(text) # Remove HTML entities
clean_text = re.sub(r'<[^>]+?>', '', clean_text) # Remove HTML tags
clean_text = re.sub(r'https?://[^\s<>"\']+', '', clean_text) # Remove URLs
clean_text = clean_text.strip() # Remove whitespaces
if clean_text: # If text remains after cleaning
# Process URL at the beginning
if text_has_urls and "://" in clean_text:
clean_text = clean_text.split("://")[0].strip()
# Process line breaks & punctuation
first_line = clean_text.split('\n', 1)[0] # Get first line
first_line = re.sub(r'[.,;:]+$', '', first_line) # Remove trailing punctuation
first_line = first_line.strip() # Remove whitespaces
# Handle uppercase
if first_line.isupper() and len(first_line) > 1:
first_line = first_line.lower().capitalize() # Downcase and capitalize first letter
# --- Trim long strings ---
processed_title = self._truncate_title(first_line) # Truncate title
# --- Decision Logic --- (Phase 2: Decide whether to use processed text or fallback)
# Condition to use processed_title: It exists AND (it's long enough OR there's no media)
# Webpage presence doesn't prevent using short text if there's no media.
use_text_title = processed_title and (len(processed_title) >= min_title_length or not message.media)
if use_text_title: return processed_title
# --- Fallback Processing --- (Phase 3: If text wasn't suitable or was discarded)
# Handle specific cases for non-meaningful original text (if text block was entered but didn't yield a usable title)
if text_was_processed and not use_text_title:
if re.search(r'(?:youtube\.com|youtu\.be)', text_stripped.lower()):
if message.web_page and message.web_page.title:
return f"🎥 YouTube: {message.web_page.title}"
else:
return "🎥 YouTube Link"
# Check if original text was just a URL and there's a webpage title
if message.web_page and message.web_page.title:
url_match = re.match(r'^\s*(https?://[^\s<>"\']+)\s*$', text_stripped)
if url_match: return f"🔗 {message.web_page.title}"
if text_has_urls: # If original text had any URL (and wasn't YouTube/Webpage with title)
return "🔗 Web link"
return None
def _generate_title(self, message: Message) -> str: #Tests: tests/postparser_gen_title.py
"""Generate a title for a message, based on its content."""
title = None
title = self._service_message_title(message)
if title is None: title = self._generate_base_title(message)
if title is None: title = self._media_message_title(message)
if title is None: title = "❓ Unknown Post"
if message.forward_origin: title = f"FWD: {title}"
return title
def _format_reply_info(self, message: Message) -> Union[str, None]:
is_pinned = (getattr(message, "service", None) and 'PINNED_MESSAGE' in str(message.service))
reply_to = getattr(message, "reply_to_message", None)
if is_pinned and reply_to:
reply_text = reply_to.text or reply_to.caption or ''
if len(reply_text) > 100:
reply_text = reply_text[:100] + '...'
return f'
'
if reply_to:
reply_text = reply_to.text or reply_to.caption or ''
if len(reply_text) > 100:
reply_text = reply_text[:100] + '...'
channel_username = getattr(reply_to.sender_chat, "username", None)
if channel_username:
reply_link = f'#{reply_to.id}'
return f'
{data["raw_message"]}')
html_data = '\n'.join(html_content)
return html_data
def _format_flags(self, flags_list: list) -> str:
if not Config['show_post_flags']: return ''
return_html = ""
if flags_list:
flags_html = ['')
return_html = ' '.join(flags_html)
return return_html
def process_message(self, message: Message) -> Dict[Any, Any]:
result = {
'channel': self.get_channel_username(message),
'message_id': message.id,
'date': datetime.timestamp(message.date) if message.date else None,
'date_formatted': message.date.strftime('%Y-%m-%d %H:%M:%S') if message.date else None,
'text': message.text or message.caption or '',
'html': {
'title': self._generate_title(message),
'body': self._generate_html_body(message),
'footer': self.generate_html_footer(message)
},
'flags': self._extract_flags(message),
'author': self._get_author_info(message),
'views': message.views,
'reactions': self._extract_reactions(message),
'media_group_id': message.media_group_id,
'service': getattr(message, "service", None),
'raw_message': str(message)
}
# Add webpage data if available
if webpage := getattr(message, "web_page", None):
result['webpage'] = {
'type': getattr(webpage, 'type', None),
'url': getattr(webpage, 'url', None),
'display_url': getattr(webpage, 'display_url', None),
'site_name': getattr(webpage, 'site_name', None),
'title': getattr(webpage, 'title', None),
'description': getattr(webpage, 'description', None),
'has_large_media': getattr(webpage, 'has_large_media', False),
'is_telegram_link': getattr(webpage, 'type', '') == 'telegram_message'
}
return result
def _sanitize_html(self, html_raw: str) -> str:
allowed_tags = ['p', 'a', 'b', 'i', 'strong',
'em', 'ul', 'ol', 'li', 'br',
'div', 'span', 'img', 'video', 'audio',
'source']
allowed_attributes = {
'a': ['href', 'title', 'target'],
'img': ['src', 'alt', 'style'],
'video': ['controls', 'src', 'style'],
'audio': ['controls', 'style'],
'source': ['src', 'type'],
'div': ['class', 'style'],
'span': ['class']
}
try:
css_sanitizer = CSSSanitizer(
allowed_css_properties=["max-width", "max-height", "object-fit", "width", "height"]
)
sanitized_html = HTMLSanitizer(
html_raw,
tags=allowed_tags,
attributes=allowed_attributes,
protocols=['http', 'https', 'tg'],
css_sanitizer=css_sanitizer,
strip=True,
)
return sanitized_html
except Exception as e:
logger.error(f"html_sanitization_error: error {str(e)}")
return html_raw
def _format_forward_info(self, message: Message) -> Union[str, None]:
if forward_origin := getattr(message, "forward_origin", None):
# Case 1: Channel or supergroup forward (has chat attribute)
if sender_chat := getattr(forward_origin, "chat", None):
forward_title = getattr(sender_chat, "title", "Unknown channel")
forward_username = getattr(sender_chat, "username", None)
if forward_username:
forward_link = f'{forward_title} (@{forward_username})'
return f''
return f''
# Case 2: Hidden user (MessageOriginHiddenUser)
if hasattr(forward_origin, "sender_user_name") and forward_origin.sender_user_name:
sender_name = forward_origin.sender_user_name
return f''
# Case 3: Regular user (MessageOriginUser)
if hasattr(forward_origin, "sender_user") and forward_origin.sender_user:
user = forward_origin.sender_user
name_parts = []
if hasattr(user, "first_name") and user.first_name:
name_parts.append(user.first_name)
if hasattr(user, "last_name") and user.last_name:
name_parts.append(user.last_name)
sender_name = " ".join(name_parts) if name_parts else "Unknown user"
if hasattr(user, "username") and user.username:
sender_name = f"{sender_name} (@{user.username})"
return f''
# Case 4: Channel without username (MessageOriginChannel)
if hasattr(forward_origin, "chat_id") and hasattr(forward_origin, "title"):
title = forward_origin.title if forward_origin.title else "Unknown channel"
return f''
# Case 5: Default for any other type of forward_origin
origin_type = getattr(forward_origin, "type", "unknown")
logger.debug(f"unhandled_forward_type: {origin_type}, data: {forward_origin}")
return f''
return None
def _get_post_text_with_urls(self, message: Message) -> Union[str, None]:
if message.text: text = message.text.html
elif message.caption: text = message.caption.html
else: return None
text = text.replace('\n', '