vvzvlad/pyrogram-bridge
1
0
Fork 0
Code Issues 6 Pull Requests Actions 8 Packages Projects Releases Wiki Activity

Refactor PostParser title generation

Browse Source
This commit is contained in:
vvzvlad
2025-04-15 18:03:16 +03:00
parent 1871843b17
commit ff9ebeaa70
3 changed files with 242 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.cursor/rules/testing.mdс
+4 -1
View File
@@ -3,4 +3,7 @@
- **Location:** Tests are located in the `tests/` directory.
- **Execution:**
- To run tests for the post parser, use the command: `pytest tests/test_post_parser.py`
- To run all tests in the project, use: `pytest tests/`
- To run all tests in the project, use: `pytest tests/`
You should run the tests yourself with the `pytest tests/` command, not ask the user to run them
After that, you have to analyze the test results and if they fall down, keep fixing the code or tests depending on the task context
post_parser.py
+107 -90
View File
@@ -136,37 +136,119 @@ class PostParser:
# Check for service messages first
if service := getattr(message, "service", None):
service_str = str(service)
if 'PINNED_MESSAGE' in service_str: return "📌 Pinned message"
elif 'NEW_CHAT_PHOTO' in service_str: return "🖼 New chat photo"
elif 'NEW_CHAT_TITLE' in service_str: return "✏️ New chat title"
elif 'VIDEO_CHAT_STARTED' in service_str: return "▶️ Video chat started"
elif 'VIDEO_CHAT_ENDED' in service_str: return "⏹ Video chat ended"
elif 'VIDEO_CHAT_SCHEDULED' in service_str: return "⏰ Video chat scheduled"
elif 'GROUP_CHAT_CREATED' in service_str: return "✨ Group chat created"
elif 'CHANNEL_CHAT_CREATED' in service_str: return "✨ Chat created"
elif 'DELETE_CHAT_PHOTO' in service_str: return "🗑️ Chat photo deleted"
elif 'NEW_CHAT_TITLE' in service_str: return "✏️ New chat title"
if 'PINNED_MESSAGE' in str(service): return "📌 Pinned message"
elif 'NEW_CHAT_PHOTO' in str(service): return "🖼 New chat photo"
elif 'NEW_CHAT_TITLE' in str(service): return "✏️ New chat title"
elif 'VIDEO_CHAT_STARTED' in str(service): return "▶️ Video chat started"
elif 'VIDEO_CHAT_ENDED' in str(service): return "⏹ Video chat ended"
elif 'VIDEO_CHAT_SCHEDULED' in str(service): return "⏰ Video chat scheduled"
elif 'GROUP_CHAT_CREATED' in str(service): return "✨ Group chat created"
elif 'CHANNEL_CHAT_CREATED' in str(service): return "✨ Chat created"
elif 'DELETE_CHAT_PHOTO' in str(service): return "🗑️ Chat photo deleted"
# Process media content
if message.media:
# Polls
# --- Text Processing --- (Phase 1: Process text if available)
text = message.text or message.caption or ''
text_stripped = text.strip()
processed_title = None
text_was_processed = False # Flag to indicate if text processing block was entered
min_title_length = 10 # Minimum length for text to be preferred over media/webpage
if text_stripped:
text_was_processed = True
text_has_urls = bool(re.search(r'https?://[^\\s<>\"\\\']+', text_stripped))
clean_text = html.unescape(text)
clean_text = re.sub(r'<[^>]+?>', '', clean_text)
clean_text = re.sub(r'https?://[^\s<>"\']+', '', clean_text)
clean_text = clean_text.strip()
if clean_text: # If text remains after cleaning
# Process URL at the beginning
if text_has_urls and "://" in clean_text:
clean_text = clean_text.split("://")[0].strip()
# Process line breaks & punctuation
first_line = clean_text.split('\n', 1)[0]
first_line = re.sub(r'[.,;:]+$', '', first_line)
first_line = first_line.strip()
# Handle uppercase
if first_line.isupper() and len(first_line) > 1:
first_line = first_line.lower().capitalize()
# --- Trim long strings (REVISED LOGIC - Find LAST space before limit) ---
cut_at = 37
max_extra_chars = 15
limit_index = cut_at + max_extra_chars # 52
if len(first_line) > cut_at:
# Define the effective limit for searching and cutting
cut_limit = min(len(first_line), limit_index)
# Find the LAST space before the cut_limit
last_space_index = first_line.rfind(' ', 0, cut_limit)
# Decide the cut index
if last_space_index != -1 and last_space_index >= cut_at:
# Found a suitable space within [cut_at, cut_limit)
cut_index = last_space_index
else:
# No suitable space found, cut at the hard limit
cut_index = cut_limit
# Slice the string
title_segment = first_line[:cut_index]
# Clean trailing punctuation/spaces from the segment AFTER slicing
title_segment = re.sub(r'[\s.,;:]+$', '', title_segment).strip()
# Only add ellipsis if the title was actually cut
# (Check against original first_line length before cleaning segment)
if len(first_line[:cut_index]) < len(first_line):
processed_title = f"{title_segment}..."
else: # Safeguard
processed_title = title_segment
else:
# Length <= cut_at, use as is
processed_title = first_line
# --- End Trim long strings (REVISED LOGIC) ---
# --- Decision Logic --- (Phase 2: Decide whether to use processed text or fallback)
# Condition to use processed_title: It exists AND (it's long enough OR there's no media)
# Webpage presence doesn't prevent using short text if there's no media.
use_text_title = processed_title and (len(processed_title) >= min_title_length or not message.media)
if use_text_title:
return processed_title
# --- Fallback Processing --- (Phase 3: If text wasn't suitable or was discarded)
# Handle specific cases for non-meaningful original text (if text block was entered but didn't yield a usable title)
if text_was_processed and not use_text_title:
if re.search(r'(?:youtube\.com|youtu\.be)', text_stripped.lower()):
return "🎥 YouTube Link"
# Check if original text was just a URL and there's a webpage title
if message.web_page and message.web_page.title:
url_match = re.match(r'^\s*(https?://[^\s<>"\']+)\s*$', text_stripped)
if url_match:
return f"🔗 {message.web_page.title}"
if text_has_urls: # If original text had any URL (and wasn't YouTube/Webpage with title)
return "🔗 Web link"
# If it only had tags or was short/whitespace, proceed to media/poll check
# Process media content (if no suitable text title)
if message.media:
if message.media == MessageMediaType.POLL:
if hasattr(message, 'poll') and hasattr(message.poll, 'question'):
poll_question = message.poll.question.strip()
if poll_question:
return f"📊 Poll: {poll_question}"
else:
return "📊 Poll"
# PDF documents
return "📊 Poll"
if message.media == MessageMediaType.DOCUMENT:
if hasattr(message.document, 'mime_type') and 'pdf' in message.document.mime_type.lower():
return "📄 PDF Document"
else:
return "📎 Document"
# Other media types
return "📎 Document"
if message.media == MessageMediaType.PHOTO: return "📷 Photo"
if message.media == MessageMediaType.VIDEO: return "🎥 Video"
if message.media == MessageMediaType.ANIMATION: return "🎞 GIF"
@@ -175,77 +257,12 @@ class PostParser:
if message.media == MessageMediaType.VIDEO_NOTE: return "📱 Video circle"
if message.media == MessageMediaType.STICKER: return "🎯 Sticker"
text = message.text or message.caption or ''
text_stripped = text.strip()
if text_stripped:
# Check if there is only HTML or URL
text_has_tags = bool(re.search(r'<[^>]+?>', text_stripped))
text_has_urls = bool(re.search(r'https?://[^\s<>"\']+', text_stripped))
# Clean text from tags and links
clean_text = html.unescape(text)
clean_text = re.sub(r'<[^>]+?>', '', clean_text)
clean_text = re.sub(r'https?://[^\s<>"\']+', '', clean_text)
clean_text = clean_text.strip()
# If after cleaning there is no meaningful content
if not clean_text:
# If there was a <br> or other special tags
if text_has_tags and not text_has_urls:
return "❓ Unknown Post"
# If there was a YouTube link
if re.search(r'(?:youtube\.com|youtu\.be)', text_stripped.lower()):
return "🎥 YouTube Link"
# Check if we have a web_page with title and text is basically just a URL
if message.web_page and message.web_page.title:
# Text is basically just a URL
url_match = re.match(r'^(https?://[^\s<>"\']+)$', text_stripped)
if url_match:
return f"🔗 {message.web_page.title}"
# Otherwise, it's a normal web link
return "🔗 Web link"
# Process URL at the beginning of the title
if text_has_urls and "://" in clean_text:
# Remove part of text after URL
clean_text = clean_text.split("://")[0].strip()
# Process line breaks - take only the first line
first_line = clean_text.split('\n', 1)[0]
first_line = re.sub(r'[.,;:]+$', '', first_line) #Remove dot
first_line = first_line.strip()
# Handle uppercase text - convert to title case if the text is all uppercase
if first_line.isupper():
first_line = first_line.lower().capitalize()
# Process long strings
cut_at = 37
max_extra_chars = 15
if len(first_line) > cut_at:
# Find the next space after cut_at, but not more than 15 chars forward
extended_cut = cut_at
for i in range(cut_at, min(cut_at + max_extra_chars, len(first_line))):
extended_cut = i
if first_line[i] == ' ':
break
title = f"{first_line[:extended_cut]}"
title = re.sub(r'[.,;:]+$', '', title)
return f"{title}..."
return first_line
# Web pages
# Web pages (if no text or media title)
if message.web_page:
if message.web_page.title:
return f"🔗 {message.web_page.title}"
else:
return "🔗 Web link"
return "🔗 Web link"
# If nothing matches
return "❓ Unknown Post"
tests/test_post_parser.py
+131 -15
View File
@@ -159,17 +159,79 @@ class TestPostParserGenerateTitle(unittest.TestCase):
def test_generate_title_caption_with_url_and_text(self):
message = self._create_mock_message(media=MessageMediaType.PHOTO, caption="Look at this photo! https://example.com/image.jpg")
self.assertEqual(self.parser._generate_title(message), "📷 Photo")
self.assertEqual(self.parser._generate_title(message), "Look at this photo!")
def test_generate_title_caption_with_uppercase_text(self):
message = self._create_mock_message(text="ЖИЗНЬ НА ОБОЯХ")
self.assertEqual(self.parser._generate_title(message), "Жизнь на обоях") #downcase
def test_generate_title_long_text_trimming(self):
long_text = "This is a very long line of text that definitely exceeds the maximum length allowed for a title, so it should be trimmed intelligently at the last space before the limit."
message = self._create_mock_message(text=long_text)
expected_title = "This is a very long line of text that..." #cut at 37
self.assertEqual(self.parser._generate_title(message), expected_title)
def test_generate_title_long_text_trimming_with_spaces(self):
cut_at = 37
max_extra = 15
limit = cut_at + max_extra # 52
# --- Test Cases Based on Correct Logic ---
# 1. Length <= cut_at (37) -> No trim
text = "This text is exactly thirty-seven chars" # len 37
message = self._create_mock_message(text=text)
self.assertEqual(self.parser._generate_title(message), text)
# 2. Length = 38, no space in check range -> NO Trim (cut index == len)
text = "This text is exactly thirty-seven charsX" # len 38
message = self._create_mock_message(text=text)
self.assertEqual(self.parser._generate_title(message), text)
# 3. Space found within range [cut_at, limit)
# 3a. Space exactly at cut_at (index 37)
text = "This text is exactly thirty-seven chars next" # len 42. Space at 37.
message = self._create_mock_message(text=text)
# Loop range(37, 42) -> i=37. first_line[37]==' '. Break. ext_cut=37. Slice [:37].
self.assertEqual(self.parser._generate_title(message), "This text is exactly thirty-seven chars...")
# 3b.
text = "This text is quite a bit longer now space_here_herehere"
message = self._create_mock_message(text=text)
# Loop range(37, 49). Finds space at i=43. Breaks. ext_cut=43. Slice [:43].
self.assertEqual(self.parser._generate_title(message), "This text is quite a bit longer now space_here_hereh...")
# 3c. Space exactly at limit - 1 (index 51)
text = "This is fifty-one characters long with the space1 here X" # len 54. Space at 51.
message = self._create_mock_message(text=text)
# Loop range(37, 52). Finds space at i=51. Breaks. ext_cut=51. Slice [:51].
self.assertEqual(self.parser._generate_title(message), "This is fifty-one characters long with the space1...")
# 3c. Space exactly at limit - 1 (index 51)
text = "This is fifty-one chara the space1 here1 X" # len 54. Space at 51.
message = self._create_mock_message(text=text)
# Loop range(37, 52). Finds space at i=51. Breaks. ext_cut=51. Slice [:51].
self.assertEqual(self.parser._generate_title(message), "This is fifty-one chara the space1 here1...")
# 4. No space found within range [cut_at, limit)
# 4a. Length > cut_at, Length < limit. No space in [cut_at, len). -> Cut at len-1
text = "JGHJHKJHKJDHfushdkjfskjdfhnksjdvnskjdnkjsdfjksdhfsdlfijoirukjvnsdkjvskufh" # len 49. cut_at=37. limit=52.
message = self._create_mock_message(text=text)
# Loop range(37, 49). No space. Finishes. ext_cut=48. Slice [:48].
self.assertEqual(self.parser._generate_title(message), "JGHJHKJHKJDHfushdkjfskjdfhnksjdvnskjdnkjsdfjksdhfsdl...")
text = "JGHJHKJHKJDHfushdkjfskjdfhnksjdvnskjdnkjsdfjksdhfs" # len 49. cut_at=37. limit=52.
message = self._create_mock_message(text=text)
self.assertEqual(self.parser._generate_title(message), "JGHJHKJHKJDHfushdkjfskjdfhnksjdvnskjdnkjsdfjksdhfs")
# 4b. Length >= limit. No space in [cut_at, limit). -> Cut at limit-1 = 51
text = "ThisIsAnEvenLongerWordWithoutAnySpacesAndDefinitelyMoreThan52Chars" # len 66. cut_at=37. limit=52.
message = self._create_mock_message(text=text)
# Loop range(37, 52). No space. Finishes. ext_cut=51. Slice [:51].
self.assertEqual(self.parser._generate_title(message), "ThisIsAnEvenLongerWordWithoutAnySpacesAndDefinitelyM...")
# 5. Trailing space/punctuation removal check
# 5a. Space found, cut segment ends with space/punct
text = "This text is quite a bit longer now, space .,;: here" # len 54. Space at 43.
message = self._create_mock_message(text=text)
# Loop range(37, 54). Finds space at i=43. Breaks. ext_cut=43. Slice [:43] is "This text is quite a bit longer now, space ".
# re.sub removes trailing " .,;: ". Result "This text is quite a bit longer now, space".
self.assertEqual(self.parser._generate_title(message), "This text is quite a bit longer now, space...")
def test_generate_title_break_word_after_limit(self):
# Test with a specific text example from the user's query
@@ -181,7 +243,8 @@ class TestPostParserGenerateTitle(unittest.TestCase):
def test_generate_title_long_text_no_space_trimming(self):
long_text = "Thisisaverylonglineoftextthatdefinitelyexceedsthemaximumlengthallowedforatitlesoitshouldbetrimmedatthelimitbecausehasnospaces."
message = self._create_mock_message(text=long_text)
expected_title = "Thisisaverylonglineoftextthatdefinitelyexceedsthema..." #cut at 30+15 symbols without space
# Corrected expected title based on new logic: cut at index 52 (min(52, len)) -> slice[:52]
expected_title = "Thisisaverylonglineoftextthatdefinitelyexceedsthemax..."
self.assertEqual(self.parser._generate_title(message), expected_title)
def test_generate_title_text_with_only_html_and_urls(self):
@@ -198,11 +261,6 @@ class TestPostParserGenerateTitle(unittest.TestCase):
message = self._create_mock_message(media=MessageMediaType.PHOTO, web_page=web_page_mock)
self.assertEqual(self.parser._generate_title(message), "📷 Photo") # Media has higher priority
def test_generate_title_webpage_preview_ignored_with_text(self):
web_page_mock = MagicMock()
message = self._create_mock_message(text="Some text", web_page=web_page_mock)
self.assertEqual(self.parser._generate_title(message), "Some text") # Text has higher priority than fallback web page
def test_generate_title_fallback_unknown(self):
message = self._create_mock_message() # No text, no media, no webpage
# Use variable to capture the actual value
@@ -219,9 +277,9 @@ class TestPostParserGenerateTitle(unittest.TestCase):
self.assertEqual(self.parser._generate_title(message), "📊 Poll: Как правильно?")
def test_generate_title_webpage_media_type(self):
# Webpage media type should be ignored for title generation, text should be used
message = self._create_mock_message(media=MessageMediaType.WEB_PAGE, text="Check this out")
self.assertEqual(self.parser._generate_title(message), "Check this out")
# Webpage media type should be ignored for title generation if there's enough text
message = self._create_mock_message(media=MessageMediaType.WEB_PAGE, text="Check this out it is long enough")
self.assertEqual(self.parser._generate_title(message), "Check this out it is long enough")
def test_generate_title_webpage_with_url_text(self):
"""Test case that reproduces real issue with VK video link."""
@@ -319,5 +377,63 @@ class TestPostParserGenerateTitle(unittest.TestCase):
title = self.parser._generate_title(message)
self.assertEqual(title, expected_output, f"Ошибка при обработке '{input_text}': получено '{title}', ожидалось '{expected_output}'")
# --- New tests for text length logic ---
def test_generate_title_media_with_short_caption(self):
"""Media title should be used if caption is short (< 10 chars)."""
message = self._create_mock_message(media=MessageMediaType.PHOTO, caption="Hi <3")
self.assertEqual(self.parser._generate_title(message), "📷 Photo")
def test_generate_title_media_with_long_text(self):
"""Text title should be used if text is long (>= 10 chars), ignoring media."""
message = self._create_mock_message(media=MessageMediaType.VIDEO, text="This is a sufficiently long text.")
self.assertEqual(self.parser._generate_title(message), "This is a sufficiently long text")
def test_generate_title_media_with_long_caption(self):
"""Text title from caption should be used if caption is long (>= 10 chars), ignoring media."""
message = self._create_mock_message(media=MessageMediaType.PHOTO, caption="This is a sufficiently long caption.")
self.assertEqual(self.parser._generate_title(message), "This is a sufficiently long caption")
def test_generate_title_no_media_with_short_text(self):
"""Text title should be used if no media and text is short."""
message = self._create_mock_message(text="Short one")
self.assertEqual(self.parser._generate_title(message), "Short one")
def test_generate_title_no_media_with_long_text(self):
"""Text title should be used if no media and text is long."""
message = self._create_mock_message(text="This is a long text without any media.")
self.assertEqual(self.parser._generate_title(message), "This is a long text without any media")
def test_generate_title_media_with_no_text(self):
"""Media title should be used if media exists and text is None."""
message = self._create_mock_message(media=MessageMediaType.STICKER, text=None)
self.assertEqual(self.parser._generate_title(message), "🎯 Sticker")
def test_generate_title_media_with_empty_text(self):
"""Media title should be used if media exists and text is empty string."""
message = self._create_mock_message(media=MessageMediaType.AUDIO, text="")
self.assertEqual(self.parser._generate_title(message), "🎵 Audio")
def test_generate_title_media_with_whitespace_text(self):
"""Media title should be used if media exists and text is only whitespace."""
message = self._create_mock_message(media=MessageMediaType.VOICE, text=" \\n ")
self.assertEqual(self.parser._generate_title(message), "🎤 Voice")
def test_generate_title_service_message_overrides_long_text(self):
"""Service message title should override even long text."""
mock_service = MagicMock()
mock_service.__str__ = MagicMock(return_value="pyrogram.enums.MessageService.PINNED_MESSAGE")
message = self._create_mock_message(service=mock_service, text="This is a long text but should be ignored.")
self.assertEqual(self.parser._generate_title(message), "📌 Pinned message")
def test_generate_title_webpage_with_short_text(self):
"""Text title should be used if web_page exists and text is short (but not just URL)."""
web_page_mock = MagicMock()
web_page_mock.title = "Web Page Title To Ignore"
message = self._create_mock_message(web_page=web_page_mock, text="Short txt")
self.assertEqual(self.parser._generate_title(message), "Short txt")
# --- End new tests ---
if __name__ == '__main__':
unittest.main()