perf(downloads): implement concurrent download queue with semaphore limiting and retry logic

Add queue-based background download system to improve performance and reliability:
- Introduce DOWNLOAD_SEMAPHORE to limit concurrent downloads to 3
- Add asyncio.Queue (maxsize 100) with dedicated worker for background processing
- Implement safe_get_messages and safe_download_media wrappers with timeout protection (30s and 120s)
- Add retry logic for KeyError auth failures with 5s backoff
- Replace synchronous sequential downloads with asynchronous queued processing
- Prevent event loop blocking by queuing files instead of immediate download
This commit is contained in:
vvzvlad
2026-01-01 14:14:26 +03:00
parent deacb8b9a0
commit 3c2b4ce544
4 changed files with 1013 additions and 119 deletions
+30
View File
@@ -83,6 +83,36 @@ class TelegramClient:
# Emergency termination
os._exit(1)
async def safe_get_messages(self, channel_id, post_id, max_retries=2):
"""Wrapper with retry logic for auth errors"""
for attempt in range(max_retries):
try:
return await asyncio.wait_for(
self.client.get_messages(channel_id, post_id),
timeout=30.0
)
except Exception as e:
if "KeyError" in str(e) and attempt < max_retries - 1:
logger.warning(f"Auth error on attempt {attempt + 1}, retrying in 5s...")
await asyncio.sleep(5)
continue
raise
async def safe_download_media(self, file_id, file_name, max_retries=2):
"""Wrapper with retry logic for download errors"""
for attempt in range(max_retries):
try:
return await asyncio.wait_for(
self.client.download_media(file_id, file_name=file_name),
timeout=120.0
)
except Exception as e:
if "KeyError" in str(e) and attempt < max_retries - 1:
logger.warning(f"Download auth error on attempt {attempt + 1}, retrying...")
await asyncio.sleep(5)
continue
raise
async def stop(self):
if self.client.is_connected:
await self.client.stop()