fix(cache): канонизация ключа канала (Durov/durov/@name -> один ключ) + one-shot миграция

Один канал жил под ключами Durov/durov/@name в трёх слоях (tgcache-файлы,
data/cache/ каталоги, SQLite-строки) -> дубли кеша, двойные походы в Telegram,
осиротевшие деревья. ID<->username НЕ унифицируем (сознательно).

- channel_key.py: canonical_channel_key (strip @, -100... числовые verbatim,
  иначе lowercase; безопасно и для API-вызовов).
- tg_cache._cache_file_path прогоняет ключ через canonical.
- post_parser.get_channel_username -> lowercase (обе ветки; канонизирует
  SQLite/media-URL/t.me). Только эти 2 строки.
- api_server.get_media: fs_channel = canonical_channel_key(channel) ПОСЛЕ проверки
  дайджеста (дайджест — по ИСХОДНОМУ url, иначе ломаются выданные ссылки); fs_channel
  во всех путях/ключах/download ниже.
- migrate_channel_keys_sync в lifespan (после init_db_sync, до client.start, через
  asyncio.to_thread): канал целиком FS-потом-SQL. Обязательный os.path.samefile
  guard ДО merge/rmtree — на case-insensitive FS Durov/durov один inode, иначе
  снесли бы кеш. FS-сбой -> SQL пропуск (строки старорегистровые, дерево видно
  свиперу, нет вечных orphan). SQL-merge PK-safe (twin -> merge added=max/mime
  non-NULL + DELETE, иначе plain UPDATE). Идемпотентно.

closes #24

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-10 03:40:51 +03:00
parent d5a38c3f13
commit 11bfd2b3ee
6 changed files with 498 additions and 17 deletions
+23 -13
View File
@@ -44,6 +44,8 @@ from file_io import (DB_PATH, init_db_sync, get_all_media_file_ids_sync,
remove_media_file_ids_sync,
get_mime_type_sync, set_mime_type_sync)
from tg_cache import cleanup_legacy_cache_files, sweep_tgcache
from channel_key import canonical_channel_key
from migrate_channel_keys import migrate_channel_keys_sync
# Global python-magic instance for MIME type detection
magic_mime = magic.Magic(mime=True)
@@ -270,6 +272,12 @@ async def lifespan(_: FastAPI):
# Initialize SQLite database (creates table if not present)
await asyncio.to_thread(init_db_sync, DB_PATH)
# One-shot migration of existing cache dirs + DB rows to the canonical channel key
# (case-insensitive username collapse). Runs AFTER init_db_sync and BEFORE client.start()
# and the background tasks. Wrapped in to_thread because it is a blocking FS-rename +
# SQLite routine that would otherwise stall the event loop. Idempotent: a re-run is a no-op.
await asyncio.to_thread(migrate_channel_keys_sync, DB_PATH, MEDIA_CACHE_DIR)
# One-shot startup maintenance of the history/chatinfo cache: drop legacy pickle files
# (which are now always a miss), age-sweep stale tgcache entries, and remove the
# pre-SQLite data/media_file_ids.json legacy dump (no longer read by any code).
@@ -1311,30 +1319,32 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, request: Re
#else:
# logger.info(f"Valid digest for media {url}: {digest}")
# Convert numeric channel ID to int if needed
channel_id: Union[str, int] = channel
if isinstance(channel, str) and channel.startswith('-100'):
channel_id = int(channel)
# Canonical filesystem/DB/API key. Computed AFTER the digest check (which must run
# against the ORIGINAL url string) so 'Durov', 'durov' and '@durov' collapse to one
# on-disk tree, one _access_updates/media_key identity and one download identity. The
# canonical form is API-safe: usernames are case-insensitive on Telegram's side and
# numeric '-100...' ids are preserved verbatim (download_media_file re-ints them).
fs_channel = canonical_channel_key(channel)
try: # Wrap the download and prepare call
import time as _time
# Pre-semaphore cache check: serve already-cached files without acquiring the semaphore
cache_path = media_cache_path(str(channel), post_id, file_unique_id)
cache_path = media_cache_path(fs_channel, post_id, file_unique_id)
if os.path.exists(cache_path) and os.path.getsize(cache_path) > 0:
# File is already in cache — skip semaphore and serve directly
logger.info(f"pre_semaphore_cache_hit: {channel}/{post_id}/{file_unique_id}")
logger.info(f"pre_semaphore_cache_hit: {fs_channel}/{post_id}/{file_unique_id}")
# Record the access time into the accumulator instead of firing a per-hit
# SQLite write. Key channel as str(channel) — see _access_updates.
_access_updates[(str(channel), post_id, file_unique_id)] = datetime.now().timestamp()
# SQLite write. Key channel as the canonical fs_channel — see _access_updates.
_access_updates[(fs_channel, post_id, file_unique_id)] = datetime.now().timestamp()
return await prepare_file_response(cache_path, request=request,
media_key=(str(channel), post_id, file_unique_id))
media_key=(fs_channel, post_id, file_unique_id))
# A file that recently kept failing is in backoff: fast-reject instead of
# occupying a scarce download slot (and a Pyrogram transmission permit) on a
# request that will very likely hang again. Cached files already returned above,
# so this only guards the live-download path.
backoff_remaining = _download_backoff_remaining((str(channel), post_id, file_unique_id))
backoff_remaining = _download_backoff_remaining((fs_channel, post_id, file_unique_id))
if backoff_remaining > 0:
logger.info(f"media_backoff_skip: {channel}/{post_id}/{file_unique_id} in backoff {backoff_remaining:.0f}s")
return Response(status_code=503, content="Media temporarily unavailable, retry later",
@@ -1369,7 +1379,7 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, request: Re
if _sem_wait > 0.5:
logger.warning(f"diag_semaphore_wait: {channel}/{post_id}/{file_unique_id} waited {_sem_wait:.3f}s for HTTP_DOWNLOAD_SEMAPHORE")
_dl_start = _time.monotonic()
file_path, delete_after = await _download_deduped(channel_id, post_id, file_unique_id)
file_path, delete_after = await _download_deduped(fs_channel, post_id, file_unique_id)
_dl_elapsed = _time.monotonic() - _dl_start
logger.info(f"diag_download_timing: {channel}/{post_id}/{file_unique_id} download_media_file took {_dl_elapsed:.3f}s (semaphore_wait={_sem_wait:.3f}s)")
finally:
@@ -1378,7 +1388,7 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, request: Re
raise HTTPException(status_code=404, detail="File not found")
if file_path:
return await prepare_file_response(file_path, request=request, delete_after=delete_after,
media_key=(str(channel), post_id, file_unique_id))
media_key=(fs_channel, post_id, file_unique_id))
except ZeroSizeFileError as e: # Catch zero-size file errors
logger.warning(f"zero_size_file_encountered: {str(e)}. Instructing client to retry.")
return Response(