api_server.py

@@ -47,6 +47,28 @@ from file_io import (DB_PATH, init_db_sync, get_all_media_file_ids_sync,
 # Global python-magic instance for MIME type detection
 magic_mime = magic.Magic(mime=True)
 
+# Media cache on-disk layout. Resolved once at import so every producer/consumer of the
+# cache agrees on the exact same absolute path format instead of re-deriving it inline.
+MEDIA_CACHE_DIR = os.path.abspath(os.path.join("data", "cache"))  # resolved once at import
+
+
+def media_cache_path(channel: str, post_id: int, file_unique_id: str | None = None) -> str:
+    """Single source of truth for the on-disk layout: <root>/<channel>/<post_id>[/<fid>].
+
+    Callers that need only the per-post directory omit ``file_unique_id``; passing it yields
+    the full per-file path. ``channel`` is stringified by the caller where it may be an int.
+    """
+    path = os.path.join(MEDIA_CACHE_DIR, str(channel), str(post_id))
+    if file_unique_id is not None:
+        path = os.path.join(path, file_unique_id)
+    return path
+
+
+# MIME types are immutable per file_unique_id, so a process-lifetime dict in front of
+# SQLite removes a to_thread + connect from every cache-hit response.
+_mime_types: dict[tuple[str, int, str], str] = {}
+_MIME_CACHE_MAX = 50_000  # crude bound; clear-all on overflow is fine at this size
+
 # Define custom exception for zero-size files
 class ZeroSizeFileError(Exception):
     """Custom exception for zero-size files found or downloaded."""
@@ -234,8 +256,7 @@ async def lifespan(_: FastAPI):
     io_executor = ThreadPoolExecutor(max_workers=Config["io_thread_pool_size"], thread_name_prefix="io")
     loop.set_default_executor(io_executor)
 
-    base_cache_dir = os.path.abspath("./data/cache")
-    os.makedirs(base_cache_dir, exist_ok=True) # Create cache directory
+    os.makedirs(MEDIA_CACHE_DIR, exist_ok=True) # Create cache directory
 
     # Initialize SQLite database (creates table if not present)
     await asyncio.to_thread(init_db_sync, DB_PATH)
@@ -448,9 +469,18 @@ async def prepare_file_response(file_path: str, request: Request, delete_after:
     media_type: str | None = None
 
     if media_key is not None:
-        # Try to load the cached MIME type from the database (avoids repeated python-magic I/O)
+        # Fast path: the in-memory MIME cache. MIME is immutable per file_unique_id, so a
+        # hit here serves the response with no to_thread hop and no SQLite connection at all.
         channel_key, post_id_key, file_unique_id_key = media_key
-        media_type = await asyncio.to_thread(get_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key)
+        media_type = _mime_types.get(media_key)
+        if not media_type:
+            # Dict miss — consult the SQLite type cache (still avoids python-magic I/O).
+            media_type = await asyncio.to_thread(get_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key)
+            if media_type:
+                # Populate the dict so the next request skips the to_thread + connect.
+                if len(_mime_types) >= _MIME_CACHE_MAX:
+                    _mime_types.clear()
+                _mime_types[media_key] = media_type
 
     if not media_type:
         # Cache miss or no media_key — detect with python-magic in a thread to avoid blocking the event loop
@@ -460,9 +490,13 @@ async def prepare_file_response(file_path: str, request: Request, delete_after:
             logger.warning(f"Failed to determine MIME type using python-magic: {str(e)}")
             media_type = None
 
-        # Persist the detected MIME type so the next request can skip python-magic
+        # Persist the detected MIME type so the next request can skip python-magic — into
+        # BOTH the SQLite type cache and the in-memory dict.
         if media_type and media_key is not None:
             await asyncio.to_thread(set_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key, media_type)
+            if len(_mime_types) >= _MIME_CACHE_MAX:
+                _mime_types.clear()
+            _mime_types[media_key] = media_type
 
     if not media_type: media_type, _ = mimetypes.guess_type(file_path)  # Fallback to mimetypes if python-magic failed
     if not media_type: media_type = "application/octet-stream"  # Final fallback to octet-stream
@@ -594,11 +628,8 @@ async def download_media_file(channel: Union[str, int], post_id: int, file_uniqu
     """
     import time as _time
     _fn_start = _time.monotonic()
-    base_cache_dir = os.path.abspath("./data/cache")
-    
     # Create nested cache structure
-    channel_dir = os.path.join(base_cache_dir, str(channel))
-    post_dir = os.path.join(channel_dir, str(post_id))
+    post_dir = media_cache_path(str(channel), post_id)
     os.makedirs(post_dir, exist_ok=True)
     
     # Convert numeric channel ID to int if needed
@@ -889,7 +920,7 @@ async def cache_media_files() -> None:
             # Load all media file ID records from SQLite
             media_files = await asyncio.to_thread(get_all_media_file_ids_sync, DB_PATH)
 
-            cache_dir = os.path.abspath("./data/cache")
+            cache_dir = MEDIA_CACHE_DIR
             updated_media_files, files_removed = await asyncio.to_thread(remove_old_cached_files_sync, media_files, cache_dir)
 
             if files_removed > 0:
@@ -923,7 +954,7 @@ def calculate_cache_stats() -> dict[str, Any]:
     Calculate cache statistics including file count, total size in MB, and time difference in days.
     Returns a dictionary with keys: 'cache_files_count', 'cache_total_size_mb', 'cache_time_diff_days', 'channels'.
     """
-    base_cache_dir = os.path.abspath("./data/cache")
+    base_cache_dir = MEDIA_CACHE_DIR
     cache_files_count = 0
     cache_total_size_bytes = 0
     channels_stats = {}
@@ -1244,10 +1275,7 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, request: Re
             import time as _time
 
             # Pre-semaphore cache check: serve already-cached files without acquiring the semaphore
-            base_cache_dir = os.path.abspath("./data/cache")
-            channel_dir = os.path.join(base_cache_dir, str(channel))
-            post_dir = os.path.join(channel_dir, str(post_id))
-            cache_path = os.path.join(post_dir, file_unique_id)
+            cache_path = media_cache_path(str(channel), post_id, file_unique_id)
             if os.path.exists(cache_path) and os.path.getsize(cache_path) > 0:
                 # File is already in cache — skip semaphore and serve directly
                 logger.info(f"pre_semaphore_cache_hit: {channel}/{post_id}/{file_unique_id}")

tests/conftest.py

@@ -22,3 +22,20 @@ sys.modules['config'] = _mock_config
 # without this pin RSS <pubDate> values drift between machines (this sandbox is MSK).
 os.environ['TZ'] = 'UTC'
 time.tzset()
+
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def _reset_media_mime_cache():
+    """Clear the process-lifetime MIME dict before each test (issue #26).
+
+    ``api_server._mime_types`` persists for the whole process, so an entry populated by one
+    test would otherwise leak into another and mask a get/magic call the next test asserts on.
+    """
+    try:
+        import api_server
+        api_server._mime_types.clear()
+    except Exception:
+        pass
+    yield

tests/test_stage2_static.py

@@ -111,6 +111,7 @@ async def test_download_atomic_race_loser_keeps_existing_final(monkeypatch, tmp_
 # --------------------------------------------------------------------------- #
 async def test_concurrent_large_video_no_partial_served(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)  # download_media_file writes under ./data/cache
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
 
     msg = SimpleNamespace(
         media=MessageMediaType.VIDEO,

tests/test_stage5_sqlite.py

@@ -46,6 +46,7 @@ def _clean_accumulator():
 @pytest.mark.asyncio
 async def test_cache_hit_records_accumulator_no_sqlite(tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     channel, post_id, fid = "testchan", 5, "fidHIT"
     cache_dir = tmp_path / "data" / "cache" / channel / str(post_id)
     cache_dir.mkdir(parents=True)
@@ -77,6 +78,7 @@ async def test_cache_hit_records_accumulator_no_sqlite(tmp_path, monkeypatch):
 @pytest.mark.asyncio
 async def test_get_media_pre_semaphore_cache_hit_no_sqlite(tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     channel, post_id, fid = "gmchan", 11, "fidGM"
     cache_dir = tmp_path / "data" / "cache" / channel / str(post_id)
     cache_dir.mkdir(parents=True)
@@ -108,6 +110,7 @@ async def test_get_media_pre_semaphore_cache_hit_no_sqlite(tmp_path, monkeypatch
 @pytest.mark.asyncio
 async def test_cache_hit_keys_channel_as_str(tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     channel_int, post_id, fid = 12345, 7, "fidINT"
     cache_dir = tmp_path / "data" / "cache" / str(channel_int) / str(post_id)
     cache_dir.mkdir(parents=True)

tests/test_stage7_integration.py

@@ -73,6 +73,7 @@ def _seed_cache(tmp_path, channel, post_id, fid, body=BODY):
 # --------------------------------------------------------------------------- #
 def test_media_route_range_prefix_0_99(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     _seed_cache(tmp_path, "chan", 3, "fidR")
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     # Keep the MIME path DB-free; the FileResponse/Range machinery is what we exercise.
@@ -90,6 +91,7 @@ def test_media_route_range_prefix_0_99(monkeypatch, tmp_path):
 
 def test_media_route_range_suffix_last_100(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     _seed_cache(tmp_path, "chan", 3, "fidS")
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     monkeypatch.setattr(api_server, "get_mime_type_sync", lambda *a, **k: None)
@@ -105,6 +107,7 @@ def test_media_route_range_suffix_last_100(monkeypatch, tmp_path):
 
 def test_media_route_range_unsatisfiable_416(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     _seed_cache(tmp_path, "chan", 3, "fidU")
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     monkeypatch.setattr(api_server, "get_mime_type_sync", lambda *a, **k: None)
@@ -215,6 +218,7 @@ async def test_ping_reports_degraded_promptly_while_slow_op_pending(monkeypatch)
 # --------------------------------------------------------------------------- #
 async def test_get_media_concurrent_shares_one_download_and_drains_registry(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     api_server._inflight.clear()
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     # Serve step is not under test here; keep it to a sentinel so we assert on dedup + registry.
@@ -246,6 +250,7 @@ async def test_get_media_concurrent_shares_one_download_and_drains_registry(monk
 
 async def test_get_media_request_cancel_does_not_stick_registry_or_hang_sibling(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     api_server._inflight.clear()
     monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True)
     sentinel = object()
@@ -298,6 +303,7 @@ async def test_get_media_request_cancel_does_not_stick_registry_or_hang_sibling(
 # --------------------------------------------------------------------------- #
 async def test_media_cache_hit_flush_updates_str_keyed_db_row(monkeypatch, tmp_path):
     monkeypatch.chdir(tmp_path)
+    monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache"))
     api_server._access_updates = {}
 
     channel, post_id, fid = "12345", 7, "fidINT"  # int-ish channel, passed as the route str

tests/test_stage_e_media_hotpath.py

@@ -0,0 +1,204 @@
+# flake8: noqa
+# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring
+# pylint: disable=redefined-outer-name, line-too-long
+"""
+Issue #26 package E — media-cache path helper + in-memory MIME cache on the hot /media path.
+
+Covers:
+- media_cache_path builds EXACTLY the pre-existing on-disk layout (path-format regression).
+- The in-memory _mime_types dict short-circuits the SQLite MIME read on a repeat hit
+  (get_mime_type_sync not called a second time; no new SQLite connection opened).
+- _mime_types overflow (>= _MIME_CACHE_MAX) clears the dict without raising and repopulates.
+"""
+import os
+
+import pytest
+
+from fastapi import FastAPI, Request
+from fastapi.testclient import TestClient
+
+import file_io
+import api_server
+
+
+BODY = bytes(range(256)) * 8  # 2048 deterministic bytes
+
+
+def _make_client(file_path, media_key=None):
+    """Drive prepare_file_response through real ASGI (as the stage-3 tests do)."""
+    app = FastAPI()
+
+    @app.get("/f")
+    async def _serve(request: Request):
+        return await api_server.prepare_file_response(file_path, request=request, media_key=media_key)
+
+    return TestClient(app)
+
+
+@pytest.fixture
+def sample_file(tmp_path):
+    fp = tmp_path / "myfile.bin"
+    fp.write_bytes(BODY)
+    return str(fp)
+
+
+# --------------------------------------------------------------------------- #
+# Task 15 — media_cache_path path-format regression.
+# --------------------------------------------------------------------------- #
+def test_media_cache_path_layout():
+    root = api_server.MEDIA_CACHE_DIR
+    # The constant must resolve to the exact absolute string every legacy site built.
+    assert root == os.path.abspath("./data/cache")
+
+    # Without file_unique_id -> the per-post directory <root>/<channel>/<post_id>.
+    assert api_server.media_cache_path("mychan", 42) == os.path.join(root, "mychan", "42")
+
+    # With file_unique_id -> the full per-file path <root>/<channel>/<post_id>/<fid>.
+    assert api_server.media_cache_path("mychan", 42, "fidABC") == os.path.join(root, "mychan", "42", "fidABC")
+
+    # Faithful drop-in: identical to the old manual assembly (incl. int channel stringified).
+    assert api_server.media_cache_path(-100123, 7, "fidABC") == os.path.join(
+        os.path.abspath("./data/cache"), "-100123", "7", "fidABC"
+    )
+
+
+# --------------------------------------------------------------------------- #
+# Task 16 — in-memory MIME cache short-circuits the SQLite read on a repeat hit.
+# --------------------------------------------------------------------------- #
+def test_second_request_served_from_dict_not_sqlite(sample_file, monkeypatch):
+    calls = {"get": 0, "magic": 0}
+
+    def fake_get(db, ch, pid, fid):
+        calls["get"] += 1
+        return "video/mp4"  # SQLite HIT on the first (dict-miss) request
+
+    def fake_magic(_path):
+        calls["magic"] += 1
+        return "application/octet-stream"
+
+    monkeypatch.setattr(api_server, "get_mime_type_sync", fake_get)
+    monkeypatch.setattr(api_server.magic_mime, "from_file", fake_magic)
+
+    key = ("chanE", 100, "fidE")
+    c = _make_client(sample_file, media_key=key)
+
+    r1 = c.get("/f")
+    r2 = c.get("/f")
+
+    assert r1.status_code == 200 and r2.status_code == 200
+    assert r1.headers["content-type"].startswith("video/mp4")
+    assert r2.headers["content-type"].startswith("video/mp4")
+    # First request populated the dict from SQLite; the second is served from the dict.
+    assert calls["get"] == 1
+    assert calls["magic"] == 0
+    assert api_server._mime_types[key] == "video/mp4"
+
+
+def test_repeat_hit_opens_no_new_db_connection(tmp_path, sample_file, monkeypatch):
+    """Acceptance recipe: monkeypatch file_io._open_db with a counter; two consecutive
+    requests for the same file — the counter must NOT increase on the second."""
+    db_path = str(tmp_path / "media.db")
+    file_io.init_db_sync(db_path)
+    # Seed a row whose MIME is already stored, so the first request reads it from SQLite
+    # (no python-magic, no set-write) and caches it in the dict.
+    file_io.upsert_media_file_id_sync(db_path, "chanDB", 55, "fidDB", 0.0)
+    file_io.set_mime_type_sync(db_path, "chanDB", 55, "fidDB", "image/png")
+
+    monkeypatch.setattr(api_server, "DB_PATH", db_path)
+
+    count = {"n": 0}
+    orig_open = file_io._open_db
+
+    def counting_open(p):
+        count["n"] += 1
+        return orig_open(p)
+
+    monkeypatch.setattr(file_io, "_open_db", counting_open)
+
+    c = _make_client(sample_file, media_key=("chanDB", 55, "fidDB"))
+
+    r1 = c.get("/f")
+    after_first = count["n"]
+    r2 = c.get("/f")
+    after_second = count["n"]
+
+    assert r1.status_code == 200 and r2.status_code == 200
+    assert r1.headers["content-type"].startswith("image/png")
+    assert r2.headers["content-type"].startswith("image/png")
+    assert after_first >= 1  # the first (dict-miss) hit did open a connection for the read
+    # The repeat hit opened NO new SQLite connection.
+    assert after_second == after_first
+
+
+# --------------------------------------------------------------------------- #
+# Task 16 — overflow clears the dict without raising, then repopulates.
+# --------------------------------------------------------------------------- #
+def test_mime_cache_overflow_clears_and_repopulates(sample_file, monkeypatch):
+    def fake_get(db, ch, pid, fid):
+        return "text/plain"
+
+    monkeypatch.setattr(api_server, "get_mime_type_sync", fake_get)
+    monkeypatch.setattr(api_server.magic_mime, "from_file", lambda _p: "text/plain")
+
+    # Fill the dict up to the bound with dummy entries.
+    api_server._mime_types.clear()
+    for i in range(api_server._MIME_CACHE_MAX):
+        api_server._mime_types[("dummy", i, "f")] = "x/y"
+    assert len(api_server._mime_types) == api_server._MIME_CACHE_MAX
+
+    key = ("chanOverflow", 1, "fidOverflow")
+    c = _make_client(sample_file, media_key=key)
+
+    r = c.get("/f")  # must clear-all on overflow, then insert the new key — no exception
+    assert r.status_code == 200
+    assert r.headers["content-type"].startswith("text/plain")
+    # Cleared then repopulated with just the one fresh key.
+    assert len(api_server._mime_types) == 1
+    assert api_server._mime_types[key] == "text/plain"
+
+    # The next request is now a clean dict hit.
+    r2 = c.get("/f")
+    assert r2.status_code == 200
+
+
+# --------------------------------------------------------------------------- #
+# Task 16 — magic cold-start path: dict-miss + SQLite-miss -> python-magic ->
+# write-through to BOTH the SQLite type cache AND the in-memory dict.
+# The other stage-E tests all make SQLite HIT (get_mime_type_sync returns a value),
+# so the magic branch's dict-write (api_server.py:499) is never exercised there.
+# --------------------------------------------------------------------------- #
+def test_magic_cold_start_writes_through_to_sqlite_and_dict(sample_file, monkeypatch):
+    key = ("chanMagic", 7, "fidMagic")
+    # Dict must be empty for this key (dict-miss).
+    api_server._mime_types.pop(key, None)
+
+    calls = {"get": 0, "set": 0, "magic": 0}
+
+    def fake_get(db, ch, pid, fid):
+        calls["get"] += 1
+        return None  # SQLite MISS -> falls through to python-magic
+
+    def fake_set(db, ch, pid, fid, mime):
+        calls["set"] += 1
+        assert (ch, pid, fid) == key
+        assert mime == "image/jpeg"
+
+    def fake_magic(_path):
+        calls["magic"] += 1
+        return "image/jpeg"
+
+    monkeypatch.setattr(api_server, "get_mime_type_sync", fake_get)
+    monkeypatch.setattr(api_server, "set_mime_type_sync", fake_set)
+    monkeypatch.setattr(api_server.magic_mime, "from_file", fake_magic)
+
+    c = _make_client(sample_file, media_key=key)
+    r = c.get("/f")
+
+    assert r.status_code == 200
+    assert r.headers["content-type"].startswith("image/jpeg")
+    # Cold start: dict-miss -> SQLite-miss -> python-magic detected the type.
+    assert calls["get"] == 1
+    assert calls["magic"] == 1
+    # Write-through to BOTH caches.
+    assert calls["set"] == 1                             # SQLite write-through
+    assert api_server._mime_types[key] == "image/jpeg"  # dict write-through