diff --git a/api_server.py b/api_server.py index 95fb23c..e76e929 100644 --- a/api_server.py +++ b/api_server.py @@ -47,6 +47,28 @@ from file_io import (DB_PATH, init_db_sync, get_all_media_file_ids_sync, # Global python-magic instance for MIME type detection magic_mime = magic.Magic(mime=True) +# Media cache on-disk layout. Resolved once at import so every producer/consumer of the +# cache agrees on the exact same absolute path format instead of re-deriving it inline. +MEDIA_CACHE_DIR = os.path.abspath(os.path.join("data", "cache")) # resolved once at import + + +def media_cache_path(channel: str, post_id: int, file_unique_id: str | None = None) -> str: + """Single source of truth for the on-disk layout: //[/]. + + Callers that need only the per-post directory omit ``file_unique_id``; passing it yields + the full per-file path. ``channel`` is stringified by the caller where it may be an int. + """ + path = os.path.join(MEDIA_CACHE_DIR, str(channel), str(post_id)) + if file_unique_id is not None: + path = os.path.join(path, file_unique_id) + return path + + +# MIME types are immutable per file_unique_id, so a process-lifetime dict in front of +# SQLite removes a to_thread + connect from every cache-hit response. +_mime_types: dict[tuple[str, int, str], str] = {} +_MIME_CACHE_MAX = 50_000 # crude bound; clear-all on overflow is fine at this size + # Define custom exception for zero-size files class ZeroSizeFileError(Exception): """Custom exception for zero-size files found or downloaded.""" @@ -234,8 +256,7 @@ async def lifespan(_: FastAPI): io_executor = ThreadPoolExecutor(max_workers=Config["io_thread_pool_size"], thread_name_prefix="io") loop.set_default_executor(io_executor) - base_cache_dir = os.path.abspath("./data/cache") - os.makedirs(base_cache_dir, exist_ok=True) # Create cache directory + os.makedirs(MEDIA_CACHE_DIR, exist_ok=True) # Create cache directory # Initialize SQLite database (creates table if not present) await asyncio.to_thread(init_db_sync, DB_PATH) @@ -448,9 +469,18 @@ async def prepare_file_response(file_path: str, request: Request, delete_after: media_type: str | None = None if media_key is not None: - # Try to load the cached MIME type from the database (avoids repeated python-magic I/O) + # Fast path: the in-memory MIME cache. MIME is immutable per file_unique_id, so a + # hit here serves the response with no to_thread hop and no SQLite connection at all. channel_key, post_id_key, file_unique_id_key = media_key - media_type = await asyncio.to_thread(get_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key) + media_type = _mime_types.get(media_key) + if not media_type: + # Dict miss — consult the SQLite type cache (still avoids python-magic I/O). + media_type = await asyncio.to_thread(get_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key) + if media_type: + # Populate the dict so the next request skips the to_thread + connect. + if len(_mime_types) >= _MIME_CACHE_MAX: + _mime_types.clear() + _mime_types[media_key] = media_type if not media_type: # Cache miss or no media_key — detect with python-magic in a thread to avoid blocking the event loop @@ -460,9 +490,13 @@ async def prepare_file_response(file_path: str, request: Request, delete_after: logger.warning(f"Failed to determine MIME type using python-magic: {str(e)}") media_type = None - # Persist the detected MIME type so the next request can skip python-magic + # Persist the detected MIME type so the next request can skip python-magic — into + # BOTH the SQLite type cache and the in-memory dict. if media_type and media_key is not None: await asyncio.to_thread(set_mime_type_sync, DB_PATH, channel_key, post_id_key, file_unique_id_key, media_type) + if len(_mime_types) >= _MIME_CACHE_MAX: + _mime_types.clear() + _mime_types[media_key] = media_type if not media_type: media_type, _ = mimetypes.guess_type(file_path) # Fallback to mimetypes if python-magic failed if not media_type: media_type = "application/octet-stream" # Final fallback to octet-stream @@ -594,11 +628,8 @@ async def download_media_file(channel: Union[str, int], post_id: int, file_uniqu """ import time as _time _fn_start = _time.monotonic() - base_cache_dir = os.path.abspath("./data/cache") - # Create nested cache structure - channel_dir = os.path.join(base_cache_dir, str(channel)) - post_dir = os.path.join(channel_dir, str(post_id)) + post_dir = media_cache_path(str(channel), post_id) os.makedirs(post_dir, exist_ok=True) # Convert numeric channel ID to int if needed @@ -889,7 +920,7 @@ async def cache_media_files() -> None: # Load all media file ID records from SQLite media_files = await asyncio.to_thread(get_all_media_file_ids_sync, DB_PATH) - cache_dir = os.path.abspath("./data/cache") + cache_dir = MEDIA_CACHE_DIR updated_media_files, files_removed = await asyncio.to_thread(remove_old_cached_files_sync, media_files, cache_dir) if files_removed > 0: @@ -923,7 +954,7 @@ def calculate_cache_stats() -> dict[str, Any]: Calculate cache statistics including file count, total size in MB, and time difference in days. Returns a dictionary with keys: 'cache_files_count', 'cache_total_size_mb', 'cache_time_diff_days', 'channels'. """ - base_cache_dir = os.path.abspath("./data/cache") + base_cache_dir = MEDIA_CACHE_DIR cache_files_count = 0 cache_total_size_bytes = 0 channels_stats = {} @@ -1244,10 +1275,7 @@ async def get_media(channel: str, post_id: int, file_unique_id: str, request: Re import time as _time # Pre-semaphore cache check: serve already-cached files without acquiring the semaphore - base_cache_dir = os.path.abspath("./data/cache") - channel_dir = os.path.join(base_cache_dir, str(channel)) - post_dir = os.path.join(channel_dir, str(post_id)) - cache_path = os.path.join(post_dir, file_unique_id) + cache_path = media_cache_path(str(channel), post_id, file_unique_id) if os.path.exists(cache_path) and os.path.getsize(cache_path) > 0: # File is already in cache — skip semaphore and serve directly logger.info(f"pre_semaphore_cache_hit: {channel}/{post_id}/{file_unique_id}") diff --git a/tests/conftest.py b/tests/conftest.py index b362efc..ebcec85 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -22,3 +22,20 @@ sys.modules['config'] = _mock_config # without this pin RSS values drift between machines (this sandbox is MSK). os.environ['TZ'] = 'UTC' time.tzset() + +import pytest + + +@pytest.fixture(autouse=True) +def _reset_media_mime_cache(): + """Clear the process-lifetime MIME dict before each test (issue #26). + + ``api_server._mime_types`` persists for the whole process, so an entry populated by one + test would otherwise leak into another and mask a get/magic call the next test asserts on. + """ + try: + import api_server + api_server._mime_types.clear() + except Exception: + pass + yield diff --git a/tests/test_stage2_static.py b/tests/test_stage2_static.py index 00ce557..0ae1edd 100644 --- a/tests/test_stage2_static.py +++ b/tests/test_stage2_static.py @@ -111,6 +111,7 @@ async def test_download_atomic_race_loser_keeps_existing_final(monkeypatch, tmp_ # --------------------------------------------------------------------------- # async def test_concurrent_large_video_no_partial_served(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) # download_media_file writes under ./data/cache + monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache")) msg = SimpleNamespace( media=MessageMediaType.VIDEO, diff --git a/tests/test_stage5_sqlite.py b/tests/test_stage5_sqlite.py index 5e9d8e5..7a4e983 100644 --- a/tests/test_stage5_sqlite.py +++ b/tests/test_stage5_sqlite.py @@ -46,6 +46,7 @@ def _clean_accumulator(): @pytest.mark.asyncio async def test_cache_hit_records_accumulator_no_sqlite(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) + monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache")) channel, post_id, fid = "testchan", 5, "fidHIT" cache_dir = tmp_path / "data" / "cache" / channel / str(post_id) cache_dir.mkdir(parents=True) @@ -77,6 +78,7 @@ async def test_cache_hit_records_accumulator_no_sqlite(tmp_path, monkeypatch): @pytest.mark.asyncio async def test_get_media_pre_semaphore_cache_hit_no_sqlite(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) + monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache")) channel, post_id, fid = "gmchan", 11, "fidGM" cache_dir = tmp_path / "data" / "cache" / channel / str(post_id) cache_dir.mkdir(parents=True) @@ -108,6 +110,7 @@ async def test_get_media_pre_semaphore_cache_hit_no_sqlite(tmp_path, monkeypatch @pytest.mark.asyncio async def test_cache_hit_keys_channel_as_str(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) + monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache")) channel_int, post_id, fid = 12345, 7, "fidINT" cache_dir = tmp_path / "data" / "cache" / str(channel_int) / str(post_id) cache_dir.mkdir(parents=True) diff --git a/tests/test_stage7_integration.py b/tests/test_stage7_integration.py index 47db84f..84befb7 100644 --- a/tests/test_stage7_integration.py +++ b/tests/test_stage7_integration.py @@ -73,6 +73,7 @@ def _seed_cache(tmp_path, channel, post_id, fid, body=BODY): # --------------------------------------------------------------------------- # def test_media_route_range_prefix_0_99(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) + monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache")) _seed_cache(tmp_path, "chan", 3, "fidR") monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True) # Keep the MIME path DB-free; the FileResponse/Range machinery is what we exercise. @@ -90,6 +91,7 @@ def test_media_route_range_prefix_0_99(monkeypatch, tmp_path): def test_media_route_range_suffix_last_100(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) + monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache")) _seed_cache(tmp_path, "chan", 3, "fidS") monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True) monkeypatch.setattr(api_server, "get_mime_type_sync", lambda *a, **k: None) @@ -105,6 +107,7 @@ def test_media_route_range_suffix_last_100(monkeypatch, tmp_path): def test_media_route_range_unsatisfiable_416(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) + monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache")) _seed_cache(tmp_path, "chan", 3, "fidU") monkeypatch.setattr(api_server, "verify_media_digest", lambda url, digest: True) monkeypatch.setattr(api_server, "get_mime_type_sync", lambda *a, **k: None) @@ -298,6 +301,7 @@ async def test_get_media_request_cancel_does_not_stick_registry_or_hang_sibling( # --------------------------------------------------------------------------- # async def test_media_cache_hit_flush_updates_str_keyed_db_row(monkeypatch, tmp_path): monkeypatch.chdir(tmp_path) + monkeypatch.setattr(api_server, "MEDIA_CACHE_DIR", str(tmp_path / "data" / "cache")) api_server._access_updates = {} channel, post_id, fid = "12345", 7, "fidINT" # int-ish channel, passed as the route str diff --git a/tests/test_stage_e_media_hotpath.py b/tests/test_stage_e_media_hotpath.py new file mode 100644 index 0000000..83edc9f --- /dev/null +++ b/tests/test_stage_e_media_hotpath.py @@ -0,0 +1,161 @@ +# flake8: noqa +# pylint: disable=protected-access, missing-function-docstring, missing-class-docstring +# pylint: disable=redefined-outer-name, line-too-long +""" +Issue #26 package E — media-cache path helper + in-memory MIME cache on the hot /media path. + +Covers: +- media_cache_path builds EXACTLY the pre-existing on-disk layout (path-format regression). +- The in-memory _mime_types dict short-circuits the SQLite MIME read on a repeat hit + (get_mime_type_sync not called a second time; no new SQLite connection opened). +- _mime_types overflow (>= _MIME_CACHE_MAX) clears the dict without raising and repopulates. +""" +import os + +import pytest + +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient + +import file_io +import api_server + + +BODY = bytes(range(256)) * 8 # 2048 deterministic bytes + + +def _make_client(file_path, media_key=None): + """Drive prepare_file_response through real ASGI (as the stage-3 tests do).""" + app = FastAPI() + + @app.get("/f") + async def _serve(request: Request): + return await api_server.prepare_file_response(file_path, request=request, media_key=media_key) + + return TestClient(app) + + +@pytest.fixture +def sample_file(tmp_path): + fp = tmp_path / "myfile.bin" + fp.write_bytes(BODY) + return str(fp) + + +# --------------------------------------------------------------------------- # +# Task 15 — media_cache_path path-format regression. +# --------------------------------------------------------------------------- # +def test_media_cache_path_layout(): + root = api_server.MEDIA_CACHE_DIR + # The constant must resolve to the exact absolute string every legacy site built. + assert root == os.path.abspath("./data/cache") + + # Without file_unique_id -> the per-post directory //. + assert api_server.media_cache_path("mychan", 42) == os.path.join(root, "mychan", "42") + + # With file_unique_id -> the full per-file path ///. + assert api_server.media_cache_path("mychan", 42, "fidABC") == os.path.join(root, "mychan", "42", "fidABC") + + # Faithful drop-in: identical to the old manual assembly (incl. int channel stringified). + assert api_server.media_cache_path(-100123, 7, "fidABC") == os.path.join( + os.path.abspath("./data/cache"), "-100123", "7", "fidABC" + ) + + +# --------------------------------------------------------------------------- # +# Task 16 — in-memory MIME cache short-circuits the SQLite read on a repeat hit. +# --------------------------------------------------------------------------- # +def test_second_request_served_from_dict_not_sqlite(sample_file, monkeypatch): + calls = {"get": 0, "magic": 0} + + def fake_get(db, ch, pid, fid): + calls["get"] += 1 + return "video/mp4" # SQLite HIT on the first (dict-miss) request + + def fake_magic(_path): + calls["magic"] += 1 + return "application/octet-stream" + + monkeypatch.setattr(api_server, "get_mime_type_sync", fake_get) + monkeypatch.setattr(api_server.magic_mime, "from_file", fake_magic) + + key = ("chanE", 100, "fidE") + c = _make_client(sample_file, media_key=key) + + r1 = c.get("/f") + r2 = c.get("/f") + + assert r1.status_code == 200 and r2.status_code == 200 + assert r1.headers["content-type"].startswith("video/mp4") + assert r2.headers["content-type"].startswith("video/mp4") + # First request populated the dict from SQLite; the second is served from the dict. + assert calls["get"] == 1 + assert calls["magic"] == 0 + assert api_server._mime_types[key] == "video/mp4" + + +def test_repeat_hit_opens_no_new_db_connection(tmp_path, sample_file, monkeypatch): + """Acceptance recipe: monkeypatch file_io._open_db with a counter; two consecutive + requests for the same file — the counter must NOT increase on the second.""" + db_path = str(tmp_path / "media.db") + file_io.init_db_sync(db_path) + # Seed a row whose MIME is already stored, so the first request reads it from SQLite + # (no python-magic, no set-write) and caches it in the dict. + file_io.upsert_media_file_id_sync(db_path, "chanDB", 55, "fidDB", 0.0) + file_io.set_mime_type_sync(db_path, "chanDB", 55, "fidDB", "image/png") + + monkeypatch.setattr(api_server, "DB_PATH", db_path) + + count = {"n": 0} + orig_open = file_io._open_db + + def counting_open(p): + count["n"] += 1 + return orig_open(p) + + monkeypatch.setattr(file_io, "_open_db", counting_open) + + c = _make_client(sample_file, media_key=("chanDB", 55, "fidDB")) + + r1 = c.get("/f") + after_first = count["n"] + r2 = c.get("/f") + after_second = count["n"] + + assert r1.status_code == 200 and r2.status_code == 200 + assert r1.headers["content-type"].startswith("image/png") + assert r2.headers["content-type"].startswith("image/png") + assert after_first >= 1 # the first (dict-miss) hit did open a connection for the read + # The repeat hit opened NO new SQLite connection. + assert after_second == after_first + + +# --------------------------------------------------------------------------- # +# Task 16 — overflow clears the dict without raising, then repopulates. +# --------------------------------------------------------------------------- # +def test_mime_cache_overflow_clears_and_repopulates(sample_file, monkeypatch): + def fake_get(db, ch, pid, fid): + return "text/plain" + + monkeypatch.setattr(api_server, "get_mime_type_sync", fake_get) + monkeypatch.setattr(api_server.magic_mime, "from_file", lambda _p: "text/plain") + + # Fill the dict up to the bound with dummy entries. + api_server._mime_types.clear() + for i in range(api_server._MIME_CACHE_MAX): + api_server._mime_types[("dummy", i, "f")] = "x/y" + assert len(api_server._mime_types) == api_server._MIME_CACHE_MAX + + key = ("chanOverflow", 1, "fidOverflow") + c = _make_client(sample_file, media_key=key) + + r = c.get("/f") # must clear-all on overflow, then insert the new key — no exception + assert r.status_code == 200 + assert r.headers["content-type"].startswith("text/plain") + # Cleared then repopulated with just the one fresh key. + assert len(api_server._mime_types) == 1 + assert api_server._mime_types[key] == "text/plain" + + # The next request is now a clean dict hit. + r2 = c.get("/f") + assert r2.status_code == 200