fix: move persistent queue state writes off the event loop (#980)

PersistentQueue.put/delete wrote the whole queue inline: serialize, write
a temp file, fsync it, rename, then fsync the directory. All of that ran
synchronously inside async callers, so on a slow or contended filesystem
a single queue mutation stalled every other request for as long as the
two fsyncs took. Adds and completions are exactly when it fires, which
matches the reported "hiccups happen when something is pushing into the
queue".

put/delete are now coroutines. The payload is still serialized on the
event loop -- it is pure CPU and sub-millisecond -- and only the write
goes to a thread, so the writer never walks live DownloadInfo objects
while the loop mutates them. Each queue gets its own single-worker
executor rather than sharing the default one, because extract_info can
hold default-executor threads for minutes and would leave state writes
queued behind exactly when they are needed.

Awaiting the write makes interleaving possible where it was not before,
so a lock now covers the mutate-write-rollback section: the invariant
that in-memory state never diverges from what is on disk is unchanged,
including the rollback when a write fails. On shutdown the queues are
drained rather than cancelled, so a write in flight still lands.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

README.md

@@ -98,7 +98,7 @@ Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a fee
 * __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`.
 * __CERTFILE__: HTTPS certificate file path.
 * __KEYFILE__: HTTPS key file path.
-* __CORS_ALLOWED_ORIGINS__: Comma-separated list of origins permitted to make cross-origin requests to the MeTube API; `*` allows all. When unset or empty, all cross-origin requests are denied. Required for browser extensions and bookmarklets — see [Sending links to MeTube](#-sending-links-to-metube).
+* __CORS_ALLOWED_ORIGINS__: Comma-separated list of origins permitted to make cross-origin requests to the MeTube API; `*` allows all. When unset or empty, all cross-origin requests are denied. Required for browser extensions and bookmarklets — see [Sending links to MeTube](#-sending-links-to-metube). Naming origins explicitly also lets them send credentials (a login cookie, or the `Authorization` header a reverse proxy checks), which `*` deliberately does not: it would let any site you visit drive your instance with your own session.
 * __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container.
 
 ## 🎛️ Configuring yt-dlp options
@@ -243,7 +243,7 @@ __Browser extensions__ allow right-clicking videos and sending them directly to
 * __Chrome:__ contributed by [Rpsl](https://github.com/rpsl) — install from the [Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or [from sources](https://github.com/Rpsl/metube-browser-extension).
 * __Firefox:__ contributed by [nanocortex](https://github.com/nanocortex) — install from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources [here](https://github.com/nanocortex/metube-firefox-addon).
 
-__Bookmarklets__ send the currently open page to MeTube with one click. Add the origins of the sites where you use them to `CORS_ALLOWED_ORIGINS`, e.g. `https://www.youtube.com,https://www.vimeo.com`. The code (Chrome and Firefox variants, contributed by [kushfest](https://github.com/kushfest) and [shoonya75](https://github.com/shoonya75)) is in the [Bookmarklets wiki page](https://github.com/alexta69/metube/wiki/Bookmarklets).
+__Bookmarklets__ send the currently open page to MeTube with one click. Add the origins of the sites where you use them to `CORS_ALLOWED_ORIGINS`, e.g. `https://www.youtube.com,https://www.vimeo.com`. If your instance sits behind authentication, list the origins individually rather than using `*` — only named origins are allowed to send credentials. The code (Chrome and Firefox variants, contributed by [kushfest](https://github.com/kushfest) and [shoonya75](https://github.com/shoonya75)) is in the [Bookmarklets wiki page](https://github.com/alexta69/metube/wiki/Bookmarklets).
 
 __iOS Shortcut:__ [rithask](https://github.com/rithask) created an [iOS shortcut](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6) for sending URLs to MeTube from Safari's share menu; it prompts for your instance address on first use.
 

app/main.py

@@ -116,6 +116,17 @@ class Config:
         if not self.URL_PREFIX.endswith('/'):
             self.URL_PREFIX += '/'
 
+        # Strip trailing slashes from the download directories. get_custom_dirs()
+        # builds the folder dropdown by removing the base path as a prefix from
+        # each subdirectory, and the base directory's own path does not carry the
+        # trailing slash — so 'DOWNLOAD_DIR=/downloads/' failed to match itself
+        # and leaked 'downloads' into the dropdown as a bogus folder option.
+        # Runs after the '%%' indirection above so AUDIO_DOWNLOAD_DIR is resolved.
+        for attr in ('DOWNLOAD_DIR', 'AUDIO_DOWNLOAD_DIR', 'TEMP_DIR', 'STATE_DIR'):
+            val = getattr(self, attr)
+            if isinstance(val, str) and len(val) > 1 and val.endswith('/'):
+                setattr(self, attr, val.rstrip('/') or '/')
+
         # A blank PUBLIC_HOST_AUDIO_URL (e.g. set empty in a compose file) bypasses the
         # default via os.environ.get, which would leave audio links root-relative and 404.
         # Fall back to the 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR. When
@@ -335,6 +346,12 @@ async def state_dir_guard(request, handler):
 
 app = web.Application(middlewares=[state_dir_guard])
 _cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
+if '*' in _cors_origins and len(_cors_origins) > 1:
+    log.warning(
+        "CORS_ALLOWED_ORIGINS mixes '*' with named origins %s. '*' wins, and credentialed "
+        "cross-origin requests stay disabled for every origin in the list. Remove '*' if you "
+        "need a bookmarklet to reach an authenticated instance.",
+        [o for o in _cors_origins if o != '*'])
 sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
 sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
 routes = web.RouteTableDef()
@@ -1063,6 +1080,30 @@ async def start(request):
 
 COOKIES_PATH = os.path.join(config.STATE_DIR, 'cookies.txt')
 
+
+def warn_if_cookiefile_shadowed():
+    """Warn before an uploaded cookies file displaces an operator-configured one.
+
+    Uploaded cookies deliberately win: the upload exists so cookies can be
+    refreshed without restarting the container, and letting YTDL_OPTIONS win
+    would leave a visible UI button doing nothing. But set_runtime_override
+    writes straight into YTDL_OPTIONS, so the configured path is gone from the
+    live config the moment an uploaded file is applied — after that, nothing
+    downstream can report the conflict (delete_cookies' has_manual_cookiefile
+    check cannot fire once the value has been replaced). This is the only point
+    where both are still visible, so it is the only place the warning can be
+    issued. Must be called before set_runtime_override. See issue #881, where
+    the silence cost the reporter days of debugging.
+    """
+    configured = config.YTDL_OPTIONS.get('cookiefile')
+    if isinstance(configured, str) and configured and configured != COOKIES_PATH:
+        log.warning(
+            'Uploaded cookies at %s take precedence over the cookiefile configured in '
+            'YTDL_OPTIONS (%s), which will not be used. Delete the uploaded cookies from '
+            'the UI to go back to the configured file.',
+            COOKIES_PATH, configured)
+
+
 @routes.post(config.URL_PREFIX + 'upload-cookies')
 async def upload_cookies(request):
     reader = await request.multipart()
@@ -1092,6 +1133,7 @@ async def upload_cookies(request):
     except OSError as exc:
         log.warning(f'Could not restrict permissions on cookies file: {exc}')
     os.replace(tmp_cookie_path, COOKIES_PATH)
+    warn_if_cookiefile_shadowed()
     config.set_runtime_override('cookiefile', COOKIES_PATH)
     log.info(f'Cookies file uploaded ({size} bytes)')
     return web.Response(text=serializer.encode({'status': 'ok', 'msg': f'Cookies uploaded ({size} bytes)'}))
@@ -1286,9 +1328,44 @@ app.router.add_route('OPTIONS', config.URL_PREFIX + 'delete-cookies', add_cors)
 
 async def on_prepare(request, response):
     origin = request.headers.get('Origin')
-    if origin and _cors_origins and ('*' in _cors_origins or origin in _cors_origins):
-        response.headers['Access-Control-Allow-Origin'] = origin
-        response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
+    if not origin or not _cors_origins:
+        return
+
+    # Naming an origin in CORS_ALLOWED_ORIGINS is a deliberate trust grant, so
+    # such an origin may send credentials: the cookie or Authorization header
+    # that a reverse proxy in front of MeTube checks. Without this a bookmarklet
+    # cannot reach an authenticated instance at all (issue #155).
+    #
+    # The '*' wildcard is emphatically not such a grant — it matches origins the
+    # operator never enumerated, including every site the user happens to visit.
+    # Echoing the origin back (which we must do, since '*' is illegal alongside
+    # credentials) and allowing credentials would let any page drive the user's
+    # instance with the user's own session. So the wildcard keeps exactly the
+    # uncredentialed behaviour it has always had, and a wildcard anywhere in the
+    # list disables credentials for every origin in it.
+    #
+    # Derived here rather than held in a second module global so the wildcard
+    # test and the membership test can never disagree about the same list.
+    wildcard = '*' in _cors_origins
+    trusted = not wildcard and origin in _cors_origins
+    if not (wildcard or trusted):
+        return
+
+    response.headers['Access-Control-Allow-Origin'] = origin
+    # Authorization rides on the same grant: allowing it under the wildcard
+    # would let an arbitrary page attempt credentials against an instance it
+    # can already reach, from inside the victim's network.
+    response.headers['Access-Control-Allow-Headers'] = 'Content-Type, Authorization' if trusted else 'Content-Type'
+    if trusted:
+        response.headers['Access-Control-Allow-Credentials'] = 'true'
+
+    # The response now differs per Origin, so a shared cache must not hand one
+    # origin's Allow-Origin to another.
+    vary = response.headers.get('Vary')
+    if not vary:
+        response.headers['Vary'] = 'Origin'
+    elif 'origin' not in (v.strip().lower() for v in vary.split(',')):
+        response.headers['Vary'] = f'{vary}, Origin'
 
 app.on_response_prepare.append(on_prepare)
 
@@ -1314,6 +1391,7 @@ if __name__ == '__main__':
 
     # Auto-detect cookie file on startup
     if os.path.exists(COOKIES_PATH):
+        warn_if_cookiefile_shadowed()
         config.set_runtime_override('cookiefile', COOKIES_PATH)
         log.info(f'Cookie file detected at {COOKIES_PATH}')
 

app/tests/test_api.py

@@ -525,3 +525,130 @@ async def test_download_blocks_state_dir_files(monkeypatch):
         (download_dir / "video.mp4").unlink(missing_ok=True)
         (download_dir / percent_filename).unlink(missing_ok=True)
         state_dir.rmdir()
+
+# --- CORS (issue #155) -------------------------------------------------------
+#
+# The security property under test: credentials are granted only to an origin
+# the operator named explicitly, and never under the '*' wildcard. Each test
+# builds a fresh Application because main.app binds to the first event loop
+# that runs it; the logic under test lives entirely in main.on_prepare, and the
+# real main.add_cors preflight handler is mounted so the preflight path is the
+# production one.
+
+async def _cors_version(request):
+    return web.Response(text="v")
+
+
+def _cors_app():
+    app = web.Application()
+    app.router.add_route("OPTIONS", "/add", main.add_cors)
+    app.router.add_get("/version", _cors_version)
+    app.on_response_prepare.append(main.on_prepare)
+    return app
+
+
+async def _cors_headers(monkeypatch, origins, origin, path="/add", method="OPTIONS"):
+    monkeypatch.setattr(main, "_cors_origins", origins)
+    async with TestClient(TestServer(_cors_app())) as client:
+        resp = await client.request(
+            method, path,
+            headers={
+                "Origin": origin,
+                "Access-Control-Request-Method": "POST",
+                "Access-Control-Request-Headers": "content-type,authorization",
+            },
+        )
+        return resp.headers
+
+
+@pytest.mark.asyncio
+async def test_cors_listed_origin_gets_credentials(monkeypatch):
+    h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "https://www.youtube.com")
+    assert h["Access-Control-Allow-Origin"] == "https://www.youtube.com"
+    assert h["Access-Control-Allow-Credentials"] == "true"
+    assert "Authorization" in h["Access-Control-Allow-Headers"]
+    assert "Origin" in h["Vary"]
+
+
+@pytest.mark.asyncio
+async def test_cors_wildcard_never_grants_credentials(monkeypatch):
+    h = await _cors_headers(monkeypatch, ["*"], "https://evil.example")
+    # The wildcard still reflects the origin, exactly as before...
+    assert h["Access-Control-Allow-Origin"] == "https://evil.example"
+    # ...but must not hand out the user's session, nor let a page attempt
+    # credentials of its own.
+    assert "Access-Control-Allow-Credentials" not in h
+    assert h["Access-Control-Allow-Headers"] == "Content-Type"
+
+
+@pytest.mark.asyncio
+async def test_cors_wildcard_mixed_with_named_origin_still_denies_credentials(monkeypatch):
+    # '*' anywhere in the list disables credentials for everyone in it, so a
+    # stray wildcard cannot silently widen a named grant.
+    h = await _cors_headers(monkeypatch, ["*", "https://www.youtube.com"], "https://www.youtube.com")
+    assert h["Access-Control-Allow-Origin"] == "https://www.youtube.com"
+    assert "Access-Control-Allow-Credentials" not in h
+    assert h["Access-Control-Allow-Headers"] == "Content-Type"
+
+
+@pytest.mark.asyncio
+async def test_cors_unlisted_origin_gets_nothing(monkeypatch):
+    h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "https://evil.example")
+    assert "Access-Control-Allow-Origin" not in h
+    assert "Access-Control-Allow-Credentials" not in h
+
+
+@pytest.mark.asyncio
+async def test_cors_disabled_by_default(monkeypatch):
+    h = await _cors_headers(monkeypatch, [], "https://www.youtube.com")
+    assert "Access-Control-Allow-Origin" not in h
+    assert "Access-Control-Allow-Credentials" not in h
+
+
+@pytest.mark.asyncio
+async def test_cors_credentials_apply_to_actual_response_not_just_preflight(monkeypatch):
+    # The browser checks Allow-Credentials on the real response too, so a
+    # preflight-only grant would still fail.
+    h = await _cors_headers(
+        monkeypatch, ["https://www.youtube.com"], "https://www.youtube.com",
+        path="/version", method="GET")
+    assert h["Access-Control-Allow-Credentials"] == "true"
+
+
+@pytest.mark.asyncio
+async def test_cors_origin_match_is_exact(monkeypatch):
+    # Substring or suffix matching here would be a bypass.
+    for impostor in (
+        "https://www.youtube.com.evil.example",
+        "https://evilwww.youtube.com",
+        "http://www.youtube.com",
+        "https://www.youtube.com:8443",
+    ):
+        h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], impostor)
+        assert "Access-Control-Allow-Origin" not in h, impostor
+        assert "Access-Control-Allow-Credentials" not in h, impostor
+
+
+@pytest.mark.asyncio
+async def test_cors_null_origin_is_not_trusted(monkeypatch):
+    # Sandboxed iframes and some file:// contexts send Origin: null.
+    h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "null")
+    assert "Access-Control-Allow-Origin" not in h
+    assert "Access-Control-Allow-Credentials" not in h
+
+
+@pytest.mark.asyncio
+async def test_cors_vary_appends_to_existing_value(monkeypatch):
+    # Static responses can already carry a Vary; clobbering it would break
+    # content negotiation.
+    monkeypatch.setattr(main, "_cors_origins", ["https://www.youtube.com"])
+
+    async def handler(request):
+        return web.Response(text="x", headers={"Vary": "Accept-Encoding"})
+
+    app = web.Application()
+    app.router.add_get("/v", handler)
+    app.on_response_prepare.append(main.on_prepare)
+    async with TestClient(TestServer(app)) as client:
+        resp = await client.get("/v", headers={"Origin": "https://www.youtube.com"})
+        assert resp.headers["Vary"] == "Accept-Encoding, Origin"

app/tests/test_config.py

@@ -77,6 +77,34 @@ class ConfigTests(unittest.TestCase):
         self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
         self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
 
+    def test_download_dirs_lose_trailing_slash(self):
+        # get_custom_dirs strips the base path as a prefix from each subdirectory,
+        # and the base directory's own path has no trailing slash -- so a trailing
+        # slash here leaked the absolute path into the folder dropdown.
+        with patch.dict(os.environ, _base_env(
+            DOWNLOAD_DIR="/downloads/",
+            AUDIO_DOWNLOAD_DIR="/audio/",
+            TEMP_DIR="/tmp/",
+            STATE_DIR="/state/",
+        ), clear=False):
+            c = Config()
+        self.assertEqual(c.DOWNLOAD_DIR, "/downloads")
+        self.assertEqual(c.AUDIO_DOWNLOAD_DIR, "/audio")
+        self.assertEqual(c.TEMP_DIR, "/tmp")
+        self.assertEqual(c.STATE_DIR, "/state")
+
+    def test_root_download_dir_survives_normalisation(self):
+        with patch.dict(os.environ, _base_env(DOWNLOAD_DIR="/", AUDIO_DOWNLOAD_DIR="///"), clear=False):
+            c = Config()
+        self.assertEqual(c.DOWNLOAD_DIR, "/")
+        self.assertEqual(c.AUDIO_DOWNLOAD_DIR, "/")
+
+    def test_download_dirs_without_trailing_slash_unchanged(self):
+        with patch.dict(os.environ, _base_env(DOWNLOAD_DIR="/downloads", AUDIO_DOWNLOAD_DIR="."), clear=False):
+            c = Config()
+        self.assertEqual(c.DOWNLOAD_DIR, "/downloads")
+        self.assertEqual(c.AUDIO_DOWNLOAD_DIR, ".")
+
     def test_ytdl_options_json_loaded(self):
         opts = {"quiet": True, "no_warnings": True}
         with patch.dict(

app/tests/test_download_queue.py

@@ -330,7 +330,7 @@ async def test_retry_restores_playlist_output_context(dq_env):
         chapter_template="",
     )
     failed_info.status = "error"
-    dq.done.put(Download(None, None, None, None, "best", "any", {}, failed_info))
+    await dq.done.put(Download(None, None, None, None, "best", "any", {}, failed_info))
 
     def fake_extract(self, extracted_url, *_args, **_kwargs):
         return {
@@ -389,7 +389,7 @@ async def test_retry_keeps_playlist_context_through_url_indirection(dq_env):
     dq = DownloadQueue(dq_env, notifier)
     url = "https://example.com/watch?v=1"
     resolved = "https://example.com/resolved?v=1"
-    dq.done.put(Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url)))
+    await dq.done.put(Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url)))
 
     def fake_extract(self, extracted_url, *_args, **_kwargs):
         if extracted_url == url:
@@ -427,7 +427,7 @@ async def test_retry_reapplies_current_options_gates(dq_env):
         ytdl_options_presets=["Still There", "Removed Preset"],
         ytdl_options_overrides={"paths": {"home": "/etc"}},
     )
-    dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
+    await dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
 
     def fake_extract(self, extracted_url, *_args, **_kwargs):
         return {
@@ -457,7 +457,7 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env):
     dq = DownloadQueue(dq_env, notifier)
     url = "https://example.com/watch?v=1"
     info = _failed_playlist_item(url, ytdl_options_overrides={"writesubtitles": True})
-    dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
+    await dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
 
     def fake_extract(self, extracted_url, *_args, **_kwargs):
         return {
@@ -481,7 +481,7 @@ async def test_retry_carries_the_sponsorblock_flag(dq_env):
     notifier = AsyncMock()
     dq = DownloadQueue(dq_env, notifier)
     url = "https://example.com/watch?v=1"
-    dq.done.put(
+    await dq.done.put(
         Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url, sponsorblock=True))
     )
 
@@ -1441,9 +1441,9 @@ async def test_post_download_cleanup_clears_filename_on_error(dq_env):
     notifier = AsyncMock()
     dq = DownloadQueue(dq_env, notifier)
     download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
-    dq.queue.put(download)
+    await dq.queue.put(download)
 
-    dq._post_download_cleanup(download)
+    await dq._post_download_cleanup(download)
 
     assert download.info.status == "error"
     assert download.info.filename is None
@@ -1456,9 +1456,9 @@ async def test_post_download_cleanup_keeps_captured_subtitles_on_error(dq_env):
     dq = DownloadQueue(dq_env, notifier)
     download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
     download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
-    dq.queue.put(download)
+    await dq.queue.put(download)
 
-    dq._post_download_cleanup(download)
+    await dq._post_download_cleanup(download)
 
     assert download.info.status == "error"
     assert download.info.filename == "en.srt"
@@ -1478,7 +1478,7 @@ async def test_clear_skips_deletion_outside_download_directory(dq_env):
     # A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
     escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
     download = _make_download(dq_env, status="finished", filename=escaping_filename)
-    dq.done.put(download)
+    await dq.done.put(download)
 
     await dq.clear([download.info.url])
 

app/tests/test_main_helpers.py

@@ -313,3 +313,40 @@ class GetCustomDirsTests(unittest.TestCase):
 
 if __name__ == "__main__":
     unittest.main()
+
+
+class WarnIfCookiefileShadowedTests(unittest.TestCase):
+    """Issue #881: an uploaded cookies file wins over an operator-configured
+    cookiefile, and used to do so with no way for anyone to notice."""
+
+    def setUp(self):
+        self._saved = main.config.YTDL_OPTIONS
+        main.config.YTDL_OPTIONS = dict(self._saved)
+
+    def tearDown(self):
+        main.config.YTDL_OPTIONS = self._saved
+
+    def test_warns_when_a_different_cookiefile_is_configured(self):
+        main.config.YTDL_OPTIONS["cookiefile"] = "/cookies/cookies.txt"
+        with self.assertLogs("main", level="WARNING") as cm:
+            main.warn_if_cookiefile_shadowed()
+        joined = "\n".join(cm.output)
+        self.assertIn("/cookies/cookies.txt", joined)
+        self.assertIn(main.COOKIES_PATH, joined)
+
+    def test_silent_when_no_cookiefile_configured(self):
+        main.config.YTDL_OPTIONS.pop("cookiefile", None)
+        with self.assertNoLogs("main", level="WARNING"):
+            main.warn_if_cookiefile_shadowed()
+
+    def test_silent_when_configured_file_is_the_uploaded_one(self):
+        # The steady state after an upload: re-running must not nag.
+        main.config.YTDL_OPTIONS["cookiefile"] = main.COOKIES_PATH
+        with self.assertNoLogs("main", level="WARNING"):
+            main.warn_if_cookiefile_shadowed()
+
+    def test_silent_on_non_string_or_empty_values(self):
+        for value in (None, "", 0, [], {}):
+            main.config.YTDL_OPTIONS["cookiefile"] = value
+            with self.assertNoLogs("main", level="WARNING"):
+                main.warn_if_cookiefile_shadowed()

app/tests/test_persistent_queue.py

@@ -2,8 +2,11 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import os
+import threading
+import time
 import shelve
 import sys
 import tempfile
@@ -69,22 +72,22 @@ def _create_legacy_shelf(path: str, *infos: DownloadInfo) -> None:
             shelf[info.url] = info
 
 
-class PersistentQueueTests(unittest.TestCase):
-    def test_put_get_delete_roundtrip(self):
+class PersistentQueueTests(unittest.IsolatedAsyncioTestCase):
+    async def test_put_get_delete_roundtrip(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "queue")
             pq = PersistentQueue("queue", path)
             dl = _FakeDownload(_make_info("http://a.example"))
-            pq.put(dl)
+            await pq.put(dl)
             self.assertTrue(os.path.exists(path + ".json"))
             self.assertTrue(pq.exists("http://a.example"))
             self.assertFalse(pq.empty())
             got = pq.get("http://a.example")
             self.assertEqual(got.info.url, "http://a.example")
-            pq.delete("http://a.example")
+            await pq.delete("http://a.example")
             self.assertFalse(pq.exists("http://a.example"))
 
-    def test_saved_items_sorted_by_timestamp(self):
+    async def test_saved_items_sorted_by_timestamp(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "queue")
             pq = PersistentQueue("queue", path)
@@ -92,16 +95,16 @@ class PersistentQueueTests(unittest.TestCase):
             b = _FakeDownload(_make_info("http://second.example"))
             a.info.timestamp = 100
             b.info.timestamp = 200
-            pq.put(a)
-            pq.put(b)
+            await pq.put(a)
+            await pq.put(b)
             keys = [k for k, _ in pq.saved_items()]
             self.assertEqual(keys, ["http://first.example", "http://second.example"])
 
-    def test_load_restores_from_json(self):
+    async def test_load_restores_from_json(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "queue")
             pq1 = PersistentQueue("queue", path)
-            pq1.put(_FakeDownload(_make_info("http://load.example")))
+            await pq1.put(_FakeDownload(_make_info("http://load.example")))
             pq2 = PersistentQueue("queue", path)
             pq2.load()
             self.assertTrue(pq2.exists("http://load.example"))
@@ -115,7 +118,7 @@ class PersistentQueueTests(unittest.TestCase):
             self.assertTrue(pq.exists("http://legacy.example"))
             self.assertTrue(os.path.exists(path + ".json"))
 
-    def test_queue_persists_only_compact_entry_subset(self):
+    async def test_queue_persists_only_compact_entry_subset(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "queue")
             pq = PersistentQueue("queue", path)
@@ -128,7 +131,7 @@ class PersistentQueueTests(unittest.TestCase):
                 "formats": [{"id": "huge"}],
                 "description": "very large payload",
             }
-            pq.put(_FakeDownload(info))
+            await pq.put(_FakeDownload(info))
 
             with open(path + ".json", encoding="utf-8") as f:
                 payload = json.load(f)
@@ -146,7 +149,7 @@ class PersistentQueueTests(unittest.TestCase):
             self.assertNotIn("formats", record["entry"])
             self.assertNotIn("description", record["entry"])
 
-    def test_completed_queue_persists_only_failed_retry_context(self):
+    async def test_completed_queue_persists_only_failed_retry_context(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "completed")
             pq = PersistentQueue("completed", path)
@@ -161,7 +164,7 @@ class PersistentQueueTests(unittest.TestCase):
                 "formats": [{"id": "huge"}],
             }
             info.filename = "done.mp4"
-            pq.put(_FakeDownload(info))
+            await pq.put(_FakeDownload(info))
 
             with open(path + ".json", encoding="utf-8") as f:
                 payload = json.load(f)
@@ -180,7 +183,7 @@ class PersistentQueueTests(unittest.TestCase):
             self.assertEqual(record["filename"], "done.mp4")
 
             info.status = "finished"
-            pq.put(_FakeDownload(info))
+            await pq.put(_FakeDownload(info))
             with open(path + ".json", encoding="utf-8") as f:
                 payload = json.load(f)
             self.assertNotIn("entry", payload["items"][0]["info"])
@@ -256,7 +259,7 @@ class PersistentQueueTests(unittest.TestCase):
             self.assertNotIn("speed", record)
             self.assertNotIn("eta", record)
 
-    def test_put_rollbacks_in_memory_queue_when_state_write_fails(self):
+    async def test_put_rollbacks_in_memory_queue_when_state_write_fails(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "queue")
             pq = PersistentQueue("queue", path)
@@ -272,18 +275,18 @@ class PersistentQueueTests(unittest.TestCase):
 
             with patch("ytdl.AtomicJsonStore.save", bad_save):
                 with self.assertRaises(OSError):
-                    pq.put(dl)
+                    await pq.put(dl)
 
             self.assertFalse(pq.exists("http://rollback.example"))
 
-    def test_put_rollbacks_to_previous_download_when_replace_fails(self):
+    async def test_put_rollbacks_to_previous_download_when_replace_fails(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "queue")
             pq = PersistentQueue("queue", path)
             first = _FakeDownload(_make_info("http://same.example"))
             second = _FakeDownload(_make_info("http://same.example"))
             second.info.title = "Replaced title"
-            pq.put(first)
+            await pq.put(first)
 
             orig_save = __import__("state_store").AtomicJsonStore.save
 
@@ -294,10 +297,68 @@ class PersistentQueueTests(unittest.TestCase):
 
             with patch("ytdl.AtomicJsonStore.save", bad_save):
                 with self.assertRaises(OSError):
-                    pq.put(second)
+                    await pq.put(second)
 
             self.assertEqual(pq.get("http://same.example").info.title, "Title")
 
 
+class StateWriteOffEventLoopTests(unittest.IsolatedAsyncioTestCase):
+    """State writes fsync twice; on a slow disk that must not stall the loop.
+
+    Before this, put()/delete() wrote inline, so a queue mutation blocked every
+    other request the server was serving for as long as the filesystem took.
+    See issue #980.
+    """
+
+    async def test_save_runs_off_the_event_loop_thread(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            pq = PersistentQueue("queue", os.path.join(tmp, "queue"))
+            self.addCleanup(pq.close)
+            loop_thread = threading.get_ident()
+            save_threads = []
+
+            orig_save = __import__("state_store").AtomicJsonStore.save
+
+            def recording_save(store, data):
+                save_threads.append(threading.get_ident())
+                return orig_save(store, data)
+
+            with patch("ytdl.AtomicJsonStore.save", recording_save):
+                await pq.put(_FakeDownload(_make_info("http://a.example")))
+
+            self.assertEqual(len(save_threads), 1)
+            self.assertNotEqual(save_threads[0], loop_thread)
+
+    async def test_a_slow_write_does_not_stall_other_coroutines(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            pq = PersistentQueue("queue", os.path.join(tmp, "queue"))
+            self.addCleanup(pq.close)
+            orig_save = __import__("state_store").AtomicJsonStore.save
+
+            def slow_save(store, data):
+                time.sleep(0.3)
+                return orig_save(store, data)
+
+            ticks = 0
+
+            async def ticker():
+                nonlocal ticks
+                while True:
+                    await asyncio.sleep(0.01)
+                    ticks += 1
+
+            ticking = asyncio.create_task(ticker())
+            try:
+                with patch("ytdl.AtomicJsonStore.save", slow_save):
+                    await pq.put(_FakeDownload(_make_info("http://a.example")))
+            finally:
+                ticking.cancel()
+
+            # An inline write would have starved the loop for the whole 0.3s and
+            # left ticks at 0.
+            self.assertGreater(ticks, 5)
+            self.assertTrue(pq.exists("http://a.example"))
+
+
 if __name__ == "__main__":
     unittest.main()

app/tests/test_ytdl_utils.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import os
 import pickle
 import signal
@@ -11,7 +12,8 @@ import threading
 import types
 import unittest
 from pathlib import Path
-from unittest.mock import MagicMock, patch
+from concurrent.futures import ThreadPoolExecutor
+from unittest.mock import AsyncMock, MagicMock, patch
 
 fake_yt_dlp = types.ModuleType("yt_dlp")
 fake_networking = types.ModuleType("yt_dlp.networking")
@@ -1176,3 +1178,62 @@ class PotProviderUrlsTests(unittest.TestCase):
 if __name__ == "__main__":
 
     unittest.main()
+
+
+class UpdateStatusFileStatTests(unittest.IsolatedAsyncioTestCase):
+    """The progress path must not touch the filesystem on the event loop.
+
+    yt-dlp reports 'filename' on every progress tick, but until the download
+    finishes that path does not exist yet -- the bytes are in 'tmpfilename'.
+    Stating it per tick meant blocking syscalls on the event loop twice a
+    second per download, always answering None. See issue #980.
+    """
+
+    async def _run_update_status(self, statuses):
+        import queue as _queue
+
+        download = _make_test_download()
+        download.download_dir = "/tmp"
+        source = _queue.Queue()
+        for status in statuses:
+            source.put(status)
+        source.put(None)
+        download.status_queue = source
+        download.loop = asyncio.get_running_loop()
+        download._executor = ThreadPoolExecutor(max_workers=1)
+        notifier = MagicMock()
+        notifier.updated = AsyncMock()
+        download.notifier = notifier
+
+        stat_calls = []
+
+        def record_exists(path):
+            stat_calls.append(path)
+            return False
+
+        try:
+            with patch("ytdl.os.path.exists", side_effect=record_exists):
+                await download.update_status()
+        finally:
+            download._executor.shutdown(wait=True)
+        return download, stat_calls
+
+    async def test_downloading_ticks_do_not_stat_the_output_file(self):
+        ticks = [
+            {"status": "downloading", "filename": "/tmp/v.mp4",
+             "tmpfilename": "/tmp/v.mp4.part", "downloaded_bytes": i}
+            for i in range(1, 6)
+        ]
+        download, stat_calls = await self._run_update_status(ticks)
+
+        self.assertEqual(stat_calls, [])
+        self.assertEqual(download.info.filename, "v.mp4")
+
+    async def test_finished_status_still_stats_the_output_file(self):
+        download, stat_calls = await self._run_update_status([
+            {"status": "downloading", "filename": "/tmp/v.mp4", "downloaded_bytes": 1},
+            {"status": "finished", "filename": "/tmp/v.mp4"},
+        ])
+
+        self.assertEqual(stat_calls, ["/tmp/v.mp4"])
+        self.assertIsNone(download.info.size)

app/ytdl.py

@@ -1027,7 +1027,17 @@ class Download:
                     if not rel_name.lower().endswith(allowed_caption_exts):
                         continue
                 self.info.filename = rel_name
-                self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
+                # Stat only on a terminal status. yt-dlp documents 'filename' as
+                # always present in a progress hook, but until the download
+                # finishes the bytes are in tmpfilename and 'filename' is a
+                # destination that does not exist yet -- so this was two
+                # blocking syscalls on the event loop, twice a second per
+                # active download, to arrive at None. A stat that takes seconds
+                # on a contended filesystem stalls every other request the
+                # server is serving. Nothing displays the size before
+                # completion: the Downloading table has no size column.
+                if status.get('status') == 'finished':
+                    self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
                 if getattr(self.info, 'download_type', '') == 'thumbnail':
                     # The thumbnail convertor always emits a .jpg, but yt-dlp may
                     # report the pre-conversion media/thumbnail extension
@@ -1101,6 +1111,19 @@ class PersistentQueue:
         self.path = f"{path}.json"
         self.store = AtomicJsonStore(self.path, kind=f"persistent_queue:{name}")
         self.dict = OrderedDict()
+        # A state write fsyncs twice (the file and its directory). On a slow or
+        # contended filesystem that is seconds, and running it inline in an
+        # async caller blocked the event loop -- every other request stalled
+        # behind a single queue mutation. One dedicated thread keeps the writes
+        # off the loop and, being single, keeps them ordered. The default
+        # executor is not usable for this: extract_info shares it and can hold
+        # its threads for minutes, which is exactly when state writes happen.
+        self._store_executor = ThreadPoolExecutor(
+            max_workers=1, thread_name_prefix=f"state-{name}")
+        # Guards the mutate-write-rollback section. The write is awaited now, so
+        # without this two callers could interleave between changing self.dict
+        # and persisting it, and a rollback could revert the wrong entry.
+        self._lock = asyncio.Lock()
 
     def load(self):
         for k, v in self.saved_items():
@@ -1141,8 +1164,14 @@ class PersistentQueue:
             for key, download in self.dict.items()
         ]
 
-    def _save_dict(self):
-        self.store.save({"items": self._serialize_items()})
+    async def _save_dict_async(self):
+        # Serialize on the event loop -- it is pure CPU and sub-millisecond --
+        # and hand the finished payload to the writer thread. Serializing in the
+        # thread instead would have it walk live DownloadInfo objects while the
+        # loop mutates them.
+        payload = {"items": self._serialize_items()}
+        await asyncio.get_running_loop().run_in_executor(
+            self._store_executor, self.store.save, payload)
 
     def _load_state_items(self):
         payload = self.store.load()
@@ -1183,32 +1212,39 @@ class PersistentQueue:
         self.store.save({"items": items})
         return items
 
-    def put(self, value):
+    async def put(self, value):
         key = value.info.url
-        old = self.dict.get(key)
-        self.dict[key] = value
-        try:
-            self._save_dict()
-        except Exception:
-            if old is None:
-                del self.dict[key]
-            else:
-                self.dict[key] = old
-            raise
-
-    def delete(self, key):
-        if key in self.dict:
-            old = self.dict[key]
-            del self.dict[key]
+        async with self._lock:
+            old = self.dict.get(key)
+            self.dict[key] = value
             try:
-                self._save_dict()
+                await self._save_dict_async()
             except Exception:
-                self.dict[key] = old
+                if old is None:
+                    del self.dict[key]
+                else:
+                    self.dict[key] = old
                 raise
 
+    async def delete(self, key):
+        async with self._lock:
+            if key in self.dict:
+                old = self.dict[key]
+                del self.dict[key]
+                try:
+                    await self._save_dict_async()
+                except Exception:
+                    self.dict[key] = old
+                    raise
+
     def empty(self):
         return not bool(self.dict)
 
+    def close(self):
+        # wait=True so a write already in flight reaches disk before the
+        # process exits; there is at most one, and it is the newest state.
+        self._store_executor.shutdown(wait=True)
+
 class DownloadQueue:
     def __init__(self, config, notifier):
         self.config = config
@@ -1385,8 +1421,8 @@ class DownloadQueue:
                 if not info.error:
                     info.error = str(exc)
                 self._unregister_scheduled(url)
-                self.queue.delete(url)
-                self.done.put(download)
+                await self.queue.delete(url)
+                await self.done.put(download)
                 await self.notifier.completed(info)
             else:
                 log.warning(
@@ -1420,9 +1456,9 @@ class DownloadQueue:
         await self.notifier.updated(info)
         bg_tasks.create_task(self.__start_download(download), name="start_download")
 
-    def _schedule_upcoming_download(self, download: Download) -> None:
+    async def _schedule_upcoming_download(self, download: Download) -> None:
         download.info.status = 'scheduled'
-        self.queue.put(download)
+        await self.queue.put(download)
         self._register_scheduled(download)
 
     def _force_start_scheduled(self, download: Download) -> None:
@@ -1441,9 +1477,9 @@ class DownloadQueue:
                 log.info(f"Download {download.info.title} was canceled, skipping start.")
                 return
             await download.start(self.notifier, self._download_executor)
-            self._post_download_cleanup(download)
+            await self._post_download_cleanup(download)
 
-    def _post_download_cleanup(self, download):
+    async def _post_download_cleanup(self, download):
         if download.info.status != 'finished':
             if download.tmpfilename and os.path.isfile(download.tmpfilename):
                 try:
@@ -1463,11 +1499,11 @@ class DownloadQueue:
                 download.info.size = None
         download.close()
         if self.queue.exists(download.info.url):
-            self.queue.delete(download.info.url)
+            await self.queue.delete(download.info.url)
             if download.canceled:
                 bg_tasks.create_task(self.notifier.canceled(download.info.url), name="notify_canceled")
             else:
-                self.done.put(download)
+                await self.done.put(download)
                 bg_tasks.create_task(self.notifier.completed(download.info), name="notify_completed")
                 try:
                     clear_after = int(self.config.CLEAR_COMPLETED_AFTER)
@@ -1575,12 +1611,12 @@ class DownloadQueue:
         )
         if auto_start is True:
             if is_upcoming:
-                self._schedule_upcoming_download(download)
+                await self._schedule_upcoming_download(download)
             else:
-                self.queue.put(download)
+                await self.queue.put(download)
                 bg_tasks.create_task(self.__start_download(download), name="start_download")
         else:
-            self.pending.put(download)
+            await self.pending.put(download)
         await self.notifier.added(dl)
 
     def __write_feed_metadata_sync(self, entry, etype, download_type, folder,
@@ -1889,7 +1925,7 @@ class DownloadQueue:
         info.status = 'error'
         info.msg = msg
         download = Download(None, None, None, None, quality, format, {}, info)
-        self.done.put(download)
+        await self.done.put(download)
         await self.notifier.completed(info)
 
     async def add(
@@ -2076,11 +2112,11 @@ class DownloadQueue:
         for id in ids:
             if self.pending.exists(id):
                 dl = self.pending.get(id)
-                self.pending.delete(id)
+                await self.pending.delete(id)
                 if getattr(dl.info, 'live_status', None) == 'is_upcoming':
-                    self._schedule_upcoming_download(dl)
+                    await self._schedule_upcoming_download(dl)
                 else:
-                    self.queue.put(dl)
+                    await self.queue.put(dl)
                     bg_tasks.create_task(self.__start_download(dl), name="start_download")
                 continue
             if self.queue.exists(id):
@@ -2096,7 +2132,7 @@ class DownloadQueue:
             # Track URL so playlist add loop won't re-queue it
             self._canceled_urls.add(id)
             if self.pending.exists(id):
-                self.pending.delete(id)
+                await self.pending.delete(id)
                 await self.notifier.canceled(id)
                 continue
             if not self.queue.exists(id):
@@ -2109,7 +2145,7 @@ class DownloadQueue:
                 dl.cancel()
             else:
                 dl.canceled = True
-                self.queue.delete(id)
+                await self.queue.delete(id)
                 await self.notifier.canceled(id)
         return {'status': 'ok'}
 
@@ -2147,7 +2183,7 @@ class DownloadQueue:
                             pass
                         except OSError as e:
                             log.warning(f'deleting file "{rel_name}" for download {id} failed with error message {e!r}')
-            self.done.delete(id)
+            await self.done.delete(id)
             await self.notifier.cleared(id)
         return {'status': 'ok'}
 
@@ -2165,3 +2201,7 @@ class DownloadQueue:
             if download.started() and download.running():
                 download.cancel()
         self._download_executor.shutdown(wait=False, cancel_futures=True)
+        # Unlike the download executor these are drained, not cancelled: a
+        # queued write is the newest state and must reach disk before exit.
+        for queue in (self.queue, self.pending, self.done):
+            queue.close()

ui/src/app/app.spec.ts

@@ -413,4 +413,64 @@ describe('App', () => {
     expect(errorSpy).toHaveBeenCalledWith('Subscription name must not be empty');
     errorSpy.mockRestore();
   });
+  // Issue #533: the server picks AUDIO_DOWNLOAD_DIR on download_type alone
+  // (ytdl.py), so the UI's choice of URL base has to use the same rule. It used
+  // to also treat any .mp3 as audio, which pointed the link at audio_download/
+  // for files the server had written to DOWNLOAD_DIR.
+  describe('download links follow the server directory rule (#533)', () => {
+    const makeDownload = (over: Partial<Download>): Download => ({
+      id: 'vid1',
+      title: 'Test',
+      url: 'https://example.com/v',
+      download_type: 'video',
+      quality: 'best',
+      format: 'any',
+      folder: '',
+      custom_name_prefix: '',
+      playlist_item_limit: 0,
+      status: 'finished',
+      msg: '',
+      percent: 100,
+      speed: 0,
+      eta: 0,
+      filename: 'song.mp4',
+      checked: false,
+      ...over,
+    } as Download);
+
+    const appWithDirs = () => {
+      const fixture = TestBed.createComponent(App);
+      const app = fixture.componentInstance;
+      const downloads = TestBed.inject(DownloadsService) as unknown as DownloadsServiceStub;
+      downloads.configuration['PUBLIC_HOST_URL'] = 'download/';
+      downloads.configuration['PUBLIC_HOST_AUDIO_URL'] = 'audio_download/';
+      return app;
+    };
+
+    it('uses the audio base for an audio download', () => {
+      const app = appWithDirs();
+      const link = app.buildDownloadLink(makeDownload({ download_type: 'audio', filename: 'song.mp3' }));
+      expect(link).toBe('audio_download/song.mp3');
+    });
+
+    it('uses the video base for an mp3 produced by a video download', () => {
+      const app = appWithDirs();
+      const link = app.buildDownloadLink(makeDownload({ download_type: 'video', filename: 'song.mp3' }));
+      expect(link).toBe('download/song.mp3');
+    });
+
+    it('uses the video base for a video download', () => {
+      const app = appWithDirs();
+      const link = app.buildDownloadLink(makeDownload({ filename: 'clip.mp4' }));
+      expect(link).toBe('download/clip.mp4');
+    });
+
+    it('applies the same rule to chapter links', () => {
+      const app = appWithDirs();
+      const dl = makeDownload({ download_type: 'video' });
+      expect(app.buildChapterDownloadLink(dl, 'ch1.mp3')).toBe('download/ch1.mp3');
+      const audio = makeDownload({ download_type: 'audio' });
+      expect(app.buildChapterDownloadLink(audio, 'ch1.mp3')).toBe('audio_download/ch1.mp3');
+    });
+  });
 });

ui/src/app/app.ts

@@ -1284,7 +1284,12 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
 
   buildDownloadLink(download: Download) {
     let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
-    if (download.download_type === 'audio' || download.filename.endsWith('.mp3')) {
+    // Must match the server's directory rule exactly: ytdl.py writes to
+    // AUDIO_DOWNLOAD_DIR on download_type alone. Treating any .mp3 as audio
+    // sent the link to audio_download/ for mp3s produced under a video-type
+    // download (a postprocessor, a preset, or a legacy record), which the
+    // server had written to DOWNLOAD_DIR -- a 404 whenever the two differ.
+    if (download.download_type === 'audio') {
       baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
     }
 
@@ -1382,7 +1387,8 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
 
   buildChapterDownloadLink(download: Download, chapterFilename: string) {
     let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
-    if (download.download_type === 'audio' || chapterFilename.endsWith('.mp3')) {
+    // Same server-side rule as buildDownloadLink above.
+    if (download.download_type === 'audio') {
       baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
     }