diff --git a/README.md b/README.md index b5e366e..0c7ed6d 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ Certain values can be set via environment variables, using the `-e` parameter on * __ALLOW_PRIVATE_ADDRESSES__: Whether to allow downloads from private, loopback, link-local and other non-global addresses. Defaults to `false`, which protects against SSRF by refusing URLs that resolve to internal hosts. Set to `true` only in trusted environments — for example when routing traffic through a proxy/VPN client in Fake-IP mode (sing-box, Clash, Mihomo), which resolves hosts to the `198.18.0.0/15` range. Enabling this disables the SSRF protection entirely, so only use it when you control the network. * __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled). +Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a feed-level `.info.json` and thumbnail when you add a playlist or channel. These reuse the template of the items they belong to — `OUTPUT_TEMPLATE_CHANNEL` or `OUTPUT_TEMPLATE_PLAYLIST` — evaluated against the feed itself, so with the defaults they land in the same folder as the videos, named after the feed. Set `allow_playlist_files` to `false` in `YTDL_OPTIONS` to skip them. + ### 🌐 Web Server & URLs * __HOST__: The host address the web server will bind to. Defaults to `0.0.0.0` (all interfaces). diff --git a/app/subscriptions.py b/app/subscriptions.py index f64e700..3c3803c 100644 --- a/app/subscriptions.py +++ b/app/subscriptions.py @@ -64,6 +64,12 @@ def _build_ydl_params( "paths": {"home": config.DOWNLOAD_DIR, "temp": config.TEMP_DIR}, **config.YTDL_OPTIONS, **(extra_opts or {}), + # A scan is a poll, not an add: it runs on a timer and queues items + # through the download queue, which writes the feed metadata itself. + # yt-dlp emits the playlist-level infojson/description/thumbnail + # regardless of `download`, so without this a writeinfojson user would + # get those files rewritten on every check interval. See issue #1040. + "allow_playlist_files": False, } params = _impersonate_opt(params) if playlistend is not None and playlistend > 0: diff --git a/app/tests/test_download_queue.py b/app/tests/test_download_queue.py index 4d7c9b6..cf5b8af 100644 --- a/app/tests/test_download_queue.py +++ b/app/tests/test_download_queue.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import os import re import tempfile @@ -89,7 +90,7 @@ def test_get_returns_tuple_of_lists(dq_env): async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env): notifier = AsyncMock() - def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, url, *_args, **_kwargs): return { "_type": "video", "id": "vid1", @@ -124,7 +125,7 @@ async def test_add_unsupported_url_recorded_as_failed_entry(dq_env): notifier = AsyncMock() url = "https://example.com/not-a-video" - def boom(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def boom(self, url, *_args, **_kwargs): raise ytdl.yt_dlp.utils.YoutubeDLError(f'Unsupported URL: {url}') dq = DownloadQueue(dq_env, notifier) @@ -167,7 +168,7 @@ async def test_add_ssrf_rejected_url_recorded_as_failed_entry(dq_env): async def test_cancel_removes_from_pending(dq_env): notifier = AsyncMock() - def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, url, *_args, **_kwargs): return { "_type": "video", "id": "vid1", @@ -204,7 +205,7 @@ async def test_cancel_before_start_marks_download_canceled(dq_env): cancelling, because its ``download.canceled`` guard was never flipped.""" notifier = AsyncMock() - def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, url, *_args, **_kwargs): return { "_type": "video", "id": "vid1", @@ -242,7 +243,7 @@ async def test_cancel_before_start_marks_download_canceled(dq_env): async def test_start_pending_moves_to_queue(dq_env): notifier = AsyncMock() - def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, url, *_args, **_kwargs): return { "_type": "video", "id": "vid1", @@ -331,7 +332,7 @@ async def test_retry_restores_playlist_output_context(dq_env): failed_info.status = "error" dq.done.put(Download(None, None, None, None, "best", "any", {}, failed_info)) - def fake_extract(self, extracted_url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, extracted_url, *_args, **_kwargs): return { "_type": "video", "id": "vid1", @@ -390,7 +391,7 @@ async def test_retry_keeps_playlist_context_through_url_indirection(dq_env): resolved = "https://example.com/resolved?v=1" dq.done.put(Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url))) - def fake_extract(self, extracted_url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, extracted_url, *_args, **_kwargs): if extracted_url == url: return {"_type": "url", "url": resolved, "id": "vid1"} return { @@ -428,7 +429,7 @@ async def test_retry_reapplies_current_options_gates(dq_env): ) dq.done.put(Download(None, None, None, None, "best", "any", {}, info)) - def fake_extract(self, extracted_url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, extracted_url, *_args, **_kwargs): return { "_type": "video", "id": "vid1", @@ -458,7 +459,7 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env): info = _failed_playlist_item(url, ytdl_options_overrides={"writesubtitles": True}) dq.done.put(Download(None, None, None, None, "best", "any", {}, info)) - def fake_extract(self, extracted_url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, extracted_url, *_args, **_kwargs): return { "_type": "video", "id": "vid1", @@ -537,7 +538,7 @@ async def test_channel_download_uses_output_template_when_channel_template_empty channel_id = "UCabcd123" - def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, url, *_args, **_kwargs): return { "_type": "playlist", "id": channel_id, @@ -586,7 +587,7 @@ async def test_playlist_download_not_treated_as_channel(dq_env): dq_env.OUTPUT_TEMPLATE_CHANNEL = "" dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s" - def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, url, *_args, **_kwargs): return { "_type": "playlist", "id": "PLxyz789", @@ -633,7 +634,7 @@ async def test_add_merges_global_preset_and_override_options(dq_env): "Preset B": {"writesubtitles": False, "ratelimit": 1000}, } - def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, url, *_args, **_kwargs): return { "_type": "video", "id": "vid2", @@ -756,11 +757,191 @@ async def test_extract_info_metube_extract_keys_win_over_preset(dq_env): assert captured_params[0]["noplaylist"] is True +def _feed_extract(feed): + """Patch for __extract_info that returns a playlist/channel feed dict.""" + + def fake_extract(self, url, *_args, **_kwargs): + return copy.deepcopy(feed) + + return fake_extract + + +_CHANNEL_FEED = { + "_type": "playlist", + "id": "UC123", + "title": "Vanessa - Videos", + "channel": "Vanessa", + "channel_id": "UC123", + "uploader": "Vanessa", + "extractor": "youtube:tab", + "extractor_key": "YoutubeTab", + "webpage_url": "https://example.com/@vanessa/videos", + "entries": [ + {"id": "v1", "title": "One", "url": "https://example.com/v1", + "webpage_url": "https://example.com/v1", "_type": "url"}, + ], +} + +_PLAYLIST_FEED = { + "_type": "playlist", + "id": "PL123", + "title": "My Playlist", + "extractor": "generic", + "extractor_key": "Generic", + "webpage_url": "https://example.com/playlist?list=PL123", + "entries": [ + {"id": "v1", "title": "One", "url": "https://example.com/v1", + "webpage_url": "https://example.com/v1", "_type": "url"}, + ], +} + + +def _written_files(root): + found = [] + for dirpath, _dirs, files in os.walk(root): + for f in files: + found.append(os.path.relpath(os.path.join(dirpath, f), root)) + return sorted(found) + + +@pytest.mark.asyncio +async def test_channel_feed_metadata_lands_beside_its_items(dq_env): + """Issues #660/#1040: the feed-level .info.json follows the same template + the items use, so it sits in the channel's own folder rather than in + DOWNLOAD_DIR under yt-dlp's pl_* default name.""" + dq_env.YTDL_OPTIONS = {"writeinfojson": True} + dq_env.OUTPUT_TEMPLATE_CHANNEL = "%(channel)s/%(title)s.%(ext)s" + + dq = DownloadQueue(dq_env, AsyncMock()) + with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_CHANNEL_FEED)), \ + patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()): + result = await dq.add( + "https://example.com/@vanessa/videos", "video", "auto", "any", "best", + "", "", 0, auto_start=False, + ) + + assert result["status"] == "ok" + assert _written_files(dq_env.DOWNLOAD_DIR) == [ + os.path.join("Vanessa", "Vanessa - Videos.info.json") + ] + + +@pytest.mark.asyncio +async def test_playlist_feed_metadata_uses_the_playlist_template(dq_env): + dq_env.YTDL_OPTIONS = {"writeinfojson": True} + dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s" + + dq = DownloadQueue(dq_env, AsyncMock()) + with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \ + patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()): + await dq.add( + "https://example.com/playlist?list=PL123", "video", "auto", "any", "best", + "", "", 0, auto_start=False, + ) + + assert _written_files(dq_env.DOWNLOAD_DIR) == [ + os.path.join("My Playlist", "My Playlist.info.json") + ] + + +@pytest.mark.asyncio +async def test_feed_metadata_honours_custom_folder(dq_env): + dq_env.YTDL_OPTIONS = {"writeinfojson": True} + dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s" + + dq = DownloadQueue(dq_env, AsyncMock()) + with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \ + patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()): + await dq.add( + "https://example.com/playlist?list=PL123", "video", "auto", "any", "best", + "Music", "", 0, auto_start=False, + ) + + assert _written_files(dq_env.DOWNLOAD_DIR) == [ + os.path.join("Music", "My Playlist", "My Playlist.info.json") + ] + + +@pytest.mark.asyncio +async def test_no_feed_metadata_without_writeinfojson(dq_env): + """Nothing new appears for users who never asked for these files.""" + dq_env.YTDL_OPTIONS = {} + + dq = DownloadQueue(dq_env, AsyncMock()) + with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \ + patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()): + await dq.add( + "https://example.com/playlist?list=PL123", "video", "auto", "any", "best", + "", "", 0, auto_start=False, + ) + + assert _written_files(dq_env.DOWNLOAD_DIR) == [] + + +@pytest.mark.asyncio +async def test_feed_metadata_can_be_turned_off_by_the_user(dq_env): + dq_env.YTDL_OPTIONS = {"writeinfojson": True, "allow_playlist_files": False} + + dq = DownloadQueue(dq_env, AsyncMock()) + with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \ + patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()): + await dq.add( + "https://example.com/playlist?list=PL123", "video", "auto", "any", "best", + "", "", 0, auto_start=False, + ) + + assert _written_files(dq_env.DOWNLOAD_DIR) == [] + + +@pytest.mark.asyncio +async def test_feed_metadata_failure_does_not_fail_the_add(dq_env): + dq_env.YTDL_OPTIONS = {"writeinfojson": True} + + dq = DownloadQueue(dq_env, AsyncMock()) + with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \ + patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()), \ + patch.object( + DownloadQueue, "_DownloadQueue__write_feed_metadata_sync", + side_effect=OSError("read-only filesystem"), + ): + result = await dq.add( + "https://example.com/playlist?list=PL123", "video", "auto", "any", "best", + "", "", 0, auto_start=False, + ) + + assert result["status"] == "ok" + assert dq.pending.exists("https://example.com/v1") + + +@pytest.mark.asyncio +async def test_extraction_pass_never_writes_feed_metadata(dq_env): + """The classification pass must not produce files: it runs before the add is + known to succeed, and yt-dlp writes playlist files regardless of `download`.""" + dq_env.YTDL_OPTIONS = {"writeinfojson": True, "allow_playlist_files": True} + captured: list = [] + + class FakeYoutubeDL: + def __init__(self, params=None): + captured.append(params) + + def extract_info(self, url, download=False): + return {"_type": "video", "id": "v", "title": "V", "url": url, "webpage_url": url} + + dq = DownloadQueue(dq_env, AsyncMock()) + with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL): + await dq.add( + "https://example.com/watch?v=1", "video", "auto", "any", "best", + "", "", 0, auto_start=False, + ) + + assert captured[0]["allow_playlist_files"] is False + + @pytest.mark.asyncio async def test_add_sets_clip_bounds_on_download_info(dq_env): notifier = AsyncMock() - def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): + def fake_extract(self, url, *_args, **_kwargs): return { "_type": "video", "id": "vid1", diff --git a/app/tests/test_subscriptions.py b/app/tests/test_subscriptions.py index fbee0db..7870cf1 100644 --- a/app/tests/test_subscriptions.py +++ b/app/tests/test_subscriptions.py @@ -1171,6 +1171,44 @@ class SubscriptionScanExtraOptsTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(captured_params[0].get("cookiefile"), "preset.txt") self.assertEqual(captured_params[0].get("extra"), "override") + async def test_scan_never_writes_playlist_sidecar_files(self): + """A subscription scan is a metadata probe. yt-dlp writes the + playlist-level infojson/description/thumbnail regardless of ``download``, + so without this a writeinfojson/writethumbnail user would get stray files + in DOWNLOAD_DIR on every check interval. Issue #1040.""" + captured_params: list = [] + fake_ydl = _make_scan_capturing_fake_ydl( + captured_params, + [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}], + ) + + with tempfile.TemporaryDirectory() as tmp: + cfg = _Config(tmp) + cfg.YTDL_OPTIONS = {"writeinfojson": True, "writethumbnail": True} + mgr = SubscriptionManager(cfg, _Queue(), _Notifier()) + + with patch("subscriptions.yt_dlp.YoutubeDL", fake_ydl, create=True): + await mgr.add_subscription( + "https://example.com/channel", + check_interval_minutes=60, + download_type="video", + codec="auto", + format="any", + quality="best", + folder="", + custom_name_prefix="", + auto_start=True, + playlist_item_limit=0, + split_by_chapters=False, + chapter_template="", + subtitle_language="en", + subtitle_mode="prefer_manual", + ytdl_options_overrides={"allow_playlist_files": True}, + ) + + self.assertTrue(captured_params) + self.assertIs(captured_params[0].get("allow_playlist_files"), False) + async def test_check_now_scan_applies_stored_subscription_presets(self): entries = [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}] diff --git a/app/ytdl.py b/app/ytdl.py index 698294d..2f51230 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -1314,6 +1314,14 @@ class DownloadQueue: 'ignore_no_formats_error': True, 'noplaylist': True, 'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR}, + # This is a classification pass, not a download. yt-dlp emits the + # feed-level infojson/description/thumbnail from + # __process_playlist_result without consulting `download`, so + # without this a writeinfojson user gets stray files here — in + # DOWNLOAD_DIR, under yt-dlp's pl_* names, even for an add that goes + # on to fail. __write_feed_metadata writes them properly once the + # feed is accepted. See issues #1040 and #660. + 'allow_playlist_files': False, } imp = user_opts.get('impersonate') if imp is not None: @@ -1377,6 +1385,81 @@ class DownloadQueue: self.pending.put(download) await self.notifier.added(dl) + def __write_feed_metadata_sync(self, entry, etype, download_type, folder, + ytdl_options_presets, ytdl_options_overrides): + """Write the feed-level .info.json/description/thumbnail for a playlist + or channel add, using the same output template its items will use. + + yt-dlp produces these from __process_playlist_result, which ignores + ``download`` — so they used to fall out of the classification pass with + yt-dlp's own pl_* names, in DOWNLOAD_DIR, ignoring the download's folder + (issue #1040) and with no way to steer them (issue #660). Doing it here + instead means the feed type is already known, so the file lands beside + the items rather than in a differently-named sibling directory. + + Re-runs yt-dlp on a copy of the feed with no entries: that reaches the + playlist-file writing without re-extracting anything or touching + yt-dlp's private write helpers. + """ + user_opts = self._build_ytdl_options(ytdl_options_presets, ytdl_options_overrides) + wants = ('writeinfojson', 'writedescription', 'writethumbnail', 'write_all_thumbnails') + if not any(user_opts.get(key) for key in wants): + return + # An explicit allow_playlist_files=false is the user asking for exactly + # this to not happen. + if user_opts.get('allow_playlist_files') is False: + return + + dldirectory, error_message = self.__calc_download_path(download_type, folder) + if error_message is not None: + return + + template = ( + self.config.OUTPUT_TEMPLATE_CHANNEL if etype == 'channel' + else self.config.OUTPUT_TEMPLATE_PLAYLIST + ) or self.config.OUTPUT_TEMPLATE + + debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG) + params = { + **user_opts, + 'quiet': not debug_logging, + 'verbose': debug_logging, + 'no_color': True, + 'skip_download': True, + 'extract_flat': True, + 'allow_playlist_files': True, + 'paths': {"home": dldirectory, "temp": self.config.TEMP_DIR}, + # Feed-level keys only; per-item names are resolved by __add_download. + 'outtmpl': { + 'pl_infojson': template, + 'pl_thumbnail': template, + 'pl_description': template, + }, + } + imp = user_opts.get('impersonate') + if imp is not None: + params['impersonate'] = yt_dlp.networking.impersonate.ImpersonateTarget.from_str(imp) + + # A copy: process_ie_result mutates entries/requested_entries, and the + # caller still needs the real feed dict to queue the items. + feed = {k: v for k, v in entry.items() if k != 'entries'} + feed['entries'] = [] + yt_dlp.YoutubeDL(params=params).process_ie_result(feed, download=False) + + async def __write_feed_metadata(self, entry, etype, download_type, folder, + ytdl_options_presets, ytdl_options_overrides): + try: + await asyncio.get_running_loop().run_in_executor( + None, + partial( + self.__write_feed_metadata_sync, entry, etype, download_type, folder, + ytdl_options_presets, ytdl_options_overrides, + ), + ) + except Exception as exc: + # Supplemental output must never fail the add. + log.warning(f'Could not write {etype} metadata files: {exc}') + async def __add_entry( self, entry, @@ -1453,6 +1536,10 @@ class DownloadQueue: entries = list(entries) total_entries = len(entries) log.info(f'{etype} detected with {total_entries} entries') + await self.__write_feed_metadata( + entry, etype, download_type, folder, + ytdl_options_presets, ytdl_options_overrides, + ) index_digits = len(str(total_entries)) results = [] if playlist_item_limit > 0: