diff --git a/README.md b/README.md index e1451b4..3a7cee2 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ Certain values can be set via environment variables, using the `-e` parameter on * __ALLOW_PRIVATE_ADDRESSES__: Whether to allow downloads from private, loopback, link-local and other non-global addresses. Defaults to `false`, which protects against SSRF by refusing URLs that resolve to internal hosts. Set to `true` only in trusted environments — for example when routing traffic through a proxy/VPN client in Fake-IP mode (sing-box, Clash, Mihomo), which resolves hosts to the `198.18.0.0/15` range. Enabling this disables the SSRF protection entirely, so only use it when you control the network. You do **not** need this to use a proxy on an internal address: a proxy configured through the `proxy` option in `YTDL_OPTIONS` (or the `*_proxy` environment variables) is always reachable at its own host and port, wherever it lives. * __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled). +A filename that would exceed the limit the filesystem accepts is shortened to fit, keeping its extension, with room left for the suffixes yt-dlp adds while downloading. Sites that put a long description in the title would otherwise fail the download outright with `File name too long`. Use `trim_file_name` in `YTDL_OPTIONS` if you want names shorter than the filesystem's own limit, or `restrictfilenames` to strip non-ASCII characters. + Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a feed-level `.info.json` and thumbnail when you add a playlist or channel. These reuse the template of the items they belong to — `OUTPUT_TEMPLATE_CHANNEL` or `OUTPUT_TEMPLATE_PLAYLIST` — evaluated against the feed itself, so with the defaults they land in the same folder as the videos, named after the feed. Set `allow_playlist_files` to `false` in `YTDL_OPTIONS` to skip them. ### 🌐 Web Server & URLs diff --git a/app/tests/test_ytdl_utils.py b/app/tests/test_ytdl_utils.py index 858a2aa..5972aee 100644 --- a/app/tests/test_ytdl_utils.py +++ b/app/tests/test_ytdl_utils.py @@ -376,6 +376,49 @@ class ConfinedYoutubeDLTests(unittest.TestCase): self.assertEqual(self._prepared_path(""), "") self.assertEqual(self._prepared_path("-"), "-") + def test_overlong_name_is_trimmed_to_fit_the_filesystem(self): + # A title long enough to blow the filename limit is what made these + # downloads fail outright with [Errno 36] File name too long. + long_path = os.path.join(self.base, "a" * 400 + ".mp4") + + result = self._prepared_path(long_path) + + name = os.path.basename(result) + self.assertTrue(name.endswith(".mp4")) + self.assertLessEqual(len(name.encode("utf-8")), 255 - 32) + self.assertEqual(os.path.dirname(result), self.base) + # The file must still be writable once yt-dlp adds its own suffixes. + self.assertLessEqual(len(f"{name}.f1229065279304024v.part".encode("utf-8")), 255) + + def test_name_within_the_limit_is_left_alone(self): + ok = os.path.join(self.base, "Ordinary Title.mp4") + self.assertEqual(self._prepared_path(ok), ok) + + def test_limit_counts_bytes_not_characters(self): + # 200 CJK characters are 600 bytes: a character count would pass this. + long_path = os.path.join(self.base, "音" * 200 + ".mp4") + + name = os.path.basename(self._prepared_path(long_path)) + + self.assertLessEqual(len(name.encode("utf-8")), 255 - 32) + # A trim landing mid-character must not leave a broken byte sequence. + self.assertEqual(name, name.encode("utf-8").decode("utf-8")) + self.assertTrue(name.endswith(".mp4")) + + def test_a_long_tail_is_not_mistaken_for_an_extension(self): + # os.path.splitext on a title containing a dot late in the string would + # otherwise "preserve" a 100-character extension and trim nothing. + long_path = os.path.join(self.base, "b" * 300 + "." + "c" * 100) + + name = os.path.basename(self._prepared_path(long_path)) + + self.assertLessEqual(len(name.encode("utf-8")), 255 - 32) + + def test_trimming_still_cannot_escape_the_download_directory(self): + escaping = os.path.join(self.base, "..", "..", "d" * 400 + ".mp4") + with self.assertRaises(ytdl.yt_dlp.utils.DownloadError): + self._prepared_path(escaping) + class SanitizeEntryForPickleTests(unittest.TestCase): def test_nested(self): diff --git a/app/ytdl.py b/app/ytdl.py index df26599..89dbdd0 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -147,6 +147,56 @@ def _sanitize_path_component(value: Any) -> Any: return value.lstrip('.').strip() or '_' +# Room left for the suffixes yt-dlp appends after prepare_filename has run: +# '.part' and '.ytdl' while the download is in flight, '.f' for a +# stream fetched on its own before merging, '-Frag' for fragmented +# downloads. A name trimmed to exactly the limit would still fail the moment +# one of those is added, which is what the '.part' in the reported errors is. +_NAME_SUFFIX_RESERVE_BYTES = 32 +# POSIX guarantees at least this much, and it is what ext4/xfs/btrfs allow. +_FALLBACK_NAME_MAX_BYTES = 255 +# Keep a recognisable stem even on a filesystem with a very short limit. +_MIN_STEM_BYTES = 16 +# Longer than this is not really an extension (a title ending in '.something'), +# so the whole name is treated as the stem rather than preserving it. +_MAX_EXT_BYTES = 16 + + +def _name_max_bytes(directory: str) -> int: + """The filesystem's filename limit, in bytes, for *directory*.""" + try: + return int(os.pathconf(directory or '.', 'PC_NAME_MAX')) + except (OSError, ValueError, AttributeError): + # The directory may not exist yet (CREATE_CUSTOM_DIRS makes it during + # the download), and pathconf is not available on every platform. + return _FALLBACK_NAME_MAX_BYTES + + +def _trim_to_name_max(path: str) -> str: + """Shorten the final component of *path* to what the filesystem accepts. + + The limit is a byte count, not a character count: a title of accented or + CJK characters hits it in half as many characters, or fewer. The extension + is preserved, since it is what decides how the file is handled afterwards. + """ + directory, name = os.path.split(path) + if not name: + return path + encoded = name.encode('utf-8', 'surrogatepass') + limit = _name_max_bytes(directory) - _NAME_SUFFIX_RESERVE_BYTES + if len(encoded) <= limit: + return path + + stem, ext = os.path.splitext(name) + ext_bytes = ext.encode('utf-8', 'surrogatepass') + if len(ext_bytes) > _MAX_EXT_BYTES: + stem, ext, ext_bytes = name, '', b'' + stem_limit = max(limit - len(ext_bytes), _MIN_STEM_BYTES) + # 'ignore' drops a multi-byte character the cut landed inside of. + trimmed = stem.encode('utf-8', 'surrogatepass')[:stem_limit].decode('utf-8', 'ignore').rstrip() + return os.path.join(directory, (trimmed or '_') + ext) + + class _ConfinedYoutubeDL(yt_dlp.YoutubeDL): """A ``YoutubeDL`` that refuses to emit any output path outside the allowed roots. @@ -170,6 +220,12 @@ class _ConfinedYoutubeDL(yt_dlp.YoutubeDL): def prepare_filename(self, *args, **kwargs): filename = super().prepare_filename(*args, **kwargs) + # Titles long enough to exceed the filesystem's filename limit are + # common on some sites, and the download fails outright when they do. + # Every output path comes through here, so trimming once keeps the + # main file, its chapter files, thumbnails and subtitles consistent. + if filename and filename != '-': + filename = _trim_to_name_max(filename) if filename and filename != '-' and self._allowed_roots: resolved = os.path.realpath(filename) if not any(_is_within_directory(root, resolved) for root in self._allowed_roots):