Compare commits

..

6 Commits

Author SHA1 Message Date
Alex Shnitman 9ca78be199 Merge PR #1025: fill missing album-artist metadata for audio downloads
Adds _AlbumArtistPostProcessor, a yt-dlp pre_process postprocessor that
fills album_artist from the '<artist> - Topic' channel/uploader signal,
falling back to the first credited artist, when album metadata exists
but no album artist is set.

Closes #1025.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-16 23:03:17 +03:00
Alex Shnitman 8071611a84 upgrade dependencies 2026-07-16 23:01:23 +03:00
Your GitHub Name 220f991fae fix: prefer topic channel for album artist 2026-07-16 12:34:03 -07:00
Alex Shnitman 6d0528783c fix: block SSRF via user-submitted URLs
User-submitted URLs were passed straight to yt-dlp's generic extractor,
letting the server fetch internal endpoints (cloud metadata, loopback,
RFC1918 hosts). Add a url_guard.validate_url check at every URL ingress
(add, subscribe, and nested playlist recursion) that rejects non-http(s)
schemes and hosts resolving to non-global addresses, while leaving bare
video IDs and search prefixes untouched.
2026-07-16 21:31:51 +03:00
Your GitHub Name c104e30451 feat: add AlbumArtistPostProcessor to fill missing album-artist metadata 2026-07-15 15:40:36 -07:00
Alex Shnitman fdfbfed5e2 fix: prevent playlist/channel title path traversal (closes GHSA-vh67-38x4-w8pc)
Sanitize path separators and .. segments in playlist/channel titles before they are baked into yt-dlp output templates, and refuse downloads whose resolved output directory escapes DOWNLOAD_DIR.
2026-07-13 23:07:39 +03:00
8 changed files with 748 additions and 306 deletions
+10
View File
@@ -19,6 +19,7 @@ import yt_dlp.networking.impersonate
import bg_tasks import bg_tasks
from dl_formats import merge_ytdl_option_layers from dl_formats import merge_ytdl_option_layers
from state_store import AtomicJsonStore, read_legacy_shelf from state_store import AtomicJsonStore, read_legacy_shelf
from url_guard import validate_url
log = logging.getLogger("subscriptions") log = logging.getLogger("subscriptions")
@@ -113,6 +114,9 @@ def extract_flat_playlist(
nested_url = _entry_video_url(ent) nested_url = _entry_video_url(ent)
if not nested_url: if not nested_url:
continue continue
# nested_url comes from remote playlist content; guard it too.
if validate_url(nested_url) is not None:
continue
nested_info, nested_entries = extract_flat_playlist( nested_info, nested_entries = extract_flat_playlist(
config, config,
nested_url, nested_url,
@@ -542,6 +546,12 @@ class SubscriptionManager:
url = self._normalize_url(url) url = self._normalize_url(url)
if not url: if not url:
return {"status": "error", "msg": "Missing URL"} return {"status": "error", "msg": "Missing URL"}
# SSRF guard: block non-http(s) schemes and internal/metadata hosts
# before yt-dlp fetches the feed. May do a DNS lookup, so run off-loop.
url_error = await asyncio.get_running_loop().run_in_executor(None, validate_url, url)
if url_error is not None:
log.warning('Rejected subscription URL "%s": %s', url, url_error)
return {"status": "error", "msg": url_error}
try: try:
title_regex_stored = validate_title_regex(title_regex) title_regex_stored = validate_title_regex(title_regex)
except re.error as exc: except re.error as exc:
+102
View File
@@ -0,0 +1,102 @@
"""Tests for the SSRF URL guard (``url_guard.validate_url``)."""
from __future__ import annotations
import socket
import unittest
from unittest import mock
from url_guard import validate_url
def _addrinfo(*addrs, family=socket.AF_INET):
return [(family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (addr, 0)) for addr in addrs]
class NonUrlInputTests(unittest.TestCase):
"""Bare IDs and yt-dlp search/extractor prefixes must pass untouched."""
def test_bare_video_id_allowed(self):
self.assertIsNone(validate_url("dQw4w9WgXcQ"))
def test_ytsearch_prefix_allowed(self):
self.assertIsNone(validate_url("ytsearch:some song"))
def test_empty_string_allowed(self):
self.assertIsNone(validate_url(""))
def test_non_string_rejected(self):
self.assertIsNotNone(validate_url(None))
class SchemeTests(unittest.TestCase):
def test_file_scheme_blocked(self):
self.assertIsNotNone(validate_url("file:///etc/passwd"))
def test_ftp_scheme_blocked(self):
self.assertIsNotNone(validate_url("ftp://example.com/x"))
def test_data_scheme_blocked(self):
self.assertIsNotNone(validate_url("data://text/plain;base64,AAAA"))
class HostnameBlocklistTests(unittest.TestCase):
def test_localhost_blocked_without_lookup(self):
with mock.patch("url_guard.socket.getaddrinfo") as gai:
self.assertIsNotNone(validate_url("http://localhost:8080/x"))
gai.assert_not_called()
def test_localhost_subdomain_blocked(self):
self.assertIsNotNone(validate_url("http://foo.localhost/x"))
def test_gcp_metadata_name_blocked(self):
self.assertIsNotNone(validate_url("http://metadata.google.internal/x"))
class AddressResolutionTests(unittest.TestCase):
def _validate_with_addrs(self, url, *addrs, family=socket.AF_INET):
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo(*addrs, family=family)):
return validate_url(url)
def test_public_https_allowed(self):
self.assertIsNone(self._validate_with_addrs("https://youtube.com/watch?v=x", "142.250.1.1"))
def test_public_http_allowed(self):
self.assertIsNone(self._validate_with_addrs("http://example.com/x", "93.184.216.34"))
def test_link_local_metadata_blocked(self):
self.assertIsNotNone(self._validate_with_addrs("http://metadata/x", "169.254.169.254"))
def test_loopback_ipv4_blocked(self):
self.assertIsNotNone(self._validate_with_addrs("http://127.0.0.1/x", "127.0.0.1"))
def test_private_rfc1918_blocked(self):
self.assertIsNotNone(self._validate_with_addrs("http://intranet/x", "10.0.0.5"))
def test_decimal_ip_form_blocked(self):
# 2852039166 == 169.254.169.254; the OS resolver normalizes it.
self.assertIsNotNone(self._validate_with_addrs("http://2852039166/x", "169.254.169.254"))
def test_ipv6_loopback_blocked(self):
self.assertIsNotNone(
self._validate_with_addrs("http://[::1]/x", "::1", family=socket.AF_INET6)
)
def test_ipv4_mapped_ipv6_metadata_blocked(self):
self.assertIsNotNone(
self._validate_with_addrs(
"http://evil/x", "::ffff:169.254.169.254", family=socket.AF_INET6
)
)
def test_mixed_public_and_private_blocked(self):
# If any resolved address is internal, reject the whole URL.
self.assertIsNotNone(self._validate_with_addrs("http://mixed/x", "142.250.1.1", "127.0.0.1"))
def test_resolution_failure_defers_to_ytdlp(self):
with mock.patch("url_guard.socket.getaddrinfo", side_effect=socket.gaierror):
self.assertIsNone(validate_url("http://does-not-resolve.example/x"))
if __name__ == "__main__":
unittest.main()
+188
View File
@@ -15,6 +15,8 @@ from unittest.mock import MagicMock, patch
fake_yt_dlp = types.ModuleType("yt_dlp") fake_yt_dlp = types.ModuleType("yt_dlp")
fake_networking = types.ModuleType("yt_dlp.networking") fake_networking = types.ModuleType("yt_dlp.networking")
fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate") fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate")
fake_postprocessor = types.ModuleType("yt_dlp.postprocessor")
fake_postprocessor_common = types.ModuleType("yt_dlp.postprocessor.common")
fake_utils = types.ModuleType("yt_dlp.utils") fake_utils = types.ModuleType("yt_dlp.utils")
@@ -24,18 +26,27 @@ class _ImpersonateTarget:
return value return value
class _PostProcessor:
def __init__(self, downloader=None):
self._downloader = downloader
fake_impersonate.ImpersonateTarget = _ImpersonateTarget fake_impersonate.ImpersonateTarget = _ImpersonateTarget
fake_networking.impersonate = fake_impersonate fake_networking.impersonate = fake_impersonate
fake_postprocessor_common.PostProcessor = _PostProcessor
# The inner ``key`` group mirrors the real ``STR_FORMAT_RE_TMPL`` so that # The inner ``key`` group mirrors the real ``STR_FORMAT_RE_TMPL`` so that
# ``_OUTTMPL_FIELD_RE`` (compiled at import time) has the named group that # ``_OUTTMPL_FIELD_RE`` (compiled at import time) has the named group that
# ``_resolve_outtmpl_fields`` reads via ``match.group('key')``. # ``_resolve_outtmpl_fields`` reads via ``match.group('key')``.
fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>(?P<key>{}))\)(?P<format>[-0-9.]*{})" fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>(?P<key>{}))\)(?P<format>[-0-9.]*{})"
fake_utils.STR_FORMAT_TYPES = "diouxXeEfFgGcrsa" fake_utils.STR_FORMAT_TYPES = "diouxXeEfFgGcrsa"
fake_yt_dlp.networking = fake_networking fake_yt_dlp.networking = fake_networking
fake_yt_dlp.postprocessor = fake_postprocessor
fake_yt_dlp.utils = fake_utils fake_yt_dlp.utils = fake_utils
sys.modules.setdefault("yt_dlp", fake_yt_dlp) sys.modules.setdefault("yt_dlp", fake_yt_dlp)
sys.modules.setdefault("yt_dlp.networking", fake_networking) sys.modules.setdefault("yt_dlp.networking", fake_networking)
sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate) sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
sys.modules.setdefault("yt_dlp.postprocessor", fake_postprocessor)
sys.modules.setdefault("yt_dlp.postprocessor.common", fake_postprocessor_common)
sys.modules.setdefault("yt_dlp.utils", fake_utils) sys.modules.setdefault("yt_dlp.utils", fake_utils)
from ytdl import ( from ytdl import (
@@ -43,6 +54,8 @@ from ytdl import (
DownloadInfo, DownloadInfo,
_compact_persisted_entry, _compact_persisted_entry,
_convert_srt_to_txt_file, _convert_srt_to_txt_file,
_AlbumArtistPostProcessor,
_output_dir_escapes,
_resolve_outtmpl_fields, _resolve_outtmpl_fields,
_sanitize_entry_for_pickle, _sanitize_entry_for_pickle,
_sanitize_path_component, _sanitize_path_component,
@@ -53,6 +66,132 @@ from ytdl import (
_has_real_ytdlp = hasattr(sys.modules.get("yt_dlp"), "YoutubeDL") _has_real_ytdlp = hasattr(sys.modules.get("yt_dlp"), "YoutubeDL")
class AlbumArtistPostProcessorTests(unittest.TestCase):
def setUp(self):
self.postprocessor = _AlbumArtistPostProcessor()
def test_fills_album_artist_from_artist(self):
info = {'album': 'CrasH Talk', 'artist': 'ScHoolboy Q'}
_, result = self.postprocessor.run(info)
self.assertEqual(result['album_artist'], 'ScHoolboy Q')
def test_uses_main_artist_for_featured_track(self):
info = {
'album': 'CrasH Talk',
'artists': ['ScHoolboy Q · Travis Scott'],
}
_, result = self.postprocessor.run(info)
self.assertEqual(result['album_artist'], 'ScHoolboy Q')
def test_uses_topic_channel_artist_for_joint_album(self):
info = {
'album': 'Watch the Throne',
'artists': ['JAY-Z', 'Kanye West'],
'channel': 'JAY-Z & Kanye West - Topic',
}
_, result = self.postprocessor.run(info)
self.assertEqual(result['album_artist'], 'JAY-Z & Kanye West')
def test_uses_topic_uploader_and_strips_suffix_for_compilation(self):
info = {
'album': 'Compilation',
'artist': 'Track Artist',
'channel': 'Regular Channel',
'uploader': 'Various Artists - Topic',
}
_, result = self.postprocessor.run(info)
self.assertEqual(result['album_artist'], 'Various Artists')
def test_regular_channel_falls_back_to_main_artist(self):
info = {
'album': 'Album',
'artist': 'Track Artist',
'channel': 'Label Channel',
}
_, result = self.postprocessor.run(info)
self.assertEqual(result['album_artist'], 'Track Artist')
def test_preserves_explicit_various_artists(self):
info = {
'album': 'Revenge of the Dreamers III',
'artist': 'J. Cole',
'album_artist': 'Various Artists',
}
_, result = self.postprocessor.run(info)
self.assertEqual(result['album_artist'], 'Various Artists')
def test_preserves_existing_album_artists_list(self):
info = {
'album': 'Album',
'artist': 'Track Artist',
'album_artists': ['Album Artist'],
}
_, result = self.postprocessor.run(info)
self.assertEqual(result['album_artists'], ['Album Artist'])
self.assertNotIn('album_artist', result)
def test_uses_first_artist_when_artist_list_has_multiple_entries(self):
info = {'album': 'Album', 'artists': ['Main Artist', 'Featured Artist']}
_, result = self.postprocessor.run(info)
self.assertEqual(result['album_artist'], 'Main Artist')
def test_does_not_fill_without_album(self):
info = {'artist': 'Standalone Artist'}
_, result = self.postprocessor.run(info)
self.assertNotIn('album_artist', result)
self.assertNotIn('album_artists', result)
def test_does_not_fill_without_artist(self):
info = {'album': 'Instrumental Album'}
_, result = self.postprocessor.run(info)
self.assertNotIn('album_artist', result)
self.assertNotIn('album_artists', result)
class AlbumArtistRegistrationTests(unittest.TestCase):
def test_audio_download_registers_pre_process_postprocessor(self):
download = _make_test_download()
download.info.download_type = 'audio'
fake_ydl = MagicMock()
with patch('ytdl.yt_dlp.YoutubeDL', return_value=fake_ydl):
result = download._make_youtube_dl({'quiet': True})
self.assertIs(result, fake_ydl)
postprocessor, = fake_ydl.add_post_processor.call_args.args
self.assertIsInstance(postprocessor, _AlbumArtistPostProcessor)
self.assertEqual(fake_ydl.add_post_processor.call_args.kwargs, {'when': 'pre_process'})
def test_video_download_does_not_register_postprocessor(self):
download = _make_test_download()
fake_ydl = MagicMock()
with patch('ytdl.yt_dlp.YoutubeDL', return_value=fake_ydl):
download._make_youtube_dl({'quiet': True})
fake_ydl.add_post_processor.assert_not_called()
class SanitizePathComponentTests(unittest.TestCase): class SanitizePathComponentTests(unittest.TestCase):
def test_replaces_windows_invalid_chars(self): def test_replaces_windows_invalid_chars(self):
self.assertEqual(_sanitize_path_component('a:b*c?d"e<f>g|h'), "a_b_c_d_e_f_g_h") self.assertEqual(_sanitize_path_component('a:b*c?d"e<f>g|h'), "a_b_c_d_e_f_g_h")
@@ -61,6 +200,24 @@ class SanitizePathComponentTests(unittest.TestCase):
self.assertIs(_sanitize_path_component(None), None) self.assertIs(_sanitize_path_component(None), None)
self.assertEqual(_sanitize_path_component(42), 42) self.assertEqual(_sanitize_path_component(42), 42)
def test_strips_path_separators_and_traversal(self):
result = _sanitize_path_component('../../../../etc/x')
self.assertNotIn('..', result)
self.assertNotIn('/', result)
self.assertNotIn('\\', result)
def test_strips_leading_absolute_path_separator(self):
result = _sanitize_path_component('/tmp/x')
self.assertFalse(result.startswith('/'))
self.assertFalse(result.startswith('\\'))
self.assertEqual(result, '_tmp_x')
def test_collapses_slashes_in_legitimate_titles(self):
self.assertEqual(_sanitize_path_component('AC/DC'), 'AC_DC')
def test_empty_after_strip_becomes_underscore(self):
self.assertEqual(_sanitize_path_component(' '), '_')
@unittest.skipUnless(_has_real_ytdlp, "requires real yt-dlp") @unittest.skipUnless(_has_real_ytdlp, "requires real yt-dlp")
class ResolveOuttmplFieldsTests(unittest.TestCase): class ResolveOuttmplFieldsTests(unittest.TestCase):
@@ -125,6 +282,37 @@ class ResolveOuttmplFieldsTests(unittest.TestCase):
) )
self.assertEqual(result, "5 - %(title)s.%(ext)s") self.assertEqual(result, "5 - %(title)s.%(ext)s")
def test_malicious_playlist_title_cannot_escape_via_template(self):
malicious_title = '/tmp/METUBE_ARBITRARY_WRITE_POC'
entry = {
'playlist_title': malicious_title,
'playlist_index': '1',
'title': 'video',
'ext': 'mp4',
}
sanitized = {k: _sanitize_path_component(v) for k, v in entry.items()}
template = '%(playlist_title)s/%(title)s.%(ext)s'
result = _resolve_outtmpl_fields(template, sanitized, ('playlist',))
marker = result.find('%(')
literal_prefix = result[:marker] if marker != -1 else result
self.assertNotIn('..', literal_prefix)
self.assertFalse(literal_prefix.startswith('/'))
self.assertFalse(literal_prefix.startswith('\\'))
class OutputDirEscapesTests(unittest.TestCase):
def setUp(self):
self.base_dir = tempfile.mkdtemp()
def test_relative_traversal_escapes(self):
self.assertTrue(_output_dir_escapes(self.base_dir, '../../tmp/x/%(title)s.%(ext)s'))
def test_absolute_path_escapes(self):
self.assertTrue(_output_dir_escapes(self.base_dir, '/tmp/x/%(title)s.%(ext)s'))
def test_normal_playlist_dir_stays_inside(self):
self.assertFalse(_output_dir_escapes(self.base_dir, 'Playlist/%(title)s.%(ext)s'))
class SanitizeEntryForPickleTests(unittest.TestCase): class SanitizeEntryForPickleTests(unittest.TestCase):
def test_nested(self): def test_nested(self):
+89
View File
@@ -0,0 +1,89 @@
"""Lightweight SSRF guard for user-submitted URLs.
MeTube hands user-submitted URLs to yt-dlp, whose generic extractor will fetch
any ``http(s)`` URL. Without a guard, an attacker can make the server fetch
internal endpoints (cloud metadata services, loopback, RFC1918 hosts, etc.) and
have the response saved to the download directory and served back.
This module provides a single cheap validator applied at every URL ingress. It
intentionally does NOT attempt DNS-rebinding pinning, redirect-chain
re-validation, or validation of every media URL yt-dlp derives from remote
metadata — network isolation (e.g. Docker) remains the backstop for those.
"""
import ipaddress
import logging
import socket
from urllib.parse import urlsplit
log = logging.getLogger('url_guard')
_ALLOWED_SCHEMES = ('http', 'https')
# Hostnames that must be blocked without needing a lookup. ``localhost`` and any
# subdomain of it are conventionally loopback, and the GCP metadata name is a
# well-known SSRF target that may resolve via a resolver we don't control.
_BLOCKED_HOSTNAMES = ('localhost', 'metadata.google.internal')
def _hostname_is_blocked(hostname: str) -> bool:
host = hostname.rstrip('.').lower()
for blocked in _BLOCKED_HOSTNAMES:
if host == blocked or host.endswith('.' + blocked):
return True
return False
def _address_is_global(addr: str) -> bool:
try:
ip = ipaddress.ip_address(addr)
except ValueError:
return False
# Unwrap IPv4-mapped/compatible IPv6 (e.g. ::ffff:169.254.169.254) so the
# embedded IPv4 address is judged on its own merits.
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
ip = ip.ipv4_mapped
return ip.is_global
def validate_url(url: str) -> str | None:
"""Return an error message if the URL is disallowed, else ``None``.
Inputs without a ``://`` scheme separator (bare video IDs, ``ytsearch:``
and other yt-dlp search/extractor prefixes) are allowed unchanged so that
non-URL entries keep working.
"""
if not isinstance(url, str):
return 'Invalid URL'
candidate = url.strip()
if '://' not in candidate:
# Not an absolute URL: bare video IDs, ytsearch: prefixes, etc.
return None
parts = urlsplit(candidate)
scheme = parts.scheme.lower()
if scheme not in _ALLOWED_SCHEMES:
return f'URL scheme "{parts.scheme}" is not allowed (only http and https)'
hostname = parts.hostname
if not hostname:
return 'URL is missing a host'
if _hostname_is_blocked(hostname):
return f'Refusing to fetch internal host "{hostname}"'
try:
addrinfo = socket.getaddrinfo(hostname, parts.port, proto=socket.IPPROTO_TCP)
except socket.gaierror:
# Let yt-dlp surface a normal resolution error rather than masking it.
return None
except (UnicodeError, ValueError):
return f'Invalid host "{hostname}"'
for family, _type, _proto, _canonname, sockaddr in addrinfo:
addr = sockaddr[0]
if not _address_is_global(addr):
return f'Refusing to fetch internal address "{addr}" for host "{hostname}"'
return None
+83 -3
View File
@@ -19,12 +19,14 @@ import types
from typing import Any, Optional from typing import Any, Optional
import yt_dlp.networking.impersonate import yt_dlp.networking.impersonate
from yt_dlp.postprocessor.common import PostProcessor
from yt_dlp.utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES from yt_dlp.utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
import bg_tasks import bg_tasks
from dl_formats import get_format, get_opts, AUDIO_FORMATS, merge_ytdl_option_layers from dl_formats import get_format, get_opts, AUDIO_FORMATS, merge_ytdl_option_layers
from datetime import datetime from datetime import datetime
from state_store import AtomicJsonStore, from_json_compatible, read_legacy_shelf, to_json_compatible from state_store import AtomicJsonStore, from_json_compatible, read_legacy_shelf, to_json_compatible
from subscriptions import _entry_id from subscriptions import _entry_id
from url_guard import validate_url
log = logging.getLogger('ytdl') log = logging.getLogger('ytdl')
@@ -52,6 +54,52 @@ _LIVE_MAX_CHECK_INTERVAL = 3600
_LIVE_PROBE_MAX_FAILURES = 5 _LIVE_PROBE_MAX_FAILURES = 5
class _AlbumArtistPostProcessor(PostProcessor):
"""Fill missing album-artist metadata from yt-dlp's album-level signals."""
_TOPIC_SUFFIX = ' - Topic'
@staticmethod
def _has_value(value: Any) -> bool:
if isinstance(value, str):
return bool(value.strip())
if isinstance(value, (list, tuple)):
return any(_AlbumArtistPostProcessor._has_value(item) for item in value)
return value is not None
@staticmethod
def _main_artist(info) -> Optional[str]:
artists = info.get('artists')
candidates = artists if isinstance(artists, list) else [info.get('artist')]
for candidate in candidates:
if not isinstance(candidate, str) or not candidate.strip():
continue
# YouTube Music uses a spaced middle dot between credited artists.
# The first credit is the primary artist for normal albums.
return candidate.split(' · ', 1)[0].strip()
return None
@classmethod
def _topic_artist(cls, info) -> Optional[str]:
for field in ('channel', 'uploader'):
value = info.get(field)
if not isinstance(value, str) or not value.endswith(cls._TOPIC_SUFFIX):
continue
if artist := value[:-len(cls._TOPIC_SUFFIX)].strip():
return artist
return None
def run(self, info):
if not self._has_value(info.get('album')):
return [], info
if self._has_value(info.get('album_artist')) or self._has_value(info.get('album_artists')):
return [], info
if artist := self._topic_artist(info) or self._main_artist(info):
info['album_artist'] = artist
return [], info
def _is_within_directory(real_base: str, real_target: str) -> bool: def _is_within_directory(real_base: str, real_target: str) -> bool:
"""True if ``real_target`` is inside (or equal to) ``real_base``. """True if ``real_target`` is inside (or equal to) ``real_base``.
@@ -72,6 +120,7 @@ def _is_within_directory(real_base: str, real_target: str) -> bool:
# sanitised when substituting playlist/channel titles into output templates so # sanitised when substituting playlist/channel titles into output templates so
# that downloads do not fail on NTFS-mounted volumes or Windows Docker hosts. # that downloads do not fail on NTFS-mounted volumes or Windows Docker hosts.
_WINDOWS_INVALID_PATH_CHARS = re.compile(r'[\\:*?"<>|]') _WINDOWS_INVALID_PATH_CHARS = re.compile(r'[\\:*?"<>|]')
_PATH_SEP_OR_TRAVERSAL = re.compile(r'[\\/]|\.\.')
def _sanitize_path_component(value: Any) -> Any: def _sanitize_path_component(value: Any) -> Any:
@@ -81,11 +130,27 @@ def _sanitize_path_component(value: Any) -> Any:
that numeric format specs (e.g. ``%(playlist_index)02d``) still work. that numeric format specs (e.g. ``%(playlist_index)02d``) still work.
Only string values are sanitised because Windows-invalid characters are Only string values are sanitised because Windows-invalid characters are
only a concern for human-readable strings (titles, channel names, etc.) only a concern for human-readable strings (titles, channel names, etc.)
that may end up as directory names. that may end up as directory names. Path separators and ``..`` segments
are also collapsed so attacker-controlled playlist/channel titles cannot
escape the download directory via the output template.
""" """
if not isinstance(value, str): if not isinstance(value, str):
return value return value
return _WINDOWS_INVALID_PATH_CHARS.sub('_', value) value = _WINDOWS_INVALID_PATH_CHARS.sub('_', value)
value = _PATH_SEP_OR_TRAVERSAL.sub('_', value)
return value.lstrip('.').strip() or '_'
def _output_dir_escapes(base_dir: str, output_template: str) -> bool:
"""True when the literal directory prefix of *output_template* resolves outside *base_dir*."""
marker = output_template.find('%(')
literal = output_template if marker == -1 else output_template[:marker]
dir_prefix = os.path.dirname(literal)
if not dir_prefix:
return False
real_base = os.path.realpath(base_dir)
real_target = os.path.realpath(os.path.join(base_dir, dir_prefix))
return not _is_within_directory(real_base, real_target)
# Regex matching yt-dlp output-template field references, e.g. ``%(title)s`` # Regex matching yt-dlp output-template field references, e.g. ``%(title)s``
@@ -528,6 +593,12 @@ class Download:
return put_status return put_status
def _make_youtube_dl(self, params):
ydl = yt_dlp.YoutubeDL(params=params)
if getattr(self.info, 'download_type', '') == 'audio':
ydl.add_post_processor(_AlbumArtistPostProcessor(ydl), when='pre_process')
return ydl
def _download(self): def _download(self):
# Run in our own process group so cancel() can SIGKILL the whole # Run in our own process group so cancel() can SIGKILL the whole
# group (yt-dlp + any ffmpeg children it spawned for merge/postproc), # group (yt-dlp + any ffmpeg children it spawned for merge/postproc),
@@ -605,7 +676,7 @@ class Download:
[(start, end)], [(start, end)],
) )
ret = yt_dlp.YoutubeDL(params=ytdl_params).download([self.info.url]) ret = self._make_youtube_dl(ytdl_params).download([self.info.url])
self.status_queue.put({'status': 'finished' if ret == 0 else 'error'}) self.status_queue.put({'status': 'finished' if ret == 0 else 'error'})
log.info(f"Finished download for: {self.info.title}") log.info(f"Finished download for: {self.info.title}")
except yt_dlp.utils.YoutubeDLError as exc: except yt_dlp.utils.YoutubeDLError as exc:
@@ -1205,6 +1276,8 @@ class DownloadQueue:
if playlist_item_limit > 0: if playlist_item_limit > 0:
log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries') log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries')
ytdl_options['playlistend'] = playlist_item_limit ytdl_options['playlistend'] = playlist_item_limit
if _output_dir_escapes(dldirectory, output):
return {'status': 'error', 'msg': 'Refusing download: resolved output path escapes the download directory'}
download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, dl.quality, dl.format, ytdl_options, dl) download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, dl.quality, dl.format, ytdl_options, dl)
is_upcoming = ( is_upcoming = (
getattr(dl, 'live_status', None) == 'is_upcoming' getattr(dl, 'live_status', None) == 'is_upcoming'
@@ -1425,6 +1498,13 @@ class DownloadQueue:
return {'status': 'ok'} return {'status': 'ok'}
else: else:
already.add(url) already.add(url)
# SSRF guard: reject non-http(s) schemes and hosts resolving to
# internal/loopback/link-local/metadata addresses before yt-dlp fetches
# anything. run_in_executor because validate_url may perform a DNS lookup.
url_error = await asyncio.get_running_loop().run_in_executor(None, validate_url, url)
if url_error is not None:
log.warning('Rejected URL "%s": %s', url, url_error)
return {'status': 'error', 'msg': url_error}
try: try:
entry = await asyncio.get_running_loop().run_in_executor( entry = await asyncio.get_running_loop().run_in_executor(
None, None,
+8 -8
View File
@@ -33,10 +33,10 @@
"@angular/platform-browser-dynamic": "^22.0.6", "@angular/platform-browser-dynamic": "^22.0.6",
"@angular/service-worker": "^22.0.6", "@angular/service-worker": "^22.0.6",
"@fortawesome/angular-fontawesome": "~4.0.0", "@fortawesome/angular-fontawesome": "~4.0.0",
"@fortawesome/fontawesome-svg-core": "^7.3.0", "@fortawesome/fontawesome-svg-core": "^7.3.1",
"@fortawesome/free-brands-svg-icons": "^7.3.0", "@fortawesome/free-brands-svg-icons": "^7.3.1",
"@fortawesome/free-regular-svg-icons": "^7.3.0", "@fortawesome/free-regular-svg-icons": "^7.3.1",
"@fortawesome/free-solid-svg-icons": "^7.3.0", "@fortawesome/free-solid-svg-icons": "^7.3.1",
"@ng-bootstrap/ng-bootstrap": "^21.0.0", "@ng-bootstrap/ng-bootstrap": "^21.0.0",
"@ng-select/ng-select": "^23.2.0", "@ng-select/ng-select": "^23.2.0",
"@popperjs/core": "^2.11.8", "@popperjs/core": "^2.11.8",
@@ -49,13 +49,13 @@
}, },
"devDependencies": { "devDependencies": {
"@angular-eslint/builder": "22.0.0", "@angular-eslint/builder": "22.0.0",
"@angular/build": "^22.0.5", "@angular/build": "^22.0.7",
"@angular/cli": "^22.0.5", "@angular/cli": "^22.0.7",
"@angular/compiler-cli": "^22.0.6", "@angular/compiler-cli": "^22.0.6",
"@angular/localize": "^22.0.6", "@angular/localize": "^22.0.6",
"@eslint/js": "^9.39.4", "@eslint/js": "^9.39.5",
"angular-eslint": "22.0.0", "angular-eslint": "22.0.0",
"eslint": "^9.39.4", "eslint": "^9.39.5",
"jsdom": "^27.4.0", "jsdom": "^27.4.0",
"typescript": "~6.0.3", "typescript": "~6.0.3",
"typescript-eslint": "8.62.0", "typescript-eslint": "8.62.0",
+258 -285
View File
File diff suppressed because it is too large Load Diff
Generated
+10 -10
View File
@@ -106,14 +106,14 @@ wheels = [
[[package]] [[package]]
name = "anyio" name = "anyio"
version = "4.14.1" version = "4.14.2"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "idna" }, { name = "idna" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" } sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" }, { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
] ]
[[package]] [[package]]
@@ -366,15 +366,15 @@ wheels = [
[[package]] [[package]]
name = "deno" name = "deno"
version = "2.9.2" version = "2.9.3"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/dc/3c/bf3a4e2c00f7eefd4ed2fd033a958cab3e92dcc539d3ab80351850cae99c/deno-2.9.2.tar.gz", hash = "sha256:3b77d7689c15fb2c47d5a3927dc8cf70dbd408bc92e20c70da079c6d78980e34", size = 8164, upload-time = "2026-07-08T14:38:54.267Z" } sdist = { url = "https://files.pythonhosted.org/packages/98/ab/638749d76881f74d100a079414745f0a0fd20bba38e27ab4a3a6495d8264/deno-2.9.3.tar.gz", hash = "sha256:73268cbac7f7c4ff1983e49420a93f3f3f2f2e4e4372436f0942d04d02c9e943", size = 8166, upload-time = "2026-07-15T15:37:50.565Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/27/fe/4a02a256f19b3f31ac17ec37f6467ecf523601ed753e6c9177865dd2bbb2/deno-2.9.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fb4bb5362d02c193f8086668789076f366416980b79277c06ea6cff73866b668", size = 42341183, upload-time = "2026-07-08T14:38:36.384Z" }, { url = "https://files.pythonhosted.org/packages/07/fa/b4e1e2b3b894ed992c78a6796a43dc775edff895239d83c0e9a10457f4f7/deno-2.9.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:1deef9fe97ab28d5d4fea07693075e9ccc62f4ea97a653047a770beb01ecd307", size = 42354026, upload-time = "2026-07-15T15:37:34.977Z" },
{ url = "https://files.pythonhosted.org/packages/5a/3e/235530ac9b9c206ee8c098e2b215f711a71d915d8d7d2be77599c8c1db4b/deno-2.9.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f6f74fd8698a8a9a31af14eed5730f1964ab50eaa0129079d11d91fa92496f60", size = 37984508, upload-time = "2026-07-08T14:38:40.006Z" }, { url = "https://files.pythonhosted.org/packages/3d/cc/4090ef7370005b740efe57ec6ff75a3471393abe9816f1459e73dbbf0429/deno-2.9.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:78635cd9f0802ce7125aa3ba6feb5775460f55d432611374f16b454ffb8a1308", size = 37997909, upload-time = "2026-07-15T15:37:38.291Z" },
{ url = "https://files.pythonhosted.org/packages/7b/27/06a166fca538c06dfae4c45dbb92c2ebf3083341141966a3d649272ea734/deno-2.9.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8b2fd9318d452b7e44c45968916670a0c5452d72796e1af9abfaa7a389e4aa6b", size = 42089854, upload-time = "2026-07-08T14:38:43.841Z" }, { url = "https://files.pythonhosted.org/packages/14/07/70b7c965bbca5f3d158859acda0e90c041f1fa98b8c9ac8eff3fafd8b96c/deno-2.9.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:7c1c669827a480ef9e2710dd762560811da35dca04a9e90b23c0518587dadb8a", size = 42100873, upload-time = "2026-07-15T15:37:41.539Z" },
{ url = "https://files.pythonhosted.org/packages/09/b4/38338550a5ca0a7bcc39991e3d48e46ce3699413460dff32f383a611d822/deno-2.9.2-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:bf0471db7238e90810bca77a8d2c28646317b5a3fd9e5cdfdf73b251fde11b83", size = 43916921, upload-time = "2026-07-08T14:38:48.057Z" }, { url = "https://files.pythonhosted.org/packages/7b/f5/0a07cc19a27476011719dd017a98950c7ccc33058c2ffc07019ba3ad9a5e/deno-2.9.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:3e724d5c8df13a90f6b50b16cc8df1acfe64c61a7736a774e11d61f3bca3a3b1", size = 43928967, upload-time = "2026-07-15T15:37:45.175Z" },
{ url = "https://files.pythonhosted.org/packages/a9/3d/c490c27e1f720baf014bb2c18b29d09b11c21d69fadfd69639e5904cdc95/deno-2.9.2-py3-none-win_amd64.whl", hash = "sha256:8e815d66b3e1314d028b2f0a05c75a331dd7025f4c825e12dd1f057207cee1f6", size = 41621982, upload-time = "2026-07-08T14:38:51.928Z" }, { url = "https://files.pythonhosted.org/packages/e1/9a/030847bd4ea6cefbb471cae2b63c9ab29a3ab6c1809484447752bbcbcd52/deno-2.9.3-py3-none-win_amd64.whl", hash = "sha256:e2ebe2b5c1a7ee9daeaf8caf14ebc33fda96b4c2399da259e761fe2f82d22fa4", size = 41630201, upload-time = "2026-07-15T15:37:48.344Z" },
] ]
[[package]] [[package]]