mirror of
https://github.com/alexta69/metube.git
synced 2026-09-21 21:45:04 +00:00
Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 9ca78be199 | |||
| 8071611a84 | |||
| 220f991fae | |||
| 6d0528783c | |||
| c104e30451 | |||
| fdfbfed5e2 |
@@ -19,6 +19,7 @@ import yt_dlp.networking.impersonate
|
||||
import bg_tasks
|
||||
from dl_formats import merge_ytdl_option_layers
|
||||
from state_store import AtomicJsonStore, read_legacy_shelf
|
||||
from url_guard import validate_url
|
||||
|
||||
log = logging.getLogger("subscriptions")
|
||||
|
||||
@@ -113,6 +114,9 @@ def extract_flat_playlist(
|
||||
nested_url = _entry_video_url(ent)
|
||||
if not nested_url:
|
||||
continue
|
||||
# nested_url comes from remote playlist content; guard it too.
|
||||
if validate_url(nested_url) is not None:
|
||||
continue
|
||||
nested_info, nested_entries = extract_flat_playlist(
|
||||
config,
|
||||
nested_url,
|
||||
@@ -542,6 +546,12 @@ class SubscriptionManager:
|
||||
url = self._normalize_url(url)
|
||||
if not url:
|
||||
return {"status": "error", "msg": "Missing URL"}
|
||||
# SSRF guard: block non-http(s) schemes and internal/metadata hosts
|
||||
# before yt-dlp fetches the feed. May do a DNS lookup, so run off-loop.
|
||||
url_error = await asyncio.get_running_loop().run_in_executor(None, validate_url, url)
|
||||
if url_error is not None:
|
||||
log.warning('Rejected subscription URL "%s": %s', url, url_error)
|
||||
return {"status": "error", "msg": url_error}
|
||||
try:
|
||||
title_regex_stored = validate_title_regex(title_regex)
|
||||
except re.error as exc:
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Tests for the SSRF URL guard (``url_guard.validate_url``)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import unittest
|
||||
from unittest import mock
|
||||
|
||||
from url_guard import validate_url
|
||||
|
||||
|
||||
def _addrinfo(*addrs, family=socket.AF_INET):
|
||||
return [(family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (addr, 0)) for addr in addrs]
|
||||
|
||||
|
||||
class NonUrlInputTests(unittest.TestCase):
|
||||
"""Bare IDs and yt-dlp search/extractor prefixes must pass untouched."""
|
||||
|
||||
def test_bare_video_id_allowed(self):
|
||||
self.assertIsNone(validate_url("dQw4w9WgXcQ"))
|
||||
|
||||
def test_ytsearch_prefix_allowed(self):
|
||||
self.assertIsNone(validate_url("ytsearch:some song"))
|
||||
|
||||
def test_empty_string_allowed(self):
|
||||
self.assertIsNone(validate_url(""))
|
||||
|
||||
def test_non_string_rejected(self):
|
||||
self.assertIsNotNone(validate_url(None))
|
||||
|
||||
|
||||
class SchemeTests(unittest.TestCase):
|
||||
def test_file_scheme_blocked(self):
|
||||
self.assertIsNotNone(validate_url("file:///etc/passwd"))
|
||||
|
||||
def test_ftp_scheme_blocked(self):
|
||||
self.assertIsNotNone(validate_url("ftp://example.com/x"))
|
||||
|
||||
def test_data_scheme_blocked(self):
|
||||
self.assertIsNotNone(validate_url("data://text/plain;base64,AAAA"))
|
||||
|
||||
|
||||
class HostnameBlocklistTests(unittest.TestCase):
|
||||
def test_localhost_blocked_without_lookup(self):
|
||||
with mock.patch("url_guard.socket.getaddrinfo") as gai:
|
||||
self.assertIsNotNone(validate_url("http://localhost:8080/x"))
|
||||
gai.assert_not_called()
|
||||
|
||||
def test_localhost_subdomain_blocked(self):
|
||||
self.assertIsNotNone(validate_url("http://foo.localhost/x"))
|
||||
|
||||
def test_gcp_metadata_name_blocked(self):
|
||||
self.assertIsNotNone(validate_url("http://metadata.google.internal/x"))
|
||||
|
||||
|
||||
class AddressResolutionTests(unittest.TestCase):
|
||||
def _validate_with_addrs(self, url, *addrs, family=socket.AF_INET):
|
||||
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo(*addrs, family=family)):
|
||||
return validate_url(url)
|
||||
|
||||
def test_public_https_allowed(self):
|
||||
self.assertIsNone(self._validate_with_addrs("https://youtube.com/watch?v=x", "142.250.1.1"))
|
||||
|
||||
def test_public_http_allowed(self):
|
||||
self.assertIsNone(self._validate_with_addrs("http://example.com/x", "93.184.216.34"))
|
||||
|
||||
def test_link_local_metadata_blocked(self):
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://metadata/x", "169.254.169.254"))
|
||||
|
||||
def test_loopback_ipv4_blocked(self):
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://127.0.0.1/x", "127.0.0.1"))
|
||||
|
||||
def test_private_rfc1918_blocked(self):
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://intranet/x", "10.0.0.5"))
|
||||
|
||||
def test_decimal_ip_form_blocked(self):
|
||||
# 2852039166 == 169.254.169.254; the OS resolver normalizes it.
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://2852039166/x", "169.254.169.254"))
|
||||
|
||||
def test_ipv6_loopback_blocked(self):
|
||||
self.assertIsNotNone(
|
||||
self._validate_with_addrs("http://[::1]/x", "::1", family=socket.AF_INET6)
|
||||
)
|
||||
|
||||
def test_ipv4_mapped_ipv6_metadata_blocked(self):
|
||||
self.assertIsNotNone(
|
||||
self._validate_with_addrs(
|
||||
"http://evil/x", "::ffff:169.254.169.254", family=socket.AF_INET6
|
||||
)
|
||||
)
|
||||
|
||||
def test_mixed_public_and_private_blocked(self):
|
||||
# If any resolved address is internal, reject the whole URL.
|
||||
self.assertIsNotNone(self._validate_with_addrs("http://mixed/x", "142.250.1.1", "127.0.0.1"))
|
||||
|
||||
def test_resolution_failure_defers_to_ytdlp(self):
|
||||
with mock.patch("url_guard.socket.getaddrinfo", side_effect=socket.gaierror):
|
||||
self.assertIsNone(validate_url("http://does-not-resolve.example/x"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -15,6 +15,8 @@ from unittest.mock import MagicMock, patch
|
||||
fake_yt_dlp = types.ModuleType("yt_dlp")
|
||||
fake_networking = types.ModuleType("yt_dlp.networking")
|
||||
fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate")
|
||||
fake_postprocessor = types.ModuleType("yt_dlp.postprocessor")
|
||||
fake_postprocessor_common = types.ModuleType("yt_dlp.postprocessor.common")
|
||||
fake_utils = types.ModuleType("yt_dlp.utils")
|
||||
|
||||
|
||||
@@ -24,18 +26,27 @@ class _ImpersonateTarget:
|
||||
return value
|
||||
|
||||
|
||||
class _PostProcessor:
|
||||
def __init__(self, downloader=None):
|
||||
self._downloader = downloader
|
||||
|
||||
|
||||
fake_impersonate.ImpersonateTarget = _ImpersonateTarget
|
||||
fake_networking.impersonate = fake_impersonate
|
||||
fake_postprocessor_common.PostProcessor = _PostProcessor
|
||||
# The inner ``key`` group mirrors the real ``STR_FORMAT_RE_TMPL`` so that
|
||||
# ``_OUTTMPL_FIELD_RE`` (compiled at import time) has the named group that
|
||||
# ``_resolve_outtmpl_fields`` reads via ``match.group('key')``.
|
||||
fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>(?P<key>{}))\)(?P<format>[-0-9.]*{})"
|
||||
fake_utils.STR_FORMAT_TYPES = "diouxXeEfFgGcrsa"
|
||||
fake_yt_dlp.networking = fake_networking
|
||||
fake_yt_dlp.postprocessor = fake_postprocessor
|
||||
fake_yt_dlp.utils = fake_utils
|
||||
sys.modules.setdefault("yt_dlp", fake_yt_dlp)
|
||||
sys.modules.setdefault("yt_dlp.networking", fake_networking)
|
||||
sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
|
||||
sys.modules.setdefault("yt_dlp.postprocessor", fake_postprocessor)
|
||||
sys.modules.setdefault("yt_dlp.postprocessor.common", fake_postprocessor_common)
|
||||
sys.modules.setdefault("yt_dlp.utils", fake_utils)
|
||||
|
||||
from ytdl import (
|
||||
@@ -43,6 +54,8 @@ from ytdl import (
|
||||
DownloadInfo,
|
||||
_compact_persisted_entry,
|
||||
_convert_srt_to_txt_file,
|
||||
_AlbumArtistPostProcessor,
|
||||
_output_dir_escapes,
|
||||
_resolve_outtmpl_fields,
|
||||
_sanitize_entry_for_pickle,
|
||||
_sanitize_path_component,
|
||||
@@ -53,6 +66,132 @@ from ytdl import (
|
||||
_has_real_ytdlp = hasattr(sys.modules.get("yt_dlp"), "YoutubeDL")
|
||||
|
||||
|
||||
class AlbumArtistPostProcessorTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.postprocessor = _AlbumArtistPostProcessor()
|
||||
|
||||
def test_fills_album_artist_from_artist(self):
|
||||
info = {'album': 'CrasH Talk', 'artist': 'ScHoolboy Q'}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'ScHoolboy Q')
|
||||
|
||||
def test_uses_main_artist_for_featured_track(self):
|
||||
info = {
|
||||
'album': 'CrasH Talk',
|
||||
'artists': ['ScHoolboy Q · Travis Scott'],
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'ScHoolboy Q')
|
||||
|
||||
def test_uses_topic_channel_artist_for_joint_album(self):
|
||||
info = {
|
||||
'album': 'Watch the Throne',
|
||||
'artists': ['JAY-Z', 'Kanye West'],
|
||||
'channel': 'JAY-Z & Kanye West - Topic',
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'JAY-Z & Kanye West')
|
||||
|
||||
def test_uses_topic_uploader_and_strips_suffix_for_compilation(self):
|
||||
info = {
|
||||
'album': 'Compilation',
|
||||
'artist': 'Track Artist',
|
||||
'channel': 'Regular Channel',
|
||||
'uploader': 'Various Artists - Topic',
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'Various Artists')
|
||||
|
||||
def test_regular_channel_falls_back_to_main_artist(self):
|
||||
info = {
|
||||
'album': 'Album',
|
||||
'artist': 'Track Artist',
|
||||
'channel': 'Label Channel',
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'Track Artist')
|
||||
|
||||
def test_preserves_explicit_various_artists(self):
|
||||
info = {
|
||||
'album': 'Revenge of the Dreamers III',
|
||||
'artist': 'J. Cole',
|
||||
'album_artist': 'Various Artists',
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'Various Artists')
|
||||
|
||||
def test_preserves_existing_album_artists_list(self):
|
||||
info = {
|
||||
'album': 'Album',
|
||||
'artist': 'Track Artist',
|
||||
'album_artists': ['Album Artist'],
|
||||
}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artists'], ['Album Artist'])
|
||||
self.assertNotIn('album_artist', result)
|
||||
|
||||
def test_uses_first_artist_when_artist_list_has_multiple_entries(self):
|
||||
info = {'album': 'Album', 'artists': ['Main Artist', 'Featured Artist']}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertEqual(result['album_artist'], 'Main Artist')
|
||||
|
||||
def test_does_not_fill_without_album(self):
|
||||
info = {'artist': 'Standalone Artist'}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertNotIn('album_artist', result)
|
||||
self.assertNotIn('album_artists', result)
|
||||
|
||||
def test_does_not_fill_without_artist(self):
|
||||
info = {'album': 'Instrumental Album'}
|
||||
|
||||
_, result = self.postprocessor.run(info)
|
||||
|
||||
self.assertNotIn('album_artist', result)
|
||||
self.assertNotIn('album_artists', result)
|
||||
|
||||
|
||||
class AlbumArtistRegistrationTests(unittest.TestCase):
|
||||
def test_audio_download_registers_pre_process_postprocessor(self):
|
||||
download = _make_test_download()
|
||||
download.info.download_type = 'audio'
|
||||
fake_ydl = MagicMock()
|
||||
|
||||
with patch('ytdl.yt_dlp.YoutubeDL', return_value=fake_ydl):
|
||||
result = download._make_youtube_dl({'quiet': True})
|
||||
|
||||
self.assertIs(result, fake_ydl)
|
||||
postprocessor, = fake_ydl.add_post_processor.call_args.args
|
||||
self.assertIsInstance(postprocessor, _AlbumArtistPostProcessor)
|
||||
self.assertEqual(fake_ydl.add_post_processor.call_args.kwargs, {'when': 'pre_process'})
|
||||
|
||||
def test_video_download_does_not_register_postprocessor(self):
|
||||
download = _make_test_download()
|
||||
fake_ydl = MagicMock()
|
||||
|
||||
with patch('ytdl.yt_dlp.YoutubeDL', return_value=fake_ydl):
|
||||
download._make_youtube_dl({'quiet': True})
|
||||
|
||||
fake_ydl.add_post_processor.assert_not_called()
|
||||
|
||||
|
||||
class SanitizePathComponentTests(unittest.TestCase):
|
||||
def test_replaces_windows_invalid_chars(self):
|
||||
self.assertEqual(_sanitize_path_component('a:b*c?d"e<f>g|h'), "a_b_c_d_e_f_g_h")
|
||||
@@ -61,6 +200,24 @@ class SanitizePathComponentTests(unittest.TestCase):
|
||||
self.assertIs(_sanitize_path_component(None), None)
|
||||
self.assertEqual(_sanitize_path_component(42), 42)
|
||||
|
||||
def test_strips_path_separators_and_traversal(self):
|
||||
result = _sanitize_path_component('../../../../etc/x')
|
||||
self.assertNotIn('..', result)
|
||||
self.assertNotIn('/', result)
|
||||
self.assertNotIn('\\', result)
|
||||
|
||||
def test_strips_leading_absolute_path_separator(self):
|
||||
result = _sanitize_path_component('/tmp/x')
|
||||
self.assertFalse(result.startswith('/'))
|
||||
self.assertFalse(result.startswith('\\'))
|
||||
self.assertEqual(result, '_tmp_x')
|
||||
|
||||
def test_collapses_slashes_in_legitimate_titles(self):
|
||||
self.assertEqual(_sanitize_path_component('AC/DC'), 'AC_DC')
|
||||
|
||||
def test_empty_after_strip_becomes_underscore(self):
|
||||
self.assertEqual(_sanitize_path_component(' '), '_')
|
||||
|
||||
|
||||
@unittest.skipUnless(_has_real_ytdlp, "requires real yt-dlp")
|
||||
class ResolveOuttmplFieldsTests(unittest.TestCase):
|
||||
@@ -125,6 +282,37 @@ class ResolveOuttmplFieldsTests(unittest.TestCase):
|
||||
)
|
||||
self.assertEqual(result, "5 - %(title)s.%(ext)s")
|
||||
|
||||
def test_malicious_playlist_title_cannot_escape_via_template(self):
|
||||
malicious_title = '/tmp/METUBE_ARBITRARY_WRITE_POC'
|
||||
entry = {
|
||||
'playlist_title': malicious_title,
|
||||
'playlist_index': '1',
|
||||
'title': 'video',
|
||||
'ext': 'mp4',
|
||||
}
|
||||
sanitized = {k: _sanitize_path_component(v) for k, v in entry.items()}
|
||||
template = '%(playlist_title)s/%(title)s.%(ext)s'
|
||||
result = _resolve_outtmpl_fields(template, sanitized, ('playlist',))
|
||||
marker = result.find('%(')
|
||||
literal_prefix = result[:marker] if marker != -1 else result
|
||||
self.assertNotIn('..', literal_prefix)
|
||||
self.assertFalse(literal_prefix.startswith('/'))
|
||||
self.assertFalse(literal_prefix.startswith('\\'))
|
||||
|
||||
|
||||
class OutputDirEscapesTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.base_dir = tempfile.mkdtemp()
|
||||
|
||||
def test_relative_traversal_escapes(self):
|
||||
self.assertTrue(_output_dir_escapes(self.base_dir, '../../tmp/x/%(title)s.%(ext)s'))
|
||||
|
||||
def test_absolute_path_escapes(self):
|
||||
self.assertTrue(_output_dir_escapes(self.base_dir, '/tmp/x/%(title)s.%(ext)s'))
|
||||
|
||||
def test_normal_playlist_dir_stays_inside(self):
|
||||
self.assertFalse(_output_dir_escapes(self.base_dir, 'Playlist/%(title)s.%(ext)s'))
|
||||
|
||||
|
||||
class SanitizeEntryForPickleTests(unittest.TestCase):
|
||||
def test_nested(self):
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
"""Lightweight SSRF guard for user-submitted URLs.
|
||||
|
||||
MeTube hands user-submitted URLs to yt-dlp, whose generic extractor will fetch
|
||||
any ``http(s)`` URL. Without a guard, an attacker can make the server fetch
|
||||
internal endpoints (cloud metadata services, loopback, RFC1918 hosts, etc.) and
|
||||
have the response saved to the download directory and served back.
|
||||
|
||||
This module provides a single cheap validator applied at every URL ingress. It
|
||||
intentionally does NOT attempt DNS-rebinding pinning, redirect-chain
|
||||
re-validation, or validation of every media URL yt-dlp derives from remote
|
||||
metadata — network isolation (e.g. Docker) remains the backstop for those.
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
log = logging.getLogger('url_guard')
|
||||
|
||||
_ALLOWED_SCHEMES = ('http', 'https')
|
||||
|
||||
# Hostnames that must be blocked without needing a lookup. ``localhost`` and any
|
||||
# subdomain of it are conventionally loopback, and the GCP metadata name is a
|
||||
# well-known SSRF target that may resolve via a resolver we don't control.
|
||||
_BLOCKED_HOSTNAMES = ('localhost', 'metadata.google.internal')
|
||||
|
||||
|
||||
def _hostname_is_blocked(hostname: str) -> bool:
|
||||
host = hostname.rstrip('.').lower()
|
||||
for blocked in _BLOCKED_HOSTNAMES:
|
||||
if host == blocked or host.endswith('.' + blocked):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _address_is_global(addr: str) -> bool:
|
||||
try:
|
||||
ip = ipaddress.ip_address(addr)
|
||||
except ValueError:
|
||||
return False
|
||||
# Unwrap IPv4-mapped/compatible IPv6 (e.g. ::ffff:169.254.169.254) so the
|
||||
# embedded IPv4 address is judged on its own merits.
|
||||
if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
|
||||
ip = ip.ipv4_mapped
|
||||
return ip.is_global
|
||||
|
||||
|
||||
def validate_url(url: str) -> str | None:
|
||||
"""Return an error message if the URL is disallowed, else ``None``.
|
||||
|
||||
Inputs without a ``://`` scheme separator (bare video IDs, ``ytsearch:``
|
||||
and other yt-dlp search/extractor prefixes) are allowed unchanged so that
|
||||
non-URL entries keep working.
|
||||
"""
|
||||
if not isinstance(url, str):
|
||||
return 'Invalid URL'
|
||||
|
||||
candidate = url.strip()
|
||||
if '://' not in candidate:
|
||||
# Not an absolute URL: bare video IDs, ytsearch: prefixes, etc.
|
||||
return None
|
||||
|
||||
parts = urlsplit(candidate)
|
||||
scheme = parts.scheme.lower()
|
||||
if scheme not in _ALLOWED_SCHEMES:
|
||||
return f'URL scheme "{parts.scheme}" is not allowed (only http and https)'
|
||||
|
||||
hostname = parts.hostname
|
||||
if not hostname:
|
||||
return 'URL is missing a host'
|
||||
|
||||
if _hostname_is_blocked(hostname):
|
||||
return f'Refusing to fetch internal host "{hostname}"'
|
||||
|
||||
try:
|
||||
addrinfo = socket.getaddrinfo(hostname, parts.port, proto=socket.IPPROTO_TCP)
|
||||
except socket.gaierror:
|
||||
# Let yt-dlp surface a normal resolution error rather than masking it.
|
||||
return None
|
||||
except (UnicodeError, ValueError):
|
||||
return f'Invalid host "{hostname}"'
|
||||
|
||||
for family, _type, _proto, _canonname, sockaddr in addrinfo:
|
||||
addr = sockaddr[0]
|
||||
if not _address_is_global(addr):
|
||||
return f'Refusing to fetch internal address "{addr}" for host "{hostname}"'
|
||||
|
||||
return None
|
||||
+83
-3
@@ -19,12 +19,14 @@ import types
|
||||
from typing import Any, Optional
|
||||
|
||||
import yt_dlp.networking.impersonate
|
||||
from yt_dlp.postprocessor.common import PostProcessor
|
||||
from yt_dlp.utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
|
||||
import bg_tasks
|
||||
from dl_formats import get_format, get_opts, AUDIO_FORMATS, merge_ytdl_option_layers
|
||||
from datetime import datetime
|
||||
from state_store import AtomicJsonStore, from_json_compatible, read_legacy_shelf, to_json_compatible
|
||||
from subscriptions import _entry_id
|
||||
from url_guard import validate_url
|
||||
|
||||
log = logging.getLogger('ytdl')
|
||||
|
||||
@@ -52,6 +54,52 @@ _LIVE_MAX_CHECK_INTERVAL = 3600
|
||||
_LIVE_PROBE_MAX_FAILURES = 5
|
||||
|
||||
|
||||
class _AlbumArtistPostProcessor(PostProcessor):
|
||||
"""Fill missing album-artist metadata from yt-dlp's album-level signals."""
|
||||
|
||||
_TOPIC_SUFFIX = ' - Topic'
|
||||
|
||||
@staticmethod
|
||||
def _has_value(value: Any) -> bool:
|
||||
if isinstance(value, str):
|
||||
return bool(value.strip())
|
||||
if isinstance(value, (list, tuple)):
|
||||
return any(_AlbumArtistPostProcessor._has_value(item) for item in value)
|
||||
return value is not None
|
||||
|
||||
@staticmethod
|
||||
def _main_artist(info) -> Optional[str]:
|
||||
artists = info.get('artists')
|
||||
candidates = artists if isinstance(artists, list) else [info.get('artist')]
|
||||
for candidate in candidates:
|
||||
if not isinstance(candidate, str) or not candidate.strip():
|
||||
continue
|
||||
# YouTube Music uses a spaced middle dot between credited artists.
|
||||
# The first credit is the primary artist for normal albums.
|
||||
return candidate.split(' · ', 1)[0].strip()
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _topic_artist(cls, info) -> Optional[str]:
|
||||
for field in ('channel', 'uploader'):
|
||||
value = info.get(field)
|
||||
if not isinstance(value, str) or not value.endswith(cls._TOPIC_SUFFIX):
|
||||
continue
|
||||
if artist := value[:-len(cls._TOPIC_SUFFIX)].strip():
|
||||
return artist
|
||||
return None
|
||||
|
||||
def run(self, info):
|
||||
if not self._has_value(info.get('album')):
|
||||
return [], info
|
||||
if self._has_value(info.get('album_artist')) or self._has_value(info.get('album_artists')):
|
||||
return [], info
|
||||
|
||||
if artist := self._topic_artist(info) or self._main_artist(info):
|
||||
info['album_artist'] = artist
|
||||
return [], info
|
||||
|
||||
|
||||
def _is_within_directory(real_base: str, real_target: str) -> bool:
|
||||
"""True if ``real_target`` is inside (or equal to) ``real_base``.
|
||||
|
||||
@@ -72,6 +120,7 @@ def _is_within_directory(real_base: str, real_target: str) -> bool:
|
||||
# sanitised when substituting playlist/channel titles into output templates so
|
||||
# that downloads do not fail on NTFS-mounted volumes or Windows Docker hosts.
|
||||
_WINDOWS_INVALID_PATH_CHARS = re.compile(r'[\\:*?"<>|]')
|
||||
_PATH_SEP_OR_TRAVERSAL = re.compile(r'[\\/]|\.\.')
|
||||
|
||||
|
||||
def _sanitize_path_component(value: Any) -> Any:
|
||||
@@ -81,11 +130,27 @@ def _sanitize_path_component(value: Any) -> Any:
|
||||
that numeric format specs (e.g. ``%(playlist_index)02d``) still work.
|
||||
Only string values are sanitised because Windows-invalid characters are
|
||||
only a concern for human-readable strings (titles, channel names, etc.)
|
||||
that may end up as directory names.
|
||||
that may end up as directory names. Path separators and ``..`` segments
|
||||
are also collapsed so attacker-controlled playlist/channel titles cannot
|
||||
escape the download directory via the output template.
|
||||
"""
|
||||
if not isinstance(value, str):
|
||||
return value
|
||||
return _WINDOWS_INVALID_PATH_CHARS.sub('_', value)
|
||||
value = _WINDOWS_INVALID_PATH_CHARS.sub('_', value)
|
||||
value = _PATH_SEP_OR_TRAVERSAL.sub('_', value)
|
||||
return value.lstrip('.').strip() or '_'
|
||||
|
||||
|
||||
def _output_dir_escapes(base_dir: str, output_template: str) -> bool:
|
||||
"""True when the literal directory prefix of *output_template* resolves outside *base_dir*."""
|
||||
marker = output_template.find('%(')
|
||||
literal = output_template if marker == -1 else output_template[:marker]
|
||||
dir_prefix = os.path.dirname(literal)
|
||||
if not dir_prefix:
|
||||
return False
|
||||
real_base = os.path.realpath(base_dir)
|
||||
real_target = os.path.realpath(os.path.join(base_dir, dir_prefix))
|
||||
return not _is_within_directory(real_base, real_target)
|
||||
|
||||
|
||||
# Regex matching yt-dlp output-template field references, e.g. ``%(title)s``
|
||||
@@ -528,6 +593,12 @@ class Download:
|
||||
|
||||
return put_status
|
||||
|
||||
def _make_youtube_dl(self, params):
|
||||
ydl = yt_dlp.YoutubeDL(params=params)
|
||||
if getattr(self.info, 'download_type', '') == 'audio':
|
||||
ydl.add_post_processor(_AlbumArtistPostProcessor(ydl), when='pre_process')
|
||||
return ydl
|
||||
|
||||
def _download(self):
|
||||
# Run in our own process group so cancel() can SIGKILL the whole
|
||||
# group (yt-dlp + any ffmpeg children it spawned for merge/postproc),
|
||||
@@ -605,7 +676,7 @@ class Download:
|
||||
[(start, end)],
|
||||
)
|
||||
|
||||
ret = yt_dlp.YoutubeDL(params=ytdl_params).download([self.info.url])
|
||||
ret = self._make_youtube_dl(ytdl_params).download([self.info.url])
|
||||
self.status_queue.put({'status': 'finished' if ret == 0 else 'error'})
|
||||
log.info(f"Finished download for: {self.info.title}")
|
||||
except yt_dlp.utils.YoutubeDLError as exc:
|
||||
@@ -1205,6 +1276,8 @@ class DownloadQueue:
|
||||
if playlist_item_limit > 0:
|
||||
log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries')
|
||||
ytdl_options['playlistend'] = playlist_item_limit
|
||||
if _output_dir_escapes(dldirectory, output):
|
||||
return {'status': 'error', 'msg': 'Refusing download: resolved output path escapes the download directory'}
|
||||
download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, dl.quality, dl.format, ytdl_options, dl)
|
||||
is_upcoming = (
|
||||
getattr(dl, 'live_status', None) == 'is_upcoming'
|
||||
@@ -1425,6 +1498,13 @@ class DownloadQueue:
|
||||
return {'status': 'ok'}
|
||||
else:
|
||||
already.add(url)
|
||||
# SSRF guard: reject non-http(s) schemes and hosts resolving to
|
||||
# internal/loopback/link-local/metadata addresses before yt-dlp fetches
|
||||
# anything. run_in_executor because validate_url may perform a DNS lookup.
|
||||
url_error = await asyncio.get_running_loop().run_in_executor(None, validate_url, url)
|
||||
if url_error is not None:
|
||||
log.warning('Rejected URL "%s": %s', url, url_error)
|
||||
return {'status': 'error', 'msg': url_error}
|
||||
try:
|
||||
entry = await asyncio.get_running_loop().run_in_executor(
|
||||
None,
|
||||
|
||||
+8
-8
@@ -33,10 +33,10 @@
|
||||
"@angular/platform-browser-dynamic": "^22.0.6",
|
||||
"@angular/service-worker": "^22.0.6",
|
||||
"@fortawesome/angular-fontawesome": "~4.0.0",
|
||||
"@fortawesome/fontawesome-svg-core": "^7.3.0",
|
||||
"@fortawesome/free-brands-svg-icons": "^7.3.0",
|
||||
"@fortawesome/free-regular-svg-icons": "^7.3.0",
|
||||
"@fortawesome/free-solid-svg-icons": "^7.3.0",
|
||||
"@fortawesome/fontawesome-svg-core": "^7.3.1",
|
||||
"@fortawesome/free-brands-svg-icons": "^7.3.1",
|
||||
"@fortawesome/free-regular-svg-icons": "^7.3.1",
|
||||
"@fortawesome/free-solid-svg-icons": "^7.3.1",
|
||||
"@ng-bootstrap/ng-bootstrap": "^21.0.0",
|
||||
"@ng-select/ng-select": "^23.2.0",
|
||||
"@popperjs/core": "^2.11.8",
|
||||
@@ -49,13 +49,13 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-eslint/builder": "22.0.0",
|
||||
"@angular/build": "^22.0.5",
|
||||
"@angular/cli": "^22.0.5",
|
||||
"@angular/build": "^22.0.7",
|
||||
"@angular/cli": "^22.0.7",
|
||||
"@angular/compiler-cli": "^22.0.6",
|
||||
"@angular/localize": "^22.0.6",
|
||||
"@eslint/js": "^9.39.4",
|
||||
"@eslint/js": "^9.39.5",
|
||||
"angular-eslint": "22.0.0",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint": "^9.39.5",
|
||||
"jsdom": "^27.4.0",
|
||||
"typescript": "~6.0.3",
|
||||
"typescript-eslint": "8.62.0",
|
||||
|
||||
Generated
+258
-285
File diff suppressed because it is too large
Load Diff
@@ -106,14 +106,14 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "anyio"
|
||||
version = "4.14.1"
|
||||
version = "4.14.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "idna" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3b/72/5562aabb8dd7181e8e860622a38bea08d17842b99ecd4c91f84ac95251b0/anyio-4.14.1.tar.gz", hash = "sha256:8d648a3544c1a700e3ff78615cd679e4c5c3f149904287e73687b2596963629e", size = 254831, upload-time = "2026-06-24T20:56:06.017Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b0/7b/90df4a0a816d98d6ea26f559d87836d494a2cf1fcf063be67df50a7bcc30/anyio-4.14.1-py3-none-any.whl", hash = "sha256:4e5533c5b8ff0a24f5d7a176cbe6877129cd183893f66b537f8f227d10527d72", size = 124875, upload-time = "2026-06-24T20:56:04.413Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -366,15 +366,15 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "deno"
|
||||
version = "2.9.2"
|
||||
version = "2.9.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/dc/3c/bf3a4e2c00f7eefd4ed2fd033a958cab3e92dcc539d3ab80351850cae99c/deno-2.9.2.tar.gz", hash = "sha256:3b77d7689c15fb2c47d5a3927dc8cf70dbd408bc92e20c70da079c6d78980e34", size = 8164, upload-time = "2026-07-08T14:38:54.267Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/98/ab/638749d76881f74d100a079414745f0a0fd20bba38e27ab4a3a6495d8264/deno-2.9.3.tar.gz", hash = "sha256:73268cbac7f7c4ff1983e49420a93f3f3f2f2e4e4372436f0942d04d02c9e943", size = 8166, upload-time = "2026-07-15T15:37:50.565Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/27/fe/4a02a256f19b3f31ac17ec37f6467ecf523601ed753e6c9177865dd2bbb2/deno-2.9.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fb4bb5362d02c193f8086668789076f366416980b79277c06ea6cff73866b668", size = 42341183, upload-time = "2026-07-08T14:38:36.384Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/3e/235530ac9b9c206ee8c098e2b215f711a71d915d8d7d2be77599c8c1db4b/deno-2.9.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f6f74fd8698a8a9a31af14eed5730f1964ab50eaa0129079d11d91fa92496f60", size = 37984508, upload-time = "2026-07-08T14:38:40.006Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/27/06a166fca538c06dfae4c45dbb92c2ebf3083341141966a3d649272ea734/deno-2.9.2-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:8b2fd9318d452b7e44c45968916670a0c5452d72796e1af9abfaa7a389e4aa6b", size = 42089854, upload-time = "2026-07-08T14:38:43.841Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/b4/38338550a5ca0a7bcc39991e3d48e46ce3699413460dff32f383a611d822/deno-2.9.2-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:bf0471db7238e90810bca77a8d2c28646317b5a3fd9e5cdfdf73b251fde11b83", size = 43916921, upload-time = "2026-07-08T14:38:48.057Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/3d/c490c27e1f720baf014bb2c18b29d09b11c21d69fadfd69639e5904cdc95/deno-2.9.2-py3-none-win_amd64.whl", hash = "sha256:8e815d66b3e1314d028b2f0a05c75a331dd7025f4c825e12dd1f057207cee1f6", size = 41621982, upload-time = "2026-07-08T14:38:51.928Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/fa/b4e1e2b3b894ed992c78a6796a43dc775edff895239d83c0e9a10457f4f7/deno-2.9.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:1deef9fe97ab28d5d4fea07693075e9ccc62f4ea97a653047a770beb01ecd307", size = 42354026, upload-time = "2026-07-15T15:37:34.977Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3d/cc/4090ef7370005b740efe57ec6ff75a3471393abe9816f1459e73dbbf0429/deno-2.9.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:78635cd9f0802ce7125aa3ba6feb5775460f55d432611374f16b454ffb8a1308", size = 37997909, upload-time = "2026-07-15T15:37:38.291Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/07/70b7c965bbca5f3d158859acda0e90c041f1fa98b8c9ac8eff3fafd8b96c/deno-2.9.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:7c1c669827a480ef9e2710dd762560811da35dca04a9e90b23c0518587dadb8a", size = 42100873, upload-time = "2026-07-15T15:37:41.539Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/7b/f5/0a07cc19a27476011719dd017a98950c7ccc33058c2ffc07019ba3ad9a5e/deno-2.9.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:3e724d5c8df13a90f6b50b16cc8df1acfe64c61a7736a774e11d61f3bca3a3b1", size = 43928967, upload-time = "2026-07-15T15:37:45.175Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/9a/030847bd4ea6cefbb471cae2b63c9ab29a3ab6c1809484447752bbcbcd52/deno-2.9.3-py3-none-win_amd64.whl", hash = "sha256:e2ebe2b5c1a7ee9daeaf8caf14ebc33fda96b4c2399da259e761fe2f82d22fa4", size = 41630201, upload-time = "2026-07-15T15:37:48.344Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
Reference in New Issue
Block a user