mirror of
https://github.com/alexta69/metube.git
synced 2026-09-21 13:35:01 +00:00
Compare commits
5 Commits
ac46fff6d9
...
2026.08.20
| Author | SHA1 | Date | |
|---|---|---|---|
| 86954784fd | |||
| 72e8f5031f | |||
| f3c464fad5 | |||
| b10bb6103a | |||
| 8c2990e68a |
@@ -725,6 +725,7 @@ def parse_download_options(post: dict) -> dict:
|
|||||||
playlist_item_limit = post.get('playlist_item_limit')
|
playlist_item_limit = post.get('playlist_item_limit')
|
||||||
auto_start = post.get('auto_start')
|
auto_start = post.get('auto_start')
|
||||||
split_by_chapters = post.get('split_by_chapters')
|
split_by_chapters = post.get('split_by_chapters')
|
||||||
|
sponsorblock = bool(post.get('sponsorblock'))
|
||||||
chapter_template = post.get('chapter_template')
|
chapter_template = post.get('chapter_template')
|
||||||
subtitle_language = post.get('subtitle_language')
|
subtitle_language = post.get('subtitle_language')
|
||||||
subtitle_mode = post.get('subtitle_mode')
|
subtitle_mode = post.get('subtitle_mode')
|
||||||
@@ -845,6 +846,7 @@ def parse_download_options(post: dict) -> dict:
|
|||||||
'playlist_item_limit': playlist_item_limit,
|
'playlist_item_limit': playlist_item_limit,
|
||||||
'auto_start': auto_start,
|
'auto_start': auto_start,
|
||||||
'split_by_chapters': split_by_chapters,
|
'split_by_chapters': split_by_chapters,
|
||||||
|
'sponsorblock': sponsorblock,
|
||||||
'chapter_template': chapter_template,
|
'chapter_template': chapter_template,
|
||||||
'subtitle_language': subtitle_language,
|
'subtitle_language': subtitle_language,
|
||||||
'subtitle_mode': subtitle_mode,
|
'subtitle_mode': subtitle_mode,
|
||||||
@@ -890,6 +892,7 @@ async def add(request):
|
|||||||
o['ytdl_options_overrides'],
|
o['ytdl_options_overrides'],
|
||||||
o['clip_start'],
|
o['clip_start'],
|
||||||
o['clip_end'],
|
o['clip_end'],
|
||||||
|
sponsorblock=o['sponsorblock'],
|
||||||
)
|
)
|
||||||
return web.Response(text=serializer.encode(status))
|
return web.Response(text=serializer.encode(status))
|
||||||
|
|
||||||
@@ -970,6 +973,7 @@ async def subscribe(request):
|
|||||||
subtitle_mode=o['subtitle_mode'],
|
subtitle_mode=o['subtitle_mode'],
|
||||||
ytdl_options_presets=o['ytdl_options_presets'],
|
ytdl_options_presets=o['ytdl_options_presets'],
|
||||||
ytdl_options_overrides=o['ytdl_options_overrides'],
|
ytdl_options_overrides=o['ytdl_options_overrides'],
|
||||||
|
sponsorblock=o['sponsorblock'],
|
||||||
title_regex=post.get('title_regex'),
|
title_regex=post.get('title_regex'),
|
||||||
skip_subscriber_only=skip_subscriber_only,
|
skip_subscriber_only=skip_subscriber_only,
|
||||||
clip_start=sub_clip_start,
|
clip_start=sub_clip_start,
|
||||||
|
|||||||
@@ -182,6 +182,7 @@ class SubscriptionInfo:
|
|||||||
auto_start: bool = True
|
auto_start: bool = True
|
||||||
playlist_item_limit: int = 0
|
playlist_item_limit: int = 0
|
||||||
split_by_chapters: bool = False
|
split_by_chapters: bool = False
|
||||||
|
sponsorblock: bool = False
|
||||||
chapter_template: str = ""
|
chapter_template: str = ""
|
||||||
subtitle_language: str = "en"
|
subtitle_language: str = "en"
|
||||||
subtitle_mode: str = "prefer_manual"
|
subtitle_mode: str = "prefer_manual"
|
||||||
@@ -242,6 +243,7 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]:
|
|||||||
"auto_start": sub.auto_start,
|
"auto_start": sub.auto_start,
|
||||||
"playlist_item_limit": sub.playlist_item_limit,
|
"playlist_item_limit": sub.playlist_item_limit,
|
||||||
"split_by_chapters": sub.split_by_chapters,
|
"split_by_chapters": sub.split_by_chapters,
|
||||||
|
"sponsorblock": sub.sponsorblock,
|
||||||
"chapter_template": sub.chapter_template,
|
"chapter_template": sub.chapter_template,
|
||||||
"subtitle_language": sub.subtitle_language,
|
"subtitle_language": sub.subtitle_language,
|
||||||
"subtitle_mode": sub.subtitle_mode,
|
"subtitle_mode": sub.subtitle_mode,
|
||||||
@@ -487,6 +489,7 @@ class SubscriptionManager:
|
|||||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||||
clip_start: Optional[float] = None,
|
clip_start: Optional[float] = None,
|
||||||
clip_end: Optional[float] = None,
|
clip_end: Optional[float] = None,
|
||||||
|
sponsorblock: bool = False,
|
||||||
) -> tuple[list[str], list[str]]:
|
) -> tuple[list[str], list[str]]:
|
||||||
queued_ids: list[str] = []
|
queued_ids: list[str] = []
|
||||||
queue_errors: list[str] = []
|
queue_errors: list[str] = []
|
||||||
@@ -519,6 +522,7 @@ class SubscriptionManager:
|
|||||||
ytdl_options_overrides,
|
ytdl_options_overrides,
|
||||||
clip_start,
|
clip_start,
|
||||||
clip_end,
|
clip_end,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
if isinstance(result, dict) and result.get("status") == "error":
|
if isinstance(result, dict) and result.get("status") == "error":
|
||||||
msg = str(result.get("msg") or f"Queueing failed for {vurl}")
|
msg = str(result.get("msg") or f"Queueing failed for {vurl}")
|
||||||
@@ -606,6 +610,7 @@ class SubscriptionManager:
|
|||||||
subtitle_mode: str,
|
subtitle_mode: str,
|
||||||
ytdl_options_presets: Optional[list[str]] = None,
|
ytdl_options_presets: Optional[list[str]] = None,
|
||||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||||
|
sponsorblock: bool = False,
|
||||||
title_regex: Any = None,
|
title_regex: Any = None,
|
||||||
skip_subscriber_only: Any = None,
|
skip_subscriber_only: Any = None,
|
||||||
clip_start: Optional[float] = None,
|
clip_start: Optional[float] = None,
|
||||||
@@ -689,6 +694,7 @@ class SubscriptionManager:
|
|||||||
auto_start=bool(auto_start),
|
auto_start=bool(auto_start),
|
||||||
playlist_item_limit=int(playlist_item_limit),
|
playlist_item_limit=int(playlist_item_limit),
|
||||||
split_by_chapters=bool(split_by_chapters),
|
split_by_chapters=bool(split_by_chapters),
|
||||||
|
sponsorblock=bool(sponsorblock),
|
||||||
chapter_template=chapter_template or "",
|
chapter_template=chapter_template or "",
|
||||||
subtitle_language=subtitle_language,
|
subtitle_language=subtitle_language,
|
||||||
subtitle_mode=subtitle_mode,
|
subtitle_mode=subtitle_mode,
|
||||||
@@ -942,6 +948,7 @@ class SubscriptionManager:
|
|||||||
dl_plimit = cur.playlist_item_limit
|
dl_plimit = cur.playlist_item_limit
|
||||||
dl_autostart = cur.auto_start
|
dl_autostart = cur.auto_start
|
||||||
dl_split = cur.split_by_chapters
|
dl_split = cur.split_by_chapters
|
||||||
|
dl_sponsorblock = cur.sponsorblock
|
||||||
dl_chapter = cur.chapter_template
|
dl_chapter = cur.chapter_template
|
||||||
dl_sublang = cur.subtitle_language
|
dl_sublang = cur.subtitle_language
|
||||||
dl_submode = cur.subtitle_mode
|
dl_submode = cur.subtitle_mode
|
||||||
@@ -1010,6 +1017,7 @@ class SubscriptionManager:
|
|||||||
playlist_item_limit=dl_plimit,
|
playlist_item_limit=dl_plimit,
|
||||||
auto_start=dl_autostart,
|
auto_start=dl_autostart,
|
||||||
split_by_chapters=dl_split,
|
split_by_chapters=dl_split,
|
||||||
|
sponsorblock=dl_sponsorblock,
|
||||||
chapter_template=dl_chapter or "",
|
chapter_template=dl_chapter or "",
|
||||||
subtitle_language=dl_sublang,
|
subtitle_language=dl_sublang,
|
||||||
subtitle_mode=dl_submode,
|
subtitle_mode=dl_submode,
|
||||||
|
|||||||
@@ -366,6 +366,25 @@ async def test_subscribe_passes_clip_bounds(mock_dqueue, monkeypatch):
|
|||||||
assert kwargs["clip_end"] == pytest.approx(204.0)
|
assert kwargs["clip_end"] == pytest.approx(204.0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_passes_sponsorblock(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
req = _json_request(
|
||||||
|
{**_valid_video_add_body(), "check_interval_minutes": 60, "sponsorblock": True}
|
||||||
|
)
|
||||||
|
resp = await main.subscribe(req)
|
||||||
|
assert resp.status == 200
|
||||||
|
assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_defaults_sponsorblock_off(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
req = _json_request({**_valid_video_add_body(), "check_interval_minutes": 60})
|
||||||
|
await main.subscribe(req)
|
||||||
|
assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_subscribe_without_clip_fields_stores_none(mock_dqueue, monkeypatch):
|
async def test_subscribe_without_clip_fields_stores_none(mock_dqueue, monkeypatch):
|
||||||
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
|||||||
@@ -476,6 +476,32 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env):
|
|||||||
assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True}
|
assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_retry_carries_the_sponsorblock_flag(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
url = "https://example.com/watch?v=1"
|
||||||
|
dq.done.put(
|
||||||
|
Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url, sponsorblock=True))
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
|
return {
|
||||||
|
"_type": "video",
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Test Video",
|
||||||
|
"url": extracted_url,
|
||||||
|
"webpage_url": extracted_url,
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||||
|
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||||
|
result = await dq.retry(url)
|
||||||
|
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert dq.queue.get(url).info.sponsorblock is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
|
|||||||
@@ -479,6 +479,69 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
|||||||
self.assertEqual(reloaded.get(sub_id).clip_start, 12.5)
|
self.assertEqual(reloaded.get(sub_id).clip_start, 12.5)
|
||||||
self.assertIsNone(reloaded.get(sub_id).clip_end)
|
self.assertIsNone(reloaded.get(sub_id).clip_end)
|
||||||
|
|
||||||
|
async def test_check_now_applies_subscription_sponsorblock(self):
|
||||||
|
"""Subscriptions download unattended, so the sponsor-segment removal has
|
||||||
|
to reach every entry the subscription queues, not just manual adds."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
queue = _Queue()
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"subscriptions.extract_flat_playlist",
|
||||||
|
side_effect=[
|
||||||
|
(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[
|
||||||
|
{"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"},
|
||||||
|
{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
):
|
||||||
|
result = await mgr.add_subscription(
|
||||||
|
"https://example.com/channel",
|
||||||
|
check_interval_minutes=60,
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
quality="best",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
auto_start=True,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
subtitle_language="en",
|
||||||
|
subtitle_mode="prefer_manual",
|
||||||
|
sponsorblock=True,
|
||||||
|
)
|
||||||
|
sub_id = result["subscription"]["id"]
|
||||||
|
self.assertTrue(mgr.get(sub_id).sponsorblock)
|
||||||
|
await mgr.check_now([sub_id])
|
||||||
|
|
||||||
|
self.assertEqual(len(queue.entries), 1)
|
||||||
|
_entry, _args, kwargs = queue.entries[0]
|
||||||
|
self.assertIs(kwargs["sponsorblock"], True)
|
||||||
|
|
||||||
|
async def test_sponsorblock_survives_reload_and_defaults_to_false(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg = _Config(tmp)
|
||||||
|
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
# Records written before the field existed simply take the default.
|
||||||
|
self.assertFalse(mgr.get(sub_id).sponsorblock)
|
||||||
|
|
||||||
|
mgr.get(sub_id).sponsorblock = True
|
||||||
|
async with mgr._lock:
|
||||||
|
mgr._save_locked()
|
||||||
|
|
||||||
|
reloaded = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
self.assertTrue(reloaded.get(sub_id).sponsorblock)
|
||||||
|
|
||||||
async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
|
async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
queue = _Queue()
|
queue = _Queue()
|
||||||
|
|||||||
+71
-36
@@ -12,7 +12,7 @@ from url_guard import (
|
|||||||
_address_allowed_at_connect,
|
_address_allowed_at_connect,
|
||||||
_address_is_global,
|
_address_is_global,
|
||||||
_guarded_getaddrinfo,
|
_guarded_getaddrinfo,
|
||||||
_proxy_endpoint,
|
_url_endpoint,
|
||||||
install_socket_guard,
|
install_socket_guard,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -121,19 +121,19 @@ class ConnectAddressPolicyTests(unittest.TestCase):
|
|||||||
self.assertFalse(_address_allowed_at_connect("::1"))
|
self.assertFalse(_address_allowed_at_connect("::1"))
|
||||||
|
|
||||||
def test_loopback_allowed_only_when_opted_in(self):
|
def test_loopback_allowed_only_when_opted_in(self):
|
||||||
self.assertTrue(_address_allowed_at_connect("127.0.0.1", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("127.0.0.1", is_allowed_endpoint=True))
|
||||||
self.assertTrue(_address_allowed_at_connect("::1", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("::1", is_allowed_endpoint=True))
|
||||||
|
|
||||||
def test_proxy_opt_in_covers_any_internal_range(self):
|
def test_proxy_opt_in_covers_any_internal_range(self):
|
||||||
# A proxy is just as legitimately on the LAN or a VPN range as on
|
# A proxy is just as legitimately on the LAN or a VPN range as on
|
||||||
# loopback (#1055): the allowance follows the operator's configured
|
# loopback (#1055): the allowance follows the operator's configured
|
||||||
# endpoint, not a particular address family.
|
# endpoint, not a particular address family.
|
||||||
self.assertTrue(_address_allowed_at_connect("10.1.20.30", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("10.1.20.30", is_allowed_endpoint=True))
|
||||||
self.assertTrue(_address_allowed_at_connect("192.168.1.10", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("192.168.1.10", is_allowed_endpoint=True))
|
||||||
self.assertTrue(_address_allowed_at_connect("fd00::1", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("fd00::1", is_allowed_endpoint=True))
|
||||||
|
|
||||||
def test_opt_in_still_rejects_non_addresses(self):
|
def test_opt_in_still_rejects_non_addresses(self):
|
||||||
self.assertFalse(_address_allowed_at_connect("not-an-ip", is_proxy_endpoint=True))
|
self.assertFalse(_address_allowed_at_connect("not-an-ip", is_allowed_endpoint=True))
|
||||||
|
|
||||||
def test_link_local_metadata_blocked(self):
|
def test_link_local_metadata_blocked(self):
|
||||||
self.assertFalse(_address_allowed_at_connect("169.254.169.254"))
|
self.assertFalse(_address_allowed_at_connect("169.254.169.254"))
|
||||||
@@ -187,36 +187,36 @@ class TunnelledIPv4Tests(unittest.TestCase):
|
|||||||
self.assertIsNotNone(validate_url("http://nat64.example/x"))
|
self.assertIsNotNone(validate_url("http://nat64.example/x"))
|
||||||
|
|
||||||
|
|
||||||
class ProxyEndpointParsingTests(unittest.TestCase):
|
class EndpointParsingTests(unittest.TestCase):
|
||||||
def test_explicit_port(self):
|
def test_explicit_port(self):
|
||||||
self.assertEqual(_proxy_endpoint("http://127.0.0.1:9050"), ("127.0.0.1", 9050))
|
self.assertEqual(_url_endpoint("http://127.0.0.1:9050"), ("127.0.0.1", 9050))
|
||||||
|
|
||||||
def test_default_port_per_scheme(self):
|
def test_default_port_per_scheme(self):
|
||||||
self.assertEqual(_proxy_endpoint("socks5://127.0.0.1"), ("127.0.0.1", 1080))
|
self.assertEqual(_url_endpoint("socks5://127.0.0.1"), ("127.0.0.1", 1080))
|
||||||
self.assertEqual(_proxy_endpoint("http://127.0.0.1"), ("127.0.0.1", 80))
|
self.assertEqual(_url_endpoint("http://127.0.0.1"), ("127.0.0.1", 80))
|
||||||
|
|
||||||
def test_bare_host_port(self):
|
def test_bare_host_port(self):
|
||||||
self.assertEqual(_proxy_endpoint("127.0.0.1:8080"), ("127.0.0.1", 8080))
|
self.assertEqual(_url_endpoint("127.0.0.1:8080"), ("127.0.0.1", 8080))
|
||||||
|
|
||||||
def test_hostname_lowercased(self):
|
def test_hostname_lowercased(self):
|
||||||
self.assertEqual(_proxy_endpoint("http://LocalHost.:9050"), ("localhost", 9050))
|
self.assertEqual(_url_endpoint("http://LocalHost.:9050"), ("localhost", 9050))
|
||||||
|
|
||||||
def test_ipv6_literal(self):
|
def test_ipv6_literal(self):
|
||||||
self.assertEqual(_proxy_endpoint("http://[::1]:9050"), ("::1", 9050))
|
self.assertEqual(_url_endpoint("http://[::1]:9050"), ("::1", 9050))
|
||||||
|
|
||||||
def test_empty_and_invalid(self):
|
def test_empty_and_invalid(self):
|
||||||
self.assertIsNone(_proxy_endpoint(""))
|
self.assertIsNone(_url_endpoint(""))
|
||||||
self.assertIsNone(_proxy_endpoint(" "))
|
self.assertIsNone(_url_endpoint(" "))
|
||||||
self.assertIsNone(_proxy_endpoint(None))
|
self.assertIsNone(_url_endpoint(None))
|
||||||
self.assertIsNone(_proxy_endpoint("http://"))
|
self.assertIsNone(_url_endpoint("http://"))
|
||||||
|
|
||||||
|
|
||||||
class GuardedGetaddrinfoTests(unittest.TestCase):
|
class GuardedGetaddrinfoTests(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
# Default state: no proxy configured, so no loopback destination allowed.
|
# Default state: no proxy configured, so no loopback destination allowed.
|
||||||
saved = set(url_guard._allowed_proxy_endpoints)
|
saved = set(url_guard._allowed_endpoints)
|
||||||
url_guard._allowed_proxy_endpoints = set()
|
url_guard._allowed_endpoints = set()
|
||||||
self.addCleanup(lambda: setattr(url_guard, "_allowed_proxy_endpoints", saved))
|
self.addCleanup(lambda: setattr(url_guard, "_allowed_endpoints", saved))
|
||||||
|
|
||||||
def test_internal_only_raises(self):
|
def test_internal_only_raises(self):
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")):
|
||||||
@@ -235,27 +235,27 @@ class GuardedGetaddrinfoTests(unittest.TestCase):
|
|||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
_guarded_getaddrinfo("127.0.0.1", 9999)
|
_guarded_getaddrinfo("127.0.0.1", 9999)
|
||||||
|
|
||||||
def test_loopback_allowed_at_configured_proxy_endpoint(self):
|
def test_loopback_allowed_at_configured_url_endpoint(self):
|
||||||
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)}
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
results = _guarded_getaddrinfo("127.0.0.1", 9050)
|
results = _guarded_getaddrinfo("127.0.0.1", 9050)
|
||||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
def test_loopback_blocked_at_other_port_on_proxy_host(self):
|
def test_loopback_blocked_at_other_port_on_proxy_host(self):
|
||||||
# Same host as the proxy, different port: still off limits.
|
# Same host as the proxy, different port: still off limits.
|
||||||
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)}
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
_guarded_getaddrinfo("127.0.0.1", 9999)
|
_guarded_getaddrinfo("127.0.0.1", 9999)
|
||||||
|
|
||||||
def test_proxy_reachable_by_hostname(self):
|
def test_proxy_reachable_by_hostname(self):
|
||||||
url_guard._allowed_proxy_endpoints = {("localhost", 9050)}
|
url_guard._allowed_endpoints = {("localhost", 9050)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
results = _guarded_getaddrinfo("localhost", 9050)
|
results = _guarded_getaddrinfo("localhost", 9050)
|
||||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
def test_string_port_is_normalised(self):
|
def test_string_port_is_normalised(self):
|
||||||
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)}
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
results = _guarded_getaddrinfo("127.0.0.1", "9050")
|
results = _guarded_getaddrinfo("127.0.0.1", "9050")
|
||||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
@@ -263,22 +263,38 @@ class GuardedGetaddrinfoTests(unittest.TestCase):
|
|||||||
def test_lan_proxy_reachable(self):
|
def test_lan_proxy_reachable(self):
|
||||||
# #1055: a socks5 proxy on the LAN, refused while the allowance was
|
# #1055: a socks5 proxy on the LAN, refused while the allowance was
|
||||||
# loopback-only, which pushed operators to ALLOW_PRIVATE_ADDRESSES.
|
# loopback-only, which pushed operators to ALLOW_PRIVATE_ADDRESSES.
|
||||||
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)}
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
||||||
results = _guarded_getaddrinfo("10.1.20.30", 1080)
|
results = _guarded_getaddrinfo("10.1.20.30", 1080)
|
||||||
self.assertEqual([r[4][0] for r in results], ["10.1.20.30"])
|
self.assertEqual([r[4][0] for r in results], ["10.1.20.30"])
|
||||||
|
|
||||||
def test_other_lan_host_still_blocked(self):
|
def test_other_lan_host_still_blocked(self):
|
||||||
# The allowance is the proxy's endpoint, not its subnet.
|
# The allowance is the proxy's endpoint, not its subnet.
|
||||||
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)}
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.31")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.31")):
|
||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
_guarded_getaddrinfo("10.1.20.31", 1080)
|
_guarded_getaddrinfo("10.1.20.31", 1080)
|
||||||
|
|
||||||
|
def test_pot_provider_reachable_on_loopback(self):
|
||||||
|
# #1064: the bundled PO token provider listens on loopback, and blocking
|
||||||
|
# it left every default install downloading YouTube without a token.
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 4416)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
results = _guarded_getaddrinfo("127.0.0.1", 4416)
|
||||||
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
|
def test_other_loopback_service_still_blocked(self):
|
||||||
|
# MeTube's own port is one hop away on the same interface: allowing the
|
||||||
|
# token provider must not allow the rest of loopback.
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 4416)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
with self.assertRaises(socket.gaierror):
|
||||||
|
_guarded_getaddrinfo("127.0.0.1", 8081)
|
||||||
|
|
||||||
def test_proxy_address_not_borrowable_by_another_host(self):
|
def test_proxy_address_not_borrowable_by_another_host(self):
|
||||||
# Matching is on the configured host string: a manifest URL that resolves
|
# Matching is on the configured host string: a manifest URL that resolves
|
||||||
# to the proxy's address under its own name gets no allowance.
|
# to the proxy's address under its own name gets no allowance.
|
||||||
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)}
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
_guarded_getaddrinfo("evil.example", 1080)
|
_guarded_getaddrinfo("evil.example", 1080)
|
||||||
@@ -312,9 +328,9 @@ class AllowPrivateBypassTests(unittest.TestCase):
|
|||||||
|
|
||||||
class InstallSocketGuardTests(unittest.TestCase):
|
class InstallSocketGuardTests(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
original, saved = socket.getaddrinfo, set(url_guard._allowed_proxy_endpoints)
|
original, saved = socket.getaddrinfo, set(url_guard._allowed_endpoints)
|
||||||
self.addCleanup(lambda: setattr(socket, "getaddrinfo", original))
|
self.addCleanup(lambda: setattr(socket, "getaddrinfo", original))
|
||||||
self.addCleanup(lambda: setattr(url_guard, "_allowed_proxy_endpoints", saved))
|
self.addCleanup(lambda: setattr(url_guard, "_allowed_endpoints", saved))
|
||||||
# Keep the host's own environment out of the assertions below.
|
# Keep the host's own environment out of the assertions below.
|
||||||
patcher = mock.patch("url_guard.urllib.request.getproxies", return_value={})
|
patcher = mock.patch("url_guard.urllib.request.getproxies", return_value={})
|
||||||
self.getproxies = patcher.start()
|
self.getproxies = patcher.start()
|
||||||
@@ -329,26 +345,45 @@ class InstallSocketGuardTests(unittest.TestCase):
|
|||||||
|
|
||||||
def test_no_proxy_means_no_loopback_allowance(self):
|
def test_no_proxy_means_no_loopback_allowance(self):
|
||||||
install_socket_guard()
|
install_socket_guard()
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, set())
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
def test_explicit_proxy_is_registered(self):
|
def test_explicit_proxy_is_registered(self):
|
||||||
install_socket_guard(proxy_urls=("socks5://127.0.0.1:9050",))
|
install_socket_guard(proxy_urls=("socks5://127.0.0.1:9050",))
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, {("127.0.0.1", 9050)})
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 9050)})
|
||||||
|
|
||||||
def test_unset_proxy_option_is_ignored(self):
|
def test_unset_proxy_option_is_ignored(self):
|
||||||
# ytdl_opts.get('proxy') is None when the operator configured no proxy.
|
# ytdl_opts.get('proxy') is None when the operator configured no proxy.
|
||||||
install_socket_guard(proxy_urls=(None,))
|
install_socket_guard(proxy_urls=(None,))
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, set())
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
def test_environment_proxies_are_registered(self):
|
def test_environment_proxies_are_registered(self):
|
||||||
self.getproxies.return_value = {"http": "http://127.0.0.1:8080"}
|
self.getproxies.return_value = {"http": "http://127.0.0.1:8080"}
|
||||||
install_socket_guard()
|
install_socket_guard()
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, {("127.0.0.1", 8080)})
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 8080)})
|
||||||
|
|
||||||
|
def test_service_url_is_registered(self):
|
||||||
|
install_socket_guard(service_urls=("http://127.0.0.1:4416",))
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 4416)})
|
||||||
|
|
||||||
|
def test_service_and_proxy_endpoints_coexist(self):
|
||||||
|
install_socket_guard(
|
||||||
|
proxy_urls=("socks5://10.1.20.30:1080",),
|
||||||
|
service_urls=("http://127.0.0.1:4416",),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
url_guard._allowed_endpoints,
|
||||||
|
{("10.1.20.30", 1080), ("127.0.0.1", 4416)},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_service_urls_reset_between_installs(self):
|
||||||
|
install_socket_guard(service_urls=("http://127.0.0.1:4416",))
|
||||||
|
install_socket_guard()
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
def test_endpoints_reset_between_installs(self):
|
def test_endpoints_reset_between_installs(self):
|
||||||
install_socket_guard(proxy_urls=("http://127.0.0.1:8080",))
|
install_socket_guard(proxy_urls=("http://127.0.0.1:8080",))
|
||||||
install_socket_guard(proxy_urls=(None,))
|
install_socket_guard(proxy_urls=(None,))
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, set())
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -77,6 +77,7 @@ from ytdl import (
|
|||||||
MusicMetadataPreProcessor,
|
MusicMetadataPreProcessor,
|
||||||
_compact_persisted_entry,
|
_compact_persisted_entry,
|
||||||
_convert_srt_to_txt_file,
|
_convert_srt_to_txt_file,
|
||||||
|
_pot_provider_urls,
|
||||||
_AlbumArtistPostProcessor,
|
_AlbumArtistPostProcessor,
|
||||||
_resolve_outtmpl_fields,
|
_resolve_outtmpl_fields,
|
||||||
_sanitize_entry_for_pickle,
|
_sanitize_entry_for_pickle,
|
||||||
@@ -678,6 +679,69 @@ class DownloadResultTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _capture_ytdl_params(download: Download) -> dict:
|
||||||
|
"""Run ``_download`` far enough to capture the params it builds."""
|
||||||
|
fake_ydl = MagicMock()
|
||||||
|
fake_ydl.download.return_value = 0
|
||||||
|
download.status_queue = types.SimpleNamespace(put=lambda _: None)
|
||||||
|
|
||||||
|
with patch('ytdl.install_socket_guard'), \
|
||||||
|
patch.object(Download, '_make_youtube_dl', return_value=fake_ydl) as make:
|
||||||
|
download._download()
|
||||||
|
|
||||||
|
params, = make.call_args.args
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
class SponsorBlockPostprocessorTests(unittest.TestCase):
|
||||||
|
def test_no_sponsorblock_postprocessors_when_disabled(self):
|
||||||
|
download = _make_test_download()
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
keys = [pp['key'] for pp in params.get('postprocessors', [])]
|
||||||
|
self.assertNotIn('SponsorBlock', keys)
|
||||||
|
self.assertNotIn('ModifyChapters', keys)
|
||||||
|
|
||||||
|
def test_sponsorblock_pair_matches_the_cli(self):
|
||||||
|
download = _make_test_download()
|
||||||
|
download.info.sponsorblock = True
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
params['postprocessors'],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'key': 'SponsorBlock',
|
||||||
|
'categories': ['sponsor'],
|
||||||
|
'when': 'after_filter',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'key': 'ModifyChapters',
|
||||||
|
'remove_sponsor_segments': ['sponsor'],
|
||||||
|
'force_keyframes': False,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_segment_removal_runs_before_the_chapter_split(self):
|
||||||
|
# yt-dlp runs same-stage postprocessors in list order, so ModifyChapters
|
||||||
|
# has to rewrite the chapter list before FFmpegSplitChapters cuts the
|
||||||
|
# file up -- the order the CLI builds for
|
||||||
|
# --sponsorblock-remove sponsor --split-chapters.
|
||||||
|
download = _make_test_download()
|
||||||
|
download.info.sponsorblock = True
|
||||||
|
download.info.split_by_chapters = True
|
||||||
|
download.info.chapter_template = '%(section_number)s.%(ext)s'
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
keys = [pp['key'] for pp in params['postprocessors']]
|
||||||
|
self.assertEqual(keys, ['SponsorBlock', 'ModifyChapters', 'FFmpegSplitChapters'])
|
||||||
|
self.assertEqual(params['outtmpl']['chapter'], '%(section_number)s.%(ext)s')
|
||||||
|
|
||||||
|
|
||||||
class ProgressThrottleTests(unittest.TestCase):
|
class ProgressThrottleTests(unittest.TestCase):
|
||||||
def test_downloading_ticks_are_throttled(self):
|
def test_downloading_ticks_are_throttled(self):
|
||||||
dl = _make_test_download()
|
dl = _make_test_download()
|
||||||
@@ -1071,5 +1135,44 @@ class ShortTitleForFailedUrlTests(unittest.TestCase):
|
|||||||
self.assertEqual(_short_title_for_failed_url(malformed), malformed)
|
self.assertEqual(_short_title_for_failed_url(malformed), malformed)
|
||||||
|
|
||||||
|
|
||||||
|
class PotProviderUrlsTests(unittest.TestCase):
|
||||||
|
"""#1064: the connect-time guard must let the download reach the PO token
|
||||||
|
provider, so it has to know every endpoint yt-dlp might dial for one."""
|
||||||
|
|
||||||
|
def test_bundled_provider_by_default(self):
|
||||||
|
self.assertEqual(_pot_provider_urls({}), ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
def test_configured_base_url_is_added(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtubepot-bgutilhttp": {"base_url": ["http://pot:4416"]}},
|
||||||
|
})
|
||||||
|
# The bundled server runs regardless, so both stay reachable.
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416", "http://pot:4416"))
|
||||||
|
|
||||||
|
def test_deprecated_base_url_arg_is_honoured(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtube": {"getpot_bgutil_baseurl": ["http://pot:4416"]}},
|
||||||
|
})
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416", "http://pot:4416"))
|
||||||
|
|
||||||
|
def test_unrelated_extractor_args_are_ignored(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtube": {"player_client": ["web"]}},
|
||||||
|
})
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
def test_malformed_extractor_args_do_not_raise(self):
|
||||||
|
# YTDL_OPTIONS is operator-supplied JSON and reaches here unvalidated.
|
||||||
|
for opts in (
|
||||||
|
{"extractor_args": None},
|
||||||
|
{"extractor_args": "youtube:player_client=web"},
|
||||||
|
{"extractor_args": {"youtubepot-bgutilhttp": "http://pot:4416"}},
|
||||||
|
{"extractor_args": {"youtubepot-bgutilhttp": {"base_url": []}}},
|
||||||
|
):
|
||||||
|
with self.subTest(opts=opts):
|
||||||
|
self.assertEqual(_pot_provider_urls(opts), ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
+52
-38
@@ -37,8 +37,8 @@ log = logging.getLogger('url_guard')
|
|||||||
|
|
||||||
_ALLOWED_SCHEMES = ('http', 'https')
|
_ALLOWED_SCHEMES = ('http', 'https')
|
||||||
|
|
||||||
# Ports to assume when a configured proxy URL omits one, per proxy scheme.
|
# Ports to assume when a configured endpoint URL omits one, per scheme.
|
||||||
_PROXY_DEFAULT_PORTS = {
|
_SCHEME_DEFAULT_PORTS = {
|
||||||
'http': 80,
|
'http': 80,
|
||||||
'https': 443,
|
'https': 443,
|
||||||
'socks4': 1080,
|
'socks4': 1080,
|
||||||
@@ -129,31 +129,32 @@ def _address_is_global(addr: str) -> bool:
|
|||||||
return bool(ips) and all(ip.is_global for ip in ips)
|
return bool(ips) and all(ip.is_global for ip in ips)
|
||||||
|
|
||||||
|
|
||||||
def _address_allowed_at_connect(addr: str, is_proxy_endpoint: bool = False) -> bool:
|
def _address_allowed_at_connect(addr: str, is_allowed_endpoint: bool = False) -> bool:
|
||||||
"""True if *addr* may be connected to at download time.
|
"""True if *addr* may be connected to at download time.
|
||||||
|
|
||||||
Permits global addresses, and anything at all when the destination is an
|
Permits global addresses, and anything at all when the destination is an
|
||||||
operator-configured proxy (see ``_is_proxy_endpoint``). Internal addresses
|
endpoint the operator or the image configured — a proxy, or the PO token
|
||||||
are otherwise refused with no blanket exception: media URLs that yt-dlp
|
provider (see ``_is_allowed_endpoint``). Internal addresses are otherwise
|
||||||
derives from a remote manifest are attacker-controlled and reach this policy
|
refused with no blanket exception: media URLs that yt-dlp derives from a
|
||||||
without passing ``validate_url``, so any range opened here is a range a
|
remote manifest are attacker-controlled and reach this policy without passing
|
||||||
hostile playlist can read from the server's own network. Blocks link-local
|
``validate_url``, so any range opened here is a range a hostile playlist can
|
||||||
|
read from the server's own network. Blocks link-local
|
||||||
(cloud metadata at 169.254.169.254), private (RFC1918), loopback,
|
(cloud metadata at 169.254.169.254), private (RFC1918), loopback,
|
||||||
unique-local and every other non-global range.
|
unique-local and every other non-global range.
|
||||||
"""
|
"""
|
||||||
ips = _ips_to_judge(addr)
|
ips = _ips_to_judge(addr)
|
||||||
if not ips:
|
if not ips:
|
||||||
return False
|
return False
|
||||||
return is_proxy_endpoint or all(ip.is_global for ip in ips)
|
return is_allowed_endpoint or all(ip.is_global for ip in ips)
|
||||||
|
|
||||||
|
|
||||||
def _proxy_endpoint(proxy_url: str):
|
def _url_endpoint(url: str):
|
||||||
"""Parse a proxy URL into a ``(hostname, port)`` pair, or ``None`` if it has
|
"""Parse a configured URL into a ``(hostname, port)`` pair, or ``None`` if it
|
||||||
no usable host. Used to scope the internal-address allowance to that endpoint
|
has no usable host. Used to scope the internal-address allowance to that
|
||||||
alone."""
|
endpoint alone."""
|
||||||
if not isinstance(proxy_url, str) or not proxy_url.strip():
|
if not isinstance(url, str) or not url.strip():
|
||||||
return None
|
return None
|
||||||
candidate = proxy_url.strip()
|
candidate = url.strip()
|
||||||
if '://' not in candidate:
|
if '://' not in candidate:
|
||||||
# Bare host:port, as accepted by the *_proxy environment variables.
|
# Bare host:port, as accepted by the *_proxy environment variables.
|
||||||
candidate = '//' + candidate
|
candidate = '//' + candidate
|
||||||
@@ -165,23 +166,28 @@ def _proxy_endpoint(proxy_url: str):
|
|||||||
if not hostname:
|
if not hostname:
|
||||||
return None
|
return None
|
||||||
if port is None:
|
if port is None:
|
||||||
port = _PROXY_DEFAULT_PORTS.get(parts.scheme.lower())
|
port = _SCHEME_DEFAULT_PORTS.get(parts.scheme.lower())
|
||||||
return (hostname.rstrip('.').lower(), port)
|
return (hostname.rstrip('.').lower(), port)
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoints(urls) -> set:
|
||||||
|
"""The parseable endpoints among *urls*, dropping any that name no host."""
|
||||||
|
return {ep for ep in map(_url_endpoint, urls) if ep is not None}
|
||||||
|
|
||||||
|
|
||||||
def _collect_proxy_endpoints(proxy_urls) -> set:
|
def _collect_proxy_endpoints(proxy_urls) -> set:
|
||||||
"""Endpoints of every proxy this download may legitimately dial: the explicit
|
"""Endpoints of every proxy this download may legitimately dial: the explicit
|
||||||
yt-dlp ``proxy`` option plus the ``*_proxy`` environment variables yt-dlp falls
|
yt-dlp ``proxy`` option plus the ``*_proxy`` environment variables yt-dlp falls
|
||||||
back to. All are operator-configured, unlike the URLs inside fetched media."""
|
back to. All are operator-configured, unlike the URLs inside fetched media."""
|
||||||
candidates = list(proxy_urls) + list(urllib.request.getproxies().values())
|
candidates = list(proxy_urls) + list(urllib.request.getproxies().values())
|
||||||
return {ep for ep in map(_proxy_endpoint, candidates) if ep is not None}
|
return _endpoints(candidates)
|
||||||
|
|
||||||
|
|
||||||
# Captured at import so re-installing the guard never wraps the wrapper.
|
# Captured at import so re-installing the guard never wraps the wrapper.
|
||||||
_real_getaddrinfo = socket.getaddrinfo
|
_real_getaddrinfo = socket.getaddrinfo
|
||||||
|
|
||||||
# Populated by install_socket_guard; empty means no internal destination is allowed.
|
# Populated by install_socket_guard; empty means no internal destination is allowed.
|
||||||
_allowed_proxy_endpoints: set = set()
|
_allowed_endpoints: set = set()
|
||||||
|
|
||||||
|
|
||||||
def _normalise_port(port):
|
def _normalise_port(port):
|
||||||
@@ -196,28 +202,29 @@ def _normalise_port(port):
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
|
|
||||||
def _is_proxy_endpoint(host, port) -> bool:
|
def _is_allowed_endpoint(host, port) -> bool:
|
||||||
"""True when host:port is exactly an endpoint the operator configured as a
|
"""True when host:port is exactly one of the endpoints this download is
|
||||||
proxy. Matching is on the configured host *string*, not on the resolved
|
configured to dial — a proxy or the PO token provider. Matching is on the
|
||||||
address, so a hostile media URL cannot borrow the allowance by resolving to
|
configured host *string*, not on the resolved address, so a hostile media URL
|
||||||
the same address under a different name."""
|
cannot borrow the allowance by resolving to the same address under a
|
||||||
if not _allowed_proxy_endpoints or host is None:
|
different name."""
|
||||||
|
if not _allowed_endpoints or host is None:
|
||||||
return False
|
return False
|
||||||
return (str(host).rstrip('.').lower(), _normalise_port(port)) in _allowed_proxy_endpoints
|
return (str(host).rstrip('.').lower(), _normalise_port(port)) in _allowed_endpoints
|
||||||
|
|
||||||
|
|
||||||
def _guarded_getaddrinfo(host, *args, **kwargs):
|
def _guarded_getaddrinfo(host, *args, **kwargs):
|
||||||
results = _real_getaddrinfo(host, *args, **kwargs)
|
results = _real_getaddrinfo(host, *args, **kwargs)
|
||||||
# Mirrors getaddrinfo(host, port, ...): port is the first optional argument.
|
# Mirrors getaddrinfo(host, port, ...): port is the first optional argument.
|
||||||
port = args[0] if args else kwargs.get('port')
|
port = args[0] if args else kwargs.get('port')
|
||||||
is_proxy = _is_proxy_endpoint(host, port)
|
is_configured = _is_allowed_endpoint(host, port)
|
||||||
allowed = [r for r in results if _address_allowed_at_connect(r[4][0], is_proxy)]
|
allowed = [r for r in results if _address_allowed_at_connect(r[4][0], is_configured)]
|
||||||
if not allowed:
|
if not allowed:
|
||||||
raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}')
|
raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}')
|
||||||
return allowed
|
return allowed
|
||||||
|
|
||||||
|
|
||||||
def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
|
def install_socket_guard(allow_private: bool = False, proxy_urls=(), service_urls=()) -> None:
|
||||||
"""Enforce the no-internal-hosts policy at actual connection time.
|
"""Enforce the no-internal-hosts policy at actual connection time.
|
||||||
|
|
||||||
``validate_url`` only checks the *submitted* URL string; yt-dlp then follows
|
``validate_url`` only checks the *submitted* URL string; yt-dlp then follows
|
||||||
@@ -230,12 +237,16 @@ def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
|
|||||||
isolation as the backstop.
|
isolation as the backstop.
|
||||||
|
|
||||||
*proxy_urls* are the operator's configured proxies (yt-dlp's ``proxy`` option;
|
*proxy_urls* are the operator's configured proxies (yt-dlp's ``proxy`` option;
|
||||||
the ``*_proxy`` environment variables are picked up automatically). A proxy is
|
the ``*_proxy`` environment variables are picked up automatically), and
|
||||||
reachable at its own host:port wherever it lives — loopback, the LAN, a VPN
|
*service_urls* the helper services the download itself has to reach — the PO
|
||||||
range — and nothing else internal is. That costs proxied setups nothing and
|
token provider this image ships and starts on loopback. Each is reachable at
|
||||||
gives away nothing: yt-dlp resolves the proxy itself at exactly that host:port,
|
its own host:port wherever it lives — loopback, the LAN, a VPN range — and
|
||||||
and a media URL is either handed to the proxy unresolved or resolved on its own
|
nothing else internal is. That costs those setups nothing and gives away
|
||||||
merits — never inheriting the proxy's allowance.
|
little: yt-dlp dials each at exactly that host:port, and a media URL is either
|
||||||
|
handed to the proxy unresolved or resolved on its own merits — never
|
||||||
|
inheriting the allowance. A hostile media URL naming an allowed endpoint
|
||||||
|
reaches only what is listening there: a proxy that would have fetched it
|
||||||
|
anyway, or a token server with two endpoints and nothing to read.
|
||||||
|
|
||||||
When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not
|
When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not
|
||||||
installed at all, so proxy/VPN setups that route through private or Fake-IP
|
installed at all, so proxy/VPN setups that route through private or Fake-IP
|
||||||
@@ -243,10 +254,13 @@ def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
|
|||||||
"""
|
"""
|
||||||
if allow_private:
|
if allow_private:
|
||||||
return
|
return
|
||||||
_allowed_proxy_endpoints.clear()
|
proxy_endpoints = _collect_proxy_endpoints(proxy_urls)
|
||||||
_allowed_proxy_endpoints.update(_collect_proxy_endpoints(proxy_urls))
|
service_endpoints = _endpoints(service_urls) - proxy_endpoints
|
||||||
for host, port in sorted(_allowed_proxy_endpoints, key=lambda ep: (ep[0], ep[1] or 0)):
|
_allowed_endpoints.clear()
|
||||||
log.info(f'Allowing connections to configured proxy {host}:{port}')
|
_allowed_endpoints.update(proxy_endpoints | service_endpoints)
|
||||||
|
for label, endpoints in (('proxy', proxy_endpoints), ('service', service_endpoints)):
|
||||||
|
for host, port in sorted(endpoints, key=lambda ep: (ep[0], ep[1] or 0)):
|
||||||
|
log.info(f'Allowing connections to configured {label} {host}:{port}')
|
||||||
socket.getaddrinfo = _guarded_getaddrinfo
|
socket.getaddrinfo = _guarded_getaddrinfo
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+75
-5
@@ -92,6 +92,36 @@ class _DownloadYtdlLogger:
|
|||||||
# vanish in the child can deadlock it silently before it does any work. This
|
# vanish in the child can deadlock it silently before it does any work. This
|
||||||
# app creates background threads (executors, notifier callbacks) well before
|
# app creates background threads (executors, notifier callbacks) well before
|
||||||
# any download starts, so forcing fork there reproduces exactly that hazard.
|
# any download starts, so forcing fork there reproduces exactly that hazard.
|
||||||
|
# The image ships yt-dlp's bgutil PO token provider and starts it on loopback
|
||||||
|
# (docker-entrypoint.sh); the plugin dials this URL unless pointed elsewhere.
|
||||||
|
# Without a token YouTube serves 403s, so the connect-time guard has to let the
|
||||||
|
# download subprocess reach it.
|
||||||
|
_POT_PROVIDER_DEFAULT_URL = 'http://127.0.0.1:4416'
|
||||||
|
|
||||||
|
# extractor-arg keys the bgutil HTTP provider reads its base URL from: the
|
||||||
|
# current one first, then the deprecated form it still honours.
|
||||||
|
_POT_PROVIDER_BASE_URL_ARGS = (
|
||||||
|
('youtubepot-bgutilhttp', 'base_url'),
|
||||||
|
('youtube', 'getpot_bgutil_baseurl'),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _pot_provider_urls(ytdl_opts: dict) -> tuple:
|
||||||
|
"""Every PO token provider endpoint this download may dial: the bundled one,
|
||||||
|
plus any the operator pointed yt-dlp at through ``extractor_args``. The
|
||||||
|
bundled server runs either way, so it stays allowed even when a base URL is
|
||||||
|
configured."""
|
||||||
|
urls = [_POT_PROVIDER_DEFAULT_URL]
|
||||||
|
extractor_args = ytdl_opts.get('extractor_args')
|
||||||
|
if isinstance(extractor_args, dict):
|
||||||
|
for ie_key, arg in _POT_PROVIDER_BASE_URL_ARGS:
|
||||||
|
section = extractor_args.get(ie_key)
|
||||||
|
values = section.get(arg) if isinstance(section, dict) else None
|
||||||
|
if values:
|
||||||
|
urls.append(values[0])
|
||||||
|
return tuple(urls)
|
||||||
|
|
||||||
|
|
||||||
_MP_CTX = (
|
_MP_CTX = (
|
||||||
multiprocessing.get_context("fork")
|
multiprocessing.get_context("fork")
|
||||||
if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods()
|
if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods()
|
||||||
@@ -453,6 +483,7 @@ class DownloadInfo:
|
|||||||
clip_end=None,
|
clip_end=None,
|
||||||
live_status=None,
|
live_status=None,
|
||||||
live_release_timestamp=None,
|
live_release_timestamp=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
||||||
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
||||||
@@ -472,6 +503,7 @@ class DownloadInfo:
|
|||||||
self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None
|
self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None
|
||||||
self.playlist_item_limit = playlist_item_limit
|
self.playlist_item_limit = playlist_item_limit
|
||||||
self.split_by_chapters = split_by_chapters
|
self.split_by_chapters = split_by_chapters
|
||||||
|
self.sponsorblock = sponsorblock
|
||||||
self.chapter_template = chapter_template
|
self.chapter_template = chapter_template
|
||||||
self.subtitle_language = subtitle_language
|
self.subtitle_language = subtitle_language
|
||||||
self.subtitle_mode = subtitle_mode
|
self.subtitle_mode = subtitle_mode
|
||||||
@@ -541,6 +573,8 @@ class DownloadInfo:
|
|||||||
self.playlist_item_limit = 0
|
self.playlist_item_limit = 0
|
||||||
if not hasattr(self, "split_by_chapters"):
|
if not hasattr(self, "split_by_chapters"):
|
||||||
self.split_by_chapters = False
|
self.split_by_chapters = False
|
||||||
|
if not hasattr(self, "sponsorblock"):
|
||||||
|
self.sponsorblock = False
|
||||||
if not hasattr(self, "chapter_template"):
|
if not hasattr(self, "chapter_template"):
|
||||||
self.chapter_template = ""
|
self.chapter_template = ""
|
||||||
if not hasattr(self, "subtitle_language"):
|
if not hasattr(self, "subtitle_language"):
|
||||||
@@ -585,6 +619,7 @@ _PERSISTED_DOWNLOAD_FIELDS = (
|
|||||||
"custom_name_prefix",
|
"custom_name_prefix",
|
||||||
"playlist_item_limit",
|
"playlist_item_limit",
|
||||||
"split_by_chapters",
|
"split_by_chapters",
|
||||||
|
"sponsorblock",
|
||||||
"chapter_template",
|
"chapter_template",
|
||||||
"subtitle_language",
|
"subtitle_language",
|
||||||
"subtitle_mode",
|
"subtitle_mode",
|
||||||
@@ -764,10 +799,15 @@ class Download:
|
|||||||
# Re-validate every outbound connection at fetch time. validate_url only
|
# Re-validate every outbound connection at fetch time. validate_url only
|
||||||
# saw the submitted URL string; this catches redirects, DNS rebinding and
|
# saw the submitted URL string; this catches redirects, DNS rebinding and
|
||||||
# attacker-controlled media URLs pulled from a remote manifest, none of
|
# attacker-controlled media URLs pulled from a remote manifest, none of
|
||||||
# which it can see. The configured proxy is passed so that a proxy on an
|
# which it can see. The configured proxy and the PO token provider are
|
||||||
# internal address stays reachable at its own host:port without opening up
|
# passed so that each stays reachable at its own host:port without opening
|
||||||
# anything else. Skipped when ALLOW_PRIVATE_ADDRESSES trusts the environment.
|
# up anything else. Skipped when ALLOW_PRIVATE_ADDRESSES trusts the
|
||||||
install_socket_guard(self.allow_private, proxy_urls=(self.ytdl_opts.get('proxy'),))
|
# environment.
|
||||||
|
install_socket_guard(
|
||||||
|
self.allow_private,
|
||||||
|
proxy_urls=(self.ytdl_opts.get('proxy'),),
|
||||||
|
service_urls=_pot_provider_urls(self.ytdl_opts),
|
||||||
|
)
|
||||||
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
||||||
# Bound outside the try so the except branch can read what was captured
|
# Bound outside the try so the except branch can read what was captured
|
||||||
# before the error was raised.
|
# before the error was raised.
|
||||||
@@ -822,6 +862,27 @@ class Download:
|
|||||||
# this logger, so a user-supplied one must not replace it.
|
# this logger, so a user-supplied one must not replace it.
|
||||||
ytdl_params['logger'] = ytdl_logger
|
ytdl_params['logger'] = ytdl_logger
|
||||||
|
|
||||||
|
# SponsorBlock: mark sponsor segments and cut them out, the same
|
||||||
|
# postprocessor pair the CLI's --sponsorblock-remove sponsor builds.
|
||||||
|
# This has to stay above the chapter-splitting block: yt-dlp runs
|
||||||
|
# same-stage postprocessors in list order, and ModifyChapters must
|
||||||
|
# rewrite the chapter list before FFmpegSplitChapters cuts the file
|
||||||
|
# up, or the chapter files keep the sponsor segments and the
|
||||||
|
# removal desyncs the remaining chapter timings.
|
||||||
|
if getattr(self.info, 'sponsorblock', False):
|
||||||
|
if 'postprocessors' not in ytdl_params:
|
||||||
|
ytdl_params['postprocessors'] = []
|
||||||
|
ytdl_params['postprocessors'].append({
|
||||||
|
'key': 'SponsorBlock',
|
||||||
|
'categories': ['sponsor'],
|
||||||
|
'when': 'after_filter',
|
||||||
|
})
|
||||||
|
ytdl_params['postprocessors'].append({
|
||||||
|
'key': 'ModifyChapters',
|
||||||
|
'remove_sponsor_segments': ['sponsor'],
|
||||||
|
'force_keyframes': False,
|
||||||
|
})
|
||||||
|
|
||||||
# Add chapter splitting options if enabled
|
# Add chapter splitting options if enabled
|
||||||
if self.info.split_by_chapters:
|
if self.info.split_by_chapters:
|
||||||
ytdl_params['outtmpl']['chapter'] = self.info.chapter_template
|
ytdl_params['outtmpl']['chapter'] = self.info.chapter_template
|
||||||
@@ -1619,6 +1680,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen=None,
|
_add_gen=None,
|
||||||
retry_entry=None,
|
retry_entry=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if not entry:
|
if not entry:
|
||||||
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
||||||
@@ -1662,6 +1724,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
retry_entry,
|
retry_entry,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
elif etype == 'playlist' or etype == 'channel':
|
elif etype == 'playlist' or etype == 'channel':
|
||||||
if etype == 'playlist' and self.__is_channel_extraction(entry):
|
if etype == 'playlist' and self.__is_channel_extraction(entry):
|
||||||
@@ -1729,6 +1792,7 @@ class DownloadQueue:
|
|||||||
clip_end,
|
clip_end,
|
||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if any(res['status'] == 'error' for res in results):
|
if any(res['status'] == 'error' for res in results):
|
||||||
@@ -1769,6 +1833,7 @@ class DownloadQueue:
|
|||||||
clip_end=clip_end,
|
clip_end=clip_end,
|
||||||
live_status=entry.get('live_status'),
|
live_status=entry.get('live_status'),
|
||||||
live_release_timestamp=entry.get('release_timestamp'),
|
live_release_timestamp=entry.get('release_timestamp'),
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
await self.__add_download(dl, auto_start)
|
await self.__add_download(dl, auto_start)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
@@ -1849,13 +1914,14 @@ class DownloadQueue:
|
|||||||
already=None,
|
already=None,
|
||||||
_add_gen=None,
|
_add_gen=None,
|
||||||
retry_entry=None,
|
retry_entry=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if ytdl_options_presets is None:
|
if ytdl_options_presets is None:
|
||||||
ytdl_options_presets = []
|
ytdl_options_presets = []
|
||||||
log.info(
|
log.info(
|
||||||
f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} '
|
f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} '
|
||||||
f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} '
|
f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} '
|
||||||
f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=}'
|
f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=} {sponsorblock=}'
|
||||||
)
|
)
|
||||||
if already is None:
|
if already is None:
|
||||||
_add_gen = self._add_generation
|
_add_gen = self._add_generation
|
||||||
@@ -1918,6 +1984,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
retry_entry,
|
retry_entry,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def retry(self, id):
|
async def retry(self, id):
|
||||||
@@ -1954,6 +2021,7 @@ class DownloadQueue:
|
|||||||
info.clip_start,
|
info.clip_start,
|
||||||
info.clip_end,
|
info.clip_end,
|
||||||
retry_entry=info.entry,
|
retry_entry=info.entry,
|
||||||
|
sponsorblock=info.sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def add_entry(
|
async def add_entry(
|
||||||
@@ -1975,6 +2043,7 @@ class DownloadQueue:
|
|||||||
ytdl_options_overrides=None,
|
ytdl_options_overrides=None,
|
||||||
clip_start=None,
|
clip_start=None,
|
||||||
clip_end=None,
|
clip_end=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if ytdl_options_presets is None:
|
if ytdl_options_presets is None:
|
||||||
ytdl_options_presets = []
|
ytdl_options_presets = []
|
||||||
@@ -2000,6 +2069,7 @@ class DownloadQueue:
|
|||||||
clip_end,
|
clip_end,
|
||||||
already,
|
already,
|
||||||
None,
|
None,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def start_pending(self, ids):
|
async def start_pending(self, ids):
|
||||||
|
|||||||
@@ -399,6 +399,16 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="col-12">
|
<div class="col-12">
|
||||||
<div class="row g-2 align-items-center">
|
<div class="row g-2 align-items-center">
|
||||||
|
<div class="col-auto">
|
||||||
|
<div class="form-check form-switch">
|
||||||
|
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-sponsorblock"
|
||||||
|
name="sponsorblock" [(ngModel)]="sponsorblock" (change)="sponsorblockChanged()"
|
||||||
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
|
<label class="form-check-label" for="checkbox-sponsorblock"
|
||||||
|
ngbPopover="Cut out sponsor segments using SponsorBlock's crowd-sourced markers (YouTube only)."
|
||||||
|
triggers="hover" container="body">Remove sponsor segments</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div class="col-auto">
|
<div class="col-auto">
|
||||||
<div class="form-check form-switch">
|
<div class="form-check form-switch">
|
||||||
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
||||||
|
|||||||
@@ -86,6 +86,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
autoStart: boolean;
|
autoStart: boolean;
|
||||||
playlistItemLimit!: number;
|
playlistItemLimit!: number;
|
||||||
splitByChapters: boolean;
|
splitByChapters: boolean;
|
||||||
|
sponsorblock: boolean;
|
||||||
chapterTemplate: string;
|
chapterTemplate: string;
|
||||||
clipStart = '';
|
clipStart = '';
|
||||||
clipEnd = '';
|
clipEnd = '';
|
||||||
@@ -259,6 +260,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.quality = this.cookieService.get('metube_quality') || 'best';
|
this.quality = this.cookieService.get('metube_quality') || 'best';
|
||||||
this.autoStart = this.cookieService.get('metube_auto_start') !== 'false';
|
this.autoStart = this.cookieService.get('metube_auto_start') !== 'false';
|
||||||
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
||||||
|
this.sponsorblock = this.cookieService.get('metube_sponsorblock') === 'true';
|
||||||
// Will be set from backend configuration, use empty string as placeholder
|
// Will be set from backend configuration, use empty string as placeholder
|
||||||
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
||||||
this.clipStart = this.cookieService.get('metube_clip_start') || '';
|
this.clipStart = this.cookieService.get('metube_clip_start') || '';
|
||||||
@@ -855,6 +857,10 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sponsorblockChanged() {
|
||||||
|
this.cookieService.set('metube_sponsorblock', this.sponsorblock ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
|
}
|
||||||
|
|
||||||
splitByChaptersChanged() {
|
splitByChaptersChanged() {
|
||||||
this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
}
|
}
|
||||||
@@ -1111,6 +1117,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
playlistItemLimit: overrides.playlistItemLimit ?? this.playlistItemLimit,
|
playlistItemLimit: overrides.playlistItemLimit ?? this.playlistItemLimit,
|
||||||
autoStart: overrides.autoStart ?? this.autoStart,
|
autoStart: overrides.autoStart ?? this.autoStart,
|
||||||
splitByChapters: overrides.splitByChapters ?? this.splitByChapters,
|
splitByChapters: overrides.splitByChapters ?? this.splitByChapters,
|
||||||
|
sponsorblock: overrides.sponsorblock ?? this.sponsorblock,
|
||||||
chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate,
|
chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate,
|
||||||
subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage,
|
subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage,
|
||||||
subtitleMode: overrides.subtitleMode ?? this.subtitleMode,
|
subtitleMode: overrides.subtitleMode ?? this.subtitleMode,
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ export interface Download {
|
|||||||
custom_name_prefix: string;
|
custom_name_prefix: string;
|
||||||
playlist_item_limit: number;
|
playlist_item_limit: number;
|
||||||
split_by_chapters?: boolean;
|
split_by_chapters?: boolean;
|
||||||
|
sponsorblock?: boolean;
|
||||||
chapter_template?: string;
|
chapter_template?: string;
|
||||||
subtitle_language?: string;
|
subtitle_language?: string;
|
||||||
subtitle_mode?: string;
|
subtitle_mode?: string;
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ function basePayload(): AddDownloadPayload {
|
|||||||
playlistItemLimit: 0,
|
playlistItemLimit: 0,
|
||||||
autoStart: true,
|
autoStart: true,
|
||||||
splitByChapters: false,
|
splitByChapters: false,
|
||||||
|
sponsorblock: false,
|
||||||
chapterTemplate: '',
|
chapterTemplate: '',
|
||||||
subtitleLanguage: 'en',
|
subtitleLanguage: 'en',
|
||||||
subtitleMode: 'prefer_manual',
|
subtitleMode: 'prefer_manual',
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ export interface AddDownloadPayload {
|
|||||||
playlistItemLimit: number;
|
playlistItemLimit: number;
|
||||||
autoStart: boolean;
|
autoStart: boolean;
|
||||||
splitByChapters: boolean;
|
splitByChapters: boolean;
|
||||||
|
sponsorblock: boolean;
|
||||||
chapterTemplate: string;
|
chapterTemplate: string;
|
||||||
subtitleLanguage: string;
|
subtitleLanguage: string;
|
||||||
subtitleMode: string;
|
subtitleMode: string;
|
||||||
@@ -148,6 +149,7 @@ export class DownloadsService {
|
|||||||
playlist_item_limit: payload.playlistItemLimit,
|
playlist_item_limit: payload.playlistItemLimit,
|
||||||
auto_start: payload.autoStart,
|
auto_start: payload.autoStart,
|
||||||
split_by_chapters: payload.splitByChapters,
|
split_by_chapters: payload.splitByChapters,
|
||||||
|
sponsorblock: payload.sponsorblock,
|
||||||
chapter_template: payload.chapterTemplate,
|
chapter_template: payload.chapterTemplate,
|
||||||
subtitle_language: payload.subtitleLanguage,
|
subtitle_language: payload.subtitleLanguage,
|
||||||
subtitle_mode: payload.subtitleMode,
|
subtitle_mode: payload.subtitleMode,
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import { TestBed } from '@angular/core/testing';
|
||||||
|
import { provideHttpClient } from '@angular/common/http';
|
||||||
|
import { provideHttpClientTesting, HttpTestingController } from '@angular/common/http/testing';
|
||||||
|
import { Subject } from 'rxjs';
|
||||||
|
import { SubscriptionsService, SubscribePayload } from './subscriptions.service';
|
||||||
|
import { MeTubeSocket } from './metube-socket.service';
|
||||||
|
|
||||||
|
class MeTubeSocketStub {
|
||||||
|
private subjects: Record<string, Subject<string>> = {};
|
||||||
|
|
||||||
|
fromEvent(event: string) {
|
||||||
|
if (!this.subjects[event]) {
|
||||||
|
this.subjects[event] = new Subject<string>();
|
||||||
|
}
|
||||||
|
return this.subjects[event].asObservable();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function basePayload(): SubscribePayload {
|
||||||
|
return {
|
||||||
|
url: 'https://example.com/channel',
|
||||||
|
downloadType: 'video',
|
||||||
|
codec: 'auto',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
customNamePrefix: '',
|
||||||
|
playlistItemLimit: 0,
|
||||||
|
autoStart: true,
|
||||||
|
splitByChapters: false,
|
||||||
|
sponsorblock: false,
|
||||||
|
chapterTemplate: '',
|
||||||
|
subtitleLanguage: 'en',
|
||||||
|
subtitleMode: 'prefer_manual',
|
||||||
|
ytdlOptionsPresets: [],
|
||||||
|
ytdlOptionsOverrides: '',
|
||||||
|
clipStart: '',
|
||||||
|
clipEnd: '',
|
||||||
|
checkIntervalMinutes: 60,
|
||||||
|
titleRegex: '',
|
||||||
|
skipSubscriberOnly: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('SubscriptionsService', () => {
|
||||||
|
let httpMock: HttpTestingController;
|
||||||
|
let service: SubscriptionsService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await TestBed.configureTestingModule({
|
||||||
|
providers: [
|
||||||
|
SubscriptionsService,
|
||||||
|
provideHttpClient(),
|
||||||
|
provideHttpClientTesting(),
|
||||||
|
{ provide: MeTubeSocket, useValue: new MeTubeSocketStub() },
|
||||||
|
],
|
||||||
|
}).compileComponents();
|
||||||
|
|
||||||
|
service = TestBed.inject(SubscriptionsService);
|
||||||
|
httpMock = TestBed.inject(HttpTestingController);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('subscribe() carries the sponsorblock flag', () => {
|
||||||
|
service.subscribe({ ...basePayload(), sponsorblock: true }).subscribe();
|
||||||
|
const req = httpMock.expectOne('subscribe');
|
||||||
|
expect(req.request.method).toBe('POST');
|
||||||
|
expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: true }));
|
||||||
|
req.flush({ status: 'ok' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('subscribe() sends the flag off by default', () => {
|
||||||
|
service.subscribe(basePayload()).subscribe();
|
||||||
|
const req = httpMock.expectOne('subscribe');
|
||||||
|
expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: false }));
|
||||||
|
req.flush({ status: 'ok' });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -92,6 +92,7 @@ export class SubscriptionsService {
|
|||||||
playlist_item_limit: payload.playlistItemLimit,
|
playlist_item_limit: payload.playlistItemLimit,
|
||||||
auto_start: payload.autoStart,
|
auto_start: payload.autoStart,
|
||||||
split_by_chapters: payload.splitByChapters,
|
split_by_chapters: payload.splitByChapters,
|
||||||
|
sponsorblock: payload.sponsorblock,
|
||||||
chapter_template: payload.chapterTemplate,
|
chapter_template: payload.chapterTemplate,
|
||||||
subtitle_language: payload.subtitleLanguage,
|
subtitle_language: payload.subtitleLanguage,
|
||||||
subtitle_mode: payload.subtitleMode,
|
subtitle_mode: payload.subtitleMode,
|
||||||
|
|||||||
@@ -1236,11 +1236,11 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "yt-dlp"
|
name = "yt-dlp"
|
||||||
version = "2026.7.4"
|
version = "2026.8.19"
|
||||||
source = { registry = "https://pypi.org/simple" }
|
source = { registry = "https://pypi.org/simple" }
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/47/c5/9972af4b472b0d55badf841ebafd2f98944cb0ae0f46e11d01f363ea5b91/yt_dlp-2026.7.4.tar.gz", hash = "sha256:b094813404f87a9dd2186f00815231df32e5fd8a5403be0f807b3bb2d21a4432", size = 3049326, upload-time = "2026-07-04T22:42:14.837Z" }
|
sdist = { url = "https://files.pythonhosted.org/packages/1e/e0/832fa4ca334b766a06933a196066edc3dba37cdb6f14cd98d59bcc69a4b4/yt_dlp-2026.8.19.tar.gz", hash = "sha256:9e213e48cea35c66b378e4447903f118f6392a5fa380a2b6d7070ec86f4e0af1", size = 3052025, upload-time = "2026-08-19T23:48:59.291Z" }
|
||||||
wheels = [
|
wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/f9/8a/cd4c9b02c10c563adfe78118310129641900e1cd6de888cfae2452072696/yt_dlp-2026.7.4-py3-none-any.whl", hash = "sha256:f11f2b11d5a8ac4059f9bdf29fa4407dc7c6bb00c5097e95ca22a7a9db518266", size = 3184705, upload-time = "2026-07-04T22:42:12.989Z" },
|
{ url = "https://files.pythonhosted.org/packages/69/b2/8cd1613f56eed7ceb64fbd4df3f1c01246bfb098e6f398228bafda22b80b/yt_dlp-2026.8.19-py3-none-any.whl", hash = "sha256:1d57897e94c6665a0a6f9bc54b34e584284e32c034ffab3a7df25d8f7b24eedf", size = 3185533, upload-time = "2026-08-19T23:48:56.925Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.optional-dependencies]
|
[package.optional-dependencies]
|
||||||
|
|||||||
Reference in New Issue
Block a user