Compare commits

...

4 Commits

Author SHA1 Message Date
Alex Shnitman 72e8f5031f Merge PR #1028: first-class SponsorBlock toggle
Adds a "Remove sponsor segments" switch to the shared options panel,
building the same SponsorBlock + ModifyChapters pair the CLI's
--sponsorblock-remove sponsor does, and carries the flag through
subscriptions so the panel's control applies to both forms.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-18 15:47:02 +02:00
Alex Shnitman f3c464fad5 fix: let the download reach the PO token provider (closes #1064)
The image ships yt-dlp's bgutil PO token provider and starts it on
loopback, where the plugin dials it at http://127.0.0.1:4416. Since
482381d scoped the connect-time allowance to the configured proxy, the
download subprocess could no longer resolve it:

    Refusing to connect to non-global address for host '127.0.0.1'

which surfaces as the plugin's "Error reaching GET .../ping". Metadata
extraction runs in the main process and installs no guard, so titles kept
resolving while the download itself ran without a token — and YouTube
increasingly answers those with 403.

The allowance already had the right shape for this; it was just named for
its only user. Endpoints the operator or the image configured are now
allowed as a class: install_socket_guard takes service_urls alongside
proxy_urls, and ytdl derives them from the bundled default plus any
base_url set through the youtubepot-bgutilhttp (or the deprecated youtube
getpot_bgutil_baseurl) extractor argument. The bundled server runs either
way, so it stays allowed when a base URL is configured.

Matching stays exact host:port on the configured string, so nothing else
on loopback opens up: a hostile media URL naming the endpoint reaches a
token server with two endpoints and nothing worth reading.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-18 15:45:06 +02:00
tjelite1986 b10bb6103a feat: carry the SponsorBlock toggle into subscriptions
Subscriptions download unattended, which is where skipping sponsor reads
is most useful, so the flag now travels the same path the other download
options take: stored on SubscriptionInfo, persisted in the record, and
passed to add_entry for every entry a check queues.

Like the clip bounds, it is set when the subscription is created; the
update endpoint's field list is unchanged.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-17 10:39:11 +02:00
tjelite1986 8c2990e68a feat: first-class SponsorBlock toggle
A "Remove sponsor segments" switch next to "Split by chapters" queues
the download with the same postprocessor pair the CLI's
--sponsorblock-remove sponsor builds (SponsorBlock + ModifyChapters).
The flag persists as a cookie like the other form options, survives in
the queue records, and is carried into retries.

The pair is registered above the chapter-splitting block: yt-dlp runs
same-stage postprocessors in list order, so ModifyChapters has to
rewrite the chapter list before FFmpegSplitChapters cuts the file up,
matching what the CLI builds for --sponsorblock-remove sponsor
--split-chapters. With both toggles on the other way around the chapter
files keep the sponsor segments and the removal desyncs the remaining
chapter timings.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-17 10:39:10 +02:00
16 changed files with 520 additions and 79 deletions
+4
View File
@@ -725,6 +725,7 @@ def parse_download_options(post: dict) -> dict:
playlist_item_limit = post.get('playlist_item_limit') playlist_item_limit = post.get('playlist_item_limit')
auto_start = post.get('auto_start') auto_start = post.get('auto_start')
split_by_chapters = post.get('split_by_chapters') split_by_chapters = post.get('split_by_chapters')
sponsorblock = bool(post.get('sponsorblock'))
chapter_template = post.get('chapter_template') chapter_template = post.get('chapter_template')
subtitle_language = post.get('subtitle_language') subtitle_language = post.get('subtitle_language')
subtitle_mode = post.get('subtitle_mode') subtitle_mode = post.get('subtitle_mode')
@@ -845,6 +846,7 @@ def parse_download_options(post: dict) -> dict:
'playlist_item_limit': playlist_item_limit, 'playlist_item_limit': playlist_item_limit,
'auto_start': auto_start, 'auto_start': auto_start,
'split_by_chapters': split_by_chapters, 'split_by_chapters': split_by_chapters,
'sponsorblock': sponsorblock,
'chapter_template': chapter_template, 'chapter_template': chapter_template,
'subtitle_language': subtitle_language, 'subtitle_language': subtitle_language,
'subtitle_mode': subtitle_mode, 'subtitle_mode': subtitle_mode,
@@ -890,6 +892,7 @@ async def add(request):
o['ytdl_options_overrides'], o['ytdl_options_overrides'],
o['clip_start'], o['clip_start'],
o['clip_end'], o['clip_end'],
sponsorblock=o['sponsorblock'],
) )
return web.Response(text=serializer.encode(status)) return web.Response(text=serializer.encode(status))
@@ -970,6 +973,7 @@ async def subscribe(request):
subtitle_mode=o['subtitle_mode'], subtitle_mode=o['subtitle_mode'],
ytdl_options_presets=o['ytdl_options_presets'], ytdl_options_presets=o['ytdl_options_presets'],
ytdl_options_overrides=o['ytdl_options_overrides'], ytdl_options_overrides=o['ytdl_options_overrides'],
sponsorblock=o['sponsorblock'],
title_regex=post.get('title_regex'), title_regex=post.get('title_regex'),
skip_subscriber_only=skip_subscriber_only, skip_subscriber_only=skip_subscriber_only,
clip_start=sub_clip_start, clip_start=sub_clip_start,
+8
View File
@@ -182,6 +182,7 @@ class SubscriptionInfo:
auto_start: bool = True auto_start: bool = True
playlist_item_limit: int = 0 playlist_item_limit: int = 0
split_by_chapters: bool = False split_by_chapters: bool = False
sponsorblock: bool = False
chapter_template: str = "" chapter_template: str = ""
subtitle_language: str = "en" subtitle_language: str = "en"
subtitle_mode: str = "prefer_manual" subtitle_mode: str = "prefer_manual"
@@ -242,6 +243,7 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]:
"auto_start": sub.auto_start, "auto_start": sub.auto_start,
"playlist_item_limit": sub.playlist_item_limit, "playlist_item_limit": sub.playlist_item_limit,
"split_by_chapters": sub.split_by_chapters, "split_by_chapters": sub.split_by_chapters,
"sponsorblock": sub.sponsorblock,
"chapter_template": sub.chapter_template, "chapter_template": sub.chapter_template,
"subtitle_language": sub.subtitle_language, "subtitle_language": sub.subtitle_language,
"subtitle_mode": sub.subtitle_mode, "subtitle_mode": sub.subtitle_mode,
@@ -487,6 +489,7 @@ class SubscriptionManager:
ytdl_options_overrides: Optional[dict[str, Any]] = None, ytdl_options_overrides: Optional[dict[str, Any]] = None,
clip_start: Optional[float] = None, clip_start: Optional[float] = None,
clip_end: Optional[float] = None, clip_end: Optional[float] = None,
sponsorblock: bool = False,
) -> tuple[list[str], list[str]]: ) -> tuple[list[str], list[str]]:
queued_ids: list[str] = [] queued_ids: list[str] = []
queue_errors: list[str] = [] queue_errors: list[str] = []
@@ -519,6 +522,7 @@ class SubscriptionManager:
ytdl_options_overrides, ytdl_options_overrides,
clip_start, clip_start,
clip_end, clip_end,
sponsorblock=sponsorblock,
) )
if isinstance(result, dict) and result.get("status") == "error": if isinstance(result, dict) and result.get("status") == "error":
msg = str(result.get("msg") or f"Queueing failed for {vurl}") msg = str(result.get("msg") or f"Queueing failed for {vurl}")
@@ -606,6 +610,7 @@ class SubscriptionManager:
subtitle_mode: str, subtitle_mode: str,
ytdl_options_presets: Optional[list[str]] = None, ytdl_options_presets: Optional[list[str]] = None,
ytdl_options_overrides: Optional[dict[str, Any]] = None, ytdl_options_overrides: Optional[dict[str, Any]] = None,
sponsorblock: bool = False,
title_regex: Any = None, title_regex: Any = None,
skip_subscriber_only: Any = None, skip_subscriber_only: Any = None,
clip_start: Optional[float] = None, clip_start: Optional[float] = None,
@@ -689,6 +694,7 @@ class SubscriptionManager:
auto_start=bool(auto_start), auto_start=bool(auto_start),
playlist_item_limit=int(playlist_item_limit), playlist_item_limit=int(playlist_item_limit),
split_by_chapters=bool(split_by_chapters), split_by_chapters=bool(split_by_chapters),
sponsorblock=bool(sponsorblock),
chapter_template=chapter_template or "", chapter_template=chapter_template or "",
subtitle_language=subtitle_language, subtitle_language=subtitle_language,
subtitle_mode=subtitle_mode, subtitle_mode=subtitle_mode,
@@ -942,6 +948,7 @@ class SubscriptionManager:
dl_plimit = cur.playlist_item_limit dl_plimit = cur.playlist_item_limit
dl_autostart = cur.auto_start dl_autostart = cur.auto_start
dl_split = cur.split_by_chapters dl_split = cur.split_by_chapters
dl_sponsorblock = cur.sponsorblock
dl_chapter = cur.chapter_template dl_chapter = cur.chapter_template
dl_sublang = cur.subtitle_language dl_sublang = cur.subtitle_language
dl_submode = cur.subtitle_mode dl_submode = cur.subtitle_mode
@@ -1010,6 +1017,7 @@ class SubscriptionManager:
playlist_item_limit=dl_plimit, playlist_item_limit=dl_plimit,
auto_start=dl_autostart, auto_start=dl_autostart,
split_by_chapters=dl_split, split_by_chapters=dl_split,
sponsorblock=dl_sponsorblock,
chapter_template=dl_chapter or "", chapter_template=dl_chapter or "",
subtitle_language=dl_sublang, subtitle_language=dl_sublang,
subtitle_mode=dl_submode, subtitle_mode=dl_submode,
+19
View File
@@ -366,6 +366,25 @@ async def test_subscribe_passes_clip_bounds(mock_dqueue, monkeypatch):
assert kwargs["clip_end"] == pytest.approx(204.0) assert kwargs["clip_end"] == pytest.approx(204.0)
@pytest.mark.asyncio
async def test_subscribe_passes_sponsorblock(mock_dqueue, monkeypatch):
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
req = _json_request(
{**_valid_video_add_body(), "check_interval_minutes": 60, "sponsorblock": True}
)
resp = await main.subscribe(req)
assert resp.status == 200
assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is True
@pytest.mark.asyncio
async def test_subscribe_defaults_sponsorblock_off(mock_dqueue, monkeypatch):
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
req = _json_request({**_valid_video_add_body(), "check_interval_minutes": 60})
await main.subscribe(req)
assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is False
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_subscribe_without_clip_fields_stores_none(mock_dqueue, monkeypatch): async def test_subscribe_without_clip_fields_stores_none(mock_dqueue, monkeypatch):
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"})) monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
+26
View File
@@ -476,6 +476,32 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env):
assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True} assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True}
@pytest.mark.asyncio
async def test_retry_carries_the_sponsorblock_flag(dq_env):
notifier = AsyncMock()
dq = DownloadQueue(dq_env, notifier)
url = "https://example.com/watch?v=1"
dq.done.put(
Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url, sponsorblock=True))
)
def fake_extract(self, extracted_url, *_args, **_kwargs):
return {
"_type": "video",
"id": "vid1",
"title": "Test Video",
"url": extracted_url,
"webpage_url": extracted_url,
}
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
result = await dq.retry(url)
assert result["status"] == "ok"
assert dq.queue.get(url).info.sponsorblock is True
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env): async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
notifier = AsyncMock() notifier = AsyncMock()
+63
View File
@@ -479,6 +479,69 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(reloaded.get(sub_id).clip_start, 12.5) self.assertEqual(reloaded.get(sub_id).clip_start, 12.5)
self.assertIsNone(reloaded.get(sub_id).clip_end) self.assertIsNone(reloaded.get(sub_id).clip_end)
async def test_check_now_applies_subscription_sponsorblock(self):
"""Subscriptions download unattended, so the sponsor-segment removal has
to reach every entry the subscription queues, not just manual adds."""
with tempfile.TemporaryDirectory() as tmp:
queue = _Queue()
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
with patch(
"subscriptions.extract_flat_playlist",
side_effect=[
(
{"_type": "channel", "title": "Channel"},
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
),
(
{"_type": "channel", "title": "Channel"},
[
{"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"},
{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
],
),
],
):
result = await mgr.add_subscription(
"https://example.com/channel",
check_interval_minutes=60,
download_type="video",
codec="auto",
format="any",
quality="best",
folder="",
custom_name_prefix="",
auto_start=True,
playlist_item_limit=0,
split_by_chapters=False,
chapter_template="",
subtitle_language="en",
subtitle_mode="prefer_manual",
sponsorblock=True,
)
sub_id = result["subscription"]["id"]
self.assertTrue(mgr.get(sub_id).sponsorblock)
await mgr.check_now([sub_id])
self.assertEqual(len(queue.entries), 1)
_entry, _args, kwargs = queue.entries[0]
self.assertIs(kwargs["sponsorblock"], True)
async def test_sponsorblock_survives_reload_and_defaults_to_false(self):
with tempfile.TemporaryDirectory() as tmp:
cfg = _Config(tmp)
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
sub_id = await self._add_one_subscription(mgr)
# Records written before the field existed simply take the default.
self.assertFalse(mgr.get(sub_id).sponsorblock)
mgr.get(sub_id).sponsorblock = True
async with mgr._lock:
mgr._save_locked()
reloaded = SubscriptionManager(cfg, _Queue(), _Notifier())
self.assertTrue(reloaded.get(sub_id).sponsorblock)
async def test_check_now_queues_subscriber_only_when_skip_disabled(self): async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
queue = _Queue() queue = _Queue()
+71 -36
View File
@@ -12,7 +12,7 @@ from url_guard import (
_address_allowed_at_connect, _address_allowed_at_connect,
_address_is_global, _address_is_global,
_guarded_getaddrinfo, _guarded_getaddrinfo,
_proxy_endpoint, _url_endpoint,
install_socket_guard, install_socket_guard,
) )
@@ -121,19 +121,19 @@ class ConnectAddressPolicyTests(unittest.TestCase):
self.assertFalse(_address_allowed_at_connect("::1")) self.assertFalse(_address_allowed_at_connect("::1"))
def test_loopback_allowed_only_when_opted_in(self): def test_loopback_allowed_only_when_opted_in(self):
self.assertTrue(_address_allowed_at_connect("127.0.0.1", is_proxy_endpoint=True)) self.assertTrue(_address_allowed_at_connect("127.0.0.1", is_allowed_endpoint=True))
self.assertTrue(_address_allowed_at_connect("::1", is_proxy_endpoint=True)) self.assertTrue(_address_allowed_at_connect("::1", is_allowed_endpoint=True))
def test_proxy_opt_in_covers_any_internal_range(self): def test_proxy_opt_in_covers_any_internal_range(self):
# A proxy is just as legitimately on the LAN or a VPN range as on # A proxy is just as legitimately on the LAN or a VPN range as on
# loopback (#1055): the allowance follows the operator's configured # loopback (#1055): the allowance follows the operator's configured
# endpoint, not a particular address family. # endpoint, not a particular address family.
self.assertTrue(_address_allowed_at_connect("10.1.20.30", is_proxy_endpoint=True)) self.assertTrue(_address_allowed_at_connect("10.1.20.30", is_allowed_endpoint=True))
self.assertTrue(_address_allowed_at_connect("192.168.1.10", is_proxy_endpoint=True)) self.assertTrue(_address_allowed_at_connect("192.168.1.10", is_allowed_endpoint=True))
self.assertTrue(_address_allowed_at_connect("fd00::1", is_proxy_endpoint=True)) self.assertTrue(_address_allowed_at_connect("fd00::1", is_allowed_endpoint=True))
def test_opt_in_still_rejects_non_addresses(self): def test_opt_in_still_rejects_non_addresses(self):
self.assertFalse(_address_allowed_at_connect("not-an-ip", is_proxy_endpoint=True)) self.assertFalse(_address_allowed_at_connect("not-an-ip", is_allowed_endpoint=True))
def test_link_local_metadata_blocked(self): def test_link_local_metadata_blocked(self):
self.assertFalse(_address_allowed_at_connect("169.254.169.254")) self.assertFalse(_address_allowed_at_connect("169.254.169.254"))
@@ -187,36 +187,36 @@ class TunnelledIPv4Tests(unittest.TestCase):
self.assertIsNotNone(validate_url("http://nat64.example/x")) self.assertIsNotNone(validate_url("http://nat64.example/x"))
class ProxyEndpointParsingTests(unittest.TestCase): class EndpointParsingTests(unittest.TestCase):
def test_explicit_port(self): def test_explicit_port(self):
self.assertEqual(_proxy_endpoint("http://127.0.0.1:9050"), ("127.0.0.1", 9050)) self.assertEqual(_url_endpoint("http://127.0.0.1:9050"), ("127.0.0.1", 9050))
def test_default_port_per_scheme(self): def test_default_port_per_scheme(self):
self.assertEqual(_proxy_endpoint("socks5://127.0.0.1"), ("127.0.0.1", 1080)) self.assertEqual(_url_endpoint("socks5://127.0.0.1"), ("127.0.0.1", 1080))
self.assertEqual(_proxy_endpoint("http://127.0.0.1"), ("127.0.0.1", 80)) self.assertEqual(_url_endpoint("http://127.0.0.1"), ("127.0.0.1", 80))
def test_bare_host_port(self): def test_bare_host_port(self):
self.assertEqual(_proxy_endpoint("127.0.0.1:8080"), ("127.0.0.1", 8080)) self.assertEqual(_url_endpoint("127.0.0.1:8080"), ("127.0.0.1", 8080))
def test_hostname_lowercased(self): def test_hostname_lowercased(self):
self.assertEqual(_proxy_endpoint("http://LocalHost.:9050"), ("localhost", 9050)) self.assertEqual(_url_endpoint("http://LocalHost.:9050"), ("localhost", 9050))
def test_ipv6_literal(self): def test_ipv6_literal(self):
self.assertEqual(_proxy_endpoint("http://[::1]:9050"), ("::1", 9050)) self.assertEqual(_url_endpoint("http://[::1]:9050"), ("::1", 9050))
def test_empty_and_invalid(self): def test_empty_and_invalid(self):
self.assertIsNone(_proxy_endpoint("")) self.assertIsNone(_url_endpoint(""))
self.assertIsNone(_proxy_endpoint(" ")) self.assertIsNone(_url_endpoint(" "))
self.assertIsNone(_proxy_endpoint(None)) self.assertIsNone(_url_endpoint(None))
self.assertIsNone(_proxy_endpoint("http://")) self.assertIsNone(_url_endpoint("http://"))
class GuardedGetaddrinfoTests(unittest.TestCase): class GuardedGetaddrinfoTests(unittest.TestCase):
def setUp(self): def setUp(self):
# Default state: no proxy configured, so no loopback destination allowed. # Default state: no proxy configured, so no loopback destination allowed.
saved = set(url_guard._allowed_proxy_endpoints) saved = set(url_guard._allowed_endpoints)
url_guard._allowed_proxy_endpoints = set() url_guard._allowed_endpoints = set()
self.addCleanup(lambda: setattr(url_guard, "_allowed_proxy_endpoints", saved)) self.addCleanup(lambda: setattr(url_guard, "_allowed_endpoints", saved))
def test_internal_only_raises(self): def test_internal_only_raises(self):
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")): with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")):
@@ -235,27 +235,27 @@ class GuardedGetaddrinfoTests(unittest.TestCase):
with self.assertRaises(socket.gaierror): with self.assertRaises(socket.gaierror):
_guarded_getaddrinfo("127.0.0.1", 9999) _guarded_getaddrinfo("127.0.0.1", 9999)
def test_loopback_allowed_at_configured_proxy_endpoint(self): def test_loopback_allowed_at_configured_url_endpoint(self):
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)} url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")): with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
results = _guarded_getaddrinfo("127.0.0.1", 9050) results = _guarded_getaddrinfo("127.0.0.1", 9050)
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"]) self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
def test_loopback_blocked_at_other_port_on_proxy_host(self): def test_loopback_blocked_at_other_port_on_proxy_host(self):
# Same host as the proxy, different port: still off limits. # Same host as the proxy, different port: still off limits.
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)} url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")): with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
with self.assertRaises(socket.gaierror): with self.assertRaises(socket.gaierror):
_guarded_getaddrinfo("127.0.0.1", 9999) _guarded_getaddrinfo("127.0.0.1", 9999)
def test_proxy_reachable_by_hostname(self): def test_proxy_reachable_by_hostname(self):
url_guard._allowed_proxy_endpoints = {("localhost", 9050)} url_guard._allowed_endpoints = {("localhost", 9050)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")): with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
results = _guarded_getaddrinfo("localhost", 9050) results = _guarded_getaddrinfo("localhost", 9050)
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"]) self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
def test_string_port_is_normalised(self): def test_string_port_is_normalised(self):
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)} url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")): with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
results = _guarded_getaddrinfo("127.0.0.1", "9050") results = _guarded_getaddrinfo("127.0.0.1", "9050")
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"]) self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
@@ -263,22 +263,38 @@ class GuardedGetaddrinfoTests(unittest.TestCase):
def test_lan_proxy_reachable(self): def test_lan_proxy_reachable(self):
# #1055: a socks5 proxy on the LAN, refused while the allowance was # #1055: a socks5 proxy on the LAN, refused while the allowance was
# loopback-only, which pushed operators to ALLOW_PRIVATE_ADDRESSES. # loopback-only, which pushed operators to ALLOW_PRIVATE_ADDRESSES.
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)} url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")): with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
results = _guarded_getaddrinfo("10.1.20.30", 1080) results = _guarded_getaddrinfo("10.1.20.30", 1080)
self.assertEqual([r[4][0] for r in results], ["10.1.20.30"]) self.assertEqual([r[4][0] for r in results], ["10.1.20.30"])
def test_other_lan_host_still_blocked(self): def test_other_lan_host_still_blocked(self):
# The allowance is the proxy's endpoint, not its subnet. # The allowance is the proxy's endpoint, not its subnet.
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)} url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.31")): with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.31")):
with self.assertRaises(socket.gaierror): with self.assertRaises(socket.gaierror):
_guarded_getaddrinfo("10.1.20.31", 1080) _guarded_getaddrinfo("10.1.20.31", 1080)
def test_pot_provider_reachable_on_loopback(self):
# #1064: the bundled PO token provider listens on loopback, and blocking
# it left every default install downloading YouTube without a token.
url_guard._allowed_endpoints = {("127.0.0.1", 4416)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
results = _guarded_getaddrinfo("127.0.0.1", 4416)
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
def test_other_loopback_service_still_blocked(self):
# MeTube's own port is one hop away on the same interface: allowing the
# token provider must not allow the rest of loopback.
url_guard._allowed_endpoints = {("127.0.0.1", 4416)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
with self.assertRaises(socket.gaierror):
_guarded_getaddrinfo("127.0.0.1", 8081)
def test_proxy_address_not_borrowable_by_another_host(self): def test_proxy_address_not_borrowable_by_another_host(self):
# Matching is on the configured host string: a manifest URL that resolves # Matching is on the configured host string: a manifest URL that resolves
# to the proxy's address under its own name gets no allowance. # to the proxy's address under its own name gets no allowance.
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)} url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")): with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
with self.assertRaises(socket.gaierror): with self.assertRaises(socket.gaierror):
_guarded_getaddrinfo("evil.example", 1080) _guarded_getaddrinfo("evil.example", 1080)
@@ -312,9 +328,9 @@ class AllowPrivateBypassTests(unittest.TestCase):
class InstallSocketGuardTests(unittest.TestCase): class InstallSocketGuardTests(unittest.TestCase):
def setUp(self): def setUp(self):
original, saved = socket.getaddrinfo, set(url_guard._allowed_proxy_endpoints) original, saved = socket.getaddrinfo, set(url_guard._allowed_endpoints)
self.addCleanup(lambda: setattr(socket, "getaddrinfo", original)) self.addCleanup(lambda: setattr(socket, "getaddrinfo", original))
self.addCleanup(lambda: setattr(url_guard, "_allowed_proxy_endpoints", saved)) self.addCleanup(lambda: setattr(url_guard, "_allowed_endpoints", saved))
# Keep the host's own environment out of the assertions below. # Keep the host's own environment out of the assertions below.
patcher = mock.patch("url_guard.urllib.request.getproxies", return_value={}) patcher = mock.patch("url_guard.urllib.request.getproxies", return_value={})
self.getproxies = patcher.start() self.getproxies = patcher.start()
@@ -329,26 +345,45 @@ class InstallSocketGuardTests(unittest.TestCase):
def test_no_proxy_means_no_loopback_allowance(self): def test_no_proxy_means_no_loopback_allowance(self):
install_socket_guard() install_socket_guard()
self.assertEqual(url_guard._allowed_proxy_endpoints, set()) self.assertEqual(url_guard._allowed_endpoints, set())
def test_explicit_proxy_is_registered(self): def test_explicit_proxy_is_registered(self):
install_socket_guard(proxy_urls=("socks5://127.0.0.1:9050",)) install_socket_guard(proxy_urls=("socks5://127.0.0.1:9050",))
self.assertEqual(url_guard._allowed_proxy_endpoints, {("127.0.0.1", 9050)}) self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 9050)})
def test_unset_proxy_option_is_ignored(self): def test_unset_proxy_option_is_ignored(self):
# ytdl_opts.get('proxy') is None when the operator configured no proxy. # ytdl_opts.get('proxy') is None when the operator configured no proxy.
install_socket_guard(proxy_urls=(None,)) install_socket_guard(proxy_urls=(None,))
self.assertEqual(url_guard._allowed_proxy_endpoints, set()) self.assertEqual(url_guard._allowed_endpoints, set())
def test_environment_proxies_are_registered(self): def test_environment_proxies_are_registered(self):
self.getproxies.return_value = {"http": "http://127.0.0.1:8080"} self.getproxies.return_value = {"http": "http://127.0.0.1:8080"}
install_socket_guard() install_socket_guard()
self.assertEqual(url_guard._allowed_proxy_endpoints, {("127.0.0.1", 8080)}) self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 8080)})
def test_service_url_is_registered(self):
install_socket_guard(service_urls=("http://127.0.0.1:4416",))
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 4416)})
def test_service_and_proxy_endpoints_coexist(self):
install_socket_guard(
proxy_urls=("socks5://10.1.20.30:1080",),
service_urls=("http://127.0.0.1:4416",),
)
self.assertEqual(
url_guard._allowed_endpoints,
{("10.1.20.30", 1080), ("127.0.0.1", 4416)},
)
def test_service_urls_reset_between_installs(self):
install_socket_guard(service_urls=("http://127.0.0.1:4416",))
install_socket_guard()
self.assertEqual(url_guard._allowed_endpoints, set())
def test_endpoints_reset_between_installs(self): def test_endpoints_reset_between_installs(self):
install_socket_guard(proxy_urls=("http://127.0.0.1:8080",)) install_socket_guard(proxy_urls=("http://127.0.0.1:8080",))
install_socket_guard(proxy_urls=(None,)) install_socket_guard(proxy_urls=(None,))
self.assertEqual(url_guard._allowed_proxy_endpoints, set()) self.assertEqual(url_guard._allowed_endpoints, set())
if __name__ == "__main__": if __name__ == "__main__":
+103
View File
@@ -77,6 +77,7 @@ from ytdl import (
MusicMetadataPreProcessor, MusicMetadataPreProcessor,
_compact_persisted_entry, _compact_persisted_entry,
_convert_srt_to_txt_file, _convert_srt_to_txt_file,
_pot_provider_urls,
_AlbumArtistPostProcessor, _AlbumArtistPostProcessor,
_resolve_outtmpl_fields, _resolve_outtmpl_fields,
_sanitize_entry_for_pickle, _sanitize_entry_for_pickle,
@@ -678,6 +679,69 @@ class DownloadResultTests(unittest.TestCase):
) )
def _capture_ytdl_params(download: Download) -> dict:
"""Run ``_download`` far enough to capture the params it builds."""
fake_ydl = MagicMock()
fake_ydl.download.return_value = 0
download.status_queue = types.SimpleNamespace(put=lambda _: None)
with patch('ytdl.install_socket_guard'), \
patch.object(Download, '_make_youtube_dl', return_value=fake_ydl) as make:
download._download()
params, = make.call_args.args
return params
class SponsorBlockPostprocessorTests(unittest.TestCase):
def test_no_sponsorblock_postprocessors_when_disabled(self):
download = _make_test_download()
params = _capture_ytdl_params(download)
keys = [pp['key'] for pp in params.get('postprocessors', [])]
self.assertNotIn('SponsorBlock', keys)
self.assertNotIn('ModifyChapters', keys)
def test_sponsorblock_pair_matches_the_cli(self):
download = _make_test_download()
download.info.sponsorblock = True
params = _capture_ytdl_params(download)
self.assertEqual(
params['postprocessors'],
[
{
'key': 'SponsorBlock',
'categories': ['sponsor'],
'when': 'after_filter',
},
{
'key': 'ModifyChapters',
'remove_sponsor_segments': ['sponsor'],
'force_keyframes': False,
},
],
)
def test_segment_removal_runs_before_the_chapter_split(self):
# yt-dlp runs same-stage postprocessors in list order, so ModifyChapters
# has to rewrite the chapter list before FFmpegSplitChapters cuts the
# file up -- the order the CLI builds for
# --sponsorblock-remove sponsor --split-chapters.
download = _make_test_download()
download.info.sponsorblock = True
download.info.split_by_chapters = True
download.info.chapter_template = '%(section_number)s.%(ext)s'
params = _capture_ytdl_params(download)
keys = [pp['key'] for pp in params['postprocessors']]
self.assertEqual(keys, ['SponsorBlock', 'ModifyChapters', 'FFmpegSplitChapters'])
self.assertEqual(params['outtmpl']['chapter'], '%(section_number)s.%(ext)s')
class ProgressThrottleTests(unittest.TestCase): class ProgressThrottleTests(unittest.TestCase):
def test_downloading_ticks_are_throttled(self): def test_downloading_ticks_are_throttled(self):
dl = _make_test_download() dl = _make_test_download()
@@ -1071,5 +1135,44 @@ class ShortTitleForFailedUrlTests(unittest.TestCase):
self.assertEqual(_short_title_for_failed_url(malformed), malformed) self.assertEqual(_short_title_for_failed_url(malformed), malformed)
class PotProviderUrlsTests(unittest.TestCase):
"""#1064: the connect-time guard must let the download reach the PO token
provider, so it has to know every endpoint yt-dlp might dial for one."""
def test_bundled_provider_by_default(self):
self.assertEqual(_pot_provider_urls({}), ("http://127.0.0.1:4416",))
def test_configured_base_url_is_added(self):
urls = _pot_provider_urls({
"extractor_args": {"youtubepot-bgutilhttp": {"base_url": ["http://pot:4416"]}},
})
# The bundled server runs regardless, so both stay reachable.
self.assertEqual(urls, ("http://127.0.0.1:4416", "http://pot:4416"))
def test_deprecated_base_url_arg_is_honoured(self):
urls = _pot_provider_urls({
"extractor_args": {"youtube": {"getpot_bgutil_baseurl": ["http://pot:4416"]}},
})
self.assertEqual(urls, ("http://127.0.0.1:4416", "http://pot:4416"))
def test_unrelated_extractor_args_are_ignored(self):
urls = _pot_provider_urls({
"extractor_args": {"youtube": {"player_client": ["web"]}},
})
self.assertEqual(urls, ("http://127.0.0.1:4416",))
def test_malformed_extractor_args_do_not_raise(self):
# YTDL_OPTIONS is operator-supplied JSON and reaches here unvalidated.
for opts in (
{"extractor_args": None},
{"extractor_args": "youtube:player_client=web"},
{"extractor_args": {"youtubepot-bgutilhttp": "http://pot:4416"}},
{"extractor_args": {"youtubepot-bgutilhttp": {"base_url": []}}},
):
with self.subTest(opts=opts):
self.assertEqual(_pot_provider_urls(opts), ("http://127.0.0.1:4416",))
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+52 -38
View File
@@ -37,8 +37,8 @@ log = logging.getLogger('url_guard')
_ALLOWED_SCHEMES = ('http', 'https') _ALLOWED_SCHEMES = ('http', 'https')
# Ports to assume when a configured proxy URL omits one, per proxy scheme. # Ports to assume when a configured endpoint URL omits one, per scheme.
_PROXY_DEFAULT_PORTS = { _SCHEME_DEFAULT_PORTS = {
'http': 80, 'http': 80,
'https': 443, 'https': 443,
'socks4': 1080, 'socks4': 1080,
@@ -129,31 +129,32 @@ def _address_is_global(addr: str) -> bool:
return bool(ips) and all(ip.is_global for ip in ips) return bool(ips) and all(ip.is_global for ip in ips)
def _address_allowed_at_connect(addr: str, is_proxy_endpoint: bool = False) -> bool: def _address_allowed_at_connect(addr: str, is_allowed_endpoint: bool = False) -> bool:
"""True if *addr* may be connected to at download time. """True if *addr* may be connected to at download time.
Permits global addresses, and anything at all when the destination is an Permits global addresses, and anything at all when the destination is an
operator-configured proxy (see ``_is_proxy_endpoint``). Internal addresses endpoint the operator or the image configured a proxy, or the PO token
are otherwise refused with no blanket exception: media URLs that yt-dlp provider (see ``_is_allowed_endpoint``). Internal addresses are otherwise
derives from a remote manifest are attacker-controlled and reach this policy refused with no blanket exception: media URLs that yt-dlp derives from a
without passing ``validate_url``, so any range opened here is a range a remote manifest are attacker-controlled and reach this policy without passing
hostile playlist can read from the server's own network. Blocks link-local ``validate_url``, so any range opened here is a range a hostile playlist can
read from the server's own network. Blocks link-local
(cloud metadata at 169.254.169.254), private (RFC1918), loopback, (cloud metadata at 169.254.169.254), private (RFC1918), loopback,
unique-local and every other non-global range. unique-local and every other non-global range.
""" """
ips = _ips_to_judge(addr) ips = _ips_to_judge(addr)
if not ips: if not ips:
return False return False
return is_proxy_endpoint or all(ip.is_global for ip in ips) return is_allowed_endpoint or all(ip.is_global for ip in ips)
def _proxy_endpoint(proxy_url: str): def _url_endpoint(url: str):
"""Parse a proxy URL into a ``(hostname, port)`` pair, or ``None`` if it has """Parse a configured URL into a ``(hostname, port)`` pair, or ``None`` if it
no usable host. Used to scope the internal-address allowance to that endpoint has no usable host. Used to scope the internal-address allowance to that
alone.""" endpoint alone."""
if not isinstance(proxy_url, str) or not proxy_url.strip(): if not isinstance(url, str) or not url.strip():
return None return None
candidate = proxy_url.strip() candidate = url.strip()
if '://' not in candidate: if '://' not in candidate:
# Bare host:port, as accepted by the *_proxy environment variables. # Bare host:port, as accepted by the *_proxy environment variables.
candidate = '//' + candidate candidate = '//' + candidate
@@ -165,23 +166,28 @@ def _proxy_endpoint(proxy_url: str):
if not hostname: if not hostname:
return None return None
if port is None: if port is None:
port = _PROXY_DEFAULT_PORTS.get(parts.scheme.lower()) port = _SCHEME_DEFAULT_PORTS.get(parts.scheme.lower())
return (hostname.rstrip('.').lower(), port) return (hostname.rstrip('.').lower(), port)
def _endpoints(urls) -> set:
"""The parseable endpoints among *urls*, dropping any that name no host."""
return {ep for ep in map(_url_endpoint, urls) if ep is not None}
def _collect_proxy_endpoints(proxy_urls) -> set: def _collect_proxy_endpoints(proxy_urls) -> set:
"""Endpoints of every proxy this download may legitimately dial: the explicit """Endpoints of every proxy this download may legitimately dial: the explicit
yt-dlp ``proxy`` option plus the ``*_proxy`` environment variables yt-dlp falls yt-dlp ``proxy`` option plus the ``*_proxy`` environment variables yt-dlp falls
back to. All are operator-configured, unlike the URLs inside fetched media.""" back to. All are operator-configured, unlike the URLs inside fetched media."""
candidates = list(proxy_urls) + list(urllib.request.getproxies().values()) candidates = list(proxy_urls) + list(urllib.request.getproxies().values())
return {ep for ep in map(_proxy_endpoint, candidates) if ep is not None} return _endpoints(candidates)
# Captured at import so re-installing the guard never wraps the wrapper. # Captured at import so re-installing the guard never wraps the wrapper.
_real_getaddrinfo = socket.getaddrinfo _real_getaddrinfo = socket.getaddrinfo
# Populated by install_socket_guard; empty means no internal destination is allowed. # Populated by install_socket_guard; empty means no internal destination is allowed.
_allowed_proxy_endpoints: set = set() _allowed_endpoints: set = set()
def _normalise_port(port): def _normalise_port(port):
@@ -196,28 +202,29 @@ def _normalise_port(port):
return port return port
def _is_proxy_endpoint(host, port) -> bool: def _is_allowed_endpoint(host, port) -> bool:
"""True when host:port is exactly an endpoint the operator configured as a """True when host:port is exactly one of the endpoints this download is
proxy. Matching is on the configured host *string*, not on the resolved configured to dial a proxy or the PO token provider. Matching is on the
address, so a hostile media URL cannot borrow the allowance by resolving to configured host *string*, not on the resolved address, so a hostile media URL
the same address under a different name.""" cannot borrow the allowance by resolving to the same address under a
if not _allowed_proxy_endpoints or host is None: different name."""
if not _allowed_endpoints or host is None:
return False return False
return (str(host).rstrip('.').lower(), _normalise_port(port)) in _allowed_proxy_endpoints return (str(host).rstrip('.').lower(), _normalise_port(port)) in _allowed_endpoints
def _guarded_getaddrinfo(host, *args, **kwargs): def _guarded_getaddrinfo(host, *args, **kwargs):
results = _real_getaddrinfo(host, *args, **kwargs) results = _real_getaddrinfo(host, *args, **kwargs)
# Mirrors getaddrinfo(host, port, ...): port is the first optional argument. # Mirrors getaddrinfo(host, port, ...): port is the first optional argument.
port = args[0] if args else kwargs.get('port') port = args[0] if args else kwargs.get('port')
is_proxy = _is_proxy_endpoint(host, port) is_configured = _is_allowed_endpoint(host, port)
allowed = [r for r in results if _address_allowed_at_connect(r[4][0], is_proxy)] allowed = [r for r in results if _address_allowed_at_connect(r[4][0], is_configured)]
if not allowed: if not allowed:
raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}') raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}')
return allowed return allowed
def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None: def install_socket_guard(allow_private: bool = False, proxy_urls=(), service_urls=()) -> None:
"""Enforce the no-internal-hosts policy at actual connection time. """Enforce the no-internal-hosts policy at actual connection time.
``validate_url`` only checks the *submitted* URL string; yt-dlp then follows ``validate_url`` only checks the *submitted* URL string; yt-dlp then follows
@@ -230,12 +237,16 @@ def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
isolation as the backstop. isolation as the backstop.
*proxy_urls* are the operator's configured proxies (yt-dlp's ``proxy`` option; *proxy_urls* are the operator's configured proxies (yt-dlp's ``proxy`` option;
the ``*_proxy`` environment variables are picked up automatically). A proxy is the ``*_proxy`` environment variables are picked up automatically), and
reachable at its own host:port wherever it lives loopback, the LAN, a VPN *service_urls* the helper services the download itself has to reach the PO
range and nothing else internal is. That costs proxied setups nothing and token provider this image ships and starts on loopback. Each is reachable at
gives away nothing: yt-dlp resolves the proxy itself at exactly that host:port, its own host:port wherever it lives loopback, the LAN, a VPN range and
and a media URL is either handed to the proxy unresolved or resolved on its own nothing else internal is. That costs those setups nothing and gives away
merits never inheriting the proxy's allowance. little: yt-dlp dials each at exactly that host:port, and a media URL is either
handed to the proxy unresolved or resolved on its own merits never
inheriting the allowance. A hostile media URL naming an allowed endpoint
reaches only what is listening there: a proxy that would have fetched it
anyway, or a token server with two endpoints and nothing to read.
When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not
installed at all, so proxy/VPN setups that route through private or Fake-IP installed at all, so proxy/VPN setups that route through private or Fake-IP
@@ -243,10 +254,13 @@ def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
""" """
if allow_private: if allow_private:
return return
_allowed_proxy_endpoints.clear() proxy_endpoints = _collect_proxy_endpoints(proxy_urls)
_allowed_proxy_endpoints.update(_collect_proxy_endpoints(proxy_urls)) service_endpoints = _endpoints(service_urls) - proxy_endpoints
for host, port in sorted(_allowed_proxy_endpoints, key=lambda ep: (ep[0], ep[1] or 0)): _allowed_endpoints.clear()
log.info(f'Allowing connections to configured proxy {host}:{port}') _allowed_endpoints.update(proxy_endpoints | service_endpoints)
for label, endpoints in (('proxy', proxy_endpoints), ('service', service_endpoints)):
for host, port in sorted(endpoints, key=lambda ep: (ep[0], ep[1] or 0)):
log.info(f'Allowing connections to configured {label} {host}:{port}')
socket.getaddrinfo = _guarded_getaddrinfo socket.getaddrinfo = _guarded_getaddrinfo
+75 -5
View File
@@ -92,6 +92,36 @@ class _DownloadYtdlLogger:
# vanish in the child can deadlock it silently before it does any work. This # vanish in the child can deadlock it silently before it does any work. This
# app creates background threads (executors, notifier callbacks) well before # app creates background threads (executors, notifier callbacks) well before
# any download starts, so forcing fork there reproduces exactly that hazard. # any download starts, so forcing fork there reproduces exactly that hazard.
# The image ships yt-dlp's bgutil PO token provider and starts it on loopback
# (docker-entrypoint.sh); the plugin dials this URL unless pointed elsewhere.
# Without a token YouTube serves 403s, so the connect-time guard has to let the
# download subprocess reach it.
_POT_PROVIDER_DEFAULT_URL = 'http://127.0.0.1:4416'
# extractor-arg keys the bgutil HTTP provider reads its base URL from: the
# current one first, then the deprecated form it still honours.
_POT_PROVIDER_BASE_URL_ARGS = (
('youtubepot-bgutilhttp', 'base_url'),
('youtube', 'getpot_bgutil_baseurl'),
)
def _pot_provider_urls(ytdl_opts: dict) -> tuple:
"""Every PO token provider endpoint this download may dial: the bundled one,
plus any the operator pointed yt-dlp at through ``extractor_args``. The
bundled server runs either way, so it stays allowed even when a base URL is
configured."""
urls = [_POT_PROVIDER_DEFAULT_URL]
extractor_args = ytdl_opts.get('extractor_args')
if isinstance(extractor_args, dict):
for ie_key, arg in _POT_PROVIDER_BASE_URL_ARGS:
section = extractor_args.get(ie_key)
values = section.get(arg) if isinstance(section, dict) else None
if values:
urls.append(values[0])
return tuple(urls)
_MP_CTX = ( _MP_CTX = (
multiprocessing.get_context("fork") multiprocessing.get_context("fork")
if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods() if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods()
@@ -453,6 +483,7 @@ class DownloadInfo:
clip_end=None, clip_end=None,
live_status=None, live_status=None,
live_release_timestamp=None, live_release_timestamp=None,
sponsorblock=False,
): ):
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
@@ -472,6 +503,7 @@ class DownloadInfo:
self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None
self.playlist_item_limit = playlist_item_limit self.playlist_item_limit = playlist_item_limit
self.split_by_chapters = split_by_chapters self.split_by_chapters = split_by_chapters
self.sponsorblock = sponsorblock
self.chapter_template = chapter_template self.chapter_template = chapter_template
self.subtitle_language = subtitle_language self.subtitle_language = subtitle_language
self.subtitle_mode = subtitle_mode self.subtitle_mode = subtitle_mode
@@ -541,6 +573,8 @@ class DownloadInfo:
self.playlist_item_limit = 0 self.playlist_item_limit = 0
if not hasattr(self, "split_by_chapters"): if not hasattr(self, "split_by_chapters"):
self.split_by_chapters = False self.split_by_chapters = False
if not hasattr(self, "sponsorblock"):
self.sponsorblock = False
if not hasattr(self, "chapter_template"): if not hasattr(self, "chapter_template"):
self.chapter_template = "" self.chapter_template = ""
if not hasattr(self, "subtitle_language"): if not hasattr(self, "subtitle_language"):
@@ -585,6 +619,7 @@ _PERSISTED_DOWNLOAD_FIELDS = (
"custom_name_prefix", "custom_name_prefix",
"playlist_item_limit", "playlist_item_limit",
"split_by_chapters", "split_by_chapters",
"sponsorblock",
"chapter_template", "chapter_template",
"subtitle_language", "subtitle_language",
"subtitle_mode", "subtitle_mode",
@@ -764,10 +799,15 @@ class Download:
# Re-validate every outbound connection at fetch time. validate_url only # Re-validate every outbound connection at fetch time. validate_url only
# saw the submitted URL string; this catches redirects, DNS rebinding and # saw the submitted URL string; this catches redirects, DNS rebinding and
# attacker-controlled media URLs pulled from a remote manifest, none of # attacker-controlled media URLs pulled from a remote manifest, none of
# which it can see. The configured proxy is passed so that a proxy on an # which it can see. The configured proxy and the PO token provider are
# internal address stays reachable at its own host:port without opening up # passed so that each stays reachable at its own host:port without opening
# anything else. Skipped when ALLOW_PRIVATE_ADDRESSES trusts the environment. # up anything else. Skipped when ALLOW_PRIVATE_ADDRESSES trusts the
install_socket_guard(self.allow_private, proxy_urls=(self.ytdl_opts.get('proxy'),)) # environment.
install_socket_guard(
self.allow_private,
proxy_urls=(self.ytdl_opts.get('proxy'),),
service_urls=_pot_provider_urls(self.ytdl_opts),
)
log.info(f"Starting download for: {self.info.title} ({self.info.url})") log.info(f"Starting download for: {self.info.title} ({self.info.url})")
# Bound outside the try so the except branch can read what was captured # Bound outside the try so the except branch can read what was captured
# before the error was raised. # before the error was raised.
@@ -822,6 +862,27 @@ class Download:
# this logger, so a user-supplied one must not replace it. # this logger, so a user-supplied one must not replace it.
ytdl_params['logger'] = ytdl_logger ytdl_params['logger'] = ytdl_logger
# SponsorBlock: mark sponsor segments and cut them out, the same
# postprocessor pair the CLI's --sponsorblock-remove sponsor builds.
# This has to stay above the chapter-splitting block: yt-dlp runs
# same-stage postprocessors in list order, and ModifyChapters must
# rewrite the chapter list before FFmpegSplitChapters cuts the file
# up, or the chapter files keep the sponsor segments and the
# removal desyncs the remaining chapter timings.
if getattr(self.info, 'sponsorblock', False):
if 'postprocessors' not in ytdl_params:
ytdl_params['postprocessors'] = []
ytdl_params['postprocessors'].append({
'key': 'SponsorBlock',
'categories': ['sponsor'],
'when': 'after_filter',
})
ytdl_params['postprocessors'].append({
'key': 'ModifyChapters',
'remove_sponsor_segments': ['sponsor'],
'force_keyframes': False,
})
# Add chapter splitting options if enabled # Add chapter splitting options if enabled
if self.info.split_by_chapters: if self.info.split_by_chapters:
ytdl_params['outtmpl']['chapter'] = self.info.chapter_template ytdl_params['outtmpl']['chapter'] = self.info.chapter_template
@@ -1619,6 +1680,7 @@ class DownloadQueue:
already, already,
_add_gen=None, _add_gen=None,
retry_entry=None, retry_entry=None,
sponsorblock=False,
): ):
if not entry: if not entry:
return {'status': 'error', 'msg': "Invalid/empty data was given."} return {'status': 'error', 'msg': "Invalid/empty data was given."}
@@ -1662,6 +1724,7 @@ class DownloadQueue:
already, already,
_add_gen, _add_gen,
retry_entry, retry_entry,
sponsorblock=sponsorblock,
) )
elif etype == 'playlist' or etype == 'channel': elif etype == 'playlist' or etype == 'channel':
if etype == 'playlist' and self.__is_channel_extraction(entry): if etype == 'playlist' and self.__is_channel_extraction(entry):
@@ -1729,6 +1792,7 @@ class DownloadQueue:
clip_end, clip_end,
already, already,
_add_gen, _add_gen,
sponsorblock=sponsorblock,
) )
) )
if any(res['status'] == 'error' for res in results): if any(res['status'] == 'error' for res in results):
@@ -1769,6 +1833,7 @@ class DownloadQueue:
clip_end=clip_end, clip_end=clip_end,
live_status=entry.get('live_status'), live_status=entry.get('live_status'),
live_release_timestamp=entry.get('release_timestamp'), live_release_timestamp=entry.get('release_timestamp'),
sponsorblock=sponsorblock,
) )
await self.__add_download(dl, auto_start) await self.__add_download(dl, auto_start)
return {'status': 'ok'} return {'status': 'ok'}
@@ -1849,13 +1914,14 @@ class DownloadQueue:
already=None, already=None,
_add_gen=None, _add_gen=None,
retry_entry=None, retry_entry=None,
sponsorblock=False,
): ):
if ytdl_options_presets is None: if ytdl_options_presets is None:
ytdl_options_presets = [] ytdl_options_presets = []
log.info( log.info(
f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} ' f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} '
f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} ' f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} '
f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=}' f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=} {sponsorblock=}'
) )
if already is None: if already is None:
_add_gen = self._add_generation _add_gen = self._add_generation
@@ -1918,6 +1984,7 @@ class DownloadQueue:
already, already,
_add_gen, _add_gen,
retry_entry, retry_entry,
sponsorblock=sponsorblock,
) )
async def retry(self, id): async def retry(self, id):
@@ -1954,6 +2021,7 @@ class DownloadQueue:
info.clip_start, info.clip_start,
info.clip_end, info.clip_end,
retry_entry=info.entry, retry_entry=info.entry,
sponsorblock=info.sponsorblock,
) )
async def add_entry( async def add_entry(
@@ -1975,6 +2043,7 @@ class DownloadQueue:
ytdl_options_overrides=None, ytdl_options_overrides=None,
clip_start=None, clip_start=None,
clip_end=None, clip_end=None,
sponsorblock=False,
): ):
if ytdl_options_presets is None: if ytdl_options_presets is None:
ytdl_options_presets = [] ytdl_options_presets = []
@@ -2000,6 +2069,7 @@ class DownloadQueue:
clip_end, clip_end,
already, already,
None, None,
sponsorblock=sponsorblock,
) )
async def start_pending(self, ids): async def start_pending(self, ids):
+10
View File
@@ -399,6 +399,16 @@
</div> </div>
<div class="col-12"> <div class="col-12">
<div class="row g-2 align-items-center"> <div class="row g-2 align-items-center">
<div class="col-auto">
<div class="form-check form-switch">
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-sponsorblock"
name="sponsorblock" [(ngModel)]="sponsorblock" (change)="sponsorblockChanged()"
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
<label class="form-check-label" for="checkbox-sponsorblock"
ngbPopover="Cut out sponsor segments using SponsorBlock's crowd-sourced markers (YouTube only)."
triggers="hover" container="body">Remove sponsor segments</label>
</div>
</div>
<div class="col-auto"> <div class="col-auto">
<div class="form-check form-switch"> <div class="form-check form-switch">
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters" <input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
+7
View File
@@ -86,6 +86,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
autoStart: boolean; autoStart: boolean;
playlistItemLimit!: number; playlistItemLimit!: number;
splitByChapters: boolean; splitByChapters: boolean;
sponsorblock: boolean;
chapterTemplate: string; chapterTemplate: string;
clipStart = ''; clipStart = '';
clipEnd = ''; clipEnd = '';
@@ -259,6 +260,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
this.quality = this.cookieService.get('metube_quality') || 'best'; this.quality = this.cookieService.get('metube_quality') || 'best';
this.autoStart = this.cookieService.get('metube_auto_start') !== 'false'; this.autoStart = this.cookieService.get('metube_auto_start') !== 'false';
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true'; this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
this.sponsorblock = this.cookieService.get('metube_sponsorblock') === 'true';
// Will be set from backend configuration, use empty string as placeholder // Will be set from backend configuration, use empty string as placeholder
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || ''; this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
this.clipStart = this.cookieService.get('metube_clip_start') || ''; this.clipStart = this.cookieService.get('metube_clip_start') || '';
@@ -855,6 +857,10 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: this.settingsCookieExpiryDays }); this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
} }
sponsorblockChanged() {
this.cookieService.set('metube_sponsorblock', this.sponsorblock ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
}
splitByChaptersChanged() { splitByChaptersChanged() {
this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: this.settingsCookieExpiryDays }); this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
} }
@@ -1111,6 +1117,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
playlistItemLimit: overrides.playlistItemLimit ?? this.playlistItemLimit, playlistItemLimit: overrides.playlistItemLimit ?? this.playlistItemLimit,
autoStart: overrides.autoStart ?? this.autoStart, autoStart: overrides.autoStart ?? this.autoStart,
splitByChapters: overrides.splitByChapters ?? this.splitByChapters, splitByChapters: overrides.splitByChapters ?? this.splitByChapters,
sponsorblock: overrides.sponsorblock ?? this.sponsorblock,
chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate, chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate,
subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage, subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage,
subtitleMode: overrides.subtitleMode ?? this.subtitleMode, subtitleMode: overrides.subtitleMode ?? this.subtitleMode,
+1
View File
@@ -11,6 +11,7 @@ export interface Download {
custom_name_prefix: string; custom_name_prefix: string;
playlist_item_limit: number; playlist_item_limit: number;
split_by_chapters?: boolean; split_by_chapters?: boolean;
sponsorblock?: boolean;
chapter_template?: string; chapter_template?: string;
subtitle_language?: string; subtitle_language?: string;
subtitle_mode?: string; subtitle_mode?: string;
@@ -36,6 +36,7 @@ function basePayload(): AddDownloadPayload {
playlistItemLimit: 0, playlistItemLimit: 0,
autoStart: true, autoStart: true,
splitByChapters: false, splitByChapters: false,
sponsorblock: false,
chapterTemplate: '', chapterTemplate: '',
subtitleLanguage: 'en', subtitleLanguage: 'en',
subtitleMode: 'prefer_manual', subtitleMode: 'prefer_manual',
+2
View File
@@ -17,6 +17,7 @@ export interface AddDownloadPayload {
playlistItemLimit: number; playlistItemLimit: number;
autoStart: boolean; autoStart: boolean;
splitByChapters: boolean; splitByChapters: boolean;
sponsorblock: boolean;
chapterTemplate: string; chapterTemplate: string;
subtitleLanguage: string; subtitleLanguage: string;
subtitleMode: string; subtitleMode: string;
@@ -148,6 +149,7 @@ export class DownloadsService {
playlist_item_limit: payload.playlistItemLimit, playlist_item_limit: payload.playlistItemLimit,
auto_start: payload.autoStart, auto_start: payload.autoStart,
split_by_chapters: payload.splitByChapters, split_by_chapters: payload.splitByChapters,
sponsorblock: payload.sponsorblock,
chapter_template: payload.chapterTemplate, chapter_template: payload.chapterTemplate,
subtitle_language: payload.subtitleLanguage, subtitle_language: payload.subtitleLanguage,
subtitle_mode: payload.subtitleMode, subtitle_mode: payload.subtitleMode,
@@ -0,0 +1,77 @@
import { TestBed } from '@angular/core/testing';
import { provideHttpClient } from '@angular/common/http';
import { provideHttpClientTesting, HttpTestingController } from '@angular/common/http/testing';
import { Subject } from 'rxjs';
import { SubscriptionsService, SubscribePayload } from './subscriptions.service';
import { MeTubeSocket } from './metube-socket.service';
class MeTubeSocketStub {
private subjects: Record<string, Subject<string>> = {};
fromEvent(event: string) {
if (!this.subjects[event]) {
this.subjects[event] = new Subject<string>();
}
return this.subjects[event].asObservable();
}
}
function basePayload(): SubscribePayload {
return {
url: 'https://example.com/channel',
downloadType: 'video',
codec: 'auto',
quality: 'best',
format: 'any',
folder: '',
customNamePrefix: '',
playlistItemLimit: 0,
autoStart: true,
splitByChapters: false,
sponsorblock: false,
chapterTemplate: '',
subtitleLanguage: 'en',
subtitleMode: 'prefer_manual',
ytdlOptionsPresets: [],
ytdlOptionsOverrides: '',
clipStart: '',
clipEnd: '',
checkIntervalMinutes: 60,
titleRegex: '',
skipSubscriberOnly: false,
};
}
describe('SubscriptionsService', () => {
let httpMock: HttpTestingController;
let service: SubscriptionsService;
beforeEach(async () => {
await TestBed.configureTestingModule({
providers: [
SubscriptionsService,
provideHttpClient(),
provideHttpClientTesting(),
{ provide: MeTubeSocket, useValue: new MeTubeSocketStub() },
],
}).compileComponents();
service = TestBed.inject(SubscriptionsService);
httpMock = TestBed.inject(HttpTestingController);
});
it('subscribe() carries the sponsorblock flag', () => {
service.subscribe({ ...basePayload(), sponsorblock: true }).subscribe();
const req = httpMock.expectOne('subscribe');
expect(req.request.method).toBe('POST');
expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: true }));
req.flush({ status: 'ok' });
});
it('subscribe() sends the flag off by default', () => {
service.subscribe(basePayload()).subscribe();
const req = httpMock.expectOne('subscribe');
expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: false }));
req.flush({ status: 'ok' });
});
});
@@ -92,6 +92,7 @@ export class SubscriptionsService {
playlist_item_limit: payload.playlistItemLimit, playlist_item_limit: payload.playlistItemLimit,
auto_start: payload.autoStart, auto_start: payload.autoStart,
split_by_chapters: payload.splitByChapters, split_by_chapters: payload.splitByChapters,
sponsorblock: payload.sponsorblock,
chapter_template: payload.chapterTemplate, chapter_template: payload.chapterTemplate,
subtitle_language: payload.subtitleLanguage, subtitle_language: payload.subtitleLanguage,
subtitle_mode: payload.subtitleMode, subtitle_mode: payload.subtitleMode,