diff --git a/app/main.py b/app/main.py index a94cbd6..e48ab43 100644 --- a/app/main.py +++ b/app/main.py @@ -725,6 +725,7 @@ def parse_download_options(post: dict) -> dict: playlist_item_limit = post.get('playlist_item_limit') auto_start = post.get('auto_start') split_by_chapters = post.get('split_by_chapters') + sponsorblock = bool(post.get('sponsorblock')) chapter_template = post.get('chapter_template') subtitle_language = post.get('subtitle_language') subtitle_mode = post.get('subtitle_mode') @@ -845,6 +846,7 @@ def parse_download_options(post: dict) -> dict: 'playlist_item_limit': playlist_item_limit, 'auto_start': auto_start, 'split_by_chapters': split_by_chapters, + 'sponsorblock': sponsorblock, 'chapter_template': chapter_template, 'subtitle_language': subtitle_language, 'subtitle_mode': subtitle_mode, @@ -890,6 +892,7 @@ async def add(request): o['ytdl_options_overrides'], o['clip_start'], o['clip_end'], + sponsorblock=o['sponsorblock'], ) return web.Response(text=serializer.encode(status)) @@ -970,6 +973,7 @@ async def subscribe(request): subtitle_mode=o['subtitle_mode'], ytdl_options_presets=o['ytdl_options_presets'], ytdl_options_overrides=o['ytdl_options_overrides'], + sponsorblock=o['sponsorblock'], title_regex=post.get('title_regex'), skip_subscriber_only=skip_subscriber_only, clip_start=sub_clip_start, diff --git a/app/subscriptions.py b/app/subscriptions.py index 1cf7764..5afe878 100644 --- a/app/subscriptions.py +++ b/app/subscriptions.py @@ -182,6 +182,7 @@ class SubscriptionInfo: auto_start: bool = True playlist_item_limit: int = 0 split_by_chapters: bool = False + sponsorblock: bool = False chapter_template: str = "" subtitle_language: str = "en" subtitle_mode: str = "prefer_manual" @@ -242,6 +243,7 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]: "auto_start": sub.auto_start, "playlist_item_limit": sub.playlist_item_limit, "split_by_chapters": sub.split_by_chapters, + "sponsorblock": sub.sponsorblock, "chapter_template": sub.chapter_template, "subtitle_language": sub.subtitle_language, "subtitle_mode": sub.subtitle_mode, @@ -487,6 +489,7 @@ class SubscriptionManager: ytdl_options_overrides: Optional[dict[str, Any]] = None, clip_start: Optional[float] = None, clip_end: Optional[float] = None, + sponsorblock: bool = False, ) -> tuple[list[str], list[str]]: queued_ids: list[str] = [] queue_errors: list[str] = [] @@ -519,6 +522,7 @@ class SubscriptionManager: ytdl_options_overrides, clip_start, clip_end, + sponsorblock=sponsorblock, ) if isinstance(result, dict) and result.get("status") == "error": msg = str(result.get("msg") or f"Queueing failed for {vurl}") @@ -606,6 +610,7 @@ class SubscriptionManager: subtitle_mode: str, ytdl_options_presets: Optional[list[str]] = None, ytdl_options_overrides: Optional[dict[str, Any]] = None, + sponsorblock: bool = False, title_regex: Any = None, skip_subscriber_only: Any = None, clip_start: Optional[float] = None, @@ -689,6 +694,7 @@ class SubscriptionManager: auto_start=bool(auto_start), playlist_item_limit=int(playlist_item_limit), split_by_chapters=bool(split_by_chapters), + sponsorblock=bool(sponsorblock), chapter_template=chapter_template or "", subtitle_language=subtitle_language, subtitle_mode=subtitle_mode, @@ -942,6 +948,7 @@ class SubscriptionManager: dl_plimit = cur.playlist_item_limit dl_autostart = cur.auto_start dl_split = cur.split_by_chapters + dl_sponsorblock = cur.sponsorblock dl_chapter = cur.chapter_template dl_sublang = cur.subtitle_language dl_submode = cur.subtitle_mode @@ -1010,6 +1017,7 @@ class SubscriptionManager: playlist_item_limit=dl_plimit, auto_start=dl_autostart, split_by_chapters=dl_split, + sponsorblock=dl_sponsorblock, chapter_template=dl_chapter or "", subtitle_language=dl_sublang, subtitle_mode=dl_submode, diff --git a/app/tests/test_api.py b/app/tests/test_api.py index ceb4124..0ed7ee0 100644 --- a/app/tests/test_api.py +++ b/app/tests/test_api.py @@ -366,6 +366,25 @@ async def test_subscribe_passes_clip_bounds(mock_dqueue, monkeypatch): assert kwargs["clip_end"] == pytest.approx(204.0) +@pytest.mark.asyncio +async def test_subscribe_passes_sponsorblock(mock_dqueue, monkeypatch): + monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"})) + req = _json_request( + {**_valid_video_add_body(), "check_interval_minutes": 60, "sponsorblock": True} + ) + resp = await main.subscribe(req) + assert resp.status == 200 + assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is True + + +@pytest.mark.asyncio +async def test_subscribe_defaults_sponsorblock_off(mock_dqueue, monkeypatch): + monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"})) + req = _json_request({**_valid_video_add_body(), "check_interval_minutes": 60}) + await main.subscribe(req) + assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is False + + @pytest.mark.asyncio async def test_subscribe_without_clip_fields_stores_none(mock_dqueue, monkeypatch): monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"})) diff --git a/app/tests/test_download_queue.py b/app/tests/test_download_queue.py index 8c85be2..66b5a0d 100644 --- a/app/tests/test_download_queue.py +++ b/app/tests/test_download_queue.py @@ -476,6 +476,32 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env): assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True} +@pytest.mark.asyncio +async def test_retry_carries_the_sponsorblock_flag(dq_env): + notifier = AsyncMock() + dq = DownloadQueue(dq_env, notifier) + url = "https://example.com/watch?v=1" + dq.done.put( + Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url, sponsorblock=True)) + ) + + def fake_extract(self, extracted_url, *_args, **_kwargs): + return { + "_type": "video", + "id": "vid1", + "title": "Test Video", + "url": extracted_url, + "webpage_url": extracted_url, + } + + with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \ + patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()): + result = await dq.retry(url) + + assert result["status"] == "ok" + assert dq.queue.get(url).info.sponsorblock is True + + @pytest.mark.asyncio async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env): notifier = AsyncMock() diff --git a/app/tests/test_subscriptions.py b/app/tests/test_subscriptions.py index 1934d36..cf72fba 100644 --- a/app/tests/test_subscriptions.py +++ b/app/tests/test_subscriptions.py @@ -479,6 +479,69 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(reloaded.get(sub_id).clip_start, 12.5) self.assertIsNone(reloaded.get(sub_id).clip_end) + async def test_check_now_applies_subscription_sponsorblock(self): + """Subscriptions download unattended, so the sponsor-segment removal has + to reach every entry the subscription queues, not just manual adds.""" + with tempfile.TemporaryDirectory() as tmp: + queue = _Queue() + mgr = SubscriptionManager(_Config(tmp), queue, _Notifier()) + + with patch( + "subscriptions.extract_flat_playlist", + side_effect=[ + ( + {"_type": "channel", "title": "Channel"}, + [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}], + ), + ( + {"_type": "channel", "title": "Channel"}, + [ + {"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"}, + {"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}, + ], + ), + ], + ): + result = await mgr.add_subscription( + "https://example.com/channel", + check_interval_minutes=60, + download_type="video", + codec="auto", + format="any", + quality="best", + folder="", + custom_name_prefix="", + auto_start=True, + playlist_item_limit=0, + split_by_chapters=False, + chapter_template="", + subtitle_language="en", + subtitle_mode="prefer_manual", + sponsorblock=True, + ) + sub_id = result["subscription"]["id"] + self.assertTrue(mgr.get(sub_id).sponsorblock) + await mgr.check_now([sub_id]) + + self.assertEqual(len(queue.entries), 1) + _entry, _args, kwargs = queue.entries[0] + self.assertIs(kwargs["sponsorblock"], True) + + async def test_sponsorblock_survives_reload_and_defaults_to_false(self): + with tempfile.TemporaryDirectory() as tmp: + cfg = _Config(tmp) + mgr = SubscriptionManager(cfg, _Queue(), _Notifier()) + sub_id = await self._add_one_subscription(mgr) + # Records written before the field existed simply take the default. + self.assertFalse(mgr.get(sub_id).sponsorblock) + + mgr.get(sub_id).sponsorblock = True + async with mgr._lock: + mgr._save_locked() + + reloaded = SubscriptionManager(cfg, _Queue(), _Notifier()) + self.assertTrue(reloaded.get(sub_id).sponsorblock) + async def test_check_now_queues_subscriber_only_when_skip_disabled(self): with tempfile.TemporaryDirectory() as tmp: queue = _Queue() diff --git a/app/tests/test_ytdl_utils.py b/app/tests/test_ytdl_utils.py index 358a487..434a7ef 100644 --- a/app/tests/test_ytdl_utils.py +++ b/app/tests/test_ytdl_utils.py @@ -679,6 +679,69 @@ class DownloadResultTests(unittest.TestCase): ) +def _capture_ytdl_params(download: Download) -> dict: + """Run ``_download`` far enough to capture the params it builds.""" + fake_ydl = MagicMock() + fake_ydl.download.return_value = 0 + download.status_queue = types.SimpleNamespace(put=lambda _: None) + + with patch('ytdl.install_socket_guard'), \ + patch.object(Download, '_make_youtube_dl', return_value=fake_ydl) as make: + download._download() + + params, = make.call_args.args + return params + + +class SponsorBlockPostprocessorTests(unittest.TestCase): + def test_no_sponsorblock_postprocessors_when_disabled(self): + download = _make_test_download() + + params = _capture_ytdl_params(download) + + keys = [pp['key'] for pp in params.get('postprocessors', [])] + self.assertNotIn('SponsorBlock', keys) + self.assertNotIn('ModifyChapters', keys) + + def test_sponsorblock_pair_matches_the_cli(self): + download = _make_test_download() + download.info.sponsorblock = True + + params = _capture_ytdl_params(download) + + self.assertEqual( + params['postprocessors'], + [ + { + 'key': 'SponsorBlock', + 'categories': ['sponsor'], + 'when': 'after_filter', + }, + { + 'key': 'ModifyChapters', + 'remove_sponsor_segments': ['sponsor'], + 'force_keyframes': False, + }, + ], + ) + + def test_segment_removal_runs_before_the_chapter_split(self): + # yt-dlp runs same-stage postprocessors in list order, so ModifyChapters + # has to rewrite the chapter list before FFmpegSplitChapters cuts the + # file up -- the order the CLI builds for + # --sponsorblock-remove sponsor --split-chapters. + download = _make_test_download() + download.info.sponsorblock = True + download.info.split_by_chapters = True + download.info.chapter_template = '%(section_number)s.%(ext)s' + + params = _capture_ytdl_params(download) + + keys = [pp['key'] for pp in params['postprocessors']] + self.assertEqual(keys, ['SponsorBlock', 'ModifyChapters', 'FFmpegSplitChapters']) + self.assertEqual(params['outtmpl']['chapter'], '%(section_number)s.%(ext)s') + + class ProgressThrottleTests(unittest.TestCase): def test_downloading_ticks_are_throttled(self): dl = _make_test_download() diff --git a/app/ytdl.py b/app/ytdl.py index f7258a2..21a5182 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -483,6 +483,7 @@ class DownloadInfo: clip_end=None, live_status=None, live_release_timestamp=None, + sponsorblock=False, ): self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}' self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}' @@ -502,6 +503,7 @@ class DownloadInfo: self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None self.playlist_item_limit = playlist_item_limit self.split_by_chapters = split_by_chapters + self.sponsorblock = sponsorblock self.chapter_template = chapter_template self.subtitle_language = subtitle_language self.subtitle_mode = subtitle_mode @@ -571,6 +573,8 @@ class DownloadInfo: self.playlist_item_limit = 0 if not hasattr(self, "split_by_chapters"): self.split_by_chapters = False + if not hasattr(self, "sponsorblock"): + self.sponsorblock = False if not hasattr(self, "chapter_template"): self.chapter_template = "" if not hasattr(self, "subtitle_language"): @@ -615,6 +619,7 @@ _PERSISTED_DOWNLOAD_FIELDS = ( "custom_name_prefix", "playlist_item_limit", "split_by_chapters", + "sponsorblock", "chapter_template", "subtitle_language", "subtitle_mode", @@ -857,6 +862,27 @@ class Download: # this logger, so a user-supplied one must not replace it. ytdl_params['logger'] = ytdl_logger + # SponsorBlock: mark sponsor segments and cut them out, the same + # postprocessor pair the CLI's --sponsorblock-remove sponsor builds. + # This has to stay above the chapter-splitting block: yt-dlp runs + # same-stage postprocessors in list order, and ModifyChapters must + # rewrite the chapter list before FFmpegSplitChapters cuts the file + # up, or the chapter files keep the sponsor segments and the + # removal desyncs the remaining chapter timings. + if getattr(self.info, 'sponsorblock', False): + if 'postprocessors' not in ytdl_params: + ytdl_params['postprocessors'] = [] + ytdl_params['postprocessors'].append({ + 'key': 'SponsorBlock', + 'categories': ['sponsor'], + 'when': 'after_filter', + }) + ytdl_params['postprocessors'].append({ + 'key': 'ModifyChapters', + 'remove_sponsor_segments': ['sponsor'], + 'force_keyframes': False, + }) + # Add chapter splitting options if enabled if self.info.split_by_chapters: ytdl_params['outtmpl']['chapter'] = self.info.chapter_template @@ -1654,6 +1680,7 @@ class DownloadQueue: already, _add_gen=None, retry_entry=None, + sponsorblock=False, ): if not entry: return {'status': 'error', 'msg': "Invalid/empty data was given."} @@ -1697,6 +1724,7 @@ class DownloadQueue: already, _add_gen, retry_entry, + sponsorblock=sponsorblock, ) elif etype == 'playlist' or etype == 'channel': if etype == 'playlist' and self.__is_channel_extraction(entry): @@ -1764,6 +1792,7 @@ class DownloadQueue: clip_end, already, _add_gen, + sponsorblock=sponsorblock, ) ) if any(res['status'] == 'error' for res in results): @@ -1804,6 +1833,7 @@ class DownloadQueue: clip_end=clip_end, live_status=entry.get('live_status'), live_release_timestamp=entry.get('release_timestamp'), + sponsorblock=sponsorblock, ) await self.__add_download(dl, auto_start) return {'status': 'ok'} @@ -1884,13 +1914,14 @@ class DownloadQueue: already=None, _add_gen=None, retry_entry=None, + sponsorblock=False, ): if ytdl_options_presets is None: ytdl_options_presets = [] log.info( f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} ' f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} ' - f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=}' + f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=} {sponsorblock=}' ) if already is None: _add_gen = self._add_generation @@ -1953,6 +1984,7 @@ class DownloadQueue: already, _add_gen, retry_entry, + sponsorblock=sponsorblock, ) async def retry(self, id): @@ -1989,6 +2021,7 @@ class DownloadQueue: info.clip_start, info.clip_end, retry_entry=info.entry, + sponsorblock=info.sponsorblock, ) async def add_entry( @@ -2010,6 +2043,7 @@ class DownloadQueue: ytdl_options_overrides=None, clip_start=None, clip_end=None, + sponsorblock=False, ): if ytdl_options_presets is None: ytdl_options_presets = [] @@ -2035,6 +2069,7 @@ class DownloadQueue: clip_end, already, None, + sponsorblock=sponsorblock, ) async def start_pending(self, ids): diff --git a/ui/src/app/app.html b/ui/src/app/app.html index 0f705cb..f9b6dbb 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -399,6 +399,16 @@
+
+
+ + +
+
> = {}; + + fromEvent(event: string) { + if (!this.subjects[event]) { + this.subjects[event] = new Subject(); + } + return this.subjects[event].asObservable(); + } +} + +function basePayload(): SubscribePayload { + return { + url: 'https://example.com/channel', + downloadType: 'video', + codec: 'auto', + quality: 'best', + format: 'any', + folder: '', + customNamePrefix: '', + playlistItemLimit: 0, + autoStart: true, + splitByChapters: false, + sponsorblock: false, + chapterTemplate: '', + subtitleLanguage: 'en', + subtitleMode: 'prefer_manual', + ytdlOptionsPresets: [], + ytdlOptionsOverrides: '', + clipStart: '', + clipEnd: '', + checkIntervalMinutes: 60, + titleRegex: '', + skipSubscriberOnly: false, + }; +} + +describe('SubscriptionsService', () => { + let httpMock: HttpTestingController; + let service: SubscriptionsService; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + providers: [ + SubscriptionsService, + provideHttpClient(), + provideHttpClientTesting(), + { provide: MeTubeSocket, useValue: new MeTubeSocketStub() }, + ], + }).compileComponents(); + + service = TestBed.inject(SubscriptionsService); + httpMock = TestBed.inject(HttpTestingController); + }); + + it('subscribe() carries the sponsorblock flag', () => { + service.subscribe({ ...basePayload(), sponsorblock: true }).subscribe(); + const req = httpMock.expectOne('subscribe'); + expect(req.request.method).toBe('POST'); + expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: true })); + req.flush({ status: 'ok' }); + }); + + it('subscribe() sends the flag off by default', () => { + service.subscribe(basePayload()).subscribe(); + const req = httpMock.expectOne('subscribe'); + expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: false })); + req.flush({ status: 'ok' }); + }); +}); diff --git a/ui/src/app/services/subscriptions.service.ts b/ui/src/app/services/subscriptions.service.ts index 3199db8..eb6ff67 100644 --- a/ui/src/app/services/subscriptions.service.ts +++ b/ui/src/app/services/subscriptions.service.ts @@ -92,6 +92,7 @@ export class SubscriptionsService { playlist_item_limit: payload.playlistItemLimit, auto_start: payload.autoStart, split_by_chapters: payload.splitByChapters, + sponsorblock: payload.sponsorblock, chapter_template: payload.chapterTemplate, subtitle_language: payload.subtitleLanguage, subtitle_mode: payload.subtitleMode,