mirror of
https://github.com/alexta69/metube.git
synced 2026-09-21 13:35:01 +00:00
Compare commits
14 Commits
2026.07.24
...
2026.08.04
| Author | SHA1 | Date | |
|---|---|---|---|
| 482381d6b9 | |||
| 0445f5858b | |||
| 6551f7ad58 | |||
| 06c63ec6e5 | |||
| d66b04ccf5 | |||
| 2744f36b44 | |||
| ff1b73a576 | |||
| 1a09dbd686 | |||
| 08dccd98fb | |||
| 1f20aaee94 | |||
| 8a29f3a084 | |||
| 1839e5484d | |||
| fceac97033 | |||
| a13762aa61 |
@@ -15,7 +15,7 @@ jobs:
|
||||
token: ${{ secrets.AUTOUPDATE_PAT }}
|
||||
-
|
||||
name: Set up Python
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.13'
|
||||
-
|
||||
|
||||
@@ -96,7 +96,23 @@ release the same day. **Master is continuously released** — a PR must be
|
||||
release-ready exactly as merged; there is no stabilization window for follow-up
|
||||
fixes.
|
||||
|
||||
## Code style
|
||||
## Commit messages
|
||||
|
||||
A commit that resolves an issue must close it, with a GitHub closing keyword in
|
||||
parentheses at the end of the subject line:
|
||||
|
||||
```
|
||||
fix: stop metadata probes from writing playlist sidecar files (closes #1040)
|
||||
```
|
||||
|
||||
Because master is the default branch and is released on every push, the issue
|
||||
closes at the moment the fix ships, and keeps a permanent link to the commit that
|
||||
fixed it. A bare `(#1040)` is only a reference — and reads as a pull-request
|
||||
number — so it does not count; the keyword is what closes the issue.
|
||||
|
||||
Auto-closing leaves only a commit stub on the issue, which is not an answer to
|
||||
whoever reported it. Post an explanatory comment as well: what the cause was, what
|
||||
changed, and anything the reporter needs to do differently.
|
||||
|
||||
Follow `.editorconfig`:
|
||||
- Python: 4-space indent
|
||||
|
||||
@@ -10,7 +10,7 @@ Key capabilities:
|
||||
* Download playlists and channels, with configurable output and download options.
|
||||
* [Subscribe](https://github.com/alexta69/metube/wiki/Subscriptions) to channels and playlists, periodically check for new items, and queue new uploads automatically.
|
||||
|
||||

|
||||

|
||||
|
||||
## 🐳 Run using Docker
|
||||
|
||||
@@ -83,6 +83,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
||||
* __ALLOW_PRIVATE_ADDRESSES__: Whether to allow downloads from private, loopback, link-local and other non-global addresses. Defaults to `false`, which protects against SSRF by refusing URLs that resolve to internal hosts. Set to `true` only in trusted environments — for example when routing traffic through a proxy/VPN client in Fake-IP mode (sing-box, Clash, Mihomo), which resolves hosts to the `198.18.0.0/15` range. Enabling this disables the SSRF protection entirely, so only use it when you control the network.
|
||||
* __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled).
|
||||
|
||||
Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a feed-level `.info.json` and thumbnail when you add a playlist or channel. These reuse the template of the items they belong to — `OUTPUT_TEMPLATE_CHANNEL` or `OUTPUT_TEMPLATE_PLAYLIST` — evaluated against the feed itself, so with the defaults they land in the same folder as the videos, named after the feed. Set `allow_playlist_files` to `false` in `YTDL_OPTIONS` to skip them.
|
||||
|
||||
### 🌐 Web Server & URLs
|
||||
|
||||
* __HOST__: The host address the web server will bind to. Defaults to `0.0.0.0` (all interfaces).
|
||||
|
||||
+19
@@ -893,6 +893,17 @@ async def cancel_add(request):
|
||||
return web.Response(text=serializer.encode({'status': 'ok'}), content_type='application/json')
|
||||
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'retry')
|
||||
async def retry(request):
|
||||
# Singular by design, unlike the 'ids' batch endpoints: a retry re-extracts
|
||||
# the URL, so it can fail per item, and the caller removes that item's done
|
||||
# record only once it is confirmed re-queued. A batch form would have to
|
||||
# report per-id results for the caller to know which ones to remove.
|
||||
post = await _read_json_request(request)
|
||||
status = await dqueue.retry(_require_id(post))
|
||||
return web.Response(text=serializer.encode(status), content_type='application/json')
|
||||
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'subscribe')
|
||||
async def subscribe(request):
|
||||
post = await _read_json_request(request)
|
||||
@@ -985,6 +996,13 @@ async def subscriptions_check(request):
|
||||
result = await submgr.check_now([str(i) for i in ids] if ids else None)
|
||||
return web.Response(text=serializer.encode(result))
|
||||
|
||||
def _require_id(post: dict) -> str:
|
||||
id = post.get('id')
|
||||
if not isinstance(id, str) or not id:
|
||||
raise web.HTTPBadRequest(reason="'id' must be a non-empty string")
|
||||
return id
|
||||
|
||||
|
||||
def _require_id_list(post: dict) -> list:
|
||||
ids = post.get('ids')
|
||||
if not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
|
||||
@@ -1227,6 +1245,7 @@ async def add_cors(request):
|
||||
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'add', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'cancel-add', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'retry', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscribe', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions/update', add_cors)
|
||||
|
||||
+33
-2
@@ -64,6 +64,12 @@ def _build_ydl_params(
|
||||
"paths": {"home": config.DOWNLOAD_DIR, "temp": config.TEMP_DIR},
|
||||
**config.YTDL_OPTIONS,
|
||||
**(extra_opts or {}),
|
||||
# A scan is a poll, not an add: it runs on a timer and queues items
|
||||
# through the download queue, which writes the feed metadata itself.
|
||||
# yt-dlp emits the playlist-level infojson/description/thumbnail
|
||||
# regardless of `download`, so without this a writeinfojson user would
|
||||
# get those files rewritten on every check interval. See issue #1040.
|
||||
"allow_playlist_files": False,
|
||||
}
|
||||
params = _impersonate_opt(params)
|
||||
if playlistend is not None and playlistend > 0:
|
||||
@@ -287,6 +293,24 @@ def validate_title_regex(value: Any) -> str:
|
||||
return s
|
||||
|
||||
|
||||
# The name is a display label the user picks; it is persisted and broadcast to
|
||||
# every connected client, so keep it a bounded single-line string.
|
||||
SUBSCRIPTION_NAME_MAX_LENGTH = 200
|
||||
|
||||
|
||||
def validate_subscription_name(value: Any) -> str:
|
||||
"""Return a stored subscription name, or raise ValueError if unusable."""
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("name must be a string")
|
||||
# Collapse newlines/tabs so a pasted title can't break the table layout.
|
||||
name = " ".join(value.split())
|
||||
if not name:
|
||||
raise ValueError("name must not be empty")
|
||||
if len(name) > SUBSCRIPTION_NAME_MAX_LENGTH:
|
||||
raise ValueError(f"name must be at most {SUBSCRIPTION_NAME_MAX_LENGTH} characters")
|
||||
return name
|
||||
|
||||
|
||||
def _coerce_bool(value: Any) -> bool:
|
||||
"""Accept JSON booleans and common string forms used by API clients."""
|
||||
if isinstance(value, bool):
|
||||
@@ -674,6 +698,13 @@ class SubscriptionManager:
|
||||
return {"status": "ok"}
|
||||
|
||||
async def update_subscription(self, sub_id: str, changes: dict) -> dict:
|
||||
validated_name: Optional[str] = None
|
||||
if "name" in changes:
|
||||
try:
|
||||
validated_name = validate_subscription_name(changes["name"])
|
||||
except ValueError as exc:
|
||||
return {"status": "error", "msg": str(exc)}
|
||||
|
||||
validated_tr: Optional[str] = None
|
||||
if "title_regex" in changes:
|
||||
try:
|
||||
@@ -722,8 +753,8 @@ class SubscriptionManager:
|
||||
sub.enabled = validated_enabled
|
||||
if interval_set:
|
||||
sub.check_interval_minutes = validated_interval
|
||||
if "name" in changes and changes["name"]:
|
||||
sub.name = str(changes["name"])
|
||||
if validated_name is not None:
|
||||
sub.name = validated_name
|
||||
if validated_tr is not None:
|
||||
sub.title_regex = validated_tr
|
||||
if skip_so_set:
|
||||
|
||||
@@ -20,6 +20,7 @@ def mock_dqueue(monkeypatch):
|
||||
d = MagicMock()
|
||||
d.initialize = AsyncMock(return_value=None)
|
||||
d.add = AsyncMock(return_value={"status": "ok"})
|
||||
d.retry = AsyncMock(return_value={"status": "ok"})
|
||||
d.cancel = AsyncMock(return_value={"status": "ok"})
|
||||
d.clear = AsyncMock(return_value={"status": "ok"})
|
||||
d.start_pending = AsyncMock(return_value={"status": "ok"})
|
||||
@@ -69,6 +70,22 @@ async def test_add_ok(mock_dqueue):
|
||||
mock_dqueue.add.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_passes_failed_download_id(mock_dqueue):
|
||||
req = _json_request({"id": "https://example.com/watch?v=1"})
|
||||
resp = await main.retry(req)
|
||||
assert resp.status == 200
|
||||
mock_dqueue.retry.assert_awaited_once_with("https://example.com/watch?v=1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("body", [{}, {"id": ""}, {"id": ["a"]}, {"ids": ["a"]}])
|
||||
async def test_retry_rejects_missing_or_non_string_id(mock_dqueue, body):
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.retry(_json_request(body))
|
||||
mock_dqueue.retry.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_passes_preset_and_overrides(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "YTDL_OPTIONS_PRESETS", {"Preset A": {"writesubtitles": True}})
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
@@ -89,7 +90,7 @@ def test_get_returns_tuple_of_lists(dq_env):
|
||||
async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
@@ -124,7 +125,7 @@ async def test_add_unsupported_url_recorded_as_failed_entry(dq_env):
|
||||
notifier = AsyncMock()
|
||||
url = "https://example.com/not-a-video"
|
||||
|
||||
def boom(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def boom(self, url, *_args, **_kwargs):
|
||||
raise ytdl.yt_dlp.utils.YoutubeDLError(f'Unsupported URL: {url}')
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
@@ -167,7 +168,7 @@ async def test_add_ssrf_rejected_url_recorded_as_failed_entry(dq_env):
|
||||
async def test_cancel_removes_from_pending(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
@@ -204,7 +205,7 @@ async def test_cancel_before_start_marks_download_canceled(dq_env):
|
||||
cancelling, because its ``download.canceled`` guard was never flipped."""
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
@@ -242,7 +243,7 @@ async def test_cancel_before_start_marks_download_canceled(dq_env):
|
||||
async def test_start_pending_moves_to_queue(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
@@ -302,6 +303,179 @@ async def test_add_entry_queues_single_video_without_reextracting(dq_env):
|
||||
assert dq.pending.exists("https://example.com/watch?v=1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_restores_playlist_output_context(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
url = "https://example.com/watch?v=1"
|
||||
failed_info = DownloadInfo(
|
||||
id="vid1",
|
||||
title="Test Video",
|
||||
url=url,
|
||||
quality="best",
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
error="temporary failure",
|
||||
entry={
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "My Playlist",
|
||||
"playlist_count": 10,
|
||||
},
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
)
|
||||
failed_info.status = "error"
|
||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, failed_info))
|
||||
|
||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": extracted_url,
|
||||
"webpage_url": extracted_url,
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
result = await dq.retry(url)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
queued = dq.queue.get(url)
|
||||
assert queued.output_template == "My Playlist/%(title)s.%(ext)s"
|
||||
assert queued.info.entry["playlist_index"] == "01"
|
||||
assert queued.info.entry["playlist_title"] == "My Playlist"
|
||||
|
||||
|
||||
def _failed_playlist_item(url, **overrides):
|
||||
"""A done-list entry for a playlist item that failed mid-download."""
|
||||
info = DownloadInfo(
|
||||
id="vid1",
|
||||
title="Test Video",
|
||||
url=url,
|
||||
quality="best",
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
error="temporary failure",
|
||||
entry={
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "My Playlist",
|
||||
"playlist_count": 10,
|
||||
},
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
**overrides,
|
||||
)
|
||||
info.status = "error"
|
||||
return info
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_keeps_playlist_context_through_url_indirection(dq_env):
|
||||
# extract_flat=True makes yt-dlp hand back url/url_transparent results
|
||||
# unprocessed, so __add_entry recurses into add() a second time. The retry
|
||||
# context has to survive that hop or the item lands in the root directory.
|
||||
notifier = AsyncMock()
|
||||
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
url = "https://example.com/watch?v=1"
|
||||
resolved = "https://example.com/resolved?v=1"
|
||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url)))
|
||||
|
||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||
if extracted_url == url:
|
||||
return {"_type": "url", "url": resolved, "id": "vid1"}
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": extracted_url,
|
||||
"webpage_url": extracted_url,
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
result = await dq.retry(url)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
queued = dq.queue.get(resolved)
|
||||
assert queued.output_template == "My Playlist/%(title)s.%(ext)s"
|
||||
assert queued.info.entry["playlist_title"] == "My Playlist"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_reapplies_current_options_gates(dq_env):
|
||||
# The stored options passed parse_download_options when first submitted, but
|
||||
# the configuration can have changed since; retry must not resurrect
|
||||
# overrides or presets the current configuration no longer allows.
|
||||
notifier = AsyncMock()
|
||||
dq_env.ALLOW_YTDL_OPTIONS_OVERRIDES = False
|
||||
dq_env.YTDL_OPTIONS_PRESETS = {"Still There": {"writesubtitles": True}}
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
url = "https://example.com/watch?v=1"
|
||||
info = _failed_playlist_item(
|
||||
url,
|
||||
ytdl_options_presets=["Still There", "Removed Preset"],
|
||||
ytdl_options_overrides={"paths": {"home": "/etc"}},
|
||||
)
|
||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
||||
|
||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": extracted_url,
|
||||
"webpage_url": extracted_url,
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
result = await dq.retry(url)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
queued = dq.queue.get(url)
|
||||
assert queued.info.ytdl_options_overrides == {}
|
||||
assert queued.info.ytdl_options_presets == ["Still There"]
|
||||
assert queued.ytdl_opts.get("paths", {}).get("home") != "/etc"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_retry_keeps_overrides_while_still_allowed(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq_env.ALLOW_YTDL_OPTIONS_OVERRIDES = True
|
||||
dq_env.YTDL_OPTIONS_PRESETS = {}
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
url = "https://example.com/watch?v=1"
|
||||
info = _failed_playlist_item(url, ytdl_options_overrides={"writesubtitles": True})
|
||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
||||
|
||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": extracted_url,
|
||||
"webpage_url": extracted_url,
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
result = await dq.retry(url)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
||||
notifier = AsyncMock()
|
||||
@@ -364,7 +538,7 @@ async def test_channel_download_uses_output_template_when_channel_template_empty
|
||||
|
||||
channel_id = "UCabcd123"
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "playlist",
|
||||
"id": channel_id,
|
||||
@@ -413,7 +587,7 @@ async def test_playlist_download_not_treated_as_channel(dq_env):
|
||||
dq_env.OUTPUT_TEMPLATE_CHANNEL = ""
|
||||
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "playlist",
|
||||
"id": "PLxyz789",
|
||||
@@ -460,7 +634,7 @@ async def test_add_merges_global_preset_and_override_options(dq_env):
|
||||
"Preset B": {"writesubtitles": False, "ratelimit": 1000},
|
||||
}
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid2",
|
||||
@@ -583,11 +757,191 @@ async def test_extract_info_metube_extract_keys_win_over_preset(dq_env):
|
||||
assert captured_params[0]["noplaylist"] is True
|
||||
|
||||
|
||||
def _feed_extract(feed):
|
||||
"""Patch for __extract_info that returns a playlist/channel feed dict."""
|
||||
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return copy.deepcopy(feed)
|
||||
|
||||
return fake_extract
|
||||
|
||||
|
||||
_CHANNEL_FEED = {
|
||||
"_type": "playlist",
|
||||
"id": "UC123",
|
||||
"title": "Vanessa - Videos",
|
||||
"channel": "Vanessa",
|
||||
"channel_id": "UC123",
|
||||
"uploader": "Vanessa",
|
||||
"extractor": "youtube:tab",
|
||||
"extractor_key": "YoutubeTab",
|
||||
"webpage_url": "https://example.com/@vanessa/videos",
|
||||
"entries": [
|
||||
{"id": "v1", "title": "One", "url": "https://example.com/v1",
|
||||
"webpage_url": "https://example.com/v1", "_type": "url"},
|
||||
],
|
||||
}
|
||||
|
||||
_PLAYLIST_FEED = {
|
||||
"_type": "playlist",
|
||||
"id": "PL123",
|
||||
"title": "My Playlist",
|
||||
"extractor": "generic",
|
||||
"extractor_key": "Generic",
|
||||
"webpage_url": "https://example.com/playlist?list=PL123",
|
||||
"entries": [
|
||||
{"id": "v1", "title": "One", "url": "https://example.com/v1",
|
||||
"webpage_url": "https://example.com/v1", "_type": "url"},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _written_files(root):
|
||||
found = []
|
||||
for dirpath, _dirs, files in os.walk(root):
|
||||
for f in files:
|
||||
found.append(os.path.relpath(os.path.join(dirpath, f), root))
|
||||
return sorted(found)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_feed_metadata_lands_beside_its_items(dq_env):
|
||||
"""Issues #660/#1040: the feed-level .info.json follows the same template
|
||||
the items use, so it sits in the channel's own folder rather than in
|
||||
DOWNLOAD_DIR under yt-dlp's pl_* default name."""
|
||||
dq_env.YTDL_OPTIONS = {"writeinfojson": True}
|
||||
dq_env.OUTPUT_TEMPLATE_CHANNEL = "%(channel)s/%(title)s.%(ext)s"
|
||||
|
||||
dq = DownloadQueue(dq_env, AsyncMock())
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_CHANNEL_FEED)), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
result = await dq.add(
|
||||
"https://example.com/@vanessa/videos", "video", "auto", "any", "best",
|
||||
"", "", 0, auto_start=False,
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert _written_files(dq_env.DOWNLOAD_DIR) == [
|
||||
os.path.join("Vanessa", "Vanessa - Videos.info.json")
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_playlist_feed_metadata_uses_the_playlist_template(dq_env):
|
||||
dq_env.YTDL_OPTIONS = {"writeinfojson": True}
|
||||
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||
|
||||
dq = DownloadQueue(dq_env, AsyncMock())
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
await dq.add(
|
||||
"https://example.com/playlist?list=PL123", "video", "auto", "any", "best",
|
||||
"", "", 0, auto_start=False,
|
||||
)
|
||||
|
||||
assert _written_files(dq_env.DOWNLOAD_DIR) == [
|
||||
os.path.join("My Playlist", "My Playlist.info.json")
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feed_metadata_honours_custom_folder(dq_env):
|
||||
dq_env.YTDL_OPTIONS = {"writeinfojson": True}
|
||||
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||
|
||||
dq = DownloadQueue(dq_env, AsyncMock())
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
await dq.add(
|
||||
"https://example.com/playlist?list=PL123", "video", "auto", "any", "best",
|
||||
"Music", "", 0, auto_start=False,
|
||||
)
|
||||
|
||||
assert _written_files(dq_env.DOWNLOAD_DIR) == [
|
||||
os.path.join("Music", "My Playlist", "My Playlist.info.json")
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_feed_metadata_without_writeinfojson(dq_env):
|
||||
"""Nothing new appears for users who never asked for these files."""
|
||||
dq_env.YTDL_OPTIONS = {}
|
||||
|
||||
dq = DownloadQueue(dq_env, AsyncMock())
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
await dq.add(
|
||||
"https://example.com/playlist?list=PL123", "video", "auto", "any", "best",
|
||||
"", "", 0, auto_start=False,
|
||||
)
|
||||
|
||||
assert _written_files(dq_env.DOWNLOAD_DIR) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feed_metadata_can_be_turned_off_by_the_user(dq_env):
|
||||
dq_env.YTDL_OPTIONS = {"writeinfojson": True, "allow_playlist_files": False}
|
||||
|
||||
dq = DownloadQueue(dq_env, AsyncMock())
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
await dq.add(
|
||||
"https://example.com/playlist?list=PL123", "video", "auto", "any", "best",
|
||||
"", "", 0, auto_start=False,
|
||||
)
|
||||
|
||||
assert _written_files(dq_env.DOWNLOAD_DIR) == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feed_metadata_failure_does_not_fail_the_add(dq_env):
|
||||
dq_env.YTDL_OPTIONS = {"writeinfojson": True}
|
||||
|
||||
dq = DownloadQueue(dq_env, AsyncMock())
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", _feed_extract(_PLAYLIST_FEED)), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()), \
|
||||
patch.object(
|
||||
DownloadQueue, "_DownloadQueue__write_feed_metadata_sync",
|
||||
side_effect=OSError("read-only filesystem"),
|
||||
):
|
||||
result = await dq.add(
|
||||
"https://example.com/playlist?list=PL123", "video", "auto", "any", "best",
|
||||
"", "", 0, auto_start=False,
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert dq.pending.exists("https://example.com/v1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extraction_pass_never_writes_feed_metadata(dq_env):
|
||||
"""The classification pass must not produce files: it runs before the add is
|
||||
known to succeed, and yt-dlp writes playlist files regardless of `download`."""
|
||||
dq_env.YTDL_OPTIONS = {"writeinfojson": True, "allow_playlist_files": True}
|
||||
captured: list = []
|
||||
|
||||
class FakeYoutubeDL:
|
||||
def __init__(self, params=None):
|
||||
captured.append(params)
|
||||
|
||||
def extract_info(self, url, download=False):
|
||||
return {"_type": "video", "id": "v", "title": "V", "url": url, "webpage_url": url}
|
||||
|
||||
dq = DownloadQueue(dq_env, AsyncMock())
|
||||
with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
|
||||
await dq.add(
|
||||
"https://example.com/watch?v=1", "video", "auto", "any", "best",
|
||||
"", "", 0, auto_start=False,
|
||||
)
|
||||
|
||||
assert captured[0]["allow_playlist_files"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_sets_clip_bounds_on_download_info(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
def fake_extract(self, url, *_args, **_kwargs):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
|
||||
@@ -146,12 +146,12 @@ class PersistentQueueTests(unittest.TestCase):
|
||||
self.assertNotIn("formats", record["entry"])
|
||||
self.assertNotIn("description", record["entry"])
|
||||
|
||||
def test_completed_queue_does_not_persist_entry_or_transient_progress(self):
|
||||
def test_completed_queue_persists_only_failed_retry_context(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "completed")
|
||||
pq = PersistentQueue("completed", path)
|
||||
info = _make_info("http://done.example")
|
||||
info.status = "finished"
|
||||
info.status = "error"
|
||||
info.percent = 88
|
||||
info.speed = 123
|
||||
info.eta = 9
|
||||
@@ -167,12 +167,24 @@ class PersistentQueueTests(unittest.TestCase):
|
||||
payload = json.load(f)
|
||||
|
||||
record = payload["items"][0]["info"]
|
||||
self.assertNotIn("entry", record)
|
||||
self.assertEqual(
|
||||
record["entry"],
|
||||
{
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
},
|
||||
)
|
||||
self.assertNotIn("percent", record)
|
||||
self.assertNotIn("speed", record)
|
||||
self.assertNotIn("eta", record)
|
||||
self.assertEqual(record["filename"], "done.mp4")
|
||||
|
||||
info.status = "finished"
|
||||
pq.put(_FakeDownload(info))
|
||||
with open(path + ".json", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
self.assertNotIn("entry", payload["items"][0]["info"])
|
||||
|
||||
def test_invalid_json_is_quarantined_and_legacy_is_imported(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
|
||||
@@ -821,6 +821,76 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(upd["subscription"]["title_regex"], "foo|bar")
|
||||
self.assertEqual(mgr.list_all()[0].title_regex, "foo|bar")
|
||||
|
||||
async def _add_one_subscription(self, mgr):
|
||||
with patch(
|
||||
"subscriptions.extract_flat_playlist",
|
||||
return_value=(
|
||||
{"_type": "channel", "title": "Videos"},
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
),
|
||||
):
|
||||
result = await mgr.add_subscription(
|
||||
"https://example.com/playlist?list=UULFabc",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
)
|
||||
return result["subscription"]["id"]
|
||||
|
||||
async def test_update_subscription_renames(self):
|
||||
"""Issue #1044: UULF-style uploads playlists all come back named 'Videos',
|
||||
so the user needs to be able to relabel them."""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||
sub_id = await self._add_one_subscription(mgr)
|
||||
self.assertEqual(mgr.list_all()[0].name, "Videos")
|
||||
|
||||
upd = await mgr.update_subscription(sub_id, {"name": " Jane's uploads \n"})
|
||||
self.assertEqual(upd["status"], "ok")
|
||||
# Surrounding and interior whitespace is collapsed to keep the name
|
||||
# a single-line label.
|
||||
self.assertEqual(upd["subscription"]["name"], "Jane's uploads")
|
||||
self.assertEqual(mgr.list_all()[0].name, "Jane's uploads")
|
||||
|
||||
async def test_update_subscription_rename_survives_reload(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg = _Config(tmp)
|
||||
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||
sub_id = await self._add_one_subscription(mgr)
|
||||
await mgr.update_subscription(sub_id, {"name": "Renamed"})
|
||||
|
||||
reloaded = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||
self.assertEqual(reloaded.get(sub_id).name, "Renamed")
|
||||
|
||||
async def test_update_subscription_rejects_unusable_name(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||
sub_id = await self._add_one_subscription(mgr)
|
||||
|
||||
for bad in ("", " ", "\n\t", 42, None, ["a"], "x" * 201):
|
||||
upd = await mgr.update_subscription(sub_id, {"name": bad})
|
||||
self.assertEqual(upd["status"], "error", f"expected {bad!r} to be rejected")
|
||||
self.assertEqual(mgr.list_all()[0].name, "Videos")
|
||||
|
||||
async def test_update_subscription_accepts_name_at_length_limit(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||
sub_id = await self._add_one_subscription(mgr)
|
||||
|
||||
upd = await mgr.update_subscription(sub_id, {"name": "x" * 200})
|
||||
self.assertEqual(upd["status"], "ok")
|
||||
self.assertEqual(mgr.list_all()[0].name, "x" * 200)
|
||||
|
||||
async def test_update_subscription_skip_subscriber_only(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
queue = _Queue()
|
||||
@@ -1101,6 +1171,44 @@ class SubscriptionScanExtraOptsTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(captured_params[0].get("cookiefile"), "preset.txt")
|
||||
self.assertEqual(captured_params[0].get("extra"), "override")
|
||||
|
||||
async def test_scan_never_writes_playlist_sidecar_files(self):
|
||||
"""A subscription scan is a metadata probe. yt-dlp writes the
|
||||
playlist-level infojson/description/thumbnail regardless of ``download``,
|
||||
so without this a writeinfojson/writethumbnail user would get stray files
|
||||
in DOWNLOAD_DIR on every check interval. Issue #1040."""
|
||||
captured_params: list = []
|
||||
fake_ydl = _make_scan_capturing_fake_ydl(
|
||||
captured_params,
|
||||
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg = _Config(tmp)
|
||||
cfg.YTDL_OPTIONS = {"writeinfojson": True, "writethumbnail": True}
|
||||
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||
|
||||
with patch("subscriptions.yt_dlp.YoutubeDL", fake_ydl, create=True):
|
||||
await mgr.add_subscription(
|
||||
"https://example.com/channel",
|
||||
check_interval_minutes=60,
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
auto_start=True,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
subtitle_language="en",
|
||||
subtitle_mode="prefer_manual",
|
||||
ytdl_options_overrides={"allow_playlist_files": True},
|
||||
)
|
||||
|
||||
self.assertTrue(captured_params)
|
||||
self.assertIs(captured_params[0].get("allow_playlist_files"), False)
|
||||
|
||||
async def test_check_now_scan_applies_stored_subscription_presets(self):
|
||||
entries = [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}]
|
||||
|
||||
|
||||
+111
-16
@@ -11,6 +11,7 @@ from url_guard import (
|
||||
validate_url,
|
||||
_address_allowed_at_connect,
|
||||
_guarded_getaddrinfo,
|
||||
_proxy_endpoint,
|
||||
install_socket_guard,
|
||||
)
|
||||
|
||||
@@ -106,15 +107,25 @@ class AddressResolutionTests(unittest.TestCase):
|
||||
|
||||
|
||||
class ConnectAddressPolicyTests(unittest.TestCase):
|
||||
"""Connect-time policy: allow global + loopback, block everything else."""
|
||||
"""Connect-time policy: allow global, plus loopback only when the caller has
|
||||
established that this destination is the operator's configured proxy."""
|
||||
|
||||
def test_global_allowed(self):
|
||||
self.assertTrue(_address_allowed_at_connect("142.250.1.1"))
|
||||
|
||||
def test_loopback_allowed(self):
|
||||
# Loopback stays reachable so locally-configured proxies keep working.
|
||||
self.assertTrue(_address_allowed_at_connect("127.0.0.1"))
|
||||
self.assertTrue(_address_allowed_at_connect("::1"))
|
||||
def test_loopback_blocked_by_default(self):
|
||||
# A blanket loopback allowance is what let manifest-derived media URLs
|
||||
# reach services on the server's own loopback interface.
|
||||
self.assertFalse(_address_allowed_at_connect("127.0.0.1"))
|
||||
self.assertFalse(_address_allowed_at_connect("::1"))
|
||||
|
||||
def test_loopback_allowed_only_when_opted_in(self):
|
||||
self.assertTrue(_address_allowed_at_connect("127.0.0.1", allow_loopback=True))
|
||||
self.assertTrue(_address_allowed_at_connect("::1", allow_loopback=True))
|
||||
|
||||
def test_opt_in_does_not_widen_beyond_loopback(self):
|
||||
self.assertFalse(_address_allowed_at_connect("169.254.169.254", allow_loopback=True))
|
||||
self.assertFalse(_address_allowed_at_connect("10.0.0.5", allow_loopback=True))
|
||||
|
||||
def test_link_local_metadata_blocked(self):
|
||||
self.assertFalse(_address_allowed_at_connect("169.254.169.254"))
|
||||
@@ -127,7 +138,37 @@ class ConnectAddressPolicyTests(unittest.TestCase):
|
||||
self.assertFalse(_address_allowed_at_connect("::ffff:169.254.169.254"))
|
||||
|
||||
|
||||
class ProxyEndpointParsingTests(unittest.TestCase):
|
||||
def test_explicit_port(self):
|
||||
self.assertEqual(_proxy_endpoint("http://127.0.0.1:9050"), ("127.0.0.1", 9050))
|
||||
|
||||
def test_default_port_per_scheme(self):
|
||||
self.assertEqual(_proxy_endpoint("socks5://127.0.0.1"), ("127.0.0.1", 1080))
|
||||
self.assertEqual(_proxy_endpoint("http://127.0.0.1"), ("127.0.0.1", 80))
|
||||
|
||||
def test_bare_host_port(self):
|
||||
self.assertEqual(_proxy_endpoint("127.0.0.1:8080"), ("127.0.0.1", 8080))
|
||||
|
||||
def test_hostname_lowercased(self):
|
||||
self.assertEqual(_proxy_endpoint("http://LocalHost.:9050"), ("localhost", 9050))
|
||||
|
||||
def test_ipv6_literal(self):
|
||||
self.assertEqual(_proxy_endpoint("http://[::1]:9050"), ("::1", 9050))
|
||||
|
||||
def test_empty_and_invalid(self):
|
||||
self.assertIsNone(_proxy_endpoint(""))
|
||||
self.assertIsNone(_proxy_endpoint(" "))
|
||||
self.assertIsNone(_proxy_endpoint(None))
|
||||
self.assertIsNone(_proxy_endpoint("http://"))
|
||||
|
||||
|
||||
class GuardedGetaddrinfoTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
# Default state: no proxy configured, so no loopback destination allowed.
|
||||
saved = set(url_guard._allowed_loopback_endpoints)
|
||||
url_guard._allowed_loopback_endpoints = set()
|
||||
self.addCleanup(lambda: setattr(url_guard, "_allowed_loopback_endpoints", saved))
|
||||
|
||||
def test_internal_only_raises(self):
|
||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")):
|
||||
with self.assertRaises(socket.gaierror):
|
||||
@@ -139,9 +180,35 @@ class GuardedGetaddrinfoTests(unittest.TestCase):
|
||||
results = _guarded_getaddrinfo("mixed", 80)
|
||||
self.assertEqual([r[4][0] for r in results], ["142.250.1.1"])
|
||||
|
||||
def test_loopback_passes(self):
|
||||
def test_loopback_blocked_without_matching_proxy(self):
|
||||
# The advisory case: an m3u8 segment URL pointing at a loopback service.
|
||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||
results = _guarded_getaddrinfo("localproxy", 9050)
|
||||
with self.assertRaises(socket.gaierror):
|
||||
_guarded_getaddrinfo("127.0.0.1", 9999)
|
||||
|
||||
def test_loopback_allowed_at_configured_proxy_endpoint(self):
|
||||
url_guard._allowed_loopback_endpoints = {("127.0.0.1", 9050)}
|
||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||
results = _guarded_getaddrinfo("127.0.0.1", 9050)
|
||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||
|
||||
def test_loopback_blocked_at_other_port_on_proxy_host(self):
|
||||
# Same host as the proxy, different port: still off limits.
|
||||
url_guard._allowed_loopback_endpoints = {("127.0.0.1", 9050)}
|
||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||
with self.assertRaises(socket.gaierror):
|
||||
_guarded_getaddrinfo("127.0.0.1", 9999)
|
||||
|
||||
def test_proxy_reachable_by_hostname(self):
|
||||
url_guard._allowed_loopback_endpoints = {("localhost", 9050)}
|
||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||
results = _guarded_getaddrinfo("localhost", 9050)
|
||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||
|
||||
def test_string_port_is_normalised(self):
|
||||
url_guard._allowed_loopback_endpoints = {("127.0.0.1", 9050)}
|
||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||
results = _guarded_getaddrinfo("127.0.0.1", "9050")
|
||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||
|
||||
|
||||
@@ -172,16 +239,44 @@ class AllowPrivateBypassTests(unittest.TestCase):
|
||||
|
||||
|
||||
class InstallSocketGuardTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
original, saved = socket.getaddrinfo, set(url_guard._allowed_loopback_endpoints)
|
||||
self.addCleanup(lambda: setattr(socket, "getaddrinfo", original))
|
||||
self.addCleanup(lambda: setattr(url_guard, "_allowed_loopback_endpoints", saved))
|
||||
# Keep the host's own environment out of the assertions below.
|
||||
patcher = mock.patch("url_guard.urllib.request.getproxies", return_value={})
|
||||
self.getproxies = patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def test_install_replaces_and_is_idempotent(self):
|
||||
original = socket.getaddrinfo
|
||||
try:
|
||||
install_socket_guard()
|
||||
self.assertIs(socket.getaddrinfo, url_guard._guarded_getaddrinfo)
|
||||
# Re-installing must not wrap the wrapper (real fn captured at import).
|
||||
install_socket_guard()
|
||||
self.assertIs(socket.getaddrinfo, url_guard._guarded_getaddrinfo)
|
||||
finally:
|
||||
socket.getaddrinfo = original
|
||||
install_socket_guard()
|
||||
self.assertIs(socket.getaddrinfo, url_guard._guarded_getaddrinfo)
|
||||
# Re-installing must not wrap the wrapper (real fn captured at import).
|
||||
install_socket_guard()
|
||||
self.assertIs(socket.getaddrinfo, url_guard._guarded_getaddrinfo)
|
||||
|
||||
def test_no_proxy_means_no_loopback_allowance(self):
|
||||
install_socket_guard()
|
||||
self.assertEqual(url_guard._allowed_loopback_endpoints, set())
|
||||
|
||||
def test_explicit_proxy_is_registered(self):
|
||||
install_socket_guard(proxy_urls=("socks5://127.0.0.1:9050",))
|
||||
self.assertEqual(url_guard._allowed_loopback_endpoints, {("127.0.0.1", 9050)})
|
||||
|
||||
def test_unset_proxy_option_is_ignored(self):
|
||||
# ytdl_opts.get('proxy') is None when the operator configured no proxy.
|
||||
install_socket_guard(proxy_urls=(None,))
|
||||
self.assertEqual(url_guard._allowed_loopback_endpoints, set())
|
||||
|
||||
def test_environment_proxies_are_registered(self):
|
||||
self.getproxies.return_value = {"http": "http://127.0.0.1:8080"}
|
||||
install_socket_guard()
|
||||
self.assertEqual(url_guard._allowed_loopback_endpoints, {("127.0.0.1", 8080)})
|
||||
|
||||
def test_endpoints_reset_between_installs(self):
|
||||
install_socket_guard(proxy_urls=("http://127.0.0.1:8080",))
|
||||
install_socket_guard(proxy_urls=(None,))
|
||||
self.assertEqual(url_guard._allowed_loopback_endpoints, set())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+96
-12
@@ -30,12 +30,23 @@ all of these:
|
||||
import ipaddress
|
||||
import logging
|
||||
import socket
|
||||
import urllib.request
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
log = logging.getLogger('url_guard')
|
||||
|
||||
_ALLOWED_SCHEMES = ('http', 'https')
|
||||
|
||||
# Ports to assume when a configured proxy URL omits one, per proxy scheme.
|
||||
_PROXY_DEFAULT_PORTS = {
|
||||
'http': 80,
|
||||
'https': 443,
|
||||
'socks4': 1080,
|
||||
'socks4a': 1080,
|
||||
'socks5': 1080,
|
||||
'socks5h': 1080,
|
||||
}
|
||||
|
||||
# Hostnames that must be blocked without needing a lookup. ``localhost`` and any
|
||||
# subdomain of it are conventionally loopback, and the GCP metadata name is a
|
||||
# well-known SSRF target that may resolve via a resolver we don't control.
|
||||
@@ -68,40 +79,107 @@ def _address_is_global(addr: str) -> bool:
|
||||
return ip is not None and ip.is_global
|
||||
|
||||
|
||||
def _address_allowed_at_connect(addr: str) -> bool:
|
||||
def _address_allowed_at_connect(addr: str, allow_loopback: bool = False) -> bool:
|
||||
"""True if *addr* may be connected to at download time.
|
||||
|
||||
Permits global addresses and loopback — loopback so that locally-configured
|
||||
proxies (e.g. ``proxy: http://127.0.0.1:9050``) keep working. Blocks the SSRF
|
||||
targets that matter: link-local (cloud metadata at 169.254.169.254), private
|
||||
(RFC1918), unique-local and every other non-global, non-loopback range.
|
||||
Permits global addresses only. Loopback is permitted just for the specific
|
||||
host:port of an operator-configured proxy (see ``_loopback_endpoint_allowed``),
|
||||
never as a blanket rule: media URLs that yt-dlp derives from a remote manifest
|
||||
are attacker-controlled and reach this policy without passing ``validate_url``,
|
||||
so a general loopback allowance would let a hostile playlist read any service
|
||||
on the server's loopback interface. Blocks link-local (cloud metadata at
|
||||
169.254.169.254), private (RFC1918), unique-local and every other non-global
|
||||
range.
|
||||
"""
|
||||
ip = _normalise_ip(addr)
|
||||
return ip is not None and (ip.is_global or ip.is_loopback)
|
||||
if ip is None:
|
||||
return False
|
||||
return ip.is_global or (allow_loopback and ip.is_loopback)
|
||||
|
||||
|
||||
def _proxy_endpoint(proxy_url: str):
|
||||
"""Parse a proxy URL into a ``(hostname, port)`` pair, or ``None`` if it has
|
||||
no usable host. Used to scope the loopback allowance to that endpoint alone."""
|
||||
if not isinstance(proxy_url, str) or not proxy_url.strip():
|
||||
return None
|
||||
candidate = proxy_url.strip()
|
||||
if '://' not in candidate:
|
||||
# Bare host:port, as accepted by the *_proxy environment variables.
|
||||
candidate = '//' + candidate
|
||||
try:
|
||||
parts = urlsplit(candidate)
|
||||
hostname, port = parts.hostname, parts.port
|
||||
except ValueError:
|
||||
return None
|
||||
if not hostname:
|
||||
return None
|
||||
if port is None:
|
||||
port = _PROXY_DEFAULT_PORTS.get(parts.scheme.lower())
|
||||
return (hostname.rstrip('.').lower(), port)
|
||||
|
||||
|
||||
def _collect_proxy_endpoints(proxy_urls) -> set:
|
||||
"""Endpoints of every proxy this download may legitimately dial: the explicit
|
||||
yt-dlp ``proxy`` option plus the ``*_proxy`` environment variables yt-dlp falls
|
||||
back to. All are operator-configured, unlike the URLs inside fetched media."""
|
||||
candidates = list(proxy_urls) + list(urllib.request.getproxies().values())
|
||||
return {ep for ep in map(_proxy_endpoint, candidates) if ep is not None}
|
||||
|
||||
|
||||
# Captured at import so re-installing the guard never wraps the wrapper.
|
||||
_real_getaddrinfo = socket.getaddrinfo
|
||||
|
||||
# Populated by install_socket_guard; empty means no loopback destination is allowed.
|
||||
_allowed_loopback_endpoints: set = set()
|
||||
|
||||
|
||||
def _normalise_port(port):
|
||||
if isinstance(port, str):
|
||||
try:
|
||||
return int(port)
|
||||
except ValueError:
|
||||
try:
|
||||
return socket.getservbyname(port)
|
||||
except OSError:
|
||||
return None
|
||||
return port
|
||||
|
||||
|
||||
def _loopback_endpoint_allowed(host, port) -> bool:
|
||||
if not _allowed_loopback_endpoints or host is None:
|
||||
return False
|
||||
return (str(host).rstrip('.').lower(), _normalise_port(port)) in _allowed_loopback_endpoints
|
||||
|
||||
|
||||
def _guarded_getaddrinfo(host, *args, **kwargs):
|
||||
results = _real_getaddrinfo(host, *args, **kwargs)
|
||||
allowed = [r for r in results if _address_allowed_at_connect(r[4][0])]
|
||||
# Mirrors getaddrinfo(host, port, ...): port is the first optional argument.
|
||||
port = args[0] if args else kwargs.get('port')
|
||||
allow_loopback = _loopback_endpoint_allowed(host, port)
|
||||
allowed = [r for r in results if _address_allowed_at_connect(r[4][0], allow_loopback)]
|
||||
if not allowed:
|
||||
raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}')
|
||||
return allowed
|
||||
|
||||
|
||||
def install_socket_guard(allow_private: bool = False) -> None:
|
||||
def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
|
||||
"""Enforce the no-internal-hosts policy at actual connection time.
|
||||
|
||||
``validate_url`` only checks the *submitted* URL string; yt-dlp then follows
|
||||
HTTP redirects and resolves media URLs from remote metadata without
|
||||
re-validating them. Installing this in the download subprocess re-checks
|
||||
every resolved address at connect time, covering redirects and DNS rebinding
|
||||
for any networking backend that resolves through Python's socket module
|
||||
(urllib, requests). Native resolvers — notably curl_cffi/libcurl used by
|
||||
``--impersonate`` — bypass this and rely on network isolation as the backstop.
|
||||
every resolved address at connect time, covering redirects, DNS rebinding and
|
||||
manifest-derived media URLs for any networking backend that resolves through
|
||||
Python's socket module (urllib, requests). Native resolvers — notably
|
||||
curl_cffi/libcurl used by ``--impersonate`` — bypass this and rely on network
|
||||
isolation as the backstop.
|
||||
|
||||
*proxy_urls* are the operator's configured proxies (yt-dlp's ``proxy`` option;
|
||||
the ``*_proxy`` environment variables are picked up automatically). A proxy on
|
||||
loopback is reachable at its own host:port, and nothing else on loopback is.
|
||||
That costs proxied setups nothing: yt-dlp resolves the proxy itself at exactly
|
||||
that host:port, and a media URL is either handed to the proxy unresolved or
|
||||
resolved on its own merits — never inheriting the proxy's allowance.
|
||||
|
||||
When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not
|
||||
installed at all, so proxy/VPN setups that route through private or Fake-IP
|
||||
@@ -109,6 +187,12 @@ def install_socket_guard(allow_private: bool = False) -> None:
|
||||
"""
|
||||
if allow_private:
|
||||
return
|
||||
_allowed_loopback_endpoints.clear()
|
||||
_allowed_loopback_endpoints.update(_collect_proxy_endpoints(proxy_urls))
|
||||
for host, port in sorted(_allowed_loopback_endpoints, key=lambda ep: (ep[0], ep[1] or 0)):
|
||||
ip = _normalise_ip(host)
|
||||
if ip is not None and ip.is_loopback:
|
||||
log.info(f'Allowing connections to configured loopback proxy {host}:{port}')
|
||||
socket.getaddrinfo = _guarded_getaddrinfo
|
||||
|
||||
|
||||
|
||||
+154
-13
@@ -510,7 +510,7 @@ def _short_title_for_failed_url(url: str) -> str:
|
||||
return hostname or url
|
||||
|
||||
|
||||
_COMPACT_ENTRY_EXTRA_KEYS = frozenset(("n_entries", "__last_playlist_index"))
|
||||
_COMPACT_ENTRY_EXTRA_KEYS = frozenset(("n_entries", "__last_playlist_index", "track_number"))
|
||||
|
||||
|
||||
def _compact_persisted_entry(entry: Any) -> Optional[dict[str, Any]]:
|
||||
@@ -657,10 +657,12 @@ class Download:
|
||||
except OSError:
|
||||
pass
|
||||
# Re-validate every outbound connection at fetch time. validate_url only
|
||||
# saw the submitted URL string; this catches redirects and DNS rebinding
|
||||
# to internal hosts (cloud metadata, RFC1918) that it cannot. Skipped when
|
||||
# ALLOW_PRIVATE_ADDRESSES trusts the environment (e.g. Fake-IP proxies).
|
||||
install_socket_guard(self.allow_private)
|
||||
# saw the submitted URL string; this catches redirects, DNS rebinding and
|
||||
# attacker-controlled media URLs pulled from a remote manifest, none of
|
||||
# which it can see. The configured proxy is passed so that a proxy on
|
||||
# loopback stays reachable at its own address without opening up the rest
|
||||
# of loopback. Skipped when ALLOW_PRIVATE_ADDRESSES trusts the environment.
|
||||
install_socket_guard(self.allow_private, proxy_urls=(self.ytdl_opts.get('proxy'),))
|
||||
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
||||
try:
|
||||
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
||||
@@ -944,8 +946,12 @@ class PersistentQueue:
|
||||
]
|
||||
return sorted(items, key=lambda item: item[1].timestamp)
|
||||
|
||||
def _should_persist_entry(self) -> bool:
|
||||
return self.identifier != "completed"
|
||||
def _should_persist_entry(self, info: DownloadInfo | dict[str, Any]) -> bool:
|
||||
# Failed downloads need their compact playlist/channel context so a
|
||||
# retry after a server restart still resolves the original outtmpl.
|
||||
# Successful completed entries continue to omit extractor metadata.
|
||||
status = info.get("status") if isinstance(info, dict) else info.status
|
||||
return self.identifier != "completed" or status == "error"
|
||||
|
||||
def _serialize_items(self):
|
||||
return [
|
||||
@@ -953,7 +959,7 @@ class PersistentQueue:
|
||||
"key": key,
|
||||
"info": _download_info_to_record(
|
||||
download.info,
|
||||
include_entry=self._should_persist_entry(),
|
||||
include_entry=self._should_persist_entry(download.info),
|
||||
),
|
||||
}
|
||||
for key, download in self.dict.items()
|
||||
@@ -972,7 +978,7 @@ class PersistentQueue:
|
||||
"key": item["key"],
|
||||
"info": _download_info_to_record(
|
||||
_download_info_from_record(item["info"]),
|
||||
include_entry=self._should_persist_entry(),
|
||||
include_entry=self._should_persist_entry(item["info"]),
|
||||
),
|
||||
}
|
||||
for item in items
|
||||
@@ -993,7 +999,7 @@ class PersistentQueue:
|
||||
"key": key,
|
||||
"info": _download_info_to_record(
|
||||
value,
|
||||
include_entry=self._should_persist_entry(),
|
||||
include_entry=self._should_persist_entry(value),
|
||||
),
|
||||
}
|
||||
for key, value in sorted(legacy_items, key=lambda item: item[1].timestamp)
|
||||
@@ -1310,6 +1316,14 @@ class DownloadQueue:
|
||||
'ignore_no_formats_error': True,
|
||||
'noplaylist': True,
|
||||
'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR},
|
||||
# This is a classification pass, not a download. yt-dlp emits the
|
||||
# feed-level infojson/description/thumbnail from
|
||||
# __process_playlist_result without consulting `download`, so
|
||||
# without this a writeinfojson user gets stray files here — in
|
||||
# DOWNLOAD_DIR, under yt-dlp's pl_* names, even for an add that goes
|
||||
# on to fail. __write_feed_metadata writes them properly once the
|
||||
# feed is accepted. See issues #1040 and #660.
|
||||
'allow_playlist_files': False,
|
||||
}
|
||||
imp = user_opts.get('impersonate')
|
||||
if imp is not None:
|
||||
@@ -1373,6 +1387,81 @@ class DownloadQueue:
|
||||
self.pending.put(download)
|
||||
await self.notifier.added(dl)
|
||||
|
||||
def __write_feed_metadata_sync(self, entry, etype, download_type, folder,
|
||||
ytdl_options_presets, ytdl_options_overrides):
|
||||
"""Write the feed-level .info.json/description/thumbnail for a playlist
|
||||
or channel add, using the same output template its items will use.
|
||||
|
||||
yt-dlp produces these from __process_playlist_result, which ignores
|
||||
``download`` — so they used to fall out of the classification pass with
|
||||
yt-dlp's own pl_* names, in DOWNLOAD_DIR, ignoring the download's folder
|
||||
(issue #1040) and with no way to steer them (issue #660). Doing it here
|
||||
instead means the feed type is already known, so the file lands beside
|
||||
the items rather than in a differently-named sibling directory.
|
||||
|
||||
Re-runs yt-dlp on a copy of the feed with no entries: that reaches the
|
||||
playlist-file writing without re-extracting anything or touching
|
||||
yt-dlp's private write helpers.
|
||||
"""
|
||||
user_opts = self._build_ytdl_options(ytdl_options_presets, ytdl_options_overrides)
|
||||
wants = ('writeinfojson', 'writedescription', 'writethumbnail', 'write_all_thumbnails')
|
||||
if not any(user_opts.get(key) for key in wants):
|
||||
return
|
||||
# An explicit allow_playlist_files=false is the user asking for exactly
|
||||
# this to not happen.
|
||||
if user_opts.get('allow_playlist_files') is False:
|
||||
return
|
||||
|
||||
dldirectory, error_message = self.__calc_download_path(download_type, folder)
|
||||
if error_message is not None:
|
||||
return
|
||||
|
||||
template = (
|
||||
self.config.OUTPUT_TEMPLATE_CHANNEL if etype == 'channel'
|
||||
else self.config.OUTPUT_TEMPLATE_PLAYLIST
|
||||
) or self.config.OUTPUT_TEMPLATE
|
||||
|
||||
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
||||
params = {
|
||||
**user_opts,
|
||||
'quiet': not debug_logging,
|
||||
'verbose': debug_logging,
|
||||
'no_color': True,
|
||||
'skip_download': True,
|
||||
'extract_flat': True,
|
||||
'allow_playlist_files': True,
|
||||
'paths': {"home": dldirectory, "temp": self.config.TEMP_DIR},
|
||||
# Feed-level keys only; per-item names are resolved by __add_download.
|
||||
'outtmpl': {
|
||||
'pl_infojson': template,
|
||||
'pl_thumbnail': template,
|
||||
'pl_description': template,
|
||||
},
|
||||
}
|
||||
imp = user_opts.get('impersonate')
|
||||
if imp is not None:
|
||||
params['impersonate'] = yt_dlp.networking.impersonate.ImpersonateTarget.from_str(imp)
|
||||
|
||||
# A copy: process_ie_result mutates entries/requested_entries, and the
|
||||
# caller still needs the real feed dict to queue the items.
|
||||
feed = {k: v for k, v in entry.items() if k != 'entries'}
|
||||
feed['entries'] = []
|
||||
yt_dlp.YoutubeDL(params=params).process_ie_result(feed, download=False)
|
||||
|
||||
async def __write_feed_metadata(self, entry, etype, download_type, folder,
|
||||
ytdl_options_presets, ytdl_options_overrides):
|
||||
try:
|
||||
await asyncio.get_running_loop().run_in_executor(
|
||||
None,
|
||||
partial(
|
||||
self.__write_feed_metadata_sync, entry, etype, download_type, folder,
|
||||
ytdl_options_presets, ytdl_options_overrides,
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
# Supplemental output must never fail the add.
|
||||
log.warning(f'Could not write {etype} metadata files: {exc}')
|
||||
|
||||
async def __add_entry(
|
||||
self,
|
||||
entry,
|
||||
@@ -1394,6 +1483,7 @@ class DownloadQueue:
|
||||
clip_end,
|
||||
already,
|
||||
_add_gen=None,
|
||||
retry_entry=None,
|
||||
):
|
||||
if not entry:
|
||||
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
||||
@@ -1412,6 +1502,10 @@ class DownloadQueue:
|
||||
|
||||
if etype.startswith('url'):
|
||||
log.debug('Processing as a url')
|
||||
# retry_entry must ride along: extraction can hand back an
|
||||
# unprocessed url/url_transparent result, and dropping the retry
|
||||
# context here would send the retried item back to the root
|
||||
# directory instead of its original playlist folder.
|
||||
return await self.add(
|
||||
entry['url'],
|
||||
download_type,
|
||||
@@ -1432,6 +1526,7 @@ class DownloadQueue:
|
||||
clip_end,
|
||||
already,
|
||||
_add_gen,
|
||||
retry_entry,
|
||||
)
|
||||
elif etype == 'playlist' or etype == 'channel':
|
||||
if etype == 'playlist' and self.__is_channel_extraction(entry):
|
||||
@@ -1443,6 +1538,10 @@ class DownloadQueue:
|
||||
entries = list(entries)
|
||||
total_entries = len(entries)
|
||||
log.info(f'{etype} detected with {total_entries} entries')
|
||||
await self.__write_feed_metadata(
|
||||
entry, etype, download_type, folder,
|
||||
ytdl_options_presets, ytdl_options_overrides,
|
||||
)
|
||||
index_digits = len(str(total_entries))
|
||||
results = []
|
||||
if playlist_item_limit > 0:
|
||||
@@ -1559,6 +1658,7 @@ class DownloadQueue:
|
||||
ytdl_options_overrides,
|
||||
clip_start,
|
||||
clip_end,
|
||||
entry=None,
|
||||
):
|
||||
"""Surface a URL that failed before a DownloadInfo could be created (unsupported
|
||||
URL, SSRF-rejected, extraction error) as a failed entry in the done list, so the
|
||||
@@ -1575,7 +1675,7 @@ class DownloadQueue:
|
||||
folder=folder,
|
||||
custom_name_prefix=custom_name_prefix,
|
||||
error=msg,
|
||||
entry=None,
|
||||
entry=entry,
|
||||
playlist_item_limit=playlist_item_limit,
|
||||
split_by_chapters=split_by_chapters,
|
||||
chapter_template=chapter_template,
|
||||
@@ -1613,6 +1713,7 @@ class DownloadQueue:
|
||||
clip_end=None,
|
||||
already=None,
|
||||
_add_gen=None,
|
||||
retry_entry=None,
|
||||
):
|
||||
if ytdl_options_presets is None:
|
||||
ytdl_options_presets = []
|
||||
@@ -1641,7 +1742,7 @@ class DownloadQueue:
|
||||
url, url_error, download_type, codec, format, quality, folder,
|
||||
custom_name_prefix, playlist_item_limit, split_by_chapters, chapter_template,
|
||||
subtitle_language, subtitle_mode, ytdl_options_presets, ytdl_options_overrides,
|
||||
clip_start, clip_end,
|
||||
clip_start, clip_end, retry_entry,
|
||||
)
|
||||
return {'status': 'error', 'msg': url_error}
|
||||
try:
|
||||
@@ -1655,9 +1756,12 @@ class DownloadQueue:
|
||||
url, msg, download_type, codec, format, quality, folder,
|
||||
custom_name_prefix, playlist_item_limit, split_by_chapters, chapter_template,
|
||||
subtitle_language, subtitle_mode, ytdl_options_presets, ytdl_options_overrides,
|
||||
clip_start, clip_end,
|
||||
clip_start, clip_end, retry_entry,
|
||||
)
|
||||
return {'status': 'error', 'msg': msg}
|
||||
retry_context = _compact_persisted_entry(retry_entry)
|
||||
if isinstance(entry, dict) and retry_context is not None:
|
||||
entry = {**entry, **copy.deepcopy(retry_context)}
|
||||
return await self.__add_entry(
|
||||
entry,
|
||||
download_type,
|
||||
@@ -1678,6 +1782,43 @@ class DownloadQueue:
|
||||
clip_end,
|
||||
already,
|
||||
_add_gen,
|
||||
retry_entry,
|
||||
)
|
||||
|
||||
async def retry(self, id):
|
||||
if not self.done.exists(id):
|
||||
return {'status': 'error', 'msg': 'Failed download no longer exists.'}
|
||||
|
||||
info = self.done.get(id).info
|
||||
if info.status != 'error':
|
||||
return {'status': 'error', 'msg': 'Only failed downloads can be retried.'}
|
||||
|
||||
# The stored options were validated by parse_download_options when the
|
||||
# download was first submitted, but the configuration can have changed
|
||||
# since. Re-apply the same gates here so a retry can't resurrect
|
||||
# overrides or presets the current configuration no longer allows.
|
||||
overrides = info.ytdl_options_overrides if self.config.ALLOW_YTDL_OPTIONS_OVERRIDES else {}
|
||||
presets = [p for p in info.ytdl_options_presets if p in self.config.YTDL_OPTIONS_PRESETS]
|
||||
|
||||
return await self.add(
|
||||
info.url,
|
||||
info.download_type,
|
||||
info.codec,
|
||||
info.format,
|
||||
info.quality,
|
||||
info.folder,
|
||||
info.custom_name_prefix,
|
||||
info.playlist_item_limit,
|
||||
True,
|
||||
info.split_by_chapters,
|
||||
info.chapter_template,
|
||||
info.subtitle_language,
|
||||
info.subtitle_mode,
|
||||
presets,
|
||||
overrides,
|
||||
info.clip_start,
|
||||
info.clip_end,
|
||||
retry_entry=info.entry,
|
||||
)
|
||||
|
||||
async def add_entry(
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 885 KiB After Width: | Height: | Size: 1.9 MiB |
+27
-1
@@ -958,7 +958,33 @@
|
||||
[disabled]="downloads.loading"
|
||||
[attr.aria-label]="'Select subscription ' + entry[1].name" />
|
||||
</td>
|
||||
<td>{{ entry[1].name }}</td>
|
||||
<td>
|
||||
@if (editingNameId === entry[0]) {
|
||||
<div class="d-flex flex-wrap gap-1 align-items-center">
|
||||
<input type="text"
|
||||
class="form-control form-control-sm flex-grow-1"
|
||||
[name]="'subName' + entry[0]"
|
||||
[(ngModel)]="nameEditDraft"
|
||||
[maxlength]="subscriptionNameMaxLength"
|
||||
[disabled]="downloads.loading"
|
||||
[attr.aria-label]="'Subscription name for ' + entry[1].name" />
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary"
|
||||
(click)="saveName(entry[0])"
|
||||
[disabled]="downloads.loading">Save</button>
|
||||
<button type="button" class="btn btn-sm btn-outline-secondary"
|
||||
(click)="cancelEditName()"
|
||||
[disabled]="downloads.loading">Cancel</button>
|
||||
</div>
|
||||
} @else {
|
||||
<div class="d-flex flex-wrap gap-1 align-items-center">
|
||||
<span class="text-break">{{ entry[1].name }}</span>
|
||||
<button type="button" class="btn btn-link btn-sm p-0"
|
||||
(click)="beginEditName(entry[0], entry[1].name)"
|
||||
[disabled]="downloads.loading"
|
||||
ngbTooltip="Rename this subscription (display name only; does not affect the download folder)">Edit</button>
|
||||
</div>
|
||||
}
|
||||
</td>
|
||||
<td class="text-break"><a [href]="entry[1].url" target="_blank" rel="noopener">{{ entry[1].url }}</a></td>
|
||||
<td>
|
||||
@if (editingTitleRegexId === entry[0]) {
|
||||
|
||||
+70
-1
@@ -19,6 +19,7 @@ class DownloadsServiceStub {
|
||||
customDirsChanged = new Subject<Record<string, string[]>>();
|
||||
ytdlOptionsChanged = new Subject<Record<string, unknown>>();
|
||||
updated = new Subject<void>();
|
||||
retryCalls: string[] = [];
|
||||
|
||||
getCookieStatus() {
|
||||
return of({ status: 'ok', has_cookies: false });
|
||||
@@ -32,6 +33,11 @@ class DownloadsServiceStub {
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
retry(id: string) {
|
||||
this.retryCalls.push(id);
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
cancelAdd() {
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
@@ -75,7 +81,10 @@ class SubscriptionsServiceStub {
|
||||
return of({});
|
||||
}
|
||||
|
||||
update() {
|
||||
updateCalls: [string, unknown][] = [];
|
||||
|
||||
update(id: string, changes: unknown) {
|
||||
this.updateCalls.push([id, changes]);
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
@@ -269,6 +278,33 @@ describe('App', () => {
|
||||
expect(payload.clipEnd).toBe('1:20');
|
||||
});
|
||||
|
||||
it('retries a failed download by its server-side queue id', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const download = {
|
||||
id: 'vid1',
|
||||
title: 'Test Video',
|
||||
url: 'https://example.com/v',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'error',
|
||||
msg: 'temporary failure',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: false,
|
||||
};
|
||||
|
||||
app.retryDownload(download.url, download);
|
||||
|
||||
expect(downloads.retryCalls).toEqual([download.url]);
|
||||
});
|
||||
|
||||
it('blocks subscribe with invalid title regex', () => {
|
||||
const toasts = TestBed.inject(ToastService);
|
||||
const errorSpy = vi.spyOn(toasts, 'error').mockImplementation(() => undefined);
|
||||
@@ -282,4 +318,37 @@ describe('App', () => {
|
||||
expect(errorSpy).toHaveBeenCalledWith('Invalid subscription title filter (regex)');
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
|
||||
it('renames a subscription and closes the inline editor', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
|
||||
app.beginEditName('sub1', 'Videos');
|
||||
expect(app.editingNameId).toBe('sub1');
|
||||
expect(app.nameEditDraft).toBe('Videos');
|
||||
|
||||
app.nameEditDraft = ' Jane uploads ';
|
||||
app.saveName('sub1');
|
||||
|
||||
expect(subs.updateCalls).toEqual([['sub1', { name: 'Jane uploads' }]]);
|
||||
expect(app.editingNameId).toBeNull();
|
||||
});
|
||||
|
||||
it('blocks renaming a subscription to an empty name', () => {
|
||||
const toasts = TestBed.inject(ToastService);
|
||||
const errorSpy = vi.spyOn(toasts, 'error').mockImplementation(() => undefined);
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
|
||||
app.beginEditName('sub1', 'Videos');
|
||||
app.nameEditDraft = ' ';
|
||||
app.saveName('sub1');
|
||||
|
||||
expect(subs.updateCalls.length).toBe(0);
|
||||
expect(app.editingNameId).toBe('sub1');
|
||||
expect(errorSpy).toHaveBeenCalledWith('Subscription name must not be empty');
|
||||
errorSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
+32
-22
@@ -102,6 +102,9 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
||||
skipSubscriberOnly = false;
|
||||
editingTitleRegexId: string | null = null;
|
||||
titleRegexEditDraft = '';
|
||||
editingNameId: string | null = null;
|
||||
nameEditDraft = '';
|
||||
readonly subscriptionNameMaxLength = 200;
|
||||
cachedSubs: [string, SubscriptionRow][] = [];
|
||||
selectedSubscriptionIds = new Set<string>();
|
||||
checkingSubscriptionIds = new Set<string>();
|
||||
@@ -663,6 +666,34 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
||||
});
|
||||
}
|
||||
|
||||
beginEditName(id: string, current: string | undefined) {
|
||||
this.editingNameId = id;
|
||||
this.nameEditDraft = current ?? '';
|
||||
this.cdr.markForCheck();
|
||||
}
|
||||
|
||||
cancelEditName() {
|
||||
this.editingNameId = null;
|
||||
this.nameEditDraft = '';
|
||||
this.cdr.markForCheck();
|
||||
}
|
||||
|
||||
saveName(id: string) {
|
||||
const name = (this.nameEditDraft || '').trim();
|
||||
if (!name) {
|
||||
this.toasts.error('Subscription name must not be empty');
|
||||
return;
|
||||
}
|
||||
this.subscriptionsSvc.update(id, { name }).subscribe((res) => {
|
||||
const error = this.getStatusError(res);
|
||||
if (error) {
|
||||
this.toasts.error(error || 'Update subscription failed');
|
||||
return;
|
||||
}
|
||||
this.cancelEditName();
|
||||
});
|
||||
}
|
||||
|
||||
deleteSubscription(id: string) {
|
||||
this.subscriptionsSvc.delete([id]).subscribe((res) => {
|
||||
const error = this.getStatusError(res);
|
||||
@@ -1146,30 +1177,9 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
||||
}
|
||||
|
||||
retryDownload(key: string, download: Download) {
|
||||
const payload = this.buildAddPayload({
|
||||
url: download.url,
|
||||
downloadType: download.download_type,
|
||||
codec: download.codec,
|
||||
quality: download.quality,
|
||||
format: download.format,
|
||||
folder: download.folder,
|
||||
customNamePrefix: download.custom_name_prefix,
|
||||
playlistItemLimit: download.playlist_item_limit,
|
||||
autoStart: true,
|
||||
splitByChapters: download.split_by_chapters,
|
||||
chapterTemplate: download.chapter_template,
|
||||
subtitleLanguage: download.subtitle_language,
|
||||
subtitleMode: download.subtitle_mode,
|
||||
ytdlOptionsPresets: download.ytdl_options_presets?.length
|
||||
? [...download.ytdl_options_presets]
|
||||
: [],
|
||||
ytdlOptionsOverrides: download.ytdl_options_overrides ? JSON.stringify(download.ytdl_options_overrides) : '',
|
||||
clipStart: download.clip_start != null ? String(download.clip_start) : '',
|
||||
clipEnd: download.clip_end != null ? String(download.clip_end) : '',
|
||||
});
|
||||
// Only remove the done-list record once the retry is confirmed queued —
|
||||
// deleting it eagerly would silently lose history if the re-add fails.
|
||||
this.downloads.add(payload)
|
||||
this.downloads.retry(key)
|
||||
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||
.subscribe((status: Status) => {
|
||||
if (status.status === 'error') {
|
||||
|
||||
@@ -117,6 +117,14 @@ describe('DownloadsService', () => {
|
||||
req.flush({ presets: ['Preset A'] });
|
||||
});
|
||||
|
||||
it('retry() posts the failed download id', () => {
|
||||
service.retry('https://example.com/v').subscribe();
|
||||
const req = httpMock.expectOne('retry');
|
||||
expect(req.request.method).toBe('POST');
|
||||
expect(req.request.body).toEqual({ id: 'https://example.com/v' });
|
||||
req.flush({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('cancelAdd posts to cancel-add', () => {
|
||||
service.cancelAdd().subscribe();
|
||||
const req = httpMock.expectOne('cancel-add');
|
||||
|
||||
@@ -169,6 +169,12 @@ export class DownloadsService {
|
||||
);
|
||||
}
|
||||
|
||||
public retry(id: string) {
|
||||
return this.http.post<Status>('retry', { id: id }).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
|
||||
public startById(ids: string[]) {
|
||||
return this.http.post<Status>('start', {ids: ids}).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
|
||||
Reference in New Issue
Block a user