mirror of
https://github.com/alexta69/metube.git
synced 2026-09-21 21:45:04 +00:00
Compare commits
23 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c9c507f939 | |||
| 327e1eb4b8 | |||
| 82e966caaf | |||
| 346da19108 | |||
| b74185b2af | |||
| c393e0195b | |||
| 86954784fd | |||
| 72e8f5031f | |||
| f3c464fad5 | |||
| b10bb6103a | |||
| 8c2990e68a | |||
| ac46fff6d9 | |||
| 05c21326b3 | |||
| 99da62dcbb | |||
| d0ad36baad | |||
| d2095caea2 | |||
| e15aff3339 | |||
| fccd207799 | |||
| 6461924bf8 | |||
| c68fcaddd1 | |||
| a4454ac460 | |||
| 75fe1f0c11 | |||
| 5826d0dc2b |
@@ -62,6 +62,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
|||||||
* __CUSTOM_DIRS__: Whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a dropdown appears next to the Add button to specify the download directory. Defaults to `true`.
|
* __CUSTOM_DIRS__: Whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a dropdown appears next to the Add button to specify the download directory. Defaults to `true`.
|
||||||
* __CREATE_CUSTOM_DIRS__: Whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector supports free-text input, and the specified directory will be created recursively. Defaults to `true`.
|
* __CREATE_CUSTOM_DIRS__: Whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector supports free-text input, and the specified directory will be created recursively. Defaults to `true`.
|
||||||
* __CUSTOM_DIRS_EXCLUDE_REGEX__: Regular expression to exclude some custom directories from the dropdown. Empty regex disables exclusion. Defaults to `(^|/)[.@].*$`, which means directories starting with `.` or `@`.
|
* __CUSTOM_DIRS_EXCLUDE_REGEX__: Regular expression to exclude some custom directories from the dropdown. Empty regex disables exclusion. Defaults to `(^|/)[.@].*$`, which means directories starting with `.` or `@`.
|
||||||
|
* __DEFAULT_FOLDER__: Custom directory to pre-select in the download folder field, relative to __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__), for when most downloads go to the same place. It is only a starting value — the field stays editable, so any other folder can still be picked per download. Requires __CUSTOM_DIRS__; ignored with a warning otherwise. Defaults to empty, i.e. the base download directory.
|
||||||
* __DOWNLOAD_DIRS_INDEXABLE__: If `true`, the download directories (__DOWNLOAD_DIR__ and __AUDIO_DOWNLOAD_DIR__) are indexable on the web server. Defaults to `false`.
|
* __DOWNLOAD_DIRS_INDEXABLE__: If `true`, the download directories (__DOWNLOAD_DIR__ and __AUDIO_DOWNLOAD_DIR__) are indexable on the web server. Defaults to `false`.
|
||||||
* __STATE_DIR__: Path to where MeTube will store its persistent state files (`queue.json`, `pending.json`, `completed.json`, `subscriptions.json`). Defaults to `/downloads/.metube` in the Docker image, and `.` otherwise.
|
* __STATE_DIR__: Path to where MeTube will store its persistent state files (`queue.json`, `pending.json`, `completed.json`, `subscriptions.json`). Defaults to `/downloads/.metube` in the Docker image, and `.` otherwise.
|
||||||
* __TEMP_DIR__: Path where intermediary download files will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise.
|
* __TEMP_DIR__: Path where intermediary download files will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise.
|
||||||
@@ -83,6 +84,8 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
|||||||
* __ALLOW_PRIVATE_ADDRESSES__: Whether to allow downloads from private, loopback, link-local and other non-global addresses. Defaults to `false`, which protects against SSRF by refusing URLs that resolve to internal hosts. Set to `true` only in trusted environments — for example when routing traffic through a proxy/VPN client in Fake-IP mode (sing-box, Clash, Mihomo), which resolves hosts to the `198.18.0.0/15` range. Enabling this disables the SSRF protection entirely, so only use it when you control the network. You do **not** need this to use a proxy on an internal address: a proxy configured through the `proxy` option in `YTDL_OPTIONS` (or the `*_proxy` environment variables) is always reachable at its own host and port, wherever it lives.
|
* __ALLOW_PRIVATE_ADDRESSES__: Whether to allow downloads from private, loopback, link-local and other non-global addresses. Defaults to `false`, which protects against SSRF by refusing URLs that resolve to internal hosts. Set to `true` only in trusted environments — for example when routing traffic through a proxy/VPN client in Fake-IP mode (sing-box, Clash, Mihomo), which resolves hosts to the `198.18.0.0/15` range. Enabling this disables the SSRF protection entirely, so only use it when you control the network. You do **not** need this to use a proxy on an internal address: a proxy configured through the `proxy` option in `YTDL_OPTIONS` (or the `*_proxy` environment variables) is always reachable at its own host and port, wherever it lives.
|
||||||
* __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled).
|
* __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled).
|
||||||
|
|
||||||
|
A filename that would exceed the limit the filesystem accepts is shortened to fit, keeping its extension, with room left for the suffixes yt-dlp adds while downloading. Sites that put a long description in the title would otherwise fail the download outright with `File name too long`. Use `trim_file_name` in `YTDL_OPTIONS` if you want names shorter than the filesystem's own limit, or `restrictfilenames` to strip non-ASCII characters.
|
||||||
|
|
||||||
Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a feed-level `.info.json` and thumbnail when you add a playlist or channel. These reuse the template of the items they belong to — `OUTPUT_TEMPLATE_CHANNEL` or `OUTPUT_TEMPLATE_PLAYLIST` — evaluated against the feed itself, so with the defaults they land in the same folder as the videos, named after the feed. Set `allow_playlist_files` to `false` in `YTDL_OPTIONS` to skip them.
|
Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a feed-level `.info.json` and thumbnail when you add a playlist or channel. These reuse the template of the items they belong to — `OUTPUT_TEMPLATE_CHANNEL` or `OUTPUT_TEMPLATE_PLAYLIST` — evaluated against the feed itself, so with the defaults they land in the same folder as the videos, named after the feed. Set `allow_playlist_files` to `false` in `YTDL_OPTIONS` to skip them.
|
||||||
|
|
||||||
### 🌐 Web Server & URLs
|
### 🌐 Web Server & URLs
|
||||||
@@ -95,7 +98,7 @@ Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a fee
|
|||||||
* __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`.
|
* __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`.
|
||||||
* __CERTFILE__: HTTPS certificate file path.
|
* __CERTFILE__: HTTPS certificate file path.
|
||||||
* __KEYFILE__: HTTPS key file path.
|
* __KEYFILE__: HTTPS key file path.
|
||||||
* __CORS_ALLOWED_ORIGINS__: Comma-separated list of origins permitted to make cross-origin requests to the MeTube API; `*` allows all. When unset or empty, all cross-origin requests are denied. Required for browser extensions and bookmarklets — see [Sending links to MeTube](#-sending-links-to-metube).
|
* __CORS_ALLOWED_ORIGINS__: Comma-separated list of origins permitted to make cross-origin requests to the MeTube API; `*` allows all. When unset or empty, all cross-origin requests are denied. Required for browser extensions and bookmarklets — see [Sending links to MeTube](#-sending-links-to-metube). Naming origins explicitly also lets them send credentials (a login cookie, or the `Authorization` header a reverse proxy checks), which `*` deliberately does not: it would let any site you visit drive your instance with your own session.
|
||||||
* __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container.
|
* __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container.
|
||||||
|
|
||||||
## 🎛️ Configuring yt-dlp options
|
## 🎛️ Configuring yt-dlp options
|
||||||
@@ -240,7 +243,7 @@ __Browser extensions__ allow right-clicking videos and sending them directly to
|
|||||||
* __Chrome:__ contributed by [Rpsl](https://github.com/rpsl) — install from the [Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or [from sources](https://github.com/Rpsl/metube-browser-extension).
|
* __Chrome:__ contributed by [Rpsl](https://github.com/rpsl) — install from the [Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or [from sources](https://github.com/Rpsl/metube-browser-extension).
|
||||||
* __Firefox:__ contributed by [nanocortex](https://github.com/nanocortex) — install from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources [here](https://github.com/nanocortex/metube-firefox-addon).
|
* __Firefox:__ contributed by [nanocortex](https://github.com/nanocortex) — install from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources [here](https://github.com/nanocortex/metube-firefox-addon).
|
||||||
|
|
||||||
__Bookmarklets__ send the currently open page to MeTube with one click. Add the origins of the sites where you use them to `CORS_ALLOWED_ORIGINS`, e.g. `https://www.youtube.com,https://www.vimeo.com`. The code (Chrome and Firefox variants, contributed by [kushfest](https://github.com/kushfest) and [shoonya75](https://github.com/shoonya75)) is in the [Bookmarklets wiki page](https://github.com/alexta69/metube/wiki/Bookmarklets).
|
__Bookmarklets__ send the currently open page to MeTube with one click. Add the origins of the sites where you use them to `CORS_ALLOWED_ORIGINS`, e.g. `https://www.youtube.com,https://www.vimeo.com`. If your instance sits behind authentication, list the origins individually rather than using `*` — only named origins are allowed to send credentials. The code (Chrome and Firefox variants, contributed by [kushfest](https://github.com/kushfest) and [shoonya75](https://github.com/shoonya75)) is in the [Bookmarklets wiki page](https://github.com/alexta69/metube/wiki/Bookmarklets).
|
||||||
|
|
||||||
__iOS Shortcut:__ [rithask](https://github.com/rithask) created an [iOS shortcut](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6) for sending URLs to MeTube from Safari's share menu; it prompts for your instance address on first use.
|
__iOS Shortcut:__ [rithask](https://github.com/rithask) created an [iOS shortcut](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6) for sending URLs to MeTube from Safari's share menu; it prompts for your instance address on first use.
|
||||||
|
|
||||||
|
|||||||
+99
-3
@@ -62,6 +62,7 @@ class Config:
|
|||||||
'CUSTOM_DIRS': 'true',
|
'CUSTOM_DIRS': 'true',
|
||||||
'CREATE_CUSTOM_DIRS': 'true',
|
'CREATE_CUSTOM_DIRS': 'true',
|
||||||
'CUSTOM_DIRS_EXCLUDE_REGEX': r'(^|/)[.@].*$',
|
'CUSTOM_DIRS_EXCLUDE_REGEX': r'(^|/)[.@].*$',
|
||||||
|
'DEFAULT_FOLDER': '',
|
||||||
'DELETE_FILE_ON_TRASHCAN': 'false',
|
'DELETE_FILE_ON_TRASHCAN': 'false',
|
||||||
'STATE_DIR': '.',
|
'STATE_DIR': '.',
|
||||||
'URL_PREFIX': '',
|
'URL_PREFIX': '',
|
||||||
@@ -115,6 +116,17 @@ class Config:
|
|||||||
if not self.URL_PREFIX.endswith('/'):
|
if not self.URL_PREFIX.endswith('/'):
|
||||||
self.URL_PREFIX += '/'
|
self.URL_PREFIX += '/'
|
||||||
|
|
||||||
|
# Strip trailing slashes from the download directories. get_custom_dirs()
|
||||||
|
# builds the folder dropdown by removing the base path as a prefix from
|
||||||
|
# each subdirectory, and the base directory's own path does not carry the
|
||||||
|
# trailing slash — so 'DOWNLOAD_DIR=/downloads/' failed to match itself
|
||||||
|
# and leaked 'downloads' into the dropdown as a bogus folder option.
|
||||||
|
# Runs after the '%%' indirection above so AUDIO_DOWNLOAD_DIR is resolved.
|
||||||
|
for attr in ('DOWNLOAD_DIR', 'AUDIO_DOWNLOAD_DIR', 'TEMP_DIR', 'STATE_DIR'):
|
||||||
|
val = getattr(self, attr)
|
||||||
|
if isinstance(val, str) and len(val) > 1 and val.endswith('/'):
|
||||||
|
setattr(self, attr, val.rstrip('/') or '/')
|
||||||
|
|
||||||
# A blank PUBLIC_HOST_AUDIO_URL (e.g. set empty in a compose file) bypasses the
|
# A blank PUBLIC_HOST_AUDIO_URL (e.g. set empty in a compose file) bypasses the
|
||||||
# default via os.environ.get, which would leave audio links root-relative and 404.
|
# default via os.environ.get, which would leave audio links root-relative and 404.
|
||||||
# Fall back to the 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR. When
|
# Fall back to the 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR. When
|
||||||
@@ -127,6 +139,18 @@ class Config:
|
|||||||
if val and not val.endswith('/'):
|
if val and not val.endswith('/'):
|
||||||
setattr(self, attr, val + '/')
|
setattr(self, attr, val + '/')
|
||||||
|
|
||||||
|
# DEFAULT_FOLDER only pre-fills the form's folder field, which the UI
|
||||||
|
# does not even show without CUSTOM_DIRS. Sending one anyway would fail
|
||||||
|
# every download on the server's own folder check, so drop it and say so
|
||||||
|
# rather than leaving the user with a form that cannot submit.
|
||||||
|
self.DEFAULT_FOLDER = self.DEFAULT_FOLDER.strip().strip('/')
|
||||||
|
if self.DEFAULT_FOLDER and not self.CUSTOM_DIRS:
|
||||||
|
log.warning(
|
||||||
|
'Ignoring DEFAULT_FOLDER "%s" because CUSTOM_DIRS is not enabled',
|
||||||
|
self.DEFAULT_FOLDER,
|
||||||
|
)
|
||||||
|
self.DEFAULT_FOLDER = ''
|
||||||
|
|
||||||
# Convert relative addresses to absolute addresses to prevent the failure of file address comparison
|
# Convert relative addresses to absolute addresses to prevent the failure of file address comparison
|
||||||
if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'):
|
if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'):
|
||||||
self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve())
|
self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve())
|
||||||
@@ -187,6 +211,7 @@ class Config:
|
|||||||
_FRONTEND_KEYS = (
|
_FRONTEND_KEYS = (
|
||||||
'CUSTOM_DIRS',
|
'CUSTOM_DIRS',
|
||||||
'CREATE_CUSTOM_DIRS',
|
'CREATE_CUSTOM_DIRS',
|
||||||
|
'DEFAULT_FOLDER',
|
||||||
'OUTPUT_TEMPLATE_CHAPTER',
|
'OUTPUT_TEMPLATE_CHAPTER',
|
||||||
'PUBLIC_HOST_URL',
|
'PUBLIC_HOST_URL',
|
||||||
'PUBLIC_HOST_AUDIO_URL',
|
'PUBLIC_HOST_AUDIO_URL',
|
||||||
@@ -321,6 +346,12 @@ async def state_dir_guard(request, handler):
|
|||||||
|
|
||||||
app = web.Application(middlewares=[state_dir_guard])
|
app = web.Application(middlewares=[state_dir_guard])
|
||||||
_cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
|
_cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
|
||||||
|
if '*' in _cors_origins and len(_cors_origins) > 1:
|
||||||
|
log.warning(
|
||||||
|
"CORS_ALLOWED_ORIGINS mixes '*' with named origins %s. '*' wins, and credentialed "
|
||||||
|
"cross-origin requests stay disabled for every origin in the list. Remove '*' if you "
|
||||||
|
"need a bookmarklet to reach an authenticated instance.",
|
||||||
|
[o for o in _cors_origins if o != '*'])
|
||||||
sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
|
sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
|
||||||
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
||||||
routes = web.RouteTableDef()
|
routes = web.RouteTableDef()
|
||||||
@@ -711,6 +742,7 @@ def parse_download_options(post: dict) -> dict:
|
|||||||
playlist_item_limit = post.get('playlist_item_limit')
|
playlist_item_limit = post.get('playlist_item_limit')
|
||||||
auto_start = post.get('auto_start')
|
auto_start = post.get('auto_start')
|
||||||
split_by_chapters = post.get('split_by_chapters')
|
split_by_chapters = post.get('split_by_chapters')
|
||||||
|
sponsorblock = bool(post.get('sponsorblock'))
|
||||||
chapter_template = post.get('chapter_template')
|
chapter_template = post.get('chapter_template')
|
||||||
subtitle_language = post.get('subtitle_language')
|
subtitle_language = post.get('subtitle_language')
|
||||||
subtitle_mode = post.get('subtitle_mode')
|
subtitle_mode = post.get('subtitle_mode')
|
||||||
@@ -831,6 +863,7 @@ def parse_download_options(post: dict) -> dict:
|
|||||||
'playlist_item_limit': playlist_item_limit,
|
'playlist_item_limit': playlist_item_limit,
|
||||||
'auto_start': auto_start,
|
'auto_start': auto_start,
|
||||||
'split_by_chapters': split_by_chapters,
|
'split_by_chapters': split_by_chapters,
|
||||||
|
'sponsorblock': sponsorblock,
|
||||||
'chapter_template': chapter_template,
|
'chapter_template': chapter_template,
|
||||||
'subtitle_language': subtitle_language,
|
'subtitle_language': subtitle_language,
|
||||||
'subtitle_mode': subtitle_mode,
|
'subtitle_mode': subtitle_mode,
|
||||||
@@ -876,6 +909,7 @@ async def add(request):
|
|||||||
o['ytdl_options_overrides'],
|
o['ytdl_options_overrides'],
|
||||||
o['clip_start'],
|
o['clip_start'],
|
||||||
o['clip_end'],
|
o['clip_end'],
|
||||||
|
sponsorblock=o['sponsorblock'],
|
||||||
)
|
)
|
||||||
return web.Response(text=serializer.encode(status))
|
return web.Response(text=serializer.encode(status))
|
||||||
|
|
||||||
@@ -956,6 +990,7 @@ async def subscribe(request):
|
|||||||
subtitle_mode=o['subtitle_mode'],
|
subtitle_mode=o['subtitle_mode'],
|
||||||
ytdl_options_presets=o['ytdl_options_presets'],
|
ytdl_options_presets=o['ytdl_options_presets'],
|
||||||
ytdl_options_overrides=o['ytdl_options_overrides'],
|
ytdl_options_overrides=o['ytdl_options_overrides'],
|
||||||
|
sponsorblock=o['sponsorblock'],
|
||||||
title_regex=post.get('title_regex'),
|
title_regex=post.get('title_regex'),
|
||||||
skip_subscriber_only=skip_subscriber_only,
|
skip_subscriber_only=skip_subscriber_only,
|
||||||
clip_start=sub_clip_start,
|
clip_start=sub_clip_start,
|
||||||
@@ -1045,6 +1080,30 @@ async def start(request):
|
|||||||
|
|
||||||
COOKIES_PATH = os.path.join(config.STATE_DIR, 'cookies.txt')
|
COOKIES_PATH = os.path.join(config.STATE_DIR, 'cookies.txt')
|
||||||
|
|
||||||
|
|
||||||
|
def warn_if_cookiefile_shadowed():
|
||||||
|
"""Warn before an uploaded cookies file displaces an operator-configured one.
|
||||||
|
|
||||||
|
Uploaded cookies deliberately win: the upload exists so cookies can be
|
||||||
|
refreshed without restarting the container, and letting YTDL_OPTIONS win
|
||||||
|
would leave a visible UI button doing nothing. But set_runtime_override
|
||||||
|
writes straight into YTDL_OPTIONS, so the configured path is gone from the
|
||||||
|
live config the moment an uploaded file is applied — after that, nothing
|
||||||
|
downstream can report the conflict (delete_cookies' has_manual_cookiefile
|
||||||
|
check cannot fire once the value has been replaced). This is the only point
|
||||||
|
where both are still visible, so it is the only place the warning can be
|
||||||
|
issued. Must be called before set_runtime_override. See issue #881, where
|
||||||
|
the silence cost the reporter days of debugging.
|
||||||
|
"""
|
||||||
|
configured = config.YTDL_OPTIONS.get('cookiefile')
|
||||||
|
if isinstance(configured, str) and configured and configured != COOKIES_PATH:
|
||||||
|
log.warning(
|
||||||
|
'Uploaded cookies at %s take precedence over the cookiefile configured in '
|
||||||
|
'YTDL_OPTIONS (%s), which will not be used. Delete the uploaded cookies from '
|
||||||
|
'the UI to go back to the configured file.',
|
||||||
|
COOKIES_PATH, configured)
|
||||||
|
|
||||||
|
|
||||||
@routes.post(config.URL_PREFIX + 'upload-cookies')
|
@routes.post(config.URL_PREFIX + 'upload-cookies')
|
||||||
async def upload_cookies(request):
|
async def upload_cookies(request):
|
||||||
reader = await request.multipart()
|
reader = await request.multipart()
|
||||||
@@ -1074,6 +1133,7 @@ async def upload_cookies(request):
|
|||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
log.warning(f'Could not restrict permissions on cookies file: {exc}')
|
log.warning(f'Could not restrict permissions on cookies file: {exc}')
|
||||||
os.replace(tmp_cookie_path, COOKIES_PATH)
|
os.replace(tmp_cookie_path, COOKIES_PATH)
|
||||||
|
warn_if_cookiefile_shadowed()
|
||||||
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
||||||
log.info(f'Cookies file uploaded ({size} bytes)')
|
log.info(f'Cookies file uploaded ({size} bytes)')
|
||||||
return web.Response(text=serializer.encode({'status': 'ok', 'msg': f'Cookies uploaded ({size} bytes)'}))
|
return web.Response(text=serializer.encode({'status': 'ok', 'msg': f'Cookies uploaded ({size} bytes)'}))
|
||||||
@@ -1268,9 +1328,44 @@ app.router.add_route('OPTIONS', config.URL_PREFIX + 'delete-cookies', add_cors)
|
|||||||
|
|
||||||
async def on_prepare(request, response):
|
async def on_prepare(request, response):
|
||||||
origin = request.headers.get('Origin')
|
origin = request.headers.get('Origin')
|
||||||
if origin and _cors_origins and ('*' in _cors_origins or origin in _cors_origins):
|
if not origin or not _cors_origins:
|
||||||
response.headers['Access-Control-Allow-Origin'] = origin
|
return
|
||||||
response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
|
|
||||||
|
# Naming an origin in CORS_ALLOWED_ORIGINS is a deliberate trust grant, so
|
||||||
|
# such an origin may send credentials: the cookie or Authorization header
|
||||||
|
# that a reverse proxy in front of MeTube checks. Without this a bookmarklet
|
||||||
|
# cannot reach an authenticated instance at all (issue #155).
|
||||||
|
#
|
||||||
|
# The '*' wildcard is emphatically not such a grant — it matches origins the
|
||||||
|
# operator never enumerated, including every site the user happens to visit.
|
||||||
|
# Echoing the origin back (which we must do, since '*' is illegal alongside
|
||||||
|
# credentials) and allowing credentials would let any page drive the user's
|
||||||
|
# instance with the user's own session. So the wildcard keeps exactly the
|
||||||
|
# uncredentialed behaviour it has always had, and a wildcard anywhere in the
|
||||||
|
# list disables credentials for every origin in it.
|
||||||
|
#
|
||||||
|
# Derived here rather than held in a second module global so the wildcard
|
||||||
|
# test and the membership test can never disagree about the same list.
|
||||||
|
wildcard = '*' in _cors_origins
|
||||||
|
trusted = not wildcard and origin in _cors_origins
|
||||||
|
if not (wildcard or trusted):
|
||||||
|
return
|
||||||
|
|
||||||
|
response.headers['Access-Control-Allow-Origin'] = origin
|
||||||
|
# Authorization rides on the same grant: allowing it under the wildcard
|
||||||
|
# would let an arbitrary page attempt credentials against an instance it
|
||||||
|
# can already reach, from inside the victim's network.
|
||||||
|
response.headers['Access-Control-Allow-Headers'] = 'Content-Type, Authorization' if trusted else 'Content-Type'
|
||||||
|
if trusted:
|
||||||
|
response.headers['Access-Control-Allow-Credentials'] = 'true'
|
||||||
|
|
||||||
|
# The response now differs per Origin, so a shared cache must not hand one
|
||||||
|
# origin's Allow-Origin to another.
|
||||||
|
vary = response.headers.get('Vary')
|
||||||
|
if not vary:
|
||||||
|
response.headers['Vary'] = 'Origin'
|
||||||
|
elif 'origin' not in (v.strip().lower() for v in vary.split(',')):
|
||||||
|
response.headers['Vary'] = f'{vary}, Origin'
|
||||||
|
|
||||||
app.on_response_prepare.append(on_prepare)
|
app.on_response_prepare.append(on_prepare)
|
||||||
|
|
||||||
@@ -1296,6 +1391,7 @@ if __name__ == '__main__':
|
|||||||
|
|
||||||
# Auto-detect cookie file on startup
|
# Auto-detect cookie file on startup
|
||||||
if os.path.exists(COOKIES_PATH):
|
if os.path.exists(COOKIES_PATH):
|
||||||
|
warn_if_cookiefile_shadowed()
|
||||||
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
||||||
log.info(f'Cookie file detected at {COOKIES_PATH}')
|
log.info(f'Cookie file detected at {COOKIES_PATH}')
|
||||||
|
|
||||||
|
|||||||
@@ -182,6 +182,7 @@ class SubscriptionInfo:
|
|||||||
auto_start: bool = True
|
auto_start: bool = True
|
||||||
playlist_item_limit: int = 0
|
playlist_item_limit: int = 0
|
||||||
split_by_chapters: bool = False
|
split_by_chapters: bool = False
|
||||||
|
sponsorblock: bool = False
|
||||||
chapter_template: str = ""
|
chapter_template: str = ""
|
||||||
subtitle_language: str = "en"
|
subtitle_language: str = "en"
|
||||||
subtitle_mode: str = "prefer_manual"
|
subtitle_mode: str = "prefer_manual"
|
||||||
@@ -242,6 +243,7 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]:
|
|||||||
"auto_start": sub.auto_start,
|
"auto_start": sub.auto_start,
|
||||||
"playlist_item_limit": sub.playlist_item_limit,
|
"playlist_item_limit": sub.playlist_item_limit,
|
||||||
"split_by_chapters": sub.split_by_chapters,
|
"split_by_chapters": sub.split_by_chapters,
|
||||||
|
"sponsorblock": sub.sponsorblock,
|
||||||
"chapter_template": sub.chapter_template,
|
"chapter_template": sub.chapter_template,
|
||||||
"subtitle_language": sub.subtitle_language,
|
"subtitle_language": sub.subtitle_language,
|
||||||
"subtitle_mode": sub.subtitle_mode,
|
"subtitle_mode": sub.subtitle_mode,
|
||||||
@@ -487,6 +489,7 @@ class SubscriptionManager:
|
|||||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||||
clip_start: Optional[float] = None,
|
clip_start: Optional[float] = None,
|
||||||
clip_end: Optional[float] = None,
|
clip_end: Optional[float] = None,
|
||||||
|
sponsorblock: bool = False,
|
||||||
) -> tuple[list[str], list[str]]:
|
) -> tuple[list[str], list[str]]:
|
||||||
queued_ids: list[str] = []
|
queued_ids: list[str] = []
|
||||||
queue_errors: list[str] = []
|
queue_errors: list[str] = []
|
||||||
@@ -519,6 +522,7 @@ class SubscriptionManager:
|
|||||||
ytdl_options_overrides,
|
ytdl_options_overrides,
|
||||||
clip_start,
|
clip_start,
|
||||||
clip_end,
|
clip_end,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
if isinstance(result, dict) and result.get("status") == "error":
|
if isinstance(result, dict) and result.get("status") == "error":
|
||||||
msg = str(result.get("msg") or f"Queueing failed for {vurl}")
|
msg = str(result.get("msg") or f"Queueing failed for {vurl}")
|
||||||
@@ -606,6 +610,7 @@ class SubscriptionManager:
|
|||||||
subtitle_mode: str,
|
subtitle_mode: str,
|
||||||
ytdl_options_presets: Optional[list[str]] = None,
|
ytdl_options_presets: Optional[list[str]] = None,
|
||||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||||
|
sponsorblock: bool = False,
|
||||||
title_regex: Any = None,
|
title_regex: Any = None,
|
||||||
skip_subscriber_only: Any = None,
|
skip_subscriber_only: Any = None,
|
||||||
clip_start: Optional[float] = None,
|
clip_start: Optional[float] = None,
|
||||||
@@ -689,6 +694,7 @@ class SubscriptionManager:
|
|||||||
auto_start=bool(auto_start),
|
auto_start=bool(auto_start),
|
||||||
playlist_item_limit=int(playlist_item_limit),
|
playlist_item_limit=int(playlist_item_limit),
|
||||||
split_by_chapters=bool(split_by_chapters),
|
split_by_chapters=bool(split_by_chapters),
|
||||||
|
sponsorblock=bool(sponsorblock),
|
||||||
chapter_template=chapter_template or "",
|
chapter_template=chapter_template or "",
|
||||||
subtitle_language=subtitle_language,
|
subtitle_language=subtitle_language,
|
||||||
subtitle_mode=subtitle_mode,
|
subtitle_mode=subtitle_mode,
|
||||||
@@ -942,6 +948,7 @@ class SubscriptionManager:
|
|||||||
dl_plimit = cur.playlist_item_limit
|
dl_plimit = cur.playlist_item_limit
|
||||||
dl_autostart = cur.auto_start
|
dl_autostart = cur.auto_start
|
||||||
dl_split = cur.split_by_chapters
|
dl_split = cur.split_by_chapters
|
||||||
|
dl_sponsorblock = cur.sponsorblock
|
||||||
dl_chapter = cur.chapter_template
|
dl_chapter = cur.chapter_template
|
||||||
dl_sublang = cur.subtitle_language
|
dl_sublang = cur.subtitle_language
|
||||||
dl_submode = cur.subtitle_mode
|
dl_submode = cur.subtitle_mode
|
||||||
@@ -1010,6 +1017,7 @@ class SubscriptionManager:
|
|||||||
playlist_item_limit=dl_plimit,
|
playlist_item_limit=dl_plimit,
|
||||||
auto_start=dl_autostart,
|
auto_start=dl_autostart,
|
||||||
split_by_chapters=dl_split,
|
split_by_chapters=dl_split,
|
||||||
|
sponsorblock=dl_sponsorblock,
|
||||||
chapter_template=dl_chapter or "",
|
chapter_template=dl_chapter or "",
|
||||||
subtitle_language=dl_sublang,
|
subtitle_language=dl_sublang,
|
||||||
subtitle_mode=dl_submode,
|
subtitle_mode=dl_submode,
|
||||||
|
|||||||
@@ -366,6 +366,25 @@ async def test_subscribe_passes_clip_bounds(mock_dqueue, monkeypatch):
|
|||||||
assert kwargs["clip_end"] == pytest.approx(204.0)
|
assert kwargs["clip_end"] == pytest.approx(204.0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_passes_sponsorblock(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
req = _json_request(
|
||||||
|
{**_valid_video_add_body(), "check_interval_minutes": 60, "sponsorblock": True}
|
||||||
|
)
|
||||||
|
resp = await main.subscribe(req)
|
||||||
|
assert resp.status == 200
|
||||||
|
assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_defaults_sponsorblock_off(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
req = _json_request({**_valid_video_add_body(), "check_interval_minutes": 60})
|
||||||
|
await main.subscribe(req)
|
||||||
|
assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is False
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_subscribe_without_clip_fields_stores_none(mock_dqueue, monkeypatch):
|
async def test_subscribe_without_clip_fields_stores_none(mock_dqueue, monkeypatch):
|
||||||
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
@@ -506,3 +525,130 @@ async def test_download_blocks_state_dir_files(monkeypatch):
|
|||||||
(download_dir / "video.mp4").unlink(missing_ok=True)
|
(download_dir / "video.mp4").unlink(missing_ok=True)
|
||||||
(download_dir / percent_filename).unlink(missing_ok=True)
|
(download_dir / percent_filename).unlink(missing_ok=True)
|
||||||
state_dir.rmdir()
|
state_dir.rmdir()
|
||||||
|
|
||||||
|
# --- CORS (issue #155) -------------------------------------------------------
|
||||||
|
#
|
||||||
|
# The security property under test: credentials are granted only to an origin
|
||||||
|
# the operator named explicitly, and never under the '*' wildcard. Each test
|
||||||
|
# builds a fresh Application because main.app binds to the first event loop
|
||||||
|
# that runs it; the logic under test lives entirely in main.on_prepare, and the
|
||||||
|
# real main.add_cors preflight handler is mounted so the preflight path is the
|
||||||
|
# production one.
|
||||||
|
|
||||||
|
async def _cors_version(request):
|
||||||
|
return web.Response(text="v")
|
||||||
|
|
||||||
|
|
||||||
|
def _cors_app():
|
||||||
|
app = web.Application()
|
||||||
|
app.router.add_route("OPTIONS", "/add", main.add_cors)
|
||||||
|
app.router.add_get("/version", _cors_version)
|
||||||
|
app.on_response_prepare.append(main.on_prepare)
|
||||||
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
async def _cors_headers(monkeypatch, origins, origin, path="/add", method="OPTIONS"):
|
||||||
|
monkeypatch.setattr(main, "_cors_origins", origins)
|
||||||
|
async with TestClient(TestServer(_cors_app())) as client:
|
||||||
|
resp = await client.request(
|
||||||
|
method, path,
|
||||||
|
headers={
|
||||||
|
"Origin": origin,
|
||||||
|
"Access-Control-Request-Method": "POST",
|
||||||
|
"Access-Control-Request-Headers": "content-type,authorization",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return resp.headers
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_listed_origin_gets_credentials(monkeypatch):
|
||||||
|
h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "https://www.youtube.com")
|
||||||
|
assert h["Access-Control-Allow-Origin"] == "https://www.youtube.com"
|
||||||
|
assert h["Access-Control-Allow-Credentials"] == "true"
|
||||||
|
assert "Authorization" in h["Access-Control-Allow-Headers"]
|
||||||
|
assert "Origin" in h["Vary"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_wildcard_never_grants_credentials(monkeypatch):
|
||||||
|
h = await _cors_headers(monkeypatch, ["*"], "https://evil.example")
|
||||||
|
# The wildcard still reflects the origin, exactly as before...
|
||||||
|
assert h["Access-Control-Allow-Origin"] == "https://evil.example"
|
||||||
|
# ...but must not hand out the user's session, nor let a page attempt
|
||||||
|
# credentials of its own.
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
assert h["Access-Control-Allow-Headers"] == "Content-Type"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_wildcard_mixed_with_named_origin_still_denies_credentials(monkeypatch):
|
||||||
|
# '*' anywhere in the list disables credentials for everyone in it, so a
|
||||||
|
# stray wildcard cannot silently widen a named grant.
|
||||||
|
h = await _cors_headers(monkeypatch, ["*", "https://www.youtube.com"], "https://www.youtube.com")
|
||||||
|
assert h["Access-Control-Allow-Origin"] == "https://www.youtube.com"
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
assert h["Access-Control-Allow-Headers"] == "Content-Type"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_unlisted_origin_gets_nothing(monkeypatch):
|
||||||
|
h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "https://evil.example")
|
||||||
|
assert "Access-Control-Allow-Origin" not in h
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_disabled_by_default(monkeypatch):
|
||||||
|
h = await _cors_headers(monkeypatch, [], "https://www.youtube.com")
|
||||||
|
assert "Access-Control-Allow-Origin" not in h
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_credentials_apply_to_actual_response_not_just_preflight(monkeypatch):
|
||||||
|
# The browser checks Allow-Credentials on the real response too, so a
|
||||||
|
# preflight-only grant would still fail.
|
||||||
|
h = await _cors_headers(
|
||||||
|
monkeypatch, ["https://www.youtube.com"], "https://www.youtube.com",
|
||||||
|
path="/version", method="GET")
|
||||||
|
assert h["Access-Control-Allow-Credentials"] == "true"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_origin_match_is_exact(monkeypatch):
|
||||||
|
# Substring or suffix matching here would be a bypass.
|
||||||
|
for impostor in (
|
||||||
|
"https://www.youtube.com.evil.example",
|
||||||
|
"https://evilwww.youtube.com",
|
||||||
|
"http://www.youtube.com",
|
||||||
|
"https://www.youtube.com:8443",
|
||||||
|
):
|
||||||
|
h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], impostor)
|
||||||
|
assert "Access-Control-Allow-Origin" not in h, impostor
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h, impostor
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_null_origin_is_not_trusted(monkeypatch):
|
||||||
|
# Sandboxed iframes and some file:// contexts send Origin: null.
|
||||||
|
h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "null")
|
||||||
|
assert "Access-Control-Allow-Origin" not in h
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_vary_appends_to_existing_value(monkeypatch):
|
||||||
|
# Static responses can already carry a Vary; clobbering it would break
|
||||||
|
# content negotiation.
|
||||||
|
monkeypatch.setattr(main, "_cors_origins", ["https://www.youtube.com"])
|
||||||
|
|
||||||
|
async def handler(request):
|
||||||
|
return web.Response(text="x", headers={"Vary": "Accept-Encoding"})
|
||||||
|
|
||||||
|
app = web.Application()
|
||||||
|
app.router.add_get("/v", handler)
|
||||||
|
app.on_response_prepare.append(main.on_prepare)
|
||||||
|
async with TestClient(TestServer(app)) as client:
|
||||||
|
resp = await client.get("/v", headers={"Origin": "https://www.youtube.com"})
|
||||||
|
assert resp.headers["Vary"] == "Accept-Encoding, Origin"
|
||||||
|
|||||||
@@ -77,6 +77,34 @@ class ConfigTests(unittest.TestCase):
|
|||||||
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
||||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
|
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
|
||||||
|
|
||||||
|
def test_download_dirs_lose_trailing_slash(self):
|
||||||
|
# get_custom_dirs strips the base path as a prefix from each subdirectory,
|
||||||
|
# and the base directory's own path has no trailing slash -- so a trailing
|
||||||
|
# slash here leaked the absolute path into the folder dropdown.
|
||||||
|
with patch.dict(os.environ, _base_env(
|
||||||
|
DOWNLOAD_DIR="/downloads/",
|
||||||
|
AUDIO_DOWNLOAD_DIR="/audio/",
|
||||||
|
TEMP_DIR="/tmp/",
|
||||||
|
STATE_DIR="/state/",
|
||||||
|
), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DOWNLOAD_DIR, "/downloads")
|
||||||
|
self.assertEqual(c.AUDIO_DOWNLOAD_DIR, "/audio")
|
||||||
|
self.assertEqual(c.TEMP_DIR, "/tmp")
|
||||||
|
self.assertEqual(c.STATE_DIR, "/state")
|
||||||
|
|
||||||
|
def test_root_download_dir_survives_normalisation(self):
|
||||||
|
with patch.dict(os.environ, _base_env(DOWNLOAD_DIR="/", AUDIO_DOWNLOAD_DIR="///"), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DOWNLOAD_DIR, "/")
|
||||||
|
self.assertEqual(c.AUDIO_DOWNLOAD_DIR, "/")
|
||||||
|
|
||||||
|
def test_download_dirs_without_trailing_slash_unchanged(self):
|
||||||
|
with patch.dict(os.environ, _base_env(DOWNLOAD_DIR="/downloads", AUDIO_DOWNLOAD_DIR="."), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DOWNLOAD_DIR, "/downloads")
|
||||||
|
self.assertEqual(c.AUDIO_DOWNLOAD_DIR, ".")
|
||||||
|
|
||||||
def test_ytdl_options_json_loaded(self):
|
def test_ytdl_options_json_loaded(self):
|
||||||
opts = {"quiet": True, "no_warnings": True}
|
opts = {"quiet": True, "no_warnings": True}
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
@@ -115,6 +143,28 @@ class ConfigTests(unittest.TestCase):
|
|||||||
self.assertNotIn("HOST", safe)
|
self.assertNotIn("HOST", safe)
|
||||||
self.assertEqual(safe["ALLOW_YTDL_OPTIONS_OVERRIDES"], False)
|
self.assertEqual(safe["ALLOW_YTDL_OPTIONS_OVERRIDES"], False)
|
||||||
|
|
||||||
|
def test_default_folder_empty_by_default(self):
|
||||||
|
with patch.dict(os.environ, _base_env(), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DEFAULT_FOLDER, "")
|
||||||
|
|
||||||
|
def test_default_folder_is_trimmed_and_reaches_the_frontend(self):
|
||||||
|
with patch.dict(os.environ, _base_env(DEFAULT_FOLDER=" /youtube/ "), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DEFAULT_FOLDER, "youtube")
|
||||||
|
self.assertEqual(c.frontend_safe()["DEFAULT_FOLDER"], "youtube")
|
||||||
|
|
||||||
|
def test_default_folder_ignored_without_custom_dirs(self):
|
||||||
|
# The folder field is not shown at all without CUSTOM_DIRS, and sending
|
||||||
|
# a folder anyway is rejected by the download path check.
|
||||||
|
with patch.dict(
|
||||||
|
os.environ,
|
||||||
|
_base_env(DEFAULT_FOLDER="youtube", CUSTOM_DIRS="false"),
|
||||||
|
clear=False,
|
||||||
|
):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DEFAULT_FOLDER, "")
|
||||||
|
|
||||||
def test_allow_ytdl_options_overrides_boolean_loaded(self):
|
def test_allow_ytdl_options_overrides_boolean_loaded(self):
|
||||||
with patch.dict(os.environ, _base_env(ALLOW_YTDL_OPTIONS_OVERRIDES="true"), clear=False):
|
with patch.dict(os.environ, _base_env(ALLOW_YTDL_OPTIONS_OVERRIDES="true"), clear=False):
|
||||||
c = Config()
|
c = Config()
|
||||||
|
|||||||
@@ -330,7 +330,7 @@ async def test_retry_restores_playlist_output_context(dq_env):
|
|||||||
chapter_template="",
|
chapter_template="",
|
||||||
)
|
)
|
||||||
failed_info.status = "error"
|
failed_info.status = "error"
|
||||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, failed_info))
|
await dq.done.put(Download(None, None, None, None, "best", "any", {}, failed_info))
|
||||||
|
|
||||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
return {
|
return {
|
||||||
@@ -389,7 +389,7 @@ async def test_retry_keeps_playlist_context_through_url_indirection(dq_env):
|
|||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
url = "https://example.com/watch?v=1"
|
url = "https://example.com/watch?v=1"
|
||||||
resolved = "https://example.com/resolved?v=1"
|
resolved = "https://example.com/resolved?v=1"
|
||||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url)))
|
await dq.done.put(Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url)))
|
||||||
|
|
||||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
if extracted_url == url:
|
if extracted_url == url:
|
||||||
@@ -427,7 +427,7 @@ async def test_retry_reapplies_current_options_gates(dq_env):
|
|||||||
ytdl_options_presets=["Still There", "Removed Preset"],
|
ytdl_options_presets=["Still There", "Removed Preset"],
|
||||||
ytdl_options_overrides={"paths": {"home": "/etc"}},
|
ytdl_options_overrides={"paths": {"home": "/etc"}},
|
||||||
)
|
)
|
||||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
await dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
||||||
|
|
||||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
return {
|
return {
|
||||||
@@ -457,7 +457,7 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env):
|
|||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
url = "https://example.com/watch?v=1"
|
url = "https://example.com/watch?v=1"
|
||||||
info = _failed_playlist_item(url, ytdl_options_overrides={"writesubtitles": True})
|
info = _failed_playlist_item(url, ytdl_options_overrides={"writesubtitles": True})
|
||||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
await dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
||||||
|
|
||||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
return {
|
return {
|
||||||
@@ -476,6 +476,32 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env):
|
|||||||
assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True}
|
assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_retry_carries_the_sponsorblock_flag(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
url = "https://example.com/watch?v=1"
|
||||||
|
await dq.done.put(
|
||||||
|
Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url, sponsorblock=True))
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
|
return {
|
||||||
|
"_type": "video",
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Test Video",
|
||||||
|
"url": extracted_url,
|
||||||
|
"webpage_url": extracted_url,
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||||
|
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||||
|
result = await dq.retry(url)
|
||||||
|
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert dq.queue.get(url).info.sponsorblock is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
@@ -625,6 +651,82 @@ async def test_playlist_download_not_treated_as_channel(dq_env):
|
|||||||
assert download.output_template.startswith("My Playlist/")
|
assert download.output_template.startswith("My Playlist/")
|
||||||
|
|
||||||
|
|
||||||
|
def _channel_extraction(entry_id, **extra):
|
||||||
|
"""A channel yt-dlp reported as a playlist, addressed by *entry_id*."""
|
||||||
|
return {
|
||||||
|
"_type": "playlist",
|
||||||
|
"id": entry_id,
|
||||||
|
"channel_id": "UCabcd123",
|
||||||
|
"channel": "Odin",
|
||||||
|
"title": "Odin",
|
||||||
|
**extra,
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Salvia Plath - Pondering",
|
||||||
|
"url": "https://example.com/watch?v=1",
|
||||||
|
"webpage_url": "https://example.com/watch?v=1",
|
||||||
|
"channel": "Odin",
|
||||||
|
"upload_date": "20130804",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _add_and_get_template(dq_env, extraction, url):
|
||||||
|
dq_env.OUTPUT_TEMPLATE = "%(channel)s [YT]/%(title)s.%(ext)s"
|
||||||
|
dq_env.OUTPUT_TEMPLATE_CHANNEL = ""
|
||||||
|
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||||
|
|
||||||
|
def fake_extract(self, _url, *_args, **_kwargs):
|
||||||
|
return extraction
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, AsyncMock())
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||||
|
result = await dq.add(url, "video", "auto", "any", "best", "", "", 0, auto_start=False)
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
return dq.pending.get("https://example.com/watch?v=1").output_template
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bare_handle_channel_url_is_treated_as_a_channel(dq_env):
|
||||||
|
"""A channel addressed as /@handle reports its id as the handle, not the
|
||||||
|
channel id, and was falling through to OUTPUT_TEMPLATE_PLAYLIST."""
|
||||||
|
template = await _add_and_get_template(
|
||||||
|
dq_env,
|
||||||
|
_channel_extraction("@odin", uploader_id="@odin"),
|
||||||
|
"https://www.youtube.com/@odin",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert template.startswith("Odin [YT]/")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_legacy_vanity_channel_url_is_treated_as_a_channel(dq_env):
|
||||||
|
"""A legacy /c/Name URL reports the vanity name as its id, while
|
||||||
|
uploader_id is still the handle."""
|
||||||
|
template = await _add_and_get_template(
|
||||||
|
dq_env,
|
||||||
|
_channel_extraction("Odin", uploader_id="@odin"),
|
||||||
|
"https://www.youtube.com/c/Odin",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert template.startswith("Odin [YT]/")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_playlist_with_owner_uploader_id_is_still_a_playlist(dq_env):
|
||||||
|
"""A real playlist carries its owner's channel_id and uploader_id, but its
|
||||||
|
own id matches neither, so it must keep the playlist template."""
|
||||||
|
template = await _add_and_get_template(
|
||||||
|
dq_env,
|
||||||
|
_channel_extraction("PLxyz789", uploader_id="@odin", title="My Playlist"),
|
||||||
|
"https://www.youtube.com/playlist?list=PLxyz789",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert template.startswith("My Playlist/")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_add_merges_global_preset_and_override_options(dq_env):
|
async def test_add_merges_global_preset_and_override_options(dq_env):
|
||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
@@ -1339,9 +1441,9 @@ async def test_post_download_cleanup_clears_filename_on_error(dq_env):
|
|||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
|
download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
|
||||||
dq.queue.put(download)
|
await dq.queue.put(download)
|
||||||
|
|
||||||
dq._post_download_cleanup(download)
|
await dq._post_download_cleanup(download)
|
||||||
|
|
||||||
assert download.info.status == "error"
|
assert download.info.status == "error"
|
||||||
assert download.info.filename is None
|
assert download.info.filename is None
|
||||||
@@ -1354,9 +1456,9 @@ async def test_post_download_cleanup_keeps_captured_subtitles_on_error(dq_env):
|
|||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
|
download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
|
||||||
download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
|
download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
|
||||||
dq.queue.put(download)
|
await dq.queue.put(download)
|
||||||
|
|
||||||
dq._post_download_cleanup(download)
|
await dq._post_download_cleanup(download)
|
||||||
|
|
||||||
assert download.info.status == "error"
|
assert download.info.status == "error"
|
||||||
assert download.info.filename == "en.srt"
|
assert download.info.filename == "en.srt"
|
||||||
@@ -1376,7 +1478,7 @@ async def test_clear_skips_deletion_outside_download_directory(dq_env):
|
|||||||
# A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
|
# A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
|
||||||
escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
|
escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
|
||||||
download = _make_download(dq_env, status="finished", filename=escaping_filename)
|
download = _make_download(dq_env, status="finished", filename=escaping_filename)
|
||||||
dq.done.put(download)
|
await dq.done.put(download)
|
||||||
|
|
||||||
await dq.clear([download.info.url])
|
await dq.clear([download.info.url])
|
||||||
|
|
||||||
|
|||||||
@@ -313,3 +313,40 @@ class GetCustomDirsTests(unittest.TestCase):
|
|||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
||||||
|
|
||||||
|
class WarnIfCookiefileShadowedTests(unittest.TestCase):
|
||||||
|
"""Issue #881: an uploaded cookies file wins over an operator-configured
|
||||||
|
cookiefile, and used to do so with no way for anyone to notice."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._saved = main.config.YTDL_OPTIONS
|
||||||
|
main.config.YTDL_OPTIONS = dict(self._saved)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
main.config.YTDL_OPTIONS = self._saved
|
||||||
|
|
||||||
|
def test_warns_when_a_different_cookiefile_is_configured(self):
|
||||||
|
main.config.YTDL_OPTIONS["cookiefile"] = "/cookies/cookies.txt"
|
||||||
|
with self.assertLogs("main", level="WARNING") as cm:
|
||||||
|
main.warn_if_cookiefile_shadowed()
|
||||||
|
joined = "\n".join(cm.output)
|
||||||
|
self.assertIn("/cookies/cookies.txt", joined)
|
||||||
|
self.assertIn(main.COOKIES_PATH, joined)
|
||||||
|
|
||||||
|
def test_silent_when_no_cookiefile_configured(self):
|
||||||
|
main.config.YTDL_OPTIONS.pop("cookiefile", None)
|
||||||
|
with self.assertNoLogs("main", level="WARNING"):
|
||||||
|
main.warn_if_cookiefile_shadowed()
|
||||||
|
|
||||||
|
def test_silent_when_configured_file_is_the_uploaded_one(self):
|
||||||
|
# The steady state after an upload: re-running must not nag.
|
||||||
|
main.config.YTDL_OPTIONS["cookiefile"] = main.COOKIES_PATH
|
||||||
|
with self.assertNoLogs("main", level="WARNING"):
|
||||||
|
main.warn_if_cookiefile_shadowed()
|
||||||
|
|
||||||
|
def test_silent_on_non_string_or_empty_values(self):
|
||||||
|
for value in (None, "", 0, [], {}):
|
||||||
|
main.config.YTDL_OPTIONS["cookiefile"] = value
|
||||||
|
with self.assertNoLogs("main", level="WARNING"):
|
||||||
|
main.warn_if_cookiefile_shadowed()
|
||||||
|
|||||||
@@ -2,8 +2,11 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
import shelve
|
import shelve
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -69,22 +72,22 @@ def _create_legacy_shelf(path: str, *infos: DownloadInfo) -> None:
|
|||||||
shelf[info.url] = info
|
shelf[info.url] = info
|
||||||
|
|
||||||
|
|
||||||
class PersistentQueueTests(unittest.TestCase):
|
class PersistentQueueTests(unittest.IsolatedAsyncioTestCase):
|
||||||
def test_put_get_delete_roundtrip(self):
|
async def test_put_get_delete_roundtrip(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
dl = _FakeDownload(_make_info("http://a.example"))
|
dl = _FakeDownload(_make_info("http://a.example"))
|
||||||
pq.put(dl)
|
await pq.put(dl)
|
||||||
self.assertTrue(os.path.exists(path + ".json"))
|
self.assertTrue(os.path.exists(path + ".json"))
|
||||||
self.assertTrue(pq.exists("http://a.example"))
|
self.assertTrue(pq.exists("http://a.example"))
|
||||||
self.assertFalse(pq.empty())
|
self.assertFalse(pq.empty())
|
||||||
got = pq.get("http://a.example")
|
got = pq.get("http://a.example")
|
||||||
self.assertEqual(got.info.url, "http://a.example")
|
self.assertEqual(got.info.url, "http://a.example")
|
||||||
pq.delete("http://a.example")
|
await pq.delete("http://a.example")
|
||||||
self.assertFalse(pq.exists("http://a.example"))
|
self.assertFalse(pq.exists("http://a.example"))
|
||||||
|
|
||||||
def test_saved_items_sorted_by_timestamp(self):
|
async def test_saved_items_sorted_by_timestamp(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
@@ -92,16 +95,16 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
b = _FakeDownload(_make_info("http://second.example"))
|
b = _FakeDownload(_make_info("http://second.example"))
|
||||||
a.info.timestamp = 100
|
a.info.timestamp = 100
|
||||||
b.info.timestamp = 200
|
b.info.timestamp = 200
|
||||||
pq.put(a)
|
await pq.put(a)
|
||||||
pq.put(b)
|
await pq.put(b)
|
||||||
keys = [k for k, _ in pq.saved_items()]
|
keys = [k for k, _ in pq.saved_items()]
|
||||||
self.assertEqual(keys, ["http://first.example", "http://second.example"])
|
self.assertEqual(keys, ["http://first.example", "http://second.example"])
|
||||||
|
|
||||||
def test_load_restores_from_json(self):
|
async def test_load_restores_from_json(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq1 = PersistentQueue("queue", path)
|
pq1 = PersistentQueue("queue", path)
|
||||||
pq1.put(_FakeDownload(_make_info("http://load.example")))
|
await pq1.put(_FakeDownload(_make_info("http://load.example")))
|
||||||
pq2 = PersistentQueue("queue", path)
|
pq2 = PersistentQueue("queue", path)
|
||||||
pq2.load()
|
pq2.load()
|
||||||
self.assertTrue(pq2.exists("http://load.example"))
|
self.assertTrue(pq2.exists("http://load.example"))
|
||||||
@@ -115,7 +118,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
self.assertTrue(pq.exists("http://legacy.example"))
|
self.assertTrue(pq.exists("http://legacy.example"))
|
||||||
self.assertTrue(os.path.exists(path + ".json"))
|
self.assertTrue(os.path.exists(path + ".json"))
|
||||||
|
|
||||||
def test_queue_persists_only_compact_entry_subset(self):
|
async def test_queue_persists_only_compact_entry_subset(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
@@ -128,7 +131,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
"formats": [{"id": "huge"}],
|
"formats": [{"id": "huge"}],
|
||||||
"description": "very large payload",
|
"description": "very large payload",
|
||||||
}
|
}
|
||||||
pq.put(_FakeDownload(info))
|
await pq.put(_FakeDownload(info))
|
||||||
|
|
||||||
with open(path + ".json", encoding="utf-8") as f:
|
with open(path + ".json", encoding="utf-8") as f:
|
||||||
payload = json.load(f)
|
payload = json.load(f)
|
||||||
@@ -146,7 +149,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
self.assertNotIn("formats", record["entry"])
|
self.assertNotIn("formats", record["entry"])
|
||||||
self.assertNotIn("description", record["entry"])
|
self.assertNotIn("description", record["entry"])
|
||||||
|
|
||||||
def test_completed_queue_persists_only_failed_retry_context(self):
|
async def test_completed_queue_persists_only_failed_retry_context(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "completed")
|
path = os.path.join(tmp, "completed")
|
||||||
pq = PersistentQueue("completed", path)
|
pq = PersistentQueue("completed", path)
|
||||||
@@ -161,7 +164,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
"formats": [{"id": "huge"}],
|
"formats": [{"id": "huge"}],
|
||||||
}
|
}
|
||||||
info.filename = "done.mp4"
|
info.filename = "done.mp4"
|
||||||
pq.put(_FakeDownload(info))
|
await pq.put(_FakeDownload(info))
|
||||||
|
|
||||||
with open(path + ".json", encoding="utf-8") as f:
|
with open(path + ".json", encoding="utf-8") as f:
|
||||||
payload = json.load(f)
|
payload = json.load(f)
|
||||||
@@ -180,7 +183,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
self.assertEqual(record["filename"], "done.mp4")
|
self.assertEqual(record["filename"], "done.mp4")
|
||||||
|
|
||||||
info.status = "finished"
|
info.status = "finished"
|
||||||
pq.put(_FakeDownload(info))
|
await pq.put(_FakeDownload(info))
|
||||||
with open(path + ".json", encoding="utf-8") as f:
|
with open(path + ".json", encoding="utf-8") as f:
|
||||||
payload = json.load(f)
|
payload = json.load(f)
|
||||||
self.assertNotIn("entry", payload["items"][0]["info"])
|
self.assertNotIn("entry", payload["items"][0]["info"])
|
||||||
@@ -256,7 +259,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
self.assertNotIn("speed", record)
|
self.assertNotIn("speed", record)
|
||||||
self.assertNotIn("eta", record)
|
self.assertNotIn("eta", record)
|
||||||
|
|
||||||
def test_put_rollbacks_in_memory_queue_when_state_write_fails(self):
|
async def test_put_rollbacks_in_memory_queue_when_state_write_fails(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
@@ -272,18 +275,18 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
|
|
||||||
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
||||||
with self.assertRaises(OSError):
|
with self.assertRaises(OSError):
|
||||||
pq.put(dl)
|
await pq.put(dl)
|
||||||
|
|
||||||
self.assertFalse(pq.exists("http://rollback.example"))
|
self.assertFalse(pq.exists("http://rollback.example"))
|
||||||
|
|
||||||
def test_put_rollbacks_to_previous_download_when_replace_fails(self):
|
async def test_put_rollbacks_to_previous_download_when_replace_fails(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
first = _FakeDownload(_make_info("http://same.example"))
|
first = _FakeDownload(_make_info("http://same.example"))
|
||||||
second = _FakeDownload(_make_info("http://same.example"))
|
second = _FakeDownload(_make_info("http://same.example"))
|
||||||
second.info.title = "Replaced title"
|
second.info.title = "Replaced title"
|
||||||
pq.put(first)
|
await pq.put(first)
|
||||||
|
|
||||||
orig_save = __import__("state_store").AtomicJsonStore.save
|
orig_save = __import__("state_store").AtomicJsonStore.save
|
||||||
|
|
||||||
@@ -294,10 +297,68 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
|
|
||||||
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
||||||
with self.assertRaises(OSError):
|
with self.assertRaises(OSError):
|
||||||
pq.put(second)
|
await pq.put(second)
|
||||||
|
|
||||||
self.assertEqual(pq.get("http://same.example").info.title, "Title")
|
self.assertEqual(pq.get("http://same.example").info.title, "Title")
|
||||||
|
|
||||||
|
|
||||||
|
class StateWriteOffEventLoopTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
"""State writes fsync twice; on a slow disk that must not stall the loop.
|
||||||
|
|
||||||
|
Before this, put()/delete() wrote inline, so a queue mutation blocked every
|
||||||
|
other request the server was serving for as long as the filesystem took.
|
||||||
|
See issue #980.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def test_save_runs_off_the_event_loop_thread(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
pq = PersistentQueue("queue", os.path.join(tmp, "queue"))
|
||||||
|
self.addCleanup(pq.close)
|
||||||
|
loop_thread = threading.get_ident()
|
||||||
|
save_threads = []
|
||||||
|
|
||||||
|
orig_save = __import__("state_store").AtomicJsonStore.save
|
||||||
|
|
||||||
|
def recording_save(store, data):
|
||||||
|
save_threads.append(threading.get_ident())
|
||||||
|
return orig_save(store, data)
|
||||||
|
|
||||||
|
with patch("ytdl.AtomicJsonStore.save", recording_save):
|
||||||
|
await pq.put(_FakeDownload(_make_info("http://a.example")))
|
||||||
|
|
||||||
|
self.assertEqual(len(save_threads), 1)
|
||||||
|
self.assertNotEqual(save_threads[0], loop_thread)
|
||||||
|
|
||||||
|
async def test_a_slow_write_does_not_stall_other_coroutines(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
pq = PersistentQueue("queue", os.path.join(tmp, "queue"))
|
||||||
|
self.addCleanup(pq.close)
|
||||||
|
orig_save = __import__("state_store").AtomicJsonStore.save
|
||||||
|
|
||||||
|
def slow_save(store, data):
|
||||||
|
time.sleep(0.3)
|
||||||
|
return orig_save(store, data)
|
||||||
|
|
||||||
|
ticks = 0
|
||||||
|
|
||||||
|
async def ticker():
|
||||||
|
nonlocal ticks
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(0.01)
|
||||||
|
ticks += 1
|
||||||
|
|
||||||
|
ticking = asyncio.create_task(ticker())
|
||||||
|
try:
|
||||||
|
with patch("ytdl.AtomicJsonStore.save", slow_save):
|
||||||
|
await pq.put(_FakeDownload(_make_info("http://a.example")))
|
||||||
|
finally:
|
||||||
|
ticking.cancel()
|
||||||
|
|
||||||
|
# An inline write would have starved the loop for the whole 0.3s and
|
||||||
|
# left ticks at 0.
|
||||||
|
self.assertGreater(ticks, 5)
|
||||||
|
self.assertTrue(pq.exists("http://a.example"))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -479,6 +479,69 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
|||||||
self.assertEqual(reloaded.get(sub_id).clip_start, 12.5)
|
self.assertEqual(reloaded.get(sub_id).clip_start, 12.5)
|
||||||
self.assertIsNone(reloaded.get(sub_id).clip_end)
|
self.assertIsNone(reloaded.get(sub_id).clip_end)
|
||||||
|
|
||||||
|
async def test_check_now_applies_subscription_sponsorblock(self):
|
||||||
|
"""Subscriptions download unattended, so the sponsor-segment removal has
|
||||||
|
to reach every entry the subscription queues, not just manual adds."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
queue = _Queue()
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"subscriptions.extract_flat_playlist",
|
||||||
|
side_effect=[
|
||||||
|
(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[
|
||||||
|
{"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"},
|
||||||
|
{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
):
|
||||||
|
result = await mgr.add_subscription(
|
||||||
|
"https://example.com/channel",
|
||||||
|
check_interval_minutes=60,
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
quality="best",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
auto_start=True,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
subtitle_language="en",
|
||||||
|
subtitle_mode="prefer_manual",
|
||||||
|
sponsorblock=True,
|
||||||
|
)
|
||||||
|
sub_id = result["subscription"]["id"]
|
||||||
|
self.assertTrue(mgr.get(sub_id).sponsorblock)
|
||||||
|
await mgr.check_now([sub_id])
|
||||||
|
|
||||||
|
self.assertEqual(len(queue.entries), 1)
|
||||||
|
_entry, _args, kwargs = queue.entries[0]
|
||||||
|
self.assertIs(kwargs["sponsorblock"], True)
|
||||||
|
|
||||||
|
async def test_sponsorblock_survives_reload_and_defaults_to_false(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg = _Config(tmp)
|
||||||
|
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
# Records written before the field existed simply take the default.
|
||||||
|
self.assertFalse(mgr.get(sub_id).sponsorblock)
|
||||||
|
|
||||||
|
mgr.get(sub_id).sponsorblock = True
|
||||||
|
async with mgr._lock:
|
||||||
|
mgr._save_locked()
|
||||||
|
|
||||||
|
reloaded = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
self.assertTrue(reloaded.get(sub_id).sponsorblock)
|
||||||
|
|
||||||
async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
|
async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
queue = _Queue()
|
queue = _Queue()
|
||||||
|
|||||||
+71
-36
@@ -12,7 +12,7 @@ from url_guard import (
|
|||||||
_address_allowed_at_connect,
|
_address_allowed_at_connect,
|
||||||
_address_is_global,
|
_address_is_global,
|
||||||
_guarded_getaddrinfo,
|
_guarded_getaddrinfo,
|
||||||
_proxy_endpoint,
|
_url_endpoint,
|
||||||
install_socket_guard,
|
install_socket_guard,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -121,19 +121,19 @@ class ConnectAddressPolicyTests(unittest.TestCase):
|
|||||||
self.assertFalse(_address_allowed_at_connect("::1"))
|
self.assertFalse(_address_allowed_at_connect("::1"))
|
||||||
|
|
||||||
def test_loopback_allowed_only_when_opted_in(self):
|
def test_loopback_allowed_only_when_opted_in(self):
|
||||||
self.assertTrue(_address_allowed_at_connect("127.0.0.1", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("127.0.0.1", is_allowed_endpoint=True))
|
||||||
self.assertTrue(_address_allowed_at_connect("::1", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("::1", is_allowed_endpoint=True))
|
||||||
|
|
||||||
def test_proxy_opt_in_covers_any_internal_range(self):
|
def test_proxy_opt_in_covers_any_internal_range(self):
|
||||||
# A proxy is just as legitimately on the LAN or a VPN range as on
|
# A proxy is just as legitimately on the LAN or a VPN range as on
|
||||||
# loopback (#1055): the allowance follows the operator's configured
|
# loopback (#1055): the allowance follows the operator's configured
|
||||||
# endpoint, not a particular address family.
|
# endpoint, not a particular address family.
|
||||||
self.assertTrue(_address_allowed_at_connect("10.1.20.30", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("10.1.20.30", is_allowed_endpoint=True))
|
||||||
self.assertTrue(_address_allowed_at_connect("192.168.1.10", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("192.168.1.10", is_allowed_endpoint=True))
|
||||||
self.assertTrue(_address_allowed_at_connect("fd00::1", is_proxy_endpoint=True))
|
self.assertTrue(_address_allowed_at_connect("fd00::1", is_allowed_endpoint=True))
|
||||||
|
|
||||||
def test_opt_in_still_rejects_non_addresses(self):
|
def test_opt_in_still_rejects_non_addresses(self):
|
||||||
self.assertFalse(_address_allowed_at_connect("not-an-ip", is_proxy_endpoint=True))
|
self.assertFalse(_address_allowed_at_connect("not-an-ip", is_allowed_endpoint=True))
|
||||||
|
|
||||||
def test_link_local_metadata_blocked(self):
|
def test_link_local_metadata_blocked(self):
|
||||||
self.assertFalse(_address_allowed_at_connect("169.254.169.254"))
|
self.assertFalse(_address_allowed_at_connect("169.254.169.254"))
|
||||||
@@ -187,36 +187,36 @@ class TunnelledIPv4Tests(unittest.TestCase):
|
|||||||
self.assertIsNotNone(validate_url("http://nat64.example/x"))
|
self.assertIsNotNone(validate_url("http://nat64.example/x"))
|
||||||
|
|
||||||
|
|
||||||
class ProxyEndpointParsingTests(unittest.TestCase):
|
class EndpointParsingTests(unittest.TestCase):
|
||||||
def test_explicit_port(self):
|
def test_explicit_port(self):
|
||||||
self.assertEqual(_proxy_endpoint("http://127.0.0.1:9050"), ("127.0.0.1", 9050))
|
self.assertEqual(_url_endpoint("http://127.0.0.1:9050"), ("127.0.0.1", 9050))
|
||||||
|
|
||||||
def test_default_port_per_scheme(self):
|
def test_default_port_per_scheme(self):
|
||||||
self.assertEqual(_proxy_endpoint("socks5://127.0.0.1"), ("127.0.0.1", 1080))
|
self.assertEqual(_url_endpoint("socks5://127.0.0.1"), ("127.0.0.1", 1080))
|
||||||
self.assertEqual(_proxy_endpoint("http://127.0.0.1"), ("127.0.0.1", 80))
|
self.assertEqual(_url_endpoint("http://127.0.0.1"), ("127.0.0.1", 80))
|
||||||
|
|
||||||
def test_bare_host_port(self):
|
def test_bare_host_port(self):
|
||||||
self.assertEqual(_proxy_endpoint("127.0.0.1:8080"), ("127.0.0.1", 8080))
|
self.assertEqual(_url_endpoint("127.0.0.1:8080"), ("127.0.0.1", 8080))
|
||||||
|
|
||||||
def test_hostname_lowercased(self):
|
def test_hostname_lowercased(self):
|
||||||
self.assertEqual(_proxy_endpoint("http://LocalHost.:9050"), ("localhost", 9050))
|
self.assertEqual(_url_endpoint("http://LocalHost.:9050"), ("localhost", 9050))
|
||||||
|
|
||||||
def test_ipv6_literal(self):
|
def test_ipv6_literal(self):
|
||||||
self.assertEqual(_proxy_endpoint("http://[::1]:9050"), ("::1", 9050))
|
self.assertEqual(_url_endpoint("http://[::1]:9050"), ("::1", 9050))
|
||||||
|
|
||||||
def test_empty_and_invalid(self):
|
def test_empty_and_invalid(self):
|
||||||
self.assertIsNone(_proxy_endpoint(""))
|
self.assertIsNone(_url_endpoint(""))
|
||||||
self.assertIsNone(_proxy_endpoint(" "))
|
self.assertIsNone(_url_endpoint(" "))
|
||||||
self.assertIsNone(_proxy_endpoint(None))
|
self.assertIsNone(_url_endpoint(None))
|
||||||
self.assertIsNone(_proxy_endpoint("http://"))
|
self.assertIsNone(_url_endpoint("http://"))
|
||||||
|
|
||||||
|
|
||||||
class GuardedGetaddrinfoTests(unittest.TestCase):
|
class GuardedGetaddrinfoTests(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
# Default state: no proxy configured, so no loopback destination allowed.
|
# Default state: no proxy configured, so no loopback destination allowed.
|
||||||
saved = set(url_guard._allowed_proxy_endpoints)
|
saved = set(url_guard._allowed_endpoints)
|
||||||
url_guard._allowed_proxy_endpoints = set()
|
url_guard._allowed_endpoints = set()
|
||||||
self.addCleanup(lambda: setattr(url_guard, "_allowed_proxy_endpoints", saved))
|
self.addCleanup(lambda: setattr(url_guard, "_allowed_endpoints", saved))
|
||||||
|
|
||||||
def test_internal_only_raises(self):
|
def test_internal_only_raises(self):
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")):
|
||||||
@@ -235,27 +235,27 @@ class GuardedGetaddrinfoTests(unittest.TestCase):
|
|||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
_guarded_getaddrinfo("127.0.0.1", 9999)
|
_guarded_getaddrinfo("127.0.0.1", 9999)
|
||||||
|
|
||||||
def test_loopback_allowed_at_configured_proxy_endpoint(self):
|
def test_loopback_allowed_at_configured_url_endpoint(self):
|
||||||
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)}
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
results = _guarded_getaddrinfo("127.0.0.1", 9050)
|
results = _guarded_getaddrinfo("127.0.0.1", 9050)
|
||||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
def test_loopback_blocked_at_other_port_on_proxy_host(self):
|
def test_loopback_blocked_at_other_port_on_proxy_host(self):
|
||||||
# Same host as the proxy, different port: still off limits.
|
# Same host as the proxy, different port: still off limits.
|
||||||
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)}
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
_guarded_getaddrinfo("127.0.0.1", 9999)
|
_guarded_getaddrinfo("127.0.0.1", 9999)
|
||||||
|
|
||||||
def test_proxy_reachable_by_hostname(self):
|
def test_proxy_reachable_by_hostname(self):
|
||||||
url_guard._allowed_proxy_endpoints = {("localhost", 9050)}
|
url_guard._allowed_endpoints = {("localhost", 9050)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
results = _guarded_getaddrinfo("localhost", 9050)
|
results = _guarded_getaddrinfo("localhost", 9050)
|
||||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
def test_string_port_is_normalised(self):
|
def test_string_port_is_normalised(self):
|
||||||
url_guard._allowed_proxy_endpoints = {("127.0.0.1", 9050)}
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
results = _guarded_getaddrinfo("127.0.0.1", "9050")
|
results = _guarded_getaddrinfo("127.0.0.1", "9050")
|
||||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
@@ -263,22 +263,38 @@ class GuardedGetaddrinfoTests(unittest.TestCase):
|
|||||||
def test_lan_proxy_reachable(self):
|
def test_lan_proxy_reachable(self):
|
||||||
# #1055: a socks5 proxy on the LAN, refused while the allowance was
|
# #1055: a socks5 proxy on the LAN, refused while the allowance was
|
||||||
# loopback-only, which pushed operators to ALLOW_PRIVATE_ADDRESSES.
|
# loopback-only, which pushed operators to ALLOW_PRIVATE_ADDRESSES.
|
||||||
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)}
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
||||||
results = _guarded_getaddrinfo("10.1.20.30", 1080)
|
results = _guarded_getaddrinfo("10.1.20.30", 1080)
|
||||||
self.assertEqual([r[4][0] for r in results], ["10.1.20.30"])
|
self.assertEqual([r[4][0] for r in results], ["10.1.20.30"])
|
||||||
|
|
||||||
def test_other_lan_host_still_blocked(self):
|
def test_other_lan_host_still_blocked(self):
|
||||||
# The allowance is the proxy's endpoint, not its subnet.
|
# The allowance is the proxy's endpoint, not its subnet.
|
||||||
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)}
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.31")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.31")):
|
||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
_guarded_getaddrinfo("10.1.20.31", 1080)
|
_guarded_getaddrinfo("10.1.20.31", 1080)
|
||||||
|
|
||||||
|
def test_pot_provider_reachable_on_loopback(self):
|
||||||
|
# #1064: the bundled PO token provider listens on loopback, and blocking
|
||||||
|
# it left every default install downloading YouTube without a token.
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 4416)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
results = _guarded_getaddrinfo("127.0.0.1", 4416)
|
||||||
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
|
def test_other_loopback_service_still_blocked(self):
|
||||||
|
# MeTube's own port is one hop away on the same interface: allowing the
|
||||||
|
# token provider must not allow the rest of loopback.
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 4416)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
with self.assertRaises(socket.gaierror):
|
||||||
|
_guarded_getaddrinfo("127.0.0.1", 8081)
|
||||||
|
|
||||||
def test_proxy_address_not_borrowable_by_another_host(self):
|
def test_proxy_address_not_borrowable_by_another_host(self):
|
||||||
# Matching is on the configured host string: a manifest URL that resolves
|
# Matching is on the configured host string: a manifest URL that resolves
|
||||||
# to the proxy's address under its own name gets no allowance.
|
# to the proxy's address under its own name gets no allowance.
|
||||||
url_guard._allowed_proxy_endpoints = {("10.1.20.30", 1080)}
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
_guarded_getaddrinfo("evil.example", 1080)
|
_guarded_getaddrinfo("evil.example", 1080)
|
||||||
@@ -312,9 +328,9 @@ class AllowPrivateBypassTests(unittest.TestCase):
|
|||||||
|
|
||||||
class InstallSocketGuardTests(unittest.TestCase):
|
class InstallSocketGuardTests(unittest.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
original, saved = socket.getaddrinfo, set(url_guard._allowed_proxy_endpoints)
|
original, saved = socket.getaddrinfo, set(url_guard._allowed_endpoints)
|
||||||
self.addCleanup(lambda: setattr(socket, "getaddrinfo", original))
|
self.addCleanup(lambda: setattr(socket, "getaddrinfo", original))
|
||||||
self.addCleanup(lambda: setattr(url_guard, "_allowed_proxy_endpoints", saved))
|
self.addCleanup(lambda: setattr(url_guard, "_allowed_endpoints", saved))
|
||||||
# Keep the host's own environment out of the assertions below.
|
# Keep the host's own environment out of the assertions below.
|
||||||
patcher = mock.patch("url_guard.urllib.request.getproxies", return_value={})
|
patcher = mock.patch("url_guard.urllib.request.getproxies", return_value={})
|
||||||
self.getproxies = patcher.start()
|
self.getproxies = patcher.start()
|
||||||
@@ -329,26 +345,45 @@ class InstallSocketGuardTests(unittest.TestCase):
|
|||||||
|
|
||||||
def test_no_proxy_means_no_loopback_allowance(self):
|
def test_no_proxy_means_no_loopback_allowance(self):
|
||||||
install_socket_guard()
|
install_socket_guard()
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, set())
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
def test_explicit_proxy_is_registered(self):
|
def test_explicit_proxy_is_registered(self):
|
||||||
install_socket_guard(proxy_urls=("socks5://127.0.0.1:9050",))
|
install_socket_guard(proxy_urls=("socks5://127.0.0.1:9050",))
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, {("127.0.0.1", 9050)})
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 9050)})
|
||||||
|
|
||||||
def test_unset_proxy_option_is_ignored(self):
|
def test_unset_proxy_option_is_ignored(self):
|
||||||
# ytdl_opts.get('proxy') is None when the operator configured no proxy.
|
# ytdl_opts.get('proxy') is None when the operator configured no proxy.
|
||||||
install_socket_guard(proxy_urls=(None,))
|
install_socket_guard(proxy_urls=(None,))
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, set())
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
def test_environment_proxies_are_registered(self):
|
def test_environment_proxies_are_registered(self):
|
||||||
self.getproxies.return_value = {"http": "http://127.0.0.1:8080"}
|
self.getproxies.return_value = {"http": "http://127.0.0.1:8080"}
|
||||||
install_socket_guard()
|
install_socket_guard()
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, {("127.0.0.1", 8080)})
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 8080)})
|
||||||
|
|
||||||
|
def test_service_url_is_registered(self):
|
||||||
|
install_socket_guard(service_urls=("http://127.0.0.1:4416",))
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 4416)})
|
||||||
|
|
||||||
|
def test_service_and_proxy_endpoints_coexist(self):
|
||||||
|
install_socket_guard(
|
||||||
|
proxy_urls=("socks5://10.1.20.30:1080",),
|
||||||
|
service_urls=("http://127.0.0.1:4416",),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
url_guard._allowed_endpoints,
|
||||||
|
{("10.1.20.30", 1080), ("127.0.0.1", 4416)},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_service_urls_reset_between_installs(self):
|
||||||
|
install_socket_guard(service_urls=("http://127.0.0.1:4416",))
|
||||||
|
install_socket_guard()
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
def test_endpoints_reset_between_installs(self):
|
def test_endpoints_reset_between_installs(self):
|
||||||
install_socket_guard(proxy_urls=("http://127.0.0.1:8080",))
|
install_socket_guard(proxy_urls=("http://127.0.0.1:8080",))
|
||||||
install_socket_guard(proxy_urls=(None,))
|
install_socket_guard(proxy_urls=(None,))
|
||||||
self.assertEqual(url_guard._allowed_proxy_endpoints, set())
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import pickle
|
import pickle
|
||||||
import signal
|
import signal
|
||||||
@@ -11,7 +12,8 @@ import threading
|
|||||||
import types
|
import types
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import MagicMock, patch
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
fake_yt_dlp = types.ModuleType("yt_dlp")
|
fake_yt_dlp = types.ModuleType("yt_dlp")
|
||||||
fake_networking = types.ModuleType("yt_dlp.networking")
|
fake_networking = types.ModuleType("yt_dlp.networking")
|
||||||
@@ -50,6 +52,7 @@ class _YoutubeDL:
|
|||||||
|
|
||||||
|
|
||||||
fake_utils.DownloadError = type("DownloadError", (Exception,), {})
|
fake_utils.DownloadError = type("DownloadError", (Exception,), {})
|
||||||
|
fake_utils.YoutubeDLError = fake_utils.DownloadError
|
||||||
fake_yt_dlp.YoutubeDL = _YoutubeDL
|
fake_yt_dlp.YoutubeDL = _YoutubeDL
|
||||||
fake_impersonate.ImpersonateTarget = _ImpersonateTarget
|
fake_impersonate.ImpersonateTarget = _ImpersonateTarget
|
||||||
fake_networking.impersonate = fake_impersonate
|
fake_networking.impersonate = fake_impersonate
|
||||||
@@ -76,6 +79,7 @@ from ytdl import (
|
|||||||
MusicMetadataPreProcessor,
|
MusicMetadataPreProcessor,
|
||||||
_compact_persisted_entry,
|
_compact_persisted_entry,
|
||||||
_convert_srt_to_txt_file,
|
_convert_srt_to_txt_file,
|
||||||
|
_pot_provider_urls,
|
||||||
_AlbumArtistPostProcessor,
|
_AlbumArtistPostProcessor,
|
||||||
_resolve_outtmpl_fields,
|
_resolve_outtmpl_fields,
|
||||||
_sanitize_entry_for_pickle,
|
_sanitize_entry_for_pickle,
|
||||||
@@ -376,6 +380,49 @@ class ConfinedYoutubeDLTests(unittest.TestCase):
|
|||||||
self.assertEqual(self._prepared_path(""), "")
|
self.assertEqual(self._prepared_path(""), "")
|
||||||
self.assertEqual(self._prepared_path("-"), "-")
|
self.assertEqual(self._prepared_path("-"), "-")
|
||||||
|
|
||||||
|
def test_overlong_name_is_trimmed_to_fit_the_filesystem(self):
|
||||||
|
# A title long enough to blow the filename limit is what made these
|
||||||
|
# downloads fail outright with [Errno 36] File name too long.
|
||||||
|
long_path = os.path.join(self.base, "a" * 400 + ".mp4")
|
||||||
|
|
||||||
|
result = self._prepared_path(long_path)
|
||||||
|
|
||||||
|
name = os.path.basename(result)
|
||||||
|
self.assertTrue(name.endswith(".mp4"))
|
||||||
|
self.assertLessEqual(len(name.encode("utf-8")), 255 - 32)
|
||||||
|
self.assertEqual(os.path.dirname(result), self.base)
|
||||||
|
# The file must still be writable once yt-dlp adds its own suffixes.
|
||||||
|
self.assertLessEqual(len(f"{name}.f1229065279304024v.part".encode("utf-8")), 255)
|
||||||
|
|
||||||
|
def test_name_within_the_limit_is_left_alone(self):
|
||||||
|
ok = os.path.join(self.base, "Ordinary Title.mp4")
|
||||||
|
self.assertEqual(self._prepared_path(ok), ok)
|
||||||
|
|
||||||
|
def test_limit_counts_bytes_not_characters(self):
|
||||||
|
# 200 CJK characters are 600 bytes: a character count would pass this.
|
||||||
|
long_path = os.path.join(self.base, "音" * 200 + ".mp4")
|
||||||
|
|
||||||
|
name = os.path.basename(self._prepared_path(long_path))
|
||||||
|
|
||||||
|
self.assertLessEqual(len(name.encode("utf-8")), 255 - 32)
|
||||||
|
# A trim landing mid-character must not leave a broken byte sequence.
|
||||||
|
self.assertEqual(name, name.encode("utf-8").decode("utf-8"))
|
||||||
|
self.assertTrue(name.endswith(".mp4"))
|
||||||
|
|
||||||
|
def test_a_long_tail_is_not_mistaken_for_an_extension(self):
|
||||||
|
# os.path.splitext on a title containing a dot late in the string would
|
||||||
|
# otherwise "preserve" a 100-character extension and trim nothing.
|
||||||
|
long_path = os.path.join(self.base, "b" * 300 + "." + "c" * 100)
|
||||||
|
|
||||||
|
name = os.path.basename(self._prepared_path(long_path))
|
||||||
|
|
||||||
|
self.assertLessEqual(len(name.encode("utf-8")), 255 - 32)
|
||||||
|
|
||||||
|
def test_trimming_still_cannot_escape_the_download_directory(self):
|
||||||
|
escaping = os.path.join(self.base, "..", "..", "d" * 400 + ".mp4")
|
||||||
|
with self.assertRaises(ytdl.yt_dlp.utils.DownloadError):
|
||||||
|
self._prepared_path(escaping)
|
||||||
|
|
||||||
|
|
||||||
class SanitizeEntryForPickleTests(unittest.TestCase):
|
class SanitizeEntryForPickleTests(unittest.TestCase):
|
||||||
def test_nested(self):
|
def test_nested(self):
|
||||||
@@ -434,6 +481,269 @@ def _make_test_download() -> Download:
|
|||||||
return Download("/tmp", "/tmp", "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info)
|
return Download("/tmp", "/tmp", "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadLoggerTests(unittest.TestCase):
|
||||||
|
def test_routes_messages_and_retains_only_non_empty_warnings(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='DEBUG') as logs:
|
||||||
|
logger.debug('debug detail')
|
||||||
|
logger.warning(' useful warning ')
|
||||||
|
logger.warning(' ')
|
||||||
|
logger.error('error detail')
|
||||||
|
|
||||||
|
self.assertEqual(logger.warnings, ['useful warning'])
|
||||||
|
self.assertIn('DEBUG:ytdl:debug detail', logs.output)
|
||||||
|
self.assertIn('WARNING:ytdl: useful warning ', logs.output)
|
||||||
|
self.assertIn('ERROR:ytdl:error detail', logs.output)
|
||||||
|
|
||||||
|
def test_retains_only_the_last_distinct_warnings(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
cap = ytdl._MAX_RETAINED_WARNINGS
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING') as logs:
|
||||||
|
for index in range(cap + 3):
|
||||||
|
logger.warning(f'fragment {index} not found')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
logger.warnings,
|
||||||
|
[f'fragment {index} not found' for index in range(3, cap + 3)],
|
||||||
|
)
|
||||||
|
# Every warning still reaches the log; only the retained list is bounded.
|
||||||
|
self.assertEqual(len(logs.output), cap + 3)
|
||||||
|
|
||||||
|
def test_repeated_warning_is_retained_once(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING'):
|
||||||
|
logger.warning('Requested format is not available')
|
||||||
|
logger.warning('Only images are available for download')
|
||||||
|
logger.warning('Requested format is not available')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
logger.warnings,
|
||||||
|
['Requested format is not available', 'Only images are available for download'],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_failure_message_puts_the_error_last(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING'):
|
||||||
|
logger.warning('Only images are available for download')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
logger.failure_message('ERROR: [youtube] u2HSc2Ym1Vk: No video formats found!'),
|
||||||
|
'Only images are available for download\n'
|
||||||
|
'ERROR: [youtube] u2HSc2Ym1Vk: No video formats found!',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_failure_message_skips_a_last_warning_that_repeats_the_error(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING'):
|
||||||
|
logger.warning('Video unavailable')
|
||||||
|
# yt-dlp labels errors but hands warnings to the logger unlabelled,
|
||||||
|
# so the same text can arrive through both routes.
|
||||||
|
logger.warning('Requested format is not available')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
logger.failure_message('ERROR: Requested format is not available'),
|
||||||
|
'Video unavailable\nERROR: Requested format is not available',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_failure_message_without_warnings_is_the_error_alone(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
self.assertEqual(logger.failure_message('ERROR: boom'), 'ERROR: boom')
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadResultTests(unittest.TestCase):
|
||||||
|
def _run_download(self, result=0, warnings=(), error=None):
|
||||||
|
download = _make_test_download()
|
||||||
|
statuses = []
|
||||||
|
download.status_queue = types.SimpleNamespace(put=statuses.append)
|
||||||
|
captured_params = {}
|
||||||
|
|
||||||
|
class FakeYoutubeDL:
|
||||||
|
def download(self, urls):
|
||||||
|
self.urls = urls
|
||||||
|
for warning in warnings:
|
||||||
|
captured_params['logger'].warning(warning)
|
||||||
|
if error is not None:
|
||||||
|
raise error
|
||||||
|
return result
|
||||||
|
|
||||||
|
def make_youtube_dl(params):
|
||||||
|
captured_params.update(params)
|
||||||
|
return FakeYoutubeDL()
|
||||||
|
|
||||||
|
with patch.object(download, '_make_youtube_dl', side_effect=make_youtube_dl), \
|
||||||
|
patch('ytdl.install_socket_guard'), \
|
||||||
|
patch('ytdl.os.setpgrp'):
|
||||||
|
download._download()
|
||||||
|
|
||||||
|
return statuses, captured_params
|
||||||
|
|
||||||
|
def test_nonzero_result_includes_warning_context_and_forwards_logs(self):
|
||||||
|
warnings = [
|
||||||
|
'The uploader has blocked this video in your country',
|
||||||
|
'No video formats found',
|
||||||
|
]
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING') as logs:
|
||||||
|
statuses, params = self._run_download(result=1, warnings=warnings)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1],
|
||||||
|
{'status': 'error', 'msg': '\n'.join(warnings)},
|
||||||
|
)
|
||||||
|
self.assertIs(params['logger'].__class__, ytdl._DownloadYtdlLogger)
|
||||||
|
for warning in warnings:
|
||||||
|
self.assertTrue(any(warning in entry for entry in logs.output))
|
||||||
|
|
||||||
|
def test_nonzero_result_without_warning_uses_fallback_message(self):
|
||||||
|
statuses, _ = self._run_download(result=2)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1],
|
||||||
|
{'status': 'error', 'msg': 'yt-dlp failed with exit code 2'},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_warning_does_not_change_success_status(self):
|
||||||
|
statuses, _ = self._run_download(result=0, warnings=['A recoverable warning'])
|
||||||
|
|
||||||
|
self.assertEqual(statuses[-1], {'status': 'finished'})
|
||||||
|
|
||||||
|
def test_youtube_dl_error_carries_the_warnings_that_explain_it(self):
|
||||||
|
# The sequence from issue #1047: yt-dlp raises DownloadError, so the
|
||||||
|
# warnings naming the real cause only reach the user if the exception
|
||||||
|
# branch carries them too.
|
||||||
|
statuses, _ = self._run_download(
|
||||||
|
warnings=[
|
||||||
|
'[youtube] Video unavailable. This video contains content from bryhuangpub,'
|
||||||
|
' who has blocked it from display on this website or application',
|
||||||
|
'Only images are available for download. use --list-formats to see them',
|
||||||
|
'Requested format is not available',
|
||||||
|
],
|
||||||
|
error=ytdl.yt_dlp.utils.YoutubeDLError(
|
||||||
|
'ERROR: [youtube] u2HSc2Ym1Vk: No video formats found!'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1],
|
||||||
|
{
|
||||||
|
'status': 'error',
|
||||||
|
'msg': '[youtube] Video unavailable. This video contains content from bryhuangpub,'
|
||||||
|
' who has blocked it from display on this website or application\n'
|
||||||
|
'Only images are available for download. use --list-formats to see them\n'
|
||||||
|
'Requested format is not available\n'
|
||||||
|
'ERROR: [youtube] u2HSc2Ym1Vk: No video formats found!',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_youtube_dl_error_drops_a_last_warning_that_repeats_it(self):
|
||||||
|
statuses, _ = self._run_download(
|
||||||
|
warnings=['Earlier warning', 'Requested format is not available'],
|
||||||
|
error=ytdl.yt_dlp.utils.YoutubeDLError('ERROR: Requested format is not available'),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1],
|
||||||
|
{
|
||||||
|
'status': 'error',
|
||||||
|
'msg': 'Earlier warning\nERROR: Requested format is not available',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_youtube_dl_error_message_is_bounded(self):
|
||||||
|
cap = ytdl._MAX_RETAINED_WARNINGS
|
||||||
|
statuses, _ = self._run_download(
|
||||||
|
warnings=[f'fragment {index} not found' for index in range(cap + 4)],
|
||||||
|
error=ytdl.yt_dlp.utils.YoutubeDLError('ERROR: giving up'),
|
||||||
|
)
|
||||||
|
|
||||||
|
msg = statuses[-1]['msg']
|
||||||
|
self.assertEqual(
|
||||||
|
msg.split('\n'),
|
||||||
|
[f'fragment {index} not found' for index in range(4, cap + 4)] + ['ERROR: giving up'],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_nonzero_result_message_is_bounded(self):
|
||||||
|
cap = ytdl._MAX_RETAINED_WARNINGS
|
||||||
|
statuses, _ = self._run_download(
|
||||||
|
result=1,
|
||||||
|
warnings=[f'fragment {index} not found' for index in range(cap + 4)],
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1]['msg'].split('\n'),
|
||||||
|
[f'fragment {index} not found' for index in range(4, cap + 4)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _capture_ytdl_params(download: Download) -> dict:
|
||||||
|
"""Run ``_download`` far enough to capture the params it builds."""
|
||||||
|
fake_ydl = MagicMock()
|
||||||
|
fake_ydl.download.return_value = 0
|
||||||
|
download.status_queue = types.SimpleNamespace(put=lambda _: None)
|
||||||
|
|
||||||
|
with patch('ytdl.install_socket_guard'), \
|
||||||
|
patch.object(Download, '_make_youtube_dl', return_value=fake_ydl) as make:
|
||||||
|
download._download()
|
||||||
|
|
||||||
|
params, = make.call_args.args
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
class SponsorBlockPostprocessorTests(unittest.TestCase):
|
||||||
|
def test_no_sponsorblock_postprocessors_when_disabled(self):
|
||||||
|
download = _make_test_download()
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
keys = [pp['key'] for pp in params.get('postprocessors', [])]
|
||||||
|
self.assertNotIn('SponsorBlock', keys)
|
||||||
|
self.assertNotIn('ModifyChapters', keys)
|
||||||
|
|
||||||
|
def test_sponsorblock_pair_matches_the_cli(self):
|
||||||
|
download = _make_test_download()
|
||||||
|
download.info.sponsorblock = True
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
params['postprocessors'],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'key': 'SponsorBlock',
|
||||||
|
'categories': ['sponsor'],
|
||||||
|
'when': 'after_filter',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'key': 'ModifyChapters',
|
||||||
|
'remove_sponsor_segments': ['sponsor'],
|
||||||
|
'force_keyframes': False,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_segment_removal_runs_before_the_chapter_split(self):
|
||||||
|
# yt-dlp runs same-stage postprocessors in list order, so ModifyChapters
|
||||||
|
# has to rewrite the chapter list before FFmpegSplitChapters cuts the
|
||||||
|
# file up -- the order the CLI builds for
|
||||||
|
# --sponsorblock-remove sponsor --split-chapters.
|
||||||
|
download = _make_test_download()
|
||||||
|
download.info.sponsorblock = True
|
||||||
|
download.info.split_by_chapters = True
|
||||||
|
download.info.chapter_template = '%(section_number)s.%(ext)s'
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
keys = [pp['key'] for pp in params['postprocessors']]
|
||||||
|
self.assertEqual(keys, ['SponsorBlock', 'ModifyChapters', 'FFmpegSplitChapters'])
|
||||||
|
self.assertEqual(params['outtmpl']['chapter'], '%(section_number)s.%(ext)s')
|
||||||
|
|
||||||
|
|
||||||
class ProgressThrottleTests(unittest.TestCase):
|
class ProgressThrottleTests(unittest.TestCase):
|
||||||
def test_downloading_ticks_are_throttled(self):
|
def test_downloading_ticks_are_throttled(self):
|
||||||
dl = _make_test_download()
|
dl = _make_test_download()
|
||||||
@@ -827,5 +1137,103 @@ class ShortTitleForFailedUrlTests(unittest.TestCase):
|
|||||||
self.assertEqual(_short_title_for_failed_url(malformed), malformed)
|
self.assertEqual(_short_title_for_failed_url(malformed), malformed)
|
||||||
|
|
||||||
|
|
||||||
|
class PotProviderUrlsTests(unittest.TestCase):
|
||||||
|
"""#1064: the connect-time guard must let the download reach the PO token
|
||||||
|
provider, so it has to know every endpoint yt-dlp might dial for one."""
|
||||||
|
|
||||||
|
def test_bundled_provider_by_default(self):
|
||||||
|
self.assertEqual(_pot_provider_urls({}), ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
def test_configured_base_url_is_added(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtubepot-bgutilhttp": {"base_url": ["http://pot:4416"]}},
|
||||||
|
})
|
||||||
|
# The bundled server runs regardless, so both stay reachable.
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416", "http://pot:4416"))
|
||||||
|
|
||||||
|
def test_deprecated_base_url_arg_is_honoured(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtube": {"getpot_bgutil_baseurl": ["http://pot:4416"]}},
|
||||||
|
})
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416", "http://pot:4416"))
|
||||||
|
|
||||||
|
def test_unrelated_extractor_args_are_ignored(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtube": {"player_client": ["web"]}},
|
||||||
|
})
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
def test_malformed_extractor_args_do_not_raise(self):
|
||||||
|
# YTDL_OPTIONS is operator-supplied JSON and reaches here unvalidated.
|
||||||
|
for opts in (
|
||||||
|
{"extractor_args": None},
|
||||||
|
{"extractor_args": "youtube:player_client=web"},
|
||||||
|
{"extractor_args": {"youtubepot-bgutilhttp": "http://pot:4416"}},
|
||||||
|
{"extractor_args": {"youtubepot-bgutilhttp": {"base_url": []}}},
|
||||||
|
):
|
||||||
|
with self.subTest(opts=opts):
|
||||||
|
self.assertEqual(_pot_provider_urls(opts), ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateStatusFileStatTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
"""The progress path must not touch the filesystem on the event loop.
|
||||||
|
|
||||||
|
yt-dlp reports 'filename' on every progress tick, but until the download
|
||||||
|
finishes that path does not exist yet -- the bytes are in 'tmpfilename'.
|
||||||
|
Stating it per tick meant blocking syscalls on the event loop twice a
|
||||||
|
second per download, always answering None. See issue #980.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def _run_update_status(self, statuses):
|
||||||
|
import queue as _queue
|
||||||
|
|
||||||
|
download = _make_test_download()
|
||||||
|
download.download_dir = "/tmp"
|
||||||
|
source = _queue.Queue()
|
||||||
|
for status in statuses:
|
||||||
|
source.put(status)
|
||||||
|
source.put(None)
|
||||||
|
download.status_queue = source
|
||||||
|
download.loop = asyncio.get_running_loop()
|
||||||
|
download._executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
notifier = MagicMock()
|
||||||
|
notifier.updated = AsyncMock()
|
||||||
|
download.notifier = notifier
|
||||||
|
|
||||||
|
stat_calls = []
|
||||||
|
|
||||||
|
def record_exists(path):
|
||||||
|
stat_calls.append(path)
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
with patch("ytdl.os.path.exists", side_effect=record_exists):
|
||||||
|
await download.update_status()
|
||||||
|
finally:
|
||||||
|
download._executor.shutdown(wait=True)
|
||||||
|
return download, stat_calls
|
||||||
|
|
||||||
|
async def test_downloading_ticks_do_not_stat_the_output_file(self):
|
||||||
|
ticks = [
|
||||||
|
{"status": "downloading", "filename": "/tmp/v.mp4",
|
||||||
|
"tmpfilename": "/tmp/v.mp4.part", "downloaded_bytes": i}
|
||||||
|
for i in range(1, 6)
|
||||||
|
]
|
||||||
|
download, stat_calls = await self._run_update_status(ticks)
|
||||||
|
|
||||||
|
self.assertEqual(stat_calls, [])
|
||||||
|
self.assertEqual(download.info.filename, "v.mp4")
|
||||||
|
|
||||||
|
async def test_finished_status_still_stats_the_output_file(self):
|
||||||
|
download, stat_calls = await self._run_update_status([
|
||||||
|
{"status": "downloading", "filename": "/tmp/v.mp4", "downloaded_bytes": 1},
|
||||||
|
{"status": "finished", "filename": "/tmp/v.mp4"},
|
||||||
|
])
|
||||||
|
|
||||||
|
self.assertEqual(stat_calls, ["/tmp/v.mp4"])
|
||||||
|
self.assertIsNone(download.info.size)
|
||||||
|
|||||||
+52
-38
@@ -37,8 +37,8 @@ log = logging.getLogger('url_guard')
|
|||||||
|
|
||||||
_ALLOWED_SCHEMES = ('http', 'https')
|
_ALLOWED_SCHEMES = ('http', 'https')
|
||||||
|
|
||||||
# Ports to assume when a configured proxy URL omits one, per proxy scheme.
|
# Ports to assume when a configured endpoint URL omits one, per scheme.
|
||||||
_PROXY_DEFAULT_PORTS = {
|
_SCHEME_DEFAULT_PORTS = {
|
||||||
'http': 80,
|
'http': 80,
|
||||||
'https': 443,
|
'https': 443,
|
||||||
'socks4': 1080,
|
'socks4': 1080,
|
||||||
@@ -129,31 +129,32 @@ def _address_is_global(addr: str) -> bool:
|
|||||||
return bool(ips) and all(ip.is_global for ip in ips)
|
return bool(ips) and all(ip.is_global for ip in ips)
|
||||||
|
|
||||||
|
|
||||||
def _address_allowed_at_connect(addr: str, is_proxy_endpoint: bool = False) -> bool:
|
def _address_allowed_at_connect(addr: str, is_allowed_endpoint: bool = False) -> bool:
|
||||||
"""True if *addr* may be connected to at download time.
|
"""True if *addr* may be connected to at download time.
|
||||||
|
|
||||||
Permits global addresses, and anything at all when the destination is an
|
Permits global addresses, and anything at all when the destination is an
|
||||||
operator-configured proxy (see ``_is_proxy_endpoint``). Internal addresses
|
endpoint the operator or the image configured — a proxy, or the PO token
|
||||||
are otherwise refused with no blanket exception: media URLs that yt-dlp
|
provider (see ``_is_allowed_endpoint``). Internal addresses are otherwise
|
||||||
derives from a remote manifest are attacker-controlled and reach this policy
|
refused with no blanket exception: media URLs that yt-dlp derives from a
|
||||||
without passing ``validate_url``, so any range opened here is a range a
|
remote manifest are attacker-controlled and reach this policy without passing
|
||||||
hostile playlist can read from the server's own network. Blocks link-local
|
``validate_url``, so any range opened here is a range a hostile playlist can
|
||||||
|
read from the server's own network. Blocks link-local
|
||||||
(cloud metadata at 169.254.169.254), private (RFC1918), loopback,
|
(cloud metadata at 169.254.169.254), private (RFC1918), loopback,
|
||||||
unique-local and every other non-global range.
|
unique-local and every other non-global range.
|
||||||
"""
|
"""
|
||||||
ips = _ips_to_judge(addr)
|
ips = _ips_to_judge(addr)
|
||||||
if not ips:
|
if not ips:
|
||||||
return False
|
return False
|
||||||
return is_proxy_endpoint or all(ip.is_global for ip in ips)
|
return is_allowed_endpoint or all(ip.is_global for ip in ips)
|
||||||
|
|
||||||
|
|
||||||
def _proxy_endpoint(proxy_url: str):
|
def _url_endpoint(url: str):
|
||||||
"""Parse a proxy URL into a ``(hostname, port)`` pair, or ``None`` if it has
|
"""Parse a configured URL into a ``(hostname, port)`` pair, or ``None`` if it
|
||||||
no usable host. Used to scope the internal-address allowance to that endpoint
|
has no usable host. Used to scope the internal-address allowance to that
|
||||||
alone."""
|
endpoint alone."""
|
||||||
if not isinstance(proxy_url, str) or not proxy_url.strip():
|
if not isinstance(url, str) or not url.strip():
|
||||||
return None
|
return None
|
||||||
candidate = proxy_url.strip()
|
candidate = url.strip()
|
||||||
if '://' not in candidate:
|
if '://' not in candidate:
|
||||||
# Bare host:port, as accepted by the *_proxy environment variables.
|
# Bare host:port, as accepted by the *_proxy environment variables.
|
||||||
candidate = '//' + candidate
|
candidate = '//' + candidate
|
||||||
@@ -165,23 +166,28 @@ def _proxy_endpoint(proxy_url: str):
|
|||||||
if not hostname:
|
if not hostname:
|
||||||
return None
|
return None
|
||||||
if port is None:
|
if port is None:
|
||||||
port = _PROXY_DEFAULT_PORTS.get(parts.scheme.lower())
|
port = _SCHEME_DEFAULT_PORTS.get(parts.scheme.lower())
|
||||||
return (hostname.rstrip('.').lower(), port)
|
return (hostname.rstrip('.').lower(), port)
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoints(urls) -> set:
|
||||||
|
"""The parseable endpoints among *urls*, dropping any that name no host."""
|
||||||
|
return {ep for ep in map(_url_endpoint, urls) if ep is not None}
|
||||||
|
|
||||||
|
|
||||||
def _collect_proxy_endpoints(proxy_urls) -> set:
|
def _collect_proxy_endpoints(proxy_urls) -> set:
|
||||||
"""Endpoints of every proxy this download may legitimately dial: the explicit
|
"""Endpoints of every proxy this download may legitimately dial: the explicit
|
||||||
yt-dlp ``proxy`` option plus the ``*_proxy`` environment variables yt-dlp falls
|
yt-dlp ``proxy`` option plus the ``*_proxy`` environment variables yt-dlp falls
|
||||||
back to. All are operator-configured, unlike the URLs inside fetched media."""
|
back to. All are operator-configured, unlike the URLs inside fetched media."""
|
||||||
candidates = list(proxy_urls) + list(urllib.request.getproxies().values())
|
candidates = list(proxy_urls) + list(urllib.request.getproxies().values())
|
||||||
return {ep for ep in map(_proxy_endpoint, candidates) if ep is not None}
|
return _endpoints(candidates)
|
||||||
|
|
||||||
|
|
||||||
# Captured at import so re-installing the guard never wraps the wrapper.
|
# Captured at import so re-installing the guard never wraps the wrapper.
|
||||||
_real_getaddrinfo = socket.getaddrinfo
|
_real_getaddrinfo = socket.getaddrinfo
|
||||||
|
|
||||||
# Populated by install_socket_guard; empty means no internal destination is allowed.
|
# Populated by install_socket_guard; empty means no internal destination is allowed.
|
||||||
_allowed_proxy_endpoints: set = set()
|
_allowed_endpoints: set = set()
|
||||||
|
|
||||||
|
|
||||||
def _normalise_port(port):
|
def _normalise_port(port):
|
||||||
@@ -196,28 +202,29 @@ def _normalise_port(port):
|
|||||||
return port
|
return port
|
||||||
|
|
||||||
|
|
||||||
def _is_proxy_endpoint(host, port) -> bool:
|
def _is_allowed_endpoint(host, port) -> bool:
|
||||||
"""True when host:port is exactly an endpoint the operator configured as a
|
"""True when host:port is exactly one of the endpoints this download is
|
||||||
proxy. Matching is on the configured host *string*, not on the resolved
|
configured to dial — a proxy or the PO token provider. Matching is on the
|
||||||
address, so a hostile media URL cannot borrow the allowance by resolving to
|
configured host *string*, not on the resolved address, so a hostile media URL
|
||||||
the same address under a different name."""
|
cannot borrow the allowance by resolving to the same address under a
|
||||||
if not _allowed_proxy_endpoints or host is None:
|
different name."""
|
||||||
|
if not _allowed_endpoints or host is None:
|
||||||
return False
|
return False
|
||||||
return (str(host).rstrip('.').lower(), _normalise_port(port)) in _allowed_proxy_endpoints
|
return (str(host).rstrip('.').lower(), _normalise_port(port)) in _allowed_endpoints
|
||||||
|
|
||||||
|
|
||||||
def _guarded_getaddrinfo(host, *args, **kwargs):
|
def _guarded_getaddrinfo(host, *args, **kwargs):
|
||||||
results = _real_getaddrinfo(host, *args, **kwargs)
|
results = _real_getaddrinfo(host, *args, **kwargs)
|
||||||
# Mirrors getaddrinfo(host, port, ...): port is the first optional argument.
|
# Mirrors getaddrinfo(host, port, ...): port is the first optional argument.
|
||||||
port = args[0] if args else kwargs.get('port')
|
port = args[0] if args else kwargs.get('port')
|
||||||
is_proxy = _is_proxy_endpoint(host, port)
|
is_configured = _is_allowed_endpoint(host, port)
|
||||||
allowed = [r for r in results if _address_allowed_at_connect(r[4][0], is_proxy)]
|
allowed = [r for r in results if _address_allowed_at_connect(r[4][0], is_configured)]
|
||||||
if not allowed:
|
if not allowed:
|
||||||
raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}')
|
raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}')
|
||||||
return allowed
|
return allowed
|
||||||
|
|
||||||
|
|
||||||
def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
|
def install_socket_guard(allow_private: bool = False, proxy_urls=(), service_urls=()) -> None:
|
||||||
"""Enforce the no-internal-hosts policy at actual connection time.
|
"""Enforce the no-internal-hosts policy at actual connection time.
|
||||||
|
|
||||||
``validate_url`` only checks the *submitted* URL string; yt-dlp then follows
|
``validate_url`` only checks the *submitted* URL string; yt-dlp then follows
|
||||||
@@ -230,12 +237,16 @@ def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
|
|||||||
isolation as the backstop.
|
isolation as the backstop.
|
||||||
|
|
||||||
*proxy_urls* are the operator's configured proxies (yt-dlp's ``proxy`` option;
|
*proxy_urls* are the operator's configured proxies (yt-dlp's ``proxy`` option;
|
||||||
the ``*_proxy`` environment variables are picked up automatically). A proxy is
|
the ``*_proxy`` environment variables are picked up automatically), and
|
||||||
reachable at its own host:port wherever it lives — loopback, the LAN, a VPN
|
*service_urls* the helper services the download itself has to reach — the PO
|
||||||
range — and nothing else internal is. That costs proxied setups nothing and
|
token provider this image ships and starts on loopback. Each is reachable at
|
||||||
gives away nothing: yt-dlp resolves the proxy itself at exactly that host:port,
|
its own host:port wherever it lives — loopback, the LAN, a VPN range — and
|
||||||
and a media URL is either handed to the proxy unresolved or resolved on its own
|
nothing else internal is. That costs those setups nothing and gives away
|
||||||
merits — never inheriting the proxy's allowance.
|
little: yt-dlp dials each at exactly that host:port, and a media URL is either
|
||||||
|
handed to the proxy unresolved or resolved on its own merits — never
|
||||||
|
inheriting the allowance. A hostile media URL naming an allowed endpoint
|
||||||
|
reaches only what is listening there: a proxy that would have fetched it
|
||||||
|
anyway, or a token server with two endpoints and nothing to read.
|
||||||
|
|
||||||
When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not
|
When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not
|
||||||
installed at all, so proxy/VPN setups that route through private or Fake-IP
|
installed at all, so proxy/VPN setups that route through private or Fake-IP
|
||||||
@@ -243,10 +254,13 @@ def install_socket_guard(allow_private: bool = False, proxy_urls=()) -> None:
|
|||||||
"""
|
"""
|
||||||
if allow_private:
|
if allow_private:
|
||||||
return
|
return
|
||||||
_allowed_proxy_endpoints.clear()
|
proxy_endpoints = _collect_proxy_endpoints(proxy_urls)
|
||||||
_allowed_proxy_endpoints.update(_collect_proxy_endpoints(proxy_urls))
|
service_endpoints = _endpoints(service_urls) - proxy_endpoints
|
||||||
for host, port in sorted(_allowed_proxy_endpoints, key=lambda ep: (ep[0], ep[1] or 0)):
|
_allowed_endpoints.clear()
|
||||||
log.info(f'Allowing connections to configured proxy {host}:{port}')
|
_allowed_endpoints.update(proxy_endpoints | service_endpoints)
|
||||||
|
for label, endpoints in (('proxy', proxy_endpoints), ('service', service_endpoints)):
|
||||||
|
for host, port in sorted(endpoints, key=lambda ep: (ep[0], ep[1] or 0)):
|
||||||
|
log.info(f'Allowing connections to configured {label} {host}:{port}')
|
||||||
socket.getaddrinfo = _guarded_getaddrinfo
|
socket.getaddrinfo = _guarded_getaddrinfo
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+295
-50
@@ -32,6 +32,55 @@ from urllib.parse import urlsplit
|
|||||||
|
|
||||||
log = logging.getLogger('ytdl')
|
log = logging.getLogger('ytdl')
|
||||||
|
|
||||||
|
|
||||||
|
# Fragmented and live downloads can emit a warning per fragment, and the joined
|
||||||
|
# text is persisted with the completed queue and broadcast to every client, so
|
||||||
|
# only the last few distinct warnings are kept.
|
||||||
|
_MAX_RETAINED_WARNINGS = 5
|
||||||
|
|
||||||
|
_REPORT_LABEL_RE = re.compile(r'^(?:ERROR|WARNING):\s*')
|
||||||
|
|
||||||
|
|
||||||
|
def _report_body(message):
|
||||||
|
"""yt-dlp labels errors with an ``ERROR:`` prefix but hands warnings to the
|
||||||
|
logger unlabelled, so compare the two with any such label removed."""
|
||||||
|
return _REPORT_LABEL_RE.sub('', message).strip()
|
||||||
|
|
||||||
|
|
||||||
|
class _DownloadYtdlLogger:
|
||||||
|
"""Forward yt-dlp output while retaining warnings for failed downloads."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._warnings = collections.deque(maxlen=_MAX_RETAINED_WARNINGS)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def warnings(self):
|
||||||
|
return list(self._warnings)
|
||||||
|
|
||||||
|
def debug(self, msg):
|
||||||
|
log.debug('%s', msg)
|
||||||
|
|
||||||
|
def warning(self, msg):
|
||||||
|
log.warning('%s', msg)
|
||||||
|
if msg is not None and (warning := str(msg).strip()) and warning not in self._warnings:
|
||||||
|
self._warnings.append(warning)
|
||||||
|
|
||||||
|
def error(self, msg):
|
||||||
|
log.error('%s', msg)
|
||||||
|
|
||||||
|
def failure_message(self, error_text):
|
||||||
|
"""Retained warnings followed by *error_text*, kept last so the actual
|
||||||
|
error stays prominent under the context that explains it."""
|
||||||
|
lines = self.warnings
|
||||||
|
error_text = (error_text or '').strip()
|
||||||
|
if not error_text:
|
||||||
|
return '\n'.join(lines)
|
||||||
|
if lines and _report_body(lines[-1]) == _report_body(error_text):
|
||||||
|
lines.pop()
|
||||||
|
lines.append(error_text)
|
||||||
|
return '\n'.join(lines)
|
||||||
|
|
||||||
|
|
||||||
# Python 3.14 switches the default multiprocessing start method on Linux
|
# Python 3.14 switches the default multiprocessing start method on Linux
|
||||||
# (this app's only supported deployment target, per the Dockerfile) from fork
|
# (this app's only supported deployment target, per the Dockerfile) from fork
|
||||||
# to forkserver. Download._download relies on inheriting process state the
|
# to forkserver. Download._download relies on inheriting process state the
|
||||||
@@ -43,6 +92,36 @@ log = logging.getLogger('ytdl')
|
|||||||
# vanish in the child can deadlock it silently before it does any work. This
|
# vanish in the child can deadlock it silently before it does any work. This
|
||||||
# app creates background threads (executors, notifier callbacks) well before
|
# app creates background threads (executors, notifier callbacks) well before
|
||||||
# any download starts, so forcing fork there reproduces exactly that hazard.
|
# any download starts, so forcing fork there reproduces exactly that hazard.
|
||||||
|
# The image ships yt-dlp's bgutil PO token provider and starts it on loopback
|
||||||
|
# (docker-entrypoint.sh); the plugin dials this URL unless pointed elsewhere.
|
||||||
|
# Without a token YouTube serves 403s, so the connect-time guard has to let the
|
||||||
|
# download subprocess reach it.
|
||||||
|
_POT_PROVIDER_DEFAULT_URL = 'http://127.0.0.1:4416'
|
||||||
|
|
||||||
|
# extractor-arg keys the bgutil HTTP provider reads its base URL from: the
|
||||||
|
# current one first, then the deprecated form it still honours.
|
||||||
|
_POT_PROVIDER_BASE_URL_ARGS = (
|
||||||
|
('youtubepot-bgutilhttp', 'base_url'),
|
||||||
|
('youtube', 'getpot_bgutil_baseurl'),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _pot_provider_urls(ytdl_opts: dict) -> tuple:
|
||||||
|
"""Every PO token provider endpoint this download may dial: the bundled one,
|
||||||
|
plus any the operator pointed yt-dlp at through ``extractor_args``. The
|
||||||
|
bundled server runs either way, so it stays allowed even when a base URL is
|
||||||
|
configured."""
|
||||||
|
urls = [_POT_PROVIDER_DEFAULT_URL]
|
||||||
|
extractor_args = ytdl_opts.get('extractor_args')
|
||||||
|
if isinstance(extractor_args, dict):
|
||||||
|
for ie_key, arg in _POT_PROVIDER_BASE_URL_ARGS:
|
||||||
|
section = extractor_args.get(ie_key)
|
||||||
|
values = section.get(arg) if isinstance(section, dict) else None
|
||||||
|
if values:
|
||||||
|
urls.append(values[0])
|
||||||
|
return tuple(urls)
|
||||||
|
|
||||||
|
|
||||||
_MP_CTX = (
|
_MP_CTX = (
|
||||||
multiprocessing.get_context("fork")
|
multiprocessing.get_context("fork")
|
||||||
if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods()
|
if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods()
|
||||||
@@ -147,6 +226,56 @@ def _sanitize_path_component(value: Any) -> Any:
|
|||||||
return value.lstrip('.').strip() or '_'
|
return value.lstrip('.').strip() or '_'
|
||||||
|
|
||||||
|
|
||||||
|
# Room left for the suffixes yt-dlp appends after prepare_filename has run:
|
||||||
|
# '.part' and '.ytdl' while the download is in flight, '.f<format_id>' for a
|
||||||
|
# stream fetched on its own before merging, '-Frag<n>' for fragmented
|
||||||
|
# downloads. A name trimmed to exactly the limit would still fail the moment
|
||||||
|
# one of those is added, which is what the '.part' in the reported errors is.
|
||||||
|
_NAME_SUFFIX_RESERVE_BYTES = 32
|
||||||
|
# POSIX guarantees at least this much, and it is what ext4/xfs/btrfs allow.
|
||||||
|
_FALLBACK_NAME_MAX_BYTES = 255
|
||||||
|
# Keep a recognisable stem even on a filesystem with a very short limit.
|
||||||
|
_MIN_STEM_BYTES = 16
|
||||||
|
# Longer than this is not really an extension (a title ending in '.something'),
|
||||||
|
# so the whole name is treated as the stem rather than preserving it.
|
||||||
|
_MAX_EXT_BYTES = 16
|
||||||
|
|
||||||
|
|
||||||
|
def _name_max_bytes(directory: str) -> int:
|
||||||
|
"""The filesystem's filename limit, in bytes, for *directory*."""
|
||||||
|
try:
|
||||||
|
return int(os.pathconf(directory or '.', 'PC_NAME_MAX'))
|
||||||
|
except (OSError, ValueError, AttributeError):
|
||||||
|
# The directory may not exist yet (CREATE_CUSTOM_DIRS makes it during
|
||||||
|
# the download), and pathconf is not available on every platform.
|
||||||
|
return _FALLBACK_NAME_MAX_BYTES
|
||||||
|
|
||||||
|
|
||||||
|
def _trim_to_name_max(path: str) -> str:
|
||||||
|
"""Shorten the final component of *path* to what the filesystem accepts.
|
||||||
|
|
||||||
|
The limit is a byte count, not a character count: a title of accented or
|
||||||
|
CJK characters hits it in half as many characters, or fewer. The extension
|
||||||
|
is preserved, since it is what decides how the file is handled afterwards.
|
||||||
|
"""
|
||||||
|
directory, name = os.path.split(path)
|
||||||
|
if not name:
|
||||||
|
return path
|
||||||
|
encoded = name.encode('utf-8', 'surrogatepass')
|
||||||
|
limit = _name_max_bytes(directory) - _NAME_SUFFIX_RESERVE_BYTES
|
||||||
|
if len(encoded) <= limit:
|
||||||
|
return path
|
||||||
|
|
||||||
|
stem, ext = os.path.splitext(name)
|
||||||
|
ext_bytes = ext.encode('utf-8', 'surrogatepass')
|
||||||
|
if len(ext_bytes) > _MAX_EXT_BYTES:
|
||||||
|
stem, ext, ext_bytes = name, '', b''
|
||||||
|
stem_limit = max(limit - len(ext_bytes), _MIN_STEM_BYTES)
|
||||||
|
# 'ignore' drops a multi-byte character the cut landed inside of.
|
||||||
|
trimmed = stem.encode('utf-8', 'surrogatepass')[:stem_limit].decode('utf-8', 'ignore').rstrip()
|
||||||
|
return os.path.join(directory, (trimmed or '_') + ext)
|
||||||
|
|
||||||
|
|
||||||
class _ConfinedYoutubeDL(yt_dlp.YoutubeDL):
|
class _ConfinedYoutubeDL(yt_dlp.YoutubeDL):
|
||||||
"""A ``YoutubeDL`` that refuses to emit any output path outside the allowed roots.
|
"""A ``YoutubeDL`` that refuses to emit any output path outside the allowed roots.
|
||||||
|
|
||||||
@@ -170,6 +299,12 @@ class _ConfinedYoutubeDL(yt_dlp.YoutubeDL):
|
|||||||
|
|
||||||
def prepare_filename(self, *args, **kwargs):
|
def prepare_filename(self, *args, **kwargs):
|
||||||
filename = super().prepare_filename(*args, **kwargs)
|
filename = super().prepare_filename(*args, **kwargs)
|
||||||
|
# Titles long enough to exceed the filesystem's filename limit are
|
||||||
|
# common on some sites, and the download fails outright when they do.
|
||||||
|
# Every output path comes through here, so trimming once keeps the
|
||||||
|
# main file, its chapter files, thumbnails and subtitles consistent.
|
||||||
|
if filename and filename != '-':
|
||||||
|
filename = _trim_to_name_max(filename)
|
||||||
if filename and filename != '-' and self._allowed_roots:
|
if filename and filename != '-' and self._allowed_roots:
|
||||||
resolved = os.path.realpath(filename)
|
resolved = os.path.realpath(filename)
|
||||||
if not any(_is_within_directory(root, resolved) for root in self._allowed_roots):
|
if not any(_is_within_directory(root, resolved) for root in self._allowed_roots):
|
||||||
@@ -348,6 +483,7 @@ class DownloadInfo:
|
|||||||
clip_end=None,
|
clip_end=None,
|
||||||
live_status=None,
|
live_status=None,
|
||||||
live_release_timestamp=None,
|
live_release_timestamp=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
||||||
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
||||||
@@ -367,6 +503,7 @@ class DownloadInfo:
|
|||||||
self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None
|
self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None
|
||||||
self.playlist_item_limit = playlist_item_limit
|
self.playlist_item_limit = playlist_item_limit
|
||||||
self.split_by_chapters = split_by_chapters
|
self.split_by_chapters = split_by_chapters
|
||||||
|
self.sponsorblock = sponsorblock
|
||||||
self.chapter_template = chapter_template
|
self.chapter_template = chapter_template
|
||||||
self.subtitle_language = subtitle_language
|
self.subtitle_language = subtitle_language
|
||||||
self.subtitle_mode = subtitle_mode
|
self.subtitle_mode = subtitle_mode
|
||||||
@@ -436,6 +573,8 @@ class DownloadInfo:
|
|||||||
self.playlist_item_limit = 0
|
self.playlist_item_limit = 0
|
||||||
if not hasattr(self, "split_by_chapters"):
|
if not hasattr(self, "split_by_chapters"):
|
||||||
self.split_by_chapters = False
|
self.split_by_chapters = False
|
||||||
|
if not hasattr(self, "sponsorblock"):
|
||||||
|
self.sponsorblock = False
|
||||||
if not hasattr(self, "chapter_template"):
|
if not hasattr(self, "chapter_template"):
|
||||||
self.chapter_template = ""
|
self.chapter_template = ""
|
||||||
if not hasattr(self, "subtitle_language"):
|
if not hasattr(self, "subtitle_language"):
|
||||||
@@ -480,6 +619,7 @@ _PERSISTED_DOWNLOAD_FIELDS = (
|
|||||||
"custom_name_prefix",
|
"custom_name_prefix",
|
||||||
"playlist_item_limit",
|
"playlist_item_limit",
|
||||||
"split_by_chapters",
|
"split_by_chapters",
|
||||||
|
"sponsorblock",
|
||||||
"chapter_template",
|
"chapter_template",
|
||||||
"subtitle_language",
|
"subtitle_language",
|
||||||
"subtitle_mode",
|
"subtitle_mode",
|
||||||
@@ -659,11 +799,19 @@ class Download:
|
|||||||
# Re-validate every outbound connection at fetch time. validate_url only
|
# Re-validate every outbound connection at fetch time. validate_url only
|
||||||
# saw the submitted URL string; this catches redirects, DNS rebinding and
|
# saw the submitted URL string; this catches redirects, DNS rebinding and
|
||||||
# attacker-controlled media URLs pulled from a remote manifest, none of
|
# attacker-controlled media URLs pulled from a remote manifest, none of
|
||||||
# which it can see. The configured proxy is passed so that a proxy on an
|
# which it can see. The configured proxy and the PO token provider are
|
||||||
# internal address stays reachable at its own host:port without opening up
|
# passed so that each stays reachable at its own host:port without opening
|
||||||
# anything else. Skipped when ALLOW_PRIVATE_ADDRESSES trusts the environment.
|
# up anything else. Skipped when ALLOW_PRIVATE_ADDRESSES trusts the
|
||||||
install_socket_guard(self.allow_private, proxy_urls=(self.ytdl_opts.get('proxy'),))
|
# environment.
|
||||||
|
install_socket_guard(
|
||||||
|
self.allow_private,
|
||||||
|
proxy_urls=(self.ytdl_opts.get('proxy'),),
|
||||||
|
service_urls=_pot_provider_urls(self.ytdl_opts),
|
||||||
|
)
|
||||||
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
||||||
|
# Bound outside the try so the except branch can read what was captured
|
||||||
|
# before the error was raised.
|
||||||
|
ytdl_logger = _DownloadYtdlLogger()
|
||||||
try:
|
try:
|
||||||
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
||||||
put_status = self._make_progress_hook()
|
put_status = self._make_progress_hook()
|
||||||
@@ -710,6 +858,30 @@ class Download:
|
|||||||
'postprocessor_hooks': [put_status_postprocessor],
|
'postprocessor_hooks': [put_status_postprocessor],
|
||||||
**self.ytdl_opts,
|
**self.ytdl_opts,
|
||||||
}
|
}
|
||||||
|
# Set after the ytdl_opts merge: the failure messages below depend on
|
||||||
|
# this logger, so a user-supplied one must not replace it.
|
||||||
|
ytdl_params['logger'] = ytdl_logger
|
||||||
|
|
||||||
|
# SponsorBlock: mark sponsor segments and cut them out, the same
|
||||||
|
# postprocessor pair the CLI's --sponsorblock-remove sponsor builds.
|
||||||
|
# This has to stay above the chapter-splitting block: yt-dlp runs
|
||||||
|
# same-stage postprocessors in list order, and ModifyChapters must
|
||||||
|
# rewrite the chapter list before FFmpegSplitChapters cuts the file
|
||||||
|
# up, or the chapter files keep the sponsor segments and the
|
||||||
|
# removal desyncs the remaining chapter timings.
|
||||||
|
if getattr(self.info, 'sponsorblock', False):
|
||||||
|
if 'postprocessors' not in ytdl_params:
|
||||||
|
ytdl_params['postprocessors'] = []
|
||||||
|
ytdl_params['postprocessors'].append({
|
||||||
|
'key': 'SponsorBlock',
|
||||||
|
'categories': ['sponsor'],
|
||||||
|
'when': 'after_filter',
|
||||||
|
})
|
||||||
|
ytdl_params['postprocessors'].append({
|
||||||
|
'key': 'ModifyChapters',
|
||||||
|
'remove_sponsor_segments': ['sponsor'],
|
||||||
|
'force_keyframes': False,
|
||||||
|
})
|
||||||
|
|
||||||
# Add chapter splitting options if enabled
|
# Add chapter splitting options if enabled
|
||||||
if self.info.split_by_chapters:
|
if self.info.split_by_chapters:
|
||||||
@@ -732,11 +904,15 @@ class Download:
|
|||||||
)
|
)
|
||||||
|
|
||||||
ret = self._make_youtube_dl(ytdl_params).download([self.info.url])
|
ret = self._make_youtube_dl(ytdl_params).download([self.info.url])
|
||||||
self.status_queue.put({'status': 'finished' if ret == 0 else 'error'})
|
if ret == 0:
|
||||||
|
self.status_queue.put({'status': 'finished'})
|
||||||
|
else:
|
||||||
|
msg = '\n'.join(ytdl_logger.warnings) or f'yt-dlp failed with exit code {ret}'
|
||||||
|
self.status_queue.put({'status': 'error', 'msg': msg})
|
||||||
log.info(f"Finished download for: {self.info.title}")
|
log.info(f"Finished download for: {self.info.title}")
|
||||||
except yt_dlp.utils.YoutubeDLError as exc:
|
except yt_dlp.utils.YoutubeDLError as exc:
|
||||||
log.error(f"Download error for {self.info.title}: {str(exc)}")
|
log.error(f"Download error for {self.info.title}: {str(exc)}")
|
||||||
self.status_queue.put({'status': 'error', 'msg': str(exc)})
|
self.status_queue.put({'status': 'error', 'msg': ytdl_logger.failure_message(str(exc))})
|
||||||
|
|
||||||
async def start(self, notifier, executor=None):
|
async def start(self, notifier, executor=None):
|
||||||
log.info(f"Preparing download for: {self.info.title}")
|
log.info(f"Preparing download for: {self.info.title}")
|
||||||
@@ -851,7 +1027,17 @@ class Download:
|
|||||||
if not rel_name.lower().endswith(allowed_caption_exts):
|
if not rel_name.lower().endswith(allowed_caption_exts):
|
||||||
continue
|
continue
|
||||||
self.info.filename = rel_name
|
self.info.filename = rel_name
|
||||||
self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
|
# Stat only on a terminal status. yt-dlp documents 'filename' as
|
||||||
|
# always present in a progress hook, but until the download
|
||||||
|
# finishes the bytes are in tmpfilename and 'filename' is a
|
||||||
|
# destination that does not exist yet -- so this was two
|
||||||
|
# blocking syscalls on the event loop, twice a second per
|
||||||
|
# active download, to arrive at None. A stat that takes seconds
|
||||||
|
# on a contended filesystem stalls every other request the
|
||||||
|
# server is serving. Nothing displays the size before
|
||||||
|
# completion: the Downloading table has no size column.
|
||||||
|
if status.get('status') == 'finished':
|
||||||
|
self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
|
||||||
if getattr(self.info, 'download_type', '') == 'thumbnail':
|
if getattr(self.info, 'download_type', '') == 'thumbnail':
|
||||||
# The thumbnail convertor always emits a .jpg, but yt-dlp may
|
# The thumbnail convertor always emits a .jpg, but yt-dlp may
|
||||||
# report the pre-conversion media/thumbnail extension
|
# report the pre-conversion media/thumbnail extension
|
||||||
@@ -925,6 +1111,19 @@ class PersistentQueue:
|
|||||||
self.path = f"{path}.json"
|
self.path = f"{path}.json"
|
||||||
self.store = AtomicJsonStore(self.path, kind=f"persistent_queue:{name}")
|
self.store = AtomicJsonStore(self.path, kind=f"persistent_queue:{name}")
|
||||||
self.dict = OrderedDict()
|
self.dict = OrderedDict()
|
||||||
|
# A state write fsyncs twice (the file and its directory). On a slow or
|
||||||
|
# contended filesystem that is seconds, and running it inline in an
|
||||||
|
# async caller blocked the event loop -- every other request stalled
|
||||||
|
# behind a single queue mutation. One dedicated thread keeps the writes
|
||||||
|
# off the loop and, being single, keeps them ordered. The default
|
||||||
|
# executor is not usable for this: extract_info shares it and can hold
|
||||||
|
# its threads for minutes, which is exactly when state writes happen.
|
||||||
|
self._store_executor = ThreadPoolExecutor(
|
||||||
|
max_workers=1, thread_name_prefix=f"state-{name}")
|
||||||
|
# Guards the mutate-write-rollback section. The write is awaited now, so
|
||||||
|
# without this two callers could interleave between changing self.dict
|
||||||
|
# and persisting it, and a rollback could revert the wrong entry.
|
||||||
|
self._lock = asyncio.Lock()
|
||||||
|
|
||||||
def load(self):
|
def load(self):
|
||||||
for k, v in self.saved_items():
|
for k, v in self.saved_items():
|
||||||
@@ -965,8 +1164,14 @@ class PersistentQueue:
|
|||||||
for key, download in self.dict.items()
|
for key, download in self.dict.items()
|
||||||
]
|
]
|
||||||
|
|
||||||
def _save_dict(self):
|
async def _save_dict_async(self):
|
||||||
self.store.save({"items": self._serialize_items()})
|
# Serialize on the event loop -- it is pure CPU and sub-millisecond --
|
||||||
|
# and hand the finished payload to the writer thread. Serializing in the
|
||||||
|
# thread instead would have it walk live DownloadInfo objects while the
|
||||||
|
# loop mutates them.
|
||||||
|
payload = {"items": self._serialize_items()}
|
||||||
|
await asyncio.get_running_loop().run_in_executor(
|
||||||
|
self._store_executor, self.store.save, payload)
|
||||||
|
|
||||||
def _load_state_items(self):
|
def _load_state_items(self):
|
||||||
payload = self.store.load()
|
payload = self.store.load()
|
||||||
@@ -1007,32 +1212,39 @@ class PersistentQueue:
|
|||||||
self.store.save({"items": items})
|
self.store.save({"items": items})
|
||||||
return items
|
return items
|
||||||
|
|
||||||
def put(self, value):
|
async def put(self, value):
|
||||||
key = value.info.url
|
key = value.info.url
|
||||||
old = self.dict.get(key)
|
async with self._lock:
|
||||||
self.dict[key] = value
|
old = self.dict.get(key)
|
||||||
try:
|
self.dict[key] = value
|
||||||
self._save_dict()
|
|
||||||
except Exception:
|
|
||||||
if old is None:
|
|
||||||
del self.dict[key]
|
|
||||||
else:
|
|
||||||
self.dict[key] = old
|
|
||||||
raise
|
|
||||||
|
|
||||||
def delete(self, key):
|
|
||||||
if key in self.dict:
|
|
||||||
old = self.dict[key]
|
|
||||||
del self.dict[key]
|
|
||||||
try:
|
try:
|
||||||
self._save_dict()
|
await self._save_dict_async()
|
||||||
except Exception:
|
except Exception:
|
||||||
self.dict[key] = old
|
if old is None:
|
||||||
|
del self.dict[key]
|
||||||
|
else:
|
||||||
|
self.dict[key] = old
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
async def delete(self, key):
|
||||||
|
async with self._lock:
|
||||||
|
if key in self.dict:
|
||||||
|
old = self.dict[key]
|
||||||
|
del self.dict[key]
|
||||||
|
try:
|
||||||
|
await self._save_dict_async()
|
||||||
|
except Exception:
|
||||||
|
self.dict[key] = old
|
||||||
|
raise
|
||||||
|
|
||||||
def empty(self):
|
def empty(self):
|
||||||
return not bool(self.dict)
|
return not bool(self.dict)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
# wait=True so a write already in flight reaches disk before the
|
||||||
|
# process exits; there is at most one, and it is the newest state.
|
||||||
|
self._store_executor.shutdown(wait=True)
|
||||||
|
|
||||||
class DownloadQueue:
|
class DownloadQueue:
|
||||||
def __init__(self, config, notifier):
|
def __init__(self, config, notifier):
|
||||||
self.config = config
|
self.config = config
|
||||||
@@ -1063,13 +1275,33 @@ class DownloadQueue:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def __is_channel_extraction(entry):
|
def __is_channel_extraction(entry):
|
||||||
"""Return True when yt-dlp reported a channel tab as a playlist.
|
"""Return True when yt-dlp reported a channel as a playlist.
|
||||||
|
|
||||||
YouTube channel tabs are extracted with ``_type: 'playlist'`` but set
|
A channel *tab* -- ``/channel/UC...``, ``/@handle/videos``, and the
|
||||||
``id`` equal to ``channel_id``; real playlists keep a distinct id.
|
streams, shorts and playlists tabs -- is extracted with ``id`` equal to
|
||||||
|
``channel_id``. A channel addressed without a tab keeps the form it was
|
||||||
|
asked for instead: ``@handle`` for a handle URL and the vanity name for
|
||||||
|
a legacy ``/c/`` URL. Both of those match ``uploader_id``, which is the
|
||||||
|
handle either way, so compare against it as well.
|
||||||
|
|
||||||
|
A real playlist has an id of its own and matches neither, even though
|
||||||
|
it also carries its owner's ``channel_id``.
|
||||||
"""
|
"""
|
||||||
channel_id = entry.get('channel_id')
|
channel_id = entry.get('channel_id')
|
||||||
return bool(channel_id) and entry.get('id') == channel_id
|
entry_id = entry.get('id')
|
||||||
|
if not channel_id or not entry_id:
|
||||||
|
return False
|
||||||
|
if entry_id == channel_id:
|
||||||
|
return True
|
||||||
|
uploader_id = entry.get('uploader_id')
|
||||||
|
if not uploader_id:
|
||||||
|
return False
|
||||||
|
# Compared without case because a legacy vanity name and the handle it
|
||||||
|
# became need not agree on it. No playlist id can collide here: those
|
||||||
|
# are 'PL...', 'OLAK...' and the like, never a handle.
|
||||||
|
handle = uploader_id.casefold()
|
||||||
|
entry_id = entry_id.casefold()
|
||||||
|
return handle in (entry_id, f'@{entry_id}')
|
||||||
|
|
||||||
async def __import_queue(self):
|
async def __import_queue(self):
|
||||||
for k, v in self.queue.saved_items():
|
for k, v in self.queue.saved_items():
|
||||||
@@ -1189,8 +1421,8 @@ class DownloadQueue:
|
|||||||
if not info.error:
|
if not info.error:
|
||||||
info.error = str(exc)
|
info.error = str(exc)
|
||||||
self._unregister_scheduled(url)
|
self._unregister_scheduled(url)
|
||||||
self.queue.delete(url)
|
await self.queue.delete(url)
|
||||||
self.done.put(download)
|
await self.done.put(download)
|
||||||
await self.notifier.completed(info)
|
await self.notifier.completed(info)
|
||||||
else:
|
else:
|
||||||
log.warning(
|
log.warning(
|
||||||
@@ -1224,9 +1456,9 @@ class DownloadQueue:
|
|||||||
await self.notifier.updated(info)
|
await self.notifier.updated(info)
|
||||||
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
||||||
|
|
||||||
def _schedule_upcoming_download(self, download: Download) -> None:
|
async def _schedule_upcoming_download(self, download: Download) -> None:
|
||||||
download.info.status = 'scheduled'
|
download.info.status = 'scheduled'
|
||||||
self.queue.put(download)
|
await self.queue.put(download)
|
||||||
self._register_scheduled(download)
|
self._register_scheduled(download)
|
||||||
|
|
||||||
def _force_start_scheduled(self, download: Download) -> None:
|
def _force_start_scheduled(self, download: Download) -> None:
|
||||||
@@ -1245,9 +1477,9 @@ class DownloadQueue:
|
|||||||
log.info(f"Download {download.info.title} was canceled, skipping start.")
|
log.info(f"Download {download.info.title} was canceled, skipping start.")
|
||||||
return
|
return
|
||||||
await download.start(self.notifier, self._download_executor)
|
await download.start(self.notifier, self._download_executor)
|
||||||
self._post_download_cleanup(download)
|
await self._post_download_cleanup(download)
|
||||||
|
|
||||||
def _post_download_cleanup(self, download):
|
async def _post_download_cleanup(self, download):
|
||||||
if download.info.status != 'finished':
|
if download.info.status != 'finished':
|
||||||
if download.tmpfilename and os.path.isfile(download.tmpfilename):
|
if download.tmpfilename and os.path.isfile(download.tmpfilename):
|
||||||
try:
|
try:
|
||||||
@@ -1267,11 +1499,11 @@ class DownloadQueue:
|
|||||||
download.info.size = None
|
download.info.size = None
|
||||||
download.close()
|
download.close()
|
||||||
if self.queue.exists(download.info.url):
|
if self.queue.exists(download.info.url):
|
||||||
self.queue.delete(download.info.url)
|
await self.queue.delete(download.info.url)
|
||||||
if download.canceled:
|
if download.canceled:
|
||||||
bg_tasks.create_task(self.notifier.canceled(download.info.url), name="notify_canceled")
|
bg_tasks.create_task(self.notifier.canceled(download.info.url), name="notify_canceled")
|
||||||
else:
|
else:
|
||||||
self.done.put(download)
|
await self.done.put(download)
|
||||||
bg_tasks.create_task(self.notifier.completed(download.info), name="notify_completed")
|
bg_tasks.create_task(self.notifier.completed(download.info), name="notify_completed")
|
||||||
try:
|
try:
|
||||||
clear_after = int(self.config.CLEAR_COMPLETED_AFTER)
|
clear_after = int(self.config.CLEAR_COMPLETED_AFTER)
|
||||||
@@ -1379,12 +1611,12 @@ class DownloadQueue:
|
|||||||
)
|
)
|
||||||
if auto_start is True:
|
if auto_start is True:
|
||||||
if is_upcoming:
|
if is_upcoming:
|
||||||
self._schedule_upcoming_download(download)
|
await self._schedule_upcoming_download(download)
|
||||||
else:
|
else:
|
||||||
self.queue.put(download)
|
await self.queue.put(download)
|
||||||
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
||||||
else:
|
else:
|
||||||
self.pending.put(download)
|
await self.pending.put(download)
|
||||||
await self.notifier.added(dl)
|
await self.notifier.added(dl)
|
||||||
|
|
||||||
def __write_feed_metadata_sync(self, entry, etype, download_type, folder,
|
def __write_feed_metadata_sync(self, entry, etype, download_type, folder,
|
||||||
@@ -1484,6 +1716,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen=None,
|
_add_gen=None,
|
||||||
retry_entry=None,
|
retry_entry=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if not entry:
|
if not entry:
|
||||||
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
||||||
@@ -1527,6 +1760,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
retry_entry,
|
retry_entry,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
elif etype == 'playlist' or etype == 'channel':
|
elif etype == 'playlist' or etype == 'channel':
|
||||||
if etype == 'playlist' and self.__is_channel_extraction(entry):
|
if etype == 'playlist' and self.__is_channel_extraction(entry):
|
||||||
@@ -1594,6 +1828,7 @@ class DownloadQueue:
|
|||||||
clip_end,
|
clip_end,
|
||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if any(res['status'] == 'error' for res in results):
|
if any(res['status'] == 'error' for res in results):
|
||||||
@@ -1634,6 +1869,7 @@ class DownloadQueue:
|
|||||||
clip_end=clip_end,
|
clip_end=clip_end,
|
||||||
live_status=entry.get('live_status'),
|
live_status=entry.get('live_status'),
|
||||||
live_release_timestamp=entry.get('release_timestamp'),
|
live_release_timestamp=entry.get('release_timestamp'),
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
await self.__add_download(dl, auto_start)
|
await self.__add_download(dl, auto_start)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
@@ -1689,7 +1925,7 @@ class DownloadQueue:
|
|||||||
info.status = 'error'
|
info.status = 'error'
|
||||||
info.msg = msg
|
info.msg = msg
|
||||||
download = Download(None, None, None, None, quality, format, {}, info)
|
download = Download(None, None, None, None, quality, format, {}, info)
|
||||||
self.done.put(download)
|
await self.done.put(download)
|
||||||
await self.notifier.completed(info)
|
await self.notifier.completed(info)
|
||||||
|
|
||||||
async def add(
|
async def add(
|
||||||
@@ -1714,13 +1950,14 @@ class DownloadQueue:
|
|||||||
already=None,
|
already=None,
|
||||||
_add_gen=None,
|
_add_gen=None,
|
||||||
retry_entry=None,
|
retry_entry=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if ytdl_options_presets is None:
|
if ytdl_options_presets is None:
|
||||||
ytdl_options_presets = []
|
ytdl_options_presets = []
|
||||||
log.info(
|
log.info(
|
||||||
f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} '
|
f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} '
|
||||||
f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} '
|
f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} '
|
||||||
f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=}'
|
f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=} {sponsorblock=}'
|
||||||
)
|
)
|
||||||
if already is None:
|
if already is None:
|
||||||
_add_gen = self._add_generation
|
_add_gen = self._add_generation
|
||||||
@@ -1783,6 +2020,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
retry_entry,
|
retry_entry,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def retry(self, id):
|
async def retry(self, id):
|
||||||
@@ -1819,6 +2057,7 @@ class DownloadQueue:
|
|||||||
info.clip_start,
|
info.clip_start,
|
||||||
info.clip_end,
|
info.clip_end,
|
||||||
retry_entry=info.entry,
|
retry_entry=info.entry,
|
||||||
|
sponsorblock=info.sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def add_entry(
|
async def add_entry(
|
||||||
@@ -1840,6 +2079,7 @@ class DownloadQueue:
|
|||||||
ytdl_options_overrides=None,
|
ytdl_options_overrides=None,
|
||||||
clip_start=None,
|
clip_start=None,
|
||||||
clip_end=None,
|
clip_end=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if ytdl_options_presets is None:
|
if ytdl_options_presets is None:
|
||||||
ytdl_options_presets = []
|
ytdl_options_presets = []
|
||||||
@@ -1865,17 +2105,18 @@ class DownloadQueue:
|
|||||||
clip_end,
|
clip_end,
|
||||||
already,
|
already,
|
||||||
None,
|
None,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def start_pending(self, ids):
|
async def start_pending(self, ids):
|
||||||
for id in ids:
|
for id in ids:
|
||||||
if self.pending.exists(id):
|
if self.pending.exists(id):
|
||||||
dl = self.pending.get(id)
|
dl = self.pending.get(id)
|
||||||
self.pending.delete(id)
|
await self.pending.delete(id)
|
||||||
if getattr(dl.info, 'live_status', None) == 'is_upcoming':
|
if getattr(dl.info, 'live_status', None) == 'is_upcoming':
|
||||||
self._schedule_upcoming_download(dl)
|
await self._schedule_upcoming_download(dl)
|
||||||
else:
|
else:
|
||||||
self.queue.put(dl)
|
await self.queue.put(dl)
|
||||||
bg_tasks.create_task(self.__start_download(dl), name="start_download")
|
bg_tasks.create_task(self.__start_download(dl), name="start_download")
|
||||||
continue
|
continue
|
||||||
if self.queue.exists(id):
|
if self.queue.exists(id):
|
||||||
@@ -1891,7 +2132,7 @@ class DownloadQueue:
|
|||||||
# Track URL so playlist add loop won't re-queue it
|
# Track URL so playlist add loop won't re-queue it
|
||||||
self._canceled_urls.add(id)
|
self._canceled_urls.add(id)
|
||||||
if self.pending.exists(id):
|
if self.pending.exists(id):
|
||||||
self.pending.delete(id)
|
await self.pending.delete(id)
|
||||||
await self.notifier.canceled(id)
|
await self.notifier.canceled(id)
|
||||||
continue
|
continue
|
||||||
if not self.queue.exists(id):
|
if not self.queue.exists(id):
|
||||||
@@ -1904,7 +2145,7 @@ class DownloadQueue:
|
|||||||
dl.cancel()
|
dl.cancel()
|
||||||
else:
|
else:
|
||||||
dl.canceled = True
|
dl.canceled = True
|
||||||
self.queue.delete(id)
|
await self.queue.delete(id)
|
||||||
await self.notifier.canceled(id)
|
await self.notifier.canceled(id)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
|
|
||||||
@@ -1942,7 +2183,7 @@ class DownloadQueue:
|
|||||||
pass
|
pass
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
log.warning(f'deleting file "{rel_name}" for download {id} failed with error message {e!r}')
|
log.warning(f'deleting file "{rel_name}" for download {id} failed with error message {e!r}')
|
||||||
self.done.delete(id)
|
await self.done.delete(id)
|
||||||
await self.notifier.cleared(id)
|
await self.notifier.cleared(id)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
|
|
||||||
@@ -1960,3 +2201,7 @@ class DownloadQueue:
|
|||||||
if download.started() and download.running():
|
if download.started() and download.running():
|
||||||
download.cancel()
|
download.cancel()
|
||||||
self._download_executor.shutdown(wait=False, cancel_futures=True)
|
self._download_executor.shutdown(wait=False, cancel_futures=True)
|
||||||
|
# Unlike the download executor these are drained, not cancelled: a
|
||||||
|
# queued write is the newest state and must reach disk before exit.
|
||||||
|
for queue in (self.queue, self.pending, self.done):
|
||||||
|
queue.close()
|
||||||
|
|||||||
+13
-1
@@ -399,6 +399,16 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="col-12">
|
<div class="col-12">
|
||||||
<div class="row g-2 align-items-center">
|
<div class="row g-2 align-items-center">
|
||||||
|
<div class="col-auto">
|
||||||
|
<div class="form-check form-switch">
|
||||||
|
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-sponsorblock"
|
||||||
|
name="sponsorblock" [(ngModel)]="sponsorblock" (change)="sponsorblockChanged()"
|
||||||
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
|
<label class="form-check-label" for="checkbox-sponsorblock"
|
||||||
|
ngbPopover="Cut out sponsor segments using SponsorBlock's crowd-sourced markers (YouTube only)."
|
||||||
|
triggers="hover" container="body">Remove sponsor segments</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div class="col-auto">
|
<div class="col-auto">
|
||||||
<div class="form-check form-switch">
|
<div class="form-check form-switch">
|
||||||
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
||||||
@@ -693,6 +703,7 @@
|
|||||||
<app-select-all-checkbox #queueMasterCheckboxRef [id]="'queue'" [list]="downloads.queue" (changed)="queueSelectionChanged($event)" />
|
<app-select-all-checkbox #queueMasterCheckboxRef [id]="'queue'" [list]="downloads.queue" (changed)="queueSelectionChanged($event)" />
|
||||||
</th>
|
</th>
|
||||||
<th scope="col">Video</th>
|
<th scope="col">Video</th>
|
||||||
|
<th scope="col" style="width: 7rem;">Format</th>
|
||||||
<th scope="col" style="width: 8rem;">Speed</th>
|
<th scope="col" style="width: 8rem;">Speed</th>
|
||||||
<th scope="col" style="width: 7rem;">ETA</th>
|
<th scope="col" style="width: 7rem;">ETA</th>
|
||||||
<th scope="col" style="width: 6rem;"></th>
|
<th scope="col" style="width: 6rem;"></th>
|
||||||
@@ -726,6 +737,7 @@
|
|||||||
}
|
}
|
||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
|
<td class="text-nowrap">{{ formatLabel(download.value) }}</td>
|
||||||
<td>{{ download.value.speed | speed }}</td>
|
<td>{{ download.value.speed | speed }}</td>
|
||||||
<td>{{ download.value.eta | eta }}</td>
|
<td>{{ download.value.eta | eta }}</td>
|
||||||
<td>
|
<td>
|
||||||
@@ -757,7 +769,7 @@
|
|||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<th scope="col" style="width: 1rem;">
|
<th scope="col" style="width: 1rem;">
|
||||||
<app-select-all-checkbox #doneMasterCheckboxRef [id]="'done'" [list]="downloads.done" (changed)="doneSelectionChanged($event)" />
|
<app-select-all-checkbox #doneMasterCheckboxRef [id]="'done'" [list]="downloads.done" [orderedIds]="cachedSortedDoneIds" (changed)="doneSelectionChanged($event)" />
|
||||||
</th>
|
</th>
|
||||||
<th scope="col">Video</th>
|
<th scope="col">Video</th>
|
||||||
<th scope="col">Type</th>
|
<th scope="col">Type</th>
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import { DownloadsService } from './services/downloads.service';
|
|||||||
import { SubscriptionsService } from './services/subscriptions.service';
|
import { SubscriptionsService } from './services/subscriptions.service';
|
||||||
import { ToastService } from './services/toast.service';
|
import { ToastService } from './services/toast.service';
|
||||||
import { CookieService } from 'ngx-cookie-service';
|
import { CookieService } from 'ngx-cookie-service';
|
||||||
|
import { Download } from './interfaces';
|
||||||
|
|
||||||
class DownloadsServiceStub {
|
class DownloadsServiceStub {
|
||||||
loading = false;
|
loading = false;
|
||||||
@@ -148,6 +149,25 @@ describe('App', () => {
|
|||||||
expect(app).toBeTruthy();
|
expect(app).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('pre-fills the download folder from DEFAULT_FOLDER', () => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
fixture.detectChanges();
|
||||||
|
|
||||||
|
downloads.configurationChanged.next({ DEFAULT_FOLDER: 'youtube' });
|
||||||
|
|
||||||
|
expect(fixture.componentInstance.folder).toBe('youtube');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not overwrite a folder the user already typed', () => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
fixture.detectChanges();
|
||||||
|
fixture.componentInstance.folder = 'music';
|
||||||
|
|
||||||
|
downloads.configurationChanged.next({ DEFAULT_FOLDER: 'youtube' });
|
||||||
|
|
||||||
|
expect(fixture.componentInstance.folder).toBe('music');
|
||||||
|
});
|
||||||
|
|
||||||
it('asIsOrder returns a stable comparator value (insertion order preserved)', () => {
|
it('asIsOrder returns a stable comparator value (insertion order preserved)', () => {
|
||||||
const fixture = TestBed.createComponent(App);
|
const fixture = TestBed.createComponent(App);
|
||||||
const app = fixture.componentInstance;
|
const app = fixture.componentInstance;
|
||||||
@@ -229,6 +249,46 @@ describe('App', () => {
|
|||||||
expect(root.textContent).toContain('starts in');
|
expect(root.textContent).toContain('starts in');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('shows the queued format in the Downloading table', () => {
|
||||||
|
downloads.queue.set('https://example.com/v', {
|
||||||
|
id: 'v1',
|
||||||
|
title: 'Some Video',
|
||||||
|
url: 'https://example.com/v',
|
||||||
|
download_type: 'audio',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'flac',
|
||||||
|
folder: '',
|
||||||
|
custom_name_prefix: '',
|
||||||
|
playlist_item_limit: 0,
|
||||||
|
status: 'downloading',
|
||||||
|
msg: '',
|
||||||
|
percent: 10,
|
||||||
|
speed: 0,
|
||||||
|
eta: 0,
|
||||||
|
filename: '',
|
||||||
|
checked: false,
|
||||||
|
});
|
||||||
|
downloads.queueChanged.next();
|
||||||
|
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
fixture.detectChanges();
|
||||||
|
|
||||||
|
const row = (fixture.nativeElement as HTMLElement).querySelector('tbody tr');
|
||||||
|
expect(row?.textContent).toContain('FLAC');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('labels formats the way the form does, and copes with an unknown one', () => {
|
||||||
|
const app = TestBed.createComponent(App).componentInstance;
|
||||||
|
const base = { format: '' } as Download;
|
||||||
|
|
||||||
|
expect(app.formatLabel({ ...base, format: 'any' })).toBe('Auto');
|
||||||
|
expect(app.formatLabel({ ...base, format: 'mp4' })).toBe('MP4');
|
||||||
|
expect(app.formatLabel({ ...base, format: 'srt' })).toBe('SRT');
|
||||||
|
// A format from a record older than the option list still reads sensibly.
|
||||||
|
expect(app.formatLabel({ ...base, format: 'mkv' })).toBe('MKV');
|
||||||
|
expect(app.formatLabel(base)).toBe('-');
|
||||||
|
});
|
||||||
|
|
||||||
it('includes titleRegex in subscribe payload', () => {
|
it('includes titleRegex in subscribe payload', () => {
|
||||||
const fixture = TestBed.createComponent(App);
|
const fixture = TestBed.createComponent(App);
|
||||||
const app = fixture.componentInstance;
|
const app = fixture.componentInstance;
|
||||||
@@ -353,4 +413,64 @@ describe('App', () => {
|
|||||||
expect(errorSpy).toHaveBeenCalledWith('Subscription name must not be empty');
|
expect(errorSpy).toHaveBeenCalledWith('Subscription name must not be empty');
|
||||||
errorSpy.mockRestore();
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
// Issue #533: the server picks AUDIO_DOWNLOAD_DIR on download_type alone
|
||||||
|
// (ytdl.py), so the UI's choice of URL base has to use the same rule. It used
|
||||||
|
// to also treat any .mp3 as audio, which pointed the link at audio_download/
|
||||||
|
// for files the server had written to DOWNLOAD_DIR.
|
||||||
|
describe('download links follow the server directory rule (#533)', () => {
|
||||||
|
const makeDownload = (over: Partial<Download>): Download => ({
|
||||||
|
id: 'vid1',
|
||||||
|
title: 'Test',
|
||||||
|
url: 'https://example.com/v',
|
||||||
|
download_type: 'video',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
custom_name_prefix: '',
|
||||||
|
playlist_item_limit: 0,
|
||||||
|
status: 'finished',
|
||||||
|
msg: '',
|
||||||
|
percent: 100,
|
||||||
|
speed: 0,
|
||||||
|
eta: 0,
|
||||||
|
filename: 'song.mp4',
|
||||||
|
checked: false,
|
||||||
|
...over,
|
||||||
|
} as Download);
|
||||||
|
|
||||||
|
const appWithDirs = () => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
const app = fixture.componentInstance;
|
||||||
|
const downloads = TestBed.inject(DownloadsService) as unknown as DownloadsServiceStub;
|
||||||
|
downloads.configuration['PUBLIC_HOST_URL'] = 'download/';
|
||||||
|
downloads.configuration['PUBLIC_HOST_AUDIO_URL'] = 'audio_download/';
|
||||||
|
return app;
|
||||||
|
};
|
||||||
|
|
||||||
|
it('uses the audio base for an audio download', () => {
|
||||||
|
const app = appWithDirs();
|
||||||
|
const link = app.buildDownloadLink(makeDownload({ download_type: 'audio', filename: 'song.mp3' }));
|
||||||
|
expect(link).toBe('audio_download/song.mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the video base for an mp3 produced by a video download', () => {
|
||||||
|
const app = appWithDirs();
|
||||||
|
const link = app.buildDownloadLink(makeDownload({ download_type: 'video', filename: 'song.mp3' }));
|
||||||
|
expect(link).toBe('download/song.mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the video base for a video download', () => {
|
||||||
|
const app = appWithDirs();
|
||||||
|
const link = app.buildDownloadLink(makeDownload({ filename: 'clip.mp4' }));
|
||||||
|
expect(link).toBe('download/clip.mp4');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('applies the same rule to chapter links', () => {
|
||||||
|
const app = appWithDirs();
|
||||||
|
const dl = makeDownload({ download_type: 'video' });
|
||||||
|
expect(app.buildChapterDownloadLink(dl, 'ch1.mp3')).toBe('download/ch1.mp3');
|
||||||
|
const audio = makeDownload({ download_type: 'audio' });
|
||||||
|
expect(app.buildChapterDownloadLink(audio, 'ch1.mp3')).toBe('audio_download/ch1.mp3');
|
||||||
|
});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
+41
-2
@@ -86,6 +86,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
autoStart: boolean;
|
autoStart: boolean;
|
||||||
playlistItemLimit!: number;
|
playlistItemLimit!: number;
|
||||||
splitByChapters: boolean;
|
splitByChapters: boolean;
|
||||||
|
sponsorblock: boolean;
|
||||||
chapterTemplate: string;
|
chapterTemplate: string;
|
||||||
clipStart = '';
|
clipStart = '';
|
||||||
clipEnd = '';
|
clipEnd = '';
|
||||||
@@ -137,6 +138,9 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
sortAscending = false;
|
sortAscending = false;
|
||||||
expandedErrors: Set<string> = new Set<string>();
|
expandedErrors: Set<string> = new Set<string>();
|
||||||
cachedSortedDone: [string, Download][] = [];
|
cachedSortedDone: [string, Download][] = [];
|
||||||
|
// The done ids in rendered order, so a shift-click range follows the sort
|
||||||
|
// the user is looking at rather than the map's insertion order.
|
||||||
|
cachedSortedDoneIds: string[] = [];
|
||||||
lastCopiedErrorId: string | null = null;
|
lastCopiedErrorId: string | null = null;
|
||||||
private previousDownloadType = 'video';
|
private previousDownloadType = 'video';
|
||||||
private addRequestSub?: Subscription;
|
private addRequestSub?: Subscription;
|
||||||
@@ -256,6 +260,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.quality = this.cookieService.get('metube_quality') || 'best';
|
this.quality = this.cookieService.get('metube_quality') || 'best';
|
||||||
this.autoStart = this.cookieService.get('metube_auto_start') !== 'false';
|
this.autoStart = this.cookieService.get('metube_auto_start') !== 'false';
|
||||||
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
||||||
|
this.sponsorblock = this.cookieService.get('metube_sponsorblock') === 'true';
|
||||||
// Will be set from backend configuration, use empty string as placeholder
|
// Will be set from backend configuration, use empty string as placeholder
|
||||||
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
||||||
this.clipStart = this.cookieService.get('metube_clip_start') || '';
|
this.clipStart = this.cookieService.get('metube_clip_start') || '';
|
||||||
@@ -434,6 +439,12 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
if (!Number.isNaN(playlistItemLimit) && playlistItemLimit > 0) {
|
if (!Number.isNaN(playlistItemLimit) && playlistItemLimit > 0) {
|
||||||
this.playlistItemLimit = playlistItemLimit;
|
this.playlistItemLimit = playlistItemLimit;
|
||||||
}
|
}
|
||||||
|
// Pre-fill the download folder, unless the user has already typed one
|
||||||
|
// this session. The server drops DEFAULT_FOLDER when CUSTOM_DIRS is
|
||||||
|
// off, so there is nothing to guard against here.
|
||||||
|
if (!this.folder) {
|
||||||
|
this.folder = String(config['DEFAULT_FOLDER'] ?? '');
|
||||||
|
}
|
||||||
// Set chapter template from backend config if not already set by cookie
|
// Set chapter template from backend config if not already set by cookie
|
||||||
if (!this.chapterTemplate) {
|
if (!this.chapterTemplate) {
|
||||||
this.chapterTemplate = config['OUTPUT_TEMPLATE_CHAPTER'];
|
this.chapterTemplate = config['OUTPUT_TEMPLATE_CHAPTER'];
|
||||||
@@ -846,6 +857,10 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sponsorblockChanged() {
|
||||||
|
this.cookieService.set('metube_sponsorblock', this.sponsorblock ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
|
}
|
||||||
|
|
||||||
splitByChaptersChanged() {
|
splitByChaptersChanged() {
|
||||||
this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
}
|
}
|
||||||
@@ -909,6 +924,22 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
return type.charAt(0).toUpperCase() + type.slice(1);
|
return type.charAt(0).toUpperCase() + type.slice(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The format the download was queued with, labelled the way the form labels
|
||||||
|
// it, so a queued item can be told apart while it is still downloading.
|
||||||
|
formatLabel(download: Download): string {
|
||||||
|
const format = (download.format || '').trim();
|
||||||
|
if (!format) {
|
||||||
|
return '-';
|
||||||
|
}
|
||||||
|
const options: Option[] = [
|
||||||
|
...this.videoFormats,
|
||||||
|
...this.audioFormats,
|
||||||
|
...this.captionFormats,
|
||||||
|
...this.thumbnailFormats,
|
||||||
|
];
|
||||||
|
return options.find(o => o.id === format)?.text ?? format.toUpperCase();
|
||||||
|
}
|
||||||
|
|
||||||
formatCodecLabel(download: Download): string {
|
formatCodecLabel(download: Download): string {
|
||||||
if (download.download_type !== 'video') {
|
if (download.download_type !== 'video') {
|
||||||
const format = (download.format || '').toUpperCase();
|
const format = (download.format || '').toUpperCase();
|
||||||
@@ -1086,6 +1117,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
playlistItemLimit: overrides.playlistItemLimit ?? this.playlistItemLimit,
|
playlistItemLimit: overrides.playlistItemLimit ?? this.playlistItemLimit,
|
||||||
autoStart: overrides.autoStart ?? this.autoStart,
|
autoStart: overrides.autoStart ?? this.autoStart,
|
||||||
splitByChapters: overrides.splitByChapters ?? this.splitByChapters,
|
splitByChapters: overrides.splitByChapters ?? this.splitByChapters,
|
||||||
|
sponsorblock: overrides.sponsorblock ?? this.sponsorblock,
|
||||||
chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate,
|
chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate,
|
||||||
subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage,
|
subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage,
|
||||||
subtitleMode: overrides.subtitleMode ?? this.subtitleMode,
|
subtitleMode: overrides.subtitleMode ?? this.subtitleMode,
|
||||||
@@ -1252,7 +1284,12 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
|
|
||||||
buildDownloadLink(download: Download) {
|
buildDownloadLink(download: Download) {
|
||||||
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
||||||
if (download.download_type === 'audio' || download.filename.endsWith('.mp3')) {
|
// Must match the server's directory rule exactly: ytdl.py writes to
|
||||||
|
// AUDIO_DOWNLOAD_DIR on download_type alone. Treating any .mp3 as audio
|
||||||
|
// sent the link to audio_download/ for mp3s produced under a video-type
|
||||||
|
// download (a postprocessor, a preset, or a legacy record), which the
|
||||||
|
// server had written to DOWNLOAD_DIR -- a 404 whenever the two differ.
|
||||||
|
if (download.download_type === 'audio') {
|
||||||
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1350,7 +1387,8 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
|
|
||||||
buildChapterDownloadLink(download: Download, chapterFilename: string) {
|
buildChapterDownloadLink(download: Download, chapterFilename: string) {
|
||||||
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
||||||
if (download.download_type === 'audio' || chapterFilename.endsWith('.mp3')) {
|
// Same server-side rule as buildDownloadLink above.
|
||||||
|
if (download.download_type === 'audio') {
|
||||||
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1532,6 +1570,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
result.reverse();
|
result.reverse();
|
||||||
}
|
}
|
||||||
this.cachedSortedDone = result;
|
this.cachedSortedDone = result;
|
||||||
|
this.cachedSortedDoneIds = result.map(([key]) => key);
|
||||||
}
|
}
|
||||||
|
|
||||||
toggleErrorDetail(id: string) {
|
toggleErrorDetail(id: string) {
|
||||||
|
|||||||
@@ -2,6 +2,38 @@ import { TestBed } from '@angular/core/testing';
|
|||||||
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||||
import { Checkable } from '../interfaces';
|
import { Checkable } from '../interfaces';
|
||||||
|
|
||||||
|
function makeList(ids: string[]): Map<string, Checkable> {
|
||||||
|
const list = new Map<string, Checkable>();
|
||||||
|
for (const id of ids) {
|
||||||
|
list.set(id, { checked: false });
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeMaster(list: Map<string, Checkable>, orderedIds: string[] | null = null) {
|
||||||
|
const fixture = TestBed.createComponent(SelectAllCheckboxComponent);
|
||||||
|
fixture.componentRef.setInput('id', 'queue');
|
||||||
|
fixture.componentRef.setInput('list', list);
|
||||||
|
if (orderedIds) {
|
||||||
|
fixture.componentRef.setInput('orderedIds', orderedIds);
|
||||||
|
}
|
||||||
|
fixture.detectChanges();
|
||||||
|
return fixture;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simulates what the item checkbox does: ngModel writes the new state, then
|
||||||
|
// the change handler reports the click to the master.
|
||||||
|
function clickItem(
|
||||||
|
master: SelectAllCheckboxComponent,
|
||||||
|
list: Map<string, Checkable>,
|
||||||
|
id: string,
|
||||||
|
shift = false,
|
||||||
|
) {
|
||||||
|
const item = list.get(id)!;
|
||||||
|
item.checked = !item.checked;
|
||||||
|
master.selectionChanged(id, shift);
|
||||||
|
}
|
||||||
|
|
||||||
describe('SelectAllCheckboxComponent', () => {
|
describe('SelectAllCheckboxComponent', () => {
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
await TestBed.configureTestingModule({
|
await TestBed.configureTestingModule({
|
||||||
@@ -20,4 +52,87 @@ describe('SelectAllCheckboxComponent', () => {
|
|||||||
fixture.componentInstance.clicked();
|
fixture.componentInstance.clicked();
|
||||||
expect(list.get('u1')?.checked).toBe(true);
|
expect(list.get('u1')?.checked).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('shift-click checks every item between the two clicks', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3', 'u4', 'u5']);
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u2');
|
||||||
|
clickItem(master, list, 'u4', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, true, true, true, false]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extends upwards as well as downwards', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3', 'u4']);
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u4');
|
||||||
|
clickItem(master, list, 'u2', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, true, true, true]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shift-clicking a checked box clears the range', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3']);
|
||||||
|
list.forEach((item) => (item.checked = true));
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u1');
|
||||||
|
clickItem(master, list, 'u3', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, false, false]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('follows the rendered order, not the map order', () => {
|
||||||
|
// The done list renders newest-first, so its rendered order is not the
|
||||||
|
// order the entries sit in the map. u2 lies inside the range on screen
|
||||||
|
// and outside it in the map, which is what separates the two.
|
||||||
|
const list = makeList(['u1', 'u2', 'u3', 'u4']);
|
||||||
|
const master = makeMaster(list, ['u4', 'u2', 'u3', 'u1']).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u4');
|
||||||
|
clickItem(master, list, 'u3', true);
|
||||||
|
|
||||||
|
// u1 (rendered last) stays clear; u2 is swept up with the range.
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, true, true, true]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a plain click after a range starts a new anchor', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3', 'u4']);
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u1');
|
||||||
|
clickItem(master, list, 'u2', true);
|
||||||
|
clickItem(master, list, 'u4');
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([true, true, false, true]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('select-all clears the anchor so the next shift-click is a plain toggle', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3']);
|
||||||
|
const fixture = makeMaster(list);
|
||||||
|
const master = fixture.componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u1');
|
||||||
|
master.selected = true;
|
||||||
|
master.clicked();
|
||||||
|
master.selected = false;
|
||||||
|
master.clicked();
|
||||||
|
clickItem(master, list, 'u3', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, false, true]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ignores a range whose anchor row is gone', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3']);
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u1');
|
||||||
|
// The anchor finishes downloading and leaves the queue.
|
||||||
|
list.delete('u1');
|
||||||
|
clickItem(master, list, 'u3', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, true]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -20,17 +20,33 @@ import { FormsModule } from "@angular/forms";
|
|||||||
export class SelectAllCheckboxComponent {
|
export class SelectAllCheckboxComponent {
|
||||||
readonly id = input.required<string>();
|
readonly id = input.required<string>();
|
||||||
readonly list = input.required<Map<string, Checkable>>();
|
readonly list = input.required<Map<string, Checkable>>();
|
||||||
|
// The ids in the order the rows are rendered. The done list is sorted for
|
||||||
|
// display, so its order is not the map's insertion order, and a range
|
||||||
|
// selection has to follow what the user sees. Left unset, the map order is
|
||||||
|
// the rendered order.
|
||||||
|
readonly orderedIds = input<string[] | null>(null);
|
||||||
readonly changed = output<number>();
|
readonly changed = output<number>();
|
||||||
|
|
||||||
readonly masterCheckbox = viewChild.required<ElementRef>('masterCheckbox');
|
readonly masterCheckbox = viewChild.required<ElementRef>('masterCheckbox');
|
||||||
selected!: boolean;
|
selected!: boolean;
|
||||||
|
|
||||||
|
// The item a range extends from: the last one toggled on its own.
|
||||||
|
private anchorId: string | null = null;
|
||||||
|
|
||||||
clicked() {
|
clicked() {
|
||||||
this.list().forEach(item => item.checked = this.selected);
|
this.list().forEach(item => item.checked = this.selected);
|
||||||
|
// Select-all is not a position, so there is nothing to extend from next.
|
||||||
|
this.anchorId = null;
|
||||||
this.selectionChanged();
|
this.selectionChanged();
|
||||||
}
|
}
|
||||||
|
|
||||||
selectionChanged() {
|
selectionChanged(id?: string, extend = false) {
|
||||||
|
if (id !== undefined) {
|
||||||
|
if (extend && this.anchorId !== null && this.anchorId !== id) {
|
||||||
|
this.applyRange(this.anchorId, id);
|
||||||
|
}
|
||||||
|
this.anchorId = id;
|
||||||
|
}
|
||||||
const masterCheckbox = this.masterCheckbox();
|
const masterCheckbox = this.masterCheckbox();
|
||||||
if (!masterCheckbox)
|
if (!masterCheckbox)
|
||||||
return;
|
return;
|
||||||
@@ -40,4 +56,27 @@ export class SelectAllCheckboxComponent {
|
|||||||
masterCheckbox.nativeElement.indeterminate = checked > 0 && checked < this.list().size;
|
masterCheckbox.nativeElement.indeterminate = checked > 0 && checked < this.list().size;
|
||||||
this.changed.emit(checked);
|
this.changed.emit(checked);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Everything between the anchor and the just-clicked row takes the state the
|
||||||
|
// click produced, so shift-clicking a checked box clears the range and
|
||||||
|
// shift-clicking an unchecked one fills it.
|
||||||
|
private applyRange(fromId: string, toId: string) {
|
||||||
|
const ids = this.orderedIds() ?? Array.from(this.list().keys());
|
||||||
|
const from = ids.indexOf(fromId);
|
||||||
|
const to = ids.indexOf(toId);
|
||||||
|
// A row can disappear between two clicks (a download finishing moves it
|
||||||
|
// from the queue to the done list); without both ends there is no range.
|
||||||
|
if (from < 0 || to < 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const target = this.list().get(toId)?.checked ?? false;
|
||||||
|
const start = Math.min(from, to);
|
||||||
|
const end = Math.max(from, to);
|
||||||
|
for (let i = start; i <= end; i++) {
|
||||||
|
const item = this.list().get(ids[i]);
|
||||||
|
if (item) {
|
||||||
|
item.checked = target;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,4 +22,33 @@ describe('ItemCheckboxComponent', () => {
|
|||||||
itemFixture.detectChanges();
|
itemFixture.detectChanges();
|
||||||
expect(itemFixture.componentInstance).toBeTruthy();
|
expect(itemFixture.componentInstance).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reports the shift modifier from the click to the master', () => {
|
||||||
|
const masterFixture = TestBed.createComponent(SelectAllCheckboxComponent);
|
||||||
|
masterFixture.componentRef.setInput('id', 'q');
|
||||||
|
masterFixture.componentRef.setInput('list', new Map());
|
||||||
|
masterFixture.detectChanges();
|
||||||
|
const master = masterFixture.componentInstance;
|
||||||
|
const reported: [string | undefined, boolean | undefined][] = [];
|
||||||
|
master.selectionChanged = (id?: string, extend?: boolean) => {
|
||||||
|
reported.push([id, extend]);
|
||||||
|
};
|
||||||
|
|
||||||
|
const itemFixture = TestBed.createComponent(ItemCheckboxComponent);
|
||||||
|
itemFixture.componentRef.setInput('id', 'row1');
|
||||||
|
itemFixture.componentRef.setInput('master', master);
|
||||||
|
itemFixture.componentRef.setInput('checkable', { checked: false });
|
||||||
|
itemFixture.detectChanges();
|
||||||
|
const item = itemFixture.componentInstance;
|
||||||
|
|
||||||
|
item.clicked(new MouseEvent('click', { shiftKey: true }));
|
||||||
|
item.changed();
|
||||||
|
// The modifier must not stick to the next toggle.
|
||||||
|
item.changed();
|
||||||
|
|
||||||
|
expect(reported).toEqual([
|
||||||
|
['row1', true],
|
||||||
|
['row1', false],
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import { FormsModule } from '@angular/forms';
|
|||||||
selector: 'app-item-checkbox',
|
selector: 'app-item-checkbox',
|
||||||
template: `
|
template: `
|
||||||
<div class="form-check">
|
<div class="form-check">
|
||||||
<input type="checkbox" class="form-check-input" id="{{master().id()}}-{{id()}}-select" [(ngModel)]="checkable().checked" (change)="master().selectionChanged()" [attr.aria-label]="'Select item ' + id()">
|
<input type="checkbox" class="form-check-input" id="{{master().id()}}-{{id()}}-select" [(ngModel)]="checkable().checked" (click)="clicked($event)" (change)="changed()" [attr.aria-label]="'Select item ' + id()">
|
||||||
<label class="form-check-label visually-hidden" for="{{master().id()}}-{{id()}}-select">Select item</label>
|
<label class="form-check-label visually-hidden" for="{{master().id()}}-{{id()}}-select">Select item</label>
|
||||||
</div>
|
</div>
|
||||||
`,
|
`,
|
||||||
@@ -22,4 +22,19 @@ export class ItemCheckboxComponent {
|
|||||||
readonly id = input.required<string>();
|
readonly id = input.required<string>();
|
||||||
readonly master = input.required<SelectAllCheckboxComponent>();
|
readonly master = input.required<SelectAllCheckboxComponent>();
|
||||||
readonly checkable = input.required<Checkable>();
|
readonly checkable = input.required<Checkable>();
|
||||||
|
|
||||||
|
// click fires before change, so the modifier is recorded here and read once
|
||||||
|
// ngModel has written the new state into the checkable. Keyboard activation
|
||||||
|
// fires change without a click, which is a plain toggle.
|
||||||
|
private extend = false;
|
||||||
|
|
||||||
|
clicked(event: MouseEvent) {
|
||||||
|
this.extend = event.shiftKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
changed() {
|
||||||
|
const extend = this.extend;
|
||||||
|
this.extend = false;
|
||||||
|
this.master().selectionChanged(this.id(), extend);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ export interface Download {
|
|||||||
custom_name_prefix: string;
|
custom_name_prefix: string;
|
||||||
playlist_item_limit: number;
|
playlist_item_limit: number;
|
||||||
split_by_chapters?: boolean;
|
split_by_chapters?: boolean;
|
||||||
|
sponsorblock?: boolean;
|
||||||
chapter_template?: string;
|
chapter_template?: string;
|
||||||
subtitle_language?: string;
|
subtitle_language?: string;
|
||||||
subtitle_mode?: string;
|
subtitle_mode?: string;
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ function basePayload(): AddDownloadPayload {
|
|||||||
playlistItemLimit: 0,
|
playlistItemLimit: 0,
|
||||||
autoStart: true,
|
autoStart: true,
|
||||||
splitByChapters: false,
|
splitByChapters: false,
|
||||||
|
sponsorblock: false,
|
||||||
chapterTemplate: '',
|
chapterTemplate: '',
|
||||||
subtitleLanguage: 'en',
|
subtitleLanguage: 'en',
|
||||||
subtitleMode: 'prefer_manual',
|
subtitleMode: 'prefer_manual',
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ export interface AddDownloadPayload {
|
|||||||
playlistItemLimit: number;
|
playlistItemLimit: number;
|
||||||
autoStart: boolean;
|
autoStart: boolean;
|
||||||
splitByChapters: boolean;
|
splitByChapters: boolean;
|
||||||
|
sponsorblock: boolean;
|
||||||
chapterTemplate: string;
|
chapterTemplate: string;
|
||||||
subtitleLanguage: string;
|
subtitleLanguage: string;
|
||||||
subtitleMode: string;
|
subtitleMode: string;
|
||||||
@@ -148,6 +149,7 @@ export class DownloadsService {
|
|||||||
playlist_item_limit: payload.playlistItemLimit,
|
playlist_item_limit: payload.playlistItemLimit,
|
||||||
auto_start: payload.autoStart,
|
auto_start: payload.autoStart,
|
||||||
split_by_chapters: payload.splitByChapters,
|
split_by_chapters: payload.splitByChapters,
|
||||||
|
sponsorblock: payload.sponsorblock,
|
||||||
chapter_template: payload.chapterTemplate,
|
chapter_template: payload.chapterTemplate,
|
||||||
subtitle_language: payload.subtitleLanguage,
|
subtitle_language: payload.subtitleLanguage,
|
||||||
subtitle_mode: payload.subtitleMode,
|
subtitle_mode: payload.subtitleMode,
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import { TestBed } from '@angular/core/testing';
|
||||||
|
import { provideHttpClient } from '@angular/common/http';
|
||||||
|
import { provideHttpClientTesting, HttpTestingController } from '@angular/common/http/testing';
|
||||||
|
import { Subject } from 'rxjs';
|
||||||
|
import { SubscriptionsService, SubscribePayload } from './subscriptions.service';
|
||||||
|
import { MeTubeSocket } from './metube-socket.service';
|
||||||
|
|
||||||
|
class MeTubeSocketStub {
|
||||||
|
private subjects: Record<string, Subject<string>> = {};
|
||||||
|
|
||||||
|
fromEvent(event: string) {
|
||||||
|
if (!this.subjects[event]) {
|
||||||
|
this.subjects[event] = new Subject<string>();
|
||||||
|
}
|
||||||
|
return this.subjects[event].asObservable();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function basePayload(): SubscribePayload {
|
||||||
|
return {
|
||||||
|
url: 'https://example.com/channel',
|
||||||
|
downloadType: 'video',
|
||||||
|
codec: 'auto',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
customNamePrefix: '',
|
||||||
|
playlistItemLimit: 0,
|
||||||
|
autoStart: true,
|
||||||
|
splitByChapters: false,
|
||||||
|
sponsorblock: false,
|
||||||
|
chapterTemplate: '',
|
||||||
|
subtitleLanguage: 'en',
|
||||||
|
subtitleMode: 'prefer_manual',
|
||||||
|
ytdlOptionsPresets: [],
|
||||||
|
ytdlOptionsOverrides: '',
|
||||||
|
clipStart: '',
|
||||||
|
clipEnd: '',
|
||||||
|
checkIntervalMinutes: 60,
|
||||||
|
titleRegex: '',
|
||||||
|
skipSubscriberOnly: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('SubscriptionsService', () => {
|
||||||
|
let httpMock: HttpTestingController;
|
||||||
|
let service: SubscriptionsService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await TestBed.configureTestingModule({
|
||||||
|
providers: [
|
||||||
|
SubscriptionsService,
|
||||||
|
provideHttpClient(),
|
||||||
|
provideHttpClientTesting(),
|
||||||
|
{ provide: MeTubeSocket, useValue: new MeTubeSocketStub() },
|
||||||
|
],
|
||||||
|
}).compileComponents();
|
||||||
|
|
||||||
|
service = TestBed.inject(SubscriptionsService);
|
||||||
|
httpMock = TestBed.inject(HttpTestingController);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('subscribe() carries the sponsorblock flag', () => {
|
||||||
|
service.subscribe({ ...basePayload(), sponsorblock: true }).subscribe();
|
||||||
|
const req = httpMock.expectOne('subscribe');
|
||||||
|
expect(req.request.method).toBe('POST');
|
||||||
|
expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: true }));
|
||||||
|
req.flush({ status: 'ok' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('subscribe() sends the flag off by default', () => {
|
||||||
|
service.subscribe(basePayload()).subscribe();
|
||||||
|
const req = httpMock.expectOne('subscribe');
|
||||||
|
expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: false }));
|
||||||
|
req.flush({ status: 'ok' });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -92,6 +92,7 @@ export class SubscriptionsService {
|
|||||||
playlist_item_limit: payload.playlistItemLimit,
|
playlist_item_limit: payload.playlistItemLimit,
|
||||||
auto_start: payload.autoStart,
|
auto_start: payload.autoStart,
|
||||||
split_by_chapters: payload.splitByChapters,
|
split_by_chapters: payload.splitByChapters,
|
||||||
|
sponsorblock: payload.sponsorblock,
|
||||||
chapter_template: payload.chapterTemplate,
|
chapter_template: payload.chapterTemplate,
|
||||||
subtitle_language: payload.subtitleLanguage,
|
subtitle_language: payload.subtitleLanguage,
|
||||||
subtitle_mode: payload.subtitleMode,
|
subtitle_mode: payload.subtitleMode,
|
||||||
|
|||||||
@@ -1236,11 +1236,11 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "yt-dlp"
|
name = "yt-dlp"
|
||||||
version = "2026.7.4"
|
version = "2026.8.19"
|
||||||
source = { registry = "https://pypi.org/simple" }
|
source = { registry = "https://pypi.org/simple" }
|
||||||
sdist = { url = "https://files.pythonhosted.org/packages/47/c5/9972af4b472b0d55badf841ebafd2f98944cb0ae0f46e11d01f363ea5b91/yt_dlp-2026.7.4.tar.gz", hash = "sha256:b094813404f87a9dd2186f00815231df32e5fd8a5403be0f807b3bb2d21a4432", size = 3049326, upload-time = "2026-07-04T22:42:14.837Z" }
|
sdist = { url = "https://files.pythonhosted.org/packages/1e/e0/832fa4ca334b766a06933a196066edc3dba37cdb6f14cd98d59bcc69a4b4/yt_dlp-2026.8.19.tar.gz", hash = "sha256:9e213e48cea35c66b378e4447903f118f6392a5fa380a2b6d7070ec86f4e0af1", size = 3052025, upload-time = "2026-08-19T23:48:59.291Z" }
|
||||||
wheels = [
|
wheels = [
|
||||||
{ url = "https://files.pythonhosted.org/packages/f9/8a/cd4c9b02c10c563adfe78118310129641900e1cd6de888cfae2452072696/yt_dlp-2026.7.4-py3-none-any.whl", hash = "sha256:f11f2b11d5a8ac4059f9bdf29fa4407dc7c6bb00c5097e95ca22a7a9db518266", size = 3184705, upload-time = "2026-07-04T22:42:12.989Z" },
|
{ url = "https://files.pythonhosted.org/packages/69/b2/8cd1613f56eed7ceb64fbd4df3f1c01246bfb098e6f398228bafda22b80b/yt_dlp-2026.8.19-py3-none-any.whl", hash = "sha256:1d57897e94c6665a0a6f9bc54b34e584284e32c034ffab3a7df25d8f7b24eedf", size = 3185533, upload-time = "2026-08-19T23:48:56.925Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.optional-dependencies]
|
[package.optional-dependencies]
|
||||||
|
|||||||
Reference in New Issue
Block a user