mirror of
https://github.com/alexta69/metube.git
synced 2026-09-21 13:35:01 +00:00
Compare commits
46 Commits
2026.07.27
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| 300bf79b52 | |||
| ec01524527 | |||
| a6d81d4513 | |||
| 6708a88229 | |||
| 3767d6073b | |||
| 5cac98a6d1 | |||
| 3516872513 | |||
| 79388370e9 | |||
| 1251613f45 | |||
| 70d19759e8 | |||
| f11b376ce7 | |||
| c9c507f939 | |||
| 327e1eb4b8 | |||
| 82e966caaf | |||
| 346da19108 | |||
| b74185b2af | |||
| c393e0195b | |||
| 86954784fd | |||
| 72e8f5031f | |||
| f3c464fad5 | |||
| b10bb6103a | |||
| 8c2990e68a | |||
| ac46fff6d9 | |||
| 05c21326b3 | |||
| 99da62dcbb | |||
| d0ad36baad | |||
| d2095caea2 | |||
| e15aff3339 | |||
| fccd207799 | |||
| 6461924bf8 | |||
| c68fcaddd1 | |||
| a4454ac460 | |||
| 75fe1f0c11 | |||
| aac9c63a36 | |||
| 3444b1605b | |||
| 59cf84ae1a | |||
| 99b6452c8e | |||
| 5826d0dc2b | |||
| c8fb5bbc27 | |||
| 56194f0bf3 | |||
| de57484fc9 | |||
| 7082858237 | |||
| 97d1cc865c | |||
| 482381d6b9 | |||
| 0445f5858b | |||
| 6551f7ad58 |
@@ -96,7 +96,23 @@ release the same day. **Master is continuously released** — a PR must be
|
|||||||
release-ready exactly as merged; there is no stabilization window for follow-up
|
release-ready exactly as merged; there is no stabilization window for follow-up
|
||||||
fixes.
|
fixes.
|
||||||
|
|
||||||
## Code style
|
## Commit messages
|
||||||
|
|
||||||
|
A commit that resolves an issue must close it, with a GitHub closing keyword in
|
||||||
|
parentheses at the end of the subject line:
|
||||||
|
|
||||||
|
```
|
||||||
|
fix: stop metadata probes from writing playlist sidecar files (closes #1040)
|
||||||
|
```
|
||||||
|
|
||||||
|
Because master is the default branch and is released on every push, the issue
|
||||||
|
closes at the moment the fix ships, and keeps a permanent link to the commit that
|
||||||
|
fixed it. A bare `(#1040)` is only a reference — and reads as a pull-request
|
||||||
|
number — so it does not count; the keyword is what closes the issue.
|
||||||
|
|
||||||
|
Auto-closing leaves only a commit stub on the issue, which is not an answer to
|
||||||
|
whoever reported it. Post an explanatory comment as well: what the cause was, what
|
||||||
|
changed, and anything the reporter needs to do differently.
|
||||||
|
|
||||||
Follow `.editorconfig`:
|
Follow `.editorconfig`:
|
||||||
- Python: 4-space indent
|
- Python: 4-space indent
|
||||||
|
|||||||
+13
-1
@@ -2,7 +2,19 @@
|
|||||||
# has lagged behind and resolved to a Node patch older than the Angular CLI's
|
# has lagged behind and resolved to a Node patch older than the Angular CLI's
|
||||||
# minimum supported version, breaking the build. node:22-alpine currently
|
# minimum supported version, breaking the build. node:22-alpine currently
|
||||||
# satisfies @angular/cli's >=22.22.3 requirement.
|
# satisfies @angular/cli's >=22.22.3 requirement.
|
||||||
FROM node:22-alpine AS builder
|
#
|
||||||
|
# Pinned further, to a digest: Docker Hub rebuilt node:22-alpine on 2026-09-17
|
||||||
|
# with the same Node (22.23.2) but refreshed Alpine layers, and that rebuild
|
||||||
|
# dies under QEMU while cross-building the arm64 leg — `qemu: uncaught target
|
||||||
|
# signal 4 (Illegal instruction)`, exit 132, during `pnpm install`. Three
|
||||||
|
# consecutive release builds failed identically on it while every other input
|
||||||
|
# (runner image, binfmt digest, pnpm version, lockfile) was unchanged.
|
||||||
|
#
|
||||||
|
# This digest is the last image known to cross-build cleanly (built 2026-07-29).
|
||||||
|
# It ships nothing: this stage is thrown away and only ui/dist is copied out.
|
||||||
|
# Unpin once the arm64 leg builds natively instead of under emulation, or once
|
||||||
|
# a later node:22-alpine is confirmed to survive QEMU.
|
||||||
|
FROM node:22-alpine@sha256:c610fcdfb1d5b4740dd70c284ed3cb16bb857e0f7166196e36a5501df7a3aa32 AS builder
|
||||||
|
|
||||||
WORKDIR /metube
|
WORKDIR /metube
|
||||||
COPY ui ./
|
COPY ui ./
|
||||||
|
|||||||
@@ -59,9 +59,10 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
|||||||
|
|
||||||
* __DOWNLOAD_DIR__: Path to where the downloads will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise.
|
* __DOWNLOAD_DIR__: Path to where the downloads will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise.
|
||||||
* __AUDIO_DOWNLOAD_DIR__: Path to where audio-only downloads will be saved, if you wish to separate them from the video downloads. Defaults to the value of `DOWNLOAD_DIR`.
|
* __AUDIO_DOWNLOAD_DIR__: Path to where audio-only downloads will be saved, if you wish to separate them from the video downloads. Defaults to the value of `DOWNLOAD_DIR`.
|
||||||
* __CUSTOM_DIRS__: Whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a dropdown appears next to the Add button to specify the download directory. Defaults to `true`.
|
* __CUSTOM_DIRS__: Whether to enable downloading videos into custom directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__). When enabled, a **Download Folder** field appears under **Advanced Options**, where the directory for each download can be specified. Defaults to `true`.
|
||||||
* __CREATE_CUSTOM_DIRS__: Whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector supports free-text input, and the specified directory will be created recursively. Defaults to `true`.
|
* __CREATE_CUSTOM_DIRS__: Whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector supports free-text input, and the specified directory will be created recursively. Defaults to `true`.
|
||||||
* __CUSTOM_DIRS_EXCLUDE_REGEX__: Regular expression to exclude some custom directories from the dropdown. Empty regex disables exclusion. Defaults to `(^|/)[.@].*$`, which means directories starting with `.` or `@`.
|
* __CUSTOM_DIRS_EXCLUDE_REGEX__: Regular expression to exclude some custom directories from the folder field's suggestions. Empty regex disables exclusion. Defaults to `(^|/)[.@].*$`, which means directories starting with `.` or `@`.
|
||||||
|
* __DEFAULT_FOLDER__: Custom directory to pre-select in the download folder field, relative to __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__), for when most downloads go to the same place. It is only a starting value — the field stays editable, so any other folder can still be picked per download. Requires __CUSTOM_DIRS__; ignored with a warning otherwise. Defaults to empty, i.e. the base download directory.
|
||||||
* __DOWNLOAD_DIRS_INDEXABLE__: If `true`, the download directories (__DOWNLOAD_DIR__ and __AUDIO_DOWNLOAD_DIR__) are indexable on the web server. Defaults to `false`.
|
* __DOWNLOAD_DIRS_INDEXABLE__: If `true`, the download directories (__DOWNLOAD_DIR__ and __AUDIO_DOWNLOAD_DIR__) are indexable on the web server. Defaults to `false`.
|
||||||
* __STATE_DIR__: Path to where MeTube will store its persistent state files (`queue.json`, `pending.json`, `completed.json`, `subscriptions.json`). Defaults to `/downloads/.metube` in the Docker image, and `.` otherwise.
|
* __STATE_DIR__: Path to where MeTube will store its persistent state files (`queue.json`, `pending.json`, `completed.json`, `subscriptions.json`). Defaults to `/downloads/.metube` in the Docker image, and `.` otherwise.
|
||||||
* __TEMP_DIR__: Path where intermediary download files will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise.
|
* __TEMP_DIR__: Path where intermediary download files will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise.
|
||||||
@@ -80,14 +81,16 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
|||||||
* __YTDL_OPTIONS_PRESETS__: Named bundles of yt-dlp options, selectable per download in the UI. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for format and examples.
|
* __YTDL_OPTIONS_PRESETS__: Named bundles of yt-dlp options, selectable per download in the UI. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for format and examples.
|
||||||
* __YTDL_OPTIONS_PRESETS_FILE__: Path to a JSON file containing presets. Monitored and reloaded automatically on changes. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options).
|
* __YTDL_OPTIONS_PRESETS_FILE__: Path to a JSON file containing presets. Monitored and reloaded automatically on changes. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options).
|
||||||
* __ALLOW_YTDL_OPTIONS_OVERRIDES__: Whether to show a free-text field in the UI for per-download yt-dlp option overrides. Defaults to `false`. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for details and security considerations.
|
* __ALLOW_YTDL_OPTIONS_OVERRIDES__: Whether to show a free-text field in the UI for per-download yt-dlp option overrides. Defaults to `false`. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for details and security considerations.
|
||||||
* __ALLOW_PRIVATE_ADDRESSES__: Whether to allow downloads from private, loopback, link-local and other non-global addresses. Defaults to `false`, which protects against SSRF by refusing URLs that resolve to internal hosts. Set to `true` only in trusted environments — for example when routing traffic through a proxy/VPN client in Fake-IP mode (sing-box, Clash, Mihomo), which resolves hosts to the `198.18.0.0/15` range. Enabling this disables the SSRF protection entirely, so only use it when you control the network.
|
* __ALLOW_PRIVATE_ADDRESSES__: Whether to allow downloads from private, loopback, link-local and other non-global addresses. Defaults to `false`, which protects against SSRF by refusing URLs that resolve to internal hosts. Set to `true` only in trusted environments — for example when routing traffic through a proxy/VPN client in Fake-IP mode (sing-box, Clash, Mihomo), which resolves hosts to the `198.18.0.0/15` range. Enabling this disables the SSRF protection entirely, so only use it when you control the network. You do **not** need this to use a proxy on an internal address: a proxy configured through the `proxy` option in `YTDL_OPTIONS` (or the `*_proxy` environment variables) is always reachable at its own host and port, wherever it lives. Nor do you need it for a proxy that resolves hostnames itself (an HTTP proxy, `socks5`, `socks5h` or `socks4a`): MeTube leaves those lookups to the proxy rather than resolving submitted URLs locally, so none leak and proxy-only hosts still work.
|
||||||
* __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled).
|
* __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled).
|
||||||
|
|
||||||
|
A filename that would exceed the limit the filesystem accepts is shortened to fit, keeping its extension, with room left for the suffixes yt-dlp adds while downloading. Sites that put a long description in the title would otherwise fail the download outright with `File name too long`. Use `trim_file_name` in `YTDL_OPTIONS` if you want names shorter than the filesystem's own limit, or `restrictfilenames` to strip non-ASCII characters.
|
||||||
|
|
||||||
Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a feed-level `.info.json` and thumbnail when you add a playlist or channel. These reuse the template of the items they belong to — `OUTPUT_TEMPLATE_CHANNEL` or `OUTPUT_TEMPLATE_PLAYLIST` — evaluated against the feed itself, so with the defaults they land in the same folder as the videos, named after the feed. Set `allow_playlist_files` to `false` in `YTDL_OPTIONS` to skip them.
|
Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a feed-level `.info.json` and thumbnail when you add a playlist or channel. These reuse the template of the items they belong to — `OUTPUT_TEMPLATE_CHANNEL` or `OUTPUT_TEMPLATE_PLAYLIST` — evaluated against the feed itself, so with the defaults they land in the same folder as the videos, named after the feed. Set `allow_playlist_files` to `false` in `YTDL_OPTIONS` to skip them.
|
||||||
|
|
||||||
### 🌐 Web Server & URLs
|
### 🌐 Web Server & URLs
|
||||||
|
|
||||||
* __HOST__: The host address the web server will bind to. Defaults to `0.0.0.0` (all interfaces).
|
* __HOST__: The host address the web server will bind to. Defaults to `0.0.0.0`, which is every IPv4 interface but no IPv6 one. Set it to `*` (or leave it empty) to listen on both stacks, or to `::` for IPv6 only — `::` does not also accept IPv4, whatever the host's `bindv6only` setting says.
|
||||||
* __PORT__: The port number the web server will listen on. Defaults to `8081`.
|
* __PORT__: The port number the web server will listen on. Defaults to `8081`.
|
||||||
* __URL_PREFIX__: Base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
|
* __URL_PREFIX__: Base path for the web server (for use when hosting behind a reverse proxy). Defaults to `/`.
|
||||||
* __PUBLIC_HOST_URL__: Base URL for the download links shown in the UI for completed files. By default, MeTube serves them under its own URL. If your download directory is accessible on another URL and you want the download links to be based there, use this variable to set it.
|
* __PUBLIC_HOST_URL__: Base URL for the download links shown in the UI for completed files. By default, MeTube serves them under its own URL. If your download directory is accessible on another URL and you want the download links to be based there, use this variable to set it.
|
||||||
@@ -95,7 +98,7 @@ Enabling `writeinfojson` or `writethumbnail` in `YTDL_OPTIONS` also writes a fee
|
|||||||
* __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`.
|
* __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`.
|
||||||
* __CERTFILE__: HTTPS certificate file path.
|
* __CERTFILE__: HTTPS certificate file path.
|
||||||
* __KEYFILE__: HTTPS key file path.
|
* __KEYFILE__: HTTPS key file path.
|
||||||
* __CORS_ALLOWED_ORIGINS__: Comma-separated list of origins permitted to make cross-origin requests to the MeTube API; `*` allows all. When unset or empty, all cross-origin requests are denied. Required for browser extensions and bookmarklets — see [Sending links to MeTube](#-sending-links-to-metube).
|
* __CORS_ALLOWED_ORIGINS__: Comma-separated list of origins permitted to make cross-origin requests to the MeTube API; `*` allows all. When unset or empty, all cross-origin requests are denied. Required for browser extensions and bookmarklets — see [Sending links to MeTube](#-sending-links-to-metube). Naming origins explicitly also lets them send credentials (a login cookie, or the `Authorization` header a reverse proxy checks), which `*` deliberately does not: it would let any site you visit drive your instance with your own session.
|
||||||
* __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container.
|
* __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container.
|
||||||
|
|
||||||
## 🎛️ Configuring yt-dlp options
|
## 🎛️ Configuring yt-dlp options
|
||||||
@@ -240,10 +243,12 @@ __Browser extensions__ allow right-clicking videos and sending them directly to
|
|||||||
* __Chrome:__ contributed by [Rpsl](https://github.com/rpsl) — install from the [Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or [from sources](https://github.com/Rpsl/metube-browser-extension).
|
* __Chrome:__ contributed by [Rpsl](https://github.com/rpsl) — install from the [Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or [from sources](https://github.com/Rpsl/metube-browser-extension).
|
||||||
* __Firefox:__ contributed by [nanocortex](https://github.com/nanocortex) — install from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources [here](https://github.com/nanocortex/metube-firefox-addon).
|
* __Firefox:__ contributed by [nanocortex](https://github.com/nanocortex) — install from [Firefox Addons](https://addons.mozilla.org/en-US/firefox/addon/metube-downloader) or get sources [here](https://github.com/nanocortex/metube-firefox-addon).
|
||||||
|
|
||||||
__Bookmarklets__ send the currently open page to MeTube with one click. Add the origins of the sites where you use them to `CORS_ALLOWED_ORIGINS`, e.g. `https://www.youtube.com,https://www.vimeo.com`. The code (Chrome and Firefox variants, contributed by [kushfest](https://github.com/kushfest) and [shoonya75](https://github.com/shoonya75)) is in the [Bookmarklets wiki page](https://github.com/alexta69/metube/wiki/Bookmarklets).
|
__Bookmarklets__ send the currently open page to MeTube with one click. Add the origins of the sites where you use them to `CORS_ALLOWED_ORIGINS`, e.g. `https://www.youtube.com,https://www.vimeo.com`. If your instance sits behind authentication, list the origins individually rather than using `*` — only named origins are allowed to send credentials. The code (Chrome and Firefox variants, contributed by [kushfest](https://github.com/kushfest) and [shoonya75](https://github.com/shoonya75)) is in the [Bookmarklets wiki page](https://github.com/alexta69/metube/wiki/Bookmarklets).
|
||||||
|
|
||||||
__iOS Shortcut:__ [rithask](https://github.com/rithask) created an [iOS shortcut](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6) for sending URLs to MeTube from Safari's share menu; it prompts for your instance address on first use.
|
__iOS Shortcut:__ [rithask](https://github.com/rithask) created an [iOS shortcut](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6) for sending URLs to MeTube from Safari's share menu; it prompts for your instance address on first use.
|
||||||
|
|
||||||
|
__Android:__ [sagheerys](https://github.com/sagheerys) created [MeTube Mobile](https://github.com/sagheerys/metube-mobile), two apps that take links from Android's share menu: Lite pulls finished downloads to the phone and removes them from the server, Super keeps them on the server and streams them.
|
||||||
|
|
||||||
__Raycast:__ [dotvhs](https://github.com/dotvhs) has created an [extension for Raycast](https://www.raycast.com/dot/metube) for adding videos to MeTube directly from Raycast.
|
__Raycast:__ [dotvhs](https://github.com/dotvhs) has created an [extension for Raycast](https://www.raycast.com/dot/metube) for adding videos to MeTube directly from Raycast.
|
||||||
|
|
||||||
## 🎵 Pairing with a music tagger
|
## 🎵 Pairing with a music tagger
|
||||||
|
|||||||
+130
-8
@@ -62,6 +62,7 @@ class Config:
|
|||||||
'CUSTOM_DIRS': 'true',
|
'CUSTOM_DIRS': 'true',
|
||||||
'CREATE_CUSTOM_DIRS': 'true',
|
'CREATE_CUSTOM_DIRS': 'true',
|
||||||
'CUSTOM_DIRS_EXCLUDE_REGEX': r'(^|/)[.@].*$',
|
'CUSTOM_DIRS_EXCLUDE_REGEX': r'(^|/)[.@].*$',
|
||||||
|
'DEFAULT_FOLDER': '',
|
||||||
'DELETE_FILE_ON_TRASHCAN': 'false',
|
'DELETE_FILE_ON_TRASHCAN': 'false',
|
||||||
'STATE_DIR': '.',
|
'STATE_DIR': '.',
|
||||||
'URL_PREFIX': '',
|
'URL_PREFIX': '',
|
||||||
@@ -112,9 +113,32 @@ class Config:
|
|||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
setattr(self, k, v in ('true', 'True', 'on', '1'))
|
setattr(self, k, v in ('true', 'True', 'on', '1'))
|
||||||
|
|
||||||
|
# aiohttp hands HOST straight to getaddrinfo, which has no notion of a
|
||||||
|
# '*' wildcard: the lookup fails and takes the server down at startup
|
||||||
|
# with an opaque DNS error. '*' is nevertheless what people reach for
|
||||||
|
# when they want to serve both IP stacks, while the value that actually
|
||||||
|
# does it -- an empty string, which asyncio expands to one listening
|
||||||
|
# socket per address family -- is undiscoverable. Accept '*' as the
|
||||||
|
# spelling for "every interface, both stacks". Note that '::' on its own
|
||||||
|
# is IPv6-only regardless of the host's bindv6only setting, because
|
||||||
|
# asyncio always sets IPV6_V6ONLY on the sockets it binds.
|
||||||
|
if self.HOST.strip() == '*':
|
||||||
|
self.HOST = ''
|
||||||
|
|
||||||
if not self.URL_PREFIX.endswith('/'):
|
if not self.URL_PREFIX.endswith('/'):
|
||||||
self.URL_PREFIX += '/'
|
self.URL_PREFIX += '/'
|
||||||
|
|
||||||
|
# Strip trailing slashes from the download directories. get_custom_dirs()
|
||||||
|
# builds the folder dropdown by removing the base path as a prefix from
|
||||||
|
# each subdirectory, and the base directory's own path does not carry the
|
||||||
|
# trailing slash — so 'DOWNLOAD_DIR=/downloads/' failed to match itself
|
||||||
|
# and leaked 'downloads' into the dropdown as a bogus folder option.
|
||||||
|
# Runs after the '%%' indirection above so AUDIO_DOWNLOAD_DIR is resolved.
|
||||||
|
for attr in ('DOWNLOAD_DIR', 'AUDIO_DOWNLOAD_DIR', 'TEMP_DIR', 'STATE_DIR'):
|
||||||
|
val = getattr(self, attr)
|
||||||
|
if isinstance(val, str) and len(val) > 1 and val.endswith('/'):
|
||||||
|
setattr(self, attr, val.rstrip('/') or '/')
|
||||||
|
|
||||||
# A blank PUBLIC_HOST_AUDIO_URL (e.g. set empty in a compose file) bypasses the
|
# A blank PUBLIC_HOST_AUDIO_URL (e.g. set empty in a compose file) bypasses the
|
||||||
# default via os.environ.get, which would leave audio links root-relative and 404.
|
# default via os.environ.get, which would leave audio links root-relative and 404.
|
||||||
# Fall back to the 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR. When
|
# Fall back to the 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR. When
|
||||||
@@ -127,6 +151,18 @@ class Config:
|
|||||||
if val and not val.endswith('/'):
|
if val and not val.endswith('/'):
|
||||||
setattr(self, attr, val + '/')
|
setattr(self, attr, val + '/')
|
||||||
|
|
||||||
|
# DEFAULT_FOLDER only pre-fills the form's folder field, which the UI
|
||||||
|
# does not even show without CUSTOM_DIRS. Sending one anyway would fail
|
||||||
|
# every download on the server's own folder check, so drop it and say so
|
||||||
|
# rather than leaving the user with a form that cannot submit.
|
||||||
|
self.DEFAULT_FOLDER = self.DEFAULT_FOLDER.strip().strip('/')
|
||||||
|
if self.DEFAULT_FOLDER and not self.CUSTOM_DIRS:
|
||||||
|
log.warning(
|
||||||
|
'Ignoring DEFAULT_FOLDER "%s" because CUSTOM_DIRS is not enabled',
|
||||||
|
self.DEFAULT_FOLDER,
|
||||||
|
)
|
||||||
|
self.DEFAULT_FOLDER = ''
|
||||||
|
|
||||||
# Convert relative addresses to absolute addresses to prevent the failure of file address comparison
|
# Convert relative addresses to absolute addresses to prevent the failure of file address comparison
|
||||||
if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'):
|
if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'):
|
||||||
self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve())
|
self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve())
|
||||||
@@ -187,6 +223,7 @@ class Config:
|
|||||||
_FRONTEND_KEYS = (
|
_FRONTEND_KEYS = (
|
||||||
'CUSTOM_DIRS',
|
'CUSTOM_DIRS',
|
||||||
'CREATE_CUSTOM_DIRS',
|
'CREATE_CUSTOM_DIRS',
|
||||||
|
'DEFAULT_FOLDER',
|
||||||
'OUTPUT_TEMPLATE_CHAPTER',
|
'OUTPUT_TEMPLATE_CHAPTER',
|
||||||
'PUBLIC_HOST_URL',
|
'PUBLIC_HOST_URL',
|
||||||
'PUBLIC_HOST_AUDIO_URL',
|
'PUBLIC_HOST_AUDIO_URL',
|
||||||
@@ -321,6 +358,12 @@ async def state_dir_guard(request, handler):
|
|||||||
|
|
||||||
app = web.Application(middlewares=[state_dir_guard])
|
app = web.Application(middlewares=[state_dir_guard])
|
||||||
_cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
|
_cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
|
||||||
|
if '*' in _cors_origins and len(_cors_origins) > 1:
|
||||||
|
log.warning(
|
||||||
|
"CORS_ALLOWED_ORIGINS mixes '*' with named origins %s. '*' wins, and credentialed "
|
||||||
|
"cross-origin requests stay disabled for every origin in the list. Remove '*' if you "
|
||||||
|
"need a bookmarklet to reach an authenticated instance.",
|
||||||
|
[o for o in _cors_origins if o != '*'])
|
||||||
sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
|
sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
|
||||||
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
||||||
routes = web.RouteTableDef()
|
routes = web.RouteTableDef()
|
||||||
@@ -711,6 +754,7 @@ def parse_download_options(post: dict) -> dict:
|
|||||||
playlist_item_limit = post.get('playlist_item_limit')
|
playlist_item_limit = post.get('playlist_item_limit')
|
||||||
auto_start = post.get('auto_start')
|
auto_start = post.get('auto_start')
|
||||||
split_by_chapters = post.get('split_by_chapters')
|
split_by_chapters = post.get('split_by_chapters')
|
||||||
|
sponsorblock = bool(post.get('sponsorblock'))
|
||||||
chapter_template = post.get('chapter_template')
|
chapter_template = post.get('chapter_template')
|
||||||
subtitle_language = post.get('subtitle_language')
|
subtitle_language = post.get('subtitle_language')
|
||||||
subtitle_mode = post.get('subtitle_mode')
|
subtitle_mode = post.get('subtitle_mode')
|
||||||
@@ -831,6 +875,7 @@ def parse_download_options(post: dict) -> dict:
|
|||||||
'playlist_item_limit': playlist_item_limit,
|
'playlist_item_limit': playlist_item_limit,
|
||||||
'auto_start': auto_start,
|
'auto_start': auto_start,
|
||||||
'split_by_chapters': split_by_chapters,
|
'split_by_chapters': split_by_chapters,
|
||||||
|
'sponsorblock': sponsorblock,
|
||||||
'chapter_template': chapter_template,
|
'chapter_template': chapter_template,
|
||||||
'subtitle_language': subtitle_language,
|
'subtitle_language': subtitle_language,
|
||||||
'subtitle_mode': subtitle_mode,
|
'subtitle_mode': subtitle_mode,
|
||||||
@@ -876,6 +921,7 @@ async def add(request):
|
|||||||
o['ytdl_options_overrides'],
|
o['ytdl_options_overrides'],
|
||||||
o['clip_start'],
|
o['clip_start'],
|
||||||
o['clip_end'],
|
o['clip_end'],
|
||||||
|
sponsorblock=o['sponsorblock'],
|
||||||
)
|
)
|
||||||
return web.Response(text=serializer.encode(status))
|
return web.Response(text=serializer.encode(status))
|
||||||
|
|
||||||
@@ -917,9 +963,6 @@ async def subscribe(request):
|
|||||||
raise web.HTTPBadRequest(reason='check_interval_minutes must be an integer') from exc
|
raise web.HTTPBadRequest(reason='check_interval_minutes must be an integer') from exc
|
||||||
if cic < 1:
|
if cic < 1:
|
||||||
raise web.HTTPBadRequest(reason='check_interval_minutes must be at least 1')
|
raise web.HTTPBadRequest(reason='check_interval_minutes must be at least 1')
|
||||||
if o.get('clip_start') is not None or o.get('clip_end') is not None:
|
|
||||||
raise web.HTTPBadRequest(reason='clip options are not supported for subscriptions')
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
skip_subscriber_only = coerce_optional_bool(
|
skip_subscriber_only = coerce_optional_bool(
|
||||||
post.get('skip_subscriber_only'),
|
post.get('skip_subscriber_only'),
|
||||||
@@ -929,6 +972,19 @@ async def subscribe(request):
|
|||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise web.HTTPBadRequest(reason=str(exc)) from exc
|
raise web.HTTPBadRequest(reason=str(exc)) from exc
|
||||||
|
|
||||||
|
# A t= timestamp in the URL means "start playing here" and parse_download_options
|
||||||
|
# turns it into a clip start, which is right for a one-off download of that video.
|
||||||
|
# A subscription URL is a channel or playlist, so a timestamp left on it says
|
||||||
|
# nothing about the videos it will yield — honour clip fields only when the
|
||||||
|
# caller supplied them explicitly, rather than silently clipping every future
|
||||||
|
# download. The t= param is still stripped from the stored URL.
|
||||||
|
clip_given = (
|
||||||
|
_clip_field_provided_in_post(post.get('clip_start'))
|
||||||
|
or _clip_field_provided_in_post(post.get('clip_end'))
|
||||||
|
)
|
||||||
|
sub_clip_start = o['clip_start'] if clip_given else None
|
||||||
|
sub_clip_end = o['clip_end'] if clip_given else None
|
||||||
|
|
||||||
result = await submgr.add_subscription(
|
result = await submgr.add_subscription(
|
||||||
o['url'],
|
o['url'],
|
||||||
check_interval_minutes=cic,
|
check_interval_minutes=cic,
|
||||||
@@ -946,8 +1002,11 @@ async def subscribe(request):
|
|||||||
subtitle_mode=o['subtitle_mode'],
|
subtitle_mode=o['subtitle_mode'],
|
||||||
ytdl_options_presets=o['ytdl_options_presets'],
|
ytdl_options_presets=o['ytdl_options_presets'],
|
||||||
ytdl_options_overrides=o['ytdl_options_overrides'],
|
ytdl_options_overrides=o['ytdl_options_overrides'],
|
||||||
|
sponsorblock=o['sponsorblock'],
|
||||||
title_regex=post.get('title_regex'),
|
title_regex=post.get('title_regex'),
|
||||||
skip_subscriber_only=skip_subscriber_only,
|
skip_subscriber_only=skip_subscriber_only,
|
||||||
|
clip_start=sub_clip_start,
|
||||||
|
clip_end=sub_clip_end,
|
||||||
)
|
)
|
||||||
return web.Response(text=serializer.encode(result))
|
return web.Response(text=serializer.encode(result))
|
||||||
|
|
||||||
@@ -967,7 +1026,7 @@ async def subscriptions_update(request):
|
|||||||
k: v
|
k: v
|
||||||
for k, v in post.items()
|
for k, v in post.items()
|
||||||
if k != 'id'
|
if k != 'id'
|
||||||
and k in ('enabled', 'check_interval_minutes', 'name', 'title_regex', 'skip_subscriber_only')
|
and k in ('enabled', 'check_interval_minutes', 'name', 'folder', 'title_regex', 'skip_subscriber_only')
|
||||||
}
|
}
|
||||||
if not changes:
|
if not changes:
|
||||||
raise web.HTTPBadRequest(reason='no valid fields to update')
|
raise web.HTTPBadRequest(reason='no valid fields to update')
|
||||||
@@ -1033,6 +1092,30 @@ async def start(request):
|
|||||||
|
|
||||||
COOKIES_PATH = os.path.join(config.STATE_DIR, 'cookies.txt')
|
COOKIES_PATH = os.path.join(config.STATE_DIR, 'cookies.txt')
|
||||||
|
|
||||||
|
|
||||||
|
def warn_if_cookiefile_shadowed():
|
||||||
|
"""Warn before an uploaded cookies file displaces an operator-configured one.
|
||||||
|
|
||||||
|
Uploaded cookies deliberately win: the upload exists so cookies can be
|
||||||
|
refreshed without restarting the container, and letting YTDL_OPTIONS win
|
||||||
|
would leave a visible UI button doing nothing. But set_runtime_override
|
||||||
|
writes straight into YTDL_OPTIONS, so the configured path is gone from the
|
||||||
|
live config the moment an uploaded file is applied — after that, nothing
|
||||||
|
downstream can report the conflict (delete_cookies' has_manual_cookiefile
|
||||||
|
check cannot fire once the value has been replaced). This is the only point
|
||||||
|
where both are still visible, so it is the only place the warning can be
|
||||||
|
issued. Must be called before set_runtime_override. See issue #881, where
|
||||||
|
the silence cost the reporter days of debugging.
|
||||||
|
"""
|
||||||
|
configured = config.YTDL_OPTIONS.get('cookiefile')
|
||||||
|
if isinstance(configured, str) and configured and configured != COOKIES_PATH:
|
||||||
|
log.warning(
|
||||||
|
'Uploaded cookies at %s take precedence over the cookiefile configured in '
|
||||||
|
'YTDL_OPTIONS (%s), which will not be used. Delete the uploaded cookies from '
|
||||||
|
'the UI to go back to the configured file.',
|
||||||
|
COOKIES_PATH, configured)
|
||||||
|
|
||||||
|
|
||||||
@routes.post(config.URL_PREFIX + 'upload-cookies')
|
@routes.post(config.URL_PREFIX + 'upload-cookies')
|
||||||
async def upload_cookies(request):
|
async def upload_cookies(request):
|
||||||
reader = await request.multipart()
|
reader = await request.multipart()
|
||||||
@@ -1062,6 +1145,7 @@ async def upload_cookies(request):
|
|||||||
except OSError as exc:
|
except OSError as exc:
|
||||||
log.warning(f'Could not restrict permissions on cookies file: {exc}')
|
log.warning(f'Could not restrict permissions on cookies file: {exc}')
|
||||||
os.replace(tmp_cookie_path, COOKIES_PATH)
|
os.replace(tmp_cookie_path, COOKIES_PATH)
|
||||||
|
warn_if_cookiefile_shadowed()
|
||||||
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
||||||
log.info(f'Cookies file uploaded ({size} bytes)')
|
log.info(f'Cookies file uploaded ({size} bytes)')
|
||||||
return web.Response(text=serializer.encode({'status': 'ok', 'msg': f'Cookies uploaded ({size} bytes)'}))
|
return web.Response(text=serializer.encode({'status': 'ok', 'msg': f'Cookies uploaded ({size} bytes)'}))
|
||||||
@@ -1256,9 +1340,44 @@ app.router.add_route('OPTIONS', config.URL_PREFIX + 'delete-cookies', add_cors)
|
|||||||
|
|
||||||
async def on_prepare(request, response):
|
async def on_prepare(request, response):
|
||||||
origin = request.headers.get('Origin')
|
origin = request.headers.get('Origin')
|
||||||
if origin and _cors_origins and ('*' in _cors_origins or origin in _cors_origins):
|
if not origin or not _cors_origins:
|
||||||
response.headers['Access-Control-Allow-Origin'] = origin
|
return
|
||||||
response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
|
|
||||||
|
# Naming an origin in CORS_ALLOWED_ORIGINS is a deliberate trust grant, so
|
||||||
|
# such an origin may send credentials: the cookie or Authorization header
|
||||||
|
# that a reverse proxy in front of MeTube checks. Without this a bookmarklet
|
||||||
|
# cannot reach an authenticated instance at all (issue #155).
|
||||||
|
#
|
||||||
|
# The '*' wildcard is emphatically not such a grant — it matches origins the
|
||||||
|
# operator never enumerated, including every site the user happens to visit.
|
||||||
|
# Echoing the origin back (which we must do, since '*' is illegal alongside
|
||||||
|
# credentials) and allowing credentials would let any page drive the user's
|
||||||
|
# instance with the user's own session. So the wildcard keeps exactly the
|
||||||
|
# uncredentialed behaviour it has always had, and a wildcard anywhere in the
|
||||||
|
# list disables credentials for every origin in it.
|
||||||
|
#
|
||||||
|
# Derived here rather than held in a second module global so the wildcard
|
||||||
|
# test and the membership test can never disagree about the same list.
|
||||||
|
wildcard = '*' in _cors_origins
|
||||||
|
trusted = not wildcard and origin in _cors_origins
|
||||||
|
if not (wildcard or trusted):
|
||||||
|
return
|
||||||
|
|
||||||
|
response.headers['Access-Control-Allow-Origin'] = origin
|
||||||
|
# Authorization rides on the same grant: allowing it under the wildcard
|
||||||
|
# would let an arbitrary page attempt credentials against an instance it
|
||||||
|
# can already reach, from inside the victim's network.
|
||||||
|
response.headers['Access-Control-Allow-Headers'] = 'Content-Type, Authorization' if trusted else 'Content-Type'
|
||||||
|
if trusted:
|
||||||
|
response.headers['Access-Control-Allow-Credentials'] = 'true'
|
||||||
|
|
||||||
|
# The response now differs per Origin, so a shared cache must not hand one
|
||||||
|
# origin's Allow-Origin to another.
|
||||||
|
vary = response.headers.get('Vary')
|
||||||
|
if not vary:
|
||||||
|
response.headers['Vary'] = 'Origin'
|
||||||
|
elif 'origin' not in (v.strip().lower() for v in vary.split(',')):
|
||||||
|
response.headers['Vary'] = f'{vary}, Origin'
|
||||||
|
|
||||||
app.on_response_prepare.append(on_prepare)
|
app.on_response_prepare.append(on_prepare)
|
||||||
|
|
||||||
@@ -1279,11 +1398,14 @@ def isAccessLogEnabled():
|
|||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
logging.getLogger().setLevel(parseLogLevel(config.LOGLEVEL) or logging.INFO)
|
logging.getLogger().setLevel(parseLogLevel(config.LOGLEVEL) or logging.INFO)
|
||||||
log.info(f"Listening on {config.HOST}:{config.PORT}")
|
# An empty HOST binds every interface on both stacks; print the '*' spelling
|
||||||
|
# that selects it rather than a bare ':8081'.
|
||||||
|
log.info(f"Listening on {config.HOST or '*'}:{config.PORT}")
|
||||||
|
|
||||||
|
|
||||||
# Auto-detect cookie file on startup
|
# Auto-detect cookie file on startup
|
||||||
if os.path.exists(COOKIES_PATH):
|
if os.path.exists(COOKIES_PATH):
|
||||||
|
warn_if_cookiefile_shadowed()
|
||||||
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
||||||
log.info(f'Cookie file detected at {COOKIES_PATH}')
|
log.info(f'Cookie file detected at {COOKIES_PATH}')
|
||||||
|
|
||||||
|
|||||||
+87
-4
@@ -19,7 +19,7 @@ import yt_dlp.networking.impersonate
|
|||||||
import bg_tasks
|
import bg_tasks
|
||||||
from dl_formats import merge_ytdl_option_layers
|
from dl_formats import merge_ytdl_option_layers
|
||||||
from state_store import AtomicJsonStore, read_legacy_shelf
|
from state_store import AtomicJsonStore, read_legacy_shelf
|
||||||
from url_guard import validate_url
|
from url_guard import validate_url, download_proxies
|
||||||
|
|
||||||
log = logging.getLogger("subscriptions")
|
log = logging.getLogger("subscriptions")
|
||||||
|
|
||||||
@@ -116,12 +116,18 @@ def extract_flat_playlist(
|
|||||||
if media_entries:
|
if media_entries:
|
||||||
return info, media_entries
|
return info, media_entries
|
||||||
if _depth < 1:
|
if _depth < 1:
|
||||||
|
proxies = download_proxies({**config.YTDL_OPTIONS, **(extra_opts or {})})
|
||||||
for ent in entries[:5]:
|
for ent in entries[:5]:
|
||||||
nested_url = _entry_video_url(ent)
|
nested_url = _entry_video_url(ent)
|
||||||
if not nested_url:
|
if not nested_url:
|
||||||
continue
|
continue
|
||||||
# nested_url comes from remote playlist content; guard it too.
|
# nested_url comes from remote playlist content; guard it too,
|
||||||
if validate_url(nested_url, allow_private=getattr(config, "ALLOW_PRIVATE_ADDRESSES", False)) is not None:
|
# against the same proxy map this scan is using.
|
||||||
|
if validate_url(
|
||||||
|
nested_url,
|
||||||
|
allow_private=getattr(config, "ALLOW_PRIVATE_ADDRESSES", False),
|
||||||
|
proxies=proxies,
|
||||||
|
) is not None:
|
||||||
continue
|
continue
|
||||||
nested_info, nested_entries = extract_flat_playlist(
|
nested_info, nested_entries = extract_flat_playlist(
|
||||||
config,
|
config,
|
||||||
@@ -182,6 +188,7 @@ class SubscriptionInfo:
|
|||||||
auto_start: bool = True
|
auto_start: bool = True
|
||||||
playlist_item_limit: int = 0
|
playlist_item_limit: int = 0
|
||||||
split_by_chapters: bool = False
|
split_by_chapters: bool = False
|
||||||
|
sponsorblock: bool = False
|
||||||
chapter_template: str = ""
|
chapter_template: str = ""
|
||||||
subtitle_language: str = "en"
|
subtitle_language: str = "en"
|
||||||
subtitle_mode: str = "prefer_manual"
|
subtitle_mode: str = "prefer_manual"
|
||||||
@@ -189,6 +196,13 @@ class SubscriptionInfo:
|
|||||||
ytdl_options_overrides: dict[str, Any] = field(default_factory=dict)
|
ytdl_options_overrides: dict[str, Any] = field(default_factory=dict)
|
||||||
title_regex: str = ""
|
title_regex: str = ""
|
||||||
skip_subscriber_only: bool = False
|
skip_subscriber_only: bool = False
|
||||||
|
# A fixed range applied to every video the subscription downloads. Only
|
||||||
|
# sensible for channels with a consistent format (a standing intro, a fixed
|
||||||
|
# sponsor read); left unset, videos download whole. Old stored records
|
||||||
|
# predate these fields and take the defaults — _from_stored filters by
|
||||||
|
# field name, so nothing needs migrating.
|
||||||
|
clip_start: Optional[float] = None
|
||||||
|
clip_end: Optional[float] = None
|
||||||
last_checked: Optional[float] = None
|
last_checked: Optional[float] = None
|
||||||
seen_ids: list[str] = field(default_factory=list)
|
seen_ids: list[str] = field(default_factory=list)
|
||||||
error: Optional[str] = None
|
error: Optional[str] = None
|
||||||
@@ -211,6 +225,8 @@ class SubscriptionInfo:
|
|||||||
"folder": self.folder,
|
"folder": self.folder,
|
||||||
"title_regex": self.title_regex,
|
"title_regex": self.title_regex,
|
||||||
"skip_subscriber_only": self.skip_subscriber_only,
|
"skip_subscriber_only": self.skip_subscriber_only,
|
||||||
|
"clip_start": self.clip_start,
|
||||||
|
"clip_end": self.clip_end,
|
||||||
"last_checked": self.last_checked,
|
"last_checked": self.last_checked,
|
||||||
"seen_count": len(self.seen_ids),
|
"seen_count": len(self.seen_ids),
|
||||||
"error": self.error,
|
"error": self.error,
|
||||||
@@ -233,6 +249,7 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]:
|
|||||||
"auto_start": sub.auto_start,
|
"auto_start": sub.auto_start,
|
||||||
"playlist_item_limit": sub.playlist_item_limit,
|
"playlist_item_limit": sub.playlist_item_limit,
|
||||||
"split_by_chapters": sub.split_by_chapters,
|
"split_by_chapters": sub.split_by_chapters,
|
||||||
|
"sponsorblock": sub.sponsorblock,
|
||||||
"chapter_template": sub.chapter_template,
|
"chapter_template": sub.chapter_template,
|
||||||
"subtitle_language": sub.subtitle_language,
|
"subtitle_language": sub.subtitle_language,
|
||||||
"subtitle_mode": sub.subtitle_mode,
|
"subtitle_mode": sub.subtitle_mode,
|
||||||
@@ -240,6 +257,8 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]:
|
|||||||
"ytdl_options_overrides": sub.ytdl_options_overrides,
|
"ytdl_options_overrides": sub.ytdl_options_overrides,
|
||||||
"title_regex": sub.title_regex,
|
"title_regex": sub.title_regex,
|
||||||
"skip_subscriber_only": sub.skip_subscriber_only,
|
"skip_subscriber_only": sub.skip_subscriber_only,
|
||||||
|
"clip_start": sub.clip_start,
|
||||||
|
"clip_end": sub.clip_end,
|
||||||
"last_checked": sub.last_checked,
|
"last_checked": sub.last_checked,
|
||||||
"seen_ids": list(sub.seen_ids),
|
"seen_ids": list(sub.seen_ids),
|
||||||
"error": sub.error,
|
"error": sub.error,
|
||||||
@@ -311,6 +330,33 @@ def validate_subscription_name(value: Any) -> str:
|
|||||||
return name
|
return name
|
||||||
|
|
||||||
|
|
||||||
|
def validate_subscription_folder(value: Any) -> str:
|
||||||
|
"""Return a stored subscription folder, or raise ValueError if unusable.
|
||||||
|
|
||||||
|
The folder is relative to the configured download directory, and the
|
||||||
|
authoritative check still happens at download time in ``DownloadQueue`` —
|
||||||
|
that is where ``CUSTOM_DIRS``, ``CREATE_CUSTOM_DIRS`` and the
|
||||||
|
resolves-inside-the-base-directory rule live, and where the directory is
|
||||||
|
created. This rejects only values that could never be valid, so an edit is
|
||||||
|
refused while the user is looking at it rather than silently failing every
|
||||||
|
check from then on. An empty folder is valid and means the base directory.
|
||||||
|
"""
|
||||||
|
if value is None:
|
||||||
|
return ""
|
||||||
|
if not isinstance(value, str):
|
||||||
|
raise ValueError("folder must be a string")
|
||||||
|
folder = value.strip()
|
||||||
|
if not folder:
|
||||||
|
return ""
|
||||||
|
if os.path.isabs(folder):
|
||||||
|
raise ValueError("folder must be relative to the download directory")
|
||||||
|
# Check both separators: the value is stored as typed, and a Windows-style
|
||||||
|
# path would otherwise carry an unexamined '..' past this point.
|
||||||
|
if any(part == ".." for part in folder.replace("\\", "/").split("/")):
|
||||||
|
raise ValueError('folder must not contain ".."')
|
||||||
|
return folder
|
||||||
|
|
||||||
|
|
||||||
def _coerce_bool(value: Any) -> bool:
|
def _coerce_bool(value: Any) -> bool:
|
||||||
"""Accept JSON booleans and common string forms used by API clients."""
|
"""Accept JSON booleans and common string forms used by API clients."""
|
||||||
if isinstance(value, bool):
|
if isinstance(value, bool):
|
||||||
@@ -447,6 +493,9 @@ class SubscriptionManager:
|
|||||||
subtitle_mode: str,
|
subtitle_mode: str,
|
||||||
ytdl_options_presets: Optional[list[str]] = None,
|
ytdl_options_presets: Optional[list[str]] = None,
|
||||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||||
|
clip_start: Optional[float] = None,
|
||||||
|
clip_end: Optional[float] = None,
|
||||||
|
sponsorblock: bool = False,
|
||||||
) -> tuple[list[str], list[str]]:
|
) -> tuple[list[str], list[str]]:
|
||||||
queued_ids: list[str] = []
|
queued_ids: list[str] = []
|
||||||
queue_errors: list[str] = []
|
queue_errors: list[str] = []
|
||||||
@@ -477,6 +526,9 @@ class SubscriptionManager:
|
|||||||
subtitle_mode,
|
subtitle_mode,
|
||||||
presets,
|
presets,
|
||||||
ytdl_options_overrides,
|
ytdl_options_overrides,
|
||||||
|
clip_start,
|
||||||
|
clip_end,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
if isinstance(result, dict) and result.get("status") == "error":
|
if isinstance(result, dict) and result.get("status") == "error":
|
||||||
msg = str(result.get("msg") or f"Queueing failed for {vurl}")
|
msg = str(result.get("msg") or f"Queueing failed for {vurl}")
|
||||||
@@ -564,16 +616,27 @@ class SubscriptionManager:
|
|||||||
subtitle_mode: str,
|
subtitle_mode: str,
|
||||||
ytdl_options_presets: Optional[list[str]] = None,
|
ytdl_options_presets: Optional[list[str]] = None,
|
||||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||||
|
sponsorblock: bool = False,
|
||||||
title_regex: Any = None,
|
title_regex: Any = None,
|
||||||
skip_subscriber_only: Any = None,
|
skip_subscriber_only: Any = None,
|
||||||
|
clip_start: Optional[float] = None,
|
||||||
|
clip_end: Optional[float] = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
url = self._normalize_url(url)
|
url = self._normalize_url(url)
|
||||||
if not url:
|
if not url:
|
||||||
return {"status": "error", "msg": "Missing URL"}
|
return {"status": "error", "msg": "Missing URL"}
|
||||||
# SSRF guard: block non-http(s) schemes and internal/metadata hosts
|
# SSRF guard: block non-http(s) schemes and internal/metadata hosts
|
||||||
# before yt-dlp fetches the feed. May do a DNS lookup, so run off-loop.
|
# before yt-dlp fetches the feed. May do a DNS lookup, so run off-loop.
|
||||||
|
# The scan's own options pick the proxy, so a feed fetched through one
|
||||||
|
# is not resolved here — see validate_url.
|
||||||
|
proxies = download_proxies({
|
||||||
|
**self.config.YTDL_OPTIONS,
|
||||||
|
**self._scan_extra_opts(ytdl_options_presets, ytdl_options_overrides),
|
||||||
|
})
|
||||||
url_error = await asyncio.get_running_loop().run_in_executor(
|
url_error = await asyncio.get_running_loop().run_in_executor(
|
||||||
None, partial(validate_url, url, allow_private=getattr(self.config, "ALLOW_PRIVATE_ADDRESSES", False)))
|
None, partial(validate_url, url,
|
||||||
|
allow_private=getattr(self.config, "ALLOW_PRIVATE_ADDRESSES", False),
|
||||||
|
proxies=proxies))
|
||||||
if url_error is not None:
|
if url_error is not None:
|
||||||
log.warning('Rejected subscription URL "%s": %s', url, url_error)
|
log.warning('Rejected subscription URL "%s": %s', url, url_error)
|
||||||
return {"status": "error", "msg": url_error}
|
return {"status": "error", "msg": url_error}
|
||||||
@@ -645,6 +708,7 @@ class SubscriptionManager:
|
|||||||
auto_start=bool(auto_start),
|
auto_start=bool(auto_start),
|
||||||
playlist_item_limit=int(playlist_item_limit),
|
playlist_item_limit=int(playlist_item_limit),
|
||||||
split_by_chapters=bool(split_by_chapters),
|
split_by_chapters=bool(split_by_chapters),
|
||||||
|
sponsorblock=bool(sponsorblock),
|
||||||
chapter_template=chapter_template or "",
|
chapter_template=chapter_template or "",
|
||||||
subtitle_language=subtitle_language,
|
subtitle_language=subtitle_language,
|
||||||
subtitle_mode=subtitle_mode,
|
subtitle_mode=subtitle_mode,
|
||||||
@@ -652,6 +716,8 @@ class SubscriptionManager:
|
|||||||
ytdl_options_overrides=dict(ytdl_options_overrides or {}),
|
ytdl_options_overrides=dict(ytdl_options_overrides or {}),
|
||||||
title_regex=title_regex_stored,
|
title_regex=title_regex_stored,
|
||||||
skip_subscriber_only=skip_so,
|
skip_subscriber_only=skip_so,
|
||||||
|
clip_start=clip_start,
|
||||||
|
clip_end=clip_end,
|
||||||
last_checked=time.time(),
|
last_checked=time.time(),
|
||||||
seen_ids=list(dict.fromkeys(all_ids)),
|
seen_ids=list(dict.fromkeys(all_ids)),
|
||||||
error=None,
|
error=None,
|
||||||
@@ -705,6 +771,13 @@ class SubscriptionManager:
|
|||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return {"status": "error", "msg": str(exc)}
|
return {"status": "error", "msg": str(exc)}
|
||||||
|
|
||||||
|
validated_folder: Optional[str] = None
|
||||||
|
if "folder" in changes:
|
||||||
|
try:
|
||||||
|
validated_folder = validate_subscription_folder(changes["folder"])
|
||||||
|
except ValueError as exc:
|
||||||
|
return {"status": "error", "msg": str(exc)}
|
||||||
|
|
||||||
validated_tr: Optional[str] = None
|
validated_tr: Optional[str] = None
|
||||||
if "title_regex" in changes:
|
if "title_regex" in changes:
|
||||||
try:
|
try:
|
||||||
@@ -755,6 +828,10 @@ class SubscriptionManager:
|
|||||||
sub.check_interval_minutes = validated_interval
|
sub.check_interval_minutes = validated_interval
|
||||||
if validated_name is not None:
|
if validated_name is not None:
|
||||||
sub.name = validated_name
|
sub.name = validated_name
|
||||||
|
if validated_folder is not None:
|
||||||
|
# Applies to future downloads only; files already downloaded
|
||||||
|
# stay where they are.
|
||||||
|
sub.folder = validated_folder
|
||||||
if validated_tr is not None:
|
if validated_tr is not None:
|
||||||
sub.title_regex = validated_tr
|
sub.title_regex = validated_tr
|
||||||
if skip_so_set:
|
if skip_so_set:
|
||||||
@@ -885,6 +962,7 @@ class SubscriptionManager:
|
|||||||
dl_plimit = cur.playlist_item_limit
|
dl_plimit = cur.playlist_item_limit
|
||||||
dl_autostart = cur.auto_start
|
dl_autostart = cur.auto_start
|
||||||
dl_split = cur.split_by_chapters
|
dl_split = cur.split_by_chapters
|
||||||
|
dl_sponsorblock = cur.sponsorblock
|
||||||
dl_chapter = cur.chapter_template
|
dl_chapter = cur.chapter_template
|
||||||
dl_sublang = cur.subtitle_language
|
dl_sublang = cur.subtitle_language
|
||||||
dl_submode = cur.subtitle_mode
|
dl_submode = cur.subtitle_mode
|
||||||
@@ -892,6 +970,8 @@ class SubscriptionManager:
|
|||||||
dl_ytdl_overrides = dict(cur.ytdl_options_overrides)
|
dl_ytdl_overrides = dict(cur.ytdl_options_overrides)
|
||||||
dl_title_regex = cur.title_regex or ""
|
dl_title_regex = cur.title_regex or ""
|
||||||
dl_skip_subscriber_only = bool(cur.skip_subscriber_only)
|
dl_skip_subscriber_only = bool(cur.skip_subscriber_only)
|
||||||
|
dl_clip_start = cur.clip_start
|
||||||
|
dl_clip_end = cur.clip_end
|
||||||
|
|
||||||
new_entries: list[dict] = []
|
new_entries: list[dict] = []
|
||||||
for ent in entries:
|
for ent in entries:
|
||||||
@@ -951,11 +1031,14 @@ class SubscriptionManager:
|
|||||||
playlist_item_limit=dl_plimit,
|
playlist_item_limit=dl_plimit,
|
||||||
auto_start=dl_autostart,
|
auto_start=dl_autostart,
|
||||||
split_by_chapters=dl_split,
|
split_by_chapters=dl_split,
|
||||||
|
sponsorblock=dl_sponsorblock,
|
||||||
chapter_template=dl_chapter or "",
|
chapter_template=dl_chapter or "",
|
||||||
subtitle_language=dl_sublang,
|
subtitle_language=dl_sublang,
|
||||||
subtitle_mode=dl_submode,
|
subtitle_mode=dl_submode,
|
||||||
ytdl_options_presets=dl_ytdl_presets,
|
ytdl_options_presets=dl_ytdl_presets,
|
||||||
ytdl_options_overrides=dl_ytdl_overrides,
|
ytdl_options_overrides=dl_ytdl_overrides,
|
||||||
|
clip_start=dl_clip_start,
|
||||||
|
clip_end=dl_clip_end,
|
||||||
)
|
)
|
||||||
log.info(
|
log.info(
|
||||||
"Subscription check finished for %s: %d new, %d filtered, %d subscriber_skipped, %d queued, %d failed",
|
"Subscription check finished for %s: %d new, %d filtered, %d subscriber_skipped, %d queued, %d failed",
|
||||||
|
|||||||
+230
-4
@@ -349,17 +349,89 @@ async def test_add_passes_clip_bounds_to_queue(mock_dqueue):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_subscribe_rejects_clip_options(mock_dqueue, monkeypatch):
|
async def test_subscribe_passes_clip_bounds(mock_dqueue, monkeypatch):
|
||||||
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock())
|
"""Issue #1049: a subscription's options apply to every future download, and
|
||||||
|
clip bounds were the one option carved out of that."""
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
req = _json_request(
|
req = _json_request(
|
||||||
{
|
{
|
||||||
**_valid_video_add_body(clip_start="10"),
|
**_valid_video_add_body(clip_start="2:26", clip_end="3:24"),
|
||||||
"check_interval_minutes": 60,
|
"check_interval_minutes": 60,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
resp = await main.subscribe(req)
|
||||||
|
assert resp.status == 200
|
||||||
|
kwargs = main.submgr.add_subscription.await_args.kwargs
|
||||||
|
assert kwargs["clip_start"] == pytest.approx(146.0)
|
||||||
|
assert kwargs["clip_end"] == pytest.approx(204.0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_passes_sponsorblock(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
req = _json_request(
|
||||||
|
{**_valid_video_add_body(), "check_interval_minutes": 60, "sponsorblock": True}
|
||||||
|
)
|
||||||
|
resp = await main.subscribe(req)
|
||||||
|
assert resp.status == 200
|
||||||
|
assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_defaults_sponsorblock_off(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
req = _json_request({**_valid_video_add_body(), "check_interval_minutes": 60})
|
||||||
|
await main.subscribe(req)
|
||||||
|
assert main.submgr.add_subscription.await_args.kwargs["sponsorblock"] is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_without_clip_fields_stores_none(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
req = _json_request({**_valid_video_add_body(), "check_interval_minutes": 60})
|
||||||
|
await main.subscribe(req)
|
||||||
|
kwargs = main.submgr.add_subscription.await_args.kwargs
|
||||||
|
assert kwargs["clip_start"] is None
|
||||||
|
assert kwargs["clip_end"] is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_ignores_t_param_in_url(mock_dqueue, monkeypatch):
|
||||||
|
"""A t= timestamp means "start here" for a one-off download of that video.
|
||||||
|
On a channel or playlist URL it says nothing about the videos it yields, so
|
||||||
|
it must not silently clip every future download."""
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
body = _valid_video_add_body()
|
||||||
|
# t= is only honoured on YouTube hosts, so this must be one to exercise it.
|
||||||
|
body["url"] = "https://www.youtube.com/@somechannel?t=90"
|
||||||
|
req = _json_request({**body, "check_interval_minutes": 60})
|
||||||
|
await main.subscribe(req)
|
||||||
|
kwargs = main.submgr.add_subscription.await_args.kwargs
|
||||||
|
assert kwargs["clip_start"] is None
|
||||||
|
assert kwargs["clip_end"] is None
|
||||||
|
# The timestamp is still stripped from the URL that gets stored.
|
||||||
|
assert "t=90" not in main.submgr.add_subscription.await_args.args[0]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_explicit_clip_wins_over_t_param(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
body = _valid_video_add_body(clip_start="30")
|
||||||
|
body["url"] = "https://www.youtube.com/@somechannel?t=90"
|
||||||
|
req = _json_request({**body, "check_interval_minutes": 60})
|
||||||
|
await main.subscribe(req)
|
||||||
|
kwargs = main.submgr.add_subscription.await_args.kwargs
|
||||||
|
assert kwargs["clip_start"] == pytest.approx(30.0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscribe_still_rejects_clips_for_non_media(mock_dqueue, monkeypatch):
|
||||||
|
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock(return_value={"status": "ok"}))
|
||||||
|
body = _valid_video_add_body(clip_start="10")
|
||||||
|
body["download_type"] = "thumbnail"
|
||||||
|
req = _json_request({**body, "check_interval_minutes": 60})
|
||||||
with pytest.raises(web.HTTPBadRequest):
|
with pytest.raises(web.HTTPBadRequest):
|
||||||
await main.subscribe(req)
|
await main.subscribe(req)
|
||||||
main.submgr.add_subscription.assert_not_awaited()
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -380,6 +452,33 @@ async def test_subscriptions_update_invalid_interval_returns_error_not_500(mock_
|
|||||||
assert body["status"] == "error"
|
assert body["status"] == "error"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscriptions_update_accepts_folder(monkeypatch, mock_dqueue):
|
||||||
|
"""Issue #1052: folder was absent from the route's accepted fields, so a
|
||||||
|
folder-only update was rejected outright as having nothing to update."""
|
||||||
|
submgr = MagicMock()
|
||||||
|
submgr.update_subscription = AsyncMock(return_value={"status": "ok"})
|
||||||
|
monkeypatch.setattr(main, "submgr", submgr)
|
||||||
|
|
||||||
|
req = _json_request({"id": "abc", "folder": "channels/jane"})
|
||||||
|
resp = await main.subscriptions_update(req)
|
||||||
|
|
||||||
|
assert resp.status == 200
|
||||||
|
submgr.update_subscription.assert_awaited_once_with("abc", {"folder": "channels/jane"})
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscriptions_update_still_drops_unknown_fields(monkeypatch, mock_dqueue):
|
||||||
|
submgr = MagicMock()
|
||||||
|
submgr.update_subscription = AsyncMock(return_value={"status": "ok"})
|
||||||
|
monkeypatch.setattr(main, "submgr", submgr)
|
||||||
|
|
||||||
|
req = _json_request({"id": "abc", "seen_ids": ["x"], "url": "https://evil.example"})
|
||||||
|
with pytest.raises(web.HTTPBadRequest):
|
||||||
|
await main.subscriptions_update(req)
|
||||||
|
submgr.update_subscription.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
def test_is_within_state_dir_blocks_state_subtree():
|
def test_is_within_state_dir_blocks_state_subtree():
|
||||||
state_dir = main._STATE_DIR_REAL
|
state_dir = main._STATE_DIR_REAL
|
||||||
assert main._is_within_state_dir(state_dir)
|
assert main._is_within_state_dir(state_dir)
|
||||||
@@ -426,3 +525,130 @@ async def test_download_blocks_state_dir_files(monkeypatch):
|
|||||||
(download_dir / "video.mp4").unlink(missing_ok=True)
|
(download_dir / "video.mp4").unlink(missing_ok=True)
|
||||||
(download_dir / percent_filename).unlink(missing_ok=True)
|
(download_dir / percent_filename).unlink(missing_ok=True)
|
||||||
state_dir.rmdir()
|
state_dir.rmdir()
|
||||||
|
|
||||||
|
# --- CORS (issue #155) -------------------------------------------------------
|
||||||
|
#
|
||||||
|
# The security property under test: credentials are granted only to an origin
|
||||||
|
# the operator named explicitly, and never under the '*' wildcard. Each test
|
||||||
|
# builds a fresh Application because main.app binds to the first event loop
|
||||||
|
# that runs it; the logic under test lives entirely in main.on_prepare, and the
|
||||||
|
# real main.add_cors preflight handler is mounted so the preflight path is the
|
||||||
|
# production one.
|
||||||
|
|
||||||
|
async def _cors_version(request):
|
||||||
|
return web.Response(text="v")
|
||||||
|
|
||||||
|
|
||||||
|
def _cors_app():
|
||||||
|
app = web.Application()
|
||||||
|
app.router.add_route("OPTIONS", "/add", main.add_cors)
|
||||||
|
app.router.add_get("/version", _cors_version)
|
||||||
|
app.on_response_prepare.append(main.on_prepare)
|
||||||
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
async def _cors_headers(monkeypatch, origins, origin, path="/add", method="OPTIONS"):
|
||||||
|
monkeypatch.setattr(main, "_cors_origins", origins)
|
||||||
|
async with TestClient(TestServer(_cors_app())) as client:
|
||||||
|
resp = await client.request(
|
||||||
|
method, path,
|
||||||
|
headers={
|
||||||
|
"Origin": origin,
|
||||||
|
"Access-Control-Request-Method": "POST",
|
||||||
|
"Access-Control-Request-Headers": "content-type,authorization",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return resp.headers
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_listed_origin_gets_credentials(monkeypatch):
|
||||||
|
h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "https://www.youtube.com")
|
||||||
|
assert h["Access-Control-Allow-Origin"] == "https://www.youtube.com"
|
||||||
|
assert h["Access-Control-Allow-Credentials"] == "true"
|
||||||
|
assert "Authorization" in h["Access-Control-Allow-Headers"]
|
||||||
|
assert "Origin" in h["Vary"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_wildcard_never_grants_credentials(monkeypatch):
|
||||||
|
h = await _cors_headers(monkeypatch, ["*"], "https://evil.example")
|
||||||
|
# The wildcard still reflects the origin, exactly as before...
|
||||||
|
assert h["Access-Control-Allow-Origin"] == "https://evil.example"
|
||||||
|
# ...but must not hand out the user's session, nor let a page attempt
|
||||||
|
# credentials of its own.
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
assert h["Access-Control-Allow-Headers"] == "Content-Type"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_wildcard_mixed_with_named_origin_still_denies_credentials(monkeypatch):
|
||||||
|
# '*' anywhere in the list disables credentials for everyone in it, so a
|
||||||
|
# stray wildcard cannot silently widen a named grant.
|
||||||
|
h = await _cors_headers(monkeypatch, ["*", "https://www.youtube.com"], "https://www.youtube.com")
|
||||||
|
assert h["Access-Control-Allow-Origin"] == "https://www.youtube.com"
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
assert h["Access-Control-Allow-Headers"] == "Content-Type"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_unlisted_origin_gets_nothing(monkeypatch):
|
||||||
|
h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "https://evil.example")
|
||||||
|
assert "Access-Control-Allow-Origin" not in h
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_disabled_by_default(monkeypatch):
|
||||||
|
h = await _cors_headers(monkeypatch, [], "https://www.youtube.com")
|
||||||
|
assert "Access-Control-Allow-Origin" not in h
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_credentials_apply_to_actual_response_not_just_preflight(monkeypatch):
|
||||||
|
# The browser checks Allow-Credentials on the real response too, so a
|
||||||
|
# preflight-only grant would still fail.
|
||||||
|
h = await _cors_headers(
|
||||||
|
monkeypatch, ["https://www.youtube.com"], "https://www.youtube.com",
|
||||||
|
path="/version", method="GET")
|
||||||
|
assert h["Access-Control-Allow-Credentials"] == "true"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_origin_match_is_exact(monkeypatch):
|
||||||
|
# Substring or suffix matching here would be a bypass.
|
||||||
|
for impostor in (
|
||||||
|
"https://www.youtube.com.evil.example",
|
||||||
|
"https://evilwww.youtube.com",
|
||||||
|
"http://www.youtube.com",
|
||||||
|
"https://www.youtube.com:8443",
|
||||||
|
):
|
||||||
|
h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], impostor)
|
||||||
|
assert "Access-Control-Allow-Origin" not in h, impostor
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h, impostor
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_null_origin_is_not_trusted(monkeypatch):
|
||||||
|
# Sandboxed iframes and some file:// contexts send Origin: null.
|
||||||
|
h = await _cors_headers(monkeypatch, ["https://www.youtube.com"], "null")
|
||||||
|
assert "Access-Control-Allow-Origin" not in h
|
||||||
|
assert "Access-Control-Allow-Credentials" not in h
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_cors_vary_appends_to_existing_value(monkeypatch):
|
||||||
|
# Static responses can already carry a Vary; clobbering it would break
|
||||||
|
# content negotiation.
|
||||||
|
monkeypatch.setattr(main, "_cors_origins", ["https://www.youtube.com"])
|
||||||
|
|
||||||
|
async def handler(request):
|
||||||
|
return web.Response(text="x", headers={"Vary": "Accept-Encoding"})
|
||||||
|
|
||||||
|
app = web.Application()
|
||||||
|
app.router.add_get("/v", handler)
|
||||||
|
app.on_response_prepare.append(main.on_prepare)
|
||||||
|
async with TestClient(TestServer(app)) as client:
|
||||||
|
resp = await client.get("/v", headers={"Origin": "https://www.youtube.com"})
|
||||||
|
assert resp.headers["Vary"] == "Accept-Encoding, Origin"
|
||||||
|
|||||||
@@ -51,6 +51,24 @@ class ConfigTests(unittest.TestCase):
|
|||||||
self.assertEqual(c.PUBLIC_HOST_URL, "")
|
self.assertEqual(c.PUBLIC_HOST_URL, "")
|
||||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "")
|
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "")
|
||||||
|
|
||||||
|
def test_host_wildcard_becomes_empty_for_dual_stack(self):
|
||||||
|
# Regression: aiohttp passes HOST to getaddrinfo, which does not resolve
|
||||||
|
# '*' -- the server died at startup on a DNS error. '*' now selects the
|
||||||
|
# empty string, the only value asyncio expands to a listening socket per
|
||||||
|
# address family.
|
||||||
|
for raw in ("*", " * "):
|
||||||
|
with self.subTest(raw=raw):
|
||||||
|
with patch.dict(os.environ, _base_env(HOST=raw), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.HOST, "")
|
||||||
|
|
||||||
|
def test_host_literal_addresses_are_untouched(self):
|
||||||
|
for raw in ("0.0.0.0", "::", "127.0.0.1", ""):
|
||||||
|
with self.subTest(raw=raw):
|
||||||
|
with patch.dict(os.environ, _base_env(HOST=raw), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.HOST, raw)
|
||||||
|
|
||||||
def test_blank_audio_host_falls_back_to_audio_download_route(self):
|
def test_blank_audio_host_falls_back_to_audio_download_route(self):
|
||||||
# Regression: a present-but-blank PUBLIC_HOST_AUDIO_URL must not stay empty
|
# Regression: a present-but-blank PUBLIC_HOST_AUDIO_URL must not stay empty
|
||||||
# (which produced root-relative, 404ing audio links). It falls back to the
|
# (which produced root-relative, 404ing audio links). It falls back to the
|
||||||
@@ -77,6 +95,34 @@ class ConfigTests(unittest.TestCase):
|
|||||||
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
||||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
|
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
|
||||||
|
|
||||||
|
def test_download_dirs_lose_trailing_slash(self):
|
||||||
|
# get_custom_dirs strips the base path as a prefix from each subdirectory,
|
||||||
|
# and the base directory's own path has no trailing slash -- so a trailing
|
||||||
|
# slash here leaked the absolute path into the folder dropdown.
|
||||||
|
with patch.dict(os.environ, _base_env(
|
||||||
|
DOWNLOAD_DIR="/downloads/",
|
||||||
|
AUDIO_DOWNLOAD_DIR="/audio/",
|
||||||
|
TEMP_DIR="/tmp/",
|
||||||
|
STATE_DIR="/state/",
|
||||||
|
), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DOWNLOAD_DIR, "/downloads")
|
||||||
|
self.assertEqual(c.AUDIO_DOWNLOAD_DIR, "/audio")
|
||||||
|
self.assertEqual(c.TEMP_DIR, "/tmp")
|
||||||
|
self.assertEqual(c.STATE_DIR, "/state")
|
||||||
|
|
||||||
|
def test_root_download_dir_survives_normalisation(self):
|
||||||
|
with patch.dict(os.environ, _base_env(DOWNLOAD_DIR="/", AUDIO_DOWNLOAD_DIR="///"), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DOWNLOAD_DIR, "/")
|
||||||
|
self.assertEqual(c.AUDIO_DOWNLOAD_DIR, "/")
|
||||||
|
|
||||||
|
def test_download_dirs_without_trailing_slash_unchanged(self):
|
||||||
|
with patch.dict(os.environ, _base_env(DOWNLOAD_DIR="/downloads", AUDIO_DOWNLOAD_DIR="."), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DOWNLOAD_DIR, "/downloads")
|
||||||
|
self.assertEqual(c.AUDIO_DOWNLOAD_DIR, ".")
|
||||||
|
|
||||||
def test_ytdl_options_json_loaded(self):
|
def test_ytdl_options_json_loaded(self):
|
||||||
opts = {"quiet": True, "no_warnings": True}
|
opts = {"quiet": True, "no_warnings": True}
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
@@ -115,6 +161,28 @@ class ConfigTests(unittest.TestCase):
|
|||||||
self.assertNotIn("HOST", safe)
|
self.assertNotIn("HOST", safe)
|
||||||
self.assertEqual(safe["ALLOW_YTDL_OPTIONS_OVERRIDES"], False)
|
self.assertEqual(safe["ALLOW_YTDL_OPTIONS_OVERRIDES"], False)
|
||||||
|
|
||||||
|
def test_default_folder_empty_by_default(self):
|
||||||
|
with patch.dict(os.environ, _base_env(), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DEFAULT_FOLDER, "")
|
||||||
|
|
||||||
|
def test_default_folder_is_trimmed_and_reaches_the_frontend(self):
|
||||||
|
with patch.dict(os.environ, _base_env(DEFAULT_FOLDER=" /youtube/ "), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DEFAULT_FOLDER, "youtube")
|
||||||
|
self.assertEqual(c.frontend_safe()["DEFAULT_FOLDER"], "youtube")
|
||||||
|
|
||||||
|
def test_default_folder_ignored_without_custom_dirs(self):
|
||||||
|
# The folder field is not shown at all without CUSTOM_DIRS, and sending
|
||||||
|
# a folder anyway is rejected by the download path check.
|
||||||
|
with patch.dict(
|
||||||
|
os.environ,
|
||||||
|
_base_env(DEFAULT_FOLDER="youtube", CUSTOM_DIRS="false"),
|
||||||
|
clear=False,
|
||||||
|
):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DEFAULT_FOLDER, "")
|
||||||
|
|
||||||
def test_allow_ytdl_options_overrides_boolean_loaded(self):
|
def test_allow_ytdl_options_overrides_boolean_loaded(self):
|
||||||
with patch.dict(os.environ, _base_env(ALLOW_YTDL_OPTIONS_OVERRIDES="true"), clear=False):
|
with patch.dict(os.environ, _base_env(ALLOW_YTDL_OPTIONS_OVERRIDES="true"), clear=False):
|
||||||
c = Config()
|
c = Config()
|
||||||
|
|||||||
@@ -2,9 +2,11 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import copy
|
import copy
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import socket
|
||||||
import tempfile
|
import tempfile
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
@@ -27,6 +29,9 @@ def dq_env():
|
|||||||
cfg.AUDIO_DOWNLOAD_DIR = dl
|
cfg.AUDIO_DOWNLOAD_DIR = dl
|
||||||
cfg.TEMP_DIR = dl
|
cfg.TEMP_DIR = dl
|
||||||
cfg.MAX_CONCURRENT_DOWNLOADS = "3"
|
cfg.MAX_CONCURRENT_DOWNLOADS = "3"
|
||||||
|
# Explicit: an unset attribute on a MagicMock is truthy, which would
|
||||||
|
# make validate_url bypass every SSRF check it is asked to run.
|
||||||
|
cfg.ALLOW_PRIVATE_ADDRESSES = False
|
||||||
cfg.YTDL_OPTIONS = {}
|
cfg.YTDL_OPTIONS = {}
|
||||||
cfg.YTDL_OPTIONS_PRESETS = {}
|
cfg.YTDL_OPTIONS_PRESETS = {}
|
||||||
cfg.CUSTOM_DIRS = True
|
cfg.CUSTOM_DIRS = True
|
||||||
@@ -164,6 +169,43 @@ async def test_add_ssrf_rejected_url_recorded_as_failed_entry(dq_env):
|
|||||||
notifier.completed.assert_awaited()
|
notifier.completed.assert_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_add_does_not_resolve_hostname_when_proxied(dq_env):
|
||||||
|
"""With a remote-DNS proxy configured, adding a URL must not look its host
|
||||||
|
up here: that both leaks the hostname to the local resolver and fails closed
|
||||||
|
when only the proxy can resolve it (issue #1079)."""
|
||||||
|
dq_env.YTDL_OPTIONS = {"proxy": "socks5h://tor:9050"}
|
||||||
|
notifier = AsyncMock()
|
||||||
|
|
||||||
|
def fake_extract(self, url, *_args, **_kwargs):
|
||||||
|
return {"_type": "video", "id": "vid1", "title": "t", "webpage_url": url}
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch("url_guard.socket.getaddrinfo", side_effect=AssertionError("resolved")), \
|
||||||
|
patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||||
|
result = await dq.add(
|
||||||
|
"https://only-the-proxy-can-resolve.invalid/x",
|
||||||
|
"video", "auto", "any", "best", "", "", 0, auto_start=False,
|
||||||
|
)
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
dq.close()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_add_resolves_hostname_when_not_proxied(dq_env):
|
||||||
|
"""Without a proxy the address check still runs and still rejects."""
|
||||||
|
notifier = AsyncMock()
|
||||||
|
url = "https://internal.invalid/x"
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch("url_guard.socket.getaddrinfo",
|
||||||
|
return_value=[(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "",
|
||||||
|
("169.254.169.254", 0))]):
|
||||||
|
result = await dq.add(url, "video", "auto", "any", "best", "", "", 0, auto_start=False)
|
||||||
|
assert result["status"] == "error"
|
||||||
|
dq.close()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_cancel_removes_from_pending(dq_env):
|
async def test_cancel_removes_from_pending(dq_env):
|
||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
@@ -270,6 +312,47 @@ async def test_start_pending_moves_to_queue(dq_env):
|
|||||||
with patch.object(DownloadQueue, "_DownloadQueue__start_download", AsyncMock()):
|
with patch.object(DownloadQueue, "_DownloadQueue__start_download", AsyncMock()):
|
||||||
await dq.start_pending([url])
|
await dq.start_pending([url])
|
||||||
assert not dq.pending.exists(url)
|
assert not dq.pending.exists(url)
|
||||||
|
# It is in the queue now and starts on its own, so it must not keep
|
||||||
|
# advertising the Start button the UI draws for 'pending' (#1081), and the
|
||||||
|
# client has to be told before the concurrency slot frees up.
|
||||||
|
assert dq.queue.get(url).info.status == "queued"
|
||||||
|
assert notifier.updated.await_args[0][0].status == "queued"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_queued_download_is_not_offered_as_startable(dq_env):
|
||||||
|
"""A download waiting on a MAX_CONCURRENT_DOWNLOADS slot used to sit at
|
||||||
|
'pending', which is the status the UI draws a Start button for — but
|
||||||
|
start_pending has nothing to do for an item already in the queue, so the
|
||||||
|
button silently did nothing and still reported success (#1081).
|
||||||
|
"""
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq_env.MAX_CONCURRENT_DOWNLOADS = "1"
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
released = asyncio.Event()
|
||||||
|
|
||||||
|
def fake_extract(self, url, *_args, **_kwargs):
|
||||||
|
return {"_type": "video", "id": url[-1], "title": f"Video {url[-1]}",
|
||||||
|
"url": url, "webpage_url": url}
|
||||||
|
|
||||||
|
async def blocking_start(self, notifier_, executor=None):
|
||||||
|
await released.wait()
|
||||||
|
|
||||||
|
first, second = "https://example.com/v1", "https://example.com/v2"
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||||
|
patch("ytdl.Download.start", blocking_start), \
|
||||||
|
patch("ytdl.Download.close", lambda self: None):
|
||||||
|
for url in (first, second):
|
||||||
|
await dq.add(url, "video", "auto", "any", "best", "", "", 0, auto_start=True)
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
# The first holds the only slot; the second is waiting behind it.
|
||||||
|
assert dq.queue.get(second).info.status == "queued"
|
||||||
|
assert not dq.pending.exists(second)
|
||||||
|
|
||||||
|
released.set()
|
||||||
|
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
@@ -330,7 +413,7 @@ async def test_retry_restores_playlist_output_context(dq_env):
|
|||||||
chapter_template="",
|
chapter_template="",
|
||||||
)
|
)
|
||||||
failed_info.status = "error"
|
failed_info.status = "error"
|
||||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, failed_info))
|
await dq.done.put(Download(None, None, None, None, "best", "any", {}, failed_info))
|
||||||
|
|
||||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
return {
|
return {
|
||||||
@@ -389,7 +472,7 @@ async def test_retry_keeps_playlist_context_through_url_indirection(dq_env):
|
|||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
url = "https://example.com/watch?v=1"
|
url = "https://example.com/watch?v=1"
|
||||||
resolved = "https://example.com/resolved?v=1"
|
resolved = "https://example.com/resolved?v=1"
|
||||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url)))
|
await dq.done.put(Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url)))
|
||||||
|
|
||||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
if extracted_url == url:
|
if extracted_url == url:
|
||||||
@@ -427,7 +510,7 @@ async def test_retry_reapplies_current_options_gates(dq_env):
|
|||||||
ytdl_options_presets=["Still There", "Removed Preset"],
|
ytdl_options_presets=["Still There", "Removed Preset"],
|
||||||
ytdl_options_overrides={"paths": {"home": "/etc"}},
|
ytdl_options_overrides={"paths": {"home": "/etc"}},
|
||||||
)
|
)
|
||||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
await dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
||||||
|
|
||||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
return {
|
return {
|
||||||
@@ -457,7 +540,7 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env):
|
|||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
url = "https://example.com/watch?v=1"
|
url = "https://example.com/watch?v=1"
|
||||||
info = _failed_playlist_item(url, ytdl_options_overrides={"writesubtitles": True})
|
info = _failed_playlist_item(url, ytdl_options_overrides={"writesubtitles": True})
|
||||||
dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
await dq.done.put(Download(None, None, None, None, "best", "any", {}, info))
|
||||||
|
|
||||||
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
return {
|
return {
|
||||||
@@ -476,6 +559,32 @@ async def test_retry_keeps_overrides_while_still_allowed(dq_env):
|
|||||||
assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True}
|
assert dq.queue.get(url).info.ytdl_options_overrides == {"writesubtitles": True}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_retry_carries_the_sponsorblock_flag(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
url = "https://example.com/watch?v=1"
|
||||||
|
await dq.done.put(
|
||||||
|
Download(None, None, None, None, "best", "any", {}, _failed_playlist_item(url, sponsorblock=True))
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_extract(self, extracted_url, *_args, **_kwargs):
|
||||||
|
return {
|
||||||
|
"_type": "video",
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Test Video",
|
||||||
|
"url": extracted_url,
|
||||||
|
"webpage_url": extracted_url,
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||||
|
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||||
|
result = await dq.retry(url)
|
||||||
|
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert dq.queue.get(url).info.sponsorblock is True
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
@@ -625,6 +734,82 @@ async def test_playlist_download_not_treated_as_channel(dq_env):
|
|||||||
assert download.output_template.startswith("My Playlist/")
|
assert download.output_template.startswith("My Playlist/")
|
||||||
|
|
||||||
|
|
||||||
|
def _channel_extraction(entry_id, **extra):
|
||||||
|
"""A channel yt-dlp reported as a playlist, addressed by *entry_id*."""
|
||||||
|
return {
|
||||||
|
"_type": "playlist",
|
||||||
|
"id": entry_id,
|
||||||
|
"channel_id": "UCabcd123",
|
||||||
|
"channel": "Odin",
|
||||||
|
"title": "Odin",
|
||||||
|
**extra,
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Salvia Plath - Pondering",
|
||||||
|
"url": "https://example.com/watch?v=1",
|
||||||
|
"webpage_url": "https://example.com/watch?v=1",
|
||||||
|
"channel": "Odin",
|
||||||
|
"upload_date": "20130804",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _add_and_get_template(dq_env, extraction, url):
|
||||||
|
dq_env.OUTPUT_TEMPLATE = "%(channel)s [YT]/%(title)s.%(ext)s"
|
||||||
|
dq_env.OUTPUT_TEMPLATE_CHANNEL = ""
|
||||||
|
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||||
|
|
||||||
|
def fake_extract(self, _url, *_args, **_kwargs):
|
||||||
|
return extraction
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, AsyncMock())
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||||
|
result = await dq.add(url, "video", "auto", "any", "best", "", "", 0, auto_start=False)
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
return dq.pending.get("https://example.com/watch?v=1").output_template
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_bare_handle_channel_url_is_treated_as_a_channel(dq_env):
|
||||||
|
"""A channel addressed as /@handle reports its id as the handle, not the
|
||||||
|
channel id, and was falling through to OUTPUT_TEMPLATE_PLAYLIST."""
|
||||||
|
template = await _add_and_get_template(
|
||||||
|
dq_env,
|
||||||
|
_channel_extraction("@odin", uploader_id="@odin"),
|
||||||
|
"https://www.youtube.com/@odin",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert template.startswith("Odin [YT]/")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_legacy_vanity_channel_url_is_treated_as_a_channel(dq_env):
|
||||||
|
"""A legacy /c/Name URL reports the vanity name as its id, while
|
||||||
|
uploader_id is still the handle."""
|
||||||
|
template = await _add_and_get_template(
|
||||||
|
dq_env,
|
||||||
|
_channel_extraction("Odin", uploader_id="@odin"),
|
||||||
|
"https://www.youtube.com/c/Odin",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert template.startswith("Odin [YT]/")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_playlist_with_owner_uploader_id_is_still_a_playlist(dq_env):
|
||||||
|
"""A real playlist carries its owner's channel_id and uploader_id, but its
|
||||||
|
own id matches neither, so it must keep the playlist template."""
|
||||||
|
template = await _add_and_get_template(
|
||||||
|
dq_env,
|
||||||
|
_channel_extraction("PLxyz789", uploader_id="@odin", title="My Playlist"),
|
||||||
|
"https://www.youtube.com/playlist?list=PLxyz789",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert template.startswith("My Playlist/")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_add_merges_global_preset_and_override_options(dq_env):
|
async def test_add_merges_global_preset_and_override_options(dq_env):
|
||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
@@ -1057,7 +1242,7 @@ async def test_probe_scheduled_starts_when_live(dq_env):
|
|||||||
|
|
||||||
assert url not in dq._scheduled_probe_at
|
assert url not in dq._scheduled_probe_at
|
||||||
assert download.info.live_status == "is_live"
|
assert download.info.live_status == "is_live"
|
||||||
assert download.info.status == "pending"
|
assert download.info.status == "queued"
|
||||||
start_mock.assert_called_once_with(download)
|
start_mock.assert_called_once_with(download)
|
||||||
|
|
||||||
|
|
||||||
@@ -1202,7 +1387,7 @@ async def test_probe_recovers_after_transient_then_starts(dq_env):
|
|||||||
|
|
||||||
assert url not in dq._scheduled_probe_at
|
assert url not in dq._scheduled_probe_at
|
||||||
assert url not in dq._scheduled_probe_failures
|
assert url not in dq._scheduled_probe_failures
|
||||||
assert download.info.status == "pending"
|
assert download.info.status == "queued"
|
||||||
# Placeholder error/msg cleared now that a real download is starting.
|
# Placeholder error/msg cleared now that a real download is starting.
|
||||||
assert download.info.error is None
|
assert download.info.error is None
|
||||||
assert download.info.msg is None
|
assert download.info.msg is None
|
||||||
@@ -1339,9 +1524,9 @@ async def test_post_download_cleanup_clears_filename_on_error(dq_env):
|
|||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
|
download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
|
||||||
dq.queue.put(download)
|
await dq.queue.put(download)
|
||||||
|
|
||||||
dq._post_download_cleanup(download)
|
await dq._post_download_cleanup(download)
|
||||||
|
|
||||||
assert download.info.status == "error"
|
assert download.info.status == "error"
|
||||||
assert download.info.filename is None
|
assert download.info.filename is None
|
||||||
@@ -1354,9 +1539,9 @@ async def test_post_download_cleanup_keeps_captured_subtitles_on_error(dq_env):
|
|||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
|
download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
|
||||||
download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
|
download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
|
||||||
dq.queue.put(download)
|
await dq.queue.put(download)
|
||||||
|
|
||||||
dq._post_download_cleanup(download)
|
await dq._post_download_cleanup(download)
|
||||||
|
|
||||||
assert download.info.status == "error"
|
assert download.info.status == "error"
|
||||||
assert download.info.filename == "en.srt"
|
assert download.info.filename == "en.srt"
|
||||||
@@ -1376,7 +1561,7 @@ async def test_clear_skips_deletion_outside_download_directory(dq_env):
|
|||||||
# A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
|
# A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
|
||||||
escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
|
escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
|
||||||
download = _make_download(dq_env, status="finished", filename=escaping_filename)
|
download = _make_download(dq_env, status="finished", filename=escaping_filename)
|
||||||
dq.done.put(download)
|
await dq.done.put(download)
|
||||||
|
|
||||||
await dq.clear([download.info.url])
|
await dq.clear([download.info.url])
|
||||||
|
|
||||||
|
|||||||
@@ -313,3 +313,40 @@ class GetCustomDirsTests(unittest.TestCase):
|
|||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
||||||
|
|
||||||
|
class WarnIfCookiefileShadowedTests(unittest.TestCase):
|
||||||
|
"""Issue #881: an uploaded cookies file wins over an operator-configured
|
||||||
|
cookiefile, and used to do so with no way for anyone to notice."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._saved = main.config.YTDL_OPTIONS
|
||||||
|
main.config.YTDL_OPTIONS = dict(self._saved)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
main.config.YTDL_OPTIONS = self._saved
|
||||||
|
|
||||||
|
def test_warns_when_a_different_cookiefile_is_configured(self):
|
||||||
|
main.config.YTDL_OPTIONS["cookiefile"] = "/cookies/cookies.txt"
|
||||||
|
with self.assertLogs("main", level="WARNING") as cm:
|
||||||
|
main.warn_if_cookiefile_shadowed()
|
||||||
|
joined = "\n".join(cm.output)
|
||||||
|
self.assertIn("/cookies/cookies.txt", joined)
|
||||||
|
self.assertIn(main.COOKIES_PATH, joined)
|
||||||
|
|
||||||
|
def test_silent_when_no_cookiefile_configured(self):
|
||||||
|
main.config.YTDL_OPTIONS.pop("cookiefile", None)
|
||||||
|
with self.assertNoLogs("main", level="WARNING"):
|
||||||
|
main.warn_if_cookiefile_shadowed()
|
||||||
|
|
||||||
|
def test_silent_when_configured_file_is_the_uploaded_one(self):
|
||||||
|
# The steady state after an upload: re-running must not nag.
|
||||||
|
main.config.YTDL_OPTIONS["cookiefile"] = main.COOKIES_PATH
|
||||||
|
with self.assertNoLogs("main", level="WARNING"):
|
||||||
|
main.warn_if_cookiefile_shadowed()
|
||||||
|
|
||||||
|
def test_silent_on_non_string_or_empty_values(self):
|
||||||
|
for value in (None, "", 0, [], {}):
|
||||||
|
main.config.YTDL_OPTIONS["cookiefile"] = value
|
||||||
|
with self.assertNoLogs("main", level="WARNING"):
|
||||||
|
main.warn_if_cookiefile_shadowed()
|
||||||
|
|||||||
@@ -2,8 +2,11 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
import shelve
|
import shelve
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
@@ -69,22 +72,22 @@ def _create_legacy_shelf(path: str, *infos: DownloadInfo) -> None:
|
|||||||
shelf[info.url] = info
|
shelf[info.url] = info
|
||||||
|
|
||||||
|
|
||||||
class PersistentQueueTests(unittest.TestCase):
|
class PersistentQueueTests(unittest.IsolatedAsyncioTestCase):
|
||||||
def test_put_get_delete_roundtrip(self):
|
async def test_put_get_delete_roundtrip(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
dl = _FakeDownload(_make_info("http://a.example"))
|
dl = _FakeDownload(_make_info("http://a.example"))
|
||||||
pq.put(dl)
|
await pq.put(dl)
|
||||||
self.assertTrue(os.path.exists(path + ".json"))
|
self.assertTrue(os.path.exists(path + ".json"))
|
||||||
self.assertTrue(pq.exists("http://a.example"))
|
self.assertTrue(pq.exists("http://a.example"))
|
||||||
self.assertFalse(pq.empty())
|
self.assertFalse(pq.empty())
|
||||||
got = pq.get("http://a.example")
|
got = pq.get("http://a.example")
|
||||||
self.assertEqual(got.info.url, "http://a.example")
|
self.assertEqual(got.info.url, "http://a.example")
|
||||||
pq.delete("http://a.example")
|
await pq.delete("http://a.example")
|
||||||
self.assertFalse(pq.exists("http://a.example"))
|
self.assertFalse(pq.exists("http://a.example"))
|
||||||
|
|
||||||
def test_saved_items_sorted_by_timestamp(self):
|
async def test_saved_items_sorted_by_timestamp(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
@@ -92,16 +95,16 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
b = _FakeDownload(_make_info("http://second.example"))
|
b = _FakeDownload(_make_info("http://second.example"))
|
||||||
a.info.timestamp = 100
|
a.info.timestamp = 100
|
||||||
b.info.timestamp = 200
|
b.info.timestamp = 200
|
||||||
pq.put(a)
|
await pq.put(a)
|
||||||
pq.put(b)
|
await pq.put(b)
|
||||||
keys = [k for k, _ in pq.saved_items()]
|
keys = [k for k, _ in pq.saved_items()]
|
||||||
self.assertEqual(keys, ["http://first.example", "http://second.example"])
|
self.assertEqual(keys, ["http://first.example", "http://second.example"])
|
||||||
|
|
||||||
def test_load_restores_from_json(self):
|
async def test_load_restores_from_json(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq1 = PersistentQueue("queue", path)
|
pq1 = PersistentQueue("queue", path)
|
||||||
pq1.put(_FakeDownload(_make_info("http://load.example")))
|
await pq1.put(_FakeDownload(_make_info("http://load.example")))
|
||||||
pq2 = PersistentQueue("queue", path)
|
pq2 = PersistentQueue("queue", path)
|
||||||
pq2.load()
|
pq2.load()
|
||||||
self.assertTrue(pq2.exists("http://load.example"))
|
self.assertTrue(pq2.exists("http://load.example"))
|
||||||
@@ -115,7 +118,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
self.assertTrue(pq.exists("http://legacy.example"))
|
self.assertTrue(pq.exists("http://legacy.example"))
|
||||||
self.assertTrue(os.path.exists(path + ".json"))
|
self.assertTrue(os.path.exists(path + ".json"))
|
||||||
|
|
||||||
def test_queue_persists_only_compact_entry_subset(self):
|
async def test_queue_persists_only_compact_entry_subset(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
@@ -128,7 +131,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
"formats": [{"id": "huge"}],
|
"formats": [{"id": "huge"}],
|
||||||
"description": "very large payload",
|
"description": "very large payload",
|
||||||
}
|
}
|
||||||
pq.put(_FakeDownload(info))
|
await pq.put(_FakeDownload(info))
|
||||||
|
|
||||||
with open(path + ".json", encoding="utf-8") as f:
|
with open(path + ".json", encoding="utf-8") as f:
|
||||||
payload = json.load(f)
|
payload = json.load(f)
|
||||||
@@ -146,7 +149,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
self.assertNotIn("formats", record["entry"])
|
self.assertNotIn("formats", record["entry"])
|
||||||
self.assertNotIn("description", record["entry"])
|
self.assertNotIn("description", record["entry"])
|
||||||
|
|
||||||
def test_completed_queue_persists_only_failed_retry_context(self):
|
async def test_completed_queue_persists_only_failed_retry_context(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "completed")
|
path = os.path.join(tmp, "completed")
|
||||||
pq = PersistentQueue("completed", path)
|
pq = PersistentQueue("completed", path)
|
||||||
@@ -161,7 +164,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
"formats": [{"id": "huge"}],
|
"formats": [{"id": "huge"}],
|
||||||
}
|
}
|
||||||
info.filename = "done.mp4"
|
info.filename = "done.mp4"
|
||||||
pq.put(_FakeDownload(info))
|
await pq.put(_FakeDownload(info))
|
||||||
|
|
||||||
with open(path + ".json", encoding="utf-8") as f:
|
with open(path + ".json", encoding="utf-8") as f:
|
||||||
payload = json.load(f)
|
payload = json.load(f)
|
||||||
@@ -180,7 +183,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
self.assertEqual(record["filename"], "done.mp4")
|
self.assertEqual(record["filename"], "done.mp4")
|
||||||
|
|
||||||
info.status = "finished"
|
info.status = "finished"
|
||||||
pq.put(_FakeDownload(info))
|
await pq.put(_FakeDownload(info))
|
||||||
with open(path + ".json", encoding="utf-8") as f:
|
with open(path + ".json", encoding="utf-8") as f:
|
||||||
payload = json.load(f)
|
payload = json.load(f)
|
||||||
self.assertNotIn("entry", payload["items"][0]["info"])
|
self.assertNotIn("entry", payload["items"][0]["info"])
|
||||||
@@ -256,7 +259,7 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
self.assertNotIn("speed", record)
|
self.assertNotIn("speed", record)
|
||||||
self.assertNotIn("eta", record)
|
self.assertNotIn("eta", record)
|
||||||
|
|
||||||
def test_put_rollbacks_in_memory_queue_when_state_write_fails(self):
|
async def test_put_rollbacks_in_memory_queue_when_state_write_fails(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
@@ -272,18 +275,18 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
|
|
||||||
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
||||||
with self.assertRaises(OSError):
|
with self.assertRaises(OSError):
|
||||||
pq.put(dl)
|
await pq.put(dl)
|
||||||
|
|
||||||
self.assertFalse(pq.exists("http://rollback.example"))
|
self.assertFalse(pq.exists("http://rollback.example"))
|
||||||
|
|
||||||
def test_put_rollbacks_to_previous_download_when_replace_fails(self):
|
async def test_put_rollbacks_to_previous_download_when_replace_fails(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue")
|
path = os.path.join(tmp, "queue")
|
||||||
pq = PersistentQueue("queue", path)
|
pq = PersistentQueue("queue", path)
|
||||||
first = _FakeDownload(_make_info("http://same.example"))
|
first = _FakeDownload(_make_info("http://same.example"))
|
||||||
second = _FakeDownload(_make_info("http://same.example"))
|
second = _FakeDownload(_make_info("http://same.example"))
|
||||||
second.info.title = "Replaced title"
|
second.info.title = "Replaced title"
|
||||||
pq.put(first)
|
await pq.put(first)
|
||||||
|
|
||||||
orig_save = __import__("state_store").AtomicJsonStore.save
|
orig_save = __import__("state_store").AtomicJsonStore.save
|
||||||
|
|
||||||
@@ -294,10 +297,68 @@ class PersistentQueueTests(unittest.TestCase):
|
|||||||
|
|
||||||
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
||||||
with self.assertRaises(OSError):
|
with self.assertRaises(OSError):
|
||||||
pq.put(second)
|
await pq.put(second)
|
||||||
|
|
||||||
self.assertEqual(pq.get("http://same.example").info.title, "Title")
|
self.assertEqual(pq.get("http://same.example").info.title, "Title")
|
||||||
|
|
||||||
|
|
||||||
|
class StateWriteOffEventLoopTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
"""State writes fsync twice; on a slow disk that must not stall the loop.
|
||||||
|
|
||||||
|
Before this, put()/delete() wrote inline, so a queue mutation blocked every
|
||||||
|
other request the server was serving for as long as the filesystem took.
|
||||||
|
See issue #980.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def test_save_runs_off_the_event_loop_thread(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
pq = PersistentQueue("queue", os.path.join(tmp, "queue"))
|
||||||
|
self.addCleanup(pq.close)
|
||||||
|
loop_thread = threading.get_ident()
|
||||||
|
save_threads = []
|
||||||
|
|
||||||
|
orig_save = __import__("state_store").AtomicJsonStore.save
|
||||||
|
|
||||||
|
def recording_save(store, data):
|
||||||
|
save_threads.append(threading.get_ident())
|
||||||
|
return orig_save(store, data)
|
||||||
|
|
||||||
|
with patch("ytdl.AtomicJsonStore.save", recording_save):
|
||||||
|
await pq.put(_FakeDownload(_make_info("http://a.example")))
|
||||||
|
|
||||||
|
self.assertEqual(len(save_threads), 1)
|
||||||
|
self.assertNotEqual(save_threads[0], loop_thread)
|
||||||
|
|
||||||
|
async def test_a_slow_write_does_not_stall_other_coroutines(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
pq = PersistentQueue("queue", os.path.join(tmp, "queue"))
|
||||||
|
self.addCleanup(pq.close)
|
||||||
|
orig_save = __import__("state_store").AtomicJsonStore.save
|
||||||
|
|
||||||
|
def slow_save(store, data):
|
||||||
|
time.sleep(0.3)
|
||||||
|
return orig_save(store, data)
|
||||||
|
|
||||||
|
ticks = 0
|
||||||
|
|
||||||
|
async def ticker():
|
||||||
|
nonlocal ticks
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(0.01)
|
||||||
|
ticks += 1
|
||||||
|
|
||||||
|
ticking = asyncio.create_task(ticker())
|
||||||
|
try:
|
||||||
|
with patch("ytdl.AtomicJsonStore.save", slow_save):
|
||||||
|
await pq.put(_FakeDownload(_make_info("http://a.example")))
|
||||||
|
finally:
|
||||||
|
ticking.cancel()
|
||||||
|
|
||||||
|
# An inline write would have starved the loop for the whole 0.3s and
|
||||||
|
# left ticks at 0.
|
||||||
|
self.assertGreater(ticks, 5)
|
||||||
|
self.assertTrue(pq.exists("http://a.example"))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -409,6 +409,139 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
|||||||
self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
|
self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
|
||||||
self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"])
|
self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"])
|
||||||
|
|
||||||
|
async def test_check_now_applies_subscription_clip_bounds(self):
|
||||||
|
"""Issue #1049: clip bounds were the one download option a subscription
|
||||||
|
could not carry, so they must reach every entry it queues."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
queue = _Queue()
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"subscriptions.extract_flat_playlist",
|
||||||
|
side_effect=[
|
||||||
|
(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[
|
||||||
|
{"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"},
|
||||||
|
{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
):
|
||||||
|
result = await mgr.add_subscription(
|
||||||
|
"https://example.com/channel",
|
||||||
|
check_interval_minutes=60,
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
quality="best",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
auto_start=True,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
subtitle_language="en",
|
||||||
|
subtitle_mode="prefer_manual",
|
||||||
|
clip_start=30.0,
|
||||||
|
clip_end=90.0,
|
||||||
|
)
|
||||||
|
sub_id = result["subscription"]["id"]
|
||||||
|
self.assertEqual(mgr.get(sub_id).clip_start, 30.0)
|
||||||
|
self.assertEqual(mgr.get(sub_id).clip_end, 90.0)
|
||||||
|
await mgr.check_now([sub_id])
|
||||||
|
|
||||||
|
self.assertEqual(len(queue.entries), 1)
|
||||||
|
_entry, args, _kwargs = queue.entries[0]
|
||||||
|
# add_entry(entry, download_type, ..., ytdl_options_overrides, clip_start, clip_end)
|
||||||
|
self.assertEqual(args[-2], 30.0)
|
||||||
|
self.assertEqual(args[-1], 90.0)
|
||||||
|
|
||||||
|
async def test_clip_bounds_survive_reload_and_default_to_none(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg = _Config(tmp)
|
||||||
|
queue = _Queue()
|
||||||
|
mgr = SubscriptionManager(cfg, queue, _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
# Records written before these fields existed simply take the defaults.
|
||||||
|
self.assertIsNone(mgr.get(sub_id).clip_start)
|
||||||
|
self.assertIsNone(mgr.get(sub_id).clip_end)
|
||||||
|
|
||||||
|
mgr.get(sub_id).clip_start = 12.5
|
||||||
|
async with mgr._lock:
|
||||||
|
mgr._save_locked()
|
||||||
|
|
||||||
|
reloaded = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
self.assertEqual(reloaded.get(sub_id).clip_start, 12.5)
|
||||||
|
self.assertIsNone(reloaded.get(sub_id).clip_end)
|
||||||
|
|
||||||
|
async def test_check_now_applies_subscription_sponsorblock(self):
|
||||||
|
"""Subscriptions download unattended, so the sponsor-segment removal has
|
||||||
|
to reach every entry the subscription queues, not just manual adds."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
queue = _Queue()
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"subscriptions.extract_flat_playlist",
|
||||||
|
side_effect=[
|
||||||
|
(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[
|
||||||
|
{"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"},
|
||||||
|
{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
|
||||||
|
],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
):
|
||||||
|
result = await mgr.add_subscription(
|
||||||
|
"https://example.com/channel",
|
||||||
|
check_interval_minutes=60,
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
quality="best",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
auto_start=True,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
subtitle_language="en",
|
||||||
|
subtitle_mode="prefer_manual",
|
||||||
|
sponsorblock=True,
|
||||||
|
)
|
||||||
|
sub_id = result["subscription"]["id"]
|
||||||
|
self.assertTrue(mgr.get(sub_id).sponsorblock)
|
||||||
|
await mgr.check_now([sub_id])
|
||||||
|
|
||||||
|
self.assertEqual(len(queue.entries), 1)
|
||||||
|
_entry, _args, kwargs = queue.entries[0]
|
||||||
|
self.assertIs(kwargs["sponsorblock"], True)
|
||||||
|
|
||||||
|
async def test_sponsorblock_survives_reload_and_defaults_to_false(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg = _Config(tmp)
|
||||||
|
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
# Records written before the field existed simply take the default.
|
||||||
|
self.assertFalse(mgr.get(sub_id).sponsorblock)
|
||||||
|
|
||||||
|
mgr.get(sub_id).sponsorblock = True
|
||||||
|
async with mgr._lock:
|
||||||
|
mgr._save_locked()
|
||||||
|
|
||||||
|
reloaded = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
self.assertTrue(reloaded.get(sub_id).sponsorblock)
|
||||||
|
|
||||||
async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
|
async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
queue = _Queue()
|
queue = _Queue()
|
||||||
@@ -891,6 +1024,75 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
|||||||
self.assertEqual(upd["status"], "ok")
|
self.assertEqual(upd["status"], "ok")
|
||||||
self.assertEqual(mgr.list_all()[0].name, "x" * 200)
|
self.assertEqual(mgr.list_all()[0].name, "x" * 200)
|
||||||
|
|
||||||
|
async def test_update_subscription_changes_folder(self):
|
||||||
|
"""Issue #1052: the folder was settable at creation and then frozen,
|
||||||
|
because it was never added to the fields the update route accepts."""
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
self.assertEqual(mgr.list_all()[0].folder, "")
|
||||||
|
|
||||||
|
upd = await mgr.update_subscription(sub_id, {"folder": " channels/jane "})
|
||||||
|
self.assertEqual(upd["status"], "ok")
|
||||||
|
self.assertEqual(upd["subscription"]["folder"], "channels/jane")
|
||||||
|
self.assertEqual(mgr.list_all()[0].folder, "channels/jane")
|
||||||
|
|
||||||
|
async def test_update_subscription_folder_survives_reload(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg = _Config(tmp)
|
||||||
|
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
await mgr.update_subscription(sub_id, {"folder": "archive"})
|
||||||
|
|
||||||
|
reloaded = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
self.assertEqual(reloaded.get(sub_id).folder, "archive")
|
||||||
|
|
||||||
|
async def test_update_subscription_clears_folder(self):
|
||||||
|
# An empty folder is valid and means the base download directory.
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
await mgr.update_subscription(sub_id, {"folder": "archive"})
|
||||||
|
|
||||||
|
upd = await mgr.update_subscription(sub_id, {"folder": " "})
|
||||||
|
self.assertEqual(upd["status"], "ok")
|
||||||
|
self.assertEqual(mgr.list_all()[0].folder, "")
|
||||||
|
|
||||||
|
async def test_update_subscription_rejects_unusable_folder(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
await mgr.update_subscription(sub_id, {"folder": "keep"})
|
||||||
|
|
||||||
|
bad_values = (
|
||||||
|
"/etc",
|
||||||
|
"/absolute/path",
|
||||||
|
"../escape",
|
||||||
|
"nested/../../escape",
|
||||||
|
"windows\\..\\escape",
|
||||||
|
42,
|
||||||
|
["a"],
|
||||||
|
)
|
||||||
|
for bad in bad_values:
|
||||||
|
upd = await mgr.update_subscription(sub_id, {"folder": bad})
|
||||||
|
self.assertEqual(upd["status"], "error", f"expected {bad!r} to be rejected")
|
||||||
|
self.assertEqual(mgr.list_all()[0].folder, "keep")
|
||||||
|
|
||||||
|
async def test_update_subscription_folder_leaves_other_fields_alone(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||||
|
sub_id = await self._add_one_subscription(mgr)
|
||||||
|
before = mgr.get(sub_id)
|
||||||
|
name, interval, enabled = before.name, before.check_interval_minutes, before.enabled
|
||||||
|
|
||||||
|
await mgr.update_subscription(sub_id, {"folder": "only/this"})
|
||||||
|
|
||||||
|
after = mgr.get(sub_id)
|
||||||
|
self.assertEqual(after.folder, "only/this")
|
||||||
|
self.assertEqual(after.name, name)
|
||||||
|
self.assertEqual(after.check_interval_minutes, interval)
|
||||||
|
self.assertEqual(after.enabled, enabled)
|
||||||
|
|
||||||
async def test_update_subscription_skip_subscriber_only(self):
|
async def test_update_subscription_skip_subscriber_only(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
queue = _Queue()
|
queue = _Queue()
|
||||||
|
|||||||
+323
-16
@@ -10,7 +10,10 @@ import url_guard
|
|||||||
from url_guard import (
|
from url_guard import (
|
||||||
validate_url,
|
validate_url,
|
||||||
_address_allowed_at_connect,
|
_address_allowed_at_connect,
|
||||||
|
_address_is_global,
|
||||||
_guarded_getaddrinfo,
|
_guarded_getaddrinfo,
|
||||||
|
_url_endpoint,
|
||||||
|
download_proxies,
|
||||||
install_socket_guard,
|
install_socket_guard,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -106,15 +109,32 @@ class AddressResolutionTests(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
class ConnectAddressPolicyTests(unittest.TestCase):
|
class ConnectAddressPolicyTests(unittest.TestCase):
|
||||||
"""Connect-time policy: allow global + loopback, block everything else."""
|
"""Connect-time policy: allow global, plus anything at a destination the
|
||||||
|
caller has established is the operator's configured proxy."""
|
||||||
|
|
||||||
def test_global_allowed(self):
|
def test_global_allowed(self):
|
||||||
self.assertTrue(_address_allowed_at_connect("142.250.1.1"))
|
self.assertTrue(_address_allowed_at_connect("142.250.1.1"))
|
||||||
|
|
||||||
def test_loopback_allowed(self):
|
def test_loopback_blocked_by_default(self):
|
||||||
# Loopback stays reachable so locally-configured proxies keep working.
|
# A blanket loopback allowance is what let manifest-derived media URLs
|
||||||
self.assertTrue(_address_allowed_at_connect("127.0.0.1"))
|
# reach services on the server's own loopback interface.
|
||||||
self.assertTrue(_address_allowed_at_connect("::1"))
|
self.assertFalse(_address_allowed_at_connect("127.0.0.1"))
|
||||||
|
self.assertFalse(_address_allowed_at_connect("::1"))
|
||||||
|
|
||||||
|
def test_loopback_allowed_only_when_opted_in(self):
|
||||||
|
self.assertTrue(_address_allowed_at_connect("127.0.0.1", is_allowed_endpoint=True))
|
||||||
|
self.assertTrue(_address_allowed_at_connect("::1", is_allowed_endpoint=True))
|
||||||
|
|
||||||
|
def test_proxy_opt_in_covers_any_internal_range(self):
|
||||||
|
# A proxy is just as legitimately on the LAN or a VPN range as on
|
||||||
|
# loopback (#1055): the allowance follows the operator's configured
|
||||||
|
# endpoint, not a particular address family.
|
||||||
|
self.assertTrue(_address_allowed_at_connect("10.1.20.30", is_allowed_endpoint=True))
|
||||||
|
self.assertTrue(_address_allowed_at_connect("192.168.1.10", is_allowed_endpoint=True))
|
||||||
|
self.assertTrue(_address_allowed_at_connect("fd00::1", is_allowed_endpoint=True))
|
||||||
|
|
||||||
|
def test_opt_in_still_rejects_non_addresses(self):
|
||||||
|
self.assertFalse(_address_allowed_at_connect("not-an-ip", is_allowed_endpoint=True))
|
||||||
|
|
||||||
def test_link_local_metadata_blocked(self):
|
def test_link_local_metadata_blocked(self):
|
||||||
self.assertFalse(_address_allowed_at_connect("169.254.169.254"))
|
self.assertFalse(_address_allowed_at_connect("169.254.169.254"))
|
||||||
@@ -127,7 +147,78 @@ class ConnectAddressPolicyTests(unittest.TestCase):
|
|||||||
self.assertFalse(_address_allowed_at_connect("::ffff:169.254.169.254"))
|
self.assertFalse(_address_allowed_at_connect("::ffff:169.254.169.254"))
|
||||||
|
|
||||||
|
|
||||||
|
class TunnelledIPv4Tests(unittest.TestCase):
|
||||||
|
"""IPv6 transition forms that carry an IPv4 address the outer address hides.
|
||||||
|
|
||||||
|
``is_global`` looks only at the outer address, so a form that tunnels an
|
||||||
|
internal IPv4 has to be unwrapped before it is judged (GHSA-5mq5-qr7m-f4wx).
|
||||||
|
"""
|
||||||
|
|
||||||
|
def test_nat64_well_known_prefix_blocked(self):
|
||||||
|
# 2000::/3 global unicast on its face; carries the metadata address.
|
||||||
|
self.assertFalse(_address_is_global("64:ff9b::a9fe:a9fe"))
|
||||||
|
self.assertFalse(_address_is_global("64:ff9b::7f00:1"))
|
||||||
|
self.assertFalse(_address_allowed_at_connect("64:ff9b::a9fe:a9fe"))
|
||||||
|
|
||||||
|
def test_nat64_carrying_a_public_address_allowed(self):
|
||||||
|
self.assertTrue(_address_is_global("64:ff9b::8.8.8.8"))
|
||||||
|
|
||||||
|
def test_ipv4_compatible_form_blocked(self):
|
||||||
|
# The deprecated ::/96 form, likewise global-looking to is_global.
|
||||||
|
self.assertFalse(_address_is_global("::a9fe:a9fe"))
|
||||||
|
self.assertFalse(_address_allowed_at_connect("::a9fe:a9fe"))
|
||||||
|
|
||||||
|
def test_sixtofour_and_teredo_stay_blocked(self):
|
||||||
|
# Python rejects these ranges wholesale. Unwrapping must not promote a
|
||||||
|
# blocked address to an allowed one just because the payload is global.
|
||||||
|
self.assertFalse(_address_is_global("2002:a9fe:a9fe::"))
|
||||||
|
self.assertFalse(_address_is_global("2002:0808:0808::"))
|
||||||
|
self.assertFalse(_address_is_global("2001:0:4136:e378:8000:63bf:3fff:fdd2"))
|
||||||
|
|
||||||
|
def test_plain_addresses_unaffected(self):
|
||||||
|
self.assertTrue(_address_is_global("142.250.1.1"))
|
||||||
|
self.assertTrue(_address_is_global("2607:f8b0:4004:c07::64"))
|
||||||
|
self.assertFalse(_address_is_global("not-an-ip"))
|
||||||
|
|
||||||
|
def test_tunnelled_form_blocked_at_ingress(self):
|
||||||
|
with mock.patch(
|
||||||
|
"url_guard.socket.getaddrinfo",
|
||||||
|
return_value=_addrinfo("64:ff9b::a9fe:a9fe", family=socket.AF_INET6),
|
||||||
|
):
|
||||||
|
self.assertIsNotNone(validate_url("http://nat64.example/x"))
|
||||||
|
|
||||||
|
|
||||||
|
class EndpointParsingTests(unittest.TestCase):
|
||||||
|
def test_explicit_port(self):
|
||||||
|
self.assertEqual(_url_endpoint("http://127.0.0.1:9050"), ("127.0.0.1", 9050))
|
||||||
|
|
||||||
|
def test_default_port_per_scheme(self):
|
||||||
|
self.assertEqual(_url_endpoint("socks5://127.0.0.1"), ("127.0.0.1", 1080))
|
||||||
|
self.assertEqual(_url_endpoint("http://127.0.0.1"), ("127.0.0.1", 80))
|
||||||
|
|
||||||
|
def test_bare_host_port(self):
|
||||||
|
self.assertEqual(_url_endpoint("127.0.0.1:8080"), ("127.0.0.1", 8080))
|
||||||
|
|
||||||
|
def test_hostname_lowercased(self):
|
||||||
|
self.assertEqual(_url_endpoint("http://LocalHost.:9050"), ("localhost", 9050))
|
||||||
|
|
||||||
|
def test_ipv6_literal(self):
|
||||||
|
self.assertEqual(_url_endpoint("http://[::1]:9050"), ("::1", 9050))
|
||||||
|
|
||||||
|
def test_empty_and_invalid(self):
|
||||||
|
self.assertIsNone(_url_endpoint(""))
|
||||||
|
self.assertIsNone(_url_endpoint(" "))
|
||||||
|
self.assertIsNone(_url_endpoint(None))
|
||||||
|
self.assertIsNone(_url_endpoint("http://"))
|
||||||
|
|
||||||
|
|
||||||
class GuardedGetaddrinfoTests(unittest.TestCase):
|
class GuardedGetaddrinfoTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
# Default state: no proxy configured, so no loopback destination allowed.
|
||||||
|
saved = set(url_guard._allowed_endpoints)
|
||||||
|
url_guard._allowed_endpoints = set()
|
||||||
|
self.addCleanup(lambda: setattr(url_guard, "_allowed_endpoints", saved))
|
||||||
|
|
||||||
def test_internal_only_raises(self):
|
def test_internal_only_raises(self):
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("169.254.169.254")):
|
||||||
with self.assertRaises(socket.gaierror):
|
with self.assertRaises(socket.gaierror):
|
||||||
@@ -139,11 +230,180 @@ class GuardedGetaddrinfoTests(unittest.TestCase):
|
|||||||
results = _guarded_getaddrinfo("mixed", 80)
|
results = _guarded_getaddrinfo("mixed", 80)
|
||||||
self.assertEqual([r[4][0] for r in results], ["142.250.1.1"])
|
self.assertEqual([r[4][0] for r in results], ["142.250.1.1"])
|
||||||
|
|
||||||
def test_loopback_passes(self):
|
def test_loopback_blocked_without_matching_proxy(self):
|
||||||
|
# The advisory case: an m3u8 segment URL pointing at a loopback service.
|
||||||
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
results = _guarded_getaddrinfo("localproxy", 9050)
|
with self.assertRaises(socket.gaierror):
|
||||||
|
_guarded_getaddrinfo("127.0.0.1", 9999)
|
||||||
|
|
||||||
|
def test_loopback_allowed_at_configured_url_endpoint(self):
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
results = _guarded_getaddrinfo("127.0.0.1", 9050)
|
||||||
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
|
def test_loopback_blocked_at_other_port_on_proxy_host(self):
|
||||||
|
# Same host as the proxy, different port: still off limits.
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
with self.assertRaises(socket.gaierror):
|
||||||
|
_guarded_getaddrinfo("127.0.0.1", 9999)
|
||||||
|
|
||||||
|
def test_proxy_reachable_by_hostname(self):
|
||||||
|
url_guard._allowed_endpoints = {("localhost", 9050)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
results = _guarded_getaddrinfo("localhost", 9050)
|
||||||
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
|
def test_string_port_is_normalised(self):
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 9050)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
results = _guarded_getaddrinfo("127.0.0.1", "9050")
|
||||||
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
|
def test_lan_proxy_reachable(self):
|
||||||
|
# #1055: a socks5 proxy on the LAN, refused while the allowance was
|
||||||
|
# loopback-only, which pushed operators to ALLOW_PRIVATE_ADDRESSES.
|
||||||
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
||||||
|
results = _guarded_getaddrinfo("10.1.20.30", 1080)
|
||||||
|
self.assertEqual([r[4][0] for r in results], ["10.1.20.30"])
|
||||||
|
|
||||||
|
def test_other_lan_host_still_blocked(self):
|
||||||
|
# The allowance is the proxy's endpoint, not its subnet.
|
||||||
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.31")):
|
||||||
|
with self.assertRaises(socket.gaierror):
|
||||||
|
_guarded_getaddrinfo("10.1.20.31", 1080)
|
||||||
|
|
||||||
|
def test_pot_provider_reachable_on_loopback(self):
|
||||||
|
# #1064: the bundled PO token provider listens on loopback, and blocking
|
||||||
|
# it left every default install downloading YouTube without a token.
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 4416)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
results = _guarded_getaddrinfo("127.0.0.1", 4416)
|
||||||
|
self.assertEqual([r[4][0] for r in results], ["127.0.0.1"])
|
||||||
|
|
||||||
|
def test_other_loopback_service_still_blocked(self):
|
||||||
|
# MeTube's own port is one hop away on the same interface: allowing the
|
||||||
|
# token provider must not allow the rest of loopback.
|
||||||
|
url_guard._allowed_endpoints = {("127.0.0.1", 4416)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("127.0.0.1")):
|
||||||
|
with self.assertRaises(socket.gaierror):
|
||||||
|
_guarded_getaddrinfo("127.0.0.1", 8081)
|
||||||
|
|
||||||
|
def test_proxy_address_not_borrowable_by_another_host(self):
|
||||||
|
# Matching is on the configured host string: a manifest URL that resolves
|
||||||
|
# to the proxy's address under its own name gets no allowance.
|
||||||
|
url_guard._allowed_endpoints = {("10.1.20.30", 1080)}
|
||||||
|
with mock.patch("url_guard._real_getaddrinfo", return_value=_addrinfo("10.1.20.30")):
|
||||||
|
with self.assertRaises(socket.gaierror):
|
||||||
|
_guarded_getaddrinfo("evil.example", 1080)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadProxiesTests(unittest.TestCase):
|
||||||
|
"""The proxy map is assembled the way YoutubeDL.proxies assembles it."""
|
||||||
|
|
||||||
|
def test_explicit_proxy_option_replaces_environment(self):
|
||||||
|
with mock.patch("url_guard.urllib.request.getproxies",
|
||||||
|
return_value={"http": "http://env:3128", "no": "example.com"}):
|
||||||
|
self.assertEqual(download_proxies({"proxy": "socks5h://tor:9050"}),
|
||||||
|
{"all": "socks5h://tor:9050"})
|
||||||
|
|
||||||
|
def test_empty_proxy_option_means_no_proxy(self):
|
||||||
|
self.assertEqual(download_proxies({"proxy": ""}), {"all": "__noproxy__"})
|
||||||
|
|
||||||
|
def test_environment_used_when_no_proxy_option(self):
|
||||||
|
with mock.patch("url_guard.urllib.request.getproxies",
|
||||||
|
return_value={"http": "http://env:3128"}):
|
||||||
|
# http_proxy alone also covers https, as in yt-dlp.
|
||||||
|
self.assertEqual(download_proxies({}),
|
||||||
|
{"http": "http://env:3128", "https": "http://env:3128"})
|
||||||
|
|
||||||
|
def test_no_options_at_all(self):
|
||||||
|
with mock.patch("url_guard.urllib.request.getproxies", return_value={}):
|
||||||
|
self.assertEqual(download_proxies(), {})
|
||||||
|
|
||||||
|
|
||||||
|
class ProxiedHostnameTests(unittest.TestCase):
|
||||||
|
"""A proxy that resolves hostnames itself makes a local lookup both wrong
|
||||||
|
and harmful, so the address check is skipped for hostnames behind one."""
|
||||||
|
|
||||||
|
def _validate(self, url, proxies):
|
||||||
|
with mock.patch("url_guard.socket.getaddrinfo") as gai:
|
||||||
|
gai.side_effect = AssertionError("resolved a hostname that the proxy resolves")
|
||||||
|
return validate_url(url, proxies=proxies)
|
||||||
|
|
||||||
|
def test_socks5h_hostname_not_resolved(self):
|
||||||
|
self.assertIsNone(self._validate("https://youtube.com/x", {"all": "socks5h://tor:9050"}))
|
||||||
|
|
||||||
|
def test_plain_socks5_treated_as_remote_dns(self):
|
||||||
|
# yt-dlp rewrites socks5 to socks5h on every request for compatibility.
|
||||||
|
self.assertIsNone(self._validate("https://youtube.com/x", {"all": "socks5://tor:9050"}))
|
||||||
|
|
||||||
|
def test_socks4a_hostname_not_resolved(self):
|
||||||
|
self.assertIsNone(self._validate("https://youtube.com/x", {"all": "socks4a://tor:9050"}))
|
||||||
|
|
||||||
|
def test_http_proxy_hostname_not_resolved(self):
|
||||||
|
self.assertIsNone(self._validate("https://youtube.com/x", {"https": "http://squid:3128"}))
|
||||||
|
|
||||||
|
def test_scheme_less_proxy_treated_as_http(self):
|
||||||
|
self.assertIsNone(self._validate("https://youtube.com/x", {"all": "squid:3128"}))
|
||||||
|
|
||||||
|
def test_per_scheme_entry_selected(self):
|
||||||
|
# Only http is proxied here, so an https URL keeps the check.
|
||||||
|
proxies = {"http": "http://squid:3128"}
|
||||||
|
self.assertIsNone(self._validate("http://youtube.com/x", proxies))
|
||||||
|
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo("10.0.0.5")):
|
||||||
|
self.assertIsNotNone(validate_url("https://youtube.com/x", proxies=proxies))
|
||||||
|
|
||||||
|
def test_socks4_still_resolved_locally(self):
|
||||||
|
# SOCKS4 resolves in this process, so the address check still applies.
|
||||||
|
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo("10.0.0.5")):
|
||||||
|
self.assertIsNotNone(validate_url("https://youtube.com/x",
|
||||||
|
proxies={"all": "socks4://tor:9050"}))
|
||||||
|
|
||||||
|
def test_noproxy_host_still_resolved(self):
|
||||||
|
# A host excluded from the proxy is fetched directly, so it is checked.
|
||||||
|
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo("169.254.169.254")):
|
||||||
|
self.assertIsNotNone(validate_url(
|
||||||
|
"http://metadata.internal/x",
|
||||||
|
proxies={"all": "socks5h://tor:9050", "no": "metadata.internal"}))
|
||||||
|
|
||||||
|
def test_noproxy_marker_keeps_check(self):
|
||||||
|
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo("10.0.0.5")):
|
||||||
|
self.assertIsNotNone(validate_url("https://youtube.com/x",
|
||||||
|
proxies={"all": "__noproxy__"}))
|
||||||
|
|
||||||
|
def test_ip_literal_still_checked_behind_a_proxy(self):
|
||||||
|
# An IP literal needs no name resolution, so nothing leaks by judging it
|
||||||
|
# and the proxy changes nothing: the request names this address either
|
||||||
|
# way. Unmocked on purpose — getaddrinfo on a literal never hits DNS.
|
||||||
|
self.assertIsNotNone(validate_url("http://169.254.169.254/latest/meta-data/",
|
||||||
|
proxies={"all": "socks5h://tor:9050"}))
|
||||||
|
|
||||||
|
def test_ipv6_literal_still_checked_behind_a_proxy(self):
|
||||||
|
self.assertIsNotNone(validate_url("http://[::1]:8080/x",
|
||||||
|
proxies={"all": "socks5h://tor:9050"}))
|
||||||
|
|
||||||
|
def test_blocked_hostname_still_blocked_behind_a_proxy(self):
|
||||||
|
self.assertIsNotNone(validate_url("http://localhost:8080/x",
|
||||||
|
proxies={"all": "socks5h://tor:9050"}))
|
||||||
|
|
||||||
|
def test_scheme_still_enforced_behind_a_proxy(self):
|
||||||
|
self.assertIsNotNone(validate_url("file:///etc/passwd",
|
||||||
|
proxies={"all": "socks5h://tor:9050"}))
|
||||||
|
|
||||||
|
def test_unparseable_proxy_keeps_check(self):
|
||||||
|
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo("10.0.0.5")):
|
||||||
|
self.assertIsNotNone(validate_url("https://youtube.com/x",
|
||||||
|
proxies={"all": "gopher://weird:70"}))
|
||||||
|
|
||||||
|
def test_no_proxies_argument_keeps_check(self):
|
||||||
|
# The default: callers that pass nothing get today's behaviour.
|
||||||
|
with mock.patch("url_guard.socket.getaddrinfo", return_value=_addrinfo("10.0.0.5")):
|
||||||
|
self.assertIsNotNone(validate_url("https://youtube.com/x"))
|
||||||
|
|
||||||
|
|
||||||
class AllowPrivateBypassTests(unittest.TestCase):
|
class AllowPrivateBypassTests(unittest.TestCase):
|
||||||
"""ALLOW_PRIVATE_ADDRESSES: trusted proxy/VPN environments opt out of the
|
"""ALLOW_PRIVATE_ADDRESSES: trusted proxy/VPN environments opt out of the
|
||||||
@@ -172,16 +432,63 @@ class AllowPrivateBypassTests(unittest.TestCase):
|
|||||||
|
|
||||||
|
|
||||||
class InstallSocketGuardTests(unittest.TestCase):
|
class InstallSocketGuardTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
original, saved = socket.getaddrinfo, set(url_guard._allowed_endpoints)
|
||||||
|
self.addCleanup(lambda: setattr(socket, "getaddrinfo", original))
|
||||||
|
self.addCleanup(lambda: setattr(url_guard, "_allowed_endpoints", saved))
|
||||||
|
# Keep the host's own environment out of the assertions below.
|
||||||
|
patcher = mock.patch("url_guard.urllib.request.getproxies", return_value={})
|
||||||
|
self.getproxies = patcher.start()
|
||||||
|
self.addCleanup(patcher.stop)
|
||||||
|
|
||||||
def test_install_replaces_and_is_idempotent(self):
|
def test_install_replaces_and_is_idempotent(self):
|
||||||
original = socket.getaddrinfo
|
install_socket_guard()
|
||||||
try:
|
self.assertIs(socket.getaddrinfo, url_guard._guarded_getaddrinfo)
|
||||||
install_socket_guard()
|
# Re-installing must not wrap the wrapper (real fn captured at import).
|
||||||
self.assertIs(socket.getaddrinfo, url_guard._guarded_getaddrinfo)
|
install_socket_guard()
|
||||||
# Re-installing must not wrap the wrapper (real fn captured at import).
|
self.assertIs(socket.getaddrinfo, url_guard._guarded_getaddrinfo)
|
||||||
install_socket_guard()
|
|
||||||
self.assertIs(socket.getaddrinfo, url_guard._guarded_getaddrinfo)
|
def test_no_proxy_means_no_loopback_allowance(self):
|
||||||
finally:
|
install_socket_guard()
|
||||||
socket.getaddrinfo = original
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
|
def test_explicit_proxy_is_registered(self):
|
||||||
|
install_socket_guard(proxy_urls=("socks5://127.0.0.1:9050",))
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 9050)})
|
||||||
|
|
||||||
|
def test_unset_proxy_option_is_ignored(self):
|
||||||
|
# ytdl_opts.get('proxy') is None when the operator configured no proxy.
|
||||||
|
install_socket_guard(proxy_urls=(None,))
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
|
def test_environment_proxies_are_registered(self):
|
||||||
|
self.getproxies.return_value = {"http": "http://127.0.0.1:8080"}
|
||||||
|
install_socket_guard()
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 8080)})
|
||||||
|
|
||||||
|
def test_service_url_is_registered(self):
|
||||||
|
install_socket_guard(service_urls=("http://127.0.0.1:4416",))
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, {("127.0.0.1", 4416)})
|
||||||
|
|
||||||
|
def test_service_and_proxy_endpoints_coexist(self):
|
||||||
|
install_socket_guard(
|
||||||
|
proxy_urls=("socks5://10.1.20.30:1080",),
|
||||||
|
service_urls=("http://127.0.0.1:4416",),
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
url_guard._allowed_endpoints,
|
||||||
|
{("10.1.20.30", 1080), ("127.0.0.1", 4416)},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_service_urls_reset_between_installs(self):
|
||||||
|
install_socket_guard(service_urls=("http://127.0.0.1:4416",))
|
||||||
|
install_socket_guard()
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
|
def test_endpoints_reset_between_installs(self):
|
||||||
|
install_socket_guard(proxy_urls=("http://127.0.0.1:8080",))
|
||||||
|
install_socket_guard(proxy_urls=(None,))
|
||||||
|
self.assertEqual(url_guard._allowed_endpoints, set())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import os
|
import os
|
||||||
import pickle
|
import pickle
|
||||||
import signal
|
import signal
|
||||||
@@ -11,7 +12,8 @@ import threading
|
|||||||
import types
|
import types
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from unittest.mock import MagicMock, patch
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
fake_yt_dlp = types.ModuleType("yt_dlp")
|
fake_yt_dlp = types.ModuleType("yt_dlp")
|
||||||
fake_networking = types.ModuleType("yt_dlp.networking")
|
fake_networking = types.ModuleType("yt_dlp.networking")
|
||||||
@@ -50,6 +52,7 @@ class _YoutubeDL:
|
|||||||
|
|
||||||
|
|
||||||
fake_utils.DownloadError = type("DownloadError", (Exception,), {})
|
fake_utils.DownloadError = type("DownloadError", (Exception,), {})
|
||||||
|
fake_utils.YoutubeDLError = fake_utils.DownloadError
|
||||||
fake_yt_dlp.YoutubeDL = _YoutubeDL
|
fake_yt_dlp.YoutubeDL = _YoutubeDL
|
||||||
fake_impersonate.ImpersonateTarget = _ImpersonateTarget
|
fake_impersonate.ImpersonateTarget = _ImpersonateTarget
|
||||||
fake_networking.impersonate = fake_impersonate
|
fake_networking.impersonate = fake_impersonate
|
||||||
@@ -76,6 +79,7 @@ from ytdl import (
|
|||||||
MusicMetadataPreProcessor,
|
MusicMetadataPreProcessor,
|
||||||
_compact_persisted_entry,
|
_compact_persisted_entry,
|
||||||
_convert_srt_to_txt_file,
|
_convert_srt_to_txt_file,
|
||||||
|
_pot_provider_urls,
|
||||||
_AlbumArtistPostProcessor,
|
_AlbumArtistPostProcessor,
|
||||||
_resolve_outtmpl_fields,
|
_resolve_outtmpl_fields,
|
||||||
_sanitize_entry_for_pickle,
|
_sanitize_entry_for_pickle,
|
||||||
@@ -376,6 +380,49 @@ class ConfinedYoutubeDLTests(unittest.TestCase):
|
|||||||
self.assertEqual(self._prepared_path(""), "")
|
self.assertEqual(self._prepared_path(""), "")
|
||||||
self.assertEqual(self._prepared_path("-"), "-")
|
self.assertEqual(self._prepared_path("-"), "-")
|
||||||
|
|
||||||
|
def test_overlong_name_is_trimmed_to_fit_the_filesystem(self):
|
||||||
|
# A title long enough to blow the filename limit is what made these
|
||||||
|
# downloads fail outright with [Errno 36] File name too long.
|
||||||
|
long_path = os.path.join(self.base, "a" * 400 + ".mp4")
|
||||||
|
|
||||||
|
result = self._prepared_path(long_path)
|
||||||
|
|
||||||
|
name = os.path.basename(result)
|
||||||
|
self.assertTrue(name.endswith(".mp4"))
|
||||||
|
self.assertLessEqual(len(name.encode("utf-8")), 255 - 32)
|
||||||
|
self.assertEqual(os.path.dirname(result), self.base)
|
||||||
|
# The file must still be writable once yt-dlp adds its own suffixes.
|
||||||
|
self.assertLessEqual(len(f"{name}.f1229065279304024v.part".encode("utf-8")), 255)
|
||||||
|
|
||||||
|
def test_name_within_the_limit_is_left_alone(self):
|
||||||
|
ok = os.path.join(self.base, "Ordinary Title.mp4")
|
||||||
|
self.assertEqual(self._prepared_path(ok), ok)
|
||||||
|
|
||||||
|
def test_limit_counts_bytes_not_characters(self):
|
||||||
|
# 200 CJK characters are 600 bytes: a character count would pass this.
|
||||||
|
long_path = os.path.join(self.base, "音" * 200 + ".mp4")
|
||||||
|
|
||||||
|
name = os.path.basename(self._prepared_path(long_path))
|
||||||
|
|
||||||
|
self.assertLessEqual(len(name.encode("utf-8")), 255 - 32)
|
||||||
|
# A trim landing mid-character must not leave a broken byte sequence.
|
||||||
|
self.assertEqual(name, name.encode("utf-8").decode("utf-8"))
|
||||||
|
self.assertTrue(name.endswith(".mp4"))
|
||||||
|
|
||||||
|
def test_a_long_tail_is_not_mistaken_for_an_extension(self):
|
||||||
|
# os.path.splitext on a title containing a dot late in the string would
|
||||||
|
# otherwise "preserve" a 100-character extension and trim nothing.
|
||||||
|
long_path = os.path.join(self.base, "b" * 300 + "." + "c" * 100)
|
||||||
|
|
||||||
|
name = os.path.basename(self._prepared_path(long_path))
|
||||||
|
|
||||||
|
self.assertLessEqual(len(name.encode("utf-8")), 255 - 32)
|
||||||
|
|
||||||
|
def test_trimming_still_cannot_escape_the_download_directory(self):
|
||||||
|
escaping = os.path.join(self.base, "..", "..", "d" * 400 + ".mp4")
|
||||||
|
with self.assertRaises(ytdl.yt_dlp.utils.DownloadError):
|
||||||
|
self._prepared_path(escaping)
|
||||||
|
|
||||||
|
|
||||||
class SanitizeEntryForPickleTests(unittest.TestCase):
|
class SanitizeEntryForPickleTests(unittest.TestCase):
|
||||||
def test_nested(self):
|
def test_nested(self):
|
||||||
@@ -434,6 +481,269 @@ def _make_test_download() -> Download:
|
|||||||
return Download("/tmp", "/tmp", "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info)
|
return Download("/tmp", "/tmp", "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info)
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadLoggerTests(unittest.TestCase):
|
||||||
|
def test_routes_messages_and_retains_only_non_empty_warnings(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='DEBUG') as logs:
|
||||||
|
logger.debug('debug detail')
|
||||||
|
logger.warning(' useful warning ')
|
||||||
|
logger.warning(' ')
|
||||||
|
logger.error('error detail')
|
||||||
|
|
||||||
|
self.assertEqual(logger.warnings, ['useful warning'])
|
||||||
|
self.assertIn('DEBUG:ytdl:debug detail', logs.output)
|
||||||
|
self.assertIn('WARNING:ytdl: useful warning ', logs.output)
|
||||||
|
self.assertIn('ERROR:ytdl:error detail', logs.output)
|
||||||
|
|
||||||
|
def test_retains_only_the_last_distinct_warnings(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
cap = ytdl._MAX_RETAINED_WARNINGS
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING') as logs:
|
||||||
|
for index in range(cap + 3):
|
||||||
|
logger.warning(f'fragment {index} not found')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
logger.warnings,
|
||||||
|
[f'fragment {index} not found' for index in range(3, cap + 3)],
|
||||||
|
)
|
||||||
|
# Every warning still reaches the log; only the retained list is bounded.
|
||||||
|
self.assertEqual(len(logs.output), cap + 3)
|
||||||
|
|
||||||
|
def test_repeated_warning_is_retained_once(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING'):
|
||||||
|
logger.warning('Requested format is not available')
|
||||||
|
logger.warning('Only images are available for download')
|
||||||
|
logger.warning('Requested format is not available')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
logger.warnings,
|
||||||
|
['Requested format is not available', 'Only images are available for download'],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_failure_message_puts_the_error_last(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING'):
|
||||||
|
logger.warning('Only images are available for download')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
logger.failure_message('ERROR: [youtube] u2HSc2Ym1Vk: No video formats found!'),
|
||||||
|
'Only images are available for download\n'
|
||||||
|
'ERROR: [youtube] u2HSc2Ym1Vk: No video formats found!',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_failure_message_skips_a_last_warning_that_repeats_the_error(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING'):
|
||||||
|
logger.warning('Video unavailable')
|
||||||
|
# yt-dlp labels errors but hands warnings to the logger unlabelled,
|
||||||
|
# so the same text can arrive through both routes.
|
||||||
|
logger.warning('Requested format is not available')
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
logger.failure_message('ERROR: Requested format is not available'),
|
||||||
|
'Video unavailable\nERROR: Requested format is not available',
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_failure_message_without_warnings_is_the_error_alone(self):
|
||||||
|
logger = ytdl._DownloadYtdlLogger()
|
||||||
|
|
||||||
|
self.assertEqual(logger.failure_message('ERROR: boom'), 'ERROR: boom')
|
||||||
|
|
||||||
|
|
||||||
|
class DownloadResultTests(unittest.TestCase):
|
||||||
|
def _run_download(self, result=0, warnings=(), error=None):
|
||||||
|
download = _make_test_download()
|
||||||
|
statuses = []
|
||||||
|
download.status_queue = types.SimpleNamespace(put=statuses.append)
|
||||||
|
captured_params = {}
|
||||||
|
|
||||||
|
class FakeYoutubeDL:
|
||||||
|
def download(self, urls):
|
||||||
|
self.urls = urls
|
||||||
|
for warning in warnings:
|
||||||
|
captured_params['logger'].warning(warning)
|
||||||
|
if error is not None:
|
||||||
|
raise error
|
||||||
|
return result
|
||||||
|
|
||||||
|
def make_youtube_dl(params):
|
||||||
|
captured_params.update(params)
|
||||||
|
return FakeYoutubeDL()
|
||||||
|
|
||||||
|
with patch.object(download, '_make_youtube_dl', side_effect=make_youtube_dl), \
|
||||||
|
patch('ytdl.install_socket_guard'), \
|
||||||
|
patch('ytdl.os.setpgrp'):
|
||||||
|
download._download()
|
||||||
|
|
||||||
|
return statuses, captured_params
|
||||||
|
|
||||||
|
def test_nonzero_result_includes_warning_context_and_forwards_logs(self):
|
||||||
|
warnings = [
|
||||||
|
'The uploader has blocked this video in your country',
|
||||||
|
'No video formats found',
|
||||||
|
]
|
||||||
|
|
||||||
|
with self.assertLogs('ytdl', level='WARNING') as logs:
|
||||||
|
statuses, params = self._run_download(result=1, warnings=warnings)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1],
|
||||||
|
{'status': 'error', 'msg': '\n'.join(warnings)},
|
||||||
|
)
|
||||||
|
self.assertIs(params['logger'].__class__, ytdl._DownloadYtdlLogger)
|
||||||
|
for warning in warnings:
|
||||||
|
self.assertTrue(any(warning in entry for entry in logs.output))
|
||||||
|
|
||||||
|
def test_nonzero_result_without_warning_uses_fallback_message(self):
|
||||||
|
statuses, _ = self._run_download(result=2)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1],
|
||||||
|
{'status': 'error', 'msg': 'yt-dlp failed with exit code 2'},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_warning_does_not_change_success_status(self):
|
||||||
|
statuses, _ = self._run_download(result=0, warnings=['A recoverable warning'])
|
||||||
|
|
||||||
|
self.assertEqual(statuses[-1], {'status': 'finished'})
|
||||||
|
|
||||||
|
def test_youtube_dl_error_carries_the_warnings_that_explain_it(self):
|
||||||
|
# The sequence from issue #1047: yt-dlp raises DownloadError, so the
|
||||||
|
# warnings naming the real cause only reach the user if the exception
|
||||||
|
# branch carries them too.
|
||||||
|
statuses, _ = self._run_download(
|
||||||
|
warnings=[
|
||||||
|
'[youtube] Video unavailable. This video contains content from bryhuangpub,'
|
||||||
|
' who has blocked it from display on this website or application',
|
||||||
|
'Only images are available for download. use --list-formats to see them',
|
||||||
|
'Requested format is not available',
|
||||||
|
],
|
||||||
|
error=ytdl.yt_dlp.utils.YoutubeDLError(
|
||||||
|
'ERROR: [youtube] u2HSc2Ym1Vk: No video formats found!'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1],
|
||||||
|
{
|
||||||
|
'status': 'error',
|
||||||
|
'msg': '[youtube] Video unavailable. This video contains content from bryhuangpub,'
|
||||||
|
' who has blocked it from display on this website or application\n'
|
||||||
|
'Only images are available for download. use --list-formats to see them\n'
|
||||||
|
'Requested format is not available\n'
|
||||||
|
'ERROR: [youtube] u2HSc2Ym1Vk: No video formats found!',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_youtube_dl_error_drops_a_last_warning_that_repeats_it(self):
|
||||||
|
statuses, _ = self._run_download(
|
||||||
|
warnings=['Earlier warning', 'Requested format is not available'],
|
||||||
|
error=ytdl.yt_dlp.utils.YoutubeDLError('ERROR: Requested format is not available'),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1],
|
||||||
|
{
|
||||||
|
'status': 'error',
|
||||||
|
'msg': 'Earlier warning\nERROR: Requested format is not available',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_youtube_dl_error_message_is_bounded(self):
|
||||||
|
cap = ytdl._MAX_RETAINED_WARNINGS
|
||||||
|
statuses, _ = self._run_download(
|
||||||
|
warnings=[f'fragment {index} not found' for index in range(cap + 4)],
|
||||||
|
error=ytdl.yt_dlp.utils.YoutubeDLError('ERROR: giving up'),
|
||||||
|
)
|
||||||
|
|
||||||
|
msg = statuses[-1]['msg']
|
||||||
|
self.assertEqual(
|
||||||
|
msg.split('\n'),
|
||||||
|
[f'fragment {index} not found' for index in range(4, cap + 4)] + ['ERROR: giving up'],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_nonzero_result_message_is_bounded(self):
|
||||||
|
cap = ytdl._MAX_RETAINED_WARNINGS
|
||||||
|
statuses, _ = self._run_download(
|
||||||
|
result=1,
|
||||||
|
warnings=[f'fragment {index} not found' for index in range(cap + 4)],
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
statuses[-1]['msg'].split('\n'),
|
||||||
|
[f'fragment {index} not found' for index in range(4, cap + 4)],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _capture_ytdl_params(download: Download) -> dict:
|
||||||
|
"""Run ``_download`` far enough to capture the params it builds."""
|
||||||
|
fake_ydl = MagicMock()
|
||||||
|
fake_ydl.download.return_value = 0
|
||||||
|
download.status_queue = types.SimpleNamespace(put=lambda _: None)
|
||||||
|
|
||||||
|
with patch('ytdl.install_socket_guard'), \
|
||||||
|
patch.object(Download, '_make_youtube_dl', return_value=fake_ydl) as make:
|
||||||
|
download._download()
|
||||||
|
|
||||||
|
params, = make.call_args.args
|
||||||
|
return params
|
||||||
|
|
||||||
|
|
||||||
|
class SponsorBlockPostprocessorTests(unittest.TestCase):
|
||||||
|
def test_no_sponsorblock_postprocessors_when_disabled(self):
|
||||||
|
download = _make_test_download()
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
keys = [pp['key'] for pp in params.get('postprocessors', [])]
|
||||||
|
self.assertNotIn('SponsorBlock', keys)
|
||||||
|
self.assertNotIn('ModifyChapters', keys)
|
||||||
|
|
||||||
|
def test_sponsorblock_pair_matches_the_cli(self):
|
||||||
|
download = _make_test_download()
|
||||||
|
download.info.sponsorblock = True
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
params['postprocessors'],
|
||||||
|
[
|
||||||
|
{
|
||||||
|
'key': 'SponsorBlock',
|
||||||
|
'categories': ['sponsor'],
|
||||||
|
'when': 'after_filter',
|
||||||
|
},
|
||||||
|
{
|
||||||
|
'key': 'ModifyChapters',
|
||||||
|
'remove_sponsor_segments': ['sponsor'],
|
||||||
|
'force_keyframes': False,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_segment_removal_runs_before_the_chapter_split(self):
|
||||||
|
# yt-dlp runs same-stage postprocessors in list order, so ModifyChapters
|
||||||
|
# has to rewrite the chapter list before FFmpegSplitChapters cuts the
|
||||||
|
# file up -- the order the CLI builds for
|
||||||
|
# --sponsorblock-remove sponsor --split-chapters.
|
||||||
|
download = _make_test_download()
|
||||||
|
download.info.sponsorblock = True
|
||||||
|
download.info.split_by_chapters = True
|
||||||
|
download.info.chapter_template = '%(section_number)s.%(ext)s'
|
||||||
|
|
||||||
|
params = _capture_ytdl_params(download)
|
||||||
|
|
||||||
|
keys = [pp['key'] for pp in params['postprocessors']]
|
||||||
|
self.assertEqual(keys, ['SponsorBlock', 'ModifyChapters', 'FFmpegSplitChapters'])
|
||||||
|
self.assertEqual(params['outtmpl']['chapter'], '%(section_number)s.%(ext)s')
|
||||||
|
|
||||||
|
|
||||||
class ProgressThrottleTests(unittest.TestCase):
|
class ProgressThrottleTests(unittest.TestCase):
|
||||||
def test_downloading_ticks_are_throttled(self):
|
def test_downloading_ticks_are_throttled(self):
|
||||||
dl = _make_test_download()
|
dl = _make_test_download()
|
||||||
@@ -467,6 +777,80 @@ class ProgressThrottleTests(unittest.TestCase):
|
|||||||
self.assertIn("error", statuses)
|
self.assertIn("error", statuses)
|
||||||
|
|
||||||
|
|
||||||
|
class PostprocessorHookTests(unittest.TestCase):
|
||||||
|
"""The postprocessing phase must be visible without ever displacing the
|
||||||
|
terminal 'finished' status that _post_download_cleanup keys off."""
|
||||||
|
|
||||||
|
def _hook(self):
|
||||||
|
dl = _make_test_download()
|
||||||
|
forwarded = []
|
||||||
|
dl.status_queue = types.SimpleNamespace(put=forwarded.append)
|
||||||
|
return dl._make_postprocessor_hook(), forwarded
|
||||||
|
|
||||||
|
def test_postprocessor_start_reports_the_phase(self):
|
||||||
|
hook, forwarded = self._hook()
|
||||||
|
hook({"postprocessor": "VideoConvertor", "status": "started", "info_dict": {}})
|
||||||
|
self.assertEqual(forwarded, [{"status": "postprocessing"}])
|
||||||
|
|
||||||
|
def test_postprocessor_finish_is_not_reported(self):
|
||||||
|
# Only 'started' moves the UI; a PP finishing says nothing about what
|
||||||
|
# comes next, and an extra broadcast per PP buys nothing.
|
||||||
|
hook, forwarded = self._hook()
|
||||||
|
hook({"postprocessor": "VideoConvertor", "status": "finished", "info_dict": {}})
|
||||||
|
self.assertEqual(forwarded, [])
|
||||||
|
|
||||||
|
def test_move_files_still_reports_finished_with_the_final_path(self):
|
||||||
|
hook, forwarded = self._hook()
|
||||||
|
hook({
|
||||||
|
"postprocessor": "MoveFiles",
|
||||||
|
"status": "finished",
|
||||||
|
"info_dict": {"filepath": "/tmp/video.mp4"},
|
||||||
|
})
|
||||||
|
self.assertEqual(forwarded, [{"status": "finished", "filename": "/tmp/video.mp4"}])
|
||||||
|
|
||||||
|
def test_nothing_is_reported_after_move_files_finished(self):
|
||||||
|
# Regression guard: _post_download_cleanup turns any final status other
|
||||||
|
# than 'finished' into an error, so an 'after_move' postprocessor
|
||||||
|
# starting up must not overwrite it and fail the download.
|
||||||
|
hook, forwarded = self._hook()
|
||||||
|
hook({
|
||||||
|
"postprocessor": "MoveFiles",
|
||||||
|
"status": "finished",
|
||||||
|
"info_dict": {"filepath": "/tmp/video.mp4"},
|
||||||
|
})
|
||||||
|
hook({"postprocessor": "SomeAfterMovePP", "status": "started", "info_dict": {}})
|
||||||
|
self.assertEqual([item["status"] for item in forwarded], ["finished"])
|
||||||
|
|
||||||
|
def test_realistic_sequence_ends_finished(self):
|
||||||
|
hook, forwarded = self._hook()
|
||||||
|
for pp in ("Merger", "VideoConvertor", "Metadata"):
|
||||||
|
hook({"postprocessor": pp, "status": "started", "info_dict": {}})
|
||||||
|
hook({"postprocessor": pp, "status": "finished", "info_dict": {}})
|
||||||
|
hook({"postprocessor": "MoveFiles", "status": "started", "info_dict": {}})
|
||||||
|
hook({
|
||||||
|
"postprocessor": "MoveFiles",
|
||||||
|
"status": "finished",
|
||||||
|
"info_dict": {"filepath": "/tmp/video.mp4"},
|
||||||
|
})
|
||||||
|
|
||||||
|
statuses = [item["status"] for item in forwarded]
|
||||||
|
self.assertEqual(statuses[-1], "finished")
|
||||||
|
self.assertEqual(statuses.count("finished"), 1)
|
||||||
|
self.assertTrue(all(st == "postprocessing" for st in statuses[:-1]))
|
||||||
|
|
||||||
|
def test_split_chapters_still_captures_files_without_a_status(self):
|
||||||
|
hook, forwarded = self._hook()
|
||||||
|
hook({
|
||||||
|
"postprocessor": "SplitChapters",
|
||||||
|
"status": "finished",
|
||||||
|
"info_dict": {"chapters": [{"filepath": "/tmp/ch1.mp4"}, {"filepath": "/tmp/ch2.mp4"}]},
|
||||||
|
})
|
||||||
|
self.assertEqual(
|
||||||
|
forwarded,
|
||||||
|
[{"chapter_file": "/tmp/ch1.mp4"}, {"chapter_file": "/tmp/ch2.mp4"}],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class CancelProcessGroupTests(unittest.TestCase):
|
class CancelProcessGroupTests(unittest.TestCase):
|
||||||
# cancel() now sends SIGINT first (so yt-dlp/ffmpeg can finalize the
|
# cancel() now sends SIGINT first (so yt-dlp/ffmpeg can finalize the
|
||||||
# partial file) and schedules a SIGKILL escalation via the event loop
|
# partial file) and schedules a SIGKILL escalation via the event loop
|
||||||
@@ -827,5 +1211,142 @@ class ShortTitleForFailedUrlTests(unittest.TestCase):
|
|||||||
self.assertEqual(_short_title_for_failed_url(malformed), malformed)
|
self.assertEqual(_short_title_for_failed_url(malformed), malformed)
|
||||||
|
|
||||||
|
|
||||||
|
class PotProviderUrlsTests(unittest.TestCase):
|
||||||
|
"""#1064: the connect-time guard must let the download reach the PO token
|
||||||
|
provider, so it has to know every endpoint yt-dlp might dial for one."""
|
||||||
|
|
||||||
|
def test_bundled_provider_by_default(self):
|
||||||
|
self.assertEqual(_pot_provider_urls({}), ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
def test_configured_base_url_is_added(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtubepot-bgutilhttp": {"base_url": ["http://pot:4416"]}},
|
||||||
|
})
|
||||||
|
# The bundled server runs regardless, so both stay reachable.
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416", "http://pot:4416"))
|
||||||
|
|
||||||
|
def test_deprecated_base_url_arg_is_honoured(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtube": {"getpot_bgutil_baseurl": ["http://pot:4416"]}},
|
||||||
|
})
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416", "http://pot:4416"))
|
||||||
|
|
||||||
|
def test_unrelated_extractor_args_are_ignored(self):
|
||||||
|
urls = _pot_provider_urls({
|
||||||
|
"extractor_args": {"youtube": {"player_client": ["web"]}},
|
||||||
|
})
|
||||||
|
self.assertEqual(urls, ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
def test_malformed_extractor_args_do_not_raise(self):
|
||||||
|
# YTDL_OPTIONS is operator-supplied JSON and reaches here unvalidated.
|
||||||
|
for opts in (
|
||||||
|
{"extractor_args": None},
|
||||||
|
{"extractor_args": "youtube:player_client=web"},
|
||||||
|
{"extractor_args": {"youtubepot-bgutilhttp": "http://pot:4416"}},
|
||||||
|
{"extractor_args": {"youtubepot-bgutilhttp": {"base_url": []}}},
|
||||||
|
):
|
||||||
|
with self.subTest(opts=opts):
|
||||||
|
self.assertEqual(_pot_provider_urls(opts), ("http://127.0.0.1:4416",))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|
||||||
|
|
||||||
|
class UpdateStatusFileStatTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
"""The progress path must not touch the filesystem on the event loop.
|
||||||
|
|
||||||
|
yt-dlp reports 'filename' on every progress tick, but until the download
|
||||||
|
finishes that path does not exist yet -- the bytes are in 'tmpfilename'.
|
||||||
|
Stating it per tick meant blocking syscalls on the event loop twice a
|
||||||
|
second per download, always answering None. See issue #980.
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def _run_update_status(self, statuses):
|
||||||
|
import queue as _queue
|
||||||
|
|
||||||
|
download = _make_test_download()
|
||||||
|
download.download_dir = "/tmp"
|
||||||
|
source = _queue.Queue()
|
||||||
|
for status in statuses:
|
||||||
|
source.put(status)
|
||||||
|
source.put(None)
|
||||||
|
download.status_queue = source
|
||||||
|
download.loop = asyncio.get_running_loop()
|
||||||
|
download._executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
notifier = MagicMock()
|
||||||
|
# updated() is handed the same DownloadInfo every time, so the status has
|
||||||
|
# to be copied out when the call happens -- reading it off the recorded
|
||||||
|
# call args afterwards only ever shows the final value.
|
||||||
|
notifier.broadcast_statuses = []
|
||||||
|
notifier.updated = AsyncMock(
|
||||||
|
side_effect=lambda info: notifier.broadcast_statuses.append(info.status)
|
||||||
|
)
|
||||||
|
download.notifier = notifier
|
||||||
|
|
||||||
|
stat_calls = []
|
||||||
|
|
||||||
|
def record_exists(path):
|
||||||
|
stat_calls.append(path)
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
with patch("ytdl.os.path.exists", side_effect=record_exists):
|
||||||
|
await download.update_status()
|
||||||
|
finally:
|
||||||
|
download._executor.shutdown(wait=True)
|
||||||
|
return download, stat_calls
|
||||||
|
|
||||||
|
async def test_downloading_ticks_do_not_stat_the_output_file(self):
|
||||||
|
ticks = [
|
||||||
|
{"status": "downloading", "filename": "/tmp/v.mp4",
|
||||||
|
"tmpfilename": "/tmp/v.mp4.part", "downloaded_bytes": i}
|
||||||
|
for i in range(1, 6)
|
||||||
|
]
|
||||||
|
download, stat_calls = await self._run_update_status(ticks)
|
||||||
|
|
||||||
|
self.assertEqual(stat_calls, [])
|
||||||
|
self.assertEqual(download.info.filename, "v.mp4")
|
||||||
|
|
||||||
|
async def test_finished_status_still_stats_the_output_file(self):
|
||||||
|
download, stat_calls = await self._run_update_status([
|
||||||
|
{"status": "downloading", "filename": "/tmp/v.mp4", "downloaded_bytes": 1},
|
||||||
|
{"status": "finished", "filename": "/tmp/v.mp4"},
|
||||||
|
])
|
||||||
|
|
||||||
|
self.assertEqual(stat_calls, ["/tmp/v.mp4"])
|
||||||
|
self.assertIsNone(download.info.size)
|
||||||
|
|
||||||
|
async def test_repeated_postprocessing_status_is_broadcast_once(self):
|
||||||
|
# yt-dlp's metaclass wraps run() once per class in a postprocessor's MRO,
|
||||||
|
# so one whose subclass overrides run (FFmpegCopyStream, among others)
|
||||||
|
# reports 'started' twice. Observed live on a 13s libx264 re-encode.
|
||||||
|
download, _ = await self._run_update_status([
|
||||||
|
{"status": "downloading", "downloaded_bytes": 1},
|
||||||
|
{"status": "postprocessing"},
|
||||||
|
{"status": "postprocessing"},
|
||||||
|
{"status": "postprocessing"},
|
||||||
|
{"status": "finished", "filename": "/tmp/v.mp4"},
|
||||||
|
])
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
download.notifier.broadcast_statuses,
|
||||||
|
["downloading", "postprocessing", "finished"],
|
||||||
|
)
|
||||||
|
|
||||||
|
async def test_postprocessing_is_announced_again_after_the_download_resumes(self):
|
||||||
|
# An audio download runs pre_process postprocessors before the bytes
|
||||||
|
# arrive, so 'postprocessing' legitimately appears on both sides of the
|
||||||
|
# download. Deduping against the live status keeps the second one.
|
||||||
|
download, _ = await self._run_update_status([
|
||||||
|
{"status": "postprocessing"},
|
||||||
|
{"status": "downloading", "downloaded_bytes": 1},
|
||||||
|
{"status": "postprocessing"},
|
||||||
|
{"status": "finished", "filename": "/tmp/v.mp4"},
|
||||||
|
])
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
download.notifier.broadcast_statuses,
|
||||||
|
["postprocessing", "downloading", "postprocessing", "finished"],
|
||||||
|
)
|
||||||
|
|||||||
+265
-17
@@ -25,17 +25,43 @@ all of these:
|
|||||||
impact, blind SSRF, since the extraction response is not written to disk).
|
impact, blind SSRF, since the extraction response is not written to disk).
|
||||||
* Native resolvers (curl_cffi/libcurl via ``--impersonate``) resolve outside
|
* Native resolvers (curl_cffi/libcurl via ``--impersonate``) resolve outside
|
||||||
Python's socket module and bypass the connect-time guard entirely.
|
Python's socket module and bypass the connect-time guard entirely.
|
||||||
|
* When a proxy carries the fetch and resolves hostnames itself (an HTTP proxy,
|
||||||
|
``socks5h``, ``socks4a``, or the plain ``socks5`` yt-dlp rewrites to
|
||||||
|
``socks5h``), ``validate_url`` cannot check where a *hostname* leads: the
|
||||||
|
proxy resolves it on its own network, and looking it up here would both
|
||||||
|
describe the wrong network and leak the hostname to the local resolver. The
|
||||||
|
address check is skipped for those, and what the proxy's network exposes is
|
||||||
|
the proxy's to police. Hosts written as IP literals are still checked, and
|
||||||
|
the connect-time guard still covers everything dialled directly.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import ipaddress
|
import ipaddress
|
||||||
import logging
|
import logging
|
||||||
import socket
|
import socket
|
||||||
|
import urllib.request
|
||||||
from urllib.parse import urlsplit
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
log = logging.getLogger('url_guard')
|
log = logging.getLogger('url_guard')
|
||||||
|
|
||||||
_ALLOWED_SCHEMES = ('http', 'https')
|
_ALLOWED_SCHEMES = ('http', 'https')
|
||||||
|
|
||||||
|
# Ports to assume when a configured endpoint URL omits one, per scheme.
|
||||||
|
_SCHEME_DEFAULT_PORTS = {
|
||||||
|
'http': 80,
|
||||||
|
'https': 443,
|
||||||
|
'socks4': 1080,
|
||||||
|
'socks4a': 1080,
|
||||||
|
'socks5': 1080,
|
||||||
|
'socks5h': 1080,
|
||||||
|
}
|
||||||
|
|
||||||
|
# Proxy schemes that hand the destination hostname to the proxy instead of
|
||||||
|
# resolving it here. yt-dlp rewrites a scheme-less proxy to ``http`` and plain
|
||||||
|
# ``socks5`` to ``socks5h`` on every request (``clean_proxies``), so only SOCKS4
|
||||||
|
# — and the non-standard ``socks`` alias yt-dlp maps onto it — still resolves
|
||||||
|
# locally and is therefore absent from this list.
|
||||||
|
_REMOTE_DNS_PROXY_SCHEMES = ('http', 'https', 'socks5', 'socks5h', 'socks4a')
|
||||||
|
|
||||||
# Hostnames that must be blocked without needing a lookup. ``localhost`` and any
|
# Hostnames that must be blocked without needing a lookup. ``localhost`` and any
|
||||||
# subdomain of it are conventionally loopback, and the GCP metadata name is a
|
# subdomain of it are conventionally loopback, and the GCP metadata name is a
|
||||||
# well-known SSRF target that may resolve via a resolver we don't control.
|
# well-known SSRF target that may resolve via a resolver we don't control.
|
||||||
@@ -50,6 +76,19 @@ def _hostname_is_blocked(hostname: str) -> bool:
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
# IPv6 ranges that tunnel an IPv4 address at a fixed offset. ``is_global``
|
||||||
|
# judges only the outer address, so an internal IPv4 wrapped in one of these can
|
||||||
|
# pass a check the bare address would fail — 64:ff9b::a9fe:a9fe carries the cloud
|
||||||
|
# metadata address but sits in the 2000::/3 global unicast range.
|
||||||
|
_NAT64_WELL_KNOWN_PREFIX = ipaddress.ip_network('64:ff9b::/96')
|
||||||
|
_IPV4_COMPATIBLE = ipaddress.ip_network('::/96')
|
||||||
|
|
||||||
|
# ``::`` and ``::1`` sit inside ::/96 without being IPv4-compatible addresses
|
||||||
|
# (RFC 4291 reserves both), and 0.0.0.0/8 is not a routable destination anyway.
|
||||||
|
# Reading a tunnelled address out of them would just misdescribe them.
|
||||||
|
_UNUSABLE_IPV4 = ipaddress.ip_network('0.0.0.0/8')
|
||||||
|
|
||||||
|
|
||||||
def _normalise_ip(addr: str):
|
def _normalise_ip(addr: str):
|
||||||
"""Parse *addr*, unwrapping IPv4-mapped IPv6 (e.g. ``::ffff:169.254.169.254``)
|
"""Parse *addr*, unwrapping IPv4-mapped IPv6 (e.g. ``::ffff:169.254.169.254``)
|
||||||
so the embedded IPv4 address is judged on its own merits. Returns ``None``
|
so the embedded IPv4 address is judged on its own merits. Returns ``None``
|
||||||
@@ -63,45 +102,228 @@ def _normalise_ip(addr: str):
|
|||||||
return ip
|
return ip
|
||||||
|
|
||||||
|
|
||||||
def _address_is_global(addr: str) -> bool:
|
def _tunnelled_ipv4(ip):
|
||||||
ip = _normalise_ip(addr)
|
"""The IPv4 address an IPv6 transition form tunnels, or ``None``.
|
||||||
return ip is not None and ip.is_global
|
|
||||||
|
Covers 6to4 (``2002::/16``), Teredo (``2001::/32``), the NAT64 well-known
|
||||||
|
prefix (``64:ff9b::/96``) and the deprecated IPv4-compatible form
|
||||||
|
(``::/96``). IPv4-mapped is handled by ``_normalise_ip`` instead: that form
|
||||||
|
*is* its embedded address rather than a tunnel to it.
|
||||||
|
"""
|
||||||
|
if not isinstance(ip, ipaddress.IPv6Address):
|
||||||
|
return None
|
||||||
|
if ip.sixtofour is not None:
|
||||||
|
return ip.sixtofour
|
||||||
|
if ip.teredo is not None:
|
||||||
|
return ip.teredo[1]
|
||||||
|
if ip in _NAT64_WELL_KNOWN_PREFIX or ip in _IPV4_COMPATIBLE:
|
||||||
|
tunnelled = ipaddress.ip_address(int(ip) & 0xFFFFFFFF)
|
||||||
|
return None if tunnelled in _UNUSABLE_IPV4 else tunnelled
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _address_allowed_at_connect(addr: str) -> bool:
|
def _ips_to_judge(addr: str) -> tuple:
|
||||||
"""True if *addr* may be connected to at download time.
|
"""Every address a verdict on *addr* has to account for: the address itself
|
||||||
|
plus any IPv4 it tunnels. Empty when *addr* is not a valid IP literal.
|
||||||
|
|
||||||
Permits global addresses and loopback — loopback so that locally-configured
|
A tunnelled address is judged on *both* halves, so unwrapping can only ever
|
||||||
proxies (e.g. ``proxy: http://127.0.0.1:9050``) keep working. Blocks the SSRF
|
tighten the verdict. Returning the embedded address alone would be a way in:
|
||||||
targets that matter: link-local (cloud metadata at 169.254.169.254), private
|
Python already rejects all of 2002::/16 and 2001::/32, and replacing
|
||||||
(RFC1918), unique-local and every other non-global, non-loopback range.
|
``2002:0808:0808::`` with the global 8.8.8.8 would turn an address the guard
|
||||||
|
blocks today into an allowed one.
|
||||||
"""
|
"""
|
||||||
ip = _normalise_ip(addr)
|
ip = _normalise_ip(addr)
|
||||||
return ip is not None and (ip.is_global or ip.is_loopback)
|
if ip is None:
|
||||||
|
return ()
|
||||||
|
tunnelled = _tunnelled_ipv4(ip)
|
||||||
|
return (ip,) if tunnelled is None else (ip, tunnelled)
|
||||||
|
|
||||||
|
|
||||||
|
def _address_is_global(addr: str) -> bool:
|
||||||
|
ips = _ips_to_judge(addr)
|
||||||
|
return bool(ips) and all(ip.is_global for ip in ips)
|
||||||
|
|
||||||
|
|
||||||
|
def _address_allowed_at_connect(addr: str, is_allowed_endpoint: bool = False) -> bool:
|
||||||
|
"""True if *addr* may be connected to at download time.
|
||||||
|
|
||||||
|
Permits global addresses, and anything at all when the destination is an
|
||||||
|
endpoint the operator or the image configured — a proxy, or the PO token
|
||||||
|
provider (see ``_is_allowed_endpoint``). Internal addresses are otherwise
|
||||||
|
refused with no blanket exception: media URLs that yt-dlp derives from a
|
||||||
|
remote manifest are attacker-controlled and reach this policy without passing
|
||||||
|
``validate_url``, so any range opened here is a range a hostile playlist can
|
||||||
|
read from the server's own network. Blocks link-local
|
||||||
|
(cloud metadata at 169.254.169.254), private (RFC1918), loopback,
|
||||||
|
unique-local and every other non-global range.
|
||||||
|
"""
|
||||||
|
ips = _ips_to_judge(addr)
|
||||||
|
if not ips:
|
||||||
|
return False
|
||||||
|
return is_allowed_endpoint or all(ip.is_global for ip in ips)
|
||||||
|
|
||||||
|
|
||||||
|
def _url_endpoint(url: str):
|
||||||
|
"""Parse a configured URL into a ``(hostname, port)`` pair, or ``None`` if it
|
||||||
|
has no usable host. Used to scope the internal-address allowance to that
|
||||||
|
endpoint alone."""
|
||||||
|
if not isinstance(url, str) or not url.strip():
|
||||||
|
return None
|
||||||
|
candidate = url.strip()
|
||||||
|
if '://' not in candidate:
|
||||||
|
# Bare host:port, as accepted by the *_proxy environment variables.
|
||||||
|
candidate = '//' + candidate
|
||||||
|
try:
|
||||||
|
parts = urlsplit(candidate)
|
||||||
|
hostname, port = parts.hostname, parts.port
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
if not hostname:
|
||||||
|
return None
|
||||||
|
if port is None:
|
||||||
|
port = _SCHEME_DEFAULT_PORTS.get(parts.scheme.lower())
|
||||||
|
return (hostname.rstrip('.').lower(), port)
|
||||||
|
|
||||||
|
|
||||||
|
def _endpoints(urls) -> set:
|
||||||
|
"""The parseable endpoints among *urls*, dropping any that name no host."""
|
||||||
|
return {ep for ep in map(_url_endpoint, urls) if ep is not None}
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_proxy_endpoints(proxy_urls) -> set:
|
||||||
|
"""Endpoints of every proxy this download may legitimately dial: the explicit
|
||||||
|
yt-dlp ``proxy`` option plus the ``*_proxy`` environment variables yt-dlp falls
|
||||||
|
back to. All are operator-configured, unlike the URLs inside fetched media."""
|
||||||
|
candidates = list(proxy_urls) + list(urllib.request.getproxies().values())
|
||||||
|
return _endpoints(candidates)
|
||||||
|
|
||||||
|
|
||||||
|
def _proxy_scheme(proxy: str) -> str:
|
||||||
|
"""The scheme of a configured proxy URL.
|
||||||
|
|
||||||
|
Defaults to ``http`` for the bare ``host:port`` form the ``*_proxy``
|
||||||
|
variables accept, which is the same default yt-dlp applies to them.
|
||||||
|
"""
|
||||||
|
if not isinstance(proxy, str):
|
||||||
|
return ''
|
||||||
|
candidate = proxy.strip()
|
||||||
|
if '://' not in candidate:
|
||||||
|
return 'http' if candidate else ''
|
||||||
|
return urlsplit(candidate).scheme.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def download_proxies(ytdl_opts=None) -> dict:
|
||||||
|
"""The proxy map a fetch will use, assembled as ``YoutubeDL.proxies`` does.
|
||||||
|
|
||||||
|
An explicit yt-dlp ``proxy`` option replaces the environment wholesale —
|
||||||
|
including any ``no_proxy`` exceptions — while without one the ``*_proxy``
|
||||||
|
variables apply as they stand. Mirrored rather than imported because
|
||||||
|
``YoutubeDL.proxies`` is only reachable from a constructed instance, and
|
||||||
|
because this decides whether a security check runs: a quiet upstream change
|
||||||
|
should leave the check in place rather than silently skip it.
|
||||||
|
"""
|
||||||
|
opts_proxy = (ytdl_opts or {}).get('proxy')
|
||||||
|
if opts_proxy is not None:
|
||||||
|
# '' means "no proxy, ignore the environment", which yt-dlp spells
|
||||||
|
# '__noproxy__' internally.
|
||||||
|
return {'all': opts_proxy or '__noproxy__'}
|
||||||
|
proxies = urllib.request.getproxies()
|
||||||
|
# compat, as in yt-dlp: http_proxy alone also covers https.
|
||||||
|
if 'http' in proxies and 'https' not in proxies:
|
||||||
|
proxies['https'] = proxies['http']
|
||||||
|
return proxies
|
||||||
|
|
||||||
|
|
||||||
|
def _proxy_resolves_remotely(parts, proxies) -> bool:
|
||||||
|
"""True when the proxy carrying this URL looks its hostname up itself.
|
||||||
|
|
||||||
|
Follows yt-dlp's ``select_proxy``: ``no_proxy`` exclusions first, then the
|
||||||
|
per-scheme entry, then the catch-all. Anything unrecognised answers False,
|
||||||
|
so an unparseable or unusual configuration keeps the address check.
|
||||||
|
"""
|
||||||
|
if not proxies:
|
||||||
|
return False
|
||||||
|
hostname = parts.hostname
|
||||||
|
if not hostname:
|
||||||
|
return False
|
||||||
|
no_proxy = proxies.get('no')
|
||||||
|
if no_proxy:
|
||||||
|
hostport = hostname if parts.port is None else f'{hostname}:{parts.port}'
|
||||||
|
try:
|
||||||
|
if urllib.request.proxy_bypass_environment(hostport, {'no': no_proxy}):
|
||||||
|
return False
|
||||||
|
except (ValueError, UnicodeError):
|
||||||
|
return False
|
||||||
|
proxy = proxies.get(parts.scheme.lower()) or proxies.get('all')
|
||||||
|
if not proxy or proxy == '__noproxy__':
|
||||||
|
return False
|
||||||
|
return _proxy_scheme(proxy) in _REMOTE_DNS_PROXY_SCHEMES
|
||||||
|
|
||||||
|
|
||||||
# Captured at import so re-installing the guard never wraps the wrapper.
|
# Captured at import so re-installing the guard never wraps the wrapper.
|
||||||
_real_getaddrinfo = socket.getaddrinfo
|
_real_getaddrinfo = socket.getaddrinfo
|
||||||
|
|
||||||
|
# Populated by install_socket_guard; empty means no internal destination is allowed.
|
||||||
|
_allowed_endpoints: set = set()
|
||||||
|
|
||||||
|
|
||||||
|
def _normalise_port(port):
|
||||||
|
if isinstance(port, str):
|
||||||
|
try:
|
||||||
|
return int(port)
|
||||||
|
except ValueError:
|
||||||
|
try:
|
||||||
|
return socket.getservbyname(port)
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
return port
|
||||||
|
|
||||||
|
|
||||||
|
def _is_allowed_endpoint(host, port) -> bool:
|
||||||
|
"""True when host:port is exactly one of the endpoints this download is
|
||||||
|
configured to dial — a proxy or the PO token provider. Matching is on the
|
||||||
|
configured host *string*, not on the resolved address, so a hostile media URL
|
||||||
|
cannot borrow the allowance by resolving to the same address under a
|
||||||
|
different name."""
|
||||||
|
if not _allowed_endpoints or host is None:
|
||||||
|
return False
|
||||||
|
return (str(host).rstrip('.').lower(), _normalise_port(port)) in _allowed_endpoints
|
||||||
|
|
||||||
|
|
||||||
def _guarded_getaddrinfo(host, *args, **kwargs):
|
def _guarded_getaddrinfo(host, *args, **kwargs):
|
||||||
results = _real_getaddrinfo(host, *args, **kwargs)
|
results = _real_getaddrinfo(host, *args, **kwargs)
|
||||||
allowed = [r for r in results if _address_allowed_at_connect(r[4][0])]
|
# Mirrors getaddrinfo(host, port, ...): port is the first optional argument.
|
||||||
|
port = args[0] if args else kwargs.get('port')
|
||||||
|
is_configured = _is_allowed_endpoint(host, port)
|
||||||
|
allowed = [r for r in results if _address_allowed_at_connect(r[4][0], is_configured)]
|
||||||
if not allowed:
|
if not allowed:
|
||||||
raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}')
|
raise socket.gaierror(f'Refusing to connect to non-global address for host {host!r}')
|
||||||
return allowed
|
return allowed
|
||||||
|
|
||||||
|
|
||||||
def install_socket_guard(allow_private: bool = False) -> None:
|
def install_socket_guard(allow_private: bool = False, proxy_urls=(), service_urls=()) -> None:
|
||||||
"""Enforce the no-internal-hosts policy at actual connection time.
|
"""Enforce the no-internal-hosts policy at actual connection time.
|
||||||
|
|
||||||
``validate_url`` only checks the *submitted* URL string; yt-dlp then follows
|
``validate_url`` only checks the *submitted* URL string; yt-dlp then follows
|
||||||
HTTP redirects and resolves media URLs from remote metadata without
|
HTTP redirects and resolves media URLs from remote metadata without
|
||||||
re-validating them. Installing this in the download subprocess re-checks
|
re-validating them. Installing this in the download subprocess re-checks
|
||||||
every resolved address at connect time, covering redirects and DNS rebinding
|
every resolved address at connect time, covering redirects, DNS rebinding and
|
||||||
for any networking backend that resolves through Python's socket module
|
manifest-derived media URLs for any networking backend that resolves through
|
||||||
(urllib, requests). Native resolvers — notably curl_cffi/libcurl used by
|
Python's socket module (urllib, requests). Native resolvers — notably
|
||||||
``--impersonate`` — bypass this and rely on network isolation as the backstop.
|
curl_cffi/libcurl used by ``--impersonate`` — bypass this and rely on network
|
||||||
|
isolation as the backstop.
|
||||||
|
|
||||||
|
*proxy_urls* are the operator's configured proxies (yt-dlp's ``proxy`` option;
|
||||||
|
the ``*_proxy`` environment variables are picked up automatically), and
|
||||||
|
*service_urls* the helper services the download itself has to reach — the PO
|
||||||
|
token provider this image ships and starts on loopback. Each is reachable at
|
||||||
|
its own host:port wherever it lives — loopback, the LAN, a VPN range — and
|
||||||
|
nothing else internal is. That costs those setups nothing and gives away
|
||||||
|
little: yt-dlp dials each at exactly that host:port, and a media URL is either
|
||||||
|
handed to the proxy unresolved or resolved on its own merits — never
|
||||||
|
inheriting the allowance. A hostile media URL naming an allowed endpoint
|
||||||
|
reaches only what is listening there: a proxy that would have fetched it
|
||||||
|
anyway, or a token server with two endpoints and nothing to read.
|
||||||
|
|
||||||
When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not
|
When *allow_private* is set (``ALLOW_PRIVATE_ADDRESSES``), the guard is not
|
||||||
installed at all, so proxy/VPN setups that route through private or Fake-IP
|
installed at all, so proxy/VPN setups that route through private or Fake-IP
|
||||||
@@ -109,10 +331,17 @@ def install_socket_guard(allow_private: bool = False) -> None:
|
|||||||
"""
|
"""
|
||||||
if allow_private:
|
if allow_private:
|
||||||
return
|
return
|
||||||
|
proxy_endpoints = _collect_proxy_endpoints(proxy_urls)
|
||||||
|
service_endpoints = _endpoints(service_urls) - proxy_endpoints
|
||||||
|
_allowed_endpoints.clear()
|
||||||
|
_allowed_endpoints.update(proxy_endpoints | service_endpoints)
|
||||||
|
for label, endpoints in (('proxy', proxy_endpoints), ('service', service_endpoints)):
|
||||||
|
for host, port in sorted(endpoints, key=lambda ep: (ep[0], ep[1] or 0)):
|
||||||
|
log.info(f'Allowing connections to configured {label} {host}:{port}')
|
||||||
socket.getaddrinfo = _guarded_getaddrinfo
|
socket.getaddrinfo = _guarded_getaddrinfo
|
||||||
|
|
||||||
|
|
||||||
def validate_url(url: str, allow_private: bool = False) -> str | None:
|
def validate_url(url: str, allow_private: bool = False, proxies: dict | None = None) -> str | None:
|
||||||
"""Return an error message if the URL is disallowed, else ``None``.
|
"""Return an error message if the URL is disallowed, else ``None``.
|
||||||
|
|
||||||
Inputs without a ``://`` scheme separator (bare video IDs, ``ytsearch:``
|
Inputs without a ``://`` scheme separator (bare video IDs, ``ytsearch:``
|
||||||
@@ -123,6 +352,10 @@ def validate_url(url: str, allow_private: bool = False) -> str | None:
|
|||||||
and internal-address checks are skipped so that trusted proxy/VPN setups —
|
and internal-address checks are skipped so that trusted proxy/VPN setups —
|
||||||
e.g. Fake-IP clients that resolve YouTube to ``198.18.0.0/15`` — can be used.
|
e.g. Fake-IP clients that resolve YouTube to ``198.18.0.0/15`` — can be used.
|
||||||
Scheme validation (http/https only) still applies.
|
Scheme validation (http/https only) still applies.
|
||||||
|
|
||||||
|
*proxies* is the proxy map the fetch will use (see ``download_proxies``).
|
||||||
|
When it routes this URL through a proxy that resolves hostnames itself, the
|
||||||
|
address check is skipped — see the comment at that branch for why.
|
||||||
"""
|
"""
|
||||||
if not isinstance(url, str):
|
if not isinstance(url, str):
|
||||||
return 'Invalid URL'
|
return 'Invalid URL'
|
||||||
@@ -148,6 +381,21 @@ def validate_url(url: str, allow_private: bool = False) -> str | None:
|
|||||||
if _hostname_is_blocked(hostname):
|
if _hostname_is_blocked(hostname):
|
||||||
return f'Refusing to fetch internal host "{hostname}"'
|
return f'Refusing to fetch internal host "{hostname}"'
|
||||||
|
|
||||||
|
# A host written as an IP literal needs no lookup, so it is judged directly
|
||||||
|
# whatever the proxy setup: nothing leaves this process, and the address is
|
||||||
|
# exactly the one the request will name.
|
||||||
|
if _normalise_ip(hostname) is None and _proxy_resolves_remotely(parts, proxies):
|
||||||
|
# The proxy resolves this hostname on its own network, so a lookup here
|
||||||
|
# answers a different question than the one that matters, and asking it
|
||||||
|
# is itself the harm: it leaks the hostname of every queued URL to the
|
||||||
|
# local resolver, which is the single thing a SOCKS/Tor setup exists to
|
||||||
|
# prevent. It also fails closed against resolvers this process cannot
|
||||||
|
# reach — a container pointed at the proxy's own DNS port resolves
|
||||||
|
# nothing here and every add is refused. What the proxy's network
|
||||||
|
# exposes is the proxy's to police; the connect-time guard still holds
|
||||||
|
# everything that is dialled directly.
|
||||||
|
return None
|
||||||
|
|
||||||
try:
|
try:
|
||||||
addrinfo = socket.getaddrinfo(hostname, parts.port, proto=socket.IPPROTO_TCP)
|
addrinfo = socket.getaddrinfo(hostname, parts.port, proto=socket.IPPROTO_TCP)
|
||||||
except socket.gaierror:
|
except socket.gaierror:
|
||||||
|
|||||||
+386
-83
@@ -27,11 +27,60 @@ from music_metadata import MusicMetadataPreProcessor
|
|||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from state_store import AtomicJsonStore, from_json_compatible, read_legacy_shelf, to_json_compatible
|
from state_store import AtomicJsonStore, from_json_compatible, read_legacy_shelf, to_json_compatible
|
||||||
from subscriptions import _entry_id
|
from subscriptions import _entry_id
|
||||||
from url_guard import validate_url, install_socket_guard
|
from url_guard import validate_url, install_socket_guard, download_proxies
|
||||||
from urllib.parse import urlsplit
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
log = logging.getLogger('ytdl')
|
log = logging.getLogger('ytdl')
|
||||||
|
|
||||||
|
|
||||||
|
# Fragmented and live downloads can emit a warning per fragment, and the joined
|
||||||
|
# text is persisted with the completed queue and broadcast to every client, so
|
||||||
|
# only the last few distinct warnings are kept.
|
||||||
|
_MAX_RETAINED_WARNINGS = 5
|
||||||
|
|
||||||
|
_REPORT_LABEL_RE = re.compile(r'^(?:ERROR|WARNING):\s*')
|
||||||
|
|
||||||
|
|
||||||
|
def _report_body(message):
|
||||||
|
"""yt-dlp labels errors with an ``ERROR:`` prefix but hands warnings to the
|
||||||
|
logger unlabelled, so compare the two with any such label removed."""
|
||||||
|
return _REPORT_LABEL_RE.sub('', message).strip()
|
||||||
|
|
||||||
|
|
||||||
|
class _DownloadYtdlLogger:
|
||||||
|
"""Forward yt-dlp output while retaining warnings for failed downloads."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self._warnings = collections.deque(maxlen=_MAX_RETAINED_WARNINGS)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def warnings(self):
|
||||||
|
return list(self._warnings)
|
||||||
|
|
||||||
|
def debug(self, msg):
|
||||||
|
log.debug('%s', msg)
|
||||||
|
|
||||||
|
def warning(self, msg):
|
||||||
|
log.warning('%s', msg)
|
||||||
|
if msg is not None and (warning := str(msg).strip()) and warning not in self._warnings:
|
||||||
|
self._warnings.append(warning)
|
||||||
|
|
||||||
|
def error(self, msg):
|
||||||
|
log.error('%s', msg)
|
||||||
|
|
||||||
|
def failure_message(self, error_text):
|
||||||
|
"""Retained warnings followed by *error_text*, kept last so the actual
|
||||||
|
error stays prominent under the context that explains it."""
|
||||||
|
lines = self.warnings
|
||||||
|
error_text = (error_text or '').strip()
|
||||||
|
if not error_text:
|
||||||
|
return '\n'.join(lines)
|
||||||
|
if lines and _report_body(lines[-1]) == _report_body(error_text):
|
||||||
|
lines.pop()
|
||||||
|
lines.append(error_text)
|
||||||
|
return '\n'.join(lines)
|
||||||
|
|
||||||
|
|
||||||
# Python 3.14 switches the default multiprocessing start method on Linux
|
# Python 3.14 switches the default multiprocessing start method on Linux
|
||||||
# (this app's only supported deployment target, per the Dockerfile) from fork
|
# (this app's only supported deployment target, per the Dockerfile) from fork
|
||||||
# to forkserver. Download._download relies on inheriting process state the
|
# to forkserver. Download._download relies on inheriting process state the
|
||||||
@@ -43,6 +92,36 @@ log = logging.getLogger('ytdl')
|
|||||||
# vanish in the child can deadlock it silently before it does any work. This
|
# vanish in the child can deadlock it silently before it does any work. This
|
||||||
# app creates background threads (executors, notifier callbacks) well before
|
# app creates background threads (executors, notifier callbacks) well before
|
||||||
# any download starts, so forcing fork there reproduces exactly that hazard.
|
# any download starts, so forcing fork there reproduces exactly that hazard.
|
||||||
|
# The image ships yt-dlp's bgutil PO token provider and starts it on loopback
|
||||||
|
# (docker-entrypoint.sh); the plugin dials this URL unless pointed elsewhere.
|
||||||
|
# Without a token YouTube serves 403s, so the connect-time guard has to let the
|
||||||
|
# download subprocess reach it.
|
||||||
|
_POT_PROVIDER_DEFAULT_URL = 'http://127.0.0.1:4416'
|
||||||
|
|
||||||
|
# extractor-arg keys the bgutil HTTP provider reads its base URL from: the
|
||||||
|
# current one first, then the deprecated form it still honours.
|
||||||
|
_POT_PROVIDER_BASE_URL_ARGS = (
|
||||||
|
('youtubepot-bgutilhttp', 'base_url'),
|
||||||
|
('youtube', 'getpot_bgutil_baseurl'),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _pot_provider_urls(ytdl_opts: dict) -> tuple:
|
||||||
|
"""Every PO token provider endpoint this download may dial: the bundled one,
|
||||||
|
plus any the operator pointed yt-dlp at through ``extractor_args``. The
|
||||||
|
bundled server runs either way, so it stays allowed even when a base URL is
|
||||||
|
configured."""
|
||||||
|
urls = [_POT_PROVIDER_DEFAULT_URL]
|
||||||
|
extractor_args = ytdl_opts.get('extractor_args')
|
||||||
|
if isinstance(extractor_args, dict):
|
||||||
|
for ie_key, arg in _POT_PROVIDER_BASE_URL_ARGS:
|
||||||
|
section = extractor_args.get(ie_key)
|
||||||
|
values = section.get(arg) if isinstance(section, dict) else None
|
||||||
|
if values:
|
||||||
|
urls.append(values[0])
|
||||||
|
return tuple(urls)
|
||||||
|
|
||||||
|
|
||||||
_MP_CTX = (
|
_MP_CTX = (
|
||||||
multiprocessing.get_context("fork")
|
multiprocessing.get_context("fork")
|
||||||
if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods()
|
if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods()
|
||||||
@@ -147,6 +226,56 @@ def _sanitize_path_component(value: Any) -> Any:
|
|||||||
return value.lstrip('.').strip() or '_'
|
return value.lstrip('.').strip() or '_'
|
||||||
|
|
||||||
|
|
||||||
|
# Room left for the suffixes yt-dlp appends after prepare_filename has run:
|
||||||
|
# '.part' and '.ytdl' while the download is in flight, '.f<format_id>' for a
|
||||||
|
# stream fetched on its own before merging, '-Frag<n>' for fragmented
|
||||||
|
# downloads. A name trimmed to exactly the limit would still fail the moment
|
||||||
|
# one of those is added, which is what the '.part' in the reported errors is.
|
||||||
|
_NAME_SUFFIX_RESERVE_BYTES = 32
|
||||||
|
# POSIX guarantees at least this much, and it is what ext4/xfs/btrfs allow.
|
||||||
|
_FALLBACK_NAME_MAX_BYTES = 255
|
||||||
|
# Keep a recognisable stem even on a filesystem with a very short limit.
|
||||||
|
_MIN_STEM_BYTES = 16
|
||||||
|
# Longer than this is not really an extension (a title ending in '.something'),
|
||||||
|
# so the whole name is treated as the stem rather than preserving it.
|
||||||
|
_MAX_EXT_BYTES = 16
|
||||||
|
|
||||||
|
|
||||||
|
def _name_max_bytes(directory: str) -> int:
|
||||||
|
"""The filesystem's filename limit, in bytes, for *directory*."""
|
||||||
|
try:
|
||||||
|
return int(os.pathconf(directory or '.', 'PC_NAME_MAX'))
|
||||||
|
except (OSError, ValueError, AttributeError):
|
||||||
|
# The directory may not exist yet (CREATE_CUSTOM_DIRS makes it during
|
||||||
|
# the download), and pathconf is not available on every platform.
|
||||||
|
return _FALLBACK_NAME_MAX_BYTES
|
||||||
|
|
||||||
|
|
||||||
|
def _trim_to_name_max(path: str) -> str:
|
||||||
|
"""Shorten the final component of *path* to what the filesystem accepts.
|
||||||
|
|
||||||
|
The limit is a byte count, not a character count: a title of accented or
|
||||||
|
CJK characters hits it in half as many characters, or fewer. The extension
|
||||||
|
is preserved, since it is what decides how the file is handled afterwards.
|
||||||
|
"""
|
||||||
|
directory, name = os.path.split(path)
|
||||||
|
if not name:
|
||||||
|
return path
|
||||||
|
encoded = name.encode('utf-8', 'surrogatepass')
|
||||||
|
limit = _name_max_bytes(directory) - _NAME_SUFFIX_RESERVE_BYTES
|
||||||
|
if len(encoded) <= limit:
|
||||||
|
return path
|
||||||
|
|
||||||
|
stem, ext = os.path.splitext(name)
|
||||||
|
ext_bytes = ext.encode('utf-8', 'surrogatepass')
|
||||||
|
if len(ext_bytes) > _MAX_EXT_BYTES:
|
||||||
|
stem, ext, ext_bytes = name, '', b''
|
||||||
|
stem_limit = max(limit - len(ext_bytes), _MIN_STEM_BYTES)
|
||||||
|
# 'ignore' drops a multi-byte character the cut landed inside of.
|
||||||
|
trimmed = stem.encode('utf-8', 'surrogatepass')[:stem_limit].decode('utf-8', 'ignore').rstrip()
|
||||||
|
return os.path.join(directory, (trimmed or '_') + ext)
|
||||||
|
|
||||||
|
|
||||||
class _ConfinedYoutubeDL(yt_dlp.YoutubeDL):
|
class _ConfinedYoutubeDL(yt_dlp.YoutubeDL):
|
||||||
"""A ``YoutubeDL`` that refuses to emit any output path outside the allowed roots.
|
"""A ``YoutubeDL`` that refuses to emit any output path outside the allowed roots.
|
||||||
|
|
||||||
@@ -170,6 +299,12 @@ class _ConfinedYoutubeDL(yt_dlp.YoutubeDL):
|
|||||||
|
|
||||||
def prepare_filename(self, *args, **kwargs):
|
def prepare_filename(self, *args, **kwargs):
|
||||||
filename = super().prepare_filename(*args, **kwargs)
|
filename = super().prepare_filename(*args, **kwargs)
|
||||||
|
# Titles long enough to exceed the filesystem's filename limit are
|
||||||
|
# common on some sites, and the download fails outright when they do.
|
||||||
|
# Every output path comes through here, so trimming once keeps the
|
||||||
|
# main file, its chapter files, thumbnails and subtitles consistent.
|
||||||
|
if filename and filename != '-':
|
||||||
|
filename = _trim_to_name_max(filename)
|
||||||
if filename and filename != '-' and self._allowed_roots:
|
if filename and filename != '-' and self._allowed_roots:
|
||||||
resolved = os.path.realpath(filename)
|
resolved = os.path.realpath(filename)
|
||||||
if not any(_is_within_directory(root, resolved) for root in self._allowed_roots):
|
if not any(_is_within_directory(root, resolved) for root in self._allowed_roots):
|
||||||
@@ -348,6 +483,7 @@ class DownloadInfo:
|
|||||||
clip_end=None,
|
clip_end=None,
|
||||||
live_status=None,
|
live_status=None,
|
||||||
live_release_timestamp=None,
|
live_release_timestamp=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
||||||
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
||||||
@@ -359,6 +495,12 @@ class DownloadInfo:
|
|||||||
self.folder = folder
|
self.folder = folder
|
||||||
self.custom_name_prefix = custom_name_prefix
|
self.custom_name_prefix = custom_name_prefix
|
||||||
self.msg = self.percent = self.speed = self.eta = None
|
self.msg = self.percent = self.speed = self.eta = None
|
||||||
|
# 'pending' means "waiting for the user to press Start" — an item added
|
||||||
|
# with auto_start=False, sitting in self.pending. A download that is in
|
||||||
|
# self.queue waiting for a MAX_CONCURRENT_DOWNLOADS slot is 'queued'
|
||||||
|
# instead: it starts on its own and there is nothing to press. Keeping
|
||||||
|
# both under one name left the UI showing a Start button that silently
|
||||||
|
# did nothing (#1081).
|
||||||
self.status = "pending"
|
self.status = "pending"
|
||||||
self.size = None
|
self.size = None
|
||||||
self.timestamp = time.time_ns()
|
self.timestamp = time.time_ns()
|
||||||
@@ -367,6 +509,7 @@ class DownloadInfo:
|
|||||||
self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None
|
self.entry = _sanitize_entry_for_pickle(entry) if entry is not None else None
|
||||||
self.playlist_item_limit = playlist_item_limit
|
self.playlist_item_limit = playlist_item_limit
|
||||||
self.split_by_chapters = split_by_chapters
|
self.split_by_chapters = split_by_chapters
|
||||||
|
self.sponsorblock = sponsorblock
|
||||||
self.chapter_template = chapter_template
|
self.chapter_template = chapter_template
|
||||||
self.subtitle_language = subtitle_language
|
self.subtitle_language = subtitle_language
|
||||||
self.subtitle_mode = subtitle_mode
|
self.subtitle_mode = subtitle_mode
|
||||||
@@ -436,6 +579,8 @@ class DownloadInfo:
|
|||||||
self.playlist_item_limit = 0
|
self.playlist_item_limit = 0
|
||||||
if not hasattr(self, "split_by_chapters"):
|
if not hasattr(self, "split_by_chapters"):
|
||||||
self.split_by_chapters = False
|
self.split_by_chapters = False
|
||||||
|
if not hasattr(self, "sponsorblock"):
|
||||||
|
self.sponsorblock = False
|
||||||
if not hasattr(self, "chapter_template"):
|
if not hasattr(self, "chapter_template"):
|
||||||
self.chapter_template = ""
|
self.chapter_template = ""
|
||||||
if not hasattr(self, "subtitle_language"):
|
if not hasattr(self, "subtitle_language"):
|
||||||
@@ -480,6 +625,7 @@ _PERSISTED_DOWNLOAD_FIELDS = (
|
|||||||
"custom_name_prefix",
|
"custom_name_prefix",
|
||||||
"playlist_item_limit",
|
"playlist_item_limit",
|
||||||
"split_by_chapters",
|
"split_by_chapters",
|
||||||
|
"sponsorblock",
|
||||||
"chapter_template",
|
"chapter_template",
|
||||||
"subtitle_language",
|
"subtitle_language",
|
||||||
"subtitle_mode",
|
"subtitle_mode",
|
||||||
@@ -631,6 +777,59 @@ class Download:
|
|||||||
|
|
||||||
return put_status
|
return put_status
|
||||||
|
|
||||||
|
def _make_postprocessor_hook(self):
|
||||||
|
# yt-dlp reports the download 'finished' as soon as the media bytes
|
||||||
|
# have landed, but the ffmpeg work that follows -- merging,
|
||||||
|
# re-encoding, chapter splitting, sponsor removal -- routinely takes
|
||||||
|
# longer than the download itself and said nothing until now, leaving
|
||||||
|
# the row on a full, frozen progress bar for minutes. Report the phase
|
||||||
|
# so the UI can run its indeterminate bar instead.
|
||||||
|
#
|
||||||
|
# Latched off once MoveFiles reports finished: that branch emits the
|
||||||
|
# terminal 'finished', and _post_download_cleanup turns any other
|
||||||
|
# final status into an error, so nothing may overwrite it. yt-dlp does
|
||||||
|
# run an 'after_move' stage after MoveFiles, but every postprocessor
|
||||||
|
# MeTube configures is 'after_filter' or 'post_process', both of which
|
||||||
|
# precede it. Each postprocessor emits exactly one started/finished
|
||||||
|
# pair, so this needs no throttling.
|
||||||
|
postprocessing_done = False
|
||||||
|
|
||||||
|
def put_status_postprocessor(d):
|
||||||
|
nonlocal postprocessing_done
|
||||||
|
if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished':
|
||||||
|
postprocessing_done = True
|
||||||
|
filepath = d['info_dict']['filepath']
|
||||||
|
if '__finaldir' in d['info_dict']:
|
||||||
|
finaldir = d['info_dict']['__finaldir']
|
||||||
|
filename = os.path.join(finaldir, os.path.basename(filepath))
|
||||||
|
else:
|
||||||
|
filename = filepath
|
||||||
|
self.status_queue.put({'status': 'finished', 'filename': filename})
|
||||||
|
# For captions-only downloads, yt-dlp may still report a media-like
|
||||||
|
# filepath in MoveFiles. Capture subtitle outputs explicitly so the
|
||||||
|
# UI can link to real caption files.
|
||||||
|
if getattr(self.info, 'download_type', '') == 'captions':
|
||||||
|
requested_subtitles = d.get('info_dict', {}).get('requested_subtitles', {}) or {}
|
||||||
|
for subtitle in requested_subtitles.values():
|
||||||
|
if isinstance(subtitle, dict) and subtitle.get('filepath'):
|
||||||
|
self.status_queue.put({'subtitle_file': subtitle['filepath']})
|
||||||
|
|
||||||
|
# Capture all chapter files when SplitChapters finishes
|
||||||
|
elif d.get('postprocessor') == 'SplitChapters' and d.get('status') == 'finished':
|
||||||
|
chapters = d.get('info_dict', {}).get('chapters', [])
|
||||||
|
if chapters:
|
||||||
|
for chapter in chapters:
|
||||||
|
if isinstance(chapter, dict) and 'filepath' in chapter:
|
||||||
|
log.info(f"Captured chapter file: {chapter['filepath']}")
|
||||||
|
self.status_queue.put({'chapter_file': chapter['filepath']})
|
||||||
|
else:
|
||||||
|
log.warning("SplitChapters finished but no chapter files found in info_dict")
|
||||||
|
|
||||||
|
elif d.get('status') == 'started' and not postprocessing_done:
|
||||||
|
self.status_queue.put({'status': 'postprocessing'})
|
||||||
|
|
||||||
|
return put_status_postprocessor
|
||||||
|
|
||||||
def _make_youtube_dl(self, params):
|
def _make_youtube_dl(self, params):
|
||||||
ydl = _ConfinedYoutubeDL(
|
ydl = _ConfinedYoutubeDL(
|
||||||
params=params,
|
params=params,
|
||||||
@@ -657,43 +856,25 @@ class Download:
|
|||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
# Re-validate every outbound connection at fetch time. validate_url only
|
# Re-validate every outbound connection at fetch time. validate_url only
|
||||||
# saw the submitted URL string; this catches redirects and DNS rebinding
|
# saw the submitted URL string; this catches redirects, DNS rebinding and
|
||||||
# to internal hosts (cloud metadata, RFC1918) that it cannot. Skipped when
|
# attacker-controlled media URLs pulled from a remote manifest, none of
|
||||||
# ALLOW_PRIVATE_ADDRESSES trusts the environment (e.g. Fake-IP proxies).
|
# which it can see. The configured proxy and the PO token provider are
|
||||||
install_socket_guard(self.allow_private)
|
# passed so that each stays reachable at its own host:port without opening
|
||||||
|
# up anything else. Skipped when ALLOW_PRIVATE_ADDRESSES trusts the
|
||||||
|
# environment.
|
||||||
|
install_socket_guard(
|
||||||
|
self.allow_private,
|
||||||
|
proxy_urls=(self.ytdl_opts.get('proxy'),),
|
||||||
|
service_urls=_pot_provider_urls(self.ytdl_opts),
|
||||||
|
)
|
||||||
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
||||||
|
# Bound outside the try so the except branch can read what was captured
|
||||||
|
# before the error was raised.
|
||||||
|
ytdl_logger = _DownloadYtdlLogger()
|
||||||
try:
|
try:
|
||||||
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
||||||
put_status = self._make_progress_hook()
|
put_status = self._make_progress_hook()
|
||||||
|
put_status_postprocessor = self._make_postprocessor_hook()
|
||||||
def put_status_postprocessor(d):
|
|
||||||
if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished':
|
|
||||||
filepath = d['info_dict']['filepath']
|
|
||||||
if '__finaldir' in d['info_dict']:
|
|
||||||
finaldir = d['info_dict']['__finaldir']
|
|
||||||
filename = os.path.join(finaldir, os.path.basename(filepath))
|
|
||||||
else:
|
|
||||||
filename = filepath
|
|
||||||
self.status_queue.put({'status': 'finished', 'filename': filename})
|
|
||||||
# For captions-only downloads, yt-dlp may still report a media-like
|
|
||||||
# filepath in MoveFiles. Capture subtitle outputs explicitly so the
|
|
||||||
# UI can link to real caption files.
|
|
||||||
if getattr(self.info, 'download_type', '') == 'captions':
|
|
||||||
requested_subtitles = d.get('info_dict', {}).get('requested_subtitles', {}) or {}
|
|
||||||
for subtitle in requested_subtitles.values():
|
|
||||||
if isinstance(subtitle, dict) and subtitle.get('filepath'):
|
|
||||||
self.status_queue.put({'subtitle_file': subtitle['filepath']})
|
|
||||||
|
|
||||||
# Capture all chapter files when SplitChapters finishes
|
|
||||||
elif d.get('postprocessor') == 'SplitChapters' and d.get('status') == 'finished':
|
|
||||||
chapters = d.get('info_dict', {}).get('chapters', [])
|
|
||||||
if chapters:
|
|
||||||
for chapter in chapters:
|
|
||||||
if isinstance(chapter, dict) and 'filepath' in chapter:
|
|
||||||
log.info(f"Captured chapter file: {chapter['filepath']}")
|
|
||||||
self.status_queue.put({'chapter_file': chapter['filepath']})
|
|
||||||
else:
|
|
||||||
log.warning("SplitChapters finished but no chapter files found in info_dict")
|
|
||||||
|
|
||||||
ytdl_params = {
|
ytdl_params = {
|
||||||
'quiet': not debug_logging,
|
'quiet': not debug_logging,
|
||||||
@@ -708,6 +889,30 @@ class Download:
|
|||||||
'postprocessor_hooks': [put_status_postprocessor],
|
'postprocessor_hooks': [put_status_postprocessor],
|
||||||
**self.ytdl_opts,
|
**self.ytdl_opts,
|
||||||
}
|
}
|
||||||
|
# Set after the ytdl_opts merge: the failure messages below depend on
|
||||||
|
# this logger, so a user-supplied one must not replace it.
|
||||||
|
ytdl_params['logger'] = ytdl_logger
|
||||||
|
|
||||||
|
# SponsorBlock: mark sponsor segments and cut them out, the same
|
||||||
|
# postprocessor pair the CLI's --sponsorblock-remove sponsor builds.
|
||||||
|
# This has to stay above the chapter-splitting block: yt-dlp runs
|
||||||
|
# same-stage postprocessors in list order, and ModifyChapters must
|
||||||
|
# rewrite the chapter list before FFmpegSplitChapters cuts the file
|
||||||
|
# up, or the chapter files keep the sponsor segments and the
|
||||||
|
# removal desyncs the remaining chapter timings.
|
||||||
|
if getattr(self.info, 'sponsorblock', False):
|
||||||
|
if 'postprocessors' not in ytdl_params:
|
||||||
|
ytdl_params['postprocessors'] = []
|
||||||
|
ytdl_params['postprocessors'].append({
|
||||||
|
'key': 'SponsorBlock',
|
||||||
|
'categories': ['sponsor'],
|
||||||
|
'when': 'after_filter',
|
||||||
|
})
|
||||||
|
ytdl_params['postprocessors'].append({
|
||||||
|
'key': 'ModifyChapters',
|
||||||
|
'remove_sponsor_segments': ['sponsor'],
|
||||||
|
'force_keyframes': False,
|
||||||
|
})
|
||||||
|
|
||||||
# Add chapter splitting options if enabled
|
# Add chapter splitting options if enabled
|
||||||
if self.info.split_by_chapters:
|
if self.info.split_by_chapters:
|
||||||
@@ -730,11 +935,15 @@ class Download:
|
|||||||
)
|
)
|
||||||
|
|
||||||
ret = self._make_youtube_dl(ytdl_params).download([self.info.url])
|
ret = self._make_youtube_dl(ytdl_params).download([self.info.url])
|
||||||
self.status_queue.put({'status': 'finished' if ret == 0 else 'error'})
|
if ret == 0:
|
||||||
|
self.status_queue.put({'status': 'finished'})
|
||||||
|
else:
|
||||||
|
msg = '\n'.join(ytdl_logger.warnings) or f'yt-dlp failed with exit code {ret}'
|
||||||
|
self.status_queue.put({'status': 'error', 'msg': msg})
|
||||||
log.info(f"Finished download for: {self.info.title}")
|
log.info(f"Finished download for: {self.info.title}")
|
||||||
except yt_dlp.utils.YoutubeDLError as exc:
|
except yt_dlp.utils.YoutubeDLError as exc:
|
||||||
log.error(f"Download error for {self.info.title}: {str(exc)}")
|
log.error(f"Download error for {self.info.title}: {str(exc)}")
|
||||||
self.status_queue.put({'status': 'error', 'msg': str(exc)})
|
self.status_queue.put({'status': 'error', 'msg': ytdl_logger.failure_message(str(exc))})
|
||||||
|
|
||||||
async def start(self, notifier, executor=None):
|
async def start(self, notifier, executor=None):
|
||||||
log.info(f"Preparing download for: {self.info.title}")
|
log.info(f"Preparing download for: {self.info.title}")
|
||||||
@@ -849,7 +1058,17 @@ class Download:
|
|||||||
if not rel_name.lower().endswith(allowed_caption_exts):
|
if not rel_name.lower().endswith(allowed_caption_exts):
|
||||||
continue
|
continue
|
||||||
self.info.filename = rel_name
|
self.info.filename = rel_name
|
||||||
self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
|
# Stat only on a terminal status. yt-dlp documents 'filename' as
|
||||||
|
# always present in a progress hook, but until the download
|
||||||
|
# finishes the bytes are in tmpfilename and 'filename' is a
|
||||||
|
# destination that does not exist yet -- so this was two
|
||||||
|
# blocking syscalls on the event loop, twice a second per
|
||||||
|
# active download, to arrive at None. A stat that takes seconds
|
||||||
|
# on a contended filesystem stalls every other request the
|
||||||
|
# server is serving. Nothing displays the size before
|
||||||
|
# completion: the Downloading table has no size column.
|
||||||
|
if status.get('status') == 'finished':
|
||||||
|
self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
|
||||||
if getattr(self.info, 'download_type', '') == 'thumbnail':
|
if getattr(self.info, 'download_type', '') == 'thumbnail':
|
||||||
# The thumbnail convertor always emits a .jpg, but yt-dlp may
|
# The thumbnail convertor always emits a .jpg, but yt-dlp may
|
||||||
# report the pre-conversion media/thumbnail extension
|
# report the pre-conversion media/thumbnail extension
|
||||||
@@ -902,6 +1121,17 @@ class Download:
|
|||||||
self.info.size = file_size
|
self.info.size = file_size
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# yt-dlp's postprocessor metaclass wraps run() once per class in the
|
||||||
|
# MRO, so a postprocessor whose subclass overrides run reports
|
||||||
|
# started twice (FFmpegCopyStream, among others). Nothing has changed
|
||||||
|
# between the two, so drop the repeat rather than re-encode and
|
||||||
|
# rebroadcast the same state to every connected client. Compared
|
||||||
|
# against the live status, not a flag in the hook, so the phase is
|
||||||
|
# still announced when a pre_process postprocessor ran before the
|
||||||
|
# download and 'downloading' came in between.
|
||||||
|
if status['status'] == 'postprocessing' and self.info.status == 'postprocessing':
|
||||||
|
continue
|
||||||
|
|
||||||
self.info.status = status['status']
|
self.info.status = status['status']
|
||||||
self.info.msg = status.get('msg')
|
self.info.msg = status.get('msg')
|
||||||
if 'downloaded_bytes' in status:
|
if 'downloaded_bytes' in status:
|
||||||
@@ -923,6 +1153,19 @@ class PersistentQueue:
|
|||||||
self.path = f"{path}.json"
|
self.path = f"{path}.json"
|
||||||
self.store = AtomicJsonStore(self.path, kind=f"persistent_queue:{name}")
|
self.store = AtomicJsonStore(self.path, kind=f"persistent_queue:{name}")
|
||||||
self.dict = OrderedDict()
|
self.dict = OrderedDict()
|
||||||
|
# A state write fsyncs twice (the file and its directory). On a slow or
|
||||||
|
# contended filesystem that is seconds, and running it inline in an
|
||||||
|
# async caller blocked the event loop -- every other request stalled
|
||||||
|
# behind a single queue mutation. One dedicated thread keeps the writes
|
||||||
|
# off the loop and, being single, keeps them ordered. The default
|
||||||
|
# executor is not usable for this: extract_info shares it and can hold
|
||||||
|
# its threads for minutes, which is exactly when state writes happen.
|
||||||
|
self._store_executor = ThreadPoolExecutor(
|
||||||
|
max_workers=1, thread_name_prefix=f"state-{name}")
|
||||||
|
# Guards the mutate-write-rollback section. The write is awaited now, so
|
||||||
|
# without this two callers could interleave between changing self.dict
|
||||||
|
# and persisting it, and a rollback could revert the wrong entry.
|
||||||
|
self._lock = asyncio.Lock()
|
||||||
|
|
||||||
def load(self):
|
def load(self):
|
||||||
for k, v in self.saved_items():
|
for k, v in self.saved_items():
|
||||||
@@ -963,8 +1206,14 @@ class PersistentQueue:
|
|||||||
for key, download in self.dict.items()
|
for key, download in self.dict.items()
|
||||||
]
|
]
|
||||||
|
|
||||||
def _save_dict(self):
|
async def _save_dict_async(self):
|
||||||
self.store.save({"items": self._serialize_items()})
|
# Serialize on the event loop -- it is pure CPU and sub-millisecond --
|
||||||
|
# and hand the finished payload to the writer thread. Serializing in the
|
||||||
|
# thread instead would have it walk live DownloadInfo objects while the
|
||||||
|
# loop mutates them.
|
||||||
|
payload = {"items": self._serialize_items()}
|
||||||
|
await asyncio.get_running_loop().run_in_executor(
|
||||||
|
self._store_executor, self.store.save, payload)
|
||||||
|
|
||||||
def _load_state_items(self):
|
def _load_state_items(self):
|
||||||
payload = self.store.load()
|
payload = self.store.load()
|
||||||
@@ -1005,32 +1254,39 @@ class PersistentQueue:
|
|||||||
self.store.save({"items": items})
|
self.store.save({"items": items})
|
||||||
return items
|
return items
|
||||||
|
|
||||||
def put(self, value):
|
async def put(self, value):
|
||||||
key = value.info.url
|
key = value.info.url
|
||||||
old = self.dict.get(key)
|
async with self._lock:
|
||||||
self.dict[key] = value
|
old = self.dict.get(key)
|
||||||
try:
|
self.dict[key] = value
|
||||||
self._save_dict()
|
|
||||||
except Exception:
|
|
||||||
if old is None:
|
|
||||||
del self.dict[key]
|
|
||||||
else:
|
|
||||||
self.dict[key] = old
|
|
||||||
raise
|
|
||||||
|
|
||||||
def delete(self, key):
|
|
||||||
if key in self.dict:
|
|
||||||
old = self.dict[key]
|
|
||||||
del self.dict[key]
|
|
||||||
try:
|
try:
|
||||||
self._save_dict()
|
await self._save_dict_async()
|
||||||
except Exception:
|
except Exception:
|
||||||
self.dict[key] = old
|
if old is None:
|
||||||
|
del self.dict[key]
|
||||||
|
else:
|
||||||
|
self.dict[key] = old
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
async def delete(self, key):
|
||||||
|
async with self._lock:
|
||||||
|
if key in self.dict:
|
||||||
|
old = self.dict[key]
|
||||||
|
del self.dict[key]
|
||||||
|
try:
|
||||||
|
await self._save_dict_async()
|
||||||
|
except Exception:
|
||||||
|
self.dict[key] = old
|
||||||
|
raise
|
||||||
|
|
||||||
def empty(self):
|
def empty(self):
|
||||||
return not bool(self.dict)
|
return not bool(self.dict)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
# wait=True so a write already in flight reaches disk before the
|
||||||
|
# process exits; there is at most one, and it is the newest state.
|
||||||
|
self._store_executor.shutdown(wait=True)
|
||||||
|
|
||||||
class DownloadQueue:
|
class DownloadQueue:
|
||||||
def __init__(self, config, notifier):
|
def __init__(self, config, notifier):
|
||||||
self.config = config
|
self.config = config
|
||||||
@@ -1061,13 +1317,33 @@ class DownloadQueue:
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def __is_channel_extraction(entry):
|
def __is_channel_extraction(entry):
|
||||||
"""Return True when yt-dlp reported a channel tab as a playlist.
|
"""Return True when yt-dlp reported a channel as a playlist.
|
||||||
|
|
||||||
YouTube channel tabs are extracted with ``_type: 'playlist'`` but set
|
A channel *tab* -- ``/channel/UC...``, ``/@handle/videos``, and the
|
||||||
``id`` equal to ``channel_id``; real playlists keep a distinct id.
|
streams, shorts and playlists tabs -- is extracted with ``id`` equal to
|
||||||
|
``channel_id``. A channel addressed without a tab keeps the form it was
|
||||||
|
asked for instead: ``@handle`` for a handle URL and the vanity name for
|
||||||
|
a legacy ``/c/`` URL. Both of those match ``uploader_id``, which is the
|
||||||
|
handle either way, so compare against it as well.
|
||||||
|
|
||||||
|
A real playlist has an id of its own and matches neither, even though
|
||||||
|
it also carries its owner's ``channel_id``.
|
||||||
"""
|
"""
|
||||||
channel_id = entry.get('channel_id')
|
channel_id = entry.get('channel_id')
|
||||||
return bool(channel_id) and entry.get('id') == channel_id
|
entry_id = entry.get('id')
|
||||||
|
if not channel_id or not entry_id:
|
||||||
|
return False
|
||||||
|
if entry_id == channel_id:
|
||||||
|
return True
|
||||||
|
uploader_id = entry.get('uploader_id')
|
||||||
|
if not uploader_id:
|
||||||
|
return False
|
||||||
|
# Compared without case because a legacy vanity name and the handle it
|
||||||
|
# became need not agree on it. No playlist id can collide here: those
|
||||||
|
# are 'PL...', 'OLAK...' and the like, never a handle.
|
||||||
|
handle = uploader_id.casefold()
|
||||||
|
entry_id = entry_id.casefold()
|
||||||
|
return handle in (entry_id, f'@{entry_id}')
|
||||||
|
|
||||||
async def __import_queue(self):
|
async def __import_queue(self):
|
||||||
for k, v in self.queue.saved_items():
|
for k, v in self.queue.saved_items():
|
||||||
@@ -1187,8 +1463,8 @@ class DownloadQueue:
|
|||||||
if not info.error:
|
if not info.error:
|
||||||
info.error = str(exc)
|
info.error = str(exc)
|
||||||
self._unregister_scheduled(url)
|
self._unregister_scheduled(url)
|
||||||
self.queue.delete(url)
|
await self.queue.delete(url)
|
||||||
self.done.put(download)
|
await self.done.put(download)
|
||||||
await self.notifier.completed(info)
|
await self.notifier.completed(info)
|
||||||
else:
|
else:
|
||||||
log.warning(
|
log.warning(
|
||||||
@@ -1214,7 +1490,7 @@ class DownloadQueue:
|
|||||||
return
|
return
|
||||||
|
|
||||||
self._unregister_scheduled(url)
|
self._unregister_scheduled(url)
|
||||||
info.status = 'pending'
|
info.status = 'queued'
|
||||||
# Clear the "scheduled to start at ..." placeholder now that the stream
|
# Clear the "scheduled to start at ..." placeholder now that the stream
|
||||||
# is live and a real download is about to begin.
|
# is live and a real download is about to begin.
|
||||||
info.error = None
|
info.error = None
|
||||||
@@ -1222,14 +1498,14 @@ class DownloadQueue:
|
|||||||
await self.notifier.updated(info)
|
await self.notifier.updated(info)
|
||||||
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
||||||
|
|
||||||
def _schedule_upcoming_download(self, download: Download) -> None:
|
async def _schedule_upcoming_download(self, download: Download) -> None:
|
||||||
download.info.status = 'scheduled'
|
download.info.status = 'scheduled'
|
||||||
self.queue.put(download)
|
await self.queue.put(download)
|
||||||
self._register_scheduled(download)
|
self._register_scheduled(download)
|
||||||
|
|
||||||
def _force_start_scheduled(self, download: Download) -> None:
|
def _force_start_scheduled(self, download: Download) -> None:
|
||||||
self._unregister_scheduled(download.info.url)
|
self._unregister_scheduled(download.info.url)
|
||||||
download.info.status = 'pending'
|
download.info.status = 'queued'
|
||||||
download.info.error = None
|
download.info.error = None
|
||||||
download.info.msg = None
|
download.info.msg = None
|
||||||
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
||||||
@@ -1243,9 +1519,9 @@ class DownloadQueue:
|
|||||||
log.info(f"Download {download.info.title} was canceled, skipping start.")
|
log.info(f"Download {download.info.title} was canceled, skipping start.")
|
||||||
return
|
return
|
||||||
await download.start(self.notifier, self._download_executor)
|
await download.start(self.notifier, self._download_executor)
|
||||||
self._post_download_cleanup(download)
|
await self._post_download_cleanup(download)
|
||||||
|
|
||||||
def _post_download_cleanup(self, download):
|
async def _post_download_cleanup(self, download):
|
||||||
if download.info.status != 'finished':
|
if download.info.status != 'finished':
|
||||||
if download.tmpfilename and os.path.isfile(download.tmpfilename):
|
if download.tmpfilename and os.path.isfile(download.tmpfilename):
|
||||||
try:
|
try:
|
||||||
@@ -1265,11 +1541,11 @@ class DownloadQueue:
|
|||||||
download.info.size = None
|
download.info.size = None
|
||||||
download.close()
|
download.close()
|
||||||
if self.queue.exists(download.info.url):
|
if self.queue.exists(download.info.url):
|
||||||
self.queue.delete(download.info.url)
|
await self.queue.delete(download.info.url)
|
||||||
if download.canceled:
|
if download.canceled:
|
||||||
bg_tasks.create_task(self.notifier.canceled(download.info.url), name="notify_canceled")
|
bg_tasks.create_task(self.notifier.canceled(download.info.url), name="notify_canceled")
|
||||||
else:
|
else:
|
||||||
self.done.put(download)
|
await self.done.put(download)
|
||||||
bg_tasks.create_task(self.notifier.completed(download.info), name="notify_completed")
|
bg_tasks.create_task(self.notifier.completed(download.info), name="notify_completed")
|
||||||
try:
|
try:
|
||||||
clear_after = int(self.config.CLEAR_COMPLETED_AFTER)
|
clear_after = int(self.config.CLEAR_COMPLETED_AFTER)
|
||||||
@@ -1377,12 +1653,13 @@ class DownloadQueue:
|
|||||||
)
|
)
|
||||||
if auto_start is True:
|
if auto_start is True:
|
||||||
if is_upcoming:
|
if is_upcoming:
|
||||||
self._schedule_upcoming_download(download)
|
await self._schedule_upcoming_download(download)
|
||||||
else:
|
else:
|
||||||
self.queue.put(download)
|
download.info.status = 'queued'
|
||||||
|
await self.queue.put(download)
|
||||||
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
||||||
else:
|
else:
|
||||||
self.pending.put(download)
|
await self.pending.put(download)
|
||||||
await self.notifier.added(dl)
|
await self.notifier.added(dl)
|
||||||
|
|
||||||
def __write_feed_metadata_sync(self, entry, etype, download_type, folder,
|
def __write_feed_metadata_sync(self, entry, etype, download_type, folder,
|
||||||
@@ -1482,6 +1759,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen=None,
|
_add_gen=None,
|
||||||
retry_entry=None,
|
retry_entry=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if not entry:
|
if not entry:
|
||||||
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
return {'status': 'error', 'msg': "Invalid/empty data was given."}
|
||||||
@@ -1525,6 +1803,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
retry_entry,
|
retry_entry,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
elif etype == 'playlist' or etype == 'channel':
|
elif etype == 'playlist' or etype == 'channel':
|
||||||
if etype == 'playlist' and self.__is_channel_extraction(entry):
|
if etype == 'playlist' and self.__is_channel_extraction(entry):
|
||||||
@@ -1592,6 +1871,7 @@ class DownloadQueue:
|
|||||||
clip_end,
|
clip_end,
|
||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
if any(res['status'] == 'error' for res in results):
|
if any(res['status'] == 'error' for res in results):
|
||||||
@@ -1632,6 +1912,7 @@ class DownloadQueue:
|
|||||||
clip_end=clip_end,
|
clip_end=clip_end,
|
||||||
live_status=entry.get('live_status'),
|
live_status=entry.get('live_status'),
|
||||||
live_release_timestamp=entry.get('release_timestamp'),
|
live_release_timestamp=entry.get('release_timestamp'),
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
await self.__add_download(dl, auto_start)
|
await self.__add_download(dl, auto_start)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
@@ -1687,7 +1968,7 @@ class DownloadQueue:
|
|||||||
info.status = 'error'
|
info.status = 'error'
|
||||||
info.msg = msg
|
info.msg = msg
|
||||||
download = Download(None, None, None, None, quality, format, {}, info)
|
download = Download(None, None, None, None, quality, format, {}, info)
|
||||||
self.done.put(download)
|
await self.done.put(download)
|
||||||
await self.notifier.completed(info)
|
await self.notifier.completed(info)
|
||||||
|
|
||||||
async def add(
|
async def add(
|
||||||
@@ -1712,13 +1993,14 @@ class DownloadQueue:
|
|||||||
already=None,
|
already=None,
|
||||||
_add_gen=None,
|
_add_gen=None,
|
||||||
retry_entry=None,
|
retry_entry=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if ytdl_options_presets is None:
|
if ytdl_options_presets is None:
|
||||||
ytdl_options_presets = []
|
ytdl_options_presets = []
|
||||||
log.info(
|
log.info(
|
||||||
f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} '
|
f'adding {url}: {download_type=} {codec=} {format=} {quality=} {already=} {folder=} {custom_name_prefix=} '
|
||||||
f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} '
|
f'{playlist_item_limit=} {auto_start=} {split_by_chapters=} {chapter_template=} '
|
||||||
f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=}'
|
f'{subtitle_language=} {subtitle_mode=} {ytdl_options_presets=} {clip_start=} {clip_end=} {sponsorblock=}'
|
||||||
)
|
)
|
||||||
if already is None:
|
if already is None:
|
||||||
_add_gen = self._add_generation
|
_add_gen = self._add_generation
|
||||||
@@ -1732,8 +2014,15 @@ class DownloadQueue:
|
|||||||
# SSRF guard: reject non-http(s) schemes and hosts resolving to
|
# SSRF guard: reject non-http(s) schemes and hosts resolving to
|
||||||
# internal/loopback/link-local/metadata addresses before yt-dlp fetches
|
# internal/loopback/link-local/metadata addresses before yt-dlp fetches
|
||||||
# anything. run_in_executor because validate_url may perform a DNS lookup.
|
# anything. run_in_executor because validate_url may perform a DNS lookup.
|
||||||
|
# The merged options decide the proxy, same as the connect-time guard
|
||||||
|
# reads `proxy` from them — a proxied fetch resolves at the proxy, so
|
||||||
|
# the address check is skipped rather than leaking the hostname here.
|
||||||
|
proxies = download_proxies(
|
||||||
|
self._build_ytdl_options(ytdl_options_presets, ytdl_options_overrides))
|
||||||
url_error = await asyncio.get_running_loop().run_in_executor(
|
url_error = await asyncio.get_running_loop().run_in_executor(
|
||||||
None, partial(validate_url, url, allow_private=self.config.ALLOW_PRIVATE_ADDRESSES))
|
None, partial(validate_url, url,
|
||||||
|
allow_private=self.config.ALLOW_PRIVATE_ADDRESSES,
|
||||||
|
proxies=proxies))
|
||||||
if url_error is not None:
|
if url_error is not None:
|
||||||
log.warning('Rejected URL "%s": %s', url, url_error)
|
log.warning('Rejected URL "%s": %s', url, url_error)
|
||||||
await self.__record_add_failure(
|
await self.__record_add_failure(
|
||||||
@@ -1781,6 +2070,7 @@ class DownloadQueue:
|
|||||||
already,
|
already,
|
||||||
_add_gen,
|
_add_gen,
|
||||||
retry_entry,
|
retry_entry,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def retry(self, id):
|
async def retry(self, id):
|
||||||
@@ -1817,6 +2107,7 @@ class DownloadQueue:
|
|||||||
info.clip_start,
|
info.clip_start,
|
||||||
info.clip_end,
|
info.clip_end,
|
||||||
retry_entry=info.entry,
|
retry_entry=info.entry,
|
||||||
|
sponsorblock=info.sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def add_entry(
|
async def add_entry(
|
||||||
@@ -1838,6 +2129,7 @@ class DownloadQueue:
|
|||||||
ytdl_options_overrides=None,
|
ytdl_options_overrides=None,
|
||||||
clip_start=None,
|
clip_start=None,
|
||||||
clip_end=None,
|
clip_end=None,
|
||||||
|
sponsorblock=False,
|
||||||
):
|
):
|
||||||
if ytdl_options_presets is None:
|
if ytdl_options_presets is None:
|
||||||
ytdl_options_presets = []
|
ytdl_options_presets = []
|
||||||
@@ -1863,17 +2155,24 @@ class DownloadQueue:
|
|||||||
clip_end,
|
clip_end,
|
||||||
already,
|
already,
|
||||||
None,
|
None,
|
||||||
|
sponsorblock=sponsorblock,
|
||||||
)
|
)
|
||||||
|
|
||||||
async def start_pending(self, ids):
|
async def start_pending(self, ids):
|
||||||
for id in ids:
|
for id in ids:
|
||||||
if self.pending.exists(id):
|
if self.pending.exists(id):
|
||||||
dl = self.pending.get(id)
|
dl = self.pending.get(id)
|
||||||
self.pending.delete(id)
|
await self.pending.delete(id)
|
||||||
if getattr(dl.info, 'live_status', None) == 'is_upcoming':
|
if getattr(dl.info, 'live_status', None) == 'is_upcoming':
|
||||||
self._schedule_upcoming_download(dl)
|
await self._schedule_upcoming_download(dl)
|
||||||
else:
|
else:
|
||||||
self.queue.put(dl)
|
dl.info.status = 'queued'
|
||||||
|
await self.queue.put(dl)
|
||||||
|
# Tell the client it moved out of 'pending' now, not when a
|
||||||
|
# slot frees up: with MAX_CONCURRENT_DOWNLOADS saturated the
|
||||||
|
# wait is unbounded, and until this lands the row still
|
||||||
|
# offers the Start button it has already outgrown.
|
||||||
|
await self.notifier.updated(dl.info)
|
||||||
bg_tasks.create_task(self.__start_download(dl), name="start_download")
|
bg_tasks.create_task(self.__start_download(dl), name="start_download")
|
||||||
continue
|
continue
|
||||||
if self.queue.exists(id):
|
if self.queue.exists(id):
|
||||||
@@ -1889,7 +2188,7 @@ class DownloadQueue:
|
|||||||
# Track URL so playlist add loop won't re-queue it
|
# Track URL so playlist add loop won't re-queue it
|
||||||
self._canceled_urls.add(id)
|
self._canceled_urls.add(id)
|
||||||
if self.pending.exists(id):
|
if self.pending.exists(id):
|
||||||
self.pending.delete(id)
|
await self.pending.delete(id)
|
||||||
await self.notifier.canceled(id)
|
await self.notifier.canceled(id)
|
||||||
continue
|
continue
|
||||||
if not self.queue.exists(id):
|
if not self.queue.exists(id):
|
||||||
@@ -1902,7 +2201,7 @@ class DownloadQueue:
|
|||||||
dl.cancel()
|
dl.cancel()
|
||||||
else:
|
else:
|
||||||
dl.canceled = True
|
dl.canceled = True
|
||||||
self.queue.delete(id)
|
await self.queue.delete(id)
|
||||||
await self.notifier.canceled(id)
|
await self.notifier.canceled(id)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
|
|
||||||
@@ -1940,7 +2239,7 @@ class DownloadQueue:
|
|||||||
pass
|
pass
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
log.warning(f'deleting file "{rel_name}" for download {id} failed with error message {e!r}')
|
log.warning(f'deleting file "{rel_name}" for download {id} failed with error message {e!r}')
|
||||||
self.done.delete(id)
|
await self.done.delete(id)
|
||||||
await self.notifier.cleared(id)
|
await self.notifier.cleared(id)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
|
|
||||||
@@ -1958,3 +2257,7 @@ class DownloadQueue:
|
|||||||
if download.started() and download.running():
|
if download.started() and download.running():
|
||||||
download.cancel()
|
download.cancel()
|
||||||
self._download_executor.shutdown(wait=False, cancel_futures=True)
|
self._download_executor.shutdown(wait=False, cancel_futures=True)
|
||||||
|
# Unlike the download executor these are drained, not cancelled: a
|
||||||
|
# queued write is the newest state and must reach disk before exit.
|
||||||
|
for queue in (self.queue, self.pending, self.done):
|
||||||
|
queue.close()
|
||||||
|
|||||||
+23
-23
@@ -24,41 +24,41 @@
|
|||||||
"packageManager": "pnpm@11.5.2",
|
"packageManager": "pnpm@11.5.2",
|
||||||
"private": true,
|
"private": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@angular/animations": "^22.0.6",
|
"@angular/animations": "^22.1.6",
|
||||||
"@angular/common": "^22.0.6",
|
"@angular/common": "^22.1.6",
|
||||||
"@angular/compiler": "^22.0.6",
|
"@angular/compiler": "^22.1.6",
|
||||||
"@angular/core": "^22.0.6",
|
"@angular/core": "^22.1.6",
|
||||||
"@angular/forms": "^22.0.6",
|
"@angular/forms": "^22.1.6",
|
||||||
"@angular/platform-browser": "^22.0.6",
|
"@angular/platform-browser": "^22.1.6",
|
||||||
"@angular/platform-browser-dynamic": "^22.0.6",
|
"@angular/platform-browser-dynamic": "^22.1.6",
|
||||||
"@angular/service-worker": "^22.0.6",
|
"@angular/service-worker": "^22.1.6",
|
||||||
"@fortawesome/angular-fontawesome": "~4.0.0",
|
"@fortawesome/angular-fontawesome": "~5.1.0",
|
||||||
"@fortawesome/fontawesome-svg-core": "^7.3.1",
|
"@fortawesome/fontawesome-svg-core": "^7.3.1",
|
||||||
"@fortawesome/free-brands-svg-icons": "^7.3.1",
|
"@fortawesome/free-brands-svg-icons": "^7.3.1",
|
||||||
"@fortawesome/free-regular-svg-icons": "^7.3.1",
|
"@fortawesome/free-regular-svg-icons": "^7.3.1",
|
||||||
"@fortawesome/free-solid-svg-icons": "^7.3.1",
|
"@fortawesome/free-solid-svg-icons": "^7.3.1",
|
||||||
"@ng-bootstrap/ng-bootstrap": "^21.0.0",
|
"@ng-bootstrap/ng-bootstrap": "^21.0.0",
|
||||||
"@ng-select/ng-select": "^23.2.0",
|
"@ng-select/ng-select": "^24.1.2",
|
||||||
"@popperjs/core": "^2.11.8",
|
"@popperjs/core": "^2.11.8",
|
||||||
"bootstrap": "^5.3.8",
|
"bootstrap": "^5.3.8",
|
||||||
"ngx-cookie-service": "^22.0.0",
|
"ngx-cookie-service": "^22.0.0",
|
||||||
"ngx-socket-io": "~4.10.0",
|
"ngx-socket-io": "~4.11.1",
|
||||||
"rxjs": "~7.8.2",
|
"rxjs": "~7.8.2",
|
||||||
"tslib": "^2.8.1",
|
"tslib": "^2.8.1",
|
||||||
"zone.js": "0.15.0"
|
"zone.js": "0.16.3"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@angular-eslint/builder": "22.0.0",
|
"@angular-eslint/builder": "22.5.0",
|
||||||
"@angular/build": "^22.0.7",
|
"@angular/build": "^22.1.8",
|
||||||
"@angular/cli": "^22.0.7",
|
"@angular/cli": "^22.1.8",
|
||||||
"@angular/compiler-cli": "^22.0.6",
|
"@angular/compiler-cli": "^22.1.6",
|
||||||
"@angular/localize": "^22.0.6",
|
"@angular/localize": "^22.1.6",
|
||||||
"@eslint/js": "^9.39.5",
|
"@eslint/js": "^10.0.1",
|
||||||
"angular-eslint": "22.0.0",
|
"angular-eslint": "22.5.0",
|
||||||
"eslint": "^9.39.5",
|
"eslint": "^10.10.0",
|
||||||
"jsdom": "^27.4.0",
|
"jsdom": "^30.0.1",
|
||||||
"typescript": "~6.0.3",
|
"typescript": "~6.0.3",
|
||||||
"typescript-eslint": "8.62.0",
|
"typescript-eslint": "8.70.0",
|
||||||
"vitest": "^4.1.10"
|
"vitest": "^4.1.11"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
+2358
-2678
File diff suppressed because it is too large
Load Diff
+421
-381
@@ -399,6 +399,16 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="col-12">
|
<div class="col-12">
|
||||||
<div class="row g-2 align-items-center">
|
<div class="row g-2 align-items-center">
|
||||||
|
<div class="col-auto">
|
||||||
|
<div class="form-check form-switch">
|
||||||
|
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-sponsorblock"
|
||||||
|
name="sponsorblock" [(ngModel)]="sponsorblock" (change)="sponsorblockChanged()"
|
||||||
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
|
<label class="form-check-label" for="checkbox-sponsorblock"
|
||||||
|
ngbPopover="Cut out sponsor segments using SponsorBlock's crowd-sourced markers (YouTube only)."
|
||||||
|
triggers="hover" container="body">Remove sponsor segments</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div class="col-auto">
|
<div class="col-auto">
|
||||||
<div class="form-check form-switch">
|
<div class="form-check form-switch">
|
||||||
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
||||||
@@ -680,397 +690,427 @@
|
|||||||
Connecting to server...
|
Connecting to server...
|
||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
<div class="metube-section-header">Downloading</div>
|
<div class="metube-section-header">
|
||||||
<div class="px-2 py-3 border-bottom">
|
<button type="button" class="metube-section-toggle" (click)="toggleDownloadingCollapsed()" [attr.aria-expanded]="!downloadingCollapsed">
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled #queueDelSelected (click)="delSelectedDownloads('queue')"><fa-icon [icon]="faTrashAlt" /> Cancel selected</button>
|
<span>Downloading</span>
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled #queueDownloadSelected (click)="startSelectedDownloads('queue')"><fa-icon [icon]="faDownload" /> Download selected</button>
|
<fa-icon [icon]="downloadingCollapsed ? faChevronRight : faChevronDown" class="metube-section-chevron" />
|
||||||
</div>
|
|
||||||
<div class="overflow-auto">
|
|
||||||
<table class="table">
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th scope="col" style="width: 1rem;">
|
|
||||||
<app-select-all-checkbox #queueMasterCheckboxRef [id]="'queue'" [list]="downloads.queue" (changed)="queueSelectionChanged($event)" />
|
|
||||||
</th>
|
|
||||||
<th scope="col">Video</th>
|
|
||||||
<th scope="col" style="width: 8rem;">Speed</th>
|
|
||||||
<th scope="col" style="width: 7rem;">ETA</th>
|
|
||||||
<th scope="col" style="width: 6rem;"></th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
@for (download of downloads.queue | keyvalue: asIsOrder; track download.key) {
|
|
||||||
<tr [class.disabled]='download.value.deleting'>
|
|
||||||
<td>
|
|
||||||
<app-item-checkbox [id]="download.key" [master]="queueMasterCheckboxRef" [checkable]="download.value" />
|
|
||||||
</td>
|
|
||||||
<td title="{{ download.value.filename }}">
|
|
||||||
<div class="d-flex flex-column flex-sm-row align-items-center row-gap-2 column-gap-3">
|
|
||||||
<div class="d-flex align-items-center flex-wrap gap-2">
|
|
||||||
<span>{{ download.value.title }}</span>
|
|
||||||
@if (download.value.live_status === 'is_live' && download.value.status !== 'scheduled') {
|
|
||||||
<span class="badge bg-danger">LIVE</span>
|
|
||||||
}
|
|
||||||
</div>
|
|
||||||
@if (download.value.status === 'scheduled') {
|
|
||||||
<span class="badge bg-warning text-dark">
|
|
||||||
<fa-icon [icon]="faClock" />
|
|
||||||
Waiting for stream
|
|
||||||
@if (liveCountdownSeconds(download.value); as secs) {
|
|
||||||
- starts in {{ secs | eta }}
|
|
||||||
}
|
|
||||||
</span>
|
|
||||||
} @else {
|
|
||||||
<ngb-progressbar height="1.5rem" [showValue]="download.value.status !== 'preparing'" [striped]="download.value.status === 'preparing'" [animated]="download.value.status === 'preparing'" type="success"
|
|
||||||
[value]="download.value.status === 'preparing' ? 100 : download.value.percent" class="download-progressbar" />
|
|
||||||
}
|
|
||||||
</div>
|
|
||||||
</td>
|
|
||||||
<td>{{ download.value.speed | speed }}</td>
|
|
||||||
<td>{{ download.value.eta | eta }}</td>
|
|
||||||
<td>
|
|
||||||
<div class="d-flex">
|
|
||||||
@if (download.value.status === 'pending' || download.value.status === 'scheduled') {
|
|
||||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Start download for ' + download.value.title" (click)="downloadItemByKey(download.key)"><fa-icon [icon]="faDownload" /></button>
|
|
||||||
}
|
|
||||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Remove ' + download.value.title + ' from queue'" (click)="delDownload('queue', download.key)"><fa-icon [icon]="faTrashAlt" /></button>
|
|
||||||
<a href="{{download.value.url}}" target="_blank" class="btn btn-link" [attr.aria-label]="'Open source URL for ' + download.value.title"><fa-icon [icon]="faExternalLinkAlt" /></a>
|
|
||||||
</div>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="metube-section-header">Completed</div>
|
|
||||||
<div class="px-2 py-3 border-bottom">
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" (click)="toggleSortOrder()" ngbTooltip="{{ sortAscending ? 'Oldest first' : 'Newest first' }}"><fa-icon [icon]="sortAscending ? faSortAmountUp : faSortAmountDown" /> {{ sortAscending ? 'Oldest first' : 'Newest first' }}</button>
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled #doneDelSelected (click)="delSelectedDownloads('done')"><fa-icon [icon]="faTrashAlt" /> Clear selected</button>
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" [disabled]="!hasCompletedDone" (click)="clearCompletedDownloads()"><fa-icon [icon]="faCheckCircle" /> Clear completed</button>
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" [disabled]="!hasFailedDone" (click)="clearFailedDownloads()"><fa-icon [icon]="faTimesCircle" /> Clear failed</button>
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" [disabled]="!hasFailedDone" (click)="retryFailedDownloads()"><fa-icon [icon]="faRedoAlt" /> Retry failed</button>
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled #doneDownloadSelected (click)="downloadSelectedFiles()"><fa-icon [icon]="faDownload" /> Download Selected</button>
|
|
||||||
</div>
|
|
||||||
<div class="overflow-auto">
|
|
||||||
<table class="table">
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th scope="col" style="width: 1rem;">
|
|
||||||
<app-select-all-checkbox #doneMasterCheckboxRef [id]="'done'" [list]="downloads.done" (changed)="doneSelectionChanged($event)" />
|
|
||||||
</th>
|
|
||||||
<th scope="col">Video</th>
|
|
||||||
<th scope="col">Type</th>
|
|
||||||
<th scope="col">Quality</th>
|
|
||||||
<th scope="col">Codec / Format</th>
|
|
||||||
<th scope="col">File Size</th>
|
|
||||||
<th scope="col">Downloaded</th>
|
|
||||||
<th scope="col" style="width: 8rem;"></th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
@for (entry of cachedSortedDone; track entry[0]) {
|
|
||||||
<tr [class.disabled]='entry[1].deleting'>
|
|
||||||
<td>
|
|
||||||
<app-item-checkbox [id]="entry[0]" [master]="doneMasterCheckboxRef" [checkable]="entry[1]" />
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
<div style="display: inline-block; width: 1.5rem;">
|
|
||||||
@if (entry[1].status === 'finished') {
|
|
||||||
<fa-icon [icon]="faCheckCircle" class="text-success" />
|
|
||||||
}
|
|
||||||
@if (entry[1].status === 'error') {
|
|
||||||
<button type="button" class="btn btn-link p-0"
|
|
||||||
(click)="toggleErrorDetail(entry[0])"
|
|
||||||
[attr.aria-label]="'Toggle error details for ' + entry[1].title"
|
|
||||||
[attr.aria-expanded]="isErrorExpanded(entry[0])">
|
|
||||||
<fa-icon [icon]="faTimesCircle" class="text-danger" />
|
|
||||||
</button>
|
|
||||||
}
|
|
||||||
</div>
|
|
||||||
<span ngbTooltip="{{buildResultItemTooltip(entry[1])}}">@if (!!entry[1].filename) {
|
|
||||||
<a href="{{buildDownloadLink(entry[1])}}" target="_blank">{{ entry[1].title }}</a>
|
|
||||||
} @else {
|
|
||||||
@if (entry[1].status === 'error') {
|
|
||||||
<button type="button" class="btn btn-link p-0 text-start align-baseline" (click)="toggleErrorDetail(entry[0])">
|
|
||||||
{{entry[1].title}}
|
|
||||||
@if (!isErrorExpanded(entry[0])) {
|
|
||||||
<small class="text-danger ms-2">
|
|
||||||
<fa-icon [icon]="faChevronRight" size="xs" class="me-1" />Click for details
|
|
||||||
</small>
|
|
||||||
}
|
|
||||||
</button>
|
|
||||||
} @else {
|
|
||||||
<span>{{entry[1].title}}</span>
|
|
||||||
}
|
|
||||||
}</span>
|
|
||||||
@if (entry[1].status === 'error' && isErrorExpanded(entry[0])) {
|
|
||||||
<div class="alert alert-danger py-2 px-3 mt-2 mb-0 small" style="border-left: 4px solid var(--bs-danger);">
|
|
||||||
<div class="d-flex justify-content-between align-items-start">
|
|
||||||
<div class="flex-grow-1">
|
|
||||||
@if (entry[1].msg) {
|
|
||||||
<div class="mb-1"><strong>Message:</strong> {{entry[1].msg}}</div>
|
|
||||||
}
|
|
||||||
@if (entry[1].error) {
|
|
||||||
<div class="mb-1"><strong>Error:</strong> {{entry[1].error}}</div>
|
|
||||||
}
|
|
||||||
<div class="text-muted" style="word-break: break-all;"><strong>URL:</strong> {{entry[1].url}}</div>
|
|
||||||
</div>
|
|
||||||
<button type="button" class="btn btn-sm btn-outline-danger ms-2 flex-shrink-0"
|
|
||||||
(click)="copyErrorMessage(entry[0], entry[1]); $event.stopPropagation()"
|
|
||||||
ngbTooltip="Copy error details to clipboard">
|
|
||||||
@if (lastCopiedErrorId === entry[0]) {
|
|
||||||
<span class="text-success">Copied!</span>
|
|
||||||
} @else {
|
|
||||||
<fa-icon [icon]="faCopy" />
|
|
||||||
}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
}
|
|
||||||
</td>
|
|
||||||
<td class="text-nowrap">
|
|
||||||
{{ downloadTypeLabel(entry[1]) }}
|
|
||||||
</td>
|
|
||||||
<td class="text-nowrap">
|
|
||||||
{{ formatQualityLabel(entry[1]) }}
|
|
||||||
</td>
|
|
||||||
<td class="text-nowrap">
|
|
||||||
{{ formatCodecLabel(entry[1]) }}
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
@if (entry[1].size) {
|
|
||||||
<span>{{ entry[1].size | fileSize }}</span>
|
|
||||||
}
|
|
||||||
</td>
|
|
||||||
<td class="text-nowrap">
|
|
||||||
@if (entry[1].timestamp) {
|
|
||||||
<span>{{ entry[1].timestamp / 1000000 | date:'yyyy-MM-dd HH:mm' }}</span>
|
|
||||||
}
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
<div class="d-flex">
|
|
||||||
@if (entry[1].status === 'error') {
|
|
||||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Retry download for ' + entry[1].title" (click)="retryDownload(entry[0], entry[1])"><fa-icon [icon]="faRedoAlt" /></button>
|
|
||||||
}
|
|
||||||
@if (entry[1].filename) {
|
|
||||||
<a href="{{buildDownloadLink(entry[1])}}" download class="btn btn-link" [attr.aria-label]="'Download result file for ' + entry[1].title"><fa-icon [icon]="faDownload" /></a>
|
|
||||||
}
|
|
||||||
@if (entry[1].filename && canShareDownloads()) {
|
|
||||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Share result file for ' + entry[1].title" (click)="shareDownload(entry[1])"><fa-icon [icon]="faShareNodes" /></button>
|
|
||||||
}
|
|
||||||
<a href="{{entry[1].url}}" target="_blank" class="btn btn-link" [attr.aria-label]="'Open source URL for ' + entry[1].title"><fa-icon [icon]="faExternalLinkAlt" /></a>
|
|
||||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Delete completed item ' + entry[1].title" (click)="delDownload('done', entry[0])"><fa-icon [icon]="faTrashAlt" /></button>
|
|
||||||
</div>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
@if (entry[1].chapter_files && entry[1].chapter_files.length > 0) {
|
|
||||||
@for (chapterFile of entry[1].chapter_files; track chapterFile.filename) {
|
|
||||||
<tr [class.disabled]='entry[1].deleting'>
|
|
||||||
<td></td>
|
|
||||||
<td>
|
|
||||||
<div style="padding-left: 2rem;">
|
|
||||||
<fa-icon [icon]="faCheckCircle" class="text-success me-2" />
|
|
||||||
<a href="{{buildChapterDownloadLink(entry[1], chapterFile.filename)}}" target="_blank" [attr.aria-label]="'Open chapter file ' + getChapterFileName(chapterFile.filename)">{{
|
|
||||||
getChapterFileName(chapterFile.filename) }}</a>
|
|
||||||
</div>
|
|
||||||
</td>
|
|
||||||
<td></td>
|
|
||||||
<td></td>
|
|
||||||
<td></td>
|
|
||||||
<td>
|
|
||||||
@if (chapterFile.size) {
|
|
||||||
<span>{{ chapterFile.size | fileSize }}</span>
|
|
||||||
}
|
|
||||||
</td>
|
|
||||||
<td></td>
|
|
||||||
<td>
|
|
||||||
<div class="d-flex">
|
|
||||||
<a href="{{buildChapterDownloadLink(entry[1], chapterFile.filename)}}" download [attr.aria-label]="'Download chapter file ' + getChapterFileName(chapterFile.filename)"
|
|
||||||
class="btn btn-link"><fa-icon [icon]="faDownload" /></a>
|
|
||||||
</div>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="metube-section-header">Subscriptions</div>
|
|
||||||
<div class="px-2 py-3 border-bottom">
|
|
||||||
@if (checkingAllSubscriptions) {
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled>
|
|
||||||
<span class="spinner-border spinner-border-sm me-2" role="status" aria-hidden="true"></span>Check all now
|
|
||||||
</button>
|
|
||||||
} @else {
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4"
|
|
||||||
(click)="checkAllSubscriptions()"
|
|
||||||
[disabled]="downloads.loading || cachedSubs.length === 0 || checkingSelectedSubscriptions">
|
|
||||||
<fa-icon [icon]="faRedoAlt" /> Check all now
|
|
||||||
</button>
|
|
||||||
}
|
|
||||||
@if (checkingSelectedSubscriptions) {
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled>
|
|
||||||
<span class="spinner-border spinner-border-sm me-2" role="status" aria-hidden="true"></span>Check selected
|
|
||||||
</button>
|
|
||||||
} @else {
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4"
|
|
||||||
(click)="checkSelectedSubscriptions()"
|
|
||||||
[disabled]="downloads.loading || selectedSubscriptionIds.size === 0 || checkingAllSubscriptions">
|
|
||||||
<fa-icon [icon]="faRedoAlt" /> Check selected
|
|
||||||
</button>
|
|
||||||
}
|
|
||||||
<button type="button" class="btn btn-link text-decoration-none px-0 me-4"
|
|
||||||
(click)="deleteSelectedSubscriptions()"
|
|
||||||
[disabled]="downloads.loading || selectedSubscriptionIds.size === 0">
|
|
||||||
<fa-icon [icon]="faTrashAlt" /> Delete selected
|
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
<div class="overflow-auto">
|
@if (!downloadingCollapsed) {
|
||||||
<table class="table">
|
<div class="px-2 py-3 border-bottom">
|
||||||
<thead>
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled #queueDelSelected (click)="delSelectedDownloads('queue')"><fa-icon [icon]="faTrashAlt" /> Cancel selected</button>
|
||||||
<tr>
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled #queueDownloadSelected (click)="startSelectedDownloads('queue')"><fa-icon [icon]="faDownload" /> Download selected</button>
|
||||||
<th scope="col" style="width: 1rem;">
|
</div>
|
||||||
<input type="checkbox" class="form-check-input"
|
<div class="overflow-auto">
|
||||||
[checked]="allSubsSelected()"
|
<table class="table">
|
||||||
(change)="toggleSubMaster($event)"
|
<thead>
|
||||||
[disabled]="downloads.loading || cachedSubs.length === 0"
|
|
||||||
aria-label="Select all subscriptions" />
|
|
||||||
</th>
|
|
||||||
<th scope="col">Name</th>
|
|
||||||
<th scope="col">URL</th>
|
|
||||||
<th scope="col" class="text-nowrap"><span class="help-title" ngbPopover="Subscriptions only — which new video titles to queue when this feed is checked. Does not affect manual downloads." triggers="click" autoClose="outside" container="body">Filter</span></th>
|
|
||||||
<th scope="col" class="text-nowrap">Interval (min)</th>
|
|
||||||
<th scope="col" class="text-nowrap">Last checked</th>
|
|
||||||
<th scope="col">Status</th>
|
|
||||||
<th scope="col" style="width: 8rem;"></th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
@for (entry of cachedSubs; track entry[0]) {
|
|
||||||
<tr>
|
<tr>
|
||||||
<td>
|
<th scope="col" style="width: 1rem;">
|
||||||
<input type="checkbox" class="form-check-input"
|
<app-select-all-checkbox #queueMasterCheckboxRef [id]="'queue'" [list]="downloads.queue" (changed)="queueSelectionChanged($event)" />
|
||||||
[checked]="isSubSelected(entry[0])"
|
</th>
|
||||||
(change)="toggleSubSelected(entry[0])"
|
<th scope="col">Video</th>
|
||||||
[disabled]="downloads.loading"
|
<th scope="col" style="width: 7rem;">Format</th>
|
||||||
[attr.aria-label]="'Select subscription ' + entry[1].name" />
|
<th scope="col" style="width: 8rem;">Speed</th>
|
||||||
</td>
|
<th scope="col" style="width: 7rem;">ETA</th>
|
||||||
<td>
|
<th scope="col" style="width: 6rem;"></th>
|
||||||
@if (editingNameId === entry[0]) {
|
</tr>
|
||||||
<div class="d-flex flex-wrap gap-1 align-items-center">
|
</thead>
|
||||||
<input type="text"
|
<tbody>
|
||||||
class="form-control form-control-sm flex-grow-1"
|
@for (download of downloads.queue | keyvalue: asIsOrder; track download.key) {
|
||||||
[name]="'subName' + entry[0]"
|
<tr [class.disabled]='download.value.deleting'>
|
||||||
[(ngModel)]="nameEditDraft"
|
<td>
|
||||||
[maxlength]="subscriptionNameMaxLength"
|
<app-item-checkbox [id]="download.key" [master]="queueMasterCheckboxRef" [checkable]="download.value" />
|
||||||
[disabled]="downloads.loading"
|
</td>
|
||||||
[attr.aria-label]="'Subscription name for ' + entry[1].name" />
|
<td title="{{ download.value.filename }}">
|
||||||
<button type="button" class="btn btn-sm btn-outline-secondary"
|
<div class="d-flex flex-column flex-sm-row align-items-center row-gap-2 column-gap-3">
|
||||||
(click)="saveName(entry[0])"
|
<div class="d-flex align-items-center flex-wrap gap-2">
|
||||||
[disabled]="downloads.loading">Save</button>
|
<span>{{ download.value.title }}</span>
|
||||||
<button type="button" class="btn btn-sm btn-outline-secondary"
|
@if (download.value.live_status === 'is_live' && download.value.status !== 'scheduled') {
|
||||||
(click)="cancelEditName()"
|
<span class="badge bg-danger">LIVE</span>
|
||||||
[disabled]="downloads.loading">Cancel</button>
|
}
|
||||||
</div>
|
@if (download.value.status === 'queued') {
|
||||||
} @else {
|
<span class="badge bg-secondary">Queued</span>
|
||||||
<div class="d-flex flex-wrap gap-1 align-items-center">
|
}
|
||||||
<span class="text-break">{{ entry[1].name }}</span>
|
</div>
|
||||||
<button type="button" class="btn btn-link btn-sm p-0"
|
@if (download.value.status === 'scheduled') {
|
||||||
(click)="beginEditName(entry[0], entry[1].name)"
|
<span class="badge bg-warning text-dark">
|
||||||
[disabled]="downloads.loading"
|
<fa-icon [icon]="faClock" />
|
||||||
ngbTooltip="Rename this subscription (display name only; does not affect the download folder)">Edit</button>
|
Waiting for stream
|
||||||
</div>
|
@if (liveCountdownSeconds(download.value); as secs) {
|
||||||
}
|
- starts in {{ secs | eta }}
|
||||||
</td>
|
}
|
||||||
<td class="text-break"><a [href]="entry[1].url" target="_blank" rel="noopener">{{ entry[1].url }}</a></td>
|
</span>
|
||||||
<td>
|
|
||||||
@if (editingTitleRegexId === entry[0]) {
|
|
||||||
<div class="d-flex flex-wrap gap-1 align-items-center">
|
|
||||||
<input type="text"
|
|
||||||
class="form-control form-control-sm flex-grow-1"
|
|
||||||
[name]="'subTitleRegex' + entry[0]"
|
|
||||||
[(ngModel)]="titleRegexEditDraft"
|
|
||||||
[disabled]="downloads.loading" />
|
|
||||||
<button type="button" class="btn btn-sm btn-outline-secondary"
|
|
||||||
(click)="saveTitleRegex(entry[0])"
|
|
||||||
[disabled]="downloads.loading">Save</button>
|
|
||||||
<button type="button" class="btn btn-sm btn-outline-secondary"
|
|
||||||
(click)="cancelEditTitleRegex()"
|
|
||||||
[disabled]="downloads.loading">Cancel</button>
|
|
||||||
</div>
|
|
||||||
} @else {
|
|
||||||
<div class="d-flex flex-wrap gap-1 align-items-center">
|
|
||||||
<span class="text-muted small text-break"
|
|
||||||
[class.text-secondary]="!entry[1].title_regex">{{ entry[1].title_regex || '—' }}</span>
|
|
||||||
<button type="button" class="btn btn-link btn-sm p-0"
|
|
||||||
(click)="beginEditTitleRegex(entry[0], entry[1].title_regex)"
|
|
||||||
[disabled]="downloads.loading"
|
|
||||||
ngbTooltip="Edit subscription title filter (subscriptions only; not for one-off downloads)">Edit</button>
|
|
||||||
</div>
|
|
||||||
}
|
|
||||||
</td>
|
|
||||||
<td>{{ entry[1].check_interval_minutes }}</td>
|
|
||||||
<td class="text-nowrap">
|
|
||||||
@if (entry[1].last_checked !== null) {
|
|
||||||
<span>{{ entry[1].last_checked! * 1000 | date:'yyyy-MM-dd HH:mm:ss' }}</span>
|
|
||||||
} @else {
|
|
||||||
<span class="text-muted">—</span>
|
|
||||||
}
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
@if (entry[1].error) {
|
|
||||||
<span class="text-danger small">{{ entry[1].error }}</span>
|
|
||||||
} @else if (entry[1].enabled) {
|
|
||||||
<span class="text-success">Active</span>
|
|
||||||
} @else {
|
|
||||||
<span class="text-secondary">Paused</span>
|
|
||||||
}
|
|
||||||
</td>
|
|
||||||
<td>
|
|
||||||
<div class="d-flex flex-wrap gap-1">
|
|
||||||
@if (isSubscriptionChecking(entry[0])) {
|
|
||||||
<button type="button" class="btn btn-link btn-sm p-0 me-2"
|
|
||||||
disabled
|
|
||||||
[attr.aria-label]="'Checking ' + entry[1].name"
|
|
||||||
ngbTooltip="Checking now">
|
|
||||||
<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span>
|
|
||||||
</button>
|
|
||||||
} @else {
|
|
||||||
<button type="button" class="btn btn-link btn-sm p-0 me-2"
|
|
||||||
(click)="checkSubscriptionNow(entry[0])"
|
|
||||||
[disabled]="downloads.loading"
|
|
||||||
[attr.aria-label]="'Check now ' + entry[1].name"
|
|
||||||
ngbTooltip="Check now">
|
|
||||||
<fa-icon [icon]="faRedoAlt" />
|
|
||||||
</button>
|
|
||||||
}
|
|
||||||
<button type="button" class="btn btn-link btn-sm p-0 me-2"
|
|
||||||
(click)="toggleSubscriptionEnabled(entry[1])"
|
|
||||||
[disabled]="downloads.loading"
|
|
||||||
[attr.aria-label]="(entry[1].enabled ? 'Pause ' : 'Resume ') + entry[1].name"
|
|
||||||
[ngbTooltip]="entry[1].enabled ? 'Pause' : 'Resume'">
|
|
||||||
@if (entry[1].enabled) {
|
|
||||||
<fa-icon [icon]="faPause" />
|
|
||||||
} @else {
|
} @else {
|
||||||
<fa-icon [icon]="faPlay" />
|
<ngb-progressbar height="1.5rem" [showValue]="!isIndeterminate(download.value)" [striped]="isIndeterminate(download.value)" [animated]="isIndeterminate(download.value)" type="success"
|
||||||
|
[value]="isIndeterminate(download.value) ? 100 : download.value.percent" class="download-progressbar">
|
||||||
|
@if (download.value.status === 'postprocessing') {
|
||||||
|
<span>Post-processing</span>
|
||||||
|
}
|
||||||
|
</ngb-progressbar>
|
||||||
}
|
}
|
||||||
</button>
|
</div>
|
||||||
<button type="button" class="btn btn-link btn-sm p-0 text-danger"
|
</td>
|
||||||
(click)="deleteSubscription(entry[0])"
|
<td class="text-nowrap">{{ formatLabel(download.value) }}</td>
|
||||||
[disabled]="downloads.loading"
|
<td>{{ download.value.speed | speed }}</td>
|
||||||
[attr.aria-label]="'Delete subscription ' + entry[1].name">
|
<td>{{ download.value.eta | eta }}</td>
|
||||||
<fa-icon [icon]="faTrashAlt" />
|
<td>
|
||||||
</button>
|
<div class="d-flex justify-content-end">
|
||||||
|
@if (download.value.status === 'pending' || download.value.status === 'scheduled') {
|
||||||
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Start download for ' + download.value.title" (click)="downloadItemByKey(download.key)"><fa-icon [icon]="faDownload" /></button>
|
||||||
|
}
|
||||||
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Remove ' + download.value.title + ' from queue'" (click)="delDownload('queue', download.key)"><fa-icon [icon]="faTrashAlt" /></button>
|
||||||
|
<a href="{{download.value.url}}" target="_blank" class="btn btn-link" [attr.aria-label]="'Open source URL for ' + download.value.title"><fa-icon [icon]="faExternalLinkAlt" /></a>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
|
||||||
|
<div class="metube-section-header">
|
||||||
|
<button type="button" class="metube-section-toggle" (click)="toggleCompletedCollapsed()" [attr.aria-expanded]="!completedCollapsed">
|
||||||
|
<span>Completed</span>
|
||||||
|
<fa-icon [icon]="completedCollapsed ? faChevronRight : faChevronDown" class="metube-section-chevron" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
@if (!completedCollapsed) {
|
||||||
|
<div class="px-2 py-3 border-bottom">
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" (click)="toggleSortOrder()" ngbTooltip="{{ sortAscending ? 'Oldest first' : 'Newest first' }}"><fa-icon [icon]="sortAscending ? faSortAmountUp : faSortAmountDown" /> {{ sortAscending ? 'Oldest first' : 'Newest first' }}</button>
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled #doneDelSelected (click)="delSelectedDownloads('done')"><fa-icon [icon]="faTrashAlt" /> Clear selected</button>
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" [disabled]="!hasCompletedDone" (click)="clearCompletedDownloads()"><fa-icon [icon]="faCheckCircle" /> Clear completed</button>
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" [disabled]="!hasFailedDone" (click)="clearFailedDownloads()"><fa-icon [icon]="faTimesCircle" /> Clear failed</button>
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" [disabled]="!hasFailedDone" (click)="retryFailedDownloads()"><fa-icon [icon]="faRedoAlt" /> Retry failed</button>
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled #doneDownloadSelected (click)="downloadSelectedFiles()"><fa-icon [icon]="faDownload" /> Download Selected</button>
|
||||||
|
</div>
|
||||||
|
<div class="overflow-auto">
|
||||||
|
<table class="table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th scope="col" style="width: 1rem;">
|
||||||
|
<app-select-all-checkbox #doneMasterCheckboxRef [id]="'done'" [list]="downloads.done" [orderedIds]="cachedSortedDoneIds" (changed)="doneSelectionChanged($event)" />
|
||||||
|
</th>
|
||||||
|
<th scope="col">Video</th>
|
||||||
|
<th scope="col">Type</th>
|
||||||
|
<th scope="col">Quality</th>
|
||||||
|
<th scope="col">Codec / Format</th>
|
||||||
|
<th scope="col">File Size</th>
|
||||||
|
<th scope="col">Downloaded</th>
|
||||||
|
<th scope="col" style="width: 8rem;"></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
@for (entry of cachedSortedDone; track entry[0]) {
|
||||||
|
<tr [class.disabled]='entry[1].deleting'>
|
||||||
|
<td>
|
||||||
|
<app-item-checkbox [id]="entry[0]" [master]="doneMasterCheckboxRef" [checkable]="entry[1]" />
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<div style="display: inline-block; width: 1.5rem;">
|
||||||
|
@if (entry[1].status === 'finished') {
|
||||||
|
<fa-icon [icon]="faCheckCircle" class="text-success" />
|
||||||
|
}
|
||||||
|
@if (entry[1].status === 'error') {
|
||||||
|
<button type="button" class="btn btn-link p-0"
|
||||||
|
(click)="toggleErrorDetail(entry[0])"
|
||||||
|
[attr.aria-label]="'Toggle error details for ' + entry[1].title"
|
||||||
|
[attr.aria-expanded]="isErrorExpanded(entry[0])">
|
||||||
|
<fa-icon [icon]="faTimesCircle" class="text-danger" />
|
||||||
|
</button>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
<span ngbTooltip="{{buildResultItemTooltip(entry[1])}}">@if (!!entry[1].filename) {
|
||||||
|
<a href="{{buildDownloadLink(entry[1])}}" target="_blank">{{ entry[1].title }}</a>
|
||||||
|
} @else {
|
||||||
|
@if (entry[1].status === 'error') {
|
||||||
|
<button type="button" class="btn btn-link p-0 text-start align-baseline" (click)="toggleErrorDetail(entry[0])">
|
||||||
|
{{entry[1].title}}
|
||||||
|
@if (!isErrorExpanded(entry[0])) {
|
||||||
|
<small class="text-danger ms-2">
|
||||||
|
<fa-icon [icon]="faChevronRight" size="xs" class="me-1" />Click for details
|
||||||
|
</small>
|
||||||
|
}
|
||||||
|
</button>
|
||||||
|
} @else {
|
||||||
|
<span>{{entry[1].title}}</span>
|
||||||
|
}
|
||||||
|
}</span>
|
||||||
|
@if (entry[1].status === 'error' && isErrorExpanded(entry[0])) {
|
||||||
|
<div class="alert alert-danger py-2 px-3 mt-2 mb-0 small" style="border-left: 4px solid var(--bs-danger);">
|
||||||
|
<div class="d-flex justify-content-between align-items-start">
|
||||||
|
<div class="flex-grow-1">
|
||||||
|
@if (entry[1].msg) {
|
||||||
|
<div class="mb-1"><strong>Message:</strong> {{entry[1].msg}}</div>
|
||||||
|
}
|
||||||
|
@if (entry[1].error) {
|
||||||
|
<div class="mb-1"><strong>Error:</strong> {{entry[1].error}}</div>
|
||||||
|
}
|
||||||
|
<div class="text-muted" style="word-break: break-all;"><strong>URL:</strong> {{entry[1].url}}</div>
|
||||||
|
</div>
|
||||||
|
<button type="button" class="btn btn-sm btn-outline-danger ms-2 flex-shrink-0"
|
||||||
|
(click)="copyErrorMessage(entry[0], entry[1]); $event.stopPropagation()"
|
||||||
|
ngbTooltip="Copy error details to clipboard">
|
||||||
|
@if (lastCopiedErrorId === entry[0]) {
|
||||||
|
<span class="text-success">Copied!</span>
|
||||||
|
} @else {
|
||||||
|
<fa-icon [icon]="faCopy" />
|
||||||
|
}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td class="text-nowrap">
|
||||||
|
{{ downloadTypeLabel(entry[1]) }}
|
||||||
|
</td>
|
||||||
|
<td class="text-nowrap">
|
||||||
|
{{ formatQualityLabel(entry[1]) }}
|
||||||
|
</td>
|
||||||
|
<td class="text-nowrap">
|
||||||
|
{{ formatCodecLabel(entry[1]) }}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
@if (entry[1].size) {
|
||||||
|
<span>{{ entry[1].size | fileSize }}</span>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td class="text-nowrap">
|
||||||
|
@if (entry[1].timestamp) {
|
||||||
|
<span>{{ entry[1].timestamp / 1000000 | date:'yyyy-MM-dd HH:mm' }}</span>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<div class="d-flex justify-content-end">
|
||||||
|
@if (entry[1].status === 'error') {
|
||||||
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Retry download for ' + entry[1].title" (click)="retryDownload(entry[0], entry[1])"><fa-icon [icon]="faRedoAlt" /></button>
|
||||||
|
}
|
||||||
|
@if (entry[1].filename) {
|
||||||
|
<a href="{{buildDownloadLink(entry[1])}}" download class="btn btn-link" [attr.aria-label]="'Download result file for ' + entry[1].title"><fa-icon [icon]="faDownload" /></a>
|
||||||
|
}
|
||||||
|
@if (entry[1].filename && canShareDownloads()) {
|
||||||
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Share result file for ' + entry[1].title" (click)="shareDownload(entry[1])"><fa-icon [icon]="faShareNodes" /></button>
|
||||||
|
}
|
||||||
|
<a href="{{entry[1].url}}" target="_blank" class="btn btn-link" [attr.aria-label]="'Open source URL for ' + entry[1].title"><fa-icon [icon]="faExternalLinkAlt" /></a>
|
||||||
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Delete completed item ' + entry[1].title" (click)="delDownload('done', entry[0])"><fa-icon [icon]="faTrashAlt" /></button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
@if (entry[1].chapter_files && entry[1].chapter_files.length > 0) {
|
||||||
|
@for (chapterFile of entry[1].chapter_files; track chapterFile.filename) {
|
||||||
|
<tr [class.disabled]='entry[1].deleting'>
|
||||||
|
<td></td>
|
||||||
|
<td>
|
||||||
|
<div style="padding-left: 2rem;">
|
||||||
|
<fa-icon [icon]="faCheckCircle" class="text-success me-2" />
|
||||||
|
<a href="{{buildChapterDownloadLink(entry[1], chapterFile.filename)}}" target="_blank" [attr.aria-label]="'Open chapter file ' + getChapterFileName(chapterFile.filename)">{{
|
||||||
|
getChapterFileName(chapterFile.filename) }}</a>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
<td></td>
|
||||||
|
<td></td>
|
||||||
|
<td></td>
|
||||||
|
<td>
|
||||||
|
@if (chapterFile.size) {
|
||||||
|
<span>{{ chapterFile.size | fileSize }}</span>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td></td>
|
||||||
|
<td>
|
||||||
|
<div class="d-flex">
|
||||||
|
<a href="{{buildChapterDownloadLink(entry[1], chapterFile.filename)}}" download [attr.aria-label]="'Download chapter file ' + getChapterFileName(chapterFile.filename)"
|
||||||
|
class="btn btn-link"><fa-icon [icon]="faDownload" /></a>
|
||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
}
|
||||||
}
|
}
|
||||||
</tbody>
|
}
|
||||||
</table>
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
|
||||||
|
<div class="metube-section-header">
|
||||||
|
<button type="button" class="metube-section-toggle" (click)="toggleSubscriptionsCollapsed()" [attr.aria-expanded]="!subscriptionsCollapsed">
|
||||||
|
<span>Subscriptions</span>
|
||||||
|
<fa-icon [icon]="subscriptionsCollapsed ? faChevronRight : faChevronDown" class="metube-section-chevron" />
|
||||||
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
@if (!subscriptionsCollapsed) {
|
||||||
|
<div class="px-2 py-3 border-bottom">
|
||||||
|
@if (checkingAllSubscriptions) {
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled>
|
||||||
|
<span class="spinner-border spinner-border-sm me-2" role="status" aria-hidden="true"></span>Check all now
|
||||||
|
</button>
|
||||||
|
} @else {
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4"
|
||||||
|
(click)="checkAllSubscriptions()"
|
||||||
|
[disabled]="downloads.loading || cachedSubs.length === 0 || checkingSelectedSubscriptions">
|
||||||
|
<fa-icon [icon]="faRedoAlt" /> Check all now
|
||||||
|
</button>
|
||||||
|
}
|
||||||
|
@if (checkingSelectedSubscriptions) {
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4" disabled>
|
||||||
|
<span class="spinner-border spinner-border-sm me-2" role="status" aria-hidden="true"></span>Check selected
|
||||||
|
</button>
|
||||||
|
} @else {
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4"
|
||||||
|
(click)="checkSelectedSubscriptions()"
|
||||||
|
[disabled]="downloads.loading || selectedSubscriptionIds.size === 0 || checkingAllSubscriptions">
|
||||||
|
<fa-icon [icon]="faRedoAlt" /> Check selected
|
||||||
|
</button>
|
||||||
|
}
|
||||||
|
<button type="button" class="btn btn-link text-decoration-none px-0 me-4"
|
||||||
|
(click)="deleteSelectedSubscriptions()"
|
||||||
|
[disabled]="downloads.loading || selectedSubscriptionIds.size === 0">
|
||||||
|
<fa-icon [icon]="faTrashAlt" /> Delete selected
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="overflow-auto">
|
||||||
|
<table class="table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th scope="col" style="width: 1rem;">
|
||||||
|
<input type="checkbox" class="form-check-input"
|
||||||
|
[checked]="allSubsSelected()"
|
||||||
|
(change)="toggleSubMaster($event)"
|
||||||
|
[disabled]="downloads.loading || cachedSubs.length === 0"
|
||||||
|
aria-label="Select all subscriptions" />
|
||||||
|
</th>
|
||||||
|
<th scope="col">Name</th>
|
||||||
|
<th scope="col">URL</th>
|
||||||
|
<th scope="col" class="text-nowrap"><span class="help-title" ngbPopover="Subscriptions only — which new video titles to queue when this feed is checked. Does not affect manual downloads." triggers="click" autoClose="outside" container="body">Filter</span></th>
|
||||||
|
<th scope="col" class="text-nowrap">Interval (min)</th>
|
||||||
|
<th scope="col" class="text-nowrap">Last checked</th>
|
||||||
|
<th scope="col">Status</th>
|
||||||
|
<th scope="col" style="width: 8rem;"></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
@for (entry of cachedSubs; track entry[0]) {
|
||||||
|
<tr>
|
||||||
|
<td>
|
||||||
|
<input type="checkbox" class="form-check-input"
|
||||||
|
[checked]="isSubSelected(entry[0])"
|
||||||
|
(change)="toggleSubSelected(entry[0])"
|
||||||
|
[disabled]="downloads.loading"
|
||||||
|
[attr.aria-label]="'Select subscription ' + entry[1].name" />
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
@if (editingNameId === entry[0]) {
|
||||||
|
<div class="d-flex flex-wrap gap-1 align-items-center">
|
||||||
|
<input type="text"
|
||||||
|
class="form-control form-control-sm flex-grow-1"
|
||||||
|
[name]="'subName' + entry[0]"
|
||||||
|
[(ngModel)]="nameEditDraft"
|
||||||
|
[maxlength]="subscriptionNameMaxLength"
|
||||||
|
[disabled]="downloads.loading"
|
||||||
|
[attr.aria-label]="'Subscription name for ' + entry[1].name" />
|
||||||
|
<button type="button" class="btn btn-sm btn-outline-secondary"
|
||||||
|
(click)="saveName(entry[0])"
|
||||||
|
[disabled]="downloads.loading">Save</button>
|
||||||
|
<button type="button" class="btn btn-sm btn-outline-secondary"
|
||||||
|
(click)="cancelEditName()"
|
||||||
|
[disabled]="downloads.loading">Cancel</button>
|
||||||
|
</div>
|
||||||
|
} @else {
|
||||||
|
<div class="d-flex flex-wrap gap-1 align-items-center">
|
||||||
|
<span class="text-break">{{ entry[1].name }}</span>
|
||||||
|
<button type="button" class="btn btn-link btn-sm p-0"
|
||||||
|
(click)="beginEditName(entry[0], entry[1].name)"
|
||||||
|
[disabled]="downloads.loading"
|
||||||
|
ngbTooltip="Rename this subscription (display name only; does not affect the download folder)">Edit</button>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td class="text-break"><a [href]="entry[1].url" target="_blank" rel="noopener">{{ entry[1].url }}</a></td>
|
||||||
|
<td>
|
||||||
|
@if (editingTitleRegexId === entry[0]) {
|
||||||
|
<div class="d-flex flex-wrap gap-1 align-items-center">
|
||||||
|
<input type="text"
|
||||||
|
class="form-control form-control-sm flex-grow-1"
|
||||||
|
[name]="'subTitleRegex' + entry[0]"
|
||||||
|
[(ngModel)]="titleRegexEditDraft"
|
||||||
|
[disabled]="downloads.loading" />
|
||||||
|
<button type="button" class="btn btn-sm btn-outline-secondary"
|
||||||
|
(click)="saveTitleRegex(entry[0])"
|
||||||
|
[disabled]="downloads.loading">Save</button>
|
||||||
|
<button type="button" class="btn btn-sm btn-outline-secondary"
|
||||||
|
(click)="cancelEditTitleRegex()"
|
||||||
|
[disabled]="downloads.loading">Cancel</button>
|
||||||
|
</div>
|
||||||
|
} @else {
|
||||||
|
<div class="d-flex flex-wrap gap-1 align-items-center">
|
||||||
|
<span class="text-muted small text-break"
|
||||||
|
[class.text-secondary]="!entry[1].title_regex">{{ entry[1].title_regex || '—' }}</span>
|
||||||
|
<button type="button" class="btn btn-link btn-sm p-0"
|
||||||
|
(click)="beginEditTitleRegex(entry[0], entry[1].title_regex)"
|
||||||
|
[disabled]="downloads.loading"
|
||||||
|
ngbTooltip="Edit subscription title filter (subscriptions only; not for one-off downloads)">Edit</button>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td>{{ entry[1].check_interval_minutes }}</td>
|
||||||
|
<td class="text-nowrap">
|
||||||
|
@if (entry[1].last_checked !== null) {
|
||||||
|
<span>{{ entry[1].last_checked! * 1000 | date:'yyyy-MM-dd HH:mm:ss' }}</span>
|
||||||
|
} @else {
|
||||||
|
<span class="text-muted">—</span>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
@if (entry[1].error) {
|
||||||
|
<span class="text-danger small">{{ entry[1].error }}</span>
|
||||||
|
} @else if (entry[1].enabled) {
|
||||||
|
<span class="text-success">Active</span>
|
||||||
|
} @else {
|
||||||
|
<span class="text-secondary">Paused</span>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<div class="d-flex flex-wrap gap-1">
|
||||||
|
@if (isSubscriptionChecking(entry[0])) {
|
||||||
|
<button type="button" class="btn btn-link btn-sm p-0 me-2"
|
||||||
|
disabled
|
||||||
|
[attr.aria-label]="'Checking ' + entry[1].name"
|
||||||
|
ngbTooltip="Checking now">
|
||||||
|
<span class="spinner-border spinner-border-sm" role="status" aria-hidden="true"></span>
|
||||||
|
</button>
|
||||||
|
} @else {
|
||||||
|
<button type="button" class="btn btn-link btn-sm p-0 me-2"
|
||||||
|
(click)="checkSubscriptionNow(entry[0])"
|
||||||
|
[disabled]="downloads.loading"
|
||||||
|
[attr.aria-label]="'Check now ' + entry[1].name"
|
||||||
|
ngbTooltip="Check now">
|
||||||
|
<fa-icon [icon]="faRedoAlt" />
|
||||||
|
</button>
|
||||||
|
}
|
||||||
|
<button type="button" class="btn btn-link btn-sm p-0 me-2"
|
||||||
|
(click)="toggleSubscriptionEnabled(entry[1])"
|
||||||
|
[disabled]="downloads.loading"
|
||||||
|
[attr.aria-label]="(entry[1].enabled ? 'Pause ' : 'Resume ') + entry[1].name"
|
||||||
|
[ngbTooltip]="entry[1].enabled ? 'Pause' : 'Resume'">
|
||||||
|
@if (entry[1].enabled) {
|
||||||
|
<fa-icon [icon]="faPause" />
|
||||||
|
} @else {
|
||||||
|
<fa-icon [icon]="faPlay" />
|
||||||
|
}
|
||||||
|
</button>
|
||||||
|
<button type="button" class="btn btn-link btn-sm p-0 text-danger"
|
||||||
|
(click)="deleteSubscription(entry[0])"
|
||||||
|
[disabled]="downloads.loading"
|
||||||
|
[attr.aria-label]="'Delete subscription ' + entry[1].name">
|
||||||
|
<fa-icon [icon]="faTrashAlt" />
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
</main><!-- /.container -->
|
</main><!-- /.container -->
|
||||||
|
|
||||||
<footer class="footer navbar-dark bg-dark py-3 mt-5">
|
<footer class="footer navbar-dark bg-dark py-3 mt-5">
|
||||||
|
|||||||
@@ -10,6 +10,28 @@
|
|||||||
padding: 0.5rem 0
|
padding: 0.5rem 0
|
||||||
margin-top: 3.5rem
|
margin-top: 3.5rem
|
||||||
|
|
||||||
|
.metube-section-toggle
|
||||||
|
// Positioned so it paints above the header's full-bleed :before overlay.
|
||||||
|
position: relative
|
||||||
|
display: flex
|
||||||
|
align-items: center
|
||||||
|
// Title left, chevron against the right edge, so all three section titles
|
||||||
|
// stay on the same left margin whether or not a chevron is present.
|
||||||
|
justify-content: space-between
|
||||||
|
gap: 0.75rem
|
||||||
|
width: 100%
|
||||||
|
padding: 0
|
||||||
|
border: 0
|
||||||
|
background: none
|
||||||
|
color: inherit
|
||||||
|
font: inherit
|
||||||
|
text-align: left
|
||||||
|
|
||||||
|
.metube-section-chevron
|
||||||
|
font-size: 1.1rem
|
||||||
|
width: 1.1rem
|
||||||
|
color: var(--bs-secondary-color)
|
||||||
|
|
||||||
.metube-section-header:before
|
.metube-section-header:before
|
||||||
content: ""
|
content: ""
|
||||||
position: absolute
|
position: absolute
|
||||||
|
|||||||
+249
-3
@@ -6,6 +6,7 @@ import { DownloadsService } from './services/downloads.service';
|
|||||||
import { SubscriptionsService } from './services/subscriptions.service';
|
import { SubscriptionsService } from './services/subscriptions.service';
|
||||||
import { ToastService } from './services/toast.service';
|
import { ToastService } from './services/toast.service';
|
||||||
import { CookieService } from 'ngx-cookie-service';
|
import { CookieService } from 'ngx-cookie-service';
|
||||||
|
import { Download } from './interfaces';
|
||||||
|
|
||||||
class DownloadsServiceStub {
|
class DownloadsServiceStub {
|
||||||
loading = false;
|
loading = false;
|
||||||
@@ -148,6 +149,50 @@ describe('App', () => {
|
|||||||
expect(app).toBeTruthy();
|
expect(app).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('pre-fills the download folder from DEFAULT_FOLDER', () => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
fixture.detectChanges();
|
||||||
|
|
||||||
|
downloads.configurationChanged.next({ DEFAULT_FOLDER: 'youtube' });
|
||||||
|
|
||||||
|
expect(fixture.componentInstance.folder).toBe('youtube');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('does not overwrite a folder the user already typed', () => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
fixture.detectChanges();
|
||||||
|
fixture.componentInstance.folder = 'music';
|
||||||
|
|
||||||
|
downloads.configurationChanged.next({ DEFAULT_FOLDER: 'youtube' });
|
||||||
|
|
||||||
|
expect(fixture.componentInstance.folder).toBe('music');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('collapses each section independently and remembers it (#1070)', () => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
fixture.detectChanges();
|
||||||
|
const app = fixture.componentInstance;
|
||||||
|
const cookies = TestBed.inject(CookieService);
|
||||||
|
|
||||||
|
expect(app.downloadingCollapsed).toBe(false);
|
||||||
|
expect(app.completedCollapsed).toBe(false);
|
||||||
|
expect(app.subscriptionsCollapsed).toBe(false);
|
||||||
|
|
||||||
|
app.toggleCompletedCollapsed();
|
||||||
|
|
||||||
|
expect(app.completedCollapsed).toBe(true);
|
||||||
|
expect(app.downloadingCollapsed).toBe(false);
|
||||||
|
expect(app.subscriptionsCollapsed).toBe(false);
|
||||||
|
expect(cookies.get('metube_completed_collapsed')).toBe('true');
|
||||||
|
|
||||||
|
// A fresh component picks the state back up from the cookie.
|
||||||
|
const restored = TestBed.createComponent(App);
|
||||||
|
restored.detectChanges();
|
||||||
|
expect(restored.componentInstance.completedCollapsed).toBe(true);
|
||||||
|
expect(restored.componentInstance.downloadingCollapsed).toBe(false);
|
||||||
|
expect(restored.componentInstance.subscriptionsCollapsed).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it('asIsOrder returns a stable comparator value (insertion order preserved)', () => {
|
it('asIsOrder returns a stable comparator value (insertion order preserved)', () => {
|
||||||
const fixture = TestBed.createComponent(App);
|
const fixture = TestBed.createComponent(App);
|
||||||
const app = fixture.componentInstance;
|
const app = fixture.componentInstance;
|
||||||
@@ -229,6 +274,46 @@ describe('App', () => {
|
|||||||
expect(root.textContent).toContain('starts in');
|
expect(root.textContent).toContain('starts in');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('shows the queued format in the Downloading table', () => {
|
||||||
|
downloads.queue.set('https://example.com/v', {
|
||||||
|
id: 'v1',
|
||||||
|
title: 'Some Video',
|
||||||
|
url: 'https://example.com/v',
|
||||||
|
download_type: 'audio',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'flac',
|
||||||
|
folder: '',
|
||||||
|
custom_name_prefix: '',
|
||||||
|
playlist_item_limit: 0,
|
||||||
|
status: 'downloading',
|
||||||
|
msg: '',
|
||||||
|
percent: 10,
|
||||||
|
speed: 0,
|
||||||
|
eta: 0,
|
||||||
|
filename: '',
|
||||||
|
checked: false,
|
||||||
|
});
|
||||||
|
downloads.queueChanged.next();
|
||||||
|
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
fixture.detectChanges();
|
||||||
|
|
||||||
|
const row = (fixture.nativeElement as HTMLElement).querySelector('tbody tr');
|
||||||
|
expect(row?.textContent).toContain('FLAC');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('labels formats the way the form does, and copes with an unknown one', () => {
|
||||||
|
const app = TestBed.createComponent(App).componentInstance;
|
||||||
|
const base = { format: '' } as Download;
|
||||||
|
|
||||||
|
expect(app.formatLabel({ ...base, format: 'any' })).toBe('Auto');
|
||||||
|
expect(app.formatLabel({ ...base, format: 'mp4' })).toBe('MP4');
|
||||||
|
expect(app.formatLabel({ ...base, format: 'srt' })).toBe('SRT');
|
||||||
|
// A format from a record older than the option list still reads sensibly.
|
||||||
|
expect(app.formatLabel({ ...base, format: 'mkv' })).toBe('MKV');
|
||||||
|
expect(app.formatLabel(base)).toBe('-');
|
||||||
|
});
|
||||||
|
|
||||||
it('includes titleRegex in subscribe payload', () => {
|
it('includes titleRegex in subscribe payload', () => {
|
||||||
const fixture = TestBed.createComponent(App);
|
const fixture = TestBed.createComponent(App);
|
||||||
const app = fixture.componentInstance;
|
const app = fixture.componentInstance;
|
||||||
@@ -254,7 +339,9 @@ describe('App', () => {
|
|||||||
expect(payload.skipSubscriberOnly).toBe(true);
|
expect(payload.skipSubscriberOnly).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it('omits clip fields from subscribe payload', () => {
|
it('passes clip fields through to the subscribe payload', () => {
|
||||||
|
// #1049: a subscription's options apply to all its future downloads, and
|
||||||
|
// clip bounds used to be stripped out on the way.
|
||||||
const fixture = TestBed.createComponent(App);
|
const fixture = TestBed.createComponent(App);
|
||||||
const app = fixture.componentInstance;
|
const app = fixture.componentInstance;
|
||||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||||
@@ -264,8 +351,8 @@ describe('App', () => {
|
|||||||
app.addSubscription();
|
app.addSubscription();
|
||||||
expect(subs.subscribeCalls.length).toBe(1);
|
expect(subs.subscribeCalls.length).toBe(1);
|
||||||
const payload = subs.subscribeCalls[0] as Record<string, unknown>;
|
const payload = subs.subscribeCalls[0] as Record<string, unknown>;
|
||||||
expect('clipStart' in payload).toBe(false);
|
expect(payload['clipStart']).toBe('1:00');
|
||||||
expect('clipEnd' in payload).toBe(false);
|
expect(payload['clipEnd']).toBe('2:00');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('buildAddPayload includes clip times', () => {
|
it('buildAddPayload includes clip times', () => {
|
||||||
@@ -351,4 +438,163 @@ describe('App', () => {
|
|||||||
expect(errorSpy).toHaveBeenCalledWith('Subscription name must not be empty');
|
expect(errorSpy).toHaveBeenCalledWith('Subscription name must not be empty');
|
||||||
errorSpy.mockRestore();
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
|
// Issue #533: the server picks AUDIO_DOWNLOAD_DIR on download_type alone
|
||||||
|
// (ytdl.py), so the UI's choice of URL base has to use the same rule. It used
|
||||||
|
// to also treat any .mp3 as audio, which pointed the link at audio_download/
|
||||||
|
// for files the server had written to DOWNLOAD_DIR.
|
||||||
|
describe('download links follow the server directory rule (#533)', () => {
|
||||||
|
const makeDownload = (over: Partial<Download>): Download => ({
|
||||||
|
id: 'vid1',
|
||||||
|
title: 'Test',
|
||||||
|
url: 'https://example.com/v',
|
||||||
|
download_type: 'video',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
custom_name_prefix: '',
|
||||||
|
playlist_item_limit: 0,
|
||||||
|
status: 'finished',
|
||||||
|
msg: '',
|
||||||
|
percent: 100,
|
||||||
|
speed: 0,
|
||||||
|
eta: 0,
|
||||||
|
filename: 'song.mp4',
|
||||||
|
checked: false,
|
||||||
|
...over,
|
||||||
|
} as Download);
|
||||||
|
|
||||||
|
const appWithDirs = () => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
const app = fixture.componentInstance;
|
||||||
|
const downloads = TestBed.inject(DownloadsService) as unknown as DownloadsServiceStub;
|
||||||
|
downloads.configuration['PUBLIC_HOST_URL'] = 'download/';
|
||||||
|
downloads.configuration['PUBLIC_HOST_AUDIO_URL'] = 'audio_download/';
|
||||||
|
return app;
|
||||||
|
};
|
||||||
|
|
||||||
|
it('uses the audio base for an audio download', () => {
|
||||||
|
const app = appWithDirs();
|
||||||
|
const link = app.buildDownloadLink(makeDownload({ download_type: 'audio', filename: 'song.mp3' }));
|
||||||
|
expect(link).toBe('audio_download/song.mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the video base for an mp3 produced by a video download', () => {
|
||||||
|
const app = appWithDirs();
|
||||||
|
const link = app.buildDownloadLink(makeDownload({ download_type: 'video', filename: 'song.mp3' }));
|
||||||
|
expect(link).toBe('download/song.mp3');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('uses the video base for a video download', () => {
|
||||||
|
const app = appWithDirs();
|
||||||
|
const link = app.buildDownloadLink(makeDownload({ filename: 'clip.mp4' }));
|
||||||
|
expect(link).toBe('download/clip.mp4');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('applies the same rule to chapter links', () => {
|
||||||
|
const app = appWithDirs();
|
||||||
|
const dl = makeDownload({ download_type: 'video' });
|
||||||
|
expect(app.buildChapterDownloadLink(dl, 'ch1.mp3')).toBe('download/ch1.mp3');
|
||||||
|
const audio = makeDownload({ download_type: 'audio' });
|
||||||
|
expect(app.buildChapterDownloadLink(audio, 'ch1.mp3')).toBe('audio_download/ch1.mp3');
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Issue #424: ffmpeg work after the bytes land (merge, re-encode, split) used
|
||||||
|
// to leave the row on a full, frozen bar with the item counted as neither
|
||||||
|
// active nor queued.
|
||||||
|
describe('post-processing is visible (#424)', () => {
|
||||||
|
const queueEntry = (status: string): Download => ({
|
||||||
|
id: 'vid1',
|
||||||
|
title: 'Test',
|
||||||
|
url: 'https://example.com/v',
|
||||||
|
download_type: 'video',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
custom_name_prefix: '',
|
||||||
|
playlist_item_limit: 0,
|
||||||
|
status,
|
||||||
|
msg: '',
|
||||||
|
percent: 100,
|
||||||
|
speed: 0,
|
||||||
|
eta: 0,
|
||||||
|
filename: '',
|
||||||
|
checked: false,
|
||||||
|
} as Download);
|
||||||
|
|
||||||
|
it('runs the bar indeterminate while preparing or post-processing', () => {
|
||||||
|
const app = TestBed.createComponent(App).componentInstance;
|
||||||
|
expect(app.isIndeterminate(queueEntry('preparing'))).toBe(true);
|
||||||
|
expect(app.isIndeterminate(queueEntry('postprocessing'))).toBe(true);
|
||||||
|
expect(app.isIndeterminate(queueEntry('downloading'))).toBe(false);
|
||||||
|
expect(app.isIndeterminate(queueEntry('pending'))).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('labels the bar and counts the item as active', () => {
|
||||||
|
// The component subscribes to queueChanged on construction, so the entry
|
||||||
|
// has to be announced after it exists or updateMetrics never runs.
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
downloads.queue.set('https://example.com/v', queueEntry('postprocessing'));
|
||||||
|
downloads.queueChanged.next();
|
||||||
|
fixture.detectChanges();
|
||||||
|
|
||||||
|
expect((fixture.nativeElement as HTMLElement).textContent).toContain('Post-processing');
|
||||||
|
expect(fixture.componentInstance.activeDownloads).toBe(1);
|
||||||
|
expect(fixture.componentInstance.queuedDownloads).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
// Issue #1081: a download waiting for a concurrency slot ('queued') starts on
|
||||||
|
// its own, so it must not offer the Start button that a 'pending' row — one
|
||||||
|
// added with auto-start off — legitimately has.
|
||||||
|
describe('queued rows do not offer a dead Start button (#1081)', () => {
|
||||||
|
const queueEntry = (status: string): Download => ({
|
||||||
|
id: 'vid1',
|
||||||
|
title: 'Test',
|
||||||
|
url: 'https://example.com/v',
|
||||||
|
download_type: 'video',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
custom_name_prefix: '',
|
||||||
|
playlist_item_limit: 0,
|
||||||
|
status,
|
||||||
|
msg: '',
|
||||||
|
percent: 0,
|
||||||
|
speed: 0,
|
||||||
|
eta: 0,
|
||||||
|
filename: '',
|
||||||
|
checked: false,
|
||||||
|
} as Download);
|
||||||
|
|
||||||
|
const render = (status: string) => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
downloads.queue.set('https://example.com/v', queueEntry(status));
|
||||||
|
downloads.queueChanged.next();
|
||||||
|
fixture.detectChanges();
|
||||||
|
return fixture;
|
||||||
|
};
|
||||||
|
|
||||||
|
it('hides Start for a queued row but keeps it for a pending one', () => {
|
||||||
|
const queued = render('queued');
|
||||||
|
expect(
|
||||||
|
(queued.nativeElement as HTMLElement).querySelector('[aria-label="Start download for Test"]')
|
||||||
|
).toBeNull();
|
||||||
|
|
||||||
|
downloads.queue.clear();
|
||||||
|
|
||||||
|
const pending = render('pending');
|
||||||
|
expect(
|
||||||
|
(pending.nativeElement as HTMLElement).querySelector('[aria-label="Start download for Test"]')
|
||||||
|
).not.toBeNull();
|
||||||
|
});
|
||||||
|
|
||||||
|
it('says why the row is idle and counts it as queued', () => {
|
||||||
|
const fixture = render('queued');
|
||||||
|
expect((fixture.nativeElement as HTMLElement).textContent).toContain('Queued');
|
||||||
|
expect(fixture.componentInstance.queuedDownloads).toBe(1);
|
||||||
|
expect(fixture.componentInstance.activeDownloads).toBe(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|||||||
+73
-9
@@ -86,6 +86,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
autoStart: boolean;
|
autoStart: boolean;
|
||||||
playlistItemLimit!: number;
|
playlistItemLimit!: number;
|
||||||
splitByChapters: boolean;
|
splitByChapters: boolean;
|
||||||
|
sponsorblock: boolean;
|
||||||
chapterTemplate: string;
|
chapterTemplate: string;
|
||||||
clipStart = '';
|
clipStart = '';
|
||||||
clipEnd = '';
|
clipEnd = '';
|
||||||
@@ -135,8 +136,14 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
metubeVersion: string | null = null;
|
metubeVersion: string | null = null;
|
||||||
isAdvancedOpen = false;
|
isAdvancedOpen = false;
|
||||||
sortAscending = false;
|
sortAscending = false;
|
||||||
|
downloadingCollapsed = false;
|
||||||
|
completedCollapsed = false;
|
||||||
|
subscriptionsCollapsed = false;
|
||||||
expandedErrors: Set<string> = new Set<string>();
|
expandedErrors: Set<string> = new Set<string>();
|
||||||
cachedSortedDone: [string, Download][] = [];
|
cachedSortedDone: [string, Download][] = [];
|
||||||
|
// The done ids in rendered order, so a shift-click range follows the sort
|
||||||
|
// the user is looking at rather than the map's insertion order.
|
||||||
|
cachedSortedDoneIds: string[] = [];
|
||||||
lastCopiedErrorId: string | null = null;
|
lastCopiedErrorId: string | null = null;
|
||||||
private previousDownloadType = 'video';
|
private previousDownloadType = 'video';
|
||||||
private addRequestSub?: Subscription;
|
private addRequestSub?: Subscription;
|
||||||
@@ -256,6 +263,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.quality = this.cookieService.get('metube_quality') || 'best';
|
this.quality = this.cookieService.get('metube_quality') || 'best';
|
||||||
this.autoStart = this.cookieService.get('metube_auto_start') !== 'false';
|
this.autoStart = this.cookieService.get('metube_auto_start') !== 'false';
|
||||||
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
this.splitByChapters = this.cookieService.get('metube_split_chapters') === 'true';
|
||||||
|
this.sponsorblock = this.cookieService.get('metube_sponsorblock') === 'true';
|
||||||
// Will be set from backend configuration, use empty string as placeholder
|
// Will be set from backend configuration, use empty string as placeholder
|
||||||
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
this.chapterTemplate = this.cookieService.get('metube_chapter_template') || '';
|
||||||
this.clipStart = this.cookieService.get('metube_clip_start') || '';
|
this.clipStart = this.cookieService.get('metube_clip_start') || '';
|
||||||
@@ -284,6 +292,9 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.previousDownloadType = this.downloadType;
|
this.previousDownloadType = this.downloadType;
|
||||||
this.saveSelection(this.downloadType);
|
this.saveSelection(this.downloadType);
|
||||||
this.sortAscending = this.cookieService.get('metube_sort_ascending') === 'true';
|
this.sortAscending = this.cookieService.get('metube_sort_ascending') === 'true';
|
||||||
|
this.downloadingCollapsed = this.cookieService.get('metube_downloading_collapsed') === 'true';
|
||||||
|
this.completedCollapsed = this.cookieService.get('metube_completed_collapsed') === 'true';
|
||||||
|
this.subscriptionsCollapsed = this.cookieService.get('metube_subscriptions_collapsed') === 'true';
|
||||||
|
|
||||||
const ci = parseInt(this.cookieService.get('metube_check_interval') || '', 10);
|
const ci = parseInt(this.cookieService.get('metube_check_interval') || '', 10);
|
||||||
if (!Number.isNaN(ci) && ci >= 1) {
|
if (!Number.isNaN(ci) && ci >= 1) {
|
||||||
@@ -434,6 +445,12 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
if (!Number.isNaN(playlistItemLimit) && playlistItemLimit > 0) {
|
if (!Number.isNaN(playlistItemLimit) && playlistItemLimit > 0) {
|
||||||
this.playlistItemLimit = playlistItemLimit;
|
this.playlistItemLimit = playlistItemLimit;
|
||||||
}
|
}
|
||||||
|
// Pre-fill the download folder, unless the user has already typed one
|
||||||
|
// this session. The server drops DEFAULT_FOLDER when CUSTOM_DIRS is
|
||||||
|
// off, so there is nothing to guard against here.
|
||||||
|
if (!this.folder) {
|
||||||
|
this.folder = String(config['DEFAULT_FOLDER'] ?? '');
|
||||||
|
}
|
||||||
// Set chapter template from backend config if not already set by cookie
|
// Set chapter template from backend config if not already set by cookie
|
||||||
if (!this.chapterTemplate) {
|
if (!this.chapterTemplate) {
|
||||||
this.chapterTemplate = config['OUTPUT_TEMPLATE_CHAPTER'];
|
this.chapterTemplate = config['OUTPUT_TEMPLATE_CHAPTER'];
|
||||||
@@ -601,14 +618,10 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) {
|
if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
// Subscriptions do not support clip ranges (backend rejects clip fields).
|
|
||||||
const { clipStart: _clipStart, clipEnd: _clipEnd, ...subscribeBase } = payload;
|
|
||||||
void _clipStart;
|
|
||||||
void _clipEnd;
|
|
||||||
this.subscribeInProgress = true;
|
this.subscribeInProgress = true;
|
||||||
this.subscriptionsSvc
|
this.subscriptionsSvc
|
||||||
.subscribe({
|
.subscribe({
|
||||||
...subscribeBase,
|
...payload,
|
||||||
checkIntervalMinutes: this.checkIntervalMinutes,
|
checkIntervalMinutes: this.checkIntervalMinutes,
|
||||||
titleRegex: tr,
|
titleRegex: tr,
|
||||||
skipSubscriberOnly: this.skipSubscriberOnly,
|
skipSubscriberOnly: this.skipSubscriberOnly,
|
||||||
@@ -850,6 +863,10 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
this.cookieService.set('metube_auto_start', this.autoStart ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
}
|
}
|
||||||
|
|
||||||
|
sponsorblockChanged() {
|
||||||
|
this.cookieService.set('metube_sponsorblock', this.sponsorblock ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
|
}
|
||||||
|
|
||||||
splitByChaptersChanged() {
|
splitByChaptersChanged() {
|
||||||
this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
this.cookieService.set('metube_split_chapters', this.splitByChapters ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
}
|
}
|
||||||
@@ -913,6 +930,22 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
return type.charAt(0).toUpperCase() + type.slice(1);
|
return type.charAt(0).toUpperCase() + type.slice(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The format the download was queued with, labelled the way the form labels
|
||||||
|
// it, so a queued item can be told apart while it is still downloading.
|
||||||
|
formatLabel(download: Download): string {
|
||||||
|
const format = (download.format || '').trim();
|
||||||
|
if (!format) {
|
||||||
|
return '-';
|
||||||
|
}
|
||||||
|
const options: Option[] = [
|
||||||
|
...this.videoFormats,
|
||||||
|
...this.audioFormats,
|
||||||
|
...this.captionFormats,
|
||||||
|
...this.thumbnailFormats,
|
||||||
|
];
|
||||||
|
return options.find(o => o.id === format)?.text ?? format.toUpperCase();
|
||||||
|
}
|
||||||
|
|
||||||
formatCodecLabel(download: Download): string {
|
formatCodecLabel(download: Download): string {
|
||||||
if (download.download_type !== 'video') {
|
if (download.download_type !== 'video') {
|
||||||
const format = (download.format || '').toUpperCase();
|
const format = (download.format || '').toUpperCase();
|
||||||
@@ -1090,6 +1123,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
playlistItemLimit: overrides.playlistItemLimit ?? this.playlistItemLimit,
|
playlistItemLimit: overrides.playlistItemLimit ?? this.playlistItemLimit,
|
||||||
autoStart: overrides.autoStart ?? this.autoStart,
|
autoStart: overrides.autoStart ?? this.autoStart,
|
||||||
splitByChapters: overrides.splitByChapters ?? this.splitByChapters,
|
splitByChapters: overrides.splitByChapters ?? this.splitByChapters,
|
||||||
|
sponsorblock: overrides.sponsorblock ?? this.sponsorblock,
|
||||||
chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate,
|
chapterTemplate: overrides.chapterTemplate ?? this.chapterTemplate,
|
||||||
subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage,
|
subtitleLanguage: overrides.subtitleLanguage ?? this.subtitleLanguage,
|
||||||
subtitleMode: overrides.subtitleMode ?? this.subtitleMode,
|
subtitleMode: overrides.subtitleMode ?? this.subtitleMode,
|
||||||
@@ -1156,6 +1190,14 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.downloads.startById([id]).subscribe((res) => this.handleActionResult(res, 'Start download failed'));
|
this.downloads.startById([id]).subscribe((res) => this.handleActionResult(res, 'Start download failed'));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 'preparing' (yt-dlp starting up) and 'postprocessing' (ffmpeg merging,
|
||||||
|
// re-encoding or splitting once the bytes have landed) both have real work in
|
||||||
|
// flight with no percentage to report, so the bar runs animated at full width
|
||||||
|
// instead of showing a number that cannot move.
|
||||||
|
isIndeterminate(download: Download): boolean {
|
||||||
|
return download.status === 'preparing' || download.status === 'postprocessing';
|
||||||
|
}
|
||||||
|
|
||||||
liveCountdownSeconds(download: Download): number | null {
|
liveCountdownSeconds(download: Download): number | null {
|
||||||
const ts = download.live_release_timestamp;
|
const ts = download.live_release_timestamp;
|
||||||
if (ts == null || download.status !== 'scheduled') {
|
if (ts == null || download.status !== 'scheduled') {
|
||||||
@@ -1256,7 +1298,12 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
|
|
||||||
buildDownloadLink(download: Download) {
|
buildDownloadLink(download: Download) {
|
||||||
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
||||||
if (download.download_type === 'audio' || download.filename.endsWith('.mp3')) {
|
// Must match the server's directory rule exactly: ytdl.py writes to
|
||||||
|
// AUDIO_DOWNLOAD_DIR on download_type alone. Treating any .mp3 as audio
|
||||||
|
// sent the link to audio_download/ for mp3s produced under a video-type
|
||||||
|
// download (a postprocessor, a preset, or a legacy record), which the
|
||||||
|
// server had written to DOWNLOAD_DIR -- a 404 whenever the two differ.
|
||||||
|
if (download.download_type === 'audio') {
|
||||||
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1354,7 +1401,8 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
|
|
||||||
buildChapterDownloadLink(download: Download, chapterFilename: string) {
|
buildChapterDownloadLink(download: Download, chapterFilename: string) {
|
||||||
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
let baseDir = this.downloads.configuration["PUBLIC_HOST_URL"];
|
||||||
if (download.download_type === 'audio' || chapterFilename.endsWith('.mp3')) {
|
// Same server-side rule as buildDownloadLink above.
|
||||||
|
if (download.download_type === 'audio') {
|
||||||
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
baseDir = this.downloads.configuration["PUBLIC_HOST_AUDIO_URL"];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1527,6 +1575,21 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.rebuildSortedDone();
|
this.rebuildSortedDone();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
toggleDownloadingCollapsed() {
|
||||||
|
this.downloadingCollapsed = !this.downloadingCollapsed;
|
||||||
|
this.cookieService.set('metube_downloading_collapsed', this.downloadingCollapsed ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
|
}
|
||||||
|
|
||||||
|
toggleCompletedCollapsed() {
|
||||||
|
this.completedCollapsed = !this.completedCollapsed;
|
||||||
|
this.cookieService.set('metube_completed_collapsed', this.completedCollapsed ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
|
}
|
||||||
|
|
||||||
|
toggleSubscriptionsCollapsed() {
|
||||||
|
this.subscriptionsCollapsed = !this.subscriptionsCollapsed;
|
||||||
|
this.cookieService.set('metube_subscriptions_collapsed', this.subscriptionsCollapsed ? 'true' : 'false', { expires: this.settingsCookieExpiryDays });
|
||||||
|
}
|
||||||
|
|
||||||
private rebuildSortedDone() {
|
private rebuildSortedDone() {
|
||||||
const result: [string, Download][] = [];
|
const result: [string, Download][] = [];
|
||||||
this.downloads.done.forEach((dl, key) => {
|
this.downloads.done.forEach((dl, key) => {
|
||||||
@@ -1536,6 +1599,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
result.reverse();
|
result.reverse();
|
||||||
}
|
}
|
||||||
this.cachedSortedDone = result;
|
this.cachedSortedDone = result;
|
||||||
|
this.cachedSortedDoneIds = result.map(([key]) => key);
|
||||||
}
|
}
|
||||||
|
|
||||||
toggleErrorDetail(id: string) {
|
toggleErrorDetail(id: string) {
|
||||||
@@ -1662,9 +1726,9 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
if (download.status === 'downloading') {
|
if (download.status === 'downloading') {
|
||||||
active++;
|
active++;
|
||||||
speed += download.speed || 0;
|
speed += download.speed || 0;
|
||||||
} else if (download.status === 'preparing') {
|
} else if (download.status === 'preparing' || download.status === 'postprocessing') {
|
||||||
active++;
|
active++;
|
||||||
} else if (download.status === 'pending' || download.status === 'scheduled') {
|
} else if (download.status === 'queued' || download.status === 'pending' || download.status === 'scheduled') {
|
||||||
queued++;
|
queued++;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -2,6 +2,38 @@ import { TestBed } from '@angular/core/testing';
|
|||||||
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||||
import { Checkable } from '../interfaces';
|
import { Checkable } from '../interfaces';
|
||||||
|
|
||||||
|
function makeList(ids: string[]): Map<string, Checkable> {
|
||||||
|
const list = new Map<string, Checkable>();
|
||||||
|
for (const id of ids) {
|
||||||
|
list.set(id, { checked: false });
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeMaster(list: Map<string, Checkable>, orderedIds: string[] | null = null) {
|
||||||
|
const fixture = TestBed.createComponent(SelectAllCheckboxComponent);
|
||||||
|
fixture.componentRef.setInput('id', 'queue');
|
||||||
|
fixture.componentRef.setInput('list', list);
|
||||||
|
if (orderedIds) {
|
||||||
|
fixture.componentRef.setInput('orderedIds', orderedIds);
|
||||||
|
}
|
||||||
|
fixture.detectChanges();
|
||||||
|
return fixture;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Simulates what the item checkbox does: ngModel writes the new state, then
|
||||||
|
// the change handler reports the click to the master.
|
||||||
|
function clickItem(
|
||||||
|
master: SelectAllCheckboxComponent,
|
||||||
|
list: Map<string, Checkable>,
|
||||||
|
id: string,
|
||||||
|
shift = false,
|
||||||
|
) {
|
||||||
|
const item = list.get(id)!;
|
||||||
|
item.checked = !item.checked;
|
||||||
|
master.selectionChanged(id, shift);
|
||||||
|
}
|
||||||
|
|
||||||
describe('SelectAllCheckboxComponent', () => {
|
describe('SelectAllCheckboxComponent', () => {
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
await TestBed.configureTestingModule({
|
await TestBed.configureTestingModule({
|
||||||
@@ -20,4 +52,87 @@ describe('SelectAllCheckboxComponent', () => {
|
|||||||
fixture.componentInstance.clicked();
|
fixture.componentInstance.clicked();
|
||||||
expect(list.get('u1')?.checked).toBe(true);
|
expect(list.get('u1')?.checked).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('shift-click checks every item between the two clicks', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3', 'u4', 'u5']);
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u2');
|
||||||
|
clickItem(master, list, 'u4', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, true, true, true, false]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('extends upwards as well as downwards', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3', 'u4']);
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u4');
|
||||||
|
clickItem(master, list, 'u2', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, true, true, true]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('shift-clicking a checked box clears the range', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3']);
|
||||||
|
list.forEach((item) => (item.checked = true));
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u1');
|
||||||
|
clickItem(master, list, 'u3', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, false, false]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('follows the rendered order, not the map order', () => {
|
||||||
|
// The done list renders newest-first, so its rendered order is not the
|
||||||
|
// order the entries sit in the map. u2 lies inside the range on screen
|
||||||
|
// and outside it in the map, which is what separates the two.
|
||||||
|
const list = makeList(['u1', 'u2', 'u3', 'u4']);
|
||||||
|
const master = makeMaster(list, ['u4', 'u2', 'u3', 'u1']).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u4');
|
||||||
|
clickItem(master, list, 'u3', true);
|
||||||
|
|
||||||
|
// u1 (rendered last) stays clear; u2 is swept up with the range.
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, true, true, true]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('a plain click after a range starts a new anchor', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3', 'u4']);
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u1');
|
||||||
|
clickItem(master, list, 'u2', true);
|
||||||
|
clickItem(master, list, 'u4');
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([true, true, false, true]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('select-all clears the anchor so the next shift-click is a plain toggle', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3']);
|
||||||
|
const fixture = makeMaster(list);
|
||||||
|
const master = fixture.componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u1');
|
||||||
|
master.selected = true;
|
||||||
|
master.clicked();
|
||||||
|
master.selected = false;
|
||||||
|
master.clicked();
|
||||||
|
clickItem(master, list, 'u3', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, false, true]);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('ignores a range whose anchor row is gone', () => {
|
||||||
|
const list = makeList(['u1', 'u2', 'u3']);
|
||||||
|
const master = makeMaster(list).componentInstance;
|
||||||
|
|
||||||
|
clickItem(master, list, 'u1');
|
||||||
|
// The anchor finishes downloading and leaves the queue.
|
||||||
|
list.delete('u1');
|
||||||
|
clickItem(master, list, 'u3', true);
|
||||||
|
|
||||||
|
expect([...list.values()].map((i) => i.checked)).toEqual([false, true]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -20,17 +20,33 @@ import { FormsModule } from "@angular/forms";
|
|||||||
export class SelectAllCheckboxComponent {
|
export class SelectAllCheckboxComponent {
|
||||||
readonly id = input.required<string>();
|
readonly id = input.required<string>();
|
||||||
readonly list = input.required<Map<string, Checkable>>();
|
readonly list = input.required<Map<string, Checkable>>();
|
||||||
|
// The ids in the order the rows are rendered. The done list is sorted for
|
||||||
|
// display, so its order is not the map's insertion order, and a range
|
||||||
|
// selection has to follow what the user sees. Left unset, the map order is
|
||||||
|
// the rendered order.
|
||||||
|
readonly orderedIds = input<string[] | null>(null);
|
||||||
readonly changed = output<number>();
|
readonly changed = output<number>();
|
||||||
|
|
||||||
readonly masterCheckbox = viewChild.required<ElementRef>('masterCheckbox');
|
readonly masterCheckbox = viewChild.required<ElementRef>('masterCheckbox');
|
||||||
selected!: boolean;
|
selected!: boolean;
|
||||||
|
|
||||||
|
// The item a range extends from: the last one toggled on its own.
|
||||||
|
private anchorId: string | null = null;
|
||||||
|
|
||||||
clicked() {
|
clicked() {
|
||||||
this.list().forEach(item => item.checked = this.selected);
|
this.list().forEach(item => item.checked = this.selected);
|
||||||
|
// Select-all is not a position, so there is nothing to extend from next.
|
||||||
|
this.anchorId = null;
|
||||||
this.selectionChanged();
|
this.selectionChanged();
|
||||||
}
|
}
|
||||||
|
|
||||||
selectionChanged() {
|
selectionChanged(id?: string, extend = false) {
|
||||||
|
if (id !== undefined) {
|
||||||
|
if (extend && this.anchorId !== null && this.anchorId !== id) {
|
||||||
|
this.applyRange(this.anchorId, id);
|
||||||
|
}
|
||||||
|
this.anchorId = id;
|
||||||
|
}
|
||||||
const masterCheckbox = this.masterCheckbox();
|
const masterCheckbox = this.masterCheckbox();
|
||||||
if (!masterCheckbox)
|
if (!masterCheckbox)
|
||||||
return;
|
return;
|
||||||
@@ -40,4 +56,27 @@ export class SelectAllCheckboxComponent {
|
|||||||
masterCheckbox.nativeElement.indeterminate = checked > 0 && checked < this.list().size;
|
masterCheckbox.nativeElement.indeterminate = checked > 0 && checked < this.list().size;
|
||||||
this.changed.emit(checked);
|
this.changed.emit(checked);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Everything between the anchor and the just-clicked row takes the state the
|
||||||
|
// click produced, so shift-clicking a checked box clears the range and
|
||||||
|
// shift-clicking an unchecked one fills it.
|
||||||
|
private applyRange(fromId: string, toId: string) {
|
||||||
|
const ids = this.orderedIds() ?? Array.from(this.list().keys());
|
||||||
|
const from = ids.indexOf(fromId);
|
||||||
|
const to = ids.indexOf(toId);
|
||||||
|
// A row can disappear between two clicks (a download finishing moves it
|
||||||
|
// from the queue to the done list); without both ends there is no range.
|
||||||
|
if (from < 0 || to < 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const target = this.list().get(toId)?.checked ?? false;
|
||||||
|
const start = Math.min(from, to);
|
||||||
|
const end = Math.max(from, to);
|
||||||
|
for (let i = start; i <= end; i++) {
|
||||||
|
const item = this.list().get(ids[i]);
|
||||||
|
if (item) {
|
||||||
|
item.checked = target;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -22,4 +22,33 @@ describe('ItemCheckboxComponent', () => {
|
|||||||
itemFixture.detectChanges();
|
itemFixture.detectChanges();
|
||||||
expect(itemFixture.componentInstance).toBeTruthy();
|
expect(itemFixture.componentInstance).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('reports the shift modifier from the click to the master', () => {
|
||||||
|
const masterFixture = TestBed.createComponent(SelectAllCheckboxComponent);
|
||||||
|
masterFixture.componentRef.setInput('id', 'q');
|
||||||
|
masterFixture.componentRef.setInput('list', new Map());
|
||||||
|
masterFixture.detectChanges();
|
||||||
|
const master = masterFixture.componentInstance;
|
||||||
|
const reported: [string | undefined, boolean | undefined][] = [];
|
||||||
|
master.selectionChanged = (id?: string, extend?: boolean) => {
|
||||||
|
reported.push([id, extend]);
|
||||||
|
};
|
||||||
|
|
||||||
|
const itemFixture = TestBed.createComponent(ItemCheckboxComponent);
|
||||||
|
itemFixture.componentRef.setInput('id', 'row1');
|
||||||
|
itemFixture.componentRef.setInput('master', master);
|
||||||
|
itemFixture.componentRef.setInput('checkable', { checked: false });
|
||||||
|
itemFixture.detectChanges();
|
||||||
|
const item = itemFixture.componentInstance;
|
||||||
|
|
||||||
|
item.clicked(new MouseEvent('click', { shiftKey: true }));
|
||||||
|
item.changed();
|
||||||
|
// The modifier must not stick to the next toggle.
|
||||||
|
item.changed();
|
||||||
|
|
||||||
|
expect(reported).toEqual([
|
||||||
|
['row1', true],
|
||||||
|
['row1', false],
|
||||||
|
]);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -7,14 +7,14 @@ import { FormsModule } from '@angular/forms';
|
|||||||
selector: 'app-item-checkbox',
|
selector: 'app-item-checkbox',
|
||||||
template: `
|
template: `
|
||||||
<div class="form-check">
|
<div class="form-check">
|
||||||
<input type="checkbox" class="form-check-input" id="{{master().id()}}-{{id()}}-select" [(ngModel)]="checkable().checked" (change)="master().selectionChanged()" [attr.aria-label]="'Select item ' + id()">
|
<input type="checkbox" class="form-check-input" id="{{master().id()}}-{{id()}}-select" [(ngModel)]="checkable().checked" (click)="clicked($event)" (change)="changed()" [attr.aria-label]="'Select item ' + id()">
|
||||||
<label class="form-check-label visually-hidden" for="{{master().id()}}-{{id()}}-select">Select item</label>
|
<label class="form-check-label visually-hidden" for="{{master().id()}}-{{id()}}-select">Select item</label>
|
||||||
</div>
|
</div>
|
||||||
`,
|
`,
|
||||||
// Shared Checkable objects are mutated in place; Eager preserves pre-v22 behavior.
|
// Shared Checkable objects are mutated in place; Eager preserves pre-v22 behavior.
|
||||||
// eslint-disable-next-line @angular-eslint/prefer-on-push-component-change-detection
|
// eslint-disable-next-line @angular-eslint/prefer-on-push-component-change-detection
|
||||||
changeDetection: ChangeDetectionStrategy.Eager,
|
changeDetection: ChangeDetectionStrategy.Eager,
|
||||||
imports: [
|
imports: [
|
||||||
FormsModule
|
FormsModule
|
||||||
]
|
]
|
||||||
})
|
})
|
||||||
@@ -22,4 +22,19 @@ export class ItemCheckboxComponent {
|
|||||||
readonly id = input.required<string>();
|
readonly id = input.required<string>();
|
||||||
readonly master = input.required<SelectAllCheckboxComponent>();
|
readonly master = input.required<SelectAllCheckboxComponent>();
|
||||||
readonly checkable = input.required<Checkable>();
|
readonly checkable = input.required<Checkable>();
|
||||||
|
|
||||||
|
// click fires before change, so the modifier is recorded here and read once
|
||||||
|
// ngModel has written the new state into the checkable. Keyboard activation
|
||||||
|
// fires change without a click, which is a plain toggle.
|
||||||
|
private extend = false;
|
||||||
|
|
||||||
|
clicked(event: MouseEvent) {
|
||||||
|
this.extend = event.shiftKey;
|
||||||
|
}
|
||||||
|
|
||||||
|
changed() {
|
||||||
|
const extend = this.extend;
|
||||||
|
this.extend = false;
|
||||||
|
this.master().selectionChanged(this.id(), extend);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ export interface Download {
|
|||||||
custom_name_prefix: string;
|
custom_name_prefix: string;
|
||||||
playlist_item_limit: number;
|
playlist_item_limit: number;
|
||||||
split_by_chapters?: boolean;
|
split_by_chapters?: boolean;
|
||||||
|
sponsorblock?: boolean;
|
||||||
chapter_template?: string;
|
chapter_template?: string;
|
||||||
subtitle_language?: string;
|
subtitle_language?: string;
|
||||||
subtitle_mode?: string;
|
subtitle_mode?: string;
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ export interface SubscriptionRow {
|
|||||||
folder: string;
|
folder: string;
|
||||||
title_regex?: string;
|
title_regex?: string;
|
||||||
skip_subscriber_only?: boolean;
|
skip_subscriber_only?: boolean;
|
||||||
|
clip_start?: number | null;
|
||||||
|
clip_end?: number | null;
|
||||||
last_checked: number | null;
|
last_checked: number | null;
|
||||||
seen_count: number;
|
seen_count: number;
|
||||||
error: string | null;
|
error: string | null;
|
||||||
|
|||||||
@@ -36,6 +36,7 @@ function basePayload(): AddDownloadPayload {
|
|||||||
playlistItemLimit: 0,
|
playlistItemLimit: 0,
|
||||||
autoStart: true,
|
autoStart: true,
|
||||||
splitByChapters: false,
|
splitByChapters: false,
|
||||||
|
sponsorblock: false,
|
||||||
chapterTemplate: '',
|
chapterTemplate: '',
|
||||||
subtitleLanguage: 'en',
|
subtitleLanguage: 'en',
|
||||||
subtitleMode: 'prefer_manual',
|
subtitleMode: 'prefer_manual',
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ export interface AddDownloadPayload {
|
|||||||
playlistItemLimit: number;
|
playlistItemLimit: number;
|
||||||
autoStart: boolean;
|
autoStart: boolean;
|
||||||
splitByChapters: boolean;
|
splitByChapters: boolean;
|
||||||
|
sponsorblock: boolean;
|
||||||
chapterTemplate: string;
|
chapterTemplate: string;
|
||||||
subtitleLanguage: string;
|
subtitleLanguage: string;
|
||||||
subtitleMode: string;
|
subtitleMode: string;
|
||||||
@@ -148,6 +149,7 @@ export class DownloadsService {
|
|||||||
playlist_item_limit: payload.playlistItemLimit,
|
playlist_item_limit: payload.playlistItemLimit,
|
||||||
auto_start: payload.autoStart,
|
auto_start: payload.autoStart,
|
||||||
split_by_chapters: payload.splitByChapters,
|
split_by_chapters: payload.splitByChapters,
|
||||||
|
sponsorblock: payload.sponsorblock,
|
||||||
chapter_template: payload.chapterTemplate,
|
chapter_template: payload.chapterTemplate,
|
||||||
subtitle_language: payload.subtitleLanguage,
|
subtitle_language: payload.subtitleLanguage,
|
||||||
subtitle_mode: payload.subtitleMode,
|
subtitle_mode: payload.subtitleMode,
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import { TestBed } from '@angular/core/testing';
|
||||||
|
import { provideHttpClient } from '@angular/common/http';
|
||||||
|
import { provideHttpClientTesting, HttpTestingController } from '@angular/common/http/testing';
|
||||||
|
import { Subject } from 'rxjs';
|
||||||
|
import { SubscriptionsService, SubscribePayload } from './subscriptions.service';
|
||||||
|
import { MeTubeSocket } from './metube-socket.service';
|
||||||
|
|
||||||
|
class MeTubeSocketStub {
|
||||||
|
private subjects: Record<string, Subject<string>> = {};
|
||||||
|
|
||||||
|
fromEvent(event: string) {
|
||||||
|
if (!this.subjects[event]) {
|
||||||
|
this.subjects[event] = new Subject<string>();
|
||||||
|
}
|
||||||
|
return this.subjects[event].asObservable();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function basePayload(): SubscribePayload {
|
||||||
|
return {
|
||||||
|
url: 'https://example.com/channel',
|
||||||
|
downloadType: 'video',
|
||||||
|
codec: 'auto',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
customNamePrefix: '',
|
||||||
|
playlistItemLimit: 0,
|
||||||
|
autoStart: true,
|
||||||
|
splitByChapters: false,
|
||||||
|
sponsorblock: false,
|
||||||
|
chapterTemplate: '',
|
||||||
|
subtitleLanguage: 'en',
|
||||||
|
subtitleMode: 'prefer_manual',
|
||||||
|
ytdlOptionsPresets: [],
|
||||||
|
ytdlOptionsOverrides: '',
|
||||||
|
clipStart: '',
|
||||||
|
clipEnd: '',
|
||||||
|
checkIntervalMinutes: 60,
|
||||||
|
titleRegex: '',
|
||||||
|
skipSubscriberOnly: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe('SubscriptionsService', () => {
|
||||||
|
let httpMock: HttpTestingController;
|
||||||
|
let service: SubscriptionsService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
await TestBed.configureTestingModule({
|
||||||
|
providers: [
|
||||||
|
SubscriptionsService,
|
||||||
|
provideHttpClient(),
|
||||||
|
provideHttpClientTesting(),
|
||||||
|
{ provide: MeTubeSocket, useValue: new MeTubeSocketStub() },
|
||||||
|
],
|
||||||
|
}).compileComponents();
|
||||||
|
|
||||||
|
service = TestBed.inject(SubscriptionsService);
|
||||||
|
httpMock = TestBed.inject(HttpTestingController);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('subscribe() carries the sponsorblock flag', () => {
|
||||||
|
service.subscribe({ ...basePayload(), sponsorblock: true }).subscribe();
|
||||||
|
const req = httpMock.expectOne('subscribe');
|
||||||
|
expect(req.request.method).toBe('POST');
|
||||||
|
expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: true }));
|
||||||
|
req.flush({ status: 'ok' });
|
||||||
|
});
|
||||||
|
|
||||||
|
it('subscribe() sends the flag off by default', () => {
|
||||||
|
service.subscribe(basePayload()).subscribe();
|
||||||
|
const req = httpMock.expectOne('subscribe');
|
||||||
|
expect(req.request.body).toEqual(expect.objectContaining({ sponsorblock: false }));
|
||||||
|
req.flush({ status: 'ok' });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -81,28 +81,35 @@ export class SubscriptionsService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
subscribe(payload: SubscribePayload) {
|
subscribe(payload: SubscribePayload) {
|
||||||
return this.http
|
const body: Record<string, unknown> = {
|
||||||
.post<Status>('subscribe', {
|
url: payload.url,
|
||||||
url: payload.url,
|
download_type: payload.downloadType,
|
||||||
download_type: payload.downloadType,
|
codec: payload.codec,
|
||||||
codec: payload.codec,
|
quality: payload.quality,
|
||||||
quality: payload.quality,
|
format: payload.format,
|
||||||
format: payload.format,
|
folder: payload.folder,
|
||||||
folder: payload.folder,
|
custom_name_prefix: payload.customNamePrefix,
|
||||||
custom_name_prefix: payload.customNamePrefix,
|
playlist_item_limit: payload.playlistItemLimit,
|
||||||
playlist_item_limit: payload.playlistItemLimit,
|
auto_start: payload.autoStart,
|
||||||
auto_start: payload.autoStart,
|
split_by_chapters: payload.splitByChapters,
|
||||||
split_by_chapters: payload.splitByChapters,
|
sponsorblock: payload.sponsorblock,
|
||||||
chapter_template: payload.chapterTemplate,
|
chapter_template: payload.chapterTemplate,
|
||||||
subtitle_language: payload.subtitleLanguage,
|
subtitle_language: payload.subtitleLanguage,
|
||||||
subtitle_mode: payload.subtitleMode,
|
subtitle_mode: payload.subtitleMode,
|
||||||
ytdl_options_presets: payload.ytdlOptionsPresets,
|
ytdl_options_presets: payload.ytdlOptionsPresets,
|
||||||
ytdl_options_overrides: payload.ytdlOptionsOverrides,
|
ytdl_options_overrides: payload.ytdlOptionsOverrides,
|
||||||
check_interval_minutes: payload.checkIntervalMinutes,
|
check_interval_minutes: payload.checkIntervalMinutes,
|
||||||
title_regex: payload.titleRegex,
|
title_regex: payload.titleRegex,
|
||||||
skip_subscriber_only: payload.skipSubscriberOnly,
|
skip_subscriber_only: payload.skipSubscriberOnly,
|
||||||
})
|
};
|
||||||
.pipe(catchError((err) => this.handleHTTPError(err)));
|
// Send the clip fields only when actually filled in. The backend treats an
|
||||||
|
// absent field as "not requested", which is what stops a t= timestamp on the
|
||||||
|
// subscribed URL from clipping every future download.
|
||||||
|
const cs = payload.clipStart?.trim();
|
||||||
|
const ce = payload.clipEnd?.trim();
|
||||||
|
if (cs) body['clip_start'] = cs;
|
||||||
|
if (ce) body['clip_end'] = ce;
|
||||||
|
return this.http.post<Status>('subscribe', body).pipe(catchError((err) => this.handleHTTPError(err)));
|
||||||
}
|
}
|
||||||
|
|
||||||
delete(ids: string[]) {
|
delete(ids: string[]) {
|
||||||
|
|||||||
Reference in New Issue
Block a user