mirror of
https://github.com/alexta69/metube.git
synced 2026-09-21 21:45:04 +00:00
Compare commits
47 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 3ea4732c5d | |||
| e2c777842e | |||
| c34a18de7a | |||
| d6bcf182c5 | |||
| ad90609c9b | |||
| 5315630ab0 | |||
| 54463baf0e | |||
| b00d4785ee | |||
| 96e88a3555 | |||
| 49a46a7d1c | |||
| 961b54aa83 | |||
| e0549d6c24 | |||
| f315b75bb2 | |||
| c2c129db61 | |||
| 363f159a0a | |||
| 38c0ca22f4 | |||
| 24ae8f0742 | |||
| 0a946cc352 | |||
| 51fd203b71 | |||
| d136344c26 | |||
| 33f1412fac | |||
| ce897ee009 | |||
| dd1b4c2436 | |||
| 8752b500d6 | |||
| 04b9366764 | |||
| b73e95f405 | |||
| 64d0d62878 | |||
| 37f7af0555 | |||
| 5aa7d033e2 | |||
| 5429200fba | |||
| 72d60ea55a | |||
| a9b2e07a59 | |||
| d157444877 | |||
| e30a24ff70 | |||
| ee20512410 | |||
| 897d52cd0d | |||
| baa72c0e94 | |||
| 66d8fa570b | |||
| cf2d2dd465 | |||
| 0b5617e96c | |||
| 56c0ad3b5f | |||
| 4478d1394e | |||
| ad92607a21 | |||
| 6ff364aacf | |||
| 39a8948976 | |||
| f0348581c2 | |||
| e2773db65a |
@@ -12,7 +12,7 @@ jobs:
|
|||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
- name: Set up Node.js
|
- name: Set up Node.js
|
||||||
uses: actions/setup-node@v6
|
uses: actions/setup-node@v6
|
||||||
with:
|
with:
|
||||||
@@ -59,7 +59,7 @@ jobs:
|
|||||||
run: echo "date=$(date +'%Y.%m.%d')" >> "$GITHUB_OUTPUT"
|
run: echo "date=$(date +'%Y.%m.%d')" >> "$GITHUB_OUTPUT"
|
||||||
-
|
-
|
||||||
name: Checkout
|
name: Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
-
|
-
|
||||||
name: Set up QEMU
|
name: Set up QEMU
|
||||||
uses: docker/setup-qemu-action@v4
|
uses: docker/setup-qemu-action@v4
|
||||||
@@ -118,7 +118,7 @@ jobs:
|
|||||||
id: date
|
id: date
|
||||||
run: echo "date=$(date +'%Y.%m.%d')" >> $GITHUB_OUTPUT
|
run: echo "date=$(date +'%Y.%m.%d')" >> $GITHUB_OUTPUT
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
fetch-depth: 0
|
fetch-depth: 0
|
||||||
- name: Get commits since last release
|
- name: Get commits since last release
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ jobs:
|
|||||||
steps:
|
steps:
|
||||||
-
|
-
|
||||||
name: Checkout
|
name: Checkout
|
||||||
uses: actions/checkout@v6
|
uses: actions/checkout@v7
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.AUTOUPDATE_PAT }}
|
token: ${{ secrets.AUTOUPDATE_PAT }}
|
||||||
-
|
-
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ If an addition would exceed the limit, trim existing prose elsewhere — prefer
|
|||||||
## Tech stack
|
## Tech stack
|
||||||
|
|
||||||
- **Backend:** Python 3.13+, aiohttp, python-socketio 5.x, yt-dlp
|
- **Backend:** Python 3.13+, aiohttp, python-socketio 5.x, yt-dlp
|
||||||
- **Frontend:** Angular 21, TypeScript, Bootstrap 5, SASS, ngx-socket-io
|
- **Frontend:** Angular 22, TypeScript, Bootstrap 5, SASS, ngx-socket-io
|
||||||
- **Package managers:** uv (Python), pnpm (frontend)
|
- **Package managers:** uv (Python), pnpm (frontend)
|
||||||
- **Container:** Multi-stage Docker (Node builder + Python runtime), multi-arch (amd64/arm64)
|
- **Container:** Multi-stage Docker (Node builder + Python runtime), multi-arch (amd64/arm64)
|
||||||
|
|
||||||
|
|||||||
+7
-2
@@ -1,4 +1,8 @@
|
|||||||
FROM node:lts-alpine AS builder
|
# Pinned to a major version rather than the lts-alpine floating tag: that tag
|
||||||
|
# has lagged behind and resolved to a Node patch older than the Angular CLI's
|
||||||
|
# minimum supported version, breaking the build. node:22-alpine currently
|
||||||
|
# satisfies @angular/cli's >=22.22.3 requirement.
|
||||||
|
FROM node:22-alpine AS builder
|
||||||
|
|
||||||
WORKDIR /metube
|
WORKDIR /metube
|
||||||
COPY ui ./
|
COPY ui ./
|
||||||
@@ -66,7 +70,8 @@ ENV TEMP_DIR=/downloads
|
|||||||
ENV PORT=8081
|
ENV PORT=8081
|
||||||
VOLUME /downloads
|
VOLUME /downloads
|
||||||
EXPOSE 8081
|
EXPOSE 8081
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD curl -fsS "http://localhost:${PORT}/" || exit 1
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
||||||
|
CMD case "$HTTPS" in true|True|on|1) curl -fsSk "https://localhost:${PORT}/";; *) curl -fsS "http://localhost:${PORT}/";; esac || exit 1
|
||||||
|
|
||||||
# Add build-time argument for version
|
# Add build-time argument for version
|
||||||
ARG VERSION=dev
|
ARG VERSION=dev
|
||||||
|
|||||||
@@ -71,6 +71,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
|||||||
* __YTDL_OPTIONS_PRESETS__: Named bundles of yt-dlp options, selectable per download in the UI. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for format and examples.
|
* __YTDL_OPTIONS_PRESETS__: Named bundles of yt-dlp options, selectable per download in the UI. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for format and examples.
|
||||||
* __YTDL_OPTIONS_PRESETS_FILE__: Path to a JSON file containing presets. Monitored and reloaded automatically on changes. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options).
|
* __YTDL_OPTIONS_PRESETS_FILE__: Path to a JSON file containing presets. Monitored and reloaded automatically on changes. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options).
|
||||||
* __ALLOW_YTDL_OPTIONS_OVERRIDES__: Whether to show a free-text field in the UI for per-download yt-dlp option overrides. Defaults to `false`. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for details and security considerations.
|
* __ALLOW_YTDL_OPTIONS_OVERRIDES__: Whether to show a free-text field in the UI for per-download yt-dlp option overrides. Defaults to `false`. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for details and security considerations.
|
||||||
|
* __YTDL_NIGHTLY_UPDATE_TIME__: If set, will cause MeTube to use [nightly yt-dlp builds](https://github.com/yt-dlp/yt-dlp-nightly-builds) instead of the stable releases. Set to the time (`HH:MM`, 24-hour) when you want the daily upgrades and MeTube restart to happen. Defaults to empty (disabled).
|
||||||
|
|
||||||
### 🌐 Web Server & URLs
|
### 🌐 Web Server & URLs
|
||||||
|
|
||||||
@@ -346,7 +347,7 @@ example.com {
|
|||||||
|
|
||||||
## 🔄 Updating yt-dlp
|
## 🔄 Updating yt-dlp
|
||||||
|
|
||||||
MeTube is powered by [yt-dlp](https://github.com/yt-dlp/yt-dlp), which requires frequent updates as video sites change their layouts. A nightly build automatically publishes a new Docker image whenever a new yt-dlp version is available, so keep your container up to date — [watchtower](https://github.com/nicholas-fedor/watchtower) works well for this.
|
MeTube is powered by [yt-dlp](https://github.com/yt-dlp/yt-dlp), which requires frequent updates as video sites change their layouts. A new MeTube Docker image is published automatically when a new yt-dlp stable release is available, so keep your container up to date — [watchtower](https://github.com/nicholas-fedor/watchtower) works well for this. To follow yt-dlp's nightly channel instead, set `YTDL_NIGHTLY_UPDATE_TIME`.
|
||||||
|
|
||||||
## 🔧 Troubleshooting and submitting issues
|
## 🔧 Troubleshooting and submitting issues
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,26 @@
|
|||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
log = logging.getLogger("bg_tasks")
|
||||||
|
_TASKS: set[asyncio.Task] = set()
|
||||||
|
|
||||||
|
|
||||||
|
def create_task(coro, *, name: str | None = None) -> asyncio.Task:
|
||||||
|
"""create_task that keeps a strong reference and logs unexpected failures.
|
||||||
|
|
||||||
|
A bare ``asyncio.create_task(...)`` is only weakly referenced by the event
|
||||||
|
loop; if nothing else holds the returned Task, it can be garbage collected
|
||||||
|
mid-flight. Keeping a module-level strong reference (removed once the task
|
||||||
|
finishes) avoids that, and the done-callback surfaces otherwise-silent
|
||||||
|
failures.
|
||||||
|
"""
|
||||||
|
task = asyncio.get_running_loop().create_task(coro, name=name)
|
||||||
|
_TASKS.add(task)
|
||||||
|
|
||||||
|
def _done(t: asyncio.Task) -> None:
|
||||||
|
_TASKS.discard(t)
|
||||||
|
if not t.cancelled() and t.exception() is not None:
|
||||||
|
log.error("Background task %s failed", t.get_name(), exc_info=t.exception())
|
||||||
|
|
||||||
|
task.add_done_callback(_done)
|
||||||
|
return task
|
||||||
+33
-1
@@ -3,6 +3,21 @@ import copy
|
|||||||
AUDIO_FORMATS = ("m4a", "mp3", "opus", "wav", "flac")
|
AUDIO_FORMATS = ("m4a", "mp3", "opus", "wav", "flac")
|
||||||
CAPTION_MODES = ("auto_only", "manual_only", "prefer_manual", "prefer_auto")
|
CAPTION_MODES = ("auto_only", "manual_only", "prefer_manual", "prefer_auto")
|
||||||
|
|
||||||
|
|
||||||
|
def merge_ytdl_option_layers(presets, overrides, presets_config) -> dict:
|
||||||
|
"""Overlay named presets (in order) then per-item overrides onto a fresh dict.
|
||||||
|
|
||||||
|
Does NOT include any base ``YTDL_OPTIONS`` — callers layer this on top of
|
||||||
|
their own base (a per-download build adds the global base; a subscription
|
||||||
|
scan relies on ``**config.YTDL_OPTIONS`` already being present in its
|
||||||
|
params). ``presets_config`` maps a preset name to its options dict.
|
||||||
|
"""
|
||||||
|
merged: dict = {}
|
||||||
|
for name in presets or []:
|
||||||
|
merged.update(presets_config.get(name, {}))
|
||||||
|
merged.update(overrides or {})
|
||||||
|
return merged
|
||||||
|
|
||||||
CODEC_FILTER_MAP = {
|
CODEC_FILTER_MAP = {
|
||||||
'h264': "[vcodec~='^(h264|avc)']",
|
'h264': "[vcodec~='^(h264|avc)']",
|
||||||
'h265': "[vcodec~='^(h265|hevc)']",
|
'h265': "[vcodec~='^(h265|hevc)']",
|
||||||
@@ -43,6 +58,10 @@ def get_format(download_type: str, codec: str, format: str, quality: str) -> str
|
|||||||
quality = (quality or "best").strip().lower()
|
quality = (quality or "best").strip().lower()
|
||||||
|
|
||||||
if format.startswith("custom:"):
|
if format.startswith("custom:"):
|
||||||
|
# Unreachable via the HTTP API (format is validated against a fixed
|
||||||
|
# set in main.py), but legacy persisted downloads may carry a
|
||||||
|
# custom: format from before that validation existed; removing this
|
||||||
|
# would crash PersistentQueue.load() for those records.
|
||||||
return format[7:]
|
return format[7:]
|
||||||
|
|
||||||
if download_type == "thumbnail":
|
if download_type == "thumbnail":
|
||||||
@@ -137,7 +156,20 @@ def get_opts(
|
|||||||
requested_subtitle_format = (format or "srt").lower()
|
requested_subtitle_format = (format or "srt").lower()
|
||||||
if requested_subtitle_format == "txt":
|
if requested_subtitle_format == "txt":
|
||||||
requested_subtitle_format = "srt"
|
requested_subtitle_format = "srt"
|
||||||
opts["subtitlesformat"] = requested_subtitle_format
|
opts["subtitlesformat"] = f"{requested_subtitle_format}/best"
|
||||||
|
if requested_subtitle_format in ("srt", "vtt"):
|
||||||
|
# subtitlesformat above is only a preference: if the extractor
|
||||||
|
# doesn't natively offer this ext (e.g. YouTube has no native srt),
|
||||||
|
# yt-dlp silently falls back to whatever it has. ffmpeg can only
|
||||||
|
# convert to srt/vtt/ass/lrc, so only guarantee the requested
|
||||||
|
# container for those; other formats stay best-effort.
|
||||||
|
postprocessors.append(
|
||||||
|
{
|
||||||
|
"key": "FFmpegSubtitlesConvertor",
|
||||||
|
"format": requested_subtitle_format,
|
||||||
|
"when": "before_dl",
|
||||||
|
}
|
||||||
|
)
|
||||||
if mode == "manual_only":
|
if mode == "manual_only":
|
||||||
opts["writesubtitles"] = True
|
opts["writesubtitles"] = True
|
||||||
opts["writeautomaticsub"] = False
|
opts["writeautomaticsub"] = False
|
||||||
|
|||||||
+180
-23
@@ -4,8 +4,10 @@
|
|||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
import asyncio
|
import asyncio
|
||||||
|
from datetime import datetime, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
from aiohttp.web import GracefulExit
|
||||||
from aiohttp.log import access_logger
|
from aiohttp.log import access_logger
|
||||||
import ssl
|
import ssl
|
||||||
import socket
|
import socket
|
||||||
@@ -14,15 +16,33 @@ import logging
|
|||||||
import json
|
import json
|
||||||
import pathlib
|
import pathlib
|
||||||
import re
|
import re
|
||||||
|
import time
|
||||||
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
||||||
from watchfiles import DefaultFilter, Change, awatch
|
from watchfiles import DefaultFilter, Change, awatch
|
||||||
|
|
||||||
|
import bg_tasks
|
||||||
from ytdl import DownloadQueueNotifier, DownloadQueue, Download
|
from ytdl import DownloadQueueNotifier, DownloadQueue, Download
|
||||||
from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo, coerce_optional_bool
|
from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo, coerce_optional_bool
|
||||||
from yt_dlp.version import __version__ as yt_dlp_version
|
from yt_dlp.version import __version__ as yt_dlp_version
|
||||||
|
|
||||||
log = logging.getLogger('main')
|
log = logging.getLogger('main')
|
||||||
|
|
||||||
|
_NIGHTLY_TIME_RE = re.compile(r'^([01]\d|2[0-3]):[0-5]\d$')
|
||||||
|
_RESTART_FOR_UPDATE = False
|
||||||
|
|
||||||
|
def _request_graceful_exit() -> None:
|
||||||
|
raise GracefulExit()
|
||||||
|
|
||||||
|
|
||||||
|
def seconds_until_next_daily_time(time_hhmm: str, now: datetime | None = None) -> float:
|
||||||
|
"""Seconds until the next occurrence of HH:MM in local time."""
|
||||||
|
now = now or datetime.now()
|
||||||
|
hour, minute = map(int, time_hhmm.split(':'))
|
||||||
|
target = now.replace(hour=hour, minute=minute, second=0, microsecond=0)
|
||||||
|
if target <= now:
|
||||||
|
target += timedelta(days=1)
|
||||||
|
return (target - now).total_seconds()
|
||||||
|
|
||||||
def parseLogLevel(logLevel):
|
def parseLogLevel(logLevel):
|
||||||
if not isinstance(logLevel, str):
|
if not isinstance(logLevel, str):
|
||||||
return None
|
return None
|
||||||
@@ -73,6 +93,7 @@ class Config:
|
|||||||
'MAX_CONCURRENT_DOWNLOADS': '3',
|
'MAX_CONCURRENT_DOWNLOADS': '3',
|
||||||
'LOGLEVEL': 'INFO',
|
'LOGLEVEL': 'INFO',
|
||||||
'ENABLE_ACCESSLOG': 'false',
|
'ENABLE_ACCESSLOG': 'false',
|
||||||
|
'YTDL_NIGHTLY_UPDATE_TIME': '',
|
||||||
}
|
}
|
||||||
|
|
||||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'ALLOW_YTDL_OPTIONS_OVERRIDES')
|
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'ALLOW_YTDL_OPTIONS_OVERRIDES')
|
||||||
@@ -93,6 +114,13 @@ class Config:
|
|||||||
if not self.URL_PREFIX.endswith('/'):
|
if not self.URL_PREFIX.endswith('/'):
|
||||||
self.URL_PREFIX += '/'
|
self.URL_PREFIX += '/'
|
||||||
|
|
||||||
|
# A blank PUBLIC_HOST_AUDIO_URL (e.g. set empty in a compose file) bypasses the
|
||||||
|
# default via os.environ.get, which would leave audio links root-relative and 404.
|
||||||
|
# Fall back to the 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR. When
|
||||||
|
# PUBLIC_HOST_URL is also blank we leave it blank to preserve serving from web root.
|
||||||
|
if not self.PUBLIC_HOST_AUDIO_URL and self.PUBLIC_HOST_URL:
|
||||||
|
self.PUBLIC_HOST_AUDIO_URL = self._DEFAULTS['PUBLIC_HOST_AUDIO_URL']
|
||||||
|
|
||||||
for attr in ('PUBLIC_HOST_URL', 'PUBLIC_HOST_AUDIO_URL'):
|
for attr in ('PUBLIC_HOST_URL', 'PUBLIC_HOST_AUDIO_URL'):
|
||||||
val = getattr(self, attr)
|
val = getattr(self, attr)
|
||||||
if val and not val.endswith('/'):
|
if val and not val.endswith('/'):
|
||||||
@@ -104,6 +132,21 @@ class Config:
|
|||||||
if self.YTDL_OPTIONS_PRESETS_FILE and self.YTDL_OPTIONS_PRESETS_FILE.startswith('.'):
|
if self.YTDL_OPTIONS_PRESETS_FILE and self.YTDL_OPTIONS_PRESETS_FILE.startswith('.'):
|
||||||
self.YTDL_OPTIONS_PRESETS_FILE = str(Path(self.YTDL_OPTIONS_PRESETS_FILE).resolve())
|
self.YTDL_OPTIONS_PRESETS_FILE = str(Path(self.YTDL_OPTIONS_PRESETS_FILE).resolve())
|
||||||
|
|
||||||
|
if self.YTDL_NIGHTLY_UPDATE_TIME and not _NIGHTLY_TIME_RE.match(self.YTDL_NIGHTLY_UPDATE_TIME):
|
||||||
|
log.error(
|
||||||
|
'Environment variable "YTDL_NIGHTLY_UPDATE_TIME" must be HH:MM (24-hour), got "%s"',
|
||||||
|
self.YTDL_NIGHTLY_UPDATE_TIME,
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
self._validate_int('MAX_CONCURRENT_DOWNLOADS', minimum=1)
|
||||||
|
self._validate_int('PORT', minimum=1, maximum=65535)
|
||||||
|
self._validate_int('CLEAR_COMPLETED_AFTER', minimum=0)
|
||||||
|
self._validate_int('DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT', minimum=0)
|
||||||
|
self._validate_int('SUBSCRIPTION_DEFAULT_CHECK_INTERVAL', minimum=1)
|
||||||
|
self._validate_int('SUBSCRIPTION_SCAN_PLAYLIST_END', minimum=1)
|
||||||
|
self._validate_int('SUBSCRIPTION_MAX_SEEN_IDS', minimum=1)
|
||||||
|
|
||||||
self._runtime_overrides = {}
|
self._runtime_overrides = {}
|
||||||
|
|
||||||
success,_ = self.load_ytdl_options()
|
success,_ = self.load_ytdl_options()
|
||||||
@@ -113,6 +156,20 @@ class Config:
|
|||||||
if not success:
|
if not success:
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
|
def _validate_int(self, key, *, minimum=None, maximum=None):
|
||||||
|
raw = getattr(self, key)
|
||||||
|
try:
|
||||||
|
value = int(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
log.error('Environment variable "%s" must be an integer, got "%s"', key, raw)
|
||||||
|
sys.exit(1)
|
||||||
|
if minimum is not None and value < minimum:
|
||||||
|
log.error('Environment variable "%s" must be >= %d, got "%s"', key, minimum, raw)
|
||||||
|
sys.exit(1)
|
||||||
|
if maximum is not None and value > maximum:
|
||||||
|
log.error('Environment variable "%s" must be <= %d, got "%s"', key, maximum, raw)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
def set_runtime_override(self, key, value):
|
def set_runtime_override(self, key, value):
|
||||||
self._runtime_overrides[key] = value
|
self._runtime_overrides[key] = value
|
||||||
self.YTDL_OPTIONS[key] = value
|
self.YTDL_OPTIONS[key] = value
|
||||||
@@ -215,7 +272,13 @@ logging.getLogger().setLevel(parseLogLevel(str(config.LOGLEVEL)) or logging.INFO
|
|||||||
|
|
||||||
class ObjectSerializer(json.JSONEncoder):
|
class ObjectSerializer(json.JSONEncoder):
|
||||||
def default(self, obj):
|
def default(self, obj):
|
||||||
# First try to use __dict__ for custom objects
|
# Prefer an explicit client-facing view when the object provides one
|
||||||
|
# (e.g. DownloadInfo / SubscriptionInfo) so server-only or bulky fields
|
||||||
|
# are never broadcast to browser clients.
|
||||||
|
to_public = getattr(obj, 'to_public_dict', None)
|
||||||
|
if callable(to_public):
|
||||||
|
return to_public()
|
||||||
|
# Fall back to __dict__ for other custom objects
|
||||||
if hasattr(obj, '__dict__'):
|
if hasattr(obj, '__dict__'):
|
||||||
return obj.__dict__
|
return obj.__dict__
|
||||||
# Convert iterables (generators, dict_items, etc.) to lists
|
# Convert iterables (generators, dict_items, etc.) to lists
|
||||||
@@ -229,7 +292,33 @@ class ObjectSerializer(json.JSONEncoder):
|
|||||||
return json.JSONEncoder.default(self, obj)
|
return json.JSONEncoder.default(self, obj)
|
||||||
|
|
||||||
serializer = ObjectSerializer()
|
serializer = ObjectSerializer()
|
||||||
app = web.Application()
|
|
||||||
|
_STATE_DIR_REAL = os.path.realpath(config.STATE_DIR)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_within_state_dir(real_target: str) -> bool:
|
||||||
|
return real_target == _STATE_DIR_REAL or real_target.startswith(_STATE_DIR_REAL + os.sep)
|
||||||
|
|
||||||
|
|
||||||
|
@web.middleware
|
||||||
|
async def state_dir_guard(request, handler):
|
||||||
|
for prefix, base in (
|
||||||
|
(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR),
|
||||||
|
(config.URL_PREFIX + 'audio_download/', config.AUDIO_DOWNLOAD_DIR),
|
||||||
|
):
|
||||||
|
if request.path.startswith(prefix):
|
||||||
|
# request.path is already percent-decoded by aiohttp; decoding it
|
||||||
|
# again would mangle a download whose filename contains a literal
|
||||||
|
# '%' (e.g. "%" turning into a truncated escape) into a false 404.
|
||||||
|
rel = request.path[len(prefix):]
|
||||||
|
target = os.path.realpath(os.path.join(base, rel))
|
||||||
|
if _is_within_state_dir(target):
|
||||||
|
raise web.HTTPNotFound()
|
||||||
|
break
|
||||||
|
return await handler(request)
|
||||||
|
|
||||||
|
|
||||||
|
app = web.Application(middlewares=[state_dir_guard])
|
||||||
_cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
|
_cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
|
||||||
sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
|
sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
|
||||||
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
||||||
@@ -345,12 +434,20 @@ def _clip_field_provided_in_post(raw) -> bool:
|
|||||||
|
|
||||||
|
|
||||||
def _extract_t_query_from_url(url: str) -> tuple[str, float | None]:
|
def _extract_t_query_from_url(url: str) -> tuple[str, float | None]:
|
||||||
"""If ``t=`` is present and parseable, return URL without ``t`` and start seconds."""
|
"""If ``t=`` is present and parseable, return URL without ``t`` and start seconds.
|
||||||
|
|
||||||
|
Restricted to YouTube hosts: ``t`` is a generic query parameter name that
|
||||||
|
other sites may use for unrelated purposes, so rewriting it there would
|
||||||
|
silently mutate the URL and inject a bogus clip start.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
parsed = urlparse(url)
|
parsed = urlparse(url)
|
||||||
params = parse_qs(parsed.query)
|
params = parse_qs(parsed.query)
|
||||||
except Exception:
|
except Exception:
|
||||||
return url, None
|
return url, None
|
||||||
|
host = (parsed.hostname or '').lower()
|
||||||
|
if not (host in ('youtu.be', 'youtube.com') or host.endswith('.youtube.com')):
|
||||||
|
return url, None
|
||||||
t_values = params.get('t')
|
t_values = params.get('t')
|
||||||
if not t_values:
|
if not t_values:
|
||||||
return url, None
|
return url, None
|
||||||
@@ -465,8 +562,19 @@ class Notifier(DownloadQueueNotifier):
|
|||||||
await sio.emit('cleared', serializer.encode(id))
|
await sio.emit('cleared', serializer.encode(id))
|
||||||
|
|
||||||
dqueue = DownloadQueue(config, Notifier())
|
dqueue = DownloadQueue(config, Notifier())
|
||||||
app.on_startup.append(lambda app: dqueue.initialize())
|
|
||||||
app.on_cleanup.append(lambda app: Download.shutdown_manager())
|
|
||||||
|
async def _download_queue_startup(app):
|
||||||
|
await dqueue.initialize()
|
||||||
|
|
||||||
|
|
||||||
|
async def _shutdown_download_manager(app):
|
||||||
|
dqueue.close()
|
||||||
|
Download.shutdown_manager()
|
||||||
|
|
||||||
|
|
||||||
|
app.on_startup.append(_download_queue_startup)
|
||||||
|
app.on_cleanup.append(_shutdown_download_manager)
|
||||||
|
|
||||||
|
|
||||||
class MetubeSubscriptionNotifier(SubscriptionNotifier):
|
class MetubeSubscriptionNotifier(SubscriptionNotifier):
|
||||||
@@ -486,7 +594,13 @@ class MetubeSubscriptionNotifier(SubscriptionNotifier):
|
|||||||
|
|
||||||
|
|
||||||
submgr = SubscriptionManager(config, dqueue, MetubeSubscriptionNotifier())
|
submgr = SubscriptionManager(config, dqueue, MetubeSubscriptionNotifier())
|
||||||
app.on_cleanup.append(lambda app: submgr.close())
|
|
||||||
|
|
||||||
|
async def _shutdown_subscriptions(app):
|
||||||
|
submgr.close()
|
||||||
|
|
||||||
|
|
||||||
|
app.on_cleanup.append(_shutdown_subscriptions)
|
||||||
|
|
||||||
|
|
||||||
async def _subscription_loop_startup(app):
|
async def _subscription_loop_startup(app):
|
||||||
@@ -496,6 +610,26 @@ async def _subscription_loop_startup(app):
|
|||||||
|
|
||||||
app.on_startup.append(_subscription_loop_startup)
|
app.on_startup.append(_subscription_loop_startup)
|
||||||
|
|
||||||
|
|
||||||
|
async def _schedule_nightly_update() -> None:
|
||||||
|
global _RESTART_FOR_UPDATE
|
||||||
|
time_hhmm = config.YTDL_NIGHTLY_UPDATE_TIME
|
||||||
|
if not time_hhmm:
|
||||||
|
return
|
||||||
|
delay = seconds_until_next_daily_time(time_hhmm)
|
||||||
|
log.info('Next yt-dlp nightly update in %.0f seconds (at %s local time)', delay, time_hhmm)
|
||||||
|
await asyncio.sleep(delay)
|
||||||
|
log.info('Scheduled yt-dlp nightly update: requesting restart')
|
||||||
|
_RESTART_FOR_UPDATE = True
|
||||||
|
asyncio.get_running_loop().call_soon(_request_graceful_exit)
|
||||||
|
|
||||||
|
|
||||||
|
async def _start_nightly_update_schedule(app):
|
||||||
|
bg_tasks.create_task(_schedule_nightly_update(), name="nightly_update_schedule")
|
||||||
|
|
||||||
|
|
||||||
|
app.on_startup.append(_start_nightly_update_schedule)
|
||||||
|
|
||||||
class FileOpsFilter(DefaultFilter):
|
class FileOpsFilter(DefaultFilter):
|
||||||
def __call__(self, change_type: int, path: str) -> bool:
|
def __call__(self, change_type: int, path: str) -> bool:
|
||||||
# Check if this path matches our YTDL_OPTIONS_FILE
|
# Check if this path matches our YTDL_OPTIONS_FILE
|
||||||
@@ -540,10 +674,14 @@ async def watch_files():
|
|||||||
await sio.emit('ytdl_options_changed', serializer.encode(result))
|
await sio.emit('ytdl_options_changed', serializer.encode(result))
|
||||||
|
|
||||||
log.info(f'Starting Watch File: {config.YTDL_OPTIONS_FILE}')
|
log.info(f'Starting Watch File: {config.YTDL_OPTIONS_FILE}')
|
||||||
asyncio.create_task(_watch_files())
|
bg_tasks.create_task(_watch_files(), name="watch_ytdl_options_file")
|
||||||
|
|
||||||
|
async def _watch_files_startup(app):
|
||||||
|
await watch_files()
|
||||||
|
|
||||||
|
|
||||||
if config.YTDL_OPTIONS_FILE:
|
if config.YTDL_OPTIONS_FILE:
|
||||||
app.on_startup.append(lambda app: watch_files())
|
app.on_startup.append(_watch_files_startup)
|
||||||
|
|
||||||
|
|
||||||
async def _read_json_request(request: web.Request) -> dict:
|
async def _read_json_request(request: web.Request) -> dict:
|
||||||
@@ -761,10 +899,7 @@ async def cancel_add(request):
|
|||||||
@routes.post(config.URL_PREFIX + 'subscribe')
|
@routes.post(config.URL_PREFIX + 'subscribe')
|
||||||
async def subscribe(request):
|
async def subscribe(request):
|
||||||
post = await _read_json_request(request)
|
post = await _read_json_request(request)
|
||||||
try:
|
|
||||||
o = parse_download_options(post)
|
o = parse_download_options(post)
|
||||||
except web.HTTPBadRequest:
|
|
||||||
raise
|
|
||||||
cic = post.get('check_interval_minutes')
|
cic = post.get('check_interval_minutes')
|
||||||
if cic is None:
|
if cic is None:
|
||||||
cic = config.SUBSCRIPTION_DEFAULT_CHECK_INTERVAL
|
cic = config.SUBSCRIPTION_DEFAULT_CHECK_INTERVAL
|
||||||
@@ -853,13 +988,20 @@ async def subscriptions_check(request):
|
|||||||
result = await submgr.check_now([str(i) for i in ids] if ids else None)
|
result = await submgr.check_now([str(i) for i in ids] if ids else None)
|
||||||
return web.Response(text=serializer.encode(result))
|
return web.Response(text=serializer.encode(result))
|
||||||
|
|
||||||
|
def _require_id_list(post: dict) -> list:
|
||||||
|
ids = post.get('ids')
|
||||||
|
if not isinstance(ids, list) or not ids or not all(isinstance(i, str) for i in ids):
|
||||||
|
raise web.HTTPBadRequest(reason="'ids' must be a non-empty list of strings")
|
||||||
|
return ids
|
||||||
|
|
||||||
|
|
||||||
@routes.post(config.URL_PREFIX + 'delete')
|
@routes.post(config.URL_PREFIX + 'delete')
|
||||||
async def delete(request):
|
async def delete(request):
|
||||||
post = await _read_json_request(request)
|
post = await _read_json_request(request)
|
||||||
ids = post.get('ids')
|
ids = _require_id_list(post)
|
||||||
where = post.get('where')
|
where = post.get('where')
|
||||||
if not ids or where not in ['queue', 'done']:
|
if where not in ['queue', 'done']:
|
||||||
log.error("Bad request: missing 'ids' or incorrect 'where' value")
|
log.error("Bad request: incorrect 'where' value")
|
||||||
raise web.HTTPBadRequest()
|
raise web.HTTPBadRequest()
|
||||||
status = await (dqueue.cancel(ids) if where == 'queue' else dqueue.clear(ids))
|
status = await (dqueue.cancel(ids) if where == 'queue' else dqueue.clear(ids))
|
||||||
log.info(f"Download delete request processed for ids: {ids}, where: {where}")
|
log.info(f"Download delete request processed for ids: {ids}, where: {where}")
|
||||||
@@ -868,7 +1010,7 @@ async def delete(request):
|
|||||||
@routes.post(config.URL_PREFIX + 'start')
|
@routes.post(config.URL_PREFIX + 'start')
|
||||||
async def start(request):
|
async def start(request):
|
||||||
post = await _read_json_request(request)
|
post = await _read_json_request(request)
|
||||||
ids = post.get('ids')
|
ids = _require_id_list(post)
|
||||||
log.info(f"Received request to start pending downloads for ids: {ids}")
|
log.info(f"Received request to start pending downloads for ids: {ids}")
|
||||||
status = await dqueue.start_pending(ids)
|
status = await dqueue.start_pending(ids)
|
||||||
return web.Response(text=serializer.encode(status))
|
return web.Response(text=serializer.encode(status))
|
||||||
@@ -898,6 +1040,12 @@ async def upload_cookies(request):
|
|||||||
tmp_cookie_path = f"{COOKIES_PATH}.tmp"
|
tmp_cookie_path = f"{COOKIES_PATH}.tmp"
|
||||||
with open(tmp_cookie_path, 'wb') as f:
|
with open(tmp_cookie_path, 'wb') as f:
|
||||||
f.write(content)
|
f.write(content)
|
||||||
|
# Cookies are sensitive auth material; restrict to owner read/write only
|
||||||
|
# (the container's default umask would otherwise leave them group/world readable).
|
||||||
|
try:
|
||||||
|
os.chmod(tmp_cookie_path, 0o600)
|
||||||
|
except OSError as exc:
|
||||||
|
log.warning(f'Could not restrict permissions on cookies file: {exc}')
|
||||||
os.replace(tmp_cookie_path, COOKIES_PATH)
|
os.replace(tmp_cookie_path, COOKIES_PATH)
|
||||||
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
||||||
log.info(f'Cookies file uploaded ({size} bytes)')
|
log.info(f'Cookies file uploaded ({size} bytes)')
|
||||||
@@ -942,12 +1090,15 @@ async def cookie_status(request):
|
|||||||
async def history(request):
|
async def history(request):
|
||||||
history = { 'done': [], 'queue': [], 'pending': []}
|
history = { 'done': [], 'queue': [], 'pending': []}
|
||||||
|
|
||||||
for _, v in dqueue.queue.saved_items():
|
# Served from the in-memory queues (like the socket 'all' event) rather
|
||||||
history['queue'].append(v)
|
# than saved_items(), which reloads and re-compacts the on-disk state on
|
||||||
for _, v in dqueue.done.saved_items():
|
# every call.
|
||||||
history['done'].append(v)
|
for _, v in dqueue.queue.items():
|
||||||
for _, v in dqueue.pending.saved_items():
|
history['queue'].append(v.info)
|
||||||
history['pending'].append(v)
|
for _, v in dqueue.done.items():
|
||||||
|
history['done'].append(v.info)
|
||||||
|
for _, v in dqueue.pending.items():
|
||||||
|
history['pending'].append(v.info)
|
||||||
|
|
||||||
log.info("Sending download history")
|
log.info("Sending download history")
|
||||||
return web.Response(text=serializer.encode(history))
|
return web.Response(text=serializer.encode(history))
|
||||||
@@ -959,13 +1110,17 @@ async def connect(sid, environ):
|
|||||||
await sio.emit('subscriptions_all', serializer.encode([s.to_public_dict() for s in submgr.list_all()]), to=sid)
|
await sio.emit('subscriptions_all', serializer.encode([s.to_public_dict() for s in submgr.list_all()]), to=sid)
|
||||||
await sio.emit('configuration', serializer.encode(config.frontend_safe()), to=sid)
|
await sio.emit('configuration', serializer.encode(config.frontend_safe()), to=sid)
|
||||||
if config.CUSTOM_DIRS:
|
if config.CUSTOM_DIRS:
|
||||||
await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)
|
# get_custom_dirs() can walk the whole download tree on a cache miss;
|
||||||
|
# keep that off the event loop so a large library doesn't stall every
|
||||||
|
# client's connect handshake.
|
||||||
|
dirs = await asyncio.get_running_loop().run_in_executor(None, get_custom_dirs)
|
||||||
|
await sio.emit('custom_dirs', serializer.encode(dirs), to=sid)
|
||||||
if config.YTDL_OPTIONS_FILE:
|
if config.YTDL_OPTIONS_FILE:
|
||||||
await sio.emit('ytdl_options_changed', serializer.encode(get_options_update_time()), to=sid)
|
await sio.emit('ytdl_options_changed', serializer.encode(get_options_update_time()), to=sid)
|
||||||
|
|
||||||
def get_custom_dirs():
|
def get_custom_dirs():
|
||||||
cache_ttl_seconds = 5
|
cache_ttl_seconds = 5
|
||||||
now = asyncio.get_running_loop().time()
|
now = time.monotonic()
|
||||||
cache_key = (
|
cache_key = (
|
||||||
config.DOWNLOAD_DIR,
|
config.DOWNLOAD_DIR,
|
||||||
config.AUDIO_DOWNLOAD_DIR,
|
config.AUDIO_DOWNLOAD_DIR,
|
||||||
@@ -1122,3 +1277,5 @@ if __name__ == '__main__':
|
|||||||
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), ssl_context=ssl_context, access_log=isAccessLogEnabled())
|
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), ssl_context=ssl_context, access_log=isAccessLogEnabled())
|
||||||
else:
|
else:
|
||||||
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), access_log=isAccessLogEnabled())
|
web.run_app(app, host=config.HOST, port=int(config.PORT), reuse_port=supports_reuse_port(), access_log=isAccessLogEnabled())
|
||||||
|
if _RESTART_FOR_UPDATE:
|
||||||
|
sys.exit(42)
|
||||||
|
|||||||
+84
-3
@@ -2,6 +2,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import base64
|
import base64
|
||||||
import collections.abc
|
import collections.abc
|
||||||
|
import errno
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
@@ -17,6 +18,25 @@ STATE_SCHEMA_VERSION = 2
|
|||||||
_BYTES_MARKER = "__metube_bytes__"
|
_BYTES_MARKER = "__metube_bytes__"
|
||||||
_DATETIME_MARKER = "__metube_datetime__"
|
_DATETIME_MARKER = "__metube_datetime__"
|
||||||
|
|
||||||
|
# Errnos that signal the filesystem cannot support the temp-file + rename
|
||||||
|
# atomic-write strategy (for example an NFS-backed state dir returning EPERM on
|
||||||
|
# mkstemp). These are safe to fall back on because they mean the atomic
|
||||||
|
# mechanism is unavailable, not that the data write itself failed. Errors like
|
||||||
|
# ENOSPC/EIO are deliberately excluded so a genuine storage failure surfaces
|
||||||
|
# instead of silently truncating an existing good state file.
|
||||||
|
_ATOMIC_UNSUPPORTED_ERRNOS = frozenset(
|
||||||
|
e
|
||||||
|
for e in (
|
||||||
|
errno.EPERM,
|
||||||
|
errno.EACCES,
|
||||||
|
errno.ENOSYS,
|
||||||
|
errno.EINVAL,
|
||||||
|
getattr(errno, "EOPNOTSUPP", None),
|
||||||
|
getattr(errno, "ENOTSUP", None),
|
||||||
|
)
|
||||||
|
if e is not None
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def to_json_compatible(value: Any) -> Any:
|
def to_json_compatible(value: Any) -> Any:
|
||||||
if value is None or isinstance(value, (bool, int, float, str)):
|
if value is None or isinstance(value, (bool, int, float, str)):
|
||||||
@@ -62,6 +82,7 @@ class AtomicJsonStore:
|
|||||||
self.path = path
|
self.path = path
|
||||||
self.kind = kind
|
self.kind = kind
|
||||||
self.schema_version = schema_version
|
self.schema_version = schema_version
|
||||||
|
self._direct_write_fallback_warned = False
|
||||||
|
|
||||||
def _ensure_parent(self) -> None:
|
def _ensure_parent(self) -> None:
|
||||||
parent = os.path.dirname(self.path)
|
parent = os.path.dirname(self.path)
|
||||||
@@ -96,6 +117,16 @@ class AtomicJsonStore:
|
|||||||
def save(self, data: dict[str, Any]) -> None:
|
def save(self, data: dict[str, Any]) -> None:
|
||||||
self._ensure_parent()
|
self._ensure_parent()
|
||||||
payload = self._build_payload(data)
|
payload = self._build_payload(data)
|
||||||
|
try:
|
||||||
|
self._atomic_write(payload)
|
||||||
|
except OSError as exc:
|
||||||
|
if exc.errno not in _ATOMIC_UNSUPPORTED_ERRNOS:
|
||||||
|
raise
|
||||||
|
self._warn_direct_write_fallback(exc)
|
||||||
|
self._direct_write(payload)
|
||||||
|
|
||||||
|
def _atomic_write(self, payload: dict[str, Any]) -> None:
|
||||||
|
text = self._serialize(payload)
|
||||||
parent = os.path.dirname(self.path) or "."
|
parent = os.path.dirname(self.path) or "."
|
||||||
fd, tmp_path = tempfile.mkstemp(
|
fd, tmp_path = tempfile.mkstemp(
|
||||||
prefix=f".{os.path.basename(self.path)}.",
|
prefix=f".{os.path.basename(self.path)}.",
|
||||||
@@ -105,10 +136,9 @@ class AtomicJsonStore:
|
|||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||||
json.dump(payload, f, ensure_ascii=False, separators=(",", ":"))
|
f.write(text)
|
||||||
f.write("\n")
|
|
||||||
f.flush()
|
f.flush()
|
||||||
os.fsync(f.fileno())
|
self._best_effort_fsync(f.fileno())
|
||||||
os.replace(tmp_path, self.path)
|
os.replace(tmp_path, self.path)
|
||||||
self._fsync_directory(parent)
|
self._fsync_directory(parent)
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -118,6 +148,57 @@ class AtomicJsonStore:
|
|||||||
pass
|
pass
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
def _direct_write(self, payload: dict[str, Any]) -> None:
|
||||||
|
# Serialize before truncating so a serialization failure never destroys
|
||||||
|
# the existing state file (the atomic path gets this for free via its
|
||||||
|
# temp file).
|
||||||
|
text = self._serialize(payload)
|
||||||
|
# Create with 0o600 so the fallback keeps the owner-only permissions the
|
||||||
|
# atomic path gets from mkstemp; state files can contain URLs and
|
||||||
|
# per-download option overrides that must not leak on shared mounts.
|
||||||
|
fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||||
|
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||||
|
# The 0o600 mode above only applies when the file is created; force
|
||||||
|
# it on rewrites too so an existing, broadly-permissioned state file
|
||||||
|
# is tightened to match the atomic path. Best-effort because some
|
||||||
|
# network filesystems reject chmod, and that must not re-crash save.
|
||||||
|
try:
|
||||||
|
os.fchmod(f.fileno(), 0o600)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
f.write(text)
|
||||||
|
f.flush()
|
||||||
|
self._best_effort_fsync(f.fileno())
|
||||||
|
# Make the new directory entry durable too, matching the atomic path.
|
||||||
|
self._fsync_directory(os.path.dirname(self.path) or ".")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _best_effort_fsync(fileno: int) -> None:
|
||||||
|
# Tolerate fsync being unsupported on the underlying filesystem (for
|
||||||
|
# example a network mount that returns EINVAL/ENOSYS), but let genuine
|
||||||
|
# storage failures such as ENOSPC/EIO surface so a non-durable write is
|
||||||
|
# never reported as success. An unsupported fsync must not by itself
|
||||||
|
# abandon the atomic rename path.
|
||||||
|
try:
|
||||||
|
os.fsync(fileno)
|
||||||
|
except OSError as exc:
|
||||||
|
if exc.errno not in _ATOMIC_UNSUPPORTED_ERRNOS:
|
||||||
|
raise
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _serialize(payload: dict[str, Any]) -> str:
|
||||||
|
return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + "\n"
|
||||||
|
|
||||||
|
def _warn_direct_write_fallback(self, exc: OSError) -> None:
|
||||||
|
if self._direct_write_fallback_warned:
|
||||||
|
return
|
||||||
|
self._direct_write_fallback_warned = True
|
||||||
|
log.warning(
|
||||||
|
"Atomic state write failed for %s (%s); falling back to direct write",
|
||||||
|
self.path,
|
||||||
|
exc,
|
||||||
|
)
|
||||||
|
|
||||||
def quarantine_invalid_file(self, exc: Exception) -> None:
|
def quarantine_invalid_file(self, exc: Exception) -> None:
|
||||||
if not os.path.exists(self.path):
|
if not os.path.exists(self.path):
|
||||||
return
|
return
|
||||||
|
|||||||
+135
-19
@@ -11,14 +11,22 @@ import time
|
|||||||
import types
|
import types
|
||||||
import uuid
|
import uuid
|
||||||
from dataclasses import dataclass, field, fields
|
from dataclasses import dataclass, field, fields
|
||||||
|
from functools import partial
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
import yt_dlp
|
import yt_dlp
|
||||||
import yt_dlp.networking.impersonate
|
import yt_dlp.networking.impersonate
|
||||||
|
import bg_tasks
|
||||||
|
from dl_formats import merge_ytdl_option_layers
|
||||||
from state_store import AtomicJsonStore, read_legacy_shelf
|
from state_store import AtomicJsonStore, read_legacy_shelf
|
||||||
|
|
||||||
log = logging.getLogger("subscriptions")
|
log = logging.getLogger("subscriptions")
|
||||||
|
|
||||||
|
# How many subscription feeds to scan at once. Bounded so one slow/hung feed
|
||||||
|
# doesn't serialize the rest, without bursting a large subscription list at the
|
||||||
|
# extractor (which risks rate-limiting / bot detection).
|
||||||
|
_MAX_CONCURRENT_CHECKS = 4
|
||||||
|
|
||||||
VIDEO_ONLY_MSG = (
|
VIDEO_ONLY_MSG = (
|
||||||
"This URL points to a single video, not a channel or playlist. Use Download instead."
|
"This URL points to a single video, not a channel or playlist. Use Download instead."
|
||||||
)
|
)
|
||||||
@@ -42,7 +50,9 @@ def _impersonate_opt(ytdl_options: dict) -> dict:
|
|||||||
return opts
|
return opts
|
||||||
|
|
||||||
|
|
||||||
def _build_ydl_params(config, *, playlistend: Optional[int] = None) -> dict:
|
def _build_ydl_params(
|
||||||
|
config, *, playlistend: Optional[int] = None, extra_opts: Optional[dict[str, Any]] = None
|
||||||
|
) -> dict:
|
||||||
params: dict[str, Any] = {
|
params: dict[str, Any] = {
|
||||||
"quiet": not logging.getLogger().isEnabledFor(logging.DEBUG),
|
"quiet": not logging.getLogger().isEnabledFor(logging.DEBUG),
|
||||||
"verbose": logging.getLogger().isEnabledFor(logging.DEBUG),
|
"verbose": logging.getLogger().isEnabledFor(logging.DEBUG),
|
||||||
@@ -52,6 +62,7 @@ def _build_ydl_params(config, *, playlistend: Optional[int] = None) -> dict:
|
|||||||
"lazy_playlist": True,
|
"lazy_playlist": True,
|
||||||
"paths": {"home": config.DOWNLOAD_DIR, "temp": config.TEMP_DIR},
|
"paths": {"home": config.DOWNLOAD_DIR, "temp": config.TEMP_DIR},
|
||||||
**config.YTDL_OPTIONS,
|
**config.YTDL_OPTIONS,
|
||||||
|
**(extra_opts or {}),
|
||||||
}
|
}
|
||||||
params = _impersonate_opt(params)
|
params = _impersonate_opt(params)
|
||||||
if playlistend is not None and playlistend > 0:
|
if playlistend is not None and playlistend > 0:
|
||||||
@@ -76,9 +87,11 @@ def _is_media_entry(entry: Any) -> bool:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
|
|
||||||
def extract_flat_playlist(config, url: str, playlistend: int, *, _depth: int = 0):
|
def extract_flat_playlist(
|
||||||
|
config, url: str, playlistend: int, *, extra_opts: Optional[dict[str, Any]] = None, _depth: int = 0
|
||||||
|
):
|
||||||
"""Return (info_dict, entries_list) for playlist/channel URLs."""
|
"""Return (info_dict, entries_list) for playlist/channel URLs."""
|
||||||
params = _build_ydl_params(config, playlistend=playlistend)
|
params = _build_ydl_params(config, playlistend=playlistend, extra_opts=extra_opts)
|
||||||
with yt_dlp.YoutubeDL(params=params) as ydl:
|
with yt_dlp.YoutubeDL(params=params) as ydl:
|
||||||
info = ydl.extract_info(url, download=False)
|
info = ydl.extract_info(url, download=False)
|
||||||
if not info:
|
if not info:
|
||||||
@@ -104,6 +117,7 @@ def extract_flat_playlist(config, url: str, playlistend: int, *, _depth: int = 0
|
|||||||
config,
|
config,
|
||||||
nested_url,
|
nested_url,
|
||||||
playlistend,
|
playlistend,
|
||||||
|
extra_opts=extra_opts,
|
||||||
_depth=_depth + 1,
|
_depth=_depth + 1,
|
||||||
)
|
)
|
||||||
if nested_entries:
|
if nested_entries:
|
||||||
@@ -312,6 +326,7 @@ class SubscriptionManager:
|
|||||||
self._subs: dict[str, SubscriptionInfo] = {}
|
self._subs: dict[str, SubscriptionInfo] = {}
|
||||||
self._url_index: dict[str, str] = {} # normalized url -> id
|
self._url_index: dict[str, str] = {} # normalized url -> id
|
||||||
self._pending_urls: set[str] = set()
|
self._pending_urls: set[str] = set()
|
||||||
|
self._checks_in_flight: set[str] = set() # subscription ids being checked right now
|
||||||
self._lock = asyncio.Lock()
|
self._lock = asyncio.Lock()
|
||||||
self._loop_task: Optional[asyncio.Task] = None
|
self._loop_task: Optional[asyncio.Task] = None
|
||||||
self._load_all()
|
self._load_all()
|
||||||
@@ -369,6 +384,23 @@ class SubscriptionManager:
|
|||||||
def _save_locked(self) -> None:
|
def _save_locked(self) -> None:
|
||||||
self._store.save({"items": [_subscription_to_record(sub) for sub in self._subs.values()]})
|
self._store.save({"items": [_subscription_to_record(sub) for sub in self._subs.values()]})
|
||||||
|
|
||||||
|
def _scan_extra_opts(
|
||||||
|
self,
|
||||||
|
ytdl_options_presets: Optional[list[str]],
|
||||||
|
ytdl_options_overrides: Optional[dict[str, Any]],
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Merge configured presets (in order) with per-subscription overrides.
|
||||||
|
|
||||||
|
Applied on top of the global YTDL_OPTIONS when scanning a
|
||||||
|
subscription's feed, so cookies/impersonation/etc. configured via a
|
||||||
|
preset or override also take effect during the flat-playlist scan,
|
||||||
|
not just the eventual per-video download. (The global YTDL_OPTIONS base
|
||||||
|
is already spread into the scan params by ``_build_ydl_params``.)
|
||||||
|
"""
|
||||||
|
return merge_ytdl_option_layers(
|
||||||
|
ytdl_options_presets, ytdl_options_overrides, self.config.YTDL_OPTIONS_PRESETS
|
||||||
|
)
|
||||||
|
|
||||||
async def _queue_subscription_entries(
|
async def _queue_subscription_entries(
|
||||||
self,
|
self,
|
||||||
entries: list[dict],
|
entries: list[dict],
|
||||||
@@ -435,12 +467,8 @@ class SubscriptionManager:
|
|||||||
def start_background_loop(self) -> None:
|
def start_background_loop(self) -> None:
|
||||||
if self._loop_task is not None and not self._loop_task.done():
|
if self._loop_task is not None and not self._loop_task.done():
|
||||||
return
|
return
|
||||||
self._loop_task = asyncio.create_task(self._periodic_loop())
|
# bg_tasks.create_task already logs unexpected task failures with the name.
|
||||||
self._loop_task.add_done_callback(
|
self._loop_task = bg_tasks.create_task(self._periodic_loop(), name="subscription_loop")
|
||||||
lambda t: log.error("Subscription loop failed: %s", t.exception())
|
|
||||||
if not t.cancelled() and t.exception()
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _periodic_loop(self) -> None:
|
async def _periodic_loop(self) -> None:
|
||||||
while True:
|
while True:
|
||||||
@@ -464,9 +492,31 @@ class SubscriptionManager:
|
|||||||
if now - sub.last_checked < interval_sec:
|
if now - sub.last_checked < interval_sec:
|
||||||
continue
|
continue
|
||||||
due.append(sub)
|
due.append(sub)
|
||||||
for sub in due:
|
await self._check_many(due)
|
||||||
|
|
||||||
|
async def _check_many(self, subs: list[SubscriptionInfo]) -> None:
|
||||||
|
"""Check subscriptions with bounded concurrency so one slow feed does
|
||||||
|
not serialize the rest. Failures are isolated per subscription."""
|
||||||
|
if not subs:
|
||||||
|
return
|
||||||
|
sem = asyncio.Semaphore(_MAX_CONCURRENT_CHECKS)
|
||||||
|
|
||||||
|
async def _guarded(sub: SubscriptionInfo) -> None:
|
||||||
|
async with sem:
|
||||||
await self._check_one_unlocked(sub)
|
await self._check_one_unlocked(sub)
|
||||||
|
|
||||||
|
results = await asyncio.gather(
|
||||||
|
*(_guarded(sub) for sub in subs), return_exceptions=True
|
||||||
|
)
|
||||||
|
for sub, result in zip(subs, results):
|
||||||
|
if isinstance(result, Exception):
|
||||||
|
log.error(
|
||||||
|
"Subscription check crashed for %s: %s",
|
||||||
|
sub.name,
|
||||||
|
result,
|
||||||
|
exc_info=result,
|
||||||
|
)
|
||||||
|
|
||||||
async def add_subscription(
|
async def add_subscription(
|
||||||
self,
|
self,
|
||||||
url: str,
|
url: str,
|
||||||
@@ -512,8 +562,12 @@ class SubscriptionManager:
|
|||||||
|
|
||||||
try:
|
try:
|
||||||
scan_first = max(int(getattr(self.config, "SUBSCRIPTION_SCAN_PLAYLIST_END", 50)), 1)
|
scan_first = max(int(getattr(self.config, "SUBSCRIPTION_SCAN_PLAYLIST_END", 50)), 1)
|
||||||
|
scan_extra_opts = self._scan_extra_opts(ytdl_options_presets, ytdl_options_overrides)
|
||||||
try:
|
try:
|
||||||
info, entries = extract_flat_playlist(self.config, url, scan_first)
|
info, entries = await asyncio.get_running_loop().run_in_executor(
|
||||||
|
None,
|
||||||
|
partial(extract_flat_playlist, self.config, url, scan_first, extra_opts=scan_extra_opts),
|
||||||
|
)
|
||||||
except yt_dlp.utils.YoutubeDLError as exc:
|
except yt_dlp.utils.YoutubeDLError as exc:
|
||||||
return {"status": "error", "msg": str(exc)}
|
return {"status": "error", "msg": str(exc)}
|
||||||
|
|
||||||
@@ -628,6 +682,24 @@ class SubscriptionManager:
|
|||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
return {"status": "error", "msg": str(exc)}
|
return {"status": "error", "msg": str(exc)}
|
||||||
|
|
||||||
|
enabled_set = False
|
||||||
|
validated_enabled = False
|
||||||
|
if "enabled" in changes:
|
||||||
|
try:
|
||||||
|
validated_enabled = _coerce_bool(changes["enabled"])
|
||||||
|
enabled_set = True
|
||||||
|
except ValueError as exc:
|
||||||
|
return {"status": "error", "msg": str(exc)}
|
||||||
|
|
||||||
|
interval_set = False
|
||||||
|
validated_interval = 0
|
||||||
|
if "check_interval_minutes" in changes:
|
||||||
|
try:
|
||||||
|
validated_interval = max(1, int(changes["check_interval_minutes"]))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return {"status": "error", "msg": "check_interval_minutes must be an integer"}
|
||||||
|
interval_set = True
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
sub = self._subs.get(sub_id)
|
sub = self._subs.get(sub_id)
|
||||||
if not sub:
|
if not sub:
|
||||||
@@ -635,10 +707,10 @@ class SubscriptionManager:
|
|||||||
previous = copy.deepcopy(sub)
|
previous = copy.deepcopy(sub)
|
||||||
old_enabled = sub.enabled
|
old_enabled = sub.enabled
|
||||||
|
|
||||||
if "enabled" in changes:
|
if enabled_set:
|
||||||
sub.enabled = _coerce_bool(changes["enabled"])
|
sub.enabled = validated_enabled
|
||||||
if "check_interval_minutes" in changes:
|
if interval_set:
|
||||||
sub.check_interval_minutes = max(1, int(changes["check_interval_minutes"]))
|
sub.check_interval_minutes = validated_interval
|
||||||
if "name" in changes and changes["name"]:
|
if "name" in changes and changes["name"]:
|
||||||
sub.name = str(changes["name"])
|
sub.name = str(changes["name"])
|
||||||
if validated_tr is not None:
|
if validated_tr is not None:
|
||||||
@@ -672,22 +744,45 @@ class SubscriptionManager:
|
|||||||
"Manual subscription check requested for %d subscription(s)",
|
"Manual subscription check requested for %d subscription(s)",
|
||||||
len(targets),
|
len(targets),
|
||||||
)
|
)
|
||||||
for sub in targets:
|
await self._check_many(targets)
|
||||||
await self._check_one_unlocked(sub)
|
|
||||||
return {"status": "ok"}
|
return {"status": "ok"}
|
||||||
|
|
||||||
async def _check_one_unlocked(self, sub: SubscriptionInfo) -> None:
|
async def _check_one_unlocked(self, sub: SubscriptionInfo) -> None:
|
||||||
|
sid = sub.id
|
||||||
|
# Prevent overlapping checks for the same subscription (e.g. the periodic
|
||||||
|
# loop and a manual check-now firing together), which could double-queue
|
||||||
|
# entries and drop seen_ids via a read-modify-write race.
|
||||||
|
async with self._lock:
|
||||||
|
if sid in self._checks_in_flight:
|
||||||
|
log.info("Subscription check already in progress for %s, skipping", sub.name)
|
||||||
|
return
|
||||||
|
self._checks_in_flight.add(sid)
|
||||||
|
try:
|
||||||
|
await self._check_one_inner(sub)
|
||||||
|
finally:
|
||||||
|
async with self._lock:
|
||||||
|
self._checks_in_flight.discard(sid)
|
||||||
|
|
||||||
|
async def _check_one_inner(self, sub: SubscriptionInfo) -> None:
|
||||||
sid = sub.id
|
sid = sub.id
|
||||||
scan = int(getattr(self.config, "SUBSCRIPTION_SCAN_PLAYLIST_END", 50))
|
scan = int(getattr(self.config, "SUBSCRIPTION_SCAN_PLAYLIST_END", 50))
|
||||||
|
# ytdl_options_presets/overrides are set at subscription creation and
|
||||||
|
# never mutated afterwards (update_subscription doesn't allow it), so
|
||||||
|
# reading them off `sub` here without holding the lock is safe.
|
||||||
|
scan_extra_opts = self._scan_extra_opts(sub.ytdl_options_presets, sub.ytdl_options_overrides)
|
||||||
log.info("Checking subscription: %s", sub.name)
|
log.info("Checking subscription: %s", sub.name)
|
||||||
try:
|
try:
|
||||||
info, entries = extract_flat_playlist(self.config, sub.url, scan)
|
info, entries = await asyncio.get_running_loop().run_in_executor(
|
||||||
|
None,
|
||||||
|
partial(extract_flat_playlist, self.config, sub.url, scan, extra_opts=scan_extra_opts),
|
||||||
|
)
|
||||||
except yt_dlp.utils.YoutubeDLError as exc:
|
except yt_dlp.utils.YoutubeDLError as exc:
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
cur = self._subs.get(sid)
|
cur = self._subs.get(sid)
|
||||||
if cur:
|
if cur:
|
||||||
previous = copy.deepcopy(cur)
|
previous = copy.deepcopy(cur)
|
||||||
cur.error = str(exc)
|
cur.error = str(exc)
|
||||||
|
cur.last_checked = time.time()
|
||||||
try:
|
try:
|
||||||
self._save_locked()
|
self._save_locked()
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -700,12 +795,13 @@ class SubscriptionManager:
|
|||||||
entries = [ent for ent in entries if _is_media_entry(ent)]
|
entries = [ent for ent in entries if _is_media_entry(ent)]
|
||||||
|
|
||||||
etype = (info or {}).get("_type") or "video"
|
etype = (info or {}).get("_type") or "video"
|
||||||
if etype == "video" or not entries:
|
if etype == "video":
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
cur = self._subs.get(sid)
|
cur = self._subs.get(sid)
|
||||||
if cur:
|
if cur:
|
||||||
previous = copy.deepcopy(cur)
|
previous = copy.deepcopy(cur)
|
||||||
cur.error = VIDEO_ONLY_MSG
|
cur.error = VIDEO_ONLY_MSG
|
||||||
|
cur.last_checked = time.time()
|
||||||
try:
|
try:
|
||||||
self._save_locked()
|
self._save_locked()
|
||||||
except Exception:
|
except Exception:
|
||||||
@@ -715,6 +811,22 @@ class SubscriptionManager:
|
|||||||
log.warning("Subscription %s no longer resolves to a subscribable feed", sub.name)
|
log.warning("Subscription %s no longer resolves to a subscribable feed", sub.name)
|
||||||
await self.notifier.subscription_updated(sub)
|
await self.notifier.subscription_updated(sub)
|
||||||
return
|
return
|
||||||
|
if not entries:
|
||||||
|
async with self._lock:
|
||||||
|
cur = self._subs.get(sid)
|
||||||
|
if cur:
|
||||||
|
previous = copy.deepcopy(cur)
|
||||||
|
cur.last_checked = time.time()
|
||||||
|
cur.error = None
|
||||||
|
try:
|
||||||
|
self._save_locked()
|
||||||
|
except Exception:
|
||||||
|
self._subs[sid] = previous
|
||||||
|
raise
|
||||||
|
sub = cur
|
||||||
|
log.warning("Subscription check finished for %s: No entries found", sub.name)
|
||||||
|
await self.notifier.subscription_updated(sub)
|
||||||
|
return
|
||||||
|
|
||||||
async with self._lock:
|
async with self._lock:
|
||||||
cur = self._subs.get(sid)
|
cur = self._subs.get(sid)
|
||||||
@@ -744,6 +856,10 @@ class SubscriptionManager:
|
|||||||
eid = _entry_id(ent)
|
eid = _entry_id(ent)
|
||||||
if not eid:
|
if not eid:
|
||||||
continue
|
continue
|
||||||
|
# Seen entries that are currently live are deliberately re-queued:
|
||||||
|
# a stream first seen as 'upcoming' must still be captured once it
|
||||||
|
# goes live. The download queue dedups by URL while a capture is
|
||||||
|
# in flight, so this can't double-queue an active capture.
|
||||||
if eid in seen and ent.get("live_status") != "is_live":
|
if eid in seen and ent.get("live_status") != "is_live":
|
||||||
continue
|
continue
|
||||||
new_entries.append(ent)
|
new_entries.append(ent)
|
||||||
|
|||||||
+125
-3
@@ -3,10 +3,14 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
from unittest.mock import AsyncMock, MagicMock
|
from unittest.mock import AsyncMock, MagicMock
|
||||||
|
from urllib.parse import quote
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from aiohttp import web
|
from aiohttp import web
|
||||||
|
from aiohttp.test_utils import TestClient, TestServer
|
||||||
|
|
||||||
import main
|
import main
|
||||||
|
|
||||||
@@ -17,6 +21,7 @@ def mock_dqueue(monkeypatch):
|
|||||||
d.initialize = AsyncMock(return_value=None)
|
d.initialize = AsyncMock(return_value=None)
|
||||||
d.add = AsyncMock(return_value={"status": "ok"})
|
d.add = AsyncMock(return_value={"status": "ok"})
|
||||||
d.cancel = AsyncMock(return_value={"status": "ok"})
|
d.cancel = AsyncMock(return_value={"status": "ok"})
|
||||||
|
d.clear = AsyncMock(return_value={"status": "ok"})
|
||||||
d.start_pending = AsyncMock(return_value={"status": "ok"})
|
d.start_pending = AsyncMock(return_value={"status": "ok"})
|
||||||
d.cancel_add = MagicMock()
|
d.cancel_add = MagicMock()
|
||||||
d.queue = MagicMock()
|
d.queue = MagicMock()
|
||||||
@@ -25,6 +30,9 @@ def mock_dqueue(monkeypatch):
|
|||||||
d.queue.saved_items = MagicMock(return_value=[])
|
d.queue.saved_items = MagicMock(return_value=[])
|
||||||
d.done.saved_items = MagicMock(return_value=[])
|
d.done.saved_items = MagicMock(return_value=[])
|
||||||
d.pending.saved_items = MagicMock(return_value=[])
|
d.pending.saved_items = MagicMock(return_value=[])
|
||||||
|
d.queue.items = MagicMock(return_value=[])
|
||||||
|
d.done.items = MagicMock(return_value=[])
|
||||||
|
d.pending.items = MagicMock(return_value=[])
|
||||||
d.get = MagicMock(return_value=([], []))
|
d.get = MagicMock(return_value=([], []))
|
||||||
monkeypatch.setattr(main, "dqueue", d)
|
monkeypatch.setattr(main, "dqueue", d)
|
||||||
return d
|
return d
|
||||||
@@ -212,11 +220,35 @@ async def test_start_pending(mock_dqueue):
|
|||||||
mock_dqueue.start_pending.assert_awaited_once_with(["a"])
|
mock_dqueue.start_pending.assert_awaited_once_with(["a"])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize("body", [{}, {"ids": "abc"}, {"ids": []}, {"ids": [1, 2]}])
|
||||||
|
async def test_start_rejects_malformed_ids(mock_dqueue, body):
|
||||||
|
req = _json_request(body)
|
||||||
|
with pytest.raises(web.HTTPBadRequest):
|
||||||
|
await main.start(req)
|
||||||
|
mock_dqueue.start_pending.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"body",
|
||||||
|
[
|
||||||
|
{"where": "queue"},
|
||||||
|
{"where": "queue", "ids": "abc"},
|
||||||
|
{"where": "queue", "ids": []},
|
||||||
|
{"where": "queue", "ids": [1, 2]},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
async def test_delete_rejects_malformed_ids(mock_dqueue, body):
|
||||||
|
req = _json_request(body)
|
||||||
|
with pytest.raises(web.HTTPBadRequest):
|
||||||
|
await main.delete(req)
|
||||||
|
mock_dqueue.cancel.assert_not_awaited()
|
||||||
|
mock_dqueue.clear.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_history_shape(mock_dqueue):
|
async def test_history_shape(mock_dqueue):
|
||||||
mock_dqueue.queue.saved_items.return_value = []
|
|
||||||
mock_dqueue.done.saved_items.return_value = []
|
|
||||||
mock_dqueue.pending.saved_items.return_value = []
|
|
||||||
req = MagicMock(spec=web.Request)
|
req = MagicMock(spec=web.Request)
|
||||||
resp = await main.history(req)
|
resp = await main.history(req)
|
||||||
assert resp.status == 200
|
assert resp.status == 200
|
||||||
@@ -224,6 +256,30 @@ async def test_history_shape(mock_dqueue):
|
|||||||
assert set(data.keys()) == {"done", "queue", "pending"}
|
assert set(data.keys()) == {"done", "queue", "pending"}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_history_reads_in_memory_queues_not_disk_state(mock_dqueue):
|
||||||
|
fake_queue_dl = MagicMock()
|
||||||
|
fake_queue_dl.info = {"id": "q1", "title": "Queued"}
|
||||||
|
fake_done_dl = MagicMock()
|
||||||
|
fake_done_dl.info = {"id": "d1", "title": "Done"}
|
||||||
|
fake_pending_dl = MagicMock()
|
||||||
|
fake_pending_dl.info = {"id": "p1", "title": "Pending"}
|
||||||
|
mock_dqueue.queue.items.return_value = [("q1", fake_queue_dl)]
|
||||||
|
mock_dqueue.done.items.return_value = [("d1", fake_done_dl)]
|
||||||
|
mock_dqueue.pending.items.return_value = [("p1", fake_pending_dl)]
|
||||||
|
|
||||||
|
req = MagicMock(spec=web.Request)
|
||||||
|
resp = await main.history(req)
|
||||||
|
assert resp.status == 200
|
||||||
|
data = json.loads(resp.text)
|
||||||
|
assert [item["id"] for item in data["queue"]] == ["q1"]
|
||||||
|
assert [item["id"] for item in data["done"]] == ["d1"]
|
||||||
|
assert [item["id"] for item in data["pending"]] == ["p1"]
|
||||||
|
mock_dqueue.queue.saved_items.assert_not_called()
|
||||||
|
mock_dqueue.done.saved_items.assert_not_called()
|
||||||
|
mock_dqueue.pending.saved_items.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_version_json(mock_dqueue):
|
async def test_version_json(mock_dqueue):
|
||||||
req = MagicMock(spec=web.Request)
|
req = MagicMock(spec=web.Request)
|
||||||
@@ -306,3 +362,69 @@ async def test_subscribe_rejects_clip_options(mock_dqueue, monkeypatch):
|
|||||||
with pytest.raises(web.HTTPBadRequest):
|
with pytest.raises(web.HTTPBadRequest):
|
||||||
await main.subscribe(req)
|
await main.subscribe(req)
|
||||||
main.submgr.add_subscription.assert_not_awaited()
|
main.submgr.add_subscription.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscriptions_update_invalid_enabled_returns_error_not_500(mock_dqueue):
|
||||||
|
req = _json_request({"id": "nonexistent", "enabled": "maybe"})
|
||||||
|
resp = await main.subscriptions_update(req)
|
||||||
|
assert resp.status == 200
|
||||||
|
body = json.loads(resp.text)
|
||||||
|
assert body["status"] == "error"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_subscriptions_update_invalid_interval_returns_error_not_500(mock_dqueue):
|
||||||
|
req = _json_request({"id": "nonexistent", "check_interval_minutes": "abc"})
|
||||||
|
resp = await main.subscriptions_update(req)
|
||||||
|
assert resp.status == 200
|
||||||
|
body = json.loads(resp.text)
|
||||||
|
assert body["status"] == "error"
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_within_state_dir_blocks_state_subtree():
|
||||||
|
state_dir = main._STATE_DIR_REAL
|
||||||
|
assert main._is_within_state_dir(state_dir)
|
||||||
|
assert main._is_within_state_dir(os.path.join(state_dir, "cookies.txt"))
|
||||||
|
assert main._is_within_state_dir(os.path.join(state_dir, "queue", "item.json"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_within_state_dir_allows_sibling_downloads():
|
||||||
|
download_dir = os.path.realpath(main.config.DOWNLOAD_DIR)
|
||||||
|
assert not main._is_within_state_dir(os.path.join(download_dir, "video.mp4"))
|
||||||
|
assert not main._is_within_state_dir("/tmp/unrelated/video.mp4")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_download_blocks_state_dir_files(monkeypatch):
|
||||||
|
download_dir = Path(main.config.DOWNLOAD_DIR)
|
||||||
|
state_dir = download_dir / ".metube"
|
||||||
|
state_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
(state_dir / "cookies.txt").write_text("# Netscape HTTP Cookie File\n", encoding="utf-8")
|
||||||
|
(download_dir / "video.mp4").write_bytes(b"video")
|
||||||
|
# request.path is already percent-decoded by aiohttp; state_dir_guard must
|
||||||
|
# not decode it a second time, or a filename containing a literal '%'
|
||||||
|
# gets mangled into a false 404.
|
||||||
|
percent_filename = "100% done.mp4"
|
||||||
|
(download_dir / percent_filename).write_bytes(b"percent video")
|
||||||
|
|
||||||
|
monkeypatch.setattr(main.config, "STATE_DIR", str(state_dir))
|
||||||
|
monkeypatch.setattr(main, "_STATE_DIR_REAL", os.path.realpath(str(state_dir)))
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with TestClient(TestServer(main.app)) as client:
|
||||||
|
blocked = await client.get("/download/.metube/cookies.txt")
|
||||||
|
assert blocked.status == 404
|
||||||
|
|
||||||
|
allowed = await client.get("/download/video.mp4")
|
||||||
|
assert allowed.status == 200
|
||||||
|
assert await allowed.read() == b"video"
|
||||||
|
|
||||||
|
percent_resp = await client.get("/download/" + quote(percent_filename))
|
||||||
|
assert percent_resp.status == 200
|
||||||
|
assert await percent_resp.read() == b"percent video"
|
||||||
|
finally:
|
||||||
|
(state_dir / "cookies.txt").unlink(missing_ok=True)
|
||||||
|
(download_dir / "video.mp4").unlink(missing_ok=True)
|
||||||
|
(download_dir / percent_filename).unlink(missing_ok=True)
|
||||||
|
state_dir.rmdir()
|
||||||
|
|||||||
@@ -0,0 +1,55 @@
|
|||||||
|
"""Tests for the ``bg_tasks.create_task`` strong-reference/logging helper."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import bg_tasks
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_create_task_removes_itself_from_registry_on_success():
|
||||||
|
async def _ok():
|
||||||
|
return 42
|
||||||
|
|
||||||
|
task = bg_tasks.create_task(_ok(), name="ok_task")
|
||||||
|
assert task in bg_tasks._TASKS
|
||||||
|
result = await task
|
||||||
|
assert result == 42
|
||||||
|
assert task not in bg_tasks._TASKS
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_create_task_logs_unhandled_exception(caplog):
|
||||||
|
async def _boom():
|
||||||
|
raise ValueError("kaboom")
|
||||||
|
|
||||||
|
with caplog.at_level(logging.ERROR, logger="bg_tasks"):
|
||||||
|
task = bg_tasks.create_task(_boom(), name="boom_task")
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
await task
|
||||||
|
# Let the done-callback (scheduled via call_soon) run.
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
assert task not in bg_tasks._TASKS
|
||||||
|
assert any("boom_task" in record.message for record in caplog.records)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_create_task_does_not_log_on_cancellation(caplog):
|
||||||
|
async def _sleep_forever():
|
||||||
|
await asyncio.sleep(10)
|
||||||
|
|
||||||
|
with caplog.at_level(logging.ERROR, logger="bg_tasks"):
|
||||||
|
task = bg_tasks.create_task(_sleep_forever(), name="cancel_task")
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
task.cancel()
|
||||||
|
with pytest.raises(asyncio.CancelledError):
|
||||||
|
await task
|
||||||
|
await asyncio.sleep(0)
|
||||||
|
|
||||||
|
assert task not in bg_tasks._TASKS
|
||||||
|
assert not any("cancel_task" in record.message for record in caplog.records)
|
||||||
@@ -51,6 +51,19 @@ class ConfigTests(unittest.TestCase):
|
|||||||
self.assertEqual(c.PUBLIC_HOST_URL, "")
|
self.assertEqual(c.PUBLIC_HOST_URL, "")
|
||||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "")
|
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "")
|
||||||
|
|
||||||
|
def test_blank_audio_host_falls_back_to_audio_download_route(self):
|
||||||
|
# Regression: a present-but-blank PUBLIC_HOST_AUDIO_URL must not stay empty
|
||||||
|
# (which produced root-relative, 404ing audio links). It falls back to the
|
||||||
|
# 'audio_download/' route that serves AUDIO_DOWNLOAD_DIR.
|
||||||
|
with patch.dict(
|
||||||
|
os.environ,
|
||||||
|
_base_env(PUBLIC_HOST_URL="https://ytdl.example.com", PUBLIC_HOST_AUDIO_URL=""),
|
||||||
|
clear=False,
|
||||||
|
):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
||||||
|
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "audio_download/")
|
||||||
|
|
||||||
def test_public_host_url_already_slashed_unchanged(self):
|
def test_public_host_url_already_slashed_unchanged(self):
|
||||||
with patch.dict(
|
with patch.dict(
|
||||||
os.environ,
|
os.environ,
|
||||||
@@ -107,6 +120,74 @@ class ConfigTests(unittest.TestCase):
|
|||||||
c = Config()
|
c = Config()
|
||||||
self.assertTrue(c.ALLOW_YTDL_OPTIONS_OVERRIDES)
|
self.assertTrue(c.ALLOW_YTDL_OPTIONS_OVERRIDES)
|
||||||
|
|
||||||
|
def test_ytdl_nightly_update_time_empty_default(self):
|
||||||
|
with patch.dict(os.environ, _base_env(YTDL_NIGHTLY_UPDATE_TIME=""), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.YTDL_NIGHTLY_UPDATE_TIME, "")
|
||||||
|
|
||||||
|
def test_ytdl_nightly_update_time_valid(self):
|
||||||
|
with patch.dict(os.environ, _base_env(YTDL_NIGHTLY_UPDATE_TIME="04:00"), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.YTDL_NIGHTLY_UPDATE_TIME, "04:00")
|
||||||
|
|
||||||
|
def test_ytdl_nightly_update_time_invalid_exits(self):
|
||||||
|
for bad in ("25:00", "4am", "12:60"):
|
||||||
|
with patch.dict(os.environ, _base_env(YTDL_NIGHTLY_UPDATE_TIME=bad), clear=False):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
Config()
|
||||||
|
|
||||||
|
def test_invalid_max_concurrent_downloads_exits(self):
|
||||||
|
for bad in ("0", "-1", "abc"):
|
||||||
|
with patch.dict(os.environ, _base_env(MAX_CONCURRENT_DOWNLOADS=bad), clear=False):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
Config()
|
||||||
|
|
||||||
|
def test_invalid_port_exits(self):
|
||||||
|
for bad in ("0", "70000", "notaport"):
|
||||||
|
with patch.dict(os.environ, _base_env(PORT=bad), clear=False):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
Config()
|
||||||
|
|
||||||
|
def test_invalid_clear_completed_after_exits(self):
|
||||||
|
for bad in ("-5", "soon"):
|
||||||
|
with patch.dict(os.environ, _base_env(CLEAR_COMPLETED_AFTER=bad), clear=False):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
Config()
|
||||||
|
|
||||||
|
def test_clear_completed_after_zero_allowed(self):
|
||||||
|
with patch.dict(os.environ, _base_env(CLEAR_COMPLETED_AFTER="0"), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.CLEAR_COMPLETED_AFTER, "0")
|
||||||
|
|
||||||
|
def test_invalid_default_option_playlist_item_limit_exits(self):
|
||||||
|
for bad in ("-1", "many"):
|
||||||
|
with patch.dict(os.environ, _base_env(DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT=bad), clear=False):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
Config()
|
||||||
|
|
||||||
|
def test_default_option_playlist_item_limit_zero_allowed(self):
|
||||||
|
with patch.dict(os.environ, _base_env(DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT="0"), clear=False):
|
||||||
|
c = Config()
|
||||||
|
self.assertEqual(c.DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT, "0")
|
||||||
|
|
||||||
|
def test_invalid_subscription_default_check_interval_exits(self):
|
||||||
|
for bad in ("0", "-1", "often"):
|
||||||
|
with patch.dict(os.environ, _base_env(SUBSCRIPTION_DEFAULT_CHECK_INTERVAL=bad), clear=False):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
Config()
|
||||||
|
|
||||||
|
def test_invalid_subscription_scan_playlist_end_exits(self):
|
||||||
|
for bad in ("0", "-1", "all"):
|
||||||
|
with patch.dict(os.environ, _base_env(SUBSCRIPTION_SCAN_PLAYLIST_END=bad), clear=False):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
Config()
|
||||||
|
|
||||||
|
def test_invalid_subscription_max_seen_ids_exits(self):
|
||||||
|
for bad in ("0", "-1", "unlimited"):
|
||||||
|
with patch.dict(os.environ, _base_env(SUBSCRIPTION_MAX_SEEN_IDS=bad), clear=False):
|
||||||
|
with self.assertRaises(SystemExit):
|
||||||
|
Config()
|
||||||
|
|
||||||
def test_runtime_override_roundtrip(self):
|
def test_runtime_override_roundtrip(self):
|
||||||
with patch.dict(os.environ, _base_env(), clear=False):
|
with patch.dict(os.environ, _base_env(), clear=False):
|
||||||
c = Config()
|
c = Config()
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from app.dl_formats import (
|
|||||||
_normalize_subtitle_language,
|
_normalize_subtitle_language,
|
||||||
get_format,
|
get_format,
|
||||||
get_opts,
|
get_opts,
|
||||||
|
merge_ytdl_option_layers,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -118,7 +119,31 @@ class DlFormatsTests(unittest.TestCase):
|
|||||||
|
|
||||||
def test_get_opts_captions_txt_maps_to_srt_format(self):
|
def test_get_opts_captions_txt_maps_to_srt_format(self):
|
||||||
opts = get_opts("captions", "auto", "txt", "best", {})
|
opts = get_opts("captions", "auto", "txt", "best", {})
|
||||||
self.assertEqual(opts["subtitlesformat"], "srt")
|
self.assertEqual(opts["subtitlesformat"], "srt/best")
|
||||||
|
keys = [p["key"] for p in opts["postprocessors"]]
|
||||||
|
self.assertIn("FFmpegSubtitlesConvertor", keys)
|
||||||
|
convertor = next(p for p in opts["postprocessors"] if p["key"] == "FFmpegSubtitlesConvertor")
|
||||||
|
self.assertEqual(convertor["format"], "srt")
|
||||||
|
|
||||||
|
def test_get_opts_captions_srt_guarantees_convertor(self):
|
||||||
|
opts = get_opts("captions", "auto", "srt", "best", {})
|
||||||
|
self.assertEqual(opts["subtitlesformat"], "srt/best")
|
||||||
|
keys = [p["key"] for p in opts["postprocessors"]]
|
||||||
|
self.assertIn("FFmpegSubtitlesConvertor", keys)
|
||||||
|
|
||||||
|
def test_get_opts_captions_vtt_guarantees_convertor(self):
|
||||||
|
opts = get_opts("captions", "auto", "vtt", "best", {})
|
||||||
|
self.assertEqual(opts["subtitlesformat"], "vtt/best")
|
||||||
|
keys = [p["key"] for p in opts["postprocessors"]]
|
||||||
|
self.assertIn("FFmpegSubtitlesConvertor", keys)
|
||||||
|
convertor = next(p for p in opts["postprocessors"] if p["key"] == "FFmpegSubtitlesConvertor")
|
||||||
|
self.assertEqual(convertor["format"], "vtt")
|
||||||
|
|
||||||
|
def test_get_opts_captions_ttml_has_no_convertor(self):
|
||||||
|
opts = get_opts("captions", "auto", "ttml", "best", {})
|
||||||
|
self.assertEqual(opts["subtitlesformat"], "ttml/best")
|
||||||
|
keys = [p["key"] for p in opts["postprocessors"]]
|
||||||
|
self.assertNotIn("FFmpegSubtitlesConvertor", keys)
|
||||||
|
|
||||||
def test_get_opts_merges_existing_postprocessors(self):
|
def test_get_opts_merges_existing_postprocessors(self):
|
||||||
opts = get_opts("audio", "auto", "opus", "best", {"postprocessors": [{"key": "SponsorBlock"}]})
|
opts = get_opts("audio", "auto", "opus", "best", {"postprocessors": [{"key": "SponsorBlock"}]})
|
||||||
@@ -135,5 +160,23 @@ class DlFormatsTests(unittest.TestCase):
|
|||||||
self.assertEqual(_normalize_subtitle_language(" "), "en")
|
self.assertEqual(_normalize_subtitle_language(" "), "en")
|
||||||
|
|
||||||
|
|
||||||
|
class MergeYtdlOptionLayersTests(unittest.TestCase):
|
||||||
|
def test_presets_applied_in_order_then_overrides(self):
|
||||||
|
presets_config = {
|
||||||
|
"a": {"x": 1, "y": 1},
|
||||||
|
"b": {"y": 2, "z": 2},
|
||||||
|
}
|
||||||
|
merged = merge_ytdl_option_layers(["a", "b"], {"z": 3, "w": 4}, presets_config)
|
||||||
|
# b overrides a's y; explicit overrides win over presets.
|
||||||
|
self.assertEqual(merged, {"x": 1, "y": 2, "z": 3, "w": 4})
|
||||||
|
|
||||||
|
def test_no_base_options_included(self):
|
||||||
|
# The helper only produces the preset/override layer, never base opts.
|
||||||
|
self.assertEqual(merge_ytdl_option_layers(None, None, {}), {})
|
||||||
|
|
||||||
|
def test_unknown_preset_names_ignored(self):
|
||||||
|
self.assertEqual(merge_ytdl_option_layers(["missing"], {"a": 1}, {}), {"a": 1})
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -3,12 +3,14 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import tempfile
|
import tempfile
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
import time
|
||||||
|
|
||||||
from ytdl import DownloadQueue
|
from ytdl import Download, DownloadInfo, DownloadQueue
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -45,6 +47,37 @@ def test_cancel_add_increments_generation(dq_env):
|
|||||||
assert dq._add_generation == before + 1
|
assert dq._add_generation == before + 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_queue_has_dedicated_executor_sized_from_config(dq_env):
|
||||||
|
notifier = MagicMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
assert dq._download_executor is not None
|
||||||
|
assert dq._download_executor._max_workers == 2 * int(dq_env.MAX_CONCURRENT_DOWNLOADS) + 2
|
||||||
|
dq.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_close_cancels_running_downloads_before_shutdown(dq_env):
|
||||||
|
notifier = MagicMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
|
||||||
|
running = MagicMock()
|
||||||
|
running.started.return_value = True
|
||||||
|
running.running.return_value = True
|
||||||
|
idle = MagicMock()
|
||||||
|
idle.started.return_value = False
|
||||||
|
idle.running.return_value = False
|
||||||
|
|
||||||
|
dq.queue.dict["u-running"] = running
|
||||||
|
dq.queue.dict["u-idle"] = idle
|
||||||
|
|
||||||
|
dq.close()
|
||||||
|
|
||||||
|
# The active download's subprocess group is killed; the not-started one is
|
||||||
|
# left alone. Executor is shut down afterwards.
|
||||||
|
running.cancel.assert_called_once()
|
||||||
|
idle.cancel.assert_not_called()
|
||||||
|
assert dq._download_executor._shutdown
|
||||||
|
|
||||||
|
|
||||||
def test_get_returns_tuple_of_lists(dq_env):
|
def test_get_returns_tuple_of_lists(dq_env):
|
||||||
notifier = MagicMock()
|
notifier = MagicMock()
|
||||||
dq = DownloadQueue(dq_env, notifier)
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
@@ -221,6 +254,155 @@ async def test_add_entry_queues_single_video_without_reextracting(dq_env):
|
|||||||
assert dq.pending.exists("https://example.com/watch?v=1")
|
assert dq.pending.exists("https://example.com/watch?v=1")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
entry = {
|
||||||
|
"_type": "video",
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Original Title",
|
||||||
|
"url": "https://example.com/watch?v=1",
|
||||||
|
"webpage_url": "https://example.com/watch?v=1",
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")):
|
||||||
|
first = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=False)
|
||||||
|
assert first["status"] == "ok"
|
||||||
|
assert "msg" not in first
|
||||||
|
|
||||||
|
dupe_entry = {**entry, "title": "Different Title"}
|
||||||
|
second = await dq.add_entry(dupe_entry, "audio", "auto", "mp3", "best", "", "", 0, auto_start=False)
|
||||||
|
|
||||||
|
assert second["status"] == "ok"
|
||||||
|
assert "Already in queue" in second["msg"]
|
||||||
|
# The original pending download's options must survive untouched.
|
||||||
|
pending_dl = dq.pending.get("https://example.com/watch?v=1")
|
||||||
|
assert pending_dl.info.download_type == "video"
|
||||||
|
assert pending_dl.info.title == "Original Title"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_add_entry_duplicate_while_queued_is_skipped(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
entry = {
|
||||||
|
"_type": "video",
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Test Video",
|
||||||
|
"url": "https://example.com/watch?v=1",
|
||||||
|
"webpage_url": "https://example.com/watch?v=1",
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")), \
|
||||||
|
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||||
|
first = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=True)
|
||||||
|
assert first["status"] == "ok"
|
||||||
|
assert dq.queue.exists("https://example.com/watch?v=1")
|
||||||
|
|
||||||
|
second = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=True)
|
||||||
|
|
||||||
|
assert second["status"] == "ok"
|
||||||
|
assert "Already in queue" in second["msg"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_channel_download_uses_output_template_when_channel_template_empty(dq_env):
|
||||||
|
"""Channel tabs reported as playlists must honor OUTPUT_TEMPLATE when OUTPUT_TEMPLATE_CHANNEL is empty."""
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq_env.OUTPUT_TEMPLATE = "%(channel)s [YT]/%(title)s.%(ext)s"
|
||||||
|
dq_env.OUTPUT_TEMPLATE_CHANNEL = ""
|
||||||
|
dq_env.OUTPUT_TEMPLATE_PLAYLIST = ""
|
||||||
|
|
||||||
|
channel_id = "UCabcd123"
|
||||||
|
|
||||||
|
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||||
|
return {
|
||||||
|
"_type": "playlist",
|
||||||
|
"id": channel_id,
|
||||||
|
"channel_id": channel_id,
|
||||||
|
"channel": "Odin",
|
||||||
|
"title": "Odin - Videos",
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Salvia Plath - Pondering",
|
||||||
|
"url": "https://example.com/watch?v=1",
|
||||||
|
"webpage_url": "https://example.com/watch?v=1",
|
||||||
|
"channel": "Odin",
|
||||||
|
"upload_date": "20130804",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||||
|
result = await dq.add(
|
||||||
|
"https://www.youtube.com/@odin/videos",
|
||||||
|
"video",
|
||||||
|
"auto",
|
||||||
|
"any",
|
||||||
|
"best",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
0,
|
||||||
|
auto_start=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
url = "https://example.com/watch?v=1"
|
||||||
|
assert dq.pending.exists(url)
|
||||||
|
download = dq.pending.get(url)
|
||||||
|
assert download.output_template.startswith("Odin [YT]/")
|
||||||
|
assert "Odin - Videos" not in download.output_template
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_playlist_download_not_treated_as_channel(dq_env):
|
||||||
|
"""Real playlists (id != channel_id) must not be promoted to channel downloads."""
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq_env.OUTPUT_TEMPLATE = "%(channel)s [YT]/%(title)s.%(ext)s"
|
||||||
|
dq_env.OUTPUT_TEMPLATE_CHANNEL = ""
|
||||||
|
dq_env.OUTPUT_TEMPLATE_PLAYLIST = "%(playlist_title)s/%(title)s.%(ext)s"
|
||||||
|
|
||||||
|
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||||
|
return {
|
||||||
|
"_type": "playlist",
|
||||||
|
"id": "PLxyz789",
|
||||||
|
"channel_id": "UCabcd123",
|
||||||
|
"channel": "Odin",
|
||||||
|
"title": "My Playlist",
|
||||||
|
"entries": [
|
||||||
|
{
|
||||||
|
"id": "vid1",
|
||||||
|
"title": "Test Video",
|
||||||
|
"url": "https://example.com/watch?v=1",
|
||||||
|
"webpage_url": "https://example.com/watch?v=1",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||||
|
result = await dq.add(
|
||||||
|
"https://www.youtube.com/playlist?list=PLxyz789",
|
||||||
|
"video",
|
||||||
|
"auto",
|
||||||
|
"any",
|
||||||
|
"best",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
0,
|
||||||
|
auto_start=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
url = "https://example.com/watch?v=1"
|
||||||
|
assert dq.pending.exists(url)
|
||||||
|
download = dq.pending.get(url)
|
||||||
|
assert download.output_template.startswith("My Playlist/")
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_add_merges_global_preset_and_override_options(dq_env):
|
async def test_add_merges_global_preset_and_override_options(dq_env):
|
||||||
notifier = AsyncMock()
|
notifier = AsyncMock()
|
||||||
@@ -386,3 +568,381 @@ async def test_add_sets_clip_bounds_on_download_info(dq_env):
|
|||||||
download = dq.pending.get("https://example.com/clip")
|
download = dq.pending.get("https://example.com/clip")
|
||||||
assert download.info.clip_start == 10.0
|
assert download.info.clip_start == 10.0
|
||||||
assert download.info.clip_end == 99.5
|
assert download.info.clip_end == 99.5
|
||||||
|
|
||||||
|
|
||||||
|
def _upcoming_entry(url: str, *, release_timestamp: float | None = None) -> dict:
|
||||||
|
return {
|
||||||
|
"_type": "video",
|
||||||
|
"id": "live1",
|
||||||
|
"title": "Upcoming Stream",
|
||||||
|
"url": url,
|
||||||
|
"webpage_url": url,
|
||||||
|
"live_status": "is_upcoming",
|
||||||
|
"release_timestamp": release_timestamp if release_timestamp is not None else time.time() + 3600,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_add_upcoming_stream_scheduled_without_starting(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
url = "https://example.com/live-upcoming"
|
||||||
|
start_mock = AsyncMock()
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||||
|
result = await dq.add_entry(
|
||||||
|
_upcoming_entry(url),
|
||||||
|
"video",
|
||||||
|
"auto",
|
||||||
|
"any",
|
||||||
|
"best",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
0,
|
||||||
|
auto_start=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert dq.queue.exists(url)
|
||||||
|
download = dq.queue.get(url)
|
||||||
|
assert download.info.status == "scheduled"
|
||||||
|
assert download.info.live_status == "is_upcoming"
|
||||||
|
assert download.info.live_release_timestamp is not None
|
||||||
|
start_mock.assert_not_called()
|
||||||
|
assert url in dq._scheduled_probe_at
|
||||||
|
# The "scheduled to start at ..." message must include a UTC offset
|
||||||
|
# (a naive datetime's %z would render as an empty string here).
|
||||||
|
assert re.search(r"[+-]\d{4}$", download.info.error)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_probe_scheduled_starts_when_live(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
url = "https://example.com/live-upcoming"
|
||||||
|
start_mock = AsyncMock()
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||||
|
await dq.add_entry(
|
||||||
|
_upcoming_entry(url),
|
||||||
|
"video",
|
||||||
|
"auto",
|
||||||
|
"any",
|
||||||
|
"best",
|
||||||
|
"",
|
||||||
|
"",
|
||||||
|
0,
|
||||||
|
auto_start=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
download = dq.queue.get(url)
|
||||||
|
|
||||||
|
def fake_probe_extract(self, probe_url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||||
|
assert probe_url == url
|
||||||
|
return {
|
||||||
|
"_type": "video",
|
||||||
|
"id": "live1",
|
||||||
|
"title": "Live Now",
|
||||||
|
"url": url,
|
||||||
|
"webpage_url": url,
|
||||||
|
"live_status": "is_live",
|
||||||
|
"formats": [{"format_id": "22"}],
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_probe_extract), \
|
||||||
|
patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||||
|
await dq._probe_scheduled_download(download)
|
||||||
|
|
||||||
|
assert url not in dq._scheduled_probe_at
|
||||||
|
assert download.info.live_status == "is_live"
|
||||||
|
assert download.info.status == "pending"
|
||||||
|
start_mock.assert_called_once_with(download)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_import_scheduled_re_registers_monitor(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
url = "https://example.com/live-restart"
|
||||||
|
release = time.time() + 7200
|
||||||
|
|
||||||
|
info = DownloadInfo(
|
||||||
|
id="live1",
|
||||||
|
title="Upcoming Stream",
|
||||||
|
url=url,
|
||||||
|
quality="best",
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
error=None,
|
||||||
|
entry=None,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
live_status="is_upcoming",
|
||||||
|
live_release_timestamp=release,
|
||||||
|
)
|
||||||
|
info.status = "scheduled"
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
start_mock = AsyncMock()
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||||
|
await dq._DownloadQueue__add_download(info, True)
|
||||||
|
|
||||||
|
assert dq.queue.exists(url)
|
||||||
|
assert dq.queue.get(url).info.status == "scheduled"
|
||||||
|
assert url in dq._scheduled_probe_at
|
||||||
|
start_mock.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_probe_transient_error_retries_without_failing(dq_env):
|
||||||
|
"""A single probe failure must not abandon the scheduled stream."""
|
||||||
|
import ytdl
|
||||||
|
|
||||||
|
notifier = AsyncMock()
|
||||||
|
url = "https://example.com/live-transient"
|
||||||
|
start_mock = AsyncMock()
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||||
|
await dq.add_entry(
|
||||||
|
_upcoming_entry(url),
|
||||||
|
"video", "auto", "any", "best", "", "", 0,
|
||||||
|
auto_start=True,
|
||||||
|
)
|
||||||
|
download = dq.queue.get(url)
|
||||||
|
|
||||||
|
def boom(self, *args, **kwargs):
|
||||||
|
raise ytdl.yt_dlp.utils.YoutubeDLError("temporary network glitch")
|
||||||
|
|
||||||
|
before = time.time()
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", boom):
|
||||||
|
await dq._probe_scheduled_download(download)
|
||||||
|
|
||||||
|
# Still scheduled, still monitored, probe rescheduled into the future.
|
||||||
|
assert download.info.status == "scheduled"
|
||||||
|
assert url in dq._scheduled_probe_at
|
||||||
|
assert dq._scheduled_probe_at[url] >= before
|
||||||
|
assert dq._scheduled_probe_failures[url] == 1
|
||||||
|
notifier.completed.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_probe_gives_up_after_max_failures(dq_env):
|
||||||
|
import ytdl
|
||||||
|
|
||||||
|
notifier = AsyncMock()
|
||||||
|
url = "https://example.com/live-dead"
|
||||||
|
start_mock = AsyncMock()
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||||
|
await dq.add_entry(
|
||||||
|
_upcoming_entry(url),
|
||||||
|
"video", "auto", "any", "best", "", "", 0,
|
||||||
|
auto_start=True,
|
||||||
|
)
|
||||||
|
download = dq.queue.get(url)
|
||||||
|
|
||||||
|
def boom(self, *args, **kwargs):
|
||||||
|
raise ytdl.yt_dlp.utils.YoutubeDLError("stream was deleted")
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", boom):
|
||||||
|
for _ in range(ytdl._LIVE_PROBE_MAX_FAILURES):
|
||||||
|
await dq._probe_scheduled_download(download)
|
||||||
|
|
||||||
|
assert url not in dq._scheduled_probe_at
|
||||||
|
assert not dq.queue.exists(url)
|
||||||
|
assert dq.done.exists(url)
|
||||||
|
assert download.info.status == "error"
|
||||||
|
notifier.completed.assert_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_probe_recovers_after_transient_then_starts(dq_env):
|
||||||
|
"""A transient failure followed by a successful live probe should start the download."""
|
||||||
|
import ytdl
|
||||||
|
|
||||||
|
notifier = AsyncMock()
|
||||||
|
url = "https://example.com/live-recover"
|
||||||
|
start_mock = AsyncMock()
|
||||||
|
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||||
|
await dq.add_entry(
|
||||||
|
_upcoming_entry(url),
|
||||||
|
"video", "auto", "any", "best", "", "", 0,
|
||||||
|
auto_start=True,
|
||||||
|
)
|
||||||
|
download = dq.queue.get(url)
|
||||||
|
# The scheduling placeholder error is set on add.
|
||||||
|
assert download.info.error
|
||||||
|
|
||||||
|
def boom(self, *args, **kwargs):
|
||||||
|
raise ytdl.yt_dlp.utils.YoutubeDLError("temporary glitch")
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", boom):
|
||||||
|
await dq._probe_scheduled_download(download)
|
||||||
|
assert dq._scheduled_probe_failures[url] == 1
|
||||||
|
|
||||||
|
def live_now(self, *args, **kwargs):
|
||||||
|
return {
|
||||||
|
"_type": "video", "id": "live1", "title": "Live Now",
|
||||||
|
"url": url, "webpage_url": url, "live_status": "is_live",
|
||||||
|
"formats": [{"format_id": "22"}],
|
||||||
|
}
|
||||||
|
|
||||||
|
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", live_now), \
|
||||||
|
patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||||
|
await dq._probe_scheduled_download(download)
|
||||||
|
|
||||||
|
assert url not in dq._scheduled_probe_at
|
||||||
|
assert url not in dq._scheduled_probe_failures
|
||||||
|
assert download.info.status == "pending"
|
||||||
|
# Placeholder error/msg cleared now that a real download is starting.
|
||||||
|
assert download.info.error is None
|
||||||
|
assert download.info.msg is None
|
||||||
|
start_mock.assert_called_once_with(download)
|
||||||
|
|
||||||
|
|
||||||
|
def test_seconds_until_next_probe_none_when_empty(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
assert dq._seconds_until_next_probe() is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_calc_download_path_allows_subfolder(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
path, err = dq._DownloadQueue__calc_download_path("video", "sub/dir")
|
||||||
|
assert err is None
|
||||||
|
assert os.path.realpath(path) == os.path.join(os.path.realpath(dq_env.DOWNLOAD_DIR), "sub", "dir")
|
||||||
|
|
||||||
|
|
||||||
|
def test_calc_download_path_rejects_sibling_prefix_escape(dq_env):
|
||||||
|
"""A folder resolving to a sibling sharing a name prefix must be rejected.
|
||||||
|
|
||||||
|
Regression test: ``startswith`` would have accepted ``../downloads-secret``
|
||||||
|
when the base directory is ``.../downloads``.
|
||||||
|
"""
|
||||||
|
notifier = AsyncMock()
|
||||||
|
base = os.path.realpath(dq_env.DOWNLOAD_DIR)
|
||||||
|
sibling = base + "-secret"
|
||||||
|
os.makedirs(sibling, exist_ok=True)
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
escape_folder = os.path.join("..", os.path.basename(sibling), "x")
|
||||||
|
path, err = dq._DownloadQueue__calc_download_path("video", escape_folder)
|
||||||
|
assert path is None
|
||||||
|
assert err is not None and err["status"] == "error"
|
||||||
|
|
||||||
|
|
||||||
|
def test_calc_download_path_rejects_parent_escape(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
path, err = dq._DownloadQueue__calc_download_path("video", "../../etc")
|
||||||
|
assert path is None
|
||||||
|
assert err is not None and err["status"] == "error"
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_info_to_public_dict_excludes_server_only_fields():
|
||||||
|
info = DownloadInfo(
|
||||||
|
id="vid1",
|
||||||
|
title="Test Video",
|
||||||
|
url="https://example.com/watch?v=1",
|
||||||
|
quality="best",
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
error=None,
|
||||||
|
entry={"id": "vid1", "huge": "x" * 100000},
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
)
|
||||||
|
info.subtitle_files = [{"filename": "a.srt", "size": 10}]
|
||||||
|
public = info.to_public_dict()
|
||||||
|
assert "entry" not in public
|
||||||
|
assert "subtitle_files" not in public
|
||||||
|
# Client-facing fields are still present.
|
||||||
|
assert public["url"] == "https://example.com/watch?v=1"
|
||||||
|
assert public["title"] == "Test Video"
|
||||||
|
assert public["status"] == "pending"
|
||||||
|
|
||||||
|
|
||||||
|
def _make_download(dq_env, *, download_type="video", status="downloading", filename=None):
|
||||||
|
info = DownloadInfo(
|
||||||
|
id="id1",
|
||||||
|
title="t",
|
||||||
|
url="http://example.com/v",
|
||||||
|
quality="best",
|
||||||
|
download_type=download_type,
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
error=None,
|
||||||
|
entry=None,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
)
|
||||||
|
info.status = status
|
||||||
|
info.filename = filename
|
||||||
|
info.size = 123 if filename else None
|
||||||
|
return Download(
|
||||||
|
dq_env.DOWNLOAD_DIR, dq_env.TEMP_DIR, "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_post_download_cleanup_clears_filename_on_error(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
|
||||||
|
dq.queue.put(download)
|
||||||
|
|
||||||
|
dq._post_download_cleanup(download)
|
||||||
|
|
||||||
|
assert download.info.status == "error"
|
||||||
|
assert download.info.filename is None
|
||||||
|
assert download.info.size is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_post_download_cleanup_keeps_captured_subtitles_on_error(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
|
||||||
|
download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
|
||||||
|
dq.queue.put(download)
|
||||||
|
|
||||||
|
dq._post_download_cleanup(download)
|
||||||
|
|
||||||
|
assert download.info.status == "error"
|
||||||
|
assert download.info.filename == "en.srt"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_clear_skips_deletion_outside_download_directory(dq_env):
|
||||||
|
notifier = AsyncMock()
|
||||||
|
dq_env.DELETE_FILE_ON_TRASHCAN = True
|
||||||
|
dq = DownloadQueue(dq_env, notifier)
|
||||||
|
|
||||||
|
outside_dir = tempfile.mkdtemp()
|
||||||
|
outside_file = os.path.join(outside_dir, "outside.txt")
|
||||||
|
with open(outside_file, "w") as f:
|
||||||
|
f.write("do not delete me")
|
||||||
|
|
||||||
|
# A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
|
||||||
|
escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
|
||||||
|
download = _make_download(dq_env, status="finished", filename=escaping_filename)
|
||||||
|
dq.done.put(download)
|
||||||
|
|
||||||
|
await dq.clear([download.info.url])
|
||||||
|
|
||||||
|
assert os.path.exists(outside_file)
|
||||||
|
assert not dq.done.exists(download.info.url)
|
||||||
|
|||||||
@@ -220,42 +220,61 @@ class ParseDownloadOptionsTests(unittest.TestCase):
|
|||||||
|
|
||||||
def test_clip_url_t_param_strips_query_and_sets_start(self):
|
def test_clip_url_t_param_strips_query_and_sets_start(self):
|
||||||
parsed = main.parse_download_options({
|
parsed = main.parse_download_options({
|
||||||
"url": "https://example.com/watch?v=1&t=855s",
|
"url": "https://www.youtube.com/watch?v=1&t=855s",
|
||||||
"download_type": "video",
|
"download_type": "video",
|
||||||
"codec": "auto",
|
"codec": "auto",
|
||||||
"format": "any",
|
"format": "any",
|
||||||
"quality": "best",
|
"quality": "best",
|
||||||
})
|
})
|
||||||
self.assertEqual(parsed["url"], "https://example.com/watch?v=1")
|
self.assertEqual(parsed["url"], "https://www.youtube.com/watch?v=1")
|
||||||
self.assertEqual(parsed["clip_start"], 855.0)
|
self.assertEqual(parsed["clip_start"], 855.0)
|
||||||
self.assertIsNone(parsed["clip_end"])
|
self.assertIsNone(parsed["clip_end"])
|
||||||
|
|
||||||
def test_clip_explicit_start_wins_over_url_t(self):
|
def test_clip_explicit_start_wins_over_url_t(self):
|
||||||
parsed = main.parse_download_options({
|
parsed = main.parse_download_options({
|
||||||
"url": "https://example.com/watch?v=1&t=100",
|
"url": "https://www.youtube.com/watch?v=1&t=100",
|
||||||
"download_type": "video",
|
"download_type": "video",
|
||||||
"codec": "auto",
|
"codec": "auto",
|
||||||
"format": "any",
|
"format": "any",
|
||||||
"quality": "best",
|
"quality": "best",
|
||||||
"clip_start": "50",
|
"clip_start": "50",
|
||||||
})
|
})
|
||||||
self.assertEqual(parsed["url"], "https://example.com/watch?v=1")
|
self.assertEqual(parsed["url"], "https://www.youtube.com/watch?v=1")
|
||||||
self.assertEqual(parsed["clip_start"], 50.0)
|
self.assertEqual(parsed["clip_start"], 50.0)
|
||||||
self.assertIsNone(parsed["clip_end"])
|
self.assertIsNone(parsed["clip_end"])
|
||||||
|
|
||||||
def test_clip_end_only_sets_start_zero_and_strips_url_t(self):
|
def test_clip_end_only_sets_start_zero_and_strips_url_t(self):
|
||||||
parsed = main.parse_download_options({
|
parsed = main.parse_download_options({
|
||||||
"url": "https://example.com/watch?v=1&t=999",
|
"url": "https://www.youtube.com/watch?v=1&t=999",
|
||||||
"download_type": "video",
|
"download_type": "video",
|
||||||
"codec": "auto",
|
"codec": "auto",
|
||||||
"format": "any",
|
"format": "any",
|
||||||
"quality": "best",
|
"quality": "best",
|
||||||
"clip_end": "60",
|
"clip_end": "60",
|
||||||
})
|
})
|
||||||
self.assertEqual(parsed["url"], "https://example.com/watch?v=1")
|
self.assertEqual(parsed["url"], "https://www.youtube.com/watch?v=1")
|
||||||
self.assertEqual(parsed["clip_start"], 0.0)
|
self.assertEqual(parsed["clip_start"], 0.0)
|
||||||
self.assertEqual(parsed["clip_end"], 60.0)
|
self.assertEqual(parsed["clip_end"], 60.0)
|
||||||
|
|
||||||
|
def test_clip_url_t_param_ignored_on_non_youtube_host(self):
|
||||||
|
# 't' is a generic query param name; only rewrite it on YouTube hosts
|
||||||
|
# so an unrelated site's URL isn't silently mutated with a bogus clip.
|
||||||
|
parsed = main.parse_download_options({
|
||||||
|
"url": "https://example.com/watch?v=1&t=855s",
|
||||||
|
"download_type": "video",
|
||||||
|
"codec": "auto",
|
||||||
|
"format": "any",
|
||||||
|
"quality": "best",
|
||||||
|
})
|
||||||
|
self.assertEqual(parsed["url"], "https://example.com/watch?v=1&t=855s")
|
||||||
|
self.assertIsNone(parsed["clip_start"])
|
||||||
|
self.assertIsNone(parsed["clip_end"])
|
||||||
|
|
||||||
|
def test_extract_t_query_youtu_be_short_host(self):
|
||||||
|
cleaned, start = main._extract_t_query_from_url("https://youtu.be/abc123?t=90")
|
||||||
|
self.assertEqual(cleaned, "https://youtu.be/abc123")
|
||||||
|
self.assertEqual(start, 90.0)
|
||||||
|
|
||||||
def test_clip_rejects_end_before_start(self):
|
def test_clip_rejects_end_before_start(self):
|
||||||
with self.assertRaises(main.web.HTTPBadRequest):
|
with self.assertRaises(main.web.HTTPBadRequest):
|
||||||
main.parse_download_options({
|
main.parse_download_options({
|
||||||
@@ -280,5 +299,17 @@ class ParseDownloadOptionsTests(unittest.TestCase):
|
|||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
class GetCustomDirsTests(unittest.TestCase):
|
||||||
|
def test_works_without_a_running_event_loop(self):
|
||||||
|
# get_custom_dirs() used to time its cache via
|
||||||
|
# asyncio.get_running_loop().time(), which raises RuntimeError outside
|
||||||
|
# a running loop (e.g. when called from a plain executor thread). It
|
||||||
|
# must work from a synchronous context too.
|
||||||
|
result = main.get_custom_dirs()
|
||||||
|
self.assertIn("download_dir", result)
|
||||||
|
self.assertIn("audio_download_dir", result)
|
||||||
|
self.assertIn("", result["download_dir"])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -0,0 +1,29 @@
|
|||||||
|
"""Tests for nightly yt-dlp update scheduling helpers."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from datetime import datetime
|
||||||
|
|
||||||
|
from main import seconds_until_next_daily_time
|
||||||
|
|
||||||
|
|
||||||
|
class NightlyUpdateTests(unittest.TestCase):
|
||||||
|
def test_seconds_until_later_today(self):
|
||||||
|
now = datetime(2026, 6, 4, 10, 0, 0)
|
||||||
|
delay = seconds_until_next_daily_time("15:30", now)
|
||||||
|
self.assertEqual(delay, 5 * 3600 + 30 * 60)
|
||||||
|
|
||||||
|
def test_seconds_until_wraps_to_next_day(self):
|
||||||
|
now = datetime(2026, 6, 4, 18, 0, 0)
|
||||||
|
delay = seconds_until_next_daily_time("04:00", now)
|
||||||
|
self.assertEqual(delay, 10 * 3600)
|
||||||
|
|
||||||
|
def test_seconds_until_same_minute_is_next_day(self):
|
||||||
|
now = datetime(2026, 6, 4, 4, 0, 30)
|
||||||
|
delay = seconds_until_next_daily_time("04:00", now)
|
||||||
|
self.assertAlmostEqual(delay, 24 * 3600 - 30, delta=1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -4,6 +4,7 @@ import os
|
|||||||
import tempfile
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
from state_store import AtomicJsonStore, from_json_compatible, to_json_compatible
|
from state_store import AtomicJsonStore, from_json_compatible, to_json_compatible
|
||||||
|
|
||||||
@@ -21,6 +22,135 @@ class StateStoreTests(unittest.TestCase):
|
|||||||
self.assertEqual(payload["schema_version"], 2)
|
self.assertEqual(payload["schema_version"], 2)
|
||||||
self.assertEqual(payload["items"][0]["info"]["title"], "hello")
|
self.assertEqual(payload["items"][0]["info"]["title"], "hello")
|
||||||
|
|
||||||
|
def test_save_falls_back_to_direct_write_when_mkstemp_fails(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = os.path.join(tmp, "queue.json")
|
||||||
|
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||||
|
|
||||||
|
with self.assertLogs("state_store", level="WARNING") as logs:
|
||||||
|
with patch(
|
||||||
|
"state_store.tempfile.mkstemp",
|
||||||
|
side_effect=PermissionError(1, "Operation not permitted"),
|
||||||
|
):
|
||||||
|
store.save({"items": [{"key": "a"}]})
|
||||||
|
|
||||||
|
self.assertTrue(os.path.exists(path))
|
||||||
|
self.assertTrue(any(path in message for message in logs.output))
|
||||||
|
# Fallback keeps owner-only permissions, matching the atomic path.
|
||||||
|
self.assertEqual(os.stat(path).st_mode & 0o777, 0o600)
|
||||||
|
payload = store.load()
|
||||||
|
self.assertEqual(payload["items"], [{"key": "a"}])
|
||||||
|
|
||||||
|
def test_fallback_tightens_permissions_on_existing_file(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = os.path.join(tmp, "queue.json")
|
||||||
|
with open(path, "w", encoding="utf-8") as f:
|
||||||
|
f.write("{}")
|
||||||
|
os.chmod(path, 0o644)
|
||||||
|
|
||||||
|
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||||
|
with patch(
|
||||||
|
"state_store.tempfile.mkstemp",
|
||||||
|
side_effect=PermissionError(1, "Operation not permitted"),
|
||||||
|
):
|
||||||
|
store.save({"items": [{"key": "a"}]})
|
||||||
|
|
||||||
|
self.assertEqual(os.stat(path).st_mode & 0o777, 0o600)
|
||||||
|
self.assertEqual(store.load()["items"], [{"key": "a"}])
|
||||||
|
|
||||||
|
def test_save_falls_back_to_direct_write_when_replace_fails(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = os.path.join(tmp, "queue.json")
|
||||||
|
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"state_store.os.replace",
|
||||||
|
side_effect=PermissionError(1, "Operation not permitted"),
|
||||||
|
):
|
||||||
|
store.save({"items": [{"key": "a"}]})
|
||||||
|
|
||||||
|
self.assertTrue(os.path.exists(path))
|
||||||
|
payload = store.load()
|
||||||
|
self.assertEqual(payload["items"], [{"key": "a"}])
|
||||||
|
self.assertEqual([], [name for name in os.listdir(tmp) if name.endswith(".tmp")])
|
||||||
|
|
||||||
|
def test_save_reraises_when_atomic_and_direct_write_fail(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = os.path.join(tmp, "queue.json")
|
||||||
|
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"state_store.tempfile.mkstemp",
|
||||||
|
side_effect=PermissionError(1, "Operation not permitted"),
|
||||||
|
):
|
||||||
|
with patch(
|
||||||
|
"state_store.os.open",
|
||||||
|
side_effect=PermissionError(13, "Permission denied"),
|
||||||
|
):
|
||||||
|
with self.assertRaises(PermissionError) as ctx:
|
||||||
|
store.save({"items": [{"key": "a"}]})
|
||||||
|
|
||||||
|
self.assertEqual(ctx.exception.errno, 13)
|
||||||
|
self.assertFalse(os.path.exists(path))
|
||||||
|
|
||||||
|
def test_unsupported_fsync_keeps_atomic_path(self):
|
||||||
|
# fsync being unsupported (EINVAL/ENOSYS) must not by itself trigger the
|
||||||
|
# direct-write fallback; the atomic temp-file + rename path still runs.
|
||||||
|
import errno as _errno
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = os.path.join(tmp, "queue.json")
|
||||||
|
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"state_store.os.fsync",
|
||||||
|
side_effect=OSError(_errno.EINVAL, "Invalid argument"),
|
||||||
|
):
|
||||||
|
with self.assertNoLogs("state_store", level="WARNING"):
|
||||||
|
store.save({"items": [{"key": "a"}]})
|
||||||
|
|
||||||
|
self.assertEqual(store.load()["items"], [{"key": "a"}])
|
||||||
|
self.assertEqual([], [name for name in os.listdir(tmp) if name.endswith(".tmp")])
|
||||||
|
|
||||||
|
def test_save_reraises_and_preserves_state_on_non_atomic_errno(self):
|
||||||
|
# A storage failure such as ENOSPC is not an "atomic unavailable"
|
||||||
|
# signal, so it must surface instead of falling back to a direct write
|
||||||
|
# that would truncate the existing good state file.
|
||||||
|
import errno as _errno
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = os.path.join(tmp, "queue.json")
|
||||||
|
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||||
|
store.save({"items": [{"key": "good"}]})
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"state_store.tempfile.mkstemp",
|
||||||
|
side_effect=OSError(_errno.ENOSPC, "No space left on device"),
|
||||||
|
):
|
||||||
|
with self.assertRaises(OSError) as ctx:
|
||||||
|
store.save({"items": [{"key": "new"}]})
|
||||||
|
|
||||||
|
self.assertEqual(ctx.exception.errno, _errno.ENOSPC)
|
||||||
|
# Existing state is untouched.
|
||||||
|
self.assertEqual(store.load()["items"], [{"key": "good"}])
|
||||||
|
|
||||||
|
def test_serialization_failure_preserves_existing_state(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = os.path.join(tmp, "queue.json")
|
||||||
|
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||||
|
store.save({"items": [{"key": "good"}]})
|
||||||
|
|
||||||
|
# Even on the fallback path, a non-serializable payload must raise
|
||||||
|
# before the existing good state file is touched.
|
||||||
|
with patch(
|
||||||
|
"state_store.tempfile.mkstemp",
|
||||||
|
side_effect=PermissionError(1, "Operation not permitted"),
|
||||||
|
):
|
||||||
|
with self.assertRaises(TypeError):
|
||||||
|
store.save({"items": object()})
|
||||||
|
|
||||||
|
self.assertEqual(store.load()["items"], [{"key": "good"}])
|
||||||
|
|
||||||
def test_invalid_file_is_quarantined(self):
|
def test_invalid_file_is_quarantined(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
path = os.path.join(tmp, "queue.json")
|
path = os.path.join(tmp, "queue.json")
|
||||||
|
|||||||
@@ -1,10 +1,12 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shelve
|
import shelve
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
import types
|
import types
|
||||||
import unittest
|
import unittest
|
||||||
from unittest.mock import patch
|
from unittest.mock import patch
|
||||||
@@ -29,6 +31,7 @@ sys.modules.setdefault("yt_dlp.networking", fake_networking)
|
|||||||
sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
|
sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
|
||||||
|
|
||||||
from subscriptions import (
|
from subscriptions import (
|
||||||
|
SubscriptionInfo,
|
||||||
SubscriptionManager,
|
SubscriptionManager,
|
||||||
_is_subscriber_only_entry,
|
_is_subscriber_only_entry,
|
||||||
coerce_optional_bool,
|
coerce_optional_bool,
|
||||||
@@ -44,6 +47,7 @@ class _Config:
|
|||||||
self.DOWNLOAD_DIR = state_dir
|
self.DOWNLOAD_DIR = state_dir
|
||||||
self.TEMP_DIR = state_dir
|
self.TEMP_DIR = state_dir
|
||||||
self.YTDL_OPTIONS = {}
|
self.YTDL_OPTIONS = {}
|
||||||
|
self.YTDL_OPTIONS_PRESETS = {}
|
||||||
|
|
||||||
|
|
||||||
class _Queue:
|
class _Queue:
|
||||||
@@ -571,8 +575,16 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
sub_id = result["subscription"]["id"]
|
sub_id = result["subscription"]["id"]
|
||||||
with self.assertRaises(ValueError):
|
update_result = await mgr.update_subscription(sub_id, {"enabled": "maybe"})
|
||||||
await mgr.update_subscription(sub_id, {"enabled": "maybe"})
|
self.assertEqual(update_result["status"], "error")
|
||||||
|
stored = mgr.get(sub_id)
|
||||||
|
self.assertTrue(stored.enabled)
|
||||||
|
|
||||||
|
update_result = await mgr.update_subscription(
|
||||||
|
sub_id, {"check_interval_minutes": "abc"}
|
||||||
|
)
|
||||||
|
self.assertEqual(update_result["status"], "error")
|
||||||
|
self.assertEqual(mgr.get(sub_id).check_interval_minutes, 60)
|
||||||
|
|
||||||
async def test_add_subscription_rejects_invalid_title_regex(self):
|
async def test_add_subscription_rejects_invalid_title_regex(self):
|
||||||
with tempfile.TemporaryDirectory() as tmp:
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
@@ -1012,6 +1024,223 @@ class ExtractFlatPlaylistTests(unittest.TestCase):
|
|||||||
self.assertEqual(info.get("_type"), "playlist")
|
self.assertEqual(info.get("_type"), "playlist")
|
||||||
self.assertEqual([entry["webpage_url"] for entry in entries], ["https://example.com/v1"])
|
self.assertEqual([entry["webpage_url"] for entry in entries], ["https://example.com/v1"])
|
||||||
|
|
||||||
|
def test_extra_opts_applied_on_top_of_config_options(self):
|
||||||
|
captured: dict = {}
|
||||||
|
|
||||||
|
class _FakeYDL:
|
||||||
|
def __init__(self, params):
|
||||||
|
captured.update(params)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc, tb):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def extract_info(self, url, download=False):
|
||||||
|
return {"_type": "video"}
|
||||||
|
|
||||||
|
cfg = _Config(tempfile.mkdtemp())
|
||||||
|
with patch("subscriptions.yt_dlp.YoutubeDL", _FakeYDL, create=True):
|
||||||
|
extract_flat_playlist(cfg, "https://example.com/v1", 50, extra_opts={"cookiefile": "x"})
|
||||||
|
|
||||||
|
self.assertEqual(captured.get("cookiefile"), "x")
|
||||||
|
|
||||||
|
|
||||||
|
def _make_scan_capturing_fake_ydl(captured_params: list, entries: list[dict]):
|
||||||
|
class _FakeYDL:
|
||||||
|
def __init__(self, params):
|
||||||
|
captured_params.append(params)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc, tb):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def extract_info(self, url, download=False):
|
||||||
|
return {"_type": "channel", "title": "Channel", "entries": entries}
|
||||||
|
|
||||||
|
return _FakeYDL
|
||||||
|
|
||||||
|
|
||||||
|
class SubscriptionScanExtraOptsTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
async def test_add_subscription_scan_applies_presets_and_overrides(self):
|
||||||
|
captured_params: list = []
|
||||||
|
fake_ydl = _make_scan_capturing_fake_ydl(
|
||||||
|
captured_params,
|
||||||
|
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||||
|
)
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg = _Config(tmp)
|
||||||
|
cfg.YTDL_OPTIONS_PRESETS = {"mypreset": {"cookiefile": "preset.txt"}}
|
||||||
|
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
|
||||||
|
with patch("subscriptions.yt_dlp.YoutubeDL", fake_ydl, create=True):
|
||||||
|
await mgr.add_subscription(
|
||||||
|
"https://example.com/channel",
|
||||||
|
check_interval_minutes=60,
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
quality="best",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
auto_start=True,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
subtitle_language="en",
|
||||||
|
subtitle_mode="prefer_manual",
|
||||||
|
ytdl_options_presets=["mypreset"],
|
||||||
|
ytdl_options_overrides={"extra": "override"},
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertTrue(captured_params)
|
||||||
|
self.assertEqual(captured_params[0].get("cookiefile"), "preset.txt")
|
||||||
|
self.assertEqual(captured_params[0].get("extra"), "override")
|
||||||
|
|
||||||
|
async def test_check_now_scan_applies_stored_subscription_presets(self):
|
||||||
|
entries = [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}]
|
||||||
|
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
cfg = _Config(tmp)
|
||||||
|
cfg.YTDL_OPTIONS_PRESETS = {"mypreset": {"cookiefile": "preset.txt"}}
|
||||||
|
mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
|
||||||
|
|
||||||
|
add_captured: list = []
|
||||||
|
with patch(
|
||||||
|
"subscriptions.yt_dlp.YoutubeDL",
|
||||||
|
_make_scan_capturing_fake_ydl(add_captured, entries),
|
||||||
|
create=True,
|
||||||
|
):
|
||||||
|
result = await mgr.add_subscription(
|
||||||
|
"https://example.com/channel",
|
||||||
|
check_interval_minutes=60,
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
quality="best",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
auto_start=True,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
subtitle_language="en",
|
||||||
|
subtitle_mode="prefer_manual",
|
||||||
|
ytdl_options_presets=["mypreset"],
|
||||||
|
)
|
||||||
|
sub_id = result["subscription"]["id"]
|
||||||
|
|
||||||
|
check_captured: list = []
|
||||||
|
with patch(
|
||||||
|
"subscriptions.yt_dlp.YoutubeDL",
|
||||||
|
_make_scan_capturing_fake_ydl(check_captured, entries),
|
||||||
|
create=True,
|
||||||
|
):
|
||||||
|
await mgr.check_now([sub_id])
|
||||||
|
|
||||||
|
self.assertTrue(check_captured)
|
||||||
|
self.assertEqual(check_captured[0].get("cookiefile"), "preset.txt")
|
||||||
|
|
||||||
|
|
||||||
|
class SubscriptionEventLoopTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
async def test_check_now_does_not_block_event_loop(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
queue = _Queue()
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"subscriptions.extract_flat_playlist",
|
||||||
|
return_value=(
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||||
|
),
|
||||||
|
):
|
||||||
|
result = await mgr.add_subscription(
|
||||||
|
"https://example.com/channel",
|
||||||
|
check_interval_minutes=60,
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
quality="best",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
auto_start=True,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
subtitle_language="en",
|
||||||
|
subtitle_mode="prefer_manual",
|
||||||
|
)
|
||||||
|
sub_id = result["subscription"]["id"]
|
||||||
|
|
||||||
|
def _slow_extract(config, url, playlistend, **kwargs):
|
||||||
|
time.sleep(0.3)
|
||||||
|
return (
|
||||||
|
{"_type": "channel", "title": "Channel"},
|
||||||
|
[{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("subscriptions.extract_flat_playlist", side_effect=_slow_extract):
|
||||||
|
check_task = asyncio.ensure_future(mgr.check_now([sub_id]))
|
||||||
|
# If check_now() blocked the event loop, this would not complete
|
||||||
|
# until after the slow extraction finishes.
|
||||||
|
await asyncio.wait_for(asyncio.sleep(0.05), timeout=0.2)
|
||||||
|
self.assertFalse(check_task.done())
|
||||||
|
await check_task
|
||||||
|
|
||||||
|
async def test_check_many_isolates_a_crashing_subscription(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||||
|
|
||||||
|
good = SubscriptionInfo(id="good", name="Good", url="https://example.com/good")
|
||||||
|
bad = SubscriptionInfo(id="bad", name="Bad", url="https://example.com/bad")
|
||||||
|
other = SubscriptionInfo(id="other", name="Other", url="https://example.com/other")
|
||||||
|
|
||||||
|
checked: list[str] = []
|
||||||
|
|
||||||
|
async def fake_check(sub):
|
||||||
|
if sub.id == "bad":
|
||||||
|
raise RuntimeError("boom")
|
||||||
|
checked.append(sub.id)
|
||||||
|
|
||||||
|
with patch.object(mgr, "_check_one_unlocked", side_effect=fake_check):
|
||||||
|
# The crashing subscription must not prevent the others running.
|
||||||
|
await mgr._check_many([good, bad, other])
|
||||||
|
|
||||||
|
self.assertIn("good", checked)
|
||||||
|
self.assertIn("other", checked)
|
||||||
|
|
||||||
|
async def test_check_many_bounded_concurrency(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
|
||||||
|
subs = [
|
||||||
|
SubscriptionInfo(id=str(i), name=str(i), url=f"https://example.com/{i}")
|
||||||
|
for i in range(10)
|
||||||
|
]
|
||||||
|
|
||||||
|
import subscriptions as subs_mod
|
||||||
|
|
||||||
|
concurrent = 0
|
||||||
|
peak = 0
|
||||||
|
|
||||||
|
async def fake_check(sub):
|
||||||
|
nonlocal concurrent, peak
|
||||||
|
concurrent += 1
|
||||||
|
peak = max(peak, concurrent)
|
||||||
|
await asyncio.sleep(0.02)
|
||||||
|
concurrent -= 1
|
||||||
|
|
||||||
|
with patch.object(mgr, "_check_one_unlocked", side_effect=fake_check):
|
||||||
|
await mgr._check_many(subs)
|
||||||
|
|
||||||
|
# Never exceed the configured bound, but do run more than one at once.
|
||||||
|
self.assertLessEqual(peak, subs_mod._MAX_CONCURRENT_CHECKS)
|
||||||
|
self.assertGreater(peak, 1)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
@@ -3,12 +3,14 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import pickle
|
import pickle
|
||||||
|
import signal
|
||||||
import sys
|
import sys
|
||||||
import tempfile
|
import tempfile
|
||||||
import threading
|
import threading
|
||||||
import types
|
import types
|
||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
fake_yt_dlp = types.ModuleType("yt_dlp")
|
fake_yt_dlp = types.ModuleType("yt_dlp")
|
||||||
fake_networking = types.ModuleType("yt_dlp.networking")
|
fake_networking = types.ModuleType("yt_dlp.networking")
|
||||||
@@ -37,6 +39,7 @@ sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
|
|||||||
sys.modules.setdefault("yt_dlp.utils", fake_utils)
|
sys.modules.setdefault("yt_dlp.utils", fake_utils)
|
||||||
|
|
||||||
from ytdl import (
|
from ytdl import (
|
||||||
|
Download,
|
||||||
DownloadInfo,
|
DownloadInfo,
|
||||||
_compact_persisted_entry,
|
_compact_persisted_entry,
|
||||||
_convert_srt_to_txt_file,
|
_convert_srt_to_txt_file,
|
||||||
@@ -160,6 +163,105 @@ class SanitizeEntryForPickleTests(unittest.TestCase):
|
|||||||
self.assertEqual(out, {"z": 1, "a": 2})
|
self.assertEqual(out, {"z": 1, "a": 2})
|
||||||
|
|
||||||
|
|
||||||
|
def _make_test_download() -> Download:
|
||||||
|
info = DownloadInfo(
|
||||||
|
id="id1",
|
||||||
|
title="t",
|
||||||
|
url="http://example.com/v",
|
||||||
|
quality="best",
|
||||||
|
download_type="video",
|
||||||
|
codec="auto",
|
||||||
|
format="any",
|
||||||
|
folder="",
|
||||||
|
custom_name_prefix="",
|
||||||
|
error=None,
|
||||||
|
entry=None,
|
||||||
|
playlist_item_limit=0,
|
||||||
|
split_by_chapters=False,
|
||||||
|
chapter_template="",
|
||||||
|
)
|
||||||
|
return Download("/tmp", "/tmp", "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info)
|
||||||
|
|
||||||
|
|
||||||
|
class ProgressThrottleTests(unittest.TestCase):
|
||||||
|
def test_downloading_ticks_are_throttled(self):
|
||||||
|
dl = _make_test_download()
|
||||||
|
forwarded = []
|
||||||
|
dl.status_queue = types.SimpleNamespace(put=forwarded.append)
|
||||||
|
hook = dl._make_progress_hook()
|
||||||
|
|
||||||
|
with patch("ytdl.time.monotonic", side_effect=[100.0, 100.1, 100.6, 100.7]):
|
||||||
|
hook({"status": "downloading", "downloaded_bytes": 1})
|
||||||
|
hook({"status": "downloading", "downloaded_bytes": 2})
|
||||||
|
hook({"status": "downloading", "downloaded_bytes": 3})
|
||||||
|
hook({"status": "downloading", "downloaded_bytes": 4})
|
||||||
|
|
||||||
|
# Only the 1st and 3rd ticks are >= 0.5s apart from the last forwarded one.
|
||||||
|
self.assertEqual(len(forwarded), 2)
|
||||||
|
|
||||||
|
def test_finished_and_error_statuses_always_forwarded(self):
|
||||||
|
dl = _make_test_download()
|
||||||
|
forwarded = []
|
||||||
|
dl.status_queue = types.SimpleNamespace(put=forwarded.append)
|
||||||
|
hook = dl._make_progress_hook()
|
||||||
|
|
||||||
|
with patch("ytdl.time.monotonic", side_effect=[200.0, 200.1]):
|
||||||
|
hook({"status": "downloading"})
|
||||||
|
hook({"status": "finished"})
|
||||||
|
hook({"status": "downloading"})
|
||||||
|
hook({"status": "error", "msg": "boom"})
|
||||||
|
|
||||||
|
statuses = [item.get("status") for item in forwarded]
|
||||||
|
self.assertIn("finished", statuses)
|
||||||
|
self.assertIn("error", statuses)
|
||||||
|
|
||||||
|
|
||||||
|
class CancelProcessGroupTests(unittest.TestCase):
|
||||||
|
def test_cancel_kills_group_when_child_is_group_leader(self):
|
||||||
|
# Child successfully ran os.setpgrp(): its pgid equals its own pid.
|
||||||
|
dl = _make_test_download()
|
||||||
|
dl.proc = types.SimpleNamespace(pid=4321)
|
||||||
|
dl.status_queue = types.SimpleNamespace(put=lambda _item: None)
|
||||||
|
|
||||||
|
with patch.object(Download, "running", return_value=True), \
|
||||||
|
patch("ytdl.os.getpgid", return_value=4321) as mock_getpgid, \
|
||||||
|
patch("ytdl.os.killpg") as mock_killpg:
|
||||||
|
dl.cancel()
|
||||||
|
|
||||||
|
mock_getpgid.assert_called_once_with(4321)
|
||||||
|
mock_killpg.assert_called_once_with(4321, signal.SIGKILL)
|
||||||
|
self.assertTrue(dl.canceled)
|
||||||
|
|
||||||
|
def test_cancel_does_not_killpg_parent_group_kills_child_only(self):
|
||||||
|
# Child has NOT become its own group leader yet (pgid != pid, e.g. it is
|
||||||
|
# still in the server's process group). killpg must NOT be called — that
|
||||||
|
# would SIGKILL the whole server — and we fall back to proc.kill().
|
||||||
|
dl = _make_test_download()
|
||||||
|
dl.proc = types.SimpleNamespace(pid=4321, kill=MagicMock())
|
||||||
|
dl.status_queue = types.SimpleNamespace(put=lambda _item: None)
|
||||||
|
|
||||||
|
with patch.object(Download, "running", return_value=True), \
|
||||||
|
patch("ytdl.os.getpgid", return_value=999), \
|
||||||
|
patch("ytdl.os.killpg") as mock_killpg:
|
||||||
|
dl.cancel()
|
||||||
|
|
||||||
|
mock_killpg.assert_not_called()
|
||||||
|
dl.proc.kill.assert_called_once()
|
||||||
|
self.assertTrue(dl.canceled)
|
||||||
|
|
||||||
|
def test_cancel_falls_back_to_proc_kill_when_getpgid_unavailable(self):
|
||||||
|
dl = _make_test_download()
|
||||||
|
dl.proc = types.SimpleNamespace(pid=4321, kill=MagicMock())
|
||||||
|
dl.status_queue = types.SimpleNamespace(put=lambda _item: None)
|
||||||
|
|
||||||
|
with patch.object(Download, "running", return_value=True), \
|
||||||
|
patch("ytdl.os.getpgid", side_effect=OSError("no such process")):
|
||||||
|
dl.cancel()
|
||||||
|
|
||||||
|
dl.proc.kill.assert_called_once()
|
||||||
|
self.assertTrue(dl.canceled)
|
||||||
|
|
||||||
|
|
||||||
class ConvertSrtToTxtTests(unittest.TestCase):
|
class ConvertSrtToTxtTests(unittest.TestCase):
|
||||||
def test_basic_conversion(self):
|
def test_basic_conversion(self):
|
||||||
srt = """1
|
srt = """1
|
||||||
@@ -180,6 +282,77 @@ Second line
|
|||||||
self.assertIn("Hello world", content)
|
self.assertIn("Hello world", content)
|
||||||
self.assertIn("Second line", content)
|
self.assertIn("Second line", content)
|
||||||
|
|
||||||
|
def test_vtt_input_strips_header_and_metadata(self):
|
||||||
|
# yt-dlp can fall back to VTT even when srt/txt was requested (the
|
||||||
|
# extractor may not offer a native srt track); the converter must not
|
||||||
|
# leak VTT-only header/metadata lines into the plain-text output.
|
||||||
|
vtt = """WEBVTT
|
||||||
|
Kind: captions
|
||||||
|
Language: en
|
||||||
|
|
||||||
|
NOTE
|
||||||
|
This is a note block
|
||||||
|
|
||||||
|
1
|
||||||
|
00:00:01.000 --> 00:00:02.000
|
||||||
|
Hello <b>world</b>
|
||||||
|
|
||||||
|
2
|
||||||
|
00:00:03.000 --> 00:00:04.000
|
||||||
|
Second line
|
||||||
|
"""
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "sub.vtt"
|
||||||
|
path.write_text(vtt, encoding="utf-8")
|
||||||
|
txt_path = _convert_srt_to_txt_file(str(path))
|
||||||
|
self.assertIsNotNone(txt_path)
|
||||||
|
content = Path(txt_path).read_text(encoding="utf-8")
|
||||||
|
self.assertIn("Hello world", content)
|
||||||
|
self.assertIn("Second line", content)
|
||||||
|
self.assertNotIn("WEBVTT", content)
|
||||||
|
self.assertNotIn("Kind:", content)
|
||||||
|
self.assertNotIn("Language:", content)
|
||||||
|
self.assertNotIn("This is a note block", content)
|
||||||
|
|
||||||
|
def test_vtt_standalone_header_block_is_stripped(self):
|
||||||
|
# Some VTT files put a blank line after WEBVTT, so Kind:/Language: form
|
||||||
|
# their own block. That header block (before the first timed cue) must
|
||||||
|
# still be stripped.
|
||||||
|
vtt = """WEBVTT
|
||||||
|
|
||||||
|
Kind: captions
|
||||||
|
Language: en
|
||||||
|
|
||||||
|
1
|
||||||
|
00:00:01.000 --> 00:00:02.000
|
||||||
|
Hello world
|
||||||
|
"""
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "sub.vtt"
|
||||||
|
path.write_text(vtt, encoding="utf-8")
|
||||||
|
content = Path(_convert_srt_to_txt_file(str(path))).read_text(encoding="utf-8")
|
||||||
|
self.assertIn("Hello world", content)
|
||||||
|
self.assertNotIn("Kind:", content)
|
||||||
|
self.assertNotIn("Language:", content)
|
||||||
|
|
||||||
|
def test_cue_text_starting_with_metadata_keyword_is_kept(self):
|
||||||
|
# A real caption line beginning with "Kind:"/"Language:" must NOT be
|
||||||
|
# dropped as if it were VTT header metadata.
|
||||||
|
srt = """1
|
||||||
|
00:00:01,000 --> 00:00:02,000
|
||||||
|
Kind: regards, everyone
|
||||||
|
|
||||||
|
2
|
||||||
|
00:00:03,000 --> 00:00:04,000
|
||||||
|
Language: they spoke French
|
||||||
|
"""
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
path = Path(tmp) / "sub.srt"
|
||||||
|
path.write_text(srt, encoding="utf-8")
|
||||||
|
content = Path(_convert_srt_to_txt_file(str(path))).read_text(encoding="utf-8")
|
||||||
|
self.assertIn("Kind: regards, everyone", content)
|
||||||
|
self.assertIn("Language: they spoke French", content)
|
||||||
|
|
||||||
|
|
||||||
class DownloadInfoSetstateTests(unittest.TestCase):
|
class DownloadInfoSetstateTests(unittest.TestCase):
|
||||||
def _base_state(self, **kwargs):
|
def _base_state(self, **kwargs):
|
||||||
|
|||||||
+430
-42
@@ -9,21 +9,64 @@ from collections import OrderedDict
|
|||||||
import time
|
import time
|
||||||
import asyncio
|
import asyncio
|
||||||
import multiprocessing
|
import multiprocessing
|
||||||
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from functools import partial
|
from functools import partial
|
||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
import types
|
import types
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
|
||||||
import yt_dlp.networking.impersonate
|
import yt_dlp.networking.impersonate
|
||||||
from yt_dlp.utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
|
from yt_dlp.utils import STR_FORMAT_RE_TMPL, STR_FORMAT_TYPES
|
||||||
from dl_formats import get_format, get_opts, AUDIO_FORMATS
|
import bg_tasks
|
||||||
|
from dl_formats import get_format, get_opts, AUDIO_FORMATS, merge_ytdl_option_layers
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from state_store import AtomicJsonStore, from_json_compatible, read_legacy_shelf, to_json_compatible
|
from state_store import AtomicJsonStore, from_json_compatible, read_legacy_shelf, to_json_compatible
|
||||||
from subscriptions import _entry_id
|
from subscriptions import _entry_id
|
||||||
|
|
||||||
log = logging.getLogger('ytdl')
|
log = logging.getLogger('ytdl')
|
||||||
|
|
||||||
|
# Python 3.14 switches the default multiprocessing start method on Linux
|
||||||
|
# (this app's only supported deployment target, per the Dockerfile) from fork
|
||||||
|
# to forkserver. Download._download relies on inheriting process state the
|
||||||
|
# way fork provides, so pin it back on Linux specifically.
|
||||||
|
#
|
||||||
|
# This must NOT be widened to "prefer fork wherever available": on macOS the
|
||||||
|
# platform default has long been spawn (never fork) precisely because forking
|
||||||
|
# a multi-threaded parent is hazardous — inherited locks held by threads that
|
||||||
|
# vanish in the child can deadlock it silently before it does any work. This
|
||||||
|
# app creates background threads (executors, notifier callbacks) well before
|
||||||
|
# any download starts, so forcing fork there reproduces exactly that hazard.
|
||||||
|
_MP_CTX = (
|
||||||
|
multiprocessing.get_context("fork")
|
||||||
|
if sys.platform.startswith("linux") and "fork" in multiprocessing.get_all_start_methods()
|
||||||
|
else multiprocessing.get_context()
|
||||||
|
)
|
||||||
|
|
||||||
|
_LIVE_CHECK_INTERVAL = 60
|
||||||
|
_LIVE_MAX_CHECK_INTERVAL = 3600
|
||||||
|
# Consecutive probe failures (network blips, rate limits, transient extractor
|
||||||
|
# errors) tolerated before a scheduled live download is abandoned as errored.
|
||||||
|
_LIVE_PROBE_MAX_FAILURES = 5
|
||||||
|
|
||||||
|
|
||||||
|
def _is_within_directory(real_base: str, real_target: str) -> bool:
|
||||||
|
"""True if ``real_target`` is inside (or equal to) ``real_base``.
|
||||||
|
|
||||||
|
Both arguments must already be resolved with ``os.path.realpath``. Uses
|
||||||
|
``commonpath`` rather than ``startswith`` so a sibling directory sharing a
|
||||||
|
name prefix (e.g. base ``/downloads`` vs ``/downloads-secret``) cannot be
|
||||||
|
reached via ``../downloads-secret``.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return os.path.commonpath([real_base, real_target]) == real_base
|
||||||
|
except ValueError:
|
||||||
|
# Raised when paths are on different drives (Windows) or mix
|
||||||
|
# absolute/relative; treat as outside the base directory.
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
# Characters that are invalid in Windows/NTFS path components. These are pre-
|
# Characters that are invalid in Windows/NTFS path components. These are pre-
|
||||||
# sanitised when substituting playlist/channel titles into output templates so
|
# sanitised when substituting playlist/channel titles into output templates so
|
||||||
@@ -127,10 +170,28 @@ def _convert_srt_to_txt_file(subtitle_path: str):
|
|||||||
# Normalize newlines so cue splitting is consistent across platforms.
|
# Normalize newlines so cue splitting is consistent across platforms.
|
||||||
content = content.replace("\r\n", "\n").replace("\r", "\n")
|
content = content.replace("\r\n", "\n").replace("\r", "\n")
|
||||||
cues = []
|
cues = []
|
||||||
|
# The caption format may resolve to a VTT file even when srt/txt was
|
||||||
|
# requested (yt-dlp falls back when the extractor doesn't offer the
|
||||||
|
# requested ext). VTT header metadata (WEBVTT/NOTE/STYLE and
|
||||||
|
# "Kind:"/"Language:" fields) only ever appears BEFORE the first timed
|
||||||
|
# cue, so only strip it while still in that header region — otherwise a
|
||||||
|
# real caption line like "Kind: regards" would be dropped as metadata.
|
||||||
|
seen_cue = False
|
||||||
for block in re.split(r"\n{2,}", content):
|
for block in re.split(r"\n{2,}", content):
|
||||||
lines = [line.strip() for line in block.split("\n") if line.strip()]
|
lines = [line.strip() for line in block.split("\n") if line.strip()]
|
||||||
if not lines:
|
if not lines:
|
||||||
continue
|
continue
|
||||||
|
has_timing = any("-->" in line for line in lines)
|
||||||
|
if not seen_cue and not has_timing:
|
||||||
|
# Still in the header region: drop metadata-only blocks
|
||||||
|
# (WEBVTT/NOTE/STYLE, or blocks made entirely of "Key: value"
|
||||||
|
# header fields such as a standalone "Kind:/Language:" block).
|
||||||
|
if re.match(r"^(WEBVTT|NOTE|STYLE)\b", lines[0]) or all(
|
||||||
|
re.match(r"^[A-Za-z][\w-]*:\s", line) for line in lines
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
if has_timing:
|
||||||
|
seen_cue = True
|
||||||
if re.fullmatch(r"\d+", lines[0]):
|
if re.fullmatch(r"\d+", lines[0]):
|
||||||
lines = lines[1:]
|
lines = lines[1:]
|
||||||
if lines and "-->" in lines[0]:
|
if lines and "-->" in lines[0]:
|
||||||
@@ -194,6 +255,8 @@ class DownloadInfo:
|
|||||||
ytdl_options_overrides=None,
|
ytdl_options_overrides=None,
|
||||||
clip_start=None,
|
clip_start=None,
|
||||||
clip_end=None,
|
clip_end=None,
|
||||||
|
live_status=None,
|
||||||
|
live_release_timestamp=None,
|
||||||
):
|
):
|
||||||
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
self.id = id if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{id}'
|
||||||
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
self.title = title if len(custom_name_prefix) == 0 else f'{custom_name_prefix}.{title}'
|
||||||
@@ -220,8 +283,24 @@ class DownloadInfo:
|
|||||||
self.ytdl_options_overrides = dict(ytdl_options_overrides or {})
|
self.ytdl_options_overrides = dict(ytdl_options_overrides or {})
|
||||||
self.clip_start = clip_start
|
self.clip_start = clip_start
|
||||||
self.clip_end = clip_end
|
self.clip_end = clip_end
|
||||||
|
self.live_status = live_status
|
||||||
|
self.live_release_timestamp = live_release_timestamp
|
||||||
self.subtitle_files = []
|
self.subtitle_files = []
|
||||||
|
|
||||||
|
# Fields that are useful server-side but must not be broadcast to browser
|
||||||
|
# clients: ``entry`` is the full yt-dlp info-dict (potentially large and
|
||||||
|
# re-sent on every progress tick) and ``subtitle_files`` is only used
|
||||||
|
# internally to derive the primary caption ``filename``.
|
||||||
|
_PUBLIC_EXCLUDED_FIELDS = ("entry", "subtitle_files")
|
||||||
|
|
||||||
|
def to_public_dict(self) -> dict:
|
||||||
|
"""Return the client-facing view, omitting server-only/bulky fields."""
|
||||||
|
return {
|
||||||
|
k: v
|
||||||
|
for k, v in self.__dict__.items()
|
||||||
|
if k not in self._PUBLIC_EXCLUDED_FIELDS
|
||||||
|
}
|
||||||
|
|
||||||
def __setstate__(self, state):
|
def __setstate__(self, state):
|
||||||
"""BACKWARD COMPATIBILITY: migrate old DownloadInfo from persistent queue files."""
|
"""BACKWARD COMPATIBILITY: migrate old DownloadInfo from persistent queue files."""
|
||||||
self.__dict__.update(state)
|
self.__dict__.update(state)
|
||||||
@@ -292,6 +371,10 @@ class DownloadInfo:
|
|||||||
self.clip_start = None
|
self.clip_start = None
|
||||||
if not hasattr(self, "clip_end"):
|
if not hasattr(self, "clip_end"):
|
||||||
self.clip_end = None
|
self.clip_end = None
|
||||||
|
if not hasattr(self, "live_status"):
|
||||||
|
self.live_status = None
|
||||||
|
if not hasattr(self, "live_release_timestamp"):
|
||||||
|
self.live_release_timestamp = None
|
||||||
|
|
||||||
|
|
||||||
_PERSISTED_DOWNLOAD_FIELDS = (
|
_PERSISTED_DOWNLOAD_FIELDS = (
|
||||||
@@ -313,6 +396,8 @@ _PERSISTED_DOWNLOAD_FIELDS = (
|
|||||||
"ytdl_options_overrides",
|
"ytdl_options_overrides",
|
||||||
"clip_start",
|
"clip_start",
|
||||||
"clip_end",
|
"clip_end",
|
||||||
|
"live_status",
|
||||||
|
"live_release_timestamp",
|
||||||
"status",
|
"status",
|
||||||
"timestamp",
|
"timestamp",
|
||||||
"error",
|
"error",
|
||||||
@@ -412,12 +497,23 @@ class Download:
|
|||||||
self.proc = None
|
self.proc = None
|
||||||
self.loop = None
|
self.loop = None
|
||||||
self.notifier = None
|
self.notifier = None
|
||||||
|
self._executor = None
|
||||||
|
|
||||||
|
# Minimum interval between forwarded 'downloading' progress ticks. yt-dlp
|
||||||
|
# emits these many times per second; without throttling, each active
|
||||||
|
# download broadcasts hundreds of socket.io events/sec to every client.
|
||||||
|
_PROGRESS_THROTTLE_SECONDS = 0.5
|
||||||
|
|
||||||
|
def _make_progress_hook(self):
|
||||||
|
last_forward = 0.0
|
||||||
|
|
||||||
def _download(self):
|
|
||||||
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
|
||||||
try:
|
|
||||||
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
|
||||||
def put_status(st):
|
def put_status(st):
|
||||||
|
nonlocal last_forward
|
||||||
|
if st.get('status') == 'downloading':
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - last_forward < self._PROGRESS_THROTTLE_SECONDS:
|
||||||
|
return
|
||||||
|
last_forward = now
|
||||||
self.status_queue.put({k: v for k, v in st.items() if k in (
|
self.status_queue.put({k: v for k, v in st.items() if k in (
|
||||||
'tmpfilename',
|
'tmpfilename',
|
||||||
'filename',
|
'filename',
|
||||||
@@ -430,6 +526,22 @@ class Download:
|
|||||||
'eta',
|
'eta',
|
||||||
)})
|
)})
|
||||||
|
|
||||||
|
return put_status
|
||||||
|
|
||||||
|
def _download(self):
|
||||||
|
# Run in our own process group so cancel() can SIGKILL the whole
|
||||||
|
# group (yt-dlp + any ffmpeg children it spawned for merge/postproc),
|
||||||
|
# instead of orphaning ffmpeg when only the yt-dlp process is killed.
|
||||||
|
if hasattr(os, 'setpgrp'):
|
||||||
|
try:
|
||||||
|
os.setpgrp()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
log.info(f"Starting download for: {self.info.title} ({self.info.url})")
|
||||||
|
try:
|
||||||
|
debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
|
||||||
|
put_status = self._make_progress_hook()
|
||||||
|
|
||||||
def put_status_postprocessor(d):
|
def put_status_postprocessor(d):
|
||||||
if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished':
|
if d['postprocessor'] == 'MoveFiles' and d['status'] == 'finished':
|
||||||
filepath = d['info_dict']['filepath']
|
filepath = d['info_dict']['filepath']
|
||||||
@@ -500,19 +612,20 @@ class Download:
|
|||||||
log.error(f"Download error for {self.info.title}: {str(exc)}")
|
log.error(f"Download error for {self.info.title}: {str(exc)}")
|
||||||
self.status_queue.put({'status': 'error', 'msg': str(exc)})
|
self.status_queue.put({'status': 'error', 'msg': str(exc)})
|
||||||
|
|
||||||
async def start(self, notifier):
|
async def start(self, notifier, executor=None):
|
||||||
log.info(f"Preparing download for: {self.info.title}")
|
log.info(f"Preparing download for: {self.info.title}")
|
||||||
if Download.manager is None:
|
if Download.manager is None:
|
||||||
Download.manager = multiprocessing.Manager()
|
Download.manager = _MP_CTX.Manager()
|
||||||
self.status_queue = Download.manager.Queue()
|
self.status_queue = Download.manager.Queue()
|
||||||
self.proc = multiprocessing.Process(target=self._download)
|
self.proc = _MP_CTX.Process(target=self._download)
|
||||||
self.proc.start()
|
self.proc.start()
|
||||||
self.loop = asyncio.get_running_loop()
|
self.loop = asyncio.get_running_loop()
|
||||||
self.notifier = notifier
|
self.notifier = notifier
|
||||||
|
self._executor = executor
|
||||||
self.info.status = 'preparing'
|
self.info.status = 'preparing'
|
||||||
await self.notifier.updated(self.info)
|
await self.notifier.updated(self.info)
|
||||||
self.status_task = asyncio.create_task(self.update_status())
|
self.status_task = asyncio.create_task(self.update_status())
|
||||||
await self.loop.run_in_executor(None, self.proc.join)
|
await self.loop.run_in_executor(self._executor, self.proc.join)
|
||||||
# Signal update_status to stop and wait for it to finish
|
# Signal update_status to stop and wait for it to finish
|
||||||
# so that all status updates (including MoveFiles with correct
|
# so that all status updates (including MoveFiles with correct
|
||||||
# file size) are processed before _post_download_cleanup runs.
|
# file size) are processed before _post_download_cleanup runs.
|
||||||
@@ -523,6 +636,22 @@ class Download:
|
|||||||
def cancel(self):
|
def cancel(self):
|
||||||
log.info(f"Cancelling download: {self.info.title}")
|
log.info(f"Cancelling download: {self.info.title}")
|
||||||
if self.running():
|
if self.running():
|
||||||
|
killed_group = False
|
||||||
|
try:
|
||||||
|
pgid = os.getpgid(self.proc.pid)
|
||||||
|
# Only kill the whole group (yt-dlp + any ffmpeg children) when
|
||||||
|
# the child actually became its own group leader via
|
||||||
|
# os.setpgrp() in _download() — that sets its pgid equal to its
|
||||||
|
# own pid. If it hasn't run setpgrp() yet, or setpgrp() failed,
|
||||||
|
# its pgid is still the SERVER's group and killpg would SIGKILL
|
||||||
|
# the entire MeTube process (PID 1 in Docker). Fall back to
|
||||||
|
# killing just the child process by pid in that case.
|
||||||
|
if pgid == self.proc.pid:
|
||||||
|
os.killpg(pgid, signal.SIGKILL)
|
||||||
|
killed_group = True
|
||||||
|
except (OSError, AttributeError):
|
||||||
|
pass
|
||||||
|
if not killed_group:
|
||||||
try:
|
try:
|
||||||
self.proc.kill()
|
self.proc.kill()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -547,7 +676,13 @@ class Download:
|
|||||||
|
|
||||||
async def update_status(self):
|
async def update_status(self):
|
||||||
while True:
|
while True:
|
||||||
status = await self.loop.run_in_executor(None, self.status_queue.get)
|
try:
|
||||||
|
status = await self.loop.run_in_executor(self._executor, self.status_queue.get)
|
||||||
|
except RuntimeError:
|
||||||
|
# The download executor was shut down (server shutting down);
|
||||||
|
# stop polling instead of raising a noisy background-task error.
|
||||||
|
log.info(f"Status polling stopped (executor shut down) for: {self.info.title}")
|
||||||
|
return
|
||||||
if status is None:
|
if status is None:
|
||||||
log.info(f"Status update finished for: {self.info.title}")
|
log.info(f"Status update finished for: {self.info.title}")
|
||||||
return
|
return
|
||||||
@@ -568,7 +703,10 @@ class Download:
|
|||||||
self.info.filename = rel_name
|
self.info.filename = rel_name
|
||||||
self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
|
self.info.size = os.path.getsize(fileName) if os.path.exists(fileName) else None
|
||||||
if getattr(self.info, 'download_type', '') == 'thumbnail':
|
if getattr(self.info, 'download_type', '') == 'thumbnail':
|
||||||
self.info.filename = re.sub(r'\.webm$', '.jpg', self.info.filename)
|
# The thumbnail convertor always emits a .jpg, but yt-dlp may
|
||||||
|
# report the pre-conversion media/thumbnail extension
|
||||||
|
# (.webm/.mp4/.png/.webp/...). Normalise to .jpg regardless.
|
||||||
|
self.info.filename = os.path.splitext(self.info.filename)[0] + '.jpg'
|
||||||
|
|
||||||
# Handle chapter files
|
# Handle chapter files
|
||||||
log.debug(f"Update status for {self.info.title}: {status}")
|
log.debug(f"Update status for {self.info.title}: {status}")
|
||||||
@@ -631,8 +769,8 @@ class PersistentQueue:
|
|||||||
def __init__(self, name, path):
|
def __init__(self, name, path):
|
||||||
self.identifier = name
|
self.identifier = name
|
||||||
pdir = os.path.dirname(path)
|
pdir = os.path.dirname(path)
|
||||||
if not os.path.isdir(pdir):
|
if pdir and not os.path.isdir(pdir):
|
||||||
os.mkdir(pdir)
|
os.makedirs(pdir, exist_ok=True)
|
||||||
self.legacy_path = path
|
self.legacy_path = path
|
||||||
self.path = f"{path}.json"
|
self.path = f"{path}.json"
|
||||||
self.store = AtomicJsonStore(self.path, kind=f"persistent_queue:{name}")
|
self.store = AtomicJsonStore(self.path, kind=f"persistent_queue:{name}")
|
||||||
@@ -738,10 +876,6 @@ class PersistentQueue:
|
|||||||
self.dict[key] = old
|
self.dict[key] = old
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def next(self):
|
|
||||||
k, v = next(iter(self.dict.items()))
|
|
||||||
return k, v
|
|
||||||
|
|
||||||
def empty(self):
|
def empty(self):
|
||||||
return not bool(self.dict)
|
return not bool(self.dict)
|
||||||
|
|
||||||
@@ -754,14 +888,35 @@ class DownloadQueue:
|
|||||||
self.pending = PersistentQueue("pending", self.config.STATE_DIR + '/pending')
|
self.pending = PersistentQueue("pending", self.config.STATE_DIR + '/pending')
|
||||||
self.active_downloads = set()
|
self.active_downloads = set()
|
||||||
self.semaphore = asyncio.Semaphore(int(self.config.MAX_CONCURRENT_DOWNLOADS))
|
self.semaphore = asyncio.Semaphore(int(self.config.MAX_CONCURRENT_DOWNLOADS))
|
||||||
|
# Each active download parks two threads for its whole duration
|
||||||
|
# (proc.join + status_queue.get). A dedicated pool keeps those from
|
||||||
|
# starving the default executor, which extract_info/live-probes also use.
|
||||||
|
self._download_executor = ThreadPoolExecutor(
|
||||||
|
max_workers=2 * int(self.config.MAX_CONCURRENT_DOWNLOADS) + 2,
|
||||||
|
thread_name_prefix="dl",
|
||||||
|
)
|
||||||
self.done.load()
|
self.done.load()
|
||||||
self._add_generation = 0
|
self._add_generation = 0
|
||||||
self._canceled_urls = set() # URLs canceled during current playlist add
|
self._canceled_urls = set() # URLs canceled during current playlist add
|
||||||
|
self._scheduled_probe_at: dict[str, float] = {}
|
||||||
|
self._scheduled_probe_failures: dict[str, int] = {}
|
||||||
|
self._live_monitor_task: Optional[asyncio.Task] = None
|
||||||
|
self._live_monitor_wakeup = asyncio.Event()
|
||||||
|
|
||||||
def cancel_add(self):
|
def cancel_add(self):
|
||||||
self._add_generation += 1
|
self._add_generation += 1
|
||||||
log.info('Playlist add operation canceled by user')
|
log.info('Playlist add operation canceled by user')
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def __is_channel_extraction(entry):
|
||||||
|
"""Return True when yt-dlp reported a channel tab as a playlist.
|
||||||
|
|
||||||
|
YouTube channel tabs are extracted with ``_type: 'playlist'`` but set
|
||||||
|
``id`` equal to ``channel_id``; real playlists keep a distinct id.
|
||||||
|
"""
|
||||||
|
channel_id = entry.get('channel_id')
|
||||||
|
return bool(channel_id) and entry.get('id') == channel_id
|
||||||
|
|
||||||
async def __import_queue(self):
|
async def __import_queue(self):
|
||||||
for k, v in self.queue.saved_items():
|
for k, v in self.queue.saved_items():
|
||||||
await self.__add_download(v, True)
|
await self.__add_download(v, True)
|
||||||
@@ -772,8 +927,160 @@ class DownloadQueue:
|
|||||||
|
|
||||||
async def initialize(self):
|
async def initialize(self):
|
||||||
log.info("Initializing DownloadQueue")
|
log.info("Initializing DownloadQueue")
|
||||||
asyncio.create_task(self.__import_queue())
|
self._start_live_monitor()
|
||||||
asyncio.create_task(self.__import_pending())
|
bg_tasks.create_task(self.__import_queue(), name="import_queue")
|
||||||
|
bg_tasks.create_task(self.__import_pending(), name="import_pending")
|
||||||
|
|
||||||
|
def _start_live_monitor(self) -> None:
|
||||||
|
if self._live_monitor_task is not None and not self._live_monitor_task.done():
|
||||||
|
return
|
||||||
|
# bg_tasks.create_task already logs unexpected task failures with the name.
|
||||||
|
self._live_monitor_task = bg_tasks.create_task(self._live_monitor_loop(), name="live_monitor")
|
||||||
|
|
||||||
|
def _register_scheduled(self, download: Download) -> None:
|
||||||
|
self._scheduled_probe_at[download.info.url] = 0
|
||||||
|
self._scheduled_probe_failures.pop(download.info.url, None)
|
||||||
|
self._start_live_monitor()
|
||||||
|
self._wake_live_monitor()
|
||||||
|
|
||||||
|
def _unregister_scheduled(self, url: str) -> None:
|
||||||
|
self._scheduled_probe_at.pop(url, None)
|
||||||
|
self._scheduled_probe_failures.pop(url, None)
|
||||||
|
|
||||||
|
def _wake_live_monitor(self) -> None:
|
||||||
|
try:
|
||||||
|
self._live_monitor_wakeup.set()
|
||||||
|
except RuntimeError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def _probe_interval_seconds(self, release_timestamp: Any) -> float:
|
||||||
|
if release_timestamp is not None:
|
||||||
|
try:
|
||||||
|
diff = float(release_timestamp) - time.time()
|
||||||
|
if diff > 0:
|
||||||
|
return max(_LIVE_CHECK_INTERVAL, min(diff, _LIVE_MAX_CHECK_INTERVAL))
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
return float(_LIVE_CHECK_INTERVAL)
|
||||||
|
|
||||||
|
def _seconds_until_next_probe(self) -> Optional[float]:
|
||||||
|
"""Time until the earliest scheduled probe, or None when nothing is scheduled."""
|
||||||
|
if not self._scheduled_probe_at:
|
||||||
|
return None
|
||||||
|
return max(0.0, min(self._scheduled_probe_at.values()) - time.time())
|
||||||
|
|
||||||
|
async def _live_monitor_loop(self) -> None:
|
||||||
|
while True:
|
||||||
|
timeout = self._seconds_until_next_probe()
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(self._live_monitor_wakeup.wait(), timeout=timeout)
|
||||||
|
except asyncio.TimeoutError:
|
||||||
|
pass
|
||||||
|
self._live_monitor_wakeup.clear()
|
||||||
|
now = time.time()
|
||||||
|
due: list[Download] = []
|
||||||
|
for url, probe_at in list(self._scheduled_probe_at.items()):
|
||||||
|
if now < probe_at:
|
||||||
|
continue
|
||||||
|
if not self.queue.exists(url):
|
||||||
|
self._unregister_scheduled(url)
|
||||||
|
continue
|
||||||
|
download = self.queue.get(url)
|
||||||
|
if download.info.status != 'scheduled' or download.canceled:
|
||||||
|
self._unregister_scheduled(url)
|
||||||
|
continue
|
||||||
|
due.append(download)
|
||||||
|
for download in due:
|
||||||
|
try:
|
||||||
|
await self._probe_scheduled_download(download)
|
||||||
|
except Exception as exc:
|
||||||
|
# Defensive: _probe_scheduled_download handles its own errors,
|
||||||
|
# but never let an unexpected failure leave probe_at in the past
|
||||||
|
# (which would spin this loop) or kill the monitor task.
|
||||||
|
log.exception("Scheduled live probe crashed for %s: %s", download.info.url, exc)
|
||||||
|
if download.info.url in self._scheduled_probe_at:
|
||||||
|
self._scheduled_probe_at[download.info.url] = time.time() + _LIVE_CHECK_INTERVAL
|
||||||
|
|
||||||
|
async def _probe_scheduled_download(self, download: Download) -> None:
|
||||||
|
url = download.info.url
|
||||||
|
info = download.info
|
||||||
|
if info.status != 'scheduled' or download.canceled:
|
||||||
|
self._unregister_scheduled(url)
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
entry = await asyncio.get_running_loop().run_in_executor(
|
||||||
|
None,
|
||||||
|
partial(
|
||||||
|
self.__extract_info,
|
||||||
|
url,
|
||||||
|
getattr(info, 'ytdl_options_presets', None),
|
||||||
|
getattr(info, 'ytdl_options_overrides', {}) or {},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except Exception as exc:
|
||||||
|
# Treat all probe failures (transient network blips, rate limits,
|
||||||
|
# extractor errors) as recoverable up to a point: retry on the next
|
||||||
|
# interval and only give up after repeated consecutive failures so a
|
||||||
|
# momentary glitch doesn't abandon a stream the user is waiting for.
|
||||||
|
fails = self._scheduled_probe_failures.get(url, 0) + 1
|
||||||
|
self._scheduled_probe_failures[url] = fails
|
||||||
|
if fails >= _LIVE_PROBE_MAX_FAILURES:
|
||||||
|
log.warning(
|
||||||
|
"Giving up on scheduled live probe for %s after %d consecutive failures: %s",
|
||||||
|
info.title, fails, exc,
|
||||||
|
)
|
||||||
|
info.status = 'error'
|
||||||
|
info.msg = str(exc)
|
||||||
|
if not info.error:
|
||||||
|
info.error = str(exc)
|
||||||
|
self._unregister_scheduled(url)
|
||||||
|
self.queue.delete(url)
|
||||||
|
self.done.put(download)
|
||||||
|
await self.notifier.completed(info)
|
||||||
|
else:
|
||||||
|
log.warning(
|
||||||
|
"Scheduled live probe failed for %s (attempt %d/%d), will retry: %s",
|
||||||
|
info.title, fails, _LIVE_PROBE_MAX_FAILURES, exc,
|
||||||
|
)
|
||||||
|
self._scheduled_probe_at[url] = time.time() + _LIVE_CHECK_INTERVAL
|
||||||
|
return
|
||||||
|
|
||||||
|
# Successful probe resets the transient-failure streak.
|
||||||
|
self._scheduled_probe_failures.pop(url, None)
|
||||||
|
|
||||||
|
release_ts = entry.get('release_timestamp')
|
||||||
|
live_status = entry.get('live_status')
|
||||||
|
if release_ts is not None:
|
||||||
|
info.live_release_timestamp = release_ts
|
||||||
|
if live_status is not None:
|
||||||
|
info.live_status = live_status
|
||||||
|
|
||||||
|
if live_status == 'is_upcoming':
|
||||||
|
self._scheduled_probe_at[url] = time.time() + self._probe_interval_seconds(release_ts)
|
||||||
|
await self.notifier.updated(info)
|
||||||
|
return
|
||||||
|
|
||||||
|
self._unregister_scheduled(url)
|
||||||
|
info.status = 'pending'
|
||||||
|
# Clear the "scheduled to start at ..." placeholder now that the stream
|
||||||
|
# is live and a real download is about to begin.
|
||||||
|
info.error = None
|
||||||
|
info.msg = None
|
||||||
|
await self.notifier.updated(info)
|
||||||
|
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
||||||
|
|
||||||
|
def _schedule_upcoming_download(self, download: Download) -> None:
|
||||||
|
download.info.status = 'scheduled'
|
||||||
|
self.queue.put(download)
|
||||||
|
self._register_scheduled(download)
|
||||||
|
|
||||||
|
def _force_start_scheduled(self, download: Download) -> None:
|
||||||
|
self._unregister_scheduled(download.info.url)
|
||||||
|
download.info.status = 'pending'
|
||||||
|
download.info.error = None
|
||||||
|
download.info.msg = None
|
||||||
|
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
||||||
|
|
||||||
async def __start_download(self, download):
|
async def __start_download(self, download):
|
||||||
if download.canceled:
|
if download.canceled:
|
||||||
@@ -783,7 +1090,7 @@ class DownloadQueue:
|
|||||||
if download.canceled:
|
if download.canceled:
|
||||||
log.info(f"Download {download.info.title} was canceled, skipping start.")
|
log.info(f"Download {download.info.title} was canceled, skipping start.")
|
||||||
return
|
return
|
||||||
await download.start(self.notifier)
|
await download.start(self.notifier, self._download_executor)
|
||||||
self._post_download_cleanup(download)
|
self._post_download_cleanup(download)
|
||||||
|
|
||||||
def _post_download_cleanup(self, download):
|
def _post_download_cleanup(self, download):
|
||||||
@@ -794,22 +1101,35 @@ class DownloadQueue:
|
|||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
download.info.status = 'error'
|
download.info.status = 'error'
|
||||||
|
# A progress tick may have set filename to a temp-directory
|
||||||
|
# relative path before the error occurred; clear it so the UI
|
||||||
|
# doesn't render a broken link (or, worse, so a later trashcan
|
||||||
|
# delete doesn't act on a path outside the download directory).
|
||||||
|
# Captions downloads may still have captured valid subtitle
|
||||||
|
# files even when the overall status is 'error' — keep those.
|
||||||
|
has_captured_subtitles = bool(getattr(download.info, 'subtitle_files', None))
|
||||||
|
if not (download.info.download_type == 'captions' and has_captured_subtitles):
|
||||||
|
download.info.filename = None
|
||||||
|
download.info.size = None
|
||||||
download.close()
|
download.close()
|
||||||
if self.queue.exists(download.info.url):
|
if self.queue.exists(download.info.url):
|
||||||
self.queue.delete(download.info.url)
|
self.queue.delete(download.info.url)
|
||||||
if download.canceled:
|
if download.canceled:
|
||||||
asyncio.create_task(self.notifier.canceled(download.info.url))
|
bg_tasks.create_task(self.notifier.canceled(download.info.url), name="notify_canceled")
|
||||||
else:
|
else:
|
||||||
self.done.put(download)
|
self.done.put(download)
|
||||||
asyncio.create_task(self.notifier.completed(download.info))
|
bg_tasks.create_task(self.notifier.completed(download.info), name="notify_completed")
|
||||||
try:
|
try:
|
||||||
clear_after = int(self.config.CLEAR_COMPLETED_AFTER)
|
clear_after = int(self.config.CLEAR_COMPLETED_AFTER)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
log.error(f'CLEAR_COMPLETED_AFTER is set to an invalid value "{self.config.CLEAR_COMPLETED_AFTER}", expected an integer number of seconds')
|
log.error(f'CLEAR_COMPLETED_AFTER is set to an invalid value "{self.config.CLEAR_COMPLETED_AFTER}", expected an integer number of seconds')
|
||||||
clear_after = 0
|
clear_after = 0
|
||||||
if clear_after > 0:
|
if clear_after > 0:
|
||||||
task = asyncio.create_task(self.__auto_clear_after_delay(download.info.url, clear_after))
|
# bg_tasks.create_task already logs unexpected task failures.
|
||||||
task.add_done_callback(lambda t: log.error(f'Auto-clear task failed: {t.exception()}') if not t.cancelled() and t.exception() else None)
|
bg_tasks.create_task(
|
||||||
|
self.__auto_clear_after_delay(download.info.url, clear_after),
|
||||||
|
name="auto_clear",
|
||||||
|
)
|
||||||
|
|
||||||
async def __auto_clear_after_delay(self, url, delay_seconds):
|
async def __auto_clear_after_delay(self, url, delay_seconds):
|
||||||
await asyncio.sleep(delay_seconds)
|
await asyncio.sleep(delay_seconds)
|
||||||
@@ -820,9 +1140,9 @@ class DownloadQueue:
|
|||||||
def _build_ytdl_options(self, ytdl_options_presets=None, ytdl_options_overrides=None):
|
def _build_ytdl_options(self, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||||
"""Merge global options, presets (in order), and per-download overrides."""
|
"""Merge global options, presets (in order), and per-download overrides."""
|
||||||
opts = dict(self.config.YTDL_OPTIONS)
|
opts = dict(self.config.YTDL_OPTIONS)
|
||||||
for preset_name in ytdl_options_presets or []:
|
opts.update(merge_ytdl_option_layers(
|
||||||
opts.update(self.config.YTDL_OPTIONS_PRESETS.get(preset_name, {}))
|
ytdl_options_presets, ytdl_options_overrides, self.config.YTDL_OPTIONS_PRESETS
|
||||||
opts.update(ytdl_options_overrides or {})
|
))
|
||||||
return opts
|
return opts
|
||||||
|
|
||||||
def __extract_info(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
def __extract_info(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||||
@@ -850,7 +1170,7 @@ class DownloadQueue:
|
|||||||
return None, {'status': 'error', 'msg': 'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
|
return None, {'status': 'error', 'msg': 'A folder for the download was specified but CUSTOM_DIRS is not true in the configuration.'}
|
||||||
dldirectory = os.path.realpath(os.path.join(base_directory, folder))
|
dldirectory = os.path.realpath(os.path.join(base_directory, folder))
|
||||||
real_base_directory = os.path.realpath(base_directory)
|
real_base_directory = os.path.realpath(base_directory)
|
||||||
if not dldirectory.startswith(real_base_directory):
|
if not _is_within_directory(real_base_directory, dldirectory):
|
||||||
return None, {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'}
|
return None, {'status': 'error', 'msg': f'Folder "{folder}" must resolve inside the base download directory "{real_base_directory}"'}
|
||||||
if not os.path.isdir(dldirectory):
|
if not os.path.isdir(dldirectory):
|
||||||
if not self.config.CREATE_CUSTOM_DIRS:
|
if not self.config.CREATE_CUSTOM_DIRS:
|
||||||
@@ -886,9 +1206,16 @@ class DownloadQueue:
|
|||||||
log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries')
|
log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries')
|
||||||
ytdl_options['playlistend'] = playlist_item_limit
|
ytdl_options['playlistend'] = playlist_item_limit
|
||||||
download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, dl.quality, dl.format, ytdl_options, dl)
|
download = Download(dldirectory, self.config.TEMP_DIR, output, output_chapter, dl.quality, dl.format, ytdl_options, dl)
|
||||||
|
is_upcoming = (
|
||||||
|
getattr(dl, 'live_status', None) == 'is_upcoming'
|
||||||
|
or getattr(dl, 'status', None) == 'scheduled'
|
||||||
|
)
|
||||||
if auto_start is True:
|
if auto_start is True:
|
||||||
|
if is_upcoming:
|
||||||
|
self._schedule_upcoming_download(download)
|
||||||
|
else:
|
||||||
self.queue.put(download)
|
self.queue.put(download)
|
||||||
asyncio.create_task(self.__start_download(download))
|
bg_tasks.create_task(self.__start_download(download), name="start_download")
|
||||||
else:
|
else:
|
||||||
self.pending.put(download)
|
self.pending.put(download)
|
||||||
await self.notifier.added(dl)
|
await self.notifier.added(dl)
|
||||||
@@ -920,7 +1247,9 @@ class DownloadQueue:
|
|||||||
|
|
||||||
error = None
|
error = None
|
||||||
if "live_status" in entry and "release_timestamp" in entry and entry.get("live_status") == "is_upcoming":
|
if "live_status" in entry and "release_timestamp" in entry and entry.get("live_status") == "is_upcoming":
|
||||||
dt_ts = datetime.fromtimestamp(entry.get("release_timestamp")).strftime('%Y-%m-%d %H:%M:%S %z')
|
# astimezone() makes this an aware datetime in the server's local
|
||||||
|
# zone; a naive datetime's %z renders as an empty string.
|
||||||
|
dt_ts = datetime.fromtimestamp(entry.get("release_timestamp")).astimezone().strftime('%Y-%m-%d %H:%M:%S %z')
|
||||||
error = f"Live stream is scheduled to start at {dt_ts}"
|
error = f"Live stream is scheduled to start at {dt_ts}"
|
||||||
else:
|
else:
|
||||||
if "msg" in entry:
|
if "msg" in entry:
|
||||||
@@ -952,6 +1281,8 @@ class DownloadQueue:
|
|||||||
_add_gen,
|
_add_gen,
|
||||||
)
|
)
|
||||||
elif etype == 'playlist' or etype == 'channel':
|
elif etype == 'playlist' or etype == 'channel':
|
||||||
|
if etype == 'playlist' and self.__is_channel_extraction(entry):
|
||||||
|
etype = 'channel'
|
||||||
log.debug(f'Processing as a {etype}')
|
log.debug(f'Processing as a {etype}')
|
||||||
entries = entry['entries']
|
entries = entry['entries']
|
||||||
# Convert generator to list if needed (for len() and slicing operations)
|
# Convert generator to list if needed (for len() and slicing operations)
|
||||||
@@ -971,7 +1302,15 @@ class DownloadQueue:
|
|||||||
if "id" not in etr:
|
if "id" not in etr:
|
||||||
etr["id"] = _entry_id(etr)
|
etr["id"] = _entry_id(etr)
|
||||||
etr["_type"] = "video"
|
etr["_type"] = "video"
|
||||||
etr[etype] = entry.get("id") or entry.get("channel_id") or entry.get("channel")
|
if etype == 'channel':
|
||||||
|
etr["channel"] = (
|
||||||
|
entry.get("channel")
|
||||||
|
or entry.get("uploader")
|
||||||
|
or entry.get("title")
|
||||||
|
or entry.get("id")
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
etr["playlist"] = entry.get("id") or entry.get("channel_id") or entry.get("channel")
|
||||||
etr[f"{etype}_index"] = '{{0:0{0:d}d}}'.format(index_digits).format(index)
|
etr[f"{etype}_index"] = '{{0:0{0:d}d}}'.format(index_digits).format(index)
|
||||||
etr[f"{etype}_count"] = total_entries
|
etr[f"{etype}_count"] = total_entries
|
||||||
etr[f"{etype}_autonumber"] = index
|
etr[f"{etype}_autonumber"] = index
|
||||||
@@ -1008,13 +1347,18 @@ class DownloadQueue:
|
|||||||
if any(res['status'] == 'error' for res in results):
|
if any(res['status'] == 'error' for res in results):
|
||||||
return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
|
return {'status': 'error', 'msg': ', '.join(res['msg'] for res in results if res['status'] == 'error' and 'msg' in res)}
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
elif etype == 'video' or (etype.startswith('url') and 'id' in entry and 'title' in entry):
|
elif etype == 'video':
|
||||||
log.debug('Processing as a video')
|
log.debug('Processing as a video')
|
||||||
key = entry.get('webpage_url') or entry['url']
|
key = entry.get('webpage_url') or entry['url']
|
||||||
if key in self._canceled_urls:
|
if key in self._canceled_urls:
|
||||||
log.info(f'Skipping canceled URL: {entry.get("title") or key}')
|
log.info(f'Skipping canceled URL: {entry.get("title") or key}')
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
if not self.queue.exists(key):
|
if self.queue.exists(key) or self.pending.exists(key):
|
||||||
|
# Surface the skip instead of silently no-op'ing, and avoid
|
||||||
|
# clobbering an existing pending entry's options with a
|
||||||
|
# fresh DownloadInfo built from possibly-different args.
|
||||||
|
title = entry.get('title') or key
|
||||||
|
return {'status': 'ok', 'msg': f'Already in queue: {title}'}
|
||||||
dl = DownloadInfo(
|
dl = DownloadInfo(
|
||||||
id=entry['id'],
|
id=entry['id'],
|
||||||
title=entry.get('title') or entry['id'],
|
title=entry.get('title') or entry['id'],
|
||||||
@@ -1036,6 +1380,8 @@ class DownloadQueue:
|
|||||||
ytdl_options_overrides=ytdl_options_overrides,
|
ytdl_options_overrides=ytdl_options_overrides,
|
||||||
clip_start=clip_start,
|
clip_start=clip_start,
|
||||||
clip_end=clip_end,
|
clip_end=clip_end,
|
||||||
|
live_status=entry.get('live_status'),
|
||||||
|
live_release_timestamp=entry.get('release_timestamp'),
|
||||||
)
|
)
|
||||||
await self.__add_download(dl, auto_start)
|
await self.__add_download(dl, auto_start)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
@@ -1156,13 +1502,21 @@ class DownloadQueue:
|
|||||||
|
|
||||||
async def start_pending(self, ids):
|
async def start_pending(self, ids):
|
||||||
for id in ids:
|
for id in ids:
|
||||||
if not self.pending.exists(id):
|
if self.pending.exists(id):
|
||||||
log.warning(f'requested start for non-existent download {id}')
|
|
||||||
continue
|
|
||||||
dl = self.pending.get(id)
|
dl = self.pending.get(id)
|
||||||
self.queue.put(dl)
|
|
||||||
self.pending.delete(id)
|
self.pending.delete(id)
|
||||||
asyncio.create_task(self.__start_download(dl))
|
if getattr(dl.info, 'live_status', None) == 'is_upcoming':
|
||||||
|
self._schedule_upcoming_download(dl)
|
||||||
|
else:
|
||||||
|
self.queue.put(dl)
|
||||||
|
bg_tasks.create_task(self.__start_download(dl), name="start_download")
|
||||||
|
continue
|
||||||
|
if self.queue.exists(id):
|
||||||
|
dl = self.queue.get(id)
|
||||||
|
if dl.info.status == 'scheduled':
|
||||||
|
self._force_start_scheduled(dl)
|
||||||
|
continue
|
||||||
|
log.warning(f'requested start for non-existent download {id}')
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
|
|
||||||
async def cancel(self, ids):
|
async def cancel(self, ids):
|
||||||
@@ -1177,6 +1531,8 @@ class DownloadQueue:
|
|||||||
log.warning(f'requested cancel for non-existent download {id}')
|
log.warning(f'requested cancel for non-existent download {id}')
|
||||||
continue
|
continue
|
||||||
dl = self.queue.get(id)
|
dl = self.queue.get(id)
|
||||||
|
if dl.info.status == 'scheduled':
|
||||||
|
self._unregister_scheduled(id)
|
||||||
if dl.started():
|
if dl.started():
|
||||||
dl.cancel()
|
dl.cancel()
|
||||||
else:
|
else:
|
||||||
@@ -1192,11 +1548,33 @@ class DownloadQueue:
|
|||||||
continue
|
continue
|
||||||
if self.config.DELETE_FILE_ON_TRASHCAN:
|
if self.config.DELETE_FILE_ON_TRASHCAN:
|
||||||
dl = self.done.get(id)
|
dl = self.done.get(id)
|
||||||
|
dldirectory, calc_error = self.__calc_download_path(dl.info.download_type, dl.info.folder)
|
||||||
|
if calc_error is not None or not dldirectory:
|
||||||
|
log.warning(f'deleting files for download {id} skipped: could not resolve download directory')
|
||||||
|
else:
|
||||||
|
# Remove the primary output plus any per-chapter / per-subtitle
|
||||||
|
# outputs. Each filename is relative to the download directory.
|
||||||
|
rel_names = []
|
||||||
|
if getattr(dl.info, 'filename', None):
|
||||||
|
rel_names.append(dl.info.filename)
|
||||||
|
for extra in (getattr(dl.info, 'chapter_files', None) or []):
|
||||||
|
if isinstance(extra, dict) and extra.get('filename'):
|
||||||
|
rel_names.append(extra['filename'])
|
||||||
|
for extra in (getattr(dl.info, 'subtitle_files', None) or []):
|
||||||
|
if isinstance(extra, dict) and extra.get('filename'):
|
||||||
|
rel_names.append(extra['filename'])
|
||||||
|
real_base_directory = os.path.realpath(dldirectory)
|
||||||
|
for rel_name in rel_names:
|
||||||
|
full_path = os.path.realpath(os.path.join(dldirectory, rel_name))
|
||||||
|
if not _is_within_directory(real_base_directory, full_path):
|
||||||
|
log.warning(f'skipping deletion of "{rel_name}" for download {id}: resolves outside the download directory')
|
||||||
|
continue
|
||||||
try:
|
try:
|
||||||
dldirectory, _ = self.__calc_download_path(dl.info.download_type, dl.info.folder)
|
os.remove(full_path)
|
||||||
os.remove(os.path.join(dldirectory, dl.info.filename))
|
except FileNotFoundError:
|
||||||
except Exception as e:
|
pass
|
||||||
log.warning(f'deleting file for download {id} failed with error message {e!r}')
|
except OSError as e:
|
||||||
|
log.warning(f'deleting file "{rel_name}" for download {id} failed with error message {e!r}')
|
||||||
self.done.delete(id)
|
self.done.delete(id)
|
||||||
await self.notifier.cleared(id)
|
await self.notifier.cleared(id)
|
||||||
return {'status': 'ok'}
|
return {'status': 'ok'}
|
||||||
@@ -1205,3 +1583,13 @@ class DownloadQueue:
|
|||||||
return (list((k, v.info) for k, v in self.queue.items()) +
|
return (list((k, v.info) for k, v in self.queue.items()) +
|
||||||
list((k, v.info) for k, v in self.pending.items()),
|
list((k, v.info) for k, v in self.pending.items()),
|
||||||
list((k, v.info) for k, v in self.done.items()))
|
list((k, v.info) for k, v in self.done.items()))
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
# Kill any still-running download subprocesses (and their ffmpeg
|
||||||
|
# children) before tearing down the executor, so they aren't orphaned
|
||||||
|
# when the server exits. Their queue entries stay persisted and are
|
||||||
|
# re-imported/restarted on next startup.
|
||||||
|
for _key, download in list(self.queue.items()):
|
||||||
|
if download.started() and download.running():
|
||||||
|
download.cancel()
|
||||||
|
self._download_executor.shutdown(wait=False, cancel_futures=True)
|
||||||
|
|||||||
+55
-5
@@ -2,11 +2,54 @@
|
|||||||
|
|
||||||
PUID="${UID:-$PUID}"
|
PUID="${UID:-$PUID}"
|
||||||
PGID="${GID:-$PGID}"
|
PGID="${GID:-$PGID}"
|
||||||
|
AUDIO_DOWNLOAD_DIR="${AUDIO_DOWNLOAD_DIR:-$DOWNLOAD_DIR}"
|
||||||
|
|
||||||
echo "Setting umask to ${UMASK}"
|
echo "Setting umask to ${UMASK}"
|
||||||
umask ${UMASK}
|
umask ${UMASK}
|
||||||
echo "Creating download directory (${DOWNLOAD_DIR}), state directory (${STATE_DIR}), and temp dir (${TEMP_DIR})"
|
echo "Creating download directory (${DOWNLOAD_DIR}), audio download directory (${AUDIO_DOWNLOAD_DIR}), state directory (${STATE_DIR}), and temp dir (${TEMP_DIR})"
|
||||||
mkdir -p "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
mkdir -p "${DOWNLOAD_DIR}" "${AUDIO_DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||||
|
|
||||||
|
do_upgrade() {
|
||||||
|
echo "Upgrading yt-dlp to nightly channel..."
|
||||||
|
if ! python3 -m pip --version >/dev/null 2>&1; then
|
||||||
|
echo "pip not found; attempting ensurepip"
|
||||||
|
python3 -m ensurepip --upgrade >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
if ! python3 -m pip install -U --pre "yt-dlp[default,curl-cffi,deno]"; then
|
||||||
|
echo "Warning: yt-dlp nightly upgrade failed; continuing with existing installation"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "yt-dlp nightly upgrade complete"
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
run_supervised() {
|
||||||
|
while true; do
|
||||||
|
"$@" &
|
||||||
|
child_pid=$!
|
||||||
|
trap 'kill -TERM "$child_pid" 2>/dev/null; wait "$child_pid" 2>/dev/null' TERM INT
|
||||||
|
wait "$child_pid"
|
||||||
|
exit_code=$?
|
||||||
|
trap - TERM INT
|
||||||
|
if [ "$exit_code" -eq 42 ]; then
|
||||||
|
echo "MeTube requested yt-dlp update restart (exit 42)"
|
||||||
|
do_upgrade || true
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
return "$exit_code"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
nightly_enabled() {
|
||||||
|
[ -n "${YTDL_NIGHTLY_UPDATE_TIME}" ]
|
||||||
|
}
|
||||||
|
|
||||||
|
disable_nightly_for_non_root() {
|
||||||
|
if nightly_enabled; then
|
||||||
|
echo "YTDL_NIGHTLY_UPDATE_TIME is set but this container runs as a non-root user; nightly yt-dlp updates are not supported. Ignoring YTDL_NIGHTLY_UPDATE_TIME."
|
||||||
|
unset YTDL_NIGHTLY_UPDATE_TIME
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
if [ `id -u` -eq 0 ] && [ `id -g` -eq 0 ]; then
|
if [ `id -u` -eq 0 ] && [ `id -g` -eq 0 ]; then
|
||||||
if [ "${PUID}" -eq 0 ]; then
|
if [ "${PUID}" -eq 0 ]; then
|
||||||
@@ -14,15 +57,22 @@ if [ `id -u` -eq 0 ] && [ `id -g` -eq 0 ]; then
|
|||||||
fi
|
fi
|
||||||
if [ "${CHOWN_DIRS:-true}" != "false" ]; then
|
if [ "${CHOWN_DIRS:-true}" != "false" ]; then
|
||||||
echo "Changing ownership of download and state directories to ${PUID}:${PGID}"
|
echo "Changing ownership of download and state directories to ${PUID}:${PGID}"
|
||||||
chown -R "${PUID}":"${PGID}" /app "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
chown -R "${PUID}":"${PGID}" /app "${DOWNLOAD_DIR}" "${AUDIO_DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||||
|
fi
|
||||||
|
if nightly_enabled; then
|
||||||
|
echo "YTDL_NIGHTLY_UPDATE_TIME is set to ${YTDL_NIGHTLY_UPDATE_TIME}; upgrading yt-dlp on startup"
|
||||||
|
do_upgrade || true
|
||||||
fi
|
fi
|
||||||
echo "Starting BgUtils POT Provider"
|
echo "Starting BgUtils POT Provider"
|
||||||
gosu "${PUID}":"${PGID}" bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
|
gosu "${PUID}":"${PGID}" bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
|
||||||
echo "Running MeTube as user ${PUID}:${PGID}"
|
echo "Running MeTube as user ${PUID}:${PGID}"
|
||||||
exec gosu "${PUID}":"${PGID}" python3 app/main.py
|
run_supervised gosu "${PUID}":"${PGID}" python3 app/main.py
|
||||||
|
exit $?
|
||||||
else
|
else
|
||||||
echo "User set by docker; running MeTube as `id -u`:`id -g`"
|
echo "User set by docker; running MeTube as `id -u`:`id -g`"
|
||||||
|
disable_nightly_for_non_root
|
||||||
echo "Starting BgUtils POT Provider"
|
echo "Starting BgUtils POT Provider"
|
||||||
bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
|
bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
|
||||||
exec python3 app/main.py
|
run_supervised python3 app/main.py
|
||||||
|
exit $?
|
||||||
fi
|
fi
|
||||||
|
|||||||
+26
-25
@@ -5,7 +5,7 @@
|
|||||||
"ng": "ng",
|
"ng": "ng",
|
||||||
"start": "ng serve",
|
"start": "ng serve",
|
||||||
"build": "ng build",
|
"build": "ng build",
|
||||||
"build:watch": "ng build --watch",
|
"build:watch": "ng build --watch --configuration development",
|
||||||
"test": "ng test",
|
"test": "ng test",
|
||||||
"lint": "ng lint"
|
"lint": "ng lint"
|
||||||
},
|
},
|
||||||
@@ -21,43 +21,44 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"packageManager": "pnpm@11.5.2",
|
||||||
"private": true,
|
"private": true,
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@angular/animations": "^21.2.9",
|
"@angular/animations": "^22.0.6",
|
||||||
"@angular/common": "^21.2.9",
|
"@angular/common": "^22.0.6",
|
||||||
"@angular/compiler": "^21.2.9",
|
"@angular/compiler": "^22.0.6",
|
||||||
"@angular/core": "^21.2.9",
|
"@angular/core": "^22.0.6",
|
||||||
"@angular/forms": "^21.2.9",
|
"@angular/forms": "^22.0.6",
|
||||||
"@angular/platform-browser": "^21.2.9",
|
"@angular/platform-browser": "^22.0.6",
|
||||||
"@angular/platform-browser-dynamic": "^21.2.9",
|
"@angular/platform-browser-dynamic": "^22.0.6",
|
||||||
"@angular/service-worker": "^21.2.9",
|
"@angular/service-worker": "^22.0.6",
|
||||||
"@fortawesome/angular-fontawesome": "~4.0.0",
|
"@fortawesome/angular-fontawesome": "~4.0.0",
|
||||||
"@fortawesome/fontawesome-svg-core": "^7.2.0",
|
"@fortawesome/fontawesome-svg-core": "^7.3.0",
|
||||||
"@fortawesome/free-brands-svg-icons": "^7.2.0",
|
"@fortawesome/free-brands-svg-icons": "^7.3.0",
|
||||||
"@fortawesome/free-regular-svg-icons": "^7.2.0",
|
"@fortawesome/free-regular-svg-icons": "^7.3.0",
|
||||||
"@fortawesome/free-solid-svg-icons": "^7.2.0",
|
"@fortawesome/free-solid-svg-icons": "^7.3.0",
|
||||||
"@ng-bootstrap/ng-bootstrap": "^20.0.0",
|
"@ng-bootstrap/ng-bootstrap": "^21.0.0",
|
||||||
"@ng-select/ng-select": "^21.8.0",
|
"@ng-select/ng-select": "^23.2.0",
|
||||||
"@popperjs/core": "^2.11.8",
|
"@popperjs/core": "^2.11.8",
|
||||||
"bootstrap": "^5.3.8",
|
"bootstrap": "^5.3.8",
|
||||||
"ngx-cookie-service": "^21.3.1",
|
"ngx-cookie-service": "^22.0.0",
|
||||||
"ngx-socket-io": "~4.10.0",
|
"ngx-socket-io": "~4.10.0",
|
||||||
"rxjs": "~7.8.2",
|
"rxjs": "~7.8.2",
|
||||||
"tslib": "^2.8.1",
|
"tslib": "^2.8.1",
|
||||||
"zone.js": "0.15.0"
|
"zone.js": "0.15.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@angular-eslint/builder": "21.1.0",
|
"@angular-eslint/builder": "22.0.0",
|
||||||
"@angular/build": "^21.2.7",
|
"@angular/build": "^22.0.5",
|
||||||
"@angular/cli": "^21.2.7",
|
"@angular/cli": "^22.0.5",
|
||||||
"@angular/compiler-cli": "^21.2.9",
|
"@angular/compiler-cli": "^22.0.6",
|
||||||
"@angular/localize": "^21.2.9",
|
"@angular/localize": "^22.0.6",
|
||||||
"@eslint/js": "^9.39.4",
|
"@eslint/js": "^9.39.4",
|
||||||
"angular-eslint": "21.1.0",
|
"angular-eslint": "22.0.0",
|
||||||
"eslint": "^9.39.4",
|
"eslint": "^9.39.4",
|
||||||
"jsdom": "^27.4.0",
|
"jsdom": "^27.4.0",
|
||||||
"typescript": "~5.9.3",
|
"typescript": "~6.0.3",
|
||||||
"typescript-eslint": "8.47.0",
|
"typescript-eslint": "8.62.0",
|
||||||
"vitest": "^4.1.5"
|
"vitest": "^4.1.10"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Generated
+1832
-1885
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,6 @@
|
|||||||
import { ApplicationConfig, provideBrowserGlobalErrorListeners, isDevMode, provideZoneChangeDetection } from '@angular/core';
|
import { ApplicationConfig, provideBrowserGlobalErrorListeners, isDevMode, provideZoneChangeDetection } from '@angular/core';
|
||||||
import { provideServiceWorker } from '@angular/service-worker';
|
import { provideServiceWorker } from '@angular/service-worker';
|
||||||
import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http';
|
import { provideHttpClient, withInterceptorsFromDi, withXhr } from '@angular/common/http';
|
||||||
|
|
||||||
export const appConfig: ApplicationConfig = {
|
export const appConfig: ApplicationConfig = {
|
||||||
providers: [
|
providers: [
|
||||||
@@ -12,6 +12,6 @@ export const appConfig: ApplicationConfig = {
|
|||||||
// or after 30 seconds (whichever comes first).
|
// or after 30 seconds (whichever comes first).
|
||||||
registrationStrategy: 'registerWhenStable:30000'
|
registrationStrategy: 'registerWhenStable:30000'
|
||||||
}),
|
}),
|
||||||
provideHttpClient(withInterceptorsFromDi()),
|
provideHttpClient(withXhr(), withInterceptorsFromDi()),
|
||||||
]
|
]
|
||||||
};
|
};
|
||||||
|
|||||||
+73
-67
@@ -279,13 +279,12 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="col-12 col-md-6 col-lg-3">
|
<div class="col-12 col-md-6 col-lg-3">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Format</span>
|
<span class="input-group-text help-title" ngbPopover="Subtitle output format for captions mode." triggers="click" autoClose="outside" container="body">Format</span>
|
||||||
<select class="form-select"
|
<select class="form-select"
|
||||||
name="format"
|
name="format"
|
||||||
[(ngModel)]="format"
|
[(ngModel)]="format"
|
||||||
(change)="formatChanged()"
|
(change)="formatChanged()"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Subtitle output format for captions mode">
|
|
||||||
@for (f of formatOptions; track f.id) {
|
@for (f of formatOptions; track f.id) {
|
||||||
<option [ngValue]="f.id">{{ f.text }}</option>
|
<option [ngValue]="f.id">{{ f.text }}</option>
|
||||||
}
|
}
|
||||||
@@ -294,7 +293,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="col-12 col-md-6 col-lg-3">
|
<div class="col-12 col-md-6 col-lg-3">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Language</span>
|
<span class="input-group-text help-title" ngbPopover="Subtitle language (you can type any language code)." triggers="click" autoClose="outside" container="body">Language</span>
|
||||||
<input class="form-control"
|
<input class="form-control"
|
||||||
type="text"
|
type="text"
|
||||||
list="subtitleLanguageOptions"
|
list="subtitleLanguageOptions"
|
||||||
@@ -302,8 +301,7 @@
|
|||||||
[(ngModel)]="subtitleLanguage"
|
[(ngModel)]="subtitleLanguage"
|
||||||
(change)="subtitleLanguageChanged()"
|
(change)="subtitleLanguageChanged()"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||||
placeholder="e.g. en, es, zh-Hans"
|
placeholder="e.g. en, es, zh-Hans">
|
||||||
ngbTooltip="Subtitle language (you can type any language code)">
|
|
||||||
<datalist id="subtitleLanguageOptions">
|
<datalist id="subtitleLanguageOptions">
|
||||||
@for (lang of subtitleLanguages; track lang.id) {
|
@for (lang of subtitleLanguages; track lang.id) {
|
||||||
<option [value]="lang.id">{{ lang.text }}</option>
|
<option [value]="lang.id">{{ lang.text }}</option>
|
||||||
@@ -313,13 +311,12 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="col-12 col-md-6 col-lg-3">
|
<div class="col-12 col-md-6 col-lg-3">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Subtitle Source</span>
|
<span class="input-group-text help-title" ngbPopover="Choose manual, auto, or fallback preference for captions mode." triggers="click" autoClose="outside" container="body">Subtitle Source</span>
|
||||||
<select class="form-select"
|
<select class="form-select"
|
||||||
name="subtitleMode"
|
name="subtitleMode"
|
||||||
[(ngModel)]="subtitleMode"
|
[(ngModel)]="subtitleMode"
|
||||||
(change)="subtitleModeChanged()"
|
(change)="subtitleModeChanged()"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Choose manual, auto, or fallback preference for captions mode">
|
|
||||||
@for (mode of subtitleModes; track mode.id) {
|
@for (mode of subtitleModes; track mode.id) {
|
||||||
<option [ngValue]="mode.id">{{ mode.text }}</option>
|
<option [ngValue]="mode.id">{{ mode.text }}</option>
|
||||||
}
|
}
|
||||||
@@ -375,34 +372,29 @@
|
|||||||
<div class="row g-3">
|
<div class="row g-3">
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Download Folder</span>
|
<span class="input-group-text help-title" ngbPopover="Type to filter existing folders, or enter a new folder name." triggers="click" autoClose="outside" container="body">Download Folder</span>
|
||||||
@if (customDirs$ | async; as customDirs) {
|
<input type="text"
|
||||||
<ng-select [items]="customDirs"
|
class="form-control"
|
||||||
placeholder="Default"
|
placeholder="Default"
|
||||||
[addTag]="allowCustomDir.bind(this)"
|
name="folder"
|
||||||
addTagText="Create directory"
|
|
||||||
bindLabel="folder"
|
|
||||||
[(ngModel)]="folder"
|
[(ngModel)]="folder"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[ngbTypeahead]="searchFolder"
|
||||||
[virtualScroll]="true"
|
[editable]="!!downloads.configuration['CREATE_CUSTOM_DIRS']"
|
||||||
[clearable]="true"
|
(focus)="folderFocus$.next($any($event.target).value)"
|
||||||
[loading]="downloads.loading"
|
(click)="folderClick$.next($any($event.target).value)"
|
||||||
[searchable]="true"
|
#folderTypeahead="ngbTypeahead"
|
||||||
[closeOnSelect]="true"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Choose where to save downloads. Type to create a new folder." />
|
|
||||||
}
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Custom Name Prefix</span>
|
<span class="input-group-text help-title" ngbPopover="Add a prefix to downloaded filenames." triggers="click" autoClose="outside" container="body">Custom Name Prefix</span>
|
||||||
<input type="text"
|
<input type="text"
|
||||||
class="form-control"
|
class="form-control"
|
||||||
placeholder="Default"
|
placeholder="Default"
|
||||||
name="customNamePrefix"
|
name="customNamePrefix"
|
||||||
[(ngModel)]="customNamePrefix"
|
[(ngModel)]="customNamePrefix"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Add a prefix to downloaded filenames">
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-12">
|
<div class="col-12">
|
||||||
@@ -411,18 +403,16 @@
|
|||||||
<div class="form-check form-switch">
|
<div class="form-check form-switch">
|
||||||
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-split-chapters"
|
||||||
name="splitByChapters" [(ngModel)]="splitByChapters" (change)="splitByChaptersChanged()"
|
name="splitByChapters" [(ngModel)]="splitByChapters" (change)="splitByChaptersChanged()"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Split video into separate files by chapters">
|
|
||||||
<label class="form-check-label" for="checkbox-split-chapters">Split by chapters</label>
|
<label class="form-check-label" for="checkbox-split-chapters">Split by chapters</label>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@if (splitByChapters) {
|
@if (splitByChapters) {
|
||||||
<div class="col">
|
<div class="col">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Template</span>
|
<span class="input-group-text help-title" ngbPopover="Output template for chapter files." triggers="click" autoClose="outside" container="body">Template</span>
|
||||||
<input type="text" class="form-control" name="chapterTemplate" [(ngModel)]="chapterTemplate"
|
<input type="text" class="form-control" name="chapterTemplate" [(ngModel)]="chapterTemplate"
|
||||||
(change)="chapterTemplateChanged()" [disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
(change)="chapterTemplateChanged()" [disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Output template for chapter files">
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
@@ -431,28 +421,26 @@
|
|||||||
@if (downloadType === 'video' || downloadType === 'audio') {
|
@if (downloadType === 'video' || downloadType === 'audio') {
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Clip start</span>
|
<span class="input-group-text help-title" ngbPopover="Optional start time (seconds, M:SS, or H:MM:SS). Blank = from start or YouTube &t= in URL." triggers="click" autoClose="outside" container="body">Clip start</span>
|
||||||
<input type="text"
|
<input type="text"
|
||||||
class="form-control"
|
class="form-control"
|
||||||
name="clipStart"
|
name="clipStart"
|
||||||
[(ngModel)]="clipStart"
|
[(ngModel)]="clipStart"
|
||||||
(change)="clipStartChanged()"
|
(change)="clipStartChanged()"
|
||||||
placeholder="e.g. 2:26"
|
placeholder="e.g. 2:26"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Optional start time (seconds, M:SS, or H:MM:SS). Blank = from start or YouTube &t= in URL.">
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Clip end</span>
|
<span class="input-group-text help-title" ngbPopover="Optional end time. Blank = until end of media." triggers="click" autoClose="outside" container="body">Clip end</span>
|
||||||
<input type="text"
|
<input type="text"
|
||||||
class="form-control"
|
class="form-control"
|
||||||
name="clipEnd"
|
name="clipEnd"
|
||||||
[(ngModel)]="clipEnd"
|
[(ngModel)]="clipEnd"
|
||||||
(change)="clipEndChanged()"
|
(change)="clipEndChanged()"
|
||||||
placeholder="e.g. 3:24"
|
placeholder="e.g. 3:24"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Optional end time. Blank = until end of media.">
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
@@ -463,13 +451,12 @@
|
|||||||
<div class="row g-3">
|
<div class="row g-3">
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Auto Start</span>
|
<span class="input-group-text help-title" ngbPopover="Automatically start downloads when added." triggers="click" autoClose="outside" container="body">Auto Start</span>
|
||||||
<select class="form-select"
|
<select class="form-select"
|
||||||
name="autoStart"
|
name="autoStart"
|
||||||
[(ngModel)]="autoStart"
|
[(ngModel)]="autoStart"
|
||||||
(change)="autoStartChanged()"
|
(change)="autoStartChanged()"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Automatically start downloads when added">
|
|
||||||
<option [ngValue]="true">Yes</option>
|
<option [ngValue]="true">Yes</option>
|
||||||
<option [ngValue]="false">No</option>
|
<option [ngValue]="false">No</option>
|
||||||
</select>
|
</select>
|
||||||
@@ -477,7 +464,7 @@
|
|||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Items Limit</span>
|
<span class="input-group-text help-title" ngbPopover="Maximum number of items to download from a playlist or channel (0 = no limit)." triggers="click" autoClose="outside" container="body">Items Limit</span>
|
||||||
<input type="number"
|
<input type="number"
|
||||||
min="0"
|
min="0"
|
||||||
class="form-control"
|
class="form-control"
|
||||||
@@ -485,13 +472,12 @@
|
|||||||
name="playlistItemLimit"
|
name="playlistItemLimit"
|
||||||
(keydown)="isNumber($event)"
|
(keydown)="isNumber($event)"
|
||||||
[(ngModel)]="playlistItemLimit"
|
[(ngModel)]="playlistItemLimit"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Maximum number of items to download from a playlist or channel (0 = no limit)">
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Subscription Check (min)</span>
|
<span class="input-group-text help-title" ngbPopover="How often to poll subscriptions for new videos." triggers="click" autoClose="outside" container="body">Subscription Check (min)</span>
|
||||||
<input type="number"
|
<input type="number"
|
||||||
min="1"
|
min="1"
|
||||||
class="form-control"
|
class="form-control"
|
||||||
@@ -499,29 +485,33 @@
|
|||||||
(keydown)="isNumber($event)"
|
(keydown)="isNumber($event)"
|
||||||
[(ngModel)]="checkIntervalMinutes"
|
[(ngModel)]="checkIntervalMinutes"
|
||||||
(ngModelChange)="checkIntervalChanged()"
|
(ngModelChange)="checkIntervalChanged()"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="How often to poll subscriptions for new videos">
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Subscription Title Filter</span>
|
<span class="input-group-text help-title" ngbPopover="In subscriptions, only titles matching this Python-style regex are queued. Empty = all. Case-sensitive; use (?i) in the pattern for case-insensitive." triggers="click" autoClose="outside" container="body">Subscription Title Filter</span>
|
||||||
<input type="text"
|
<input type="text"
|
||||||
class="form-control"
|
class="form-control"
|
||||||
name="titleRegex"
|
name="titleRegex"
|
||||||
[(ngModel)]="titleRegex"
|
[(ngModel)]="titleRegex"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
||||||
placeholder="Optional regex"
|
placeholder="Optional regex">
|
||||||
ngbTooltip="In subscriptions, only titles matching this Python-style regex are queued. Empty = all. Case-sensitive; use (?i) in the pattern for case-insensitive.">
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="col-12">
|
<div class="col-12">
|
||||||
<div class="form-check form-switch">
|
<div class="form-check form-switch">
|
||||||
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-skip-subscriber-only"
|
<input class="form-check-input" type="checkbox" role="switch" id="checkbox-skip-subscriber-only"
|
||||||
name="skipSubscriberOnly" [(ngModel)]="skipSubscriberOnly"
|
name="skipSubscriberOnly" [(ngModel)]="skipSubscriberOnly"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading" />
|
||||||
ngbTooltip="When enabled, subscription checks skip videos marked members-only by yt-dlp (channel Join). Ignored for one-off downloads." />
|
<label class="form-check-label" for="checkbox-skip-subscriber-only">
|
||||||
<label class="form-check-label" for="checkbox-skip-subscriber-only">Skip members-only subscription videos</label>
|
<span class="help-title" tabindex="0" role="button"
|
||||||
|
ngbPopover="When enabled, subscription checks skip videos marked members-only by yt-dlp (channel Join). Ignored for one-off downloads."
|
||||||
|
triggers="click" autoClose="outside" container="body"
|
||||||
|
(click)="$event.preventDefault(); $event.stopPropagation()"
|
||||||
|
(keydown.enter)="$event.preventDefault(); $event.stopPropagation()"
|
||||||
|
(keydown.space)="$event.preventDefault(); $event.stopPropagation()">Skip members-only subscription videos</span>
|
||||||
|
</label>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -531,7 +521,7 @@
|
|||||||
<div class="row g-3">
|
<div class="row g-3">
|
||||||
<div class="col-12" [class.col-md-6]="allowYtdlOptionsOverrides()">
|
<div class="col-12" [class.col-md-6]="allowYtdlOptionsOverrides()">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Option Presets</span>
|
<span class="input-group-text help-title" ngbPopover="Choose one or more yt-dlp option presets configured on the server (applied in order)." triggers="click" autoClose="outside" container="body">Option Presets</span>
|
||||||
<ng-select
|
<ng-select
|
||||||
class="flex-grow-1"
|
class="flex-grow-1"
|
||||||
name="ytdlOptionsPresets"
|
name="ytdlOptionsPresets"
|
||||||
@@ -541,22 +531,20 @@
|
|||||||
placeholder="Default"
|
placeholder="Default"
|
||||||
[(ngModel)]="ytdlOptionsPresets"
|
[(ngModel)]="ytdlOptionsPresets"
|
||||||
(ngModelChange)="ytdlOptionsPresetsChanged()"
|
(ngModelChange)="ytdlOptionsPresetsChanged()"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading" />
|
||||||
ngbTooltip="Choose one or more yt-dlp option presets configured on the server (applied in order)" />
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@if (allowYtdlOptionsOverrides()) {
|
@if (allowYtdlOptionsOverrides()) {
|
||||||
<div class="col-md-6">
|
<div class="col-md-6">
|
||||||
<div class="input-group">
|
<div class="input-group">
|
||||||
<span class="input-group-text">Custom yt-dlp Options</span>
|
<span class="input-group-text help-title" ngbPopover="Optional per-download yt-dlp overrides as a JSON object." triggers="click" autoClose="outside" container="body">Custom yt-dlp Options</span>
|
||||||
<input type="text"
|
<input type="text"
|
||||||
class="form-control"
|
class="form-control"
|
||||||
placeholder='e.g. {"writesubtitles": true}'
|
placeholder='e.g. {"writesubtitles": true}'
|
||||||
name="ytdlOptionsOverrides"
|
name="ytdlOptionsOverrides"
|
||||||
[(ngModel)]="ytdlOptionsOverrides"
|
[(ngModel)]="ytdlOptionsOverrides"
|
||||||
(change)="ytdlOptionsOverridesChanged()"
|
(change)="ytdlOptionsOverridesChanged()"
|
||||||
[disabled]="addInProgress || subscribeInProgress || downloads.loading"
|
[disabled]="addInProgress || subscribeInProgress || downloads.loading">
|
||||||
ngbTooltip="Optional per-download yt-dlp overrides as a JSON object">
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
@@ -566,7 +554,7 @@
|
|||||||
<div class="settings-section-label">Tools</div>
|
<div class="settings-section-label">Tools</div>
|
||||||
<div class="row g-3">
|
<div class="row g-3">
|
||||||
<div class="col-md-4">
|
<div class="col-md-4">
|
||||||
<div class="action-group-label">Cookies</div>
|
<div class="action-group-label help-title" ngbPopover="Upload a cookies.txt file from your browser to authenticate restricted or private downloads." triggers="click" autoClose="outside" container="body">Cookies</div>
|
||||||
<input type="file" id="cookie-upload" class="d-none" accept=".txt"
|
<input type="file" id="cookie-upload" class="d-none" accept=".txt"
|
||||||
(change)="onCookieFileSelect($event)"
|
(change)="onCookieFileSelect($event)"
|
||||||
[disabled]="cookieUploadInProgress || addInProgress">
|
[disabled]="cookieUploadInProgress || addInProgress">
|
||||||
@@ -574,8 +562,7 @@
|
|||||||
<label class="btn mb-0"
|
<label class="btn mb-0"
|
||||||
[class]="hasCookies ? 'btn cookie-active-btn mb-0' : 'btn cookie-btn mb-0'"
|
[class]="hasCookies ? 'btn cookie-active-btn mb-0' : 'btn cookie-btn mb-0'"
|
||||||
[class.disabled]="cookieUploadInProgress || addInProgress"
|
[class.disabled]="cookieUploadInProgress || addInProgress"
|
||||||
for="cookie-upload"
|
for="cookie-upload">
|
||||||
ngbTooltip="Upload a cookies.txt file for authenticated downloads">
|
|
||||||
@if (cookieUploadInProgress) {
|
@if (cookieUploadInProgress) {
|
||||||
<span class="spinner-border spinner-border-sm me-2" role="status"></span>
|
<span class="spinner-border spinner-border-sm me-2" role="status"></span>
|
||||||
} @else {
|
} @else {
|
||||||
@@ -712,23 +699,38 @@
|
|||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
@for (download of downloads.queue | keyvalue: asIsOrder; track download.value.id) {
|
@for (download of downloads.queue | keyvalue: asIsOrder; track download.key) {
|
||||||
<tr [class.disabled]='download.value.deleting'>
|
<tr [class.disabled]='download.value.deleting'>
|
||||||
<td>
|
<td>
|
||||||
<app-item-checkbox [id]="download.key" [master]="queueMasterCheckboxRef" [checkable]="download.value" />
|
<app-item-checkbox [id]="download.key" [master]="queueMasterCheckboxRef" [checkable]="download.value" />
|
||||||
</td>
|
</td>
|
||||||
<td title="{{ download.value.filename }}">
|
<td title="{{ download.value.filename }}">
|
||||||
<div class="d-flex flex-column flex-sm-row align-items-center row-gap-2 column-gap-3">
|
<div class="d-flex flex-column flex-sm-row align-items-center row-gap-2 column-gap-3">
|
||||||
<div>{{ download.value.title }} </div>
|
<div class="d-flex align-items-center flex-wrap gap-2">
|
||||||
|
<span>{{ download.value.title }}</span>
|
||||||
|
@if (download.value.live_status === 'is_live' && download.value.status !== 'scheduled') {
|
||||||
|
<span class="badge bg-danger">LIVE</span>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
@if (download.value.status === 'scheduled') {
|
||||||
|
<span class="badge bg-warning text-dark">
|
||||||
|
<fa-icon [icon]="faClock" />
|
||||||
|
Waiting for stream
|
||||||
|
@if (liveCountdownSeconds(download.value); as secs) {
|
||||||
|
- starts in {{ secs | eta }}
|
||||||
|
}
|
||||||
|
</span>
|
||||||
|
} @else {
|
||||||
<ngb-progressbar height="1.5rem" [showValue]="download.value.status !== 'preparing'" [striped]="download.value.status === 'preparing'" [animated]="download.value.status === 'preparing'" type="success"
|
<ngb-progressbar height="1.5rem" [showValue]="download.value.status !== 'preparing'" [striped]="download.value.status === 'preparing'" [animated]="download.value.status === 'preparing'" type="success"
|
||||||
[value]="download.value.status === 'preparing' ? 100 : download.value.percent" class="download-progressbar" />
|
[value]="download.value.status === 'preparing' ? 100 : download.value.percent" class="download-progressbar" />
|
||||||
|
}
|
||||||
</div>
|
</div>
|
||||||
</td>
|
</td>
|
||||||
<td>{{ download.value.speed | speed }}</td>
|
<td>{{ download.value.speed | speed }}</td>
|
||||||
<td>{{ download.value.eta | eta }}</td>
|
<td>{{ download.value.eta | eta }}</td>
|
||||||
<td>
|
<td>
|
||||||
<div class="d-flex">
|
<div class="d-flex">
|
||||||
@if (download.value.status === 'pending') {
|
@if (download.value.status === 'pending' || download.value.status === 'scheduled') {
|
||||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Start download for ' + download.value.title" (click)="downloadItemByKey(download.key)"><fa-icon [icon]="faDownload" /></button>
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Start download for ' + download.value.title" (click)="downloadItemByKey(download.key)"><fa-icon [icon]="faDownload" /></button>
|
||||||
}
|
}
|
||||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Remove ' + download.value.title + ' from queue'" (click)="delDownload('queue', download.key)"><fa-icon [icon]="faTrashAlt" /></button>
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Remove ' + download.value.title + ' from queue'" (click)="delDownload('queue', download.key)"><fa-icon [icon]="faTrashAlt" /></button>
|
||||||
@@ -767,7 +769,7 @@
|
|||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
@for (entry of cachedSortedDone; track entry[1].id) {
|
@for (entry of cachedSortedDone; track entry[0]) {
|
||||||
<tr [class.disabled]='entry[1].deleting'>
|
<tr [class.disabled]='entry[1].deleting'>
|
||||||
<td>
|
<td>
|
||||||
<app-item-checkbox [id]="entry[0]" [master]="doneMasterCheckboxRef" [checkable]="entry[1]" />
|
<app-item-checkbox [id]="entry[0]" [master]="doneMasterCheckboxRef" [checkable]="entry[1]" />
|
||||||
@@ -854,6 +856,9 @@
|
|||||||
@if (entry[1].filename) {
|
@if (entry[1].filename) {
|
||||||
<a href="{{buildDownloadLink(entry[1])}}" download class="btn btn-link" [attr.aria-label]="'Download result file for ' + entry[1].title"><fa-icon [icon]="faDownload" /></a>
|
<a href="{{buildDownloadLink(entry[1])}}" download class="btn btn-link" [attr.aria-label]="'Download result file for ' + entry[1].title"><fa-icon [icon]="faDownload" /></a>
|
||||||
}
|
}
|
||||||
|
@if (entry[1].filename && canShareDownloads()) {
|
||||||
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Share result file for ' + entry[1].title" (click)="shareDownload(entry[1])"><fa-icon [icon]="faShareNodes" /></button>
|
||||||
|
}
|
||||||
<a href="{{entry[1].url}}" target="_blank" class="btn btn-link" [attr.aria-label]="'Open source URL for ' + entry[1].title"><fa-icon [icon]="faExternalLinkAlt" /></a>
|
<a href="{{entry[1].url}}" target="_blank" class="btn btn-link" [attr.aria-label]="'Open source URL for ' + entry[1].title"><fa-icon [icon]="faExternalLinkAlt" /></a>
|
||||||
<button type="button" class="btn btn-link" [attr.aria-label]="'Delete completed item ' + entry[1].title" (click)="delDownload('done', entry[0])"><fa-icon [icon]="faTrashAlt" /></button>
|
<button type="button" class="btn btn-link" [attr.aria-label]="'Delete completed item ' + entry[1].title" (click)="delDownload('done', entry[0])"><fa-icon [icon]="faTrashAlt" /></button>
|
||||||
</div>
|
</div>
|
||||||
@@ -936,8 +941,7 @@
|
|||||||
</th>
|
</th>
|
||||||
<th scope="col">Name</th>
|
<th scope="col">Name</th>
|
||||||
<th scope="col">URL</th>
|
<th scope="col">URL</th>
|
||||||
<th scope="col" class="text-nowrap"
|
<th scope="col" class="text-nowrap"><span class="help-title" ngbPopover="Subscriptions only — which new video titles to queue when this feed is checked. Does not affect manual downloads." triggers="click" autoClose="outside" container="body">Filter</span></th>
|
||||||
ngbTooltip="Subscriptions only — which new video titles to queue when this feed is checked. Does not affect manual downloads.">Sub. title filter</th>
|
|
||||||
<th scope="col" class="text-nowrap">Interval (min)</th>
|
<th scope="col" class="text-nowrap">Interval (min)</th>
|
||||||
<th scope="col" class="text-nowrap">Last checked</th>
|
<th scope="col" class="text-nowrap">Last checked</th>
|
||||||
<th scope="col">Status</th>
|
<th scope="col">Status</th>
|
||||||
@@ -1074,3 +1078,5 @@
|
|||||||
}
|
}
|
||||||
</div>
|
</div>
|
||||||
</footer>
|
</footer>
|
||||||
|
|
||||||
|
<app-toast-container />
|
||||||
|
|||||||
@@ -201,6 +201,12 @@ main
|
|||||||
color: var(--bs-secondary-color)
|
color: var(--bs-secondary-color)
|
||||||
margin-bottom: 0.4rem
|
margin-bottom: 0.4rem
|
||||||
|
|
||||||
|
.help-title
|
||||||
|
cursor: help
|
||||||
|
|
||||||
|
&:focus
|
||||||
|
outline: none
|
||||||
|
|
||||||
.cookie-status
|
.cookie-status
|
||||||
font-size: 0.8rem
|
font-size: 0.8rem
|
||||||
margin-top: 0.35rem
|
margin-top: 0.35rem
|
||||||
|
|||||||
+42
-3
@@ -4,6 +4,7 @@ import { Subject, of } from 'rxjs';
|
|||||||
import { App } from './app';
|
import { App } from './app';
|
||||||
import { DownloadsService } from './services/downloads.service';
|
import { DownloadsService } from './services/downloads.service';
|
||||||
import { SubscriptionsService } from './services/subscriptions.service';
|
import { SubscriptionsService } from './services/subscriptions.service';
|
||||||
|
import { ToastService } from './services/toast.service';
|
||||||
import { CookieService } from 'ngx-cookie-service';
|
import { CookieService } from 'ngx-cookie-service';
|
||||||
|
|
||||||
class DownloadsServiceStub {
|
class DownloadsServiceStub {
|
||||||
@@ -138,6 +139,12 @@ describe('App', () => {
|
|||||||
expect(app).toBeTruthy();
|
expect(app).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('asIsOrder returns a stable comparator value (insertion order preserved)', () => {
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
const app = fixture.componentInstance;
|
||||||
|
expect(app.asIsOrder()).toBe(0);
|
||||||
|
});
|
||||||
|
|
||||||
it('hides manual override input when disabled', () => {
|
it('hides manual override input when disabled', () => {
|
||||||
const fixture = TestBed.createComponent(App);
|
const fixture = TestBed.createComponent(App);
|
||||||
fixture.componentInstance.isAdvancedOpen = true;
|
fixture.componentInstance.isAdvancedOpen = true;
|
||||||
@@ -182,6 +189,37 @@ describe('App', () => {
|
|||||||
expect(payload.ytdlOptionsOverrides).toBe('');
|
expect(payload.ytdlOptionsOverrides).toBe('');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('shows waiting badge for scheduled live stream', () => {
|
||||||
|
downloads.queue.set('https://example.com/live', {
|
||||||
|
id: 'live1',
|
||||||
|
title: 'Upcoming Stream',
|
||||||
|
url: 'https://example.com/live',
|
||||||
|
download_type: 'video',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
custom_name_prefix: '',
|
||||||
|
playlist_item_limit: 0,
|
||||||
|
status: 'scheduled',
|
||||||
|
live_status: 'is_upcoming',
|
||||||
|
live_release_timestamp: Date.now() / 1000 + 3600,
|
||||||
|
msg: '',
|
||||||
|
percent: 0,
|
||||||
|
speed: 0,
|
||||||
|
eta: 0,
|
||||||
|
filename: '',
|
||||||
|
checked: false,
|
||||||
|
});
|
||||||
|
downloads.queueChanged.next();
|
||||||
|
|
||||||
|
const fixture = TestBed.createComponent(App);
|
||||||
|
fixture.detectChanges();
|
||||||
|
|
||||||
|
const root = fixture.nativeElement as HTMLElement;
|
||||||
|
expect(root.textContent).toContain('Waiting for stream');
|
||||||
|
expect(root.textContent).toContain('starts in');
|
||||||
|
});
|
||||||
|
|
||||||
it('includes titleRegex in subscribe payload', () => {
|
it('includes titleRegex in subscribe payload', () => {
|
||||||
const fixture = TestBed.createComponent(App);
|
const fixture = TestBed.createComponent(App);
|
||||||
const app = fixture.componentInstance;
|
const app = fixture.componentInstance;
|
||||||
@@ -232,7 +270,8 @@ describe('App', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
it('blocks subscribe with invalid title regex', () => {
|
it('blocks subscribe with invalid title regex', () => {
|
||||||
const alertSpy = vi.spyOn(window, 'alert').mockImplementation(() => undefined);
|
const toasts = TestBed.inject(ToastService);
|
||||||
|
const errorSpy = vi.spyOn(toasts, 'error').mockImplementation(() => undefined);
|
||||||
const fixture = TestBed.createComponent(App);
|
const fixture = TestBed.createComponent(App);
|
||||||
const app = fixture.componentInstance;
|
const app = fixture.componentInstance;
|
||||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||||
@@ -240,7 +279,7 @@ describe('App', () => {
|
|||||||
app.titleRegex = '[';
|
app.titleRegex = '[';
|
||||||
app.addSubscription();
|
app.addSubscription();
|
||||||
expect(subs.subscribeCalls.length).toBe(0);
|
expect(subs.subscribeCalls.length).toBe(0);
|
||||||
expect(alertSpy).toHaveBeenCalledWith('Invalid subscription title filter (regex)');
|
expect(errorSpy).toHaveBeenCalledWith('Invalid subscription title filter (regex)');
|
||||||
alertSpy.mockRestore();
|
errorSpy.mockRestore();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
+221
-128
@@ -1,17 +1,20 @@
|
|||||||
import { AsyncPipe, DatePipe, KeyValuePipe, NgTemplateOutlet } from '@angular/common';
|
import { DatePipe, KeyValuePipe, NgTemplateOutlet } from '@angular/common';
|
||||||
import { HttpClient } from '@angular/common/http';
|
import { HttpClient } from '@angular/common/http';
|
||||||
import { AfterViewInit, ChangeDetectionStrategy, ChangeDetectorRef, Component, DestroyRef, ElementRef, viewChild, inject, OnDestroy, OnInit } from '@angular/core';
|
import { AfterViewInit, ChangeDetectionStrategy, ChangeDetectorRef, Component, DestroyRef, ElementRef, viewChild, inject, OnDestroy, OnInit } from '@angular/core';
|
||||||
import { Observable, Subject, Subscription, from, map, distinctUntilChanged, finalize, mergeMap, takeUntil, tap } from 'rxjs';
|
import { Observable, OperatorFunction, Subject, Subscription, from, map, merge, debounceTime, distinctUntilChanged, filter, finalize, mergeMap, takeUntil, tap } from 'rxjs';
|
||||||
import { FormsModule } from '@angular/forms';
|
import { FormsModule } from '@angular/forms';
|
||||||
import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
|
import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
|
||||||
import { FontAwesomeModule } from '@fortawesome/angular-fontawesome';
|
import { FontAwesomeModule } from '@fortawesome/angular-fontawesome';
|
||||||
import { NgbModule } from '@ng-bootstrap/ng-bootstrap';
|
import { NgbModule, NgbTypeahead } from '@ng-bootstrap/ng-bootstrap';
|
||||||
import { NgSelectModule } from '@ng-select/ng-select';
|
import { NgSelectModule } from '@ng-select/ng-select';
|
||||||
import { faTrashAlt, faCheckCircle, faTimesCircle, faRedoAlt, faSun, faMoon, faCheck, faCircleHalfStroke, faDownload, faExternalLinkAlt, faFileImport, faFileExport, faCopy, faClock, faTachometerAlt, faSortAmountDown, faSortAmountUp, faChevronRight, faChevronDown, faUpload, faPause, faPlay } from '@fortawesome/free-solid-svg-icons';
|
import { faTrashAlt, faCheckCircle, faTimesCircle, faRedoAlt, faSun, faMoon, faCheck, faCircleHalfStroke, faDownload, faExternalLinkAlt, faFileImport, faFileExport, faCopy, faClock, faTachometerAlt, faSortAmountDown, faSortAmountUp, faChevronRight, faChevronDown, faUpload, faPause, faPlay, faShareNodes } from '@fortawesome/free-solid-svg-icons';
|
||||||
import { faGithub } from '@fortawesome/free-brands-svg-icons';
|
import { faGithub } from '@fortawesome/free-brands-svg-icons';
|
||||||
import { CookieService } from 'ngx-cookie-service';
|
import { CookieService } from 'ngx-cookie-service';
|
||||||
import { AddDownloadPayload, DownloadsService } from './services/downloads.service';
|
import { AddDownloadPayload, DownloadsService } from './services/downloads.service';
|
||||||
|
import { MeTubeSocket } from './services/metube-socket.service';
|
||||||
import { SubscriptionsService } from './services/subscriptions.service';
|
import { SubscriptionsService } from './services/subscriptions.service';
|
||||||
|
import { ToastService } from './services/toast.service';
|
||||||
|
import { BatchUrlsService, BatchUrlFilter } from './services/batch-urls.service';
|
||||||
import { SubscriptionRow } from './interfaces/subscription';
|
import { SubscriptionRow } from './interfaces/subscription';
|
||||||
import { Themes } from './theme';
|
import { Themes } from './theme';
|
||||||
import {
|
import {
|
||||||
@@ -31,7 +34,7 @@ import {
|
|||||||
State,
|
State,
|
||||||
} from './interfaces';
|
} from './interfaces';
|
||||||
import { EtaPipe, SpeedPipe, FileSizePipe } from './pipes';
|
import { EtaPipe, SpeedPipe, FileSizePipe } from './pipes';
|
||||||
import { SelectAllCheckboxComponent, ItemCheckboxComponent } from './components/';
|
import { SelectAllCheckboxComponent, ItemCheckboxComponent, ToastContainerComponent } from './components/';
|
||||||
|
|
||||||
@Component({
|
@Component({
|
||||||
selector: 'app-root',
|
selector: 'app-root',
|
||||||
@@ -40,7 +43,6 @@ import { SelectAllCheckboxComponent, ItemCheckboxComponent } from './components/
|
|||||||
FormsModule,
|
FormsModule,
|
||||||
NgTemplateOutlet,
|
NgTemplateOutlet,
|
||||||
KeyValuePipe,
|
KeyValuePipe,
|
||||||
AsyncPipe,
|
|
||||||
DatePipe,
|
DatePipe,
|
||||||
FontAwesomeModule,
|
FontAwesomeModule,
|
||||||
NgbModule,
|
NgbModule,
|
||||||
@@ -50,6 +52,7 @@ import { SelectAllCheckboxComponent, ItemCheckboxComponent } from './components/
|
|||||||
FileSizePipe,
|
FileSizePipe,
|
||||||
SelectAllCheckboxComponent,
|
SelectAllCheckboxComponent,
|
||||||
ItemCheckboxComponent,
|
ItemCheckboxComponent,
|
||||||
|
ToastContainerComponent,
|
||||||
],
|
],
|
||||||
templateUrl: './app.html',
|
templateUrl: './app.html',
|
||||||
styleUrl: './app.sass',
|
styleUrl: './app.sass',
|
||||||
@@ -57,6 +60,9 @@ import { SelectAllCheckboxComponent, ItemCheckboxComponent } from './components/
|
|||||||
export class App implements AfterViewInit, OnInit, OnDestroy {
|
export class App implements AfterViewInit, OnInit, OnDestroy {
|
||||||
downloads = inject(DownloadsService);
|
downloads = inject(DownloadsService);
|
||||||
subscriptionsSvc = inject(SubscriptionsService);
|
subscriptionsSvc = inject(SubscriptionsService);
|
||||||
|
private toasts = inject(ToastService);
|
||||||
|
private batchUrls = inject(BatchUrlsService);
|
||||||
|
private socket = inject(MeTubeSocket);
|
||||||
private cookieService = inject(CookieService);
|
private cookieService = inject(CookieService);
|
||||||
private http = inject(HttpClient);
|
private http = inject(HttpClient);
|
||||||
private cdr = inject(ChangeDetectorRef);
|
private cdr = inject(ChangeDetectorRef);
|
||||||
@@ -105,7 +111,9 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
cookieUploadInProgress = false;
|
cookieUploadInProgress = false;
|
||||||
themes: Theme[] = Themes;
|
themes: Theme[] = Themes;
|
||||||
activeTheme: Theme | undefined;
|
activeTheme: Theme | undefined;
|
||||||
customDirs$!: Observable<string[]>;
|
readonly folderTypeahead = viewChild<NgbTypeahead>('folderTypeahead');
|
||||||
|
folderFocus$ = new Subject<string>();
|
||||||
|
folderClick$ = new Subject<string>();
|
||||||
showBatchPanel = false;
|
showBatchPanel = false;
|
||||||
batchImportModalOpen = false;
|
batchImportModalOpen = false;
|
||||||
batchImportText = '';
|
batchImportText = '';
|
||||||
@@ -129,6 +137,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
lastCopiedErrorId: string | null = null;
|
lastCopiedErrorId: string | null = null;
|
||||||
private previousDownloadType = 'video';
|
private previousDownloadType = 'video';
|
||||||
private addRequestSub?: Subscription;
|
private addRequestSub?: Subscription;
|
||||||
|
private liveCountdownTimer?: ReturnType<typeof setInterval>;
|
||||||
private selectionsByType: Record<string, {
|
private selectionsByType: Record<string, {
|
||||||
codec: string;
|
codec: string;
|
||||||
format: string;
|
format: string;
|
||||||
@@ -185,6 +194,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
faUpload = faUpload;
|
faUpload = faUpload;
|
||||||
faPause = faPause;
|
faPause = faPause;
|
||||||
faPlay = faPlay;
|
faPlay = faPlay;
|
||||||
|
faShareNodes = faShareNodes;
|
||||||
subtitleLanguages = [
|
subtitleLanguages = [
|
||||||
{ id: 'en', text: 'English' },
|
{ id: 'en', text: 'English' },
|
||||||
{ id: 'ar', text: 'Arabic' },
|
{ id: 'ar', text: 'Arabic' },
|
||||||
@@ -281,6 +291,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
// Subscribe to download updates
|
// Subscribe to download updates
|
||||||
this.downloads.queueChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
this.downloads.queueChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
||||||
this.updateMetrics();
|
this.updateMetrics();
|
||||||
|
this.syncLiveCountdownTimer();
|
||||||
this.cdr.markForCheck();
|
this.cdr.markForCheck();
|
||||||
});
|
});
|
||||||
this.downloads.doneChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
this.downloads.doneChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
||||||
@@ -291,6 +302,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
// Subscribe to real-time updates
|
// Subscribe to real-time updates
|
||||||
this.downloads.updated.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
this.downloads.updated.pipe(takeUntilDestroyed(this.destroyRef)).subscribe(() => {
|
||||||
this.updateMetrics();
|
this.updateMetrics();
|
||||||
|
this.syncLiveCountdownTimer();
|
||||||
this.cdr.markForCheck();
|
this.cdr.markForCheck();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -308,7 +320,6 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.getConfiguration();
|
this.getConfiguration();
|
||||||
this.getYtdlOptionsUpdateTime();
|
this.getYtdlOptionsUpdateTime();
|
||||||
this.getYtdlOptionPresets();
|
this.getYtdlOptionPresets();
|
||||||
this.customDirs$ = this.getMatchingCustomDir();
|
|
||||||
this.setTheme(this.activeTheme!);
|
this.setTheme(this.activeTheme!);
|
||||||
|
|
||||||
this.colorSchemeMediaQuery.addEventListener('change', this.onColorSchemeChanged);
|
this.colorSchemeMediaQuery.addEventListener('change', this.onColorSchemeChanged);
|
||||||
@@ -327,20 +338,23 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
// Initialize action button states for already-loaded entries.
|
// Initialize action button states for already-loaded entries.
|
||||||
this.updateDoneActionButtons();
|
this.updateDoneActionButtons();
|
||||||
this.fetchVersionInfo();
|
this.fetchVersionInfo();
|
||||||
|
this.socket.fromEvent('connect')
|
||||||
|
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||||
|
.subscribe(() => this.fetchVersionInfo());
|
||||||
}
|
}
|
||||||
|
|
||||||
ngOnDestroy() {
|
ngOnDestroy() {
|
||||||
this.addRequestSub?.unsubscribe();
|
this.addRequestSub?.unsubscribe();
|
||||||
|
if (this.liveCountdownTimer) {
|
||||||
|
clearInterval(this.liveCountdownTimer);
|
||||||
|
}
|
||||||
this.colorSchemeMediaQuery.removeEventListener('change', this.onColorSchemeChanged);
|
this.colorSchemeMediaQuery.removeEventListener('change', this.onColorSchemeChanged);
|
||||||
}
|
}
|
||||||
|
|
||||||
// workaround to allow fetching of Map values in the order they were inserted
|
// keyvalue comparator that preserves insertion order (Angular's keyvalue
|
||||||
// https://github.com/angular/angular/issues/31420
|
// pipe sorts by key by default): https://github.com/angular/angular/issues/31420
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
asIsOrder() {
|
asIsOrder() {
|
||||||
return 1;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
qualityChanged() {
|
qualityChanged() {
|
||||||
@@ -375,34 +389,26 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
return this.downloads.configuration['ALLOW_YTDL_OPTIONS_OVERRIDES'] === true;
|
return this.downloads.configuration['ALLOW_YTDL_OPTIONS_OVERRIDES'] === true;
|
||||||
}
|
}
|
||||||
|
|
||||||
allowCustomDir(tag: string) {
|
searchFolder: OperatorFunction<string, readonly string[]> = (text$: Observable<string>) => {
|
||||||
if (this.downloads.configuration['CREATE_CUSTOM_DIRS']) {
|
const debouncedText$ = text$.pipe(debounceTime(150), distinctUntilChanged());
|
||||||
return tag;
|
const clicksWithClosedPopup$ = this.folderClick$.pipe(
|
||||||
}
|
filter(() => !this.folderTypeahead()?.isPopupOpen()),
|
||||||
return false;
|
);
|
||||||
}
|
return merge(debouncedText$, this.folderFocus$, clicksWithClosedPopup$).pipe(
|
||||||
|
map(term => {
|
||||||
|
const dirs = this.isAudioType()
|
||||||
|
? (this.downloads.customDirs?.['audio_download_dir'] ?? [])
|
||||||
|
: (this.downloads.customDirs?.['download_dir'] ?? []);
|
||||||
|
const t = (term ?? '').toLowerCase();
|
||||||
|
return (t === '' ? dirs : dirs.filter(d => d.toLowerCase().includes(t))).slice(0, 10);
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
};
|
||||||
|
|
||||||
isAudioType() {
|
isAudioType() {
|
||||||
return this.downloadType === 'audio';
|
return this.downloadType === 'audio';
|
||||||
}
|
}
|
||||||
|
|
||||||
getMatchingCustomDir() : Observable<string[]> {
|
|
||||||
return this.downloads.customDirsChanged.asObservable().pipe(
|
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
||||||
map((output: any) => {
|
|
||||||
// Keep logic consistent with app/ytdl.py
|
|
||||||
if (this.isAudioType()) {
|
|
||||||
console.debug("Showing audio-specific download directories");
|
|
||||||
return output["audio_download_dir"];
|
|
||||||
} else {
|
|
||||||
console.debug("Showing default download directories");
|
|
||||||
return output["download_dir"];
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
distinctUntilChanged((prev, curr) => JSON.stringify(prev) === JSON.stringify(curr))
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
getYtdlOptionsUpdateTime() {
|
getYtdlOptionsUpdateTime() {
|
||||||
this.downloads.ytdlOptionsChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
|
this.downloads.ytdlOptionsChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
@@ -411,7 +417,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
const date = new Date(data['update_time'] * 1000);
|
const date = new Date(data['update_time'] * 1000);
|
||||||
this.ytDlpOptionsUpdateTime=date.toLocaleString();
|
this.ytDlpOptionsUpdateTime=date.toLocaleString();
|
||||||
}else{
|
}else{
|
||||||
alert("Error reload yt-dlp options: "+data['msg']);
|
this.toasts.error("Error reloading yt-dlp options: " + data['msg']);
|
||||||
}
|
}
|
||||||
this.cdr.markForCheck();
|
this.cdr.markForCheck();
|
||||||
}
|
}
|
||||||
@@ -421,8 +427,8 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.downloads.configurationChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
|
this.downloads.configurationChanged.pipe(takeUntilDestroyed(this.destroyRef)).subscribe({
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
next: (config: any) => {
|
next: (config: any) => {
|
||||||
const playlistItemLimit = config['DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT'];
|
const playlistItemLimit = parseInt(String(config['DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT'] ?? '0'), 10);
|
||||||
if (playlistItemLimit !== '0') {
|
if (!Number.isNaN(playlistItemLimit) && playlistItemLimit > 0) {
|
||||||
this.playlistItemLimit = playlistItemLimit;
|
this.playlistItemLimit = playlistItemLimit;
|
||||||
}
|
}
|
||||||
// Set chapter template from backend config if not already set by cookie
|
// Set chapter template from backend config if not already set by cookie
|
||||||
@@ -486,11 +492,11 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
try {
|
try {
|
||||||
const parsed = JSON.parse(trimmed);
|
const parsed = JSON.parse(trimmed);
|
||||||
if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') {
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') {
|
||||||
alert('Custom yt-dlp options must be a JSON object');
|
this.toasts.error('Custom yt-dlp options must be a JSON object');
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
alert('Custom yt-dlp options must be valid JSON');
|
this.toasts.error('Custom yt-dlp options must be valid JSON');
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
@@ -517,11 +523,19 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
return status?.status === 'error' ? status.msg || null : null;
|
return status?.status === 'error' ? status.msg || null : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private handleActionResult(res: unknown, fallbackMsg: string) {
|
||||||
|
const error = this.getStatusError(res);
|
||||||
|
if (error) {
|
||||||
|
this.toasts.error(error || fallbackMsg);
|
||||||
|
}
|
||||||
|
this.cdr.markForCheck();
|
||||||
|
}
|
||||||
|
|
||||||
private refreshSubscriptionsWithAlert() {
|
private refreshSubscriptionsWithAlert() {
|
||||||
this.subscriptionsSvc.refreshList().pipe(takeUntilDestroyed(this.destroyRef)).subscribe((refreshRes) => {
|
this.subscriptionsSvc.refreshList().pipe(takeUntilDestroyed(this.destroyRef)).subscribe((refreshRes) => {
|
||||||
const error = this.getStatusError(refreshRes);
|
const error = this.getStatusError(refreshRes);
|
||||||
if (error) {
|
if (error) {
|
||||||
alert(error || 'Refresh subscriptions failed');
|
this.toasts.error(error || 'Refresh subscriptions failed');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.cdr.markForCheck();
|
this.cdr.markForCheck();
|
||||||
@@ -565,7 +579,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
}
|
}
|
||||||
const payload = this.buildAddPayload();
|
const payload = this.buildAddPayload();
|
||||||
if (!payload.url?.trim()) {
|
if (!payload.url?.trim()) {
|
||||||
alert('Please enter a URL');
|
this.toasts.error('Please enter a URL');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const tr = (this.titleRegex || '').trim();
|
const tr = (this.titleRegex || '').trim();
|
||||||
@@ -573,12 +587,12 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
try {
|
try {
|
||||||
void RegExp(tr);
|
void RegExp(tr);
|
||||||
} catch {
|
} catch {
|
||||||
alert('Invalid subscription title filter (regex)');
|
this.toasts.error('Invalid subscription title filter (regex)');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (payload.splitByChapters && !payload.chapterTemplate.includes('%(section_number)')) {
|
if (payload.splitByChapters && !payload.chapterTemplate.includes('%(section_number)')) {
|
||||||
alert('Chapter template must include %(section_number)');
|
this.toasts.error('Chapter template must include %(section_number)');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) {
|
if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) {
|
||||||
@@ -607,7 +621,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
next: (res) => {
|
next: (res) => {
|
||||||
const r = res as { status?: string; msg?: string };
|
const r = res as { status?: string; msg?: string };
|
||||||
if (r.status === 'error') {
|
if (r.status === 'error') {
|
||||||
alert(r.msg || 'Subscribe failed');
|
this.toasts.error(r.msg || 'Subscribe failed');
|
||||||
} else {
|
} else {
|
||||||
this.addUrl = '';
|
this.addUrl = '';
|
||||||
this.titleRegex = '';
|
this.titleRegex = '';
|
||||||
@@ -635,14 +649,14 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
try {
|
try {
|
||||||
void RegExp(raw);
|
void RegExp(raw);
|
||||||
} catch {
|
} catch {
|
||||||
alert('Invalid subscription title filter (regex)');
|
this.toasts.error('Invalid subscription title filter (regex)');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.subscriptionsSvc.update(id, { title_regex: raw }).subscribe((res) => {
|
this.subscriptionsSvc.update(id, { title_regex: raw }).subscribe((res) => {
|
||||||
const error = this.getStatusError(res);
|
const error = this.getStatusError(res);
|
||||||
if (error) {
|
if (error) {
|
||||||
alert(error || 'Update subscription failed');
|
this.toasts.error(error || 'Update subscription failed');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.cancelEditTitleRegex();
|
this.cancelEditTitleRegex();
|
||||||
@@ -653,7 +667,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.subscriptionsSvc.delete([id]).subscribe((res) => {
|
this.subscriptionsSvc.delete([id]).subscribe((res) => {
|
||||||
const error = this.getStatusError(res);
|
const error = this.getStatusError(res);
|
||||||
if (error) {
|
if (error) {
|
||||||
alert(error || 'Delete subscription failed');
|
this.toasts.error(error || 'Delete subscription failed');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.selectedSubscriptionIds.delete(id);
|
this.selectedSubscriptionIds.delete(id);
|
||||||
@@ -669,7 +683,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.subscriptionsSvc.delete(ids).subscribe((res) => {
|
this.subscriptionsSvc.delete(ids).subscribe((res) => {
|
||||||
const error = this.getStatusError(res);
|
const error = this.getStatusError(res);
|
||||||
if (error) {
|
if (error) {
|
||||||
alert(error || 'Delete subscriptions failed');
|
this.toasts.error(error || 'Delete subscriptions failed');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.selectedSubscriptionIds.clear();
|
this.selectedSubscriptionIds.clear();
|
||||||
@@ -695,7 +709,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
.subscribe((res) => {
|
.subscribe((res) => {
|
||||||
const error = this.getStatusError(res);
|
const error = this.getStatusError(res);
|
||||||
if (error) {
|
if (error) {
|
||||||
alert(error || 'Subscription check failed');
|
this.toasts.error(error || 'Subscription check failed');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.refreshSubscriptionsWithAlert();
|
this.refreshSubscriptionsWithAlert();
|
||||||
@@ -742,7 +756,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
.subscribe((res) => {
|
.subscribe((res) => {
|
||||||
const error = this.getStatusError(res);
|
const error = this.getStatusError(res);
|
||||||
if (error) {
|
if (error) {
|
||||||
alert(error || 'Subscription check failed');
|
this.toasts.error(error || 'Subscription check failed');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.refreshSubscriptionsWithAlert();
|
this.refreshSubscriptionsWithAlert();
|
||||||
@@ -765,7 +779,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.subscriptionsSvc.update(row.id, { enabled: !row.enabled }).subscribe((res) => {
|
this.subscriptionsSvc.update(row.id, { enabled: !row.enabled }).subscribe((res) => {
|
||||||
const error = this.getStatusError(res);
|
const error = this.getStatusError(res);
|
||||||
if (error) {
|
if (error) {
|
||||||
alert(error || 'Update subscription failed');
|
this.toasts.error(error || 'Update subscription failed');
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1062,21 +1076,24 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
|
|
||||||
// Validate chapter template if chapter splitting is enabled
|
// Validate chapter template if chapter splitting is enabled
|
||||||
if (payload.splitByChapters && !payload.chapterTemplate.includes('%(section_number)')) {
|
if (payload.splitByChapters && !payload.chapterTemplate.includes('%(section_number)')) {
|
||||||
alert('Chapter template must include %(section_number)');
|
this.toasts.error('Chapter template must include %(section_number)');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) {
|
if (!this.validateYtdlOptionsOverrides(payload.ytdlOptionsOverrides)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
console.debug('Downloading:', payload);
|
|
||||||
this.addInProgress = true;
|
this.addInProgress = true;
|
||||||
this.cancelRequested = false;
|
this.cancelRequested = false;
|
||||||
this.addRequestSub?.unsubscribe();
|
this.addRequestSub?.unsubscribe();
|
||||||
this.addRequestSub = this.downloads.add(payload).subscribe((status: Status) => {
|
this.addRequestSub = this.downloads.add(payload).subscribe((status: Status) => {
|
||||||
if (status.status === 'error' && !this.cancelRequested) {
|
if (status.status === 'error' && !this.cancelRequested) {
|
||||||
alert(`Error adding URL: ${status.msg}`);
|
this.toasts.error(`Error adding URL: ${status.msg}`);
|
||||||
} else if (status.status !== 'error') {
|
} else if (status.status !== 'error') {
|
||||||
|
// e.g. "Already in queue: ..." when the backend skipped a duplicate.
|
||||||
|
if (status.msg) {
|
||||||
|
this.toasts.info(status.msg);
|
||||||
|
}
|
||||||
this.addUrl = '';
|
this.addUrl = '';
|
||||||
}
|
}
|
||||||
this.resetAddState();
|
this.resetAddState();
|
||||||
@@ -1105,11 +1122,31 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
}
|
}
|
||||||
|
|
||||||
downloadItemByKey(id: string) {
|
downloadItemByKey(id: string) {
|
||||||
this.downloads.startById([id]).subscribe();
|
this.downloads.startById([id]).subscribe((res) => this.handleActionResult(res, 'Start download failed'));
|
||||||
|
}
|
||||||
|
|
||||||
|
liveCountdownSeconds(download: Download): number | null {
|
||||||
|
const ts = download.live_release_timestamp;
|
||||||
|
if (ts == null || download.status !== 'scheduled') {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return Math.max(0, ts - Date.now() / 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
private syncLiveCountdownTimer() {
|
||||||
|
const hasScheduled = Array.from(this.downloads.queue.values()).some(
|
||||||
|
(download) => download.status === 'scheduled',
|
||||||
|
);
|
||||||
|
if (hasScheduled && !this.liveCountdownTimer) {
|
||||||
|
this.liveCountdownTimer = setInterval(() => this.cdr.markForCheck(), 1000);
|
||||||
|
} else if (!hasScheduled && this.liveCountdownTimer) {
|
||||||
|
clearInterval(this.liveCountdownTimer);
|
||||||
|
this.liveCountdownTimer = undefined;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
retryDownload(key: string, download: Download) {
|
retryDownload(key: string, download: Download) {
|
||||||
this.addDownload({
|
const payload = this.buildAddPayload({
|
||||||
url: download.url,
|
url: download.url,
|
||||||
downloadType: download.download_type,
|
downloadType: download.download_type,
|
||||||
codec: download.codec,
|
codec: download.codec,
|
||||||
@@ -1130,27 +1167,38 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
clipStart: download.clip_start != null ? String(download.clip_start) : '',
|
clipStart: download.clip_start != null ? String(download.clip_start) : '',
|
||||||
clipEnd: download.clip_end != null ? String(download.clip_end) : '',
|
clipEnd: download.clip_end != null ? String(download.clip_end) : '',
|
||||||
});
|
});
|
||||||
|
// Only remove the done-list record once the retry is confirmed queued —
|
||||||
|
// deleting it eagerly would silently lose history if the re-add fails.
|
||||||
|
this.downloads.add(payload)
|
||||||
|
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||||
|
.subscribe((status: Status) => {
|
||||||
|
if (status.status === 'error') {
|
||||||
|
this.toasts.error(`Error retrying ${download.title}: ${status.msg}`);
|
||||||
|
this.cdr.markForCheck();
|
||||||
|
return;
|
||||||
|
}
|
||||||
this.downloads.delById('done', [key]).subscribe();
|
this.downloads.delById('done', [key]).subscribe();
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
delDownload(where: State, id: string) {
|
delDownload(where: State, id: string) {
|
||||||
this.downloads.delById(where, [id]).subscribe();
|
this.downloads.delById(where, [id]).subscribe((res) => this.handleActionResult(res, 'Delete failed'));
|
||||||
}
|
}
|
||||||
|
|
||||||
startSelectedDownloads(where: State){
|
startSelectedDownloads(where: State){
|
||||||
this.downloads.startByFilter(where, dl => !!dl.checked).subscribe();
|
this.downloads.startByFilter(where, dl => !!dl.checked).subscribe((res) => this.handleActionResult(res, 'Start download failed'));
|
||||||
}
|
}
|
||||||
|
|
||||||
delSelectedDownloads(where: State) {
|
delSelectedDownloads(where: State) {
|
||||||
this.downloads.delByFilter(where, dl => !!dl.checked).subscribe();
|
this.downloads.delByFilter(where, dl => !!dl.checked).subscribe((res) => this.handleActionResult(res, 'Delete failed'));
|
||||||
}
|
}
|
||||||
|
|
||||||
clearCompletedDownloads() {
|
clearCompletedDownloads() {
|
||||||
this.downloads.delByFilter('done', dl => dl.status === 'finished').subscribe();
|
this.downloads.delByFilter('done', dl => dl.status === 'finished').subscribe((res) => this.handleActionResult(res, 'Clear completed failed'));
|
||||||
}
|
}
|
||||||
|
|
||||||
clearFailedDownloads() {
|
clearFailedDownloads() {
|
||||||
this.downloads.delByFilter('done', dl => dl.status === 'error').subscribe();
|
this.downloads.delByFilter('done', dl => dl.status === 'error').subscribe((res) => this.handleActionResult(res, 'Clear failed downloads failed'));
|
||||||
}
|
}
|
||||||
|
|
||||||
retryFailedDownloads() {
|
retryFailedDownloads() {
|
||||||
@@ -1161,10 +1209,22 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
downloadSelectedFiles() {
|
// Chromium-based browsers silently drop programmatic downloads beyond ~10 when
|
||||||
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
// triggered in a tight loop. Trigger in batches with a short pause in between so
|
||||||
this.downloads.done.forEach((dl, _) => {
|
// large selections download cleanly. See issue #1008.
|
||||||
|
private static readonly DOWNLOAD_BATCH_SIZE = 10;
|
||||||
|
private static readonly DOWNLOAD_BATCH_DELAY_MS = 1000;
|
||||||
|
|
||||||
|
async downloadSelectedFiles() {
|
||||||
|
const selected: Download[] = [];
|
||||||
|
this.downloads.done.forEach((dl) => {
|
||||||
if (dl.status === 'finished' && dl.checked) {
|
if (dl.status === 'finished' && dl.checked) {
|
||||||
|
selected.push(dl);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
for (let i = 0; i < selected.length; i++) {
|
||||||
|
const dl = selected[i];
|
||||||
const link = document.createElement('a');
|
const link = document.createElement('a');
|
||||||
link.href = this.buildDownloadLink(dl);
|
link.href = this.buildDownloadLink(dl);
|
||||||
link.setAttribute('download', dl.filename);
|
link.setAttribute('download', dl.filename);
|
||||||
@@ -1172,8 +1232,16 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
document.body.appendChild(link);
|
document.body.appendChild(link);
|
||||||
link.click();
|
link.click();
|
||||||
document.body.removeChild(link);
|
document.body.removeChild(link);
|
||||||
|
|
||||||
|
if (
|
||||||
|
(i + 1) % App.DOWNLOAD_BATCH_SIZE === 0 &&
|
||||||
|
i + 1 < selected.length
|
||||||
|
) {
|
||||||
|
await new Promise((resolve) =>
|
||||||
|
setTimeout(resolve, App.DOWNLOAD_BATCH_DELAY_MS),
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
buildDownloadLink(download: Download) {
|
buildDownloadLink(download: Download) {
|
||||||
@@ -1189,6 +1257,80 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
return baseDir + encodeURIComponent(download.filename);
|
return baseDir + encodeURIComponent(download.filename);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Web Share API support — primarily for iOS Safari / Chrome, lets the user
|
||||||
|
// hand the downloaded file off to the platform share sheet (Photos.app,
|
||||||
|
// Files, third-party apps, AirDrop). Falls back silently to the standard
|
||||||
|
// download flow on platforms without navigator.share / canShare.
|
||||||
|
canShareDownloads(): boolean {
|
||||||
|
// navigator.share alone is not enough — Desktop Safari implements
|
||||||
|
// navigator.share but not canShare with files. We explicitly require
|
||||||
|
// both, since we always intend to share a file (not a URL).
|
||||||
|
return typeof navigator !== 'undefined'
|
||||||
|
&& typeof navigator.share === 'function'
|
||||||
|
&& typeof navigator.canShare === 'function';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Conservative warning threshold for the share sheet — iOS' actual
|
||||||
|
// refusal limit varies between ~50 MB (older versions) and ~150 MB
|
||||||
|
// (recent ones). 80 MB warns the user before the time-wasting fetch+
|
||||||
|
// copy of a too-large file that the platform will then reject.
|
||||||
|
private static readonly SHARE_SIZE_WARN_BYTES = 80 * 1024 * 1024;
|
||||||
|
|
||||||
|
async shareDownload(download: Download): Promise<void> {
|
||||||
|
if (!this.canShareDownloads()) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Pre-flight size check: warn the user about the iOS share-sheet
|
||||||
|
// soft-fail on large files, before we spend time fetching the whole
|
||||||
|
// file into memory only to have navigator.canShare reject it.
|
||||||
|
if (download.size && download.size > App.SHARE_SIZE_WARN_BYTES) {
|
||||||
|
const sizeMb = Math.round(download.size / 1024 / 1024);
|
||||||
|
const proceed = await this.toasts.confirm(
|
||||||
|
`This file is ${sizeMb} MB. iOS' share sheet often refuses files ` +
|
||||||
|
`larger than ~100 MB and the share will silently fail. ` +
|
||||||
|
`Try anyway? (Use the download button instead if it fails.)`,
|
||||||
|
'Try anyway',
|
||||||
|
'Cancel',
|
||||||
|
);
|
||||||
|
if (!proceed) return;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const response = await fetch(this.buildDownloadLink(download));
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(`HTTP ${response.status} fetching file for share`);
|
||||||
|
}
|
||||||
|
const blob = await response.blob();
|
||||||
|
const file = new File([blob], download.filename, {
|
||||||
|
type: blob.type || 'application/octet-stream',
|
||||||
|
});
|
||||||
|
const payload: ShareData = { files: [file], title: download.title };
|
||||||
|
if (!navigator.canShare(payload)) {
|
||||||
|
// The platform refused the payload — most commonly because the
|
||||||
|
// file is too large for the iOS share sheet, or the MIME type
|
||||||
|
// isn't accepted. Tell the user so they can fall back to the
|
||||||
|
// download button right next to this one instead of staring at
|
||||||
|
// a button that quietly did nothing.
|
||||||
|
console.warn('navigator.canShare rejected payload for', download.filename);
|
||||||
|
this.toasts.error(
|
||||||
|
`Your device's share sheet doesn't accept this file ` +
|
||||||
|
`(most likely because it's too large). ` +
|
||||||
|
`Please use the download button instead.`
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
await navigator.share(payload);
|
||||||
|
} catch (err) {
|
||||||
|
const e = err as { name?: string; message?: string };
|
||||||
|
// AbortError = user dismissed the share sheet → silent no-op.
|
||||||
|
if (e.name === 'AbortError') return;
|
||||||
|
console.error('Share failed:', err);
|
||||||
|
this.toasts.error(
|
||||||
|
`Share failed: ${e.message || 'unknown error'}. ` +
|
||||||
|
`Please use the download button instead.`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
buildResultItemTooltip(download: Download) {
|
buildResultItemTooltip(download: Download) {
|
||||||
const parts = [];
|
const parts = [];
|
||||||
if (download.msg) {
|
if (download.msg) {
|
||||||
@@ -1274,7 +1416,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
.map(url => url.trim())
|
.map(url => url.trim())
|
||||||
.filter(url => url.length > 0);
|
.filter(url => url.length > 0);
|
||||||
if (urls.length === 0) {
|
if (urls.length === 0) {
|
||||||
alert('No valid URLs found.');
|
this.toasts.error('No valid URLs found.');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.importInProgress = true;
|
this.importInProgress = true;
|
||||||
@@ -1339,62 +1481,13 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Export URLs based on filter: 'pending', 'completed', 'failed', or 'all'
|
// Export URLs based on filter: 'pending', 'completed', 'failed', or 'all'
|
||||||
exportBatchUrls(filter: 'pending' | 'completed' | 'failed' | 'all'): void {
|
exportBatchUrls(filter: BatchUrlFilter): void {
|
||||||
let urls: string[];
|
this.batchUrls.export(filter);
|
||||||
if (filter === 'pending') {
|
|
||||||
urls = Array.from(this.downloads.queue.values()).map(dl => dl.url);
|
|
||||||
} else if (filter === 'completed') {
|
|
||||||
// Only finished downloads in the "done" Map
|
|
||||||
urls = Array.from(this.downloads.done.values()).filter(dl => dl.status === 'finished').map(dl => dl.url);
|
|
||||||
} else if (filter === 'failed') {
|
|
||||||
// Only error downloads from the "done" Map
|
|
||||||
urls = Array.from(this.downloads.done.values()).filter(dl => dl.status === 'error').map(dl => dl.url);
|
|
||||||
} else {
|
|
||||||
// All: pending + both finished and error in done
|
|
||||||
urls = [
|
|
||||||
...Array.from(this.downloads.queue.values()).map(dl => dl.url),
|
|
||||||
...Array.from(this.downloads.done.values()).map(dl => dl.url)
|
|
||||||
];
|
|
||||||
}
|
|
||||||
if (!urls.length) {
|
|
||||||
alert('No URLs found for the selected filter.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const content = urls.join('\n');
|
|
||||||
const blob = new Blob([content], { type: 'text/plain' });
|
|
||||||
const downloadUrl = window.URL.createObjectURL(blob);
|
|
||||||
const a = document.createElement('a');
|
|
||||||
a.href = downloadUrl;
|
|
||||||
a.download = 'metube_urls.txt';
|
|
||||||
document.body.appendChild(a);
|
|
||||||
a.click();
|
|
||||||
document.body.removeChild(a);
|
|
||||||
window.URL.revokeObjectURL(downloadUrl);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Copy URLs to clipboard based on filter: 'pending', 'completed', 'failed', or 'all'
|
// Copy URLs to clipboard based on filter: 'pending', 'completed', 'failed', or 'all'
|
||||||
copyBatchUrls(filter: 'pending' | 'completed' | 'failed' | 'all'): void {
|
copyBatchUrls(filter: BatchUrlFilter): void {
|
||||||
let urls: string[];
|
this.batchUrls.copy(filter);
|
||||||
if (filter === 'pending') {
|
|
||||||
urls = Array.from(this.downloads.queue.values()).map(dl => dl.url);
|
|
||||||
} else if (filter === 'completed') {
|
|
||||||
urls = Array.from(this.downloads.done.values()).filter(dl => dl.status === 'finished').map(dl => dl.url);
|
|
||||||
} else if (filter === 'failed') {
|
|
||||||
urls = Array.from(this.downloads.done.values()).filter(dl => dl.status === 'error').map(dl => dl.url);
|
|
||||||
} else {
|
|
||||||
urls = [
|
|
||||||
...Array.from(this.downloads.queue.values()).map(dl => dl.url),
|
|
||||||
...Array.from(this.downloads.done.values()).map(dl => dl.url)
|
|
||||||
];
|
|
||||||
}
|
|
||||||
if (!urls.length) {
|
|
||||||
alert('No URLs found for the selected filter.');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const content = urls.join('\n');
|
|
||||||
navigator.clipboard.writeText(content)
|
|
||||||
.then(() => alert('URLs copied to clipboard.'))
|
|
||||||
.catch(() => alert('Failed to copy URLs.'));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fetchVersionInfo(): void {
|
fetchVersionInfo(): void {
|
||||||
@@ -1454,7 +1547,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
};
|
};
|
||||||
const fail = (err?: unknown) => {
|
const fail = (err?: unknown) => {
|
||||||
console.error('Clipboard write failed:', err);
|
console.error('Clipboard write failed:', err);
|
||||||
alert('Failed to copy to clipboard. Your browser may require HTTPS for clipboard access.');
|
this.toasts.error('Failed to copy to clipboard. Your browser may require HTTPS for clipboard access.');
|
||||||
};
|
};
|
||||||
if (navigator.clipboard?.writeText) {
|
if (navigator.clipboard?.writeText) {
|
||||||
navigator.clipboard.writeText(text).then(done).catch(fail);
|
navigator.clipboard.writeText(text).then(done).catch(fail);
|
||||||
@@ -1490,7 +1583,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.hasCookies = true;
|
this.hasCookies = true;
|
||||||
} else {
|
} else {
|
||||||
this.refreshCookieStatus();
|
this.refreshCookieStatus();
|
||||||
alert(`Error uploading cookies: ${this.formatErrorMessage(response?.msg)}`);
|
this.toasts.error(`Error uploading cookies: ${this.formatErrorMessage(response?.msg)}`);
|
||||||
}
|
}
|
||||||
this.cookieUploadInProgress = false;
|
this.cookieUploadInProgress = false;
|
||||||
input.value = '';
|
input.value = '';
|
||||||
@@ -1499,7 +1592,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
this.refreshCookieStatus();
|
this.refreshCookieStatus();
|
||||||
this.cookieUploadInProgress = false;
|
this.cookieUploadInProgress = false;
|
||||||
input.value = '';
|
input.value = '';
|
||||||
alert('Error uploading cookies.');
|
this.toasts.error('Error uploading cookies.');
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1533,11 +1626,11 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
this.refreshCookieStatus();
|
this.refreshCookieStatus();
|
||||||
alert(`Error deleting cookies: ${this.formatErrorMessage(response?.msg)}`);
|
this.toasts.error(`Error deleting cookies: ${this.formatErrorMessage(response?.msg)}`);
|
||||||
},
|
},
|
||||||
error: () => {
|
error: () => {
|
||||||
this.refreshCookieStatus();
|
this.refreshCookieStatus();
|
||||||
alert('Error deleting cookies.');
|
this.toasts.error('Error deleting cookies.');
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -1561,7 +1654,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
|
|||||||
speed += download.speed || 0;
|
speed += download.speed || 0;
|
||||||
} else if (download.status === 'preparing') {
|
} else if (download.status === 'preparing') {
|
||||||
active++;
|
active++;
|
||||||
} else if (download.status === 'pending') {
|
} else if (download.status === 'pending' || download.status === 'scheduled') {
|
||||||
queued++;
|
queued++;
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,2 +1,3 @@
|
|||||||
export { SelectAllCheckboxComponent } from './master-checkbox.component';
|
export { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||||
export { ItemCheckboxComponent } from './slave-checkbox.component';
|
export { ItemCheckboxComponent } from './slave-checkbox.component';
|
||||||
|
export { ToastContainerComponent } from './toast-container.component';
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Component, ElementRef, viewChild, output, input } from "@angular/core";
|
import { Component, ElementRef, viewChild, output, input, ChangeDetectionStrategy } from "@angular/core";
|
||||||
import { Checkable } from "../interfaces";
|
import { Checkable } from "../interfaces";
|
||||||
import { FormsModule } from "@angular/forms";
|
import { FormsModule } from "@angular/forms";
|
||||||
|
|
||||||
@@ -10,6 +10,9 @@ import { FormsModule } from "@angular/forms";
|
|||||||
<label class="form-check-label visually-hidden" for="{{id()}}-select-all">Select all</label>
|
<label class="form-check-label visually-hidden" for="{{id()}}-select-all">Select all</label>
|
||||||
</div>
|
</div>
|
||||||
`,
|
`,
|
||||||
|
// Shared Checkable objects are mutated in place; Eager preserves pre-v22 behavior.
|
||||||
|
// eslint-disable-next-line @angular-eslint/prefer-on-push-component-change-detection
|
||||||
|
changeDetection: ChangeDetectionStrategy.Eager,
|
||||||
imports: [
|
imports: [
|
||||||
FormsModule
|
FormsModule
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { Component, input } from '@angular/core';
|
import { Component, input, ChangeDetectionStrategy } from '@angular/core';
|
||||||
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||||
import { Checkable } from '../interfaces';
|
import { Checkable } from '../interfaces';
|
||||||
import { FormsModule } from '@angular/forms';
|
import { FormsModule } from '@angular/forms';
|
||||||
@@ -11,6 +11,9 @@ import { FormsModule } from '@angular/forms';
|
|||||||
<label class="form-check-label visually-hidden" for="{{master().id()}}-{{id()}}-select">Select item</label>
|
<label class="form-check-label visually-hidden" for="{{master().id()}}-{{id()}}-select">Select item</label>
|
||||||
</div>
|
</div>
|
||||||
`,
|
`,
|
||||||
|
// Shared Checkable objects are mutated in place; Eager preserves pre-v22 behavior.
|
||||||
|
// eslint-disable-next-line @angular-eslint/prefer-on-push-component-change-detection
|
||||||
|
changeDetection: ChangeDetectionStrategy.Eager,
|
||||||
imports: [
|
imports: [
|
||||||
FormsModule
|
FormsModule
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
import { ChangeDetectionStrategy, Component, inject } from '@angular/core';
|
||||||
|
import { FontAwesomeModule } from '@fortawesome/angular-fontawesome';
|
||||||
|
import { faCheckCircle, faTimesCircle, faInfoCircle, faXmark } from '@fortawesome/free-solid-svg-icons';
|
||||||
|
import { ToastService } from '../services/toast.service';
|
||||||
|
|
||||||
|
@Component({
|
||||||
|
selector: 'app-toast-container',
|
||||||
|
changeDetection: ChangeDetectionStrategy.OnPush,
|
||||||
|
imports: [FontAwesomeModule],
|
||||||
|
template: `
|
||||||
|
<div class="toast-container position-fixed top-0 end-0 p-3" style="z-index: 1100;" aria-live="polite" aria-atomic="true">
|
||||||
|
@for (toast of toasts.toasts(); track toast.id) {
|
||||||
|
<div class="toast show align-items-center border-0 mb-2"
|
||||||
|
[class.text-bg-danger]="toast.level === 'error'"
|
||||||
|
[class.text-bg-success]="toast.level === 'success'"
|
||||||
|
[class.text-bg-primary]="toast.level === 'info'"
|
||||||
|
role="alert" aria-live="assertive" aria-atomic="true">
|
||||||
|
<div class="d-flex">
|
||||||
|
<div class="toast-body d-flex align-items-start gap-2">
|
||||||
|
@if (toast.level === 'error') {
|
||||||
|
<fa-icon [icon]="faTimesCircle" class="mt-1" />
|
||||||
|
} @else if (toast.level === 'success') {
|
||||||
|
<fa-icon [icon]="faCheckCircle" class="mt-1" />
|
||||||
|
} @else {
|
||||||
|
<fa-icon [icon]="faInfoCircle" class="mt-1" />
|
||||||
|
}
|
||||||
|
<span style="white-space: pre-line;">{{ toast.message }}</span>
|
||||||
|
</div>
|
||||||
|
@if (!toast.actions) {
|
||||||
|
<button type="button" class="btn-close btn-close-white me-2 m-auto"
|
||||||
|
aria-label="Close" (click)="toasts.dismiss(toast.id)"></button>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
@if (toast.actions) {
|
||||||
|
<div class="d-flex justify-content-end gap-2 px-3 pb-2">
|
||||||
|
@for (action of toast.actions; track action.label) {
|
||||||
|
<button type="button"
|
||||||
|
class="btn btn-sm"
|
||||||
|
[class.btn-light]="!action.primary"
|
||||||
|
[class.btn-outline-light]="action.primary"
|
||||||
|
(click)="toasts.respond(toast.id, action.value)">
|
||||||
|
{{ action.label }}
|
||||||
|
</button>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
`,
|
||||||
|
})
|
||||||
|
export class ToastContainerComponent {
|
||||||
|
protected readonly toasts = inject(ToastService);
|
||||||
|
protected readonly faCheckCircle = faCheckCircle;
|
||||||
|
protected readonly faTimesCircle = faTimesCircle;
|
||||||
|
protected readonly faInfoCircle = faInfoCircle;
|
||||||
|
protected readonly faXmark = faXmark;
|
||||||
|
}
|
||||||
@@ -18,6 +18,8 @@ export interface Download {
|
|||||||
ytdl_options_overrides?: Record<string, unknown>;
|
ytdl_options_overrides?: Record<string, unknown>;
|
||||||
clip_start?: number;
|
clip_start?: number;
|
||||||
clip_end?: number;
|
clip_end?: number;
|
||||||
|
live_status?: string;
|
||||||
|
live_release_timestamp?: number;
|
||||||
status: string;
|
status: string;
|
||||||
msg: string;
|
msg: string;
|
||||||
percent: number;
|
percent: number;
|
||||||
|
|||||||
@@ -10,15 +10,16 @@ describe('FileSizePipe', () => {
|
|||||||
it('formats bytes and larger units', () => {
|
it('formats bytes and larger units', () => {
|
||||||
const pipe = new FileSizePipe();
|
const pipe = new FileSizePipe();
|
||||||
expect(pipe.transform(500)).toContain('Bytes');
|
expect(pipe.transform(500)).toContain('Bytes');
|
||||||
expect(pipe.transform(1000)).toContain('KB');
|
expect(pipe.transform(1000)).toContain('Bytes');
|
||||||
expect(pipe.transform(1000 * 1000)).toContain('MB');
|
expect(pipe.transform(1024)).toContain('KB');
|
||||||
expect(pipe.transform(1000 ** 3)).toContain('GB');
|
expect(pipe.transform(1024 ** 2)).toContain('MB');
|
||||||
|
expect(pipe.transform(1024 ** 3)).toContain('GB');
|
||||||
});
|
});
|
||||||
|
|
||||||
it('handles boundaries between units', () => {
|
it('handles boundaries between units', () => {
|
||||||
const pipe = new FileSizePipe();
|
const pipe = new FileSizePipe();
|
||||||
expect(pipe.transform(999)).toContain('Bytes');
|
expect(pipe.transform(1023)).toContain('Bytes');
|
||||||
expect(pipe.transform(1000)).toContain('KB');
|
expect(pipe.transform(1024)).toContain('KB');
|
||||||
expect(pipe.transform(1001)).toContain('KB');
|
expect(pipe.transform(1025)).toContain('KB');
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -8,9 +8,10 @@ export class FileSizePipe implements PipeTransform {
|
|||||||
if (isNaN(value) || value === 0) return '0 Bytes';
|
if (isNaN(value) || value === 0) return '0 Bytes';
|
||||||
|
|
||||||
const units = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
|
const units = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'];
|
||||||
const unitIndex = Math.floor(Math.log(value) / Math.log(1000)); // Use 1000 for common units
|
const k = 1024; // Matches SpeedPipe's base so file sizes and transfer speeds agree.
|
||||||
|
const unitIndex = Math.floor(Math.log(value) / Math.log(k));
|
||||||
|
|
||||||
const unitValue = value / Math.pow(1000, unitIndex);
|
const unitValue = value / Math.pow(k, unitIndex);
|
||||||
return `${unitValue.toFixed(2)} ${units[unitIndex]}`;
|
return `${unitValue.toFixed(2)} ${units[unitIndex]}`;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import { inject, Injectable } from '@angular/core';
|
||||||
|
import { DownloadsService } from './downloads.service';
|
||||||
|
import { ToastService } from './toast.service';
|
||||||
|
|
||||||
|
export type BatchUrlFilter = 'pending' | 'completed' | 'failed' | 'all';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Encapsulates collecting download URLs by status and exporting/copying them.
|
||||||
|
* Extracted from the main app component to keep it focused on view concerns.
|
||||||
|
*/
|
||||||
|
@Injectable({ providedIn: 'root' })
|
||||||
|
export class BatchUrlsService {
|
||||||
|
private downloads = inject(DownloadsService);
|
||||||
|
private toasts = inject(ToastService);
|
||||||
|
|
||||||
|
collect(filter: BatchUrlFilter): string[] {
|
||||||
|
const queueUrls = () => Array.from(this.downloads.queue.values()).map((dl) => dl.url);
|
||||||
|
const doneUrls = (status?: string) =>
|
||||||
|
Array.from(this.downloads.done.values())
|
||||||
|
.filter((dl) => status === undefined || dl.status === status)
|
||||||
|
.map((dl) => dl.url);
|
||||||
|
switch (filter) {
|
||||||
|
case 'pending':
|
||||||
|
return queueUrls();
|
||||||
|
case 'completed':
|
||||||
|
return doneUrls('finished');
|
||||||
|
case 'failed':
|
||||||
|
return doneUrls('error');
|
||||||
|
default:
|
||||||
|
return [...queueUrls(), ...doneUrls()];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export(filter: BatchUrlFilter): void {
|
||||||
|
const urls = this.collect(filter);
|
||||||
|
if (!urls.length) {
|
||||||
|
this.toasts.info('No URLs found for the selected filter.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const blob = new Blob([urls.join('\n')], { type: 'text/plain' });
|
||||||
|
const downloadUrl = window.URL.createObjectURL(blob);
|
||||||
|
const a = document.createElement('a');
|
||||||
|
a.href = downloadUrl;
|
||||||
|
a.download = 'metube_urls.txt';
|
||||||
|
document.body.appendChild(a);
|
||||||
|
a.click();
|
||||||
|
document.body.removeChild(a);
|
||||||
|
window.URL.revokeObjectURL(downloadUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
copy(filter: BatchUrlFilter): void {
|
||||||
|
const urls = this.collect(filter);
|
||||||
|
if (!urls.length) {
|
||||||
|
this.toasts.info('No URLs found for the selected filter.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
navigator.clipboard
|
||||||
|
.writeText(urls.join('\n'))
|
||||||
|
.then(() => this.toasts.success('URLs copied to clipboard.'))
|
||||||
|
.catch(() => this.toasts.error('Failed to copy URLs.'));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -159,6 +159,61 @@ describe('DownloadsService', () => {
|
|||||||
req.flush({});
|
req.flush({});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('delById resets deleting flag and emits error status on HTTP failure', () => {
|
||||||
|
const dl: Download = {
|
||||||
|
id: '1',
|
||||||
|
title: 't',
|
||||||
|
url: 'u1',
|
||||||
|
download_type: 'video',
|
||||||
|
quality: 'best',
|
||||||
|
format: 'any',
|
||||||
|
folder: '',
|
||||||
|
custom_name_prefix: '',
|
||||||
|
playlist_item_limit: 0,
|
||||||
|
status: 'finished',
|
||||||
|
msg: '',
|
||||||
|
percent: 0,
|
||||||
|
speed: 0,
|
||||||
|
eta: 0,
|
||||||
|
filename: '',
|
||||||
|
checked: false,
|
||||||
|
deleting: false,
|
||||||
|
};
|
||||||
|
service.queue.set('u1', dl);
|
||||||
|
let queueChangedCount = 0;
|
||||||
|
service.queueChanged.subscribe(() => queueChangedCount++);
|
||||||
|
let result: unknown;
|
||||||
|
let threw = false;
|
||||||
|
|
||||||
|
service.delById('queue', ['u1']).subscribe({
|
||||||
|
next: (res) => { result = res; },
|
||||||
|
error: () => { threw = true; },
|
||||||
|
});
|
||||||
|
expect(dl.deleting).toBe(true);
|
||||||
|
const req = httpMock.expectOne('delete');
|
||||||
|
req.flush({ msg: 'boom' }, { status: 500, statusText: 'Server Error' });
|
||||||
|
|
||||||
|
expect(threw).toBe(false);
|
||||||
|
expect(dl.deleting).toBe(false);
|
||||||
|
expect(queueChangedCount).toBeGreaterThan(0);
|
||||||
|
expect((result as { status: string }).status).toBe('error');
|
||||||
|
});
|
||||||
|
|
||||||
|
it('startById surfaces HTTP errors as a status object instead of throwing', () => {
|
||||||
|
let result: unknown;
|
||||||
|
let threw = false;
|
||||||
|
|
||||||
|
service.startById(['a']).subscribe({
|
||||||
|
next: (res) => { result = res; },
|
||||||
|
error: () => { threw = true; },
|
||||||
|
});
|
||||||
|
const req = httpMock.expectOne('start');
|
||||||
|
req.flush({ msg: 'nope' }, { status: 500, statusText: 'Server Error' });
|
||||||
|
|
||||||
|
expect(threw).toBe(false);
|
||||||
|
expect((result as { status: string }).status).toBe('error');
|
||||||
|
});
|
||||||
|
|
||||||
it('handleHTTPError extracts msg from object body', async () => {
|
it('handleHTTPError extracts msg from object body', async () => {
|
||||||
const err = new HttpErrorResponse({
|
const err = new HttpErrorResponse({
|
||||||
error: { msg: 'bad' },
|
error: { msg: 'bad' },
|
||||||
@@ -226,6 +281,15 @@ describe('DownloadsService', () => {
|
|||||||
expect(updated?.deleting).toBe(true);
|
expect(updated?.deleting).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('socket updated ignores events for urls not already in the queue', () => {
|
||||||
|
expect(service.queue.has('unknown-url')).toBe(false);
|
||||||
|
socket.emit(
|
||||||
|
'updated',
|
||||||
|
JSON.stringify({ url: 'unknown-url', title: 't', status: 'downloading' }),
|
||||||
|
);
|
||||||
|
expect(service.queue.has('unknown-url')).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
it('socket completed moves entry to done', () => {
|
it('socket completed moves entry to done', () => {
|
||||||
service.queue.set('u1', {
|
service.queue.set('u1', {
|
||||||
id: '1',
|
id: '1',
|
||||||
|
|||||||
@@ -69,8 +69,14 @@ export class DownloadsService {
|
|||||||
.subscribe((strdata: string) => {
|
.subscribe((strdata: string) => {
|
||||||
const data: Download = JSON.parse(strdata);
|
const data: Download = JSON.parse(strdata);
|
||||||
const dl: Download | undefined = this.queue.get(data.url);
|
const dl: Download | undefined = this.queue.get(data.url);
|
||||||
data.checked = !!dl?.checked;
|
// An 'added' event always precedes legitimate updates. If the row is
|
||||||
data.deleting = !!dl?.deleting;
|
// gone (canceled/completed already processed), this update is stale —
|
||||||
|
// applying it would resurrect a ghost row until the next full refresh.
|
||||||
|
if (!dl) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
data.checked = !!dl.checked;
|
||||||
|
data.deleting = !!dl.deleting;
|
||||||
this.queue.set(data.url, data);
|
this.queue.set(data.url, data);
|
||||||
this.updated.next();
|
this.updated.next();
|
||||||
});
|
});
|
||||||
@@ -164,7 +170,9 @@ export class DownloadsService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public startById(ids: string[]) {
|
public startById(ids: string[]) {
|
||||||
return this.http.post('start', {ids: ids});
|
return this.http.post<Status>('start', {ids: ids}).pipe(
|
||||||
|
catchError(this.handleHTTPError)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public delById(where: State, ids: string[]) {
|
public delById(where: State, ids: string[]) {
|
||||||
@@ -177,7 +185,22 @@ export class DownloadsService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return this.http.post('delete', {where: where, ids: ids});
|
return this.http.post<Status>('delete', {where: where, ids: ids}).pipe(
|
||||||
|
catchError((err: HttpErrorResponse) => {
|
||||||
|
// Request failed — the rows would otherwise stay disabled forever
|
||||||
|
// with no way to retry, since nothing ever clears `deleting`.
|
||||||
|
if (map) {
|
||||||
|
for (const id of ids) {
|
||||||
|
const obj = map.get(id);
|
||||||
|
if (obj) {
|
||||||
|
obj.deleting = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(where === 'queue' ? this.queueChanged : this.doneChanged).next();
|
||||||
|
return this.handleHTTPError(err);
|
||||||
|
})
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
public startByFilter(where: State, filter: (dl: Download) => boolean) {
|
public startByFilter(where: State, filter: (dl: Download) => boolean) {
|
||||||
|
|||||||
@@ -1,2 +1,4 @@
|
|||||||
export { DownloadsService } from './downloads.service';
|
export { DownloadsService } from './downloads.service';
|
||||||
export { MeTubeSocket } from './metube-socket.service';
|
export { MeTubeSocket } from './metube-socket.service';
|
||||||
|
export { ToastService } from './toast.service';
|
||||||
|
export { BatchUrlsService } from './batch-urls.service';
|
||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import { Injectable, signal } from '@angular/core';
|
||||||
|
|
||||||
|
export type ToastLevel = 'info' | 'success' | 'error';
|
||||||
|
|
||||||
|
export interface ToastAction {
|
||||||
|
label: string;
|
||||||
|
value: boolean;
|
||||||
|
primary?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Toast {
|
||||||
|
id: number;
|
||||||
|
level: ToastLevel;
|
||||||
|
message: string;
|
||||||
|
actions?: ToastAction[];
|
||||||
|
/** Resolver for confirm() toasts; resolved when the user picks an action or dismisses. */
|
||||||
|
_resolve?: (value: boolean) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Lightweight non-blocking notification service. Replaces the blocking
|
||||||
|
* window.alert()/confirm() dialogs that previously littered the app component.
|
||||||
|
*/
|
||||||
|
@Injectable({ providedIn: 'root' })
|
||||||
|
export class ToastService {
|
||||||
|
private counter = 0;
|
||||||
|
readonly toasts = signal<Toast[]>([]);
|
||||||
|
|
||||||
|
info(message: string): void {
|
||||||
|
this.show('info', message, 4000);
|
||||||
|
}
|
||||||
|
|
||||||
|
success(message: string): void {
|
||||||
|
this.show('success', message, 4000);
|
||||||
|
}
|
||||||
|
|
||||||
|
error(message: string): void {
|
||||||
|
this.show('error', message, 8000);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Show a confirmation toast with confirm/cancel actions. Resolves true when
|
||||||
|
* confirmed, false when cancelled or auto-dismissed.
|
||||||
|
*/
|
||||||
|
confirm(message: string, confirmLabel = 'OK', cancelLabel = 'Cancel'): Promise<boolean> {
|
||||||
|
return new Promise<boolean>((resolve) => {
|
||||||
|
const id = ++this.counter;
|
||||||
|
this.toasts.update((list) => [
|
||||||
|
...list,
|
||||||
|
{
|
||||||
|
id,
|
||||||
|
level: 'info',
|
||||||
|
message,
|
||||||
|
actions: [
|
||||||
|
{ label: cancelLabel, value: false },
|
||||||
|
{ label: confirmLabel, value: true, primary: true },
|
||||||
|
],
|
||||||
|
_resolve: resolve,
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
respond(id: number, value: boolean): void {
|
||||||
|
const toast = this.toasts().find((t) => t.id === id);
|
||||||
|
toast?._resolve?.(value);
|
||||||
|
this.remove(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
dismiss(id: number): void {
|
||||||
|
const toast = this.toasts().find((t) => t.id === id);
|
||||||
|
// A confirm toast dismissed without an explicit choice resolves to false.
|
||||||
|
toast?._resolve?.(false);
|
||||||
|
this.remove(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
private remove(id: number): void {
|
||||||
|
this.toasts.update((list) => list.filter((t) => t.id !== id));
|
||||||
|
}
|
||||||
|
|
||||||
|
private show(level: ToastLevel, message: string, autoDismissMs: number): void {
|
||||||
|
const id = ++this.counter;
|
||||||
|
this.toasts.update((list) => [...list, { id, level, message }]);
|
||||||
|
setTimeout(() => this.remove(id), autoDismissMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,5 +12,13 @@
|
|||||||
],
|
],
|
||||||
"exclude": [
|
"exclude": [
|
||||||
"src/**/*.spec.ts"
|
"src/**/*.spec.ts"
|
||||||
]
|
],
|
||||||
|
"angularCompilerOptions": {
|
||||||
|
"extendedDiagnostics": {
|
||||||
|
"checks": {
|
||||||
|
"nullishCoalescingNotNullable": "suppress",
|
||||||
|
"optionalChainNotNullable": "suppress"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user