fix: harden download lifecycle, subscriptions, and UI robustness

Addresses a full-project review. Backend correctness and availability:

- ytdl: cancel() only SIGKILLs the child's process group when the child
  actually became its own group leader, so a race (or failed setpgrp)
  can no longer kill the whole server; kill the group on cancel and on
  shutdown to avoid orphaned ffmpeg children
- ytdl: dedicated ThreadPoolExecutor for download supervision so active
  downloads can't starve extract_info / live probes on the default pool
- ytdl/main/subscriptions: route fire-and-forget tasks through a
  bg_tasks helper that keeps a strong ref and logs failures
- subscriptions: run flat-playlist extraction in an executor and check
  feeds with bounded concurrency so one slow feed can't block the loop;
  set last_checked on failure so broken feeds aren't retried every 60s
- main: validate ids on /start & /delete and numeric env vars at startup;
  return 400 (not 500) on bad subscriptions/update input; serve /history
  from memory; move get_custom_dirs off the event loop; restrict t=
  stripping to YouTube hosts; drop double percent-decode in state guard
- dl_formats/ytdl: enforce requested caption format via
  FFmpegSubtitlesConvertor and strip VTT header metadata only in the
  pre-cue region so real dialogue is preserved
- ytdl: throttle progress events, dedup adds against pending, clear
  filename/size on error and reject out-of-dir trashcan deletes, pin
  fork start-method on Linux only

Frontend:

- retry deletes the done record only after a successful re-add
- surface HTTP errors for delete/start and reset the deleting flag
- ignore late 'updated' events for rows no longer in the queue
- track table rows by map key; FileSizePipe uses base-1024

Also: HTTPS-aware Docker healthcheck, dead-code removal, and shared
helpers for path-containment and yt-dlp option merging. Adds/updates
unit tests throughout (250 backend tests passing).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Alex Shnitman
2026-07-12 08:08:14 +03:00
parent e2c777842e
commit 3ea4732c5d
21 changed files with 1392 additions and 166 deletions
+163 -1
View File
@@ -3,13 +3,14 @@
from __future__ import annotations
import os
import re
import tempfile
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import time
from ytdl import DownloadInfo, DownloadQueue
from ytdl import Download, DownloadInfo, DownloadQueue
@pytest.fixture
@@ -46,6 +47,37 @@ def test_cancel_add_increments_generation(dq_env):
assert dq._add_generation == before + 1
def test_download_queue_has_dedicated_executor_sized_from_config(dq_env):
notifier = MagicMock()
dq = DownloadQueue(dq_env, notifier)
assert dq._download_executor is not None
assert dq._download_executor._max_workers == 2 * int(dq_env.MAX_CONCURRENT_DOWNLOADS) + 2
dq.close()
def test_close_cancels_running_downloads_before_shutdown(dq_env):
notifier = MagicMock()
dq = DownloadQueue(dq_env, notifier)
running = MagicMock()
running.started.return_value = True
running.running.return_value = True
idle = MagicMock()
idle.started.return_value = False
idle.running.return_value = False
dq.queue.dict["u-running"] = running
dq.queue.dict["u-idle"] = idle
dq.close()
# The active download's subprocess group is killed; the not-started one is
# left alone. Executor is shut down afterwards.
running.cancel.assert_called_once()
idle.cancel.assert_not_called()
assert dq._download_executor._shutdown
def test_get_returns_tuple_of_lists(dq_env):
notifier = MagicMock()
dq = DownloadQueue(dq_env, notifier)
@@ -222,6 +254,58 @@ async def test_add_entry_queues_single_video_without_reextracting(dq_env):
assert dq.pending.exists("https://example.com/watch?v=1")
@pytest.mark.asyncio
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
notifier = AsyncMock()
dq = DownloadQueue(dq_env, notifier)
entry = {
"_type": "video",
"id": "vid1",
"title": "Original Title",
"url": "https://example.com/watch?v=1",
"webpage_url": "https://example.com/watch?v=1",
}
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")):
first = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=False)
assert first["status"] == "ok"
assert "msg" not in first
dupe_entry = {**entry, "title": "Different Title"}
second = await dq.add_entry(dupe_entry, "audio", "auto", "mp3", "best", "", "", 0, auto_start=False)
assert second["status"] == "ok"
assert "Already in queue" in second["msg"]
# The original pending download's options must survive untouched.
pending_dl = dq.pending.get("https://example.com/watch?v=1")
assert pending_dl.info.download_type == "video"
assert pending_dl.info.title == "Original Title"
@pytest.mark.asyncio
async def test_add_entry_duplicate_while_queued_is_skipped(dq_env):
notifier = AsyncMock()
dq = DownloadQueue(dq_env, notifier)
entry = {
"_type": "video",
"id": "vid1",
"title": "Test Video",
"url": "https://example.com/watch?v=1",
"webpage_url": "https://example.com/watch?v=1",
}
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")), \
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
first = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=True)
assert first["status"] == "ok"
assert dq.queue.exists("https://example.com/watch?v=1")
second = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=True)
assert second["status"] == "ok"
assert "Already in queue" in second["msg"]
@pytest.mark.asyncio
async def test_channel_download_uses_output_template_when_channel_template_empty(dq_env):
"""Channel tabs reported as playlists must honor OUTPUT_TEMPLATE when OUTPUT_TEMPLATE_CHANNEL is empty."""
@@ -526,6 +610,9 @@ async def test_add_upcoming_stream_scheduled_without_starting(dq_env):
assert download.info.live_release_timestamp is not None
start_mock.assert_not_called()
assert url in dq._scheduled_probe_at
# The "scheduled to start at ..." message must include a UTC offset
# (a naive datetime's %z would render as an empty string here).
assert re.search(r"[+-]\d{4}$", download.info.error)
@pytest.mark.asyncio
@@ -784,3 +871,78 @@ def test_download_info_to_public_dict_excludes_server_only_fields():
assert public["url"] == "https://example.com/watch?v=1"
assert public["title"] == "Test Video"
assert public["status"] == "pending"
def _make_download(dq_env, *, download_type="video", status="downloading", filename=None):
info = DownloadInfo(
id="id1",
title="t",
url="http://example.com/v",
quality="best",
download_type=download_type,
codec="auto",
format="any",
folder="",
custom_name_prefix="",
error=None,
entry=None,
playlist_item_limit=0,
split_by_chapters=False,
chapter_template="",
)
info.status = status
info.filename = filename
info.size = 123 if filename else None
return Download(
dq_env.DOWNLOAD_DIR, dq_env.TEMP_DIR, "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info
)
@pytest.mark.asyncio
async def test_post_download_cleanup_clears_filename_on_error(dq_env):
notifier = AsyncMock()
dq = DownloadQueue(dq_env, notifier)
download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
dq.queue.put(download)
dq._post_download_cleanup(download)
assert download.info.status == "error"
assert download.info.filename is None
assert download.info.size is None
@pytest.mark.asyncio
async def test_post_download_cleanup_keeps_captured_subtitles_on_error(dq_env):
notifier = AsyncMock()
dq = DownloadQueue(dq_env, notifier)
download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
dq.queue.put(download)
dq._post_download_cleanup(download)
assert download.info.status == "error"
assert download.info.filename == "en.srt"
@pytest.mark.asyncio
async def test_clear_skips_deletion_outside_download_directory(dq_env):
notifier = AsyncMock()
dq_env.DELETE_FILE_ON_TRASHCAN = True
dq = DownloadQueue(dq_env, notifier)
outside_dir = tempfile.mkdtemp()
outside_file = os.path.join(outside_dir, "outside.txt")
with open(outside_file, "w") as f:
f.write("do not delete me")
# A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
download = _make_download(dq_env, status="finished", filename=escaping_filename)
dq.done.put(download)
await dq.clear([download.info.url])
assert os.path.exists(outside_file)
assert not dq.done.exists(download.info.url)