mirror of
https://github.com/alexta69/metube.git
synced 2026-09-21 21:45:04 +00:00
fix: harden download lifecycle, subscriptions, and UI robustness
Addresses a full-project review. Backend correctness and availability: - ytdl: cancel() only SIGKILLs the child's process group when the child actually became its own group leader, so a race (or failed setpgrp) can no longer kill the whole server; kill the group on cancel and on shutdown to avoid orphaned ffmpeg children - ytdl: dedicated ThreadPoolExecutor for download supervision so active downloads can't starve extract_info / live probes on the default pool - ytdl/main/subscriptions: route fire-and-forget tasks through a bg_tasks helper that keeps a strong ref and logs failures - subscriptions: run flat-playlist extraction in an executor and check feeds with bounded concurrency so one slow feed can't block the loop; set last_checked on failure so broken feeds aren't retried every 60s - main: validate ids on /start & /delete and numeric env vars at startup; return 400 (not 500) on bad subscriptions/update input; serve /history from memory; move get_custom_dirs off the event loop; restrict t= stripping to YouTube hosts; drop double percent-decode in state guard - dl_formats/ytdl: enforce requested caption format via FFmpegSubtitlesConvertor and strip VTT header metadata only in the pre-cue region so real dialogue is preserved - ytdl: throttle progress events, dedup adds against pending, clear filename/size on error and reject out-of-dir trashcan deletes, pin fork start-method on Linux only Frontend: - retry deletes the done record only after a successful re-add - surface HTTP errors for delete/start and reset the deleting flag - ignore late 'updated' events for rows no longer in the queue - track table rows by map key; FileSizePipe uses base-1024 Also: HTTPS-aware Docker healthcheck, dead-code removal, and shared helpers for path-containment and yt-dlp option merging. Adds/updates unit tests throughout (250 backend tests passing). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -3,13 +3,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import tempfile
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import time
|
||||
|
||||
from ytdl import DownloadInfo, DownloadQueue
|
||||
from ytdl import Download, DownloadInfo, DownloadQueue
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -46,6 +47,37 @@ def test_cancel_add_increments_generation(dq_env):
|
||||
assert dq._add_generation == before + 1
|
||||
|
||||
|
||||
def test_download_queue_has_dedicated_executor_sized_from_config(dq_env):
|
||||
notifier = MagicMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
assert dq._download_executor is not None
|
||||
assert dq._download_executor._max_workers == 2 * int(dq_env.MAX_CONCURRENT_DOWNLOADS) + 2
|
||||
dq.close()
|
||||
|
||||
|
||||
def test_close_cancels_running_downloads_before_shutdown(dq_env):
|
||||
notifier = MagicMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
|
||||
running = MagicMock()
|
||||
running.started.return_value = True
|
||||
running.running.return_value = True
|
||||
idle = MagicMock()
|
||||
idle.started.return_value = False
|
||||
idle.running.return_value = False
|
||||
|
||||
dq.queue.dict["u-running"] = running
|
||||
dq.queue.dict["u-idle"] = idle
|
||||
|
||||
dq.close()
|
||||
|
||||
# The active download's subprocess group is killed; the not-started one is
|
||||
# left alone. Executor is shut down afterwards.
|
||||
running.cancel.assert_called_once()
|
||||
idle.cancel.assert_not_called()
|
||||
assert dq._download_executor._shutdown
|
||||
|
||||
|
||||
def test_get_returns_tuple_of_lists(dq_env):
|
||||
notifier = MagicMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
@@ -222,6 +254,58 @@ async def test_add_entry_queues_single_video_without_reextracting(dq_env):
|
||||
assert dq.pending.exists("https://example.com/watch?v=1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_entry_duplicate_while_pending_is_skipped_not_clobbered(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
entry = {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Original Title",
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"webpage_url": "https://example.com/watch?v=1",
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")):
|
||||
first = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=False)
|
||||
assert first["status"] == "ok"
|
||||
assert "msg" not in first
|
||||
|
||||
dupe_entry = {**entry, "title": "Different Title"}
|
||||
second = await dq.add_entry(dupe_entry, "audio", "auto", "mp3", "best", "", "", 0, auto_start=False)
|
||||
|
||||
assert second["status"] == "ok"
|
||||
assert "Already in queue" in second["msg"]
|
||||
# The original pending download's options must survive untouched.
|
||||
pending_dl = dq.pending.get("https://example.com/watch?v=1")
|
||||
assert pending_dl.info.download_type == "video"
|
||||
assert pending_dl.info.title == "Original Title"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_entry_duplicate_while_queued_is_skipped(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
entry = {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"webpage_url": "https://example.com/watch?v=1",
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", new=AsyncMock()):
|
||||
first = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=True)
|
||||
assert first["status"] == "ok"
|
||||
assert dq.queue.exists("https://example.com/watch?v=1")
|
||||
|
||||
second = await dq.add_entry(entry, "video", "auto", "any", "best", "", "", 0, auto_start=True)
|
||||
|
||||
assert second["status"] == "ok"
|
||||
assert "Already in queue" in second["msg"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_channel_download_uses_output_template_when_channel_template_empty(dq_env):
|
||||
"""Channel tabs reported as playlists must honor OUTPUT_TEMPLATE when OUTPUT_TEMPLATE_CHANNEL is empty."""
|
||||
@@ -526,6 +610,9 @@ async def test_add_upcoming_stream_scheduled_without_starting(dq_env):
|
||||
assert download.info.live_release_timestamp is not None
|
||||
start_mock.assert_not_called()
|
||||
assert url in dq._scheduled_probe_at
|
||||
# The "scheduled to start at ..." message must include a UTC offset
|
||||
# (a naive datetime's %z would render as an empty string here).
|
||||
assert re.search(r"[+-]\d{4}$", download.info.error)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -784,3 +871,78 @@ def test_download_info_to_public_dict_excludes_server_only_fields():
|
||||
assert public["url"] == "https://example.com/watch?v=1"
|
||||
assert public["title"] == "Test Video"
|
||||
assert public["status"] == "pending"
|
||||
|
||||
|
||||
def _make_download(dq_env, *, download_type="video", status="downloading", filename=None):
|
||||
info = DownloadInfo(
|
||||
id="id1",
|
||||
title="t",
|
||||
url="http://example.com/v",
|
||||
quality="best",
|
||||
download_type=download_type,
|
||||
codec="auto",
|
||||
format="any",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
error=None,
|
||||
entry=None,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
)
|
||||
info.status = status
|
||||
info.filename = filename
|
||||
info.size = 123 if filename else None
|
||||
return Download(
|
||||
dq_env.DOWNLOAD_DIR, dq_env.TEMP_DIR, "%(title)s.%(ext)s", "%(title)s.%(ext)s", "best", "any", {}, info
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_download_cleanup_clears_filename_on_error(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
download = _make_download(dq_env, status="downloading", filename="../tmp/partial.mp4")
|
||||
dq.queue.put(download)
|
||||
|
||||
dq._post_download_cleanup(download)
|
||||
|
||||
assert download.info.status == "error"
|
||||
assert download.info.filename is None
|
||||
assert download.info.size is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_download_cleanup_keeps_captured_subtitles_on_error(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
download = _make_download(dq_env, download_type="captions", status="downloading", filename="en.srt")
|
||||
download.info.subtitle_files = [{"filename": "en.srt", "size": 42}]
|
||||
dq.queue.put(download)
|
||||
|
||||
dq._post_download_cleanup(download)
|
||||
|
||||
assert download.info.status == "error"
|
||||
assert download.info.filename == "en.srt"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_skips_deletion_outside_download_directory(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq_env.DELETE_FILE_ON_TRASHCAN = True
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
|
||||
outside_dir = tempfile.mkdtemp()
|
||||
outside_file = os.path.join(outside_dir, "outside.txt")
|
||||
with open(outside_file, "w") as f:
|
||||
f.write("do not delete me")
|
||||
|
||||
# A crafted/legacy relative filename that escapes DOWNLOAD_DIR via '..'.
|
||||
escaping_filename = os.path.relpath(outside_file, dq_env.DOWNLOAD_DIR)
|
||||
download = _make_download(dq_env, status="finished", filename=escaping_filename)
|
||||
dq.done.put(download)
|
||||
|
||||
await dq.clear([download.info.url])
|
||||
|
||||
assert os.path.exists(outside_file)
|
||||
assert not dq.done.exists(download.info.url)
|
||||
|
||||
Reference in New Issue
Block a user