Tom FluxandClaude Opus 5
18bb2e420b
A DVR for YouTube subscriptions, laid out so Jellyfin presents each channel as a show and each video as an episode. Cron-driven, idempotent, with a public admin UI for a non-operator. Verified end to end on susan against three real channels: PO tokens, h264 downloads, Jellyfin resolution from local NFOs with all providers disabled, retention and tombstones. Corrections to the original design handover (specs.md documents each with the evidence, and specs.handover-original.md preserves the original): - The format sort selected 360p. Ranking acodec above res makes `bv*` prefer the combined 360p stream, which carries AAC, over the 720p video-only stream whose acodec is none. vcodec now leads, so a video without h264 at 720p yields h264 lower down rather than VP9 this hardware cannot transcode. - yt-dlp now requires a JS runtime and the yt-dlp-ejs solver scripts, which only ship with the [default] extra. Without them the n challenge fails and the mweb formats disappear entirely. - --flat-playlist carries no upload dates, so the specced client-side date filter for backfill was impossible. Backfill is RSS-first. - skipped_old was terminal, so raising a channel's retention appeared to do nothing. Added an explicit rescan. - is_upcoming premieres now defer and retry instead of being skipped forever. - TubeArchivist is gone, so the media root and the tube.jihakuz.xyz vhost were both reclaimed; the latter still pointed at its dead port. 240 offline tests, no network and no real yt-dlp invocation. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
187 lines
7.5 KiB
Python
187 lines
7.5 KiB
Python
"""Retention: candidate selection, artefact deletion, pruning, disk cap."""
|
|
|
|
from datetime import timedelta
|
|
|
|
from conftest import add_video
|
|
from youtube_automate import config, reap, util, videos
|
|
|
|
|
|
def _place(media_root, channel_dir, season, stem):
|
|
"""Create a downloaded video's full set of artefacts on disk."""
|
|
season_dir = media_root / channel_dir / f"Season {season}"
|
|
season_dir.mkdir(parents=True, exist_ok=True)
|
|
(season_dir / f"{stem}.mp4").write_bytes(b"video-bytes")
|
|
(season_dir / f"{stem}.nfo").write_text("<episodedetails/>")
|
|
(season_dir / f"{stem}.info.json").write_text("{}")
|
|
(season_dir / f"{stem}-thumb.jpg").write_bytes(b"jpg")
|
|
(season_dir / f"{stem}.en.srt").write_text("1\n")
|
|
return f"{channel_dir}/Season {season}/{stem}.mp4"
|
|
|
|
|
|
def _downloaded(conn, channel, media_root, video_id, upload_date, size=1024):
|
|
"""Place artefacts and record the row.
|
|
|
|
`size` is only ever read back out of the database (the disk-cap arithmetic
|
|
uses `size_bytes`), so the files on disk stay tiny no matter how large a
|
|
size the test claims.
|
|
"""
|
|
stem = f"clabretro - S2026E8110 - Title [{video_id}]"
|
|
rel = _place(media_root, channel["dir_name"], 2026, stem)
|
|
add_video(conn, channel["id"], video_id, upload_date=upload_date)
|
|
videos.mark_downloaded(
|
|
conn, video_id, rel_path=rel, size_bytes=size, season=2026, episode=8110,
|
|
upload_date=upload_date, duration=100, title="Title",
|
|
)
|
|
return rel
|
|
|
|
|
|
class TestCandidates:
|
|
def test_selects_only_videos_past_the_window(self, conn, settings, channel, media_root):
|
|
fresh = (util.today() - timedelta(days=3)).isoformat()
|
|
stale = (util.today() - timedelta(days=30)).isoformat()
|
|
_downloaded(conn, channel, media_root, "fresh", fresh)
|
|
_downloaded(conn, channel, media_root, "stale", stale)
|
|
|
|
due = [row["video_id"] for row in reap.candidates(conn, settings)]
|
|
assert due == ["stale"]
|
|
|
|
def test_per_channel_override_widens_the_window(
|
|
self, conn, settings, channel, media_root
|
|
):
|
|
age = (util.today() - timedelta(days=30)).isoformat()
|
|
_downloaded(conn, channel, media_root, "v1", age)
|
|
assert len(reap.candidates(conn, settings)) == 1
|
|
|
|
with conn:
|
|
conn.execute(
|
|
"UPDATE channel SET retention_days = 60 WHERE id = ?", (channel["id"],)
|
|
)
|
|
assert reap.candidates(conn, settings) == []
|
|
|
|
def test_per_channel_override_narrows_the_window(
|
|
self, conn, settings, channel, media_root
|
|
):
|
|
age = (util.today() - timedelta(days=5)).isoformat()
|
|
_downloaded(conn, channel, media_root, "v1", age)
|
|
assert reap.candidates(conn, settings) == []
|
|
|
|
with conn:
|
|
conn.execute(
|
|
"UPDATE channel SET retention_days = 2 WHERE id = ?", (channel["id"],)
|
|
)
|
|
assert len(reap.candidates(conn, settings)) == 1
|
|
|
|
def test_ignores_videos_that_are_not_downloaded(self, conn, settings, channel):
|
|
old = (util.today() - timedelta(days=99)).isoformat()
|
|
add_video(conn, channel["id"], "p", upload_date=old, state=videos.PENDING)
|
|
add_video(conn, channel["id"], "d", upload_date=old, state=videos.DELETED)
|
|
assert reap.candidates(conn, settings) == []
|
|
|
|
|
|
class TestDeletion:
|
|
def test_removes_every_sidecar(self, conn, settings, channel, media_root, monkeypatch):
|
|
monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
|
|
old = (util.today() - timedelta(days=30)).isoformat()
|
|
rel = _downloaded(conn, channel, media_root, "v1", old)
|
|
season_dir = (media_root / rel).parent
|
|
|
|
reap.run(conn, settings)
|
|
assert not season_dir.exists()
|
|
|
|
def test_leaves_a_tombstone(self, conn, settings, channel, media_root, monkeypatch):
|
|
monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
|
|
old = (util.today() - timedelta(days=30)).isoformat()
|
|
_downloaded(conn, channel, media_root, "v1", old)
|
|
|
|
reap.run(conn, settings)
|
|
row = videos.get(conn, "v1")
|
|
assert row["state"] == videos.DELETED
|
|
assert row["rel_path"] is None
|
|
|
|
def test_keeps_the_channel_directory_and_its_artwork(
|
|
self, conn, settings, channel, media_root, monkeypatch
|
|
):
|
|
"""Deleting the channel dir would make an active subscription vanish
|
|
from Jellyfin and reappear later."""
|
|
monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
|
|
channel_dir = media_root / channel["dir_name"]
|
|
channel_dir.mkdir(parents=True, exist_ok=True)
|
|
(channel_dir / "tvshow.nfo").write_text("<tvshow/>")
|
|
(channel_dir / "poster.jpg").write_bytes(b"jpg")
|
|
|
|
old = (util.today() - timedelta(days=30)).isoformat()
|
|
_downloaded(conn, channel, media_root, "v1", old)
|
|
reap.run(conn, settings)
|
|
|
|
assert channel_dir.is_dir()
|
|
assert (channel_dir / "tvshow.nfo").exists()
|
|
assert (channel_dir / "poster.jpg").exists()
|
|
|
|
def test_does_not_prune_a_season_that_still_has_videos(
|
|
self, conn, settings, channel, media_root, monkeypatch
|
|
):
|
|
monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
|
|
old = (util.today() - timedelta(days=30)).isoformat()
|
|
fresh = (util.today() - timedelta(days=1)).isoformat()
|
|
rel = _downloaded(conn, channel, media_root, "old1", old)
|
|
_downloaded(conn, channel, media_root, "new1", fresh)
|
|
|
|
reap.run(conn, settings)
|
|
season_dir = (media_root / rel).parent
|
|
assert season_dir.is_dir()
|
|
assert list(season_dir.glob("*new1*"))
|
|
assert not list(season_dir.glob("*old1*"))
|
|
|
|
def test_only_deletes_files_matching_the_stem(
|
|
self, conn, settings, channel, media_root, monkeypatch
|
|
):
|
|
monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
|
|
old = (util.today() - timedelta(days=30)).isoformat()
|
|
rel = _downloaded(conn, channel, media_root, "v1", old)
|
|
season_dir = (media_root / rel).parent
|
|
bystander = season_dir / "unrelated file.txt"
|
|
bystander.write_text("keep me")
|
|
|
|
reap.run(conn, settings)
|
|
assert bystander.exists()
|
|
|
|
|
|
class TestDiskCap:
|
|
def test_disabled_by_default(self, conn, settings, channel, media_root):
|
|
_downloaded(conn, channel, media_root, "v1", util.today().isoformat(), size=10**6)
|
|
assert reap.disk_cap_evictions(conn, settings) == []
|
|
|
|
def test_evicts_oldest_first_until_under_the_cap(
|
|
self, conn, settings, channel, media_root
|
|
):
|
|
gigabyte = 1024**3
|
|
for index, day in enumerate((10, 5, 1)):
|
|
_downloaded(
|
|
conn,
|
|
channel,
|
|
media_root,
|
|
f"v{index}",
|
|
(util.today() - timedelta(days=day)).isoformat(),
|
|
size=gigabyte,
|
|
)
|
|
settings.set("disk_cap_gb", "2")
|
|
|
|
evicted = [row["video_id"] for row in reap.disk_cap_evictions(conn, settings)]
|
|
assert evicted == ["v0"]
|
|
|
|
def test_nothing_evicted_when_under_the_cap(self, conn, settings, channel, media_root):
|
|
_downloaded(conn, channel, media_root, "v1", util.today().isoformat(), size=1024)
|
|
settings.set("disk_cap_gb", "5")
|
|
assert reap.disk_cap_evictions(conn, settings) == []
|
|
|
|
|
|
class TestEffectiveRetention:
|
|
def test_override_wins(self, settings):
|
|
assert reap.effective_retention(settings, 60) == 60
|
|
|
|
def test_none_falls_back_to_global(self, settings):
|
|
assert reap.effective_retention(settings, None) == 14
|
|
|
|
def test_zero_falls_back_to_global(self, settings):
|
|
assert reap.effective_retention(settings, 0) == 14
|