Initial implementation of youtube-automate

A DVR for YouTube subscriptions, laid out so Jellyfin presents each channel as a show and each video as an episode. Cron-driven, idempotent, with a public admin UI for a non-operator. Verified end to end on susan against three real channels: PO tokens, h264 downloads, Jellyfin resolution from local NFOs with all providers disabled, retention and tombstones. Corrections to the original design handover (specs.md documents each with the evidence, and specs.handover-original.md preserves the original): - The format sort selected 360p. Ranking acodec above res makes `bv*` prefer the combined 360p stream, which carries AAC, over the 720p video-only stream whose acodec is none. vcodec now leads, so a video without h264 at 720p yields h264 lower down rather than VP9 this hardware cannot transcode. - yt-dlp now requires a JS runtime and the yt-dlp-ejs solver scripts, which only ship with the [default] extra. Without them the n challenge fails and the mweb formats disappear entirely. - --flat-playlist carries no upload dates, so the specced client-side date filter for backfill was impossible. Backfill is RSS-first. - skipped_old was terminal, so raising a channel's retention appeared to do nothing. Added an explicit rescan. - is_upcoming premieres now defer and retry instead of being skipped forever. - TubeArchivist is gone, so the media root and the tube.jihakuz.xyz vhost were both reclaimed; the latter still pointed at its dead port. 240 offline tests, no network and no real yt-dlp invocation. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-11 21:42:48 +01:00
commit 18bb2e420b
44 changed files with 7188 additions and 0 deletions
@@ -0,0 +1,186 @@
+"""Retention: candidate selection, artefact deletion, pruning, disk cap."""
+
+from datetime import timedelta
+
+from conftest import add_video
+from youtube_automate import config, reap, util, videos
+
+
+def _place(media_root, channel_dir, season, stem):
+    """Create a downloaded video's full set of artefacts on disk."""
+    season_dir = media_root / channel_dir / f"Season {season}"
+    season_dir.mkdir(parents=True, exist_ok=True)
+    (season_dir / f"{stem}.mp4").write_bytes(b"video-bytes")
+    (season_dir / f"{stem}.nfo").write_text("<episodedetails/>")
+    (season_dir / f"{stem}.info.json").write_text("{}")
+    (season_dir / f"{stem}-thumb.jpg").write_bytes(b"jpg")
+    (season_dir / f"{stem}.en.srt").write_text("1\n")
+    return f"{channel_dir}/Season {season}/{stem}.mp4"
+
+
+def _downloaded(conn, channel, media_root, video_id, upload_date, size=1024):
+    """Place artefacts and record the row.
+
+    `size` is only ever read back out of the database (the disk-cap arithmetic
+    uses `size_bytes`), so the files on disk stay tiny no matter how large a
+    size the test claims.
+    """
+    stem = f"clabretro - S2026E8110 - Title [{video_id}]"
+    rel = _place(media_root, channel["dir_name"], 2026, stem)
+    add_video(conn, channel["id"], video_id, upload_date=upload_date)
+    videos.mark_downloaded(
+        conn, video_id, rel_path=rel, size_bytes=size, season=2026, episode=8110,
+        upload_date=upload_date, duration=100, title="Title",
+    )
+    return rel
+
+
+class TestCandidates:
+    def test_selects_only_videos_past_the_window(self, conn, settings, channel, media_root):
+        fresh = (util.today() - timedelta(days=3)).isoformat()
+        stale = (util.today() - timedelta(days=30)).isoformat()
+        _downloaded(conn, channel, media_root, "fresh", fresh)
+        _downloaded(conn, channel, media_root, "stale", stale)
+
+        due = [row["video_id"] for row in reap.candidates(conn, settings)]
+        assert due == ["stale"]
+
+    def test_per_channel_override_widens_the_window(
+        self, conn, settings, channel, media_root
+    ):
+        age = (util.today() - timedelta(days=30)).isoformat()
+        _downloaded(conn, channel, media_root, "v1", age)
+        assert len(reap.candidates(conn, settings)) == 1
+
+        with conn:
+            conn.execute(
+                "UPDATE channel SET retention_days = 60 WHERE id = ?", (channel["id"],)
+            )
+        assert reap.candidates(conn, settings) == []
+
+    def test_per_channel_override_narrows_the_window(
+        self, conn, settings, channel, media_root
+    ):
+        age = (util.today() - timedelta(days=5)).isoformat()
+        _downloaded(conn, channel, media_root, "v1", age)
+        assert reap.candidates(conn, settings) == []
+
+        with conn:
+            conn.execute(
+                "UPDATE channel SET retention_days = 2 WHERE id = ?", (channel["id"],)
+            )
+        assert len(reap.candidates(conn, settings)) == 1
+
+    def test_ignores_videos_that_are_not_downloaded(self, conn, settings, channel):
+        old = (util.today() - timedelta(days=99)).isoformat()
+        add_video(conn, channel["id"], "p", upload_date=old, state=videos.PENDING)
+        add_video(conn, channel["id"], "d", upload_date=old, state=videos.DELETED)
+        assert reap.candidates(conn, settings) == []
+
+
+class TestDeletion:
+    def test_removes_every_sidecar(self, conn, settings, channel, media_root, monkeypatch):
+        monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
+        old = (util.today() - timedelta(days=30)).isoformat()
+        rel = _downloaded(conn, channel, media_root, "v1", old)
+        season_dir = (media_root / rel).parent
+
+        reap.run(conn, settings)
+        assert not season_dir.exists()
+
+    def test_leaves_a_tombstone(self, conn, settings, channel, media_root, monkeypatch):
+        monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
+        old = (util.today() - timedelta(days=30)).isoformat()
+        _downloaded(conn, channel, media_root, "v1", old)
+
+        reap.run(conn, settings)
+        row = videos.get(conn, "v1")
+        assert row["state"] == videos.DELETED
+        assert row["rel_path"] is None
+
+    def test_keeps_the_channel_directory_and_its_artwork(
+        self, conn, settings, channel, media_root, monkeypatch
+    ):
+        """Deleting the channel dir would make an active subscription vanish
+        from Jellyfin and reappear later."""
+        monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
+        channel_dir = media_root / channel["dir_name"]
+        channel_dir.mkdir(parents=True, exist_ok=True)
+        (channel_dir / "tvshow.nfo").write_text("<tvshow/>")
+        (channel_dir / "poster.jpg").write_bytes(b"jpg")
+
+        old = (util.today() - timedelta(days=30)).isoformat()
+        _downloaded(conn, channel, media_root, "v1", old)
+        reap.run(conn, settings)
+
+        assert channel_dir.is_dir()
+        assert (channel_dir / "tvshow.nfo").exists()
+        assert (channel_dir / "poster.jpg").exists()
+
+    def test_does_not_prune_a_season_that_still_has_videos(
+        self, conn, settings, channel, media_root, monkeypatch
+    ):
+        monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
+        old = (util.today() - timedelta(days=30)).isoformat()
+        fresh = (util.today() - timedelta(days=1)).isoformat()
+        rel = _downloaded(conn, channel, media_root, "old1", old)
+        _downloaded(conn, channel, media_root, "new1", fresh)
+
+        reap.run(conn, settings)
+        season_dir = (media_root / rel).parent
+        assert season_dir.is_dir()
+        assert list(season_dir.glob("*new1*"))
+        assert not list(season_dir.glob("*old1*"))
+
+    def test_only_deletes_files_matching_the_stem(
+        self, conn, settings, channel, media_root, monkeypatch
+    ):
+        monkeypatch.setattr(config, "MEDIA_ROOT", media_root)
+        old = (util.today() - timedelta(days=30)).isoformat()
+        rel = _downloaded(conn, channel, media_root, "v1", old)
+        season_dir = (media_root / rel).parent
+        bystander = season_dir / "unrelated file.txt"
+        bystander.write_text("keep me")
+
+        reap.run(conn, settings)
+        assert bystander.exists()
+
+
+class TestDiskCap:
+    def test_disabled_by_default(self, conn, settings, channel, media_root):
+        _downloaded(conn, channel, media_root, "v1", util.today().isoformat(), size=10**6)
+        assert reap.disk_cap_evictions(conn, settings) == []
+
+    def test_evicts_oldest_first_until_under_the_cap(
+        self, conn, settings, channel, media_root
+    ):
+        gigabyte = 1024**3
+        for index, day in enumerate((10, 5, 1)):
+            _downloaded(
+                conn,
+                channel,
+                media_root,
+                f"v{index}",
+                (util.today() - timedelta(days=day)).isoformat(),
+                size=gigabyte,
+            )
+        settings.set("disk_cap_gb", "2")
+
+        evicted = [row["video_id"] for row in reap.disk_cap_evictions(conn, settings)]
+        assert evicted == ["v0"]
+
+    def test_nothing_evicted_when_under_the_cap(self, conn, settings, channel, media_root):
+        _downloaded(conn, channel, media_root, "v1", util.today().isoformat(), size=1024)
+        settings.set("disk_cap_gb", "5")
+        assert reap.disk_cap_evictions(conn, settings) == []
+
+
+class TestEffectiveRetention:
+    def test_override_wins(self, settings):
+        assert reap.effective_retention(settings, 60) == 60
+
+    def test_none_falls_back_to_global(self, settings):
+        assert reap.effective_retention(settings, None) == 14
+
+    def test_zero_falls_back_to_global(self, settings):
+        assert reap.effective_retention(settings, 0) == 14