Initial implementation of youtube-automate

A DVR for YouTube subscriptions, laid out so Jellyfin presents each channel as a show and each video as an episode. Cron-driven, idempotent, with a public admin UI for a non-operator. Verified end to end on susan against three real channels: PO tokens, h264 downloads, Jellyfin resolution from local NFOs with all providers disabled, retention and tombstones. Corrections to the original design handover (specs.md documents each with the evidence, and specs.handover-original.md preserves the original): - The format sort selected 360p. Ranking acodec above res makes `bv*` prefer the combined 360p stream, which carries AAC, over the 720p video-only stream whose acodec is none. vcodec now leads, so a video without h264 at 720p yields h264 lower down rather than VP9 this hardware cannot transcode. - yt-dlp now requires a JS runtime and the yt-dlp-ejs solver scripts, which only ship with the [default] extra. Without them the n challenge fails and the mweb formats disappear entirely. - --flat-playlist carries no upload dates, so the specced client-side date filter for backfill was impossible. Backfill is RSS-first. - skipped_old was terminal, so raising a channel's retention appeared to do nothing. Added an explicit rescan. - is_upcoming premieres now defer and retry instead of being skipped forever. - TubeArchivist is gone, so the media root and the tube.jihakuz.xyz vhost were both reclaimed; the latter still pointed at its dead port. 240 offline tests, no network and no real yt-dlp invocation. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-11 21:42:48 +01:00
commit 18bb2e420b
44 changed files with 7188 additions and 0 deletions
@@ -0,0 +1,264 @@
+"""Download worker: argument construction, rejection classification, moves."""
+
+import json
+
+from conftest import add_video
+from youtube_automate import config, download, videos
+
+
+class TestBuildArgs:
+    def _row(self, conn, channel, **kwargs):
+        add_video(conn, channel["id"], "vid1", **kwargs)
+        return conn.execute(
+            "SELECT v.*, c.dir_name, c.title AS channel_title, c.channel_id "
+            "FROM video v JOIN channel c ON c.id = v.channel_pk "
+            "WHERE v.video_id = 'vid1'"
+        ).fetchone()
+
+    def test_sort_puts_vcodec_before_res_and_res_before_acodec(
+        self, conn, settings, channel
+    ):
+        """The original spec ordering selected 360p — see specs.md §6."""
+        args = download.build_args(settings, self._row(conn, channel))
+        sort = args[args.index("-S") + 1]
+        assert sort == "vcodec:h264,res:720,acodec:aac"
+        assert sort.index("vcodec") < sort.index("res") < sort.index("acodec")
+
+    def test_format_selector_caps_height(self, conn, settings, channel):
+        args = download.build_args(settings, self._row(conn, channel))
+        assert args[args.index("-f") + 1] == "bv*[height<=720]+ba/b[height<=720]"
+
+    def test_max_height_setting_is_honoured(self, conn, settings, channel):
+        settings.set("max_height", "480")
+        args = download.build_args(settings, self._row(conn, channel))
+        assert "height<=480" in args[args.index("-f") + 1]
+        assert "res:480" in args[args.index("-S") + 1]
+
+    def test_merges_to_mp4(self, conn, settings, channel):
+        args = download.build_args(settings, self._row(conn, channel))
+        assert args[args.index("--merge-output-format") + 1] == "mp4"
+
+    def test_no_match_filter_for_uulf_rows(self, conn, settings, channel):
+        args = download.build_args(
+            settings, self._row(conn, channel, discovery_source=videos.SOURCE_UULF)
+        )
+        assert "--match-filter" not in args
+
+    def test_match_filter_applied_to_fallback_rows(self, conn, settings, channel):
+        args = download.build_args(
+            settings, self._row(conn, channel, discovery_source=videos.SOURCE_UC)
+        )
+        assert "--match-filter" in args
+        expression = args[args.index("--match-filter") + 1]
+        assert "duration>?120" in expression
+        # The `?` forms must be used so unknown values pass rather than reject.
+        assert "live_status!=?is_live" in expression
+        assert "live_status!=?is_upcoming" in expression
+        assert "!was_live" in expression
+
+    def test_min_duration_setting_flows_into_the_filter(self, conn, settings, channel):
+        settings.set("min_duration_seconds", "60")
+        args = download.build_args(
+            settings, self._row(conn, channel, discovery_source=videos.SOURCE_UC)
+        )
+        assert "duration>?60" in args[args.index("--match-filter") + 1]
+
+    def test_subtitles_can_be_disabled(self, conn, settings, channel):
+        settings.set("write_subs", "false")
+        args = download.build_args(settings, self._row(conn, channel))
+        assert "--write-subs" not in args
+
+    def test_sponsorblock_marks_rather_than_removes(self, conn, settings, channel):
+        args = download.build_args(settings, self._row(conn, channel))
+        assert "--sponsorblock-mark" in args
+        assert "--sponsorblock-remove" not in args
+        assert "--embed-chapters" in args
+
+    def test_targets_the_right_video(self, conn, settings, channel):
+        args = download.build_args(settings, self._row(conn, channel))
+        assert args[-1] == "https://www.youtube.com/watch?v=vid1"
+
+
+class TestRejectionClassification:
+    def test_not_a_rejection_when_no_marker(self):
+        assert download._classify_rejection({}, "downloading", "") is None
+
+    def test_upcoming_premiere_is_deferred_not_skipped(self):
+        outcome = download._classify_rejection(
+            {"live_status": "is_upcoming"}, "does not pass filter", ""
+        )
+        assert outcome == videos.DEFERRED
+
+    def test_live_is_skipped_permanently(self):
+        assert (
+            download._classify_rejection(
+                {"live_status": "is_live"}, "does not pass filter", ""
+            )
+            == videos.SKIPPED_LIVE
+        )
+
+    def test_past_livestream_is_skipped(self):
+        assert (
+            download._classify_rejection(
+                {"was_live": True}, "does not pass filter", ""
+            )
+            == videos.SKIPPED_LIVE
+        )
+
+    def test_otherwise_it_was_too_short(self):
+        assert (
+            download._classify_rejection({"duration": 30}, "does not pass filter", "")
+            == videos.SKIPPED_SHORT
+        )
+
+    def test_missing_info_json_still_classifies(self):
+        assert (
+            download._classify_rejection(None, "does not pass filter", "")
+            == videos.SKIPPED_SHORT
+        )
+
+
+class TestSubtitleChoice:
+    def test_prefers_plain_en(self, media_root):
+        (config.WORK_DIR / "v.en.srt").write_text("a")
+        (config.WORK_DIR / "v.en-orig.srt").write_text("b")
+        assert download._choose_subtitle("v").name == "v.en.srt"
+
+    def test_promotes_en_orig_when_alone(self, media_root):
+        (config.WORK_DIR / "v.en-orig.srt").write_text("b")
+        assert download._choose_subtitle("v").name == "v.en-orig.srt"
+
+    def test_none_when_no_subtitles(self, media_root):
+        assert download._choose_subtitle("v") is None
+
+
+class TestWorkDir:
+    def test_cleanup_removes_only_that_video(self, media_root):
+        (config.WORK_DIR / "keep.mp4").write_bytes(b"x")
+        (config.WORK_DIR / "drop.mp4").write_bytes(b"x")
+        (config.WORK_DIR / "drop.info.json").write_text("{}")
+
+        download.cleanup_work("drop")
+        assert (config.WORK_DIR / "keep.mp4").exists()
+        assert not (config.WORK_DIR / "drop.mp4").exists()
+        assert not (config.WORK_DIR / "drop.info.json").exists()
+
+    def test_recover_orphans_clears_everything_but_the_ignore_marker(self, media_root):
+        (config.WORK_DIR / "a.part").write_bytes(b"x")
+        (config.WORK_DIR / "b.mp4").write_bytes(b"x")
+
+        assert download.recover_orphans() == 2
+        assert [p.name for p in config.WORK_DIR.iterdir()] == [".ignore"]
+
+
+class TestMoveIntoPlace:
+    def _row(self, conn, channel):
+        return conn.execute(
+            "SELECT v.*, c.dir_name, c.title AS channel_title, c.channel_id "
+            "FROM video v JOIN channel c ON c.id = v.channel_pk "
+            "WHERE v.video_id = 'vid1'"
+        ).fetchone()
+
+    def _artefacts(self, video_id="vid1"):
+        (config.WORK_DIR / f"{video_id}.mp4").write_bytes(b"video-bytes")
+        (config.WORK_DIR / f"{video_id}.info.json").write_text("{}")
+        (config.WORK_DIR / f"{video_id}.jpg").write_bytes(b"jpg")
+        (config.WORK_DIR / f"{video_id}.en.srt").write_text("1\n")
+
+    def test_places_every_artefact_with_the_shared_stem(
+        self, conn, channel, media_root
+    ):
+        add_video(conn, channel["id"], "vid1", upload_date="2026-08-11")
+        self._artefacts()
+        info = {
+            "upload_date": "20260811",
+            "title": "A Title: with colon",
+            "description": "plot",
+            "duration": 600,
+        }
+
+        rel_path, size = download._move_into_place(conn, self._row(conn, channel), info)
+        placed = (media_root / rel_path).parent
+        stem = "clabretro - S2026E8110 - A Title with colon [vid1]"
+
+        assert {p.name for p in placed.iterdir()} == {
+            f"{stem}.mp4",
+            f"{stem}.nfo",
+            f"{stem}.info.json",
+            f"{stem}-thumb.jpg",
+            f"{stem}.en.srt",
+        }
+        assert size == len(b"video-bytes")
+
+    def test_work_dir_is_emptied_of_that_video(self, conn, channel, media_root):
+        add_video(conn, channel["id"], "vid1", upload_date="2026-08-11")
+        self._artefacts()
+        download._move_into_place(
+            conn, self._row(conn, channel), {"upload_date": "20260811", "title": "t"}
+        )
+        assert list(config.WORK_DIR.glob("vid1.*")) == []
+
+    def test_database_row_records_the_result(self, conn, channel, media_root):
+        add_video(conn, channel["id"], "vid1", upload_date="2026-08-11")
+        self._artefacts()
+        download._move_into_place(
+            conn,
+            self._row(conn, channel),
+            {"upload_date": "20260811", "title": "t", "duration": 600},
+        )
+        row = videos.get(conn, "vid1")
+        assert row["state"] == videos.DOWNLOADED
+        assert row["season"] == 2026
+        assert row["episode"] == 8110
+        assert row["size_bytes"] == len(b"video-bytes")
+        assert row["rel_path"].endswith(".mp4")
+
+    def test_info_json_upload_date_beats_the_feed_date(
+        self, conn, channel, media_root
+    ):
+        add_video(conn, channel["id"], "vid1", upload_date="2026-01-01")
+        self._artefacts()
+        download._move_into_place(
+            conn, self._row(conn, channel), {"upload_date": "20260811", "title": "t"}
+        )
+        row = videos.get(conn, "vid1")
+        assert row["upload_date"] == "2026-08-11"
+        assert row["episode"] == 8110
+
+    def test_missing_media_file_raises(self, conn, channel, media_root):
+        add_video(conn, channel["id"], "vid1", upload_date="2026-08-11")
+        (config.WORK_DIR / "vid1.info.json").write_text("{}")
+        try:
+            download._move_into_place(
+                conn, self._row(conn, channel), {"upload_date": "20260811", "title": "t"}
+            )
+        except FileNotFoundError:
+            pass
+        else:  # pragma: no cover
+            raise AssertionError("expected FileNotFoundError")
+
+    def test_written_nfo_matches_the_episode_number(self, conn, channel, media_root):
+        add_video(conn, channel["id"], "vid1", upload_date="2026-08-11")
+        self._artefacts()
+        rel_path, _ = download._move_into_place(
+            conn,
+            self._row(conn, channel),
+            {"upload_date": "20260811", "title": "t", "description": "d", "duration": 60},
+        )
+        nfo_path = (media_root / rel_path).with_suffix(".nfo")
+        content = nfo_path.read_text()
+        assert "<episode>8110</episode>" in content
+        assert "S2026E8110" in nfo_path.name
+
+
+class TestReadInfoJson:
+    def test_returns_none_when_absent(self, media_root):
+        assert download._read_info_json("nope") is None
+
+    def test_returns_none_on_corrupt_json(self, media_root):
+        (config.WORK_DIR / "v.info.json").write_text("{not json")
+        assert download._read_info_json("v") is None
+
+    def test_parses_valid_json(self, media_root):
+        (config.WORK_DIR / "v.info.json").write_text(json.dumps({"title": "x"}))
+        assert download._read_info_json("v")["title"] == "x"