Tom FluxandClaude Opus 5
155f05773d
Phases 1-4 of plan.md §13. Forked from youtube-automate as planned rather than
written from scratch: naming, NFO, auth, the admin UI, settings and the DB layer
came across largely unchanged, download.py is gone, and the pieces that only make
sense for a streaming library are new.
api.py YouTube Data API v3 client. The whole metadata path.
strm.py Materialising: a .strm, an .nfo and a thumbnail. Replaces the
330-line download.py, because the job is writing a URL to a file.
subsync.py The subscription mirror, most of which is refusals.
reap.py Retention, rewritten around the 30-day window and min_keep_videos.
discovery.py RSS polling plus an API-backed, resumable, bounded backfill.
proxy/ The verified PoC, moved in with a systemd unit.
330 tests, all passing, no network and no yt-dlp in any of them. The suite leans
towards the failure paths, because that is where this design can actually hurt
someone: a 403 that looks like an unsubscribe, a video that ages out and comes
back, a title that never arrives. tests/test_proxy.py replaces the two standalone
scripts under proxy/ and now drives the real make_handler(mgr, ...) rather than the
PoC's make_handler(path, done), so routing and video-id validation are covered too.
Ran it end to end against the live API and it found three real bugs.
The first was mine and the tests caught it: strm.remove pruned empty directories
up to the media root, so a channel directory whose tvshow.nfo happened to be
missing would be deleted along with the season. It only looked safe because
tvshow.nfo normally stops the walk. The prune boundary is now the channel
directory explicitly.
The other two only showed up against real data, and they compounded. The backfill
inserted rows with no title and left the RSS poll to fill them in — but RSS returns
15 entries, which for Pitch Side spans 23 days against a 30-day window, so five of
twenty episodes were named after their video ids. Worse, strm.materialise wrote
that fallback back to the database as the title, which made the row look titled and
permanently disabled the repair path. Both fixed: playlistItems.list now requests
snippet as well as contentDetails, which costs the same single quota unit and
carries the title alongside the exact publish date, and the fallback is used for the
filename without being persisted. A title that does arrive late now also removes the
badly-named files and re-queues, so the episode is rewritten rather than keeping its
video-id name forever. Verified against the live API: all twenty Pitch Side episodes
now carry real titles.
Measured on the real account: 119 subscriptions queued for approval and none added
on the first sync, then a two-channel run backfilled and materialised 26 episodes in
under seven seconds.
Two deliberate departures from plan.md, both recorded there:
min_keep_videos defaults to 5 rather than being left as an open question. Without
it 52 of 117 measured channels are empty Jellyfin series that flicker in and out as
their single video crosses the retention line, and the plan already recommended it.
The Jellyfin refresh is a bare /Library/Refresh with a comment explaining why it
must stay that way. A normal scan makes zero media probes; FullRefresh does probe,
and at 400 episodes that is 400 cold starts.
Not yet done: no systemd units are installed (needs root — deploy/deploy.sh), the
admin UI has no routes for sources or the approval queue yet, and nothing has been
pointed at the real media root.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
392 lines
14 KiB
Python
392 lines
14 KiB
Python
"""Discovery: feed parsing, the API backfill, and duration enrichment."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from datetime import date, timedelta
|
|
|
|
import pytest
|
|
|
|
from ytstream import api, discovery, util, videos
|
|
|
|
from conftest import FakeApi, add_video, make_feed, patch_api
|
|
|
|
|
|
def _entry(video_id, published, title="A video"):
|
|
return {"video_id": video_id, "published": published, "title": title}
|
|
|
|
|
|
@pytest.fixture()
|
|
def offline(monkeypatch):
|
|
"""No feed fetches and no API calls unless a test asks for them."""
|
|
monkeypatch.setattr(discovery, "fetch_feed",
|
|
lambda *a, **k: pytest.fail("unexpected feed fetch"))
|
|
patch_api(monkeypatch, discovery, FakeApi())
|
|
|
|
|
|
# --------------------------------------------------------------- feed parsing
|
|
|
|
|
|
def test_parse_entries_reads_only_atom_entries():
|
|
"""The feed-level <published> is the playlist's creation date, sometimes years
|
|
old. Treating it as a video produced a nonsense 0.01/day upload rate once."""
|
|
payload = make_feed(
|
|
[_entry("vid00000001", "2026-08-11T10:00:00+00:00")],
|
|
playlist_published="2019-03-01T00:00:00+00:00",
|
|
)
|
|
|
|
entries = discovery.parse_entries(payload)
|
|
|
|
assert len(entries) == 1
|
|
assert entries[0]["published"] == date(2026, 8, 11)
|
|
|
|
|
|
def test_parse_entries_keeps_the_exact_timestamp():
|
|
payload = make_feed([_entry("vid00000001", "2026-08-11T16:32:10+00:00")])
|
|
assert discovery.parse_entries(payload)[0]["published_at"].startswith(
|
|
"2026-08-11T16:32:10"
|
|
)
|
|
|
|
|
|
def test_parse_entries_skips_undated_entries():
|
|
payload = make_feed([_entry("vid00000001", "not-a-date"),
|
|
_entry("vid00000002", "2026-08-11T10:00:00+00:00")])
|
|
assert [e["video_id"] for e in discovery.parse_entries(payload)] == ["vid00000002"]
|
|
|
|
|
|
def test_parse_entries_rejects_garbage():
|
|
with pytest.raises(discovery.FeedUnavailable):
|
|
discovery.parse_entries(b"<not xml")
|
|
|
|
|
|
def test_empty_feed_is_not_an_error():
|
|
assert discovery.parse_entries(make_feed([])) == []
|
|
|
|
|
|
def test_feed_urls_use_the_long_form_playlist():
|
|
url = discovery.uulf_feed_url("UCjCJ2LaOIsPzOoXUTMDI3wg")
|
|
assert "playlist_id=UULFjCJ2LaOIsPzOoXUTMDI3wg" in url
|
|
assert "channel_id=UCjCJ2LaOIsPzOoXUTMDI3wg" in discovery.uc_feed_url(
|
|
"UCjCJ2LaOIsPzOoXUTMDI3wg"
|
|
)
|
|
|
|
|
|
# -------------------------------------------------------------------- polling
|
|
|
|
|
|
def _install_feed(monkeypatch, entries, *, uulf=True):
|
|
payload = make_feed(entries)
|
|
|
|
def fetch(url, timeout=30.0):
|
|
if "playlist_id=UULF" in url:
|
|
return payload if uulf else None
|
|
return payload if not uulf else None
|
|
|
|
monkeypatch.setattr(discovery, "fetch_feed", fetch)
|
|
|
|
|
|
def test_poll_queues_videos_inside_the_window(conn, settings, channel, monkeypatch):
|
|
today = util.today()
|
|
_install_feed(monkeypatch, [_entry("vid00000001", f"{today}T10:00:00+00:00")])
|
|
patch_api(monkeypatch, discovery,
|
|
FakeApi(durations={"vid00000001": {"duration": 900, "is_live": False}}))
|
|
|
|
stats = discovery.poll_channel(conn, settings, channel)
|
|
|
|
assert stats["queued"] == 1
|
|
assert stats["source"] == videos.SOURCE_UULF
|
|
row = videos.get(conn, "vid00000001")
|
|
assert row["state"] == videos.LISTED
|
|
assert row["duration"] == 900
|
|
|
|
|
|
def test_poll_marks_older_videos_skipped_old(conn, settings, channel, monkeypatch):
|
|
old = util.today() - timedelta(days=90)
|
|
_install_feed(monkeypatch, [_entry("vid00000001", f"{old}T10:00:00+00:00")])
|
|
patch_api(monkeypatch, discovery, FakeApi())
|
|
|
|
stats = discovery.poll_channel(conn, settings, channel)
|
|
|
|
assert stats["old"] == 1
|
|
assert videos.get(conn, "vid00000001")["state"] == videos.SKIPPED_OLD
|
|
|
|
|
|
def test_poll_falls_back_to_the_channel_feed(conn, settings, channel, monkeypatch):
|
|
today = util.today()
|
|
_install_feed(monkeypatch, [_entry("vid00000001", f"{today}T10:00:00+00:00")],
|
|
uulf=False)
|
|
patch_api(monkeypatch, discovery, FakeApi())
|
|
|
|
stats = discovery.poll_channel(conn, settings, channel)
|
|
|
|
assert stats["source"] == videos.SOURCE_UC
|
|
assert videos.get(conn, "vid00000001")["discovery_source"] == videos.SOURCE_UC
|
|
|
|
|
|
def test_poll_records_feed_failure_without_raising(conn, settings, channel, monkeypatch):
|
|
def boom(url, timeout=30.0):
|
|
raise discovery.FeedUnavailable("HTTP 404")
|
|
|
|
monkeypatch.setattr(discovery, "fetch_feed", boom)
|
|
|
|
stats = discovery.poll_channel(conn, settings, channel)
|
|
|
|
assert "error" in stats
|
|
row = conn.execute("SELECT * FROM channel WHERE id = ?", (channel["id"],)).fetchone()
|
|
assert row["last_poll_ok"] == 0
|
|
assert row["consecutive_poll_failures"] == 1
|
|
|
|
|
|
def test_two_of_119_channels_failing_is_survivable(conn, settings, channel, monkeypatch):
|
|
"""Measured: terminated channels stay in the subscription list and 404 here."""
|
|
monkeypatch.setattr(discovery, "fetch_feed",
|
|
lambda *a, **k: (_ for _ in ()).throw(
|
|
discovery.FeedUnavailable("HTTP 404")))
|
|
totals = discovery.poll_all(conn, settings)
|
|
assert totals["failed"] == 1
|
|
assert totals["channels"] == 1
|
|
|
|
|
|
def test_poll_fills_in_a_missing_title(conn, settings, channel, monkeypatch):
|
|
"""Backfill inserts rows with no title; the feed is where titles come from."""
|
|
today = util.today()
|
|
add_video(conn, channel["id"], "vid00000001", title="",
|
|
upload_date=today.isoformat())
|
|
_install_feed(monkeypatch,
|
|
[_entry("vid00000001", f"{today}T10:00:00+00:00", "Real Title")])
|
|
patch_api(monkeypatch, discovery, FakeApi())
|
|
|
|
stats = discovery.poll_channel(conn, settings, channel)
|
|
|
|
assert stats["titled"] == 1
|
|
assert videos.get(conn, "vid00000001")["title"] == "Real Title"
|
|
|
|
|
|
def test_uulf_repairs_a_video_the_fallback_called_short(
|
|
conn, settings, channel, monkeypatch
|
|
):
|
|
today = util.today()
|
|
add_video(conn, channel["id"], "vid00000001", state=videos.SKIPPED_SHORT,
|
|
discovery_source=videos.SOURCE_UC, upload_date=today.isoformat())
|
|
_install_feed(monkeypatch, [_entry("vid00000001", f"{today}T10:00:00+00:00")])
|
|
patch_api(monkeypatch, discovery, FakeApi())
|
|
|
|
stats = discovery.poll_channel(conn, settings, channel)
|
|
|
|
assert stats["repaired"] == 1
|
|
row = videos.get(conn, "vid00000001")
|
|
assert row["state"] == videos.LISTED
|
|
assert row["discovery_source"] == videos.SOURCE_UULF
|
|
|
|
|
|
# ---------------------------------------------------------------- enrichment
|
|
|
|
|
|
def test_enrich_filters_shorts(conn, settings, channel, monkeypatch):
|
|
"""Measured: 38 of 50 consecutive uploads on a real channel were <=120s."""
|
|
add_video(conn, channel["id"], "short000001", duration=None)
|
|
patch_api(monkeypatch, discovery,
|
|
FakeApi(durations={"short000001": {"duration": 45, "is_live": False}}))
|
|
|
|
stats = discovery.enrich_durations(conn, settings, ["short000001"])
|
|
|
|
assert stats["shorts"] == 1
|
|
assert videos.get(conn, "short000001")["state"] == videos.SKIPPED_SHORT
|
|
|
|
|
|
def test_enrich_filters_livestreams_regardless_of_duration(
|
|
conn, settings, channel, monkeypatch
|
|
):
|
|
"""Live and upcoming both report PT0S, so duration cannot be the signal."""
|
|
add_video(conn, channel["id"], "live0000001", duration=None)
|
|
patch_api(monkeypatch, discovery,
|
|
FakeApi(durations={"live0000001": {"duration": 0, "is_live": True}}))
|
|
|
|
stats = discovery.enrich_durations(conn, settings, ["live0000001"])
|
|
|
|
assert stats["live"] == 1
|
|
assert videos.get(conn, "live0000001")["state"] == videos.SKIPPED_LIVE
|
|
|
|
|
|
def test_enrich_keeps_long_videos(conn, settings, channel, monkeypatch):
|
|
add_video(conn, channel["id"], "long0000001", duration=None)
|
|
patch_api(monkeypatch, discovery,
|
|
FakeApi(durations={"long0000001": {"duration": 2790, "is_live": False}}))
|
|
|
|
discovery.enrich_durations(conn, settings, ["long0000001"])
|
|
|
|
row = videos.get(conn, "long0000001")
|
|
assert row["state"] == videos.LISTED
|
|
assert row["duration"] == 2790
|
|
|
|
|
|
def test_enrich_survives_an_api_failure(conn, settings, channel, monkeypatch):
|
|
"""A NULL duration costs a runtime display, not a working library."""
|
|
add_video(conn, channel["id"], "vid00000001", duration=None)
|
|
|
|
class Failing(FakeApi):
|
|
def durations(self, ids):
|
|
raise api.ApiError(500, "backendError", "boom")
|
|
|
|
patch_api(monkeypatch, discovery, Failing())
|
|
|
|
stats = discovery.enrich_durations(conn, settings, ["vid00000001"])
|
|
|
|
assert stats["resolved"] == 0
|
|
assert videos.get(conn, "vid00000001")["state"] == videos.LISTED
|
|
|
|
|
|
def test_enrich_with_no_ids_makes_no_call(conn, settings, monkeypatch):
|
|
fake = patch_api(monkeypatch, discovery, FakeApi())
|
|
discovery.enrich_durations(conn, settings, [])
|
|
assert fake.calls == 0
|
|
|
|
|
|
# ------------------------------------------------------------------- backfill
|
|
|
|
|
|
def test_backfill_queues_the_window(conn, settings, channel, monkeypatch):
|
|
today = util.today()
|
|
uploads = [
|
|
({"video_id": f"vid{i:08d}", "published": today - timedelta(days=i),
|
|
"published_at": f"{today - timedelta(days=i)}T00:00:00Z"}, None)
|
|
for i in range(3)
|
|
]
|
|
patch_api(monkeypatch, discovery, FakeApi(
|
|
uploads=uploads,
|
|
durations={f"vid{i:08d}": {"duration": 900, "is_live": False}
|
|
for i in range(3)}))
|
|
|
|
stats = discovery.backfill_channel(conn, settings, channel)
|
|
|
|
assert stats["queued"] == 3
|
|
assert videos.get(conn, "vid00000000")["state"] == videos.LISTED
|
|
assert conn.execute("SELECT backfilled FROM channel WHERE id = ?",
|
|
(channel["id"],)).fetchone()[0] == 1
|
|
|
|
|
|
def test_backfill_stores_the_exact_publish_time(conn, settings, channel, monkeypatch):
|
|
today = util.today()
|
|
patch_api(monkeypatch, discovery, FakeApi(uploads=[
|
|
({"video_id": "vid00000001", "published": today,
|
|
"published_at": "2026-08-11T16:32:10Z"}, None)]))
|
|
|
|
discovery.backfill_channel(conn, settings, channel)
|
|
|
|
assert videos.get(conn, "vid00000001")["published_at"] == "2026-08-11T16:32:10Z"
|
|
|
|
|
|
def test_backfill_respects_the_video_cap(conn, settings, channel, monkeypatch):
|
|
settings.set("backfill_max_videos", "2")
|
|
today = util.today()
|
|
patch_api(monkeypatch, discovery, FakeApi(uploads=[
|
|
({"video_id": f"vid{i:08d}", "published": today, "published_at": None}, None)
|
|
for i in range(10)]))
|
|
|
|
stats = discovery.backfill_channel(conn, settings, channel)
|
|
|
|
assert stats["queued"] == 2
|
|
|
|
|
|
def test_backfill_clears_the_cursor_when_complete(conn, settings, channel, monkeypatch):
|
|
patch_api(monkeypatch, discovery, FakeApi(uploads=[
|
|
({"video_id": "vid00000001", "published": util.today(),
|
|
"published_at": None}, "TOKEN")]))
|
|
|
|
discovery.backfill_channel(conn, settings, channel)
|
|
|
|
assert conn.execute("SELECT backfill_cursor FROM channel WHERE id = ?",
|
|
(channel["id"],)).fetchone()[0] is None
|
|
|
|
|
|
def test_backfill_leaves_the_flag_unset_when_the_api_is_unusable(
|
|
conn, settings, channel, monkeypatch
|
|
):
|
|
"""So it retries once a key is configured, rather than silently never running."""
|
|
with conn:
|
|
conn.execute("UPDATE channel SET backfilled = 0 WHERE id = ?", (channel["id"],))
|
|
|
|
class Failing(FakeApi):
|
|
def uploads(self, *a, **kw):
|
|
raise api.NotConfigured(403, "forbidden", "blocked")
|
|
yield # pragma: no cover
|
|
|
|
patch_api(monkeypatch, discovery, Failing())
|
|
|
|
stats = discovery.backfill_channel(conn, settings, channel)
|
|
|
|
assert "error" in stats
|
|
assert conn.execute("SELECT backfilled FROM channel WHERE id = ?",
|
|
(channel["id"],)).fetchone()[0] == 0
|
|
|
|
|
|
def test_backfill_does_not_duplicate_known_videos(conn, settings, channel, monkeypatch):
|
|
add_video(conn, channel["id"], "vid00000001")
|
|
patch_api(monkeypatch, discovery, FakeApi(uploads=[
|
|
({"video_id": "vid00000001", "published": util.today(),
|
|
"published_at": None}, None)]))
|
|
|
|
stats = discovery.backfill_channel(conn, settings, channel)
|
|
|
|
assert stats["queued"] == 0
|
|
assert conn.execute("SELECT COUNT(*) FROM video").fetchone()[0] == 1
|
|
|
|
|
|
# ------------------------------------------------------- titles (regression)
|
|
|
|
|
|
def test_backfill_takes_the_title_from_the_api(conn, settings, channel, monkeypatch):
|
|
"""Not an optimisation. RSS returns 15 entries, which for a channel posting
|
|
under one long-form video a day reaches back only ~23 days against a 30-day
|
|
window — so the oldest ~5 of every 20-episode backfill was being named after
|
|
its video id."""
|
|
patch_api(monkeypatch, discovery, FakeApi(uploads=[
|
|
({"video_id": "vid00000001", "published": util.today(),
|
|
"published_at": None, "title": "A Real Title"}, None)]))
|
|
|
|
discovery.backfill_channel(conn, settings, channel)
|
|
|
|
assert videos.get(conn, "vid00000001")["title"] == "A Real Title"
|
|
|
|
|
|
def test_backfill_tolerates_a_missing_title(conn, settings, channel, monkeypatch):
|
|
patch_api(monkeypatch, discovery, FakeApi(uploads=[
|
|
({"video_id": "vid00000001", "published": util.today(),
|
|
"published_at": None}, None)]))
|
|
|
|
discovery.backfill_channel(conn, settings, channel)
|
|
|
|
assert videos.get(conn, "vid00000001")["title"] == ""
|
|
|
|
|
|
def test_a_late_title_renames_an_already_materialised_episode(
|
|
conn, settings, media_root, channel, monkeypatch
|
|
):
|
|
"""The repair has to move the file, not just the row — otherwise the episode
|
|
keeps its video-id filename forever."""
|
|
from ytstream import strm
|
|
|
|
monkeypatch.setattr(strm, "fetch_thumbnail", lambda *a, **k: False)
|
|
today = util.today()
|
|
add_video(conn, channel["id"], "vid00000001", title="",
|
|
upload_date=today.isoformat())
|
|
first = strm.materialise(conn, settings, channel,
|
|
videos.get(conn, "vid00000001"))
|
|
assert "vid00000001]" in first["rel_path"]
|
|
old_path = media_root / first["rel_path"]
|
|
assert old_path.exists()
|
|
|
|
outcome = discovery._record(
|
|
conn, channel,
|
|
{"video_id": "vid00000001", "title": "Proper Name", "published": today,
|
|
"published_at": None},
|
|
videos.SOURCE_UULF, today - timedelta(days=30),
|
|
)
|
|
|
|
assert outcome == "titled"
|
|
assert videos.get(conn, "vid00000001")["state"] == videos.LISTED
|
|
assert not old_path.exists()
|
|
|
|
second = strm.materialise(conn, settings, channel,
|
|
videos.get(conn, "vid00000001"))
|
|
assert "Proper Name" in second["rel_path"]
|