Files
ytstream/tests/test_discovery.py
Claude 22d8828080 Never serve a partial file: it makes Jellyfin transcode
The 12s first-byte grace was the wrong trade and a real play found it
within the hour. A 2-hour upload took 3 minutes to start, played 6
seconds, and stalled. Jellyfin had run ffmpeg with -probesize 1G against
the growing stream and then transcoded to HLS with libx264.

The cause is the container. A fragmented MP4 with empty_moov has no
duration in its header, so the only way to get one is to sum every
fragment -- probing a growing file reads all of it. Jellyfin cannot
establish duration, codec or bitrate, so it abandons direct play and
transcodes a stream it also cannot seek. It was targeting 4.83 Mbps
against a source measured at 3.29: re-encoding a stream that already fit,
because it could not measure it.

The same video once complete reports SupportsDirectPlay with the exact
runtime and bitrate.

So FIRST_BYTE_GRACE defaults to infinite again, with --wait-timeout raised
to 600s for a 2-hour upload. A cold long video is slow to start, which is
accepted: the fetch outlives the request so a retry is instant, and a
retryable stall beats a transcode that wastes a gigabyte and cannot work.
Both failure modes are recorded at the constant in the order measured so
the 12s cap is not reintroduced.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-13 11:32:12 +01:00

545 lines
20 KiB
Python

"""Discovery: feed parsing, the API backfill, and duration enrichment."""
from __future__ import annotations
from datetime import date, timedelta
import pytest
from ytstream import api, discovery, util, videos
from conftest import FakeApi, add_video, make_feed, patch_api
def _entry(video_id, published, title="A video"):
return {"video_id": video_id, "published": published, "title": title}
@pytest.fixture()
def offline(monkeypatch):
"""No feed fetches and no API calls unless a test asks for them."""
monkeypatch.setattr(discovery, "fetch_feed",
lambda *a, **k: pytest.fail("unexpected feed fetch"))
patch_api(monkeypatch, discovery, FakeApi())
# --------------------------------------------------------------- feed parsing
def test_parse_entries_reads_only_atom_entries():
"""The feed-level <published> is the playlist's creation date, sometimes years
old. Treating it as a video produced a nonsense 0.01/day upload rate once."""
payload = make_feed(
[_entry("vid00000001", "2026-08-11T10:00:00+00:00")],
playlist_published="2019-03-01T00:00:00+00:00",
)
entries = discovery.parse_entries(payload)
assert len(entries) == 1
assert entries[0]["published"] == date(2026, 8, 11)
def test_parse_entries_keeps_the_exact_timestamp():
payload = make_feed([_entry("vid00000001", "2026-08-11T16:32:10+00:00")])
assert discovery.parse_entries(payload)[0]["published_at"].startswith(
"2026-08-11T16:32:10"
)
def test_parse_entries_skips_undated_entries():
payload = make_feed([_entry("vid00000001", "not-a-date"),
_entry("vid00000002", "2026-08-11T10:00:00+00:00")])
assert [e["video_id"] for e in discovery.parse_entries(payload)] == ["vid00000002"]
def test_parse_entries_rejects_garbage():
with pytest.raises(discovery.FeedUnavailable):
discovery.parse_entries(b"<not xml")
def test_empty_feed_is_not_an_error():
assert discovery.parse_entries(make_feed([])) == []
def test_feed_urls_use_the_long_form_playlist():
url = discovery.uulf_feed_url("UCjCJ2LaOIsPzOoXUTMDI3wg")
assert "playlist_id=UULFjCJ2LaOIsPzOoXUTMDI3wg" in url
assert "channel_id=UCjCJ2LaOIsPzOoXUTMDI3wg" in discovery.uc_feed_url(
"UCjCJ2LaOIsPzOoXUTMDI3wg"
)
# -------------------------------------------------------------------- polling
def _install_feed(monkeypatch, entries, *, uulf=True):
payload = make_feed(entries)
def fetch(url, timeout=30.0):
if "playlist_id=UULF" in url:
return payload if uulf else None
return payload if not uulf else None
monkeypatch.setattr(discovery, "fetch_feed", fetch)
def test_poll_queues_videos_inside_the_window(conn, settings, channel, monkeypatch):
today = util.today()
_install_feed(monkeypatch, [_entry("vid00000001", f"{today}T10:00:00+00:00")])
patch_api(monkeypatch, discovery,
FakeApi(durations={"vid00000001": {"duration": 900, "is_live": False}}))
stats = discovery.poll_channel(conn, settings, channel)
assert stats["queued"] == 1
assert stats["source"] == videos.SOURCE_UULF
row = videos.get(conn, "vid00000001")
assert row["state"] == videos.LISTED
assert row["duration"] == 900
def test_poll_marks_older_videos_skipped_old(conn, settings, channel, monkeypatch):
old = util.today() - timedelta(days=90)
_install_feed(monkeypatch, [_entry("vid00000001", f"{old}T10:00:00+00:00")])
patch_api(monkeypatch, discovery, FakeApi())
stats = discovery.poll_channel(conn, settings, channel)
assert stats["old"] == 1
assert videos.get(conn, "vid00000001")["state"] == videos.SKIPPED_OLD
def test_poll_falls_back_to_the_channel_feed(conn, settings, channel, monkeypatch):
today = util.today()
_install_feed(monkeypatch, [_entry("vid00000001", f"{today}T10:00:00+00:00")],
uulf=False)
patch_api(monkeypatch, discovery, FakeApi())
stats = discovery.poll_channel(conn, settings, channel)
assert stats["source"] == videos.SOURCE_UC
assert videos.get(conn, "vid00000001")["discovery_source"] == videos.SOURCE_UC
def test_poll_records_feed_failure_without_raising(conn, settings, channel, monkeypatch):
def boom(url, timeout=30.0):
raise discovery.FeedUnavailable("HTTP 404")
monkeypatch.setattr(discovery, "fetch_feed", boom)
stats = discovery.poll_channel(conn, settings, channel)
assert "error" in stats
row = conn.execute("SELECT * FROM channel WHERE id = ?", (channel["id"],)).fetchone()
assert row["last_poll_ok"] == 0
assert row["consecutive_poll_failures"] == 1
def test_two_of_119_channels_failing_is_survivable(conn, settings, channel, monkeypatch):
"""Measured: terminated channels stay in the subscription list and 404 here."""
monkeypatch.setattr(discovery, "fetch_feed",
lambda *a, **k: (_ for _ in ()).throw(
discovery.FeedUnavailable("HTTP 404")))
totals = discovery.poll_all(conn, settings)
assert totals["failed"] == 1
assert totals["channels"] == 1
def test_poll_fills_in_a_missing_title(conn, settings, channel, monkeypatch):
"""Backfill inserts rows with no title; the feed is where titles come from."""
today = util.today()
add_video(conn, channel["id"], "vid00000001", title="",
upload_date=today.isoformat())
_install_feed(monkeypatch,
[_entry("vid00000001", f"{today}T10:00:00+00:00", "Real Title")])
patch_api(monkeypatch, discovery, FakeApi())
stats = discovery.poll_channel(conn, settings, channel)
assert stats["titled"] == 1
assert videos.get(conn, "vid00000001")["title"] == "Real Title"
def test_uulf_repairs_a_video_the_fallback_called_short(
conn, settings, channel, monkeypatch
):
today = util.today()
add_video(conn, channel["id"], "vid00000001", state=videos.SKIPPED_SHORT,
discovery_source=videos.SOURCE_UC, upload_date=today.isoformat())
_install_feed(monkeypatch, [_entry("vid00000001", f"{today}T10:00:00+00:00")])
patch_api(monkeypatch, discovery, FakeApi())
stats = discovery.poll_channel(conn, settings, channel)
assert stats["repaired"] == 1
row = videos.get(conn, "vid00000001")
assert row["state"] == videos.LISTED
assert row["discovery_source"] == videos.SOURCE_UULF
# ---------------------------------------------------------------- enrichment
def test_enrich_filters_shorts(conn, settings, channel, monkeypatch):
"""Measured: 38 of 50 consecutive uploads on a real channel were <=120s."""
add_video(conn, channel["id"], "short000001", duration=None)
patch_api(monkeypatch, discovery,
FakeApi(durations={"short000001": {"duration": 45, "is_live": False}}))
stats = discovery.enrich_durations(conn, settings, ["short000001"])
assert stats["shorts"] == 1
assert videos.get(conn, "short000001")["state"] == videos.SKIPPED_SHORT
def test_enrich_filters_livestreams_regardless_of_duration(
conn, settings, channel, monkeypatch
):
"""Live and upcoming both report PT0S, so duration cannot be the signal."""
add_video(conn, channel["id"], "live0000001", duration=None)
patch_api(monkeypatch, discovery,
FakeApi(durations={"live0000001": {"duration": 0, "is_live": True}}))
stats = discovery.enrich_durations(conn, settings, ["live0000001"])
assert stats["live"] == 1
assert videos.get(conn, "live0000001")["state"] == videos.SKIPPED_LIVE
def test_enrich_keeps_long_videos(conn, settings, channel, monkeypatch):
add_video(conn, channel["id"], "long0000001", duration=None)
patch_api(monkeypatch, discovery,
FakeApi(durations={"long0000001": {"duration": 2790, "is_live": False}}))
discovery.enrich_durations(conn, settings, ["long0000001"])
row = videos.get(conn, "long0000001")
assert row["state"] == videos.LISTED
assert row["duration"] == 2790
def test_enrich_survives_an_api_failure(conn, settings, channel, monkeypatch):
"""A NULL duration costs a runtime display, not a working library."""
add_video(conn, channel["id"], "vid00000001", duration=None)
class Failing(FakeApi):
def durations(self, ids):
raise api.ApiError(500, "backendError", "boom")
patch_api(monkeypatch, discovery, Failing())
stats = discovery.enrich_durations(conn, settings, ["vid00000001"])
assert stats["resolved"] == 0
assert videos.get(conn, "vid00000001")["state"] == videos.LISTED
def test_enrich_with_no_ids_makes_no_call(conn, settings, monkeypatch):
fake = patch_api(monkeypatch, discovery, FakeApi())
discovery.enrich_durations(conn, settings, [])
assert fake.calls == 0
# ------------------------------------------------------------------- backfill
def test_backfill_queues_the_window(conn, settings, channel, monkeypatch):
today = util.today()
uploads = [
({"video_id": f"vid{i:08d}", "published": today - timedelta(days=i),
"published_at": f"{today - timedelta(days=i)}T00:00:00Z"}, None)
for i in range(3)
]
patch_api(monkeypatch, discovery, FakeApi(
uploads=uploads,
durations={f"vid{i:08d}": {"duration": 900, "is_live": False}
for i in range(3)}))
stats = discovery.backfill_channel(conn, settings, channel)
assert stats["queued"] == 3
assert videos.get(conn, "vid00000000")["state"] == videos.LISTED
assert conn.execute("SELECT backfilled FROM channel WHERE id = ?",
(channel["id"],)).fetchone()[0] == 1
def test_backfill_stores_the_exact_publish_time(conn, settings, channel, monkeypatch):
today = util.today()
patch_api(monkeypatch, discovery, FakeApi(uploads=[
({"video_id": "vid00000001", "published": today,
"published_at": "2026-08-11T16:32:10Z"}, None)]))
discovery.backfill_channel(conn, settings, channel)
assert videos.get(conn, "vid00000001")["published_at"] == "2026-08-11T16:32:10Z"
def test_backfill_respects_the_video_cap(conn, settings, channel, monkeypatch):
settings.set("backfill_max_videos", "2")
today = util.today()
patch_api(monkeypatch, discovery, FakeApi(uploads=[
({"video_id": f"vid{i:08d}", "published": today, "published_at": None}, None)
for i in range(10)]))
stats = discovery.backfill_channel(conn, settings, channel)
assert stats["queued"] == 2
def test_backfill_clears_the_cursor_when_complete(conn, settings, channel, monkeypatch):
patch_api(monkeypatch, discovery, FakeApi(uploads=[
({"video_id": "vid00000001", "published": util.today(),
"published_at": None}, "TOKEN")]))
discovery.backfill_channel(conn, settings, channel)
assert conn.execute("SELECT backfill_cursor FROM channel WHERE id = ?",
(channel["id"],)).fetchone()[0] is None
def test_backfill_leaves_the_flag_unset_when_the_api_is_unusable(
conn, settings, channel, monkeypatch
):
"""So it retries once a key is configured, rather than silently never running."""
with conn:
conn.execute("UPDATE channel SET backfilled = 0 WHERE id = ?", (channel["id"],))
class Failing(FakeApi):
def uploads(self, *a, **kw):
raise api.NotConfigured(403, "forbidden", "blocked")
yield # pragma: no cover
patch_api(monkeypatch, discovery, Failing())
stats = discovery.backfill_channel(conn, settings, channel)
assert "error" in stats
assert conn.execute("SELECT backfilled FROM channel WHERE id = ?",
(channel["id"],)).fetchone()[0] == 0
def test_backfill_does_not_duplicate_known_videos(conn, settings, channel, monkeypatch):
add_video(conn, channel["id"], "vid00000001")
patch_api(monkeypatch, discovery, FakeApi(uploads=[
({"video_id": "vid00000001", "published": util.today(),
"published_at": None}, None)]))
stats = discovery.backfill_channel(conn, settings, channel)
assert stats["queued"] == 0
assert conn.execute("SELECT COUNT(*) FROM video").fetchone()[0] == 1
# ------------------------------------------------------- titles (regression)
def test_backfill_takes_the_title_from_the_api(conn, settings, channel, monkeypatch):
"""Not an optimisation. RSS returns 15 entries, which for a channel posting
under one long-form video a day reaches back only ~23 days against a 30-day
window — so the oldest ~5 of every 20-episode backfill was being named after
its video id."""
patch_api(monkeypatch, discovery, FakeApi(uploads=[
({"video_id": "vid00000001", "published": util.today(),
"published_at": None, "title": "A Real Title"}, None)]))
discovery.backfill_channel(conn, settings, channel)
assert videos.get(conn, "vid00000001")["title"] == "A Real Title"
def test_backfill_tolerates_a_missing_title(conn, settings, channel, monkeypatch):
patch_api(monkeypatch, discovery, FakeApi(uploads=[
({"video_id": "vid00000001", "published": util.today(),
"published_at": None}, None)]))
discovery.backfill_channel(conn, settings, channel)
assert videos.get(conn, "vid00000001")["title"] == ""
def test_a_late_title_renames_an_already_materialised_episode(
conn, settings, media_root, channel, monkeypatch
):
"""The repair has to move the file, not just the row — otherwise the episode
keeps its video-id filename forever."""
from ytstream import strm
monkeypatch.setattr(strm, "fetch_thumbnail", lambda *a, **k: False)
today = util.today()
add_video(conn, channel["id"], "vid00000001", title="",
upload_date=today.isoformat())
first = strm.materialise(conn, settings, channel,
videos.get(conn, "vid00000001"))
assert "vid00000001]" in first["rel_path"]
old_path = media_root / first["rel_path"]
assert old_path.exists()
outcome = discovery._record(
conn, channel,
{"video_id": "vid00000001", "title": "Proper Name", "published": today,
"published_at": None},
videos.SOURCE_UULF, today - timedelta(days=30),
)
assert outcome == "titled"
assert videos.get(conn, "vid00000001")["state"] == videos.LISTED
assert not old_path.exists()
second = strm.materialise(conn, settings, channel,
videos.get(conn, "vid00000001"))
assert "Proper Name" in second["rel_path"]
def test_reclassifying_a_materialised_video_removes_its_files(
conn, settings, media_root, channel, monkeypatch
):
"""A premiere that becomes a livestream, or a duration that only resolves on a
later run, would otherwise leave files on disk with no row owning them."""
from ytstream import strm
monkeypatch.setattr(strm, "fetch_thumbnail", lambda *a, **k: False)
add_video(conn, channel["id"], "vid00000001", duration=None)
result = strm.materialise(conn, settings, channel,
videos.get(conn, "vid00000001"))
path = media_root / result["rel_path"]
assert path.exists()
patch_api(monkeypatch, discovery,
FakeApi(durations={"vid00000001": {"duration": 30, "is_live": False,
"title": "", "description": ""}}))
stats = discovery.enrich_durations(conn, settings, ["vid00000001"])
assert stats["shorts"] == 1
assert videos.get(conn, "vid00000001")["state"] == videos.SKIPPED_SHORT
assert not path.exists()
# ------------------------------------------------- the min_keep_videos floor
#
# §5 defines retention as max(retention_days, min_keep_videos newest) and gives
# the reason: a channel that uploads every six weeks has nothing inside a 30-day
# window and would appear in Jellyfin as an empty series. reap.py implemented the
# second half; discovery did not, so it only protected videos that had already
# been materialised. Measured on the real 119 subscriptions: 441 episodes but 57
# channels with none, 55 of them holding skipped_old rows.
def _old(conn, channel, count, *, start=1):
"""`count` skipped_old videos, newest first at 2026-01-{start}..."""
made = []
for index in range(count):
made.append(add_video(
conn, channel["id"], f"old{index + start:08d}",
upload_date=f"2026-01-{index + start:02d}",
state=videos.SKIPPED_OLD,
))
return made
def test_a_channel_with_nothing_in_the_window_keeps_its_newest(conn, settings, channel):
_old(conn, channel, 8)
promoted = discovery.top_up_to_min_keep(conn, settings, channel)
assert promoted == 5
listed = conn.execute(
"SELECT video_id FROM video WHERE state = ? ORDER BY upload_date DESC",
(videos.LISTED,),
).fetchall()
# The five NEWEST, not the first five found.
assert [row["video_id"] for row in listed] == [
"old00000008", "old00000007", "old00000006", "old00000005", "old00000004",
]
def test_the_floor_counts_what_is_already_there(conn, settings, channel):
"""A channel with 3 in-window videos needs only 2 older ones."""
for index in range(3):
add_video(conn, channel["id"], f"new{index:08d}", upload_date="2026-08-10")
_old(conn, channel, 6)
assert discovery.top_up_to_min_keep(conn, settings, channel) == 2
def test_a_busy_channel_is_untouched(conn, settings, channel):
for index in range(9):
add_video(conn, channel["id"], f"new{index:08d}", upload_date="2026-08-10")
_old(conn, channel, 4)
assert discovery.top_up_to_min_keep(conn, settings, channel) == 0
assert conn.execute(
"SELECT count(*) FROM video WHERE state = ?", (videos.SKIPPED_OLD,)
).fetchone()[0] == 4
def test_topping_up_is_idempotent(conn, settings, channel):
"""It runs every poll, every hour. Twice must not mean ten episodes."""
_old(conn, channel, 8)
first = discovery.top_up_to_min_keep(conn, settings, channel)
second = discovery.top_up_to_min_keep(conn, settings, channel)
assert (first, second) == (5, 0)
assert conn.execute(
"SELECT count(*) FROM video WHERE state = ?", (videos.LISTED,)
).fetchone()[0] == 5
def test_aged_out_videos_are_never_revived(conn, settings, channel):
"""They were on disk and were deleted. Reviving them presents months of old
episodes to Jellyfin as new, which is what the tombstone exists to stop."""
for index in range(6):
add_video(conn, channel["id"], f"gone{index:08d}",
upload_date=f"2026-02-{index + 1:02d}", state=videos.AGED_OUT)
assert discovery.top_up_to_min_keep(conn, settings, channel) == 0
assert conn.execute(
"SELECT count(*) FROM video WHERE state = ?", (videos.AGED_OUT,)
).fetchone()[0] == 6
def test_shorts_and_livestreams_do_not_count_towards_the_floor(conn, settings, channel):
"""They are excluded by policy and can never be episodes, so a channel whose
newest uploads are all Shorts must reach further back for long-form ones."""
for index in range(4):
add_video(conn, channel["id"], f"shrt{index:08d}",
upload_date="2026-08-11", state=videos.SKIPPED_SHORT)
add_video(conn, channel["id"], "live0000001",
upload_date="2026-08-11", state=videos.SKIPPED_LIVE)
_old(conn, channel, 7)
assert discovery.top_up_to_min_keep(conn, settings, channel) == 5
def test_a_floor_of_zero_disables_it(conn, settings, channel):
settings.set("min_keep_videos", "0")
_old(conn, channel, 6)
assert discovery.top_up_to_min_keep(conn, settings, channel) == 0
def test_a_channel_with_no_videos_at_all_stays_empty(conn, settings, channel):
"""Two of the real 119 have no long-form uploads whatsoever (their UULF
playlist 404s). There is nothing to promote and no directory should appear."""
assert discovery.top_up_to_min_keep(conn, settings, channel) == 0
def test_promoted_videos_survive_the_next_reap(conn, settings, channel):
"""The whole point: the two halves of max(window, N newest) must agree. If
reap deleted what discovery just promoted, channels would flicker hourly."""
from ytstream import reap
_old(conn, channel, 5)
discovery.top_up_to_min_keep(conn, settings, channel)
for row in conn.execute(
"SELECT video_id FROM video WHERE state = ?", (videos.LISTED,)
).fetchall():
videos.mark_materialised(
conn, row["video_id"], rel_path=f"c/{row['video_id']}.strm",
season=2026, episode=1, upload_date="2026-01-01", duration=900,
title="t",
)
assert reap.candidates(conn, settings) == []