From d3bf8d6f19a913bad6bb4531ae96489eb8e24c8a Mon Sep 17 00:00:00 2001 From: Tom Flux Date: Wed, 12 Aug 2026 17:21:35 +0100 Subject: [PATCH] Deploy it, and fix the six things installation found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both units are installed and running, 10 of 119 channels approved, 251 episodes live in Jellyfin with verified DirectPlay. 345 tests. Six problems surfaced that no test could have, and two of them were mine in the deploy scripts. deploy.sh had a circular dependency with bootstrap.sh: deploy started the units and told the operator to run bootstrap, but bootstrap refused to run until the state directory existed, which only deploy creates. The units started against a non-existent venv, failed 203/EXEC and restart-looped 17 and 21 times. deploy.sh now creates the directory, calls bootstrap itself through runuser so the venv is not left root-owned, and refuses to start units when the venv is still missing. Deno was absent, and `doctor` is the only reason we know. It is mandatory rather than nice-to-have — without a JS runtime yt-dlp cannot solve the n challenge, which youtube-automate measured on this machine as 22 formats instead of 29 plus throttling. Nothing else would have complained; playback would just have quietly degraded. bootstrap.sh now installs it and asserts yt-dlp reports it. Episodes had no synopsis at all, because materialise passed plot=None while both sources hand us descriptions for free. Now plumbed through from RSS (media:group/media:description) and from videos.list, which carries snippet.description in the call already being made for durations — so the ~40% of episodes older than RSS reaches get one too. That needed a schema v2 migration; v1 was left exactly as shipped so a fresh install and a migrated one are identical, and a test asserts it. `materialise --all` — the documented recovery from a Jellyfin metadata wipe — was itself creating duplicates. Episode numbers were re-derived each run, and next_episode() excludes the row being numbered, so re-materialising a day's videos in a different order renumbered them and orphaned the old files. One run left 102 orphaned NFOs against 251 episodes. An episode number is now permanent once assigned, and a video whose rel_path changes has its old files removed first. Running it twice is now a no-op. Two Jellyfin behaviours worth having in writing. It ignores and for episodes while reading the rest of the NFO happily, so a .strm shows no duration until first played — not fixable without probing, which is the one thing this design exists to avoid. And a plain /Library/Refresh does not reliably re-read a rewritten NFO: after rewriting all 251, fifty kept their old empty metadata. The fix is metadataRefreshMode=Default with replaceAllMetadata=false, which took plots from 201 to 251 while the proxy served zero requests. §5's prohibition on replaceAllMetadata=true still stands — that one probes. Exposed as `ytstream refresh-metadata` and run automatically after `materialise --all`. The measurement §5 has been waiting for: a full Jellyfin scan of 251 .strm files took ~119 s, about 8 minutes per 1,000 episodes, and made zero media probes. That last number is the fact the whole design rests on, now confirmed at scale on the real library rather than on seven PoC files. Co-Authored-By: Claude Opus 5 --- deploy/bootstrap.sh | 50 ++++++++++++++++-- deploy/deploy.sh | 31 +++++++++-- plan.md | 110 +++++++++++++++++++++++++++++++++++++--- tests/conftest.py | 6 ++- tests/test_discovery.py | 24 +++++++++ tests/test_strm.py | 52 +++++++++++++++++++ tests/test_videos.py | 72 ++++++++++++++++++++++++++ ytstream/api.py | 28 +++++++--- ytstream/cli.py | 30 +++++++++++ ytstream/db.py | 12 ++++- ytstream/discovery.py | 44 ++++++++++++++-- ytstream/jellyfin.py | 36 +++++++++++++ ytstream/strm.py | 25 +++++++-- ytstream/videos.py | 8 +-- 14 files changed, 490 insertions(+), 38 deletions(-) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 4d6a221..73ed82e 100755 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -1,5 +1,9 @@ #!/bin/bash -# Everything that does NOT need root. Run as susan, before deploy.sh. +# Everything that does NOT need root: the venv, yt-dlp, the POT container, the +# database. +# +# Normally invoked for you by deploy.sh, which creates the state directory first. +# Safe to run directly as susan afterwards — for instance to rebuild the venv: # # /opt/ytstream/deploy/bootstrap.sh set -euo pipefail @@ -11,8 +15,12 @@ REPO=/opt/ytstream say() { printf '\n\033[1m==> %s\033[0m\n' "$1"; } if [[ ! -d $STATE ]]; then - echo "$STATE does not exist yet — run 'sudo $REPO/deploy/deploy.sh' first," >&2 - echo "or create it with: sudo install -d -o susan -g automation -m 0770 $STATE" >&2 + echo "$STATE does not exist. Run 'sudo $REPO/deploy/deploy.sh' — it creates the" >&2 + echo "directory and then calls this script." >&2 + exit 1 +fi +if [[ ! -w $STATE ]]; then + echo "$STATE is not writable by $(id -un). Expected mode 0770 susan:automation." >&2 exit 1 fi @@ -32,6 +40,40 @@ print("POT plugin:", "MISSING" if missing else "present") raise SystemExit(1 if missing else 0) PY +say "Installing the Deno JS runtime" +# MANDATORY, not optional. Without a JS runtime yt-dlp cannot solve the `n` +# challenge, and youtube-automate measured the consequence on this machine: 22 +# formats instead of 29, and throttled downloads. yt-dlp finds it by PATH, and the +# unit pins PATH to this venv's bin, so it has to live here rather than anywhere +# more natural. +if [[ -x $VENV/bin/deno ]]; then + echo " already present: $("$VENV/bin/deno" --version | head -1)" +elif [[ -x /var/lib/youtube-automate/venv/bin/deno ]]; then + # Same machine, same architecture, already verified working. Prefer it over a + # download while youtube-automate is still installed. + install -m 0755 /var/lib/youtube-automate/venv/bin/deno "$VENV/bin/deno" + echo " copied from the youtube-automate venv: $("$VENV/bin/deno" --version | head -1)" +else + tmp=$(mktemp -d) + curl -fsSL -o "$tmp/deno.zip" \ + https://github.com/denoland/deno/releases/latest/download/deno-x86_64-unknown-linux-gnu.zip + unzip -q "$tmp/deno.zip" -d "$tmp" + install -m 0755 "$tmp/deno" "$VENV/bin/deno" + rm -rf "$tmp" + echo " installed $("$VENV/bin/deno" --version | head -1)" +fi + +say "Confirming yt-dlp can solve JS challenges" +# All three lines must appear, per youtube-automate specs.md §3. +if "$VENV/bin/yt-dlp" -v --simulate --no-warnings \ + "https://www.youtube.com/watch?v=dQw4w9WgXcQ" 2>&1 \ + | grep -q "JS runtimes: deno"; then + echo " yt-dlp reports the deno runtime" +else + echo " WARNING: yt-dlp did not report a deno JS runtime — playback may be" >&2 + echo " throttled or missing formats. Check: ytstream doctor" >&2 +fi + say "Starting the POT provider container if it is not already up" if ! curl -sf --max-time 5 http://127.0.0.1:4416/ping >/dev/null 2>&1; then docker run -d --name bgutil-provider --restart unless-stopped \ @@ -43,4 +85,4 @@ fi say "Initialising the database" "$VENV/bin/ytstream" status || true -say "Done — now run 'sudo $REPO/deploy/deploy.sh'" +say "Virtualenv ready at $VENV" diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 4d1ae35..6ae16b1 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -6,14 +6,21 @@ # # sudo /opt/ytstream/deploy/deploy.sh # -# Everything that does NOT need root — the venv, the database, the POT provider -# container, subscriptions, the API key — is handled by `bootstrap.sh` and the -# application itself. Run bootstrap.sh (as susan) first. +# This is the only command needed. It creates the state directory, builds the venv +# by calling bootstrap.sh as the service user, and only then installs and starts +# the units. +# +# That ordering is not cosmetic. An earlier version installed the units first and +# told the operator to run bootstrap.sh separately — but bootstrap.sh needs the +# state directory, which only this script can create, so neither could go first. +# The units started, failed 203/EXEC against a venv that did not exist yet, and +# restart-looped until the venv appeared. set -euo pipefail REPO=/opt/ytstream STATE=/var/lib/ytstream VENV=$STATE/venv +SERVICE_USER=susan HOSTNAME_=tube.jihakuz.xyz if [[ $EUID -ne 0 ]]; then @@ -26,7 +33,23 @@ say() { printf '\n\033[1m==> %s\033[0m\n' "$1"; } say "Creating $STATE" # root-owned directory, group-writable by `automation` so susan's cron job and the # admin server can both write the database. -install -d -o susan -g automation -m 0770 "$STATE" +install -d -o "$SERVICE_USER" -g automation -m 0770 "$STATE" + +say "Building the virtualenv" +if [[ -x $VENV/bin/ytstream ]]; then + echo " already present at $VENV" +else + # As the service user, so the venv is not left root-owned. Absolute path: + # runuser lives in /sbin, which is not always on the invoking PATH. + /sbin/runuser -u "$SERVICE_USER" -- bash "$REPO/deploy/bootstrap.sh" +fi + +# Refuse to start units that cannot possibly work. Restart=always would otherwise +# turn a missing venv into a restart loop in the journal. +if [[ ! -x $VENV/bin/ytstream ]]; then + echo "The virtualenv was not built. Fix that, then re-run $0." >&2 + exit 1 +fi say "Installing the /usr/local/bin shim" cat > /usr/local/bin/ytstream <` and `` for episodes + +It reads the rest of the NFO — `aired` and the `youtube` provider id both arrive — +but runtime comes only from a media probe, so a `.strm` episode shows **no duration +until it has been played once**. Not fixable from our side: the only way to supply +one is to probe, which means fetching every episode, which is the one thing this +design exists to avoid. Accepted limitation, stated here so nobody re-litigates it. + +### Episodes had no plot at all + +`strm.materialise` passed `plot=None`, so every synopsis was empty — while both +sources hand us descriptions for free. Now plumbed through: RSS carries +`media:group/media:description`, and `videos.list` carries `snippet.description` in +the call already being made for durations, so the ~40% of episodes older than RSS +reaches still get one. That needed a **schema v2 migration**; v1 was left exactly as +it shipped so a fresh install and a migrated one end up identical, which is asserted +by a test. + +### `materialise --all` created duplicates instead of repairing + +The documented recovery path from a metadata wipe was itself broken. Episode numbers +were re-derived on every run, and `next_episode()` excludes the row it is numbering, +so re-materialising a day's videos in a different order renumbered them — new +filenames, old files left behind. One run left **102 orphaned NFOs against 251 +episodes**. + +Two fixes: an episode number, once assigned, is now permanent and reused from the +row; and materialising a video that already has a different `rel_path` removes the +old files first. Running `materialise --all` twice in a row is now a no-op, verified +on the live tree and pinned by tests. + +### There is a safe metadata refresh, and this is it + +§5 says never to use `replaceAllMetadata=true`, and that stands. But a plain +`/Library/Refresh` does **not** reliably re-read a rewritten NFO — after rewriting +all 251, fifty kept their old empty metadata. The middle ground works: + +``` +POST /Items/{id}/Refresh?metadataRefreshMode=Default&imageRefreshMode=Default + &replaceAllMetadata=false&recursive=true +``` + +Measured across the whole 251-episode library: **plots and aired dates went from 201 +to 251, and the proxy served 0 requests.** The safety is entirely in +`replaceAllMetadata=false` — with it `true`, Jellyfin discards what it has and +re-derives from the media. Exposed as `ytstream refresh-metadata`, and run +automatically after `materialise --all`. + +### Measured on the deployed service + +| | | +|---|---| +| Channels approved (of 119 queued) | 10 | +| Episodes materialised | **251** in 70 s | +| Filtered as Shorts / livestreams | 2 / 4 | +| Tree size | 49 MB for 753 files | +| **Jellyfin full scan** | **251 episodes in ~119 s** (~8 min per 1,000) | +| **Media probes during that scan** | **0** | +| Playback via Jellyfin `PlaybackInfo` | DirectPlay h264 720p + aac, 2049 s runtime | +| Episodes with plot / aired after refresh | 251 / 251 | +| `doctor` | all fatal checks pass | diff --git a/tests/conftest.py b/tests/conftest.py index 2ddc3a6..0ec80d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -181,12 +181,16 @@ class FakeApi: if limit is not None and produced >= limit: return - def durations(self, video_ids): + def details(self, video_ids): self.duration_calls += 1 self.calls += 1 return {vid: self._durations[vid] for vid in video_ids if vid in self._durations} + # `durations` was the old name; kept so a stale caller fails loudly in tests + # rather than silently skipping enrichment. + durations = details + def channel(self, channel_id): self.calls += 1 return self._channel diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 03b0307..a58ff0c 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -389,3 +389,27 @@ def test_a_late_title_renames_an_already_materialised_episode( second = strm.materialise(conn, settings, channel, videos.get(conn, "vid00000001")) assert "Proper Name" in second["rel_path"] + + +def test_reclassifying_a_materialised_video_removes_its_files( + conn, settings, media_root, channel, monkeypatch +): + """A premiere that becomes a livestream, or a duration that only resolves on a + later run, would otherwise leave files on disk with no row owning them.""" + from ytstream import strm + + monkeypatch.setattr(strm, "fetch_thumbnail", lambda *a, **k: False) + add_video(conn, channel["id"], "vid00000001", duration=None) + result = strm.materialise(conn, settings, channel, + videos.get(conn, "vid00000001")) + path = media_root / result["rel_path"] + assert path.exists() + + patch_api(monkeypatch, discovery, + FakeApi(durations={"vid00000001": {"duration": 30, "is_live": False, + "title": "", "description": ""}})) + stats = discovery.enrich_durations(conn, settings, ["vid00000001"]) + + assert stats["shorts"] == 1 + assert videos.get(conn, "vid00000001")["state"] == videos.SKIPPED_SHORT + assert not path.exists() diff --git a/tests/test_strm.py b/tests/test_strm.py index 8498867..14d30eb 100644 --- a/tests/test_strm.py +++ b/tests/test_strm.py @@ -258,3 +258,55 @@ def test_untitled_video_does_not_get_its_id_written_back_as_a_title( assert "vid00000001]" in result["rel_path"] # ...but the row stays untitled, so a later feed poll can still repair it. assert videos.get(conn, "vid00000001")["title"] == "" + + +def test_episode_number_is_stable_across_rematerialising( + conn, settings, media_root, channel, no_thumbs +): + """`materialise --all` is the documented recovery from a Jellyfin metadata + wipe. If it renumbered episodes, the recovery would create a second copy of + every episode instead of repairing the first.""" + ids = [] + for index in range(3): + row = add_video(conn, channel["id"], f"vid{index:08d}", + upload_date="2026-08-12") + ids.append(strm.materialise(conn, settings, channel, row)["episode"]) + + again = [strm.materialise(conn, settings, channel, videos.get(conn, f"vid{i:08d}")) + ["episode"] for i in range(3)] + + assert again == ids == [8120, 8121, 8122] + + +def test_rematerialising_leaves_no_orphans(conn, settings, media_root, channel, + no_thumbs): + for index in range(3): + add_video(conn, channel["id"], f"vid{index:08d}", upload_date="2026-08-12") + for index in range(3): + strm.materialise(conn, settings, channel, videos.get(conn, f"vid{index:08d}")) + for index in range(3): + strm.materialise(conn, settings, channel, videos.get(conn, f"vid{index:08d}")) + + assert len(list(media_root.rglob("*.strm"))) == 3 + assert len([p for p in media_root.rglob("*.nfo") if p.name != "tvshow.nfo"]) == 3 + + +def test_a_renamed_episode_removes_its_old_files(conn, settings, media_root, + channel, no_thumbs): + """The late-title path renames the file; the old one must not survive.""" + row = add_video(conn, channel["id"], "vid00000001", title="", + upload_date="2026-08-12") + first = strm.materialise(conn, settings, channel, row) + old = media_root / first["rel_path"] + assert old.exists() + + with conn: + conn.execute("UPDATE video SET title = ? WHERE video_id = ?", + ("Proper Title", "vid00000001")) + second = strm.materialise(conn, settings, channel, + videos.get(conn, "vid00000001")) + + assert second["rel_path"] != first["rel_path"] + assert not old.exists() + assert (media_root / second["rel_path"]).exists() + assert len(list(media_root.rglob("*.strm"))) == 1 diff --git a/tests/test_videos.py b/tests/test_videos.py index 4bd79b6..9d3db05 100644 --- a/tests/test_videos.py +++ b/tests/test_videos.py @@ -207,3 +207,75 @@ def test_deleting_a_channel_cascades_to_its_videos(conn, channel): with conn: conn.execute("DELETE FROM channel WHERE id = ?", (channel["id"],)) assert conn.execute("SELECT COUNT(*) FROM video").fetchone()[0] == 0 + + +# --------------------------------------------------------------- migrations + + +def test_a_v1_database_migrates_to_v2(tmp_path): + """The live database on susan was created at v1. A fresh install must end up + identical to a migrated one, which is why the description column is a v2 + migration rather than an edit to the v1 script.""" + import sqlite3 as sq + + from ytstream import db + + path = tmp_path / "v1.db" + raw = sq.connect(path) + raw.executescript(db._SCHEMA_V1) + raw.execute("PRAGMA user_version = 1") + raw.commit() + raw.close() + + conn = db.connect(path) + try: + assert conn.execute("PRAGMA user_version").fetchone()[0] == db.SCHEMA_VERSION + columns = {row[1] for row in conn.execute("PRAGMA table_info(video)")} + assert "description" in columns + finally: + conn.close() + + +def test_migration_is_idempotent(tmp_path): + from ytstream import db + + path = tmp_path / "twice.db" + for _ in range(3): + conn = db.connect(path) + conn.close() + conn = db.connect(path) + try: + columns = [row[1] for row in conn.execute("PRAGMA table_info(video)")] + assert columns.count("description") == 1 + finally: + conn.close() + + +def test_fresh_and_migrated_schemas_match(tmp_path): + """A fresh v2 install and a v1 database brought forward must agree, or the two + populations diverge silently.""" + import sqlite3 as sq + + from ytstream import db + + fresh = db.connect(tmp_path / "fresh.db") + fresh_cols = [tuple(row)[1:3] for row in fresh.execute("PRAGMA table_info(video)")] + fresh.close() + + old = tmp_path / "old.db" + raw = sq.connect(old) + raw.executescript(db._SCHEMA_V1) + raw.execute("PRAGMA user_version = 1") + raw.commit() + raw.close() + migrated = db.connect(old) + migrated_cols = [tuple(row)[1:3] + for row in migrated.execute("PRAGMA table_info(video)")] + migrated.close() + + assert fresh_cols == migrated_cols + + +def test_description_survives_a_round_trip(conn, channel): + add_video(conn, channel["id"], "vid00000001", description="A synopsis") + assert videos.get(conn, "vid00000001")["description"] == "A synopsis" diff --git a/ytstream/api.py b/ytstream/api.py index aec5ba6..41715fe 100644 --- a/ytstream/api.py +++ b/ytstream/api.py @@ -295,7 +295,8 @@ class Api: yield {"video_id": video_id, "published": published, "published_at": exact, - "title": title}, next_token + "title": title, + "description": snippet.get("description") or ""}, next_token produced += 1 if limit is not None and produced >= limit: return @@ -306,22 +307,33 @@ class Api: # ---------------------------------------------------------------- durations - def durations(self, video_ids: list[str]) -> dict[str, dict]: - """{video_id: {duration, is_live}} for up to any number of ids. + def details(self, video_ids: list[str]) -> dict[str, dict]: + """{video_id: {duration, is_live, title, description}} for any number of ids. - Batched 50 per call, so 441 videos costs 9 units. `is_live` comes from the - presence of liveStreamingDetails rather than from the duration, because - live and upcoming items both report PT0S. + Batched 50 per call, so 441 videos costs 9 units. `snippet` rides along at + no extra cost and is the only way to fill in a title or description for a + video older than the ~15 entries RSS reaches back — which for a 30-day + window is a real fraction of every channel. + + `is_live` comes from the presence of liveStreamingDetails rather than from + the duration, because live and upcoming items both report PT0S. """ out: dict[str, dict] = {} for start in range(0, len(video_ids), PAGE_SIZE): batch = video_ids[start:start + PAGE_SIZE] - page = self._get("videos", part="contentDetails,liveStreamingDetails", - id=",".join(batch), maxResults=PAGE_SIZE) + page = self._get( + "videos", part="snippet,contentDetails,liveStreamingDetails", + id=",".join(batch), maxResults=PAGE_SIZE) for item in page.get("items") or []: details = item.get("contentDetails") or {} + snippet = item.get("snippet") or {} out[item["id"]] = { "duration": parse_duration(details.get("duration", "")), "is_live": bool(item.get("liveStreamingDetails")), + "title": (snippet.get("title") or "").strip(), + "description": snippet.get("description") or "", } return out + + # Kept as the old name so nothing silently changes meaning mid-refactor. + durations = details diff --git a/ytstream/cli.py b/ytstream/cli.py index 9749611..c50547d 100644 --- a/ytstream/cli.py +++ b/ytstream/cli.py @@ -360,6 +360,15 @@ def cmd_materialise(args) -> int: stats = runner.materialise_all(conn, settings, args.limit) print(f"materialised={stats['materialised']} shows={stats['shows']} " f"errors={stats['errors']}") + if stats["materialised"]: + client = jellyfin.from_settings(settings) + client.refresh() + if args.all: + # A plain scan does not reliably notice a rewritten NFO — after one + # such rewrite, 50 of 251 episodes kept their old metadata. This + # forces the re-read, and does not probe. + print("asking Jellyfin to re-read local metadata...") + client.refresh_library_metadata(config.MEDIA_ROOT) return 1 if stats["errors"] else 0 finally: conn.close() @@ -375,6 +384,23 @@ def cmd_reap(args) -> int: conn.close() +def cmd_refresh_metadata(args) -> int: + """Force Jellyfin to re-read every NFO, without probing any media.""" + conn, settings = _open() + try: + client = jellyfin.from_settings(settings) + if not client.configured: + print("Jellyfin is not configured.", file=sys.stderr) + return 1 + if not client.refresh_library_metadata(config.MEDIA_ROOT): + print(f"No Jellyfin library covers {config.MEDIA_ROOT}.", file=sys.stderr) + return 1 + print("Jellyfin is re-reading local metadata (this does not probe media).") + return 0 + finally: + conn.close() + + def cmd_run(args) -> int: try: with runner.exclusive_lock(): @@ -528,6 +554,10 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("reap", help="delete videos past the retention window").set_defaults( func=cmd_reap) + sub.add_parser("refresh-metadata", + help="make Jellyfin re-read the NFOs (never probes media)" + ).set_defaults(func=cmd_refresh_metadata) + run_cmd = sub.add_parser("run", help="sync, poll, materialise, reap (what cron calls)") run_cmd.add_argument("--channel", type=int, help="restrict to one channel") run_cmd.set_defaults(func=cmd_run) diff --git a/ytstream/db.py b/ytstream/db.py index 79fcd86..dbfdcdb 100644 --- a/ytstream/db.py +++ b/ytstream/db.py @@ -15,7 +15,7 @@ from pathlib import Path from . import config -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 _SCHEMA_V1 = """ CREATE TABLE IF NOT EXISTS channel ( @@ -121,6 +121,14 @@ CREATE TABLE IF NOT EXISTS setting ( """ +# v2: the video description, used as the NFO . Both RSS +# (media:group/media:description) and playlistItems.list (snippet.description) carry +# it for free, and without it every episode has an empty synopsis in Jellyfin. +_SCHEMA_V2 = """ +ALTER TABLE video ADD COLUMN description TEXT; +""" + + def connect(path: Path | None = None) -> sqlite3.Connection: """Open the database, applying migrations if needed.""" path = Path(path) if path is not None else config.DB_PATH @@ -144,6 +152,8 @@ def migrate(conn: sqlite3.Connection) -> int: with conn: if current < 1: conn.executescript(_SCHEMA_V1) + if current < 2: + conn.executescript(_SCHEMA_V2) # Future migrations append here, each guarded by `if current < N`. conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") return SCHEMA_VERSION diff --git a/ytstream/discovery.py b/ytstream/discovery.py index 720a42f..a412509 100644 --- a/ytstream/discovery.py +++ b/ytstream/discovery.py @@ -100,6 +100,8 @@ def parse_entries(payload: bytes) -> list[dict]: { "video_id": video_id, "title": (entry.findtext("atom:title", "", NS) or "").strip(), + "description": entry.findtext( + "media:group/media:description", "", NS) or "", "published": published_date, "published_at": published, } @@ -152,6 +154,13 @@ def _record( # backfill saw it, for instance. If a later feed supplies the title, take # it, and if the episode is already on disk under its video id, remove the # files and re-queue so it is rewritten under the real name. + if not (existing["description"] or "").strip() and entry.get("description"): + with conn: + conn.execute( + "UPDATE video SET description = ? WHERE video_id = ?", + (entry["description"], entry["video_id"]), + ) + if not (existing["title"] or "").strip() and entry["title"]: with conn: conn.execute( @@ -172,6 +181,7 @@ def _record( channel_pk=channel["id"], video_id=entry["video_id"], title=entry["title"], + description=entry.get("description") or "", upload_date=entry["published"].isoformat(), published_at=entry.get("published_at"), state=state, @@ -266,7 +276,7 @@ def enrich_durations( minimum = settings.get_int("min_duration_seconds") client = api.Api(settings.get_str("youtube_api_key")) try: - found = client.durations(video_ids) + found = client.details(video_ids) except api.ApiError as exc: # Durations are an enrichment, not a gate: a NULL duration costs a runtime # display, not a working library. @@ -275,13 +285,36 @@ def enrich_durations( for video_id, info in found.items(): videos.set_duration(conn, video_id, info["duration"]) + # Fill in title and description only when we do not already have them: the + # feed is just as good a source and this must not overwrite it. + existing = videos.get(conn, video_id) + if existing is not None: + for field in ("title", "description"): + if not (existing[field] or "").strip() and info.get(field): + with conn: + conn.execute( + f"UPDATE video SET {field} = ? WHERE video_id = ?", + (info[field], video_id), + ) stats["resolved"] += 1 + skip = None if info["is_live"]: - videos.set_state(conn, video_id, videos.SKIPPED_LIVE) - stats["live"] += 1 + skip, key = videos.SKIPPED_LIVE, "live" elif info["duration"] is not None and info["duration"] < minimum: - videos.set_state(conn, video_id, videos.SKIPPED_SHORT) - stats["shorts"] += 1 + skip, key = videos.SKIPPED_SHORT, "shorts" + + if skip: + # A video can reach this pass having already been materialised — a + # premiere that turned into a livestream, or a duration lookup that + # only succeeded on a later run. Setting the state without removing + # the files would leave them orphaned: on disk for Jellyfin to show, + # but with no row that admits to owning them. + if existing is not None and existing["state"] == videos.MATERIALISED: + strm.remove(existing) + log.info("%s reclassified as %s after materialising; files removed", + video_id, skip) + videos.set_state(conn, video_id, skip) + stats[key] += 1 return stats @@ -335,6 +368,7 @@ def backfill_channel( # backfill named after their video id, because RSS reaches # back only ~23 days against a 30-day window. title=entry.get("title") or "", + description=entry.get("description") or "", upload_date=entry["published"].isoformat(), published_at=entry["published_at"], state=videos.LISTED, diff --git a/ytstream/jellyfin.py b/ytstream/jellyfin.py index 3aaea72..587c750 100644 --- a/ytstream/jellyfin.py +++ b/ytstream/jellyfin.py @@ -138,6 +138,42 @@ class Jellyfin: except JellyfinError as exc: log.warning("jellyfin refresh failed: %s", exc) + def reread_local_metadata(self, item_id: str, *, recursive: bool = True) -> None: + """Re-read NFOs for an item without probing the media. + + A plain `/Library/Refresh` only re-reads an NFO when Jellyfin notices the + file changed, and it does not always notice: after rewriting 251 NFOs, 50 + of them kept their old (empty) metadata. This is the fix, and it is safe — + measured 2026-08-12, a recursive refresh of a 56-episode series in this + mode made **zero** requests to the proxy. + + The safety is entirely in `replaceAllMetadata=false`. With it `true`, + Jellyfin discards what it has and re-derives from the media, which for a + .strm library means fetching every episode. Never set it. + """ + try: + self._request( + "POST", f"/Items/{item_id}/Refresh", + params={ + "metadataRefreshMode": "Default", + "imageRefreshMode": "Default", + "replaceAllMetadata": "false", + "replaceAllImages": "false", + "recursive": "true" if recursive else "false", + }, + ) + except JellyfinError as exc: + log.warning("jellyfin metadata re-read failed: %s", exc) + + def refresh_library_metadata(self, path) -> bool: + """Re-read every NFO under the library covering `path`. Best effort.""" + library = self.find_library(path) + if not library or not library.get("ItemId"): + log.warning("no Jellyfin library found for %s", path) + return False + self.reread_local_metadata(library["ItemId"]) + return True + def from_settings(settings) -> Jellyfin: return Jellyfin( diff --git a/ytstream/strm.py b/ytstream/strm.py index d7790cc..9ba31db 100644 --- a/ytstream/strm.py +++ b/ytstream/strm.py @@ -106,9 +106,18 @@ def materialise( bytes to the same paths. """ upload_date = naming.parse_upload_date(video["upload_date"]) - season, episode = videos.next_episode( - conn, channel["id"], upload_date, video["video_id"] - ) + + # An episode number, once assigned, is permanent. Re-deriving it would let it + # move: next_episode() looks at the MAX for that day excluding this row, so + # re-materialising a day's videos in a different order renumbers them — and + # Jellyfin treats a renumbered episode as a different episode. Reusing the + # stored pair is what makes `materialise --all` safe to run. + if video["season"] and video["episode"]: + season, episode = video["season"], video["episode"] + else: + season, episode = videos.next_episode( + conn, channel["id"], upload_date, video["video_id"] + ) # `title` is for the filename; `stored_title` is what goes back to the # database. They differ when we have no title yet: writing the video id back # would make the row look titled, permanently disabling the repair path in @@ -120,6 +129,14 @@ def materialise( channel, season, episode, title, video["video_id"] ) + # If this video already lives somewhere else on disk, remove that copy before + # writing the new one. A title arriving late changes the filename, and without + # this the old `.strm`, `.nfo` and thumbnail stay behind — so Jellyfin shows the + # episode twice and one of the two never gets updated again. Measured: running + # `materialise --all` once left 102 orphaned NFOs against 251 episodes. + if video["rel_path"] and video["rel_path"] != relative: + remove(video) + strm_path.parent.mkdir(parents=True, exist_ok=True) # No trailing newline: some Jellyfin versions have historically been fussy # about trailing whitespace in .strm files, and there is nothing to gain. @@ -134,7 +151,7 @@ def materialise( show_title=channel["title"], season=season, episode=episode, - plot=None, + plot=video["description"], aired=upload_date.isoformat(), duration_seconds=video["duration"], video_id=video["video_id"], diff --git a/ytstream/videos.py b/ytstream/videos.py index 50db596..8e5f78a 100644 --- a/ytstream/videos.py +++ b/ytstream/videos.py @@ -66,17 +66,19 @@ def insert( discovery_source: str, duration: int | None = None, published_at: str | None = None, + description: str | None = None, ) -> None: with conn: conn.execute( "INSERT OR IGNORE INTO video " - "(video_id, channel_pk, title, upload_date, published_at, duration, " - " state, discovery_source, discovered_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "(video_id, channel_pk, title, description, upload_date, published_at, " + " duration, state, discovery_source, discovered_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( video_id, channel_pk, title, + description, upload_date, published_at, duration,