diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 4d6a221..73ed82e 100755 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -1,5 +1,9 @@ #!/bin/bash -# Everything that does NOT need root. Run as susan, before deploy.sh. +# Everything that does NOT need root: the venv, yt-dlp, the POT container, the +# database. +# +# Normally invoked for you by deploy.sh, which creates the state directory first. +# Safe to run directly as susan afterwards — for instance to rebuild the venv: # # /opt/ytstream/deploy/bootstrap.sh set -euo pipefail @@ -11,8 +15,12 @@ REPO=/opt/ytstream say() { printf '\n\033[1m==> %s\033[0m\n' "$1"; } if [[ ! -d $STATE ]]; then - echo "$STATE does not exist yet — run 'sudo $REPO/deploy/deploy.sh' first," >&2 - echo "or create it with: sudo install -d -o susan -g automation -m 0770 $STATE" >&2 + echo "$STATE does not exist. Run 'sudo $REPO/deploy/deploy.sh' — it creates the" >&2 + echo "directory and then calls this script." >&2 + exit 1 +fi +if [[ ! -w $STATE ]]; then + echo "$STATE is not writable by $(id -un). Expected mode 0770 susan:automation." >&2 exit 1 fi @@ -32,6 +40,40 @@ print("POT plugin:", "MISSING" if missing else "present") raise SystemExit(1 if missing else 0) PY +say "Installing the Deno JS runtime" +# MANDATORY, not optional. Without a JS runtime yt-dlp cannot solve the `n` +# challenge, and youtube-automate measured the consequence on this machine: 22 +# formats instead of 29, and throttled downloads. yt-dlp finds it by PATH, and the +# unit pins PATH to this venv's bin, so it has to live here rather than anywhere +# more natural. +if [[ -x $VENV/bin/deno ]]; then + echo " already present: $("$VENV/bin/deno" --version | head -1)" +elif [[ -x /var/lib/youtube-automate/venv/bin/deno ]]; then + # Same machine, same architecture, already verified working. Prefer it over a + # download while youtube-automate is still installed. + install -m 0755 /var/lib/youtube-automate/venv/bin/deno "$VENV/bin/deno" + echo " copied from the youtube-automate venv: $("$VENV/bin/deno" --version | head -1)" +else + tmp=$(mktemp -d) + curl -fsSL -o "$tmp/deno.zip" \ + https://github.com/denoland/deno/releases/latest/download/deno-x86_64-unknown-linux-gnu.zip + unzip -q "$tmp/deno.zip" -d "$tmp" + install -m 0755 "$tmp/deno" "$VENV/bin/deno" + rm -rf "$tmp" + echo " installed $("$VENV/bin/deno" --version | head -1)" +fi + +say "Confirming yt-dlp can solve JS challenges" +# All three lines must appear, per youtube-automate specs.md §3. +if "$VENV/bin/yt-dlp" -v --simulate --no-warnings \ + "https://www.youtube.com/watch?v=dQw4w9WgXcQ" 2>&1 \ + | grep -q "JS runtimes: deno"; then + echo " yt-dlp reports the deno runtime" +else + echo " WARNING: yt-dlp did not report a deno JS runtime — playback may be" >&2 + echo " throttled or missing formats. Check: ytstream doctor" >&2 +fi + say "Starting the POT provider container if it is not already up" if ! curl -sf --max-time 5 http://127.0.0.1:4416/ping >/dev/null 2>&1; then docker run -d --name bgutil-provider --restart unless-stopped \ @@ -43,4 +85,4 @@ fi say "Initialising the database" "$VENV/bin/ytstream" status || true -say "Done — now run 'sudo $REPO/deploy/deploy.sh'" +say "Virtualenv ready at $VENV" diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 4d1ae35..6ae16b1 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -6,14 +6,21 @@ # # sudo /opt/ytstream/deploy/deploy.sh # -# Everything that does NOT need root — the venv, the database, the POT provider -# container, subscriptions, the API key — is handled by `bootstrap.sh` and the -# application itself. Run bootstrap.sh (as susan) first. +# This is the only command needed. It creates the state directory, builds the venv +# by calling bootstrap.sh as the service user, and only then installs and starts +# the units. +# +# That ordering is not cosmetic. An earlier version installed the units first and +# told the operator to run bootstrap.sh separately — but bootstrap.sh needs the +# state directory, which only this script can create, so neither could go first. +# The units started, failed 203/EXEC against a venv that did not exist yet, and +# restart-looped until the venv appeared. set -euo pipefail REPO=/opt/ytstream STATE=/var/lib/ytstream VENV=$STATE/venv +SERVICE_USER=susan HOSTNAME_=tube.jihakuz.xyz if [[ $EUID -ne 0 ]]; then @@ -26,7 +33,23 @@ say() { printf '\n\033[1m==> %s\033[0m\n' "$1"; } say "Creating $STATE" # root-owned directory, group-writable by `automation` so susan's cron job and the # admin server can both write the database. -install -d -o susan -g automation -m 0770 "$STATE" +install -d -o "$SERVICE_USER" -g automation -m 0770 "$STATE" + +say "Building the virtualenv" +if [[ -x $VENV/bin/ytstream ]]; then + echo " already present at $VENV" +else + # As the service user, so the venv is not left root-owned. Absolute path: + # runuser lives in /sbin, which is not always on the invoking PATH. + /sbin/runuser -u "$SERVICE_USER" -- bash "$REPO/deploy/bootstrap.sh" +fi + +# Refuse to start units that cannot possibly work. Restart=always would otherwise +# turn a missing venv into a restart loop in the journal. +if [[ ! -x $VENV/bin/ytstream ]]; then + echo "The virtualenv was not built. Fix that, then re-run $0." >&2 + exit 1 +fi say "Installing the /usr/local/bin shim" cat > /usr/local/bin/ytstream <` and `` for episodes + +It reads the rest of the NFO — `aired` and the `youtube` provider id both arrive — +but runtime comes only from a media probe, so a `.strm` episode shows **no duration +until it has been played once**. Not fixable from our side: the only way to supply +one is to probe, which means fetching every episode, which is the one thing this +design exists to avoid. Accepted limitation, stated here so nobody re-litigates it. + +### Episodes had no plot at all + +`strm.materialise` passed `plot=None`, so every synopsis was empty — while both +sources hand us descriptions for free. Now plumbed through: RSS carries +`media:group/media:description`, and `videos.list` carries `snippet.description` in +the call already being made for durations, so the ~40% of episodes older than RSS +reaches still get one. That needed a **schema v2 migration**; v1 was left exactly as +it shipped so a fresh install and a migrated one end up identical, which is asserted +by a test. + +### `materialise --all` created duplicates instead of repairing + +The documented recovery path from a metadata wipe was itself broken. Episode numbers +were re-derived on every run, and `next_episode()` excludes the row it is numbering, +so re-materialising a day's videos in a different order renumbered them — new +filenames, old files left behind. One run left **102 orphaned NFOs against 251 +episodes**. + +Two fixes: an episode number, once assigned, is now permanent and reused from the +row; and materialising a video that already has a different `rel_path` removes the +old files first. Running `materialise --all` twice in a row is now a no-op, verified +on the live tree and pinned by tests. + +### There is a safe metadata refresh, and this is it + +§5 says never to use `replaceAllMetadata=true`, and that stands. But a plain +`/Library/Refresh` does **not** reliably re-read a rewritten NFO — after rewriting +all 251, fifty kept their old empty metadata. The middle ground works: + +``` +POST /Items/{id}/Refresh?metadataRefreshMode=Default&imageRefreshMode=Default + &replaceAllMetadata=false&recursive=true +``` + +Measured across the whole 251-episode library: **plots and aired dates went from 201 +to 251, and the proxy served 0 requests.** The safety is entirely in +`replaceAllMetadata=false` — with it `true`, Jellyfin discards what it has and +re-derives from the media. Exposed as `ytstream refresh-metadata`, and run +automatically after `materialise --all`. + +### Measured on the deployed service + +| | | +|---|---| +| Channels approved (of 119 queued) | 10 | +| Episodes materialised | **251** in 70 s | +| Filtered as Shorts / livestreams | 2 / 4 | +| Tree size | 49 MB for 753 files | +| **Jellyfin full scan** | **251 episodes in ~119 s** (~8 min per 1,000) | +| **Media probes during that scan** | **0** | +| Playback via Jellyfin `PlaybackInfo` | DirectPlay h264 720p + aac, 2049 s runtime | +| Episodes with plot / aired after refresh | 251 / 251 | +| `doctor` | all fatal checks pass | diff --git a/tests/conftest.py b/tests/conftest.py index 2ddc3a6..0ec80d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -181,12 +181,16 @@ class FakeApi: if limit is not None and produced >= limit: return - def durations(self, video_ids): + def details(self, video_ids): self.duration_calls += 1 self.calls += 1 return {vid: self._durations[vid] for vid in video_ids if vid in self._durations} + # `durations` was the old name; kept so a stale caller fails loudly in tests + # rather than silently skipping enrichment. + durations = details + def channel(self, channel_id): self.calls += 1 return self._channel diff --git a/tests/test_discovery.py b/tests/test_discovery.py index 03b0307..a58ff0c 100644 --- a/tests/test_discovery.py +++ b/tests/test_discovery.py @@ -389,3 +389,27 @@ def test_a_late_title_renames_an_already_materialised_episode( second = strm.materialise(conn, settings, channel, videos.get(conn, "vid00000001")) assert "Proper Name" in second["rel_path"] + + +def test_reclassifying_a_materialised_video_removes_its_files( + conn, settings, media_root, channel, monkeypatch +): + """A premiere that becomes a livestream, or a duration that only resolves on a + later run, would otherwise leave files on disk with no row owning them.""" + from ytstream import strm + + monkeypatch.setattr(strm, "fetch_thumbnail", lambda *a, **k: False) + add_video(conn, channel["id"], "vid00000001", duration=None) + result = strm.materialise(conn, settings, channel, + videos.get(conn, "vid00000001")) + path = media_root / result["rel_path"] + assert path.exists() + + patch_api(monkeypatch, discovery, + FakeApi(durations={"vid00000001": {"duration": 30, "is_live": False, + "title": "", "description": ""}})) + stats = discovery.enrich_durations(conn, settings, ["vid00000001"]) + + assert stats["shorts"] == 1 + assert videos.get(conn, "vid00000001")["state"] == videos.SKIPPED_SHORT + assert not path.exists() diff --git a/tests/test_strm.py b/tests/test_strm.py index 8498867..14d30eb 100644 --- a/tests/test_strm.py +++ b/tests/test_strm.py @@ -258,3 +258,55 @@ def test_untitled_video_does_not_get_its_id_written_back_as_a_title( assert "vid00000001]" in result["rel_path"] # ...but the row stays untitled, so a later feed poll can still repair it. assert videos.get(conn, "vid00000001")["title"] == "" + + +def test_episode_number_is_stable_across_rematerialising( + conn, settings, media_root, channel, no_thumbs +): + """`materialise --all` is the documented recovery from a Jellyfin metadata + wipe. If it renumbered episodes, the recovery would create a second copy of + every episode instead of repairing the first.""" + ids = [] + for index in range(3): + row = add_video(conn, channel["id"], f"vid{index:08d}", + upload_date="2026-08-12") + ids.append(strm.materialise(conn, settings, channel, row)["episode"]) + + again = [strm.materialise(conn, settings, channel, videos.get(conn, f"vid{i:08d}")) + ["episode"] for i in range(3)] + + assert again == ids == [8120, 8121, 8122] + + +def test_rematerialising_leaves_no_orphans(conn, settings, media_root, channel, + no_thumbs): + for index in range(3): + add_video(conn, channel["id"], f"vid{index:08d}", upload_date="2026-08-12") + for index in range(3): + strm.materialise(conn, settings, channel, videos.get(conn, f"vid{index:08d}")) + for index in range(3): + strm.materialise(conn, settings, channel, videos.get(conn, f"vid{index:08d}")) + + assert len(list(media_root.rglob("*.strm"))) == 3 + assert len([p for p in media_root.rglob("*.nfo") if p.name != "tvshow.nfo"]) == 3 + + +def test_a_renamed_episode_removes_its_old_files(conn, settings, media_root, + channel, no_thumbs): + """The late-title path renames the file; the old one must not survive.""" + row = add_video(conn, channel["id"], "vid00000001", title="", + upload_date="2026-08-12") + first = strm.materialise(conn, settings, channel, row) + old = media_root / first["rel_path"] + assert old.exists() + + with conn: + conn.execute("UPDATE video SET title = ? WHERE video_id = ?", + ("Proper Title", "vid00000001")) + second = strm.materialise(conn, settings, channel, + videos.get(conn, "vid00000001")) + + assert second["rel_path"] != first["rel_path"] + assert not old.exists() + assert (media_root / second["rel_path"]).exists() + assert len(list(media_root.rglob("*.strm"))) == 1 diff --git a/tests/test_videos.py b/tests/test_videos.py index 4bd79b6..9d3db05 100644 --- a/tests/test_videos.py +++ b/tests/test_videos.py @@ -207,3 +207,75 @@ def test_deleting_a_channel_cascades_to_its_videos(conn, channel): with conn: conn.execute("DELETE FROM channel WHERE id = ?", (channel["id"],)) assert conn.execute("SELECT COUNT(*) FROM video").fetchone()[0] == 0 + + +# --------------------------------------------------------------- migrations + + +def test_a_v1_database_migrates_to_v2(tmp_path): + """The live database on susan was created at v1. A fresh install must end up + identical to a migrated one, which is why the description column is a v2 + migration rather than an edit to the v1 script.""" + import sqlite3 as sq + + from ytstream import db + + path = tmp_path / "v1.db" + raw = sq.connect(path) + raw.executescript(db._SCHEMA_V1) + raw.execute("PRAGMA user_version = 1") + raw.commit() + raw.close() + + conn = db.connect(path) + try: + assert conn.execute("PRAGMA user_version").fetchone()[0] == db.SCHEMA_VERSION + columns = {row[1] for row in conn.execute("PRAGMA table_info(video)")} + assert "description" in columns + finally: + conn.close() + + +def test_migration_is_idempotent(tmp_path): + from ytstream import db + + path = tmp_path / "twice.db" + for _ in range(3): + conn = db.connect(path) + conn.close() + conn = db.connect(path) + try: + columns = [row[1] for row in conn.execute("PRAGMA table_info(video)")] + assert columns.count("description") == 1 + finally: + conn.close() + + +def test_fresh_and_migrated_schemas_match(tmp_path): + """A fresh v2 install and a v1 database brought forward must agree, or the two + populations diverge silently.""" + import sqlite3 as sq + + from ytstream import db + + fresh = db.connect(tmp_path / "fresh.db") + fresh_cols = [tuple(row)[1:3] for row in fresh.execute("PRAGMA table_info(video)")] + fresh.close() + + old = tmp_path / "old.db" + raw = sq.connect(old) + raw.executescript(db._SCHEMA_V1) + raw.execute("PRAGMA user_version = 1") + raw.commit() + raw.close() + migrated = db.connect(old) + migrated_cols = [tuple(row)[1:3] + for row in migrated.execute("PRAGMA table_info(video)")] + migrated.close() + + assert fresh_cols == migrated_cols + + +def test_description_survives_a_round_trip(conn, channel): + add_video(conn, channel["id"], "vid00000001", description="A synopsis") + assert videos.get(conn, "vid00000001")["description"] == "A synopsis" diff --git a/ytstream/api.py b/ytstream/api.py index aec5ba6..41715fe 100644 --- a/ytstream/api.py +++ b/ytstream/api.py @@ -295,7 +295,8 @@ class Api: yield {"video_id": video_id, "published": published, "published_at": exact, - "title": title}, next_token + "title": title, + "description": snippet.get("description") or ""}, next_token produced += 1 if limit is not None and produced >= limit: return @@ -306,22 +307,33 @@ class Api: # ---------------------------------------------------------------- durations - def durations(self, video_ids: list[str]) -> dict[str, dict]: - """{video_id: {duration, is_live}} for up to any number of ids. + def details(self, video_ids: list[str]) -> dict[str, dict]: + """{video_id: {duration, is_live, title, description}} for any number of ids. - Batched 50 per call, so 441 videos costs 9 units. `is_live` comes from the - presence of liveStreamingDetails rather than from the duration, because - live and upcoming items both report PT0S. + Batched 50 per call, so 441 videos costs 9 units. `snippet` rides along at + no extra cost and is the only way to fill in a title or description for a + video older than the ~15 entries RSS reaches back — which for a 30-day + window is a real fraction of every channel. + + `is_live` comes from the presence of liveStreamingDetails rather than from + the duration, because live and upcoming items both report PT0S. """ out: dict[str, dict] = {} for start in range(0, len(video_ids), PAGE_SIZE): batch = video_ids[start:start + PAGE_SIZE] - page = self._get("videos", part="contentDetails,liveStreamingDetails", - id=",".join(batch), maxResults=PAGE_SIZE) + page = self._get( + "videos", part="snippet,contentDetails,liveStreamingDetails", + id=",".join(batch), maxResults=PAGE_SIZE) for item in page.get("items") or []: details = item.get("contentDetails") or {} + snippet = item.get("snippet") or {} out[item["id"]] = { "duration": parse_duration(details.get("duration", "")), "is_live": bool(item.get("liveStreamingDetails")), + "title": (snippet.get("title") or "").strip(), + "description": snippet.get("description") or "", } return out + + # Kept as the old name so nothing silently changes meaning mid-refactor. + durations = details diff --git a/ytstream/cli.py b/ytstream/cli.py index 9749611..c50547d 100644 --- a/ytstream/cli.py +++ b/ytstream/cli.py @@ -360,6 +360,15 @@ def cmd_materialise(args) -> int: stats = runner.materialise_all(conn, settings, args.limit) print(f"materialised={stats['materialised']} shows={stats['shows']} " f"errors={stats['errors']}") + if stats["materialised"]: + client = jellyfin.from_settings(settings) + client.refresh() + if args.all: + # A plain scan does not reliably notice a rewritten NFO — after one + # such rewrite, 50 of 251 episodes kept their old metadata. This + # forces the re-read, and does not probe. + print("asking Jellyfin to re-read local metadata...") + client.refresh_library_metadata(config.MEDIA_ROOT) return 1 if stats["errors"] else 0 finally: conn.close() @@ -375,6 +384,23 @@ def cmd_reap(args) -> int: conn.close() +def cmd_refresh_metadata(args) -> int: + """Force Jellyfin to re-read every NFO, without probing any media.""" + conn, settings = _open() + try: + client = jellyfin.from_settings(settings) + if not client.configured: + print("Jellyfin is not configured.", file=sys.stderr) + return 1 + if not client.refresh_library_metadata(config.MEDIA_ROOT): + print(f"No Jellyfin library covers {config.MEDIA_ROOT}.", file=sys.stderr) + return 1 + print("Jellyfin is re-reading local metadata (this does not probe media).") + return 0 + finally: + conn.close() + + def cmd_run(args) -> int: try: with runner.exclusive_lock(): @@ -528,6 +554,10 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("reap", help="delete videos past the retention window").set_defaults( func=cmd_reap) + sub.add_parser("refresh-metadata", + help="make Jellyfin re-read the NFOs (never probes media)" + ).set_defaults(func=cmd_refresh_metadata) + run_cmd = sub.add_parser("run", help="sync, poll, materialise, reap (what cron calls)") run_cmd.add_argument("--channel", type=int, help="restrict to one channel") run_cmd.set_defaults(func=cmd_run) diff --git a/ytstream/db.py b/ytstream/db.py index 79fcd86..dbfdcdb 100644 --- a/ytstream/db.py +++ b/ytstream/db.py @@ -15,7 +15,7 @@ from pathlib import Path from . import config -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 _SCHEMA_V1 = """ CREATE TABLE IF NOT EXISTS channel ( @@ -121,6 +121,14 @@ CREATE TABLE IF NOT EXISTS setting ( """ +# v2: the video description, used as the NFO . Both RSS +# (media:group/media:description) and playlistItems.list (snippet.description) carry +# it for free, and without it every episode has an empty synopsis in Jellyfin. +_SCHEMA_V2 = """ +ALTER TABLE video ADD COLUMN description TEXT; +""" + + def connect(path: Path | None = None) -> sqlite3.Connection: """Open the database, applying migrations if needed.""" path = Path(path) if path is not None else config.DB_PATH @@ -144,6 +152,8 @@ def migrate(conn: sqlite3.Connection) -> int: with conn: if current < 1: conn.executescript(_SCHEMA_V1) + if current < 2: + conn.executescript(_SCHEMA_V2) # Future migrations append here, each guarded by `if current < N`. conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") return SCHEMA_VERSION diff --git a/ytstream/discovery.py b/ytstream/discovery.py index 720a42f..a412509 100644 --- a/ytstream/discovery.py +++ b/ytstream/discovery.py @@ -100,6 +100,8 @@ def parse_entries(payload: bytes) -> list[dict]: { "video_id": video_id, "title": (entry.findtext("atom:title", "", NS) or "").strip(), + "description": entry.findtext( + "media:group/media:description", "", NS) or "", "published": published_date, "published_at": published, } @@ -152,6 +154,13 @@ def _record( # backfill saw it, for instance. If a later feed supplies the title, take # it, and if the episode is already on disk under its video id, remove the # files and re-queue so it is rewritten under the real name. + if not (existing["description"] or "").strip() and entry.get("description"): + with conn: + conn.execute( + "UPDATE video SET description = ? WHERE video_id = ?", + (entry["description"], entry["video_id"]), + ) + if not (existing["title"] or "").strip() and entry["title"]: with conn: conn.execute( @@ -172,6 +181,7 @@ def _record( channel_pk=channel["id"], video_id=entry["video_id"], title=entry["title"], + description=entry.get("description") or "", upload_date=entry["published"].isoformat(), published_at=entry.get("published_at"), state=state, @@ -266,7 +276,7 @@ def enrich_durations( minimum = settings.get_int("min_duration_seconds") client = api.Api(settings.get_str("youtube_api_key")) try: - found = client.durations(video_ids) + found = client.details(video_ids) except api.ApiError as exc: # Durations are an enrichment, not a gate: a NULL duration costs a runtime # display, not a working library. @@ -275,13 +285,36 @@ def enrich_durations( for video_id, info in found.items(): videos.set_duration(conn, video_id, info["duration"]) + # Fill in title and description only when we do not already have them: the + # feed is just as good a source and this must not overwrite it. + existing = videos.get(conn, video_id) + if existing is not None: + for field in ("title", "description"): + if not (existing[field] or "").strip() and info.get(field): + with conn: + conn.execute( + f"UPDATE video SET {field} = ? WHERE video_id = ?", + (info[field], video_id), + ) stats["resolved"] += 1 + skip = None if info["is_live"]: - videos.set_state(conn, video_id, videos.SKIPPED_LIVE) - stats["live"] += 1 + skip, key = videos.SKIPPED_LIVE, "live" elif info["duration"] is not None and info["duration"] < minimum: - videos.set_state(conn, video_id, videos.SKIPPED_SHORT) - stats["shorts"] += 1 + skip, key = videos.SKIPPED_SHORT, "shorts" + + if skip: + # A video can reach this pass having already been materialised — a + # premiere that turned into a livestream, or a duration lookup that + # only succeeded on a later run. Setting the state without removing + # the files would leave them orphaned: on disk for Jellyfin to show, + # but with no row that admits to owning them. + if existing is not None and existing["state"] == videos.MATERIALISED: + strm.remove(existing) + log.info("%s reclassified as %s after materialising; files removed", + video_id, skip) + videos.set_state(conn, video_id, skip) + stats[key] += 1 return stats @@ -335,6 +368,7 @@ def backfill_channel( # backfill named after their video id, because RSS reaches # back only ~23 days against a 30-day window. title=entry.get("title") or "", + description=entry.get("description") or "", upload_date=entry["published"].isoformat(), published_at=entry["published_at"], state=videos.LISTED, diff --git a/ytstream/jellyfin.py b/ytstream/jellyfin.py index 3aaea72..587c750 100644 --- a/ytstream/jellyfin.py +++ b/ytstream/jellyfin.py @@ -138,6 +138,42 @@ class Jellyfin: except JellyfinError as exc: log.warning("jellyfin refresh failed: %s", exc) + def reread_local_metadata(self, item_id: str, *, recursive: bool = True) -> None: + """Re-read NFOs for an item without probing the media. + + A plain `/Library/Refresh` only re-reads an NFO when Jellyfin notices the + file changed, and it does not always notice: after rewriting 251 NFOs, 50 + of them kept their old (empty) metadata. This is the fix, and it is safe — + measured 2026-08-12, a recursive refresh of a 56-episode series in this + mode made **zero** requests to the proxy. + + The safety is entirely in `replaceAllMetadata=false`. With it `true`, + Jellyfin discards what it has and re-derives from the media, which for a + .strm library means fetching every episode. Never set it. + """ + try: + self._request( + "POST", f"/Items/{item_id}/Refresh", + params={ + "metadataRefreshMode": "Default", + "imageRefreshMode": "Default", + "replaceAllMetadata": "false", + "replaceAllImages": "false", + "recursive": "true" if recursive else "false", + }, + ) + except JellyfinError as exc: + log.warning("jellyfin metadata re-read failed: %s", exc) + + def refresh_library_metadata(self, path) -> bool: + """Re-read every NFO under the library covering `path`. Best effort.""" + library = self.find_library(path) + if not library or not library.get("ItemId"): + log.warning("no Jellyfin library found for %s", path) + return False + self.reread_local_metadata(library["ItemId"]) + return True + def from_settings(settings) -> Jellyfin: return Jellyfin( diff --git a/ytstream/strm.py b/ytstream/strm.py index d7790cc..9ba31db 100644 --- a/ytstream/strm.py +++ b/ytstream/strm.py @@ -106,9 +106,18 @@ def materialise( bytes to the same paths. """ upload_date = naming.parse_upload_date(video["upload_date"]) - season, episode = videos.next_episode( - conn, channel["id"], upload_date, video["video_id"] - ) + + # An episode number, once assigned, is permanent. Re-deriving it would let it + # move: next_episode() looks at the MAX for that day excluding this row, so + # re-materialising a day's videos in a different order renumbers them — and + # Jellyfin treats a renumbered episode as a different episode. Reusing the + # stored pair is what makes `materialise --all` safe to run. + if video["season"] and video["episode"]: + season, episode = video["season"], video["episode"] + else: + season, episode = videos.next_episode( + conn, channel["id"], upload_date, video["video_id"] + ) # `title` is for the filename; `stored_title` is what goes back to the # database. They differ when we have no title yet: writing the video id back # would make the row look titled, permanently disabling the repair path in @@ -120,6 +129,14 @@ def materialise( channel, season, episode, title, video["video_id"] ) + # If this video already lives somewhere else on disk, remove that copy before + # writing the new one. A title arriving late changes the filename, and without + # this the old `.strm`, `.nfo` and thumbnail stay behind — so Jellyfin shows the + # episode twice and one of the two never gets updated again. Measured: running + # `materialise --all` once left 102 orphaned NFOs against 251 episodes. + if video["rel_path"] and video["rel_path"] != relative: + remove(video) + strm_path.parent.mkdir(parents=True, exist_ok=True) # No trailing newline: some Jellyfin versions have historically been fussy # about trailing whitespace in .strm files, and there is nothing to gain. @@ -134,7 +151,7 @@ def materialise( show_title=channel["title"], season=season, episode=episode, - plot=None, + plot=video["description"], aired=upload_date.isoformat(), duration_seconds=video["duration"], video_id=video["video_id"], diff --git a/ytstream/videos.py b/ytstream/videos.py index 50db596..8e5f78a 100644 --- a/ytstream/videos.py +++ b/ytstream/videos.py @@ -66,17 +66,19 @@ def insert( discovery_source: str, duration: int | None = None, published_at: str | None = None, + description: str | None = None, ) -> None: with conn: conn.execute( "INSERT OR IGNORE INTO video " - "(video_id, channel_pk, title, upload_date, published_at, duration, " - " state, discovery_source, discovered_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "(video_id, channel_pk, title, description, upload_date, published_at, " + " duration, state, discovery_source, discovered_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", ( video_id, channel_pk, title, + description, upload_date, published_at, duration,