Phases 1-4 of plan.md §13. Forked from youtube-automate as planned rather than
written from scratch: naming, NFO, auth, the admin UI, settings and the DB layer
came across largely unchanged, download.py is gone, and the pieces that only make
sense for a streaming library are new.
api.py YouTube Data API v3 client. The whole metadata path.
strm.py Materialising: a .strm, an .nfo and a thumbnail. Replaces the
330-line download.py, because the job is writing a URL to a file.
subsync.py The subscription mirror, most of which is refusals.
reap.py Retention, rewritten around the 30-day window and min_keep_videos.
discovery.py RSS polling plus an API-backed, resumable, bounded backfill.
proxy/ The verified PoC, moved in with a systemd unit.
330 tests, all passing, no network and no yt-dlp in any of them. The suite leans
towards the failure paths, because that is where this design can actually hurt
someone: a 403 that looks like an unsubscribe, a video that ages out and comes
back, a title that never arrives. tests/test_proxy.py replaces the two standalone
scripts under proxy/ and now drives the real make_handler(mgr, ...) rather than the
PoC's make_handler(path, done), so routing and video-id validation are covered too.
Ran it end to end against the live API and it found three real bugs.
The first was mine and the tests caught it: strm.remove pruned empty directories
up to the media root, so a channel directory whose tvshow.nfo happened to be
missing would be deleted along with the season. It only looked safe because
tvshow.nfo normally stops the walk. The prune boundary is now the channel
directory explicitly.
The other two only showed up against real data, and they compounded. The backfill
inserted rows with no title and left the RSS poll to fill them in — but RSS returns
15 entries, which for Pitch Side spans 23 days against a 30-day window, so five of
twenty episodes were named after their video ids. Worse, strm.materialise wrote
that fallback back to the database as the title, which made the row look titled and
permanently disabled the repair path. Both fixed: playlistItems.list now requests
snippet as well as contentDetails, which costs the same single quota unit and
carries the title alongside the exact publish date, and the fallback is used for the
filename without being persisted. A title that does arrive late now also removes the
badly-named files and re-queues, so the episode is rewritten rather than keeping its
video-id name forever. Verified against the live API: all twenty Pitch Side episodes
now carry real titles.
Measured on the real account: 119 subscriptions queued for approval and none added
on the first sync, then a two-channel run backfilled and materialised 26 episodes in
under seven seconds.
Two deliberate departures from plan.md, both recorded there:
min_keep_videos defaults to 5 rather than being left as an open question. Without
it 52 of 117 measured channels are empty Jellyfin series that flicker in and out as
their single video crosses the retention line, and the plan already recommended it.
The Jellyfin refresh is a bare /Library/Refresh with a comment explaining why it
must stay that way. A normal scan makes zero media probes; FullRefresh does probe,
and at 400 episodes that is 400 cold starts.
Not yet done: no systemd units are installed (needs root — deploy/deploy.sh), the
admin UI has no routes for sources or the approval queue yet, and nothing has been
pointed at the real media root.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
302 lines
10 KiB
Python
302 lines
10 KiB
Python
"""Channel resolution, subscribe and unsubscribe."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import re
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import tempfile
|
|
import urllib.request
|
|
from pathlib import Path
|
|
|
|
from . import config, naming, nfo, util, ytdlp
|
|
from .settings import Settings
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
_CHANNEL_ID = re.compile(r"^UC[A-Za-z0-9_-]{22}$")
|
|
_HANDLE = re.compile(r"^@[A-Za-z0-9._-]+$")
|
|
|
|
# Artwork we try to pull at subscribe time. Best effort — a channel without them
|
|
# still works, it just looks plainer in Jellyfin.
|
|
_ARTWORK = (("avatar_uncropped", "poster.jpg"), ("banner_uncropped", "fanart.jpg"))
|
|
|
|
|
|
class ResolutionError(RuntimeError):
|
|
pass
|
|
|
|
|
|
def normalise_url(text: str) -> str:
|
|
"""Turn any accepted channel reference into a URL yt-dlp understands."""
|
|
text = (text or "").strip()
|
|
if not text:
|
|
raise ResolutionError("no channel given")
|
|
|
|
if text.startswith(("http://", "https://")):
|
|
return text
|
|
if _CHANNEL_ID.match(text):
|
|
return f"https://www.youtube.com/channel/{text}"
|
|
if _HANDLE.match(text):
|
|
return f"https://www.youtube.com/{text}"
|
|
if text.startswith("www.youtube.com") or text.startswith("youtube.com"):
|
|
return "https://" + text
|
|
# Bare word: assume it's a handle without the @.
|
|
if re.match(r"^[A-Za-z0-9._-]+$", text):
|
|
return f"https://www.youtube.com/@{text}"
|
|
raise ResolutionError(f"could not interpret {text!r} as a channel")
|
|
|
|
|
|
def resolve(settings: Settings, text: str) -> dict:
|
|
"""Fetch channel metadata without enumerating the uploads."""
|
|
url = normalise_url(text)
|
|
args = [
|
|
"--flat-playlist",
|
|
"--playlist-items",
|
|
"0",
|
|
"-J",
|
|
"--no-warnings",
|
|
"--ignore-config",
|
|
*ytdlp.extractor_args(settings.get_str("pot_provider_url")),
|
|
url,
|
|
]
|
|
try:
|
|
data = ytdlp.run_json(args, timeout=180)
|
|
except ytdlp.YtdlpError as exc:
|
|
raise ResolutionError(str(exc)) from exc
|
|
|
|
channel_id = data.get("channel_id") or data.get("id") or ""
|
|
if not _CHANNEL_ID.match(channel_id):
|
|
raise ResolutionError(f"no channel id found for {url}")
|
|
|
|
handle = data.get("uploader_id") or ""
|
|
if handle and not handle.startswith("@"):
|
|
handle = ""
|
|
|
|
return {
|
|
"channel_id": channel_id,
|
|
"title": (data.get("channel") or data.get("title") or channel_id).strip(),
|
|
"description": data.get("description") or "",
|
|
"handle": handle,
|
|
"thumbnails": data.get("thumbnails") or [],
|
|
}
|
|
|
|
|
|
def uulf_playlist_id(channel_id: str) -> str:
|
|
"""Long-form-only uploads playlist for a channel (specs.md §4)."""
|
|
return "UULF" + channel_id[2:]
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# artwork
|
|
|
|
|
|
def _pick_thumbnail(thumbnails: list[dict], wanted_id: str) -> str | None:
|
|
for thumb in thumbnails:
|
|
if str(thumb.get("id", "")) == wanted_id and thumb.get("url"):
|
|
return thumb["url"]
|
|
return None
|
|
|
|
|
|
def _download_image(url: str, destination: Path) -> bool:
|
|
"""Fetch an image and normalise it to JPEG via ffmpeg.
|
|
|
|
YouTube serves avatars as webp as often as jpeg, and naming a webp file
|
|
.jpg would be a lie some clients notice.
|
|
"""
|
|
try:
|
|
request = urllib.request.Request(
|
|
url, headers={"User-Agent": config.USER_AGENT}
|
|
)
|
|
with urllib.request.urlopen(request, timeout=60) as response:
|
|
payload = response.read()
|
|
except OSError as exc:
|
|
log.warning("artwork download failed (%s): %s", url, exc)
|
|
return False
|
|
|
|
with tempfile.NamedTemporaryFile(suffix=".img", delete=True) as raw:
|
|
raw.write(payload)
|
|
raw.flush()
|
|
result = subprocess.run(
|
|
["ffmpeg", "-y", "-loglevel", "error", "-i", raw.name, str(destination)],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=120,
|
|
)
|
|
if result.returncode != 0:
|
|
log.warning("artwork conversion failed: %s", result.stderr.strip()[:200])
|
|
return False
|
|
return True
|
|
|
|
|
|
def write_channel_metadata(channel_dir: Path, info: dict) -> None:
|
|
"""Write tvshow.nfo and best-effort artwork into the channel directory."""
|
|
channel_dir.mkdir(parents=True, exist_ok=True)
|
|
nfo.write(
|
|
channel_dir / "tvshow.nfo",
|
|
nfo.tvshow_nfo(info["title"], info.get("description"), info["channel_id"]),
|
|
)
|
|
for thumb_id, filename in _ARTWORK:
|
|
url = _pick_thumbnail(info.get("thumbnails") or [], thumb_id)
|
|
if url:
|
|
_download_image(url, channel_dir / filename)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# subscribe / unsubscribe
|
|
|
|
|
|
def get(conn: sqlite3.Connection, pk: int) -> sqlite3.Row | None:
|
|
return conn.execute("SELECT * FROM channel WHERE id = ?", (pk,)).fetchone()
|
|
|
|
|
|
def get_by_channel_id(conn: sqlite3.Connection, channel_id: str) -> sqlite3.Row | None:
|
|
return conn.execute(
|
|
"SELECT * FROM channel WHERE channel_id = ?", (channel_id,)
|
|
).fetchone()
|
|
|
|
|
|
def all_channels(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
|
return conn.execute("SELECT * FROM channel ORDER BY title COLLATE NOCASE").fetchall()
|
|
|
|
|
|
def _unique_dir_name(conn: sqlite3.Connection, base: str) -> str:
|
|
"""dir_name is UNIQUE; two channels can legitimately share a title."""
|
|
candidate = base
|
|
suffix = 2
|
|
while conn.execute(
|
|
"SELECT 1 FROM channel WHERE dir_name = ?", (candidate,)
|
|
).fetchone():
|
|
candidate = f"{base} ({suffix})"
|
|
suffix += 1
|
|
return candidate
|
|
|
|
|
|
def subscribe(conn: sqlite3.Connection, settings: Settings, text: str) -> sqlite3.Row:
|
|
"""Resolve, insert and lay down on-disk metadata. Raises ResolutionError."""
|
|
info = resolve(settings, text)
|
|
|
|
existing = get_by_channel_id(conn, info["channel_id"])
|
|
if existing:
|
|
raise ResolutionError(f"already subscribed to {existing['title']}")
|
|
|
|
dir_name = _unique_dir_name(
|
|
conn, naming.channel_dir_name(info["title"], info["channel_id"])
|
|
)
|
|
|
|
with conn:
|
|
cursor = conn.execute(
|
|
"INSERT INTO channel (channel_id, handle, title, description, dir_name, added_at) "
|
|
"VALUES (?, ?, ?, ?, ?, ?)",
|
|
(
|
|
info["channel_id"],
|
|
info["handle"],
|
|
info["title"],
|
|
info["description"],
|
|
dir_name,
|
|
util.utcnow_iso(),
|
|
),
|
|
)
|
|
pk = cursor.lastrowid
|
|
|
|
write_channel_metadata(config.MEDIA_ROOT / dir_name, info)
|
|
log.info("subscribed to %s (%s)", info["title"], info["channel_id"])
|
|
return get(conn, pk)
|
|
|
|
|
|
def subscribe_from_sync(
|
|
conn: sqlite3.Connection, settings: Settings, channel_id: str, title: str
|
|
) -> sqlite3.Row:
|
|
"""Subscribe a channel discovered by the subscription mirror.
|
|
|
|
Distinct from `subscribe()` because it must not touch yt-dlp: cataloguing runs
|
|
entirely on the Data API and RSS, and 119 channels' worth of yt-dlp channel
|
|
resolution is exactly the residential-IP request burst the design avoids
|
|
(plan.md §3). The API already gave us the id and title; one `channels.list`
|
|
call fills in description and artwork.
|
|
"""
|
|
from . import api as ytapi
|
|
|
|
existing = get_by_channel_id(conn, channel_id)
|
|
if existing:
|
|
return existing
|
|
|
|
info = {"channel_id": channel_id, "title": title.strip() or channel_id,
|
|
"description": "", "handle": None, "avatar_url": None}
|
|
try:
|
|
fetched = ytapi.Api(settings.get_str("youtube_api_key")).channel(channel_id)
|
|
if fetched and fetched.get("title"):
|
|
info = fetched
|
|
except ytapi.ApiError as exc:
|
|
# A channel we cannot describe is still a channel we can mirror.
|
|
log.warning("could not fetch metadata for %s: %s", channel_id, exc)
|
|
|
|
dir_name = _unique_dir_name(
|
|
conn, naming.channel_dir_name(info["title"], channel_id)
|
|
)
|
|
with conn:
|
|
cursor = conn.execute(
|
|
"INSERT INTO channel (channel_id, handle, title, description, dir_name, "
|
|
" added_at, source) VALUES (?, ?, ?, ?, ?, ?, 'youtube')",
|
|
(channel_id, info.get("handle"), info["title"],
|
|
info.get("description") or "", dir_name, util.utcnow_iso()),
|
|
)
|
|
pk = cursor.lastrowid
|
|
|
|
row = get(conn, pk)
|
|
# tvshow.nfo only. The channel directory is created lazily by the first
|
|
# episode, so a channel with nothing inside the retention window does not
|
|
# leave an empty series in Jellyfin (plan.md §5).
|
|
if info.get("avatar_url"):
|
|
_write_show_metadata(config.MEDIA_ROOT / dir_name, info)
|
|
log.info("subscribed to %s (%s) via sync", info["title"], channel_id)
|
|
return row
|
|
|
|
|
|
def _write_show_metadata(channel_dir: Path, info: dict) -> None:
|
|
"""tvshow.nfo plus a poster, for a channel resolved through the API."""
|
|
channel_dir.mkdir(parents=True, exist_ok=True)
|
|
nfo.write(
|
|
channel_dir / "tvshow.nfo",
|
|
nfo.tvshow_nfo(info["title"], info.get("description"), info["channel_id"]),
|
|
)
|
|
if info.get("avatar_url"):
|
|
_download_image(info["avatar_url"], channel_dir / "poster.jpg")
|
|
|
|
|
|
def refresh_metadata(conn: sqlite3.Connection, settings: Settings, pk: int) -> None:
|
|
"""Re-resolve a channel and rewrite tvshow.nfo if the title changed.
|
|
|
|
dir_name is deliberately never recomputed — channels rename themselves and
|
|
we do not want orphaned directories.
|
|
"""
|
|
row = get(conn, pk)
|
|
if row is None:
|
|
return
|
|
info = resolve(settings, row["channel_id"])
|
|
if info["title"] != row["title"] or info["description"] != (row["description"] or ""):
|
|
with conn:
|
|
conn.execute(
|
|
"UPDATE channel SET title = ?, description = ? WHERE id = ?",
|
|
(info["title"], info["description"], pk),
|
|
)
|
|
write_channel_metadata(config.MEDIA_ROOT / row["dir_name"], info)
|
|
|
|
|
|
def unsubscribe(conn: sqlite3.Connection, pk: int) -> str:
|
|
"""Hard delete: remove the directory tree, then the rows. Irreversible."""
|
|
row = get(conn, pk)
|
|
if row is None:
|
|
raise LookupError(f"no channel with id {pk}")
|
|
|
|
title = row["title"]
|
|
channel_dir = config.MEDIA_ROOT / row["dir_name"]
|
|
if channel_dir.is_dir():
|
|
shutil.rmtree(channel_dir, ignore_errors=True)
|
|
|
|
with conn:
|
|
conn.execute("DELETE FROM channel WHERE id = ?", (pk,))
|
|
log.info("unsubscribed from %s and removed %s", title, channel_dir)
|
|
return title
|