Initial implementation of youtube-automate
A DVR for YouTube subscriptions, laid out so Jellyfin presents each channel as a show and each video as an episode. Cron-driven, idempotent, with a public admin UI for a non-operator. Verified end to end on susan against three real channels: PO tokens, h264 downloads, Jellyfin resolution from local NFOs with all providers disabled, retention and tombstones. Corrections to the original design handover (specs.md documents each with the evidence, and specs.handover-original.md preserves the original): - The format sort selected 360p. Ranking acodec above res makes `bv*` prefer the combined 360p stream, which carries AAC, over the 720p video-only stream whose acodec is none. vcodec now leads, so a video without h264 at 720p yields h264 lower down rather than VP9 this hardware cannot transcode. - yt-dlp now requires a JS runtime and the yt-dlp-ejs solver scripts, which only ship with the [default] extra. Without them the n challenge fails and the mweb formats disappear entirely. - --flat-playlist carries no upload dates, so the specced client-side date filter for backfill was impossible. Backfill is RSS-first. - skipped_old was terminal, so raising a channel's retention appeared to do nothing. Added an explicit rescan. - is_upcoming premieres now defer and retry instead of being skipped forever. - TubeArchivist is gone, so the media root and the tube.jihakuz.xyz vhost were both reclaimed; the latter still pointed at its dead port. 240 offline tests, no network and no real yt-dlp invocation. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Tom Flux
@@ -0,0 +1,3 @@
|
||||
"""youtube-automate — a DVR for YouTube subscriptions, laid out for Jellyfin."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,4 @@
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,241 @@
|
||||
"""Channel resolution, subscribe and unsubscribe."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, naming, nfo, util, ytdlp
|
||||
from .settings import Settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
_CHANNEL_ID = re.compile(r"^UC[A-Za-z0-9_-]{22}$")
|
||||
_HANDLE = re.compile(r"^@[A-Za-z0-9._-]+$")
|
||||
|
||||
# Artwork we try to pull at subscribe time. Best effort — a channel without them
|
||||
# still works, it just looks plainer in Jellyfin.
|
||||
_ARTWORK = (("avatar_uncropped", "poster.jpg"), ("banner_uncropped", "fanart.jpg"))
|
||||
|
||||
|
||||
class ResolutionError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def normalise_url(text: str) -> str:
|
||||
"""Turn any accepted channel reference into a URL yt-dlp understands."""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
raise ResolutionError("no channel given")
|
||||
|
||||
if text.startswith(("http://", "https://")):
|
||||
return text
|
||||
if _CHANNEL_ID.match(text):
|
||||
return f"https://www.youtube.com/channel/{text}"
|
||||
if _HANDLE.match(text):
|
||||
return f"https://www.youtube.com/{text}"
|
||||
if text.startswith("www.youtube.com") or text.startswith("youtube.com"):
|
||||
return "https://" + text
|
||||
# Bare word: assume it's a handle without the @.
|
||||
if re.match(r"^[A-Za-z0-9._-]+$", text):
|
||||
return f"https://www.youtube.com/@{text}"
|
||||
raise ResolutionError(f"could not interpret {text!r} as a channel")
|
||||
|
||||
|
||||
def resolve(settings: Settings, text: str) -> dict:
|
||||
"""Fetch channel metadata without enumerating the uploads."""
|
||||
url = normalise_url(text)
|
||||
args = [
|
||||
"--flat-playlist",
|
||||
"--playlist-items",
|
||||
"0",
|
||||
"-J",
|
||||
"--no-warnings",
|
||||
"--ignore-config",
|
||||
*ytdlp.extractor_args(settings.get_str("pot_provider_url")),
|
||||
url,
|
||||
]
|
||||
try:
|
||||
data = ytdlp.run_json(args, timeout=180)
|
||||
except ytdlp.YtdlpError as exc:
|
||||
raise ResolutionError(str(exc)) from exc
|
||||
|
||||
channel_id = data.get("channel_id") or data.get("id") or ""
|
||||
if not _CHANNEL_ID.match(channel_id):
|
||||
raise ResolutionError(f"no channel id found for {url}")
|
||||
|
||||
handle = data.get("uploader_id") or ""
|
||||
if handle and not handle.startswith("@"):
|
||||
handle = ""
|
||||
|
||||
return {
|
||||
"channel_id": channel_id,
|
||||
"title": (data.get("channel") or data.get("title") or channel_id).strip(),
|
||||
"description": data.get("description") or "",
|
||||
"handle": handle,
|
||||
"thumbnails": data.get("thumbnails") or [],
|
||||
}
|
||||
|
||||
|
||||
def uulf_playlist_id(channel_id: str) -> str:
|
||||
"""Long-form-only uploads playlist for a channel (specs.md §4)."""
|
||||
return "UULF" + channel_id[2:]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# artwork
|
||||
|
||||
|
||||
def _pick_thumbnail(thumbnails: list[dict], wanted_id: str) -> str | None:
|
||||
for thumb in thumbnails:
|
||||
if str(thumb.get("id", "")) == wanted_id and thumb.get("url"):
|
||||
return thumb["url"]
|
||||
return None
|
||||
|
||||
|
||||
def _download_image(url: str, destination: Path) -> bool:
|
||||
"""Fetch an image and normalise it to JPEG via ffmpeg.
|
||||
|
||||
YouTube serves avatars as webp as often as jpeg, and naming a webp file
|
||||
.jpg would be a lie some clients notice.
|
||||
"""
|
||||
try:
|
||||
request = urllib.request.Request(
|
||||
url, headers={"User-Agent": config.USER_AGENT}
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=60) as response:
|
||||
payload = response.read()
|
||||
except OSError as exc:
|
||||
log.warning("artwork download failed (%s): %s", url, exc)
|
||||
return False
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix=".img", delete=True) as raw:
|
||||
raw.write(payload)
|
||||
raw.flush()
|
||||
result = subprocess.run(
|
||||
["ffmpeg", "-y", "-loglevel", "error", "-i", raw.name, str(destination)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
log.warning("artwork conversion failed: %s", result.stderr.strip()[:200])
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def write_channel_metadata(channel_dir: Path, info: dict) -> None:
|
||||
"""Write tvshow.nfo and best-effort artwork into the channel directory."""
|
||||
channel_dir.mkdir(parents=True, exist_ok=True)
|
||||
nfo.write(
|
||||
channel_dir / "tvshow.nfo",
|
||||
nfo.tvshow_nfo(info["title"], info.get("description"), info["channel_id"]),
|
||||
)
|
||||
for thumb_id, filename in _ARTWORK:
|
||||
url = _pick_thumbnail(info.get("thumbnails") or [], thumb_id)
|
||||
if url:
|
||||
_download_image(url, channel_dir / filename)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# subscribe / unsubscribe
|
||||
|
||||
|
||||
def get(conn: sqlite3.Connection, pk: int) -> sqlite3.Row | None:
|
||||
return conn.execute("SELECT * FROM channel WHERE id = ?", (pk,)).fetchone()
|
||||
|
||||
|
||||
def get_by_channel_id(conn: sqlite3.Connection, channel_id: str) -> sqlite3.Row | None:
|
||||
return conn.execute(
|
||||
"SELECT * FROM channel WHERE channel_id = ?", (channel_id,)
|
||||
).fetchone()
|
||||
|
||||
|
||||
def all_channels(conn: sqlite3.Connection) -> list[sqlite3.Row]:
|
||||
return conn.execute("SELECT * FROM channel ORDER BY title COLLATE NOCASE").fetchall()
|
||||
|
||||
|
||||
def _unique_dir_name(conn: sqlite3.Connection, base: str) -> str:
|
||||
"""dir_name is UNIQUE; two channels can legitimately share a title."""
|
||||
candidate = base
|
||||
suffix = 2
|
||||
while conn.execute(
|
||||
"SELECT 1 FROM channel WHERE dir_name = ?", (candidate,)
|
||||
).fetchone():
|
||||
candidate = f"{base} ({suffix})"
|
||||
suffix += 1
|
||||
return candidate
|
||||
|
||||
|
||||
def subscribe(conn: sqlite3.Connection, settings: Settings, text: str) -> sqlite3.Row:
|
||||
"""Resolve, insert and lay down on-disk metadata. Raises ResolutionError."""
|
||||
info = resolve(settings, text)
|
||||
|
||||
existing = get_by_channel_id(conn, info["channel_id"])
|
||||
if existing:
|
||||
raise ResolutionError(f"already subscribed to {existing['title']}")
|
||||
|
||||
dir_name = _unique_dir_name(
|
||||
conn, naming.channel_dir_name(info["title"], info["channel_id"])
|
||||
)
|
||||
|
||||
with conn:
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO channel (channel_id, handle, title, description, dir_name, added_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
info["channel_id"],
|
||||
info["handle"],
|
||||
info["title"],
|
||||
info["description"],
|
||||
dir_name,
|
||||
util.utcnow_iso(),
|
||||
),
|
||||
)
|
||||
pk = cursor.lastrowid
|
||||
|
||||
write_channel_metadata(config.MEDIA_ROOT / dir_name, info)
|
||||
log.info("subscribed to %s (%s)", info["title"], info["channel_id"])
|
||||
return get(conn, pk)
|
||||
|
||||
|
||||
def refresh_metadata(conn: sqlite3.Connection, settings: Settings, pk: int) -> None:
|
||||
"""Re-resolve a channel and rewrite tvshow.nfo if the title changed.
|
||||
|
||||
dir_name is deliberately never recomputed — channels rename themselves and
|
||||
we do not want orphaned directories.
|
||||
"""
|
||||
row = get(conn, pk)
|
||||
if row is None:
|
||||
return
|
||||
info = resolve(settings, row["channel_id"])
|
||||
if info["title"] != row["title"] or info["description"] != (row["description"] or ""):
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE channel SET title = ?, description = ? WHERE id = ?",
|
||||
(info["title"], info["description"], pk),
|
||||
)
|
||||
write_channel_metadata(config.MEDIA_ROOT / row["dir_name"], info)
|
||||
|
||||
|
||||
def unsubscribe(conn: sqlite3.Connection, pk: int) -> str:
|
||||
"""Hard delete: remove the directory tree, then the rows. Irreversible."""
|
||||
row = get(conn, pk)
|
||||
if row is None:
|
||||
raise LookupError(f"no channel with id {pk}")
|
||||
|
||||
title = row["title"]
|
||||
channel_dir = config.MEDIA_ROOT / row["dir_name"]
|
||||
if channel_dir.is_dir():
|
||||
shutil.rmtree(channel_dir, ignore_errors=True)
|
||||
|
||||
with conn:
|
||||
conn.execute("DELETE FROM channel WHERE id = ?", (pk,))
|
||||
log.info("unsubscribed from %s and removed %s", title, channel_dir)
|
||||
return title
|
||||
@@ -0,0 +1,372 @@
|
||||
"""Command line entry point."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import getpass
|
||||
import sys
|
||||
|
||||
from . import (
|
||||
channels,
|
||||
config,
|
||||
db,
|
||||
discovery,
|
||||
doctor,
|
||||
download,
|
||||
jellyfin,
|
||||
reap,
|
||||
runner,
|
||||
util,
|
||||
)
|
||||
from .settings import Settings
|
||||
from .web import auth
|
||||
|
||||
|
||||
def _open():
|
||||
conn = db.connect()
|
||||
return conn, Settings(conn)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# commands
|
||||
|
||||
|
||||
def cmd_doctor(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
text, code = doctor.report(doctor.run_checks(settings))
|
||||
print(text)
|
||||
return code
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_set_password(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
password = getpass.getpass("New admin password: ")
|
||||
if len(password) < 8:
|
||||
print("Password must be at least 8 characters.", file=sys.stderr)
|
||||
return 1
|
||||
if password != getpass.getpass("Repeat: "):
|
||||
print("Passwords did not match.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
settings.set("admin_password_hash", auth.hash_password(password))
|
||||
if not settings.raw("session_secret"):
|
||||
settings.set("session_secret", auth.new_secret())
|
||||
print("Admin password set.")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_set_jellyfin_key(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
key = args.key or getpass.getpass("Jellyfin API key: ")
|
||||
key = key.strip()
|
||||
if not key:
|
||||
print("No key given.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
client = jellyfin.Jellyfin(settings.get_str("jellyfin_url"), key)
|
||||
try:
|
||||
client.virtual_folders()
|
||||
except jellyfin.JellyfinError as exc:
|
||||
print(f"Key rejected by Jellyfin: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
settings.set("jellyfin_api_key", key)
|
||||
print("Jellyfin API key stored and verified.")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_setup_jellyfin_library(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
client = jellyfin.from_settings(settings)
|
||||
if not client.configured:
|
||||
print("Set jellyfin_url and the API key first.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
existing = client.find_library(config.MEDIA_ROOT)
|
||||
if existing:
|
||||
print(
|
||||
f"Library '{existing.get('Name')}' already covers {config.MEDIA_ROOT}."
|
||||
)
|
||||
return 0
|
||||
|
||||
client.create_library(config.MEDIA_ROOT)
|
||||
created = client.find_library(config.MEDIA_ROOT)
|
||||
if not created:
|
||||
print("Library creation reported success but it is not present.",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
print(f"Created library '{created.get('Name')}' for {config.MEDIA_ROOT}.")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_subscribe(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
try:
|
||||
row = channels.subscribe(conn, settings, args.url)
|
||||
except channels.ResolutionError as exc:
|
||||
print(f"Could not subscribe: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"Subscribed to {row['title']} ({row['channel_id']}) -> {row['dir_name']}/")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_unsubscribe(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
row = channels.get(conn, args.id)
|
||||
if row is None:
|
||||
print(f"No channel with id {args.id}.", file=sys.stderr)
|
||||
return 1
|
||||
if not args.yes:
|
||||
print(f"This permanently deletes {config.MEDIA_ROOT / row['dir_name']} "
|
||||
f"and all rows for {row['title']}.")
|
||||
if input("Type the channel title to confirm: ").strip() != row["title"]:
|
||||
print("Aborted.")
|
||||
return 1
|
||||
|
||||
title = channels.unsubscribe(conn, args.id)
|
||||
jellyfin.from_settings(settings).refresh()
|
||||
print(f"Unsubscribed from {title}.")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_poll(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
if args.rescan:
|
||||
rows = (
|
||||
[channels.get(conn, args.channel)]
|
||||
if args.channel
|
||||
else channels.all_channels(conn)
|
||||
)
|
||||
total = sum(
|
||||
discovery.rescan_channel(conn, settings, row) for row in rows if row
|
||||
)
|
||||
print(f"re-queued {total} previously-skipped video(s)")
|
||||
|
||||
totals = discovery.poll_all(conn, settings, args.channel)
|
||||
print(
|
||||
f"queued={totals['queued']} extended={totals['extended']} "
|
||||
f"already-known={totals['known']} outside-window={totals['old']} "
|
||||
f"repaired={totals['repaired']} poll-failures={totals['failed']}"
|
||||
)
|
||||
return 1 if totals["failed"] else 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_download(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
videos_recovered = download.recover_orphans()
|
||||
if videos_recovered:
|
||||
print(f"cleared {videos_recovered} orphan file(s) from the work dir")
|
||||
try:
|
||||
counts = download.drain(conn, settings, args.limit)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 1
|
||||
if not counts:
|
||||
print("nothing to download")
|
||||
return 0
|
||||
print(" ".join(f"{state}={count}" for state, count in sorted(counts.items())))
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_reap(args) -> int:
|
||||
conn, settings = _open()
|
||||
try:
|
||||
result = reap.run(conn, settings)
|
||||
print(f"deleted={result['deleted']} evicted={result['evicted']}")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_run(args) -> int:
|
||||
try:
|
||||
with runner.exclusive_lock():
|
||||
conn, settings = _open()
|
||||
try:
|
||||
result = runner.run(conn, settings, args.channel)
|
||||
print(runner.summarise(result))
|
||||
return 1 if "error" in result.get("download", {}) else 0
|
||||
finally:
|
||||
conn.close()
|
||||
except runner.AlreadyRunning:
|
||||
# Expected when a long backfill outlasts the hourly cron tick.
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_serve(args) -> int:
|
||||
from .web import server
|
||||
|
||||
server.serve(args.host, args.port, secure_cookies=not args.insecure_cookies)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_set_retention(args) -> int:
|
||||
conn, _ = _open()
|
||||
try:
|
||||
row = channels.get(conn, args.id)
|
||||
if row is None:
|
||||
print(f"No channel with id {args.id}.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
if args.days.lower() in ("default", "none", "clear"):
|
||||
value = None
|
||||
else:
|
||||
try:
|
||||
value = int(args.days)
|
||||
except ValueError:
|
||||
print("days must be a whole number or 'default'.", file=sys.stderr)
|
||||
return 1
|
||||
if value < 1:
|
||||
print("days must be at least 1.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE channel SET retention_days = ? WHERE id = ?", (value, args.id)
|
||||
)
|
||||
shown = "the global default" if value is None else f"{value} days"
|
||||
print(f"{row['title']} retention set to {shown}.")
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def cmd_channels(args) -> int:
|
||||
conn, _ = _open()
|
||||
try:
|
||||
rows = channels.all_channels(conn)
|
||||
if not rows:
|
||||
print("No channels subscribed.")
|
||||
return 0
|
||||
print(f"{'id':>3} {'title':<32} {'handle':<20} {'videos':>6} last poll")
|
||||
for row in rows:
|
||||
counts = conn.execute(
|
||||
"SELECT COUNT(*) FROM video WHERE channel_pk = ? AND state = 'downloaded'",
|
||||
(row["id"],),
|
||||
).fetchone()[0]
|
||||
print(
|
||||
f"{row['id']:>3} {row['title'][:32]:<32} {(row['handle'] or ''):<20} "
|
||||
f"{counts:>6} {row['last_polled_at'] or 'never'}"
|
||||
)
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="youtube-automate",
|
||||
description="A DVR for YouTube subscriptions, laid out for Jellyfin.",
|
||||
)
|
||||
parser.add_argument("-v", "--verbose", action="store_true", help="debug logging")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("doctor", help="check the installation").set_defaults(
|
||||
func=cmd_doctor
|
||||
)
|
||||
sub.add_parser("set-password", help="set the admin UI password").set_defaults(
|
||||
func=cmd_set_password
|
||||
)
|
||||
|
||||
key = sub.add_parser("set-jellyfin-key", help="store and verify the Jellyfin API key")
|
||||
key.add_argument("key", nargs="?", help="omit to be prompted (preferred)")
|
||||
key.set_defaults(func=cmd_set_jellyfin_key)
|
||||
|
||||
sub.add_parser(
|
||||
"setup-jellyfin-library", help="create the Shows library for the media root"
|
||||
).set_defaults(func=cmd_setup_jellyfin_library)
|
||||
|
||||
subscribe = sub.add_parser("subscribe", help="subscribe to a channel")
|
||||
subscribe.add_argument("url", help="channel URL, @handle, or UC... id")
|
||||
subscribe.set_defaults(func=cmd_subscribe)
|
||||
|
||||
unsubscribe = sub.add_parser(
|
||||
"unsubscribe", help="remove a channel and everything it downloaded"
|
||||
)
|
||||
unsubscribe.add_argument("id", type=int, help="channel id from `channels`")
|
||||
unsubscribe.add_argument(
|
||||
"--yes", action="store_true", help="skip the confirmation prompt"
|
||||
)
|
||||
unsubscribe.set_defaults(func=cmd_unsubscribe)
|
||||
|
||||
sub.add_parser("channels", help="list subscribed channels").set_defaults(
|
||||
func=cmd_channels
|
||||
)
|
||||
|
||||
poll = sub.add_parser("poll", help="discover new videos")
|
||||
poll.add_argument("--channel", type=int, help="restrict to one channel id")
|
||||
poll.add_argument(
|
||||
"--rescan",
|
||||
action="store_true",
|
||||
help="also re-queue videos previously skipped as too old that the "
|
||||
"current retention window now covers",
|
||||
)
|
||||
poll.set_defaults(func=cmd_poll)
|
||||
|
||||
serve = sub.add_parser("serve", help="run the admin web server")
|
||||
serve.add_argument("--host", default="127.0.0.1")
|
||||
serve.add_argument("--port", type=int, default=8085)
|
||||
serve.add_argument(
|
||||
"--insecure-cookies",
|
||||
action="store_true",
|
||||
help="omit the Secure cookie flag (local testing over plain http only)",
|
||||
)
|
||||
serve.set_defaults(func=cmd_serve)
|
||||
|
||||
run_cmd = sub.add_parser("run", help="poll, download and reap (what cron calls)")
|
||||
run_cmd.add_argument("--channel", type=int, help="restrict discovery to one channel")
|
||||
run_cmd.set_defaults(func=cmd_run)
|
||||
|
||||
sub.add_parser("reap", help="delete videos past the retention window").set_defaults(
|
||||
func=cmd_reap
|
||||
)
|
||||
|
||||
down = sub.add_parser("download", help="drain the pending queue")
|
||||
down.add_argument("--limit", type=int, help="stop after this many videos")
|
||||
down.set_defaults(func=cmd_download)
|
||||
|
||||
retention = sub.add_parser(
|
||||
"set-retention", help="set or clear a channel's retention override"
|
||||
)
|
||||
retention.add_argument("id", type=int, help="channel id")
|
||||
retention.add_argument(
|
||||
"days", help="number of days, or 'default' to clear the override"
|
||||
)
|
||||
retention.set_defaults(func=cmd_set_retention)
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args(argv)
|
||||
util.setup_logging(args.verbose)
|
||||
util.apply_umask()
|
||||
return args.func(args)
|
||||
@@ -0,0 +1,33 @@
|
||||
"""Filesystem paths and process-level constants.
|
||||
|
||||
Every path is overridable through the environment so the test suite can point the
|
||||
whole application at a tmpdir without touching the real media tree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _path(env: str, default: str) -> Path:
|
||||
return Path(os.environ.get(env, default))
|
||||
|
||||
|
||||
STATE_DIR = _path("YTA_STATE_DIR", "/var/lib/youtube-automate")
|
||||
MEDIA_ROOT = _path("YTA_MEDIA_ROOT", "/disks/Plex/YouTube")
|
||||
|
||||
DB_PATH = _path("YTA_DB_PATH", str(STATE_DIR / "subs.db"))
|
||||
LOCK_PATH = _path("YTA_LOCK_PATH", str(STATE_DIR / "run.lock"))
|
||||
VENV_BIN = _path("YTA_VENV_BIN", str(STATE_DIR / "venv" / "bin"))
|
||||
|
||||
WORK_DIR = MEDIA_ROOT / ".work"
|
||||
|
||||
# Files created by the service must stay group-readable by `mediaserver`, which is
|
||||
# how Jellyfin reaches the tree. See specs.md §2.
|
||||
UMASK = 0o002
|
||||
|
||||
# Media-adjacent sidecars we own and therefore may delete on reap.
|
||||
SIDECAR_SUFFIXES = (".nfo", "-thumb.jpg", ".en.srt", ".info.json")
|
||||
|
||||
USER_AGENT = "youtube-automate/1.0"
|
||||
@@ -0,0 +1,87 @@
|
||||
"""SQLite access and schema migrations.
|
||||
|
||||
WAL is mandatory: the hourly cron job and the long-running admin server both write.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from . import config
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
|
||||
_SCHEMA_V1 = """
|
||||
CREATE TABLE IF NOT EXISTS channel (
|
||||
id INTEGER PRIMARY KEY,
|
||||
channel_id TEXT NOT NULL UNIQUE,
|
||||
handle TEXT,
|
||||
title TEXT NOT NULL,
|
||||
description TEXT,
|
||||
dir_name TEXT NOT NULL UNIQUE,
|
||||
added_at TEXT NOT NULL,
|
||||
backfilled INTEGER NOT NULL DEFAULT 0,
|
||||
retention_days INTEGER,
|
||||
last_polled_at TEXT,
|
||||
last_poll_ok INTEGER,
|
||||
consecutive_poll_failures INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS video (
|
||||
id INTEGER PRIMARY KEY,
|
||||
video_id TEXT NOT NULL UNIQUE,
|
||||
channel_pk INTEGER NOT NULL REFERENCES channel(id) ON DELETE CASCADE,
|
||||
title TEXT,
|
||||
upload_date TEXT,
|
||||
duration INTEGER,
|
||||
season INTEGER,
|
||||
episode INTEGER,
|
||||
state TEXT NOT NULL,
|
||||
discovery_source TEXT NOT NULL,
|
||||
rel_path TEXT,
|
||||
size_bytes INTEGER,
|
||||
attempts INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
discovered_at TEXT NOT NULL,
|
||||
downloaded_at TEXT,
|
||||
deleted_at TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_video_state ON video(state);
|
||||
CREATE INDEX IF NOT EXISTS idx_video_upload_date ON video(upload_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_video_channel ON video(channel_pk);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS setting (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def connect(path: Path | None = None) -> sqlite3.Connection:
|
||||
"""Open the database, applying migrations if needed."""
|
||||
path = Path(path) if path is not None else config.DB_PATH
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
conn = sqlite3.connect(path, timeout=30.0, isolation_level=None)
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("PRAGMA busy_timeout = 30000")
|
||||
migrate(conn)
|
||||
return conn
|
||||
|
||||
|
||||
def migrate(conn: sqlite3.Connection) -> int:
|
||||
"""Bring the schema up to SCHEMA_VERSION. Idempotent."""
|
||||
current = conn.execute("PRAGMA user_version").fetchone()[0]
|
||||
if current >= SCHEMA_VERSION:
|
||||
return current
|
||||
|
||||
with conn:
|
||||
if current < 1:
|
||||
conn.executescript(_SCHEMA_V1)
|
||||
# Future migrations append here, each guarded by `if current < N`.
|
||||
conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}")
|
||||
return SCHEMA_VERSION
|
||||
@@ -0,0 +1,363 @@
|
||||
"""Discovery: RSS polling and the subscribe-time backfill.
|
||||
|
||||
Primary path is the undocumented UULF uploads playlist feed, which excludes
|
||||
Shorts and livestreams at the cheapest possible point (verified — see specs.md
|
||||
§4). The channel_id feed is the fallback, and rows discovered that way carry
|
||||
`discovery_source='uc_feed'` so the download step knows to apply the duration and
|
||||
live-status match filter.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import date, timedelta
|
||||
|
||||
from . import channels, config, util, videos, ytdlp
|
||||
from .settings import Settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
NS = {
|
||||
"atom": "http://www.w3.org/2005/Atom",
|
||||
"yt": "http://www.youtube.com/xml/schemas/2015",
|
||||
"media": "http://search.yahoo.com/mrss/",
|
||||
}
|
||||
|
||||
FEED_BASE = "https://www.youtube.com/feeds/videos.xml"
|
||||
|
||||
|
||||
class FeedUnavailable(Exception):
|
||||
"""The feed could not be fetched at all (network/5xx). Not the same as 404."""
|
||||
|
||||
|
||||
def uulf_feed_url(channel_id: str) -> str:
|
||||
return f"{FEED_BASE}?playlist_id={channels.uulf_playlist_id(channel_id)}"
|
||||
|
||||
|
||||
def uc_feed_url(channel_id: str) -> str:
|
||||
return f"{FEED_BASE}?channel_id={channel_id}"
|
||||
|
||||
|
||||
def fetch_feed(url: str, timeout: float = 30.0) -> bytes | None:
|
||||
"""Return the feed body, or None if YouTube says it doesn't exist.
|
||||
|
||||
A 404 on UULF/UUSH/UULV means "no such playlist", i.e. the channel has none
|
||||
of that kind of video — it is not an error.
|
||||
"""
|
||||
request = urllib.request.Request(url, headers={"User-Agent": config.USER_AGENT})
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code == 404:
|
||||
return None
|
||||
raise FeedUnavailable(f"HTTP {exc.code}") from exc
|
||||
except OSError as exc:
|
||||
raise FeedUnavailable(str(exc)) from exc
|
||||
|
||||
|
||||
def parse_entries(payload: bytes) -> list[dict]:
|
||||
"""Parse an Atom feed into video dicts. The feed carries no duration."""
|
||||
try:
|
||||
root = ET.fromstring(payload)
|
||||
except ET.ParseError as exc:
|
||||
raise FeedUnavailable(f"unparseable feed: {exc}") from exc
|
||||
|
||||
entries = []
|
||||
for entry in root.findall("atom:entry", NS):
|
||||
video_id = entry.findtext("yt:videoId", "", NS)
|
||||
if not video_id:
|
||||
continue
|
||||
published = entry.findtext("atom:published", "", NS)
|
||||
try:
|
||||
published_date = date.fromisoformat(published[:10])
|
||||
except ValueError:
|
||||
continue
|
||||
description = entry.findtext("media:group/media:description", "", NS)
|
||||
entries.append(
|
||||
{
|
||||
"video_id": video_id,
|
||||
"title": (entry.findtext("atom:title", "", NS) or "").strip(),
|
||||
"published": published_date,
|
||||
"description": description or "",
|
||||
}
|
||||
)
|
||||
return entries
|
||||
|
||||
|
||||
def effective_retention_days(settings: Settings, channel: sqlite3.Row) -> int:
|
||||
override = channel["retention_days"] if "retention_days" in channel.keys() else None
|
||||
if override:
|
||||
return int(override)
|
||||
return settings.get_int("retention_days")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _record(
|
||||
conn: sqlite3.Connection,
|
||||
channel: sqlite3.Row,
|
||||
entry: dict,
|
||||
source: str,
|
||||
cutoff: date,
|
||||
) -> str:
|
||||
"""Insert or repair one discovered video. Returns what happened."""
|
||||
existing = videos.get(conn, entry["video_id"])
|
||||
|
||||
if existing is not None:
|
||||
# The only repair we ever perform: a video the fallback path rejected as
|
||||
# too short, later confirmed long-form by the authoritative UULF feed.
|
||||
# Deleted rows are tombstones and are never touched here.
|
||||
if (
|
||||
source == videos.SOURCE_UULF
|
||||
and existing["state"] == videos.SKIPPED_SHORT
|
||||
and existing["discovery_source"] == videos.SOURCE_UC
|
||||
):
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE video SET state = ?, discovery_source = ?, "
|
||||
"last_error = NULL WHERE video_id = ?",
|
||||
(videos.PENDING, videos.SOURCE_UULF, entry["video_id"]),
|
||||
)
|
||||
log.info(
|
||||
"re-queued %s: UULF confirms it is long-form", entry["video_id"]
|
||||
)
|
||||
return "repaired"
|
||||
return "known"
|
||||
|
||||
state = videos.PENDING if entry["published"] >= cutoff else videos.SKIPPED_OLD
|
||||
videos.insert(
|
||||
conn,
|
||||
channel_pk=channel["id"],
|
||||
video_id=entry["video_id"],
|
||||
title=entry["title"],
|
||||
upload_date=entry["published"].isoformat(),
|
||||
state=state,
|
||||
discovery_source=source,
|
||||
)
|
||||
return "queued" if state == videos.PENDING else "old"
|
||||
|
||||
|
||||
def poll_channel(conn: sqlite3.Connection, settings: Settings, channel: sqlite3.Row) -> dict:
|
||||
"""Poll one channel. Never raises for feed problems — records them instead."""
|
||||
stats = {"queued": 0, "old": 0, "known": 0, "repaired": 0, "source": None}
|
||||
|
||||
source = videos.SOURCE_UULF
|
||||
try:
|
||||
payload = fetch_feed(uulf_feed_url(channel["channel_id"]))
|
||||
entries = parse_entries(payload) if payload else []
|
||||
if not entries:
|
||||
# UULF 404'd or came back empty — fall back to the channel feed.
|
||||
log.warning(
|
||||
"UULF feed empty for %s, falling back to channel_id feed",
|
||||
channel["title"],
|
||||
)
|
||||
source = videos.SOURCE_UC
|
||||
payload = fetch_feed(uc_feed_url(channel["channel_id"]))
|
||||
entries = parse_entries(payload) if payload else []
|
||||
except FeedUnavailable as exc:
|
||||
_record_poll_failure(conn, channel, str(exc))
|
||||
stats["error"] = str(exc)
|
||||
return stats
|
||||
|
||||
cutoff = util.today() - timedelta(days=effective_retention_days(settings, channel))
|
||||
for entry in entries:
|
||||
stats[_record(conn, channel, entry, source, cutoff)] += 1
|
||||
|
||||
stats["source"] = source
|
||||
_record_poll_success(conn, channel)
|
||||
return stats
|
||||
|
||||
|
||||
def _record_poll_success(conn: sqlite3.Connection, channel: sqlite3.Row) -> None:
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE channel SET last_polled_at = ?, last_poll_ok = 1, "
|
||||
"consecutive_poll_failures = 0 WHERE id = ?",
|
||||
(util.utcnow_iso(), channel["id"]),
|
||||
)
|
||||
|
||||
|
||||
def _record_poll_failure(conn: sqlite3.Connection, channel: sqlite3.Row, error: str) -> None:
|
||||
log.error("poll failed for %s: %s", channel["title"], error)
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE channel SET last_polled_at = ?, last_poll_ok = 0, "
|
||||
"consecutive_poll_failures = consecutive_poll_failures + 1 WHERE id = ?",
|
||||
(util.utcnow_iso(), channel["id"]),
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# backfill
|
||||
|
||||
|
||||
def _flat_playlist(settings: Settings, channel_id: str, limit: int) -> list[dict]:
|
||||
"""Reverse-chronological upload list. Entries carry no upload date."""
|
||||
args = [
|
||||
"--flat-playlist",
|
||||
"--playlist-end",
|
||||
str(limit),
|
||||
"-J",
|
||||
"--no-warnings",
|
||||
"--ignore-config",
|
||||
*ytdlp.extractor_args(settings.get_str("pot_provider_url")),
|
||||
f"https://www.youtube.com/playlist?list={channels.uulf_playlist_id(channel_id)}",
|
||||
]
|
||||
try:
|
||||
data = ytdlp.run_json(args, timeout=300)
|
||||
except ytdlp.YtdlpError as exc:
|
||||
log.warning("flat playlist failed for %s: %s", channel_id, exc)
|
||||
return []
|
||||
return [entry for entry in (data.get("entries") or []) if entry.get("id")]
|
||||
|
||||
|
||||
def _upload_date(settings: Settings, video_id: str) -> date | None:
|
||||
"""One extraction to learn a single video's upload date."""
|
||||
args = [
|
||||
"--skip-download",
|
||||
"--no-warnings",
|
||||
"--ignore-config",
|
||||
"--no-playlist",
|
||||
"--print",
|
||||
"%(upload_date)s",
|
||||
*ytdlp.extractor_args(settings.get_str("pot_provider_url")),
|
||||
f"https://www.youtube.com/watch?v={video_id}",
|
||||
]
|
||||
result = ytdlp.run(args, timeout=180)
|
||||
text = (result.stdout or "").strip().splitlines()
|
||||
if result.returncode != 0 or not text:
|
||||
return None
|
||||
try:
|
||||
from . import naming
|
||||
|
||||
return naming.parse_upload_date(text[-1])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def backfill_channel(
|
||||
conn: sqlite3.Connection, settings: Settings, channel: sqlite3.Row
|
||||
) -> dict:
|
||||
"""Queue the last `backfill_days` for a freshly subscribed channel.
|
||||
|
||||
The RSS feed is the primary source because it is the only one that carries
|
||||
upload dates — `--flat-playlist` reports `timestamp: None` for every entry.
|
||||
RSS returns ~15 items, which covers the default 7-day window for any channel
|
||||
uploading less than twice a day. Only when the feed's oldest entry is still
|
||||
inside the window do we extend via the playlist, resolving those extra dates
|
||||
one video at a time.
|
||||
"""
|
||||
days = settings.get_int("backfill_days")
|
||||
cutoff = util.today() - timedelta(days=days)
|
||||
stats = {"queued": 0, "old": 0, "known": 0, "repaired": 0, "extended": 0}
|
||||
|
||||
try:
|
||||
payload = fetch_feed(uulf_feed_url(channel["channel_id"]))
|
||||
entries = parse_entries(payload) if payload else []
|
||||
source = videos.SOURCE_UULF
|
||||
if not entries:
|
||||
source = videos.SOURCE_UC
|
||||
payload = fetch_feed(uc_feed_url(channel["channel_id"]))
|
||||
entries = parse_entries(payload) if payload else []
|
||||
except FeedUnavailable as exc:
|
||||
_record_poll_failure(conn, channel, str(exc))
|
||||
stats["error"] = str(exc)
|
||||
return stats
|
||||
|
||||
seen = set()
|
||||
for entry in entries:
|
||||
seen.add(entry["video_id"])
|
||||
stats[_record(conn, channel, entry, source, cutoff)] += 1
|
||||
|
||||
oldest = min((entry["published"] for entry in entries), default=None)
|
||||
if oldest is not None and oldest >= cutoff:
|
||||
# The feed did not reach past the window, so there may be more.
|
||||
log.info(
|
||||
"%s: RSS reaches only to %s, extending backfill via playlist",
|
||||
channel["title"],
|
||||
oldest.isoformat(),
|
||||
)
|
||||
for item in _flat_playlist(settings, channel["channel_id"], 50):
|
||||
video_id = item["id"]
|
||||
if video_id in seen or videos.exists(conn, video_id):
|
||||
continue
|
||||
upload_date = _upload_date(settings, video_id)
|
||||
if upload_date is None:
|
||||
continue
|
||||
if upload_date < cutoff:
|
||||
break # playlist is reverse-chronological; everything after is older
|
||||
videos.insert(
|
||||
conn,
|
||||
channel_pk=channel["id"],
|
||||
video_id=video_id,
|
||||
title=(item.get("title") or "").strip(),
|
||||
upload_date=upload_date.isoformat(),
|
||||
duration=item.get("duration"),
|
||||
state=videos.PENDING,
|
||||
discovery_source=videos.SOURCE_BACKFILL,
|
||||
)
|
||||
stats["extended"] += 1
|
||||
|
||||
with conn:
|
||||
conn.execute("UPDATE channel SET backfilled = 1 WHERE id = ?", (channel["id"],))
|
||||
_record_poll_success(conn, channel)
|
||||
return stats
|
||||
|
||||
|
||||
def rescan_channel(
|
||||
conn: sqlite3.Connection, settings: Settings, channel: sqlite3.Row
|
||||
) -> int:
|
||||
"""Re-queue `skipped_old` rows that the current retention window now covers.
|
||||
|
||||
`skipped_old` is judged against whatever window was in force at discovery
|
||||
time, and it is otherwise terminal. Without this, raising a channel's
|
||||
retention_days would appear to do nothing for an infrequent uploader —
|
||||
every one of their videos is already marked old. This is deliberately an
|
||||
explicit action rather than something poll does silently, because doing it
|
||||
on every poll would make `backfill_days` meaningless: it would immediately
|
||||
re-queue everything the initial backfill had deliberately left behind.
|
||||
|
||||
Tombstones (`deleted`) are never touched.
|
||||
"""
|
||||
cutoff = util.today() - timedelta(days=effective_retention_days(settings, channel))
|
||||
with conn:
|
||||
cursor = conn.execute(
|
||||
"UPDATE video SET state = ?, last_error = NULL "
|
||||
"WHERE channel_pk = ? AND state = ? AND upload_date >= ?",
|
||||
(videos.PENDING, channel["id"], videos.SKIPPED_OLD, cutoff.isoformat()),
|
||||
)
|
||||
if cursor.rowcount:
|
||||
log.info(
|
||||
"%s: re-queued %d video(s) now inside the %s window",
|
||||
channel["title"],
|
||||
cursor.rowcount,
|
||||
cutoff.isoformat(),
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def poll_all(conn: sqlite3.Connection, settings: Settings, channel_pk: int | None = None) -> dict:
|
||||
"""Backfill anything new, then poll everything. Returns aggregate counts."""
|
||||
if channel_pk is not None:
|
||||
rows = [row for row in [channels.get(conn, channel_pk)] if row is not None]
|
||||
else:
|
||||
rows = channels.all_channels(conn)
|
||||
|
||||
totals = {"queued": 0, "old": 0, "known": 0, "repaired": 0, "extended": 0, "failed": 0}
|
||||
for channel in rows:
|
||||
if not channel["backfilled"]:
|
||||
stats = backfill_channel(conn, settings, channel)
|
||||
else:
|
||||
stats = poll_channel(conn, settings, channel)
|
||||
if "error" in stats:
|
||||
totals["failed"] += 1
|
||||
for key in ("queued", "old", "known", "repaired", "extended"):
|
||||
totals[key] += stats.get(key, 0)
|
||||
log.info("%s: %s", channel["title"], stats)
|
||||
return totals
|
||||
@@ -0,0 +1,203 @@
|
||||
"""Preflight checks.
|
||||
|
||||
`doctor` is the first acceptance criterion and the thing to run when something
|
||||
breaks. Every check returns a row rather than raising, so one failure doesn't
|
||||
hide the others.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import grp
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, jellyfin, ytdlp
|
||||
from .settings import Settings
|
||||
|
||||
MEDIA_GROUP = "mediaserver"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Check:
|
||||
name: str
|
||||
ok: bool
|
||||
detail: str
|
||||
fatal: bool = True
|
||||
|
||||
|
||||
def _deno() -> Check:
|
||||
binary = config.VENV_BIN / "deno"
|
||||
if not binary.exists():
|
||||
return Check(
|
||||
"js runtime",
|
||||
False,
|
||||
f"deno not found at {binary} — yt-dlp cannot solve n challenges (specs.md §3)",
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(binary), "--version"], capture_output=True, text=True, timeout=30
|
||||
)
|
||||
except OSError as exc:
|
||||
return Check("js runtime", False, f"deno unusable: {exc}")
|
||||
if result.returncode != 0:
|
||||
return Check("js runtime", False, "deno --version failed")
|
||||
return Check("js runtime", True, result.stdout.splitlines()[0])
|
||||
|
||||
|
||||
def _ytdlp() -> Check:
|
||||
try:
|
||||
return Check("yt-dlp", True, ytdlp.version())
|
||||
except (OSError, ytdlp.YtdlpError) as exc:
|
||||
return Check("yt-dlp", False, str(exc))
|
||||
|
||||
|
||||
def _ejs() -> Check:
|
||||
try:
|
||||
from importlib.metadata import version as pkg_version
|
||||
|
||||
return Check("yt-dlp-ejs", True, pkg_version("yt-dlp-ejs"))
|
||||
except Exception:
|
||||
return Check(
|
||||
"yt-dlp-ejs",
|
||||
False,
|
||||
"not installed — reinstall with the yt-dlp[default] extra (specs.md §3)",
|
||||
)
|
||||
|
||||
|
||||
def _pot(settings: Settings) -> Check:
|
||||
url = settings.get_str("pot_provider_url")
|
||||
try:
|
||||
info = ytdlp.pot_provider_ping(url)
|
||||
except Exception as exc:
|
||||
return Check("pot provider", False, f"{url}/ping unreachable: {exc}")
|
||||
|
||||
server_version = str(info.get("version", "?"))
|
||||
installed = ytdlp.plugin_version()
|
||||
if installed and installed != server_version:
|
||||
return Check(
|
||||
"pot provider",
|
||||
False,
|
||||
f"version skew: server {server_version} vs plugin {installed}",
|
||||
)
|
||||
return Check("pot provider", True, f"up, version {server_version}")
|
||||
|
||||
|
||||
def _database() -> Check:
|
||||
try:
|
||||
from . import db
|
||||
|
||||
conn = db.connect()
|
||||
version = conn.execute("PRAGMA user_version").fetchone()[0]
|
||||
mode = conn.execute("PRAGMA journal_mode").fetchone()[0]
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
return Check("database", False, str(exc))
|
||||
if str(mode).lower() != "wal":
|
||||
return Check("database", False, f"journal_mode is {mode}, expected wal")
|
||||
return Check("database", True, f"{config.DB_PATH} (schema v{version}, {mode})")
|
||||
|
||||
|
||||
def _media_root() -> Check:
|
||||
root = config.MEDIA_ROOT
|
||||
if not root.is_dir():
|
||||
return Check("media root", False, f"{root} does not exist")
|
||||
if not os.access(root, os.W_OK | os.X_OK):
|
||||
return Check("media root", False, f"{root} is not writable")
|
||||
|
||||
info = root.stat()
|
||||
problems = []
|
||||
try:
|
||||
group = grp.getgrgid(info.st_gid).gr_name
|
||||
except KeyError:
|
||||
group = str(info.st_gid)
|
||||
if group != MEDIA_GROUP:
|
||||
problems.append(f"group is {group}, expected {MEDIA_GROUP}")
|
||||
if not info.st_mode & stat.S_ISGID:
|
||||
problems.append("setgid bit not set (new dirs won't inherit the group)")
|
||||
|
||||
if problems:
|
||||
return Check("media root", False, f"{root}: " + "; ".join(problems))
|
||||
return Check("media root", True, f"{root} ({group}, setgid)")
|
||||
|
||||
|
||||
def _work_dir() -> Check:
|
||||
work = config.WORK_DIR
|
||||
if not work.is_dir():
|
||||
return Check("work dir", False, f"{work} does not exist")
|
||||
if work.stat().st_dev != config.MEDIA_ROOT.stat().st_dev:
|
||||
return Check(
|
||||
"work dir",
|
||||
False,
|
||||
"not on the same filesystem as the media root — moves would be copies",
|
||||
)
|
||||
if not (work / ".ignore").exists():
|
||||
return Check("work dir", False, f"{work}/.ignore missing", fatal=False)
|
||||
return Check("work dir", True, f"{work} (same fs, .ignore present)")
|
||||
|
||||
|
||||
def _jellyfin(settings: Settings) -> Check:
|
||||
client = jellyfin.from_settings(settings)
|
||||
if not client.base_url:
|
||||
return Check("jellyfin", False, "jellyfin_url not set", fatal=False)
|
||||
try:
|
||||
info = client.public_info()
|
||||
except jellyfin.JellyfinError as exc:
|
||||
return Check("jellyfin", False, str(exc))
|
||||
|
||||
label = f"{info.get('ServerName', '?')} {info.get('Version', '?')}"
|
||||
if not client.api_key:
|
||||
return Check(
|
||||
"jellyfin",
|
||||
False,
|
||||
f"{label} reachable but no API key set (run set-jellyfin-key)",
|
||||
fatal=False,
|
||||
)
|
||||
try:
|
||||
library = client.find_library(config.MEDIA_ROOT)
|
||||
except jellyfin.JellyfinError as exc:
|
||||
return Check("jellyfin", False, f"API key rejected: {exc}")
|
||||
if library is None:
|
||||
return Check(
|
||||
"jellyfin",
|
||||
False,
|
||||
f"{label}, no library for {config.MEDIA_ROOT} (run setup-jellyfin-library)",
|
||||
fatal=False,
|
||||
)
|
||||
return Check("jellyfin", True, f"{label}, library '{library.get('Name')}'")
|
||||
|
||||
|
||||
def run_checks(settings: Settings) -> list[Check]:
|
||||
return [
|
||||
_ytdlp(),
|
||||
_ejs(),
|
||||
_deno(),
|
||||
_pot(settings),
|
||||
_database(),
|
||||
_media_root(),
|
||||
_work_dir(),
|
||||
_jellyfin(settings),
|
||||
]
|
||||
|
||||
|
||||
def report(checks: list[Check]) -> tuple[str, int]:
|
||||
"""Render the checks and return (text, exit_code)."""
|
||||
lines = []
|
||||
failed_fatal = 0
|
||||
for check in checks:
|
||||
if check.ok:
|
||||
mark = "ok "
|
||||
elif check.fatal:
|
||||
mark = "FAIL"
|
||||
failed_fatal += 1
|
||||
else:
|
||||
mark = "warn"
|
||||
lines.append(f" [{mark}] {check.name:<14} {check.detail}")
|
||||
|
||||
if failed_fatal:
|
||||
lines.append(f"\n{failed_fatal} fatal problem(s).")
|
||||
else:
|
||||
lines.append("\nAll fatal checks passed.")
|
||||
return "\n".join(lines), (1 if failed_fatal else 0)
|
||||
@@ -0,0 +1,330 @@
|
||||
"""The download worker.
|
||||
|
||||
One video at a time, into `.work/`, then everything moves into the season
|
||||
directory with `os.rename()` — which is atomic because the work dir shares a
|
||||
filesystem with the media root.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, jellyfin, naming, nfo, videos, ytdlp
|
||||
from .settings import Settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# yt-dlp exit code / message fragments that mean "the match filter rejected it",
|
||||
# which is a decision rather than a failure.
|
||||
_REJECT_MARKERS = (
|
||||
"does not pass filter",
|
||||
"skipping ..",
|
||||
)
|
||||
|
||||
|
||||
class DownloadOutcome:
|
||||
QUEUED = "queued"
|
||||
DONE = "downloaded"
|
||||
SKIPPED_SHORT = videos.SKIPPED_SHORT
|
||||
SKIPPED_LIVE = videos.SKIPPED_LIVE
|
||||
DEFERRED = videos.DEFERRED
|
||||
FAILED = videos.FAILED
|
||||
|
||||
|
||||
def build_args(settings: Settings, video: sqlite3.Row) -> list[str]:
|
||||
"""The yt-dlp invocation from specs.md §6."""
|
||||
max_height = settings.get_int("max_height")
|
||||
args = [
|
||||
*ytdlp.extractor_args(settings.get_str("pot_provider_url")),
|
||||
"-f",
|
||||
f"bv*[height<={max_height}]+ba/b[height<={max_height}]",
|
||||
# vcodec must outrank res: on hardware that cannot transcode, h264 at a
|
||||
# lower resolution beats VP9 at 720p. And acodec must NOT outrank res,
|
||||
# or `bv*` picks the combined 360p stream because it carries AAC.
|
||||
"-S",
|
||||
f"vcodec:h264,res:{max_height},acodec:aac",
|
||||
"--merge-output-format",
|
||||
"mp4",
|
||||
"--no-playlist",
|
||||
"--ignore-config",
|
||||
"--write-info-json",
|
||||
"--write-thumbnail",
|
||||
"--convert-thumbnails",
|
||||
"jpg",
|
||||
"--retries",
|
||||
"3",
|
||||
"--fragment-retries",
|
||||
"10",
|
||||
"--sleep-requests",
|
||||
"2",
|
||||
"--sleep-interval",
|
||||
"5",
|
||||
"--max-sleep-interval",
|
||||
"15",
|
||||
"-P",
|
||||
str(config.WORK_DIR),
|
||||
"-o",
|
||||
"%(id)s.%(ext)s",
|
||||
]
|
||||
|
||||
if settings.get_bool("write_subs"):
|
||||
args += [
|
||||
"--write-subs",
|
||||
"--write-auto-subs",
|
||||
"--sub-langs",
|
||||
settings.get_str("sub_langs"),
|
||||
"--convert-subs",
|
||||
"srt",
|
||||
]
|
||||
if settings.get_bool("sponsorblock_mark"):
|
||||
args += ["--sponsorblock-mark", "all", "--embed-chapters"]
|
||||
|
||||
# The match filter only applies to rows the fallback feed produced. UULF has
|
||||
# already excluded Shorts and livestreams for everything else.
|
||||
if video["discovery_source"] == videos.SOURCE_UC:
|
||||
minimum = settings.get_int("min_duration_seconds")
|
||||
args += [
|
||||
"--match-filter",
|
||||
f"duration>?{minimum} & live_status!=?is_live "
|
||||
f"& live_status!=?is_upcoming & !was_live",
|
||||
]
|
||||
|
||||
args.append(f"https://www.youtube.com/watch?v={video['video_id']}")
|
||||
return args
|
||||
|
||||
|
||||
def _work_files(video_id: str) -> list[Path]:
|
||||
return sorted(config.WORK_DIR.glob(f"{video_id}.*"))
|
||||
|
||||
|
||||
def cleanup_work(video_id: str) -> None:
|
||||
for path in _work_files(video_id):
|
||||
try:
|
||||
path.unlink()
|
||||
except OSError as exc: # pragma: no cover - unusual fs state
|
||||
log.warning("could not remove %s: %s", path, exc)
|
||||
|
||||
|
||||
def _classify_rejection(info: dict | None, stdout: str, stderr: str) -> str | None:
|
||||
"""Decide why the match filter rejected a video, if it did."""
|
||||
blob = f"{stdout}\n{stderr}".lower()
|
||||
if not any(marker in blob for marker in _REJECT_MARKERS):
|
||||
return None
|
||||
|
||||
live_status = (info or {}).get("live_status")
|
||||
if live_status == "is_upcoming":
|
||||
return DownloadOutcome.DEFERRED
|
||||
if live_status in ("is_live", "was_live") or (info or {}).get("was_live"):
|
||||
return DownloadOutcome.SKIPPED_LIVE
|
||||
# Duration is the only other condition in our filter.
|
||||
return DownloadOutcome.SKIPPED_SHORT
|
||||
|
||||
|
||||
def _read_info_json(video_id: str) -> dict | None:
|
||||
path = config.WORK_DIR / f"{video_id}.info.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
log.warning("unreadable info.json for %s: %s", video_id, exc)
|
||||
return None
|
||||
|
||||
|
||||
def _choose_subtitle(video_id: str) -> Path | None:
|
||||
"""`--sub-langs en.*` matches both `en` and `en-orig`, yielding two identical
|
||||
English tracks. Prefer `.en.srt`; otherwise promote `en-orig`."""
|
||||
preferred = config.WORK_DIR / f"{video_id}.en.srt"
|
||||
if preferred.exists():
|
||||
return preferred
|
||||
candidates = sorted(config.WORK_DIR.glob(f"{video_id}.en*.srt"))
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def _move_into_place(
|
||||
conn: sqlite3.Connection, video: sqlite3.Row, info: dict
|
||||
) -> tuple[str, int]:
|
||||
"""Rename every artefact into the season directory. Returns (rel_path, size)."""
|
||||
upload_date = naming.parse_upload_date(
|
||||
info.get("upload_date") or video["upload_date"]
|
||||
)
|
||||
season, episode = videos.next_episode(
|
||||
conn, video["channel_pk"], upload_date, video["video_id"]
|
||||
)
|
||||
title = (info.get("title") or video["title"] or video["video_id"]).strip()
|
||||
|
||||
stem = naming.basename(
|
||||
video["dir_name"], season, episode, title, video["video_id"]
|
||||
)
|
||||
season_dir = config.MEDIA_ROOT / video["dir_name"] / naming.season_dir_name(season)
|
||||
season_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
media_source = config.WORK_DIR / f"{video['video_id']}.mp4"
|
||||
if not media_source.exists():
|
||||
raise FileNotFoundError(f"no mp4 produced for {video['video_id']}")
|
||||
|
||||
# The episode NFO is generated into the work dir first so that a failure
|
||||
# here never leaves a half-populated season directory.
|
||||
nfo.write(
|
||||
config.WORK_DIR / f"{video['video_id']}.nfo",
|
||||
nfo.episode_nfo(
|
||||
title=title,
|
||||
show_title=video["channel_title"],
|
||||
season=season,
|
||||
episode=episode,
|
||||
plot=info.get("description"),
|
||||
aired=upload_date.isoformat(),
|
||||
duration_seconds=info.get("duration"),
|
||||
video_id=video["video_id"],
|
||||
),
|
||||
)
|
||||
|
||||
moves: list[tuple[Path, Path]] = [(media_source, season_dir / f"{stem}.mp4")]
|
||||
for suffix, target in (
|
||||
(".nfo", f"{stem}.nfo"),
|
||||
(".info.json", f"{stem}.info.json"),
|
||||
(".jpg", f"{stem}-thumb.jpg"),
|
||||
):
|
||||
source = config.WORK_DIR / f"{video['video_id']}{suffix}"
|
||||
if source.exists():
|
||||
moves.append((source, season_dir / target))
|
||||
|
||||
subtitle = _choose_subtitle(video["video_id"])
|
||||
if subtitle:
|
||||
moves.append((subtitle, season_dir / f"{stem}.en.srt"))
|
||||
|
||||
for source, destination in moves:
|
||||
source.replace(destination) # same filesystem: atomic rename
|
||||
|
||||
size = (season_dir / f"{stem}.mp4").stat().st_size
|
||||
rel_path = str(
|
||||
(season_dir / f"{stem}.mp4").relative_to(config.MEDIA_ROOT)
|
||||
)
|
||||
|
||||
videos.mark_downloaded(
|
||||
conn,
|
||||
video["video_id"],
|
||||
rel_path=rel_path,
|
||||
size_bytes=size,
|
||||
season=season,
|
||||
episode=episode,
|
||||
upload_date=upload_date.isoformat(),
|
||||
duration=info.get("duration"),
|
||||
title=title,
|
||||
)
|
||||
return rel_path, size
|
||||
|
||||
|
||||
def download_one(conn: sqlite3.Connection, settings: Settings, video: sqlite3.Row) -> str:
|
||||
"""Download a single video. Returns the resulting state."""
|
||||
video_id = video["video_id"]
|
||||
config.WORK_DIR.mkdir(parents=True, exist_ok=True)
|
||||
cleanup_work(video_id)
|
||||
|
||||
videos.set_state(conn, video_id, videos.DOWNLOADING)
|
||||
result = ytdlp.run(build_args(settings, video))
|
||||
info = _read_info_json(video_id)
|
||||
|
||||
rejection = _classify_rejection(info, result.stdout, result.stderr)
|
||||
if rejection is not None:
|
||||
cleanup_work(video_id)
|
||||
videos.set_state(conn, video_id, rejection)
|
||||
log.info("%s rejected by match filter -> %s", video_id, rejection)
|
||||
return rejection
|
||||
|
||||
if result.returncode != 0:
|
||||
error = ytdlp.first_error(result.stderr) or f"exit {result.returncode}"
|
||||
cleanup_work(video_id)
|
||||
outcome = videos.record_failure(
|
||||
conn, video_id, error, settings.get_int("max_attempts")
|
||||
)
|
||||
log.error("%s failed: %s (%s)", video_id, error, outcome)
|
||||
return videos.FAILED
|
||||
|
||||
if info is None:
|
||||
cleanup_work(video_id)
|
||||
videos.record_failure(
|
||||
conn, video_id, "no info.json produced", settings.get_int("max_attempts")
|
||||
)
|
||||
return videos.FAILED
|
||||
|
||||
try:
|
||||
rel_path, size = _move_into_place(conn, video, info)
|
||||
except Exception as exc: # noqa: BLE001 - any failure must clean up
|
||||
cleanup_work(video_id)
|
||||
videos.record_failure(
|
||||
conn, video_id, str(exc), settings.get_int("max_attempts")
|
||||
)
|
||||
log.exception("moving %s into place failed", video_id)
|
||||
return videos.FAILED
|
||||
|
||||
cleanup_work(video_id)
|
||||
_warn_if_not_h264(video_id, info)
|
||||
log.info("downloaded %s -> %s (%.1f MB)", video_id, rel_path, size / 1e6)
|
||||
return videos.DOWNLOADED
|
||||
|
||||
|
||||
def _warn_if_not_h264(video_id: str, info: dict) -> None:
|
||||
"""§6: log the cases where no h264 rendition existed, since Jellyfin will
|
||||
have to transcode them and this box cannot."""
|
||||
vcodec = str(info.get("vcodec") or "")
|
||||
acodec = str(info.get("acodec") or "")
|
||||
if vcodec and not vcodec.startswith(("avc1", "h264")):
|
||||
log.warning("%s has no h264 rendition (got %s) — will transcode", video_id, vcodec)
|
||||
if acodec and not acodec.startswith(("mp4a", "aac")):
|
||||
log.warning("%s has no aac audio (got %s) — will transcode", video_id, acodec)
|
||||
|
||||
|
||||
def drain(
|
||||
conn: sqlite3.Connection, settings: Settings, limit: int | None = None
|
||||
) -> dict:
|
||||
"""Work the queue, one video at a time. Returns per-outcome counts."""
|
||||
if not _provider_healthy(settings):
|
||||
raise RuntimeError(
|
||||
"PO token provider is not reachable — refusing to download and "
|
||||
"accumulate 403s. Check the bgutil-provider container."
|
||||
)
|
||||
|
||||
counts: dict[str, int] = {}
|
||||
queue = videos.claim_pending(conn, settings.get_int("max_attempts"), limit)
|
||||
log.info("%d video(s) in the queue", len(queue))
|
||||
|
||||
for video in queue:
|
||||
state = download_one(conn, settings, video)
|
||||
counts[state] = counts.get(state, 0) + 1
|
||||
|
||||
if counts.get(videos.DOWNLOADED):
|
||||
jellyfin.from_settings(settings).refresh()
|
||||
return counts
|
||||
|
||||
|
||||
def _provider_healthy(settings: Settings) -> bool:
|
||||
try:
|
||||
ytdlp.pot_provider_ping(settings.get_str("pot_provider_url"))
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.error("POT provider health check failed: %s", exc)
|
||||
return False
|
||||
|
||||
|
||||
def recover_orphans() -> int:
|
||||
"""Remove anything left in the work dir by a killed run."""
|
||||
if not config.WORK_DIR.is_dir():
|
||||
return 0
|
||||
removed = 0
|
||||
for path in config.WORK_DIR.iterdir():
|
||||
if path.name == ".ignore":
|
||||
continue
|
||||
try:
|
||||
if path.is_dir():
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
else:
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError: # pragma: no cover
|
||||
pass
|
||||
return removed
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Minimal Jellyfin API client.
|
||||
|
||||
Only three things are needed: check the server is alive, create the Shows library
|
||||
with internet metadata providers switched off, and trigger a refresh after we
|
||||
change the tree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from . import config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
LIBRARY_NAME = "YouTube"
|
||||
COLLECTION_TYPE = "tvshows"
|
||||
|
||||
# Metadata is supplied entirely by our own NFO sidecars, so every fetcher is
|
||||
# disabled for all three item types a Shows library resolves.
|
||||
_ITEM_TYPES = ("Series", "Season", "Episode")
|
||||
|
||||
|
||||
class JellyfinError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class Jellyfin:
|
||||
def __init__(self, base_url: str, api_key: str = "", timeout: float = 30.0):
|
||||
self.base_url = (base_url or "").rstrip("/")
|
||||
self.api_key = api_key or ""
|
||||
self.timeout = timeout
|
||||
|
||||
@property
|
||||
def configured(self) -> bool:
|
||||
return bool(self.base_url and self.api_key)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
params: dict | None = None,
|
||||
body: dict | None = None,
|
||||
):
|
||||
url = self.base_url + path
|
||||
if params:
|
||||
url += "?" + urllib.parse.urlencode(params)
|
||||
data = None
|
||||
headers = {"User-Agent": config.USER_AGENT, "Accept": "application/json"}
|
||||
if self.api_key:
|
||||
headers["X-Emby-Token"] = self.api_key
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
|
||||
request = urllib.request.Request(url, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
payload = response.read()
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise JellyfinError(f"{method} {path} -> HTTP {exc.code}") from exc
|
||||
except OSError as exc:
|
||||
raise JellyfinError(f"{method} {path} -> {exc}") from exc
|
||||
|
||||
if not payload:
|
||||
return None
|
||||
try:
|
||||
return json.loads(payload)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
def public_info(self) -> dict:
|
||||
"""Unauthenticated liveness check."""
|
||||
return self._request("GET", "/System/Info/Public") or {}
|
||||
|
||||
def virtual_folders(self) -> list[dict]:
|
||||
return self._request("GET", "/Library/VirtualFolders") or []
|
||||
|
||||
def find_library(self, path: Path | str) -> dict | None:
|
||||
target = str(path).rstrip("/")
|
||||
for folder in self.virtual_folders():
|
||||
for location in folder.get("Locations") or []:
|
||||
if str(location).rstrip("/") == target:
|
||||
return folder
|
||||
return None
|
||||
|
||||
def create_library(self, path: Path | str, name: str = LIBRARY_NAME) -> None:
|
||||
"""Create the Shows library with all internet providers disabled."""
|
||||
options = {
|
||||
"EnableInternetProviders": False,
|
||||
"SaveLocalMetadata": True,
|
||||
"EnableRealtimeMonitor": False,
|
||||
"EnableChapterImageExtraction": False,
|
||||
"PathInfos": [{"Path": str(path)}],
|
||||
"TypeOptions": [
|
||||
{
|
||||
"Type": item_type,
|
||||
"MetadataFetchers": [],
|
||||
"MetadataFetcherOrder": [],
|
||||
"ImageFetchers": [],
|
||||
"ImageFetcherOrder": [],
|
||||
}
|
||||
for item_type in _ITEM_TYPES
|
||||
],
|
||||
}
|
||||
self._request(
|
||||
"POST",
|
||||
"/Library/VirtualFolders",
|
||||
params={
|
||||
"name": name,
|
||||
"collectionType": COLLECTION_TYPE,
|
||||
"paths": str(path),
|
||||
"refreshLibrary": "false",
|
||||
},
|
||||
body={"LibraryOptions": options},
|
||||
)
|
||||
|
||||
def refresh(self) -> None:
|
||||
"""Trigger a library scan. Best effort — never fatal to the caller."""
|
||||
try:
|
||||
self._request("POST", "/Library/Refresh")
|
||||
except JellyfinError as exc:
|
||||
log.warning("jellyfin refresh failed: %s", exc)
|
||||
|
||||
|
||||
def from_settings(settings) -> Jellyfin:
|
||||
return Jellyfin(
|
||||
settings.get_str("jellyfin_url"), settings.get_str("jellyfin_api_key")
|
||||
)
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Filename sanitisation and season/episode numbering.
|
||||
|
||||
Season is the upload year; episode is ``MMDD * 10 + ordinal_within_day``. That
|
||||
scheme sorts correctly across a whole year (1 Jan is 1010, 31 Dec is 12310) and
|
||||
leaves room for ten uploads per channel per day.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import date
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
# Characters that are illegal or awkward in filenames on the platforms Jellyfin
|
||||
# clients run on. Replaced with a space rather than deleted so that "A/B" reads
|
||||
# as "A B" instead of collapsing into "AB".
|
||||
FORBIDDEN = '/\\:*?"<>|'
|
||||
|
||||
MAX_TITLE_LEN = 120
|
||||
MAX_ORDINAL = 9
|
||||
|
||||
_CONTROL = re.compile(r"[\x00-\x1f\x7f]")
|
||||
_WHITESPACE = re.compile(r"\s+")
|
||||
|
||||
|
||||
def sanitize_component(text: str, max_len: int = MAX_TITLE_LEN) -> str:
|
||||
"""Make one path component safe, collapsing whitespace and truncating."""
|
||||
text = _CONTROL.sub(" ", text or "")
|
||||
text = "".join(" " if char in FORBIDDEN else char for char in text)
|
||||
text = _WHITESPACE.sub(" ", text).strip()
|
||||
text = truncate_on_word_boundary(text, max_len)
|
||||
# A component may not begin or end with a dot or space: leading dots hide the
|
||||
# file from Jellyfin, trailing ones confuse some clients.
|
||||
text = text.strip(" .")
|
||||
return text
|
||||
|
||||
|
||||
def truncate_on_word_boundary(text: str, max_len: int) -> str:
|
||||
if len(text) <= max_len:
|
||||
return text
|
||||
cut = text[:max_len]
|
||||
space = cut.rfind(" ")
|
||||
# Only honour the word boundary if it doesn't throw away most of the name.
|
||||
if space > max_len * 0.6:
|
||||
cut = cut[:space]
|
||||
return cut.rstrip()
|
||||
|
||||
|
||||
def channel_dir_name(title: str, channel_id: str) -> str:
|
||||
"""Directory name for a channel. Stored once and never recomputed."""
|
||||
name = sanitize_component(title)
|
||||
return name or channel_id
|
||||
|
||||
|
||||
def parse_upload_date(value: str | date) -> date:
|
||||
"""Accept yt-dlp's YYYYMMDD, ISO YYYY-MM-DD, or a date."""
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
text = str(value).strip()
|
||||
if len(text) == 8 and text.isdigit():
|
||||
return date(int(text[:4]), int(text[4:6]), int(text[6:8]))
|
||||
return date.fromisoformat(text[:10])
|
||||
|
||||
|
||||
def season_for(upload_date: date) -> int:
|
||||
return upload_date.year
|
||||
|
||||
|
||||
def episode_base(upload_date: date) -> int:
|
||||
"""First episode number available on this date."""
|
||||
return (upload_date.month * 100 + upload_date.day) * 10
|
||||
|
||||
|
||||
def episode_number(upload_date: date, ordinal: int) -> int:
|
||||
"""Episode number for the nth upload on a given date (n starting at 0)."""
|
||||
if ordinal > MAX_ORDINAL:
|
||||
log.warning(
|
||||
"more than %d uploads on %s; clamping ordinal %d",
|
||||
MAX_ORDINAL + 1,
|
||||
upload_date.isoformat(),
|
||||
ordinal,
|
||||
)
|
||||
return episode_base(upload_date) + min(max(ordinal, 0), MAX_ORDINAL)
|
||||
|
||||
|
||||
def episode_range(upload_date: date) -> tuple[int, int]:
|
||||
"""Inclusive (low, high) episode numbers belonging to this date."""
|
||||
base = episode_base(upload_date)
|
||||
return base, base + MAX_ORDINAL
|
||||
|
||||
|
||||
def season_dir_name(season: int) -> str:
|
||||
return f"Season {season}"
|
||||
|
||||
|
||||
def basename(channel_dir: str, season: int, episode: int, title: str, video_id: str) -> str:
|
||||
"""Filename stem shared by the media file and every sidecar.
|
||||
|
||||
The [video_id] suffix guarantees uniqueness regardless of title collisions.
|
||||
"""
|
||||
safe_title = sanitize_component(title) or video_id
|
||||
return f"{channel_dir} - S{season}E{episode} - {safe_title} [{video_id}]"
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Kodi-style NFO sidecars.
|
||||
|
||||
Video descriptions are hostile input — they contain ampersands, angle brackets,
|
||||
emoji, ASCII art and control characters — so these are always built with
|
||||
ElementTree's serialiser and never by string formatting.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
# XML 1.0 forbids most control characters outright; ElementTree will happily
|
||||
# serialise them and produce a document no parser will read back. Written as a
|
||||
# raw string so `re` interprets the escapes, not Python.
|
||||
_ILLEGAL_XML = re.compile(
|
||||
r"[^\x09\x0a\x0d\x20--�\U00010000-\U0010ffff]"
|
||||
)
|
||||
|
||||
|
||||
def clean_text(value: str | None) -> str:
|
||||
return _ILLEGAL_XML.sub("", value or "")
|
||||
|
||||
|
||||
def _child(parent: ET.Element, tag: str, text: str | None) -> ET.Element:
|
||||
element = ET.SubElement(parent, tag)
|
||||
element.text = clean_text(text)
|
||||
return element
|
||||
|
||||
|
||||
def _serialise(root: ET.Element) -> bytes:
|
||||
ET.indent(root, space=" ")
|
||||
return ET.tostring(root, encoding="utf-8", xml_declaration=True)
|
||||
|
||||
|
||||
def tvshow_nfo(title: str, plot: str | None, channel_id: str) -> bytes:
|
||||
root = ET.Element("tvshow")
|
||||
_child(root, "title", title)
|
||||
_child(root, "plot", plot)
|
||||
_child(root, "studio", "YouTube")
|
||||
unique = _child(root, "uniqueid", channel_id)
|
||||
unique.set("type", "youtube")
|
||||
unique.set("default", "true")
|
||||
return _serialise(root)
|
||||
|
||||
|
||||
def episode_nfo(
|
||||
*,
|
||||
title: str,
|
||||
show_title: str,
|
||||
season: int,
|
||||
episode: int,
|
||||
plot: str | None,
|
||||
aired: str,
|
||||
duration_seconds: int | None,
|
||||
video_id: str,
|
||||
) -> bytes:
|
||||
root = ET.Element("episodedetails")
|
||||
_child(root, "title", title)
|
||||
_child(root, "showtitle", show_title)
|
||||
_child(root, "season", str(season))
|
||||
_child(root, "episode", str(episode))
|
||||
_child(root, "plot", plot)
|
||||
_child(root, "aired", aired)
|
||||
if duration_seconds:
|
||||
# Kodi/Jellyfin expect <runtime> in whole minutes.
|
||||
_child(root, "runtime", str(max(1, round(duration_seconds / 60))))
|
||||
_child(root, "studio", "YouTube")
|
||||
unique = _child(root, "uniqueid", video_id)
|
||||
unique.set("type", "youtube")
|
||||
unique.set("default", "true")
|
||||
return _serialise(root)
|
||||
|
||||
|
||||
def write(path: Path, payload: bytes) -> None:
|
||||
"""Write atomically so a crash never leaves Jellyfin a half-written NFO."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_name(path.name + ".tmp")
|
||||
temporary.write_bytes(payload)
|
||||
temporary.replace(path)
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Retention: delete videos that have aged out.
|
||||
|
||||
Reap is a purely local operation. Jellyfin watch-state protection was considered
|
||||
and declined (specs.md §15), so nothing here needs the Jellyfin API except the
|
||||
refresh at the end — and that is best effort.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, jellyfin, util, videos
|
||||
from .settings import Settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def effective_retention(settings: Settings, channel_override: int | None) -> int:
|
||||
return int(channel_override) if channel_override else settings.get_int("retention_days")
|
||||
|
||||
|
||||
def candidates(conn: sqlite3.Connection, settings: Settings) -> list[sqlite3.Row]:
|
||||
"""Downloaded videos past their channel's effective retention window."""
|
||||
rows = conn.execute(
|
||||
"SELECT v.*, c.dir_name, c.retention_days AS channel_retention, "
|
||||
"c.title AS channel_title "
|
||||
"FROM video v JOIN channel c ON c.id = v.channel_pk "
|
||||
"WHERE v.state = ? ORDER BY v.upload_date ASC",
|
||||
(videos.DOWNLOADED,),
|
||||
).fetchall()
|
||||
|
||||
today = util.today()
|
||||
due = []
|
||||
for row in rows:
|
||||
days = effective_retention(settings, row["channel_retention"])
|
||||
if row["upload_date"] and row["upload_date"] < (today - timedelta(days=days)).isoformat():
|
||||
due.append(row)
|
||||
return due
|
||||
|
||||
|
||||
def _delete_artefacts(rel_path: str) -> int:
|
||||
"""Remove the media file and every sidecar sharing its stem."""
|
||||
media = config.MEDIA_ROOT / rel_path
|
||||
season_dir = media.parent
|
||||
stem = media.stem # includes the [videoid], so it cannot collide
|
||||
|
||||
removed = 0
|
||||
if season_dir.is_dir():
|
||||
for path in season_dir.iterdir():
|
||||
if path.name.startswith(stem):
|
||||
try:
|
||||
path.unlink()
|
||||
removed += 1
|
||||
except OSError as exc:
|
||||
log.warning("could not delete %s: %s", path, exc)
|
||||
return removed
|
||||
|
||||
|
||||
def _prune_empty_season(season_dir: Path) -> None:
|
||||
"""Remove the season directory once nothing is left in it.
|
||||
|
||||
The channel directory is deliberately kept even when it holds no seasons:
|
||||
it still carries tvshow.nfo and the artwork, and deleting it would make an
|
||||
active subscription vanish from Jellyfin and come back later.
|
||||
"""
|
||||
if not season_dir.is_dir() or season_dir == config.MEDIA_ROOT:
|
||||
return
|
||||
try:
|
||||
next(season_dir.iterdir())
|
||||
except StopIteration:
|
||||
try:
|
||||
season_dir.rmdir()
|
||||
log.info("pruned empty season directory %s", season_dir)
|
||||
except OSError as exc: # pragma: no cover
|
||||
log.warning("could not prune %s: %s", season_dir, exc)
|
||||
except OSError: # pragma: no cover
|
||||
pass
|
||||
|
||||
|
||||
def delete_video(conn: sqlite3.Connection, video: sqlite3.Row) -> bool:
|
||||
"""Delete one video's files and leave a tombstone row."""
|
||||
rel_path = video["rel_path"]
|
||||
if not rel_path:
|
||||
videos.mark_deleted(conn, video["video_id"])
|
||||
return False
|
||||
|
||||
season_dir = (config.MEDIA_ROOT / rel_path).parent
|
||||
removed = _delete_artefacts(rel_path)
|
||||
_prune_empty_season(season_dir)
|
||||
videos.mark_deleted(conn, video["video_id"])
|
||||
log.info(
|
||||
"reaped %s (%s, uploaded %s): %d file(s)",
|
||||
video["video_id"],
|
||||
video["channel_title"],
|
||||
video["upload_date"],
|
||||
removed,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def disk_cap_evictions(
|
||||
conn: sqlite3.Connection, settings: Settings
|
||||
) -> list[sqlite3.Row]:
|
||||
"""Oldest-first list of videos to evict to get back under the cap."""
|
||||
cap_gb = settings.get_int("disk_cap_gb")
|
||||
if cap_gb <= 0:
|
||||
return []
|
||||
|
||||
cap_bytes = cap_gb * 1024**3
|
||||
rows = conn.execute(
|
||||
"SELECT v.*, c.dir_name, c.title AS channel_title "
|
||||
"FROM video v JOIN channel c ON c.id = v.channel_pk "
|
||||
"WHERE v.state = ? ORDER BY v.upload_date ASC",
|
||||
(videos.DOWNLOADED,),
|
||||
).fetchall()
|
||||
|
||||
total = sum(row["size_bytes"] or 0 for row in rows)
|
||||
if total <= cap_bytes:
|
||||
return []
|
||||
|
||||
evict = []
|
||||
for row in rows:
|
||||
if total <= cap_bytes:
|
||||
break
|
||||
evict.append(row)
|
||||
total -= row["size_bytes"] or 0
|
||||
log.info("disk cap %d GB exceeded; evicting %d video(s)", cap_gb, len(evict))
|
||||
return evict
|
||||
|
||||
|
||||
def run(conn: sqlite3.Connection, settings: Settings) -> dict:
|
||||
"""Age-out pass plus optional disk-cap eviction."""
|
||||
deleted = 0
|
||||
for video in candidates(conn, settings):
|
||||
if delete_video(conn, video):
|
||||
deleted += 1
|
||||
|
||||
evicted = 0
|
||||
for video in disk_cap_evictions(conn, settings):
|
||||
if delete_video(conn, video):
|
||||
evicted += 1
|
||||
|
||||
if deleted or evicted:
|
||||
# Without this, Jellyfin shows ghost episodes until its own scheduled scan.
|
||||
jellyfin.from_settings(settings).refresh()
|
||||
|
||||
return {"deleted": deleted, "evicted": evicted}
|
||||
@@ -0,0 +1,96 @@
|
||||
"""`run` orchestration: poll, then download, then reap — under a lock."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import fcntl
|
||||
import logging
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, discovery, download, reap, util, videos
|
||||
from .settings import Settings
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AlreadyRunning(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def exclusive_lock(path: Path | None = None):
|
||||
"""Non-blocking flock. Raises AlreadyRunning if another run holds it.
|
||||
|
||||
The cron schedule is hourly and a large backfill can outlast that, so
|
||||
overlapping runs are expected and must be a silent no-op rather than two
|
||||
workers fighting over the same queue.
|
||||
"""
|
||||
path = path or config.LOCK_PATH
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
handle = path.open("w")
|
||||
try:
|
||||
try:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as exc:
|
||||
raise AlreadyRunning(f"another run holds {path}") from exc
|
||||
yield
|
||||
finally:
|
||||
with contextlib.suppress(OSError):
|
||||
fcntl.flock(handle, fcntl.LOCK_UN)
|
||||
handle.close()
|
||||
|
||||
|
||||
def recover(conn: sqlite3.Connection) -> dict:
|
||||
"""Undo the effects of a killed run before doing anything else."""
|
||||
requeued = videos.recover_downloading(conn)
|
||||
orphans = download.recover_orphans()
|
||||
if requeued or orphans:
|
||||
log.info(
|
||||
"crash recovery: %d row(s) back to pending, %d orphan file(s) cleared",
|
||||
requeued,
|
||||
orphans,
|
||||
)
|
||||
return {"requeued": requeued, "orphans": orphans}
|
||||
|
||||
|
||||
def run(
|
||||
conn: sqlite3.Connection, settings: Settings, channel_pk: int | None = None
|
||||
) -> dict:
|
||||
"""One full cycle. Assumes the caller holds the lock."""
|
||||
result = {"recovered": recover(conn)}
|
||||
|
||||
result["poll"] = discovery.poll_all(conn, settings, channel_pk)
|
||||
|
||||
try:
|
||||
result["download"] = download.drain(conn, settings)
|
||||
except RuntimeError as exc:
|
||||
# The POT provider being down is loud and fatal for this run, but the
|
||||
# poll results are still worth keeping.
|
||||
log.error("%s", exc)
|
||||
result["download"] = {"error": str(exc)}
|
||||
return result
|
||||
|
||||
result["reap"] = reap.run(conn, settings)
|
||||
settings.set("last_run_at", util.utcnow_iso())
|
||||
return result
|
||||
|
||||
|
||||
def summarise(result: dict) -> str:
|
||||
poll = result.get("poll", {})
|
||||
down = result.get("download", {})
|
||||
reaped = result.get("reap", {})
|
||||
parts = [
|
||||
f"discovered={poll.get('queued', 0)}",
|
||||
f"repaired={poll.get('repaired', 0)}",
|
||||
f"poll_failures={poll.get('failed', 0)}",
|
||||
f"downloaded={down.get(videos.DOWNLOADED, 0)}",
|
||||
f"failed={down.get(videos.FAILED, 0)}",
|
||||
f"skipped={down.get(videos.SKIPPED_SHORT, 0) + down.get(videos.SKIPPED_LIVE, 0)}",
|
||||
f"deferred={down.get(videos.DEFERRED, 0)}",
|
||||
f"reaped={reaped.get('deleted', 0)}",
|
||||
f"evicted={reaped.get('evicted', 0)}",
|
||||
]
|
||||
if "error" in down:
|
||||
parts.append(f"ERROR={down['error']}")
|
||||
return " ".join(parts)
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Typed settings accessors backed by the `setting` key/value table.
|
||||
|
||||
A missing key must never crash anything, so every read falls back to the default
|
||||
and every malformed stored value falls back to the default too.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from urllib.parse import urlparse
|
||||
|
||||
DEFAULTS: dict[str, str] = {
|
||||
"retention_days": "14",
|
||||
"backfill_days": "7",
|
||||
"max_height": "720",
|
||||
"min_duration_seconds": "120",
|
||||
"sponsorblock_mark": "true",
|
||||
"write_subs": "true",
|
||||
"sub_langs": "en.*",
|
||||
"jellyfin_url": "http://127.0.0.1:8096",
|
||||
"jellyfin_api_key": "",
|
||||
"pot_provider_url": "http://127.0.0.1:4416",
|
||||
"max_attempts": "5",
|
||||
"disk_cap_gb": "0",
|
||||
}
|
||||
|
||||
# Editable through the settings form. Everything else in the table is internal.
|
||||
EDITABLE = tuple(DEFAULTS)
|
||||
|
||||
# Never rendered, never settable through the web form.
|
||||
SECRET_KEYS = ("admin_password_hash", "session_secret")
|
||||
|
||||
# Shown as a masked value rather than plaintext.
|
||||
MASKED_KEYS = ("jellyfin_api_key",)
|
||||
|
||||
_INT_KEYS = (
|
||||
"retention_days",
|
||||
"backfill_days",
|
||||
"max_height",
|
||||
"min_duration_seconds",
|
||||
"max_attempts",
|
||||
"disk_cap_gb",
|
||||
)
|
||||
_BOOL_KEYS = ("sponsorblock_mark", "write_subs")
|
||||
_URL_KEYS = ("jellyfin_url", "pot_provider_url")
|
||||
|
||||
_TRUE = {"1", "true", "yes", "on"}
|
||||
_FALSE = {"0", "false", "no", "off", ""}
|
||||
|
||||
|
||||
class Settings:
|
||||
def __init__(self, conn: sqlite3.Connection):
|
||||
self.conn = conn
|
||||
|
||||
def raw(self, key: str) -> str:
|
||||
row = self.conn.execute(
|
||||
"SELECT value FROM setting WHERE key = ?", (key,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return DEFAULTS.get(key, "")
|
||||
return row["value"]
|
||||
|
||||
def get_str(self, key: str) -> str:
|
||||
return self.raw(key)
|
||||
|
||||
def get_int(self, key: str) -> int:
|
||||
try:
|
||||
return int(self.raw(key))
|
||||
except (TypeError, ValueError):
|
||||
return int(DEFAULTS.get(key, "0") or 0)
|
||||
|
||||
def get_bool(self, key: str) -> bool:
|
||||
value = self.raw(key).strip().lower()
|
||||
if value in _TRUE:
|
||||
return True
|
||||
if value in _FALSE:
|
||||
return False
|
||||
return DEFAULTS.get(key, "false").lower() in _TRUE
|
||||
|
||||
def set(self, key: str, value: str) -> None:
|
||||
with self.conn:
|
||||
self.conn.execute(
|
||||
"INSERT INTO setting (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(key, str(value)),
|
||||
)
|
||||
|
||||
def all_editable(self) -> dict[str, str]:
|
||||
return {key: self.raw(key) for key in EDITABLE}
|
||||
|
||||
|
||||
def validate(key: str, value: str) -> tuple[bool, str]:
|
||||
"""Validate one submitted setting.
|
||||
|
||||
Returns (ok, message). On failure the caller re-renders the form with the
|
||||
message inline rather than raising.
|
||||
"""
|
||||
value = value.strip()
|
||||
|
||||
if key in _INT_KEYS:
|
||||
try:
|
||||
number = int(value)
|
||||
except ValueError:
|
||||
return False, "must be a whole number"
|
||||
if number < 0:
|
||||
return False, "must be zero or greater"
|
||||
if key == "retention_days" and number < 1:
|
||||
return False, "must be at least 1 day"
|
||||
if key == "max_height" and number < 144:
|
||||
return False, "must be at least 144"
|
||||
return True, ""
|
||||
|
||||
if key in _BOOL_KEYS:
|
||||
if value.lower() not in _TRUE | _FALSE:
|
||||
return False, "must be true or false"
|
||||
return True, ""
|
||||
|
||||
if key in _URL_KEYS:
|
||||
parsed = urlparse(value)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.netloc:
|
||||
return False, "must be a http:// or https:// URL"
|
||||
return True, ""
|
||||
|
||||
if key == "sub_langs":
|
||||
if not value:
|
||||
return False, "must not be empty"
|
||||
return True, ""
|
||||
|
||||
if key == "jellyfin_api_key":
|
||||
return True, ""
|
||||
|
||||
return key in DEFAULTS, "unknown setting"
|
||||
|
||||
|
||||
def validate_all(submitted: dict[str, str]) -> dict[str, str]:
|
||||
"""Return {key: error} for everything that failed validation."""
|
||||
errors: dict[str, str] = {}
|
||||
for key, value in submitted.items():
|
||||
if key not in EDITABLE:
|
||||
continue
|
||||
ok, message = validate(key, value)
|
||||
if not ok:
|
||||
errors[key] = message
|
||||
return errors
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Small shared helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def utcnow_iso() -> str:
|
||||
return utcnow().replace(microsecond=0).isoformat()
|
||||
|
||||
|
||||
def today() -> date:
|
||||
return utcnow().date()
|
||||
|
||||
|
||||
def apply_umask() -> None:
|
||||
"""Ensure files land group-writable so Jellyfin's group can read them."""
|
||||
os.umask(config.UMASK)
|
||||
|
||||
|
||||
def setup_logging(verbose: bool = False) -> None:
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG if verbose else logging.INFO,
|
||||
format="%(asctime)s %(levelname)-7s %(name)s: %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
# yt-dlp and urllib are noisy at debug level and we drive them deliberately.
|
||||
logging.getLogger("urllib3").setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def human_bytes(value: int | None) -> str:
|
||||
size = float(value or 0)
|
||||
for unit in ("B", "KB", "MB", "GB", "TB"):
|
||||
if size < 1024 or unit == "TB":
|
||||
return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
|
||||
size /= 1024
|
||||
return f"{size:.1f} TB"
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Video row helpers and the state machine.
|
||||
|
||||
States (specs.md §8):
|
||||
|
||||
pending discovered, queued
|
||||
downloading claimed by a worker; recovered to pending on startup
|
||||
downloaded on disk, rel_path set
|
||||
deleted aged out — tombstone, never re-downloaded
|
||||
deferred premiere/upcoming, retried by later polls
|
||||
skipped_short below min_duration_seconds; repaired if later seen in UULF
|
||||
skipped_live livestream, never retried
|
||||
skipped_old already outside the window when discovered
|
||||
failed download error, retried up to max_attempts
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
from datetime import date
|
||||
|
||||
from . import naming, util
|
||||
|
||||
PENDING = "pending"
|
||||
DOWNLOADING = "downloading"
|
||||
DOWNLOADED = "downloaded"
|
||||
DELETED = "deleted"
|
||||
DEFERRED = "deferred"
|
||||
SKIPPED_SHORT = "skipped_short"
|
||||
SKIPPED_LIVE = "skipped_live"
|
||||
SKIPPED_OLD = "skipped_old"
|
||||
FAILED = "failed"
|
||||
|
||||
# States that mean "we have made a final negative decision about this video".
|
||||
# A deleted row is a tombstone and must never be resurrected by any code path.
|
||||
TERMINAL = (DELETED, SKIPPED_LIVE, SKIPPED_OLD)
|
||||
|
||||
SOURCE_UULF = "uulf_feed"
|
||||
SOURCE_UC = "uc_feed"
|
||||
SOURCE_BACKFILL = "backfill"
|
||||
|
||||
|
||||
def get(conn: sqlite3.Connection, video_id: str) -> sqlite3.Row | None:
|
||||
return conn.execute(
|
||||
"SELECT * FROM video WHERE video_id = ?", (video_id,)
|
||||
).fetchone()
|
||||
|
||||
|
||||
def exists(conn: sqlite3.Connection, video_id: str) -> bool:
|
||||
return get(conn, video_id) is not None
|
||||
|
||||
|
||||
def insert(
|
||||
conn: sqlite3.Connection,
|
||||
*,
|
||||
channel_pk: int,
|
||||
video_id: str,
|
||||
title: str,
|
||||
upload_date: str | None,
|
||||
state: str,
|
||||
discovery_source: str,
|
||||
duration: int | None = None,
|
||||
) -> None:
|
||||
with conn:
|
||||
conn.execute(
|
||||
"INSERT OR IGNORE INTO video "
|
||||
"(video_id, channel_pk, title, upload_date, duration, state, "
|
||||
" discovery_source, discovered_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
video_id,
|
||||
channel_pk,
|
||||
title,
|
||||
upload_date,
|
||||
duration,
|
||||
state,
|
||||
discovery_source,
|
||||
util.utcnow_iso(),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def set_state(
|
||||
conn: sqlite3.Connection, video_id: str, state: str, *, error: str | None = None
|
||||
) -> None:
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE video SET state = ?, last_error = ? WHERE video_id = ?",
|
||||
(state, error, video_id),
|
||||
)
|
||||
|
||||
|
||||
def record_failure(conn: sqlite3.Connection, video_id: str, error: str, max_attempts: int) -> str:
|
||||
"""Bump attempts and decide whether to keep retrying."""
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE video SET attempts = attempts + 1, last_error = ? WHERE video_id = ?",
|
||||
(error[:500], video_id),
|
||||
)
|
||||
row = conn.execute(
|
||||
"SELECT attempts FROM video WHERE video_id = ?", (video_id,)
|
||||
).fetchone()
|
||||
attempts = row["attempts"] if row else max_attempts
|
||||
state = FAILED
|
||||
conn.execute(
|
||||
"UPDATE video SET state = ? WHERE video_id = ?", (state, video_id)
|
||||
)
|
||||
return "exhausted" if attempts >= max_attempts else state
|
||||
|
||||
|
||||
def mark_downloaded(
|
||||
conn: sqlite3.Connection,
|
||||
video_id: str,
|
||||
*,
|
||||
rel_path: str,
|
||||
size_bytes: int,
|
||||
season: int,
|
||||
episode: int,
|
||||
upload_date: str,
|
||||
duration: int | None,
|
||||
title: str,
|
||||
) -> None:
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE video SET state = ?, rel_path = ?, size_bytes = ?, season = ?, "
|
||||
"episode = ?, upload_date = ?, duration = ?, title = ?, "
|
||||
"downloaded_at = ?, last_error = NULL WHERE video_id = ?",
|
||||
(
|
||||
DOWNLOADED,
|
||||
rel_path,
|
||||
size_bytes,
|
||||
season,
|
||||
episode,
|
||||
upload_date,
|
||||
duration,
|
||||
title,
|
||||
util.utcnow_iso(),
|
||||
video_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def mark_deleted(conn: sqlite3.Connection, video_id: str) -> None:
|
||||
"""Keep the row — it is the tombstone that prevents re-download."""
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE video SET state = ?, rel_path = NULL, size_bytes = NULL, "
|
||||
"deleted_at = ? WHERE video_id = ?",
|
||||
(DELETED, util.utcnow_iso(), video_id),
|
||||
)
|
||||
|
||||
|
||||
def next_episode(
|
||||
conn: sqlite3.Connection, channel_pk: int, upload_date: date, video_id: str
|
||||
) -> tuple[int, int]:
|
||||
"""Assign (season, episode) for a video.
|
||||
|
||||
The ordinal is computed against what is already in the database for this
|
||||
channel and date — never against the current batch — so it stays stable
|
||||
across runs and across crashes mid-batch.
|
||||
"""
|
||||
season = naming.season_for(upload_date)
|
||||
low, high = naming.episode_range(upload_date)
|
||||
row = conn.execute(
|
||||
"SELECT MAX(episode) AS top FROM video "
|
||||
"WHERE channel_pk = ? AND season = ? AND episode BETWEEN ? AND ? "
|
||||
"AND video_id != ?",
|
||||
(channel_pk, season, low, high, video_id),
|
||||
).fetchone()
|
||||
|
||||
top = row["top"] if row and row["top"] is not None else None
|
||||
if top is None:
|
||||
return season, low
|
||||
if top >= high:
|
||||
# More than ten uploads in a day; naming.episode_number logs the clamp.
|
||||
return season, high
|
||||
return season, top + 1
|
||||
|
||||
|
||||
def claim_pending(
|
||||
conn: sqlite3.Connection, max_attempts: int, limit: int | None = None
|
||||
) -> list[sqlite3.Row]:
|
||||
"""Queue: pending rows, plus failed rows that still have attempts left."""
|
||||
sql = (
|
||||
"SELECT v.*, c.dir_name, c.title AS channel_title, c.channel_id "
|
||||
"FROM video v JOIN channel c ON c.id = v.channel_pk "
|
||||
"WHERE (v.state = ? OR (v.state = ? AND v.attempts < ?)) "
|
||||
"ORDER BY v.upload_date ASC, v.discovered_at ASC"
|
||||
)
|
||||
params: list = [PENDING, FAILED, max_attempts]
|
||||
if limit:
|
||||
sql += " LIMIT ?"
|
||||
params.append(limit)
|
||||
return conn.execute(sql, params).fetchall()
|
||||
|
||||
|
||||
def recover_downloading(conn: sqlite3.Connection) -> int:
|
||||
"""Crash recovery: anything left claimed goes back on the queue."""
|
||||
with conn:
|
||||
cursor = conn.execute(
|
||||
"UPDATE video SET state = ? WHERE state = ?", (PENDING, DOWNLOADING)
|
||||
)
|
||||
return cursor.rowcount
|
||||
|
||||
|
||||
def queue_depth(conn: sqlite3.Connection) -> int:
|
||||
return conn.execute(
|
||||
"SELECT COUNT(*) FROM video WHERE state IN (?, ?, ?)",
|
||||
(PENDING, FAILED, DOWNLOADING),
|
||||
).fetchone()[0]
|
||||
@@ -0,0 +1 @@
|
||||
"""Admin web UI."""
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Password hashing, session cookies and CSRF tokens.
|
||||
|
||||
The admin UI is publicly reachable over HTTPS, so this has to be real. The design
|
||||
goal from specs.md §11 is "log in once per device, effectively never again",
|
||||
which means a long-lived signed cookie rather than HTTP basic auth.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
|
||||
SCRYPT_N = 2**14
|
||||
SCRYPT_R = 8
|
||||
SCRYPT_P = 1
|
||||
DKLEN = 32
|
||||
|
||||
SESSION_MAX_AGE = 365 * 24 * 3600 # one year
|
||||
COOKIE_NAME = "yta_session"
|
||||
|
||||
# Login throttling: after this many consecutive failures from one address, refuse
|
||||
# for LOCKOUT_SECONDS regardless of whether the password is right.
|
||||
MAX_FAILURES = 5
|
||||
LOCKOUT_SECONDS = 60
|
||||
|
||||
|
||||
def _b64(raw: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
|
||||
def _unb64(text: str) -> bytes:
|
||||
padding = "=" * (-len(text) % 4)
|
||||
return base64.urlsafe_b64decode(text + padding)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# passwords
|
||||
|
||||
|
||||
def hash_password(password: str, *, salt: bytes | None = None) -> str:
|
||||
salt = salt if salt is not None else secrets.token_bytes(16)
|
||||
derived = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=salt,
|
||||
n=SCRYPT_N,
|
||||
r=SCRYPT_R,
|
||||
p=SCRYPT_P,
|
||||
dklen=DKLEN,
|
||||
)
|
||||
return f"scrypt${SCRYPT_N}${SCRYPT_R}${SCRYPT_P}${_b64(salt)}${_b64(derived)}"
|
||||
|
||||
|
||||
def verify_password(stored: str, password: str) -> bool:
|
||||
if not stored:
|
||||
return False
|
||||
try:
|
||||
scheme, n, r, p, salt_b64, hash_b64 = stored.split("$")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
derived = hashlib.scrypt(
|
||||
password.encode("utf-8"),
|
||||
salt=_unb64(salt_b64),
|
||||
n=int(n),
|
||||
r=int(r),
|
||||
p=int(p),
|
||||
dklen=len(_unb64(hash_b64)),
|
||||
)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return hmac.compare_digest(derived, _unb64(hash_b64))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# sessions
|
||||
|
||||
|
||||
def new_secret() -> str:
|
||||
return _b64(secrets.token_bytes(32))
|
||||
|
||||
|
||||
def _sign(secret: str, payload: bytes) -> str:
|
||||
return _b64(hmac.new(secret.encode("utf-8"), payload, hashlib.sha256).digest())
|
||||
|
||||
|
||||
def issue_session(secret: str, *, issued_at: float | None = None) -> str:
|
||||
payload = json.dumps(
|
||||
{"iat": int(issued_at if issued_at is not None else time.time())},
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return f"{_b64(payload)}.{_sign(secret, payload)}"
|
||||
|
||||
|
||||
def verify_session(secret: str, token: str, *, now: float | None = None) -> bool:
|
||||
if not token or not secret:
|
||||
return False
|
||||
try:
|
||||
payload_b64, signature = token.split(".", 1)
|
||||
payload = _unb64(payload_b64)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
if not hmac.compare_digest(_sign(secret, payload), signature):
|
||||
return False
|
||||
try:
|
||||
issued_at = int(json.loads(payload)["iat"])
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return False
|
||||
age = (now if now is not None else time.time()) - issued_at
|
||||
return 0 <= age <= SESSION_MAX_AGE
|
||||
|
||||
|
||||
def cookie_header(token: str, *, secure: bool = True) -> str:
|
||||
parts = [
|
||||
f"{COOKIE_NAME}={token}",
|
||||
"Path=/",
|
||||
"HttpOnly",
|
||||
"SameSite=Lax",
|
||||
f"Max-Age={SESSION_MAX_AGE}",
|
||||
]
|
||||
if secure:
|
||||
parts.insert(2, "Secure")
|
||||
return "; ".join(parts)
|
||||
|
||||
|
||||
def clear_cookie_header() -> str:
|
||||
return f"{COOKIE_NAME}=; Path=/; HttpOnly; SameSite=Lax; Max-Age=0"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# CSRF
|
||||
|
||||
|
||||
def csrf_token(secret: str, session_token: str) -> str:
|
||||
return _sign(secret, b"csrf:" + session_token.encode("utf-8"))
|
||||
|
||||
|
||||
def verify_csrf(secret: str, session_token: str, submitted: str) -> bool:
|
||||
if not submitted:
|
||||
return False
|
||||
return hmac.compare_digest(csrf_token(secret, session_token), submitted)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# throttling
|
||||
|
||||
|
||||
class LoginThrottle:
|
||||
"""In-memory consecutive-failure tracker keyed by remote address."""
|
||||
|
||||
def __init__(self, max_failures: int = MAX_FAILURES, lockout: int = LOCKOUT_SECONDS):
|
||||
self.max_failures = max_failures
|
||||
self.lockout = lockout
|
||||
self._state: dict[str, tuple[int, float]] = {}
|
||||
|
||||
def locked(self, key: str, *, now: float | None = None) -> bool:
|
||||
failures, last = self._state.get(key, (0, 0.0))
|
||||
if failures < self.max_failures:
|
||||
return False
|
||||
elapsed = (now if now is not None else time.time()) - last
|
||||
if elapsed >= self.lockout:
|
||||
self._state.pop(key, None)
|
||||
return False
|
||||
return True
|
||||
|
||||
def record_failure(self, key: str, *, now: float | None = None) -> None:
|
||||
failures, _ = self._state.get(key, (0, 0.0))
|
||||
self._state[key] = (failures + 1, now if now is not None else time.time())
|
||||
|
||||
def record_success(self, key: str) -> None:
|
||||
self._state.pop(key, None)
|
||||
@@ -0,0 +1,400 @@
|
||||
"""The admin HTTP server.
|
||||
|
||||
Stdlib only. Binds to localhost; nginx terminates TLS in front of it at
|
||||
tube.jihakuz.xyz. Because that hostname is public, this carries real
|
||||
authentication, CSRF tokens on every state-changing request, and login
|
||||
throttling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import http.cookies
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import urllib.parse
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
from .. import channels, config, db, discovery, jellyfin, util, videos, ytdlp
|
||||
from ..settings import EDITABLE, MASKED_KEYS, Settings, validate_all
|
||||
from . import auth, templates
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
MAX_BODY = 64 * 1024
|
||||
|
||||
|
||||
class AdminServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self, address, handler, *, secure_cookies: bool = True):
|
||||
super().__init__(address, handler)
|
||||
self.throttle = auth.LoginThrottle()
|
||||
self.secure_cookies = secure_cookies
|
||||
self.db_lock = threading.Lock()
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "youtube-automate"
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
# ---------------------------------------------------------------- utils
|
||||
|
||||
def log_message(self, fmt, *args): # noqa: A003 - stdlib signature
|
||||
log.debug("%s - %s", self.client_address[0], fmt % args)
|
||||
|
||||
def _client_key(self) -> str:
|
||||
"""Real client address, since we always sit behind nginx."""
|
||||
forwarded = self.headers.get("X-Forwarded-For", "")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()
|
||||
return self.client_address[0]
|
||||
|
||||
def _send(self, status: int, body: bytes, headers: dict | None = None) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "text/html; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("Referrer-Policy", "same-origin")
|
||||
self.send_header("X-Frame-Options", "DENY")
|
||||
for key, value in (headers or {}).items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _redirect(self, location: str, headers: dict | None = None) -> None:
|
||||
combined = {"Location": location}
|
||||
combined.update(headers or {})
|
||||
self._send(303, b"", combined)
|
||||
|
||||
def _json(self, status: int, payload: dict) -> None:
|
||||
body = json.dumps(payload, indent=2).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _form(self) -> dict[str, str]:
|
||||
length = int(self.headers.get("Content-Length") or 0)
|
||||
if length <= 0 or length > MAX_BODY:
|
||||
return {}
|
||||
raw = self.rfile.read(length).decode("utf-8", "replace")
|
||||
return {
|
||||
key: values[-1]
|
||||
for key, values in urllib.parse.parse_qs(raw, keep_blank_values=True).items()
|
||||
}
|
||||
|
||||
def _cookie_token(self) -> str:
|
||||
header = self.headers.get("Cookie")
|
||||
if not header:
|
||||
return ""
|
||||
jar = http.cookies.SimpleCookie()
|
||||
try:
|
||||
jar.load(header)
|
||||
except http.cookies.CookieError:
|
||||
return ""
|
||||
morsel = jar.get(auth.COOKIE_NAME)
|
||||
return morsel.value if morsel else ""
|
||||
|
||||
# ------------------------------------------------------------- session
|
||||
|
||||
def _open(self):
|
||||
conn = db.connect()
|
||||
return conn, Settings(conn)
|
||||
|
||||
def _secret(self, settings: Settings) -> str:
|
||||
secret = settings.raw("session_secret")
|
||||
if not secret:
|
||||
secret = auth.new_secret()
|
||||
settings.set("session_secret", secret)
|
||||
return secret
|
||||
|
||||
def _authenticated(self, settings: Settings) -> str | None:
|
||||
"""Return the session token if the request is signed in, else None."""
|
||||
token = self._cookie_token()
|
||||
if token and auth.verify_session(self._secret(settings), token):
|
||||
return token
|
||||
return None
|
||||
|
||||
def _check_csrf(self, settings: Settings, token: str, form: dict) -> bool:
|
||||
return auth.verify_csrf(self._secret(settings), token, form.get("csrf", ""))
|
||||
|
||||
# ---------------------------------------------------------------- GET
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802 - stdlib signature
|
||||
path = urllib.parse.urlparse(self.path).path.rstrip("/") or "/"
|
||||
conn, settings = self._open()
|
||||
try:
|
||||
token = self._authenticated(settings)
|
||||
|
||||
# /health was specced as unauthenticated back when the UI was going
|
||||
# to be tailnet-only. On a public hostname it is gratuitous
|
||||
# fingerprinting surface (yt-dlp version, queue depth), so it needs
|
||||
# a session like everything else.
|
||||
if path == "/health":
|
||||
if not token:
|
||||
return self._json(401, {"error": "authentication required"})
|
||||
return self._health(conn, settings)
|
||||
|
||||
if path == "/login":
|
||||
if token:
|
||||
return self._redirect("/")
|
||||
return self._send(200, templates.login_page(self._login_hint(settings)))
|
||||
|
||||
if not token:
|
||||
return self._redirect("/login")
|
||||
|
||||
if path == "/":
|
||||
return self._send(200, self._render_index(conn, settings, token))
|
||||
|
||||
return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _login_hint(self, settings: Settings) -> str | None:
|
||||
if not settings.raw("admin_password_hash"):
|
||||
return "No password is set yet. Run `youtube-automate set-password` on susan."
|
||||
return None
|
||||
|
||||
def _health(self, conn, settings: Settings) -> None:
|
||||
try:
|
||||
ytdlp_version = ytdlp.version()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
ytdlp_version = f"error: {exc}"
|
||||
try:
|
||||
ytdlp.pot_provider_ping(settings.get_str("pot_provider_url"))
|
||||
pot_up = True
|
||||
except Exception: # noqa: BLE001
|
||||
pot_up = False
|
||||
|
||||
self._json(
|
||||
200,
|
||||
{
|
||||
"yt_dlp_version": ytdlp_version,
|
||||
"pot_provider_up": pot_up,
|
||||
"last_run_at": settings.raw("last_run_at") or None,
|
||||
"queue_depth": videos.queue_depth(conn),
|
||||
"channels": len(channels.all_channels(conn)),
|
||||
},
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------- POST
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802 - stdlib signature
|
||||
path = urllib.parse.urlparse(self.path).path.rstrip("/") or "/"
|
||||
conn, settings = self._open()
|
||||
try:
|
||||
form = self._form()
|
||||
|
||||
if path == "/login":
|
||||
return self._login(settings, form)
|
||||
|
||||
token = self._authenticated(settings)
|
||||
if not token:
|
||||
return self._redirect("/login")
|
||||
|
||||
if not self._check_csrf(settings, token, form):
|
||||
log.warning("CSRF check failed for %s from %s", path, self._client_key())
|
||||
return self._send(
|
||||
400,
|
||||
templates.page(
|
||||
"Bad request",
|
||||
"<h1>Bad request</h1><p>Invalid form token. "
|
||||
'<a href="/">Go back</a> and try again.</p>',
|
||||
),
|
||||
)
|
||||
|
||||
if path == "/logout":
|
||||
return self._redirect("/login", {"Set-Cookie": auth.clear_cookie_header()})
|
||||
if path == "/channels":
|
||||
return self._add_channel(conn, settings, token, form)
|
||||
if path == "/settings":
|
||||
return self._save_settings(conn, settings, token, form)
|
||||
|
||||
parts = path.strip("/").split("/")
|
||||
if len(parts) == 3 and parts[0] == "channels" and parts[1].isdigit():
|
||||
pk = int(parts[1])
|
||||
if parts[2] == "delete":
|
||||
return self._delete_channel(conn, settings, pk)
|
||||
if parts[2] == "retention":
|
||||
return self._set_retention(conn, pk, form)
|
||||
if parts[2] == "rescan":
|
||||
return self._rescan(conn, settings, pk)
|
||||
|
||||
return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _login(self, settings: Settings, form: dict) -> None:
|
||||
key = self._client_key()
|
||||
if self.server.throttle.locked(key):
|
||||
return self._send(
|
||||
429, templates.login_page("Too many attempts. Wait a minute.")
|
||||
)
|
||||
|
||||
stored = settings.raw("admin_password_hash")
|
||||
if stored and auth.verify_password(stored, form.get("password", "")):
|
||||
self.server.throttle.record_success(key)
|
||||
token = auth.issue_session(self._secret(settings))
|
||||
return self._redirect(
|
||||
"/",
|
||||
{
|
||||
"Set-Cookie": auth.cookie_header(
|
||||
token, secure=self.server.secure_cookies
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
self.server.throttle.record_failure(key)
|
||||
log.warning("failed login from %s", key)
|
||||
return self._send(
|
||||
401, templates.login_page(self._login_hint(settings) or "Wrong password.")
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------ actions
|
||||
|
||||
def _add_channel(self, conn, settings: Settings, token: str, form: dict) -> None:
|
||||
url = (form.get("url") or "").strip()
|
||||
try:
|
||||
row = channels.subscribe(conn, settings, url)
|
||||
except channels.ResolutionError as exc:
|
||||
body = self._render_index(conn, settings, token, add_error=str(exc))
|
||||
return self._send(400, body)
|
||||
|
||||
self._spawn_backfill(row["id"])
|
||||
return self._redirect("/")
|
||||
|
||||
def _spawn_backfill(self, channel_pk: int) -> None:
|
||||
"""Kick off discovery immediately rather than waiting for the hourly cron."""
|
||||
try:
|
||||
subprocess.Popen( # noqa: S603
|
||||
[sys.executable, "-m", "youtube_automate", "run", "--channel",
|
||||
str(channel_pk)],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
start_new_session=True,
|
||||
)
|
||||
except OSError as exc: # pragma: no cover
|
||||
log.warning("could not spawn backfill for channel %d: %s", channel_pk, exc)
|
||||
|
||||
def _delete_channel(self, conn, settings: Settings, pk: int) -> None:
|
||||
try:
|
||||
channels.unsubscribe(conn, pk)
|
||||
except LookupError:
|
||||
return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
|
||||
jellyfin.from_settings(settings).refresh()
|
||||
return self._redirect("/")
|
||||
|
||||
def _set_retention(self, conn, pk: int, form: dict) -> None:
|
||||
raw = (form.get("days") or "").strip()
|
||||
value: int | None
|
||||
if not raw:
|
||||
value = None
|
||||
else:
|
||||
try:
|
||||
value = max(1, int(raw))
|
||||
except ValueError:
|
||||
return self._redirect("/")
|
||||
with conn:
|
||||
conn.execute(
|
||||
"UPDATE channel SET retention_days = ? WHERE id = ?", (value, pk)
|
||||
)
|
||||
return self._redirect("/")
|
||||
|
||||
def _rescan(self, conn, settings: Settings, pk: int) -> None:
|
||||
channel = channels.get(conn, pk)
|
||||
if channel is None:
|
||||
return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
|
||||
discovery.rescan_channel(conn, settings, channel)
|
||||
self._spawn_backfill(pk)
|
||||
return self._redirect("/")
|
||||
|
||||
def _save_settings(self, conn, settings: Settings, token: str, form: dict) -> None:
|
||||
submitted = {key: form.get(key, "") for key in EDITABLE if key in form}
|
||||
|
||||
# A blank masked field means "keep what is stored", not "clear it".
|
||||
for key in MASKED_KEYS:
|
||||
if key in submitted and not submitted[key].strip():
|
||||
submitted.pop(key)
|
||||
|
||||
errors = validate_all(submitted)
|
||||
if errors:
|
||||
body = self._render_index(
|
||||
conn, settings, token, settings_errors=errors, submitted=submitted
|
||||
)
|
||||
return self._send(400, body)
|
||||
|
||||
for key, value in submitted.items():
|
||||
settings.set(key, value.strip())
|
||||
return self._redirect("/")
|
||||
|
||||
# ------------------------------------------------------------- render
|
||||
|
||||
def _render_index(
|
||||
self,
|
||||
conn,
|
||||
settings: Settings,
|
||||
token: str,
|
||||
*,
|
||||
add_error: str | None = None,
|
||||
settings_errors: dict | None = None,
|
||||
submitted: dict | None = None,
|
||||
) -> bytes:
|
||||
global_retention = settings.get_int("retention_days")
|
||||
rows = []
|
||||
for channel in channels.all_channels(conn):
|
||||
stats = conn.execute(
|
||||
"SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes, "
|
||||
"MAX(upload_date) AS latest FROM video "
|
||||
"WHERE channel_pk = ? AND state = ?",
|
||||
(channel["id"], videos.DOWNLOADED),
|
||||
).fetchone()
|
||||
latest_any = conn.execute(
|
||||
"SELECT MAX(upload_date) AS latest FROM video WHERE channel_pk = ?",
|
||||
(channel["id"],),
|
||||
).fetchone()
|
||||
rows.append(
|
||||
{
|
||||
"id": channel["id"],
|
||||
"title": channel["title"],
|
||||
"handle": channel["handle"],
|
||||
"channel_id": channel["channel_id"],
|
||||
"retention_days": channel["retention_days"],
|
||||
"global_retention": global_retention,
|
||||
"last_polled_at": channel["last_polled_at"],
|
||||
"last_poll_ok": channel["last_poll_ok"],
|
||||
"consecutive_poll_failures": channel["consecutive_poll_failures"],
|
||||
"downloaded": stats["n"],
|
||||
"bytes": stats["bytes"],
|
||||
"latest": latest_any["latest"],
|
||||
}
|
||||
)
|
||||
|
||||
values = settings.all_editable()
|
||||
if submitted:
|
||||
values.update(submitted)
|
||||
|
||||
return templates.index_page(
|
||||
channels=rows,
|
||||
settings_values=values,
|
||||
settings_errors=settings_errors or {},
|
||||
csrf=auth.csrf_token(self._secret(settings), token),
|
||||
add_error=add_error,
|
||||
queue_depth=videos.queue_depth(conn),
|
||||
)
|
||||
|
||||
|
||||
def serve(host: str = "127.0.0.1", port: int = 8085, *, secure_cookies: bool = True) -> None:
|
||||
util.apply_umask()
|
||||
server = AdminServer((host, port), Handler, secure_cookies=secure_cookies)
|
||||
log.info("admin server listening on http://%s:%d", host, port)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt: # pragma: no cover
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Server-rendered HTML. One embedded stylesheet, no JavaScript beyond a
|
||||
confirm() on the destructive buttons."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
from datetime import date
|
||||
|
||||
from .. import util
|
||||
from ..settings import DEFAULTS, EDITABLE, MASKED_KEYS
|
||||
|
||||
STYLE = """
|
||||
:root {
|
||||
--bg: #14161a; --panel: #1c1f26; --line: #2c313b; --text: #e6e8ec;
|
||||
--muted: #99a0ae; --accent: #6aa9ff; --warn: #ffb454; --bad: #ff6b6b;
|
||||
--good: #6ade9b;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; background: var(--bg); color: var(--text);
|
||||
font: 15px/1.5 ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif; }
|
||||
main { max-width: 62rem; margin: 0 auto; padding: 1.5rem 1rem 4rem; }
|
||||
h1 { font-size: 1.4rem; margin: 0; }
|
||||
h2 { font-size: 1.05rem; margin: 2rem 0 .75rem; color: var(--muted);
|
||||
text-transform: uppercase; letter-spacing: .06em; }
|
||||
header { display: flex; align-items: baseline; justify-content: space-between;
|
||||
gap: 1rem; border-bottom: 1px solid var(--line); padding-bottom: .75rem; }
|
||||
header .sub { color: var(--muted); font-size: .85rem; }
|
||||
a { color: var(--accent); }
|
||||
.panel { background: var(--panel); border: 1px solid var(--line);
|
||||
border-radius: 10px; padding: 1rem; }
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th, td { text-align: left; padding: .55rem .5rem; border-bottom: 1px solid var(--line);
|
||||
vertical-align: middle; }
|
||||
th { color: var(--muted); font-weight: 600; font-size: .78rem;
|
||||
text-transform: uppercase; letter-spacing: .05em; }
|
||||
td.num { text-align: right; font-variant-numeric: tabular-nums; }
|
||||
.muted { color: var(--muted); }
|
||||
.badge { display: inline-block; padding: .1rem .45rem; border-radius: 999px;
|
||||
font-size: .75rem; border: 1px solid currentColor; }
|
||||
.badge.warn { color: var(--warn); }
|
||||
.badge.good { color: var(--good); }
|
||||
.badge.bad { color: var(--bad); }
|
||||
input[type=text], input[type=password], input[type=number], select {
|
||||
background: #12141a; color: var(--text); border: 1px solid var(--line);
|
||||
border-radius: 7px; padding: .45rem .55rem; font: inherit; width: 100%; }
|
||||
button { background: var(--accent); color: #0b1017; border: 0; border-radius: 7px;
|
||||
padding: .5rem .9rem; font: inherit; font-weight: 600; cursor: pointer; }
|
||||
button.secondary { background: #2b3140; color: var(--text); }
|
||||
button.danger { background: transparent; color: var(--bad);
|
||||
border: 1px solid var(--bad); font-weight: 500; padding: .3rem .6rem; }
|
||||
button.link { background: transparent; color: var(--accent); border: 0;
|
||||
padding: .3rem .4rem; font-weight: 500; }
|
||||
form.inline { display: inline; }
|
||||
.row { display: flex; gap: .6rem; align-items: center; }
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(15rem, 1fr));
|
||||
gap: .85rem; }
|
||||
label { display: block; font-size: .82rem; color: var(--muted);
|
||||
margin-bottom: .25rem; }
|
||||
.field { margin-bottom: .3rem; }
|
||||
.error { color: var(--bad); font-size: .8rem; margin-top: .2rem; }
|
||||
.flash { border-radius: 8px; padding: .6rem .8rem; margin-bottom: 1rem;
|
||||
border: 1px solid; }
|
||||
.flash.ok { color: var(--good); border-color: var(--good); }
|
||||
.flash.bad { color: var(--bad); border-color: var(--bad); }
|
||||
.login { max-width: 21rem; margin: 6rem auto; }
|
||||
footer { margin-top: 2.5rem; color: var(--muted); font-size: .8rem; }
|
||||
@media (max-width: 40rem) {
|
||||
th.hide, td.hide { display: none; }
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def _e(value) -> str:
|
||||
return html.escape("" if value is None else str(value), quote=True)
|
||||
|
||||
|
||||
def page(title: str, body: str) -> bytes:
|
||||
return f"""<!doctype html>
|
||||
<html lang="en"><head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{_e(title)}</title>
|
||||
<style>{STYLE}</style>
|
||||
</head><body><main>{body}</main></body></html>""".encode("utf-8")
|
||||
|
||||
|
||||
def login_page(error: str | None = None) -> bytes:
|
||||
alert = f'<div class="flash bad">{_e(error)}</div>' if error else ""
|
||||
body = f"""
|
||||
<div class="login">
|
||||
<h1>youtube-automate</h1>
|
||||
<p class="muted">Sign in to manage subscriptions.</p>
|
||||
{alert}
|
||||
<form method="post" action="/login" class="panel">
|
||||
<div class="field">
|
||||
<label for="password">Password</label>
|
||||
<input type="password" id="password" name="password" autofocus
|
||||
autocomplete="current-password">
|
||||
</div>
|
||||
<div style="margin-top:.8rem"><button type="submit">Sign in</button></div>
|
||||
</form>
|
||||
<footer>You will stay signed in on this device for a year.</footer>
|
||||
</div>"""
|
||||
return page("Sign in — youtube-automate", body)
|
||||
|
||||
|
||||
def _channel_row(channel: dict, csrf: str) -> str:
|
||||
failures = channel["consecutive_poll_failures"]
|
||||
if failures > 2:
|
||||
badge = f'<span class="badge bad">{failures} failed polls</span>'
|
||||
elif channel["last_poll_ok"] == 0:
|
||||
badge = '<span class="badge warn">last poll failed</span>'
|
||||
else:
|
||||
badge = ""
|
||||
|
||||
retention = channel["retention_days"]
|
||||
retention_value = "" if retention is None else str(retention)
|
||||
placeholder = f"default ({channel['global_retention']})"
|
||||
|
||||
return f"""
|
||||
<tr>
|
||||
<td>
|
||||
<strong>{_e(channel['title'])}</strong> {badge}<br>
|
||||
<span class="muted">{_e(channel['handle'] or channel['channel_id'])}</span>
|
||||
</td>
|
||||
<td class="num">{channel['downloaded']}</td>
|
||||
<td class="num hide">{_e(util.human_bytes(channel['bytes']))}</td>
|
||||
<td class="hide muted">{_e(channel['latest'] or '—')}</td>
|
||||
<td class="hide muted">{_e(channel['last_polled_at'] or 'never')}</td>
|
||||
<td>
|
||||
<form method="post" action="/channels/{channel['id']}/retention" class="row">
|
||||
<input type="hidden" name="csrf" value="{_e(csrf)}">
|
||||
<input type="number" name="days" min="1" style="width:6.5rem"
|
||||
value="{_e(retention_value)}" placeholder="{_e(placeholder)}">
|
||||
<button class="link" type="submit">save</button>
|
||||
</form>
|
||||
</td>
|
||||
<td>
|
||||
<form method="post" action="/channels/{channel['id']}/rescan" class="inline">
|
||||
<input type="hidden" name="csrf" value="{_e(csrf)}">
|
||||
<button class="link" type="submit"
|
||||
title="Re-queue videos previously skipped as too old that the current
|
||||
retention window now covers">rescan</button>
|
||||
</form>
|
||||
<form method="post" action="/channels/{channel['id']}/delete" class="inline"
|
||||
onsubmit="return confirm('Permanently delete {_e(channel['title'])} and every video downloaded for it? This cannot be undone.');">
|
||||
<input type="hidden" name="csrf" value="{_e(csrf)}">
|
||||
<button class="danger" type="submit">remove</button>
|
||||
</form>
|
||||
</td>
|
||||
</tr>"""
|
||||
|
||||
|
||||
def _settings_form(values: dict, errors: dict, csrf: str) -> str:
|
||||
fields = []
|
||||
for key in EDITABLE:
|
||||
value = values.get(key, DEFAULTS[key])
|
||||
error = errors.get(key)
|
||||
if key in MASKED_KEYS and value:
|
||||
shown, placeholder = "", "stored — leave blank to keep"
|
||||
else:
|
||||
shown, placeholder = value, ""
|
||||
input_type = "password" if key in MASKED_KEYS else "text"
|
||||
fields.append(
|
||||
f"""<div class="field">
|
||||
<label for="{_e(key)}">{_e(key.replace('_', ' '))}</label>
|
||||
<input type="{input_type}" id="{_e(key)}" name="{_e(key)}"
|
||||
value="{_e(shown)}" placeholder="{_e(placeholder)}" autocomplete="off">
|
||||
{f'<div class="error">{_e(error)}</div>' if error else ''}
|
||||
</div>"""
|
||||
)
|
||||
|
||||
return f"""
|
||||
<form method="post" action="/settings" class="panel">
|
||||
<input type="hidden" name="csrf" value="{_e(csrf)}">
|
||||
<div class="grid">{''.join(fields)}</div>
|
||||
<div style="margin-top:1rem"><button type="submit">Save settings</button></div>
|
||||
</form>"""
|
||||
|
||||
|
||||
def index_page(
|
||||
*,
|
||||
channels: list[dict],
|
||||
settings_values: dict,
|
||||
settings_errors: dict,
|
||||
csrf: str,
|
||||
flash: tuple[str, str] | None = None,
|
||||
add_error: str | None = None,
|
||||
queue_depth: int = 0,
|
||||
) -> bytes:
|
||||
flash_html = ""
|
||||
if flash:
|
||||
kind, message = flash
|
||||
flash_html = f'<div class="flash {kind}">{_e(message)}</div>'
|
||||
|
||||
if channels:
|
||||
rows = "".join(_channel_row(channel, csrf) for channel in channels)
|
||||
table = f"""
|
||||
<div class="panel">
|
||||
<table>
|
||||
<thead><tr>
|
||||
<th>Channel</th><th class="num">On disk</th><th class="num hide">Size</th>
|
||||
<th class="hide">Latest upload</th><th class="hide">Last poll</th>
|
||||
<th>Retention (days)</th><th></th>
|
||||
</tr></thead>
|
||||
<tbody>{rows}</tbody>
|
||||
</table>
|
||||
</div>"""
|
||||
else:
|
||||
table = '<div class="panel muted">No channels yet. Add one below.</div>'
|
||||
|
||||
add_error_html = f'<div class="error">{_e(add_error)}</div>' if add_error else ""
|
||||
|
||||
body = f"""
|
||||
<header>
|
||||
<h1>youtube-automate</h1>
|
||||
<div class="sub">
|
||||
{len(channels)} channel(s) · {queue_depth} queued
|
||||
· <form method="post" action="/logout" class="inline">
|
||||
<input type="hidden" name="csrf" value="{_e(csrf)}">
|
||||
<button class="link" type="submit">sign out</button>
|
||||
</form>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{flash_html}
|
||||
|
||||
<h2>Channels</h2>
|
||||
{table}
|
||||
|
||||
<h2>Add a channel</h2>
|
||||
<form method="post" action="/channels" class="panel">
|
||||
<input type="hidden" name="csrf" value="{_e(csrf)}">
|
||||
<div class="row">
|
||||
<input type="text" name="url" placeholder="https://www.youtube.com/@handle, @handle, or UC..." autocomplete="off">
|
||||
<button type="submit">Add</button>
|
||||
</div>
|
||||
{add_error_html}
|
||||
</form>
|
||||
|
||||
<h2>Settings</h2>
|
||||
{_settings_form(settings_values, settings_errors, csrf)}
|
||||
|
||||
<footer>Downloads run hourly. Videos are deleted once they pass the retention
|
||||
window for their channel — this is a DVR, not an archive.</footer>"""
|
||||
return page("youtube-automate", body)
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Thin wrapper around the venv's yt-dlp binary.
|
||||
|
||||
Everything that shells out to yt-dlp goes through here so the PATH handling (Deno
|
||||
must be discoverable — see specs.md §3) and the shared extractor args live in one
|
||||
place.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from . import config
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class YtdlpError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def binary() -> Path:
|
||||
return config.VENV_BIN / "yt-dlp"
|
||||
|
||||
|
||||
def environment() -> dict[str, str]:
|
||||
"""Env for a yt-dlp subprocess.
|
||||
|
||||
yt-dlp locates the JS runtime by searching PATH, so the venv's bin directory
|
||||
must come first — that is where Deno lives.
|
||||
"""
|
||||
env = dict(os.environ)
|
||||
env["PATH"] = f"{config.VENV_BIN}:{env.get('PATH', '')}"
|
||||
return env
|
||||
|
||||
|
||||
def version() -> str:
|
||||
result = subprocess.run(
|
||||
[str(binary()), "--version"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment(),
|
||||
timeout=60,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise YtdlpError(result.stderr.strip() or "yt-dlp --version failed")
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def extractor_args(pot_provider_url: str) -> list[str]:
|
||||
return [
|
||||
"--extractor-args",
|
||||
"youtube:player_client=default,mweb",
|
||||
"--extractor-args",
|
||||
f"youtubepot-bgutilhttp:base_url={pot_provider_url}",
|
||||
]
|
||||
|
||||
|
||||
def run_json(args: list[str], timeout: int = 300) -> dict:
|
||||
"""Run yt-dlp with -J and parse the single JSON document it prints."""
|
||||
cmd = [str(binary()), *args]
|
||||
log.debug("yt-dlp %s", " ".join(args))
|
||||
result = subprocess.run(
|
||||
cmd, capture_output=True, text=True, env=environment(), timeout=timeout
|
||||
)
|
||||
if result.returncode != 0:
|
||||
raise YtdlpError(first_error(result.stderr) or "yt-dlp failed")
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise YtdlpError(f"could not parse yt-dlp output: {exc}") from exc
|
||||
|
||||
|
||||
def run(args: list[str], timeout: int = 7200) -> subprocess.CompletedProcess:
|
||||
"""Run yt-dlp for its side effects, returning the completed process."""
|
||||
cmd = [str(binary()), *args]
|
||||
log.debug("yt-dlp %s", " ".join(args))
|
||||
return subprocess.run(
|
||||
cmd, capture_output=True, text=True, env=environment(), timeout=timeout
|
||||
)
|
||||
|
||||
|
||||
def first_error(stderr: str) -> str:
|
||||
for line in (stderr or "").splitlines():
|
||||
if line.startswith("ERROR:"):
|
||||
return line[len("ERROR:") :].strip()
|
||||
return (stderr or "").strip().splitlines()[-1] if stderr.strip() else ""
|
||||
|
||||
|
||||
def pot_provider_ping(base_url: str, timeout: float = 5.0) -> dict:
|
||||
"""GET /ping on the bgutil provider. Raises on any failure."""
|
||||
url = base_url.rstrip("/") + "/ping"
|
||||
request = urllib.request.Request(url, headers={"User-Agent": config.USER_AGENT})
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
|
||||
|
||||
def plugin_version() -> str | None:
|
||||
"""Installed bgutil plugin version, for comparison against the server's."""
|
||||
try:
|
||||
from importlib.metadata import version as pkg_version
|
||||
|
||||
return pkg_version("bgutil-ytdlp-pot-provider")
|
||||
except Exception: # pragma: no cover - only when the plugin is absent
|
||||
return None
|
||||
Reference in New Issue
Block a user