Initial implementation of youtube-automate

A DVR for YouTube subscriptions, laid out so Jellyfin presents each channel as a show and each video as an episode. Cron-driven, idempotent, with a public admin UI for a non-operator. Verified end to end on susan against three real channels: PO tokens, h264 downloads, Jellyfin resolution from local NFOs with all providers disabled, retention and tombstones. Corrections to the original design handover (specs.md documents each with the evidence, and specs.handover-original.md preserves the original): - The format sort selected 360p. Ranking acodec above res makes `bv*` prefer the combined 360p stream, which carries AAC, over the 720p video-only stream whose acodec is none. vcodec now leads, so a video without h264 at 720p yields h264 lower down rather than VP9 this hardware cannot transcode. - yt-dlp now requires a JS runtime and the yt-dlp-ejs solver scripts, which only ship with the [default] extra. Without them the n challenge fails and the mweb formats disappear entirely. - --flat-playlist carries no upload dates, so the specced client-side date filter for backfill was impossible. Backfill is RSS-first. - skipped_old was terminal, so raising a channel's retention appeared to do nothing. Added an explicit rescan. - is_upcoming premieres now defer and retry instead of being skipped forever. - TubeArchivist is gone, so the media root and the tube.jihakuz.xyz vhost were both reclaimed; the latter still pointed at its dead port. 240 offline tests, no network and no real yt-dlp invocation. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-11 21:42:48 +01:00
commit 18bb2e420b
44 changed files with 7188 additions and 0 deletions
@@ -0,0 +1,400 @@
+"""The admin HTTP server.
+
+Stdlib only. Binds to localhost; nginx terminates TLS in front of it at
+tube.jihakuz.xyz. Because that hostname is public, this carries real
+authentication, CSRF tokens on every state-changing request, and login
+throttling.
+"""
+
+from __future__ import annotations
+
+import http.cookies
+import json
+import logging
+import subprocess
+import sys
+import threading
+import urllib.parse
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+from .. import channels, config, db, discovery, jellyfin, util, videos, ytdlp
+from ..settings import EDITABLE, MASKED_KEYS, Settings, validate_all
+from . import auth, templates
+
+log = logging.getLogger(__name__)
+
+MAX_BODY = 64 * 1024
+
+
+class AdminServer(ThreadingHTTPServer):
+    daemon_threads = True
+    allow_reuse_address = True
+
+    def __init__(self, address, handler, *, secure_cookies: bool = True):
+        super().__init__(address, handler)
+        self.throttle = auth.LoginThrottle()
+        self.secure_cookies = secure_cookies
+        self.db_lock = threading.Lock()
+
+
+class Handler(BaseHTTPRequestHandler):
+    server_version = "youtube-automate"
+    protocol_version = "HTTP/1.1"
+
+    # ---------------------------------------------------------------- utils
+
+    def log_message(self, fmt, *args):  # noqa: A003 - stdlib signature
+        log.debug("%s - %s", self.client_address[0], fmt % args)
+
+    def _client_key(self) -> str:
+        """Real client address, since we always sit behind nginx."""
+        forwarded = self.headers.get("X-Forwarded-For", "")
+        if forwarded:
+            return forwarded.split(",")[0].strip()
+        return self.client_address[0]
+
+    def _send(self, status: int, body: bytes, headers: dict | None = None) -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("X-Content-Type-Options", "nosniff")
+        self.send_header("Referrer-Policy", "same-origin")
+        self.send_header("X-Frame-Options", "DENY")
+        for key, value in (headers or {}).items():
+            self.send_header(key, value)
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _redirect(self, location: str, headers: dict | None = None) -> None:
+        combined = {"Location": location}
+        combined.update(headers or {})
+        self._send(303, b"", combined)
+
+    def _json(self, status: int, payload: dict) -> None:
+        body = json.dumps(payload, indent=2).encode("utf-8")
+        self.send_response(status)
+        self.send_header("Content-Type", "application/json")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _form(self) -> dict[str, str]:
+        length = int(self.headers.get("Content-Length") or 0)
+        if length <= 0 or length > MAX_BODY:
+            return {}
+        raw = self.rfile.read(length).decode("utf-8", "replace")
+        return {
+            key: values[-1]
+            for key, values in urllib.parse.parse_qs(raw, keep_blank_values=True).items()
+        }
+
+    def _cookie_token(self) -> str:
+        header = self.headers.get("Cookie")
+        if not header:
+            return ""
+        jar = http.cookies.SimpleCookie()
+        try:
+            jar.load(header)
+        except http.cookies.CookieError:
+            return ""
+        morsel = jar.get(auth.COOKIE_NAME)
+        return morsel.value if morsel else ""
+
+    # ------------------------------------------------------------- session
+
+    def _open(self):
+        conn = db.connect()
+        return conn, Settings(conn)
+
+    def _secret(self, settings: Settings) -> str:
+        secret = settings.raw("session_secret")
+        if not secret:
+            secret = auth.new_secret()
+            settings.set("session_secret", secret)
+        return secret
+
+    def _authenticated(self, settings: Settings) -> str | None:
+        """Return the session token if the request is signed in, else None."""
+        token = self._cookie_token()
+        if token and auth.verify_session(self._secret(settings), token):
+            return token
+        return None
+
+    def _check_csrf(self, settings: Settings, token: str, form: dict) -> bool:
+        return auth.verify_csrf(self._secret(settings), token, form.get("csrf", ""))
+
+    # ---------------------------------------------------------------- GET
+
+    def do_GET(self) -> None:  # noqa: N802 - stdlib signature
+        path = urllib.parse.urlparse(self.path).path.rstrip("/") or "/"
+        conn, settings = self._open()
+        try:
+            token = self._authenticated(settings)
+
+            # /health was specced as unauthenticated back when the UI was going
+            # to be tailnet-only. On a public hostname it is gratuitous
+            # fingerprinting surface (yt-dlp version, queue depth), so it needs
+            # a session like everything else.
+            if path == "/health":
+                if not token:
+                    return self._json(401, {"error": "authentication required"})
+                return self._health(conn, settings)
+
+            if path == "/login":
+                if token:
+                    return self._redirect("/")
+                return self._send(200, templates.login_page(self._login_hint(settings)))
+
+            if not token:
+                return self._redirect("/login")
+
+            if path == "/":
+                return self._send(200, self._render_index(conn, settings, token))
+
+            return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
+        finally:
+            conn.close()
+
+    def _login_hint(self, settings: Settings) -> str | None:
+        if not settings.raw("admin_password_hash"):
+            return "No password is set yet. Run `youtube-automate set-password` on susan."
+        return None
+
+    def _health(self, conn, settings: Settings) -> None:
+        try:
+            ytdlp_version = ytdlp.version()
+        except Exception as exc:  # noqa: BLE001
+            ytdlp_version = f"error: {exc}"
+        try:
+            ytdlp.pot_provider_ping(settings.get_str("pot_provider_url"))
+            pot_up = True
+        except Exception:  # noqa: BLE001
+            pot_up = False
+
+        self._json(
+            200,
+            {
+                "yt_dlp_version": ytdlp_version,
+                "pot_provider_up": pot_up,
+                "last_run_at": settings.raw("last_run_at") or None,
+                "queue_depth": videos.queue_depth(conn),
+                "channels": len(channels.all_channels(conn)),
+            },
+        )
+
+    # --------------------------------------------------------------- POST
+
+    def do_POST(self) -> None:  # noqa: N802 - stdlib signature
+        path = urllib.parse.urlparse(self.path).path.rstrip("/") or "/"
+        conn, settings = self._open()
+        try:
+            form = self._form()
+
+            if path == "/login":
+                return self._login(settings, form)
+
+            token = self._authenticated(settings)
+            if not token:
+                return self._redirect("/login")
+
+            if not self._check_csrf(settings, token, form):
+                log.warning("CSRF check failed for %s from %s", path, self._client_key())
+                return self._send(
+                    400,
+                    templates.page(
+                        "Bad request",
+                        "<h1>Bad request</h1><p>Invalid form token. "
+                        '<a href="/">Go back</a> and try again.</p>',
+                    ),
+                )
+
+            if path == "/logout":
+                return self._redirect("/login", {"Set-Cookie": auth.clear_cookie_header()})
+            if path == "/channels":
+                return self._add_channel(conn, settings, token, form)
+            if path == "/settings":
+                return self._save_settings(conn, settings, token, form)
+
+            parts = path.strip("/").split("/")
+            if len(parts) == 3 and parts[0] == "channels" and parts[1].isdigit():
+                pk = int(parts[1])
+                if parts[2] == "delete":
+                    return self._delete_channel(conn, settings, pk)
+                if parts[2] == "retention":
+                    return self._set_retention(conn, pk, form)
+                if parts[2] == "rescan":
+                    return self._rescan(conn, settings, pk)
+
+            return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
+        finally:
+            conn.close()
+
+    def _login(self, settings: Settings, form: dict) -> None:
+        key = self._client_key()
+        if self.server.throttle.locked(key):
+            return self._send(
+                429, templates.login_page("Too many attempts. Wait a minute.")
+            )
+
+        stored = settings.raw("admin_password_hash")
+        if stored and auth.verify_password(stored, form.get("password", "")):
+            self.server.throttle.record_success(key)
+            token = auth.issue_session(self._secret(settings))
+            return self._redirect(
+                "/",
+                {
+                    "Set-Cookie": auth.cookie_header(
+                        token, secure=self.server.secure_cookies
+                    )
+                },
+            )
+
+        self.server.throttle.record_failure(key)
+        log.warning("failed login from %s", key)
+        return self._send(
+            401, templates.login_page(self._login_hint(settings) or "Wrong password.")
+        )
+
+    # ------------------------------------------------------------ actions
+
+    def _add_channel(self, conn, settings: Settings, token: str, form: dict) -> None:
+        url = (form.get("url") or "").strip()
+        try:
+            row = channels.subscribe(conn, settings, url)
+        except channels.ResolutionError as exc:
+            body = self._render_index(conn, settings, token, add_error=str(exc))
+            return self._send(400, body)
+
+        self._spawn_backfill(row["id"])
+        return self._redirect("/")
+
+    def _spawn_backfill(self, channel_pk: int) -> None:
+        """Kick off discovery immediately rather than waiting for the hourly cron."""
+        try:
+            subprocess.Popen(  # noqa: S603
+                [sys.executable, "-m", "youtube_automate", "run", "--channel",
+                 str(channel_pk)],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                start_new_session=True,
+            )
+        except OSError as exc:  # pragma: no cover
+            log.warning("could not spawn backfill for channel %d: %s", channel_pk, exc)
+
+    def _delete_channel(self, conn, settings: Settings, pk: int) -> None:
+        try:
+            channels.unsubscribe(conn, pk)
+        except LookupError:
+            return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
+        jellyfin.from_settings(settings).refresh()
+        return self._redirect("/")
+
+    def _set_retention(self, conn, pk: int, form: dict) -> None:
+        raw = (form.get("days") or "").strip()
+        value: int | None
+        if not raw:
+            value = None
+        else:
+            try:
+                value = max(1, int(raw))
+            except ValueError:
+                return self._redirect("/")
+        with conn:
+            conn.execute(
+                "UPDATE channel SET retention_days = ? WHERE id = ?", (value, pk)
+            )
+        return self._redirect("/")
+
+    def _rescan(self, conn, settings: Settings, pk: int) -> None:
+        channel = channels.get(conn, pk)
+        if channel is None:
+            return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
+        discovery.rescan_channel(conn, settings, channel)
+        self._spawn_backfill(pk)
+        return self._redirect("/")
+
+    def _save_settings(self, conn, settings: Settings, token: str, form: dict) -> None:
+        submitted = {key: form.get(key, "") for key in EDITABLE if key in form}
+
+        # A blank masked field means "keep what is stored", not "clear it".
+        for key in MASKED_KEYS:
+            if key in submitted and not submitted[key].strip():
+                submitted.pop(key)
+
+        errors = validate_all(submitted)
+        if errors:
+            body = self._render_index(
+                conn, settings, token, settings_errors=errors, submitted=submitted
+            )
+            return self._send(400, body)
+
+        for key, value in submitted.items():
+            settings.set(key, value.strip())
+        return self._redirect("/")
+
+    # ------------------------------------------------------------- render
+
+    def _render_index(
+        self,
+        conn,
+        settings: Settings,
+        token: str,
+        *,
+        add_error: str | None = None,
+        settings_errors: dict | None = None,
+        submitted: dict | None = None,
+    ) -> bytes:
+        global_retention = settings.get_int("retention_days")
+        rows = []
+        for channel in channels.all_channels(conn):
+            stats = conn.execute(
+                "SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes, "
+                "MAX(upload_date) AS latest FROM video "
+                "WHERE channel_pk = ? AND state = ?",
+                (channel["id"], videos.DOWNLOADED),
+            ).fetchone()
+            latest_any = conn.execute(
+                "SELECT MAX(upload_date) AS latest FROM video WHERE channel_pk = ?",
+                (channel["id"],),
+            ).fetchone()
+            rows.append(
+                {
+                    "id": channel["id"],
+                    "title": channel["title"],
+                    "handle": channel["handle"],
+                    "channel_id": channel["channel_id"],
+                    "retention_days": channel["retention_days"],
+                    "global_retention": global_retention,
+                    "last_polled_at": channel["last_polled_at"],
+                    "last_poll_ok": channel["last_poll_ok"],
+                    "consecutive_poll_failures": channel["consecutive_poll_failures"],
+                    "downloaded": stats["n"],
+                    "bytes": stats["bytes"],
+                    "latest": latest_any["latest"],
+                }
+            )
+
+        values = settings.all_editable()
+        if submitted:
+            values.update(submitted)
+
+        return templates.index_page(
+            channels=rows,
+            settings_values=values,
+            settings_errors=settings_errors or {},
+            csrf=auth.csrf_token(self._secret(settings), token),
+            add_error=add_error,
+            queue_depth=videos.queue_depth(conn),
+        )
+
+
+def serve(host: str = "127.0.0.1", port: int = 8085, *, secure_cookies: bool = True) -> None:
+    util.apply_umask()
+    server = AdminServer((host, port), Handler, secure_cookies=secure_cookies)
+    log.info("admin server listening on http://%s:%d", host, port)
+    try:
+        server.serve_forever()
+    except KeyboardInterrupt:  # pragma: no cover
+        pass
+    finally:
+        server.server_close()