Tom FluxandClaude Opus 5
96f8d89442
The admin UI accepted the password, issued a valid session cookie, then
bounced straight back to the login page with no error shown.
http.cookies.SimpleCookie.load() silently discards the remainder of a Cookie
header the moment it meets a value it considers illegal, rather than raising
or skipping just that entry. A neighbouring cookie with a JSON-ish value such
as prefs={"a":1} is enough. Everything after it — including our session —
becomes invisible, so a perfectly good login looked like a failed one.
curl never showed it because curl only sends the one cookie. A browser sends
every cookie on the domain, and tube.jihakuz.xyz previously served
TubeArchivist alongside several sibling services on jihakuz.xyz.
Replaced with a tolerant hand-rolled parser and covered it with regression
tests for nine hostile neighbour values.
Also log successful logins at INFO. Only failures were logged, which made
"password rejected" and "session did not stick" indistinguishable from the
journal and sent the diagnosis down the wrong path.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
398 lines
15 KiB
Python
398 lines
15 KiB
Python
"""The admin HTTP server.
|
|
|
|
Stdlib only. Binds to localhost; nginx terminates TLS in front of it at
|
|
tube.jihakuz.xyz. Because that hostname is public, this carries real
|
|
authentication, CSRF tokens on every state-changing request, and login
|
|
throttling.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import urllib.parse
|
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
|
|
from .. import channels, config, db, discovery, jellyfin, util, videos, ytdlp
|
|
from ..settings import EDITABLE, MASKED_KEYS, Settings, validate_all
|
|
from . import auth, templates
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
MAX_BODY = 64 * 1024
|
|
|
|
|
|
class AdminServer(ThreadingHTTPServer):
|
|
daemon_threads = True
|
|
allow_reuse_address = True
|
|
|
|
def __init__(self, address, handler, *, secure_cookies: bool = True):
|
|
super().__init__(address, handler)
|
|
self.throttle = auth.LoginThrottle()
|
|
self.secure_cookies = secure_cookies
|
|
self.db_lock = threading.Lock()
|
|
|
|
|
|
class Handler(BaseHTTPRequestHandler):
|
|
server_version = "youtube-automate"
|
|
protocol_version = "HTTP/1.1"
|
|
|
|
# ---------------------------------------------------------------- utils
|
|
|
|
def log_message(self, fmt, *args): # noqa: A003 - stdlib signature
|
|
log.debug("%s - %s", self.client_address[0], fmt % args)
|
|
|
|
def _client_key(self) -> str:
|
|
"""Real client address, since we always sit behind nginx."""
|
|
forwarded = self.headers.get("X-Forwarded-For", "")
|
|
if forwarded:
|
|
return forwarded.split(",")[0].strip()
|
|
return self.client_address[0]
|
|
|
|
def _send(self, status: int, body: bytes, headers: dict | None = None) -> None:
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "text/html; charset=utf-8")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.send_header("X-Content-Type-Options", "nosniff")
|
|
self.send_header("Referrer-Policy", "same-origin")
|
|
self.send_header("X-Frame-Options", "DENY")
|
|
for key, value in (headers or {}).items():
|
|
self.send_header(key, value)
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def _redirect(self, location: str, headers: dict | None = None) -> None:
|
|
combined = {"Location": location}
|
|
combined.update(headers or {})
|
|
self._send(303, b"", combined)
|
|
|
|
def _json(self, status: int, payload: dict) -> None:
|
|
body = json.dumps(payload, indent=2).encode("utf-8")
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", "application/json")
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def _form(self) -> dict[str, str]:
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
if length <= 0 or length > MAX_BODY:
|
|
return {}
|
|
raw = self.rfile.read(length).decode("utf-8", "replace")
|
|
return {
|
|
key: values[-1]
|
|
for key, values in urllib.parse.parse_qs(raw, keep_blank_values=True).items()
|
|
}
|
|
|
|
def _cookie_token(self) -> str:
|
|
return auth.cookie_value(self.headers.get("Cookie") or "")
|
|
|
|
# ------------------------------------------------------------- session
|
|
|
|
def _open(self):
|
|
conn = db.connect()
|
|
return conn, Settings(conn)
|
|
|
|
def _secret(self, settings: Settings) -> str:
|
|
secret = settings.raw("session_secret")
|
|
if not secret:
|
|
secret = auth.new_secret()
|
|
settings.set("session_secret", secret)
|
|
return secret
|
|
|
|
def _authenticated(self, settings: Settings) -> str | None:
|
|
"""Return the session token if the request is signed in, else None."""
|
|
token = self._cookie_token()
|
|
if token and auth.verify_session(self._secret(settings), token):
|
|
return token
|
|
return None
|
|
|
|
def _check_csrf(self, settings: Settings, token: str, form: dict) -> bool:
|
|
return auth.verify_csrf(self._secret(settings), token, form.get("csrf", ""))
|
|
|
|
# ---------------------------------------------------------------- GET
|
|
|
|
def do_GET(self) -> None: # noqa: N802 - stdlib signature
|
|
path = urllib.parse.urlparse(self.path).path.rstrip("/") or "/"
|
|
conn, settings = self._open()
|
|
try:
|
|
token = self._authenticated(settings)
|
|
|
|
# /health was specced as unauthenticated back when the UI was going
|
|
# to be tailnet-only. On a public hostname it is gratuitous
|
|
# fingerprinting surface (yt-dlp version, queue depth), so it needs
|
|
# a session like everything else.
|
|
if path == "/health":
|
|
if not token:
|
|
return self._json(401, {"error": "authentication required"})
|
|
return self._health(conn, settings)
|
|
|
|
if path == "/login":
|
|
if token:
|
|
return self._redirect("/")
|
|
return self._send(200, templates.login_page(self._login_hint(settings)))
|
|
|
|
if not token:
|
|
return self._redirect("/login")
|
|
|
|
if path == "/":
|
|
return self._send(200, self._render_index(conn, settings, token))
|
|
|
|
return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
|
|
finally:
|
|
conn.close()
|
|
|
|
def _login_hint(self, settings: Settings) -> str | None:
|
|
if not settings.raw("admin_password_hash"):
|
|
return "No password is set yet. Run `youtube-automate set-password` on susan."
|
|
return None
|
|
|
|
def _health(self, conn, settings: Settings) -> None:
|
|
try:
|
|
ytdlp_version = ytdlp.version()
|
|
except Exception as exc: # noqa: BLE001
|
|
ytdlp_version = f"error: {exc}"
|
|
try:
|
|
ytdlp.pot_provider_ping(settings.get_str("pot_provider_url"))
|
|
pot_up = True
|
|
except Exception: # noqa: BLE001
|
|
pot_up = False
|
|
|
|
self._json(
|
|
200,
|
|
{
|
|
"yt_dlp_version": ytdlp_version,
|
|
"pot_provider_up": pot_up,
|
|
"last_run_at": settings.raw("last_run_at") or None,
|
|
"queue_depth": videos.queue_depth(conn),
|
|
"channels": len(channels.all_channels(conn)),
|
|
},
|
|
)
|
|
|
|
# --------------------------------------------------------------- POST
|
|
|
|
def do_POST(self) -> None: # noqa: N802 - stdlib signature
|
|
path = urllib.parse.urlparse(self.path).path.rstrip("/") or "/"
|
|
conn, settings = self._open()
|
|
try:
|
|
form = self._form()
|
|
|
|
if path == "/login":
|
|
return self._login(settings, form)
|
|
|
|
token = self._authenticated(settings)
|
|
if not token:
|
|
return self._redirect("/login")
|
|
|
|
if not self._check_csrf(settings, token, form):
|
|
log.warning("CSRF check failed for %s from %s", path, self._client_key())
|
|
return self._send(
|
|
400,
|
|
templates.page(
|
|
"Bad request",
|
|
"<h1>Bad request</h1><p>Invalid form token. "
|
|
'<a href="/">Go back</a> and try again.</p>',
|
|
),
|
|
)
|
|
|
|
if path == "/logout":
|
|
return self._redirect("/login", {"Set-Cookie": auth.clear_cookie_header()})
|
|
if path == "/channels":
|
|
return self._add_channel(conn, settings, token, form)
|
|
if path == "/settings":
|
|
return self._save_settings(conn, settings, token, form)
|
|
|
|
parts = path.strip("/").split("/")
|
|
if len(parts) == 3 and parts[0] == "channels" and parts[1].isdigit():
|
|
pk = int(parts[1])
|
|
if parts[2] == "delete":
|
|
return self._delete_channel(conn, settings, pk)
|
|
if parts[2] == "retention":
|
|
return self._set_retention(conn, pk, form)
|
|
if parts[2] == "rescan":
|
|
return self._rescan(conn, settings, pk)
|
|
|
|
return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
|
|
finally:
|
|
conn.close()
|
|
|
|
def _login(self, settings: Settings, form: dict) -> None:
|
|
key = self._client_key()
|
|
if self.server.throttle.locked(key):
|
|
return self._send(
|
|
429, templates.login_page("Too many attempts. Wait a minute.")
|
|
)
|
|
|
|
stored = settings.raw("admin_password_hash")
|
|
if not stored:
|
|
log.warning("login attempted from %s but no password is set", key)
|
|
|
|
if stored and auth.verify_password(stored, form.get("password", "")):
|
|
self.server.throttle.record_success(key)
|
|
# Logged at INFO so "did my login work?" is answerable from the
|
|
# journal. A rejected login only ever re-renders the form, which
|
|
# looks identical to a session that failed to stick.
|
|
log.info("successful login from %s", key)
|
|
token = auth.issue_session(self._secret(settings))
|
|
return self._redirect(
|
|
"/",
|
|
{
|
|
"Set-Cookie": auth.cookie_header(
|
|
token, secure=self.server.secure_cookies
|
|
)
|
|
},
|
|
)
|
|
|
|
self.server.throttle.record_failure(key)
|
|
log.warning("failed login from %s", key)
|
|
return self._send(
|
|
401, templates.login_page(self._login_hint(settings) or "Wrong password.")
|
|
)
|
|
|
|
# ------------------------------------------------------------ actions
|
|
|
|
def _add_channel(self, conn, settings: Settings, token: str, form: dict) -> None:
|
|
url = (form.get("url") or "").strip()
|
|
try:
|
|
row = channels.subscribe(conn, settings, url)
|
|
except channels.ResolutionError as exc:
|
|
body = self._render_index(conn, settings, token, add_error=str(exc))
|
|
return self._send(400, body)
|
|
|
|
self._spawn_backfill(row["id"])
|
|
return self._redirect("/")
|
|
|
|
def _spawn_backfill(self, channel_pk: int) -> None:
|
|
"""Kick off discovery immediately rather than waiting for the hourly cron."""
|
|
try:
|
|
subprocess.Popen( # noqa: S603
|
|
[sys.executable, "-m", "youtube_automate", "run", "--channel",
|
|
str(channel_pk)],
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
start_new_session=True,
|
|
)
|
|
except OSError as exc: # pragma: no cover
|
|
log.warning("could not spawn backfill for channel %d: %s", channel_pk, exc)
|
|
|
|
def _delete_channel(self, conn, settings: Settings, pk: int) -> None:
|
|
try:
|
|
channels.unsubscribe(conn, pk)
|
|
except LookupError:
|
|
return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
|
|
jellyfin.from_settings(settings).refresh()
|
|
return self._redirect("/")
|
|
|
|
def _set_retention(self, conn, pk: int, form: dict) -> None:
|
|
raw = (form.get("days") or "").strip()
|
|
value: int | None
|
|
if not raw:
|
|
value = None
|
|
else:
|
|
try:
|
|
value = max(1, int(raw))
|
|
except ValueError:
|
|
return self._redirect("/")
|
|
with conn:
|
|
conn.execute(
|
|
"UPDATE channel SET retention_days = ? WHERE id = ?", (value, pk)
|
|
)
|
|
return self._redirect("/")
|
|
|
|
def _rescan(self, conn, settings: Settings, pk: int) -> None:
|
|
channel = channels.get(conn, pk)
|
|
if channel is None:
|
|
return self._send(404, templates.page("Not found", "<h1>Not found</h1>"))
|
|
discovery.rescan_channel(conn, settings, channel)
|
|
self._spawn_backfill(pk)
|
|
return self._redirect("/")
|
|
|
|
def _save_settings(self, conn, settings: Settings, token: str, form: dict) -> None:
|
|
submitted = {key: form.get(key, "") for key in EDITABLE if key in form}
|
|
|
|
# A blank masked field means "keep what is stored", not "clear it".
|
|
for key in MASKED_KEYS:
|
|
if key in submitted and not submitted[key].strip():
|
|
submitted.pop(key)
|
|
|
|
errors = validate_all(submitted)
|
|
if errors:
|
|
body = self._render_index(
|
|
conn, settings, token, settings_errors=errors, submitted=submitted
|
|
)
|
|
return self._send(400, body)
|
|
|
|
for key, value in submitted.items():
|
|
settings.set(key, value.strip())
|
|
return self._redirect("/")
|
|
|
|
# ------------------------------------------------------------- render
|
|
|
|
def _render_index(
|
|
self,
|
|
conn,
|
|
settings: Settings,
|
|
token: str,
|
|
*,
|
|
add_error: str | None = None,
|
|
settings_errors: dict | None = None,
|
|
submitted: dict | None = None,
|
|
) -> bytes:
|
|
global_retention = settings.get_int("retention_days")
|
|
rows = []
|
|
for channel in channels.all_channels(conn):
|
|
stats = conn.execute(
|
|
"SELECT COUNT(*) AS n, COALESCE(SUM(size_bytes), 0) AS bytes, "
|
|
"MAX(upload_date) AS latest FROM video "
|
|
"WHERE channel_pk = ? AND state = ?",
|
|
(channel["id"], videos.DOWNLOADED),
|
|
).fetchone()
|
|
latest_any = conn.execute(
|
|
"SELECT MAX(upload_date) AS latest FROM video WHERE channel_pk = ?",
|
|
(channel["id"],),
|
|
).fetchone()
|
|
rows.append(
|
|
{
|
|
"id": channel["id"],
|
|
"title": channel["title"],
|
|
"handle": channel["handle"],
|
|
"channel_id": channel["channel_id"],
|
|
"retention_days": channel["retention_days"],
|
|
"global_retention": global_retention,
|
|
"last_polled_at": channel["last_polled_at"],
|
|
"last_poll_ok": channel["last_poll_ok"],
|
|
"consecutive_poll_failures": channel["consecutive_poll_failures"],
|
|
"downloaded": stats["n"],
|
|
"bytes": stats["bytes"],
|
|
"latest": latest_any["latest"],
|
|
}
|
|
)
|
|
|
|
values = settings.all_editable()
|
|
if submitted:
|
|
values.update(submitted)
|
|
|
|
return templates.index_page(
|
|
channels=rows,
|
|
settings_values=values,
|
|
settings_errors=settings_errors or {},
|
|
csrf=auth.csrf_token(self._secret(settings), token),
|
|
add_error=add_error,
|
|
queue_depth=videos.queue_depth(conn),
|
|
)
|
|
|
|
|
|
def serve(host: str = "127.0.0.1", port: int = 8085, *, secure_cookies: bool = True) -> None:
|
|
util.apply_umask()
|
|
server = AdminServer((host, port), Handler, secure_cookies=secure_cookies)
|
|
log.info("admin server listening on http://%s:%d", host, port)
|
|
try:
|
|
server.serve_forever()
|
|
except KeyboardInterrupt: # pragma: no cover
|
|
pass
|
|
finally:
|
|
server.server_close()
|