commit 18bb2e420bc769e3ca22456c2f2e73fc277bee4b Author: Tom Flux Date: Tue Aug 11 21:42:48 2026 +0100 Initial implementation of youtube-automate A DVR for YouTube subscriptions, laid out so Jellyfin presents each channel as a show and each video as an episode. Cron-driven, idempotent, with a public admin UI for a non-operator. Verified end to end on susan against three real channels: PO tokens, h264 downloads, Jellyfin resolution from local NFOs with all providers disabled, retention and tombstones. Corrections to the original design handover (specs.md documents each with the evidence, and specs.handover-original.md preserves the original): - The format sort selected 360p. Ranking acodec above res makes `bv*` prefer the combined 360p stream, which carries AAC, over the 720p video-only stream whose acodec is none. vcodec now leads, so a video without h264 at 720p yields h264 lower down rather than VP9 this hardware cannot transcode. - yt-dlp now requires a JS runtime and the yt-dlp-ejs solver scripts, which only ship with the [default] extra. Without them the n challenge fails and the mweb formats disappear entirely. - --flat-playlist carries no upload dates, so the specced client-side date filter for backfill was impossible. Backfill is RSS-first. - skipped_old was terminal, so raising a channel's retention appeared to do nothing. Added an explicit rescan. - is_upcoming premieres now defer and retry instead of being skipped forever. - TubeArchivist is gone, so the media root and the tube.jihakuz.xyz vhost were both reclaimed; the latter still pointed at its dead port. 240 offline tests, no network and no real yt-dlp invocation. Co-Authored-By: Claude Opus 5 (1M context) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dfcfab8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.venv/ diff --git a/deploy/crontab.fragment b/deploy/crontab.fragment new file mode 100644 index 0000000..1cf18cb --- /dev/null +++ b/deploy/crontab.fragment @@ -0,0 +1,13 @@ +# youtube-automate — add these to susan's crontab (`crontab -e`). +# +# HC_API_URL is already set at the top of susan's crontab; these entries follow +# the existing one-UUID-per-job convention. +# +# `sg mediaserver` matches how the radio jobs run and guarantees new files are +# group-owned by mediaserver even if the invoking shell's primary group differs. + +# Poll, download and reap. Hourly is ample for single-digit channels. +17 * * * * runitor -uuid 41a4d61a-7743-43d9-9b5d-d37d536e4726 -- sg mediaserver "/usr/local/bin/youtube-automate run" + +# Keep yt-dlp current, then re-run doctor. Mondays at 04:40. +40 4 * * 1 runitor -uuid 721e4cf0-d796-48e7-a184-79d21e1ba373 -- /opt/youtube-automate/deploy/update-ytdlp.sh diff --git a/deploy/deploy.sh b/deploy/deploy.sh new file mode 100755 index 0000000..20d4bbc --- /dev/null +++ b/deploy/deploy.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# Root-requiring installation steps for youtube-automate. +# +# susan has no passwordless sudo, so these are collected here for the operator +# to run in one go: +# +# sudo /opt/youtube-automate/deploy/deploy.sh +# +# Everything that does NOT need root (the venv, the database, the POT provider +# container, subscriptions) is already handled by the application itself. +set -euo pipefail + +REPO=/opt/youtube-automate +VENV=/var/lib/youtube-automate/venv +HOSTNAME_=tube.jihakuz.xyz + +if [[ $EUID -ne 0 ]]; then + echo "This script needs root. Run: sudo $0" >&2 + exit 1 +fi + +say() { printf '\n\033[1m==> %s\033[0m\n' "$1"; } + +say "Installing the /usr/local/bin shim" +cat > /usr/local/bin/youtube-automate </dev/null | grep -qv "$HOSTNAME_\$"; then + echo " $HOSTNAME_ is already served by an existing vhost." + echo " Skipping the standalone file — run deploy/fix-nginx-tube.sh instead." +else + install -m 0644 "$REPO/deploy/nginx-tube.jihakuz.xyz.conf" \ + "/etc/nginx/sites-available/$HOSTNAME_" + ln -sfn "/etc/nginx/sites-available/$HOSTNAME_" \ + "/etc/nginx/sites-enabled/$HOSTNAME_" + nginx -t + systemctl reload nginx +fi + +cat < Done. Remaining manual steps, in order: + + 1. Issue the certificate (needs port 80 reachable from the internet): + + sudo certbot --nginx -d $HOSTNAME_ + + 2. Set the admin password (prompts; never pass it as an argument): + + youtube-automate set-password + + 3. Add the two cron entries as susan (NOT root): + + crontab -e + # then paste $REPO/deploy/crontab.fragment + + 4. Confirm everything is healthy: + + youtube-automate doctor + +Note: the bgutil POT provider container starts itself on boot via +--restart unless-stopped, so it needs nothing here. If it is ever missing: + + docker run --name bgutil-provider -d --restart unless-stopped --init \\ + -p 127.0.0.1:4416:4416 brainicism/bgutil-ytdlp-pot-provider:1.3.1-deno +EOF diff --git a/deploy/fix-nginx-tube.sh b/deploy/fix-nginx-tube.sh new file mode 100755 index 0000000..4d50b12 --- /dev/null +++ b/deploy/fix-nginx-tube.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Repoint the existing tube.jihakuz.xyz vhost at youtube-automate. +# +# Why this exists: tube.jihakuz.xyz was already served by a leftover +# TubeArchivist server block inside /etc/nginx/sites-available/jihakuz.xyz, +# proxying to 127.0.0.1:8003. That block loads before the standalone vhost +# deploy.sh installs (nginx takes the first server block matching a name), so +# every request went to the dead TubeArchivist port and returned 502. +# +# That old block already owns the Let's Encrypt certificate for the hostname, +# so the right fix is to repoint it rather than duplicate it — no second +# certbot run needed. +# +# sudo /opt/youtube-automate/deploy/fix-nginx-tube.sh +set -euo pipefail + +CONF=/etc/nginx/sites-available/jihakuz.xyz +STANDALONE=/etc/nginx/sites-enabled/tube.jihakuz.xyz +OLD_PORT=8003 +NEW_PORT=8085 + +if [[ $EUID -ne 0 ]]; then + echo "This script needs root. Run: sudo $0" >&2 + exit 1 +fi + +if ! grep -q "127.0.0.1:${OLD_PORT}" "$CONF"; then + if grep -q "127.0.0.1:${NEW_PORT}" "$CONF"; then + echo "Already repointed at ${NEW_PORT}; nothing to do." + exit 0 + fi + echo "Did not find 127.0.0.1:${OLD_PORT} in $CONF — nothing to change." >&2 + exit 1 +fi + +BACKUP="${CONF}.bak-$(date +%Y%m%d%H%M%S)" +cp -a "$CONF" "$BACKUP" +echo "==> Backed up $CONF to $BACKUP" + +# X-Forwarded-For is not cosmetic: the app throttles failed logins per client +# address and reads it from that header. Without it every attempt looks like it +# came from nginx itself, so one attacker would lock out everybody. +sed -i \ + "s|proxy_pass http://127.0.0.1:${OLD_PORT};|proxy_pass http://127.0.0.1:${NEW_PORT};\n\t\tproxy_http_version 1.1;\n\t\tproxy_set_header Host \$host;\n\t\tproxy_set_header X-Real-IP \$remote_addr;\n\t\tproxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;\n\t\tproxy_set_header X-Forwarded-Proto \$scheme;|" \ + "$CONF" +echo "==> Repointed tube.jihakuz.xyz at 127.0.0.1:${NEW_PORT}" + +# The standalone vhost is now redundant and would only produce a +# "conflicting server name" warning. +if [[ -L "$STANDALONE" || -f "$STANDALONE" ]]; then + rm -f "$STANDALONE" + echo "==> Removed the redundant standalone vhost $STANDALONE" +fi + +if ! nginx -t; then + echo "!! nginx config test failed — restoring the backup" >&2 + cp -a "$BACKUP" "$CONF" + nginx -t + exit 1 +fi + +systemctl reload nginx +echo "==> nginx reloaded" +echo +echo "Verify with:" +echo " curl -sI https://tube.jihakuz.xyz/ | head -1 # expect 303 -> /login" diff --git a/deploy/nginx-tube.jihakuz.xyz.conf b/deploy/nginx-tube.jihakuz.xyz.conf new file mode 100644 index 0000000..451e1ae --- /dev/null +++ b/deploy/nginx-tube.jihakuz.xyz.conf @@ -0,0 +1,49 @@ +# youtube-automate admin UI — tube.jihakuz.xyz +# +# NOTE: on susan this file is NOT the vhost in use. tube.jihakuz.xyz was already +# served by a leftover TubeArchivist server block inside +# sites-available/jihakuz.xyz (proxying to the now-dead 127.0.0.1:8003), and +# nginx uses the first server block matching a name. That block also already +# owns the Let's Encrypt certificate, so the fix was to repoint it — see +# deploy/fix-nginx-tube.sh. This file is kept as the reference config for a +# clean install on a host without that history. +# +# Install this as /etc/nginx/sites-available/tube.jihakuz.xyz and symlink it into +# sites-enabled, then run: +# +# sudo certbot --nginx -d tube.jihakuz.xyz +# +# certbot rewrites this file to add the TLS server block and the 80->443 +# redirect, matching how the other vhosts on susan are set up. +# +# The DNS record and njal.la update key for tube.jihakuz.xyz already exist in +# ~/.local/bin/update-dns.sh, so no DNS work is needed. + +server { + listen 80; + listen [::]:80; + server_name tube.jihakuz.xyz; + + # Small admin forms only; nothing here accepts uploads. + client_max_body_size 256k; + + # Belt and braces — the app sets these too, but a misconfigured upstream + # should not be able to drop them. + add_header X-Content-Type-Options nosniff always; + add_header X-Frame-Options DENY always; + add_header Referrer-Policy same-origin always; + + location / { + proxy_pass http://127.0.0.1:8085; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + # The app throttles failed logins per client address and reads it from + # this header, so it must be set correctly. + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_read_timeout 120s; + } +} diff --git a/deploy/update-ytdlp.sh b/deploy/update-ytdlp.sh new file mode 100755 index 0000000..62a549d --- /dev/null +++ b/deploy/update-ytdlp.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Weekly yt-dlp update, run under runitor from susan's crontab. +# +# specs.md §12 calls a stale yt-dlp the single most likely cause of "everything +# broke", so this is monitored with its own Healthchecks UUID. It deliberately +# does NOT touch the bgutil plugin: that is pinned to match the container tag, +# and upgrading one half alone is how you get silent version skew. +# +# doctor runs afterwards so a successful install that nonetheless leaves the +# stack broken still fails the check. +set -euo pipefail + +VENV=/var/lib/youtube-automate/venv +UV=/home/susan/.local/bin/uv + +"$UV" pip install --python "$VENV" --quiet --upgrade 'yt-dlp[default]' + +echo "yt-dlp now: $("$VENV/bin/yt-dlp" --version)" + +exec "$VENV/bin/youtube-automate" doctor diff --git a/deploy/youtube-automate.service b/deploy/youtube-automate.service new file mode 100644 index 0000000..3e79df5 --- /dev/null +++ b/deploy/youtube-automate.service @@ -0,0 +1,31 @@ +[Unit] +Description=youtube-automate admin server +Documentation=file:///opt/youtube-automate/specs.md +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +# Group=mediaserver and UMask=0002 are load-bearing: Jellyfin reaches the media +# tree only through the mediaserver group, and anything this process writes must +# stay group-readable. See specs.md §2. +User=susan +Group=mediaserver +UMask=0002 + +WorkingDirectory=/opt/youtube-automate +ExecStart=/var/lib/youtube-automate/venv/bin/youtube-automate serve --host 127.0.0.1 --port 8085 + +Restart=always +RestartSec=5 + +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=full +ProtectControlGroups=true +ProtectKernelTunables=true +RestrictSUIDSGID=true +ReadWritePaths=/var/lib/youtube-automate /disks/Plex/YouTube + +[Install] +WantedBy=multi-user.target diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e442a47 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[project] +name = "youtube-automate" +version = "1.0.0" +description = "A DVR for YouTube subscriptions, laid out for Jellyfin" +requires-python = ">=3.11" + +# Pins that matter are documented in specs.md §3: +# - the [default] extra is what ships yt-dlp-ejs (the JS challenge solver) +# - curl-cffi must stay below 0.16 or yt-dlp rejects it as unsupported +# - the bgutil plugin must match the container tag +dependencies = [ + "yt-dlp[default]", + "bgutil-ytdlp-pot-provider==1.3.1", + "curl-cffi<0.16", +] + +[project.optional-dependencies] +dev = ["pytest>=8"] + +[project.scripts] +youtube-automate = "youtube_automate.cli:main" + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools] +packages = ["youtube_automate", "youtube_automate.web"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/specs.handover-original.md b/specs.handover-original.md new file mode 100644 index 0000000..d929df2 --- /dev/null +++ b/specs.handover-original.md @@ -0,0 +1,656 @@ +# `ytsubs` — implementation handover + +**Target machine:** `susan` +**Audience:** a Claude Code agent implementing this from scratch +**Status:** design agreed, not started + +--- + +## 1. What we're building + +A self-hosted pipeline that keeps a rolling 14-day window of recent YouTube uploads from a +handful of subscribed channels, laid out on disk so Jellyfin presents each channel as a TV +show and each video as an episode. A small web UI lets a non-operator (the requester's +brother) subscribe, unsubscribe, and tweak settings. + +The mental model is **a DVR for YouTube subscriptions**, not an archiver. Videos are +disposable. If a video ages out or a channel is unsubscribed, it goes away permanently. + +### Explicitly in scope + +- Poll subscribed channels for new uploads +- Download at ≤720p, h264/AAC, into a Jellyfin-friendly layout +- Generate Kodi-style `.nfo` metadata + artwork (no Jellyfin plugin) +- Delete videos older than the retention window +- Web admin: add/remove channels, edit settings +- Cron-driven, idempotent, monitored via Healthchecks + +### Explicitly out of scope — do not build these + +- **Cookie handling of any kind.** No browser extension, no cookie-receiving API, no + `cookies.txt`. See §3. +- Members-only, age-restricted, or private content (follows from the above) +- Migration from the existing TubeArchivist install +- Multi-user support, authentication beyond a single shared credential +- Transcoding of any kind (see §6 — this matters on this hardware) + +--- + +## 2. Environment facts you need + +- **susan**: Linux rack server, dual Xeon X58, 12 threads, **no AVX**. Old and slow. + Software video transcoding is effectively off the table. +- Already running on susan: Jellyfin, Docker, nginx (Let's Encrypt), Netdata, + Healthchecks (`hc.jihakuz.xyz`), `runitor` for cron wrapping. +- susan is on a **residential IP**. This matters — YouTube flags datacenter IPs far more + aggressively. Do not move any part of this to `victoria` (the Linode VPS). +- Tailnet: Headscale at `headscale.jihakuz.xyz`. The admin UI should bind to the tailnet + interface rather than being published publicly (see §9). +- `/disks/Plex/YouTube` is **already in use by TubeArchivist**. Do not touch it. Use a + fresh root (§5). +- Existing conventions to match: state under `/var/lib//`, deployed executables + under `/usr/local/bin/`, bare git repos under `/disks/git-repos/`. + +### Proposed paths + +| Purpose | Path | +|---|---| +| Source (bare repo) | `/disks/git-repos/youtube-automate.git` | +| Checkout | `/opt/youtube-automate` | +| Entry point | `/usr/local/bin/youtube-automate` | +| State DB | `/var/lib/youtube-automate/subs.db` | +| Lock file | `/var/lib/youtube-automate/run.lock` | +| Media root | `/disks/Plex/YouTube/` | +| Scratch/work dir | `/disks/Plex/YouTube/.ingest/` | + +The work dir **must** be on the same filesystem as the media root so finished downloads +move into place with an atomic `rename()` rather than a copy. Jellyfin should be +configured to ignore `.ingest` (it's dot-prefixed, which Jellyfin skips by default — +verify). + +--- + +## 3. YouTube access: PO tokens, not cookies + +This was the main open question in design and the answer is that it collapses. + +yt-dlp's current recommended setup for reliable downloads is the **mweb client plus a PO +Token Provider plugin** supplying tokens for GVS (Google Video Server) requests. No +account, no cookies, no browser. Cookies only unlock age-restricted / members-only / +private content, which is out of scope, and carry a real risk of the Google account being +banned. + +### Setup + +1. Run the provider as a Docker sidecar: +!! tom: needs a way of starting up when system reboots + + ``` + docker run --name bgutil-provider -d --restart unless-stopped --init \ + -p 127.0.0.1:4416:4416 brainicism/bgutil-ytdlp-pot-provider + ``` + +2. Install the matching plugin into the same Python environment as yt-dlp: +!! tom: do this in uv venv + + ``` + pip install bgutil-ytdlp-pot-provider + ``` + + The plugin and server versions should match. + +3. Verify before writing any other code: + + ``` + yt-dlp -v 'https://www.youtube.com/watch?v=' -F + ``` + + You must see a line like + `[debug] [youtube] [pot] PO Token Providers: bgutil:http-1.x.x (external), ...` + If you don't, stop and fix this first — everything downstream depends on it. + +4. `ytsubs` should health-check the provider before each download batch (a plain HTTP + GET against `http://127.0.0.1:4416/ping`) and fail loudly if it's down, rather than + silently accumulating 403s. + +### Caveats to be aware of + +- The bgutil README carries a warning that PO tokens no longer bypass the "Sign in to + confirm you're not a bot" interstitial in most cases. That is a **different failure + mode** (IP reputation) from the 403s on format URLs that this solves. Residential IP + + low request volume + sleep intervals should keep us clear of it. If it does start + firing, that's the point at which cookies get reconsidered — not before. +- **yt-dlp's YouTube handling churns every few weeks.** Client recommendations, + extractor-arg names, and which clients need tokens all move. Before implementing, read + the current `PO Token Guide` and `Extractors` pages on the yt-dlp wiki and adjust the + flags in §6 accordingly. Do not treat the flags in this document as authoritative — + treat them as the shape of the answer. +- Keep yt-dlp itself updated (weekly `pip install -U yt-dlp` under the same runitor + wrapper). A stale yt-dlp is the single most likely cause of "everything broke". + +--- + +## 4. Discovery: RSS feeds + +YouTube publishes an unauthenticated Atom feed per channel. Use the **undocumented +`UULF` playlist variant**, which returns long-form videos only — no Shorts, no +livestreams: + +``` +https://www.youtube.com/feeds/videos.xml?playlist_id=UULF +``` + +So `UCabc123...` → `https://www.youtube.com/feeds/videos.xml?playlist_id=UULFabc123...` + +This does most of the Shorts/livestream filtering for free, at the cheapest possible +point in the pipeline. + +Related prefixes, for reference: `UU` all uploads, `UUSH` shorts, `UULV` livestreams, +`UUMF`/`UUMO` members-only. + +### Robustness + +These prefixes are undocumented and there have been reports through 2026 of intermittent +failures and missing entries in YouTube's native feeds. So: + +- If the `UULF` feed 404s or returns zero entries, **fall back** to + `?channel_id=UC...` and record `discovery_source = 'uc_feed'` on the resulting rows. +- Rows discovered via the fallback path get the duration/live filter applied at download + time (§6). Rows from `UULF` don't need it. +- Track consecutive poll failures per channel and surface the count in the admin UI. Two + channels quietly failing for a month is the realistic failure mode here. + +### Parsing + +`xml.etree.ElementTree`, no dependencies. Namespaces: + +```python +NS = { + "atom": "http://www.w3.org/2005/Atom", + "yt": "http://www.youtube.com/xml/schemas/2015", + "media": "http://search.yahoo.com/mrss/", +} +``` + +Per `atom:entry`: `yt:videoId`, `atom:title`, `atom:published`, +`media:group/media:description`. The feed does **not** carry duration. + +### Backfill on subscribe + +New subscriptions pull the last **7 days**. The RSS feed only returns ~15 items, which +can be under 7 days for a prolific channel. For the initial backfill only, use: + +``` +yt-dlp --flat-playlist --playlist-end 50 -J \ + 'https://www.youtube.com/playlist?list=UULF<...>' +``` + +and date-filter the entries client-side. Steady-state polling uses the RSS feed. + +### Getting the subscription list in + +There is no need for a Google account or a subscriptions RSS feed. Either the brother +pastes channel URLs into the admin page one at a time, or he does a one-off Google +Takeout export (`subscriptions.csv`) and we bulk-insert. With single-digit channels, +manual entry is fine — do not build a CSV importer unless asked. + +--- + +## 5. On-disk layout + +Media root: `/disks/Plex/YouTubeSubs/` + +``` +/disks/Plex/YouTubeSubs/ +├── .work/ # scratch, ignored by Jellyfin +└── Some Channel/ + ├── tvshow.nfo + ├── poster.jpg + ├── fanart.jpg + └── Season 2026/ + ├── Some Channel - S2026E08110 - Video Title [dQw4w9WgXcQ].mp4 + ├── Some Channel - S2026E08110 - Video Title [dQw4w9WgXcQ].nfo + ├── Some Channel - S2026E08110 - Video Title [dQw4w9WgXcQ]-thumb.jpg + ├── Some Channel - S2026E08110 - Video Title [dQw4w9WgXcQ].en.srt + └── Some Channel - S2026E08110 - Video Title [dQw4w9WgXcQ].info.json +``` + +### Season / episode numbering + +- **Season** = upload year (`2026`) +- **Episode** = `MMDD * 10 + ordinal_within_day` + +So the first video uploaded on 11 August is `8110`, the second that day is `8111`, and +the first on 12 August is `8120`. This scheme sorts correctly across the whole year +(unlike appending a suffix to a bare `MMDD`) and tolerates up to 10 uploads per channel +per day. Clamp the ordinal at 9 and log a warning if exceeded. + +The ordinal must be computed against **what's already in the DB for that channel+date**, +not against the current batch, so it's stable across runs. + +### Filename sanitisation + +Strip `/ \ : * ? " < > |`, collapse runs of whitespace, strip leading/trailing dots and +spaces, truncate the title component to 120 chars on a word boundary. The `[videoid]` +suffix guarantees uniqueness regardless. + +Store the channel directory name in the DB (`channel.dir_name`) at subscribe time and +never recompute it — channels rename themselves and we don't want orphaned directories. + +--- + +## 6. Download + +### Format selection — read this bit carefully + +susan has no AVX and cannot realistically transcode. YouTube serves 720p as VP9 or AV1 by +default. If a client can't direct-play those, Jellyfin will try to transcode and it will +be miserable. So we force h264 + AAC in mp4 at download time: + +``` +-f "bv*[height<=720]+ba/b[height<=720]" +-S "vcodec:h264,acodec:aac,res:720" +--merge-output-format mp4 +``` + +If a video genuinely has no h264 rendition at ≤720p, `-S` will fall back to VP9 rather +than fail. Log those cases; they should be rare. + +Do **not** use `--sponsorblock-remove` — it requires cutting and re-encoding. Use +`--sponsorblock-mark all` with `--embed-chapters`, which writes chapter markers only and +is free. + +### Full invocation + +``` +yt-dlp \ + --extractor-args "youtube:player_client=default,mweb" \ + --extractor-args "youtubepot-bgutilhttp:base_url=http://127.0.0.1:4416" \ + -f "bv*[height<=720]+ba/b[height<=720]" \ + -S "vcodec:h264,acodec:aac,res:720" \ + --merge-output-format mp4 \ + --no-playlist \ + --write-info-json \ + --write-thumbnail --convert-thumbnails jpg \ + --write-auto-subs --sub-langs "en.*" --convert-subs srt \ + --sponsorblock-mark all --embed-chapters \ + --retries 3 --fragment-retries 10 \ + --sleep-requests 2 --sleep-interval 5 --max-sleep-interval 15 \ + -P "/disks/Plex/YouTubeSubs/.work" \ + -o "%(id)s.%(ext)s" \ + "https://www.youtube.com/watch?v=" +``` + +Verify the `youtubepot-bgutilhttp:base_url` arg name against the current plugin README — +this is the kind of thing that gets renamed. + +### Match filter (fallback-discovered videos only) + +For rows with `discovery_source = 'uc_feed'`, append: + +``` +--match-filter "duration>?{min_duration_seconds} & live_status!=is_live & live_status!=is_upcoming & !was_live" +``` + +The `>?` form allows videos with unknown duration through rather than rejecting them. +`min_duration_seconds` defaults to 120 (Shorts can now run to 3 minutes, but so can +legitimate short videos — this is the tradeoff, and it's why `UULF` is the primary path). + +A video rejected by the match filter should be recorded as `skipped_short` / +`skipped_live` and **never retried**. + +### Concurrency and politeness + +One download at a time. Single-digit channels over a 14-day window is a small workload; +there is no reason to be aggressive and every reason not to be. + +### After a successful download + +1. Read the `.info.json` from `.work/` +2. Compute season/episode/filename +3. Generate the episode `.nfo` (§7) +4. `os.rename()` all artefacts into the season directory +5. Update the DB row: `state='downloaded'`, `rel_path`, `downloaded_at` +6. After the whole batch, trigger a Jellyfin library refresh: + `POST {jellyfin_url}/Library/Refresh` with header `X-Emby-Token: {api_key}` + +If any step 1–4 fails, clean up `.work/` for that video ID and mark `failed` with +`attempts += 1`. Give up after 5 attempts and surface it in the UI. + +--- + +## 7. Metadata — NFO files, no Jellyfin plugin + +Jellyfin reads Kodi-style NFO sidecars natively. Configure the library as **Shows**, +disable all internet metadata providers, enable *Prefer local metadata* and *Save +artwork/metadata into media folders*. Do not write or install a metadata provider plugin. + +### `tvshow.nfo` (per channel, written at subscribe time, refreshed on title change) + +```xml + + + Some Channel + Channel description from yt-dlp. + YouTube + UCabc123... + +``` + +### Episode `.nfo` (one per video, filename matches the media file) + +```xml + + + Video Title + Some Channel + 2026 + 8110 + Video description. + 2026-08-11 + 12 + YouTube + dQw4w9WgXcQ + +``` + +`runtime` is in **minutes**. Escape all text content properly — video descriptions +contain everything. + +### Artwork + +At subscribe time, `yt-dlp --flat-playlist --playlist-items 0 -J ` returns a +`thumbnails` array containing entries with `id` values like `avatar_uncropped` and +`banner_uncropped`. Download the avatar to `poster.jpg` and the banner to `fanart.jpg`. +Treat both as best-effort — if they're missing, carry on without them. + +Per-episode thumbnails come from `--write-thumbnail`; rename to `-thumb.jpg`. + +--- + +## 8. Data model + +SQLite at `/var/lib/ytsubs/subs.db`. **Enable WAL mode** — the cron job and the web +server both write. + +```sql +PRAGMA journal_mode = WAL; +PRAGMA foreign_keys = ON; + +CREATE TABLE channel ( + id INTEGER PRIMARY KEY, + channel_id TEXT NOT NULL UNIQUE, -- UC... + handle TEXT, -- @handle, informational + title TEXT NOT NULL, + description TEXT, + dir_name TEXT NOT NULL UNIQUE, -- sanitised, immutable after creation + added_at TEXT NOT NULL, + backfilled INTEGER NOT NULL DEFAULT 0, + last_polled_at TEXT, + last_poll_ok INTEGER, + consecutive_poll_failures INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE video ( + id INTEGER PRIMARY KEY, + video_id TEXT NOT NULL UNIQUE, + channel_pk INTEGER NOT NULL REFERENCES channel(id) ON DELETE CASCADE, + title TEXT, + upload_date TEXT, -- YYYY-MM-DD + duration INTEGER, + season INTEGER, + episode INTEGER, + state TEXT NOT NULL, + discovery_source TEXT NOT NULL, -- 'uulf_feed' | 'uc_feed' | 'backfill' + rel_path TEXT, -- relative to media root, NULL unless downloaded + size_bytes INTEGER, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + discovered_at TEXT NOT NULL, + downloaded_at TEXT, + deleted_at TEXT +); + +CREATE INDEX idx_video_state ON video(state); +CREATE INDEX idx_video_upload_date ON video(upload_date); + +CREATE TABLE setting ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +``` + +### `video.state` values + +| State | Meaning | Retried? | +|---|---|---| +| `pending` | discovered, queued | — | +| `downloading` | claimed by a worker | recovered on startup → `pending` | +| `downloaded` | on disk, `rel_path` set | — | +| `deleted` | aged out; tombstone prevents re-download | never | +| `skipped_short` | rejected by match filter | never | +| `skipped_live` | rejected by match filter | never | +| `skipped_old` | discovered but already outside the window | never | +| `failed` | download error | yes, up to 5 attempts | + +The tombstone behaviour is the important part: a `deleted` row must never be +re-discovered and re-downloaded. This is why we keep our own state table rather than +using yt-dlp's `--download-archive`. + +### Settings (all editable from the admin page) + +Store as strings; provide typed accessors with defaults so a missing key never crashes. + +| Key | Default | Notes | +|---|---|---| +| `retention_days` | `14` | delete videos with `upload_date` older than this | +| `backfill_days` | `7` | window pulled when a channel is added | +| `max_height` | `720` | | +| `min_duration_seconds` | `120` | fallback-path Shorts filter | +| `sponsorblock_mark` | `true` | | +| `write_subs` | `true` | | +| `sub_langs` | `en.*` | | +| `respect_jellyfin_watch_state` | `true` | §10 | +| `jellyfin_url` | `http://127.0.0.1:8096` | | +| `jellyfin_api_key` | *(empty)* | masked in the UI | +| `jellyfin_user_id` | *(empty)* | | +| `pot_provider_url` | `http://127.0.0.1:4416` | | +| `max_attempts` | `5` | | +| `disk_cap_gb` | `0` | `0` = disabled; if set, evict oldest downloaded videos | + +Validate on save (integers parse, URLs well-formed) and re-render the form with an inline +error rather than 500-ing. + +--- + +## 9. Entry point and scheduling + +Single executable `ytsubs` with subcommands: + +| Command | Purpose | +|---|---| +| `ytsubs run` | poll → download → reap. This is what cron calls. | +| `ytsubs poll [--channel ID]` | discovery only | +| `ytsubs download` | drain the pending queue | +| `ytsubs reap` | retention pass | +| `ytsubs serve` | admin HTTP server (systemd unit) | +| `ytsubs subscribe ` / `unsubscribe ` | CLI equivalents, useful for debugging | +| `ytsubs doctor` | check yt-dlp version, POT provider reachable, DB writable, media root writable, Jellyfin reachable | + +`run` takes a **non-blocking `flock`** on `/var/lib/ytsubs/run.lock` and exits 0 silently +if already held. On startup, reset any `downloading` rows to `pending` (crash recovery). + +### Cron + +``` +17 * * * * runitor -uuid -- /usr/local/bin/ytsubs run +``` + +Hourly is ample for single-digit channels. Register the check in Healthchecks with a +generous grace period. + +### systemd unit for the admin server + +Standard `simple` unit, `Restart=always`, running as the same unprivileged user that owns +the DB and media root. When the user subscribes to a channel via the web UI, the handler +should spawn `ytsubs run --channel ` detached so the backfill starts immediately +rather than waiting up to an hour. + +### nginx + +Bind the admin server to `127.0.0.1:8085`. Front it with nginx on the **tailnet +interface only** — `yt.jihakuz.xyz` resolving inside Headscale, HTTP basic auth on top. +Given the brother will be on the tailnet anyway, there's no reason to expose this +publicly, and it removes the need for a Let's Encrypt cert and any real CSRF story. +Confirm this before implementing; if it does need to be public, add proper CSRF tokens. + +--- + +## 10. Retention and deletion + +### Aging out (`ytsubs reap`) + +Candidates: `state = 'downloaded'` and `upload_date < today - retention_days`. + +Before deleting, if `respect_jellyfin_watch_state` is on: + +1. `GET {jellyfin_url}/Items?userId={user_id}&recursive=true&includeItemTypes=Episode&fields=Path,UserData` + with header `X-Emby-Token: {api_key}` — one call per reap, build a `path → UserData` map. +2. **Skip** any candidate where `UserData.PlaybackPositionTicks > 0 and not UserData.Played` + (part-watched) or `UserData.IsFavorite` is true. +3. Apply a hard backstop: delete anyway once `upload_date` is older than + `retention_days * 2`, so a half-watched video doesn't live forever. + +Nothing wrecks "set and forget" like a video disappearing at the 20-minute mark. This +check is worth the complexity. + +Deletion removes the media file plus its `.nfo`, `-thumb.jpg`, `.srt`, and `.info.json` +siblings, then prunes the season directory if empty. Set `state='deleted'`, +`deleted_at`, and `rel_path=NULL`. **Keep the row** — it's the tombstone. + +If the Jellyfin API is unreachable, skip the whole reap for that run and log it. Do not +delete blindly. + +### Unsubscribe + +Hard delete, as agreed: `shutil.rmtree()` the channel directory, then `DELETE FROM +channel` (cascades to `video`). Re-subscribing later starts from scratch and re-downloads +the last 7 days. Put a confirmation step in the UI — this is destructive and irreversible. + +### Disk cap (optional, `disk_cap_gb`) + +If enabled, after each reap, sum `size_bytes` over `downloaded` rows and evict oldest-by- +`upload_date` until under the cap, honouring the same watch-state protection. + +--- + +## 11. Admin UI + +Stdlib only — `http.server.ThreadingHTTPServer` + `BaseHTTPRequestHandler`. No Flask, no +FastAPI, no npm. Server-rendered HTML with a single embedded ` +
{body}
""".encode("utf-8") + + +def login_page(error: str | None = None) -> bytes: + alert = f'
{_e(error)}
' if error else "" + body = f""" +""" + return page("Sign in — youtube-automate", body) + + +def _channel_row(channel: dict, csrf: str) -> str: + failures = channel["consecutive_poll_failures"] + if failures > 2: + badge = f'{failures} failed polls' + elif channel["last_poll_ok"] == 0: + badge = 'last poll failed' + else: + badge = "" + + retention = channel["retention_days"] + retention_value = "" if retention is None else str(retention) + placeholder = f"default ({channel['global_retention']})" + + return f""" + + + {_e(channel['title'])} {badge}
+ {_e(channel['handle'] or channel['channel_id'])} + + {channel['downloaded']} + {_e(util.human_bytes(channel['bytes']))} + {_e(channel['latest'] or '—')} + {_e(channel['last_polled_at'] or 'never')} + +
+ + + +
+ + +
+ + +
+
+ + +
+ +""" + + +def _settings_form(values: dict, errors: dict, csrf: str) -> str: + fields = [] + for key in EDITABLE: + value = values.get(key, DEFAULTS[key]) + error = errors.get(key) + if key in MASKED_KEYS and value: + shown, placeholder = "", "stored — leave blank to keep" + else: + shown, placeholder = value, "" + input_type = "password" if key in MASKED_KEYS else "text" + fields.append( + f"""
+ + + {f'
{_e(error)}
' if error else ''} +
""" + ) + + return f""" +
+ +
{''.join(fields)}
+
+
""" + + +def index_page( + *, + channels: list[dict], + settings_values: dict, + settings_errors: dict, + csrf: str, + flash: tuple[str, str] | None = None, + add_error: str | None = None, + queue_depth: int = 0, +) -> bytes: + flash_html = "" + if flash: + kind, message = flash + flash_html = f'
{_e(message)}
' + + if channels: + rows = "".join(_channel_row(channel, csrf) for channel in channels) + table = f""" +
+ + + + + + +{rows} +
ChannelOn diskSizeLatest uploadLast pollRetention (days)
+
""" + else: + table = '
No channels yet. Add one below.
' + + add_error_html = f'
{_e(add_error)}
' if add_error else "" + + body = f""" +
+

youtube-automate

+
+ {len(channels)} channel(s) · {queue_depth} queued + ·
+ + +
+
+
+ +{flash_html} + +

Channels

+{table} + +

Add a channel

+
+ +
+ + +
+ {add_error_html} +
+ +

Settings

+{_settings_form(settings_values, settings_errors, csrf)} + +""" + return page("youtube-automate", body) diff --git a/youtube_automate/ytdlp.py b/youtube_automate/ytdlp.py new file mode 100644 index 0000000..856390f --- /dev/null +++ b/youtube_automate/ytdlp.py @@ -0,0 +1,109 @@ +"""Thin wrapper around the venv's yt-dlp binary. + +Everything that shells out to yt-dlp goes through here so the PATH handling (Deno +must be discoverable — see specs.md §3) and the shared extractor args live in one +place. +""" + +from __future__ import annotations + +import json +import logging +import os +import subprocess +import urllib.request +from pathlib import Path + +from . import config + +log = logging.getLogger(__name__) + + +class YtdlpError(RuntimeError): + pass + + +def binary() -> Path: + return config.VENV_BIN / "yt-dlp" + + +def environment() -> dict[str, str]: + """Env for a yt-dlp subprocess. + + yt-dlp locates the JS runtime by searching PATH, so the venv's bin directory + must come first — that is where Deno lives. + """ + env = dict(os.environ) + env["PATH"] = f"{config.VENV_BIN}:{env.get('PATH', '')}" + return env + + +def version() -> str: + result = subprocess.run( + [str(binary()), "--version"], + capture_output=True, + text=True, + env=environment(), + timeout=60, + ) + if result.returncode != 0: + raise YtdlpError(result.stderr.strip() or "yt-dlp --version failed") + return result.stdout.strip() + + +def extractor_args(pot_provider_url: str) -> list[str]: + return [ + "--extractor-args", + "youtube:player_client=default,mweb", + "--extractor-args", + f"youtubepot-bgutilhttp:base_url={pot_provider_url}", + ] + + +def run_json(args: list[str], timeout: int = 300) -> dict: + """Run yt-dlp with -J and parse the single JSON document it prints.""" + cmd = [str(binary()), *args] + log.debug("yt-dlp %s", " ".join(args)) + result = subprocess.run( + cmd, capture_output=True, text=True, env=environment(), timeout=timeout + ) + if result.returncode != 0: + raise YtdlpError(first_error(result.stderr) or "yt-dlp failed") + try: + return json.loads(result.stdout) + except json.JSONDecodeError as exc: + raise YtdlpError(f"could not parse yt-dlp output: {exc}") from exc + + +def run(args: list[str], timeout: int = 7200) -> subprocess.CompletedProcess: + """Run yt-dlp for its side effects, returning the completed process.""" + cmd = [str(binary()), *args] + log.debug("yt-dlp %s", " ".join(args)) + return subprocess.run( + cmd, capture_output=True, text=True, env=environment(), timeout=timeout + ) + + +def first_error(stderr: str) -> str: + for line in (stderr or "").splitlines(): + if line.startswith("ERROR:"): + return line[len("ERROR:") :].strip() + return (stderr or "").strip().splitlines()[-1] if stderr.strip() else "" + + +def pot_provider_ping(base_url: str, timeout: float = 5.0) -> dict: + """GET /ping on the bgutil provider. Raises on any failure.""" + url = base_url.rstrip("/") + "/ping" + request = urllib.request.Request(url, headers={"User-Agent": config.USER_AGENT}) + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + + +def plugin_version() -> str | None: + """Installed bgutil plugin version, for comparison against the server's.""" + try: + from importlib.metadata import version as pkg_version + + return pkg_version("bgutil-ytdlp-pot-provider") + except Exception: # pragma: no cover - only when the plugin is absent + return None