diff options
| author | Tom Flux <tom@tomflux.xyz> | 2026-07-07 11:28:06 +0100 |
|---|---|---|
| committer | Tom Flux <tom@tomflux.xyz> | 2026-07-07 11:28:06 +0100 |
| commit | 7e446fe491a0acdad2b808f23938581a149c8d4c (patch) | |
| tree | 025a2771ca041a35f6405f218ff5ffe265878896 | |
| parent | 1596ead0c56d82f5478972383e27474c69ef8ce6 (diff) | |
Dynamic config
| -rwxr-xr-x | overnight_transcoder.py | 406 |
1 files changed, 362 insertions, 44 deletions
diff --git a/overnight_transcoder.py b/overnight_transcoder.py index cf54bbe..6c38724 100755 --- a/overnight_transcoder.py +++ b/overnight_transcoder.py @@ -33,6 +33,7 @@ parser.add_argument("--dry", action="store_true", help="Dry run - don't transcod parser.add_argument("--stats", action="store_true", help="Show cache statistics and exit") parser.add_argument("--failed", action="store_true", help="Show failed transcodes and exit") parser.add_argument("--retry-failed", action="store_true", help="Reset failed files to pending for retry") +parser.add_argument("--reset-skipped", action="store_true", help="Reset skipped_larger + already_efficient files to pending (use after changing tier thresholds)") parser.add_argument("--clear-cache", action="store_true", help="Clear the cache and exit") parser.add_argument("--verbose", "-v", action="store_true", help="Verbose debug output") parser.add_argument("--json-log", action="store_true", help="Output structured JSON logs (one per line)") @@ -60,12 +61,37 @@ CONFIG = { "cutoff_time": time(6, 30), # Don't start new encodes after 06:30 "cleanup_days": 7, # Delete originals after 7 days - # FFmpeg settings - "crf": 20, - "preset": "fast", - "audio_codec": "ac3", - "audio_bitrate": "640k", "threads": 12, + + # --- Dynamic video encode tiers --- + # Settings are chosen per-file based on source resolution and bitrate. + # Each tier: (min_height, crf, preset, skip_below_bps) + # - min_height: applies to files at least this tall (first match wins, so keep descending) + # - crf/preset: x265 settings for this tier + # - skip_below_bps: if the source VIDEO bitrate is already below this, skip the file + # entirely (status 'already_efficient') — the expected saving is too + # small to be worth an encode slot. Tuned so we skip anything likely + # to save <~15%. + "encode_tiers": [ + (2160, 22, "medium", 12_000_000), # 4K + (1080, 23, "medium", 6_000_000), # 1080p + ( 720, 24, "medium", 3_500_000), # 720p + ( 0, 25, "medium", 1_500_000), # SD / everything smaller + ], + + # --- Dynamic audio handling --- + # Copy the source audio stream untouched when it's already an efficient codec at a + # sane bitrate. Only re-encode when the source is a bloated/lossless codec, and even + # then match the target codec to the channel count. This avoids the classic regression + # of upconverting e.g. AAC 256k stereo to AC3 640k and inflating the file. + "audio_copy_codecs": ["aac", "ac3", "eac3", "opus", "mp3"], + "audio_copy_max_bps_stereo": 384_000, # copy stereo/mono at or below this + "audio_copy_max_bps_surround": 768_000, # copy 5.1/7.1 at or below this + # Re-encode targets (only used when copy conditions aren't met): + "audio_reencode_stereo_codec": "aac", + "audio_reencode_stereo_bitrate": "192k", + "audio_reencode_surround_codec": "ac3", + "audio_reencode_surround_bitrate": "640k", } @@ -91,7 +117,7 @@ def init_db() -> sqlite3.Connection: scanned_at REAL, -- Transcode info (NULL if not transcoded) - status TEXT DEFAULT 'pending', -- pending, transcoding, done, failed, stalled, skipped_larger + status TEXT DEFAULT 'pending', -- pending, transcoding, done, failed, stalled, skipped_larger, already_efficient transcoded_at REAL, transcoded_size INTEGER, transcode_duration_secs REAL, @@ -145,7 +171,190 @@ def get_audio_codec(filepath: str) -> Optional[str]: return None -def get_cached_file(conn: sqlite3.Connection, filepath: str, size: int, mtime: float) -> Optional[dict]: +def probe_media(filepath: str) -> dict: + """ + Single-shot probe of the first video and first audio stream plus container info. + Returns a dict with whatever could be determined; missing fields are None. + + Fields: width, height, video_codec, video_bitrate (bps), duration (s), + audio_codec, audio_channels, audio_bitrate (bps), + container_bitrate (bps), size (bytes). + """ + info = { + "width": None, "height": None, "video_codec": None, "video_bitrate": None, + "duration": None, "audio_codec": None, "audio_channels": None, + "audio_bitrate": None, "container_bitrate": None, "size": None, + } + try: + info["size"] = os.path.getsize(filepath) + except OSError: + pass + + try: + result = subprocess.run([ + "ffprobe", "-v", "quiet", + "-print_format", "json", + "-show_format", + "-show_streams", + filepath + ], capture_output=True, text=True, timeout=60) + + if result.returncode != 0: + return info + + data = json.loads(result.stdout) + fmt = data.get("format", {}) + + # Container-level duration and bitrate + try: + info["duration"] = float(fmt.get("duration")) if fmt.get("duration") else None + except (TypeError, ValueError): + pass + try: + info["container_bitrate"] = int(fmt.get("bit_rate")) if fmt.get("bit_rate") else None + except (TypeError, ValueError): + pass + + for stream in data.get("streams", []): + stype = stream.get("codec_type") + if stype == "video" and info["video_codec"] is None: + info["video_codec"] = (stream.get("codec_name") or "").lower() or None + info["width"] = stream.get("width") + info["height"] = stream.get("height") + try: + if stream.get("bit_rate"): + info["video_bitrate"] = int(stream["bit_rate"]) + except (TypeError, ValueError): + pass + # Stream duration can be present even when format duration isn't + if info["duration"] is None: + try: + info["duration"] = float(stream.get("duration")) if stream.get("duration") else None + except (TypeError, ValueError): + pass + elif stype == "audio" and info["audio_codec"] is None: + info["audio_codec"] = (stream.get("codec_name") or "").lower() or None + info["audio_channels"] = stream.get("channels") + try: + if stream.get("bit_rate"): + info["audio_bitrate"] = int(stream["bit_rate"]) + except (TypeError, ValueError): + pass + except Exception as e: + log.warning(f"Failed to probe media {filepath}: {e}") + + return info + + +def estimate_video_bitrate(info: dict) -> Optional[int]: + """ + Best-effort source VIDEO bitrate in bps, used only for the skip-or-encode decision. + + Preference order: + 1. Video stream's own bit_rate (most accurate when present). + 2. Container bitrate minus known audio bitrate. + 3. (size * 8 / duration) minus known audio bitrate. + Falls back to overall rate if audio is unknown; this slightly overestimates the + video rate, which biases toward attempting rather than skipping (safe — the + skipped_larger guard is the backstop). + """ + if info.get("video_bitrate"): + return info["video_bitrate"] + + audio_bps = info.get("audio_bitrate") or 0 + + overall = info.get("container_bitrate") + if not overall and info.get("size") and info.get("duration"): + try: + overall = int(info["size"] * 8 / info["duration"]) + except (TypeError, ZeroDivisionError): + overall = None + + if overall: + est = overall - audio_bps + return est if est > 0 else overall + return None + + +def pick_video_settings(info: dict) -> dict: + """ + Decide whether to encode and with what x265 settings, based on resolution and + estimated source video bitrate. Returns: + {"action": "skip", "reason": str} -> already efficient + {"action": "encode", "crf": int, "preset": str, + "tier_height": int, "video_bitrate": int|None} -> proceed + """ + height = info.get("height") or 0 + est_bitrate = estimate_video_bitrate(info) + + # Find the first tier the file is tall enough for (tiers are descending). + tier = None + for min_height, crf, preset, skip_below in CONFIG["encode_tiers"]: + if height >= min_height: + tier = (min_height, crf, preset, skip_below) + break + if tier is None: + # Shouldn't happen (last tier is min_height 0), but be safe. + tier = CONFIG["encode_tiers"][-1] + min_height, crf, preset, skip_below = tier + + # Skip if the source video bitrate is already below this tier's floor. + # If we couldn't estimate bitrate at all, don't skip — attempt and let the + # skipped_larger guard catch a bad outcome. + if est_bitrate is not None and est_bitrate < skip_below: + return { + "action": "skip", + "reason": (f"source video ~{est_bitrate/1_000_000:.2f} Mbps is below the " + f"{skip_below/1_000_000:.1f} Mbps floor for {min_height}p+ " + f"(expected saving too small)"), + } + + return { + "action": "encode", + "crf": crf, + "preset": preset, + "tier_height": min_height, + "video_bitrate": est_bitrate, + } + + +def pick_audio_settings(info: dict) -> dict: + """ + Decide audio handling. Returns: + {"action": "copy"} -> stream-copy the source audio (fast, lossless) + {"action": "encode", "codec": str, "bitrate": str} -> re-encode + Copy when the source is already an efficient codec at a sane bitrate for its + channel count. Otherwise re-encode, matching codec to channels. + """ + codec = (info.get("audio_codec") or "").lower() + channels = info.get("audio_channels") or 2 + bitrate = info.get("audio_bitrate") # may be None + + is_surround = channels >= 3 + copy_ceiling = (CONFIG["audio_copy_max_bps_surround"] if is_surround + else CONFIG["audio_copy_max_bps_stereo"]) + + if codec in CONFIG["audio_copy_codecs"]: + # No bitrate info -> trust the codec and copy (re-encoding a good codec blind + # risks inflating it, which is exactly what we're trying to avoid). + if bitrate is None or bitrate <= copy_ceiling: + return {"action": "copy"} + + # Re-encode path: match target to channel count. + if is_surround: + return { + "action": "encode", + "codec": CONFIG["audio_reencode_surround_codec"], + "bitrate": CONFIG["audio_reencode_surround_bitrate"], + } + return { + "action": "encode", + "codec": CONFIG["audio_reencode_stereo_codec"], + "bitrate": CONFIG["audio_reencode_stereo_bitrate"], + } + + + """Get cached file info if file hasn't changed. Returns dict or None.""" cursor = conn.execute( """SELECT video_codec, audio_codec, is_hevc, status @@ -250,6 +459,7 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict: SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done_count, SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed_count, SUM(CASE WHEN status = 'skipped_larger' THEN 1 ELSE 0 END) as skipped_larger_count, + SUM(CASE WHEN status = 'already_efficient' THEN 1 ELSE 0 END) as already_efficient_count, SUM(CASE WHEN is_hevc = 1 THEN original_size ELSE 0 END) as hevc_size, SUM(CASE WHEN is_hevc = 0 AND status = 'pending' THEN original_size ELSE 0 END) as pending_size FROM files @@ -262,8 +472,9 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict: "done_count": row[3] or 0, "failed_count": row[4] or 0, "skipped_larger_count": row[5] or 0, - "hevc_size": row[6] or 0, - "pending_size": row[7] or 0, + "already_efficient_count": row[6] or 0, + "hevc_size": row[7] or 0, + "pending_size": row[8] or 0, } # Running totals @@ -612,7 +823,10 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: final_path = input_file.with_suffix(".mkv") original_size = os.path.getsize(input_path) - video_duration = get_video_duration(input_path) + + # Single probe for all decisions (resolution, bitrate, codecs, channels). + info = probe_media(input_path) + video_duration = info.get("duration") log.info("") log.info("=" * 60) @@ -621,13 +835,56 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: log.info(f" Input path: {input_path}") log.info(f" Output path: {output_path}") log.info(f" Original size: {format_bytes(original_size)}") + res_str = (f"{info['width']}x{info['height']}" + if info.get("width") and info.get("height") else "unknown") + est_vbr = estimate_video_bitrate(info) + log.info(f" Resolution: {res_str}") + log.info(f" Video codec: {info.get('video_codec') or 'unknown'}" + + (f" (~{est_vbr/1_000_000:.2f} Mbps)" if est_vbr else "")) + log.info(f" Audio: {info.get('audio_codec') or 'unknown'}, " + f"{info.get('audio_channels') or '?'}ch" + + (f", {info['audio_bitrate']//1000}k" if info.get('audio_bitrate') else "")) if video_duration: log.info(f" Duration: {format_duration(video_duration)}") log_json("transcode_start", file=input_path, size=original_size, - duration=video_duration) + duration=video_duration, + width=info.get("width"), + height=info.get("height"), + video_codec=info.get("video_codec"), + est_video_bitrate=est_vbr, + audio_codec=info.get("audio_codec"), + audio_channels=info.get("audio_channels")) + + # --- Decide video: skip (already efficient) or encode with tier settings --- + video_decision = pick_video_settings(info) + if video_decision["action"] == "skip": + reason = video_decision["reason"] + log.info("") + log.info(f" ⏭ SKIPPING (already efficient): {reason}") + log.info("") + log_json("already_efficient", file=input_path, reason=reason, + est_video_bitrate=est_vbr, height=info.get("height")) + # Terminal, source-keyed status: cached so we don't re-probe-and-skip nightly. + conn.execute( + """UPDATE files SET + status = 'already_efficient', + failure_reason = ?, + scanned_at = ? + WHERE path = ?""", + (reason, datetime.now().timestamp(), input_path) + ) + conn.commit() + return {"status": "already_efficient", "reason": reason, + "original_size": original_size, "space_saved": 0} + + crf = video_decision["crf"] + preset = video_decision["preset"] + + # --- Decide audio: copy or re-encode --- + audio_decision = pick_audio_settings(info) # Mark as transcoding in DB conn.execute("UPDATE files SET status = 'transcoding' WHERE path = ?", (input_path,)) @@ -637,12 +894,13 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: CURRENT_TRANSCODE["path"] = input_path CURRENT_TRANSCODE["start_time"] = start_time + # Record the actual settings used, for later querying / re-encode decisions. settings = { - "crf": CONFIG["crf"], - "preset": CONFIG["preset"], - "audio_codec": CONFIG["audio_codec"], - "audio_bitrate": CONFIG["audio_bitrate"], - "threads": CONFIG["threads"] + "crf": crf, + "preset": preset, + "tier_height": video_decision.get("tier_height"), + "threads": CONFIG["threads"], + "audio_action": audio_decision["action"], } # Build ffmpeg command @@ -652,14 +910,23 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: "-progress", "pipe:1", # Progress to stdout for parsing "-i", input_path, "-c:v", "libx265", - "-preset", CONFIG["preset"], - "-crf", str(CONFIG["crf"]), + "-preset", preset, + "-crf", str(crf), "-x265-params", f"pools={CONFIG['threads']}", - "-c:a", CONFIG["audio_codec"], - "-b:a", CONFIG["audio_bitrate"], - "-c:s", "copy", # Copy subtitles - str(output_path) ] + # Audio: copy or re-encode + if audio_decision["action"] == "copy": + cmd += ["-c:a", "copy"] + log.info(f" Audio: copying source stream (no re-encode)") + else: + cmd += ["-c:a", audio_decision["codec"], "-b:a", audio_decision["bitrate"]] + settings["audio_codec"] = audio_decision["codec"] + settings["audio_bitrate"] = audio_decision["bitrate"] + log.info(f" Audio: re-encoding to {audio_decision['codec']} @ {audio_decision['bitrate']}") + # Subtitles: copy + cmd += ["-c:s", "copy", str(output_path)] + + log.info(f" Encode: CRF {crf}, preset {preset} (tier {video_decision.get('tier_height')}p+)") # Log the full command for debugging cmd_str = ' '.join(f'"{c}"' if ' ' in c else c for c in cmd) @@ -668,7 +935,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: if not VERBOSE: log.info(f" (use --verbose to see full command)") log.info("") - log_json("ffmpeg_command", command=cmd_str) + log_json("ffmpeg_command", command=cmd_str, settings=settings) # Log stderr to file to prevent pipe buffer deadlock # Write to /var/lib/transcoder/ since susan may not be able to create files in /var/log/ @@ -1033,6 +1300,7 @@ def main(): print(f"Completed: {stats['done_count']}") print(f"Failed: {stats['failed_count']}") print(f"Skipped (larger): {stats['skipped_larger_count']}") + print(f"Already efficient: {stats['already_efficient_count']}") print("") print("--- Lifetime Stats ---") print(f"Total transcoded: {stats.get('total_transcoded', 0)} files") @@ -1068,12 +1336,12 @@ def main(): if args.failed: cursor = conn.execute(""" SELECT path, original_size, failure_reason, status - FROM files WHERE status IN ('failed', 'skipped_larger') - ORDER BY original_size DESC + FROM files WHERE status IN ('failed', 'skipped_larger', 'already_efficient') + ORDER BY status, original_size DESC """) rows = cursor.fetchall() print("=" * 60) - print(f"Failed / Skipped Transcodes ({len(rows)} files)") + print(f"Failed / Skipped / Already-efficient ({len(rows)} files)") print("=" * 60) for path, size, reason, status in rows: print(f"\n{path}") @@ -1081,19 +1349,35 @@ def main(): print(f" Size: {size/(1024**3):.2f} GB") print(f" Reason: {reason}") if not rows: - print("No failed or skipped transcodes!") + print("Nothing failed, skipped, or already-efficient!") print("=" * 60) conn.close() return # Handle --retry-failed flag - # Only resets 'failed' — 'skipped_larger' is deterministic and intentionally left alone. + # Only resets 'failed' — 'skipped_larger' / 'already_efficient' are deterministic + # at current settings and intentionally left alone (see --reset-skipped). if args.retry_failed: cursor = conn.execute("UPDATE files SET status = 'pending', failure_reason = NULL WHERE status = 'failed'") count = cursor.rowcount conn.commit() print(f"Reset {count} failed files to pending.") - print("(skipped_larger files are left as-is — re-encoding at the same settings would fail identically.)") + print("(skipped_larger / already_efficient left as-is — re-run at the same settings " + "would repeat. Use --reset-skipped after changing thresholds.)") + conn.close() + return + + # Handle --reset-skipped flag + # For use after changing encode_tiers thresholds: re-queues files that were skipped + # as already-efficient or that produced a larger output, so they're re-evaluated. + if args.reset_skipped: + cursor = conn.execute( + "UPDATE files SET status = 'pending', failure_reason = NULL " + "WHERE status IN ('skipped_larger', 'already_efficient')" + ) + count = cursor.rowcount + conn.commit() + print(f"Reset {count} skipped/already-efficient files to pending.") conn.close() return @@ -1108,7 +1392,11 @@ def main(): log.info(f" Verbose: {'YES' if VERBOSE else 'NO'}") log.info(f" JSON log: {'YES' if JSON_LOG else 'NO'}") log.info(f" Cutoff: {CONFIG['cutoff_time']}") - log.info(f" Settings: CRF={CONFIG['crf']}, preset={CONFIG['preset']}, audio={CONFIG['audio_codec']}@{CONFIG['audio_bitrate']}") + tier_summary = ", ".join( + f"{mh}p+→CRF{crf}/{preset}" for mh, crf, preset, _ in CONFIG["encode_tiers"] + ) + log.info(f" Encode: dynamic per-file [{tier_summary}], preset from tier, threads={CONFIG['threads']}") + log.info(f" Audio: copy when efficient, else re-encode by channel count") log.info("=" * 70) log.info("") @@ -1146,28 +1434,54 @@ def main(): if DRY_RUN: log.info("") log.info("=" * 60) - log.info("DRY RUN - Files that would be transcoded (biggest first):") + log.info("DRY RUN - Per-file decision (biggest first, top 20 shown):") log.info("=" * 60) - total_size = 0 - estimated_savings = 0 + would_encode_size = 0 + would_skip_count = 0 + would_skip_size = 0 + shown = 0 - for i, video in enumerate(to_transcode[:20], 1): # Show top 20 + for i, video in enumerate(to_transcode, 1): + info = probe_media(video["path"]) + vdec = pick_video_settings(info) + adec = pick_audio_settings(info) size_gb = video["size_gb"] - total_size += video["size"] - # Estimate ~50% compression for HEVC - est_saved = video["size"] * 0.5 - estimated_savings += est_saved + res_str = (f"{info['width']}x{info['height']}" + if info.get("width") and info.get("height") else "?") + vbr = estimate_video_bitrate(info) + vbr_str = f"{vbr/1_000_000:.1f}Mbps" if vbr else "?" + + if vdec["action"] == "skip": + would_skip_count += 1 + would_skip_size += video["size"] + if shown < 20: + shown += 1 + log.info(f"{i:2}. [{size_gb:6.2f} GB] {res_str:>9} {vbr_str:>8} SKIP — " + f"{Path(video['path']).name}") + else: + would_encode_size += video["size"] + audio_str = ("audio:copy" if adec["action"] == "copy" + else f"audio:{adec['codec']}@{adec['bitrate']}") + if shown < 20: + shown += 1 + log.info(f"{i:2}. [{size_gb:6.2f} GB] {res_str:>9} {vbr_str:>8} " + f"CRF{vdec['crf']}/{vdec['preset']} {audio_str} — " + f"{Path(video['path']).name}") - log.info(f"{i:2}. [{size_gb:6.2f} GB] {video['path']}") + if len(to_transcode) > shown: + log.info(f" ... and {len(to_transcode) - shown} more (probed for totals below)") - if len(to_transcode) > 20: - log.info(f" ... and {len(to_transcode) - 20} more files") + # Rough savings estimate only for files we'd actually encode (~40% on fat sources). + estimated_savings = would_encode_size * 0.40 log.info("") log.info("=" * 60) - log.info(f"Total to transcode: {total_size / (1024**3):.2f} GB") - log.info(f"Estimated savings (~50%): {estimated_savings / (1024**3):.2f} GB") + log.info(f"Would encode: {len(to_transcode) - would_skip_count} files " + f"({would_encode_size / (1024**3):.2f} GB)") + log.info(f"Would skip: {would_skip_count} files " + f"({would_skip_size / (1024**3):.2f} GB) — already efficient") + log.info(f"Est. savings: ~{estimated_savings / (1024**3):.2f} GB (rough, ~40% of encoded)") log.info("=" * 60) return @@ -1238,7 +1552,11 @@ def main(): # Transcode result = transcode_file(video["path"], conn) - if result and result.get("status") == "skipped_larger": + if result and result.get("status") == "already_efficient": + skipped_count += 1 + log.info(f" Skipped (already efficient): {Path(video['path']).name}") + log.info(f" Running totals: {transcoded_count} done, {skipped_count} skipped, {failed_count} failed") + elif result and result.get("status") == "skipped_larger": skipped_count += 1 log.info(f" Skipped (HEVC larger than source): {Path(video['path']).name}") log.info(f" Running totals: {transcoded_count} done, {skipped_count} skipped, {failed_count} failed") |
