diff options
| -rwxr-xr-x | overnight_transcoder.py | 372 |
1 files changed, 211 insertions, 161 deletions
diff --git a/overnight_transcoder.py b/overnight_transcoder.py index 4008c34..cf54bbe 100755 --- a/overnight_transcoder.py +++ b/overnight_transcoder.py @@ -59,10 +59,10 @@ CONFIG = { "video_extensions": [".mkv", ".mp4", ".avi", ".m4v", ".mov", ".wmv"], "cutoff_time": time(6, 30), # Don't start new encodes after 06:30 "cleanup_days": 7, # Delete originals after 7 days - + # FFmpeg settings "crf": 20, - "preset": "slow", + "preset": "fast", "audio_codec": "ac3", "audio_bitrate": "640k", "threads": 12, @@ -73,9 +73,9 @@ def init_db() -> sqlite3.Connection: """Initialize the SQLite database for caching file info.""" db_path = Path(CONFIG["db_file"]) db_path.parent.mkdir(parents=True, exist_ok=True) - + conn = sqlite3.connect(CONFIG["db_file"]) - + # Main files table - tracks all video files conn.execute(""" CREATE TABLE IF NOT EXISTS files ( @@ -89,15 +89,15 @@ def init_db() -> sqlite3.Connection: is_hevc INTEGER, first_seen REAL, scanned_at REAL, - + -- Transcode info (NULL if not transcoded) - status TEXT DEFAULT 'pending', -- pending, transcoding, done, failed, stalled + status TEXT DEFAULT 'pending', -- pending, transcoding, done, failed, stalled, skipped_larger transcoded_at REAL, transcoded_size INTEGER, transcode_duration_secs REAL, transcode_settings TEXT, -- JSON: {crf, preset, audio_codec, audio_bitrate} failure_reason TEXT, - + -- Computed (updated after transcode) space_saved INTEGER, compression_ratio REAL @@ -107,7 +107,7 @@ def init_db() -> sqlite3.Connection: conn.execute("CREATE INDEX IF NOT EXISTS idx_status ON files(status)") conn.execute("CREATE INDEX IF NOT EXISTS idx_size ON files(original_size)") conn.execute("CREATE INDEX IF NOT EXISTS idx_directory ON files(directory)") - + # Stats summary table - running totals conn.execute(""" CREATE TABLE IF NOT EXISTS stats ( @@ -119,7 +119,7 @@ def init_db() -> sqlite3.Connection: ) """) conn.execute("INSERT OR IGNORE INTO stats (id) VALUES (1)") - + conn.commit() return conn @@ -134,7 +134,7 @@ def get_audio_codec(filepath: str) -> Optional[str]: "-of", "json", filepath ], capture_output=True, text=True, timeout=30) - + if result.returncode == 0: data = json.loads(result.stdout) streams = data.get("streams", []) @@ -148,7 +148,7 @@ def get_audio_codec(filepath: str) -> Optional[str]: def get_cached_file(conn: sqlite3.Connection, filepath: str, size: int, mtime: float) -> Optional[dict]: """Get cached file info if file hasn't changed. Returns dict or None.""" cursor = conn.execute( - """SELECT video_codec, audio_codec, is_hevc, status + """SELECT video_codec, audio_codec, is_hevc, status FROM files WHERE path = ? AND original_size = ? AND mtime = ?""", (filepath, size, mtime) ) @@ -163,22 +163,22 @@ def get_cached_file(conn: sqlite3.Connection, filepath: str, size: int, mtime: f return None -def cache_file(conn: sqlite3.Connection, filepath: str, size: int, mtime: float, +def cache_file(conn: sqlite3.Connection, filepath: str, size: int, mtime: float, video_codec: str, audio_codec: str, is_hevc: bool): """Cache file info.""" p = Path(filepath) now = datetime.now().timestamp() - + # Check if file already exists (to preserve first_seen) cursor = conn.execute("SELECT first_seen FROM files WHERE path = ?", (filepath,)) row = cursor.fetchone() first_seen = row[0] if row else now - + conn.execute(""" - INSERT OR REPLACE INTO files - (path, filename, directory, original_size, mtime, video_codec, audio_codec, + INSERT OR REPLACE INTO files + (path, filename, directory, original_size, mtime, video_codec, audio_codec, is_hevc, first_seen, scanned_at, status) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, COALESCE((SELECT status FROM files WHERE path = ?), 'pending')) """, (filepath, p.name, str(p.parent), size, mtime, video_codec, audio_codec, int(is_hevc), first_seen, now, filepath)) @@ -190,15 +190,15 @@ def update_transcode_result(conn: sqlite3.Connection, filepath: str, new_path: s success: bool, failure_reason: str = None): """Update database after transcode attempt.""" now = datetime.now().timestamp() - + if success: cursor = conn.execute("SELECT original_size FROM files WHERE path = ?", (filepath,)) row = cursor.fetchone() original_size = row[0] if row else 0 - + space_saved = original_size - new_size compression_ratio = new_size / original_size if original_size > 0 else 0 - + conn.execute(""" UPDATE files SET status = 'done', @@ -210,9 +210,9 @@ def update_transcode_result(conn: sqlite3.Connection, filepath: str, new_path: s compression_ratio = ?, failure_reason = NULL WHERE path = ? - """, (now, new_size, duration_secs, json.dumps(settings), + """, (now, new_size, duration_secs, json.dumps(settings), space_saved, compression_ratio, filepath)) - + # Update running totals conn.execute(""" UPDATE stats SET @@ -229,7 +229,7 @@ def update_transcode_result(conn: sqlite3.Connection, filepath: str, new_path: s failure_reason = ? WHERE path = ? """, (failure_reason, filepath)) - + conn.commit() @@ -243,12 +243,13 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict: """Get comprehensive stats from the cache.""" # File counts and sizes cursor = conn.execute(""" - SELECT + SELECT COUNT(*) as total, SUM(CASE WHEN is_hevc = 1 THEN 1 ELSE 0 END) as hevc_count, SUM(CASE WHEN is_hevc = 0 AND status = 'pending' THEN 1 ELSE 0 END) as pending_count, SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done_count, SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed_count, + SUM(CASE WHEN status = 'skipped_larger' THEN 1 ELSE 0 END) as skipped_larger_count, SUM(CASE WHEN is_hevc = 1 THEN original_size ELSE 0 END) as hevc_size, SUM(CASE WHEN is_hevc = 0 AND status = 'pending' THEN original_size ELSE 0 END) as pending_size FROM files @@ -260,10 +261,11 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict: "pending_count": row[2] or 0, "done_count": row[3] or 0, "failed_count": row[4] or 0, - "hevc_size": row[5] or 0, - "pending_size": row[6] or 0, + "skipped_larger_count": row[5] or 0, + "hevc_size": row[6] or 0, + "pending_size": row[7] or 0, } - + # Running totals cursor = conn.execute("SELECT * FROM stats WHERE id = 1") row = cursor.fetchone() @@ -271,19 +273,19 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict: file_stats["total_transcoded"] = row[1] or 0 file_stats["total_space_saved"] = row[2] or 0 file_stats["total_transcode_time"] = row[3] or 0 - + # By directory breakdown cursor = conn.execute(""" - SELECT directory, + SELECT directory, COUNT(*) as count, SUM(CASE WHEN is_hevc = 0 AND status = 'pending' THEN original_size ELSE 0 END) as pending_size - FROM files + FROM files GROUP BY directory ORDER BY pending_size DESC LIMIT 10 """) file_stats["by_directory"] = cursor.fetchall() - + return file_stats # Ensure log directory exists (skip for /var/log which requires root) @@ -357,7 +359,7 @@ def check_disk_space(): log.info("=" * 50) log.info("Disk Space Check") log.info("=" * 50) - + checked_mounts = set() for source_dir in CONFIG["source_dirs"]: # Get mount point to avoid duplicate checks @@ -368,20 +370,20 @@ def check_disk_space(): ).stdout.strip().split('\n')[-1] except: mount = source_dir - + if mount in checked_mounts: continue checked_mounts.add(mount) - + space = get_disk_space(source_dir) if space: log.info(f" {mount}:") log.info(f" Total: {format_bytes(space['total'])}") log.info(f" Free: {format_bytes(space['free'])} ({100-space['percent_used']:.1f}% available)") log_json("disk_space", mount=mount, **space) - + if space['percent_used'] > 90: - log.warning(f" ⚠️ LOW DISK SPACE on {mount}!") + log.warning(f" ⚠️ LOW DISK SPACE on {mount}!") log.info("") @@ -394,7 +396,7 @@ def get_video_duration(filepath: str) -> Optional[float]: "-of", "json", filepath ], capture_output=True, text=True, timeout=30) - + if result.returncode == 0: data = json.loads(result.stdout) return float(data.get("format", {}).get("duration", 0)) @@ -408,13 +410,14 @@ def handle_interrupt(signum, frame): log.warning("=" * 50) log.warning("INTERRUPT RECEIVED - cleaning up...") log.warning("=" * 50) - + + elapsed = None if CURRENT_TRANSCODE["path"]: log.warning(f"Interrupted transcode: {CURRENT_TRANSCODE['path']}") if CURRENT_TRANSCODE["start_time"]: elapsed = (datetime.now() - CURRENT_TRANSCODE["start_time"]).total_seconds() log.warning(f"Elapsed time: {format_duration(elapsed)}") - + # Try to kill ffmpeg if running if CURRENT_TRANSCODE["pid"]: try: @@ -422,16 +425,16 @@ def handle_interrupt(signum, frame): log.info("Sent SIGTERM to ffmpeg process") except: pass - + # Clean up partial output temp_path = Path(CURRENT_TRANSCODE["path"]).with_suffix(".transcoding.mkv") if temp_path.exists(): log.info(f"Removing partial output: {temp_path}") temp_path.unlink() - - log_json("interrupted", + + log_json("interrupted", file=CURRENT_TRANSCODE["path"], - elapsed=elapsed if CURRENT_TRANSCODE["start_time"] else None) + elapsed=elapsed) sys.exit(1) @@ -450,7 +453,7 @@ def get_video_codec(filepath: str) -> Optional[str]: "-of", "json", filepath ], capture_output=True, text=True, timeout=30) - + if result.returncode == 0: data = json.loads(result.stdout) streams = data.get("streams", []) @@ -479,42 +482,42 @@ def find_videos_to_transcode(conn: sqlite3.Connection) -> List[Dict]: cache_misses = 0 files_scanned = 0 dirs_scanned = 0 - + scan_start = datetime.now() log.info("Scanning source directories...") - + for source_dir in CONFIG["source_dirs"]: if not os.path.exists(source_dir): log.warning(f" ⚠ Source directory not found: {source_dir}") continue - + dir_start = datetime.now() dir_files = 0 dir_size = 0 log.info(f" 📁 Scanning: {source_dir}") - + for root, dirs, files in os.walk(source_dir): # Skip cleanup directory if ".cleanup" in root: continue - + dirs_scanned += 1 - + for filename in files: ext = os.path.splitext(filename)[1].lower() if ext not in CONFIG["video_extensions"]: continue - + filepath = os.path.join(root, filename) files_scanned += 1 dir_files += 1 - + try: stat = os.stat(filepath) size = stat.st_size mtime = stat.st_mtime dir_size += size - + # Check cache first cached = get_cached_file(conn, filepath, size, mtime) if cached: @@ -530,10 +533,10 @@ def find_videos_to_transcode(conn: sqlite3.Connection) -> List[Dict]: audio_codec = get_audio_codec(filepath) is_hevc = video_codec in ["hevc", "h265"] if video_codec else False status = "done" if is_hevc else "pending" - cache_file(conn, filepath, size, mtime, + cache_file(conn, filepath, size, mtime, video_codec or "unknown", audio_codec or "unknown", is_hevc) cache_misses += 1 - + videos.append({ "path": filepath, "size": size, @@ -545,18 +548,18 @@ def find_videos_to_transcode(conn: sqlite3.Connection) -> List[Dict]: }) except OSError as e: log.warning(f" ⚠ Cannot access {filepath}: {e}") - + dir_elapsed = (datetime.now() - dir_start).total_seconds() log.info(f" Found {dir_files} videos ({format_bytes(dir_size)}) in {format_duration(dir_elapsed)}") - log_json("scan_directory", + log_json("scan_directory", directory=source_dir, files=dir_files, size=dir_size, elapsed=dir_elapsed) - + scan_elapsed = (datetime.now() - scan_start).total_seconds() cache_rate = (cache_hits / (cache_hits + cache_misses) * 100) if (cache_hits + cache_misses) > 0 else 0 - + log.info("") log.info(f"Scan complete in {format_duration(scan_elapsed)}") log.info(f" Files scanned: {files_scanned}") @@ -564,14 +567,14 @@ def find_videos_to_transcode(conn: sqlite3.Connection) -> List[Dict]: log.info(f" Cache hits: {cache_hits} ({cache_rate:.1f}%)") log.info(f" Cache misses: {cache_misses} (probed)") log.info("") - + log_json("scan_complete", files=files_scanned, dirs=dirs_scanned, cache_hits=cache_hits, cache_misses=cache_misses, elapsed=scan_elapsed) - + # Sort by size descending (biggest first) videos.sort(key=lambda x: x["size"], reverse=True) return videos @@ -587,30 +590,30 @@ def verify_output(filepath: str, min_size_ratio: float = 0.1) -> bool: """Verify the transcoded file is valid.""" if not os.path.exists(filepath): return False - + # Check file size is reasonable (at least 10% of some minimum) size = os.path.getsize(filepath) if size < 1024 * 1024: # Less than 1MB is definitely wrong return False - + # Check it's valid video with ffprobe codec = get_video_codec(filepath) if codec != "hevc": return False - + return True def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: """Transcode a single file to HEVC. Returns result dict on success.""" - + input_file = Path(input_path) output_path = input_file.with_suffix(".transcoding.mkv") final_path = input_file.with_suffix(".mkv") - + original_size = os.path.getsize(input_path) video_duration = get_video_duration(input_path) - + log.info("") log.info("=" * 60) log.info(f"TRANSCODE START: {input_file.name}") @@ -620,20 +623,20 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: log.info(f" Original size: {format_bytes(original_size)}") if video_duration: log.info(f" Duration: {format_duration(video_duration)}") - - log_json("transcode_start", + + log_json("transcode_start", file=input_path, size=original_size, duration=video_duration) - + # Mark as transcoding in DB conn.execute("UPDATE files SET status = 'transcoding' WHERE path = ?", (input_path,)) conn.commit() - + start_time = datetime.now() CURRENT_TRANSCODE["path"] = input_path CURRENT_TRANSCODE["start_time"] = start_time - + settings = { "crf": CONFIG["crf"], "preset": CONFIG["preset"], @@ -641,7 +644,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: "audio_bitrate": CONFIG["audio_bitrate"], "threads": CONFIG["threads"] } - + # Build ffmpeg command cmd = [ "ffmpeg", "-y", @@ -654,10 +657,10 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: "-x265-params", f"pools={CONFIG['threads']}", "-c:a", CONFIG["audio_codec"], "-b:a", CONFIG["audio_bitrate"], - "-c:s", "srt", # Convert subtitles to SRT (mov_text from MP4 cant copy to MKV) + "-c:s", "copy", # Copy subtitles str(output_path) ] - + # Log the full command for debugging cmd_str = ' '.join(f'"{c}"' if ' ' in c else c for c in cmd) log.info(f" FFmpeg command:") @@ -666,13 +669,14 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: log.info(f" (use --verbose to see full command)") log.info("") log_json("ffmpeg_command", command=cmd_str) - + # Log stderr to file to prevent pipe buffer deadlock - stderr_log_path = Path(CONFIG["log_dir"]) / "transcoder_ffmpeg_stderr.log" + # Write to /var/lib/transcoder/ since susan may not be able to create files in /var/log/ + stderr_log_path = Path("/var/lib/transcoder") / "ffmpeg_stderr.log" stderr_file = open(stderr_log_path, "a") stderr_file.write(f"\n{'='*60}\n{datetime.now()} - {input_path}\n{'='*60}\n") stderr_file.flush() - + try: # Use Popen for progress monitoring # stderr goes to file (not PIPE) to prevent buffer deadlock @@ -684,26 +688,26 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: ) CURRENT_TRANSCODE["pid"] = process.pid log.debug(f" FFmpeg PID: {process.pid}") - + # Monitor progress last_progress_log = datetime.now() progress_interval = 60 # Log progress every 60 seconds current_time_us = 0 - + import select last_output_size = 0 last_output_growth = datetime.now() stall_timeout = 600 # 10 minutes with no output growth = stalled - + while True: # Use select for non-blocking read with timeout ready, _, _ = select.select([process.stdout], [], [], 30) - + if ready: line = process.stdout.readline() if not line and process.poll() is not None: break - + # Parse progress output if line.startswith("out_time_us="): try: @@ -712,17 +716,17 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: pass elif process.poll() is not None: break - + # Log progress periodically now = datetime.now() if (now - last_progress_log).total_seconds() >= progress_interval: elapsed = (now - start_time).total_seconds() - + if video_duration and current_time_us > 0: current_secs = current_time_us / 1_000_000 percent = (current_secs / video_duration) * 100 eta_secs = (elapsed / percent * 100) - elapsed if percent > 0 else 0 - + log.info(f" ⏳ Progress: {percent:.1f}% | Elapsed: {format_duration(elapsed)} | ETA: {format_duration(eta_secs)}") log_json("transcode_progress", file=input_path, @@ -731,12 +735,12 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: eta=eta_secs) else: log.info(f" ⏳ Elapsed: {format_duration(elapsed)}") - + # Check output size growth and detect stalls if output_path.exists(): current_size = os.path.getsize(output_path) log.info(f" Output size: {format_bytes(current_size)}") - + if current_size > last_output_size: last_output_size = current_size last_output_growth = now @@ -746,29 +750,30 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: log.error(f" Killing ffmpeg (PID {process.pid})") process.kill() process.wait(timeout=30) - + # Mark as stalled in DB with timestamp for 7-day cooldown conn.execute( "UPDATE files SET status = 'stalled', failure_reason = ?, transcoded_at = ? WHERE path = ?", - (f"Output stalled at {format_bytes(current_size)} after {format_duration(elapsed)}", + (f"Output stalled at {format_bytes(current_size)} after {format_duration(elapsed)}", datetime.now().timestamp(), input_path) ) conn.commit() - + # Clean up partial output if output_path.exists(): output_path.unlink() log.info(f" Cleaned up partial file: {output_path}") - + stderr_file.close() + CURRENT_TRANSCODE["path"] = None return None - + last_progress_log = now - + # Wait for process to finish (stderr already going to file) process.wait(timeout=60) stderr_file.close() - + if process.returncode != 0: # Read last 1000 chars of stderr log for error reporting try: @@ -776,7 +781,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: f.seek(max(0, os.path.getsize(stderr_log_path) - 1000)) error_msg = f.read() except: - error_msg = "Unknown error (check transcoder_ffmpeg_stderr.log)" + error_msg = "Unknown error (check ffmpeg_stderr.log)" log.error("=" * 60) log.error("TRANSCODE FAILED") log.error("=" * 60) @@ -787,18 +792,18 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: if line.strip(): log.error(f" {line}") log.error("=" * 60) - + log_json("transcode_failed", file=input_path, exit_code=process.returncode, error=error_msg[-500:]) - + if output_path.exists(): output_path.unlink() update_transcode_result(conn, input_path, None, 0, 0, settings, False, error_msg[-500:]) CURRENT_TRANSCODE["path"] = None return None - + except subprocess.TimeoutExpired: log.error("=" * 60) log.error(f"TRANSCODE TIMEOUT (8 hours): {input_path}") @@ -820,18 +825,18 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: if line.strip(): log.error(f" {line}") log.error("=" * 60) - + log_json("transcode_error", file=input_path, error=str(e), traceback=traceback.format_exc()) - + if output_path.exists(): output_path.unlink() update_transcode_result(conn, input_path, None, 0, 0, settings, False, str(e)) CURRENT_TRANSCODE["path"] = None return None - + # Verify output log.info(" Verifying output file...") if not verify_output(str(output_path)): @@ -846,20 +851,20 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: else: log.error(" Output file does not exist!") log.error("=" * 60) - + log_json("transcode_verify_failed", file=input_path) if output_path.exists(): output_path.unlink() update_transcode_result(conn, input_path, None, 0, 0, settings, False, "Output verification failed") CURRENT_TRANSCODE["path"] = None return None - + new_size = os.path.getsize(output_path) duration_secs = (datetime.now() - start_time).total_seconds() space_saved = original_size - new_size compression_ratio = new_size / original_size if original_size > 0 else 0 encode_speed = video_duration / duration_secs if video_duration and duration_secs > 0 else 0 - + log.info("") log.info("=" * 60) log.info("✅ TRANSCODE COMPLETE") @@ -874,7 +879,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: log.info(f" Speed: {encode_speed:.2f}x realtime") log.info("=" * 60) log.info("") - + log_json("transcode_complete", file=input_path, original_size=original_size, @@ -883,15 +888,48 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: duration_secs=duration_secs, compression_ratio=compression_ratio, encode_speed=encode_speed) - + + # Check if transcoded file is actually smaller + if new_size >= original_size: + log.warning(f" ⚠ Transcoded file is LARGER ({format_bytes(new_size)} >= {format_bytes(original_size)})") + log.warning(f" Removing transcoded file, keeping original") + log_json("transcode_larger", file=input_path, original_size=original_size, new_size=new_size) + try: + output_path.unlink() + except Exception as e: + log.error(f" Failed to remove larger output: {e}") + # Terminal status: won't be retried automatically (deterministic at same CRF/preset). + # Written directly here because update_transcode_result() can't express this status. + conn.execute( + """UPDATE files SET + status = 'skipped_larger', + failure_reason = ?, + transcoded_at = ?, + transcoded_size = ?, + transcode_duration_secs = ?, + transcode_settings = ? + WHERE path = ?""", + (f"HEVC output larger than source ({format_bytes(new_size)} >= {format_bytes(original_size)})", + datetime.now().timestamp(), new_size, duration_secs, json.dumps(settings), input_path) + ) + conn.commit() + CURRENT_TRANSCODE["path"] = None + return { + "status": "skipped_larger", + "original_size": original_size, + "new_size": new_size, + "space_saved": 0, + "duration_secs": duration_secs, + } + # Move original to cleanup directory cleanup_dir = Path(CONFIG["cleanup_dir"]) cleanup_dir.mkdir(parents=True, exist_ok=True) - + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") cleanup_name = f"{timestamp}_{input_file.name}" cleanup_path = cleanup_dir / cleanup_name - + log.info(f" Moving original to cleanup: {cleanup_path}") try: shutil.move(input_path, cleanup_path) @@ -901,7 +939,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: log.error(f" ✗ Failed to move original: {e}") log.error(f" Traceback: {traceback.format_exc()}") # Don't fail the whole operation, but keep both files - + # Rename output to final name log.info(f" Renaming output: {output_path} -> {final_path}") try: @@ -910,7 +948,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: except Exception as e: log.error(f" ✗ Failed to rename output: {e}") final_path = output_path - + # Fix permissions - mediaserver group log.debug(" Setting permissions (mediaserver:664)") try: @@ -919,10 +957,10 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]: log.debug(" ✓ Permissions set") except Exception as e: log.warning(f" ✗ Failed to set permissions: {e}") - + # Update database with success update_transcode_result(conn, input_path, str(final_path), new_size, duration_secs, settings, True) - + CURRENT_TRANSCODE["path"] = None return { "output_path": str(final_path), @@ -938,28 +976,28 @@ def cleanup_old_files(): if not cleanup_dir.exists(): log.debug("Cleanup directory doesn't exist, skipping cleanup") return - + log.info("Checking cleanup directory for old originals...") cutoff = datetime.now().timestamp() - (CONFIG["cleanup_days"] * 24 * 3600) cutoff_date = datetime.fromtimestamp(cutoff).strftime('%Y-%m-%d %H:%M') log.debug(f" Cutoff date: {cutoff_date} ({CONFIG['cleanup_days']} days ago)") - + deleted_count = 0 deleted_size = 0 kept_count = 0 kept_size = 0 - + for filepath in cleanup_dir.iterdir(): if filepath.is_file(): mtime = filepath.stat().st_mtime size = filepath.stat().st_size - + if mtime < cutoff: try: filepath.unlink() deleted_count += 1 deleted_size += size - log.info(f" 🗑️ Deleted: {filepath.name} ({format_bytes(size)})") + log.info(f" 🗑️ Deleted: {filepath.name} ({format_bytes(size)})") log_json("cleanup_deleted", file=str(filepath), size=size) except Exception as e: log.warning(f" ⚠ Failed to delete {filepath}: {e}") @@ -968,13 +1006,13 @@ def cleanup_old_files(): kept_size += size days_old = (datetime.now().timestamp() - mtime) / 86400 log.debug(f" ⏳ Keeping: {filepath.name} ({days_old:.1f} days old)") - + if deleted_count > 0: log.info(f" Cleanup: deleted {deleted_count} files, freed {format_bytes(deleted_size)}") log_json("cleanup_complete", deleted=deleted_count, freed=deleted_size) else: log.info(f" Cleanup: nothing to delete ({kept_count} files still in retention)") - + if kept_count > 0: log.debug(f" Retention: {kept_count} files ({format_bytes(kept_size)}) waiting)") @@ -982,7 +1020,7 @@ def cleanup_old_files(): def main(): # Initialize database conn = init_db() - + # Handle --stats flag if args.stats: stats = get_cache_stats(conn) @@ -994,6 +1032,7 @@ def main(): print(f"Pending transcode: {stats['pending_count']} ({stats['pending_size']/(1024**3):.2f} GB)") print(f"Completed: {stats['done_count']}") print(f"Failed: {stats['failed_count']}") + print(f"Skipped (larger): {stats['skipped_larger_count']}") print("") print("--- Lifetime Stats ---") print(f"Total transcoded: {stats.get('total_transcoded', 0)} files") @@ -1003,7 +1042,7 @@ def main(): if stats['pending_count'] > 0: est_savings = stats['pending_size'] * 0.5 print(f"\nEstimated remaining: ~{est_savings/(1024**3):.2f} GB savings") - + if stats.get('by_directory'): print("\n--- Top Directories (by pending size) ---") for dir_path, count, pending in stats['by_directory'][:5]: @@ -1011,11 +1050,11 @@ def main(): # Shorten path for display short = dir_path.replace("/disks/Plex/", "") print(f" {short}: {pending/(1024**3):.2f} GB pending") - + print("=" * 60) conn.close() return - + # Handle --clear-cache flag if args.clear_cache: conn.execute("DELETE FROM files") @@ -1024,39 +1063,42 @@ def main(): print("Cache cleared.") conn.close() return - + # Handle --failed flag if args.failed: cursor = conn.execute(""" - SELECT path, original_size, failure_reason - FROM files WHERE status = 'failed' + SELECT path, original_size, failure_reason, status + FROM files WHERE status IN ('failed', 'skipped_larger') ORDER BY original_size DESC """) rows = cursor.fetchall() print("=" * 60) - print(f"Failed Transcodes ({len(rows)} files)") + print(f"Failed / Skipped Transcodes ({len(rows)} files)") print("=" * 60) - for path, size, reason in rows: + for path, size, reason, status in rows: print(f"\n{path}") + print(f" Status: {status}") print(f" Size: {size/(1024**3):.2f} GB") print(f" Reason: {reason}") if not rows: - print("No failed transcodes!") + print("No failed or skipped transcodes!") print("=" * 60) conn.close() return - + # Handle --retry-failed flag + # Only resets 'failed' — 'skipped_larger' is deterministic and intentionally left alone. if args.retry_failed: cursor = conn.execute("UPDATE files SET status = 'pending', failure_reason = NULL WHERE status = 'failed'") count = cursor.rowcount conn.commit() print(f"Reset {count} failed files to pending.") + print("(skipped_larger files are left as-is — re-encoding at the same settings would fail identically.)") conn.close() return - + session_start = datetime.now() - + log.info("") log.info("=" * 70) log.info(" OVERNIGHT VIDEO TRANSCODER") @@ -1069,70 +1111,72 @@ def main(): log.info(f" Settings: CRF={CONFIG['crf']}, preset={CONFIG['preset']}, audio={CONFIG['audio_codec']}@{CONFIG['audio_bitrate']}") log.info("=" * 70) log.info("") - + log_json("session_start", mode="dry_run" if DRY_RUN else "live", config=CONFIG) - + # Check disk space first check_disk_space() - + # Run cleanup first (skip in dry run) if not DRY_RUN: cleanup_old_files() - + # Find videos to transcode videos = find_videos_to_transcode(conn) - + # Filter to non-HEVC pending only (using cached codec info) + # Only 'pending' is queued; done / failed / stalled / skipped_larger all fall through. to_transcode = [] hevc_count = 0 hevc_size = 0 - + for video in videos: if video.get("is_hevc", False) or video.get("status") == "done": hevc_count += 1 hevc_size += video["size"] elif video.get("status") == "pending": to_transcode.append(video) - + # failed / stalled / skipped_larger: intentionally not queued here + log.info(f"Already HEVC: {hevc_count} files ({hevc_size / (1024**3):.2f} GB)") log.info(f"Need transcoding: {len(to_transcode)} files") - + if DRY_RUN: log.info("") log.info("=" * 60) log.info("DRY RUN - Files that would be transcoded (biggest first):") log.info("=" * 60) - + total_size = 0 estimated_savings = 0 - + for i, video in enumerate(to_transcode[:20], 1): # Show top 20 size_gb = video["size_gb"] total_size += video["size"] # Estimate ~50% compression for HEVC est_saved = video["size"] * 0.5 estimated_savings += est_saved - + log.info(f"{i:2}. [{size_gb:6.2f} GB] {video['path']}") - + if len(to_transcode) > 20: log.info(f" ... and {len(to_transcode) - 20} more files") - + log.info("") log.info("=" * 60) log.info(f"Total to transcode: {total_size / (1024**3):.2f} GB") log.info(f"Estimated savings (~50%): {estimated_savings / (1024**3):.2f} GB") log.info("=" * 60) return - + # Normal run transcoded_count = 0 total_saved = 0 failed_count = 0 skipped_count = 0 - + log.info("") log.info("=" * 60) log.info("Starting transcode queue") @@ -1140,7 +1184,7 @@ def main(): log.info(f" Files to process: {len(to_transcode)}") log.info(f" Total size: {format_bytes(sum(v['size'] for v in to_transcode))}") log.info("") - + for i, video in enumerate(to_transcode, 1): # Check cutoff time if is_past_cutoff(): @@ -1152,13 +1196,13 @@ def main(): log.info("=" * 60) log_json("cutoff_reached", remaining_files=remaining) break - + # Skip already done or failed if video.get("status") in ["done", "failed"]: log.debug(f" Skipping (status={video['status']}): {video['path']}") skipped_count += 1 continue - + # Skip stalled files unless 7 days have passed if video.get("status") == "stalled": row = conn.execute( @@ -1177,12 +1221,12 @@ def main(): log.debug(f" Skipping stalled file (no timestamp): {video['path']}") skipped_count += 1 continue - + log.info(f"[{i}/{len(to_transcode)}] Queued: {Path(video['path']).name}") log.info(f" Size: {format_bytes(video['size'])}") log.info(f" Codec: video={video.get('video_codec', 'unknown')}, audio={video.get('audio_codec', 'unknown')}") log.info(f" Path: {video['path']}") - + log_json("transcode_queued", index=i, total=len(to_transcode), @@ -1190,11 +1234,15 @@ def main(): size=video['size'], video_codec=video.get('video_codec'), audio_codec=video.get('audio_codec')) - + # Transcode result = transcode_file(video["path"], conn) - - if result: + + if result and result.get("status") == "skipped_larger": + skipped_count += 1 + log.info(f" Skipped (HEVC larger than source): {Path(video['path']).name}") + log.info(f" Running totals: {transcoded_count} done, {skipped_count} skipped, {failed_count} failed") + elif result: total_saved += result["space_saved"] transcoded_count += 1 log.info(f" Running totals: {transcoded_count} done, {format_bytes(total_saved)} saved") @@ -1202,10 +1250,10 @@ def main(): failed_count += 1 log.error(f" ❌ Transcode #{i} failed: {video['path']}") log.info(f" Running totals: {transcoded_count} done, {failed_count} failed") - + session_end = datetime.now() session_duration = (session_end - session_start).total_seconds() - + log.info("") log.info("=" * 70) log.info(" SESSION COMPLETE") @@ -1217,7 +1265,7 @@ def main(): log.info(f" Failed: {failed_count} files") log.info(f" Skipped: {skipped_count} files") log.info(f" Space saved: {format_bytes(total_saved)}") - + # Get updated lifetime stats stats = get_cache_stats(conn) log.info("") @@ -1227,17 +1275,19 @@ def main(): log.info(f" Pending: {stats.get('pending_count', 0)} files ({format_bytes(stats.get('pending_size', 0))})") log.info("=" * 70) log.info("") - + log_json("session_complete", started=session_start.isoformat(), ended=session_end.isoformat(), duration_secs=session_duration, transcoded=transcoded_count, + skipped=skipped_count, + failed=failed_count, space_saved=total_saved, lifetime_transcoded=stats.get('total_transcoded', 0), lifetime_saved=stats.get('total_space_saved', 0), pending_count=stats.get('pending_count', 0)) - + conn.close() |