tom/susan-scripts
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Don't fail if size is smaller

Browse Source
This commit is contained in:
Tom Flux
2026-07-07 11:27:21 +01:00
parent 38859f5b0a
commit 1596ead0c5
1 changed files with 211 additions and 161 deletions
Download Patch File Download Diff File Expand all files Collapse all files
overnight_transcoder.py
+66 -16
View File
@@ -62,7 +62,7 @@ CONFIG = {
# FFmpeg settings
"crf": 20,
"preset": "slow",
"preset": "fast",
"audio_codec": "ac3",
"audio_bitrate": "640k",
"threads": 12,
@@ -91,7 +91,7 @@ def init_db() -> sqlite3.Connection:
scanned_at REAL,
-- Transcode info (NULL if not transcoded)
status TEXT DEFAULT 'pending', -- pending, transcoding, done, failed, stalled
status TEXT DEFAULT 'pending', -- pending, transcoding, done, failed, stalled, skipped_larger
transcoded_at REAL,
transcoded_size INTEGER,
transcode_duration_secs REAL,
@@ -249,6 +249,7 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict:
SUM(CASE WHEN is_hevc = 0 AND status = 'pending' THEN 1 ELSE 0 END) as pending_count,
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done_count,
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed_count,
SUM(CASE WHEN status = 'skipped_larger' THEN 1 ELSE 0 END) as skipped_larger_count,
SUM(CASE WHEN is_hevc = 1 THEN original_size ELSE 0 END) as hevc_size,
SUM(CASE WHEN is_hevc = 0 AND status = 'pending' THEN original_size ELSE 0 END) as pending_size
FROM files
@@ -260,8 +261,9 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict:
"pending_count": row[2] or 0,
"done_count": row[3] or 0,
"failed_count": row[4] or 0,
"hevc_size": row[5] or 0,
"pending_size": row[6] or 0,
"skipped_larger_count": row[5] or 0,
"hevc_size": row[6] or 0,
"pending_size": row[7] or 0,
}
# Running totals
@@ -381,7 +383,7 @@ def check_disk_space():
log_json("disk_space", mount=mount, **space)
if space['percent_used'] > 90:
log.warning(f" ⚠️ LOW DISK SPACE on {mount}!")
log.warning(f" ⚠️ LOW DISK SPACE on {mount}!")
log.info("")
@@ -409,6 +411,7 @@ def handle_interrupt(signum, frame):
log.warning("INTERRUPT RECEIVED - cleaning up...")
log.warning("=" * 50)
elapsed = None
if CURRENT_TRANSCODE["path"]:
log.warning(f"Interrupted transcode: {CURRENT_TRANSCODE['path']}")
if CURRENT_TRANSCODE["start_time"]:
@@ -431,7 +434,7 @@ def handle_interrupt(signum, frame):
log_json("interrupted",
file=CURRENT_TRANSCODE["path"],
elapsed=elapsed if CURRENT_TRANSCODE["start_time"] else None)
elapsed=elapsed)
sys.exit(1)
@@ -654,7 +657,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
"-x265-params", f"pools={CONFIG['threads']}",
"-c:a", CONFIG["audio_codec"],
"-b:a", CONFIG["audio_bitrate"],
"-c:s", "srt", # Convert subtitles to SRT (mov_text from MP4 cant copy to MKV)
"-c:s", "copy", # Copy subtitles
str(output_path)
]
@@ -668,7 +671,8 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
log_json("ffmpeg_command", command=cmd_str)
# Log stderr to file to prevent pipe buffer deadlock
stderr_log_path = Path(CONFIG["log_dir"]) / "transcoder_ffmpeg_stderr.log"
# Write to /var/lib/transcoder/ since susan may not be able to create files in /var/log/
stderr_log_path = Path("/var/lib/transcoder") / "ffmpeg_stderr.log"
stderr_file = open(stderr_log_path, "a")
stderr_file.write(f"\n{'='*60}\n{datetime.now()} - {input_path}\n{'='*60}\n")
stderr_file.flush()
@@ -761,6 +765,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
log.info(f" Cleaned up partial file: {output_path}")
stderr_file.close()
CURRENT_TRANSCODE["path"] = None
return None
last_progress_log = now
@@ -776,7 +781,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
f.seek(max(0, os.path.getsize(stderr_log_path) - 1000))
error_msg = f.read()
except:
error_msg = "Unknown error (check transcoder_ffmpeg_stderr.log)"
error_msg = "Unknown error (check ffmpeg_stderr.log)"
log.error("=" * 60)
log.error("TRANSCODE FAILED")
log.error("=" * 60)
@@ -884,6 +889,39 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
compression_ratio=compression_ratio,
encode_speed=encode_speed)
# Check if transcoded file is actually smaller
if new_size >= original_size:
log.warning(f" ⚠ Transcoded file is LARGER ({format_bytes(new_size)} >= {format_bytes(original_size)})")
log.warning(f" Removing transcoded file, keeping original")
log_json("transcode_larger", file=input_path, original_size=original_size, new_size=new_size)
try:
output_path.unlink()
except Exception as e:
log.error(f" Failed to remove larger output: {e}")
# Terminal status: won't be retried automatically (deterministic at same CRF/preset).
# Written directly here because update_transcode_result() can't express this status.
conn.execute(
"""UPDATE files SET
status = 'skipped_larger',
failure_reason = ?,
transcoded_at = ?,
transcoded_size = ?,
transcode_duration_secs = ?,
transcode_settings = ?
WHERE path = ?""",
(f"HEVC output larger than source ({format_bytes(new_size)} >= {format_bytes(original_size)})",
datetime.now().timestamp(), new_size, duration_secs, json.dumps(settings), input_path)
)
conn.commit()
CURRENT_TRANSCODE["path"] = None
return {
"status": "skipped_larger",
"original_size": original_size,
"new_size": new_size,
"space_saved": 0,
"duration_secs": duration_secs,
}
# Move original to cleanup directory
cleanup_dir = Path(CONFIG["cleanup_dir"])
cleanup_dir.mkdir(parents=True, exist_ok=True)
@@ -959,7 +997,7 @@ def cleanup_old_files():
filepath.unlink()
deleted_count += 1
deleted_size += size
log.info(f" 🗑️ Deleted: {filepath.name} ({format_bytes(size)})")
log.info(f" 🗑️ Deleted: {filepath.name} ({format_bytes(size)})")
log_json("cleanup_deleted", file=str(filepath), size=size)
except Exception as e:
log.warning(f" ⚠ Failed to delete {filepath}: {e}")
@@ -994,6 +1032,7 @@ def main():
print(f"Pending transcode: {stats['pending_count']} ({stats['pending_size']/(1024**3):.2f} GB)")
print(f"Completed: {stats['done_count']}")
print(f"Failed: {stats['failed_count']}")
print(f"Skipped (larger): {stats['skipped_larger_count']}")
print("")
print("--- Lifetime Stats ---")
print(f"Total transcoded: {stats.get('total_transcoded', 0)} files")
@@ -1028,30 +1067,33 @@ def main():
# Handle --failed flag
if args.failed:
cursor = conn.execute("""
SELECT path, original_size, failure_reason
FROM files WHERE status = 'failed'
SELECT path, original_size, failure_reason, status
FROM files WHERE status IN ('failed', 'skipped_larger')
ORDER BY original_size DESC
""")
rows = cursor.fetchall()
print("=" * 60)
print(f"Failed Transcodes ({len(rows)} files)")
print(f"Failed / Skipped Transcodes ({len(rows)} files)")
print("=" * 60)
for path, size, reason in rows:
for path, size, reason, status in rows:
print(f"\n{path}")
print(f" Status: {status}")
print(f" Size: {size/(1024**3):.2f} GB")
print(f" Reason: {reason}")
if not rows:
print("No failed transcodes!")
print("No failed or skipped transcodes!")
print("=" * 60)
conn.close()
return
# Handle --retry-failed flag
# Only resets 'failed' — 'skipped_larger' is deterministic and intentionally left alone.
if args.retry_failed:
cursor = conn.execute("UPDATE files SET status = 'pending', failure_reason = NULL WHERE status = 'failed'")
count = cursor.rowcount
conn.commit()
print(f"Reset {count} failed files to pending.")
print("(skipped_larger files are left as-is — re-encoding at the same settings would fail identically.)")
conn.close()
return
@@ -1085,6 +1127,7 @@ def main():
videos = find_videos_to_transcode(conn)
# Filter to non-HEVC pending only (using cached codec info)
# Only 'pending' is queued; done / failed / stalled / skipped_larger all fall through.
to_transcode = []
hevc_count = 0
hevc_size = 0
@@ -1095,6 +1138,7 @@ def main():
hevc_size += video["size"]
elif video.get("status") == "pending":
to_transcode.append(video)
# failed / stalled / skipped_larger: intentionally not queued here
log.info(f"Already HEVC: {hevc_count} files ({hevc_size / (1024**3):.2f} GB)")
log.info(f"Need transcoding: {len(to_transcode)} files")
@@ -1194,7 +1238,11 @@ def main():
# Transcode
result = transcode_file(video["path"], conn)
if result:
if result and result.get("status") == "skipped_larger":
skipped_count += 1
log.info(f" Skipped (HEVC larger than source): {Path(video['path']).name}")
log.info(f" Running totals: {transcoded_count} done, {skipped_count} skipped, {failed_count} failed")
elif result:
total_saved += result["space_saved"]
transcoded_count += 1
log.info(f" Running totals: {transcoded_count} done, {format_bytes(total_saved)} saved")
@@ -1233,6 +1281,8 @@ def main():
ended=session_end.isoformat(),
duration_secs=session_duration,
transcoded=transcoded_count,
skipped=skipped_count,
failed=failed_count,
space_saved=total_saved,
lifetime_transcoded=stats.get('total_transcoded', 0),
lifetime_saved=stats.get('total_space_saved', 0),