Dynamic config
This commit is contained in:
+362
-44
@@ -33,6 +33,7 @@ parser.add_argument("--dry", action="store_true", help="Dry run - don't transcod
|
|||||||
parser.add_argument("--stats", action="store_true", help="Show cache statistics and exit")
|
parser.add_argument("--stats", action="store_true", help="Show cache statistics and exit")
|
||||||
parser.add_argument("--failed", action="store_true", help="Show failed transcodes and exit")
|
parser.add_argument("--failed", action="store_true", help="Show failed transcodes and exit")
|
||||||
parser.add_argument("--retry-failed", action="store_true", help="Reset failed files to pending for retry")
|
parser.add_argument("--retry-failed", action="store_true", help="Reset failed files to pending for retry")
|
||||||
|
parser.add_argument("--reset-skipped", action="store_true", help="Reset skipped_larger + already_efficient files to pending (use after changing tier thresholds)")
|
||||||
parser.add_argument("--clear-cache", action="store_true", help="Clear the cache and exit")
|
parser.add_argument("--clear-cache", action="store_true", help="Clear the cache and exit")
|
||||||
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose debug output")
|
parser.add_argument("--verbose", "-v", action="store_true", help="Verbose debug output")
|
||||||
parser.add_argument("--json-log", action="store_true", help="Output structured JSON logs (one per line)")
|
parser.add_argument("--json-log", action="store_true", help="Output structured JSON logs (one per line)")
|
||||||
@@ -60,12 +61,37 @@ CONFIG = {
|
|||||||
"cutoff_time": time(6, 30), # Don't start new encodes after 06:30
|
"cutoff_time": time(6, 30), # Don't start new encodes after 06:30
|
||||||
"cleanup_days": 7, # Delete originals after 7 days
|
"cleanup_days": 7, # Delete originals after 7 days
|
||||||
|
|
||||||
# FFmpeg settings
|
|
||||||
"crf": 20,
|
|
||||||
"preset": "fast",
|
|
||||||
"audio_codec": "ac3",
|
|
||||||
"audio_bitrate": "640k",
|
|
||||||
"threads": 12,
|
"threads": 12,
|
||||||
|
|
||||||
|
# --- Dynamic video encode tiers ---
|
||||||
|
# Settings are chosen per-file based on source resolution and bitrate.
|
||||||
|
# Each tier: (min_height, crf, preset, skip_below_bps)
|
||||||
|
# - min_height: applies to files at least this tall (first match wins, so keep descending)
|
||||||
|
# - crf/preset: x265 settings for this tier
|
||||||
|
# - skip_below_bps: if the source VIDEO bitrate is already below this, skip the file
|
||||||
|
# entirely (status 'already_efficient') — the expected saving is too
|
||||||
|
# small to be worth an encode slot. Tuned so we skip anything likely
|
||||||
|
# to save <~15%.
|
||||||
|
"encode_tiers": [
|
||||||
|
(2160, 22, "medium", 12_000_000), # 4K
|
||||||
|
(1080, 23, "medium", 6_000_000), # 1080p
|
||||||
|
( 720, 24, "medium", 3_500_000), # 720p
|
||||||
|
( 0, 25, "medium", 1_500_000), # SD / everything smaller
|
||||||
|
],
|
||||||
|
|
||||||
|
# --- Dynamic audio handling ---
|
||||||
|
# Copy the source audio stream untouched when it's already an efficient codec at a
|
||||||
|
# sane bitrate. Only re-encode when the source is a bloated/lossless codec, and even
|
||||||
|
# then match the target codec to the channel count. This avoids the classic regression
|
||||||
|
# of upconverting e.g. AAC 256k stereo to AC3 640k and inflating the file.
|
||||||
|
"audio_copy_codecs": ["aac", "ac3", "eac3", "opus", "mp3"],
|
||||||
|
"audio_copy_max_bps_stereo": 384_000, # copy stereo/mono at or below this
|
||||||
|
"audio_copy_max_bps_surround": 768_000, # copy 5.1/7.1 at or below this
|
||||||
|
# Re-encode targets (only used when copy conditions aren't met):
|
||||||
|
"audio_reencode_stereo_codec": "aac",
|
||||||
|
"audio_reencode_stereo_bitrate": "192k",
|
||||||
|
"audio_reencode_surround_codec": "ac3",
|
||||||
|
"audio_reencode_surround_bitrate": "640k",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -91,7 +117,7 @@ def init_db() -> sqlite3.Connection:
|
|||||||
scanned_at REAL,
|
scanned_at REAL,
|
||||||
|
|
||||||
-- Transcode info (NULL if not transcoded)
|
-- Transcode info (NULL if not transcoded)
|
||||||
status TEXT DEFAULT 'pending', -- pending, transcoding, done, failed, stalled, skipped_larger
|
status TEXT DEFAULT 'pending', -- pending, transcoding, done, failed, stalled, skipped_larger, already_efficient
|
||||||
transcoded_at REAL,
|
transcoded_at REAL,
|
||||||
transcoded_size INTEGER,
|
transcoded_size INTEGER,
|
||||||
transcode_duration_secs REAL,
|
transcode_duration_secs REAL,
|
||||||
@@ -145,7 +171,190 @@ def get_audio_codec(filepath: str) -> Optional[str]:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def get_cached_file(conn: sqlite3.Connection, filepath: str, size: int, mtime: float) -> Optional[dict]:
|
def probe_media(filepath: str) -> dict:
|
||||||
|
"""
|
||||||
|
Single-shot probe of the first video and first audio stream plus container info.
|
||||||
|
Returns a dict with whatever could be determined; missing fields are None.
|
||||||
|
|
||||||
|
Fields: width, height, video_codec, video_bitrate (bps), duration (s),
|
||||||
|
audio_codec, audio_channels, audio_bitrate (bps),
|
||||||
|
container_bitrate (bps), size (bytes).
|
||||||
|
"""
|
||||||
|
info = {
|
||||||
|
"width": None, "height": None, "video_codec": None, "video_bitrate": None,
|
||||||
|
"duration": None, "audio_codec": None, "audio_channels": None,
|
||||||
|
"audio_bitrate": None, "container_bitrate": None, "size": None,
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
info["size"] = os.path.getsize(filepath)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = subprocess.run([
|
||||||
|
"ffprobe", "-v", "quiet",
|
||||||
|
"-print_format", "json",
|
||||||
|
"-show_format",
|
||||||
|
"-show_streams",
|
||||||
|
filepath
|
||||||
|
], capture_output=True, text=True, timeout=60)
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
return info
|
||||||
|
|
||||||
|
data = json.loads(result.stdout)
|
||||||
|
fmt = data.get("format", {})
|
||||||
|
|
||||||
|
# Container-level duration and bitrate
|
||||||
|
try:
|
||||||
|
info["duration"] = float(fmt.get("duration")) if fmt.get("duration") else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
info["container_bitrate"] = int(fmt.get("bit_rate")) if fmt.get("bit_rate") else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
for stream in data.get("streams", []):
|
||||||
|
stype = stream.get("codec_type")
|
||||||
|
if stype == "video" and info["video_codec"] is None:
|
||||||
|
info["video_codec"] = (stream.get("codec_name") or "").lower() or None
|
||||||
|
info["width"] = stream.get("width")
|
||||||
|
info["height"] = stream.get("height")
|
||||||
|
try:
|
||||||
|
if stream.get("bit_rate"):
|
||||||
|
info["video_bitrate"] = int(stream["bit_rate"])
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
# Stream duration can be present even when format duration isn't
|
||||||
|
if info["duration"] is None:
|
||||||
|
try:
|
||||||
|
info["duration"] = float(stream.get("duration")) if stream.get("duration") else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
elif stype == "audio" and info["audio_codec"] is None:
|
||||||
|
info["audio_codec"] = (stream.get("codec_name") or "").lower() or None
|
||||||
|
info["audio_channels"] = stream.get("channels")
|
||||||
|
try:
|
||||||
|
if stream.get("bit_rate"):
|
||||||
|
info["audio_bitrate"] = int(stream["bit_rate"])
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
pass
|
||||||
|
except Exception as e:
|
||||||
|
log.warning(f"Failed to probe media {filepath}: {e}")
|
||||||
|
|
||||||
|
return info
|
||||||
|
|
||||||
|
|
||||||
|
def estimate_video_bitrate(info: dict) -> Optional[int]:
|
||||||
|
"""
|
||||||
|
Best-effort source VIDEO bitrate in bps, used only for the skip-or-encode decision.
|
||||||
|
|
||||||
|
Preference order:
|
||||||
|
1. Video stream's own bit_rate (most accurate when present).
|
||||||
|
2. Container bitrate minus known audio bitrate.
|
||||||
|
3. (size * 8 / duration) minus known audio bitrate.
|
||||||
|
Falls back to overall rate if audio is unknown; this slightly overestimates the
|
||||||
|
video rate, which biases toward attempting rather than skipping (safe — the
|
||||||
|
skipped_larger guard is the backstop).
|
||||||
|
"""
|
||||||
|
if info.get("video_bitrate"):
|
||||||
|
return info["video_bitrate"]
|
||||||
|
|
||||||
|
audio_bps = info.get("audio_bitrate") or 0
|
||||||
|
|
||||||
|
overall = info.get("container_bitrate")
|
||||||
|
if not overall and info.get("size") and info.get("duration"):
|
||||||
|
try:
|
||||||
|
overall = int(info["size"] * 8 / info["duration"])
|
||||||
|
except (TypeError, ZeroDivisionError):
|
||||||
|
overall = None
|
||||||
|
|
||||||
|
if overall:
|
||||||
|
est = overall - audio_bps
|
||||||
|
return est if est > 0 else overall
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def pick_video_settings(info: dict) -> dict:
|
||||||
|
"""
|
||||||
|
Decide whether to encode and with what x265 settings, based on resolution and
|
||||||
|
estimated source video bitrate. Returns:
|
||||||
|
{"action": "skip", "reason": str} -> already efficient
|
||||||
|
{"action": "encode", "crf": int, "preset": str,
|
||||||
|
"tier_height": int, "video_bitrate": int|None} -> proceed
|
||||||
|
"""
|
||||||
|
height = info.get("height") or 0
|
||||||
|
est_bitrate = estimate_video_bitrate(info)
|
||||||
|
|
||||||
|
# Find the first tier the file is tall enough for (tiers are descending).
|
||||||
|
tier = None
|
||||||
|
for min_height, crf, preset, skip_below in CONFIG["encode_tiers"]:
|
||||||
|
if height >= min_height:
|
||||||
|
tier = (min_height, crf, preset, skip_below)
|
||||||
|
break
|
||||||
|
if tier is None:
|
||||||
|
# Shouldn't happen (last tier is min_height 0), but be safe.
|
||||||
|
tier = CONFIG["encode_tiers"][-1]
|
||||||
|
min_height, crf, preset, skip_below = tier
|
||||||
|
|
||||||
|
# Skip if the source video bitrate is already below this tier's floor.
|
||||||
|
# If we couldn't estimate bitrate at all, don't skip — attempt and let the
|
||||||
|
# skipped_larger guard catch a bad outcome.
|
||||||
|
if est_bitrate is not None and est_bitrate < skip_below:
|
||||||
|
return {
|
||||||
|
"action": "skip",
|
||||||
|
"reason": (f"source video ~{est_bitrate/1_000_000:.2f} Mbps is below the "
|
||||||
|
f"{skip_below/1_000_000:.1f} Mbps floor for {min_height}p+ "
|
||||||
|
f"(expected saving too small)"),
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
"action": "encode",
|
||||||
|
"crf": crf,
|
||||||
|
"preset": preset,
|
||||||
|
"tier_height": min_height,
|
||||||
|
"video_bitrate": est_bitrate,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def pick_audio_settings(info: dict) -> dict:
|
||||||
|
"""
|
||||||
|
Decide audio handling. Returns:
|
||||||
|
{"action": "copy"} -> stream-copy the source audio (fast, lossless)
|
||||||
|
{"action": "encode", "codec": str, "bitrate": str} -> re-encode
|
||||||
|
Copy when the source is already an efficient codec at a sane bitrate for its
|
||||||
|
channel count. Otherwise re-encode, matching codec to channels.
|
||||||
|
"""
|
||||||
|
codec = (info.get("audio_codec") or "").lower()
|
||||||
|
channels = info.get("audio_channels") or 2
|
||||||
|
bitrate = info.get("audio_bitrate") # may be None
|
||||||
|
|
||||||
|
is_surround = channels >= 3
|
||||||
|
copy_ceiling = (CONFIG["audio_copy_max_bps_surround"] if is_surround
|
||||||
|
else CONFIG["audio_copy_max_bps_stereo"])
|
||||||
|
|
||||||
|
if codec in CONFIG["audio_copy_codecs"]:
|
||||||
|
# No bitrate info -> trust the codec and copy (re-encoding a good codec blind
|
||||||
|
# risks inflating it, which is exactly what we're trying to avoid).
|
||||||
|
if bitrate is None or bitrate <= copy_ceiling:
|
||||||
|
return {"action": "copy"}
|
||||||
|
|
||||||
|
# Re-encode path: match target to channel count.
|
||||||
|
if is_surround:
|
||||||
|
return {
|
||||||
|
"action": "encode",
|
||||||
|
"codec": CONFIG["audio_reencode_surround_codec"],
|
||||||
|
"bitrate": CONFIG["audio_reencode_surround_bitrate"],
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
"action": "encode",
|
||||||
|
"codec": CONFIG["audio_reencode_stereo_codec"],
|
||||||
|
"bitrate": CONFIG["audio_reencode_stereo_bitrate"],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
"""Get cached file info if file hasn't changed. Returns dict or None."""
|
"""Get cached file info if file hasn't changed. Returns dict or None."""
|
||||||
cursor = conn.execute(
|
cursor = conn.execute(
|
||||||
"""SELECT video_codec, audio_codec, is_hevc, status
|
"""SELECT video_codec, audio_codec, is_hevc, status
|
||||||
@@ -250,6 +459,7 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict:
|
|||||||
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done_count,
|
SUM(CASE WHEN status = 'done' THEN 1 ELSE 0 END) as done_count,
|
||||||
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed_count,
|
SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed_count,
|
||||||
SUM(CASE WHEN status = 'skipped_larger' THEN 1 ELSE 0 END) as skipped_larger_count,
|
SUM(CASE WHEN status = 'skipped_larger' THEN 1 ELSE 0 END) as skipped_larger_count,
|
||||||
|
SUM(CASE WHEN status = 'already_efficient' THEN 1 ELSE 0 END) as already_efficient_count,
|
||||||
SUM(CASE WHEN is_hevc = 1 THEN original_size ELSE 0 END) as hevc_size,
|
SUM(CASE WHEN is_hevc = 1 THEN original_size ELSE 0 END) as hevc_size,
|
||||||
SUM(CASE WHEN is_hevc = 0 AND status = 'pending' THEN original_size ELSE 0 END) as pending_size
|
SUM(CASE WHEN is_hevc = 0 AND status = 'pending' THEN original_size ELSE 0 END) as pending_size
|
||||||
FROM files
|
FROM files
|
||||||
@@ -262,8 +472,9 @@ def get_cache_stats(conn: sqlite3.Connection) -> dict:
|
|||||||
"done_count": row[3] or 0,
|
"done_count": row[3] or 0,
|
||||||
"failed_count": row[4] or 0,
|
"failed_count": row[4] or 0,
|
||||||
"skipped_larger_count": row[5] or 0,
|
"skipped_larger_count": row[5] or 0,
|
||||||
"hevc_size": row[6] or 0,
|
"already_efficient_count": row[6] or 0,
|
||||||
"pending_size": row[7] or 0,
|
"hevc_size": row[7] or 0,
|
||||||
|
"pending_size": row[8] or 0,
|
||||||
}
|
}
|
||||||
|
|
||||||
# Running totals
|
# Running totals
|
||||||
@@ -612,7 +823,10 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
|
|||||||
final_path = input_file.with_suffix(".mkv")
|
final_path = input_file.with_suffix(".mkv")
|
||||||
|
|
||||||
original_size = os.path.getsize(input_path)
|
original_size = os.path.getsize(input_path)
|
||||||
video_duration = get_video_duration(input_path)
|
|
||||||
|
# Single probe for all decisions (resolution, bitrate, codecs, channels).
|
||||||
|
info = probe_media(input_path)
|
||||||
|
video_duration = info.get("duration")
|
||||||
|
|
||||||
log.info("")
|
log.info("")
|
||||||
log.info("=" * 60)
|
log.info("=" * 60)
|
||||||
@@ -621,13 +835,56 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
|
|||||||
log.info(f" Input path: {input_path}")
|
log.info(f" Input path: {input_path}")
|
||||||
log.info(f" Output path: {output_path}")
|
log.info(f" Output path: {output_path}")
|
||||||
log.info(f" Original size: {format_bytes(original_size)}")
|
log.info(f" Original size: {format_bytes(original_size)}")
|
||||||
|
res_str = (f"{info['width']}x{info['height']}"
|
||||||
|
if info.get("width") and info.get("height") else "unknown")
|
||||||
|
est_vbr = estimate_video_bitrate(info)
|
||||||
|
log.info(f" Resolution: {res_str}")
|
||||||
|
log.info(f" Video codec: {info.get('video_codec') or 'unknown'}"
|
||||||
|
+ (f" (~{est_vbr/1_000_000:.2f} Mbps)" if est_vbr else ""))
|
||||||
|
log.info(f" Audio: {info.get('audio_codec') or 'unknown'}, "
|
||||||
|
f"{info.get('audio_channels') or '?'}ch"
|
||||||
|
+ (f", {info['audio_bitrate']//1000}k" if info.get('audio_bitrate') else ""))
|
||||||
if video_duration:
|
if video_duration:
|
||||||
log.info(f" Duration: {format_duration(video_duration)}")
|
log.info(f" Duration: {format_duration(video_duration)}")
|
||||||
|
|
||||||
log_json("transcode_start",
|
log_json("transcode_start",
|
||||||
file=input_path,
|
file=input_path,
|
||||||
size=original_size,
|
size=original_size,
|
||||||
duration=video_duration)
|
duration=video_duration,
|
||||||
|
width=info.get("width"),
|
||||||
|
height=info.get("height"),
|
||||||
|
video_codec=info.get("video_codec"),
|
||||||
|
est_video_bitrate=est_vbr,
|
||||||
|
audio_codec=info.get("audio_codec"),
|
||||||
|
audio_channels=info.get("audio_channels"))
|
||||||
|
|
||||||
|
# --- Decide video: skip (already efficient) or encode with tier settings ---
|
||||||
|
video_decision = pick_video_settings(info)
|
||||||
|
if video_decision["action"] == "skip":
|
||||||
|
reason = video_decision["reason"]
|
||||||
|
log.info("")
|
||||||
|
log.info(f" ⏭ SKIPPING (already efficient): {reason}")
|
||||||
|
log.info("")
|
||||||
|
log_json("already_efficient", file=input_path, reason=reason,
|
||||||
|
est_video_bitrate=est_vbr, height=info.get("height"))
|
||||||
|
# Terminal, source-keyed status: cached so we don't re-probe-and-skip nightly.
|
||||||
|
conn.execute(
|
||||||
|
"""UPDATE files SET
|
||||||
|
status = 'already_efficient',
|
||||||
|
failure_reason = ?,
|
||||||
|
scanned_at = ?
|
||||||
|
WHERE path = ?""",
|
||||||
|
(reason, datetime.now().timestamp(), input_path)
|
||||||
|
)
|
||||||
|
conn.commit()
|
||||||
|
return {"status": "already_efficient", "reason": reason,
|
||||||
|
"original_size": original_size, "space_saved": 0}
|
||||||
|
|
||||||
|
crf = video_decision["crf"]
|
||||||
|
preset = video_decision["preset"]
|
||||||
|
|
||||||
|
# --- Decide audio: copy or re-encode ---
|
||||||
|
audio_decision = pick_audio_settings(info)
|
||||||
|
|
||||||
# Mark as transcoding in DB
|
# Mark as transcoding in DB
|
||||||
conn.execute("UPDATE files SET status = 'transcoding' WHERE path = ?", (input_path,))
|
conn.execute("UPDATE files SET status = 'transcoding' WHERE path = ?", (input_path,))
|
||||||
@@ -637,12 +894,13 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
|
|||||||
CURRENT_TRANSCODE["path"] = input_path
|
CURRENT_TRANSCODE["path"] = input_path
|
||||||
CURRENT_TRANSCODE["start_time"] = start_time
|
CURRENT_TRANSCODE["start_time"] = start_time
|
||||||
|
|
||||||
|
# Record the actual settings used, for later querying / re-encode decisions.
|
||||||
settings = {
|
settings = {
|
||||||
"crf": CONFIG["crf"],
|
"crf": crf,
|
||||||
"preset": CONFIG["preset"],
|
"preset": preset,
|
||||||
"audio_codec": CONFIG["audio_codec"],
|
"tier_height": video_decision.get("tier_height"),
|
||||||
"audio_bitrate": CONFIG["audio_bitrate"],
|
"threads": CONFIG["threads"],
|
||||||
"threads": CONFIG["threads"]
|
"audio_action": audio_decision["action"],
|
||||||
}
|
}
|
||||||
|
|
||||||
# Build ffmpeg command
|
# Build ffmpeg command
|
||||||
@@ -652,14 +910,23 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
|
|||||||
"-progress", "pipe:1", # Progress to stdout for parsing
|
"-progress", "pipe:1", # Progress to stdout for parsing
|
||||||
"-i", input_path,
|
"-i", input_path,
|
||||||
"-c:v", "libx265",
|
"-c:v", "libx265",
|
||||||
"-preset", CONFIG["preset"],
|
"-preset", preset,
|
||||||
"-crf", str(CONFIG["crf"]),
|
"-crf", str(crf),
|
||||||
"-x265-params", f"pools={CONFIG['threads']}",
|
"-x265-params", f"pools={CONFIG['threads']}",
|
||||||
"-c:a", CONFIG["audio_codec"],
|
|
||||||
"-b:a", CONFIG["audio_bitrate"],
|
|
||||||
"-c:s", "copy", # Copy subtitles
|
|
||||||
str(output_path)
|
|
||||||
]
|
]
|
||||||
|
# Audio: copy or re-encode
|
||||||
|
if audio_decision["action"] == "copy":
|
||||||
|
cmd += ["-c:a", "copy"]
|
||||||
|
log.info(f" Audio: copying source stream (no re-encode)")
|
||||||
|
else:
|
||||||
|
cmd += ["-c:a", audio_decision["codec"], "-b:a", audio_decision["bitrate"]]
|
||||||
|
settings["audio_codec"] = audio_decision["codec"]
|
||||||
|
settings["audio_bitrate"] = audio_decision["bitrate"]
|
||||||
|
log.info(f" Audio: re-encoding to {audio_decision['codec']} @ {audio_decision['bitrate']}")
|
||||||
|
# Subtitles: copy
|
||||||
|
cmd += ["-c:s", "copy", str(output_path)]
|
||||||
|
|
||||||
|
log.info(f" Encode: CRF {crf}, preset {preset} (tier {video_decision.get('tier_height')}p+)")
|
||||||
|
|
||||||
# Log the full command for debugging
|
# Log the full command for debugging
|
||||||
cmd_str = ' '.join(f'"{c}"' if ' ' in c else c for c in cmd)
|
cmd_str = ' '.join(f'"{c}"' if ' ' in c else c for c in cmd)
|
||||||
@@ -668,7 +935,7 @@ def transcode_file(input_path: str, conn: sqlite3.Connection) -> Optional[dict]:
|
|||||||
if not VERBOSE:
|
if not VERBOSE:
|
||||||
log.info(f" (use --verbose to see full command)")
|
log.info(f" (use --verbose to see full command)")
|
||||||
log.info("")
|
log.info("")
|
||||||
log_json("ffmpeg_command", command=cmd_str)
|
log_json("ffmpeg_command", command=cmd_str, settings=settings)
|
||||||
|
|
||||||
# Log stderr to file to prevent pipe buffer deadlock
|
# Log stderr to file to prevent pipe buffer deadlock
|
||||||
# Write to /var/lib/transcoder/ since susan may not be able to create files in /var/log/
|
# Write to /var/lib/transcoder/ since susan may not be able to create files in /var/log/
|
||||||
@@ -1033,6 +1300,7 @@ def main():
|
|||||||
print(f"Completed: {stats['done_count']}")
|
print(f"Completed: {stats['done_count']}")
|
||||||
print(f"Failed: {stats['failed_count']}")
|
print(f"Failed: {stats['failed_count']}")
|
||||||
print(f"Skipped (larger): {stats['skipped_larger_count']}")
|
print(f"Skipped (larger): {stats['skipped_larger_count']}")
|
||||||
|
print(f"Already efficient: {stats['already_efficient_count']}")
|
||||||
print("")
|
print("")
|
||||||
print("--- Lifetime Stats ---")
|
print("--- Lifetime Stats ---")
|
||||||
print(f"Total transcoded: {stats.get('total_transcoded', 0)} files")
|
print(f"Total transcoded: {stats.get('total_transcoded', 0)} files")
|
||||||
@@ -1068,12 +1336,12 @@ def main():
|
|||||||
if args.failed:
|
if args.failed:
|
||||||
cursor = conn.execute("""
|
cursor = conn.execute("""
|
||||||
SELECT path, original_size, failure_reason, status
|
SELECT path, original_size, failure_reason, status
|
||||||
FROM files WHERE status IN ('failed', 'skipped_larger')
|
FROM files WHERE status IN ('failed', 'skipped_larger', 'already_efficient')
|
||||||
ORDER BY original_size DESC
|
ORDER BY status, original_size DESC
|
||||||
""")
|
""")
|
||||||
rows = cursor.fetchall()
|
rows = cursor.fetchall()
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
print(f"Failed / Skipped Transcodes ({len(rows)} files)")
|
print(f"Failed / Skipped / Already-efficient ({len(rows)} files)")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
for path, size, reason, status in rows:
|
for path, size, reason, status in rows:
|
||||||
print(f"\n{path}")
|
print(f"\n{path}")
|
||||||
@@ -1081,19 +1349,35 @@ def main():
|
|||||||
print(f" Size: {size/(1024**3):.2f} GB")
|
print(f" Size: {size/(1024**3):.2f} GB")
|
||||||
print(f" Reason: {reason}")
|
print(f" Reason: {reason}")
|
||||||
if not rows:
|
if not rows:
|
||||||
print("No failed or skipped transcodes!")
|
print("Nothing failed, skipped, or already-efficient!")
|
||||||
print("=" * 60)
|
print("=" * 60)
|
||||||
conn.close()
|
conn.close()
|
||||||
return
|
return
|
||||||
|
|
||||||
# Handle --retry-failed flag
|
# Handle --retry-failed flag
|
||||||
# Only resets 'failed' — 'skipped_larger' is deterministic and intentionally left alone.
|
# Only resets 'failed' — 'skipped_larger' / 'already_efficient' are deterministic
|
||||||
|
# at current settings and intentionally left alone (see --reset-skipped).
|
||||||
if args.retry_failed:
|
if args.retry_failed:
|
||||||
cursor = conn.execute("UPDATE files SET status = 'pending', failure_reason = NULL WHERE status = 'failed'")
|
cursor = conn.execute("UPDATE files SET status = 'pending', failure_reason = NULL WHERE status = 'failed'")
|
||||||
count = cursor.rowcount
|
count = cursor.rowcount
|
||||||
conn.commit()
|
conn.commit()
|
||||||
print(f"Reset {count} failed files to pending.")
|
print(f"Reset {count} failed files to pending.")
|
||||||
print("(skipped_larger files are left as-is — re-encoding at the same settings would fail identically.)")
|
print("(skipped_larger / already_efficient left as-is — re-run at the same settings "
|
||||||
|
"would repeat. Use --reset-skipped after changing thresholds.)")
|
||||||
|
conn.close()
|
||||||
|
return
|
||||||
|
|
||||||
|
# Handle --reset-skipped flag
|
||||||
|
# For use after changing encode_tiers thresholds: re-queues files that were skipped
|
||||||
|
# as already-efficient or that produced a larger output, so they're re-evaluated.
|
||||||
|
if args.reset_skipped:
|
||||||
|
cursor = conn.execute(
|
||||||
|
"UPDATE files SET status = 'pending', failure_reason = NULL "
|
||||||
|
"WHERE status IN ('skipped_larger', 'already_efficient')"
|
||||||
|
)
|
||||||
|
count = cursor.rowcount
|
||||||
|
conn.commit()
|
||||||
|
print(f"Reset {count} skipped/already-efficient files to pending.")
|
||||||
conn.close()
|
conn.close()
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -1108,7 +1392,11 @@ def main():
|
|||||||
log.info(f" Verbose: {'YES' if VERBOSE else 'NO'}")
|
log.info(f" Verbose: {'YES' if VERBOSE else 'NO'}")
|
||||||
log.info(f" JSON log: {'YES' if JSON_LOG else 'NO'}")
|
log.info(f" JSON log: {'YES' if JSON_LOG else 'NO'}")
|
||||||
log.info(f" Cutoff: {CONFIG['cutoff_time']}")
|
log.info(f" Cutoff: {CONFIG['cutoff_time']}")
|
||||||
log.info(f" Settings: CRF={CONFIG['crf']}, preset={CONFIG['preset']}, audio={CONFIG['audio_codec']}@{CONFIG['audio_bitrate']}")
|
tier_summary = ", ".join(
|
||||||
|
f"{mh}p+→CRF{crf}/{preset}" for mh, crf, preset, _ in CONFIG["encode_tiers"]
|
||||||
|
)
|
||||||
|
log.info(f" Encode: dynamic per-file [{tier_summary}], preset from tier, threads={CONFIG['threads']}")
|
||||||
|
log.info(f" Audio: copy when efficient, else re-encode by channel count")
|
||||||
log.info("=" * 70)
|
log.info("=" * 70)
|
||||||
log.info("")
|
log.info("")
|
||||||
|
|
||||||
@@ -1146,28 +1434,54 @@ def main():
|
|||||||
if DRY_RUN:
|
if DRY_RUN:
|
||||||
log.info("")
|
log.info("")
|
||||||
log.info("=" * 60)
|
log.info("=" * 60)
|
||||||
log.info("DRY RUN - Files that would be transcoded (biggest first):")
|
log.info("DRY RUN - Per-file decision (biggest first, top 20 shown):")
|
||||||
log.info("=" * 60)
|
log.info("=" * 60)
|
||||||
|
|
||||||
total_size = 0
|
would_encode_size = 0
|
||||||
estimated_savings = 0
|
would_skip_count = 0
|
||||||
|
would_skip_size = 0
|
||||||
|
shown = 0
|
||||||
|
|
||||||
for i, video in enumerate(to_transcode[:20], 1): # Show top 20
|
for i, video in enumerate(to_transcode, 1):
|
||||||
|
info = probe_media(video["path"])
|
||||||
|
vdec = pick_video_settings(info)
|
||||||
|
adec = pick_audio_settings(info)
|
||||||
size_gb = video["size_gb"]
|
size_gb = video["size_gb"]
|
||||||
total_size += video["size"]
|
res_str = (f"{info['width']}x{info['height']}"
|
||||||
# Estimate ~50% compression for HEVC
|
if info.get("width") and info.get("height") else "?")
|
||||||
est_saved = video["size"] * 0.5
|
vbr = estimate_video_bitrate(info)
|
||||||
estimated_savings += est_saved
|
vbr_str = f"{vbr/1_000_000:.1f}Mbps" if vbr else "?"
|
||||||
|
|
||||||
log.info(f"{i:2}. [{size_gb:6.2f} GB] {video['path']}")
|
if vdec["action"] == "skip":
|
||||||
|
would_skip_count += 1
|
||||||
|
would_skip_size += video["size"]
|
||||||
|
if shown < 20:
|
||||||
|
shown += 1
|
||||||
|
log.info(f"{i:2}. [{size_gb:6.2f} GB] {res_str:>9} {vbr_str:>8} SKIP — "
|
||||||
|
f"{Path(video['path']).name}")
|
||||||
|
else:
|
||||||
|
would_encode_size += video["size"]
|
||||||
|
audio_str = ("audio:copy" if adec["action"] == "copy"
|
||||||
|
else f"audio:{adec['codec']}@{adec['bitrate']}")
|
||||||
|
if shown < 20:
|
||||||
|
shown += 1
|
||||||
|
log.info(f"{i:2}. [{size_gb:6.2f} GB] {res_str:>9} {vbr_str:>8} "
|
||||||
|
f"CRF{vdec['crf']}/{vdec['preset']} {audio_str} — "
|
||||||
|
f"{Path(video['path']).name}")
|
||||||
|
|
||||||
if len(to_transcode) > 20:
|
if len(to_transcode) > shown:
|
||||||
log.info(f" ... and {len(to_transcode) - 20} more files")
|
log.info(f" ... and {len(to_transcode) - shown} more (probed for totals below)")
|
||||||
|
|
||||||
|
# Rough savings estimate only for files we'd actually encode (~40% on fat sources).
|
||||||
|
estimated_savings = would_encode_size * 0.40
|
||||||
|
|
||||||
log.info("")
|
log.info("")
|
||||||
log.info("=" * 60)
|
log.info("=" * 60)
|
||||||
log.info(f"Total to transcode: {total_size / (1024**3):.2f} GB")
|
log.info(f"Would encode: {len(to_transcode) - would_skip_count} files "
|
||||||
log.info(f"Estimated savings (~50%): {estimated_savings / (1024**3):.2f} GB")
|
f"({would_encode_size / (1024**3):.2f} GB)")
|
||||||
|
log.info(f"Would skip: {would_skip_count} files "
|
||||||
|
f"({would_skip_size / (1024**3):.2f} GB) — already efficient")
|
||||||
|
log.info(f"Est. savings: ~{estimated_savings / (1024**3):.2f} GB (rough, ~40% of encoded)")
|
||||||
log.info("=" * 60)
|
log.info("=" * 60)
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -1238,7 +1552,11 @@ def main():
|
|||||||
# Transcode
|
# Transcode
|
||||||
result = transcode_file(video["path"], conn)
|
result = transcode_file(video["path"], conn)
|
||||||
|
|
||||||
if result and result.get("status") == "skipped_larger":
|
if result and result.get("status") == "already_efficient":
|
||||||
|
skipped_count += 1
|
||||||
|
log.info(f" Skipped (already efficient): {Path(video['path']).name}")
|
||||||
|
log.info(f" Running totals: {transcoded_count} done, {skipped_count} skipped, {failed_count} failed")
|
||||||
|
elif result and result.get("status") == "skipped_larger":
|
||||||
skipped_count += 1
|
skipped_count += 1
|
||||||
log.info(f" Skipped (HEVC larger than source): {Path(video['path']).name}")
|
log.info(f" Skipped (HEVC larger than source): {Path(video['path']).name}")
|
||||||
log.info(f" Running totals: {transcoded_count} done, {skipped_count} skipped, {failed_count} failed")
|
log.info(f" Running totals: {transcoded_count} done, {skipped_count} skipped, {failed_count} failed")
|
||||||
|
|||||||
Reference in New Issue
Block a user