Caine
c7956ae9b2
Overnight transcoding, music discovery/import, system health reports, stats page generator, and bookmark management. Secrets stored in /etc/automation/ — not in repo.
439 lines
15 KiB
Python
439 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Susan Morning Report
|
|
Generates a system health report and emails it to Tom.
|
|
Run via cron at 06:45 (after transcoder finishes).
|
|
"""
|
|
|
|
import subprocess
|
|
import json
|
|
import os
|
|
import re
|
|
import smtplib
|
|
from email.mime.text import MIMEText
|
|
from email.mime.multipart import MIMEMultipart
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
|
|
# Config
|
|
EMAIL_CONFIG = "/etc/susan/email.json"
|
|
TRANSCODER_LOG = "/var/log/transcoder.log"
|
|
TRANSCODER_DB = "/var/lib/transcoder/cache.db"
|
|
TO_EMAIL = "tom@tomflux.xyz"
|
|
|
|
|
|
def run_cmd(cmd: str, timeout: int = 30) -> str:
|
|
"""Run a shell command and return output."""
|
|
try:
|
|
result = subprocess.run(
|
|
cmd, shell=True, capture_output=True, text=True, timeout=timeout
|
|
)
|
|
return result.stdout.strip()
|
|
except Exception as e:
|
|
return f"Error: {e}"
|
|
|
|
|
|
def get_uptime() -> dict:
|
|
"""Get system uptime info."""
|
|
uptime_raw = run_cmd("uptime -p")
|
|
load = run_cmd("cat /proc/loadavg").split()[:3]
|
|
boot_time = run_cmd("uptime -s")
|
|
|
|
return {
|
|
"uptime": uptime_raw.replace("up ", ""),
|
|
"since": boot_time,
|
|
"load_avg": f"{load[0]} / {load[1]} / {load[2]}" if len(load) >= 3 else "unknown"
|
|
}
|
|
|
|
|
|
def get_disk_space() -> list:
|
|
"""Get disk usage for important mounts."""
|
|
disks = []
|
|
# Only show root and /disks* mounts, exclude virtual filesystems
|
|
df_output = run_cmd("df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs | tail -n +2")
|
|
|
|
for line in df_output.split('\n'):
|
|
if line.strip():
|
|
parts = line.split()
|
|
if len(parts) >= 5:
|
|
mount = parts[0]
|
|
# Only include / and /disks*, /home, /var
|
|
if mount in ['/', '/home', '/var'] or mount.startswith('/disks'):
|
|
disks.append({
|
|
"mount": mount,
|
|
"size": parts[1],
|
|
"used": parts[2],
|
|
"avail": parts[3],
|
|
"percent": parts[4]
|
|
})
|
|
return disks
|
|
|
|
|
|
def get_raid_status() -> dict:
|
|
"""Get 3ware RAID status using tw_cli."""
|
|
status = {"available": False, "details": None, "drives": [], "array": None}
|
|
|
|
# Try tw_cli first
|
|
tw_output = run_cmd("sudo tw_cli /c0 show 2>/dev/null")
|
|
if "Error" not in tw_output and tw_output and "Unit" in tw_output:
|
|
status["available"] = True
|
|
status["details"] = tw_output
|
|
|
|
# Parse array status
|
|
for line in tw_output.split('\n'):
|
|
if line.startswith('u0'):
|
|
parts = line.split()
|
|
if len(parts) >= 3:
|
|
status["array"] = {
|
|
"unit": parts[0],
|
|
"type": parts[1],
|
|
"status": parts[2],
|
|
"verify": parts[4] if len(parts) > 4 and '%' in parts[4] else None
|
|
}
|
|
# Parse drive lines (p4, p5, p6, etc)
|
|
if line.startswith('p'):
|
|
parts = line.split()
|
|
if len(parts) >= 3:
|
|
status["drives"].append({
|
|
"port": parts[0],
|
|
"status": parts[1],
|
|
"size": parts[3] if len(parts) > 3 else "unknown"
|
|
})
|
|
else:
|
|
# Fallback to checking for mdadm
|
|
md_output = run_cmd("cat /proc/mdstat 2>/dev/null")
|
|
if md_output and "Error" not in md_output and "md" in md_output:
|
|
status["available"] = True
|
|
status["details"] = md_output
|
|
status["type"] = "mdadm"
|
|
|
|
return status
|
|
|
|
|
|
def get_smart_status() -> list:
|
|
"""Get SMART status for drives. Note: requires sudo for smartctl."""
|
|
drives = []
|
|
|
|
# Find physical block devices (skip loop, ram, etc)
|
|
lsblk = run_cmd("lsblk -d -o NAME,SIZE,TYPE | grep disk | grep -v loop")
|
|
for line in lsblk.split('\n'):
|
|
if line.strip():
|
|
parts = line.split()
|
|
if parts and not parts[0].startswith('loop'):
|
|
dev = f"/dev/{parts[0]}"
|
|
size = parts[1] if len(parts) > 1 else "unknown"
|
|
|
|
# Try smartctl with sudo
|
|
smart = run_cmd(f"sudo smartctl -H {dev} 2>/dev/null | grep -iE 'overall-health|result|PASSED|FAILED'")
|
|
reallocated = run_cmd(f"sudo smartctl -A {dev} 2>/dev/null | grep -i 'Reallocated_Sector'")
|
|
|
|
# Determine health status
|
|
if "PASSED" in smart:
|
|
health = "PASSED"
|
|
elif "FAILED" in smart:
|
|
health = "FAILED"
|
|
elif "sudo:" in smart or not smart:
|
|
health = "needs sudo"
|
|
elif "Unable to detect" in run_cmd(f"sudo smartctl -i {dev} 2>&1"):
|
|
health = "RAID array (skip)"
|
|
else:
|
|
health = "unknown"
|
|
|
|
# Skip RAID virtual devices (they show as large and can't be queried)
|
|
if health == "RAID array (skip)":
|
|
continue
|
|
|
|
# Also skip if smartctl says it's not a physical device
|
|
if "unknown" in health and "T" in size: # Multi-TB device with unknown = likely RAID
|
|
continue
|
|
|
|
drive_info = {
|
|
"device": dev,
|
|
"size": size,
|
|
"health": health,
|
|
}
|
|
|
|
if reallocated:
|
|
# Extract reallocated sector count (last number on line)
|
|
match = re.search(r'(\d+)\s*$', reallocated.strip())
|
|
if match:
|
|
drive_info["reallocated_sectors"] = int(match.group(1))
|
|
|
|
drives.append(drive_info)
|
|
|
|
return drives
|
|
|
|
|
|
def get_memory() -> dict:
|
|
"""Get memory usage."""
|
|
mem_output = run_cmd("free -h | grep Mem")
|
|
parts = mem_output.split()
|
|
|
|
if len(parts) >= 4:
|
|
return {
|
|
"total": parts[1],
|
|
"used": parts[2],
|
|
"available": parts[6] if len(parts) > 6 else parts[3]
|
|
}
|
|
return {"total": "unknown", "used": "unknown", "available": "unknown"}
|
|
|
|
|
|
def get_cpu_info() -> dict:
|
|
"""Get CPU info and temperature."""
|
|
info = {}
|
|
|
|
# CPU model
|
|
model = run_cmd("grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2")
|
|
info["model"] = model.strip() if model else "unknown"
|
|
|
|
# Temperatures
|
|
temps = run_cmd("sensors 2>/dev/null | grep -E 'Core|temp' | head -5")
|
|
info["temps"] = temps if temps else "sensors not available"
|
|
|
|
return info
|
|
|
|
|
|
def get_transcoder_status() -> dict:
|
|
"""Get last night's transcoder results + queue stats from DB."""
|
|
import sqlite3
|
|
|
|
status = {"ran": False, "summary": None, "queue": None}
|
|
|
|
# Get queue stats from database
|
|
db_path = Path(TRANSCODER_DB)
|
|
if db_path.exists():
|
|
try:
|
|
conn = sqlite3.connect(TRANSCODER_DB)
|
|
|
|
# Pending files
|
|
cursor = conn.execute("""
|
|
SELECT COUNT(*), SUM(original_size)
|
|
FROM files WHERE is_hevc = 0 AND status = 'pending'
|
|
""")
|
|
row = cursor.fetchone()
|
|
pending_count = row[0] or 0
|
|
pending_size = row[1] or 0
|
|
|
|
# Already HEVC
|
|
cursor = conn.execute("SELECT COUNT(*) FROM files WHERE is_hevc = 1")
|
|
hevc_count = cursor.fetchone()[0] or 0
|
|
|
|
# Lifetime stats
|
|
cursor = conn.execute("SELECT total_files_transcoded, total_space_saved FROM stats WHERE id = 1")
|
|
row = cursor.fetchone()
|
|
lifetime_transcoded = row[0] if row else 0
|
|
lifetime_saved = row[1] if row else 0
|
|
|
|
# Failed count
|
|
cursor = conn.execute("SELECT COUNT(*) FROM files WHERE status = 'failed'")
|
|
failed_count = cursor.fetchone()[0] or 0
|
|
|
|
conn.close()
|
|
|
|
status["queue"] = {
|
|
"pending_count": pending_count,
|
|
"pending_size": pending_size,
|
|
"pending_size_human": f"{pending_size / (1024**3):.1f} GB" if pending_size else "0 GB",
|
|
"hevc_count": hevc_count,
|
|
"failed_count": failed_count,
|
|
"lifetime_transcoded": lifetime_transcoded,
|
|
"lifetime_saved": lifetime_saved,
|
|
"lifetime_saved_human": f"{lifetime_saved / (1024**3):.1f} GB" if lifetime_saved else "0 GB"
|
|
}
|
|
except Exception as e:
|
|
status["queue"] = {"error": str(e)}
|
|
|
|
# Check log for last night's run
|
|
log_path = Path(TRANSCODER_LOG)
|
|
if log_path.exists():
|
|
try:
|
|
log_content = log_path.read_text()
|
|
except PermissionError:
|
|
log_content = run_cmd(f"sudo cat {TRANSCODER_LOG} 2>/dev/null")
|
|
if "Error" in log_content:
|
|
log_content = ""
|
|
|
|
if log_content:
|
|
# Find the last SESSION COMPLETE block
|
|
sessions = re.findall(
|
|
r'SESSION COMPLETE.*?Transcoded:\s+(\d+).*?Failed:\s+(\d+).*?Space saved:\s+([\d.]+\s+\w+)',
|
|
log_content, re.DOTALL
|
|
)
|
|
|
|
if sessions:
|
|
last = sessions[-1]
|
|
status["ran"] = True
|
|
status["transcoded"] = int(last[0])
|
|
status["failed"] = int(last[1])
|
|
status["space_saved"] = last[2]
|
|
status["summary"] = f"{last[0]} transcoded, {last[1]} failed, {last[2]} saved"
|
|
else:
|
|
today = datetime.now().strftime("%Y-%m-%d")
|
|
yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
|
|
if today in log_content or yesterday in log_content:
|
|
status["ran"] = True
|
|
status["summary"] = "Ran (check log for details)"
|
|
|
|
return status
|
|
|
|
|
|
def get_failed_services() -> list:
|
|
"""Get any failed systemd services."""
|
|
output = run_cmd("systemctl --failed --no-pager --plain | grep -E '\.service|\.socket|\.mount' | awk '{print $1}'")
|
|
failed = []
|
|
for line in output.split('\n'):
|
|
name = line.strip()
|
|
if name and not name.startswith('UNIT') and '●' not in name:
|
|
failed.append(name)
|
|
return failed
|
|
|
|
|
|
def generate_report() -> str:
|
|
"""Generate the full morning report."""
|
|
now = datetime.now()
|
|
|
|
report = []
|
|
report.append("=" * 50)
|
|
report.append(f"🖥️ SUSAN MORNING REPORT")
|
|
report.append(f"📅 {now.strftime('%A, %B %d %Y at %H:%M')}")
|
|
report.append("=" * 50)
|
|
report.append("")
|
|
|
|
# Uptime
|
|
uptime = get_uptime()
|
|
report.append("⏱️ UPTIME")
|
|
report.append(f" Up: {uptime['uptime']}")
|
|
report.append(f" Since: {uptime['since']}")
|
|
report.append(f" Load: {uptime['load_avg']}")
|
|
report.append("")
|
|
|
|
# Memory
|
|
mem = get_memory()
|
|
report.append("🧠 MEMORY")
|
|
report.append(f" Used: {mem['used']} / {mem['total']}")
|
|
report.append(f" Available: {mem['available']}")
|
|
report.append("")
|
|
|
|
# Disk Space
|
|
disks = get_disk_space()
|
|
report.append("💾 DISK SPACE")
|
|
for disk in disks:
|
|
warn = " ⚠️" if disk['percent'].replace('%', '').isdigit() and int(disk['percent'].replace('%', '')) > 85 else ""
|
|
report.append(f" {disk['mount']}: {disk['used']}/{disk['size']} ({disk['percent']} used){warn}")
|
|
report.append("")
|
|
|
|
# RAID Status
|
|
raid = get_raid_status()
|
|
report.append("🔒 RAID STATUS")
|
|
if raid["available"]:
|
|
if raid.get("array"):
|
|
arr = raid["array"]
|
|
status_icon = "✅" if arr["status"] == "OK" else ("⚠️" if "VERIFY" in arr["status"] else "❌")
|
|
report.append(f" Array: {arr['type']} {status_icon} {arr['status']}")
|
|
if arr.get("verify"):
|
|
report.append(f" Verify progress: {arr['verify']}")
|
|
|
|
if raid.get("drives"):
|
|
report.append(f" Drives: {len(raid['drives'])} disks")
|
|
for drive in raid["drives"]:
|
|
d_icon = "✅" if drive["status"] == "OK" else "❌"
|
|
report.append(f" {drive['port']}: {d_icon} {drive['status']} ({drive['size']} TB)")
|
|
else:
|
|
report.append(" tw_cli not available - install 3ware tools for RAID monitoring")
|
|
report.append("")
|
|
|
|
# Drive Health (SMART)
|
|
drives = get_smart_status()
|
|
if drives:
|
|
report.append("🔧 DRIVE HEALTH (SMART)")
|
|
for drive in drives:
|
|
warn = ""
|
|
if drive.get("reallocated_sectors", 0) > 0:
|
|
warn = f" ⚠️ {drive['reallocated_sectors']} reallocated sectors!"
|
|
health_icon = "✅" if drive["health"] == "PASSED" else "❌"
|
|
report.append(f" {drive['device']} ({drive['size']}): {health_icon} {drive['health']}{warn}")
|
|
report.append("")
|
|
|
|
# Transcoder
|
|
transcoder = get_transcoder_status()
|
|
report.append("🎬 TRANSCODER")
|
|
|
|
# Last night's run
|
|
if transcoder.get("ran"):
|
|
report.append(f" Last run: ✅ {transcoder.get('summary', 'Completed')}")
|
|
elif transcoder.get("summary"):
|
|
report.append(f" Last run: ⏸️ {transcoder['summary']}")
|
|
else:
|
|
report.append(f" Last run: No data")
|
|
|
|
# Queue stats from database
|
|
if transcoder.get("queue") and not transcoder["queue"].get("error"):
|
|
q = transcoder["queue"]
|
|
report.append(f" Queue: {q['pending_count']} files ({q['pending_size_human']}) waiting")
|
|
if q['failed_count'] > 0:
|
|
report.append(f" ⚠️ Failed: {q['failed_count']} files need attention")
|
|
report.append(f" Library: {q['hevc_count']} files already HEVC")
|
|
if q['lifetime_transcoded'] > 0:
|
|
report.append(f" Lifetime: {q['lifetime_transcoded']} transcoded, {q['lifetime_saved_human']} saved")
|
|
elif transcoder.get("queue", {}).get("error"):
|
|
report.append(f" DB error: {transcoder['queue']['error']}")
|
|
|
|
report.append("")
|
|
|
|
# Failed Services
|
|
failed = get_failed_services()
|
|
report.append("🚨 SYSTEMD SERVICES")
|
|
if failed:
|
|
report.append(f" ❌ {len(failed)} failed: {', '.join(failed)}")
|
|
else:
|
|
report.append(" ✅ All services OK")
|
|
report.append("")
|
|
|
|
report.append("=" * 50)
|
|
report.append("End of report")
|
|
report.append("=" * 50)
|
|
|
|
return "\n".join(report)
|
|
|
|
|
|
def send_email(subject: str, body: str):
|
|
"""Send the report via email."""
|
|
with open(EMAIL_CONFIG) as f:
|
|
cfg = json.load(f)
|
|
|
|
smtp = cfg['smtp']
|
|
|
|
msg = MIMEMultipart()
|
|
msg['Subject'] = subject
|
|
msg['From'] = smtp['from']
|
|
msg['To'] = TO_EMAIL
|
|
|
|
# Plain text version
|
|
msg.attach(MIMEText(body, 'plain'))
|
|
|
|
server = smtplib.SMTP_SSL(smtp['server'], smtp['port'])
|
|
server.login(smtp['username'], smtp['password'])
|
|
server.sendmail(smtp['from'], [TO_EMAIL], msg.as_string())
|
|
server.quit()
|
|
|
|
|
|
def main():
|
|
report = generate_report()
|
|
|
|
# Print to stdout (for logging)
|
|
print(report)
|
|
|
|
# Email it
|
|
today = datetime.now().strftime("%Y-%m-%d")
|
|
subject = f"🖥️ Susan Morning Report - {today}"
|
|
|
|
try:
|
|
send_email(subject, report)
|
|
print("\n✅ Report emailed successfully")
|
|
except Exception as e:
|
|
print(f"\n❌ Failed to send email: {e}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|