Initial commit: susan automation scriptsHEADmaster

Overnight transcoding, music discovery/import, system health reports,
stats page generator, and bookmark management.

Secrets stored in /etc/automation/ — not in repo.

1 files changed, 438 insertions, 0 deletions

diff --git a/morning_report.py b/morning_report.py
new file mode 100644
index 0000000..a1e8103
--- /dev/null
+++ b/morning_report.py
@@ -0,0 +1,438 @@
+#!/usr/bin/env python3
+"""
+Susan Morning Report
+Generates a system health report and emails it to Tom.
+Run via cron at 06:45 (after transcoder finishes).
+"""
+
+import subprocess
+import json
+import os
+import re
+import smtplib
+from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from datetime import datetime, timedelta
+from pathlib import Path
+
+# Config
+EMAIL_CONFIG = "/etc/susan/email.json"
+TRANSCODER_LOG = "/var/log/transcoder.log"
+TRANSCODER_DB = "/var/lib/transcoder/cache.db"
+TO_EMAIL = "tom@tomflux.xyz"
+
+
+def run_cmd(cmd: str, timeout: int = 30) -> str:
+    """Run a shell command and return output."""
+    try:
+        result = subprocess.run(
+            cmd, shell=True, capture_output=True, text=True, timeout=timeout
+        )
+        return result.stdout.strip()
+    except Exception as e:
+        return f"Error: {e}"
+
+
+def get_uptime() -> dict:
+    """Get system uptime info."""
+    uptime_raw = run_cmd("uptime -p")
+    load = run_cmd("cat /proc/loadavg").split()[:3]
+    boot_time = run_cmd("uptime -s")
+    
+    return {
+        "uptime": uptime_raw.replace("up ", ""),
+        "since": boot_time,
+        "load_avg": f"{load[0]} / {load[1]} / {load[2]}" if len(load) >= 3 else "unknown"
+    }
+
+
+def get_disk_space() -> list:
+    """Get disk usage for important mounts."""
+    disks = []
+    # Only show root and /disks* mounts, exclude virtual filesystems
+    df_output = run_cmd("df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs | tail -n +2")
+    
+    for line in df_output.split('\n'):
+        if line.strip():
+            parts = line.split()
+            if len(parts) >= 5:
+                mount = parts[0]
+                # Only include / and /disks*, /home, /var
+                if mount in ['/', '/home', '/var'] or mount.startswith('/disks'):
+                    disks.append({
+                        "mount": mount,
+                        "size": parts[1],
+                        "used": parts[2],
+                        "avail": parts[3],
+                        "percent": parts[4]
+                    })
+    return disks
+
+
+def get_raid_status() -> dict:
+    """Get 3ware RAID status using tw_cli."""
+    status = {"available": False, "details": None, "drives": [], "array": None}
+    
+    # Try tw_cli first
+    tw_output = run_cmd("sudo tw_cli /c0 show 2>/dev/null")
+    if "Error" not in tw_output and tw_output and "Unit" in tw_output:
+        status["available"] = True
+        status["details"] = tw_output
+        
+        # Parse array status
+        for line in tw_output.split('\n'):
+            if line.startswith('u0'):
+                parts = line.split()
+                if len(parts) >= 3:
+                    status["array"] = {
+                        "unit": parts[0],
+                        "type": parts[1],
+                        "status": parts[2],
+                        "verify": parts[4] if len(parts) > 4 and '%' in parts[4] else None
+                    }
+            # Parse drive lines (p4, p5, p6, etc)
+            if line.startswith('p'):
+                parts = line.split()
+                if len(parts) >= 3:
+                    status["drives"].append({
+                        "port": parts[0],
+                        "status": parts[1],
+                        "size": parts[3] if len(parts) > 3 else "unknown"
+                    })
+    else:
+        # Fallback to checking for mdadm
+        md_output = run_cmd("cat /proc/mdstat 2>/dev/null")
+        if md_output and "Error" not in md_output and "md" in md_output:
+            status["available"] = True
+            status["details"] = md_output
+            status["type"] = "mdadm"
+    
+    return status
+
+
+def get_smart_status() -> list:
+    """Get SMART status for drives. Note: requires sudo for smartctl."""
+    drives = []
+    
+    # Find physical block devices (skip loop, ram, etc)
+    lsblk = run_cmd("lsblk -d -o NAME,SIZE,TYPE | grep disk | grep -v loop")
+    for line in lsblk.split('\n'):
+        if line.strip():
+            parts = line.split()
+            if parts and not parts[0].startswith('loop'):
+                dev = f"/dev/{parts[0]}"
+                size = parts[1] if len(parts) > 1 else "unknown"
+                
+                # Try smartctl with sudo
+                smart = run_cmd(f"sudo smartctl -H {dev} 2>/dev/null | grep -iE 'overall-health|result|PASSED|FAILED'")
+                reallocated = run_cmd(f"sudo smartctl -A {dev} 2>/dev/null | grep -i 'Reallocated_Sector'")
+                
+                # Determine health status
+                if "PASSED" in smart:
+                    health = "PASSED"
+                elif "FAILED" in smart:
+                    health = "FAILED"
+                elif "sudo:" in smart or not smart:
+                    health = "needs sudo"
+                elif "Unable to detect" in run_cmd(f"sudo smartctl -i {dev} 2>&1"):
+                    health = "RAID array (skip)"
+                else:
+                    health = "unknown"
+                
+                # Skip RAID virtual devices (they show as large and can't be queried)
+                if health == "RAID array (skip)":
+                    continue
+                    
+                # Also skip if smartctl says it's not a physical device
+                if "unknown" in health and "T" in size:  # Multi-TB device with unknown = likely RAID
+                    continue
+                
+                drive_info = {
+                    "device": dev,
+                    "size": size,
+                    "health": health,
+                }
+                
+                if reallocated:
+                    # Extract reallocated sector count (last number on line)
+                    match = re.search(r'(\d+)\s*$', reallocated.strip())
+                    if match:
+                        drive_info["reallocated_sectors"] = int(match.group(1))
+                
+                drives.append(drive_info)
+    
+    return drives
+
+
+def get_memory() -> dict:
+    """Get memory usage."""
+    mem_output = run_cmd("free -h | grep Mem")
+    parts = mem_output.split()
+    
+    if len(parts) >= 4:
+        return {
+            "total": parts[1],
+            "used": parts[2],
+            "available": parts[6] if len(parts) > 6 else parts[3]
+        }
+    return {"total": "unknown", "used": "unknown", "available": "unknown"}
+
+
+def get_cpu_info() -> dict:
+    """Get CPU info and temperature."""
+    info = {}
+    
+    # CPU model
+    model = run_cmd("grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2")
+    info["model"] = model.strip() if model else "unknown"
+    
+    # Temperatures
+    temps = run_cmd("sensors 2>/dev/null | grep -E 'Core|temp' | head -5")
+    info["temps"] = temps if temps else "sensors not available"
+    
+    return info
+
+
+def get_transcoder_status() -> dict:
+    """Get last night's transcoder results + queue stats from DB."""
+    import sqlite3
+    
+    status = {"ran": False, "summary": None, "queue": None}
+    
+    # Get queue stats from database
+    db_path = Path(TRANSCODER_DB)
+    if db_path.exists():
+        try:
+            conn = sqlite3.connect(TRANSCODER_DB)
+            
+            # Pending files
+            cursor = conn.execute("""
+                SELECT COUNT(*), SUM(original_size) 
+                FROM files WHERE is_hevc = 0 AND status = 'pending'
+            """)
+            row = cursor.fetchone()
+            pending_count = row[0] or 0
+            pending_size = row[1] or 0
+            
+            # Already HEVC
+            cursor = conn.execute("SELECT COUNT(*) FROM files WHERE is_hevc = 1")
+            hevc_count = cursor.fetchone()[0] or 0
+            
+            # Lifetime stats
+            cursor = conn.execute("SELECT total_files_transcoded, total_space_saved FROM stats WHERE id = 1")
+            row = cursor.fetchone()
+            lifetime_transcoded = row[0] if row else 0
+            lifetime_saved = row[1] if row else 0
+            
+            # Failed count
+            cursor = conn.execute("SELECT COUNT(*) FROM files WHERE status = 'failed'")
+            failed_count = cursor.fetchone()[0] or 0
+            
+            conn.close()
+            
+            status["queue"] = {
+                "pending_count": pending_count,
+                "pending_size": pending_size,
+                "pending_size_human": f"{pending_size / (1024**3):.1f} GB" if pending_size else "0 GB",
+                "hevc_count": hevc_count,
+                "failed_count": failed_count,
+                "lifetime_transcoded": lifetime_transcoded,
+                "lifetime_saved": lifetime_saved,
+                "lifetime_saved_human": f"{lifetime_saved / (1024**3):.1f} GB" if lifetime_saved else "0 GB"
+            }
+        except Exception as e:
+            status["queue"] = {"error": str(e)}
+    
+    # Check log for last night's run
+    log_path = Path(TRANSCODER_LOG)
+    if log_path.exists():
+        try:
+            log_content = log_path.read_text()
+        except PermissionError:
+            log_content = run_cmd(f"sudo cat {TRANSCODER_LOG} 2>/dev/null")
+            if "Error" in log_content:
+                log_content = ""
+        
+        if log_content:
+            # Find the last SESSION COMPLETE block
+            sessions = re.findall(
+                r'SESSION COMPLETE.*?Transcoded:\s+(\d+).*?Failed:\s+(\d+).*?Space saved:\s+([\d.]+\s+\w+)',
+                log_content, re.DOTALL
+            )
+            
+            if sessions:
+                last = sessions[-1]
+                status["ran"] = True
+                status["transcoded"] = int(last[0])
+                status["failed"] = int(last[1])
+                status["space_saved"] = last[2]
+                status["summary"] = f"{last[0]} transcoded, {last[1]} failed, {last[2]} saved"
+            else:
+                today = datetime.now().strftime("%Y-%m-%d")
+                yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
+                if today in log_content or yesterday in log_content:
+                    status["ran"] = True
+                    status["summary"] = "Ran (check log for details)"
+    
+    return status
+
+
+def get_failed_services() -> list:
+    """Get any failed systemd services."""
+    output = run_cmd("systemctl --failed --no-pager --plain | grep -E '\.service|\.socket|\.mount' | awk '{print $1}'")
+    failed = []
+    for line in output.split('\n'):
+        name = line.strip()
+        if name and not name.startswith('UNIT') and '●' not in name:
+            failed.append(name)
+    return failed
+
+
+def generate_report() -> str:
+    """Generate the full morning report."""
+    now = datetime.now()
+    
+    report = []
+    report.append("=" * 50)
+    report.append(f"🖥️  SUSAN MORNING REPORT")
+    report.append(f"📅 {now.strftime('%A, %B %d %Y at %H:%M')}")
+    report.append("=" * 50)
+    report.append("")
+    
+    # Uptime
+    uptime = get_uptime()
+    report.append("⏱️  UPTIME")
+    report.append(f"   Up: {uptime['uptime']}")
+    report.append(f"   Since: {uptime['since']}")
+    report.append(f"   Load: {uptime['load_avg']}")
+    report.append("")
+    
+    # Memory
+    mem = get_memory()
+    report.append("🧠 MEMORY")
+    report.append(f"   Used: {mem['used']} / {mem['total']}")
+    report.append(f"   Available: {mem['available']}")
+    report.append("")
+    
+    # Disk Space
+    disks = get_disk_space()
+    report.append("💾 DISK SPACE")
+    for disk in disks:
+        warn = " ⚠️" if disk['percent'].replace('%', '').isdigit() and int(disk['percent'].replace('%', '')) > 85 else ""
+        report.append(f"   {disk['mount']}: {disk['used']}/{disk['size']} ({disk['percent']} used){warn}")
+    report.append("")
+    
+    # RAID Status
+    raid = get_raid_status()
+    report.append("🔒 RAID STATUS")
+    if raid["available"]:
+        if raid.get("array"):
+            arr = raid["array"]
+            status_icon = "✅" if arr["status"] == "OK" else ("⚠️" if "VERIFY" in arr["status"] else "❌")
+            report.append(f"   Array: {arr['type']} {status_icon} {arr['status']}")
+            if arr.get("verify"):
+                report.append(f"   Verify progress: {arr['verify']}")
+        
+        if raid.get("drives"):
+            report.append(f"   Drives: {len(raid['drives'])} disks")
+            for drive in raid["drives"]:
+                d_icon = "✅" if drive["status"] == "OK" else "❌"
+                report.append(f"     {drive['port']}: {d_icon} {drive['status']} ({drive['size']} TB)")
+    else:
+        report.append("   tw_cli not available - install 3ware tools for RAID monitoring")
+    report.append("")
+    
+    # Drive Health (SMART)
+    drives = get_smart_status()
+    if drives:
+        report.append("🔧 DRIVE HEALTH (SMART)")
+        for drive in drives:
+            warn = ""
+            if drive.get("reallocated_sectors", 0) > 0:
+                warn = f" ⚠️ {drive['reallocated_sectors']} reallocated sectors!"
+            health_icon = "✅" if drive["health"] == "PASSED" else "❌"
+            report.append(f"   {drive['device']} ({drive['size']}): {health_icon} {drive['health']}{warn}")
+        report.append("")
+    
+    # Transcoder
+    transcoder = get_transcoder_status()
+    report.append("🎬 TRANSCODER")
+    
+    # Last night's run
+    if transcoder.get("ran"):
+        report.append(f"   Last run: ✅ {transcoder.get('summary', 'Completed')}")
+    elif transcoder.get("summary"):
+        report.append(f"   Last run: ⏸️ {transcoder['summary']}")
+    else:
+        report.append(f"   Last run: No data")
+    
+    # Queue stats from database
+    if transcoder.get("queue") and not transcoder["queue"].get("error"):
+        q = transcoder["queue"]
+        report.append(f"   Queue: {q['pending_count']} files ({q['pending_size_human']}) waiting")
+        if q['failed_count'] > 0:
+            report.append(f"   ⚠️ Failed: {q['failed_count']} files need attention")
+        report.append(f"   Library: {q['hevc_count']} files already HEVC")
+        if q['lifetime_transcoded'] > 0:
+            report.append(f"   Lifetime: {q['lifetime_transcoded']} transcoded, {q['lifetime_saved_human']} saved")
+    elif transcoder.get("queue", {}).get("error"):
+        report.append(f"   DB error: {transcoder['queue']['error']}")
+    
+    report.append("")
+    
+    # Failed Services
+    failed = get_failed_services()
+    report.append("🚨 SYSTEMD SERVICES")
+    if failed:
+        report.append(f"   ❌ {len(failed)} failed: {', '.join(failed)}")
+    else:
+        report.append("   ✅ All services OK")
+    report.append("")
+    
+    report.append("=" * 50)
+    report.append("End of report")
+    report.append("=" * 50)
+    
+    return "\n".join(report)
+
+
+def send_email(subject: str, body: str):
+    """Send the report via email."""
+    with open(EMAIL_CONFIG) as f:
+        cfg = json.load(f)
+    
+    smtp = cfg['smtp']
+    
+    msg = MIMEMultipart()
+    msg['Subject'] = subject
+    msg['From'] = smtp['from']
+    msg['To'] = TO_EMAIL
+    
+    # Plain text version
+    msg.attach(MIMEText(body, 'plain'))
+    
+    server = smtplib.SMTP_SSL(smtp['server'], smtp['port'])
+    server.login(smtp['username'], smtp['password'])
+    server.sendmail(smtp['from'], [TO_EMAIL], msg.as_string())
+    server.quit()
+
+
+def main():
+    report = generate_report()
+    
+    # Print to stdout (for logging)
+    print(report)
+    
+    # Email it
+    today = datetime.now().strftime("%Y-%m-%d")
+    subject = f"🖥️ Susan Morning Report - {today}"
+    
+    try:
+        send_email(subject, report)
+        print("\n✅ Report emailed successfully")
+    except Exception as e:
+        print(f"\n❌ Failed to send email: {e}")
+
+
+if __name__ == "__main__":
+    main()