Initial commit: susan automation scripts

Overnight transcoding, music discovery/import, system health reports,
stats page generator, and bookmark management.

Secrets stored in /etc/automation/ — not in repo.
This commit is contained in:
Caine
2026-02-15 09:41:49 +00:00
commit c7956ae9b2
13 changed files with 3660 additions and 0 deletions
+14
View File
@@ -0,0 +1,14 @@
# Secrets & config with credentials
music_config.json
music_state.json
bookmarks.json
# Generated data
discogs_labels.json
__pycache__/
*.pyc
# Editor
*.swp
*.swo
*~
+23
View File
@@ -0,0 +1,23 @@
# susan-scripts
Automation scripts for Susan (home server). Managed by Caine.
## Scripts
| Script | Purpose |
|--------|---------|
| `overnight_transcoder.py` | Nightly HEVC transcoding of video library |
| `morning_report.py` | Daily system health report (email) |
| `music_recommender.py` | Last.fm-based music discovery + Soulseek download |
| `import_music.sh` | FLAC→Opus transcoding + beets tagging for new albums |
| `transcode_album.sh` | Manual album transcode helper |
| `scrape_discogs_labels.py` | Scrape Discogs labels for music pipeline |
| `generate_stats_page.py` | Retro stats page generator |
| `decrypt_bookmarks.js` | Floccus bookmark decryption |
| `add_bookmark.js` | Add bookmark to Floccus XBEL |
| `add_bookmark_to_wishlist.js` | Add bookmark to wishlist folder |
## Setup
- Copy `music_config.example.json` to `music_config.json` and fill in credentials.
- Bookmark scripts read password from `/etc/automation/bookmarks.json`.
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env node
// Add a bookmark to Floccus encrypted bookmarks
// Usage: node add_bookmark.js <url> <title>
const crypto = require('crypto');
const fs = require('fs');
const BOOKMARKS_PATH = '/var/www/webdav/bookmarks.xbel';
const PASSWORD = process.env.BOOKMARKS_PASSWORD || JSON.parse(require('fs').readFileSync('/etc/automation/bookmarks.json', 'utf8')).password;
const url = process.argv[2];
const title = process.argv[3];
if (!url || !title) {
console.log('Usage: node add_bookmark.js <url> <title>');
process.exit(1);
}
function decrypt(data, password) {
const ciphertext = Buffer.from(data.ciphertext, 'base64');
const salt = data.salt;
const key = crypto.pbkdf2Sync(password, salt, 250000, 32, 'sha256');
const iv = ciphertext.slice(0, 16);
const encrypted = ciphertext.slice(16, -16);
const tag = ciphertext.slice(-16);
const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
decipher.setAuthTag(tag);
return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString('utf8');
}
function encrypt(plaintext, password, salt) {
const key = crypto.pbkdf2Sync(password, salt, 250000, 32, 'sha256');
const iv = crypto.randomBytes(16);
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]);
const tag = cipher.getAuthTag();
const combined = Buffer.concat([iv, encrypted, tag]);
return combined.toString('base64');
}
// Read and decrypt
const data = JSON.parse(fs.readFileSync(BOOKMARKS_PATH, 'utf8'));
let xml = decrypt(data, PASSWORD);
// Find highest ID
const idMatch = xml.match(/highestId :(\d+):/);
let highestId = idMatch ? parseInt(idMatch[1]) : 100;
const newId = highestId + 1;
// Escape XML entities
const escapeXml = (str) => str
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&apos;');
// Add bookmark
const newBookmark = `<bookmark href="${escapeXml(url)}" id="${newId}">
<title>${escapeXml(title)}</title>
</bookmark>
</xbel>`;
xml = xml.replace(/highestId :(\d+):/, `highestId :${newId}:`);
xml = xml.replace('</xbel>', newBookmark);
// Encrypt and save
const newCiphertext = encrypt(xml, PASSWORD, data.salt);
fs.writeFileSync(BOOKMARKS_PATH, JSON.stringify({ ciphertext: newCiphertext, salt: data.salt }));
console.log(`Added bookmark: ${title}`);
console.log(`URL: ${url}`);
console.log(`ID: ${newId}`);
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env node
// Add a bookmark to Music/Wishlist folder in Floccus bookmarks
// Usage: node add_bookmark_to_wishlist.js <url> <title>
const crypto = require('crypto');
const fs = require('fs');
const BOOKMARKS_PATH = '/var/www/webdav/bookmarks.xbel';
const PASSWORD = process.env.BOOKMARKS_PASSWORD || JSON.parse(require('fs').readFileSync('/etc/automation/bookmarks.json', 'utf8')).password;
const WISHLIST_FOLDER_TITLE = 'Wishlist';
const url = process.argv[2];
const title = process.argv[3];
if (!url || !title) {
console.error('Usage: node add_bookmark_to_wishlist.js <url> <title>');
process.exit(1);
}
function decrypt(data, password) {
const ciphertext = Buffer.from(data.ciphertext, 'base64');
const salt = data.salt;
const key = crypto.pbkdf2Sync(password, salt, 250000, 32, 'sha256');
const iv = ciphertext.slice(0, 16);
const encrypted = ciphertext.slice(16, -16);
const tag = ciphertext.slice(-16);
const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
decipher.setAuthTag(tag);
return Buffer.concat([decipher.update(encrypted), decipher.final()]).toString('utf8');
}
function encrypt(plaintext, password, salt) {
const key = crypto.pbkdf2Sync(password, salt, 250000, 32, 'sha256');
const iv = crypto.randomBytes(16);
const cipher = crypto.createCipheriv('aes-256-gcm', key, iv);
const encrypted = Buffer.concat([cipher.update(plaintext, 'utf8'), cipher.final()]);
const tag = cipher.getAuthTag();
return Buffer.concat([iv, encrypted, tag]).toString('base64');
}
const escapeXml = (str) => str
.replace(/&/g, '&amp;')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;')
.replace(/'/g, '&apos;');
try {
const data = JSON.parse(fs.readFileSync(BOOKMARKS_PATH, 'utf8'));
let xml = decrypt(data, PASSWORD);
// Check if bookmark already exists
if (xml.includes(escapeXml(url)) || xml.includes(url)) {
console.log('Bookmark already exists');
process.exit(0);
}
// Find highest ID
const idMatch = xml.match(/highestId :(\d+):/);
let highestId = idMatch ? parseInt(idMatch[1]) : 100;
const newId = highestId + 1;
// Find the Wishlist folder and add bookmark inside it
const wishlistPattern = /<folder id="\d+">\s*<title>Wishlist<\/title>/;
if (wishlistPattern.test(xml)) {
// Add bookmark inside existing Wishlist folder
const newBookmark = `<bookmark href="${escapeXml(url)}" id="${newId}">
<title>${escapeXml(title)}</title>
</bookmark>
</folder>`;
// Find Wishlist folder's closing tag and insert before it
xml = xml.replace(
/(<folder id="\d+">\s*<title>Wishlist<\/title>[\s\S]*?)(<\/folder>)/,
(match, folderContent, closingTag) => folderContent + newBookmark
);
} else {
console.error('Wishlist folder not found');
process.exit(1);
}
// Update highest ID
xml = xml.replace(/highestId :(\d+):/, `highestId :${newId}:`);
// Save
const newCiphertext = encrypt(xml, PASSWORD, data.salt);
fs.writeFileSync(BOOKMARKS_PATH, JSON.stringify({ ciphertext: newCiphertext, salt: data.salt }));
console.log(`Added: ${title}`);
} catch (e) {
console.error('Error:', e.message);
process.exit(1);
}
+34
View File
@@ -0,0 +1,34 @@
#!/usr/bin/env node
// Decrypt Floccus bookmarks
// Usage: node decrypt_bookmarks.js [password]
const crypto = require('crypto');
const fs = require('fs');
const BOOKMARKS_PATH = '/var/www/webdav/bookmarks.xbel';
const password = process.argv[2] || process.env.BOOKMARKS_PASSWORD || JSON.parse(require('fs').readFileSync('/etc/automation/bookmarks.json', 'utf8')).password;
const data = JSON.parse(fs.readFileSync(BOOKMARKS_PATH, 'utf8'));
const ciphertext = Buffer.from(data.ciphertext, 'base64');
const salt = data.salt; // Floccus uses salt as UTF-8 string, not hex-decoded
// Floccus encryption: PBKDF2-SHA256, 250000 iterations
const key = crypto.pbkdf2Sync(password, salt, 250000, 32, 'sha256');
// 16-byte IV at start, then ciphertext, then 16-byte GCM tag at end
const iv = ciphertext.slice(0, 16);
const encrypted = ciphertext.slice(16, -16);
const tag = ciphertext.slice(-16);
try {
const decipher = crypto.createDecipheriv('aes-256-gcm', key, iv);
decipher.setAuthTag(tag);
let decrypted = decipher.update(encrypted);
decrypted = Buffer.concat([decrypted, decipher.final()]);
console.log(decrypted.toString('utf8'));
} catch (e) {
console.error('Decryption failed:', e.message);
process.exit(1);
}
+285
View File
@@ -0,0 +1,285 @@
#!/usr/bin/env python3
"""Generate a retro stats page for Susan.
90s CS professor homepage aesthetic. Pure HTML, no JS.
Regenerate periodically via cron.
"""
import datetime
import os
import subprocess
import re
OUTPUT = "/var/www/webdav/obsidian/stats.html"
def cmd(c):
try:
return subprocess.check_output(c, shell=True, stderr=subprocess.DEVNULL, timeout=10).decode().strip()
except:
return ""
def get_uptime():
up = cmd("uptime -p")
return up.replace("up ", "") if up else "unknown"
def get_uptime_since():
return cmd("uptime -s")
def get_load():
load = cmd("cat /proc/loadavg")
return load.split()[:3] if load else ["?", "?", "?"]
def get_memory():
mem = cmd("free -h | grep Mem")
parts = mem.split()
if len(parts) >= 7:
return {"total": parts[1], "used": parts[2], "free": parts[3], "available": parts[6]}
return {"total": "?", "used": "?", "free": "?", "available": "?"}
def get_disk():
disks = []
for line in cmd("df -h /disks /home 2>/dev/null").split("\n")[1:]:
parts = line.split()
if len(parts) >= 6:
disks.append({"fs": parts[0], "size": parts[1], "used": parts[2], "avail": parts[3], "pct": parts[4], "mount": parts[5]})
return disks
def get_services():
services = []
lines = cmd("systemctl list-units --type=service --state=running --no-pager --no-legend").split("\n")
targets = ["jellyfin", "navidrome", "qbittorrent", "sonarr", "radarr", "lidarr",
"readarr", "prowlarr", "slskd", "nginx", "audiobookshelf", "openclaw-gateway"]
for line in lines:
for t in targets:
if t in line.lower():
name = line.split()[0].replace(".service", "")
services.append(name)
return sorted(services)
def get_media_counts():
films = int(cmd("find /disks/Plex/Films -maxdepth 1 -type d | wc -l") or 1) - 1
tv = int(cmd("find /disks/Plex/TV -maxdepth 1 -type d | wc -l") or 1) - 1
anime = int(cmd("find /disks/Plex/Anime -maxdepth 1 -type d | wc -l") or 1) - 1
tracks = int(cmd("find /disks/Plex/Music -type f \\( -name '*.flac' -o -name '*.mp3' -o -name '*.ogg' -o -name '*.opus' \\) | wc -l") or 0)
artists = int(cmd("ls -1d /disks/Plex/Music/*/ 2>/dev/null | grep -v -E 'venv|lib|bin|data|include|_|pyvenv' | wc -l") or 0)
return {"films": films, "tv": tv, "anime": anime, "tracks": tracks, "artists": artists}
def get_cpu():
model = cmd("grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2").strip()
cores = cmd("nproc")
return model, cores
def get_packages():
return cmd("dpkg -l | grep '^ii' | wc -l")
def get_kernel():
return cmd("uname -r")
def get_os():
return cmd("grep PRETTY_NAME /etc/os-release | cut -d'\"' -f2")
def get_install_date():
d = cmd("stat -c %w /")
if d and d != "-":
return d.split(".")[0]
return "unknown"
COUNTER_FILE = os.path.join(os.path.dirname(__file__), "..", "data", "visitor_counter.txt")
def get_and_increment_counter():
"""Read and increment a persistent visitor counter."""
os.makedirs(os.path.dirname(COUNTER_FILE), exist_ok=True)
count = 0
if os.path.exists(COUNTER_FILE):
try:
count = int(open(COUNTER_FILE).read().strip())
except:
count = 0
count += 1
with open(COUNTER_FILE, "w") as f:
f.write(str(count))
return count
now = datetime.datetime.now()
uptime = get_uptime()
uptime_since = get_uptime_since()
load = get_load()
mem = get_memory()
disks = get_disk()
services = get_services()
media = get_media_counts()
cpu_model, cpu_cores = get_cpu()
packages = get_packages()
kernel = get_kernel()
os_name = get_os()
install_date = get_install_date()
visitor_count = get_and_increment_counter()
html = f"""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<meta http-equiv="refresh" content="300">
<title>Susan - System Status</title>
<style type="text/css">
body {{
background-color: #e8e8e8;
background-image: url("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAMklEQVQYV2N89+7dfwYGBgZGRkYGBgYmBjIBE7mKR5UOPIUMDIyMjGQrHlU68BQCABPnC/0ZkzYUAAAAAElFTkSuQmCC");
font-family: "Times New Roman", Times, serif;
color: #333;
margin: 20px 40px;
font-size: 14pt;
}}
h1 {{
font-size: 22pt;
color: #003366;
border-bottom: 2px solid #003366;
padding-bottom: 4px;
}}
h2 {{
font-size: 16pt;
color: #003366;
margin-top: 24px;
}}
table {{
border-collapse: collapse;
margin: 8px 0;
}}
td, th {{
border: 2px solid #003366;
padding: 4px 10px;
text-align: left;
background-color: #f5f5f0;
}}
th {{
background-color: #d0d0c8;
font-weight: bold;
color: #003366;
}}
a {{
color: #003366;
}}
hr {{
border: none;
border-top: 2px solid #003366;
margin: 16px 0;
}}
.footer {{
margin-top: 30px;
font-size: 10pt;
color: #666;
border-top: 1px solid #999;
padding-top: 6px;
}}
.status-ok {{
color: green;
font-weight: bold;
}}
.status-warn {{
color: #cc7700;
font-weight: bold;
}}
.counter {{
font-family: "Courier New", monospace;
background-color: #000;
color: #0f0;
padding: 2px 6px;
font-size: 10pt;
}}
</style>
</head>
<body>
<h1>Susan &mdash; System Status</h1>
<hr>
<h2>General Information</h2>
<table>
<tr><th>Hostname</th><td>susan</td></tr>
<tr><th>Operating System</th><td>{os_name}</td></tr>
<tr><th>Kernel</th><td>{kernel}</td></tr>
<tr><th>Processor</th><td>{cpu_model} ({cpu_cores} cores)</td></tr>
<tr><th>Installed Packages</th><td>{packages}</td></tr>
<tr><th>First Installed</th><td>{install_date}</td></tr>
</table>
<h2>Uptime &amp; Load</h2>
<table>
<tr><th>Uptime</th><td>{uptime}</td></tr>
<tr><th>Up Since</th><td>{uptime_since}</td></tr>
<tr><th>Load Average</th><td>{load[0]}, {load[1]}, {load[2]} (1, 5, 15 min)</td></tr>
</table>
<h2>Memory</h2>
<table>
<tr><th>Total</th><th>Used</th><th>Free</th><th>Available</th></tr>
<tr><td>{mem['total']}</td><td>{mem['used']}</td><td>{mem['free']}</td><td>{mem['available']}</td></tr>
</table>
<h2>Disk Usage</h2>
<table>
<tr><th>Mount</th><th>Size</th><th>Used</th><th>Available</th><th>Use%</th></tr>
"""
for d in disks:
pct = int(d["pct"].replace("%", "")) if d["pct"].replace("%", "").isdigit() else 0
cls = "status-warn" if pct > 85 else "status-ok"
html += f'<tr><td>{d["mount"]}</td><td>{d["size"]}</td><td>{d["used"]}</td><td>{d["avail"]}</td><td class="{cls}">{d["pct"]}</td></tr>\n'
html += f"""</table>
<h2>Running Services</h2>
<table>
<tr><th>Service</th><th>Status</th></tr>
"""
for s in services:
html += f'<tr><td>{s}</td><td class="status-ok">running</td></tr>\n'
html += f"""</table>
<h2>Media Library</h2>
<table>
<tr><th>Category</th><th>Count</th></tr>
<tr><td>Films</td><td>{media['films']}</td></tr>
<tr><td>TV Shows</td><td>{media['tv']}</td></tr>
<tr><td>Anime</td><td>{media['anime']}</td></tr>
<tr><td>Music Artists</td><td>{media['artists']}</td></tr>
<tr><td>Music Tracks</td><td>{media['tracks']}</td></tr>
</table>
<hr>
<div class="footer">
<p>
You are visitor number <span class="counter">{visitor_count:06,}</span> since December 2023.
</p>
<p>
This page was last generated on <b>{now.strftime("%A, %d %B %Y at %H:%M:%S")}</b>.
<br>
No JavaScript was harmed in the making of this page.
<br>
Best viewed with any browser. Optimised for 800&times;600.
</p>
</div>
</body>
</html>
"""
os.makedirs(os.path.dirname(OUTPUT), exist_ok=True)
with open(OUTPUT, "w") as f:
f.write(html)
print(f"Generated stats page: {OUTPUT}")
print(f" Uptime: {uptime}")
print(f" Services: {len(services)}")
print(f" Films: {media['films']}, TV: {media['tv']}, Anime: {media['anime']}")
print(f" Music: {media['tracks']} tracks, {media['artists']} artists")
+131
View File
@@ -0,0 +1,131 @@
#!/bin/bash
# Import, transcode, and organize music from slskd downloads
# Usage: ./import_music.sh [--dry-run] [--keep-flac]
INGEST_DIR="/disks/Plex/Music/_ingest"
LIBRARY_DIR="/disks/Plex/Music"
GROUP="mediaserver"
BITRATE="128k"
DRY_RUN=false
KEEP_FLAC=false
for arg in "$@"; do
case $arg in
--dry-run) DRY_RUN=true ;;
--keep-flac) KEEP_FLAC=true ;;
esac
done
echo "=== Music Import Started ==="
$DRY_RUN && echo "DRY RUN MODE"
# Skip these directories
shopt -s extglob
cd "$INGEST_DIR" || exit 1
for album_dir in */; do
[[ ! -d "$album_dir" ]] && continue
dir_name="${album_dir%/}"
# Skip special dirs
[[ "$dir_name" == "incomplete" ]] && continue
[[ "$dir_name" == "downloads" ]] && continue
# Count audio files
audio_count=$(ls -1 "$album_dir"*.flac "$album_dir"*.mp3 "$album_dir"*.opus 2>/dev/null | wc -l)
[[ $audio_count -lt 2 ]] && continue
echo ""
echo "Processing: $dir_name ($audio_count files)"
# Get first audio file for metadata
first_file=$(ls "$album_dir"*.flac "$album_dir"*.mp3 2>/dev/null | head -1)
[[ -z "$first_file" ]] && continue
# Extract metadata
artist=$(ffprobe -v quiet -show_entries format_tags=artist -of default=noprint_wrappers=1:nokey=1 "$first_file" 2>/dev/null)
album_artist=$(ffprobe -v quiet -show_entries format_tags=album_artist -of default=noprint_wrappers=1:nokey=1 "$first_file" 2>/dev/null)
album=$(ffprobe -v quiet -show_entries format_tags=album -of default=noprint_wrappers=1:nokey=1 "$first_file" 2>/dev/null)
# Prefer album_artist over artist
[[ -n "$album_artist" ]] && artist="$album_artist"
# Clean for filesystem
artist=$(echo "$artist" | tr -d '<>:"/\\|?*' | sed 's/\.$//')
album=$(echo "$album" | tr -d '<>:"/\\|?*' | sed 's/\.$//')
if [[ -z "$artist" ]] || [[ -z "$album" ]]; then
echo " ⚠ Missing metadata, skipping"
continue
fi
dest_dir="$LIBRARY_DIR/$artist/$album"
echo "$artist / $album"
if $DRY_RUN; then
echo " [DRY RUN] Would transcode to $dest_dir"
continue
fi
mkdir -p "$dest_dir"
# Transcode FLACs
for flac in "$album_dir"*.flac "$album_dir"*.FLAC; do
[[ ! -f "$flac" ]] && continue
base=$(basename "$flac")
# Remove .flac extension properly
name="${base%.[Ff][Ll][Aa][Cc]}"
opus_out="$dest_dir/${name}.opus"
echo " Transcoding: $name"
ffmpeg -hide_banner -loglevel error -i "$flac" \
-c:a libopus -b:a "$BITRATE" -vbr on \
-map_metadata 0 -y "$opus_out"
[[ -f "$opus_out" ]] && chgrp "$GROUP" "$opus_out" 2>/dev/null
done
# Remove any FLACs from destination (we only want Opus there)
for flac in "$dest_dir"/*.flac "$dest_dir"/*.FLAC; do
[[ -f "$flac" ]] && rm "$flac"
done
# Copy non-FLAC audio
for audio in "$album_dir"*.mp3 "$album_dir"*.m4a "$album_dir"*.ogg; do
[[ -f "$audio" ]] && cp "$audio" "$dest_dir/"
done
# Copy artwork
for art in "$album_dir"*.jpg "$album_dir"*.png "$album_dir"cover.* "$album_dir"folder.*; do
[[ -f "$art" ]] && cp "$art" "$dest_dir/"
done
# Fix permissions
chgrp -R "$GROUP" "$dest_dir" 2>/dev/null
chmod -R g+rw "$dest_dir" 2>/dev/null
# Run beets for tagging
echo " Running beets..."
beet import -q "$dest_dir" 2>/dev/null
if [[ $? -eq 0 ]]; then
echo " ✓ Beets tagging complete"
else
echo " ⚠ Beets tagging skipped or failed (check manually)"
fi
# Verify
new_count=$(ls -1 "$dest_dir"/*.opus "$dest_dir"/*.mp3 2>/dev/null | wc -l)
if [[ $new_count -ge 3 ]]; then
echo " ✓ Imported $new_count files"
rm -rf "$INGEST_DIR/$dir_name"
echo " ✓ Cleaned source"
else
echo " ✗ Verification failed"
fi
done
echo ""
echo "=== Done ==="
+438
View File
@@ -0,0 +1,438 @@
#!/usr/bin/env python3
"""
Susan Morning Report
Generates a system health report and emails it to Tom.
Run via cron at 06:45 (after transcoder finishes).
"""
import subprocess
import json
import os
import re
import smtplib
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from datetime import datetime, timedelta
from pathlib import Path
# Config
EMAIL_CONFIG = "/etc/susan/email.json"
TRANSCODER_LOG = "/var/log/transcoder.log"
TRANSCODER_DB = "/var/lib/transcoder/cache.db"
TO_EMAIL = "tom@tomflux.xyz"
def run_cmd(cmd: str, timeout: int = 30) -> str:
"""Run a shell command and return output."""
try:
result = subprocess.run(
cmd, shell=True, capture_output=True, text=True, timeout=timeout
)
return result.stdout.strip()
except Exception as e:
return f"Error: {e}"
def get_uptime() -> dict:
"""Get system uptime info."""
uptime_raw = run_cmd("uptime -p")
load = run_cmd("cat /proc/loadavg").split()[:3]
boot_time = run_cmd("uptime -s")
return {
"uptime": uptime_raw.replace("up ", ""),
"since": boot_time,
"load_avg": f"{load[0]} / {load[1]} / {load[2]}" if len(load) >= 3 else "unknown"
}
def get_disk_space() -> list:
"""Get disk usage for important mounts."""
disks = []
# Only show root and /disks* mounts, exclude virtual filesystems
df_output = run_cmd("df -h --output=target,size,used,avail,pcent -x tmpfs -x devtmpfs -x squashfs | tail -n +2")
for line in df_output.split('\n'):
if line.strip():
parts = line.split()
if len(parts) >= 5:
mount = parts[0]
# Only include / and /disks*, /home, /var
if mount in ['/', '/home', '/var'] or mount.startswith('/disks'):
disks.append({
"mount": mount,
"size": parts[1],
"used": parts[2],
"avail": parts[3],
"percent": parts[4]
})
return disks
def get_raid_status() -> dict:
"""Get 3ware RAID status using tw_cli."""
status = {"available": False, "details": None, "drives": [], "array": None}
# Try tw_cli first
tw_output = run_cmd("sudo tw_cli /c0 show 2>/dev/null")
if "Error" not in tw_output and tw_output and "Unit" in tw_output:
status["available"] = True
status["details"] = tw_output
# Parse array status
for line in tw_output.split('\n'):
if line.startswith('u0'):
parts = line.split()
if len(parts) >= 3:
status["array"] = {
"unit": parts[0],
"type": parts[1],
"status": parts[2],
"verify": parts[4] if len(parts) > 4 and '%' in parts[4] else None
}
# Parse drive lines (p4, p5, p6, etc)
if line.startswith('p'):
parts = line.split()
if len(parts) >= 3:
status["drives"].append({
"port": parts[0],
"status": parts[1],
"size": parts[3] if len(parts) > 3 else "unknown"
})
else:
# Fallback to checking for mdadm
md_output = run_cmd("cat /proc/mdstat 2>/dev/null")
if md_output and "Error" not in md_output and "md" in md_output:
status["available"] = True
status["details"] = md_output
status["type"] = "mdadm"
return status
def get_smart_status() -> list:
"""Get SMART status for drives. Note: requires sudo for smartctl."""
drives = []
# Find physical block devices (skip loop, ram, etc)
lsblk = run_cmd("lsblk -d -o NAME,SIZE,TYPE | grep disk | grep -v loop")
for line in lsblk.split('\n'):
if line.strip():
parts = line.split()
if parts and not parts[0].startswith('loop'):
dev = f"/dev/{parts[0]}"
size = parts[1] if len(parts) > 1 else "unknown"
# Try smartctl with sudo
smart = run_cmd(f"sudo smartctl -H {dev} 2>/dev/null | grep -iE 'overall-health|result|PASSED|FAILED'")
reallocated = run_cmd(f"sudo smartctl -A {dev} 2>/dev/null | grep -i 'Reallocated_Sector'")
# Determine health status
if "PASSED" in smart:
health = "PASSED"
elif "FAILED" in smart:
health = "FAILED"
elif "sudo:" in smart or not smart:
health = "needs sudo"
elif "Unable to detect" in run_cmd(f"sudo smartctl -i {dev} 2>&1"):
health = "RAID array (skip)"
else:
health = "unknown"
# Skip RAID virtual devices (they show as large and can't be queried)
if health == "RAID array (skip)":
continue
# Also skip if smartctl says it's not a physical device
if "unknown" in health and "T" in size: # Multi-TB device with unknown = likely RAID
continue
drive_info = {
"device": dev,
"size": size,
"health": health,
}
if reallocated:
# Extract reallocated sector count (last number on line)
match = re.search(r'(\d+)\s*$', reallocated.strip())
if match:
drive_info["reallocated_sectors"] = int(match.group(1))
drives.append(drive_info)
return drives
def get_memory() -> dict:
"""Get memory usage."""
mem_output = run_cmd("free -h | grep Mem")
parts = mem_output.split()
if len(parts) >= 4:
return {
"total": parts[1],
"used": parts[2],
"available": parts[6] if len(parts) > 6 else parts[3]
}
return {"total": "unknown", "used": "unknown", "available": "unknown"}
def get_cpu_info() -> dict:
"""Get CPU info and temperature."""
info = {}
# CPU model
model = run_cmd("grep 'model name' /proc/cpuinfo | head -1 | cut -d: -f2")
info["model"] = model.strip() if model else "unknown"
# Temperatures
temps = run_cmd("sensors 2>/dev/null | grep -E 'Core|temp' | head -5")
info["temps"] = temps if temps else "sensors not available"
return info
def get_transcoder_status() -> dict:
"""Get last night's transcoder results + queue stats from DB."""
import sqlite3
status = {"ran": False, "summary": None, "queue": None}
# Get queue stats from database
db_path = Path(TRANSCODER_DB)
if db_path.exists():
try:
conn = sqlite3.connect(TRANSCODER_DB)
# Pending files
cursor = conn.execute("""
SELECT COUNT(*), SUM(original_size)
FROM files WHERE is_hevc = 0 AND status = 'pending'
""")
row = cursor.fetchone()
pending_count = row[0] or 0
pending_size = row[1] or 0
# Already HEVC
cursor = conn.execute("SELECT COUNT(*) FROM files WHERE is_hevc = 1")
hevc_count = cursor.fetchone()[0] or 0
# Lifetime stats
cursor = conn.execute("SELECT total_files_transcoded, total_space_saved FROM stats WHERE id = 1")
row = cursor.fetchone()
lifetime_transcoded = row[0] if row else 0
lifetime_saved = row[1] if row else 0
# Failed count
cursor = conn.execute("SELECT COUNT(*) FROM files WHERE status = 'failed'")
failed_count = cursor.fetchone()[0] or 0
conn.close()
status["queue"] = {
"pending_count": pending_count,
"pending_size": pending_size,
"pending_size_human": f"{pending_size / (1024**3):.1f} GB" if pending_size else "0 GB",
"hevc_count": hevc_count,
"failed_count": failed_count,
"lifetime_transcoded": lifetime_transcoded,
"lifetime_saved": lifetime_saved,
"lifetime_saved_human": f"{lifetime_saved / (1024**3):.1f} GB" if lifetime_saved else "0 GB"
}
except Exception as e:
status["queue"] = {"error": str(e)}
# Check log for last night's run
log_path = Path(TRANSCODER_LOG)
if log_path.exists():
try:
log_content = log_path.read_text()
except PermissionError:
log_content = run_cmd(f"sudo cat {TRANSCODER_LOG} 2>/dev/null")
if "Error" in log_content:
log_content = ""
if log_content:
# Find the last SESSION COMPLETE block
sessions = re.findall(
r'SESSION COMPLETE.*?Transcoded:\s+(\d+).*?Failed:\s+(\d+).*?Space saved:\s+([\d.]+\s+\w+)',
log_content, re.DOTALL
)
if sessions:
last = sessions[-1]
status["ran"] = True
status["transcoded"] = int(last[0])
status["failed"] = int(last[1])
status["space_saved"] = last[2]
status["summary"] = f"{last[0]} transcoded, {last[1]} failed, {last[2]} saved"
else:
today = datetime.now().strftime("%Y-%m-%d")
yesterday = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d")
if today in log_content or yesterday in log_content:
status["ran"] = True
status["summary"] = "Ran (check log for details)"
return status
def get_failed_services() -> list:
"""Get any failed systemd services."""
output = run_cmd("systemctl --failed --no-pager --plain | grep -E '\.service|\.socket|\.mount' | awk '{print $1}'")
failed = []
for line in output.split('\n'):
name = line.strip()
if name and not name.startswith('UNIT') and '' not in name:
failed.append(name)
return failed
def generate_report() -> str:
"""Generate the full morning report."""
now = datetime.now()
report = []
report.append("=" * 50)
report.append(f"🖥️ SUSAN MORNING REPORT")
report.append(f"📅 {now.strftime('%A, %B %d %Y at %H:%M')}")
report.append("=" * 50)
report.append("")
# Uptime
uptime = get_uptime()
report.append("⏱️ UPTIME")
report.append(f" Up: {uptime['uptime']}")
report.append(f" Since: {uptime['since']}")
report.append(f" Load: {uptime['load_avg']}")
report.append("")
# Memory
mem = get_memory()
report.append("🧠 MEMORY")
report.append(f" Used: {mem['used']} / {mem['total']}")
report.append(f" Available: {mem['available']}")
report.append("")
# Disk Space
disks = get_disk_space()
report.append("💾 DISK SPACE")
for disk in disks:
warn = " ⚠️" if disk['percent'].replace('%', '').isdigit() and int(disk['percent'].replace('%', '')) > 85 else ""
report.append(f" {disk['mount']}: {disk['used']}/{disk['size']} ({disk['percent']} used){warn}")
report.append("")
# RAID Status
raid = get_raid_status()
report.append("🔒 RAID STATUS")
if raid["available"]:
if raid.get("array"):
arr = raid["array"]
status_icon = "" if arr["status"] == "OK" else ("⚠️" if "VERIFY" in arr["status"] else "")
report.append(f" Array: {arr['type']} {status_icon} {arr['status']}")
if arr.get("verify"):
report.append(f" Verify progress: {arr['verify']}")
if raid.get("drives"):
report.append(f" Drives: {len(raid['drives'])} disks")
for drive in raid["drives"]:
d_icon = "" if drive["status"] == "OK" else ""
report.append(f" {drive['port']}: {d_icon} {drive['status']} ({drive['size']} TB)")
else:
report.append(" tw_cli not available - install 3ware tools for RAID monitoring")
report.append("")
# Drive Health (SMART)
drives = get_smart_status()
if drives:
report.append("🔧 DRIVE HEALTH (SMART)")
for drive in drives:
warn = ""
if drive.get("reallocated_sectors", 0) > 0:
warn = f" ⚠️ {drive['reallocated_sectors']} reallocated sectors!"
health_icon = "" if drive["health"] == "PASSED" else ""
report.append(f" {drive['device']} ({drive['size']}): {health_icon} {drive['health']}{warn}")
report.append("")
# Transcoder
transcoder = get_transcoder_status()
report.append("🎬 TRANSCODER")
# Last night's run
if transcoder.get("ran"):
report.append(f" Last run: ✅ {transcoder.get('summary', 'Completed')}")
elif transcoder.get("summary"):
report.append(f" Last run: ⏸️ {transcoder['summary']}")
else:
report.append(f" Last run: No data")
# Queue stats from database
if transcoder.get("queue") and not transcoder["queue"].get("error"):
q = transcoder["queue"]
report.append(f" Queue: {q['pending_count']} files ({q['pending_size_human']}) waiting")
if q['failed_count'] > 0:
report.append(f" ⚠️ Failed: {q['failed_count']} files need attention")
report.append(f" Library: {q['hevc_count']} files already HEVC")
if q['lifetime_transcoded'] > 0:
report.append(f" Lifetime: {q['lifetime_transcoded']} transcoded, {q['lifetime_saved_human']} saved")
elif transcoder.get("queue", {}).get("error"):
report.append(f" DB error: {transcoder['queue']['error']}")
report.append("")
# Failed Services
failed = get_failed_services()
report.append("🚨 SYSTEMD SERVICES")
if failed:
report.append(f"{len(failed)} failed: {', '.join(failed)}")
else:
report.append(" ✅ All services OK")
report.append("")
report.append("=" * 50)
report.append("End of report")
report.append("=" * 50)
return "\n".join(report)
def send_email(subject: str, body: str):
"""Send the report via email."""
with open(EMAIL_CONFIG) as f:
cfg = json.load(f)
smtp = cfg['smtp']
msg = MIMEMultipart()
msg['Subject'] = subject
msg['From'] = smtp['from']
msg['To'] = TO_EMAIL
# Plain text version
msg.attach(MIMEText(body, 'plain'))
server = smtplib.SMTP_SSL(smtp['server'], smtp['port'])
server.login(smtp['username'], smtp['password'])
server.sendmail(smtp['from'], [TO_EMAIL], msg.as_string())
server.quit()
def main():
report = generate_report()
# Print to stdout (for logging)
print(report)
# Email it
today = datetime.now().strftime("%Y-%m-%d")
subject = f"🖥️ Susan Morning Report - {today}"
try:
send_email(subject, report)
print("\n✅ Report emailed successfully")
except Exception as e:
print(f"\n❌ Failed to send email: {e}")
if __name__ == "__main__":
main()
+15
View File
@@ -0,0 +1,15 @@
{
"lastfm": {
"api_key": "YOUR_LASTFM_API_KEY",
"username": "tomflux"
},
"navidrome": {
"url": "https://navi.jihakuz.xyz",
"username": "YOUR_NAVIDROME_USERNAME",
"password": "YOUR_NAVIDROME_PASSWORD"
},
"slskd": {
"url": "https://music.jihakuz.xyz",
"api_key": "YOUR_SLSKD_API_KEY"
}
}
+1038
View File
File diff suppressed because it is too large Load Diff
+1245
View File
File diff suppressed because it is too large Load Diff
+183
View File
@@ -0,0 +1,183 @@
#!/usr/bin/env python3
"""Scrape releases from Discogs labels and save to JSON for the music pipeline.
Reads label URLs from Tom's bookmarks (Discogs folder) and scrapes releases.
"""
import json
import re
import subprocess
import sys
import time
from pathlib import Path
import requests
HEADERS = {
"User-Agent": "MusicRecommender/1.0 +https://github.com/openclaw"
}
SCRIPTS_DIR = Path(__file__).parent
def get_labels_from_bookmarks() -> list[tuple[int, str]]:
"""Parse bookmarks and extract Discogs label IDs and names.
Only looks in Music > Discog Labels folder.
"""
labels = []
# Run the decrypt script
try:
result = subprocess.run(
["node", str(SCRIPTS_DIR / "decrypt_bookmarks.js")],
capture_output=True,
text=True,
timeout=30
)
bookmarks_xml = result.stdout
except Exception as e:
print(f"Error reading bookmarks: {e}", file=sys.stderr)
return labels
# Find the "Discog Labels" folder section
# Look for <folder...><title>Discog Labels</title>...</folder>
folder_pattern = r'<folder[^>]*>\s*<title>Discog Labels</title>(.*?)</folder>'
folder_match = re.search(folder_pattern, bookmarks_xml, re.DOTALL | re.IGNORECASE)
if not folder_match:
print("Could not find 'Discog Labels' folder in bookmarks", file=sys.stderr)
return labels
folder_content = folder_match.group(1)
# Find Discogs label URLs within that folder
# Pattern: https://www.discogs.com/label/6458-Indochina
pattern = r'href="https://www\.discogs\.com/label/(\d+)-([^"?]+)'
for match in re.finditer(pattern, folder_content):
label_id = int(match.group(1))
label_name = match.group(2).replace("-", " ")
labels.append((label_id, label_name))
return labels
def get_label_releases(label_id: int, label_name: str, max_pages: int = 5) -> list[dict]:
"""Fetch releases from a Discogs label."""
releases = []
for page in range(1, max_pages + 1):
url = f"https://api.discogs.com/labels/{label_id}/releases?page={page}&per_page=100"
print(f" Fetching {label_name} page {page}...", file=sys.stderr)
try:
resp = requests.get(url, headers=HEADERS, timeout=30)
resp.raise_for_status()
data = resp.json()
except Exception as e:
print(f" Error: {e}", file=sys.stderr)
break
for r in data.get("releases", []):
# Skip compilations, singles, etc - focus on albums
if r.get("format") and "Album" not in str(r.get("format", "")):
# Still include if no format specified
pass
artist = r.get("artist", "Various")
title = r.get("title", "")
year = r.get("year", "")
# Clean up artist name
artist = re.sub(r'\s*\(\d+\)$', '', artist) # Remove disambiguation numbers
if artist and title and artist.lower() != "various":
releases.append({
"artist": artist,
"album": title,
"year": year,
"label": label_name,
"discogs_id": r.get("id"),
})
# Check if more pages
if page >= data.get("pagination", {}).get("pages", 1):
break
time.sleep(1) # Rate limit
return releases
def get_labels_from_config() -> list[tuple[int, str]]:
"""Get Discogs labels from music_config.json."""
labels = []
config_path = SCRIPTS_DIR / "music_config.json"
if not config_path.exists():
return labels
try:
with open(config_path) as f:
config = json.load(f)
for entry in config.get("discogs_labels", []):
url = entry.get("url", "")
name = entry.get("name", "")
# Extract ID from URL like https://www.discogs.com/label/6170-Tempa
match = re.search(r'/label/(\d+)', url)
if match and name:
labels.append((int(match.group(1)), name))
except Exception as e:
print(f"Error reading config: {e}", file=sys.stderr)
return labels
def main():
# Get labels from bookmarks and config
labels = get_labels_from_bookmarks()
config_labels = get_labels_from_config()
# Merge, avoiding duplicates by ID
seen_ids = {lid for lid, _ in labels}
for lid, lname in config_labels:
if lid not in seen_ids:
labels.append((lid, lname))
seen_ids.add(lid)
if not labels:
print("No Discogs labels found in bookmarks or config!", file=sys.stderr)
sys.exit(1)
print(f"Found {len(labels)} labels in bookmarks:", file=sys.stderr)
for lid, lname in labels:
print(f" - {lname} ({lid})", file=sys.stderr)
all_releases = []
for label_id, label_name in labels:
print(f"Scraping {label_name}...", file=sys.stderr)
releases = get_label_releases(label_id, label_name)
all_releases.extend(releases)
print(f" Found {len(releases)} releases", file=sys.stderr)
time.sleep(2) # Be nice to Discogs
# Dedupe by artist+album
seen = set()
unique = []
for r in all_releases:
key = f"{r['artist'].lower()}|{r['album'].lower()}"
if key not in seen:
seen.add(key)
unique.append(r)
output = {
"labels": [{"id": lid, "name": lname} for lid, lname in labels],
"releases": unique,
"scraped_at": time.strftime("%Y-%m-%d %H:%M:%S"),
}
print(json.dumps(output, indent=2))
if __name__ == "__main__":
main()
+81
View File
@@ -0,0 +1,81 @@
#!/bin/bash
# Transcode FLAC album to Opus, preserving metadata
# Usage: ./transcode_album.sh <source_dir> [--delete-source]
#
# Converts all FLAC files to Opus 128kbps (transparent quality, ~10x smaller)
# Preserves all metadata tags and album art
set -e
SOURCE_DIR="$1"
DELETE_SOURCE=false
BITRATE="128k"
GROUP="mediaserver"
if [[ "$2" == "--delete-source" ]]; then
DELETE_SOURCE=true
fi
if [[ -z "$SOURCE_DIR" ]] || [[ ! -d "$SOURCE_DIR" ]]; then
echo "Usage: $0 <source_directory> [--delete-source]"
exit 1
fi
# Count FLAC files
flac_count=$(find "$SOURCE_DIR" -maxdepth 1 -iname "*.flac" | wc -l)
if [[ $flac_count -eq 0 ]]; then
echo "No FLAC files found in $SOURCE_DIR"
exit 0
fi
echo "Transcoding $flac_count FLAC files to Opus ($BITRATE)..."
echo "Source: $SOURCE_DIR"
converted=0
failed=0
# Process each FLAC file
find "$SOURCE_DIR" -maxdepth 1 -iname "*.flac" | while read -r flac_file; do
filename=$(basename "$flac_file")
opus_file="${flac_file%.flac}.opus"
opus_file="${opus_file%.FLAC}.opus"
echo " Converting: $filename"
if ffmpeg -hide_banner -loglevel warning -i "$flac_file" \
-c:a libopus -b:a "$BITRATE" -vbr on \
-map_metadata 0 \
-y "$opus_file" 2>&1; then
# Preserve original timestamp
touch -r "$flac_file" "$opus_file"
# Fix permissions
chgrp "$GROUP" "$opus_file" 2>/dev/null || true
chmod 664 "$opus_file" 2>/dev/null || true
if $DELETE_SOURCE; then
rm "$flac_file"
echo " ✓ Converted and removed FLAC"
else
echo " ✓ Converted (FLAC kept)"
fi
((converted++)) || true
else
echo " ✗ Failed to convert"
((failed++)) || true
fi
done
# Also convert any MP3s that are suspiciously large (>15MB per file avg)
# Actually, skip this - MP3s are already compressed
# Handle cover art - keep it
# ffmpeg should copy embedded art automatically for opus
echo ""
echo "Done! Converted $converted files."
if $DELETE_SOURCE; then
# Calculate space saved
echo "FLAC files deleted to save space."
fi