1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
|
#!/usr/bin/env python3
"""Scrape releases from Discogs labels and save to JSON for the music pipeline.
Reads label URLs from Tom's bookmarks (Discogs folder) and scrapes releases.
"""
import json
import re
import subprocess
import sys
import time
from pathlib import Path
import requests
HEADERS = {
"User-Agent": "MusicRecommender/1.0 +https://github.com/openclaw"
}
SCRIPTS_DIR = Path(__file__).parent
def get_labels_from_bookmarks() -> list[tuple[int, str]]:
"""Parse bookmarks and extract Discogs label IDs and names.
Only looks in Music > Discog Labels folder.
"""
labels = []
# Run the decrypt script
try:
result = subprocess.run(
["node", str(SCRIPTS_DIR / "decrypt_bookmarks.js")],
capture_output=True,
text=True,
timeout=30
)
bookmarks_xml = result.stdout
except Exception as e:
print(f"Error reading bookmarks: {e}", file=sys.stderr)
return labels
# Find the "Discog Labels" folder section
# Look for <folder...><title>Discog Labels</title>...</folder>
folder_pattern = r'<folder[^>]*>\s*<title>Discog Labels</title>(.*?)</folder>'
folder_match = re.search(folder_pattern, bookmarks_xml, re.DOTALL | re.IGNORECASE)
if not folder_match:
print("Could not find 'Discog Labels' folder in bookmarks", file=sys.stderr)
return labels
folder_content = folder_match.group(1)
# Find Discogs label URLs within that folder
# Pattern: https://www.discogs.com/label/6458-Indochina
pattern = r'href="https://www\.discogs\.com/label/(\d+)-([^"?]+)'
for match in re.finditer(pattern, folder_content):
label_id = int(match.group(1))
label_name = match.group(2).replace("-", " ")
labels.append((label_id, label_name))
return labels
def get_label_releases(label_id: int, label_name: str, max_pages: int = 5) -> list[dict]:
"""Fetch releases from a Discogs label."""
releases = []
for page in range(1, max_pages + 1):
url = f"https://api.discogs.com/labels/{label_id}/releases?page={page}&per_page=100"
print(f" Fetching {label_name} page {page}...", file=sys.stderr)
try:
resp = requests.get(url, headers=HEADERS, timeout=30)
resp.raise_for_status()
data = resp.json()
except Exception as e:
print(f" Error: {e}", file=sys.stderr)
break
for r in data.get("releases", []):
# Skip compilations, singles, etc - focus on albums
if r.get("format") and "Album" not in str(r.get("format", "")):
# Still include if no format specified
pass
artist = r.get("artist", "Various")
title = r.get("title", "")
year = r.get("year", "")
# Clean up artist name
artist = re.sub(r'\s*\(\d+\)$', '', artist) # Remove disambiguation numbers
if artist and title and artist.lower() != "various":
releases.append({
"artist": artist,
"album": title,
"year": year,
"label": label_name,
"discogs_id": r.get("id"),
})
# Check if more pages
if page >= data.get("pagination", {}).get("pages", 1):
break
time.sleep(1) # Rate limit
return releases
def get_labels_from_config() -> list[tuple[int, str]]:
"""Get Discogs labels from music_config.json."""
labels = []
config_path = SCRIPTS_DIR / "music_config.json"
if not config_path.exists():
return labels
try:
with open(config_path) as f:
config = json.load(f)
for entry in config.get("discogs_labels", []):
url = entry.get("url", "")
name = entry.get("name", "")
# Extract ID from URL like https://www.discogs.com/label/6170-Tempa
match = re.search(r'/label/(\d+)', url)
if match and name:
labels.append((int(match.group(1)), name))
except Exception as e:
print(f"Error reading config: {e}", file=sys.stderr)
return labels
def main():
# Get labels from bookmarks and config
labels = get_labels_from_bookmarks()
config_labels = get_labels_from_config()
# Merge, avoiding duplicates by ID
seen_ids = {lid for lid, _ in labels}
for lid, lname in config_labels:
if lid not in seen_ids:
labels.append((lid, lname))
seen_ids.add(lid)
if not labels:
print("No Discogs labels found in bookmarks or config!", file=sys.stderr)
sys.exit(1)
print(f"Found {len(labels)} labels in bookmarks:", file=sys.stderr)
for lid, lname in labels:
print(f" - {lname} ({lid})", file=sys.stderr)
all_releases = []
for label_id, label_name in labels:
print(f"Scraping {label_name}...", file=sys.stderr)
releases = get_label_releases(label_id, label_name)
all_releases.extend(releases)
print(f" Found {len(releases)} releases", file=sys.stderr)
time.sleep(2) # Be nice to Discogs
# Dedupe by artist+album
seen = set()
unique = []
for r in all_releases:
key = f"{r['artist'].lower()}|{r['album'].lower()}"
if key not in seen:
seen.add(key)
unique.append(r)
output = {
"labels": [{"id": lid, "name": lname} for lid, lname in labels],
"releases": unique,
"scraped_at": time.strftime("%Y-%m-%d %H:%M:%S"),
}
print(json.dumps(output, indent=2))
if __name__ == "__main__":
main()
|
