Initial commit

2026-09-06 22:30:13 -04:00
commit 83627e788b
43 changed files with 43131 additions and 0 deletions
@@ -0,0 +1,137 @@
+"""Download all libretro PSP box art into a local offline bundle.
+
+The libretro-thumbnails repo is served from GitHub raw (not Cloudflare-blocked),
+so plain urllib works. Some files are Git symlinks (GitHub serves them as a tiny
+text body); ``_fetch_png`` follows those.
+
+Usage::
+
+    python3 -m app.libretro              # download everything (resumes)
+    python3 -m app.libretro --limit 50   # test
+
+Output: ``data/covers/libretro-covers.zip`` — entries named ``<norm(title)>.png``
+matching the keys used by ``covers.fetch_libretro``.
+"""
+import argparse
+import concurrent.futures
+import logging
+import os
+import sys
+import tempfile
+import threading
+import zipfile
+from pathlib import Path
+
+from . import config
+from .covers import _fetch_png, _load_index, _norm, _plain
+
+log = logging.getLogger("libretro")
+
+BUNDLE = config.DATA_DIR / "covers" / "libretro-covers.zip"
+
+
+def _region_rank(name):
+    nl = name.lower()
+    if "(usa" in nl:
+        return 0
+    if "(europe" in nl or "(australia" in nl:
+        return 1
+    if "(japan" in nl or "(asia" in nl or "(korea" in nl:
+        return 2
+    return 1  # unknown regions are treated as neutral
+
+
+def _dedupe(names):
+    """Collapse regional duplicates to one entry, preferring US covers."""
+    best = {}
+    for n in names:
+        key = _norm(_plain(n))
+        if not key:
+            continue
+        rank = _region_rank(n)
+        cur = best.get(key)
+        if cur is None or rank < cur[1]:
+            best[key] = (n, rank)
+    return best  # key -> (filename, rank)
+
+
+def scrape(limit=None, workers=8):
+    BUNDLE.parent.mkdir(parents=True, exist_ok=True)
+
+    names = _load_index(force=True).get("Named_Boxarts", [])
+    log.info("indexed %d libretro boxarts", len(names))
+    mapping = _dedupe(names)
+    log.info("%d unique titles", len(mapping))
+
+    existing = set()
+    if BUNDLE.exists():
+        try:
+            with zipfile.ZipFile(BUNDLE) as z:
+                existing = {n[:-4] for n in z.namelist() if n.endswith(".png")}
+        except Exception:  # noqa: BLE001
+            existing = set()
+
+    todo = [(k, v[0]) for k, v in sorted(mapping.items()) if k not in existing]
+    if limit:
+        todo = todo[:limit]
+    log.info("%d to download (%d already bundled)", len(todo), len(existing))
+
+    work_dir = Path(tempfile.mkdtemp(prefix="libretro-covers-"))
+    ok = fail = 0
+    done = 0
+    lock = threading.Lock()
+
+    def run(item):
+        key, fname = item
+        data = _fetch_png(fname, "Named_Boxarts", timeout=30)
+        nonlocal ok, fail, done
+        with lock:
+            done += 1
+            if data and data[:8] == b"\x89PNG\r\n\x1a\n":
+                (work_dir / f"{key}.png").write_bytes(data)
+                ok += 1
+            else:
+                fail += 1
+            if done % 100 == 0 or (not data):
+                log.info("  %d/%d (%d ok, %d failed)", done, len(todo), ok, fail)
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
+        list(pool.map(run, todo))
+
+    if ok:
+        out_tmp = BUNDLE.with_name(BUNDLE.name + ".tmp")
+        existing_bytes = {}
+        if BUNDLE.exists():
+            try:
+                with zipfile.ZipFile(BUNDLE) as zin:
+                    for n in zin.namelist():
+                        if n.endswith(".png") and n not in existing_bytes:
+                            existing_bytes[n] = zin.read(n)
+            except Exception:  # noqa: BLE001
+                existing_bytes = {}
+        with zipfile.ZipFile(out_tmp, "w", zipfile.ZIP_STORED) as z:
+            for name, data in existing_bytes.items():
+                z.writestr(name, data)
+            for f in sorted(work_dir.glob("*.png")):
+                z.write(f, arcname=f.name)
+        os.replace(out_tmp, BUNDLE)
+        log.info("done: bundle now has %d covers (%d new, %d failed)",
+                 len(existing_bytes) + ok, ok, fail)
+    else:
+        log.info("no new covers downloaded; bundle unchanged (%d failed)", fail)
+    return ok, fail
+
+
+def main(argv=None):
+    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(message)s", stream=sys.stderr)
+    ap = argparse.ArgumentParser(description="Download libretro PSP box art into a local bundle")
+    ap.add_argument("--limit", type=int, default=None, help="only download the first N covers (testing)")
+    ap.add_argument("--workers", type=int, default=8, help="parallel downloads")
+    args = ap.parse_args(argv)
+
+    ok, fail = scrape(limit=args.limit, workers=args.workers)
+    return 0 if fail == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())