"""Download all libretro PSP box art into a local offline bundle. The libretro-thumbnails repo is served from GitHub raw (not Cloudflare-blocked), so plain urllib works. Some files are Git symlinks (GitHub serves them as a tiny text body); ``_fetch_png`` follows those. Usage:: python3 -m app.libretro # download everything (resumes) python3 -m app.libretro --limit 50 # test Output: ``data/covers/libretro-covers.zip`` — entries named ``.png`` matching the keys used by ``covers.fetch_libretro``. """ import argparse import concurrent.futures import logging import os import sys import tempfile import threading import zipfile from pathlib import Path from . import config from .covers import _fetch_png, _load_index, _norm, _plain log = logging.getLogger("libretro") BUNDLE = config.DATA_DIR / "covers" / "libretro-covers.zip" def _region_rank(name): nl = name.lower() if "(usa" in nl: return 0 if "(europe" in nl or "(australia" in nl: return 1 if "(japan" in nl or "(asia" in nl or "(korea" in nl: return 2 return 1 # unknown regions are treated as neutral def _dedupe(names): """Collapse regional duplicates to one entry, preferring US covers.""" best = {} for n in names: key = _norm(_plain(n)) if not key: continue rank = _region_rank(n) cur = best.get(key) if cur is None or rank < cur[1]: best[key] = (n, rank) return best # key -> (filename, rank) def scrape(limit=None, workers=8): BUNDLE.parent.mkdir(parents=True, exist_ok=True) names = _load_index(force=True).get("Named_Boxarts", []) log.info("indexed %d libretro boxarts", len(names)) mapping = _dedupe(names) log.info("%d unique titles", len(mapping)) existing = set() if BUNDLE.exists(): try: with zipfile.ZipFile(BUNDLE) as z: existing = {n[:-4] for n in z.namelist() if n.endswith(".png")} except Exception: # noqa: BLE001 existing = set() todo = [(k, v[0]) for k, v in sorted(mapping.items()) if k not in existing] if limit: todo = todo[:limit] log.info("%d to download (%d already bundled)", len(todo), len(existing)) work_dir = Path(tempfile.mkdtemp(prefix="libretro-covers-")) ok = fail = 0 done = 0 lock = threading.Lock() def run(item): key, fname = item data = _fetch_png(fname, "Named_Boxarts", timeout=30) nonlocal ok, fail, done with lock: done += 1 if data and data[:8] == b"\x89PNG\r\n\x1a\n": (work_dir / f"{key}.png").write_bytes(data) ok += 1 else: fail += 1 if done % 100 == 0 or (not data): log.info(" %d/%d (%d ok, %d failed)", done, len(todo), ok, fail) with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool: list(pool.map(run, todo)) if ok: out_tmp = BUNDLE.with_name(BUNDLE.name + ".tmp") existing_bytes = {} if BUNDLE.exists(): try: with zipfile.ZipFile(BUNDLE) as zin: for n in zin.namelist(): if n.endswith(".png") and n not in existing_bytes: existing_bytes[n] = zin.read(n) except Exception: # noqa: BLE001 existing_bytes = {} with zipfile.ZipFile(out_tmp, "w", zipfile.ZIP_STORED) as z: for name, data in existing_bytes.items(): z.writestr(name, data) for f in sorted(work_dir.glob("*.png")): z.write(f, arcname=f.name) os.replace(out_tmp, BUNDLE) log.info("done: bundle now has %d covers (%d new, %d failed)", len(existing_bytes) + ok, ok, fail) else: log.info("no new covers downloaded; bundle unchanged (%d failed)", fail) return ok, fail def main(argv=None): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(message)s", stream=sys.stderr) ap = argparse.ArgumentParser(description="Download libretro PSP box art into a local bundle") ap.add_argument("--limit", type=int, default=None, help="only download the first N covers (testing)") ap.add_argument("--workers", type=int, default=8, help="parallel downloads") args = ap.parse_args(argv) ok, fail = scrape(limit=args.limit, workers=args.workers) return 0 if fail == 0 else 1 if __name__ == "__main__": raise SystemExit(main())