138 lines
4.5 KiB
Python
138 lines
4.5 KiB
Python
"""Download all libretro PSP box art into a local offline bundle.
|
|
|
|
The libretro-thumbnails repo is served from GitHub raw (not Cloudflare-blocked),
|
|
so plain urllib works. Some files are Git symlinks (GitHub serves them as a tiny
|
|
text body); ``_fetch_png`` follows those.
|
|
|
|
Usage::
|
|
|
|
python3 -m app.libretro # download everything (resumes)
|
|
python3 -m app.libretro --limit 50 # test
|
|
|
|
Output: ``data/covers/libretro-covers.zip`` — entries named ``<norm(title)>.png``
|
|
matching the keys used by ``covers.fetch_libretro``.
|
|
"""
|
|
import argparse
|
|
import concurrent.futures
|
|
import logging
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
from . import config
|
|
from .covers import _fetch_png, _load_index, _norm, _plain
|
|
|
|
log = logging.getLogger("libretro")
|
|
|
|
BUNDLE = config.DATA_DIR / "covers" / "libretro-covers.zip"
|
|
|
|
|
|
def _region_rank(name):
|
|
nl = name.lower()
|
|
if "(usa" in nl:
|
|
return 0
|
|
if "(europe" in nl or "(australia" in nl:
|
|
return 1
|
|
if "(japan" in nl or "(asia" in nl or "(korea" in nl:
|
|
return 2
|
|
return 1 # unknown regions are treated as neutral
|
|
|
|
|
|
def _dedupe(names):
|
|
"""Collapse regional duplicates to one entry, preferring US covers."""
|
|
best = {}
|
|
for n in names:
|
|
key = _norm(_plain(n))
|
|
if not key:
|
|
continue
|
|
rank = _region_rank(n)
|
|
cur = best.get(key)
|
|
if cur is None or rank < cur[1]:
|
|
best[key] = (n, rank)
|
|
return best # key -> (filename, rank)
|
|
|
|
|
|
def scrape(limit=None, workers=8):
|
|
BUNDLE.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
names = _load_index(force=True).get("Named_Boxarts", [])
|
|
log.info("indexed %d libretro boxarts", len(names))
|
|
mapping = _dedupe(names)
|
|
log.info("%d unique titles", len(mapping))
|
|
|
|
existing = set()
|
|
if BUNDLE.exists():
|
|
try:
|
|
with zipfile.ZipFile(BUNDLE) as z:
|
|
existing = {n[:-4] for n in z.namelist() if n.endswith(".png")}
|
|
except Exception: # noqa: BLE001
|
|
existing = set()
|
|
|
|
todo = [(k, v[0]) for k, v in sorted(mapping.items()) if k not in existing]
|
|
if limit:
|
|
todo = todo[:limit]
|
|
log.info("%d to download (%d already bundled)", len(todo), len(existing))
|
|
|
|
work_dir = Path(tempfile.mkdtemp(prefix="libretro-covers-"))
|
|
ok = fail = 0
|
|
done = 0
|
|
lock = threading.Lock()
|
|
|
|
def run(item):
|
|
key, fname = item
|
|
data = _fetch_png(fname, "Named_Boxarts", timeout=30)
|
|
nonlocal ok, fail, done
|
|
with lock:
|
|
done += 1
|
|
if data and data[:8] == b"\x89PNG\r\n\x1a\n":
|
|
(work_dir / f"{key}.png").write_bytes(data)
|
|
ok += 1
|
|
else:
|
|
fail += 1
|
|
if done % 100 == 0 or (not data):
|
|
log.info(" %d/%d (%d ok, %d failed)", done, len(todo), ok, fail)
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
|
list(pool.map(run, todo))
|
|
|
|
if ok:
|
|
out_tmp = BUNDLE.with_name(BUNDLE.name + ".tmp")
|
|
existing_bytes = {}
|
|
if BUNDLE.exists():
|
|
try:
|
|
with zipfile.ZipFile(BUNDLE) as zin:
|
|
for n in zin.namelist():
|
|
if n.endswith(".png") and n not in existing_bytes:
|
|
existing_bytes[n] = zin.read(n)
|
|
except Exception: # noqa: BLE001
|
|
existing_bytes = {}
|
|
with zipfile.ZipFile(out_tmp, "w", zipfile.ZIP_STORED) as z:
|
|
for name, data in existing_bytes.items():
|
|
z.writestr(name, data)
|
|
for f in sorted(work_dir.glob("*.png")):
|
|
z.write(f, arcname=f.name)
|
|
os.replace(out_tmp, BUNDLE)
|
|
log.info("done: bundle now has %d covers (%d new, %d failed)",
|
|
len(existing_bytes) + ok, ok, fail)
|
|
else:
|
|
log.info("no new covers downloaded; bundle unchanged (%d failed)", fail)
|
|
return ok, fail
|
|
|
|
|
|
def main(argv=None):
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(message)s", stream=sys.stderr)
|
|
ap = argparse.ArgumentParser(description="Download libretro PSP box art into a local bundle")
|
|
ap.add_argument("--limit", type=int, default=None, help="only download the first N covers (testing)")
|
|
ap.add_argument("--workers", type=int, default=8, help="parallel downloads")
|
|
args = ap.parse_args(argv)
|
|
|
|
ok, fail = scrape(limit=args.limit, workers=args.workers)
|
|
return 0 if fail == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|