Unbundle covers: fetch psppt-covers.zip from repo on demand with user prompt

.gitignore

@@ -23,4 +23,8 @@ data/nps/*.tsv
 
 # PyInstaller output
 build/
-dist/
+dist/
+
+# Offline cover bundles (large; fetched from the project release on first run)
+data/covers/*.zip
+/psppt-covers.zip

README.md

@@ -79,8 +79,10 @@ pyinstaller --clean --noconfirm psppt.spec
 
 Notes:
 
-- The build bundles the DATs, PS1 loader, and the offline cover bundles. The
-  cover zips are large (~850 MB), so builds are slow and the output is big.
+- The build bundles the DATs, PS1 loader, cover templates, and index catalogs
+  (~80 MB total). The large offline cover zips (~850 MB) are **not** bundled —
+  the app fetches them on first run (see
+  [Offline bundles](#offline-bundles--auto-download)).
 - `7z` (p7zip-full on Linux, 7-Zip on Windows) is a **runtime** dependency for
   reading store archives — it is not bundled.
 - User data (credentials, manual covers, PS1 templates, NPS cache, library DB,
@@ -170,9 +172,10 @@ Discs are matched by hash first (most precise), then serial, then filename.
 - `data/credentials.json` — cover/metadata API keys (IGDB, TheMovieDB,
   ScreenScraper, Google Custom Search). Environment variables take precedence.
 - `data/manual_covers/games|videos/` — your own cover images.
-- `data/covers/` — checked-in assets: the UMD video/game cover templates,
-  `libretro-index.json` + `openvgdb-index.json` catalogs, `libretro-covers.zip` +
-  `openvgdb-covers.zip` offline cover bundles, and the legacy cover cache.
+- `data/covers/` — checked-in assets: the UMD video/game cover templates and
+  the `libretro-index.json` + `openvgdb-index.json` catalogs. The large
+  `libretro-covers.zip` + `openvgdb-covers.zip` bundles are git-ignored and
+  distributed as `psppt-covers.zip` (see above).
 - `data/base.PBP` — a PSN PS1 classic EBOOT (any game) supplying `STARTDAT`
   + a base `PARAM.SFO`/icon for PS1 conversion (git-ignored, copyrighted).
 - `data/loader/DATA.PSP` + `data/loader/KEYS.BIN` — bundled plain-ELF POPS
@@ -223,6 +226,7 @@ QSettings under `psppt`) taking precedence where noted.
 | `PSP_CREDENTIALS` | `$PSP_USER_DATA_DIR/credentials.json` | credentials file |
 | `PSP_LIBRETRO_COVERS_URL` | *(empty)* | direct URL for `libretro-covers.zip` (else derived from store server) |
 | `PSP_OVGDB_COVERS_URL` | *(empty)* | direct URL for `openvgdb-covers.zip` (else derived from store server) |
+| `PSP_COVERS_BUNDLE_URL` | Gitea `covers` release asset | combined `psppt-covers.zip` download URL |
 
 Cover/metadata keys can also be set as env vars: `IGDB_CLIENT_ID`,
 `IGDB_CLIENT_SECRET`, `TMDB_API_KEY`, `GOOGLE_API_KEY`, `GOOGLE_CX`,
@@ -255,12 +259,23 @@ Manual covers in `data/manual_covers/videos/` always win.
 
 ### Offline bundles & auto-download
 
-The two bundles (`libretro-covers.zip`, `openvgdb-covers.zip`) are shipped in
-`data/covers/`, so a fresh install works with **no internet and no API keys**.
-If either is missing at startup, the app downloads it from the store server's
-`/api/covers/<name>.zip` endpoint (see [Server setup](#server-setup)).
+The two bundles (`libretro-covers.zip`, `openvgdb-covers.zip`) are packaged
+into one `psppt-covers.zip` (~850 MB) and attached to the repo's `covers`
+release — **not** bundled into the app binary.
 
-Rebuild them yourself:
+On first launch, if the bundle isn't installed, the app asks whether to
+download it (with a progress bar). Covers keep working online via libretro /
+OpenVGDB / ScreenScraper / IGDB without it — the bundle just makes them instant
+and offline. "Don't ask again" is remembered, and you can re-download from
+**Settings → Cover sources → Download offline cover bundle…**.
+
+Build the bundle yourself:
+
+```bash
+python3 scripts/make-covers-bundle.py   # -> psppt-covers.zip
+```
+
+Rebuild the inner zips:
 
 ```bash
 python3 -m app.libretro          # libretro-thumbnails -> libretro-covers.zip

app/config.py

@@ -76,6 +76,13 @@ STORE_TOKEN = os.environ.get("PSP_STORE_TOKEN", "")
 LIBRETRO_COVERS_URL = os.environ.get("PSP_LIBRETRO_COVERS_URL", "")
 OVGDB_COVERS_URL = os.environ.get("PSP_OVGDB_COVERS_URL", "")
 
+# Combined offline cover bundle (both inner zips) fetched on first run when the
+# user opts in. Defaults to the project's Gitea release asset.
+COVERS_BUNDLE_URL = os.environ.get(
+    "PSP_COVERS_BUNDLE_URL",
+    "https://git.ceaf.dev/quoije/psp-preservation-tool/releases/download/covers/psppt-covers.zip",
+)
+
 # archive.org item hosting the same Redump 7z set (identifier or full URL).
 ARCHIVE_ITEM = os.environ.get("PSP_ARCHIVE_ITEM", "psp-redump-roms321.com")
 

app/covers.py

@@ -53,8 +53,20 @@ _TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie"
 _TMDB_IMAGE = "https://image.tmdb.org/t/p/"
 _GOOGLE_CSE_API = "https://www.googleapis.com/customsearch/v1"
 _MANUAL_DIR = config.USER_DATA_DIR / "manual_covers"
-_OVGDB_ZIP = config.DATA_DIR / "covers" / "openvgdb-covers.zip"
-_LIBRETRO_ZIP = config.DATA_DIR / "covers" / "libretro-covers.zip"
+# Offline cover bundles live in the writable cover dir (downloaded on demand),
+# not the read-only app bundle. A source checkout keeps them in data/covers/.
+_OVGDB_ZIP = config.COVERS_DIR / "openvgdb-covers.zip"
+_LIBRETRO_ZIP = config.COVERS_DIR / "libretro-covers.zip"
+_BUNDLE_ZIP = config.COVERS_DIR / "psppt-covers.zip"
+
+
+def _bundle_zip_path(name):
+    """Resolve a cover bundle zip: writable dir first, legacy checkout second."""
+    for base in (config.COVERS_DIR, config.LEGACY_COVERS_DIR):
+        p = base / name
+        if p.exists():
+            return p
+    return config.COVERS_DIR / name
 
 _WIKI_SEARCH = "https://en.wikipedia.org/w/api.php"
 _WIKI_SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/"
@@ -158,48 +170,74 @@ def clear_cover_cache():
                     pass
 
 
-def _download_file(url, dest, timeout=600):
-    """Stream `url` to `dest` (atomic); returns True on success."""
+def _download_file(url, dest, timeout=600, progress_cb=None, cancel_event=None):
+    """Stream `url` to `dest` (atomic); returns True on success.
+
+    ``progress_cb(done, total)`` is called with bytes so far (total is 0 when
+    the server doesn't report a content length). ``cancel_event`` aborts the
+    download when set.
+    """
     req = urllib.request.Request(url, headers=_UA)
     with urllib.request.urlopen(req, timeout=timeout) as resp:
         if resp.status != 200:
             return False
+        total = int(resp.headers.get("Content-Length") or 0)
         tmp = dest.with_name(dest.name + ".tmp")
+        done = 0
         with open(tmp, "wb") as f:
             while True:
+                if cancel_event is not None and cancel_event.is_set():
+                    raise InterruptedError("download cancelled")
                 chunk = resp.read(1 << 20)
                 if not chunk:
                     break
                 f.write(chunk)
+                done += len(chunk)
+                if progress_cb is not None:
+                    progress_cb(done, total)
     os.replace(tmp, dest)
     return True
 
 
-def ensure_cover_bundles(server_url=""):
-    """Download missing offline cover bundle zips from the store server.
+def covers_bundle_present():
+    """True when the offline cover bundle is installed (both inner zips)."""
+    return (_bundle_zip_path("libretro-covers.zip").exists()
+            and _bundle_zip_path("openvgdb-covers.zip").exists())
 
-    ``server_url`` is the base URL (e.g. ``http://192.168.10.100:8000``); the
-    zips are fetched from ``<base>/api/covers/<name>.zip`` unless a direct URL
-    is set in config. Returns the names of bundles downloaded.
+
+def download_covers_bundle(url=None, progress_cb=None, cancel_event=None):
+    """Download and extract the combined cover bundle into the cover dir.
+
+    ``url`` defaults to ``config.COVERS_BUNDLE_URL``. Raises on failure; the
+    caller decides whether to surface the error.
     """
-    base = (server_url or config.STORE_SERVER_URL or "").rstrip("/")
-    downloaded = []
-    for name, path, direct in (
-        ("libretro-covers.zip", _LIBRETRO_ZIP, config.LIBRETRO_COVERS_URL),
-        ("openvgdb-covers.zip", _OVGDB_ZIP, config.OVGDB_COVERS_URL),
-    ):
-        if path.exists():
-            continue
-        url = direct or (f"{base}/api/covers/{name}" if base else "")
-        if not url:
-            continue
-        try:
-            if _download_file(url, path):
-                downloaded.append(name)
-                log.info("downloaded offline cover bundle %s", name)
-        except Exception as exc:  # noqa: BLE001
-            log.warning("could not download cover bundle %s: %s", name, exc)
-    return downloaded
+    url = (url or config.COVERS_BUNDLE_URL or "").strip()
+    if not url:
+        raise ValueError("no cover bundle URL configured")
+    config.COVERS_DIR.mkdir(parents=True, exist_ok=True)
+    _download_file(url, _BUNDLE_ZIP, timeout=3600,
+                   progress_cb=progress_cb, cancel_event=cancel_event)
+    inner = ("libretro-covers.zip", "openvgdb-covers.zip")
+    with zipfile.ZipFile(_BUNDLE_ZIP) as z:
+        missing = [n for n in inner if n not in z.namelist()]
+    if missing:
+        raise ValueError(f"cover bundle is missing: {', '.join(missing)}")
+    with zipfile.ZipFile(_BUNDLE_ZIP) as z:
+        for name in inner:
+            out = config.COVERS_DIR / name
+            tmp = out.with_name(out.name + ".tmp")
+            with z.open(name) as src, open(tmp, "wb") as dst:
+                while True:
+                    chunk = src.read(1 << 20)
+                    if not chunk:
+                        break
+                    dst.write(chunk)
+            os.replace(tmp, out)
+    # Invalidate cached namelists so lookups pick up the freshly extracted zips.
+    global _libretro_names, _ovgdb_names
+    _libretro_names = None
+    _ovgdb_names = None
+    return config.COVERS_DIR
 
 
 def cover_url(name, kind="Named_Boxarts", system="games"):
@@ -482,9 +520,10 @@ def _ovgdb_name_list():
     global _ovgdb_names
     if _ovgdb_names is None:
         _ovgdb_names = set()
-        if _OVGDB_ZIP.exists():
+        path = _bundle_zip_path("openvgdb-covers.zip")
+        if path.exists():
             try:
-                with zipfile.ZipFile(_OVGDB_ZIP) as z:
+                with zipfile.ZipFile(path) as z:
                     _ovgdb_names = {n[:-4] for n in z.namelist() if n.endswith(".jpg")}
             except Exception:  # noqa: BLE001
                 pass
@@ -535,7 +574,7 @@ def fetch_openvgdb_local(name):
     if entry is None:
         return None
     try:
-        with zipfile.ZipFile(_OVGDB_ZIP) as z:
+        with zipfile.ZipFile(_bundle_zip_path("openvgdb-covers.zip")) as z:
             data = z.read(entry)
         return data if _is_image(data) else None
     except Exception:  # noqa: BLE001
@@ -559,7 +598,7 @@ def fetch_libretro_local(name):
     if entry is None:
         return None
     try:
-        with zipfile.ZipFile(_LIBRETRO_ZIP) as z:
+        with zipfile.ZipFile(_bundle_zip_path("libretro-covers.zip")) as z:
             data = z.read(entry)
         return data if _is_image(data) else None
     except Exception:  # noqa: BLE001
@@ -570,9 +609,10 @@ def _libretro_name_list():
     global _libretro_names
     if _libretro_names is None:
         _libretro_names = set()
-        if _LIBRETRO_ZIP.exists():
+        path = _bundle_zip_path("libretro-covers.zip")
+        if path.exists():
             try:
-                with zipfile.ZipFile(_LIBRETRO_ZIP) as z:
+                with zipfile.ZipFile(path) as z:
                     _libretro_names = {n[:-4] for n in z.namelist() if n.endswith(".png")}
             except Exception:  # noqa: BLE001
                 pass

app/gui.py

@@ -1143,6 +1143,7 @@ class ScrapeDialog(QDialog):
 
 class SettingsPage(QWidget):
     saved = Signal()
+    download_covers = Signal()
 
     def __init__(self, games_dir, videos_dir, store_dir, ppsspp_dir, cloud_dir, ppsspp_exe,
                  store_server_url, store_token, store_archive_id, store_archive_sig, store_archive_user,
@@ -1310,6 +1311,9 @@ class SettingsPage(QWidget):
         self.clear_covers_btn = QPushButton("Clear cover cache (re-fetch all covers)")
         self.clear_covers_btn.clicked.connect(self._clear_cover_cache)
         form.addRow(self.clear_covers_btn)
+        self.download_bundle_btn = QPushButton("Download offline cover bundle…")
+        self.download_bundle_btn.clicked.connect(self.download_covers.emit)
+        form.addRow(self.download_bundle_btn)
         col.addWidget(box)
 
         # -- Storage ---------------------------------------------------------- #
@@ -3757,6 +3761,35 @@ class LibraryWorker(QObject):
         self.finished.emit()
 
 
+class CoversBundleWorker(QObject):
+    """Download + extract the offline cover bundle off the GUI thread."""
+    progress = Signal(int, int)   # done bytes, total bytes (0 = unknown)
+    succeeded = Signal()
+    failed = Signal(str)
+    finished = Signal()
+
+    def __init__(self):
+        super().__init__()
+        self._cancel = threading.Event()
+
+    def cancel(self):
+        self._cancel.set()
+
+    def run(self):
+        try:
+            covers.download_covers_bundle(
+                progress_cb=lambda d, t: self.progress.emit(d, t),
+                cancel_event=self._cancel)
+        except InterruptedError:
+            pass
+        except Exception as exc:  # noqa: BLE001
+            self.failed.emit(str(exc))
+        else:
+            self.succeeded.emit()
+        finally:
+            self.finished.emit()
+
+
 class SaveSyncPlanWorker(QObject):
     """Compute the save-sync plan off the GUI thread (content hashing is I/O-heavy)."""
     plan = Signal(list, bool)  # plan, switch_to_queue
@@ -4615,6 +4648,7 @@ class MainWindow(QMainWindow):
                                           self._ps1_dir, self._ps1_output_dir, self._ps1_base_pbp,
                                           self._nps_dir)
         self.settings_page.saved.connect(self._on_settings_saved)
+        self.settings_page.download_covers.connect(self._download_covers_bundle)
         return self.settings_page
 
     def _on_settings_saved(self):
@@ -5687,6 +5721,79 @@ class MainWindow(QMainWindow):
         QMessageBox.information(self, "Done", f"Renamed {renamed} file(s).")
         self.load_library()
 
+    def maybe_offer_covers_bundle(self):
+        """Offer to download the offline cover bundle once, if it's missing."""
+        if self._closing or covers.covers_bundle_present():
+            return
+        s = QSettings("psppt", "psppt")
+        if str(s.value("covers_bundle_skip", "false")).lower() == "true":
+            return
+        box = QMessageBox(self)
+        box.setWindowTitle("Offline covers")
+        box.setIcon(QMessageBox.Question)
+        box.setText(
+            "The offline cover bundle (~843 MB) isn't installed.\n\n"
+            "Download it now from the project repo? Covers keep working online "
+            "without it — the bundle just makes them instant and offline."
+        )
+        download = box.addButton("Download", QMessageBox.AcceptRole)
+        box.addButton("Not now", QMessageBox.RejectRole)
+        never = box.addButton("Don't ask again", QMessageBox.DestructiveRole)
+        box.setDefaultButton(download)
+        box.exec()
+        clicked = box.clickedButton()
+        if clicked is never:
+            s.setValue("covers_bundle_skip", "true")
+        elif clicked is download:
+            self._download_covers_bundle()
+
+    def _download_covers_bundle(self):
+        """Download + extract the offline cover bundle with a progress dialog."""
+        if self._closing or getattr(self, "_bundle_worker", None) is not None:
+            return
+        prog = QProgressDialog("Downloading cover bundle…", "Cancel", 0, 0, self)
+        prog.setWindowTitle("Offline covers")
+        prog.setWindowModality(Qt.WindowModal)
+        prog.setMinimumDuration(0)
+        prog.setAutoClose(False)
+        prog.setAutoReset(False)
+
+        result = {"ok": False, "err": ""}
+        thread = QThread(self)
+        worker = CoversBundleWorker()
+        worker.moveToThread(thread)
+        thread.started.connect(worker.run)
+        worker.progress.connect(lambda d, t: self._on_bundle_progress(prog, d, t))
+        worker.succeeded.connect(lambda: result.update(ok=True))
+        worker.failed.connect(lambda e: result.update(err=str(e)))
+        prog.canceled.connect(worker.cancel)
+        worker.finished.connect(thread.quit)
+        thread.finished.connect(lambda: self._on_bundle_done(prog, result, worker, thread))
+        self._bundle_worker = worker
+        self._bundle_thread = thread
+        thread.start()
+
+    @staticmethod
+    def _on_bundle_progress(prog, done, total):
+        if total > 0:
+            prog.setRange(0, total)
+            prog.setValue(done)
+            prog.setLabelText(
+                f"Downloading cover bundle… {done / (1 << 20):.0f} / {total / (1 << 20):.0f} MB")
+        else:
+            prog.setRange(0, 0)
+
+    def _on_bundle_done(self, prog, result, worker, thread):
+        prog.close()
+        self._bundle_worker = None
+        self._bundle_thread = None
+        if result["ok"]:
+            QMessageBox.information(self, "Offline covers", "Cover bundle installed.")
+            covers.clear_miss_cache()
+            self.load_library()
+        elif result["err"]:
+            QMessageBox.warning(self, "Offline covers", f"Download failed:\n{result['err']}")
+
     def closeEvent(self, event):
         self._closing = True
         self.settings.setValue("window_geometry", self.saveGeometry())
@@ -5695,6 +5802,11 @@ class MainWindow(QMainWindow):
                 self.worker.choose(None)
             self.worker.cancel()
         self._thread_cleanup()
+        if getattr(self, "_bundle_worker", None) is not None:
+            self._bundle_worker.cancel()
+            if getattr(self, "_bundle_thread", None) is not None and _thread_running(self._bundle_thread):
+                self._bundle_thread.quit()
+                self._bundle_thread.wait()
         self.cover_grid.shutdown()
         for t in self._meta_threads:
             if _thread_running(t):
@@ -5717,15 +5829,6 @@ class MainWindow(QMainWindow):
         event.accept()
 
 
-def _start_bundle_check(window):
-    """Download any missing offline cover bundles in the background."""
-    server = getattr(window, "_store_server_url", "") or ""
-    try:
-        covers.ensure_cover_bundles(server)
-    except Exception:  # noqa: BLE001 - background best-effort
-        pass
-
-
 def main():
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(name)-14s %(message)s")
     init_db()  # migrate legacy on-share DB/covers to local storage, then schema
@@ -5746,7 +5849,7 @@ def main():
     window.setWindowIcon(app.windowIcon())
     window.show()
     log.info("main window ready")
-    threading.Thread(target=_start_bundle_check, args=(window,), daemon=True).start()
+    QTimer.singleShot(1200, window.maybe_offer_covers_bundle)
 
     # Ctrl+C should close the window gracefully (running closeEvent's thread
     # cleanup) rather than tearing Qt down mid-thread, which segfaults.

psppt.spec

@@ -1,8 +1,8 @@
 # -*- mode: python ; coding: utf-8 -*-
 """PyInstaller spec: build psppt (onedir).
 
-Bundles the read-only resources (DATs, PS1 loader, offline cover bundles,
-icons) under ``lib/`` next to the executable. User-writable data
+Bundles the read-only resources (DATs, PS1 loader, cover templates, index
+catalogs, icons) under ``lib/`` next to the executable. User-writable data
 (credentials, manual covers, PS1 templates, NPS cache, library DB, cover
 cache) lives in the OS state dir at runtime — see ``app/config.py``.
 
@@ -15,8 +15,6 @@ datas = [
     ("data/dats", "data/dats"),
     ("data/loader", "data/loader"),
     ("data/supplement.json", "data"),
-    ("data/covers/libretro-covers.zip", "data/covers"),
-    ("data/covers/openvgdb-covers.zip", "data/covers"),
     ("data/covers/libretro-index.json", "data/covers"),
     ("data/covers/openvgdb-index.json", "data/covers"),
     ("data/covers/UMD_COVER_TEMPLATE.png", "data/covers"),

scripts/make-covers-bundle.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+"""Package the offline cover zips into a single distributable bundle.
+
+Outputs ``psppt-covers.zip`` containing ``libretro-covers.zip`` and
+``openvgdb-covers.zip``. Attach it to the Gitea ``covers`` release; the app
+downloads it on first run when the user opts in.
+
+Usage:  python3 scripts/make-covers-bundle.py [out-path]
+"""
+import sys
+import zipfile
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+COVERS = ROOT / "data" / "covers"
+INNER = ("libretro-covers.zip", "openvgdb-covers.zip")
+
+
+def main(argv):
+    out = Path(argv[1]) if len(argv) > 1 else ROOT / "psppt-covers.zip"
+    missing = [n for n in INNER if not (COVERS / n).exists()]
+    if missing:
+        print(f"missing: {', '.join(missing)} in {COVERS}", file=sys.stderr)
+        return 2
+    # Inner zips are already compressed; store them to avoid recompressing.
+    with zipfile.ZipFile(out, "w", zipfile.ZIP_STORED) as z:
+        for n in INNER:
+            print(f"adding {n} …")
+            z.write(COVERS / n, arcname=n)
+    size = out.stat().st_size / (1 << 20)
+    print(f"wrote {out} ({size:.0f} MB)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))