diff --git a/.gitignore b/.gitignore index 1d95b67..d428c11 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,8 @@ data/nps/*.tsv # PyInstaller output build/ -dist/ \ No newline at end of file +dist/ + +# Offline cover bundles (large; fetched from the project release on first run) +data/covers/*.zip +/psppt-covers.zip \ No newline at end of file diff --git a/README.md b/README.md index bd8b12e..5880218 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,10 @@ pyinstaller --clean --noconfirm psppt.spec Notes: -- The build bundles the DATs, PS1 loader, and the offline cover bundles. The - cover zips are large (~850 MB), so builds are slow and the output is big. +- The build bundles the DATs, PS1 loader, cover templates, and index catalogs + (~80 MB total). The large offline cover zips (~850 MB) are **not** bundled — + the app fetches them on first run (see + [Offline bundles](#offline-bundles--auto-download)). - `7z` (p7zip-full on Linux, 7-Zip on Windows) is a **runtime** dependency for reading store archives — it is not bundled. - User data (credentials, manual covers, PS1 templates, NPS cache, library DB, @@ -170,9 +172,10 @@ Discs are matched by hash first (most precise), then serial, then filename. - `data/credentials.json` — cover/metadata API keys (IGDB, TheMovieDB, ScreenScraper, Google Custom Search). Environment variables take precedence. - `data/manual_covers/games|videos/` — your own cover images. -- `data/covers/` — checked-in assets: the UMD video/game cover templates, - `libretro-index.json` + `openvgdb-index.json` catalogs, `libretro-covers.zip` + - `openvgdb-covers.zip` offline cover bundles, and the legacy cover cache. +- `data/covers/` — checked-in assets: the UMD video/game cover templates and + the `libretro-index.json` + `openvgdb-index.json` catalogs. The large + `libretro-covers.zip` + `openvgdb-covers.zip` bundles are git-ignored and + distributed as `psppt-covers.zip` (see above). - `data/base.PBP` — a PSN PS1 classic EBOOT (any game) supplying `STARTDAT` + a base `PARAM.SFO`/icon for PS1 conversion (git-ignored, copyrighted). - `data/loader/DATA.PSP` + `data/loader/KEYS.BIN` — bundled plain-ELF POPS @@ -223,6 +226,7 @@ QSettings under `psppt`) taking precedence where noted. | `PSP_CREDENTIALS` | `$PSP_USER_DATA_DIR/credentials.json` | credentials file | | `PSP_LIBRETRO_COVERS_URL` | *(empty)* | direct URL for `libretro-covers.zip` (else derived from store server) | | `PSP_OVGDB_COVERS_URL` | *(empty)* | direct URL for `openvgdb-covers.zip` (else derived from store server) | +| `PSP_COVERS_BUNDLE_URL` | Gitea `covers` release asset | combined `psppt-covers.zip` download URL | Cover/metadata keys can also be set as env vars: `IGDB_CLIENT_ID`, `IGDB_CLIENT_SECRET`, `TMDB_API_KEY`, `GOOGLE_API_KEY`, `GOOGLE_CX`, @@ -255,12 +259,23 @@ Manual covers in `data/manual_covers/videos/` always win. ### Offline bundles & auto-download -The two bundles (`libretro-covers.zip`, `openvgdb-covers.zip`) are shipped in -`data/covers/`, so a fresh install works with **no internet and no API keys**. -If either is missing at startup, the app downloads it from the store server's -`/api/covers/.zip` endpoint (see [Server setup](#server-setup)). +The two bundles (`libretro-covers.zip`, `openvgdb-covers.zip`) are packaged +into one `psppt-covers.zip` (~850 MB) and attached to the repo's `covers` +release — **not** bundled into the app binary. -Rebuild them yourself: +On first launch, if the bundle isn't installed, the app asks whether to +download it (with a progress bar). Covers keep working online via libretro / +OpenVGDB / ScreenScraper / IGDB without it — the bundle just makes them instant +and offline. "Don't ask again" is remembered, and you can re-download from +**Settings → Cover sources → Download offline cover bundle…**. + +Build the bundle yourself: + +```bash +python3 scripts/make-covers-bundle.py # -> psppt-covers.zip +``` + +Rebuild the inner zips: ```bash python3 -m app.libretro # libretro-thumbnails -> libretro-covers.zip diff --git a/app/config.py b/app/config.py index e12d282..d9e50f2 100644 --- a/app/config.py +++ b/app/config.py @@ -76,6 +76,13 @@ STORE_TOKEN = os.environ.get("PSP_STORE_TOKEN", "") LIBRETRO_COVERS_URL = os.environ.get("PSP_LIBRETRO_COVERS_URL", "") OVGDB_COVERS_URL = os.environ.get("PSP_OVGDB_COVERS_URL", "") +# Combined offline cover bundle (both inner zips) fetched on first run when the +# user opts in. Defaults to the project's Gitea release asset. +COVERS_BUNDLE_URL = os.environ.get( + "PSP_COVERS_BUNDLE_URL", + "https://git.ceaf.dev/quoije/psp-preservation-tool/releases/download/covers/psppt-covers.zip", +) + # archive.org item hosting the same Redump 7z set (identifier or full URL). ARCHIVE_ITEM = os.environ.get("PSP_ARCHIVE_ITEM", "psp-redump-roms321.com") diff --git a/app/covers.py b/app/covers.py index c24e45e..1deec8a 100644 --- a/app/covers.py +++ b/app/covers.py @@ -53,8 +53,20 @@ _TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie" _TMDB_IMAGE = "https://image.tmdb.org/t/p/" _GOOGLE_CSE_API = "https://www.googleapis.com/customsearch/v1" _MANUAL_DIR = config.USER_DATA_DIR / "manual_covers" -_OVGDB_ZIP = config.DATA_DIR / "covers" / "openvgdb-covers.zip" -_LIBRETRO_ZIP = config.DATA_DIR / "covers" / "libretro-covers.zip" +# Offline cover bundles live in the writable cover dir (downloaded on demand), +# not the read-only app bundle. A source checkout keeps them in data/covers/. +_OVGDB_ZIP = config.COVERS_DIR / "openvgdb-covers.zip" +_LIBRETRO_ZIP = config.COVERS_DIR / "libretro-covers.zip" +_BUNDLE_ZIP = config.COVERS_DIR / "psppt-covers.zip" + + +def _bundle_zip_path(name): + """Resolve a cover bundle zip: writable dir first, legacy checkout second.""" + for base in (config.COVERS_DIR, config.LEGACY_COVERS_DIR): + p = base / name + if p.exists(): + return p + return config.COVERS_DIR / name _WIKI_SEARCH = "https://en.wikipedia.org/w/api.php" _WIKI_SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/" @@ -158,48 +170,74 @@ def clear_cover_cache(): pass -def _download_file(url, dest, timeout=600): - """Stream `url` to `dest` (atomic); returns True on success.""" +def _download_file(url, dest, timeout=600, progress_cb=None, cancel_event=None): + """Stream `url` to `dest` (atomic); returns True on success. + + ``progress_cb(done, total)`` is called with bytes so far (total is 0 when + the server doesn't report a content length). ``cancel_event`` aborts the + download when set. + """ req = urllib.request.Request(url, headers=_UA) with urllib.request.urlopen(req, timeout=timeout) as resp: if resp.status != 200: return False + total = int(resp.headers.get("Content-Length") or 0) tmp = dest.with_name(dest.name + ".tmp") + done = 0 with open(tmp, "wb") as f: while True: + if cancel_event is not None and cancel_event.is_set(): + raise InterruptedError("download cancelled") chunk = resp.read(1 << 20) if not chunk: break f.write(chunk) + done += len(chunk) + if progress_cb is not None: + progress_cb(done, total) os.replace(tmp, dest) return True -def ensure_cover_bundles(server_url=""): - """Download missing offline cover bundle zips from the store server. +def covers_bundle_present(): + """True when the offline cover bundle is installed (both inner zips).""" + return (_bundle_zip_path("libretro-covers.zip").exists() + and _bundle_zip_path("openvgdb-covers.zip").exists()) - ``server_url`` is the base URL (e.g. ``http://192.168.10.100:8000``); the - zips are fetched from ``/api/covers/.zip`` unless a direct URL - is set in config. Returns the names of bundles downloaded. + +def download_covers_bundle(url=None, progress_cb=None, cancel_event=None): + """Download and extract the combined cover bundle into the cover dir. + + ``url`` defaults to ``config.COVERS_BUNDLE_URL``. Raises on failure; the + caller decides whether to surface the error. """ - base = (server_url or config.STORE_SERVER_URL or "").rstrip("/") - downloaded = [] - for name, path, direct in ( - ("libretro-covers.zip", _LIBRETRO_ZIP, config.LIBRETRO_COVERS_URL), - ("openvgdb-covers.zip", _OVGDB_ZIP, config.OVGDB_COVERS_URL), - ): - if path.exists(): - continue - url = direct or (f"{base}/api/covers/{name}" if base else "") - if not url: - continue - try: - if _download_file(url, path): - downloaded.append(name) - log.info("downloaded offline cover bundle %s", name) - except Exception as exc: # noqa: BLE001 - log.warning("could not download cover bundle %s: %s", name, exc) - return downloaded + url = (url or config.COVERS_BUNDLE_URL or "").strip() + if not url: + raise ValueError("no cover bundle URL configured") + config.COVERS_DIR.mkdir(parents=True, exist_ok=True) + _download_file(url, _BUNDLE_ZIP, timeout=3600, + progress_cb=progress_cb, cancel_event=cancel_event) + inner = ("libretro-covers.zip", "openvgdb-covers.zip") + with zipfile.ZipFile(_BUNDLE_ZIP) as z: + missing = [n for n in inner if n not in z.namelist()] + if missing: + raise ValueError(f"cover bundle is missing: {', '.join(missing)}") + with zipfile.ZipFile(_BUNDLE_ZIP) as z: + for name in inner: + out = config.COVERS_DIR / name + tmp = out.with_name(out.name + ".tmp") + with z.open(name) as src, open(tmp, "wb") as dst: + while True: + chunk = src.read(1 << 20) + if not chunk: + break + dst.write(chunk) + os.replace(tmp, out) + # Invalidate cached namelists so lookups pick up the freshly extracted zips. + global _libretro_names, _ovgdb_names + _libretro_names = None + _ovgdb_names = None + return config.COVERS_DIR def cover_url(name, kind="Named_Boxarts", system="games"): @@ -482,9 +520,10 @@ def _ovgdb_name_list(): global _ovgdb_names if _ovgdb_names is None: _ovgdb_names = set() - if _OVGDB_ZIP.exists(): + path = _bundle_zip_path("openvgdb-covers.zip") + if path.exists(): try: - with zipfile.ZipFile(_OVGDB_ZIP) as z: + with zipfile.ZipFile(path) as z: _ovgdb_names = {n[:-4] for n in z.namelist() if n.endswith(".jpg")} except Exception: # noqa: BLE001 pass @@ -535,7 +574,7 @@ def fetch_openvgdb_local(name): if entry is None: return None try: - with zipfile.ZipFile(_OVGDB_ZIP) as z: + with zipfile.ZipFile(_bundle_zip_path("openvgdb-covers.zip")) as z: data = z.read(entry) return data if _is_image(data) else None except Exception: # noqa: BLE001 @@ -559,7 +598,7 @@ def fetch_libretro_local(name): if entry is None: return None try: - with zipfile.ZipFile(_LIBRETRO_ZIP) as z: + with zipfile.ZipFile(_bundle_zip_path("libretro-covers.zip")) as z: data = z.read(entry) return data if _is_image(data) else None except Exception: # noqa: BLE001 @@ -570,9 +609,10 @@ def _libretro_name_list(): global _libretro_names if _libretro_names is None: _libretro_names = set() - if _LIBRETRO_ZIP.exists(): + path = _bundle_zip_path("libretro-covers.zip") + if path.exists(): try: - with zipfile.ZipFile(_LIBRETRO_ZIP) as z: + with zipfile.ZipFile(path) as z: _libretro_names = {n[:-4] for n in z.namelist() if n.endswith(".png")} except Exception: # noqa: BLE001 pass diff --git a/app/gui.py b/app/gui.py index 385d6fd..270ad30 100644 --- a/app/gui.py +++ b/app/gui.py @@ -1143,6 +1143,7 @@ class ScrapeDialog(QDialog): class SettingsPage(QWidget): saved = Signal() + download_covers = Signal() def __init__(self, games_dir, videos_dir, store_dir, ppsspp_dir, cloud_dir, ppsspp_exe, store_server_url, store_token, store_archive_id, store_archive_sig, store_archive_user, @@ -1310,6 +1311,9 @@ class SettingsPage(QWidget): self.clear_covers_btn = QPushButton("Clear cover cache (re-fetch all covers)") self.clear_covers_btn.clicked.connect(self._clear_cover_cache) form.addRow(self.clear_covers_btn) + self.download_bundle_btn = QPushButton("Download offline cover bundle…") + self.download_bundle_btn.clicked.connect(self.download_covers.emit) + form.addRow(self.download_bundle_btn) col.addWidget(box) # -- Storage ---------------------------------------------------------- # @@ -3757,6 +3761,35 @@ class LibraryWorker(QObject): self.finished.emit() +class CoversBundleWorker(QObject): + """Download + extract the offline cover bundle off the GUI thread.""" + progress = Signal(int, int) # done bytes, total bytes (0 = unknown) + succeeded = Signal() + failed = Signal(str) + finished = Signal() + + def __init__(self): + super().__init__() + self._cancel = threading.Event() + + def cancel(self): + self._cancel.set() + + def run(self): + try: + covers.download_covers_bundle( + progress_cb=lambda d, t: self.progress.emit(d, t), + cancel_event=self._cancel) + except InterruptedError: + pass + except Exception as exc: # noqa: BLE001 + self.failed.emit(str(exc)) + else: + self.succeeded.emit() + finally: + self.finished.emit() + + class SaveSyncPlanWorker(QObject): """Compute the save-sync plan off the GUI thread (content hashing is I/O-heavy).""" plan = Signal(list, bool) # plan, switch_to_queue @@ -4615,6 +4648,7 @@ class MainWindow(QMainWindow): self._ps1_dir, self._ps1_output_dir, self._ps1_base_pbp, self._nps_dir) self.settings_page.saved.connect(self._on_settings_saved) + self.settings_page.download_covers.connect(self._download_covers_bundle) return self.settings_page def _on_settings_saved(self): @@ -5687,6 +5721,79 @@ class MainWindow(QMainWindow): QMessageBox.information(self, "Done", f"Renamed {renamed} file(s).") self.load_library() + def maybe_offer_covers_bundle(self): + """Offer to download the offline cover bundle once, if it's missing.""" + if self._closing or covers.covers_bundle_present(): + return + s = QSettings("psppt", "psppt") + if str(s.value("covers_bundle_skip", "false")).lower() == "true": + return + box = QMessageBox(self) + box.setWindowTitle("Offline covers") + box.setIcon(QMessageBox.Question) + box.setText( + "The offline cover bundle (~843 MB) isn't installed.\n\n" + "Download it now from the project repo? Covers keep working online " + "without it — the bundle just makes them instant and offline." + ) + download = box.addButton("Download", QMessageBox.AcceptRole) + box.addButton("Not now", QMessageBox.RejectRole) + never = box.addButton("Don't ask again", QMessageBox.DestructiveRole) + box.setDefaultButton(download) + box.exec() + clicked = box.clickedButton() + if clicked is never: + s.setValue("covers_bundle_skip", "true") + elif clicked is download: + self._download_covers_bundle() + + def _download_covers_bundle(self): + """Download + extract the offline cover bundle with a progress dialog.""" + if self._closing or getattr(self, "_bundle_worker", None) is not None: + return + prog = QProgressDialog("Downloading cover bundle…", "Cancel", 0, 0, self) + prog.setWindowTitle("Offline covers") + prog.setWindowModality(Qt.WindowModal) + prog.setMinimumDuration(0) + prog.setAutoClose(False) + prog.setAutoReset(False) + + result = {"ok": False, "err": ""} + thread = QThread(self) + worker = CoversBundleWorker() + worker.moveToThread(thread) + thread.started.connect(worker.run) + worker.progress.connect(lambda d, t: self._on_bundle_progress(prog, d, t)) + worker.succeeded.connect(lambda: result.update(ok=True)) + worker.failed.connect(lambda e: result.update(err=str(e))) + prog.canceled.connect(worker.cancel) + worker.finished.connect(thread.quit) + thread.finished.connect(lambda: self._on_bundle_done(prog, result, worker, thread)) + self._bundle_worker = worker + self._bundle_thread = thread + thread.start() + + @staticmethod + def _on_bundle_progress(prog, done, total): + if total > 0: + prog.setRange(0, total) + prog.setValue(done) + prog.setLabelText( + f"Downloading cover bundle… {done / (1 << 20):.0f} / {total / (1 << 20):.0f} MB") + else: + prog.setRange(0, 0) + + def _on_bundle_done(self, prog, result, worker, thread): + prog.close() + self._bundle_worker = None + self._bundle_thread = None + if result["ok"]: + QMessageBox.information(self, "Offline covers", "Cover bundle installed.") + covers.clear_miss_cache() + self.load_library() + elif result["err"]: + QMessageBox.warning(self, "Offline covers", f"Download failed:\n{result['err']}") + def closeEvent(self, event): self._closing = True self.settings.setValue("window_geometry", self.saveGeometry()) @@ -5695,6 +5802,11 @@ class MainWindow(QMainWindow): self.worker.choose(None) self.worker.cancel() self._thread_cleanup() + if getattr(self, "_bundle_worker", None) is not None: + self._bundle_worker.cancel() + if getattr(self, "_bundle_thread", None) is not None and _thread_running(self._bundle_thread): + self._bundle_thread.quit() + self._bundle_thread.wait() self.cover_grid.shutdown() for t in self._meta_threads: if _thread_running(t): @@ -5717,15 +5829,6 @@ class MainWindow(QMainWindow): event.accept() -def _start_bundle_check(window): - """Download any missing offline cover bundles in the background.""" - server = getattr(window, "_store_server_url", "") or "" - try: - covers.ensure_cover_bundles(server) - except Exception: # noqa: BLE001 - background best-effort - pass - - def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(name)-14s %(message)s") init_db() # migrate legacy on-share DB/covers to local storage, then schema @@ -5746,7 +5849,7 @@ def main(): window.setWindowIcon(app.windowIcon()) window.show() log.info("main window ready") - threading.Thread(target=_start_bundle_check, args=(window,), daemon=True).start() + QTimer.singleShot(1200, window.maybe_offer_covers_bundle) # Ctrl+C should close the window gracefully (running closeEvent's thread # cleanup) rather than tearing Qt down mid-thread, which segfaults. diff --git a/data/covers/libretro-covers.zip b/data/covers/libretro-covers.zip deleted file mode 100644 index 620b7fa..0000000 Binary files a/data/covers/libretro-covers.zip and /dev/null differ diff --git a/data/covers/openvgdb-covers.zip b/data/covers/openvgdb-covers.zip deleted file mode 100644 index 6ca8b9b..0000000 Binary files a/data/covers/openvgdb-covers.zip and /dev/null differ diff --git a/psppt.spec b/psppt.spec index aff27d0..e77bef4 100644 --- a/psppt.spec +++ b/psppt.spec @@ -1,8 +1,8 @@ # -*- mode: python ; coding: utf-8 -*- """PyInstaller spec: build psppt (onedir). -Bundles the read-only resources (DATs, PS1 loader, offline cover bundles, -icons) under ``lib/`` next to the executable. User-writable data +Bundles the read-only resources (DATs, PS1 loader, cover templates, index +catalogs, icons) under ``lib/`` next to the executable. User-writable data (credentials, manual covers, PS1 templates, NPS cache, library DB, cover cache) lives in the OS state dir at runtime — see ``app/config.py``. @@ -15,8 +15,6 @@ datas = [ ("data/dats", "data/dats"), ("data/loader", "data/loader"), ("data/supplement.json", "data"), - ("data/covers/libretro-covers.zip", "data/covers"), - ("data/covers/openvgdb-covers.zip", "data/covers"), ("data/covers/libretro-index.json", "data/covers"), ("data/covers/openvgdb-index.json", "data/covers"), ("data/covers/UMD_COVER_TEMPLATE.png", "data/covers"), diff --git a/scripts/make-covers-bundle.py b/scripts/make-covers-bundle.py new file mode 100644 index 0000000..0e7a4af --- /dev/null +++ b/scripts/make-covers-bundle.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Package the offline cover zips into a single distributable bundle. + +Outputs ``psppt-covers.zip`` containing ``libretro-covers.zip`` and +``openvgdb-covers.zip``. Attach it to the Gitea ``covers`` release; the app +downloads it on first run when the user opts in. + +Usage: python3 scripts/make-covers-bundle.py [out-path] +""" +import sys +import zipfile +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +COVERS = ROOT / "data" / "covers" +INNER = ("libretro-covers.zip", "openvgdb-covers.zip") + + +def main(argv): + out = Path(argv[1]) if len(argv) > 1 else ROOT / "psppt-covers.zip" + missing = [n for n in INNER if not (COVERS / n).exists()] + if missing: + print(f"missing: {', '.join(missing)} in {COVERS}", file=sys.stderr) + return 2 + # Inner zips are already compressed; store them to avoid recompressing. + with zipfile.ZipFile(out, "w", zipfile.ZIP_STORED) as z: + for n in INNER: + print(f"adding {n} …") + z.write(COVERS / n, arcname=n) + size = out.stat().st_size / (1 << 20) + print(f"wrote {out} ({size:.0f} MB)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv))