Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
14b0b955f5 | ||
|
|
b378214e9b | ||
|
|
0110b50a07 | ||
|
|
76eb6a6ef4 | ||
|
|
58d348859a | ||
|
|
3359ecf131 | ||
|
|
38a3859a21 | ||
|
|
96e876b8ec |
@@ -24,3 +24,7 @@ data/nps/*.tsv
|
||||
# PyInstaller output
|
||||
build/
|
||||
dist/
|
||||
|
||||
# Offline cover bundles (large; fetched from the project release on first run)
|
||||
data/covers/*.zip
|
||||
/psppt-covers.zip
|
||||
@@ -79,8 +79,10 @@ pyinstaller --clean --noconfirm psppt.spec
|
||||
|
||||
Notes:
|
||||
|
||||
- The build bundles the DATs, PS1 loader, and the offline cover bundles. The
|
||||
cover zips are large (~850 MB), so builds are slow and the output is big.
|
||||
- The build bundles the DATs, PS1 loader, cover templates, and index catalogs
|
||||
(~80 MB total). The large offline cover zips (~850 MB) are **not** bundled —
|
||||
the app fetches them on first run (see
|
||||
[Offline bundles](#offline-bundles--auto-download)).
|
||||
- `7z` (p7zip-full on Linux, 7-Zip on Windows) is a **runtime** dependency for
|
||||
reading store archives — it is not bundled.
|
||||
- User data (credentials, manual covers, PS1 templates, NPS cache, library DB,
|
||||
@@ -170,9 +172,10 @@ Discs are matched by hash first (most precise), then serial, then filename.
|
||||
- `data/credentials.json` — cover/metadata API keys (IGDB, TheMovieDB,
|
||||
ScreenScraper, Google Custom Search). Environment variables take precedence.
|
||||
- `data/manual_covers/games|videos/` — your own cover images.
|
||||
- `data/covers/` — checked-in assets: the UMD video/game cover templates,
|
||||
`libretro-index.json` + `openvgdb-index.json` catalogs, `libretro-covers.zip` +
|
||||
`openvgdb-covers.zip` offline cover bundles, and the legacy cover cache.
|
||||
- `data/covers/` — checked-in assets: the UMD video/game cover templates and
|
||||
the `libretro-index.json` + `openvgdb-index.json` catalogs. The large
|
||||
`libretro-covers.zip` + `openvgdb-covers.zip` bundles are git-ignored and
|
||||
distributed as `psppt-covers.zip` (see above).
|
||||
- `data/base.PBP` — a PSN PS1 classic EBOOT (any game) supplying `STARTDAT`
|
||||
+ a base `PARAM.SFO`/icon for PS1 conversion (git-ignored, copyrighted).
|
||||
- `data/loader/DATA.PSP` + `data/loader/KEYS.BIN` — bundled plain-ELF POPS
|
||||
@@ -223,6 +226,7 @@ QSettings under `psppt`) taking precedence where noted.
|
||||
| `PSP_CREDENTIALS` | `$PSP_USER_DATA_DIR/credentials.json` | credentials file |
|
||||
| `PSP_LIBRETRO_COVERS_URL` | *(empty)* | direct URL for `libretro-covers.zip` (else derived from store server) |
|
||||
| `PSP_OVGDB_COVERS_URL` | *(empty)* | direct URL for `openvgdb-covers.zip` (else derived from store server) |
|
||||
| `PSP_COVERS_BUNDLE_URL` | Gitea `covers` release asset | combined `psppt-covers.zip` download URL |
|
||||
|
||||
Cover/metadata keys can also be set as env vars: `IGDB_CLIENT_ID`,
|
||||
`IGDB_CLIENT_SECRET`, `TMDB_API_KEY`, `GOOGLE_API_KEY`, `GOOGLE_CX`,
|
||||
@@ -255,12 +259,23 @@ Manual covers in `data/manual_covers/videos/` always win.
|
||||
|
||||
### Offline bundles & auto-download
|
||||
|
||||
The two bundles (`libretro-covers.zip`, `openvgdb-covers.zip`) are shipped in
|
||||
`data/covers/`, so a fresh install works with **no internet and no API keys**.
|
||||
If either is missing at startup, the app downloads it from the store server's
|
||||
`/api/covers/<name>.zip` endpoint (see [Server setup](#server-setup)).
|
||||
The two bundles (`libretro-covers.zip`, `openvgdb-covers.zip`) are packaged
|
||||
into one `psppt-covers.zip` (~850 MB) and attached to the repo's `covers`
|
||||
release — **not** bundled into the app binary.
|
||||
|
||||
Rebuild them yourself:
|
||||
On first launch, if the bundle isn't installed, the app asks whether to
|
||||
download it (with a progress bar). Covers keep working online via libretro /
|
||||
OpenVGDB / ScreenScraper / IGDB without it — the bundle just makes them instant
|
||||
and offline. "Don't ask again" is remembered, and you can re-download from
|
||||
**Settings → Cover sources → Download offline cover bundle…**.
|
||||
|
||||
Build the bundle yourself:
|
||||
|
||||
```bash
|
||||
python3 scripts/make-covers-bundle.py # -> psppt-covers.zip
|
||||
```
|
||||
|
||||
Rebuild the inner zips:
|
||||
|
||||
```bash
|
||||
python3 -m app.libretro # libretro-thumbnails -> libretro-covers.zip
|
||||
|
||||
@@ -76,6 +76,13 @@ STORE_TOKEN = os.environ.get("PSP_STORE_TOKEN", "")
|
||||
LIBRETRO_COVERS_URL = os.environ.get("PSP_LIBRETRO_COVERS_URL", "")
|
||||
OVGDB_COVERS_URL = os.environ.get("PSP_OVGDB_COVERS_URL", "")
|
||||
|
||||
# Combined offline cover bundle (both inner zips) fetched on first run when the
|
||||
# user opts in. Defaults to the project's Gitea release asset.
|
||||
COVERS_BUNDLE_URL = os.environ.get(
|
||||
"PSP_COVERS_BUNDLE_URL",
|
||||
"https://git.ceaf.dev/quoije/psp-preservation-tool/releases/download/covers/psppt-covers.zip",
|
||||
)
|
||||
|
||||
# archive.org item hosting the same Redump 7z set (identifier or full URL).
|
||||
ARCHIVE_ITEM = os.environ.get("PSP_ARCHIVE_ITEM", "psp-redump-roms321.com")
|
||||
|
||||
|
||||
+72
-32
@@ -53,8 +53,20 @@ _TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie"
|
||||
_TMDB_IMAGE = "https://image.tmdb.org/t/p/"
|
||||
_GOOGLE_CSE_API = "https://www.googleapis.com/customsearch/v1"
|
||||
_MANUAL_DIR = config.USER_DATA_DIR / "manual_covers"
|
||||
_OVGDB_ZIP = config.DATA_DIR / "covers" / "openvgdb-covers.zip"
|
||||
_LIBRETRO_ZIP = config.DATA_DIR / "covers" / "libretro-covers.zip"
|
||||
# Offline cover bundles live in the writable cover dir (downloaded on demand),
|
||||
# not the read-only app bundle. A source checkout keeps them in data/covers/.
|
||||
_OVGDB_ZIP = config.COVERS_DIR / "openvgdb-covers.zip"
|
||||
_LIBRETRO_ZIP = config.COVERS_DIR / "libretro-covers.zip"
|
||||
_BUNDLE_ZIP = config.COVERS_DIR / "psppt-covers.zip"
|
||||
|
||||
|
||||
def _bundle_zip_path(name):
|
||||
"""Resolve a cover bundle zip: writable dir first, legacy checkout second."""
|
||||
for base in (config.COVERS_DIR, config.LEGACY_COVERS_DIR):
|
||||
p = base / name
|
||||
if p.exists():
|
||||
return p
|
||||
return config.COVERS_DIR / name
|
||||
|
||||
_WIKI_SEARCH = "https://en.wikipedia.org/w/api.php"
|
||||
_WIKI_SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/"
|
||||
@@ -158,48 +170,74 @@ def clear_cover_cache():
|
||||
pass
|
||||
|
||||
|
||||
def _download_file(url, dest, timeout=600):
|
||||
"""Stream `url` to `dest` (atomic); returns True on success."""
|
||||
def _download_file(url, dest, timeout=600, progress_cb=None, cancel_event=None):
|
||||
"""Stream `url` to `dest` (atomic); returns True on success.
|
||||
|
||||
``progress_cb(done, total)`` is called with bytes so far (total is 0 when
|
||||
the server doesn't report a content length). ``cancel_event`` aborts the
|
||||
download when set.
|
||||
"""
|
||||
req = urllib.request.Request(url, headers=_UA)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
if resp.status != 200:
|
||||
return False
|
||||
total = int(resp.headers.get("Content-Length") or 0)
|
||||
tmp = dest.with_name(dest.name + ".tmp")
|
||||
done = 0
|
||||
with open(tmp, "wb") as f:
|
||||
while True:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
raise InterruptedError("download cancelled")
|
||||
chunk = resp.read(1 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
done += len(chunk)
|
||||
if progress_cb is not None:
|
||||
progress_cb(done, total)
|
||||
os.replace(tmp, dest)
|
||||
return True
|
||||
|
||||
|
||||
def ensure_cover_bundles(server_url=""):
|
||||
"""Download missing offline cover bundle zips from the store server.
|
||||
def covers_bundle_present():
|
||||
"""True when the offline cover bundle is installed (both inner zips)."""
|
||||
return (_bundle_zip_path("libretro-covers.zip").exists()
|
||||
and _bundle_zip_path("openvgdb-covers.zip").exists())
|
||||
|
||||
``server_url`` is the base URL (e.g. ``http://192.168.10.100:8000``); the
|
||||
zips are fetched from ``<base>/api/covers/<name>.zip`` unless a direct URL
|
||||
is set in config. Returns the names of bundles downloaded.
|
||||
|
||||
def download_covers_bundle(url=None, progress_cb=None, cancel_event=None):
|
||||
"""Download and extract the combined cover bundle into the cover dir.
|
||||
|
||||
``url`` defaults to ``config.COVERS_BUNDLE_URL``. Raises on failure; the
|
||||
caller decides whether to surface the error.
|
||||
"""
|
||||
base = (server_url or config.STORE_SERVER_URL or "").rstrip("/")
|
||||
downloaded = []
|
||||
for name, path, direct in (
|
||||
("libretro-covers.zip", _LIBRETRO_ZIP, config.LIBRETRO_COVERS_URL),
|
||||
("openvgdb-covers.zip", _OVGDB_ZIP, config.OVGDB_COVERS_URL),
|
||||
):
|
||||
if path.exists():
|
||||
continue
|
||||
url = direct or (f"{base}/api/covers/{name}" if base else "")
|
||||
url = (url or config.COVERS_BUNDLE_URL or "").strip()
|
||||
if not url:
|
||||
continue
|
||||
try:
|
||||
if _download_file(url, path):
|
||||
downloaded.append(name)
|
||||
log.info("downloaded offline cover bundle %s", name)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("could not download cover bundle %s: %s", name, exc)
|
||||
return downloaded
|
||||
raise ValueError("no cover bundle URL configured")
|
||||
config.COVERS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
_download_file(url, _BUNDLE_ZIP, timeout=3600,
|
||||
progress_cb=progress_cb, cancel_event=cancel_event)
|
||||
inner = ("libretro-covers.zip", "openvgdb-covers.zip")
|
||||
with zipfile.ZipFile(_BUNDLE_ZIP) as z:
|
||||
missing = [n for n in inner if n not in z.namelist()]
|
||||
if missing:
|
||||
raise ValueError(f"cover bundle is missing: {', '.join(missing)}")
|
||||
with zipfile.ZipFile(_BUNDLE_ZIP) as z:
|
||||
for name in inner:
|
||||
out = config.COVERS_DIR / name
|
||||
tmp = out.with_name(out.name + ".tmp")
|
||||
with z.open(name) as src, open(tmp, "wb") as dst:
|
||||
while True:
|
||||
chunk = src.read(1 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
dst.write(chunk)
|
||||
os.replace(tmp, out)
|
||||
# Invalidate cached namelists so lookups pick up the freshly extracted zips.
|
||||
global _libretro_names, _ovgdb_names
|
||||
_libretro_names = None
|
||||
_ovgdb_names = None
|
||||
return config.COVERS_DIR
|
||||
|
||||
|
||||
def cover_url(name, kind="Named_Boxarts", system="games"):
|
||||
@@ -482,9 +520,10 @@ def _ovgdb_name_list():
|
||||
global _ovgdb_names
|
||||
if _ovgdb_names is None:
|
||||
_ovgdb_names = set()
|
||||
if _OVGDB_ZIP.exists():
|
||||
path = _bundle_zip_path("openvgdb-covers.zip")
|
||||
if path.exists():
|
||||
try:
|
||||
with zipfile.ZipFile(_OVGDB_ZIP) as z:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
_ovgdb_names = {n[:-4] for n in z.namelist() if n.endswith(".jpg")}
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
@@ -535,7 +574,7 @@ def fetch_openvgdb_local(name):
|
||||
if entry is None:
|
||||
return None
|
||||
try:
|
||||
with zipfile.ZipFile(_OVGDB_ZIP) as z:
|
||||
with zipfile.ZipFile(_bundle_zip_path("openvgdb-covers.zip")) as z:
|
||||
data = z.read(entry)
|
||||
return data if _is_image(data) else None
|
||||
except Exception: # noqa: BLE001
|
||||
@@ -559,7 +598,7 @@ def fetch_libretro_local(name):
|
||||
if entry is None:
|
||||
return None
|
||||
try:
|
||||
with zipfile.ZipFile(_LIBRETRO_ZIP) as z:
|
||||
with zipfile.ZipFile(_bundle_zip_path("libretro-covers.zip")) as z:
|
||||
data = z.read(entry)
|
||||
return data if _is_image(data) else None
|
||||
except Exception: # noqa: BLE001
|
||||
@@ -570,9 +609,10 @@ def _libretro_name_list():
|
||||
global _libretro_names
|
||||
if _libretro_names is None:
|
||||
_libretro_names = set()
|
||||
if _LIBRETRO_ZIP.exists():
|
||||
path = _bundle_zip_path("libretro-covers.zip")
|
||||
if path.exists():
|
||||
try:
|
||||
with zipfile.ZipFile(_LIBRETRO_ZIP) as z:
|
||||
with zipfile.ZipFile(path) as z:
|
||||
_libretro_names = {n[:-4] for n in z.namelist() if n.endswith(".png")}
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
+161
-27
@@ -7,6 +7,7 @@ Run with::
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
from pathlib import Path
|
||||
@@ -158,7 +159,9 @@ class CoverLoader(QObject):
|
||||
else:
|
||||
self.failed.emit(self.gen, i, self.items[i]["name"])
|
||||
finally:
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
# Return promptly on cancel/exit instead of blocking the GUI thread
|
||||
# on in-flight network timeouts during shutdown.
|
||||
pool.shutdown(wait=not self._cancel.is_set(), cancel_futures=True)
|
||||
self.finished.emit()
|
||||
|
||||
|
||||
@@ -480,7 +483,7 @@ class CoverShowcase(QWidget):
|
||||
return QIcon(pm)
|
||||
|
||||
def _on_loaded(self, gen, index, image):
|
||||
if gen != self._generation:
|
||||
if self._closing or gen != self._generation:
|
||||
return
|
||||
item = self.list.item(index)
|
||||
if item is not None:
|
||||
@@ -502,7 +505,7 @@ class CoverShowcase(QWidget):
|
||||
self.play_requested.emit(data)
|
||||
|
||||
def _on_failed(self, gen, index, name):
|
||||
if gen != self._generation:
|
||||
if self._closing or gen != self._generation:
|
||||
return
|
||||
if index not in self.missing:
|
||||
self.missing.append(index)
|
||||
@@ -510,6 +513,8 @@ class CoverShowcase(QWidget):
|
||||
self._update_status()
|
||||
|
||||
def _on_loading_done(self):
|
||||
if self._closing:
|
||||
return
|
||||
sender = self.sender()
|
||||
if sender is not None and getattr(sender, "gen", None) != self._generation:
|
||||
return
|
||||
@@ -517,6 +522,8 @@ class CoverShowcase(QWidget):
|
||||
self._update_status()
|
||||
|
||||
def _update_status(self):
|
||||
if self._closing:
|
||||
return
|
||||
if self._loading:
|
||||
self.status.setText("Loading covers…")
|
||||
self.missing_changed.emit(0)
|
||||
@@ -1136,6 +1143,7 @@ class ScrapeDialog(QDialog):
|
||||
|
||||
class SettingsPage(QWidget):
|
||||
saved = Signal()
|
||||
download_covers = Signal()
|
||||
|
||||
def __init__(self, games_dir, videos_dir, store_dir, ppsspp_dir, cloud_dir, ppsspp_exe,
|
||||
store_server_url, store_token, store_archive_id, store_archive_sig, store_archive_user,
|
||||
@@ -1303,6 +1311,9 @@ class SettingsPage(QWidget):
|
||||
self.clear_covers_btn = QPushButton("Clear cover cache (re-fetch all covers)")
|
||||
self.clear_covers_btn.clicked.connect(self._clear_cover_cache)
|
||||
form.addRow(self.clear_covers_btn)
|
||||
self.download_bundle_btn = QPushButton("Download offline cover bundle…")
|
||||
self.download_bundle_btn.clicked.connect(self.download_covers.emit)
|
||||
form.addRow(self.download_bundle_btn)
|
||||
col.addWidget(box)
|
||||
|
||||
# -- Storage ---------------------------------------------------------- #
|
||||
@@ -1541,7 +1552,9 @@ class StoreCoverLoader(QObject):
|
||||
if not img.isNull():
|
||||
self.loaded.emit(self.gen, row, _fit_image(img, self.icon_w, self.icon_h))
|
||||
finally:
|
||||
pool.shutdown(wait=True, cancel_futures=True)
|
||||
# Return promptly on cancel/exit instead of blocking the GUI thread
|
||||
# on in-flight network timeouts during shutdown.
|
||||
pool.shutdown(wait=not self._cancel.is_set(), cancel_futures=True)
|
||||
self.finished.emit()
|
||||
|
||||
|
||||
@@ -1818,6 +1831,8 @@ class StorePage(QWidget):
|
||||
self._lib_cover = {} # norm -> (release_id, system)
|
||||
self._psp_norm = set() # norm of library items installed on the PSP
|
||||
self._cover_attempted = set()
|
||||
self._cover_busy = False
|
||||
self._date_busy = False
|
||||
self._generation = 0
|
||||
self._scan_gen = 0
|
||||
self._meta_gen = 0
|
||||
@@ -1845,7 +1860,7 @@ class StorePage(QWidget):
|
||||
self.source_combo.currentIndexChanged.connect(self._on_source_changed)
|
||||
bar.addWidget(self.source_combo)
|
||||
self.nps_type = QComboBox()
|
||||
self.nps_type.addItems(["PSP Games", "PS1 Classics", "PSP Themes", "PSP DLC"])
|
||||
self.nps_type.addItems(["All", "PSP Games", "PS1 Classics", "PSP Themes", "PSP DLC"])
|
||||
self.nps_type.setVisible(False)
|
||||
self.nps_type.currentIndexChanged.connect(self._on_nps_type_changed)
|
||||
bar.addWidget(self.nps_type)
|
||||
@@ -1957,7 +1972,7 @@ class StorePage(QWidget):
|
||||
if self.source_combo.currentIndex() == 1:
|
||||
return transfers.ArchiveOrgSource(self._archive_id, self._archive_sig, self._archive_user)
|
||||
if self.source_combo.currentIndex() == 3:
|
||||
ct = ("psp", "psx", "theme", "dlc")[self.nps_type.currentIndex()]
|
||||
ct = ("all", "psp", "psx", "theme", "dlc")[self.nps_type.currentIndex()]
|
||||
return transfers.NoPayStationSource(ct)
|
||||
return transfers.LocalShareSource(self._store_dir)
|
||||
|
||||
@@ -2004,9 +2019,27 @@ class StorePage(QWidget):
|
||||
if self._items:
|
||||
self._apply_filter()
|
||||
|
||||
def _begin_scan(self):
|
||||
"""Clear the stale view and invalidate in-flight cover/date batches."""
|
||||
self._generation += 1
|
||||
self._cover_busy = False
|
||||
self._date_busy = False
|
||||
self._cover_queue = []
|
||||
self._cover_next = 0
|
||||
self._date_queue = []
|
||||
self._date_next = 0
|
||||
self._cover_attempted.clear()
|
||||
for loader in self._loaders:
|
||||
loader.cancel()
|
||||
self._loaders = []
|
||||
self.grid.clear()
|
||||
self.table.setRowCount(0)
|
||||
self.status.setText("Loading…")
|
||||
|
||||
def refresh(self, force=False):
|
||||
if self._closing:
|
||||
return
|
||||
self._begin_scan()
|
||||
self._scan_gen += 1
|
||||
gen = self._scan_gen
|
||||
if self.source_combo.currentIndex() == 0: # Server
|
||||
@@ -2024,7 +2057,7 @@ class StorePage(QWidget):
|
||||
self.status.setText("Fetching store from archive.org…")
|
||||
worker = RemoteStoreScanWorker(transfers.ArchiveOrgSource(self._archive_id, self._archive_sig, self._archive_user))
|
||||
elif self.source_combo.currentIndex() == 3: # NoPayStation
|
||||
ct = ("psp", "psx", "theme", "dlc")[self.nps_type.currentIndex()]
|
||||
ct = ("all", "psp", "psx", "theme", "dlc")[self.nps_type.currentIndex()]
|
||||
log.info("store: fetching NoPayStation %s list", ct)
|
||||
self.status.setText(f"Fetching NoPayStation ({ct.upper()})…")
|
||||
worker = RemoteStoreScanWorker(transfers.NoPayStationSource(ct, force=force))
|
||||
@@ -2067,6 +2100,8 @@ class StorePage(QWidget):
|
||||
self._items = []
|
||||
self._filtered = []
|
||||
self._dirty = False
|
||||
self.grid.clear()
|
||||
self.table.setRowCount(0)
|
||||
log.warning("store unavailable: %s", message)
|
||||
self.status.setText(f"Store unavailable: {message}")
|
||||
|
||||
@@ -2167,6 +2202,8 @@ class StorePage(QWidget):
|
||||
self._cover_next = 0
|
||||
self._date_queue = []
|
||||
self._date_next = 0
|
||||
self._cover_busy = False
|
||||
self._date_busy = False
|
||||
self._cover_attempted.clear()
|
||||
self._placeholder = QIcon(_placeholder_pixmap(self.ICON_W, self.ICON_H))
|
||||
|
||||
@@ -2176,6 +2213,7 @@ class StorePage(QWidget):
|
||||
self._loaders = []
|
||||
|
||||
self.grid.clear()
|
||||
self.grid.setUpdatesEnabled(False)
|
||||
self.table.setUpdatesEnabled(False)
|
||||
self.table.setSortingEnabled(False)
|
||||
self.table.setRowCount(show_count)
|
||||
@@ -2218,10 +2256,10 @@ class StorePage(QWidget):
|
||||
rid, system = lc
|
||||
else:
|
||||
rid, system = covers.stable_key(it["name"]), it.get("system") or "games"
|
||||
p = covers.cover_path(rid, system)
|
||||
if p.exists() and covers.file_is_image(p):
|
||||
item.setIcon(QIcon(_fit_pixmap(QPixmap(str(p)), self.ICON_W, self.ICON_H)))
|
||||
elif system not in ("theme", "dlc") and rid not in self._cover_attempted:
|
||||
# Covers are decoded + scaled off the GUI thread (StoreCoverLoader).
|
||||
# Never load a QPixmap here — doing so for cached covers made the store
|
||||
# unresponsive while filling/scrolling.
|
||||
if system not in ("theme", "dlc") and rid not in self._cover_attempted:
|
||||
# Themes and DLC have no meaningful box art in libretro/IGDB, and
|
||||
# bulk-scraping hundreds of them on every switch to the DLC list
|
||||
# saturates the network/CPU and makes the store unresponsive.
|
||||
@@ -2280,6 +2318,7 @@ class StorePage(QWidget):
|
||||
def _finish_fill(self):
|
||||
if self._fill_timer is not None:
|
||||
self._fill_timer.stop()
|
||||
self.grid.setUpdatesEnabled(True)
|
||||
self.table.setUpdatesEnabled(True)
|
||||
self.table.setSortingEnabled(True)
|
||||
total = len(self._filtered)
|
||||
@@ -2292,13 +2331,14 @@ class StorePage(QWidget):
|
||||
self._load_date_batch(self.DATE_BATCH)
|
||||
|
||||
def _load_cover_batch(self, n):
|
||||
if self._closing or not self._cover_queue:
|
||||
if self._closing or not self._cover_queue or self._cover_busy:
|
||||
return
|
||||
end = min(self._cover_next + n, len(self._cover_queue))
|
||||
if self._cover_next >= end:
|
||||
return
|
||||
tasks = self._cover_queue[self._cover_next:end]
|
||||
self._cover_next = end
|
||||
self._cover_busy = True
|
||||
self._start_cover_loader(self._generation, tasks)
|
||||
|
||||
def _active_scrollbar(self):
|
||||
@@ -2329,13 +2369,14 @@ class StorePage(QWidget):
|
||||
self._fill_timer.start()
|
||||
|
||||
def _load_date_batch(self, n):
|
||||
if self._closing or not self._date_queue:
|
||||
if self._closing or not self._date_queue or self._date_busy:
|
||||
return
|
||||
end = min(self._date_next + n, len(self._date_queue))
|
||||
if self._date_next >= end:
|
||||
return
|
||||
tasks = self._date_queue[self._date_next:end]
|
||||
self._date_next = end
|
||||
self._date_busy = True
|
||||
self._start_date_loader(self._generation, tasks)
|
||||
|
||||
def _start_date_loader(self, gen, tasks):
|
||||
@@ -2346,14 +2387,24 @@ class StorePage(QWidget):
|
||||
loader.moveToThread(thread)
|
||||
thread.started.connect(loader.run)
|
||||
loader.loaded.connect(self._on_date_loaded)
|
||||
loader.finished.connect(self._on_date_batch_done)
|
||||
loader.finished.connect(thread.quit)
|
||||
thread.finished.connect(self._on_thread_finished)
|
||||
self._threads.append(thread)
|
||||
self._loaders.append(loader)
|
||||
thread.start()
|
||||
|
||||
def _on_date_batch_done(self):
|
||||
if self._closing:
|
||||
return
|
||||
loader = self.sender()
|
||||
if getattr(loader, "gen", None) != self._generation:
|
||||
return
|
||||
self._date_busy = False
|
||||
self._load_date_batch(self.DATE_BATCH)
|
||||
|
||||
def _on_date_loaded(self, gen, row, date):
|
||||
if gen != self._generation or not date:
|
||||
if self._closing or gen != self._generation or not date:
|
||||
return
|
||||
if 0 <= row < len(self._filtered):
|
||||
self._filtered[row]["release_date"] = date
|
||||
@@ -2369,14 +2420,24 @@ class StorePage(QWidget):
|
||||
loader.moveToThread(thread)
|
||||
thread.started.connect(loader.run)
|
||||
loader.loaded.connect(self._on_cover_loaded)
|
||||
loader.finished.connect(self._on_cover_batch_done)
|
||||
loader.finished.connect(thread.quit)
|
||||
thread.finished.connect(self._on_thread_finished)
|
||||
self._threads.append(thread)
|
||||
self._loaders.append(loader)
|
||||
thread.start()
|
||||
|
||||
def _on_cover_batch_done(self):
|
||||
if self._closing:
|
||||
return
|
||||
loader = self.sender()
|
||||
if getattr(loader, "gen", None) != self._generation:
|
||||
return
|
||||
self._cover_busy = False
|
||||
self._load_cover_batch(self.COVER_SCROLL_BATCH)
|
||||
|
||||
def _on_cover_loaded(self, gen, row, image):
|
||||
if gen != self._generation:
|
||||
if self._closing or gen != self._generation:
|
||||
return
|
||||
item = self.grid.item(row)
|
||||
if item is not None:
|
||||
@@ -2609,6 +2670,7 @@ class QueuePage(QWidget):
|
||||
"umd-dump": "UMD dump",
|
||||
"save-sync": "Save sync",
|
||||
"nps-download": "NoPayStation",
|
||||
"covers": "Covers bundle",
|
||||
}
|
||||
|
||||
def __init__(self, parent=None):
|
||||
@@ -4130,6 +4192,7 @@ class Ps1Page(QWidget):
|
||||
self._preview_gen = 0
|
||||
self._preview_threads = []
|
||||
self._preview_workers = []
|
||||
self._closing = False
|
||||
|
||||
layout = QVBoxLayout(self)
|
||||
bar = QHBoxLayout()
|
||||
@@ -4228,7 +4291,7 @@ class Ps1Page(QWidget):
|
||||
self._preview_workers.append(worker)
|
||||
|
||||
def _on_preview_result(self, gen, game_id, icon0, pic0, pic1):
|
||||
if gen != self._preview_gen:
|
||||
if self._closing or gen != self._preview_gen:
|
||||
return
|
||||
self.preview_icon0.setText("")
|
||||
self.preview_pic0.setText("")
|
||||
@@ -4276,6 +4339,8 @@ class Ps1Page(QWidget):
|
||||
self._thread.start()
|
||||
|
||||
def _on_results(self, entries):
|
||||
if self._closing:
|
||||
return
|
||||
# Only show discs that still need converting; converted games already
|
||||
# appear in the Library tab.
|
||||
entries = [e for e in entries if e["kind"] == "disc" and e.get("status") != "converted"]
|
||||
@@ -4291,8 +4356,23 @@ class Ps1Page(QWidget):
|
||||
self.table.selectRow(0)
|
||||
|
||||
def _on_failed(self, message):
|
||||
if not self._closing:
|
||||
self.status.setText(f"PS1 scan failed: {message}")
|
||||
|
||||
def shutdown(self):
|
||||
self._closing = True
|
||||
if self._thread is not None and _thread_running(self._thread):
|
||||
self._thread.quit()
|
||||
self._thread.wait()
|
||||
for t in self._preview_threads:
|
||||
if _thread_running(t):
|
||||
t.quit()
|
||||
t.wait()
|
||||
self._preview_threads = []
|
||||
self._preview_workers = []
|
||||
self._thread = None
|
||||
self._worker = None
|
||||
|
||||
def _convert_selected(self):
|
||||
row = self.table.currentRow()
|
||||
if row < 0 or row >= len(self._entries):
|
||||
@@ -4588,6 +4668,7 @@ class MainWindow(QMainWindow):
|
||||
self._ps1_dir, self._ps1_output_dir, self._ps1_base_pbp,
|
||||
self._nps_dir)
|
||||
self.settings_page.saved.connect(self._on_settings_saved)
|
||||
self.settings_page.download_covers.connect(self._download_covers_bundle)
|
||||
return self.settings_page
|
||||
|
||||
def _on_settings_saved(self):
|
||||
@@ -4919,7 +5000,7 @@ class MainWindow(QMainWindow):
|
||||
if not self._nps_dir:
|
||||
QMessageBox.warning(self, "No NPS output dir", "Set the NoPayStation output dir in Settings → Store.")
|
||||
return
|
||||
sub = {"psp": "PSP", "psx": "PSX", "theme": "THEME", "dlc": "DLC"}.get(src.content_type, "PSP")
|
||||
sub = {"psp": "PSP", "psx": "PSX", "theme": "THEME", "dlc": "DLC"}.get(it.get("content_type") or src.content_type, "PSP")
|
||||
dest_dir = Path(self._nps_dir) / sub
|
||||
kind = "nps-download"
|
||||
else:
|
||||
@@ -5325,6 +5406,9 @@ class MainWindow(QMainWindow):
|
||||
QTimer.singleShot(0, self._refresh_psp)
|
||||
if kind == "save-sync":
|
||||
QTimer.singleShot(0, self._refresh_saves)
|
||||
if kind == "covers":
|
||||
covers.clear_miss_cache()
|
||||
QTimer.singleShot(0, self.load_library)
|
||||
if kind == "ps1-convert":
|
||||
QTimer.singleShot(0, self.ps1_page.refresh)
|
||||
QTimer.singleShot(0, self.load_library)
|
||||
@@ -5660,6 +5744,48 @@ class MainWindow(QMainWindow):
|
||||
QMessageBox.information(self, "Done", f"Renamed {renamed} file(s).")
|
||||
self.load_library()
|
||||
|
||||
def maybe_offer_covers_bundle(self):
|
||||
"""Offer to download the offline cover bundle once, if it's missing."""
|
||||
if self._closing or covers.covers_bundle_present():
|
||||
return
|
||||
s = QSettings("psppt", "psppt")
|
||||
if str(s.value("covers_bundle_skip", "false")).lower() == "true":
|
||||
return
|
||||
box = QMessageBox(self)
|
||||
box.setWindowTitle("Offline covers")
|
||||
box.setIcon(QMessageBox.Question)
|
||||
box.setText(
|
||||
"The offline cover bundle (~843 MB) isn't installed.\n\n"
|
||||
"Download it now from the project repo? Covers keep working online "
|
||||
"without it — the bundle just makes them instant and offline."
|
||||
)
|
||||
download = box.addButton("Download", QMessageBox.AcceptRole)
|
||||
box.addButton("Not now", QMessageBox.RejectRole)
|
||||
never = box.addButton("Don't ask again", QMessageBox.DestructiveRole)
|
||||
box.setDefaultButton(download)
|
||||
box.exec()
|
||||
clicked = box.clickedButton()
|
||||
if clicked is never:
|
||||
s.setValue("covers_bundle_skip", "true")
|
||||
elif clicked is download:
|
||||
self._download_covers_bundle()
|
||||
|
||||
def _download_covers_bundle(self):
|
||||
"""Queue the offline cover bundle download in the transfer queue."""
|
||||
if self._closing:
|
||||
return
|
||||
for j in self.queue.jobs:
|
||||
if j.kind == "covers" and j.status in ("queued", "running", "cancelling"):
|
||||
return # already queued or running
|
||||
self.queue.enqueue(
|
||||
transfers.CoversBundleSource(),
|
||||
str(config.COVERS_DIR),
|
||||
None,
|
||||
"Offline cover bundle",
|
||||
kind="covers",
|
||||
)
|
||||
self.tabs.setCurrentWidget(self.queue_page)
|
||||
|
||||
def closeEvent(self, event):
|
||||
self._closing = True
|
||||
self.settings.setValue("window_geometry", self.saveGeometry())
|
||||
@@ -5685,18 +5811,11 @@ class MainWindow(QMainWindow):
|
||||
self.psp_page.shutdown()
|
||||
if hasattr(self, "saves_page"):
|
||||
self.saves_page.shutdown()
|
||||
if hasattr(self, "ps1_page"):
|
||||
self.ps1_page.shutdown()
|
||||
event.accept()
|
||||
|
||||
|
||||
def _start_bundle_check(window):
|
||||
"""Download any missing offline cover bundles in the background."""
|
||||
server = getattr(window, "_store_server_url", "") or ""
|
||||
try:
|
||||
covers.ensure_cover_bundles(server)
|
||||
except Exception: # noqa: BLE001 - background best-effort
|
||||
pass
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(name)-14s %(message)s")
|
||||
init_db() # migrate legacy on-share DB/covers to local storage, then schema
|
||||
@@ -5717,7 +5836,22 @@ def main():
|
||||
window.setWindowIcon(app.windowIcon())
|
||||
window.show()
|
||||
log.info("main window ready")
|
||||
threading.Thread(target=_start_bundle_check, args=(window,), daemon=True).start()
|
||||
QTimer.singleShot(1200, window.maybe_offer_covers_bundle)
|
||||
|
||||
# Ctrl+C should close the window gracefully (running closeEvent's thread
|
||||
# cleanup) rather than tearing Qt down mid-thread, which segfaults.
|
||||
interrupt = {"pending": False}
|
||||
|
||||
def _sigint_handler(signum, frame):
|
||||
interrupt["pending"] = True
|
||||
|
||||
signal.signal(signal.SIGINT, _sigint_handler)
|
||||
sigint_timer = QTimer()
|
||||
sigint_timer.setInterval(100)
|
||||
sigint_timer.timeout.connect(
|
||||
lambda: window.close() if interrupt["pending"] else None)
|
||||
sigint_timer.start()
|
||||
|
||||
return app.exec()
|
||||
|
||||
|
||||
|
||||
+35
-32
@@ -79,13 +79,13 @@ class TransferQueue(QObject):
|
||||
job_finished = Signal(dict)
|
||||
job_removed = Signal(int)
|
||||
|
||||
def __init__(self, parent=None):
|
||||
def __init__(self, parent=None, max_concurrent=3):
|
||||
super().__init__(parent)
|
||||
self.max_concurrent = max(1, int(max_concurrent))
|
||||
self.jobs = []
|
||||
self._pending = []
|
||||
self._current = None
|
||||
self._thread = None
|
||||
self._worker = None
|
||||
# job_id -> (QThread, JobWorker) for jobs currently running in parallel.
|
||||
self._running = {}
|
||||
# Keep strong references to every worker/thread for their full lifetime.
|
||||
# Dropping a Python ref to a QObject that was moved to another thread
|
||||
# deletes it directly and crashes (bus error).
|
||||
@@ -115,8 +115,10 @@ class TransferQueue(QObject):
|
||||
self.job_updated.emit(job.to_dict())
|
||||
self.job_finished.emit(job.to_dict())
|
||||
return
|
||||
if job.status == "running" and self._worker is not None:
|
||||
self._worker.cancel()
|
||||
if job.status == "running":
|
||||
pair = self._running.get(job_id)
|
||||
if pair is not None:
|
||||
pair[1].cancel()
|
||||
job.status = "cancelling"
|
||||
self.job_updated.emit(job.to_dict())
|
||||
|
||||
@@ -140,8 +142,8 @@ class TransferQueue(QObject):
|
||||
self.remove(jid)
|
||||
|
||||
def shutdown(self):
|
||||
if self._worker is not None:
|
||||
self._worker.cancel()
|
||||
for _t, worker in self._running.values():
|
||||
worker.cancel()
|
||||
for t in self._threads:
|
||||
try:
|
||||
if t.isRunning():
|
||||
@@ -157,30 +159,29 @@ class TransferQueue(QObject):
|
||||
return None
|
||||
|
||||
def _start_next(self):
|
||||
if self._current is not None or not self._pending:
|
||||
return
|
||||
while len(self._running) < self.max_concurrent and self._pending:
|
||||
job = self._pending.pop(0)
|
||||
self._current = job
|
||||
job.status = "running"
|
||||
log.info("starting job #%d: %s (%s)", job.id, job.title, job.kind)
|
||||
self.job_updated.emit(job.to_dict())
|
||||
|
||||
self._thread = QThread(self)
|
||||
self._worker = JobWorker(job.source, job.dest_dir, job)
|
||||
self._worker.moveToThread(self._thread)
|
||||
self._thread.started.connect(self._worker.run)
|
||||
self._worker.progress.connect(self._on_progress)
|
||||
self._worker.finished.connect(self._on_finished)
|
||||
self._worker.skipped.connect(self._on_skipped)
|
||||
self._worker.cancelled.connect(self._on_cancelled)
|
||||
self._worker.failed.connect(self._on_failed)
|
||||
for sig in (self._worker.finished, self._worker.skipped,
|
||||
self._worker.cancelled, self._worker.failed):
|
||||
sig.connect(self._thread.quit)
|
||||
self._thread.finished.connect(self._on_thread_finished)
|
||||
self._threads.append(self._thread)
|
||||
self._workers.append(self._worker)
|
||||
self._thread.start()
|
||||
thread = QThread(self)
|
||||
worker = JobWorker(job.source, job.dest_dir, job)
|
||||
worker.moveToThread(thread)
|
||||
thread.started.connect(worker.run)
|
||||
worker.progress.connect(self._on_progress)
|
||||
worker.finished.connect(self._on_finished)
|
||||
worker.skipped.connect(self._on_skipped)
|
||||
worker.cancelled.connect(self._on_cancelled)
|
||||
worker.failed.connect(self._on_failed)
|
||||
for sig in (worker.finished, worker.skipped,
|
||||
worker.cancelled, worker.failed):
|
||||
sig.connect(thread.quit)
|
||||
thread.finished.connect(self._on_thread_finished)
|
||||
self._running[job.id] = (thread, worker)
|
||||
self._threads.append(thread)
|
||||
self._workers.append(worker)
|
||||
thread.start()
|
||||
|
||||
def _on_progress(self, job_id, stage, fraction, detail):
|
||||
job = self._find(job_id)
|
||||
@@ -218,10 +219,12 @@ class TransferQueue(QObject):
|
||||
self.job_finished.emit(job.to_dict())
|
||||
|
||||
def _on_thread_finished(self):
|
||||
# Queued to the main thread once the worker thread has fully stopped,
|
||||
# so it is now safe to drop the "current" pointers and advance.
|
||||
self._current = None
|
||||
self._thread = None
|
||||
self._worker = None
|
||||
# Queued to the main thread once a worker thread has fully stopped, so
|
||||
# it is now safe to drop its pointers and advance the queue.
|
||||
thread = self.sender()
|
||||
for jid, pair in list(self._running.items()):
|
||||
if pair[0] is thread:
|
||||
del self._running[jid]
|
||||
break
|
||||
self._start_next()
|
||||
|
||||
|
||||
+39
-3
@@ -175,6 +175,33 @@ class Source:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class CoversBundleSource:
|
||||
"""Download the combined offline cover bundle (runs as a queue job)."""
|
||||
label = "psppt-covers.zip"
|
||||
|
||||
def __init__(self, url=""):
|
||||
self.url = url
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
from . import covers
|
||||
|
||||
def on_progress(done, total):
|
||||
if progress_cb is None:
|
||||
return
|
||||
frac = (done / total) if total else 0.0
|
||||
detail = (f"{done / (1 << 20):.0f} / {total / (1 << 20):.0f} MB"
|
||||
if total else "")
|
||||
progress_cb("Downloading", frac, detail)
|
||||
|
||||
try:
|
||||
result = covers.download_covers_bundle(self.url,
|
||||
progress_cb=on_progress,
|
||||
cancel_event=cancel_event)
|
||||
except InterruptedError:
|
||||
raise TransferCancelled() from None
|
||||
return str(result)
|
||||
|
||||
|
||||
class FileCopySource(Source):
|
||||
"""Copy a single local file (library -> PSP or PSP -> library).
|
||||
|
||||
@@ -479,11 +506,19 @@ class NoPayStationSource(Source):
|
||||
def __init__(self, content_type, tsv_url="", force=False):
|
||||
from . import nopaystation
|
||||
self.content_type = content_type
|
||||
self.tsv_url = tsv_url or nopaystation.tsv_url(content_type)
|
||||
self.tsv_url = tsv_url or ("" if content_type == "all" else nopaystation.tsv_url(content_type))
|
||||
self.force = force
|
||||
|
||||
def scan(self, root=None):
|
||||
from . import nopaystation
|
||||
if self.content_type == "all":
|
||||
items = []
|
||||
for ct in ("psp", "psx", "theme", "dlc"):
|
||||
try:
|
||||
items.extend(nopaystation.fetch_items(ct, force=self.force))
|
||||
except Exception: # noqa: BLE001 - skip a source that fails
|
||||
log.warning("could not fetch NoPayStation %s list", ct)
|
||||
return items
|
||||
return nopaystation.fetch_items(self.content_type, self.tsv_url, force=self.force)
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
@@ -493,6 +528,7 @@ class NoPayStationSource(Source):
|
||||
|
||||
# The bundled Python decryptor handles PSP/PSX/themes/DLC directly.
|
||||
title_id = item.get("title_id") or item.get("content_id") or ""
|
||||
item_ct = item.get("content_type") or self.content_type
|
||||
pkg_name = re.sub(r"[^A-Za-z0-9_-]", "", title_id) or "pkg"
|
||||
tmp = Path(tempfile.mkdtemp(prefix="nps-download-"))
|
||||
pkg = tmp / f"{pkg_name}.pkg"
|
||||
@@ -500,10 +536,10 @@ class NoPayStationSource(Source):
|
||||
if progress_cb is not None:
|
||||
progress_cb("Downloading", 0.0, "")
|
||||
download_url(item["pkg_url"], pkg, None, progress_cb, cancel_event)
|
||||
kind_hint = self.content_type if self.content_type in ("theme", "dlc") else None
|
||||
kind_hint = item_ct if item_ct in ("theme", "dlc") else None
|
||||
folder = nopaystation.extract_pkg(pkg, title_id, dest_dir,
|
||||
kind_hint, progress_cb, cancel_event)
|
||||
if self.content_type in ("theme", "dlc"):
|
||||
if item_ct in ("theme", "dlc"):
|
||||
_write_sidecar(folder, item)
|
||||
return folder
|
||||
finally:
|
||||
|
||||
Binary file not shown.
Binary file not shown.
+2
-4
@@ -1,8 +1,8 @@
|
||||
# -*- mode: python ; coding: utf-8 -*-
|
||||
"""PyInstaller spec: build psppt (onedir).
|
||||
|
||||
Bundles the read-only resources (DATs, PS1 loader, offline cover bundles,
|
||||
icons) under ``lib/`` next to the executable. User-writable data
|
||||
Bundles the read-only resources (DATs, PS1 loader, cover templates, index
|
||||
catalogs, icons) under ``lib/`` next to the executable. User-writable data
|
||||
(credentials, manual covers, PS1 templates, NPS cache, library DB, cover
|
||||
cache) lives in the OS state dir at runtime — see ``app/config.py``.
|
||||
|
||||
@@ -15,8 +15,6 @@ datas = [
|
||||
("data/dats", "data/dats"),
|
||||
("data/loader", "data/loader"),
|
||||
("data/supplement.json", "data"),
|
||||
("data/covers/libretro-covers.zip", "data/covers"),
|
||||
("data/covers/openvgdb-covers.zip", "data/covers"),
|
||||
("data/covers/libretro-index.json", "data/covers"),
|
||||
("data/covers/openvgdb-index.json", "data/covers"),
|
||||
("data/covers/UMD_COVER_TEMPLATE.png", "data/covers"),
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Package the offline cover zips into a single distributable bundle.
|
||||
|
||||
Outputs ``psppt-covers.zip`` containing ``libretro-covers.zip`` and
|
||||
``openvgdb-covers.zip``. Attach it to the Gitea ``covers`` release; the app
|
||||
downloads it on first run when the user opts in.
|
||||
|
||||
Usage: python3 scripts/make-covers-bundle.py [out-path]
|
||||
"""
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
COVERS = ROOT / "data" / "covers"
|
||||
INNER = ("libretro-covers.zip", "openvgdb-covers.zip")
|
||||
|
||||
|
||||
def main(argv):
|
||||
out = Path(argv[1]) if len(argv) > 1 else ROOT / "psppt-covers.zip"
|
||||
missing = [n for n in INNER if not (COVERS / n).exists()]
|
||||
if missing:
|
||||
print(f"missing: {', '.join(missing)} in {COVERS}", file=sys.stderr)
|
||||
return 2
|
||||
# Inner zips are already compressed; store them to avoid recompressing.
|
||||
with zipfile.ZipFile(out, "w", zipfile.ZIP_STORED) as z:
|
||||
for n in INNER:
|
||||
print(f"adding {n} …")
|
||||
z.write(COVERS / n, arcname=n)
|
||||
size = out.stat().st_size / (1 << 20)
|
||||
print(f"wrote {out} ({size:.0f} MB)")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
Reference in New Issue
Block a user