From ec9f5e6d8e179eed9839533b4fa14496c2691256 Mon Sep 17 00:00:00 2001 From: Mathieu Moran Date: Tue, 8 Sep 2026 23:18:23 -0400 Subject: [PATCH] Store: fix NPS refresh, scan race, thread cleanup, and matching perf --- app/gui.py | 75 +++++++++++++++++++++++++++------------------ app/info.py | 44 +++++++++++++++++--------- app/nopaystation.py | 38 +++++++++++++++++------ app/transfers.py | 5 +-- 4 files changed, 105 insertions(+), 57 deletions(-) diff --git a/app/gui.py b/app/gui.py index 5b13389..93d374b 100644 --- a/app/gui.py +++ b/app/gui.py @@ -1650,11 +1650,12 @@ class StorePage(QWidget): self._pkg2zip = "" self._items = [] self._filtered = [] - self._lib_norm = [] + self._lib_norm = set() self._lib_cover = {} # norm -> (release_id, system) - self._psp_norm = [] # norm of library items installed on the PSP + self._psp_norm = set() # norm of library items installed on the PSP self._cover_attempted = set() self._generation = 0 + self._scan_gen = 0 self._meta_gen = 0 self._closing = False self._threads = [] @@ -1694,7 +1695,7 @@ class StorePage(QWidget): self.psp_filter.currentIndexChanged.connect(self._apply_filter) bar.addWidget(self.psp_filter) refresh_btn = QPushButton("Refresh") - refresh_btn.clicked.connect(self.refresh) + refresh_btn.clicked.connect(lambda: self.refresh(force=True)) bar.addWidget(refresh_btn) self.all_check = QCheckBox("All") self.all_check.setChecked(False) @@ -1807,14 +1808,14 @@ class StorePage(QWidget): self.refresh() def set_library_items(self, items): - norm, seen, cover = [], set(), {} - psp_norm, psp_seen = [], set() + norm, seen, cover = set(), set(), {} + psp_norm, psp_seen = set(), set() for it in items or []: - # The store only contains PSP game discs, so ignore UMD videos when - # matching against the library (a game archive must never match a - # same-named movie and inherit its cover/metadata). + # The store lists PSP games/themes/DLC; ignore UMD videos (a game + # archive must never match a same-named movie) and PS1 (its titles + # are matched separately). system = it.get("system") - if system != "games": + if system not in ("games", "theme", "dlc"): continue rid = it.get("release_id") for key in (it.get("canonical"), it.get("name")): @@ -1823,21 +1824,23 @@ class StorePage(QWidget): n = matcher.normalize_name(key) if n and n not in seen: seen.add(n) - norm.append(n) + norm.add(n) if rid is not None: cover[n] = (rid, system) if it.get("on_psp") and n and n not in psp_seen: psp_seen.add(n) - psp_norm.append(n) + psp_norm.add(n) self._lib_norm = norm self._lib_cover = cover self._psp_norm = psp_norm if self._items: self._apply_filter() - def refresh(self): + def refresh(self, force=False): if self._closing: return + self._scan_gen += 1 + gen = self._scan_gen if self.source_combo.currentIndex() == 0: # Server if not self._server_url: self.status.setText("Set the store server URL in Settings") @@ -1856,7 +1859,7 @@ class StorePage(QWidget): ct = ("psp", "psx", "theme", "dlc")[self.nps_type.currentIndex()] log.info("store: fetching NoPayStation %s list", ct) self.status.setText(f"Fetching NoPayStation ({ct.upper()})…") - worker = RemoteStoreScanWorker(transfers.NoPayStationSource(ct, pkg2zip_path=self._pkg2zip)) + worker = RemoteStoreScanWorker(transfers.NoPayStationSource(ct, pkg2zip_path=self._pkg2zip, force=force)) else: if not self._store_dir: self.status.setText("Set the local store folder in Settings (Advanced)") @@ -1867,15 +1870,27 @@ class StorePage(QWidget): thread = QThread(self) worker.moveToThread(thread) thread.started.connect(worker.run) - worker.results.connect(self._on_scan_results) - worker.failed.connect(self._on_scan_failed) + worker.results.connect(lambda items, g=gen: self._on_scan_results(g, items)) + worker.failed.connect(lambda msg, g=gen: self._on_scan_failed(g, msg)) worker.finished.connect(thread.quit) + worker.finished.connect(lambda w=worker: self._on_worker_finished(w)) + thread.finished.connect(lambda t=thread: self._on_thread_finished(t)) self._threads.append(thread) self._workers.append(worker) thread.start() - def _on_scan_failed(self, message): - if self._closing: + def _on_thread_finished(self, thread): + if thread in self._threads: + self._threads.remove(thread) + thread.deleteLater() + + def _on_worker_finished(self, worker): + if worker in self._workers: + self._workers.remove(worker) + worker.deleteLater() + + def _on_scan_failed(self, gen, message): + if self._closing or gen != self._scan_gen: return self._items = [] self._filtered = [] @@ -1883,8 +1898,8 @@ class StorePage(QWidget): log.warning("store unavailable: %s", message) self.status.setText(f"Store unavailable: {message}") - def _on_scan_results(self, items): - if self._closing: + def _on_scan_results(self, gen, items): + if self._closing or gen != self._scan_gen: return self._items = items self._cover_attempted.clear() @@ -1914,15 +1929,11 @@ class StorePage(QWidget): # Exact normalized match only: region/language/version tags are # already stripped, and substring matching would false-positive # sequels (e.g. "Open Tee" matching "Open Tee 2"). - for ln in self._lib_norm: - if n == ln: - it["in_library"] = True - it["_lib_cover"] = self._lib_cover.get(ln) - break - for pn in self._psp_norm: - if n == pn: - it["on_psp"] = True - break + if n in self._lib_norm: + it["in_library"] = True + it["_lib_cover"] = self._lib_cover.get(n) + if n in self._psp_norm: + it["on_psp"] = True if psp_f == "On PSP" and not it.get("on_psp"): continue if psp_f == "Not on PSP" and it.get("on_psp"): @@ -1958,7 +1969,7 @@ class StorePage(QWidget): CHUNK = 150 # items filled per timer tick COVER_BATCH = 200 # covers fetched for the initially-visible area COVER_SCROLL_BATCH = 100 - RECENT_LIMIT = 150 # default: show only the newest N store items + RECENT_LIMIT = 300 # default: show only the newest N store items DATE_BATCH = 150 # release-date metadata fetched per batch def _populate(self, items, show_count): @@ -2120,6 +2131,7 @@ class StorePage(QWidget): thread.started.connect(loader.run) loader.loaded.connect(self._on_date_loaded) loader.finished.connect(thread.quit) + thread.finished.connect(lambda t=thread: self._on_thread_finished(t)) self._threads.append(thread) self._loaders.append(loader) thread.start() @@ -2142,6 +2154,7 @@ class StorePage(QWidget): thread.started.connect(loader.run) loader.loaded.connect(self._on_cover_loaded) loader.finished.connect(thread.quit) + thread.finished.connect(lambda t=thread: self._on_thread_finished(t)) self._threads.append(thread) self._loaders.append(loader) thread.start() @@ -2198,6 +2211,8 @@ class StorePage(QWidget): thread.started.connect(worker.run) worker.result.connect(self._on_store_meta) worker.finished.connect(thread.quit) + worker.finished.connect(lambda w=worker: self._on_worker_finished(w)) + thread.finished.connect(lambda t=thread: self._on_thread_finished(t)) self._threads.append(thread) self._workers.append(worker) thread.start() @@ -2381,7 +2396,7 @@ class StorePage(QWidget): dlg = self._scrape_dialog if dlg is not None: dlg.close() - for t in self._threads: + for t in list(self._threads): try: if t.isRunning(): t.quit() diff --git a/app/info.py b/app/info.py index 380e79e..48f9be0 100644 --- a/app/info.py +++ b/app/info.py @@ -23,6 +23,9 @@ _miss_db = {} _miss_loaded = False _MISS_TTL = 86400.0 # retry failed metadata after 1 day +# Memoised release-date table (invalidated on save/reset). +_dates_cache = None + def _load_miss_db(): global _miss_loaded @@ -122,6 +125,7 @@ def save(release_id, system, meta, manual=False, title=None): ) conn.commit() conn.close() + _invalidate_dates_cache() return True except sqlite3.Error: return False @@ -140,6 +144,7 @@ def reset(release_id, system): ) conn.commit() conn.close() + _invalidate_dates_cache() except sqlite3.Error: pass @@ -161,23 +166,32 @@ def cached_release_dates(): """Return {(release_id, system): release_date} from cached metadata only. Pure DB read — no network — so list views can show a release date without - triggering API fetches for every row. + triggering API fetches for every row. Result is memoised and invalidated + whenever metadata is saved. """ - out = {} - try: - conn = get_connection(config.DB_PATH) - rows = conn.execute("SELECT release_id, system, meta_json FROM item_info").fetchall() - conn.close() - except sqlite3.Error: - return {} - for r in rows: + global _dates_cache + if _dates_cache is None: + out = {} try: - meta = json.loads(r["meta_json"] or "{}") - except (TypeError, ValueError): - continue - if isinstance(meta, dict) and meta.get("release_date"): - out[(r["release_id"], r["system"])] = meta["release_date"] - return out + conn = get_connection(config.DB_PATH) + rows = conn.execute("SELECT release_id, system, meta_json FROM item_info").fetchall() + conn.close() + except sqlite3.Error: + return out + for r in rows: + try: + meta = json.loads(r["meta_json"] or "{}") + except (TypeError, ValueError): + continue + if isinstance(meta, dict) and meta.get("release_date"): + out[(r["release_id"], r["system"])] = meta["release_date"] + _dates_cache = out + return _dates_cache + + +def _invalidate_dates_cache(): + global _dates_cache + _dates_cache = None def get_or_fetch(release_id, system, fetch_title): diff --git a/app/nopaystation.py b/app/nopaystation.py index e620d05..14e9214 100644 --- a/app/nopaystation.py +++ b/app/nopaystation.py @@ -131,22 +131,40 @@ def parse_tsv(text: str, content_type: str) -> List[dict]: return items -def fetch_items(content_type: str, url: str = "", timeout: int = 60) -> List[dict]: - """Download (and cache) a TSV, returning parsed item dicts.""" +def fetch_items(content_type: str, url: str = "", timeout: int = 60, force: bool = False) -> List[dict]: + """Download (and cache) a TSV, returning parsed item dicts. + + ``force`` re-downloads the TSV (used by the Store's Refresh button); the + local cache is only used as a fallback when the network fails. + """ url = url or tsv_url(content_type) name = _TSV_NAMES.get(content_type, "PSP_GAMES.tsv") cache = _tsv_dir() / name - if cache.is_file(): + + def read_cache(): try: return parse_tsv(cache.read_text(encoding="utf-8", errors="replace"), content_type) except OSError: - pass - req = urllib.request.Request(url, headers={"User-Agent": "psp-preserve/0.1"}) - with urllib.request.urlopen(req, timeout=timeout) as resp: - data = resp.read() - text = data.decode("utf-8", errors="replace") - cache.write_text(text, encoding="utf-8") - return parse_tsv(text, content_type) + return None + + if not force and cache.is_file(): + items = read_cache() + if items is not None: + return items + + try: + req = urllib.request.Request(url, headers={"User-Agent": "psp-preserve/0.1"}) + with urllib.request.urlopen(req, timeout=timeout) as resp: + data = resp.read() + text = data.decode("utf-8", errors="replace") + cache.write_text(text, encoding="utf-8") + return parse_tsv(text, content_type) + except Exception: # noqa: BLE001 - fall back to the cached list on failure + items = read_cache() + if items is not None: + log.warning("could not refresh %s TSV; using cached copy", content_type) + return items + raise def find_pkg2zip(configured_path: str = "") -> Optional[Path]: diff --git a/app/transfers.py b/app/transfers.py index 06175aa..5513272 100644 --- a/app/transfers.py +++ b/app/transfers.py @@ -476,15 +476,16 @@ class NoPayStationSource(Source): label = "NoPayStation" - def __init__(self, content_type, tsv_url="", pkg2zip_path=""): + def __init__(self, content_type, tsv_url="", pkg2zip_path="", force=False): from . import nopaystation self.content_type = content_type self.tsv_url = tsv_url or nopaystation.tsv_url(content_type) self.pkg2zip = pkg2zip_path or "" + self.force = force def scan(self, root=None): from . import nopaystation - return nopaystation.fetch_items(self.content_type, self.tsv_url) + return nopaystation.fetch_items(self.content_type, self.tsv_url, force=self.force) def download(self, item, dest_dir, progress_cb=None, cancel_event=None): from . import nopaystation