Store: fix NPS refresh, scan race, thread cleanup, and matching perf

app/gui.py

@@ -1650,11 +1650,12 @@ class StorePage(QWidget):
         self._pkg2zip = ""
         self._items = []
         self._filtered = []
-        self._lib_norm = []
+        self._lib_norm = set()
         self._lib_cover = {}  # norm -> (release_id, system)
-        self._psp_norm = []   # norm of library items installed on the PSP
+        self._psp_norm = set()   # norm of library items installed on the PSP
         self._cover_attempted = set()
         self._generation = 0
+        self._scan_gen = 0
         self._meta_gen = 0
         self._closing = False
         self._threads = []
@@ -1694,7 +1695,7 @@ class StorePage(QWidget):
         self.psp_filter.currentIndexChanged.connect(self._apply_filter)
         bar.addWidget(self.psp_filter)
         refresh_btn = QPushButton("Refresh")
-        refresh_btn.clicked.connect(self.refresh)
+        refresh_btn.clicked.connect(lambda: self.refresh(force=True))
         bar.addWidget(refresh_btn)
         self.all_check = QCheckBox("All")
         self.all_check.setChecked(False)
@@ -1807,14 +1808,14 @@ class StorePage(QWidget):
         self.refresh()
 
     def set_library_items(self, items):
-        norm, seen, cover = [], set(), {}
+        norm, seen, cover = set(), set(), {}
-        psp_norm, psp_seen = [], set()
+        psp_norm, psp_seen = set(), set()
         for it in items or []:
-            # The store only contains PSP game discs, so ignore UMD videos when
+            # The store lists PSP games/themes/DLC; ignore UMD videos (a game
-            # matching against the library (a game archive must never match a
+            # archive must never match a same-named movie) and PS1 (its titles
-            # same-named movie and inherit its cover/metadata).
+            # are matched separately).
             system = it.get("system")
-            if system != "games":
+            if system not in ("games", "theme", "dlc"):
                 continue
             rid = it.get("release_id")
             for key in (it.get("canonical"), it.get("name")):
@@ -1823,21 +1824,23 @@ class StorePage(QWidget):
                 n = matcher.normalize_name(key)
                 if n and n not in seen:
                     seen.add(n)
-                    norm.append(n)
+                    norm.add(n)
                     if rid is not None:
                         cover[n] = (rid, system)
                 if it.get("on_psp") and n and n not in psp_seen:
                     psp_seen.add(n)
-                    psp_norm.append(n)
+                    psp_norm.add(n)
         self._lib_norm = norm
         self._lib_cover = cover
         self._psp_norm = psp_norm
         if self._items:
             self._apply_filter()
 
-    def refresh(self):
+    def refresh(self, force=False):
         if self._closing:
             return
+        self._scan_gen += 1
+        gen = self._scan_gen
         if self.source_combo.currentIndex() == 0:  # Server
             if not self._server_url:
                 self.status.setText("Set the store server URL in Settings")
@@ -1856,7 +1859,7 @@ class StorePage(QWidget):
             ct = ("psp", "psx", "theme", "dlc")[self.nps_type.currentIndex()]
             log.info("store: fetching NoPayStation %s list", ct)
             self.status.setText(f"Fetching NoPayStation ({ct.upper()})…")
-            worker = RemoteStoreScanWorker(transfers.NoPayStationSource(ct, pkg2zip_path=self._pkg2zip))
+            worker = RemoteStoreScanWorker(transfers.NoPayStationSource(ct, pkg2zip_path=self._pkg2zip, force=force))
         else:
             if not self._store_dir:
                 self.status.setText("Set the local store folder in Settings (Advanced)")
@@ -1867,15 +1870,27 @@ class StorePage(QWidget):
         thread = QThread(self)
         worker.moveToThread(thread)
         thread.started.connect(worker.run)
-        worker.results.connect(self._on_scan_results)
+        worker.results.connect(lambda items, g=gen: self._on_scan_results(g, items))
-        worker.failed.connect(self._on_scan_failed)
+        worker.failed.connect(lambda msg, g=gen: self._on_scan_failed(g, msg))
         worker.finished.connect(thread.quit)
+        worker.finished.connect(lambda w=worker: self._on_worker_finished(w))
+        thread.finished.connect(lambda t=thread: self._on_thread_finished(t))
         self._threads.append(thread)
         self._workers.append(worker)
         thread.start()
 
-    def _on_scan_failed(self, message):
+    def _on_thread_finished(self, thread):
-        if self._closing:
+        if thread in self._threads:
+            self._threads.remove(thread)
+        thread.deleteLater()
+
+    def _on_worker_finished(self, worker):
+        if worker in self._workers:
+            self._workers.remove(worker)
+        worker.deleteLater()
+
+    def _on_scan_failed(self, gen, message):
+        if self._closing or gen != self._scan_gen:
             return
         self._items = []
         self._filtered = []
@@ -1883,8 +1898,8 @@ class StorePage(QWidget):
         log.warning("store unavailable: %s", message)
         self.status.setText(f"Store unavailable: {message}")
 
-    def _on_scan_results(self, items):
+    def _on_scan_results(self, gen, items):
-        if self._closing:
+        if self._closing or gen != self._scan_gen:
             return
         self._items = items
         self._cover_attempted.clear()
@@ -1914,15 +1929,11 @@ class StorePage(QWidget):
                 # Exact normalized match only: region/language/version tags are
                 # already stripped, and substring matching would false-positive
                 # sequels (e.g. "Open Tee" matching "Open Tee 2").
-                for ln in self._lib_norm:
+                if n in self._lib_norm:
-                    if n == ln:
+                    it["in_library"] = True
-                        it["in_library"] = True
+                    it["_lib_cover"] = self._lib_cover.get(n)
-                        it["_lib_cover"] = self._lib_cover.get(ln)
+                if n in self._psp_norm:
-                        break
+                    it["on_psp"] = True
-                for pn in self._psp_norm:
-                    if n == pn:
-                        it["on_psp"] = True
-                        break
             if psp_f == "On PSP" and not it.get("on_psp"):
                 continue
             if psp_f == "Not on PSP" and it.get("on_psp"):
@@ -1958,7 +1969,7 @@ class StorePage(QWidget):
     CHUNK = 150          # items filled per timer tick
     COVER_BATCH = 200    # covers fetched for the initially-visible area
     COVER_SCROLL_BATCH = 100
-    RECENT_LIMIT = 150   # default: show only the newest N store items
+    RECENT_LIMIT = 300   # default: show only the newest N store items
     DATE_BATCH = 150     # release-date metadata fetched per batch
 
     def _populate(self, items, show_count):
@@ -2120,6 +2131,7 @@ class StorePage(QWidget):
         thread.started.connect(loader.run)
         loader.loaded.connect(self._on_date_loaded)
         loader.finished.connect(thread.quit)
+        thread.finished.connect(lambda t=thread: self._on_thread_finished(t))
         self._threads.append(thread)
         self._loaders.append(loader)
         thread.start()
@@ -2142,6 +2154,7 @@ class StorePage(QWidget):
         thread.started.connect(loader.run)
         loader.loaded.connect(self._on_cover_loaded)
         loader.finished.connect(thread.quit)
+        thread.finished.connect(lambda t=thread: self._on_thread_finished(t))
         self._threads.append(thread)
         self._loaders.append(loader)
         thread.start()
@@ -2198,6 +2211,8 @@ class StorePage(QWidget):
         thread.started.connect(worker.run)
         worker.result.connect(self._on_store_meta)
         worker.finished.connect(thread.quit)
+        worker.finished.connect(lambda w=worker: self._on_worker_finished(w))
+        thread.finished.connect(lambda t=thread: self._on_thread_finished(t))
         self._threads.append(thread)
         self._workers.append(worker)
         thread.start()
@@ -2381,7 +2396,7 @@ class StorePage(QWidget):
         dlg = self._scrape_dialog
         if dlg is not None:
             dlg.close()
-        for t in self._threads:
+        for t in list(self._threads):
             try:
                 if t.isRunning():
                     t.quit()

app/info.py

@@ -23,6 +23,9 @@ _miss_db = {}
 _miss_loaded = False
 _MISS_TTL = 86400.0  # retry failed metadata after 1 day
 
+# Memoised release-date table (invalidated on save/reset).
+_dates_cache = None
+
 
 def _load_miss_db():
     global _miss_loaded
@@ -122,6 +125,7 @@ def save(release_id, system, meta, manual=False, title=None):
         )
         conn.commit()
         conn.close()
+        _invalidate_dates_cache()
         return True
     except sqlite3.Error:
         return False
@@ -140,6 +144,7 @@ def reset(release_id, system):
         )
         conn.commit()
         conn.close()
+        _invalidate_dates_cache()
     except sqlite3.Error:
         pass
 
@@ -161,23 +166,32 @@ def cached_release_dates():
     """Return {(release_id, system): release_date} from cached metadata only.
 
     Pure DB read — no network — so list views can show a release date without
-    triggering API fetches for every row.
+    triggering API fetches for every row. Result is memoised and invalidated
+    whenever metadata is saved.
     """
-    out = {}
+    global _dates_cache
-    try:
+    if _dates_cache is None:
-        conn = get_connection(config.DB_PATH)
+        out = {}
-        rows = conn.execute("SELECT release_id, system, meta_json FROM item_info").fetchall()
-        conn.close()
-    except sqlite3.Error:
-        return {}
-    for r in rows:
         try:
-            meta = json.loads(r["meta_json"] or "{}")
+            conn = get_connection(config.DB_PATH)
-        except (TypeError, ValueError):
+            rows = conn.execute("SELECT release_id, system, meta_json FROM item_info").fetchall()
-            continue
+            conn.close()
-        if isinstance(meta, dict) and meta.get("release_date"):
+        except sqlite3.Error:
-            out[(r["release_id"], r["system"])] = meta["release_date"]
+            return out
-    return out
+        for r in rows:
+            try:
+                meta = json.loads(r["meta_json"] or "{}")
+            except (TypeError, ValueError):
+                continue
+            if isinstance(meta, dict) and meta.get("release_date"):
+                out[(r["release_id"], r["system"])] = meta["release_date"]
+        _dates_cache = out
+    return _dates_cache
+
+
+def _invalidate_dates_cache():
+    global _dates_cache
+    _dates_cache = None
 
 
 def get_or_fetch(release_id, system, fetch_title):

app/nopaystation.py

@@ -131,22 +131,40 @@ def parse_tsv(text: str, content_type: str) -> List[dict]:
     return items
 
 
-def fetch_items(content_type: str, url: str = "", timeout: int = 60) -> List[dict]:
+def fetch_items(content_type: str, url: str = "", timeout: int = 60, force: bool = False) -> List[dict]:
-    """Download (and cache) a TSV, returning parsed item dicts."""
+    """Download (and cache) a TSV, returning parsed item dicts.
+
+    ``force`` re-downloads the TSV (used by the Store's Refresh button); the
+    local cache is only used as a fallback when the network fails.
+    """
     url = url or tsv_url(content_type)
     name = _TSV_NAMES.get(content_type, "PSP_GAMES.tsv")
     cache = _tsv_dir() / name
-    if cache.is_file():
+
+    def read_cache():
         try:
             return parse_tsv(cache.read_text(encoding="utf-8", errors="replace"), content_type)
         except OSError:
-            pass
+            return None
-    req = urllib.request.Request(url, headers={"User-Agent": "psp-preserve/0.1"})
+
-    with urllib.request.urlopen(req, timeout=timeout) as resp:
+    if not force and cache.is_file():
-        data = resp.read()
+        items = read_cache()
-    text = data.decode("utf-8", errors="replace")
+        if items is not None:
-    cache.write_text(text, encoding="utf-8")
+            return items
-    return parse_tsv(text, content_type)
+
+    try:
+        req = urllib.request.Request(url, headers={"User-Agent": "psp-preserve/0.1"})
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            data = resp.read()
+        text = data.decode("utf-8", errors="replace")
+        cache.write_text(text, encoding="utf-8")
+        return parse_tsv(text, content_type)
+    except Exception:  # noqa: BLE001 - fall back to the cached list on failure
+        items = read_cache()
+        if items is not None:
+            log.warning("could not refresh %s TSV; using cached copy", content_type)
+            return items
+        raise
 
 
 def find_pkg2zip(configured_path: str = "") -> Optional[Path]:

app/transfers.py

@@ -476,15 +476,16 @@ class NoPayStationSource(Source):
 
     label = "NoPayStation"
 
-    def __init__(self, content_type, tsv_url="", pkg2zip_path=""):
+    def __init__(self, content_type, tsv_url="", pkg2zip_path="", force=False):
         from . import nopaystation
         self.content_type = content_type
         self.tsv_url = tsv_url or nopaystation.tsv_url(content_type)
         self.pkg2zip = pkg2zip_path or ""
+        self.force = force
 
     def scan(self, root=None):
         from . import nopaystation
-        return nopaystation.fetch_items(self.content_type, self.tsv_url)
+        return nopaystation.fetch_items(self.content_type, self.tsv_url, force=self.force)
 
     def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
         from . import nopaystation