quoije/psp-preservation-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Store: fix NPS refresh, scan race, thread cleanup, and matching perf

Browse Source
This commit is contained in:
quoije
2026-09-08 23:18:23 -04:00
parent be54ff00aa
commit ec9f5e6d8e
4 changed files with 105 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/gui.py
+45 -30
View File
@@ -1650,11 +1650,12 @@ class StorePage(QWidget):
self._pkg2zip = ""
self._items = []
self._filtered = []
self._lib_norm = []
self._lib_norm = set()
self._lib_cover = {} # norm -> (release_id, system)
self._psp_norm = [] # norm of library items installed on the PSP
self._psp_norm = set() # norm of library items installed on the PSP
self._cover_attempted = set()
self._generation = 0
self._scan_gen = 0
self._meta_gen = 0
self._closing = False
self._threads = []
@@ -1694,7 +1695,7 @@ class StorePage(QWidget):
self.psp_filter.currentIndexChanged.connect(self._apply_filter)
bar.addWidget(self.psp_filter)
refresh_btn = QPushButton("Refresh")
refresh_btn.clicked.connect(self.refresh)
refresh_btn.clicked.connect(lambda: self.refresh(force=True))
bar.addWidget(refresh_btn)
self.all_check = QCheckBox("All")
self.all_check.setChecked(False)
@@ -1807,14 +1808,14 @@ class StorePage(QWidget):
self.refresh()
def set_library_items(self, items):
norm, seen, cover = [], set(), {}
psp_norm, psp_seen = [], set()
norm, seen, cover = set(), set(), {}
psp_norm, psp_seen = set(), set()
for it in items or []:
# The store only contains PSP game discs, so ignore UMD videos when
# matching against the library (a game archive must never match a
# same-named movie and inherit its cover/metadata).
# The store lists PSP games/themes/DLC; ignore UMD videos (a game
# archive must never match a same-named movie) and PS1 (its titles
# are matched separately).
system = it.get("system")
if system != "games":
if system not in ("games", "theme", "dlc"):
continue
rid = it.get("release_id")
for key in (it.get("canonical"), it.get("name")):
@@ -1823,21 +1824,23 @@ class StorePage(QWidget):
n = matcher.normalize_name(key)
if n and n not in seen:
seen.add(n)
norm.append(n)
norm.add(n)
if rid is not None:
cover[n] = (rid, system)
if it.get("on_psp") and n and n not in psp_seen:
psp_seen.add(n)
psp_norm.append(n)
psp_norm.add(n)
self._lib_norm = norm
self._lib_cover = cover
self._psp_norm = psp_norm
if self._items:
self._apply_filter()
def refresh(self):
def refresh(self, force=False):
if self._closing:
return
self._scan_gen += 1
gen = self._scan_gen
if self.source_combo.currentIndex() == 0: # Server
if not self._server_url:
self.status.setText("Set the store server URL in Settings")
@@ -1856,7 +1859,7 @@ class StorePage(QWidget):
ct = ("psp", "psx", "theme", "dlc")[self.nps_type.currentIndex()]
log.info("store: fetching NoPayStation %s list", ct)
self.status.setText(f"Fetching NoPayStation ({ct.upper()})…")
worker = RemoteStoreScanWorker(transfers.NoPayStationSource(ct, pkg2zip_path=self._pkg2zip))
worker = RemoteStoreScanWorker(transfers.NoPayStationSource(ct, pkg2zip_path=self._pkg2zip, force=force))
else:
if not self._store_dir:
self.status.setText("Set the local store folder in Settings (Advanced)")
@@ -1867,15 +1870,27 @@ class StorePage(QWidget):
thread = QThread(self)
worker.moveToThread(thread)
thread.started.connect(worker.run)
worker.results.connect(self._on_scan_results)
worker.failed.connect(self._on_scan_failed)
worker.results.connect(lambda items, g=gen: self._on_scan_results(g, items))
worker.failed.connect(lambda msg, g=gen: self._on_scan_failed(g, msg))
worker.finished.connect(thread.quit)
worker.finished.connect(lambda w=worker: self._on_worker_finished(w))
thread.finished.connect(lambda t=thread: self._on_thread_finished(t))
self._threads.append(thread)
self._workers.append(worker)
thread.start()
def _on_scan_failed(self, message):
if self._closing:
def _on_thread_finished(self, thread):
if thread in self._threads:
self._threads.remove(thread)
thread.deleteLater()
def _on_worker_finished(self, worker):
if worker in self._workers:
self._workers.remove(worker)
worker.deleteLater()
def _on_scan_failed(self, gen, message):
if self._closing or gen != self._scan_gen:
return
self._items = []
self._filtered = []
@@ -1883,8 +1898,8 @@ class StorePage(QWidget):
log.warning("store unavailable: %s", message)
self.status.setText(f"Store unavailable: {message}")
def _on_scan_results(self, items):
if self._closing:
def _on_scan_results(self, gen, items):
if self._closing or gen != self._scan_gen:
return
self._items = items
self._cover_attempted.clear()
@@ -1914,15 +1929,11 @@ class StorePage(QWidget):
# Exact normalized match only: region/language/version tags are
# already stripped, and substring matching would false-positive
# sequels (e.g. "Open Tee" matching "Open Tee 2").
for ln in self._lib_norm:
if n == ln:
it["in_library"] = True
it["_lib_cover"] = self._lib_cover.get(ln)
break
for pn in self._psp_norm:
if n == pn:
it["on_psp"] = True
break
if n in self._lib_norm:
it["in_library"] = True
it["_lib_cover"] = self._lib_cover.get(n)
if n in self._psp_norm:
it["on_psp"] = True
if psp_f == "On PSP" and not it.get("on_psp"):
continue
if psp_f == "Not on PSP" and it.get("on_psp"):
@@ -1958,7 +1969,7 @@ class StorePage(QWidget):
CHUNK = 150 # items filled per timer tick
COVER_BATCH = 200 # covers fetched for the initially-visible area
COVER_SCROLL_BATCH = 100
RECENT_LIMIT = 150 # default: show only the newest N store items
RECENT_LIMIT = 300 # default: show only the newest N store items
DATE_BATCH = 150 # release-date metadata fetched per batch
def _populate(self, items, show_count):
@@ -2120,6 +2131,7 @@ class StorePage(QWidget):
thread.started.connect(loader.run)
loader.loaded.connect(self._on_date_loaded)
loader.finished.connect(thread.quit)
thread.finished.connect(lambda t=thread: self._on_thread_finished(t))
self._threads.append(thread)
self._loaders.append(loader)
thread.start()
@@ -2142,6 +2154,7 @@ class StorePage(QWidget):
thread.started.connect(loader.run)
loader.loaded.connect(self._on_cover_loaded)
loader.finished.connect(thread.quit)
thread.finished.connect(lambda t=thread: self._on_thread_finished(t))
self._threads.append(thread)
self._loaders.append(loader)
thread.start()
@@ -2198,6 +2211,8 @@ class StorePage(QWidget):
thread.started.connect(worker.run)
worker.result.connect(self._on_store_meta)
worker.finished.connect(thread.quit)
worker.finished.connect(lambda w=worker: self._on_worker_finished(w))
thread.finished.connect(lambda t=thread: self._on_thread_finished(t))
self._threads.append(thread)
self._workers.append(worker)
thread.start()
@@ -2381,7 +2396,7 @@ class StorePage(QWidget):
dlg = self._scrape_dialog
if dlg is not None:
dlg.close()
for t in self._threads:
for t in list(self._threads):
try:
if t.isRunning():
t.quit()
app/info.py
+29 -15
View File
@@ -23,6 +23,9 @@ _miss_db = {}
_miss_loaded = False
_MISS_TTL = 86400.0 # retry failed metadata after 1 day
# Memoised release-date table (invalidated on save/reset).
_dates_cache = None
def _load_miss_db():
global _miss_loaded
@@ -122,6 +125,7 @@ def save(release_id, system, meta, manual=False, title=None):
)
conn.commit()
conn.close()
_invalidate_dates_cache()
return True
except sqlite3.Error:
return False
@@ -140,6 +144,7 @@ def reset(release_id, system):
)
conn.commit()
conn.close()
_invalidate_dates_cache()
except sqlite3.Error:
pass
@@ -161,23 +166,32 @@ def cached_release_dates():
"""Return {(release_id, system): release_date} from cached metadata only.
Pure DB read — no network — so list views can show a release date without
triggering API fetches for every row.
triggering API fetches for every row. Result is memoised and invalidated
whenever metadata is saved.
"""
out = {}
try:
conn = get_connection(config.DB_PATH)
rows = conn.execute("SELECT release_id, system, meta_json FROM item_info").fetchall()
conn.close()
except sqlite3.Error:
return {}
for r in rows:
global _dates_cache
if _dates_cache is None:
out = {}
try:
meta = json.loads(r["meta_json"] or "{}")
except (TypeError, ValueError):
continue
if isinstance(meta, dict) and meta.get("release_date"):
out[(r["release_id"], r["system"])] = meta["release_date"]
return out
conn = get_connection(config.DB_PATH)
rows = conn.execute("SELECT release_id, system, meta_json FROM item_info").fetchall()
conn.close()
except sqlite3.Error:
return out
for r in rows:
try:
meta = json.loads(r["meta_json"] or "{}")
except (TypeError, ValueError):
continue
if isinstance(meta, dict) and meta.get("release_date"):
out[(r["release_id"], r["system"])] = meta["release_date"]
_dates_cache = out
return _dates_cache
def _invalidate_dates_cache():
global _dates_cache
_dates_cache = None
def get_or_fetch(release_id, system, fetch_title):
app/nopaystation.py
+28 -10
View File
@@ -131,22 +131,40 @@ def parse_tsv(text: str, content_type: str) -> List[dict]:
return items
def fetch_items(content_type: str, url: str = "", timeout: int = 60) -> List[dict]:
"""Download (and cache) a TSV, returning parsed item dicts."""
def fetch_items(content_type: str, url: str = "", timeout: int = 60, force: bool = False) -> List[dict]:
"""Download (and cache) a TSV, returning parsed item dicts.
``force`` re-downloads the TSV (used by the Store's Refresh button); the
local cache is only used as a fallback when the network fails.
"""
url = url or tsv_url(content_type)
name = _TSV_NAMES.get(content_type, "PSP_GAMES.tsv")
cache = _tsv_dir() / name
if cache.is_file():
def read_cache():
try:
return parse_tsv(cache.read_text(encoding="utf-8", errors="replace"), content_type)
except OSError:
pass
req = urllib.request.Request(url, headers={"User-Agent": "psp-preserve/0.1"})
with urllib.request.urlopen(req, timeout=timeout) as resp:
data = resp.read()
text = data.decode("utf-8", errors="replace")
cache.write_text(text, encoding="utf-8")
return parse_tsv(text, content_type)
return None
if not force and cache.is_file():
items = read_cache()
if items is not None:
return items
try:
req = urllib.request.Request(url, headers={"User-Agent": "psp-preserve/0.1"})
with urllib.request.urlopen(req, timeout=timeout) as resp:
data = resp.read()
text = data.decode("utf-8", errors="replace")
cache.write_text(text, encoding="utf-8")
return parse_tsv(text, content_type)
except Exception: # noqa: BLE001 - fall back to the cached list on failure
items = read_cache()
if items is not None:
log.warning("could not refresh %s TSV; using cached copy", content_type)
return items
raise
def find_pkg2zip(configured_path: str = "") -> Optional[Path]:
app/transfers.py
+3 -2
View File
@@ -476,15 +476,16 @@ class NoPayStationSource(Source):
label = "NoPayStation"
def __init__(self, content_type, tsv_url="", pkg2zip_path=""):
def __init__(self, content_type, tsv_url="", pkg2zip_path="", force=False):
from . import nopaystation
self.content_type = content_type
self.tsv_url = tsv_url or nopaystation.tsv_url(content_type)
self.pkg2zip = pkg2zip_path or ""
self.force = force
def scan(self, root=None):
from . import nopaystation
return nopaystation.fetch_items(self.content_type, self.tsv_url)
return nopaystation.fetch_items(self.content_type, self.tsv_url, force=self.force)
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
from . import nopaystation