Compare commits

..
2 Commits
Author SHA1 Message Date
quoije 99e9cdc49a Prevent 'Too many open files' crash on store source switch
build / Linux x86_64 (push) Successful in 3m53s
build / Windows x64 (Wine) (push) Successful in 4m9s
- Raise the process soft file-descriptor limit to the hard limit at startup
  (GLib aborts the process with 'Creating pipes for GWakeup: Too many open
  files' when cover/metadata loading exhausts a 1024-soft-limit environment).
- Reduce per-loader network concurrency 8 -> 6 threads to lower socket
  pressure against the store server.
2026-09-14 23:11:51 -04:00
quoije a4d132f134 Harden worker cleanup + reduce cover-loading churn
build / Linux x86_64 (push) Successful in 3m30s
build / Windows x64 (Wine) (push) Successful in 4m22s
- Library/metadata workers: delete the worker on its own thread via
  worker.finished -> worker.deleteLater (bound) instead of letting Python GC
  delete it on the GUI thread after untracking.
- Cover templates: compute the transparent-window scan outside the lock so a
  slow first scan never holds the lock and stalls other cover threads.
- Store: larger cover/date batches (500/400/300) so the store loads 3449
  archives with far fewer QThread + ThreadPoolExecutor creations.
2026-09-14 22:33:49 -04:00
2 changed files with 66 additions and 44 deletions
+6 -4
View File
@@ -797,9 +797,8 @@ def _transparent_rect(img):
def _game_template(): def _game_template():
"""Load the PSP game cover template; return ``(template, window)``.""" """Load the PSP game cover template; return ``(template, window)``."""
global _game_template_cache global _game_template_cache
if _game_template_cache is None: if _game_template_cache is not None:
with _game_template_lock: return _game_template_cache
if _game_template_cache is None:
try: try:
from PySide6.QtGui import QImage from PySide6.QtGui import QImage
except Exception: # noqa: BLE001 - headless API may lack PySide6 except Exception: # noqa: BLE001 - headless API may lack PySide6
@@ -815,10 +814,13 @@ def _game_template():
if not img.isNull(): if not img.isNull():
break break
img = img.convertToFormat(QImage.Format_ARGB32) if not img.isNull() else None img = img.convertToFormat(QImage.Format_ARGB32) if not img.isNull() else None
_game_template_cache = ( entry = (
img, img,
_transparent_rect(img) if img is not None and not img.isNull() else (0, 0, 0, 0), _transparent_rect(img) if img is not None and not img.isNull() else (0, 0, 0, 0),
) )
with _game_template_lock:
if _game_template_cache is None:
_game_template_cache = entry
return _game_template_cache return _game_template_cache
+32 -12
View File
@@ -136,7 +136,7 @@ class CoverLoader(QObject):
def run(self): def run(self):
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
pool = ThreadPoolExecutor(max_workers=8) pool = ThreadPoolExecutor(max_workers=6)
futures = { futures = {
pool.submit(load_cover, it): i pool.submit(load_cover, it): i
for i, it in enumerate(self.items) for i, it in enumerate(self.items)
@@ -172,9 +172,8 @@ _template_lock = threading.Lock()
def _umd_template(): def _umd_template():
"""Return ``(template QImage, transparent window rect)`` — loaded once.""" """Return ``(template QImage, transparent window rect)`` — loaded once."""
global _template_cache global _template_cache
if _template_cache is None: if _template_cache is not None:
with _template_lock: return _template_cache
if _template_cache is None:
img = QImage() img = QImage()
for path in ( for path in (
config.COVERS_DIR / "UMD_COVER_TEMPLATE.png", config.COVERS_DIR / "UMD_COVER_TEMPLATE.png",
@@ -185,7 +184,10 @@ def _umd_template():
if not img.isNull(): if not img.isNull():
break break
img = img.convertToFormat(QImage.Format_ARGB32) img = img.convertToFormat(QImage.Format_ARGB32)
_template_cache = (img, _transparent_window(img)) entry = (img, _transparent_window(img))
with _template_lock:
if _template_cache is None:
_template_cache = entry
return _template_cache return _template_cache
@@ -1632,7 +1634,7 @@ class StoreCoverLoader(QObject):
def run(self): def run(self):
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ThreadPoolExecutor, as_completed
pool = ThreadPoolExecutor(max_workers=8) pool = ThreadPoolExecutor(max_workers=6)
futures = { futures = {
pool.submit(load_cover, it, 8, self.quick): row pool.submit(load_cover, it, 8, self.quick): row
for row, it in self.tasks for row, it in self.tasks
@@ -2285,11 +2287,11 @@ class StorePage(QWidget):
return (0, -date_int, -mtime) return (0, -date_int, -mtime)
return (1, 0, -mtime) return (1, 0, -mtime)
CHUNK = 150 # items filled per timer tick CHUNK = 200 # items filled per timer tick
COVER_BATCH = 200 # covers fetched for the initially-visible area COVER_BATCH = 500 # covers fetched for the initially-visible area
COVER_SCROLL_BATCH = 100 COVER_SCROLL_BATCH = 400
RECENT_LIMIT = 300 # default: show only the newest N store items RECENT_LIMIT = 300 # default: show only the newest N store items
DATE_BATCH = 150 # release-date metadata fetched per batch DATE_BATCH = 300 # release-date metadata fetched per batch
def _populate(self, items, show_count): def _populate(self, items, show_count):
self._generation += 1 self._generation += 1
@@ -4876,7 +4878,9 @@ class MainWindow(QMainWindow):
thread._worker = worker thread._worker = worker
thread.started.connect(worker.run) thread.started.connect(worker.run)
worker.result.connect(self._on_library_loaded) worker.result.connect(self._on_library_loaded)
worker.finished.connect(worker.deleteLater)
worker.finished.connect(thread.quit) worker.finished.connect(thread.quit)
thread.finished.connect(thread.deleteLater)
thread.finished.connect(self._forget_library_thread) thread.finished.connect(self._forget_library_thread)
self._library_threads.append(thread) self._library_threads.append(thread)
self._library_workers.append(worker) self._library_workers.append(worker)
@@ -4898,7 +4902,6 @@ class MainWindow(QMainWindow):
self._library_threads.remove(thread) self._library_threads.remove(thread)
if worker is not None and worker in self._library_workers: if worker is not None and worker in self._library_workers:
self._library_workers.remove(worker) self._library_workers.remove(worker)
thread.deleteLater()
def _apply_library_filter(self): def _apply_library_filter(self):
if not hasattr(self, "_library_cache"): if not hasattr(self, "_library_cache"):
@@ -5034,7 +5037,9 @@ class MainWindow(QMainWindow):
thread._worker = worker thread._worker = worker
thread.started.connect(worker.run) thread.started.connect(worker.run)
worker.result.connect(self._on_metadata) worker.result.connect(self._on_metadata)
worker.finished.connect(worker.deleteLater)
worker.finished.connect(thread.quit) worker.finished.connect(thread.quit)
thread.finished.connect(thread.deleteLater)
thread.finished.connect(self._forget_meta_thread) thread.finished.connect(self._forget_meta_thread)
self._meta_threads.append(thread) self._meta_threads.append(thread)
self._meta_workers.append(worker) self._meta_workers.append(worker)
@@ -5047,7 +5052,6 @@ class MainWindow(QMainWindow):
self._meta_threads.remove(thread) self._meta_threads.remove(thread)
if worker is not None and worker in self._meta_workers: if worker is not None and worker in self._meta_workers:
self._meta_workers.remove(worker) self._meta_workers.remove(worker)
thread.deleteLater()
def _on_metadata(self, gen, meta): def _on_metadata(self, gen, meta):
if gen != self._meta_gen: if gen != self._meta_gen:
@@ -5950,8 +5954,24 @@ class MainWindow(QMainWindow):
event.accept() event.accept()
def _raise_nofile_limit():
"""Raise the soft file-descriptor limit to the hard limit.
Cover/metadata loading opens many short-lived sockets, and each worker
QThread's GLib event dispatcher creates a wakeup pipe. On systems with a
1024 soft limit this aborts the process with GLib's "Too many open files".
"""
try:
import resource
_soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
resource.setrlimit(resource.RLIMIT_NOFILE, (hard, hard))
except Exception: # noqa: BLE001 - best effort, never block startup
pass
def main(): def main():
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(name)-14s %(message)s") logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(name)-14s %(message)s")
_raise_nofile_limit()
init_db() # migrate legacy on-share DB/covers to local storage, then schema init_db() # migrate legacy on-share DB/covers to local storage, then schema
# Auto-ingest any missing DATs so scanning matches against the catalog # Auto-ingest any missing DATs so scanning matches against the catalog
# even on a fresh/reset database (no manual `python -m app.ingest` step). # even on a fresh/reset database (no manual `python -m app.ingest` step).