quoije/psp-preservation-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Run transfer queue jobs concurrently (max 3) so downloads don't block each other
build / Linux x86_64 (push) Successful in 3m15s
Details
build / Windows x64 (Wine) (push) Successful in 3m41s
Details

Browse Source
This commit is contained in:
quoije
2026-09-14 12:50:05 -04:00
parent 0110b50a07
commit b378214e9b
1 changed files with 39 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/queue.py
+39 -36
View File
@@ -79,13 +79,13 @@ class TransferQueue(QObject):
job_finished = Signal(dict) job_finished = Signal(dict)
job_removed = Signal(int) job_removed = Signal(int)
def __init__(self, parent=None): def __init__(self, parent=None, max_concurrent=3):
super().__init__(parent) super().__init__(parent)
self.max_concurrent = max(1, int(max_concurrent))
self.jobs = [] self.jobs = []
self._pending = [] self._pending = []
self._current = None # job_id -> (QThread, JobWorker) for jobs currently running in parallel.
self._thread = None self._running = {}
self._worker = None
# Keep strong references to every worker/thread for their full lifetime. # Keep strong references to every worker/thread for their full lifetime.
# Dropping a Python ref to a QObject that was moved to another thread # Dropping a Python ref to a QObject that was moved to another thread
# deletes it directly and crashes (bus error). # deletes it directly and crashes (bus error).
@@ -115,8 +115,10 @@ class TransferQueue(QObject):
self.job_updated.emit(job.to_dict()) self.job_updated.emit(job.to_dict())
self.job_finished.emit(job.to_dict()) self.job_finished.emit(job.to_dict())
return return
if job.status == "running" and self._worker is not None: if job.status == "running":
self._worker.cancel() pair = self._running.get(job_id)
if pair is not None:
pair[1].cancel()
job.status = "cancelling" job.status = "cancelling"
self.job_updated.emit(job.to_dict()) self.job_updated.emit(job.to_dict())
@@ -140,8 +142,8 @@ class TransferQueue(QObject):
self.remove(jid) self.remove(jid)
def shutdown(self): def shutdown(self):
if self._worker is not None: for _t, worker in self._running.values():
self._worker.cancel() worker.cancel()
for t in self._threads: for t in self._threads:
try: try:
if t.isRunning(): if t.isRunning():
@@ -157,30 +159,29 @@ class TransferQueue(QObject):
return None return None
def _start_next(self): def _start_next(self):
if self._current is not None or not self._pending: while len(self._running) < self.max_concurrent and self._pending:
return job = self._pending.pop(0)
job = self._pending.pop(0) job.status = "running"
self._current = job log.info("starting job #%d: %s (%s)", job.id, job.title, job.kind)
job.status = "running" self.job_updated.emit(job.to_dict())
log.info("starting job #%d: %s (%s)", job.id, job.title, job.kind)
self.job_updated.emit(job.to_dict())
self._thread = QThread(self) thread = QThread(self)
self._worker = JobWorker(job.source, job.dest_dir, job) worker = JobWorker(job.source, job.dest_dir, job)
self._worker.moveToThread(self._thread) worker.moveToThread(thread)
self._thread.started.connect(self._worker.run) thread.started.connect(worker.run)
self._worker.progress.connect(self._on_progress) worker.progress.connect(self._on_progress)
self._worker.finished.connect(self._on_finished) worker.finished.connect(self._on_finished)
self._worker.skipped.connect(self._on_skipped) worker.skipped.connect(self._on_skipped)
self._worker.cancelled.connect(self._on_cancelled) worker.cancelled.connect(self._on_cancelled)
self._worker.failed.connect(self._on_failed) worker.failed.connect(self._on_failed)
for sig in (self._worker.finished, self._worker.skipped, for sig in (worker.finished, worker.skipped,
self._worker.cancelled, self._worker.failed): worker.cancelled, worker.failed):
sig.connect(self._thread.quit) sig.connect(thread.quit)
self._thread.finished.connect(self._on_thread_finished) thread.finished.connect(self._on_thread_finished)
self._threads.append(self._thread) self._running[job.id] = (thread, worker)
self._workers.append(self._worker) self._threads.append(thread)
self._thread.start() self._workers.append(worker)
thread.start()
def _on_progress(self, job_id, stage, fraction, detail): def _on_progress(self, job_id, stage, fraction, detail):
job = self._find(job_id) job = self._find(job_id)
@@ -218,10 +219,12 @@ class TransferQueue(QObject):
self.job_finished.emit(job.to_dict()) self.job_finished.emit(job.to_dict())
def _on_thread_finished(self): def _on_thread_finished(self):
# Queued to the main thread once the worker thread has fully stopped, # Queued to the main thread once a worker thread has fully stopped, so
# so it is now safe to drop the "current" pointers and advance. # it is now safe to drop its pointers and advance the queue.
self._current = None thread = self.sender()
self._thread = None for jid, pair in list(self._running.items()):
self._worker = None if pair[0] is thread:
del self._running[jid]
break
self._start_next() self._start_next()