Phase 2: PS1 library scan, settings, conversion queue, and PS1 tab

This commit is contained in:
2026-09-07 16:20:39 -04:00
parent bb1b8dea66
commit 743d5e8e90
8 changed files with 387 additions and 9 deletions
+7
View File
@@ -50,10 +50,17 @@ ARCHIVE_LOGGED_IN_USER = os.environ.get("PSP_ARCHIVE_LOGGED_IN_USER", "")
GAMES_ROOTS = [Path(p) for p in os.environ.get("PSP_GAMES_ROOTS", "/data/umd_games").split(",") if p.strip()]
VIDEO_ROOTS = [Path(p) for p in os.environ.get("PSP_VIDEO_ROOTS", "/data/umd_videos").split(",") if p.strip()]
# PS1: source folder (raw ISO/BIN/CUE) and converted EBOOT.PBP output folder.
PS1_ROOTS = [Path(p) for p in os.environ.get("PSP_PS1_ROOTS", "/data/ps1").split(",") if p.strip()]
PS1_OUTPUT_DIR = Path(os.environ.get("PSP_PS1_OUTPUT_DIR", "/data/ps1_converted"))
# Only .iso discs are tracked; everything below is explicitly ignored.
DISC_EXTENSIONS = {".iso"}
IGNORED_EXTENSIONS = {".pkg", ".pbp", ".cso"}
# PS1 source images (scanned separately from PSP .iso discs).
PS1_EXTENSIONS = {".cue", ".iso", ".img", ".bin"}
# Parallel hashing workers for the scanner. Keep at 1 on spinning disks
# (concurrent reads cause seek thrash); raise it for SSD / fast arrays.
SCAN_WORKERS = int(os.environ.get("PSP_SCAN_WORKERS", "1"))
+14
View File
@@ -101,6 +101,20 @@ CREATE TABLE IF NOT EXISTS save_sigs (
sig TEXT NOT NULL,
PRIMARY KEY (save_id, size, mtime_ns)
);
CREATE TABLE IF NOT EXISTS ps1_entries (
id INTEGER PRIMARY KEY,
source_path TEXT NOT NULL UNIQUE,
title TEXT,
serial TEXT,
region TEXT,
disc INTEGER NOT NULL DEFAULT 1,
disc_total INTEGER NOT NULL DEFAULT 1,
output_eboot TEXT,
status TEXT NOT NULL DEFAULT 'pending',
error TEXT,
updated_at TEXT NOT NULL
);
"""
+154 -6
View File
@@ -14,13 +14,13 @@ from PySide6.QtCore import QBuffer, QByteArray, QObject, QRect, QSettings, QSize
from PySide6.QtGui import QColor, QDesktopServices, QFont, QIcon, QImage, QPainter, QPixmap
from PySide6.QtWidgets import (
QAbstractItemView, QApplication, QCheckBox, QComboBox, QDialog, QDialogButtonBox,
QFileDialog, QFormLayout, QGridLayout, QHBoxLayout, QHeaderView, QLabel, QLineEdit,
QFileDialog, QFormLayout, QGridLayout, QHBoxLayout, QHeaderView, QInputDialog, QLabel, QLineEdit,
QListWidget, QListWidgetItem, QMainWindow, QMenu, QMessageBox, QPlainTextEdit,
QProgressBar, QProgressDialog, QPushButton, QScrollArea, QSpinBox, QSplitter, QStackedWidget,
QTabWidget, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget,
)
from . import config, covers, info, iso_reader, launcher, matcher, namer, pipeline, psp, queue, saves, scanner, sources, store, transfers
from . import config, covers, info, iso_reader, launcher, matcher, namer, pipeline, ps1, psp, queue, saves, scanner, sources, store, transfers
from .db import get_connection, init_db
log = logging.getLogger("gui")
@@ -1011,7 +1011,7 @@ class SettingsPage(QWidget):
def __init__(self, games_dir, videos_dir, store_dir, ppsspp_dir, cloud_dir, ppsspp_exe,
store_server_url, store_token, store_archive_id, store_archive_sig, store_archive_user,
iso_category, parent=None):
iso_category, ps1_dir, ps1_output_dir, ps1_base_pbp, parent=None):
super().__init__(parent)
outer = QVBoxLayout(self)
outer.setContentsMargins(0, 0, 0, 0)
@@ -1025,6 +1025,9 @@ class SettingsPage(QWidget):
page, form = self._new_form()
self.games_edit = self._dir_row(form, "Games directory:", games_dir)
self.videos_edit = self._dir_row(form, "Videos directory:", videos_dir)
self.ps1_edit = self._dir_row(form, "PS1 directory:", ps1_dir)
self.ps1_out_edit = self._dir_row(form, "PS1 output directory:", ps1_output_dir)
self.ps1_base_edit = self._file_row(form, "PS1 base.pbp (PSN classic):", ps1_base_pbp)
self.auto_scan = QCheckBox("Scan directories automatically on startup")
self.auto_scan.setChecked(str(s.value("auto_scan", False)).lower() == "true")
form.addRow(self.auto_scan)
@@ -1156,7 +1159,7 @@ class SettingsPage(QWidget):
return edit
def _browse_file(self, edit):
path, _ = QFileDialog.getOpenFileName(self, "Select PPSSPP executable")
path, _ = QFileDialog.getOpenFileName(self, "Select file")
if path:
edit.setText(path)
@@ -1210,6 +1213,15 @@ class SettingsPage(QWidget):
def iso_category(self):
return self.iso_category_edit.text().strip()
def ps1_dir(self):
return self.ps1_edit.text().strip()
def ps1_output_dir(self):
return self.ps1_out_edit.text().strip()
def ps1_base_pbp(self):
return self.ps1_base_edit.text().strip()
def _save(self):
sources.save({key: edit.text() for key, edit in self.fields.items()})
covers.clear_miss_cache()
@@ -3220,6 +3232,113 @@ class SavesPage(QWidget):
self._workers = []
class Ps1ScanWorker(QObject):
results = Signal(list)
failed = Signal(str)
def __init__(self, ps1_dir, output_dir):
super().__init__()
self.ps1_dir = ps1_dir
self.output_dir = output_dir
def run(self):
try:
self.results.emit(ps1.scan_library(self.ps1_dir, self.output_dir))
except Exception as exc: # noqa: BLE001
self.failed.emit(str(exc))
class Ps1Page(QWidget):
"""PS1 tab: list raw source discs, converted EBOOTs and PSN games."""
convert_requested = Signal(dict)
def __init__(self, parent=None):
super().__init__(parent)
self._ps1_dir = ""
self._output_dir = ""
self._entries = []
self._thread = None
self._worker = None
layout = QVBoxLayout(self)
bar = QHBoxLayout()
self.status = QLabel("PS1 discs — set the PS1 folder in Settings → General.")
bar.addWidget(self.status, 1)
self.refresh_btn = QPushButton("Refresh")
self.refresh_btn.clicked.connect(self.refresh)
self.convert_btn = QPushButton("Convert selected")
self.convert_btn.clicked.connect(self._convert_selected)
bar.addWidget(self.refresh_btn)
bar.addWidget(self.convert_btn)
layout.addLayout(bar)
self.table = QTableWidget(0, 4)
self.table.setHorizontalHeaderLabels(["Type", "Name", "Serial", "Status"])
self.table.setSelectionBehavior(QAbstractItemView.SelectRows)
self.table.setEditTriggers(QAbstractItemView.NoEditTriggers)
self.table.verticalHeader().setVisible(False)
self.table.horizontalHeader().setSectionResizeMode(1, QHeaderView.Stretch)
layout.addWidget(self.table, 1)
def set_dirs(self, ps1_dir, output_dir):
self._ps1_dir = ps1_dir
self._output_dir = output_dir
def refresh(self):
if not self._ps1_dir:
self.status.setText("PS1 folder not set — configure it in Settings → General.")
self.table.setRowCount(0)
self._entries = []
return
if self._thread is not None and _thread_running(self._thread):
return
self.status.setText("Scanning PS1 folders…")
self._worker = Ps1ScanWorker(self._ps1_dir, self._output_dir)
self._thread = QThread(self)
self._worker.moveToThread(self._thread)
self._thread.started.connect(self._worker.run)
self._worker.results.connect(self._on_results)
self._worker.failed.connect(self._on_failed)
self._worker.results.connect(self._thread.quit)
self._worker.failed.connect(self._thread.quit)
self._thread.start()
def _on_results(self, entries):
self._entries = entries
self.table.setRowCount(len(entries))
for row, e in enumerate(entries):
kind = "Disc" if e["kind"] == "disc" else "EBOOT"
self.table.setItem(row, 0, QTableWidgetItem(kind))
self.table.setItem(row, 1, QTableWidgetItem(e["name"]))
self.table.setItem(row, 2, QTableWidgetItem(e.get("serial") or ""))
self.table.setItem(row, 3, QTableWidgetItem(e.get("status") or "pending"))
pending = sum(1 for e in entries if e["kind"] == "disc" and e["status"] != "converted")
self.status.setText(f"{len(entries)} item(s) — {pending} ready to convert")
def _on_failed(self, message):
self.status.setText(f"PS1 scan failed: {message}")
def _convert_selected(self):
row = self.table.currentRow()
if row < 0 or row >= len(self._entries):
return
e = self._entries[row]
if e["kind"] != "disc":
QMessageBox.information(self, "PS1", "Select a raw PS1 disc (Disc row), not an EBOOT.")
return
if e["status"] == "converted":
QMessageBox.information(self, "PS1", "This disc already has a converted EBOOT.")
return
game_id = e.get("serial") or ""
if not game_id:
text, ok = QInputDialog.getText(self, "PS1", "Game ID (e.g. SLUS00594):")
if not ok or not text.strip():
return
game_id = text.strip()
self.convert_requested.emit({"source": e["path"], "title": e["name"], "game_id": game_id})
class MainWindow(QMainWindow):
def __init__(self):
super().__init__()
@@ -3259,6 +3378,9 @@ class MainWindow(QMainWindow):
self._auto_sync_saves = str(self.settings.value("auto_sync_saves", "false")).lower() == "true"
self._ppsspp_exe = str(self.settings.value("ppsspp_exe", "") or "")
self._iso_category = str(self.settings.value("iso_category", "") or "")
self._ps1_dir = self._first_existing(config.PS1_ROOTS) or str(self.settings.value("ps1_dir", "") or "")
self._ps1_output_dir = str(self.settings.value("ps1_output_dir", str(config.PS1_OUTPUT_DIR)) or "")
self._ps1_base_pbp = str(self.settings.value("ps1_base_pbp", "") or "")
log.info("games dir: %s", self._games_dir or "(none)")
log.info("videos dir: %s", self._videos_dir or "(none)")
log.info("store: %s", self._store_server_url or self._store_dir or "(none)")
@@ -3291,6 +3413,10 @@ class MainWindow(QMainWindow):
self.saves_page = SavesPage()
self.saves_page.sync_requested.connect(self._on_saves_sync)
self.tabs.addTab(self.saves_page, "Saves")
self.ps1_page = Ps1Page()
self.ps1_page.set_dirs(self._ps1_dir, self._ps1_output_dir)
self.ps1_page.convert_requested.connect(self._on_ps1_convert)
self.tabs.addTab(self.ps1_page, "PS1")
self.tabs.addTab(self._build_settings_tab(), "Settings")
self.tabs.currentChanged.connect(self._on_tab_changed)
self._on_psp_device_changed(self.psp_page.root())
@@ -3461,7 +3587,8 @@ class MainWindow(QMainWindow):
self.settings_page = SettingsPage(self._games_dir, self._videos_dir, self._store_dir,
self._ppsspp_saves_dir, self._cloud_saves_dir, self._ppsspp_exe,
self._store_server_url, self._store_token, self._store_archive_id,
self._store_archive_sig, self._store_archive_user, self._iso_category)
self._store_archive_sig, self._store_archive_user, self._iso_category,
self._ps1_dir, self._ps1_output_dir, self._ps1_base_pbp)
self.settings_page.saved.connect(self._on_settings_saved)
return self.settings_page
@@ -3490,6 +3617,13 @@ class MainWindow(QMainWindow):
self._auto_sync_saves = self.settings_page.auto_sync_saves()
self._ppsspp_exe = self.settings_page.ppsspp_exe()
self._iso_category = self.settings_page.iso_category()
self._ps1_dir = self.settings_page.ps1_dir()
self._ps1_output_dir = self.settings_page.ps1_output_dir()
self._ps1_base_pbp = self.settings_page.ps1_base_pbp()
self.settings.setValue("ps1_dir", self._ps1_dir)
self.settings.setValue("ps1_output_dir", self._ps1_output_dir)
self.settings.setValue("ps1_base_pbp", self._ps1_base_pbp)
self.ps1_page.set_dirs(self._ps1_dir, self._ps1_output_dir)
self.settings.setValue("ppsspp_saves_dir", self._ppsspp_saves_dir)
self.settings.setValue("cloud_saves_dir", self._cloud_saves_dir)
self.settings.setValue("auto_sync_saves", self._auto_sync_saves)
@@ -3499,7 +3633,8 @@ class MainWindow(QMainWindow):
self._update_dir_status()
def _on_tab_changed(self, index):
pass # store builds eagerly and caches its view; no per-tab work needed
if hasattr(self, "ps1_page") and index == self.tabs.indexOf(self.ps1_page):
self.ps1_page.refresh()
def _update_saves_locations(self):
if not hasattr(self, "saves_page"):
@@ -3984,6 +4119,17 @@ class MainWindow(QMainWindow):
if hasattr(self, "psp_page"):
self.psp_page.refresh_inventory()
def _on_ps1_convert(self, item):
if not self._ps1_base_pbp or not Path(self._ps1_base_pbp).is_file():
QMessageBox.warning(self, "PS1", "Set a valid PS1 base.pbp path in Settings → General first.")
return
if not item.get("game_id"):
QMessageBox.warning(self, "PS1", "No game ID for this disc.")
return
out_dir = Path(self._ps1_output_dir) if self._ps1_output_dir else Path.cwd() / "ps1_converted"
source = transfers.Ps1ConvertSource(self._ps1_base_pbp)
self.queue.enqueue(source, out_dir, item, item.get("title") or item["source"], kind="ps1-convert")
def _on_queue_job_done(self, job):
if self._closing:
return
@@ -4004,6 +4150,8 @@ class MainWindow(QMainWindow):
QTimer.singleShot(0, self._refresh_psp)
if kind == "save-sync":
QTimer.singleShot(0, self._refresh_saves)
if kind == "ps1-convert":
QTimer.singleShot(0, self.ps1_page.refresh)
# -- Common ------------------------------------------------------------ #
@staticmethod
+3
View File
@@ -24,6 +24,9 @@ def detect_system(dat_name):
return "videos"
if "PlayStation Portable" in dat_name:
return "games"
# Redump PS1: "Sony - PlayStation" (assumes no PS2/PS3 DATs are present).
if "PlayStation" in dat_name:
return "ps1"
return "unknown"
+1 -1
View File
@@ -15,7 +15,7 @@ _ARTICLE_RE = re.compile(r"^(.+),\s*(the|a|an)(?:\s+-\s+(.+))?$")
def _normalize(name):
name = name.lower()
name = re.sub(r"\.(iso|cso|pbp|img|bin)$", "", name) # strip disc extension
name = re.sub(r"\.(iso|cso|pbp|img|bin|cue)$", "", name) # strip disc extension
name = re.sub(r"^\s*\d+\s*-\s*", "", name) # legacy No-Intro "#### - Name" catalog number
name = re.sub(r"\([^)]*\)", " ", name) # parenthetical tags
name = " ".join(name.split())
+2 -2
View File
@@ -62,7 +62,7 @@ def _reorder_article(title):
return title
def canonical_name(release, serial):
def canonical_name(release, serial, ext=".iso"):
name = release["name"]
region = release.get("region")
if region:
@@ -78,4 +78,4 @@ def canonical_name(release, serial):
parts.append(f"({serial})")
name = _ILLEGAL.sub(" ", " ".join(parts)).strip()
name = re.sub(r"\s+", " ", name)
return f"{name}.iso"
return f"{name}{ext}"
+158
View File
@@ -0,0 +1,158 @@
"""PS1 library helpers: discover source discs and converted EBOOT.PBPs.
This is the read/scan side of PS1 support (the conversion side lives in
``app.ps1_convert``). It finds raw PS1 disc images (``.cue`` sets and
standalone ``.iso``/``.img``), reads each disc's boot serial, and lists
converted ``EBOOT.PBP`` files in the output folder.
"""
from __future__ import annotations
import re
from pathlib import Path
from typing import List, Optional
from . import config
RAW_SECTOR = 2352
COOKED_SECTOR = 2048
EBOOT_NAME = "EBOOT.PBP"
# SYSTEM.CNF boot line: cdrom:\SLPS_017.62;1 (backslash optional in some rips)
_SERIAL_RE = re.compile(rb"cdrom:\\?([A-Za-z]{4})[_-]?(\d{3})\.(\d{2})")
def _sector_size(path: Path) -> int:
size = path.stat().st_size
if size and size % RAW_SECTOR == 0:
return RAW_SECTOR
return COOKED_SECTOR
def read_serial(image_path: Path) -> Optional[str]:
"""Read the boot serial (e.g. ``SLPS01762``) from a PS1 disc image.
Works on both raw 2352-byte/sector images (BIN/IMG) and cooked 2048-byte
images (ISO). Returns ``None`` if the boot line can't be found.
"""
image_path = Path(image_path)
if not image_path.is_file():
return None
sector_size = _sector_size(image_path)
data_off = 24 if sector_size == RAW_SECTOR else 0
buf = bytearray()
with open(image_path, "rb") as fh:
for _ in range(128): # SYSTEM.CNF is always near the start of the disc
sector = fh.read(sector_size)
if len(sector) < sector_size:
break
buf += sector[data_off:data_off + COOKED_SECTOR]
m = _SERIAL_RE.search(bytes(buf))
if m:
return (m.group(1) + m.group(2) + m.group(3)).decode().upper()
return None
def cue_data_track(cue_path: Path) -> Optional[Path]:
"""Return the first data-track image referenced by a cue sheet."""
from . import ps1_convert
try:
disc = ps1_convert.parse_cue(cue_path.read_text(encoding="utf-8", errors="replace"))
except Exception:
return None
track = disc.data_track
if track is None:
return None
return (cue_path.parent / track.file) if track.file else None
def iter_source_discs(root: Path):
"""Yield PS1 source discs under ``root`` (cue sheets + standalone images)."""
root = Path(root)
if not root.is_dir():
return
for path in sorted(root.rglob("*")):
if not path.is_file():
continue
suffix = path.suffix.lower()
if suffix == ".cue":
yield path
elif suffix in (".iso", ".img"):
# A standalone image: no cue sheet next to it (same stem).
if not (path.parent / f"{path.stem}.cue").exists():
yield path
def find_eboots(root: Path) -> List[Path]:
"""List converted/PSN ``EBOOT.PBP`` files under ``root``."""
root = Path(root)
if not root.is_dir():
return []
return [p for p in sorted(root.rglob("*")) if p.is_file() and p.name.upper() == EBOOT_NAME]
def eboot_title(pbp_path: Path) -> str:
"""Read the game title embedded in an EBOOT.PBP (falls back to folder name)."""
from . import iso_reader, ps1_convert
try:
header = ps1_convert.read_pbp_header(pbp_path)
idx = ps1_convert.SECTION_OFFSET["sfo"] // 4
sfo = ps1_convert.read_pbp_section(pbp_path, header, idx)
parsed = iso_reader.parse_sfo(sfo)
title = parsed.get("TITLE") or parsed.get("TITLE_0") or parsed.get("TITLE_00")
if title:
return str(title)
except Exception:
pass
return pbp_path.parent.name
def scan_library(ps1_dir, output_dir) -> List[dict]:
"""Return a combined view of PS1 source discs and converted EBOOTs.
Each entry is a dict with ``kind`` ("disc" or "eboot"), ``path``, ``name``,
``serial`` and ``status`` ("pending" / "converted").
"""
ps1_dir = Path(ps1_dir)
output_dir = Path(output_dir)
eboots = find_eboots(output_dir)
eboot_names = {e.name.upper() for e in eboots}
eboot_titles = {eboot_title(e).lower() for e in eboots}
entries: List[dict] = []
for src in iter_source_discs(ps1_dir):
if src.suffix.lower() == ".cue":
image = cue_data_track(src) or src
else:
image = src
serial = read_serial(image)
title = src.stem
# A disc is "converted" when an EBOOT's embedded title or filename
# matches the source disc's title/serial.
status = "pending"
if title.lower() in eboot_titles or (serial and any(serial.lower() in t for t in eboot_titles)):
status = "converted"
entries.append({
"kind": "disc",
"path": str(src),
"name": title,
"serial": serial or "",
"status": status,
})
for e in eboots:
entries.append({
"kind": "eboot",
"path": str(e),
"name": eboot_title(e),
"serial": "",
"status": "converted",
})
return entries
+48
View File
@@ -6,6 +6,7 @@ archives, a remote HTTPS store server, etc.
import json
import logging
import os
import re
import shutil
import tempfile
import urllib.parse
@@ -200,6 +201,53 @@ class FileCopySource(Source):
return dest
class Ps1ConvertSource(Source):
"""Convert a PS1 disc image (.cue/.iso) into an EBOOT.PBP.
``item`` must carry ``source`` (path to the .cue or single image),
``title`` and ``game_id``. ``base_pbp`` is a PSN PS1 classic EBOOT used to
supply the encrypted DATA.PSP loader and STARTDAT block.
"""
label = "PS1 convert"
def __init__(self, base_pbp):
self.base_pbp = Path(base_pbp)
def scan(self, root=None):
return []
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
dest_dir = Path(dest_dir)
dest_dir.mkdir(parents=True, exist_ok=True)
source = Path(item["source"])
title = str(item.get("title") or source.stem)
game_id = item.get("game_id")
if not game_id:
raise RuntimeError("missing game_id for PS1 conversion")
safe = re.sub(r'[<>:"/\\|?*]', " ", title).strip() or "PS1 game"
out_path = dest_dir / f"{safe} ({game_id}).pbp"
if out_path.exists():
raise FileExistsError(str(out_path))
def cb(done, total):
if progress_cb is not None and total:
progress_cb("Converting", min(done / total, 1.0), "")
from . import ps1_convert
try:
ps1_convert.convert(
source, title, game_id, self.base_pbp, out_path,
progress_cb=cb, cancel_event=cancel_event,
)
except ps1_convert.Cancelled:
raise TransferCancelled() from None
return out_path
class TreeCopySource(Source):
"""Copy a whole directory (save-data folder) into a destination directory."""