Initial commit
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""PSP Preservation Tool application package."""
|
||||
+346
@@ -0,0 +1,346 @@
|
||||
"""Zero-dependency HTTP API for browsing the PSP catalog.
|
||||
|
||||
Run with::
|
||||
|
||||
python3 -m app.api [--host 0.0.0.0] [--port 8000]
|
||||
|
||||
Endpoints:
|
||||
GET /api/summary counts per system (games/videos)
|
||||
GET /api/releases list; ?system=games&q=wipeout&limit=50&offset=0
|
||||
GET /api/releases/<id> single release + its rom hashes
|
||||
GET /api/store list archives in the 7z store (token-protected)
|
||||
GET /api/store/download/<name> stream one 7z archive (token-protected)
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
|
||||
from . import config, covers, pipeline, scanner, store
|
||||
from .db import get_connection, init_db
|
||||
|
||||
ROOTS = []
|
||||
_LIB_CACHE = {"ts": 0.0, "items": []}
|
||||
_LIB_LOCK = threading.Lock()
|
||||
_LIB_TTL = 60.0 # seconds a /api/library result stays fresh
|
||||
API_TOKEN = os.environ.get("PSP_API_TOKEN", "").strip()
|
||||
|
||||
|
||||
def _authorized(handler):
|
||||
"""True when the request carries a valid token (or no token is required)."""
|
||||
if not API_TOKEN:
|
||||
return True
|
||||
auth = (handler.headers.get("Authorization") or "").strip()
|
||||
return auth in (f"Bearer {API_TOKEN}", API_TOKEN)
|
||||
|
||||
|
||||
def _json(handler, obj, status=200):
|
||||
body = json.dumps(obj, indent=2).encode("utf-8")
|
||||
handler.send_response(status)
|
||||
handler.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
handler.send_header("Content-Length", str(len(body)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(body)
|
||||
|
||||
|
||||
def _qint(query, key, default):
|
||||
try:
|
||||
return int(query.get(key, [str(default)])[0])
|
||||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
def _cover_redirect(handler, conn, rid):
|
||||
row = conn.execute(
|
||||
"SELECT r.name, d.system FROM releases r JOIN dats d ON d.id = r.dat_id "
|
||||
"WHERE r.id = ?", (rid,)
|
||||
).fetchone()
|
||||
if row is None:
|
||||
_json(handler, {"error": "not found"}, 404)
|
||||
return
|
||||
path = covers.fetch_cover(row["name"], rid, row["system"], timeout=15)
|
||||
if path is None or not path.exists():
|
||||
_json(handler, {"error": "no cover available"}, 404)
|
||||
return
|
||||
body = path.read_bytes()
|
||||
ctype = "image/jpeg" if body.startswith(b"\xff\xd8\xff") else "image/png"
|
||||
handler.send_response(200)
|
||||
handler.send_header("Content-Type", ctype)
|
||||
handler.send_header("Content-Length", str(len(body)))
|
||||
handler.end_headers()
|
||||
handler.wfile.write(body)
|
||||
|
||||
|
||||
def _summary(conn):
|
||||
rows = conn.execute(
|
||||
"SELECT d.system, COUNT(*) AS releases FROM releases r "
|
||||
"JOIN dats d ON d.id = r.dat_id GROUP BY d.system ORDER BY d.system"
|
||||
).fetchall()
|
||||
return {"systems": [dict(r) for r in rows]}
|
||||
|
||||
|
||||
def _releases(conn, system=None, q=None, limit=50, offset=0):
|
||||
sql = (
|
||||
"SELECT r.id, d.system, r.name, r.region, r.serial "
|
||||
"FROM releases r JOIN dats d ON d.id = r.dat_id"
|
||||
)
|
||||
where, params = [], []
|
||||
if system:
|
||||
where.append("d.system = ?")
|
||||
params.append(system)
|
||||
if q:
|
||||
where.append("r.name LIKE ?")
|
||||
params.append(f"%{q}%")
|
||||
if where:
|
||||
sql += " WHERE " + " AND ".join(where)
|
||||
sql += " ORDER BY r.name LIMIT ? OFFSET ?"
|
||||
params += [limit, offset]
|
||||
rows = conn.execute(sql, params).fetchall()
|
||||
items = []
|
||||
for r in rows:
|
||||
d = dict(r)
|
||||
d["cover"] = f"/api/cover/{d['id']}"
|
||||
items.append(d)
|
||||
return {"items": items, "limit": limit, "offset": offset}
|
||||
|
||||
|
||||
def _release_detail(conn, rid):
|
||||
release = conn.execute(
|
||||
"SELECT r.id, d.system, r.name, r.description, r.region, r.serial "
|
||||
"FROM releases r JOIN dats d ON d.id = r.dat_id WHERE r.id = ?",
|
||||
(rid,),
|
||||
).fetchone()
|
||||
if release is None:
|
||||
return None
|
||||
roms = conn.execute(
|
||||
"SELECT name, size, crc32, md5, sha1, serial FROM roms WHERE release_id = ?",
|
||||
(rid,),
|
||||
).fetchall()
|
||||
result = dict(release)
|
||||
result["cover"] = f"/api/cover/{release['id']}"
|
||||
result["roms"] = [dict(r) for r in roms]
|
||||
return result
|
||||
|
||||
|
||||
def _store_listing():
|
||||
items = [
|
||||
{
|
||||
"name": it["name"],
|
||||
"filename": Path(it["path"]).name,
|
||||
"size": it["size"],
|
||||
"mtime_ns": it["mtime_ns"],
|
||||
}
|
||||
for it in store.scan(config.STORE_ROOT)
|
||||
]
|
||||
return {"items": items, "count": len(items)}
|
||||
|
||||
|
||||
def _store_download(handler, name):
|
||||
root = Path(config.STORE_ROOT).resolve()
|
||||
target = (Path(config.STORE_ROOT) / name).resolve()
|
||||
if not target.is_file() or root not in target.parents:
|
||||
_json(handler, {"error": "not found"}, 404)
|
||||
return
|
||||
size = target.stat().st_size
|
||||
handler.send_response(200)
|
||||
handler.send_header("Content-Type", "application/octet-stream")
|
||||
handler.send_header("Content-Length", str(size))
|
||||
handler.send_header("Content-Disposition", f'attachment; filename="{Path(name).name}"')
|
||||
handler.end_headers()
|
||||
with open(target, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(1 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
handler.wfile.write(chunk)
|
||||
|
||||
|
||||
def _covers_download(handler, name):
|
||||
"""Stream an offline cover bundle zip from ``data/covers/`` (public)."""
|
||||
root = (Path(config.DATA_DIR) / "covers").resolve()
|
||||
target = (root / name).resolve()
|
||||
if not target.is_file() or root not in target.parents:
|
||||
_json(handler, {"error": "not found"}, 404)
|
||||
return
|
||||
size = target.stat().st_size
|
||||
handler.send_response(200)
|
||||
handler.send_header("Content-Type", "application/zip")
|
||||
handler.send_header("Content-Length", str(size))
|
||||
handler.send_header("Content-Disposition", f'attachment; filename="{name}"')
|
||||
handler.end_headers()
|
||||
with open(target, "rb") as f:
|
||||
while True:
|
||||
chunk = f.read(1 << 20)
|
||||
if not chunk:
|
||||
break
|
||||
handler.wfile.write(chunk)
|
||||
|
||||
|
||||
def _infer_system(path):
|
||||
p = (path or "").lower()
|
||||
if "umd_videos" in p:
|
||||
return "videos"
|
||||
if "umd_games" in p:
|
||||
return "games"
|
||||
return None
|
||||
|
||||
|
||||
def _refresh_library(conn):
|
||||
"""Prune stale cache rows, then incrementally scan (only changed discs)."""
|
||||
stale = [
|
||||
r["id"] for r in conn.execute("SELECT id, path FROM files").fetchall()
|
||||
if not os.path.exists(r["path"])
|
||||
]
|
||||
for fid in stale:
|
||||
conn.execute("DELETE FROM files WHERE id = ?", (fid,))
|
||||
for _ in scanner.scan(ROOTS, conn=conn, hash_files=False):
|
||||
pass
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _build_library(conn):
|
||||
items = []
|
||||
seen = set()
|
||||
for row in conn.execute(
|
||||
"SELECT path, name, size, crc32, md5, sha1, serial, title FROM files ORDER BY name"
|
||||
):
|
||||
if not os.path.exists(row["path"]):
|
||||
continue
|
||||
key = (row["name"], row["size"])
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
record = {
|
||||
"path": row["path"], "name": row["name"], "size": row["size"],
|
||||
"serial": row["serial"], "title": row["title"],
|
||||
"crc32": row["crc32"], "md5": row["md5"], "sha1": row["sha1"],
|
||||
}
|
||||
ident = pipeline.identify(conn, record)
|
||||
release = ident["release"]
|
||||
kind = release["system"] if release else _infer_system(row["path"])
|
||||
items.append({
|
||||
"name": row["name"],
|
||||
"path": row["path"],
|
||||
"size": row["size"],
|
||||
"system": kind,
|
||||
"release_id": release["id"] if release else None,
|
||||
"serial": ident["serial"],
|
||||
"method": ident["method"],
|
||||
"canonical": ident["canonical"],
|
||||
"cover": f"/api/cover/{release['id']}" if release else None,
|
||||
})
|
||||
return items
|
||||
|
||||
|
||||
def _library(conn, system=None, refresh=False):
|
||||
global _LIB_CACHE
|
||||
with _LIB_LOCK:
|
||||
if refresh:
|
||||
_refresh_library(conn)
|
||||
if refresh or time.time() - _LIB_CACHE["ts"] > _LIB_TTL:
|
||||
_LIB_CACHE = {"ts": time.time(), "items": _build_library(conn)}
|
||||
items = list(_LIB_CACHE["items"])
|
||||
if system:
|
||||
items = [it for it in items if it["system"] == system]
|
||||
return {"items": items, "count": len(items)}
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "PSPPreserve/0.1"
|
||||
|
||||
def do_GET(self):
|
||||
parsed = urlparse(self.path)
|
||||
path = parsed.path.rstrip("/") or "/"
|
||||
query = parse_qs(parsed.query)
|
||||
|
||||
conn = get_connection()
|
||||
try:
|
||||
if path == "/":
|
||||
_json(self, {
|
||||
"name": "PSP Preservation Tool API",
|
||||
"endpoints": [
|
||||
"/api/summary",
|
||||
"/api/releases?system=games|videos&q=&limit=&offset=",
|
||||
"/api/releases/{id}",
|
||||
"/api/library?system=games|videos",
|
||||
"/api/cover/{id}",
|
||||
"/api/store",
|
||||
"/api/store/download/{name}",
|
||||
],
|
||||
})
|
||||
elif path == "/api/summary":
|
||||
_json(self, _summary(conn))
|
||||
elif path == "/api/releases":
|
||||
system = query.get("system", [None])[0]
|
||||
q = query.get("q", [None])[0]
|
||||
limit = min(_qint(query, "limit", 50), 500)
|
||||
offset = max(_qint(query, "offset", 0), 0)
|
||||
_json(self, _releases(conn, system, q, limit, offset))
|
||||
elif path == "/api/library":
|
||||
system = query.get("system", [None])[0]
|
||||
refresh = query.get("refresh", ["0"])[0].lower() in ("1", "true", "yes")
|
||||
_json(self, _library(conn, system, refresh))
|
||||
elif path == "/api/store":
|
||||
if not _authorized(self):
|
||||
_json(self, {"error": "unauthorized"}, 401)
|
||||
else:
|
||||
_json(self, _store_listing())
|
||||
elif re.match(r"^/api/store/download/", path):
|
||||
if not _authorized(self):
|
||||
_json(self, {"error": "unauthorized"}, 401)
|
||||
else:
|
||||
_store_download(self, unquote(path.rsplit("/", 1)[1]))
|
||||
elif re.match(r"^/api/covers/[\w.\-]+\.zip$", path):
|
||||
_covers_download(self, unquote(path.rsplit("/", 1)[1]))
|
||||
elif re.match(r"^/api/cover/\d+$", path):
|
||||
rid = int(path.rsplit("/", 1)[1])
|
||||
_cover_redirect(self, conn, rid)
|
||||
elif re.match(r"^/api/releases/\d+$", path):
|
||||
rid = int(path.rsplit("/", 1)[1])
|
||||
detail = _release_detail(conn, rid)
|
||||
if detail is None:
|
||||
_json(self, {"error": "not found"}, 404)
|
||||
else:
|
||||
_json(self, detail)
|
||||
else:
|
||||
_json(self, {"error": "not found"}, 404)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
print("[api] " + fmt % args, flush=True)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
ap = argparse.ArgumentParser(description="Run the PSP catalog HTTP API")
|
||||
ap.add_argument("--host", default="0.0.0.0")
|
||||
ap.add_argument("--port", type=int, default=8000)
|
||||
ap.add_argument("--games-root", default=None, help="directory with game .iso files")
|
||||
ap.add_argument("--videos-root", default=None, help="directory with video .iso files")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
global ROOTS
|
||||
ROOTS = [Path(p) for p in (args.games_root, args.videos_root) if p]
|
||||
if not ROOTS:
|
||||
ROOTS = config.GAMES_ROOTS + config.VIDEO_ROOTS
|
||||
|
||||
conn = init_db()
|
||||
conn.close()
|
||||
|
||||
server = ThreadingHTTPServer((args.host, args.port), Handler)
|
||||
print(f"PSP catalog API on http://{args.host}:{args.port} (roots: {[str(r) for r in ROOTS]})", flush=True)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Central configuration, driven by environment variables."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
DATA_DIR = Path(os.environ.get("PSP_DATA_DIR", str(PROJECT_ROOT / "data")))
|
||||
DATS_DIR = Path(os.environ.get("PSP_DATS_DIR", str(DATA_DIR / "dats")))
|
||||
|
||||
|
||||
def _default_state_dir():
|
||||
"""Local state directory (fast, reliable local disk).
|
||||
|
||||
SQLite locking is unreliable over network filesystems (SMB/FUSE), so the
|
||||
library DB and the cover cache live on local storage while DATs, manual
|
||||
covers and credentials stay in ``DATA_DIR`` (which may be a network share).
|
||||
"""
|
||||
xdg = os.environ.get("XDG_STATE_HOME")
|
||||
base = Path(xdg) if xdg else Path.home() / ".local" / "state"
|
||||
return base / "psp-preserve"
|
||||
|
||||
|
||||
STATE_DIR = Path(os.environ.get("PSP_STATE_DIR", str(_default_state_dir())))
|
||||
DB_PATH = Path(os.environ.get("PSP_DB_PATH", str(STATE_DIR / "library.sqlite")))
|
||||
COVERS_DIR = Path(os.environ.get("PSP_COVERS_DIR", str(STATE_DIR / "covers")))
|
||||
LEGACY_DB_PATH = DATA_DIR / "library.sqlite"
|
||||
LEGACY_COVERS_DIR = DATA_DIR / "covers"
|
||||
|
||||
# Redump 7z store (browsable in the Store tab). Env override; GUI setting wins.
|
||||
STORE_ROOT = Path(os.environ.get("PSP_STORE_DIR", "/media/downloads/Sony - PlayStation Portable"))
|
||||
|
||||
# Remote store server (HTTPS). Used by the Store tab as the primary source.
|
||||
STORE_SERVER_URL = os.environ.get("PSP_STORE_SERVER", "https://psp.ceaf.dev")
|
||||
STORE_TOKEN = os.environ.get("PSP_STORE_TOKEN", "")
|
||||
|
||||
# Optional direct URLs for the offline cover bundle zips. When unset, the GUI
|
||||
# derives them from the store server URL (/api/covers/<name>.zip).
|
||||
LIBRETRO_COVERS_URL = os.environ.get("PSP_LIBRETRO_COVERS_URL", "")
|
||||
OVGDB_COVERS_URL = os.environ.get("PSP_OVGDB_COVERS_URL", "")
|
||||
|
||||
# archive.org item hosting the same Redump 7z set (identifier or full URL).
|
||||
ARCHIVE_ITEM = os.environ.get("PSP_ARCHIVE_ITEM", "psp-redump-roms321.com")
|
||||
|
||||
# Optional archive.org login cookies (for items whose files are "private" /
|
||||
# login-required). Grab these from a logged-in browser session on archive.org.
|
||||
ARCHIVE_LOGGED_IN_SIG = os.environ.get("PSP_ARCHIVE_LOGGED_IN_SIG", "")
|
||||
ARCHIVE_LOGGED_IN_USER = os.environ.get("PSP_ARCHIVE_LOGGED_IN_USER", "")
|
||||
|
||||
# Scan roots (used by the scanner phase). Comma-separated in env.
|
||||
GAMES_ROOTS = [Path(p) for p in os.environ.get("PSP_GAMES_ROOTS", "/data/umd_games").split(",") if p.strip()]
|
||||
VIDEO_ROOTS = [Path(p) for p in os.environ.get("PSP_VIDEO_ROOTS", "/data/umd_videos").split(",") if p.strip()]
|
||||
|
||||
# Only .iso discs are tracked; everything below is explicitly ignored.
|
||||
DISC_EXTENSIONS = {".iso"}
|
||||
IGNORED_EXTENSIONS = {".pkg", ".pbp", ".cso"}
|
||||
|
||||
# Parallel hashing workers for the scanner. Keep at 1 on spinning disks
|
||||
# (concurrent reads cause seek thrash); raise it for SSD / fast arrays.
|
||||
SCAN_WORKERS = int(os.environ.get("PSP_SCAN_WORKERS", "1"))
|
||||
+1450
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,128 @@
|
||||
"""Parser for clrmamepro (.dat) files.
|
||||
|
||||
Handles the No-Intro / Redump text DAT format::
|
||||
|
||||
clrmamepro (
|
||||
name "Sony - PlayStation Portable"
|
||||
version "2021.10.24"
|
||||
)
|
||||
game (
|
||||
name "WipEout Pure (Europe)"
|
||||
region "Europe"
|
||||
serial "ULES-00011"
|
||||
rom ( name "WipEout Pure (Europe).iso" size 123 crc ABCD1234 md5 ... sha1 ... )
|
||||
)
|
||||
|
||||
``parse_dat_text`` returns ``(header, entries)``. Each entry is a dict with
|
||||
scalar keys (``name``, ``description``, ``region``, ``serial``, ...) and
|
||||
repeated nested blocks exposed as lists under their block key (``rom``).
|
||||
"""
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
_TOKEN_RE = re.compile(
|
||||
r"""
|
||||
(?P<ws>\s+)
|
||||
| (?P<lparen>\()
|
||||
| (?P<rparen>\))
|
||||
| (?P<string>"(?:\\.|[^"\\])*")
|
||||
| (?P<ident>[^"\s()]+)
|
||||
""",
|
||||
re.VERBOSE,
|
||||
)
|
||||
|
||||
|
||||
class _Token:
|
||||
__slots__ = ("kind", "value")
|
||||
|
||||
def __init__(self, kind, value):
|
||||
self.kind = kind
|
||||
self.value = value
|
||||
|
||||
|
||||
def _tokenize(text):
|
||||
tokens = []
|
||||
for match in _TOKEN_RE.finditer(text):
|
||||
kind = match.lastgroup
|
||||
value = match.group()
|
||||
if kind == "ws":
|
||||
continue
|
||||
if kind == "string":
|
||||
value = value[1:-1].replace('\\"', '"').replace("\\\\", "\\")
|
||||
kind = "string"
|
||||
elif kind == "lparen":
|
||||
kind = "lparen"
|
||||
value = "("
|
||||
elif kind == "rparen":
|
||||
kind = "rparen"
|
||||
value = ")"
|
||||
else:
|
||||
kind = "ident"
|
||||
tokens.append(_Token(kind, value))
|
||||
return tokens
|
||||
|
||||
|
||||
class _Parser:
|
||||
def __init__(self, tokens):
|
||||
self._tokens = tokens
|
||||
self._pos = 0
|
||||
|
||||
def _peek(self):
|
||||
return self._tokens[self._pos] if self._pos < len(self._tokens) else None
|
||||
|
||||
def _next(self):
|
||||
token = self._peek()
|
||||
self._pos += 1
|
||||
return token
|
||||
|
||||
def parse_block(self, name=None):
|
||||
if name is None:
|
||||
name_token = self._next()
|
||||
if name_token is None or name_token.kind != "ident":
|
||||
raise ValueError(f"expected block name at token {self._pos}")
|
||||
name = name_token.value
|
||||
lparen = self._next()
|
||||
if lparen is None or lparen.kind != "lparen":
|
||||
raise ValueError(f"expected '(' after '{name}'")
|
||||
|
||||
block = {"_type": name}
|
||||
while True:
|
||||
token = self._peek()
|
||||
if token is None:
|
||||
raise ValueError(f"unexpected EOF inside block '{name}'")
|
||||
if token.kind == "rparen":
|
||||
self._next()
|
||||
break
|
||||
if token.kind != "ident":
|
||||
raise ValueError(f"unexpected token '{token.value}' in block '{name}'")
|
||||
key = self._next().value
|
||||
nxt = self._peek()
|
||||
if nxt is not None and nxt.kind == "lparen":
|
||||
child = self.parse_block(name=key)
|
||||
block.setdefault(key, []).append(child)
|
||||
else:
|
||||
value = self._next()
|
||||
if value is None:
|
||||
raise ValueError(f"expected value for '{key}' in block '{name}'")
|
||||
block[key] = value.value
|
||||
return block
|
||||
|
||||
|
||||
def parse_dat_text(text):
|
||||
tokens = _tokenize(text)
|
||||
parser = _Parser(tokens)
|
||||
header = None
|
||||
entries = []
|
||||
while parser._peek() is not None:
|
||||
block = parser.parse_block()
|
||||
kind = block.get("_type")
|
||||
if kind == "clrmamepro":
|
||||
header = block
|
||||
elif kind in ("game", "machine"):
|
||||
entries.append(block)
|
||||
# ignore any other top-level blocks
|
||||
return header, entries
|
||||
|
||||
|
||||
def parse_dat_file(path):
|
||||
return parse_dat_text(Path(path).read_text(encoding="utf-8-sig"))
|
||||
@@ -0,0 +1,166 @@
|
||||
"""SQLite persistence layer (standard-library sqlite3, no ORM)."""
|
||||
import shutil
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from .config import COVERS_DIR, DB_PATH, LEGACY_COVERS_DIR, LEGACY_DB_PATH
|
||||
|
||||
SCHEMA = """
|
||||
CREATE TABLE IF NOT EXISTS dats (
|
||||
id INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
version TEXT,
|
||||
description TEXT,
|
||||
system TEXT NOT NULL,
|
||||
source_path TEXT,
|
||||
ingested_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS releases (
|
||||
id INTEGER PRIMARY KEY,
|
||||
dat_id INTEGER NOT NULL REFERENCES dats(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT,
|
||||
region TEXT,
|
||||
serial TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS roms (
|
||||
id INTEGER PRIMARY KEY,
|
||||
release_id INTEGER NOT NULL REFERENCES releases(id) ON DELETE CASCADE,
|
||||
name TEXT NOT NULL,
|
||||
size INTEGER,
|
||||
crc32 TEXT,
|
||||
md5 TEXT,
|
||||
sha1 TEXT,
|
||||
serial TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_releases_serial ON releases(serial);
|
||||
CREATE INDEX IF NOT EXISTS idx_releases_name ON releases(name);
|
||||
CREATE INDEX IF NOT EXISTS idx_releases_region ON releases(region);
|
||||
CREATE INDEX IF NOT EXISTS idx_roms_crc32 ON roms(crc32);
|
||||
CREATE INDEX IF NOT EXISTS idx_roms_md5 ON roms(md5);
|
||||
CREATE INDEX IF NOT EXISTS idx_roms_sha1 ON roms(sha1);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS files (
|
||||
id INTEGER PRIMARY KEY,
|
||||
path TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
mtime_ns INTEGER,
|
||||
crc32 TEXT,
|
||||
md5 TEXT,
|
||||
sha1 TEXT,
|
||||
serial TEXT,
|
||||
title TEXT,
|
||||
release_id INTEGER,
|
||||
error TEXT,
|
||||
scanned_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_files_path ON files(path);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS choices (
|
||||
serial TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
release_id INTEGER,
|
||||
PRIMARY KEY (serial, size)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS item_info (
|
||||
release_id TEXT NOT NULL,
|
||||
system TEXT NOT NULL,
|
||||
title TEXT,
|
||||
meta_json TEXT,
|
||||
manual INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TEXT NOT NULL,
|
||||
PRIMARY KEY (release_id, system)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cover_attempts (
|
||||
key TEXT NOT NULL,
|
||||
system TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
ts REAL NOT NULL,
|
||||
PRIMARY KEY (key, system)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS meta_attempts (
|
||||
key TEXT NOT NULL,
|
||||
system TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
ts REAL NOT NULL,
|
||||
PRIMARY KEY (key, system)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS save_sigs (
|
||||
save_id TEXT NOT NULL,
|
||||
size INTEGER NOT NULL,
|
||||
mtime_ns INTEGER NOT NULL,
|
||||
sig TEXT NOT NULL,
|
||||
PRIMARY KEY (save_id, size, mtime_ns)
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def get_connection(path=None):
|
||||
target = path if path is not None else DB_PATH
|
||||
Path(target).parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(target))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA foreign_keys = ON")
|
||||
conn.execute("PRAGMA busy_timeout = 10000")
|
||||
conn.execute("PRAGMA journal_mode = WAL")
|
||||
conn.execute("PRAGMA synchronous = NORMAL")
|
||||
return conn
|
||||
|
||||
|
||||
def _migrate(conn):
|
||||
"""Bring databases created before ``release_id`` up to date."""
|
||||
cols = {r["name"] for r in conn.execute("PRAGMA table_info(files)")}
|
||||
if "release_id" not in cols:
|
||||
conn.execute("ALTER TABLE files ADD COLUMN release_id INTEGER")
|
||||
|
||||
|
||||
def _integrity_ok(path):
|
||||
try:
|
||||
conn = sqlite3.connect(str(path))
|
||||
ok = conn.execute("PRAGMA integrity_check").fetchone()[0] == "ok"
|
||||
conn.close()
|
||||
return ok
|
||||
except sqlite3.Error:
|
||||
return False
|
||||
|
||||
|
||||
def ensure_state():
|
||||
"""One-time move of legacy on-share DB + cover cache onto local storage.
|
||||
|
||||
SQLite over SMB/FUSE corrupts under concurrent access; local storage is
|
||||
fast and reliable. Runs once, is a no-op afterwards.
|
||||
"""
|
||||
if DB_PATH != LEGACY_DB_PATH and not DB_PATH.exists() and LEGACY_DB_PATH.exists():
|
||||
if _integrity_ok(LEGACY_DB_PATH):
|
||||
try:
|
||||
DB_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(LEGACY_DB_PATH, DB_PATH)
|
||||
except OSError:
|
||||
pass
|
||||
if COVERS_DIR != LEGACY_COVERS_DIR and not COVERS_DIR.exists() and LEGACY_COVERS_DIR.exists():
|
||||
try:
|
||||
COVERS_DIR.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(LEGACY_COVERS_DIR, COVERS_DIR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def init_db(path=None):
|
||||
if path is None:
|
||||
ensure_state()
|
||||
target = path if path is not None else DB_PATH
|
||||
Path(target).parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = get_connection(target)
|
||||
conn.executescript(SCHEMA)
|
||||
_migrate(conn)
|
||||
conn.commit()
|
||||
return conn
|
||||
+4315
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,30 @@
|
||||
"""Streaming file hashing (stdlib only)."""
|
||||
import hashlib
|
||||
import zlib
|
||||
|
||||
CHUNK = 1 << 23 # 8 MiB read chunks
|
||||
|
||||
|
||||
class HashCancelled(Exception):
|
||||
"""Raised when hashing is cancelled mid-read."""
|
||||
|
||||
|
||||
def hash_file(path, cancel=None):
|
||||
crc = 0
|
||||
md5 = hashlib.md5()
|
||||
sha1 = hashlib.sha1()
|
||||
with open(path, "rb") as f:
|
||||
while True:
|
||||
if cancel is not None and cancel.is_set():
|
||||
raise HashCancelled()
|
||||
block = f.read(CHUNK)
|
||||
if not block:
|
||||
break
|
||||
crc = zlib.crc32(block, crc)
|
||||
md5.update(block)
|
||||
sha1.update(block)
|
||||
return {
|
||||
"crc32": f"{crc & 0xFFFFFFFF:08X}",
|
||||
"md5": md5.hexdigest().upper(),
|
||||
"sha1": sha1.hexdigest().upper(),
|
||||
}
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
"""Per-item metadata: cached fetches + user overrides.
|
||||
|
||||
Fetched metadata (game details from IGDB, movie details from TheMovieDB /
|
||||
Wikipedia) is cached in the library DB keyed by ``(release_id, system)`` so it
|
||||
is only ever requested once. Users can also hand-tag an item; the override is
|
||||
stored with ``manual=1`` and always wins over both the cache and fresh fetches.
|
||||
|
||||
All database calls are defensive: a transient I/O failure returns ``None`` /
|
||||
``False`` instead of raising, so a flaky share never crashes the UI.
|
||||
"""
|
||||
import json
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
from . import config, covers
|
||||
from .db import get_connection
|
||||
|
||||
_failed = {} # (release_id, system) -> time.time() of last failed fetch
|
||||
_FAIL_TTL = 1800.0 # don't re-fetch failed metadata for 30 minutes
|
||||
|
||||
# Persistent "no metadata" cache so a failed lookup isn't re-tried on restart.
|
||||
_miss_db = {}
|
||||
_miss_loaded = False
|
||||
_MISS_TTL = 86400.0 # retry failed metadata after 1 day
|
||||
|
||||
|
||||
def _load_miss_db():
|
||||
global _miss_loaded
|
||||
if _miss_loaded:
|
||||
return
|
||||
_miss_loaded = True
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
for r in conn.execute(
|
||||
"SELECT key, system, ts FROM meta_attempts WHERE status = 'missing'"
|
||||
):
|
||||
_miss_db[(r["key"], r["system"])] = r["ts"]
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
|
||||
|
||||
def _mark_missing(key, system):
|
||||
ts = time.time()
|
||||
_miss_db[(key, system)] = ts
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
conn.execute(
|
||||
"INSERT INTO meta_attempts (key, system, status, ts) VALUES (?,?,?,?) "
|
||||
"ON CONFLICT(key, system) DO UPDATE SET status=excluded.status, ts=excluded.ts",
|
||||
(key, system, "missing", ts),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
|
||||
|
||||
def _clear_missing(key, system):
|
||||
_miss_db.pop((key, system), None)
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
conn.execute(
|
||||
"DELETE FROM meta_attempts WHERE key = ? AND system = ?", (key, system)
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
|
||||
|
||||
def _serialize(meta):
|
||||
"""Strip non-JSON-able or helper keys before persisting."""
|
||||
return {k: v for k, v in (meta or {}).items() if k != "manual"}
|
||||
|
||||
|
||||
def load(release_id, system):
|
||||
"""Return the stored info dict (with ``manual`` set) or None."""
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
row = conn.execute(
|
||||
"SELECT title, meta_json, manual FROM item_info WHERE release_id = ? AND system = ?",
|
||||
(str(release_id), system),
|
||||
).fetchone()
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
meta = {}
|
||||
try:
|
||||
meta = json.loads(row["meta_json"] or "{}")
|
||||
except (TypeError, ValueError):
|
||||
meta = {}
|
||||
if not isinstance(meta, dict):
|
||||
meta = {}
|
||||
if row["title"]:
|
||||
meta["title"] = row["title"]
|
||||
meta["manual"] = bool(row["manual"])
|
||||
return meta
|
||||
|
||||
|
||||
def save(release_id, system, meta, manual=False, title=None):
|
||||
"""Persist info for an item. ``manual`` marks a user override."""
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO item_info (release_id, system, title, meta_json, manual, updated_at)
|
||||
VALUES (?,?,?,?,?,datetime('now'))
|
||||
ON CONFLICT(release_id, system) DO UPDATE SET
|
||||
title=excluded.title, meta_json=excluded.meta_json,
|
||||
manual=excluded.manual, updated_at=excluded.updated_at
|
||||
""",
|
||||
(
|
||||
str(release_id),
|
||||
system,
|
||||
title if title else (meta.get("title") if manual else None),
|
||||
json.dumps(_serialize(meta)),
|
||||
1 if manual else 0,
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return True
|
||||
except sqlite3.Error:
|
||||
return False
|
||||
|
||||
|
||||
def reset(release_id, system):
|
||||
"""Delete cached/overridden info so it is re-fetched next time."""
|
||||
key = (str(release_id), system)
|
||||
_failed.pop(key, None)
|
||||
_clear_missing(*key)
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
conn.execute(
|
||||
"DELETE FROM item_info WHERE release_id = ? AND system = ?",
|
||||
(str(release_id), system),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
|
||||
|
||||
def override_titles():
|
||||
"""Return {(system, str(release_id)): title} for all manual overrides."""
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
rows = conn.execute(
|
||||
"SELECT release_id, system, title FROM item_info WHERE manual = 1 AND title IS NOT NULL"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
return {}
|
||||
return {(r["system"], str(r["release_id"])): r["title"] for r in rows}
|
||||
|
||||
|
||||
def cached_release_dates():
|
||||
"""Return {(release_id, system): release_date} from cached metadata only.
|
||||
|
||||
Pure DB read — no network — so list views can show a release date without
|
||||
triggering API fetches for every row.
|
||||
"""
|
||||
out = {}
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
rows = conn.execute("SELECT release_id, system, meta_json FROM item_info").fetchall()
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
return {}
|
||||
for r in rows:
|
||||
try:
|
||||
meta = json.loads(r["meta_json"] or "{}")
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if isinstance(meta, dict) and meta.get("release_date"):
|
||||
out[(r["release_id"], r["system"])] = meta["release_date"]
|
||||
return out
|
||||
|
||||
|
||||
def get_or_fetch(release_id, system, fetch_title):
|
||||
"""Return cached info (override or fetched) or fetch-and-cache it."""
|
||||
info = load(release_id, system)
|
||||
if info is not None:
|
||||
return info
|
||||
key = (str(release_id), system)
|
||||
last = _failed.get(key)
|
||||
if last is not None and time.time() - last < _FAIL_TTL:
|
||||
return None # recently failed; don't hammer the API again
|
||||
_load_miss_db()
|
||||
miss_ts = _miss_db.get(key)
|
||||
if miss_ts is not None and time.time() - miss_ts < _MISS_TTL:
|
||||
return None
|
||||
try:
|
||||
if system == "videos":
|
||||
meta = covers.get_movie_metadata(fetch_title)
|
||||
else:
|
||||
meta = covers.get_game_metadata(fetch_title)
|
||||
except Exception: # noqa: BLE001 - network/API failures must not crash the UI
|
||||
meta = None
|
||||
if meta:
|
||||
save(release_id, system, meta, manual=False)
|
||||
meta["manual"] = False
|
||||
return meta
|
||||
_failed[key] = time.time()
|
||||
_mark_missing(*key)
|
||||
return None
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
"""Ingest clrmamepro .dat files into the SQLite library database.
|
||||
|
||||
Usage::
|
||||
|
||||
python -m app.ingest [--dats data/dats] [--db data/library.sqlite]
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, matcher
|
||||
from .dat_parser import parse_dat_file
|
||||
from .db import get_connection, init_db
|
||||
|
||||
|
||||
def _norm_hex(value):
|
||||
if not value:
|
||||
return None
|
||||
return str(value).strip().upper()
|
||||
|
||||
|
||||
def detect_system(dat_name):
|
||||
if "UMD Video" in dat_name:
|
||||
return "videos"
|
||||
if "PlayStation Portable" in dat_name:
|
||||
return "games"
|
||||
return "unknown"
|
||||
|
||||
|
||||
def ingest_dat(conn, path):
|
||||
header, entries = parse_dat_file(path)
|
||||
dat_name = header.get("name") if header else path.stem
|
||||
version = header.get("version") if header else None
|
||||
description = header.get("description") if header else None
|
||||
system = detect_system(dat_name)
|
||||
|
||||
existing = conn.execute("SELECT id FROM dats WHERE name = ?", (dat_name,)).fetchone()
|
||||
if existing:
|
||||
dat_id = existing["id"]
|
||||
conn.execute("DELETE FROM roms WHERE release_id IN (SELECT id FROM releases WHERE dat_id = ?)", (dat_id,))
|
||||
conn.execute("DELETE FROM releases WHERE dat_id = ?", (dat_id,))
|
||||
conn.execute(
|
||||
"UPDATE dats SET version=?, description=?, system=?, source_path=?, ingested_at=datetime('now') WHERE id=?",
|
||||
(version, description, system, str(path), dat_id),
|
||||
)
|
||||
else:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO dats (name, version, description, system, source_path, ingested_at) VALUES (?,?,?,?,?,datetime('now'))",
|
||||
(dat_name, version, description, system, str(path)),
|
||||
)
|
||||
dat_id = cur.lastrowid
|
||||
|
||||
release_count = 0
|
||||
rom_count = 0
|
||||
for entry in entries:
|
||||
cur = conn.execute(
|
||||
"INSERT INTO releases (dat_id, name, description, region, serial) VALUES (?,?,?,?,?)",
|
||||
(dat_id, entry.get("name"), entry.get("description"), entry.get("region"), entry.get("serial")),
|
||||
)
|
||||
release_id = cur.lastrowid
|
||||
release_count += 1
|
||||
for rom in entry.get("rom", []):
|
||||
size = rom.get("size")
|
||||
conn.execute(
|
||||
"INSERT INTO roms (release_id, name, size, crc32, md5, sha1, serial) VALUES (?,?,?,?,?,?,?)",
|
||||
(
|
||||
release_id,
|
||||
rom.get("name"),
|
||||
int(size) if size else None,
|
||||
_norm_hex(rom.get("crc")),
|
||||
_norm_hex(rom.get("md5")),
|
||||
_norm_hex(rom.get("sha1")),
|
||||
rom.get("serial"),
|
||||
),
|
||||
)
|
||||
rom_count += 1
|
||||
|
||||
matcher.clear_cache()
|
||||
return dat_name, system, release_count, rom_count
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description="Ingest clrmamepro DAT files into SQLite")
|
||||
parser.add_argument("--dats", help="directory containing .dat files (default: data/dats)")
|
||||
parser.add_argument("--db", help="SQLite database path (default: data/library.sqlite)")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
dats_dir = Path(args.dats) if args.dats else config.DATS_DIR
|
||||
db_path = Path(args.db) if args.db else config.DB_PATH
|
||||
|
||||
conn = init_db(db_path)
|
||||
files = sorted(dats_dir.glob("*.dat"))
|
||||
if not files:
|
||||
print(f"No .dat files found in {dats_dir}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
for path in files:
|
||||
dat_name, system, releases, roms = ingest_dat(conn, path)
|
||||
print(f"{dat_name} [{system}] -> {releases} releases, {roms} roms")
|
||||
|
||||
conn.commit()
|
||||
conn.close()
|
||||
print(f"\nLibrary written to {db_path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Read identifying metadata from PSP .iso images (ISO9660 + PARAM.SFO).
|
||||
|
||||
PSP UMD dumps are ISO9660 images containing ``PSP_GAME/PARAM.SFO`` (games) or
|
||||
``UMD_VIDEO/PARAM.SFO`` (videos). PARAM.SFO is a key/value blob that carries
|
||||
``DISC_ID`` (the catalogue serial, e.g. ``ULUS-10027``) and ``TITLE``.
|
||||
"""
|
||||
import re
|
||||
import struct
|
||||
|
||||
|
||||
class IsoError(Exception):
|
||||
"""Raised when an ISO cannot be parsed or lacks the expected files."""
|
||||
|
||||
|
||||
SECTOR_SIZE = 2048
|
||||
PVD_SECTOR = 16
|
||||
ROOT_RECORD_OFFSET = 156
|
||||
|
||||
|
||||
def _read_sector(f, lba):
|
||||
f.seek(lba * SECTOR_SIZE)
|
||||
data = f.read(SECTOR_SIZE)
|
||||
if len(data) < SECTOR_SIZE:
|
||||
raise IsoError(f"short read at LBA {lba}")
|
||||
return data
|
||||
|
||||
|
||||
def _parse_dir_record(buf, offset):
|
||||
"""Parse one ISO9660 directory record at ``offset`` in ``buf``.
|
||||
|
||||
Returns ``(entry, next_offset)`` where ``entry`` is
|
||||
``(name, is_dir, extent_lba, size)`` or ``None`` when ``offset`` points at
|
||||
padding.
|
||||
"""
|
||||
if offset >= len(buf):
|
||||
return None, offset
|
||||
length = buf[offset]
|
||||
if length == 0:
|
||||
return None, offset + 1
|
||||
if offset + 33 > len(buf) or offset + length > len(buf):
|
||||
raise IsoError("directory record overruns buffer")
|
||||
name_len = buf[offset + 32]
|
||||
flags = buf[offset + 25]
|
||||
extent = struct.unpack_from("<I", buf, offset + 2)[0]
|
||||
size = struct.unpack_from("<I", buf, offset + 10)[0]
|
||||
raw_name = buf[offset + 33 : offset + 33 + name_len]
|
||||
name = raw_name.decode("ascii", "replace").rstrip("\x00")
|
||||
is_dir = bool(flags & 0x02)
|
||||
return (name, is_dir, extent, size), offset + length
|
||||
|
||||
|
||||
def _list_dir(f, extent, size):
|
||||
f.seek(extent * SECTOR_SIZE)
|
||||
data = f.read(size)
|
||||
if len(data) < size:
|
||||
raise IsoError("short directory read")
|
||||
records = []
|
||||
offset = 0
|
||||
while offset < len(data):
|
||||
entry, next_offset = _parse_dir_record(data, offset)
|
||||
if entry is None:
|
||||
# Padding: skip to the start of the next 2048-byte sector.
|
||||
offset = ((offset // SECTOR_SIZE) + 1) * SECTOR_SIZE
|
||||
continue
|
||||
records.append(entry)
|
||||
offset = next_offset
|
||||
return records
|
||||
|
||||
|
||||
def _basename(name):
|
||||
return name.split(";")[0].strip()
|
||||
|
||||
|
||||
def _find_entry(f, extent, size, target):
|
||||
for name, is_dir, entry_extent, entry_size in _list_dir(f, extent, size):
|
||||
if _basename(name) == target:
|
||||
return is_dir, entry_extent, entry_size
|
||||
return None
|
||||
|
||||
|
||||
def _root_record(f):
|
||||
pvd = _read_sector(f, PVD_SECTOR)
|
||||
if pvd[0] != 1:
|
||||
raise IsoError("sector 16 is not a primary volume descriptor")
|
||||
entry, _ = _parse_dir_record(pvd, ROOT_RECORD_OFFSET)
|
||||
if entry is None:
|
||||
raise IsoError("missing root directory record")
|
||||
_, _, extent, size = entry
|
||||
return extent, size
|
||||
|
||||
|
||||
def _read_sfo_file(f, dir_name):
|
||||
root_extent, root_size = _root_record(f)
|
||||
found = _find_entry(f, root_extent, root_size, dir_name)
|
||||
if not found:
|
||||
raise IsoError(f"{dir_name} directory not found")
|
||||
is_dir, extent, size = found
|
||||
if not is_dir:
|
||||
raise IsoError(f"{dir_name} is not a directory")
|
||||
sfo = _find_entry(f, extent, size, "PARAM.SFO")
|
||||
if not sfo:
|
||||
raise IsoError(f"{dir_name}/PARAM.SFO not found")
|
||||
_, sfo_extent, sfo_size = sfo
|
||||
f.seek(sfo_extent * SECTOR_SIZE)
|
||||
data = f.read(sfo_size)
|
||||
if len(data) < sfo_size:
|
||||
raise IsoError("short PARAM.SFO read")
|
||||
return data
|
||||
|
||||
|
||||
def read_file(path, dir_name, filename):
|
||||
"""Read the bytes of ``dir_name/filename`` from an ISO image.
|
||||
|
||||
Raises ``IsoError`` when the directory or file is not found."""
|
||||
with open(path, "rb") as f:
|
||||
root_extent, root_size = _root_record(f)
|
||||
found = _find_entry(f, root_extent, root_size, dir_name)
|
||||
if not found:
|
||||
raise IsoError(f"{dir_name} directory not found")
|
||||
is_dir, extent, size = found
|
||||
if not is_dir:
|
||||
raise IsoError(f"{dir_name} is not a directory")
|
||||
entry = _find_entry(f, extent, size, filename)
|
||||
if not entry:
|
||||
raise IsoError(f"{dir_name}/{filename} not found")
|
||||
_, f_extent, f_size = entry
|
||||
f.seek(f_extent * SECTOR_SIZE)
|
||||
data = f.read(f_size)
|
||||
if len(data) < f_size:
|
||||
raise IsoError(f"short read for {dir_name}/{filename}")
|
||||
return data
|
||||
|
||||
|
||||
def parse_sfo(data):
|
||||
"""Parse a PARAM.SFO blob into a ``{key: value}`` dict.
|
||||
|
||||
SFO layout: 20-byte header, then the index table (``count * 16`` bytes at
|
||||
offset ``key_start - count * 16``), then the key table, then the data
|
||||
table. ``fmt`` 0x0004 / 0x0404 is a uint32, anything else is a string."""
|
||||
if len(data) < 20 or data[:4] != b"\x00PSF":
|
||||
raise IsoError("bad PARAM.SFO magic")
|
||||
_version, key_start, data_start, count = struct.unpack_from("<IIII", data, 4)
|
||||
if count <= 0:
|
||||
return {}
|
||||
size = len(data)
|
||||
index_start = key_start - count * 16
|
||||
if index_start < 20:
|
||||
index_start = 20
|
||||
if index_start + count * 16 > size:
|
||||
raise IsoError("bad PARAM.SFO index table")
|
||||
fields = {}
|
||||
for i in range(count):
|
||||
off = index_start + i * 16
|
||||
key_off, fmt, param_len, _max_len, data_off = struct.unpack_from("<HHIII", data, off)
|
||||
key = data[key_start + key_off :].split(b"\x00", 1)[0].decode("utf-8", "replace")
|
||||
value_start = data_start + data_off
|
||||
if value_start + param_len > size:
|
||||
continue
|
||||
raw = data[value_start : value_start + param_len]
|
||||
if fmt in (0x0004, 0x0404) and param_len == 4: # uint32
|
||||
value = struct.unpack("<I", raw)[0]
|
||||
else: # UTF-8 string
|
||||
value = raw.rstrip(b"\x00").decode("utf-8", "replace")
|
||||
fields[key] = value
|
||||
return fields
|
||||
|
||||
|
||||
_SERIAL_RE = re.compile(r"^([A-Za-z]{4})-?(\d{4,5})(.*)$")
|
||||
|
||||
|
||||
def normalize_serial(value):
|
||||
"""Normalize a PARAM.SFO DISC_ID (e.g. ULUS10021 -> ULUS-10021)."""
|
||||
if not value:
|
||||
return value
|
||||
s = str(value).strip().upper()
|
||||
m = _SERIAL_RE.match(s)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2)}{m.group(3)}"
|
||||
return s
|
||||
|
||||
|
||||
def looks_like_serial(value):
|
||||
"""True if a value looks like a real PSP catalogue number (e.g. ULES-00011)."""
|
||||
if not value:
|
||||
return False
|
||||
return bool(re.match(r"^[A-Za-z]{3,4}-?\d{3,5}(-.*)?$", str(value).strip()))
|
||||
|
||||
|
||||
def read_identity(path):
|
||||
"""Return ``{"serial": ..., "title": ..., "sfo": {...}, "kind": ...}``.
|
||||
|
||||
``kind`` is ``"game"`` when the disc has ``PSP_GAME/PARAM.SFO`` or
|
||||
``"video"`` when it has ``UMD_VIDEO/PARAM.SFO``.
|
||||
"""
|
||||
sfo = None
|
||||
kind = None
|
||||
with open(path, "rb") as f:
|
||||
for dir_name in ("PSP_GAME", "UMD_VIDEO"):
|
||||
try:
|
||||
sfo = parse_sfo(_read_sfo_file(f, dir_name))
|
||||
kind = "game" if dir_name == "PSP_GAME" else "video"
|
||||
break
|
||||
except IsoError:
|
||||
continue
|
||||
if sfo is None:
|
||||
raise IsoError("no PARAM.SFO found (tried PSP_GAME and UMD_VIDEO)")
|
||||
title = sfo.get("TITLE") or sfo.get("TITLE_0") or sfo.get("TITLE_00")
|
||||
return {
|
||||
"serial": normalize_serial(sfo.get("DISC_ID")),
|
||||
"title": title,
|
||||
"sfo": sfo,
|
||||
"kind": kind,
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Launch external applications (currently PPSSPP)."""
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def find_ppsspp():
|
||||
"""Return a candidate PPSSPP executable path, or None."""
|
||||
for name in ("ppsspp", "PPSSPP", "PPSSPPSDL", "PPSSPPQt"):
|
||||
p = shutil.which(name)
|
||||
if p:
|
||||
return p
|
||||
home = Path.home()
|
||||
for cand in (
|
||||
home / ".local" / "bin" / "ppsspp",
|
||||
home / ".local" / "bin" / "PPSSPPSDL",
|
||||
home / "Applications" / "PPSSPP" / "PPSSPP",
|
||||
Path("/usr/games/ppsspp"),
|
||||
Path("/usr/bin/ppsspp"),
|
||||
Path("/usr/bin/PPSSPPSDL"),
|
||||
Path("/snap/bin/ppsspp"),
|
||||
):
|
||||
if cand.is_file():
|
||||
return str(cand)
|
||||
return None
|
||||
|
||||
|
||||
def _env():
|
||||
env = dict(os.environ)
|
||||
# PPSSPPSDL (the SDL build) frequently crashes on Wayland; route it through
|
||||
# XWayland instead. Kept local to PPSSPP so nothing else is affected.
|
||||
env["SDL_VIDEODRIVER"] = "x11"
|
||||
env.setdefault("SDL_AUDIODRIVER", "pulseaudio")
|
||||
return env
|
||||
|
||||
|
||||
def launch_ppsspp(iso, exe=""):
|
||||
"""Launch PPSSPP with the given ISO. Returns ``(ok, message)``."""
|
||||
iso = str(iso)
|
||||
exe = (exe or "").strip()
|
||||
if exe:
|
||||
cmd = [exe, iso]
|
||||
else:
|
||||
found = find_ppsspp()
|
||||
if found:
|
||||
cmd = [found, iso]
|
||||
else:
|
||||
return False, "PPSSPP not found. Install it or set its path in Settings → Saves."
|
||||
try:
|
||||
subprocess.Popen(cmd, start_new_session=True, env=_env(),
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
return True, ""
|
||||
except OSError as exc:
|
||||
return False, str(exc)
|
||||
+137
@@ -0,0 +1,137 @@
|
||||
"""Download all libretro PSP box art into a local offline bundle.
|
||||
|
||||
The libretro-thumbnails repo is served from GitHub raw (not Cloudflare-blocked),
|
||||
so plain urllib works. Some files are Git symlinks (GitHub serves them as a tiny
|
||||
text body); ``_fetch_png`` follows those.
|
||||
|
||||
Usage::
|
||||
|
||||
python3 -m app.libretro # download everything (resumes)
|
||||
python3 -m app.libretro --limit 50 # test
|
||||
|
||||
Output: ``data/covers/libretro-covers.zip`` — entries named ``<norm(title)>.png``
|
||||
matching the keys used by ``covers.fetch_libretro``.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from . import config
|
||||
from .covers import _fetch_png, _load_index, _norm, _plain
|
||||
|
||||
log = logging.getLogger("libretro")
|
||||
|
||||
BUNDLE = config.DATA_DIR / "covers" / "libretro-covers.zip"
|
||||
|
||||
|
||||
def _region_rank(name):
|
||||
nl = name.lower()
|
||||
if "(usa" in nl:
|
||||
return 0
|
||||
if "(europe" in nl or "(australia" in nl:
|
||||
return 1
|
||||
if "(japan" in nl or "(asia" in nl or "(korea" in nl:
|
||||
return 2
|
||||
return 1 # unknown regions are treated as neutral
|
||||
|
||||
|
||||
def _dedupe(names):
|
||||
"""Collapse regional duplicates to one entry, preferring US covers."""
|
||||
best = {}
|
||||
for n in names:
|
||||
key = _norm(_plain(n))
|
||||
if not key:
|
||||
continue
|
||||
rank = _region_rank(n)
|
||||
cur = best.get(key)
|
||||
if cur is None or rank < cur[1]:
|
||||
best[key] = (n, rank)
|
||||
return best # key -> (filename, rank)
|
||||
|
||||
|
||||
def scrape(limit=None, workers=8):
|
||||
BUNDLE.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
names = _load_index(force=True).get("Named_Boxarts", [])
|
||||
log.info("indexed %d libretro boxarts", len(names))
|
||||
mapping = _dedupe(names)
|
||||
log.info("%d unique titles", len(mapping))
|
||||
|
||||
existing = set()
|
||||
if BUNDLE.exists():
|
||||
try:
|
||||
with zipfile.ZipFile(BUNDLE) as z:
|
||||
existing = {n[:-4] for n in z.namelist() if n.endswith(".png")}
|
||||
except Exception: # noqa: BLE001
|
||||
existing = set()
|
||||
|
||||
todo = [(k, v[0]) for k, v in sorted(mapping.items()) if k not in existing]
|
||||
if limit:
|
||||
todo = todo[:limit]
|
||||
log.info("%d to download (%d already bundled)", len(todo), len(existing))
|
||||
|
||||
work_dir = Path(tempfile.mkdtemp(prefix="libretro-covers-"))
|
||||
ok = fail = 0
|
||||
done = 0
|
||||
lock = threading.Lock()
|
||||
|
||||
def run(item):
|
||||
key, fname = item
|
||||
data = _fetch_png(fname, "Named_Boxarts", timeout=30)
|
||||
nonlocal ok, fail, done
|
||||
with lock:
|
||||
done += 1
|
||||
if data and data[:8] == b"\x89PNG\r\n\x1a\n":
|
||||
(work_dir / f"{key}.png").write_bytes(data)
|
||||
ok += 1
|
||||
else:
|
||||
fail += 1
|
||||
if done % 100 == 0 or (not data):
|
||||
log.info(" %d/%d (%d ok, %d failed)", done, len(todo), ok, fail)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
list(pool.map(run, todo))
|
||||
|
||||
if ok:
|
||||
out_tmp = BUNDLE.with_name(BUNDLE.name + ".tmp")
|
||||
existing_bytes = {}
|
||||
if BUNDLE.exists():
|
||||
try:
|
||||
with zipfile.ZipFile(BUNDLE) as zin:
|
||||
for n in zin.namelist():
|
||||
if n.endswith(".png") and n not in existing_bytes:
|
||||
existing_bytes[n] = zin.read(n)
|
||||
except Exception: # noqa: BLE001
|
||||
existing_bytes = {}
|
||||
with zipfile.ZipFile(out_tmp, "w", zipfile.ZIP_STORED) as z:
|
||||
for name, data in existing_bytes.items():
|
||||
z.writestr(name, data)
|
||||
for f in sorted(work_dir.glob("*.png")):
|
||||
z.write(f, arcname=f.name)
|
||||
os.replace(out_tmp, BUNDLE)
|
||||
log.info("done: bundle now has %d covers (%d new, %d failed)",
|
||||
len(existing_bytes) + ok, ok, fail)
|
||||
else:
|
||||
log.info("no new covers downloaded; bundle unchanged (%d failed)", fail)
|
||||
return ok, fail
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(message)s", stream=sys.stderr)
|
||||
ap = argparse.ArgumentParser(description="Download libretro PSP box art into a local bundle")
|
||||
ap.add_argument("--limit", type=int, default=None, help="only download the first N covers (testing)")
|
||||
ap.add_argument("--workers", type=int, default=8, help="parallel downloads")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
ok, fail = scrape(limit=args.limit, workers=args.workers)
|
||||
return 0 if fail == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
"""PSP Preservation Tool entry point (native GUI).
|
||||
|
||||
Run with::
|
||||
|
||||
python3 -m app.main
|
||||
"""
|
||||
from .gui import main
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+186
@@ -0,0 +1,186 @@
|
||||
"""Match scanned discs against the catalog: hash -> serial -> fuzzy name.
|
||||
|
||||
When hashes are unavailable (quick scans), serial matches are disambiguated by
|
||||
the disc's exact size, and fuzzy matches are narrowed the same way.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
_ARTICLE_RE = re.compile(r"^(.+),\s*(the|a|an)(?:\s+-\s+(.+))?$")
|
||||
|
||||
|
||||
def _normalize(name):
|
||||
name = name.lower()
|
||||
name = re.sub(r"\.(iso|cso|pbp|img|bin)$", "", name) # strip disc extension
|
||||
name = re.sub(r"^\s*\d+\s*-\s*", "", name) # leading No-Intro catalog number
|
||||
name = re.sub(r"\([^)]*\)", " ", name) # parenthetical tags
|
||||
name = " ".join(name.split())
|
||||
m = _ARTICLE_RE.match(name)
|
||||
if m:
|
||||
name = f"{m.group(2)} {m.group(1)}"
|
||||
if m.group(3):
|
||||
name += f" {m.group(3)}"
|
||||
name = re.sub(r"[^a-z0-9]+", " ", name)
|
||||
return " ".join(name.split())
|
||||
|
||||
|
||||
_supplement_cache = None
|
||||
_releases_cache = {} # id(conn) -> [(normalized_name, candidate)] for the fuzzy step
|
||||
|
||||
|
||||
def clear_cache():
|
||||
"""Drop cached release lists (call after re-ingesting a DAT)."""
|
||||
_releases_cache.clear()
|
||||
|
||||
|
||||
def _all_release_candidates(conn):
|
||||
"""Catalog releases + supplement entries with pre-normalized names."""
|
||||
key = id(conn)
|
||||
cands = _releases_cache.get(key)
|
||||
if cands is not None:
|
||||
return cands
|
||||
cands = [
|
||||
{
|
||||
"id": row["id"], "name": row["name"], "region": row["region"],
|
||||
"serial": row["serial"], "system": row["system"],
|
||||
}
|
||||
for row in conn.execute(
|
||||
"SELECT r.id, r.name, r.region, r.serial, d.system FROM releases r JOIN dats d ON d.id = r.dat_id"
|
||||
)
|
||||
]
|
||||
cands.extend(_supplement_release(e) for e in _load_supplement())
|
||||
cands = [(_normalize(c["name"]), c) for c in cands]
|
||||
_releases_cache[key] = cands
|
||||
return cands
|
||||
|
||||
|
||||
def _load_supplement():
|
||||
global _supplement_cache
|
||||
if _supplement_cache is None:
|
||||
path = config.DATA_DIR / "supplement.json"
|
||||
if path.exists():
|
||||
_supplement_cache = json.loads(path.read_text(encoding="utf-8"))
|
||||
else:
|
||||
_supplement_cache = []
|
||||
return _supplement_cache
|
||||
|
||||
|
||||
def _supplement_release(entry):
|
||||
return {
|
||||
"id": None,
|
||||
"name": entry.get("name"),
|
||||
"region": entry.get("region"),
|
||||
"serial": entry.get("serial"),
|
||||
"system": entry.get("system"),
|
||||
"size": entry.get("size"),
|
||||
}
|
||||
|
||||
|
||||
def _narrow(conn, candidates, size):
|
||||
"""Prefer candidates whose known disc size matches the on-disk file size."""
|
||||
if size is None or len(candidates) <= 1:
|
||||
return candidates
|
||||
matched = []
|
||||
for cand in candidates:
|
||||
if cand.get("id") is not None:
|
||||
row = conn.execute(
|
||||
"SELECT 1 FROM roms WHERE release_id = ? AND size = ? LIMIT 1",
|
||||
(cand["id"], size),
|
||||
).fetchone()
|
||||
if row:
|
||||
matched.append(cand)
|
||||
elif cand.get("size") == size:
|
||||
matched.append(cand)
|
||||
return matched or candidates
|
||||
|
||||
|
||||
def normalize_name(name):
|
||||
"""Public wrapper around the fuzzy normalizer (used by the Store)."""
|
||||
return _normalize(name)
|
||||
|
||||
|
||||
def serial_candidates(conn, record):
|
||||
"""Return all catalog releases matching the disc's serial (size-narrowed)."""
|
||||
serial = record.get("serial")
|
||||
if not serial:
|
||||
return []
|
||||
rows = conn.execute(
|
||||
"SELECT r.id, r.name, r.region, r.serial, d.system FROM releases r "
|
||||
"JOIN dats d ON d.id = r.dat_id WHERE upper(r.serial) = upper(?)",
|
||||
(serial,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
rows = conn.execute(
|
||||
"SELECT r.id, r.name, r.region, r.serial, d.system FROM roms ro "
|
||||
"JOIN releases r ON r.id = ro.release_id "
|
||||
"JOIN dats d ON d.id = r.dat_id WHERE upper(ro.serial) = upper(?)",
|
||||
(serial,),
|
||||
).fetchall()
|
||||
return _narrow(conn, [dict(r) for r in rows], record.get("size"))
|
||||
|
||||
|
||||
def match(conn, record):
|
||||
size = record.get("size")
|
||||
|
||||
# 1. Hash is the most precise key; serials are shared across revisions.
|
||||
for field in ("crc32", "sha1", "md5"):
|
||||
value = record.get(field)
|
||||
if not value:
|
||||
continue
|
||||
row = conn.execute(
|
||||
f"SELECT r.id, r.name, r.region, r.serial, d.system FROM roms ro "
|
||||
f"JOIN releases r ON r.id = ro.release_id "
|
||||
f"JOIN dats d ON d.id = r.dat_id "
|
||||
f"WHERE upper(ro.{field}) = ? LIMIT 1",
|
||||
(value.upper(),),
|
||||
).fetchone()
|
||||
if row is not None:
|
||||
return f"hash:{field}", dict(row)
|
||||
|
||||
# 2. Local supplement catalog (dumps not present in No-Intro).
|
||||
for entry in _load_supplement():
|
||||
for field in ("crc32", "sha1", "md5"):
|
||||
value = entry.get(field)
|
||||
if value and record.get(field) and value.upper() == record[field].upper():
|
||||
return f"supplement:{field}", _supplement_release(entry)
|
||||
|
||||
# 3. Serial (size-disambiguated for revisions that share a serial).
|
||||
serial = record.get("serial")
|
||||
if serial:
|
||||
candidates = serial_candidates(conn, record)
|
||||
if len(candidates) == 1:
|
||||
return "serial", candidates[0]
|
||||
if len(candidates) > 1:
|
||||
saved = conn.execute(
|
||||
"SELECT release_id FROM choices WHERE serial = ? AND size = ?",
|
||||
(serial.upper(), size),
|
||||
).fetchone()
|
||||
if saved is not None:
|
||||
for cand in candidates:
|
||||
if cand["id"] == saved["release_id"]:
|
||||
return "choice", cand
|
||||
return "unknown", None
|
||||
|
||||
# 4. Fuzzy fallback (filename + PARAM.SFO title), size-disambiguated.
|
||||
disk_norm = _normalize(record["name"])
|
||||
title_norm = _normalize(record.get("title") or "")
|
||||
|
||||
scored = []
|
||||
for cand_norm, cand in _all_release_candidates(conn):
|
||||
ratio = max(
|
||||
SequenceMatcher(None, disk_norm, cand_norm).ratio(),
|
||||
SequenceMatcher(None, title_norm, cand_norm).ratio(),
|
||||
)
|
||||
scored.append((ratio, cand))
|
||||
best_ratio = max((r for r, _ in scored), default=0.0)
|
||||
if best_ratio < 0.9:
|
||||
return "unknown", None
|
||||
top = [cand for ratio, cand in scored if ratio >= best_ratio - 0.001]
|
||||
top = _narrow(conn, top, size)
|
||||
if len(top) == 1:
|
||||
return f"fuzzy:{best_ratio:.2f}", top[0]
|
||||
return "unknown", None
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Build canonical filenames: ``Title (Region) (tags) (Serial).iso``.
|
||||
|
||||
Revision (``(v1.03)``), disc (``(Disc 1)``) and edition tags are preserved;
|
||||
language tags (``(En,Fr,De)``) are dropped; leading articles are moved back in
|
||||
front (``Sims 2, The`` -> ``The Sims 2``).
|
||||
"""
|
||||
import re
|
||||
|
||||
_ILLEGAL = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
||||
_TAG_RE = re.compile(r"\s*\(([^()]*)\)\s*$")
|
||||
_ARTICLE_RE = re.compile(r"^(.+),\s*(The|A|An)(?:\s+-\s+(.+))?$", re.IGNORECASE)
|
||||
|
||||
_LANG_CODES = {
|
||||
"en", "fr", "de", "es", "it", "nl", "pt", "sv", "no", "da", "fi",
|
||||
"ja", "ko", "zh", "pl", "ru", "el", "tr", "cs", "hu", "hi", "ar", "th",
|
||||
}
|
||||
|
||||
|
||||
def _is_language_tag(inner):
|
||||
parts = [p.strip().lower() for p in inner.split(",")]
|
||||
return bool(parts) and all(p in _LANG_CODES for p in parts)
|
||||
|
||||
|
||||
def _split_tags(name):
|
||||
"""Split trailing parenthetical tags; keep non-language tags, drop languages."""
|
||||
title = name
|
||||
kept = []
|
||||
while True:
|
||||
m = _TAG_RE.search(title)
|
||||
if not m:
|
||||
break
|
||||
inner = m.group(1).strip()
|
||||
if _is_language_tag(inner):
|
||||
title = title[: m.start()]
|
||||
else:
|
||||
kept.insert(0, f"({inner})")
|
||||
title = title[: m.start()]
|
||||
return title.strip(), kept
|
||||
|
||||
|
||||
def region_from_name(name):
|
||||
"""Extract the region from a release name (first non-language tag).
|
||||
|
||||
Used when a DAT leaves the region field empty and embeds it in the name
|
||||
(e.g. ``Oceans 11 (Japan, Europe)``).
|
||||
"""
|
||||
_, kept = _split_tags(str(name))
|
||||
if not kept:
|
||||
return ""
|
||||
inner = kept[0][1:-1].strip()
|
||||
return inner
|
||||
|
||||
|
||||
def _reorder_article(title):
|
||||
m = _ARTICLE_RE.match(title)
|
||||
if m:
|
||||
prefix, article, suffix = m.group(1), m.group(2), m.group(3)
|
||||
result = f"{article} {prefix}"
|
||||
if suffix:
|
||||
result += f" - {suffix}"
|
||||
return result
|
||||
return title
|
||||
|
||||
|
||||
def canonical_name(release, serial):
|
||||
name = release["name"]
|
||||
region = release.get("region")
|
||||
if region:
|
||||
for r in [x.strip() for x in region.split(",") if x.strip()]:
|
||||
name = re.sub(rf"\s*\(\s*{re.escape(r)}\s*\)", "", name, flags=re.IGNORECASE)
|
||||
title, tags = _split_tags(name)
|
||||
title = _reorder_article(title)
|
||||
parts = [title]
|
||||
if region and region != "World":
|
||||
parts.append(f"({region})")
|
||||
parts.extend(tags)
|
||||
if serial:
|
||||
parts.append(f"({serial})")
|
||||
name = _ILLEGAL.sub(" ", " ".join(parts)).strip()
|
||||
name = re.sub(r"\s+", " ", name)
|
||||
return f"{name}.iso"
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
"""Scrape PSP front covers from OpenVGDB (GameFAQs) into a local bundle.
|
||||
|
||||
OpenVGDB (https://github.com/OpenVGDB/OpenVGDB) is a downloadable SQLite dump
|
||||
whose PSP entries point at GameFAQs box art. GameFAQs sits behind a Cloudflare
|
||||
TLS-fingerprint challenge, so this scraper uses ``curl_cffi`` (which impersonates
|
||||
Chrome's TLS fingerprint) instead of plain urllib.
|
||||
|
||||
Usage::
|
||||
|
||||
python3 -m app.openvgdb # scrape everything
|
||||
python3 -m app.openvgdb --limit 50 # test with the first 50 covers
|
||||
python3 -m app.openvgdb --workers 8
|
||||
|
||||
Output: ``data/covers/openvgdb-covers.zip`` — entries named ``<norm(title)>.jpg``
|
||||
(``norm`` = lowercased, ``&``->``and``, non-alphanumerics stripped, parenthetical
|
||||
tags removed). The GUI reads this bundle with no network access.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sqlite3
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from . import config
|
||||
from .covers import _norm, _plain
|
||||
|
||||
log = logging.getLogger("openvgdb")
|
||||
|
||||
OVGDB_ZIP_URL = "https://github.com/OpenVGDB/OpenVGDB/releases/download/v29.0/openvgdb.zip"
|
||||
BUNDLE = config.DATA_DIR / "covers" / "openvgdb-covers.zip"
|
||||
|
||||
# Region preference for deduping the same game across regions: USA first, then
|
||||
# USA combos, then Europe, then Japan last (JP covers differ).
|
||||
_REGION_RANK = {21: 0, 27: 1, 28: 1, 29: 1, 30: 1, 38: 1, 7: 2, 31: 2, 25: 3, 13: 4}
|
||||
|
||||
|
||||
class _RateLimiter:
|
||||
"""Cap total requests per second across all worker threads."""
|
||||
|
||||
def __init__(self, per_second):
|
||||
self.interval = 1.0 / max(per_second, 0.1)
|
||||
self._lock = threading.Lock()
|
||||
self._next = 0.0
|
||||
|
||||
def wait(self):
|
||||
with self._lock:
|
||||
now = time.monotonic()
|
||||
if now < self._next:
|
||||
time.sleep(self._next - now)
|
||||
self._next = time.monotonic() + self.interval
|
||||
|
||||
|
||||
def _fetch(url, timeout=30):
|
||||
"""GET `url` impersonating Chrome (passes GameFAQs' Cloudflare challenge)."""
|
||||
from curl_cffi import requests
|
||||
resp = requests.get(url, impersonate="chrome", timeout=timeout)
|
||||
if resp.status_code != 200:
|
||||
raise RuntimeError(f"HTTP {resp.status_code} for {url}")
|
||||
return resp.content
|
||||
|
||||
|
||||
def build_index(sqlite_path):
|
||||
"""Return {norm_title: (display_title, cover_url)} preferring US regions."""
|
||||
conn = sqlite3.connect(str(sqlite_path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
rows = conn.execute(
|
||||
"SELECT releaseTitleName, regionLocalizedID, releaseCoverFront FROM RELEASES "
|
||||
"WHERE TEMPsystemShortName='PSP' AND releaseCoverFront IS NOT NULL"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
|
||||
best = {}
|
||||
for r in rows:
|
||||
title = (r["releaseTitleName"] or "").strip()
|
||||
url = (r["releaseCoverFront"] or "").strip()
|
||||
if not title or not url:
|
||||
continue
|
||||
key = _norm(_plain(title))
|
||||
if not key:
|
||||
continue
|
||||
rank = _REGION_RANK.get(r["regionLocalizedID"], 3)
|
||||
cur = best.get(key)
|
||||
if cur is None or rank < cur[2]:
|
||||
best[key] = (title, url, rank)
|
||||
return best
|
||||
|
||||
|
||||
def _existing_keys():
|
||||
"""Titles already present in the bundle (for resume)."""
|
||||
if not BUNDLE.exists():
|
||||
return set()
|
||||
try:
|
||||
with zipfile.ZipFile(BUNDLE) as z:
|
||||
return {n[:-4] for n in z.namelist() if n.endswith(".jpg")}
|
||||
except Exception: # noqa: BLE001
|
||||
return set()
|
||||
|
||||
|
||||
def _download_one(key, url, dest_dir, limiter, retries=4):
|
||||
from curl_cffi import requests
|
||||
dest = dest_dir / f"{key}.jpg"
|
||||
last = ""
|
||||
for attempt in range(1, retries + 1):
|
||||
limiter.wait()
|
||||
try:
|
||||
resp = requests.get(url, impersonate="chrome", timeout=30)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last = f"{type(exc).__name__}: {exc}"
|
||||
time.sleep(2 * attempt)
|
||||
continue
|
||||
if resp.status_code == 429:
|
||||
last = "HTTP 429"
|
||||
time.sleep(4 * attempt) # back off and let the rate limit clear
|
||||
continue
|
||||
if resp.status_code != 200:
|
||||
last = f"HTTP {resp.status_code}"
|
||||
continue
|
||||
data = resp.content
|
||||
if not data or data[:3] != b"\xff\xd8\xff": # JPEG magic
|
||||
last = "not a JPEG"
|
||||
continue
|
||||
tmp = dest.with_suffix(".tmp")
|
||||
tmp.write_bytes(data)
|
||||
os.replace(tmp, dest)
|
||||
return True, ""
|
||||
return False, last
|
||||
|
||||
|
||||
def scrape(limit=None, workers=8, rate=3.0):
|
||||
BUNDLE.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
log.info("downloading OpenVGDB dump (%s)…", OVGDB_ZIP_URL)
|
||||
tmpdir = Path(tempfile.mkdtemp(prefix="openvgdb-"))
|
||||
dump_zip = tmpdir / "openvgdb.zip"
|
||||
req = urllib.request.Request(OVGDB_ZIP_URL, headers={"User-Agent": "psp-preserve/0.1"})
|
||||
with urllib.request.urlopen(req, timeout=60) as r, open(dump_zip, "wb") as f:
|
||||
f.write(r.read())
|
||||
|
||||
log.info("extracting SQLite…")
|
||||
sqlite_path = tmpdir / "openvgdb.sqlite"
|
||||
with zipfile.ZipFile(dump_zip) as z:
|
||||
z.extract("openvgdb.sqlite", tmpdir)
|
||||
|
||||
index = build_index(sqlite_path)
|
||||
log.info("indexed %d unique PSP covers", len(index))
|
||||
|
||||
existing = _existing_keys()
|
||||
todo = [(k, v) for k, v in sorted(index.items()) if k not in existing]
|
||||
if limit:
|
||||
todo = todo[:limit]
|
||||
log.info("%d to download (%d already in bundle)", len(todo), len(existing))
|
||||
|
||||
work_dir = Path(tempfile.mkdtemp(prefix="openvgdb-covers-"))
|
||||
limiter = _RateLimiter(rate)
|
||||
ok = fail = 0
|
||||
done = 0
|
||||
lock = threading.Lock()
|
||||
|
||||
def run(item):
|
||||
key, (title, url, _rank) = item
|
||||
success, err = _download_one(key, url, work_dir, limiter)
|
||||
nonlocal ok, fail, done
|
||||
with lock:
|
||||
done += 1
|
||||
if success:
|
||||
ok += 1
|
||||
else:
|
||||
fail += 1
|
||||
if done % 50 == 0 or not success:
|
||||
log.info(" %d/%d (%d ok, %d failed)%s", done, len(todo), ok, fail,
|
||||
f" {title!r}: {err}" if not success else "")
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
list(pool.map(run, todo))
|
||||
|
||||
if ok:
|
||||
log.info("writing bundle…")
|
||||
out_tmp = BUNDLE.with_name(BUNDLE.name + ".tmp")
|
||||
existing_bytes = {}
|
||||
if BUNDLE.exists():
|
||||
try:
|
||||
with zipfile.ZipFile(BUNDLE) as zin:
|
||||
for n in zin.namelist():
|
||||
if n.endswith(".jpg") and n not in existing_bytes:
|
||||
existing_bytes[n] = zin.read(n)
|
||||
except Exception: # noqa: BLE001
|
||||
existing_bytes = {}
|
||||
with zipfile.ZipFile(out_tmp, "w", zipfile.ZIP_STORED) as z:
|
||||
for name, data in existing_bytes.items():
|
||||
z.writestr(name, data)
|
||||
for f in sorted(work_dir.glob("*.jpg")):
|
||||
z.write(f, arcname=f.name)
|
||||
os.replace(out_tmp, BUNDLE)
|
||||
log.info("done: bundle now has %d covers (%d new, %d failed this run)",
|
||||
len(existing_bytes) + ok, ok, fail)
|
||||
else:
|
||||
log.info("no new covers downloaded; bundle unchanged (%d failed)", fail)
|
||||
return ok, fail
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(message)s", stream=sys.stderr)
|
||||
ap = argparse.ArgumentParser(description="Scrape PSP covers from OpenVGDB into a local bundle")
|
||||
ap.add_argument("--limit", type=int, default=None, help="only scrape the first N covers (testing)")
|
||||
ap.add_argument("--workers", type=int, default=8, help="parallel downloads")
|
||||
ap.add_argument("--rate", type=float, default=3.0, help="max requests/second (avoid 429)")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
try:
|
||||
import curl_cffi # noqa: F401
|
||||
except ImportError:
|
||||
print("This scraper needs curl_cffi: pip install curl_cffi", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
ok, fail = scrape(limit=args.limit, workers=args.workers, rate=args.rate)
|
||||
return 0 if fail == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,16 @@
|
||||
"""High-level identification: match + canonical naming for one scanned disc."""
|
||||
from . import iso_reader, matcher, namer
|
||||
|
||||
|
||||
def identify(conn, record):
|
||||
"""Return ``{method, release, serial, canonical}`` for a scanned disc record."""
|
||||
method, release = matcher.match(conn, record)
|
||||
return identify_with_release(record, release, method)
|
||||
|
||||
|
||||
def identify_with_release(record, release, method):
|
||||
serial = record.get("serial")
|
||||
if not serial and release and iso_reader.looks_like_serial(release.get("serial")):
|
||||
serial = iso_reader.normalize_serial(release.get("serial"))
|
||||
canonical = namer.canonical_name(release, serial) if release else None
|
||||
return {"method": method, "release": release, "serial": serial, "canonical": canonical}
|
||||
+329
@@ -0,0 +1,329 @@
|
||||
"""PSP device detection and memory-stick inventory.
|
||||
|
||||
The PSP mounts as a plain USB mass-storage device whose root contains a
|
||||
``PSP/`` folder (and, on CFW, an ``ISO/`` folder). This module finds mounted
|
||||
PSP sticks, lists what is installed on them, and reports free space.
|
||||
"""
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from . import matcher
|
||||
|
||||
|
||||
def _candidate_bases():
|
||||
bases = ["/media", "/run/media", "/mnt"]
|
||||
user = os.environ.get("USER")
|
||||
if user:
|
||||
bases += [f"/media/{user}", f"/run/media/{user}"]
|
||||
return list(dict.fromkeys(bases))
|
||||
|
||||
|
||||
def is_psp_root(path):
|
||||
"""True if ``path`` looks like the root of a PSP memory stick.
|
||||
|
||||
A real stick has a ``PSP`` folder containing the firmware-created ``GAME``
|
||||
or ``SYSTEM`` subfolders, and CFW sticks also have an ``ISO`` folder. A
|
||||
plain folder named ``psp`` that only holds saves (e.g. a cloud-save
|
||||
collection) is *not* a memory stick, so it is rejected.
|
||||
"""
|
||||
p = Path(path)
|
||||
if not p.is_dir():
|
||||
return False
|
||||
psp_dir = p / "PSP"
|
||||
if psp_dir.is_dir():
|
||||
return (psp_dir / "GAME").is_dir() or (psp_dir / "SYSTEM").is_dir()
|
||||
return (p / "ISO").is_dir()
|
||||
|
||||
|
||||
def detect_mounts():
|
||||
"""Return a list of mounted filesystem roots that look like PSP sticks."""
|
||||
found = []
|
||||
seen = set()
|
||||
for base in _candidate_bases():
|
||||
base_path = Path(base)
|
||||
if not base_path.is_dir():
|
||||
continue
|
||||
try:
|
||||
children = list(base_path.iterdir())
|
||||
except OSError:
|
||||
continue
|
||||
for child in sorted(children):
|
||||
if not child.is_dir():
|
||||
continue
|
||||
try:
|
||||
key = str(child.resolve())
|
||||
except OSError:
|
||||
key = str(child)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if is_psp_root(child):
|
||||
found.append(str(child))
|
||||
return found
|
||||
|
||||
|
||||
def free_space(root):
|
||||
"""Return ``{total, used, free}`` bytes, or None on failure."""
|
||||
try:
|
||||
usage = shutil.disk_usage(root)
|
||||
except OSError:
|
||||
return None
|
||||
return {"total": usage.total, "used": usage.used, "free": usage.free}
|
||||
|
||||
|
||||
def _file_item(path, kind):
|
||||
try:
|
||||
st = path.stat()
|
||||
size = st.st_size
|
||||
mtime = int(st.st_mtime)
|
||||
except OSError:
|
||||
size = 0
|
||||
mtime = 0
|
||||
return {"name": path.name, "path": str(path), "size": size, "mtime": mtime, "kind": kind}
|
||||
|
||||
|
||||
DISC_SUFFIXES = {".iso", ".cso"}
|
||||
|
||||
|
||||
def _iso_items(root):
|
||||
"""Split installed disc images into games (ISO/), videos (ISO/VIDEO/), and
|
||||
homebrew ISOs (ISO/Homebrew/, used by Category Lite). Handles both ``.iso``
|
||||
and compressed ``.cso`` images."""
|
||||
games, videos, homebrew = [], [], []
|
||||
iso_dir = root / "ISO"
|
||||
if iso_dir.is_dir():
|
||||
for path in sorted(iso_dir.rglob("*")):
|
||||
if not path.is_file() or path.suffix.lower() not in DISC_SUFFIXES:
|
||||
continue
|
||||
rel = path.relative_to(iso_dir)
|
||||
first = rel.parts[0].lower() if rel.parts else ""
|
||||
if first in ("video", "videos"):
|
||||
videos.append(_file_item(path, "video"))
|
||||
elif "homebrew" in first:
|
||||
# Category Lite may use "Homebrew", "CAT_Homebrew", etc.
|
||||
homebrew.append(_file_item(path, "homebrew_iso"))
|
||||
else:
|
||||
item = _file_item(path, "game")
|
||||
item["category"] = rel.parts[0] if len(rel.parts) > 1 else ""
|
||||
games.append(item)
|
||||
|
||||
# UMD video ISOs may also live in PSP/VIDEO on some CFW setups.
|
||||
psp_video = root / "PSP" / "VIDEO"
|
||||
if psp_video.is_dir():
|
||||
for path in sorted(psp_video.rglob("*")):
|
||||
if path.is_file() and path.suffix.lower() in DISC_SUFFIXES:
|
||||
videos.append(_file_item(path, "video"))
|
||||
return games, videos, homebrew
|
||||
|
||||
|
||||
def _read_pbp_param_sfo(pbp_path):
|
||||
"""Extract the PARAM.SFO blob embedded in an EBOOT.PBP, or None."""
|
||||
try:
|
||||
with open(pbp_path, "rb") as fh:
|
||||
header = fh.read(0x28)
|
||||
except OSError:
|
||||
return None
|
||||
if len(header) < 0x28 or header[:4] != b"\x00PBP":
|
||||
return None
|
||||
offsets = struct.unpack_from("<8I", header, 0x08)
|
||||
start, end = offsets[0], offsets[1]
|
||||
if end <= start:
|
||||
return None
|
||||
try:
|
||||
with open(pbp_path, "rb") as fh:
|
||||
fh.seek(start)
|
||||
return fh.read(end - start)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _sfo_strings(data):
|
||||
"""Parse a PARAM.SFO blob into a ``{key: value}`` dict of strings/ints."""
|
||||
out = {}
|
||||
if len(data) < 20 or data[:4] != b"\x00PSF":
|
||||
return out
|
||||
_version, key_table, data_table, count = struct.unpack_from("<IIII", data, 0x04)
|
||||
if count <= 0:
|
||||
return out
|
||||
size = len(data)
|
||||
# The index table sits right after the header, before the key table.
|
||||
index_start = key_table - count * 16
|
||||
if index_start < 20:
|
||||
index_start = 20
|
||||
if index_start + count * 16 > size:
|
||||
return out
|
||||
for i in range(count):
|
||||
off = index_start + i * 16
|
||||
key_off, fmt, length, _max_len, data_off = struct.unpack_from("<HHIII", data, off)
|
||||
if key_table + key_off >= size:
|
||||
continue
|
||||
key = data[key_table + key_off:].split(b"\x00", 1)[0].decode("utf-8", "replace")
|
||||
value_off = data_table + data_off
|
||||
if value_off + length > size:
|
||||
continue
|
||||
raw = data[value_off:value_off + length]
|
||||
if fmt in (0x0004, 0x0404) and length == 4: # uint32
|
||||
out[key] = struct.unpack_from("<I", data, value_off)[0]
|
||||
else: # UTF-8 string
|
||||
out[key] = raw.split(b"\x00", 1)[0].decode("utf-8", "replace").strip()
|
||||
return out
|
||||
|
||||
|
||||
def _pbp_title(pbp_path):
|
||||
"""Return the TITLE from an EBOOT.PBP's PARAM.SFO, or None."""
|
||||
try:
|
||||
sfo = _read_pbp_param_sfo(pbp_path)
|
||||
if sfo:
|
||||
return _sfo_strings(sfo).get("TITLE") or None
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _homebrew_items(root):
|
||||
"""PSP/GAME/**/EBOOT.PBP (homebrew, PSN, and Category Lite folders).
|
||||
|
||||
Category Lite nests homebrew one level deeper
|
||||
(e.g. PSP/GAME/CAT_Homebrew/App/EBOOT.PBP), so search recursively. The
|
||||
display name is the title from the PBP's embedded PARAM.SFO when available,
|
||||
falling back to the folder name."""
|
||||
items = []
|
||||
game_dir = root / "PSP" / "GAME"
|
||||
if game_dir.is_dir():
|
||||
for eboot in sorted(game_dir.rglob("EBOOT.PBP")):
|
||||
if not eboot.is_file():
|
||||
continue
|
||||
try:
|
||||
st = eboot.stat()
|
||||
size = st.st_size
|
||||
mtime = int(st.st_mtime)
|
||||
except OSError:
|
||||
size = 0
|
||||
mtime = 0
|
||||
name = _pbp_title(eboot) or eboot.parent.name
|
||||
items.append({"name": name, "id": eboot.parent.name, "path": str(eboot), "size": size, "mtime": mtime, "kind": "homebrew"})
|
||||
return items
|
||||
|
||||
|
||||
def _pkg_items(root):
|
||||
""".pkg installer files at the root of the stick (CFW package installers)."""
|
||||
items = []
|
||||
try:
|
||||
entries = list(root.iterdir())
|
||||
except OSError:
|
||||
return items
|
||||
for path in sorted(entries):
|
||||
if path.is_file() and path.suffix.lower() == ".pkg":
|
||||
items.append(_file_item(path, "pkg"))
|
||||
return items
|
||||
|
||||
|
||||
def list_installed(root):
|
||||
"""Return ``{games, videos, homebrew, pkg}`` inventories for a PSP root."""
|
||||
root = Path(root)
|
||||
games, videos, homebrew_isos = _iso_items(root)
|
||||
return {
|
||||
"games": games,
|
||||
"videos": videos,
|
||||
"homebrew": _homebrew_items(root) + homebrew_isos,
|
||||
"pkg": _pkg_items(root),
|
||||
}
|
||||
|
||||
|
||||
def build_index(inventory):
|
||||
"""Build a lookup index for matching library discs against the PSP.
|
||||
|
||||
Returns ``{"raw": set, "names": set}`` where ``raw`` holds lowercase ISO
|
||||
stems (exact filename match) and ``names`` holds fuzzy-normalized stems.
|
||||
"""
|
||||
raw, names = set(), set()
|
||||
for group in ("games", "videos"):
|
||||
for it in (inventory or {}).get(group, []):
|
||||
stem = Path(it["name"]).stem
|
||||
raw.add(stem.lower())
|
||||
norm = matcher.normalize_name(stem)
|
||||
if norm:
|
||||
names.add(norm)
|
||||
return {"raw": raw, "names": names}
|
||||
|
||||
|
||||
def find_umd_iso(root):
|
||||
"""Return the path of the ``UMD9660.ISO`` at a mount root, or None."""
|
||||
try:
|
||||
entries = list(Path(root).iterdir())
|
||||
except OSError:
|
||||
return None
|
||||
for path in entries:
|
||||
if path.is_file() and path.name.lower() == "umd9660.iso":
|
||||
return path
|
||||
return None
|
||||
|
||||
|
||||
def umd_mounts():
|
||||
"""Return mounted UMD volumes (USB UMD mode).
|
||||
|
||||
In USB UMD mode the PSP exposes a single volume whose name is the disc's
|
||||
catalogue number (e.g. ``ULUS-10268``) and whose root holds one ISO named
|
||||
``UMD9660.ISO``."""
|
||||
found, seen = [], set()
|
||||
for base in _candidate_bases():
|
||||
base_path = Path(base)
|
||||
if not base_path.is_dir():
|
||||
continue
|
||||
try:
|
||||
children = list(base_path.iterdir())
|
||||
except OSError:
|
||||
continue
|
||||
for child in sorted(children):
|
||||
if not child.is_dir():
|
||||
continue
|
||||
try:
|
||||
key = str(child.resolve())
|
||||
except OSError:
|
||||
key = str(child)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
if find_umd_iso(child) is not None:
|
||||
found.append(str(child))
|
||||
return found
|
||||
|
||||
|
||||
def block_device_for_mount(root):
|
||||
"""Return the block device backing a mount point, or None."""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["findmnt", "-n", "-o", "SOURCE", "--target", str(root)],
|
||||
capture_output=True, text=True, timeout=10,
|
||||
)
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
return None
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
lines = [ln.strip() for ln in proc.stdout.splitlines() if ln.strip()]
|
||||
return lines[0] if lines else None
|
||||
|
||||
|
||||
def eject(root):
|
||||
"""Unmount a mounted PSP stick so it is safe to unplug.
|
||||
|
||||
Returns ``(ok, message)``.
|
||||
"""
|
||||
root = Path(root)
|
||||
device = block_device_for_mount(root)
|
||||
if not device:
|
||||
return False, f"No block device found for {root}"
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["udisksctl", "unmount", "-b", device],
|
||||
capture_output=True, text=True, timeout=30,
|
||||
)
|
||||
except OSError as exc:
|
||||
return False, f"udisksctl failed: {exc}"
|
||||
if proc.returncode == 0:
|
||||
return True, proc.stdout.strip() or f"Unmounted {root}"
|
||||
err = (proc.stderr or proc.stdout or "").strip()
|
||||
return False, err or f"Unmount failed (exit code {proc.returncode})"
|
||||
+227
@@ -0,0 +1,227 @@
|
||||
"""A background transfer queue with progress reporting.
|
||||
|
||||
Jobs (copy / download / dump) run one at a time on a worker thread and report
|
||||
progress back to the GUI via Qt signals.
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from PySide6.QtCore import QObject, QThread, Signal
|
||||
|
||||
from .store import TransferCancelled
|
||||
|
||||
log = logging.getLogger("queue")
|
||||
|
||||
|
||||
class TransferJob:
|
||||
_ids = 0
|
||||
|
||||
def __init__(self, kind, title, source_label, dest):
|
||||
TransferJob._ids += 1
|
||||
self.id = TransferJob._ids
|
||||
self.kind = kind
|
||||
self.title = title
|
||||
self.source_label = source_label
|
||||
self.dest = str(dest)
|
||||
self.item = None
|
||||
self.source = None
|
||||
self.dest_dir = None
|
||||
self.status = "queued" # queued/running/done/skipped/cancelled/cancelling/error
|
||||
self.stage = ""
|
||||
self.detail = ""
|
||||
self.progress = 0.0
|
||||
self.error = ""
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
"id": self.id, "kind": self.kind, "title": self.title,
|
||||
"source": self.source_label, "dest": self.dest,
|
||||
"status": self.status, "stage": self.stage, "detail": self.detail,
|
||||
"progress": self.progress, "error": self.error,
|
||||
}
|
||||
|
||||
|
||||
class JobWorker(QObject):
|
||||
progress = Signal(int, str, float, str) # job_id, stage, fraction, detail
|
||||
finished = Signal(int, object) # job_id, result
|
||||
skipped = Signal(int, str) # job_id, path (already present)
|
||||
cancelled = Signal(int) # job_id
|
||||
failed = Signal(int, str) # job_id, error
|
||||
|
||||
def __init__(self, source, dest_dir, job):
|
||||
super().__init__()
|
||||
self.source = source
|
||||
self.dest_dir = dest_dir
|
||||
self.job = job
|
||||
self._cancel = threading.Event()
|
||||
|
||||
def cancel(self):
|
||||
self._cancel.set()
|
||||
|
||||
def _progress(self, stage, fraction, detail=""):
|
||||
self.progress.emit(self.job.id, stage, fraction, detail)
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
result = self.source.download(self.job.item, self.dest_dir, self._progress, self._cancel)
|
||||
self.finished.emit(self.job.id, result)
|
||||
except TransferCancelled:
|
||||
self.cancelled.emit(self.job.id)
|
||||
except FileExistsError as exc:
|
||||
self.skipped.emit(self.job.id, str(exc))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.failed.emit(self.job.id, str(exc))
|
||||
|
||||
|
||||
class TransferQueue(QObject):
|
||||
job_added = Signal(dict)
|
||||
job_updated = Signal(dict)
|
||||
job_finished = Signal(dict)
|
||||
job_removed = Signal(int)
|
||||
|
||||
def __init__(self, parent=None):
|
||||
super().__init__(parent)
|
||||
self.jobs = []
|
||||
self._pending = []
|
||||
self._current = None
|
||||
self._thread = None
|
||||
self._worker = None
|
||||
# Keep strong references to every worker/thread for their full lifetime.
|
||||
# Dropping a Python ref to a QObject that was moved to another thread
|
||||
# deletes it directly and crashes (bus error).
|
||||
self._threads = []
|
||||
self._workers = []
|
||||
|
||||
def enqueue(self, source, dest_dir, item, title, kind="download"):
|
||||
job = TransferJob(kind, title, getattr(source, "label", "source"), dest_dir)
|
||||
job.item = item
|
||||
job.source = source
|
||||
job.dest_dir = dest_dir
|
||||
self.jobs.append(job)
|
||||
self._pending.append(job)
|
||||
log.info("queued %s (%s) -> %s", title, kind, dest_dir)
|
||||
self.job_added.emit(job.to_dict())
|
||||
self._start_next()
|
||||
return job
|
||||
|
||||
def cancel(self, job_id):
|
||||
job = self._find(job_id)
|
||||
if job is None:
|
||||
return
|
||||
if job.status == "queued":
|
||||
if job in self._pending:
|
||||
self._pending.remove(job)
|
||||
job.status = "cancelled"
|
||||
self.job_updated.emit(job.to_dict())
|
||||
self.job_finished.emit(job.to_dict())
|
||||
return
|
||||
if job.status == "running" and self._worker is not None:
|
||||
self._worker.cancel()
|
||||
job.status = "cancelling"
|
||||
self.job_updated.emit(job.to_dict())
|
||||
|
||||
def remove(self, job_id):
|
||||
"""Drop a job from the history (active jobs must be cancelled first)."""
|
||||
job = self._find(job_id)
|
||||
if job is None:
|
||||
return
|
||||
if job.status in ("running", "cancelling"):
|
||||
return
|
||||
if job in self._pending:
|
||||
self._pending.remove(job)
|
||||
self.jobs.remove(job)
|
||||
self.job_removed.emit(job_id)
|
||||
|
||||
def clear_finished(self):
|
||||
"""Remove all finished jobs from the history."""
|
||||
finished = [j.id for j in self.jobs
|
||||
if j.status in ("done", "skipped", "cancelled", "error")]
|
||||
for jid in finished:
|
||||
self.remove(jid)
|
||||
|
||||
def shutdown(self):
|
||||
if self._worker is not None:
|
||||
self._worker.cancel()
|
||||
for t in self._threads:
|
||||
try:
|
||||
if t.isRunning():
|
||||
t.quit()
|
||||
t.wait()
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
def _find(self, job_id):
|
||||
for j in self.jobs:
|
||||
if j.id == job_id:
|
||||
return j
|
||||
return None
|
||||
|
||||
def _start_next(self):
|
||||
if self._current is not None or not self._pending:
|
||||
return
|
||||
job = self._pending.pop(0)
|
||||
self._current = job
|
||||
job.status = "running"
|
||||
log.info("starting job #%d: %s (%s)", job.id, job.title, job.kind)
|
||||
self.job_updated.emit(job.to_dict())
|
||||
|
||||
self._thread = QThread(self)
|
||||
self._worker = JobWorker(job.source, job.dest_dir, job)
|
||||
self._worker.moveToThread(self._thread)
|
||||
self._thread.started.connect(self._worker.run)
|
||||
self._worker.progress.connect(self._on_progress)
|
||||
self._worker.finished.connect(self._on_finished)
|
||||
self._worker.skipped.connect(self._on_skipped)
|
||||
self._worker.cancelled.connect(self._on_cancelled)
|
||||
self._worker.failed.connect(self._on_failed)
|
||||
for sig in (self._worker.finished, self._worker.skipped,
|
||||
self._worker.cancelled, self._worker.failed):
|
||||
sig.connect(self._thread.quit)
|
||||
self._thread.finished.connect(self._on_thread_finished)
|
||||
self._threads.append(self._thread)
|
||||
self._workers.append(self._worker)
|
||||
self._thread.start()
|
||||
|
||||
def _on_progress(self, job_id, stage, fraction, detail):
|
||||
job = self._find(job_id)
|
||||
if job is None:
|
||||
return
|
||||
job.stage = stage
|
||||
job.progress = fraction
|
||||
job.detail = detail
|
||||
self.job_updated.emit(job.to_dict())
|
||||
|
||||
def _on_finished(self, job_id, result):
|
||||
self._finish_job(job_id, "done", detail=str(result) if result is not None else "")
|
||||
|
||||
def _on_skipped(self, job_id, path):
|
||||
self._finish_job(job_id, "skipped", error=path)
|
||||
|
||||
def _on_cancelled(self, job_id):
|
||||
self._finish_job(job_id, "cancelled")
|
||||
|
||||
def _on_failed(self, job_id, error):
|
||||
self._finish_job(job_id, "error", error=error)
|
||||
|
||||
def _finish_job(self, job_id, status, detail="", error=""):
|
||||
job = self._find(job_id)
|
||||
if job is not None:
|
||||
job.status = status
|
||||
if detail:
|
||||
job.detail = detail
|
||||
if error:
|
||||
job.error = error
|
||||
if status == "done":
|
||||
job.progress = 1.0
|
||||
log.info("job #%d %s: %s%s", job.id, status, job.title,
|
||||
f" — {error}" if error else "")
|
||||
self.job_finished.emit(job.to_dict())
|
||||
|
||||
def _on_thread_finished(self):
|
||||
# Queued to the main thread once the worker thread has fully stopped,
|
||||
# so it is now safe to drop the "current" pointers and advance.
|
||||
self._current = None
|
||||
self._thread = None
|
||||
self._worker = None
|
||||
self._start_next()
|
||||
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
"""PSP save-data inventory and two-way sync (PSP / PPSSPP / cloud folder).
|
||||
|
||||
PSP saves live in ``PSP/SAVEDATA/<TITLE_ID>/`` folders (``PARAM.SFO`` + data).
|
||||
PPSSPP uses the exact same layout under its own "memory stick" directory, so a
|
||||
save folder can be copied verbatim between the PSP, PPSSPP, and a cloud folder.
|
||||
"""
|
||||
import hashlib
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, iso_reader
|
||||
from .db import get_connection
|
||||
|
||||
SAVEDATA_NAME = "SAVEDATA"
|
||||
|
||||
|
||||
def default_ppsspp_saves():
|
||||
"""Best-guess PPSSPP SAVEDATA directory (Linux-first, falls back to a default)."""
|
||||
home = Path.home()
|
||||
candidates = [
|
||||
home / ".config" / "ppsspp" / "PSP" / SAVEDATA_NAME,
|
||||
home / ".local" / "share" / "ppsspp" / "PSP" / SAVEDATA_NAME,
|
||||
home / ".config" / "ppsspp" / SAVEDATA_NAME,
|
||||
home / ".var" / "app" / "org.ppsspp.PPSSPP" / "data" / "ppsspp" / "PSP" / SAVEDATA_NAME,
|
||||
home / "Documents" / "PPSSPP" / "PSP" / SAVEDATA_NAME,
|
||||
]
|
||||
for cand in candidates:
|
||||
if cand.is_dir():
|
||||
return cand
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def read_save_meta(folder):
|
||||
"""Return {title, detail, size, mtime, encrypted} for one save folder."""
|
||||
folder = Path(folder)
|
||||
title, detail = "", ""
|
||||
sfo_path = folder / "PARAM.SFO"
|
||||
if sfo_path.is_file():
|
||||
try:
|
||||
sfo = iso_reader.parse_sfo(sfo_path.read_bytes())
|
||||
except Exception: # noqa: BLE001
|
||||
sfo = {}
|
||||
title = sfo.get("SAVEDATA_TITLE") or sfo.get("TITLE") or ""
|
||||
detail = sfo.get("SAVEDATA_DETAIL") or ""
|
||||
|
||||
size = 0
|
||||
mtime = 0.0
|
||||
mtime_ns = 0
|
||||
encrypted = False
|
||||
for p in folder.rglob("*"):
|
||||
if not p.is_file():
|
||||
continue
|
||||
if p.name.upper() == "SECURE.BIN":
|
||||
encrypted = True
|
||||
try:
|
||||
st = p.stat()
|
||||
except OSError:
|
||||
continue
|
||||
size += st.st_size
|
||||
mtime = max(mtime, st.st_mtime)
|
||||
mtime_ns = max(mtime_ns, st.st_mtime_ns)
|
||||
return {"title": title or folder.name, "detail": detail, "size": size,
|
||||
"mtime": mtime, "mtime_ns": mtime_ns, "encrypted": encrypted}
|
||||
|
||||
|
||||
def list_saves(root):
|
||||
"""Return [{id, path, title, detail, size, mtime, encrypted}] for a SAVEDATA dir."""
|
||||
root = Path(root) if root else None
|
||||
if root is None or not root.is_dir():
|
||||
return []
|
||||
out = []
|
||||
for folder in sorted(root.iterdir()):
|
||||
if not folder.is_dir():
|
||||
continue
|
||||
meta = read_save_meta(folder)
|
||||
meta["id"] = folder.name
|
||||
meta["path"] = str(folder)
|
||||
out.append(meta)
|
||||
return out
|
||||
|
||||
|
||||
_sig_cache = {} # (save_id, size, mtime_ns) -> hexdigest
|
||||
_cache_loaded = False
|
||||
|
||||
|
||||
def _load_cache():
|
||||
"""Populate the signature cache from the library DB (once per process)."""
|
||||
global _cache_loaded
|
||||
if _cache_loaded:
|
||||
return
|
||||
_cache_loaded = True
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS save_sigs ("
|
||||
" save_id TEXT NOT NULL, size INTEGER NOT NULL, mtime_ns INTEGER NOT NULL,"
|
||||
" sig TEXT NOT NULL, PRIMARY KEY (save_id, size, mtime_ns))"
|
||||
)
|
||||
for r in conn.execute("SELECT save_id, size, mtime_ns, sig FROM save_sigs"):
|
||||
_sig_cache[(r["save_id"], r["size"], r["mtime_ns"])] = r["sig"]
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
|
||||
|
||||
def save_signature(folder, size=None, mtime_ns=None):
|
||||
"""Content fingerprint of a save folder (paths + bytes), mtime-independent.
|
||||
|
||||
Two saves with identical files count as the same "version" even when their
|
||||
modification times differ (after a copy, or across filesystems). Results
|
||||
are cached by ``(save_id, size, mtime_ns)`` in memory and persisted to the
|
||||
library DB, so an unchanged save is never re-hashed — even after a restart.
|
||||
"""
|
||||
folder = Path(folder)
|
||||
save_id = folder.name
|
||||
if size is None or mtime_ns is None:
|
||||
meta = read_save_meta(folder)
|
||||
size = meta["size"]
|
||||
mtime_ns = meta["mtime_ns"]
|
||||
_load_cache()
|
||||
key = (save_id, size, mtime_ns)
|
||||
cached = _sig_cache.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
h = hashlib.sha256()
|
||||
for p in sorted(folder.rglob("*")):
|
||||
if not p.is_file():
|
||||
continue
|
||||
h.update(p.relative_to(folder).as_posix().encode("utf-8", "replace"))
|
||||
h.update(b"\0")
|
||||
try:
|
||||
with open(p, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1 << 16), b""):
|
||||
h.update(chunk)
|
||||
except OSError: # noqa: BLE001
|
||||
pass
|
||||
h.update(b"\0")
|
||||
sig = h.hexdigest()
|
||||
_sig_cache[key] = sig
|
||||
try:
|
||||
conn = get_connection(config.DB_PATH)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO save_sigs (save_id, size, mtime_ns, sig) VALUES (?,?,?,?)",
|
||||
(save_id, size, mtime_ns, sig),
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
except sqlite3.Error:
|
||||
pass
|
||||
return sig
|
||||
|
||||
|
||||
def sync_plan(locations):
|
||||
"""Converge every location to the newest *version* of each save.
|
||||
|
||||
``locations`` is ``{name: root_path}`` (empty/missing roots are ignored).
|
||||
Saves are fingerprinted by content, so a location whose copy is already the
|
||||
same version is never re-synced (mtime differences are ignored).
|
||||
|
||||
Returns a list of ``{"src": path, "dst_root": path, "dst_name": save_id}``.
|
||||
"""
|
||||
per_loc = {name: {s["id"]: s for s in list_saves(root)} for name, root in locations.items() if root}
|
||||
active = [loc for loc, root in locations.items() if root]
|
||||
all_ids = set()
|
||||
for saves_map in per_loc.values():
|
||||
all_ids.update(saves_map)
|
||||
|
||||
plan = []
|
||||
for sid in sorted(all_ids):
|
||||
entries = {}
|
||||
for loc in active:
|
||||
save = per_loc[loc].get(sid)
|
||||
if save is not None:
|
||||
save["sig"] = save_signature(save["path"], save["size"], save["mtime_ns"])
|
||||
entries[loc] = save
|
||||
|
||||
# Group existing copies into distinct versions by content signature.
|
||||
versions = {}
|
||||
for save in entries.values():
|
||||
versions.setdefault(save["sig"], []).append(save)
|
||||
|
||||
# The newest version wins (best-effort tiebreak by mtime).
|
||||
newest = max(versions.values(), key=lambda group: max(s["mtime"] for s in group))[0]
|
||||
|
||||
# Replace any copy that is a different version.
|
||||
for loc, save in entries.items():
|
||||
if save["sig"] != newest["sig"]:
|
||||
plan.append({"src": newest["path"], "dst_root": locations[loc], "dst_name": sid})
|
||||
# Fill locations that don't have this save at all.
|
||||
for loc in active:
|
||||
if sid not in per_loc[loc]:
|
||||
plan.append({"src": newest["path"], "dst_root": locations[loc], "dst_name": sid})
|
||||
return plan
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""Scan, match, and report on PSP discs. Dry-run by default.
|
||||
|
||||
Usage::
|
||||
|
||||
python3 -m app.scan --roots /mnt/user/psp/ISO
|
||||
python3 -m app.scan --roots /mnt/user/psp/ISO --apply # actually rename
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, pipeline, scanner
|
||||
from .db import init_db
|
||||
|
||||
|
||||
def _roots_from_arg(value):
|
||||
if value:
|
||||
raw = value
|
||||
else:
|
||||
raw = ",".join(map(str, config.GAMES_ROOTS + config.VIDEO_ROOTS))
|
||||
return [Path(p) for p in raw.split(",") if p.strip()]
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)-7s %(message)s", stream=sys.stderr)
|
||||
|
||||
ap = argparse.ArgumentParser(description="Scan PSP .iso discs and match against the catalog")
|
||||
ap.add_argument("--roots", default=None, help="comma-separated directories to scan")
|
||||
ap.add_argument("--db", default=None, help="SQLite DB path")
|
||||
ap.add_argument("--apply", action="store_true", help="apply renames (default: dry-run)")
|
||||
ap.add_argument("--json", action="store_true", help="output JSON")
|
||||
ap.add_argument("--workers", type=int, default=config.SCAN_WORKERS, help="parallel hashing workers")
|
||||
ap.add_argument("--quick", action="store_true", help="skip full-file hashing (serial/title match only)")
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
roots = _roots_from_arg(args.roots)
|
||||
db_path = Path(args.db) if args.db else config.DB_PATH
|
||||
|
||||
conn = init_db(db_path)
|
||||
try:
|
||||
results = []
|
||||
for record in scanner.scan(roots, conn=conn, workers=args.workers, hash_files=not args.quick):
|
||||
ident = pipeline.identify(conn, record)
|
||||
results.append({**record, **ident})
|
||||
if not args.json:
|
||||
if ident["method"] == "unknown":
|
||||
print(f"UNKNOWN {record['name']} serial={record['serial']} error={record['error']}", flush=True)
|
||||
else:
|
||||
print(f"MATCH[{ident['method']:14s}] {record['name']} -> {ident['canonical']}", flush=True)
|
||||
|
||||
totals = {
|
||||
row["system"]: row["c"]
|
||||
for row in conn.execute(
|
||||
"SELECT d.system, COUNT(*) AS c FROM releases r JOIN dats d ON d.id = r.dat_id GROUP BY d.system"
|
||||
)
|
||||
}
|
||||
on_disk = {}
|
||||
for r in results:
|
||||
if r["release"]:
|
||||
on_disk.setdefault(r["release"]["system"], set()).add(r["release"]["id"])
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(
|
||||
{
|
||||
"catalog_totals": totals,
|
||||
"on_disk": {s: len(ids) for s, ids in on_disk.items()},
|
||||
"missing": {s: totals.get(s, 0) - len(ids) for s, ids in on_disk.items()},
|
||||
"discs": [
|
||||
{k: r.get(k) for k in ("name", "size", "serial", "title", "method", "canonical", "error")}
|
||||
for r in results
|
||||
],
|
||||
},
|
||||
indent=2,
|
||||
))
|
||||
return 0
|
||||
|
||||
matched = [r for r in results if r["method"] != "unknown"]
|
||||
unknown = [r for r in results if r["method"] == "unknown"]
|
||||
print(f"\nScanned: {len(results)} Matched: {len(matched)} Unknown: {len(unknown)}", flush=True)
|
||||
for system in sorted(totals):
|
||||
have = len(on_disk.get(system, set()))
|
||||
total = totals.get(system, 0)
|
||||
print(f"Catalog[{system}]: on-disk {have}/{total} missing {total - have}")
|
||||
|
||||
renames = [
|
||||
r for r in results
|
||||
if r["method"] != "unknown" and r["canonical"]
|
||||
and Path(r["path"]).name != r["canonical"]
|
||||
]
|
||||
if renames:
|
||||
print(f"\nRename plan ({len(renames)} file(s)):")
|
||||
for r in renames:
|
||||
print(f" {Path(r['path']).name}\n -> {r['canonical']}")
|
||||
if args.apply:
|
||||
for r in renames:
|
||||
src = Path(r["path"])
|
||||
dst = src.with_name(r["canonical"])
|
||||
if src.exists() and not dst.exists():
|
||||
scanner.rename_and_recache(conn, src, dst)
|
||||
print(f" renamed -> {dst.name}")
|
||||
else:
|
||||
print(" (dry-run; pass --apply to rename)")
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""Scan directories for PSP .iso discs and read their identity + hashes.
|
||||
|
||||
A persistent scan cache (in the library DB) keyed by file size + mtime means
|
||||
re-scanning only re-hashes files that actually changed; unchanged discs are
|
||||
served from cache instantly. Hashing of new/changed files runs in parallel.
|
||||
"""
|
||||
import logging
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
from . import config, hasher, iso_reader
|
||||
|
||||
log = logging.getLogger("psp.scan")
|
||||
|
||||
|
||||
def iter_discs(roots):
|
||||
seen = set()
|
||||
for root in roots:
|
||||
root = Path(root)
|
||||
if not root.exists():
|
||||
continue
|
||||
for path in sorted(root.rglob("*")):
|
||||
if not path.is_file():
|
||||
continue
|
||||
suffix = path.suffix.lower()
|
||||
if suffix in config.IGNORED_EXTENSIONS:
|
||||
continue
|
||||
if suffix not in config.DISC_EXTENSIONS:
|
||||
continue
|
||||
key = str(path.resolve())
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
yield path
|
||||
|
||||
|
||||
def _process_disc(path, hash_files=True, cancel=None):
|
||||
stat = path.stat()
|
||||
try:
|
||||
identity = iso_reader.read_identity(path)
|
||||
serial = identity.get("serial")
|
||||
title = identity.get("title")
|
||||
error = None
|
||||
except Exception as exc: # noqa: BLE001 - keep scanning on bad discs
|
||||
serial = None
|
||||
title = None
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
if hash_files:
|
||||
hashes = hasher.hash_file(path, cancel=cancel)
|
||||
else:
|
||||
hashes = {"crc32": None, "md5": None, "sha1": None}
|
||||
return {
|
||||
"path": str(path),
|
||||
"name": path.name,
|
||||
"size": stat.st_size,
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"serial": serial,
|
||||
"title": title,
|
||||
"error": error,
|
||||
**hashes,
|
||||
}
|
||||
|
||||
|
||||
def _cached_record(conn, path, stat, need_hashes=False):
|
||||
if conn is None:
|
||||
return None
|
||||
row = conn.execute(
|
||||
"SELECT size, mtime_ns, serial, title, error, crc32, md5, sha1 FROM files WHERE path = ?",
|
||||
(str(path),),
|
||||
).fetchone()
|
||||
if row is None or row["size"] != stat.st_size or row["mtime_ns"] != stat.st_mtime_ns:
|
||||
return None
|
||||
if need_hashes and not row["crc32"]:
|
||||
return None
|
||||
return {
|
||||
"path": str(path),
|
||||
"name": path.name,
|
||||
"size": stat.st_size,
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
"serial": row["serial"],
|
||||
"title": row["title"],
|
||||
"error": row["error"],
|
||||
"crc32": row["crc32"],
|
||||
"md5": row["md5"],
|
||||
"sha1": row["sha1"],
|
||||
}
|
||||
|
||||
|
||||
def _upsert(conn, rec):
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT INTO files (path, name, size, mtime_ns, crc32, md5, sha1, serial, title, error, scanned_at)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,datetime('now'))
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
name=excluded.name, size=excluded.size, mtime_ns=excluded.mtime_ns,
|
||||
crc32=excluded.crc32, md5=excluded.md5, sha1=excluded.sha1,
|
||||
serial=excluded.serial, title=excluded.title, error=excluded.error,
|
||||
scanned_at=excluded.scanned_at
|
||||
""",
|
||||
(rec["path"], rec["name"], rec["size"], rec["mtime_ns"], rec["crc32"], rec["md5"], rec["sha1"], rec["serial"], rec["title"], rec["error"]),
|
||||
)
|
||||
|
||||
|
||||
def rename_and_recache(conn, src, dst):
|
||||
"""Rename a disc and carry its cache entry over so it isn't re-hashed."""
|
||||
src.rename(dst)
|
||||
if conn is not None:
|
||||
conn.execute(
|
||||
"UPDATE files SET path = ?, name = ? WHERE path = ?",
|
||||
(str(dst), dst.name, str(src)),
|
||||
)
|
||||
|
||||
|
||||
def scan(roots, conn=None, workers=1, hash_files=True, cancel=None):
|
||||
pending = []
|
||||
cached_count = 0
|
||||
for path in iter_discs(roots):
|
||||
if cancel is not None and cancel.is_set():
|
||||
return
|
||||
stat = path.stat()
|
||||
cached = _cached_record(conn, path, stat, need_hashes=hash_files)
|
||||
if cached is not None:
|
||||
cached_count += 1
|
||||
log.info("cached %s", path.name)
|
||||
yield cached
|
||||
continue
|
||||
pending.append(path)
|
||||
|
||||
if cached_count:
|
||||
log.info("%d disc(s) served from cache", cached_count)
|
||||
if not pending:
|
||||
return
|
||||
|
||||
log.info("processing %d disc(s) with %d worker(s)...", len(pending), workers)
|
||||
|
||||
try:
|
||||
if workers and workers > 1:
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futures = {pool.submit(_process_disc, p, hash_files, cancel): p for p in pending}
|
||||
for future in as_completed(futures):
|
||||
if cancel is not None and cancel.is_set():
|
||||
return
|
||||
rec = future.result()
|
||||
log.info("scanned %s", rec["name"])
|
||||
if conn is not None:
|
||||
_upsert(conn, rec)
|
||||
yield rec
|
||||
else:
|
||||
for path in pending:
|
||||
if cancel is not None and cancel.is_set():
|
||||
return
|
||||
rec = _process_disc(path, hash_files, cancel=cancel)
|
||||
log.info("scanned %s", rec["name"])
|
||||
if conn is not None:
|
||||
_upsert(conn, rec)
|
||||
yield rec
|
||||
except hasher.HashCancelled:
|
||||
return
|
||||
@@ -0,0 +1,81 @@
|
||||
"""API credentials for the optional cover sources (IGDB, ScreenScraper, TheMovieDB).
|
||||
|
||||
Credentials live in a small JSON file next to the database so both the GUI and
|
||||
the HTTP API (running on the NAS) can share them. Environment variables take
|
||||
precedence over the file.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from . import config
|
||||
|
||||
CREDENTIALS_PATH = Path(
|
||||
os.environ.get("PSP_CREDENTIALS", str(config.DATA_DIR / "credentials.json"))
|
||||
)
|
||||
|
||||
FIELDS = [
|
||||
"igdb_client_id",
|
||||
"igdb_client_secret",
|
||||
"tmdb_api_key",
|
||||
"screenscraper_devid",
|
||||
"screenscraper_devpassword",
|
||||
"screenscraper_ssid",
|
||||
"screenscraper_sspassword",
|
||||
"screenscraper_softname",
|
||||
]
|
||||
|
||||
_ENV_MAP = {
|
||||
"igdb_client_id": "IGDB_CLIENT_ID",
|
||||
"igdb_client_secret": "IGDB_CLIENT_SECRET",
|
||||
"tmdb_api_key": "TMDB_API_KEY",
|
||||
"screenscraper_devid": "SCREENSCRAPER_DEVID",
|
||||
"screenscraper_devpassword": "SCREENSCRAPER_DEVPASSWORD",
|
||||
"screenscraper_ssid": "SCREENSCRAPER_SSID",
|
||||
"screenscraper_sspassword": "SCREENSCRAPER_SSPASSWORD",
|
||||
"screenscraper_softname": "SCREENSCRAPER_SOFTNAME",
|
||||
}
|
||||
|
||||
_cache = None
|
||||
|
||||
|
||||
def load():
|
||||
"""Return the credentials dict (cached; env vars win over the file)."""
|
||||
global _cache
|
||||
if _cache is None:
|
||||
creds = {}
|
||||
if CREDENTIALS_PATH.exists():
|
||||
try:
|
||||
creds = json.loads(CREDENTIALS_PATH.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
creds = {}
|
||||
if not isinstance(creds, dict):
|
||||
creds = {}
|
||||
for key, env in _ENV_MAP.items():
|
||||
value = os.environ.get(env)
|
||||
if value:
|
||||
creds[key] = value
|
||||
_cache = creds
|
||||
return _cache
|
||||
|
||||
|
||||
def save(values):
|
||||
"""Merge `values` into the credential file and refresh the cache."""
|
||||
global _cache
|
||||
creds = dict(load())
|
||||
for key in FIELDS:
|
||||
if key in values:
|
||||
value = (values[key] or "").strip()
|
||||
if value:
|
||||
creds[key] = value
|
||||
else:
|
||||
creds.pop(key, None)
|
||||
try:
|
||||
CREDENTIALS_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CREDENTIALS_PATH.with_name(CREDENTIALS_PATH.name + ".tmp")
|
||||
tmp.write_text(json.dumps(creds, indent=2), encoding="utf-8")
|
||||
os.replace(tmp, CREDENTIALS_PATH)
|
||||
except OSError:
|
||||
pass
|
||||
_cache = creds
|
||||
return creds
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
"""Redump 7z store: scan archives and extract the disc image inside.
|
||||
|
||||
Store archives are ``Name (Region) (tags).7z`` with the ``.iso`` (or ``.cso``)
|
||||
inside. Extraction shells out to the system ``7z`` binary.
|
||||
"""
|
||||
import logging
|
||||
import re
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger("store")
|
||||
|
||||
_ARCHIVE_EXTS = {".7z", ".zip"}
|
||||
_IMAGE_EXTS = {".iso", ".cso", ".img", ".bin"}
|
||||
|
||||
|
||||
class TransferCancelled(Exception):
|
||||
"""Raised inside a transfer when the user cancels it."""
|
||||
|
||||
|
||||
def is_archive(path):
|
||||
return Path(path).suffix.lower() in _ARCHIVE_EXTS
|
||||
|
||||
|
||||
def display_name(name):
|
||||
"""Display title of a store archive (Redump names are kept as-is)."""
|
||||
return str(name).strip()
|
||||
|
||||
|
||||
def region(name):
|
||||
"""Extract the region tag (first trailing parenthetical) from a store name."""
|
||||
s = str(name)
|
||||
tags = []
|
||||
while True:
|
||||
m = re.search(r"\s*\(([^()]*)\)\s*$", s)
|
||||
if not m:
|
||||
break
|
||||
tags.insert(0, m.group(1).strip())
|
||||
s = s[: m.start()]
|
||||
return tags[0] if tags else ""
|
||||
|
||||
|
||||
def scan(root, limit=None):
|
||||
"""Return ``[{path, name, size, mtime_ns}]`` for archives in the store."""
|
||||
root = Path(root)
|
||||
if not root.is_dir():
|
||||
return []
|
||||
items = []
|
||||
for path in sorted(root.iterdir()):
|
||||
if not path.is_file() or not is_archive(path):
|
||||
continue
|
||||
try:
|
||||
stat = path.stat()
|
||||
except OSError:
|
||||
continue
|
||||
items.append({
|
||||
"path": str(path),
|
||||
"name": path.stem,
|
||||
"size": stat.st_size,
|
||||
"mtime_ns": stat.st_mtime_ns,
|
||||
})
|
||||
if limit is not None and len(items) >= limit:
|
||||
break
|
||||
return items
|
||||
|
||||
|
||||
def peek_archive(path, cancel_event=None):
|
||||
"""Return the disc-image filename inside a 7z archive, or None."""
|
||||
path = Path(path)
|
||||
if not path.is_file():
|
||||
return None
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
["7z", "l", "-slt", str(path)],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1,
|
||||
)
|
||||
except OSError:
|
||||
return None
|
||||
name = None
|
||||
for line in proc.stdout:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
raise TransferCancelled()
|
||||
if line.startswith("Path = "):
|
||||
candidate = line[7:].strip()
|
||||
if candidate.lower().endswith(tuple(_IMAGE_EXTS)):
|
||||
name = candidate
|
||||
code = proc.wait()
|
||||
if code != 0:
|
||||
return None
|
||||
return name
|
||||
|
||||
|
||||
def extract_iso(archive, iso_name, out_dir, progress_cb=None, cancel_event=None):
|
||||
"""Extract one disc image from a 7z archive; return its path."""
|
||||
archive = Path(archive)
|
||||
out_dir = Path(out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
cmd = ["7z", "x", "-y", f"-o{out_dir}", str(archive), iso_name]
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1,
|
||||
)
|
||||
except OSError as exc:
|
||||
raise RuntimeError(f"7z is not available: {exc}") from exc
|
||||
|
||||
last_pct = 0.0
|
||||
for line in proc.stdout:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
raise TransferCancelled()
|
||||
m = re.search(r"(\d{1,3})\s*%", line)
|
||||
if m:
|
||||
last_pct = int(m.group(1)) / 100.0
|
||||
if progress_cb is not None:
|
||||
progress_cb("Extracting", last_pct, line.strip()[:80])
|
||||
code = proc.wait()
|
||||
if code != 0:
|
||||
raise RuntimeError(f"7z extraction failed (exit {code})")
|
||||
result = out_dir / iso_name
|
||||
if not result.exists():
|
||||
raise RuntimeError("7z extraction completed but the image was not found")
|
||||
return result
|
||||
@@ -0,0 +1,408 @@
|
||||
"""Pluggable transfer sources and byte-copy primitives.
|
||||
|
||||
``Source`` is the seam for different download protocols: a local share of 7z
|
||||
archives, a remote HTTPS store server, etc.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
from .store import TransferCancelled, extract_iso, peek_archive, scan
|
||||
|
||||
log = logging.getLogger("transfers")
|
||||
|
||||
_CHUNK = 1 << 22 # 4 MiB read chunks
|
||||
|
||||
|
||||
def copy_stream(src, dst, progress_cb=None, cancel_event=None):
|
||||
"""Stream-copy ``src`` -> ``dst``, reporting fraction done. Returns bytes."""
|
||||
src, dst = Path(src), Path(dst)
|
||||
total = src.stat().st_size
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
done = 0
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
raise TransferCancelled()
|
||||
block = fsrc.read(_CHUNK)
|
||||
if not block:
|
||||
break
|
||||
fdst.write(block)
|
||||
done += len(block)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb("Copying", min(done / total, 1.0), "")
|
||||
return done
|
||||
|
||||
|
||||
def move_with_progress(src, dst, progress_cb=None, cancel_event=None):
|
||||
"""Move a file: rename if possible, else stream-copy then unlink."""
|
||||
src, dst = Path(src), Path(dst)
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
if dst.exists():
|
||||
dst.unlink()
|
||||
try:
|
||||
os.replace(src, dst)
|
||||
if progress_cb is not None:
|
||||
progress_cb("Moving", 1.0, "")
|
||||
return dst
|
||||
except OSError:
|
||||
pass
|
||||
total = src.stat().st_size
|
||||
done = 0
|
||||
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||
while True:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
fdst.close()
|
||||
try:
|
||||
dst.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise TransferCancelled()
|
||||
block = fsrc.read(_CHUNK)
|
||||
if not block:
|
||||
break
|
||||
fdst.write(block)
|
||||
done += len(block)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb("Moving", min(done / total, 1.0), "")
|
||||
try:
|
||||
src.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
return dst
|
||||
|
||||
|
||||
def download_url(url, dest, headers=None, progress_cb=None, cancel_event=None):
|
||||
"""Stream-download ``url`` to ``dest`` with progress + cancel. Returns dest."""
|
||||
dest = Path(dest)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
req = urllib.request.Request(url, headers=headers or {})
|
||||
resp = None
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=30)
|
||||
total = int(resp.headers.get("Content-Length") or 0)
|
||||
done = 0
|
||||
with open(dest, "wb") as out:
|
||||
while True:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
raise TransferCancelled()
|
||||
chunk = resp.read(_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
out.write(chunk)
|
||||
done += len(chunk)
|
||||
if progress_cb is not None and total:
|
||||
progress_cb("Downloading", min(done / total, 1.0), "")
|
||||
except Exception:
|
||||
try:
|
||||
dest.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
finally:
|
||||
if resp is not None:
|
||||
resp.close()
|
||||
return dest
|
||||
|
||||
|
||||
def remove_path(path):
|
||||
"""Delete a file or directory, using the desktop trash when available."""
|
||||
path = Path(path)
|
||||
if not path.exists() and not path.is_symlink():
|
||||
return True # already gone
|
||||
try:
|
||||
import send2trash # optional dependency; best-effort trash
|
||||
send2trash.send2trash(str(path))
|
||||
return True
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
try:
|
||||
if path.is_dir() and not path.is_symlink():
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
path.unlink()
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def copy_tree(src, dst, progress_cb=None, cancel_event=None):
|
||||
"""Replace ``dst`` with a copy of directory ``src`` (atomic; safe on cancel)."""
|
||||
src, dst = Path(src), Path(dst)
|
||||
if not src.is_dir():
|
||||
raise RuntimeError(f"not a directory: {src}")
|
||||
files = [p for p in sorted(src.rglob("*")) if p.is_file()]
|
||||
total = sum(p.stat().st_size for p in files) or 1
|
||||
done = 0
|
||||
tmp = dst.with_name(dst.name + ".tmp-sync")
|
||||
if tmp.exists():
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
tmp.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
for p in files:
|
||||
if cancel_event is not None and cancel_event.is_set():
|
||||
raise TransferCancelled()
|
||||
rel = p.relative_to(src)
|
||||
target = tmp / rel
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(p, target)
|
||||
done += p.stat().st_size
|
||||
if progress_cb is not None:
|
||||
progress_cb("Syncing", min(done / total, 1.0), str(rel))
|
||||
if dst.exists():
|
||||
shutil.rmtree(dst, ignore_errors=True)
|
||||
os.replace(tmp, dst)
|
||||
except Exception:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
raise
|
||||
return dst
|
||||
|
||||
|
||||
class Source:
|
||||
"""A pluggable source of disc images (local share, HTTP, SFTP, ...)."""
|
||||
label = "source"
|
||||
|
||||
def scan(self, root=None):
|
||||
raise NotImplementedError
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class FileCopySource(Source):
|
||||
"""Copy a single local file (library -> PSP or PSP -> library).
|
||||
|
||||
The queue passes an ``item`` dict that may carry a ``dest_name`` override;
|
||||
otherwise the source's own filename is used.
|
||||
"""
|
||||
|
||||
label = "Local file"
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = Path(path)
|
||||
|
||||
def scan(self, root=None):
|
||||
return []
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
dest_dir = Path(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
dest_name = (item or {}).get("dest_name") or self.path.name
|
||||
dest = dest_dir / dest_name
|
||||
if dest.exists():
|
||||
raise FileExistsError(str(dest))
|
||||
copy_stream(self.path, dest, progress_cb, cancel_event)
|
||||
return dest
|
||||
|
||||
|
||||
class TreeCopySource(Source):
|
||||
"""Copy a whole directory (save-data folder) into a destination directory."""
|
||||
|
||||
label = "Save folder"
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = Path(path)
|
||||
|
||||
def scan(self, root=None):
|
||||
return []
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
name = (item or {}).get("dest_name") or self.path.name
|
||||
dest = Path(dest_dir) / name
|
||||
return copy_tree(self.path, dest, progress_cb, cancel_event)
|
||||
|
||||
|
||||
class RemoteStoreSource(Source):
|
||||
"""Browse and download 7z archives from a remote HTTPS store server."""
|
||||
|
||||
label = "Remote store"
|
||||
|
||||
def __init__(self, base_url, token=""):
|
||||
self.base_url = (base_url or "").rstrip("/")
|
||||
self.token = token or ""
|
||||
|
||||
def _headers(self):
|
||||
headers = {"User-Agent": "psp-preserve/0.1"}
|
||||
if self.token:
|
||||
headers["Authorization"] = f"Bearer {self.token}"
|
||||
return headers
|
||||
|
||||
def scan(self, root=None):
|
||||
if not self.base_url:
|
||||
return []
|
||||
req = urllib.request.Request(f"{self.base_url}/api/store", headers=self._headers())
|
||||
with urllib.request.urlopen(req, timeout=30) as resp:
|
||||
data = json.loads(resp.read())
|
||||
return data.get("items", []) if isinstance(data, dict) else []
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
dest_dir = Path(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
# The listing's "name" has no extension; "filename" carries the real
|
||||
# archive filename (e.g. "Game (USA).7z") used for the download URL.
|
||||
name = item.get("filename") or item["name"]
|
||||
tmp = Path(tempfile.mkdtemp(prefix="psp-store-"))
|
||||
archive = tmp / name
|
||||
try:
|
||||
url = f"{self.base_url}/api/store/download/{urllib.parse.quote(name, safe='')}"
|
||||
download_url(url, archive, self._headers(), progress_cb, cancel_event)
|
||||
iso_name = peek_archive(archive, cancel_event)
|
||||
if not iso_name:
|
||||
raise RuntimeError("No disc image found inside the archive")
|
||||
dest = dest_dir / iso_name
|
||||
if dest.exists():
|
||||
raise FileExistsError(str(dest))
|
||||
iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event)
|
||||
return move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
def _archive_item_identifier(value):
|
||||
"""Normalize an archive.org input to a bare item identifier."""
|
||||
v = (value or "").strip().rstrip("/")
|
||||
if not v:
|
||||
return ""
|
||||
if "://" in v:
|
||||
v = v.split("://", 1)[1]
|
||||
for prefix in ("archive.org/download/", "archive.org/details/", "archive.org/metadata/"):
|
||||
if prefix in v:
|
||||
return v.split(prefix, 1)[1].split("/", 1)[0]
|
||||
return v
|
||||
|
||||
|
||||
class ArchiveOrgSource(Source):
|
||||
"""Browse a public archive.org item holding the same Redump 7z set."""
|
||||
|
||||
label = "Archive.org"
|
||||
|
||||
def __init__(self, item, sig="", user=""):
|
||||
self.item = _archive_item_identifier(item)
|
||||
self.sig = (sig or "").strip()
|
||||
self.user = (user or "").strip()
|
||||
|
||||
def _base(self):
|
||||
return f"https://archive.org/download/{self.item}"
|
||||
|
||||
def _headers(self):
|
||||
headers = {"User-Agent": "psp-preserve/0.1"}
|
||||
cookie = "; ".join(
|
||||
p for p in (
|
||||
f"logged-in-sig={self.sig}" if self.sig else "",
|
||||
f"logged-in-user={self.user}" if self.user else "",
|
||||
) if p
|
||||
)
|
||||
if cookie:
|
||||
headers["Cookie"] = cookie
|
||||
return headers
|
||||
|
||||
def scan(self, root=None):
|
||||
if not self.item:
|
||||
return []
|
||||
url = f"https://archive.org/metadata/{self.item}"
|
||||
req = urllib.request.Request(url, headers=self._headers())
|
||||
with urllib.request.urlopen(req, timeout=60) as resp:
|
||||
data = json.loads(resp.read())
|
||||
items = []
|
||||
archive_count = 0
|
||||
private_count = 0
|
||||
has_auth = bool(self.sig or self.user)
|
||||
for f in (data.get("files") or []) if isinstance(data, dict) else []:
|
||||
if not isinstance(f, dict):
|
||||
continue
|
||||
name = f.get("name") or ""
|
||||
if not name.lower().endswith((".7z", ".zip")):
|
||||
continue
|
||||
archive_count += 1
|
||||
if str(f.get("private")).lower() == "true":
|
||||
private_count += 1
|
||||
if not has_auth:
|
||||
continue
|
||||
try:
|
||||
size = int(f.get("size") or 0)
|
||||
except (TypeError, ValueError):
|
||||
size = 0
|
||||
try:
|
||||
mtime_ns = int(float(f.get("mtime") or 0)) * 1_000_000_000
|
||||
except (TypeError, ValueError):
|
||||
mtime_ns = 0
|
||||
items.append({
|
||||
"name": Path(name).stem,
|
||||
"filename": name,
|
||||
"size": size,
|
||||
"mtime_ns": mtime_ns,
|
||||
})
|
||||
if private_count:
|
||||
log.warning("archive.org: %d of %d archive(s) are private%s",
|
||||
private_count, archive_count, "" if has_auth else " (login required)")
|
||||
if not items and archive_count and private_count == archive_count and not has_auth:
|
||||
raise RuntimeError(
|
||||
f"archive.org item '{self.item}': all {archive_count} archive(s) are private "
|
||||
"(login required) — try a different item."
|
||||
)
|
||||
return items
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
dest_dir = Path(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
name = item.get("filename") or item["name"]
|
||||
tmp = Path(tempfile.mkdtemp(prefix="psp-archive-"))
|
||||
archive = tmp / Path(name).name
|
||||
try:
|
||||
url = f"{self._base()}/{urllib.parse.quote(name, safe='/')}"
|
||||
try:
|
||||
download_url(url, archive, self._headers(), progress_cb, cancel_event)
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (401, 403):
|
||||
raise RuntimeError(
|
||||
f"Archive.org returned {exc.code} for '{name}' — the file is likely "
|
||||
"private (login required) and can't be downloaded anonymously."
|
||||
) from exc
|
||||
raise
|
||||
iso_name = peek_archive(archive, cancel_event)
|
||||
if not iso_name:
|
||||
raise RuntimeError("No disc image found inside the archive")
|
||||
dest = dest_dir / iso_name
|
||||
if dest.exists():
|
||||
raise FileExistsError(str(dest))
|
||||
iso_path = extract_iso(archive, iso_name, tmp, progress_cb, cancel_event)
|
||||
return move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
class LocalShareSource(Source):
|
||||
"""A local (or mounted) folder of 7z archives."""
|
||||
|
||||
label = "Local share"
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = Path(root)
|
||||
|
||||
def scan(self, root=None):
|
||||
return scan(root or self.root)
|
||||
|
||||
def download(self, item, dest_dir, progress_cb=None, cancel_event=None):
|
||||
archive = Path(item["path"])
|
||||
dest_dir = Path(dest_dir)
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
iso_name = peek_archive(archive, cancel_event)
|
||||
if not iso_name:
|
||||
raise RuntimeError("No disc image found inside the archive")
|
||||
dest = dest_dir / iso_name
|
||||
if dest.exists():
|
||||
raise FileExistsError(str(dest))
|
||||
tmp = Path(tempfile.mkdtemp(prefix="psp-store-"))
|
||||
try:
|
||||
local = tmp / archive.name
|
||||
copy_stream(archive, local, progress_cb, cancel_event)
|
||||
iso_path = extract_iso(local, iso_name, tmp, progress_cb, cancel_event)
|
||||
move_with_progress(iso_path, dest, progress_cb, cancel_event)
|
||||
return dest
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
Reference in New Issue
Block a user