From da3429d918c5cf3efa0b4f1cdd8586d74abde80c Mon Sep 17 00:00:00 2001 From: Mathieu Moran Date: Mon, 7 Sep 2026 11:55:26 -0400 Subject: [PATCH] Add pure-Python PS1 to EBOOT.PBP converter (phase 1) --- .gitignore | 4 + app/ps1_convert.py | 765 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 769 insertions(+) create mode 100644 app/ps1_convert.py diff --git a/.gitignore b/.gitignore index 992a4b7..e53645e 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ data/*.sqlite data/*.sqlite-* data/credentials.json session-transcript-2026-09-01.md + +# Copyrighted PSN EBOOTs (base.pbp) used only for local PS1 conversion testing +data/*.PBP +data/*.pbp diff --git a/app/ps1_convert.py b/app/ps1_convert.py new file mode 100644 index 0000000..0c4fff6 --- /dev/null +++ b/app/ps1_convert.py @@ -0,0 +1,765 @@ +"""PS1 -> PSP EBOOT.PBP conversion (pure Python). + +Converts a PlayStation 1 disc image (``.iso``, ``.img``, or a ``.cue`` + +``.bin`` set) into the ``EBOOT.PBP`` "PS1 classic" container that PSP POPS +firmware can launch. + +This is an independent, MIT-clean reimplementation of the popstation-md +algorithm (same format that PSXPackager/RupertAvery writes), implemented from +the published container format rather than by copying any GPL source. + +Format reference (offsets are byte-exact): + * PBP header - 40 bytes: b"\\x00PBP" magic + version + 8 uint32 section offsets + (SFO, ICON0, ICON1, PIC0, PIC1, SND0, DATA.PSP, DATA.PSAR). + * PSAR - the DATA.PSAR section holding the disc image. Starts with + b"PSISOIMG0000", game id at +0x400, TOC at +0x800, + ISO block index at +0x4000, disc data at +0x100000. + +A ``base.pbp`` (any PSN PS1 classic ``EBOOT.PBP``) is REQUIRED to supply two +blobs that we cannot synthesise: + + * DATA.PSP - the encrypted POPS loader (header section 8). + * STARTDAT - the trailing encrypted block (boot.png + PGD). + +Without a base.pbp the resulting EBOOT will not boot on real hardware; the +converter refuses to run without one. + +Compression: only level 0 (uncompressed) is implemented. LZRC (levels 1-9) is a +planned fast-follow. +""" + +from __future__ import annotations + +import argparse +import io +import os +import re +import struct +import zlib +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterator, List, Optional, Tuple + +# -------------------------------------------------------------------------- +# Constants +# -------------------------------------------------------------------------- + +PBP_MAGIC = b"\x00PBP" +PBP_HEADER_SIZE = 0x28 +PBP_HEADER_U32 = 10 # magic + version + 8 section offsets + +# Section index -> header offset (in bytes) within the 40-byte header. +SECTION_OFFSET = { + "sfo": 0x08, + "icon0": 0x0C, + "icon1": 0x10, + "pic0": 0x14, + "pic1": 0x18, + "snd0": 0x1C, + "data_psp": 0x20, + "psar": 0x24, +} +SECTION_ORDER = ["sfo", "icon0", "icon1", "pic0", "pic1", "snd0", "data_psp", "psar"] + +# PSAR (DATA.PSAR) layout. +PSAR_MAGIC = b"PSISOIMG0000" +PSAR_GAMEID_OFFSET = 0x400 +PSAR_TOC_OFFSET = 0x800 +PSAR_INDEX_OFFSET = 0x4000 +PSAR_ISO_OFFSET = 0x100000 +PSAR_ALIGNMENT = 0x10000 + +# One disc "block" in the ISO index = 16 raw 2352-byte sectors. +RAW_SECTOR = 0x930 # 2352 +COOKED_SECTOR = 0x800 # 2048 +BLOCK_SIZE = RAW_SECTOR * 16 # 0x9300 +INDEX_ENTRY_SIZE = 32 # offset u32 + length u32 + 6 unused u32 + +SYNC_PATTERN = b"\x00\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00" + +# SFO field formats (PSP PARAM.SFO). +FMT_U32 = 0x0004 +FMT_STRING = 0x0204 + +SFO_DEFAULTS = { + "BOOTABLE": (FMT_U32, 1), + "CATEGORY": (FMT_STRING, "ME"), + "DISC_VERSION": (FMT_STRING, "1.00"), + "LICENSE": (FMT_STRING, "Library programs Copyright(C) Sony ComputerEntertainment Inc."), + "PARENTAL_LEVEL": (FMT_U32, 1), + "PSP_SYSTEM_VER": (FMT_STRING, "3.71"), + "REGION": (FMT_U32, 0x8000), +} + + +class ConversionError(Exception): + """Raised when a disc cannot be converted.""" + + +class Cancelled(Exception): + """Raised internally when the caller requests cancellation.""" + + +# -------------------------------------------------------------------------- +# CUE sheet parsing +# -------------------------------------------------------------------------- + +@dataclass +class CueTrack: + number: int + mode: str # "MODE1/2048", "MODE1/2352", "MODE2/2352", "AUDIO" + file: str + file_type: str # "BINARY" | "WAVE" | ... + indexes: List[Tuple[int, int]] = field(default_factory=list) # (index, msf) + sector_size: int = 0 + + @property + def pregap_sectors(self) -> int: + """Sectors between INDEX 00 and INDEX 01 (0 if no pregap).""" + idx = dict(self.indexes) + if 0 in idx and 1 in idx: + return max(0, idx[1] - idx[0]) + return 0 + + +@dataclass +class CueDisc: + tracks: List[CueTrack] + + @property + def data_track(self) -> Optional[CueTrack]: + for t in self.tracks: + if t.mode != "AUDIO": + return t + return None + + +def _msf_to_sectors(msf: str) -> int: + m, s, f = (int(x) for x in msf.split(":")) + return (m * 60 + s) * 75 + f + + +def _sector_size_for_mode(mode: str) -> int: + if mode in ("MODE1/2048",): + return COOKED_SECTOR + if mode in ("MODE1/2352", "MODE2/2352", "AUDIO"): + return RAW_SECTOR + # CDG / MODE2/2336 etc. fall back to raw. + return RAW_SECTOR + + +def parse_cue(text: str) -> CueDisc: + """Parse a CUE sheet into a :class:`CueDisc`. + + Handles the common PS1 layouts: one or more ``FILE`` lines, ``TRACK`` + lines (``MODE1/2048``, ``MODE1/2352``, ``MODE2/2352``, ``AUDIO``) and + ``INDEX`` lines (``INDEX 00`` pregap + ``INDEX 01`` start). + """ + tracks: List[CueTrack] = [] + current_file: Optional[str] = None + current_type = "BINARY" + current_track: Optional[CueTrack] = None + + file_re = re.compile(r'^FILE\s+"([^"]+)"\s+(\w+)', re.I) + track_re = re.compile(r'^\s*TRACK\s+(\d{1,2})\s+(\S+)', re.I) + index_re = re.compile(r'^\s*INDEX\s+(\d{1,2})\s+(\d{1,2}:\d{2}:\d{2})', re.I) + + for raw in text.splitlines(): + line = raw.strip() + if not line or line.startswith(("REM", "CATALOG", "CDTEXTFILE", "PERFORMER", "TITLE", "ISRC", "FLAGS", "POSTGAP", "PREGAP")): + if line.startswith("PREGAP"): + # PREGAP is equivalent to an INDEX 00 offset. + m = re.search(r'PREGAP\s+(\d{1,2}:\d{2}:\d{2})', line, re.I) + if m and current_track is not None: + current_track.indexes.append((0, _msf_to_sectors(m.group(1)))) + continue + + m = file_re.match(line) + if m: + current_file = m.group(1) + current_type = m.group(2).upper() + current_track = None + continue + + m = track_re.match(line) + if m: + number = int(m.group(1)) + mode = m.group(2).upper() + current_track = CueTrack( + number=number, + mode=mode, + file=current_file or "", + file_type=current_type, + sector_size=_sector_size_for_mode(mode), + ) + tracks.append(current_track) + continue + + m = index_re.match(line) + if m and current_track is not None: + current_track.indexes.append((int(m.group(1)), _msf_to_sectors(m.group(2)))) + + if not tracks: + raise ConversionError("CUE sheet contains no tracks") + for t in tracks: + if not t.file: + raise ConversionError(f"track {t.number} has no FILE") + return CueDisc(tracks) + + +# -------------------------------------------------------------------------- +# Sector normalisation (cooked 2048 -> raw 2352 Mode 2 Form 1) +# -------------------------------------------------------------------------- + +def _to_bcd(value: int) -> int: + return ((value // 10) << 4) | (value % 10) + + +def _expand_cooked_sector(data: bytes, lba: int) -> bytes: + """Expand a 2048-byte cooked sector to a 2352-byte raw Mode2 Form1 sector. + + Rebuilds the 12-byte sync pattern, the 4-byte MSF header (BCD), the + 8-byte subheader, and appends 280 bytes of EDC/ECC (zero-filled; the PSP + POPS loader does not verify EDC/ECC for converted games). + """ + assert len(data) == COOKED_SECTOR + frame = lba + 150 # 150-sector lead-in offset + msf = bytes([ + _to_bcd((frame // 75 // 60) % 100), + _to_bcd((frame // 75) % 60), + _to_bcd(frame % 75), + 0x02, # mode 2 + ]) + # Subheader: file=2, channel=0, submode=0, coding=0; repeated once. + subheader = b"\x00\x00\x08\x00" * 2 + return SYNC_PATTERN + msf + subheader + data + (b"\x00" * 280) + + +def iter_raw_sectors( + path: Path, + sector_size: int, + pregap_sectors: int = 0, + progress_cb=None, + cancel_event=None, +) -> Iterator[bytes]: + """Yield 2352-byte raw sectors for one track file. + + ``sector_size`` is 2048 (cooked; expanded on the fly) or 2352 (raw). + ``pregap_sectors`` leading zero sectors are emitted first. + """ + def _cancelled(): + return cancel_event is not None and cancel_event.is_set() + + if pregap_sectors: + for _ in range(pregap_sectors): + yield b"\x00" * RAW_SECTOR + + size = path.stat().st_size + with open(path, "rb") as fh: + lba = 0 + done = 0 + while True: + if _cancelled(): + raise Cancelled() + chunk = fh.read(sector_size) + if not chunk: + break + if len(chunk) != sector_size: + chunk = chunk.ljust(sector_size, b"\x00") + if sector_size == COOKED_SECTOR: + yield _expand_cooked_sector(chunk, lba) + else: + yield chunk + lba += 1 + done += sector_size + if progress_cb: + progress_cb(done, size) + if progress_cb: + progress_cb(size, size) + + +# -------------------------------------------------------------------------- +# PARAM.SFO writer +# -------------------------------------------------------------------------- + +def build_param_sfo(entries: List[Tuple[str, int, object]]) -> bytes: + """Build a PSP PARAM.SFO from ``(key, fmt, value)`` entries. + + ``fmt`` is ``FMT_U32`` (uint32) or ``FMT_STRING`` (utf-8 string). + Layout matches the PSP format: header, index table, key table, data table. + """ + header_size = 20 + index_size = len(entries) * 16 + key_table_start = header_size + index_size + + key_table = b"".join(key.encode("utf-8") + b"\x00" for key, _, _ in entries) + key_pad = (4 - (len(key_table) % 4)) % 4 + data_table_start = key_table_start + len(key_table) + key_pad + + data_table = b"" + index_table = b"" + key_offset = 0 + data_offset = 0 + for key, fmt, value in entries: + if fmt == FMT_U32: + raw = struct.pack(" bytes: + """A complete PARAM.SFO for a converted PS1 game.""" + entries = [] + for key, (fmt, value) in SFO_DEFAULTS.items(): + entries.append((key, fmt, value)) + entries.append(("DISC_ID", FMT_STRING, game_id)) + entries.append(("TITLE", FMT_STRING, title)) + return build_param_sfo(entries) + + +# -------------------------------------------------------------------------- +# Minimal PNG generator (no external image deps) +# -------------------------------------------------------------------------- + +def _png(width: int, height: int, rgb: Tuple[int, int, int]) -> bytes: + def chunk(typ: bytes, data: bytes) -> bytes: + payload = typ + data + return struct.pack(">I", len(data)) + payload + struct.pack(">I", zlib.crc32(payload) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) # 8-bit RGB + raw = b"".join(b"\x00" + bytes(rgb) * width for _ in range(height)) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", ihdr) + + chunk(b"IDAT", zlib.compress(raw, 9)) + + chunk(b"IEND", b"") + ) + + +def default_icon0_png() -> bytes: + return _png(144, 80, (24, 24, 24)) + + +def default_pic1_png() -> bytes: + return _png(480, 272, (10, 10, 16)) + + +# -------------------------------------------------------------------------- +# PBP reading (used both for base.pbp extraction and self-test validation) +# -------------------------------------------------------------------------- + +def read_pbp_header(path: Path) -> List[int]: + with open(path, "rb") as fh: + data = fh.read(PBP_HEADER_SIZE) + if len(data) < PBP_HEADER_SIZE or data[:4] != PBP_MAGIC: + raise ConversionError(f"{path} is not a PBP file (bad magic)") + return list(struct.unpack("<10I", data)) + + +def read_pbp_section(path: Path, header: List[int], index: int) -> bytes: + start = header[index] + end = header[index + 1] if index + 1 < len(header) else None + with open(path, "rb") as fh: + fh.seek(start) + data = fh.read() if end is None else fh.read(end - start) + return data + + +def extract_data_psp(base_pbp: Path) -> bytes: + """Extract the encrypted POPS loader (DATA.PSP section) from a base PBP.""" + header = read_pbp_header(base_pbp) + idx = SECTION_OFFSET["data_psp"] // 4 # header array index (2 + section index) + data = read_pbp_section(base_pbp, header, idx) + if not data: + raise ConversionError(f"{base_pbp} has no DATA.PSP section") + return data + + +def extract_startdat(base_pbp: Path) -> bytes: + """Extract the trailing STARTDAT block (boot.png + PGD) from a base PBP. + + The STARTDAT offset is discovered the same way the reference writer does: + read the u32 at ``psar + 0x0C``, add ``0x50000``, and expect b"STARTDAT" + there. Everything from that marker to EOF is copied verbatim. + """ + header = read_pbp_header(base_pbp) + psar_offset = header[SECTION_OFFSET["psar"] // 4] # header array index + with open(base_pbp, "rb") as fh: + fh.seek(psar_offset + 0x0C) + (x,) = struct.unpack(" str: + """Format a PS1 serial for the PSAR header field. + + The PBP's PARAM.SFO uses the plain 9-char id (``SLPS01762``), while the + PSAR header stores the region-delineated form (``_SLPS_01762``). + """ + m = re.match(r"^([A-Za-z]{4})[_-]?(\d{5})$", game_id) + if m: + return f"_{m.group(1).upper()}_{m.group(2)}" + return game_id + + +def _pad_to(out, target: int) -> None: + """Write zero bytes until the stream reaches ``target`` (absolute offset).""" + delta = target - out.tell() + if delta < 0: + raise ConversionError(f"internal error: overshot offset by {-delta} bytes") + if delta: + out.write(b"\x00" * delta) + + +def _build_gameid_block(game_id: str) -> bytes: + """Best-effort PSAR 0x400 block (game id + reserved header fields). + + TODO(validate): the reference writer overlays the game id onto a fixed + 3616-byte template that also carries the PS1 TOC at +0x800. We generate a + minimal zero-filled block with the game id in place; boot on real hardware + must be verified against PSXPackager output. + """ + game_id = _format_psar_game_id(game_id).encode("ascii")[:11] + block = bytearray(PSAR_TOC_OFFSET - PSAR_GAMEID_OFFSET) # 0x400 -> 0x800 + block[: len(game_id)] = game_id + return bytes(block) + + +def _build_toc_block(title: str, total_sectors: int) -> bytes: + """Best-effort PSAR TOC block at +0x800. + + TODO(validate): the reference writer uses a fixed 11740-byte template with + the title overlaid at offset 8. We emit the title + zero padding; the TOC + format must be verified against PSXPackager output. + """ + block = bytearray(PSAR_INDEX_OFFSET - PSAR_TOC_OFFSET) # 0x800 -> 0x4000 + title = title.encode("ascii", "replace")[:0x80 - 8] + block[8: 8 + len(title)] = title + # First two u32 of the TOC are zero in the reference template. + struct.pack_into(" None: + """Write the DATA.PSAR section (uncompressed, single-disc). + + All offsets are relative to the start of the PSAR section. + """ + base = out.tell() + iso_size = disc.padded_size() + + # 0x000: PSAR signature; 0x00C: disc size hint. + out.write(PSAR_MAGIC) + out.write(struct.pack(" + # length is always BLOCK_SIZE and offsets increase linearly. + block_count = iso_size // BLOCK_SIZE + offset = 0 + for _ in range(block_count): + out.write(struct.pack(" int: + if self.cue is None: + size = self.path.stat().st_size + if size % RAW_SECTOR == 0: + return size // RAW_SECTOR + return (size + COOKED_SECTOR - 1) // COOKED_SECTOR + total = 0 + base_dir = self.path.parent + for track in self.cue.tracks: + fpath = base_dir / track.file + if fpath.is_file(): + total += track.pregap_sectors + (fpath.stat().st_size // track.sector_size) + return total + + def padded_size(self) -> int: + size = self.total_sectors() * RAW_SECTOR + if size % BLOCK_SIZE: + size += BLOCK_SIZE - (size % BLOCK_SIZE) + return size + + def iter_raw(self, progress_cb=None, cancel_event=None) -> Iterator[bytes]: + if self.cue is None: + yield from iter_raw_sectors( + self.path, + RAW_SECTOR if self.path.stat().st_size % RAW_SECTOR == 0 else COOKED_SECTOR, + progress_cb=progress_cb, + cancel_event=cancel_event, + ) + return + + base_dir = self.path.parent + for track in self.cue.tracks: + fpath = base_dir / track.file + if not fpath.is_file(): + raise ConversionError(f"track file missing: {fpath}") + yield from iter_raw_sectors( + fpath, + track.sector_size, + pregap_sectors=track.pregap_sectors, + progress_cb=progress_cb, + cancel_event=cancel_event, + ) + + +def _aligned_up(value: int, alignment: int) -> int: + return value + ((alignment - (value % alignment)) % alignment) + + +def convert( + source: Path, + title: str, + game_id: str, + base_pbp: Path, + out_path: Path, + progress_cb=None, + cancel_event=None, +) -> Path: + """Convert a PS1 disc image to ``out_path`` (an ``EBOOT.PBP``). + + ``source`` may be a ``.cue`` (multi-track supported) or a single + ``.iso``/``.img``/``.bin``. ``base_pbp`` must be a PSN PS1 classic EBOOT. + """ + if not base_pbp.is_file(): + raise ConversionError(f"base.pbp not found: {base_pbp}") + if not source.is_file(): + raise ConversionError(f"disc image not found: {source}") + + game_id = game_id.strip() + if not game_id: + raise ConversionError("a game id (e.g. SLUS00594) is required") + + if source.suffix.lower() == ".cue": + disc: DiscImage = DiscImage(parse_cue(source.read_text(encoding="utf-8", errors="replace")), source) + else: + disc = DiscImage(None, source) + + # Resources required from the base PBP. + data_psp = extract_data_psp(base_pbp) + startdat = extract_startdat(base_pbp) + + sfo = default_ps1_sfo(title, game_id) + icon0 = default_icon0_png() + pic1 = default_pic1_png() + + # Sections in header order; empty sections keep the previous offset. + sections = { + "sfo": sfo, + "icon0": icon0, + "icon1": b"", + "pic0": b"", + "pic1": pic1, + "snd0": b"", + "data_psp": data_psp, + # psar written separately, sized after everything else. + "psar": None, + } + + header = [0] * PBP_HEADER_U32 + header[0] = struct.unpack(" None: + """Validate the parts that do not need hardware: SFO round-trip, PNG + generation, PBP header round-trip, and cue parsing.""" + import tempfile + + # CUE parsing. + cue = parse_cue( + 'FILE "Game (Track 1).bin" BINARY\n' + " TRACK 01 MODE2/2352\n" + " INDEX 01 00:00:00\n" + " TRACK 02 AUDIO\n" + " INDEX 00 00:02:00\n" + " INDEX 01 00:04:00\n" + 'FILE "Game (Track 2).bin" BINARY\n' + " TRACK 03 AUDIO\n" + " INDEX 01 00:00:00\n" + ) + assert len(cue.tracks) == 3, "expected 3 tracks" + assert cue.tracks[0].mode == "MODE2/2352" + assert cue.tracks[1].pregap_sectors == 150, "expected 2s (150 sector) pregap" + assert cue.data_track is cue.tracks[0] + + # SFO round-trip through the project's own parser. + from . import iso_reader # local import to keep the module importable standalone + + sfo = default_ps1_sfo("Final Fantasy VII", "SCUS94163") + parsed = iso_reader.parse_sfo(sfo) + assert parsed.get("TITLE") == "Final Fantasy VII", parsed + assert parsed.get("DISC_ID") == "SCUS94163", parsed + assert parsed.get("CATEGORY") == "ME", parsed + assert parsed.get("BOOTABLE") == 1, parsed + + # PNG generation produces valid PNG signatures + IHDR dimensions. + icon = default_icon0_png() + assert icon[:8] == b"\x89PNG\r\n\x1a\n" + assert struct.unpack(">II", icon[16:24]) == (144, 80) + pic = default_pic1_png() + assert struct.unpack(">II", pic[16:24]) == (480, 272) + + # PBP header round-trip: write a synthetic PBP and read it back. + with tempfile.TemporaryDirectory() as td: + p = Path(td) / "test.pbp" + header = [0] * 10 + header[0] = struct.unpack(" int: + ap = argparse.ArgumentParser(description="Convert a PS1 disc image to a PSP EBOOT.PBP") + ap.add_argument("--source", help="path to .cue or a single .iso/.img/.bin") + ap.add_argument("--title", help="game title shown on the PSP") + ap.add_argument("--game-id", help="PS1 game id, e.g. SLUS00594") + ap.add_argument("--base-pbp", help="path to a PSN PS1 classic EBOOT.PBP (required)") + ap.add_argument("--out", help="output EBOOT.PBP path") + ap.add_argument("--test", action="store_true", help="run self-test and exit") + args = ap.parse_args(argv) + + if args.test: + _self_test() + return 0 + + if not all([args.source, args.title, args.game_id, args.base_pbp, args.out]): + ap.error("--source --title --game-id --base-pbp and --out are all required (or use --test)") + + try: + convert( + Path(args.source), + args.title, + args.game_id, + Path(args.base_pbp), + Path(args.out), + ) + print(f"wrote {args.out}") + return 0 + except ConversionError as exc: + print(f"error: {exc}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main())