quoije/psp-preservation-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

ps1: compress disc data (raw DEFLATE) for real-hardware boot

Browse Source 
POPS on real PSP hardware refuses to boot uncompressed (level-0) PSARs
(noted in pop-fe: 'Uncompressed EBOOTs don't work'), which caused 'the game
cannot be started'. Compress each 0x9300 disc block with raw DEFLATE and
switch the index table to the pop-fe layout (offset u32, length u16, flags
u8, SHA-1[16]); patch p1/p2 after the compressed size is known.

- _write_disc_block: deflate each block; store raw bytes when a block does
  not shrink.
- _write_psar: reserve + patch the index table and p1/p2.
- convert/convert_to_folder/CLI: complevel parameter (default 1).
- Reuse the base PBP's ICON0 and drop generated PIC1 to match popstation.
- Self-test: add a DEFLATE round-trip with index/SHA-1 checks.
This commit is contained in:
quoije
2026-09-07 17:05:31 -04:00
parent 2f1fc760a1
commit e50cd02f2a
1 changed files with 129 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/ps1_convert.py
+128 -25
View File
@@ -24,13 +24,16 @@ blobs that we cannot synthesise:
Without a base.pbp the resulting EBOOT will not boot on real hardware; the
converter refuses to run without one.
Compression: only level 0 (uncompressed) is implemented. LZRC (levels 1-9) is a
planned fast-follow.
Compression: the disc data is split into 0x9300-byte blocks and deflated
(raw DEFLATE, no zlib wrapper) at the configured level (default 1). POPS on
real PSP hardware refuses to boot uncompressed (level-0) PSARs, so level 0
is kept only for diagnostics; a valid conversion always uses level >= 1.
"""
from __future__ import annotations
import argparse
import hashlib
import io
import os
import re
@@ -365,6 +368,13 @@ def extract_sfo(base_pbp: Path) -> bytes:
return read_pbp_section(base_pbp, header, idx)
def extract_icon0(base_pbp: Path) -> bytes:
"""Return the ICON0.PNG section of a base PBP."""
header = read_pbp_header(base_pbp)
idx = SECTION_OFFSET["icon0"] // 4
return read_pbp_section(base_pbp, header, idx)
def patch_sfo(base_sfo: bytes, title: str, game_id: str) -> bytes:
"""Patch ``TITLE`` and ``DISC_ID`` in a base PARAM.SFO in place.
@@ -605,49 +615,104 @@ def _build_psar_header(disc: "DiscImage", game_id: str, title: str) -> bytes:
return bytes(block)
def _write_disc_block(
out: io.BufferedWriter,
block: bytes,
complevel: int,
data_offset: int,
indexes: bytearray,
) -> int:
"""Compress and write one 0x9300 disc block, appending its index entry.
Blocks are deflated with raw DEFLATE (the zlib header and Adler-32
trailer stripped). If a block does not shrink, the raw bytes are stored
instead. The index entry follows pop-fe: offset (u32), length (u16),
flags (u8), then the first 16 bytes of the block's SHA-1.
"""
assert len(block) == BLOCK_SIZE
compressed = zlib.compress(block, complevel)[2:-4] if complevel > 0 else block
entry = bytearray(INDEX_ENTRY_SIZE)
struct.pack_into("<I", entry, 0, data_offset)
if complevel == 0:
entry[6] = 0x01 # uncompressed block
entry[8:24] = hashlib.sha1(block).digest()[:16]
if len(compressed) >= BLOCK_SIZE:
struct.pack_into("<H", entry, 4, BLOCK_SIZE)
out.write(block)
data_offset += BLOCK_SIZE
else:
struct.pack_into("<H", entry, 4, len(compressed))
out.write(compressed)
data_offset += len(compressed)
indexes.extend(entry)
return data_offset
def _write_psar(
out: io.BufferedWriter,
disc: "DiscImage",
game_id: str,
title: str,
complevel: int = 1,
progress_cb=None,
cancel_event=None,
) -> None:
"""Write the DATA.PSAR section (uncompressed, single-disc).
"""Write the DATA.PSAR section (single-disc).
Disc data is split into 0x9300-byte blocks and deflated (``complevel`` >
0). The PSAR header's p1/p2 fields and the index table are patched after
the compressed data is written, since the compressed size is not known
up front.
All offsets are relative to the start of the PSAR section.
"""
base = out.tell()
iso_size = disc.padded_size()
block_count = iso_size // BLOCK_SIZE
# 0x000: PSAR signature; 0x00C: disc size hint.
# 0x000: PSAR signature; 0x00C: p1 (disc data end, patched below).
out.write(PSAR_MAGIC)
out.write(struct.pack("<I", iso_size + PSAR_ISO_OFFSET))
_pad_to(out, base + PSAR_GAMEID_OFFSET)
# 0x400..0x4000: game id / TOC / title (p2 at 0x1220 is patched below).
out.write(_build_psar_header(disc, game_id, title))
_pad_to(out, base + PSAR_INDEX_OFFSET)
# ISO block index: (offset, length) per 0x9300 block. Uncompressed =>
# length is always BLOCK_SIZE and offsets increase linearly.
block_count = iso_size // BLOCK_SIZE
offset = 0
# Reserve the index table; filled in once the disc data is compressed.
for _ in range(block_count):
out.write(struct.pack("<II", offset, BLOCK_SIZE))
out.write(b"\x00" * (INDEX_ENTRY_SIZE - 8))
offset += BLOCK_SIZE
out.write(b"\x00" * INDEX_ENTRY_SIZE)
_pad_to(out, base + PSAR_ISO_OFFSET)
# Disc data, block by block.
# Compress + write disc data, one 0x9300 block at a time.
indexes = bytearray()
data_offset = 0 # relative to PSAR_ISO_OFFSET
buf = bytearray()
for sector in disc.iter_raw(progress_cb=progress_cb, cancel_event=cancel_event):
buf.extend(sector)
if len(buf) == BLOCK_SIZE:
out.write(buf)
if len(buf) < BLOCK_SIZE:
continue
data_offset = _write_disc_block(out, buf, complevel, data_offset, indexes)
buf = bytearray()
if buf:
out.write(buf.ljust(BLOCK_SIZE, b"\x00"))
data_offset = _write_disc_block(out, buf.ljust(BLOCK_SIZE, b"\x00"), complevel, data_offset, indexes)
# Align the end of the disc data to 0x10 before STARTDAT (which the
# caller appends), then patch p1/p2 and the index table.
_pad_to(out, (out.tell() + 0xF) & ~0xF)
end_offset = out.tell() - base
end = out.tell()
out.seek(base + 0x0C)
out.write(struct.pack("<I", end_offset))
out.seek(base + 0x1220)
out.write(struct.pack("<I", end_offset + 0x2D31))
out.seek(base + PSAR_INDEX_OFFSET)
out.write(bytes(indexes))
out.seek(end)
class DiscImage:
@@ -734,11 +799,14 @@ def convert(
out_path: Path,
progress_cb=None,
cancel_event=None,
complevel: int = 1,
) -> Path:
"""Convert a PS1 disc image to ``out_path`` (an ``EBOOT.PBP``).
``source`` may be a ``.cue`` (multi-track supported) or a single
``.iso``/``.img``/``.bin``. ``base_pbp`` must be a PSN PS1 classic EBOOT.
``complevel`` is the DEFLATE level for the disc data (1-9; 0 = raw, which
does not boot on real hardware and is only useful for diagnostics).
"""
if not base_pbp.is_file():
raise ConversionError(f"base.pbp not found: {base_pbp}")
@@ -761,8 +829,11 @@ def convert(
# Reuse the base PBP's SFO, patched with our title + game id (the
# popstation approach), rather than rebuilding it from scratch.
sfo = patch_sfo(extract_sfo(base_pbp), title, game_id)
icon0 = default_icon0_png()
pic1 = default_pic1_png()
# Reuse the base PBP's ICON0 too. The XMB icon parser is picky about PNG
# variants, and the base icon is guaranteed to decode; the other image
# sections are left empty exactly like popstation when no assets are
# supplied alongside the conversion.
icon0 = extract_icon0(base_pbp)
# Sections in header order; empty sections keep the previous offset.
sections = {
@@ -770,7 +841,7 @@ def convert(
"icon0": icon0,
"icon1": b"",
"pic0": b"",
"pic1": pic1,
"pic1": b"",
"snd0": b"",
"data_psp": data_psp,
# psar written separately, sized after everything else.
@@ -807,7 +878,8 @@ def convert(
while fh.tell() < psar_offset:
fh.write(b"\x00")
_write_psar(fh, disc, game_id, title, progress_cb=progress_cb, cancel_event=cancel_event)
_write_psar(fh, disc, game_id, title, complevel=complevel,
progress_cb=progress_cb, cancel_event=cancel_event)
# STARTDAT goes at the very end.
fh.write(startdat)
@@ -829,20 +901,20 @@ def convert_to_folder(
out_dir: Path,
progress_cb=None,
cancel_event=None,
complevel: int = 1,
) -> Path:
"""Convert a PS1 disc into a game folder ready for /PSP/GAME.
Writes ``<out_dir>/<GAME_ID>/EBOOT.PBP`` — the same single-file output the
popstation tool produces (uncompressed/unencrypted level-0 PSAR). No
KEYS.BIN or DOCUMENT.DAT is written: popstation doesn't emit them and CFW
(ARK-4 POPS) doesn't need them.
popstation tool produces. No KEYS.BIN or DOCUMENT.DAT is written:
popstation doesn't emit them and CFW (ARK-4 POPS) doesn't need them.
"""
folder_name = re.sub(r"[^A-Za-z0-9]", "", game_id).upper() or "PS1GAME"
folder = Path(out_dir) / folder_name
folder.mkdir(parents=True, exist_ok=True)
convert(source, title, game_id, base_pbp, folder / "EBOOT.PBP",
progress_cb=progress_cb, cancel_event=cancel_event)
progress_cb=progress_cb, cancel_event=cancel_event, complevel=complevel)
return folder
@@ -941,6 +1013,35 @@ def _self_test() -> None:
assert read_pbp_header(p)[0] == header[0]
assert read_pbp_section(p, header, 2) == sfo
# PSAR compression round-trip (level 1): the disc data must survive
# raw-DEFLATE and the index table must be readable with the pop-fe
# layout (offset u32, length u16, flags u8, SHA-1[16]).
with tempfile.TemporaryDirectory() as td:
d = Path(td)
src = d / "cd.bin"
src.write_bytes(bytes(range(256)) * (BLOCK_SIZE // 256 * 3))
disc = DiscImage(None, src)
buf = io.BytesIO()
_write_psar(buf, disc, "SLUS00594", "Round Trip", complevel=1)
raw = buf.getvalue()
assert raw[:12] == PSAR_MAGIC
rebuilt = bytearray()
i = 0
while True:
e = raw[PSAR_INDEX_OFFSET + i * 32: PSAR_INDEX_OFFSET + (i + 1) * 32]
off, = struct.unpack("<I", e[0:4])
ln, = struct.unpack("<H", e[4:6])
if ln == 0:
break
data = raw[PSAR_ISO_OFFSET + off: PSAR_ISO_OFFSET + off + ln]
rebuilt += zlib.decompress(data, wbits=-15) if ln < BLOCK_SIZE else data
i += 1
assert i == 3, i
assert len(rebuilt) == disc.padded_size(), len(rebuilt)
assert bytes(rebuilt[: src.stat().st_size]) == src.read_bytes()
# first index entry carries the SHA-1 of the first raw block
assert raw[PSAR_INDEX_OFFSET + 8: PSAR_INDEX_OFFSET + 24] == hashlib.sha1(bytes(rebuilt[:BLOCK_SIZE])).digest()[:16]
print("ps1_convert self-test OK")
@@ -952,6 +1053,8 @@ def main(argv: Optional[List[str]] = None) -> int:
ap.add_argument("--base-pbp", help="path to a PSN PS1 classic EBOOT.PBP (required)")
ap.add_argument("--out", help="output EBOOT.PBP path (single file)")
ap.add_argument("--out-dir", help="output folder: writes <dir>/<GAME_ID>/EBOOT.PBP")
ap.add_argument("--complevel", type=int, default=1, choices=range(0, 10),
help="DEFLATE compression level 0-9 for the disc data (default 1)")
ap.add_argument("--test", action="store_true", help="run self-test and exit")
args = ap.parse_args(argv)
@@ -966,13 +1069,13 @@ def main(argv: Optional[List[str]] = None) -> int:
if args.out_dir:
folder = convert_to_folder(
Path(args.source), args.title, args.game_id,
Path(args.base_pbp), Path(args.out_dir),
Path(args.base_pbp), Path(args.out_dir), complevel=args.complevel,
)
print(f"wrote {folder}")
else:
convert(
Path(args.source), args.title, args.game_id,
Path(args.base_pbp), Path(args.out),
Path(args.base_pbp), Path(args.out), complevel=args.complevel,
)
print(f"wrote {args.out}")
return 0