ps1: compress disc data (raw DEFLATE) for real-hardware boot

POPS on real PSP hardware refuses to boot uncompressed (level-0) PSARs
(noted in pop-fe: 'Uncompressed EBOOTs don't work'), which caused 'the game
cannot be started'. Compress each 0x9300 disc block with raw DEFLATE and
switch the index table to the pop-fe layout (offset u32, length u16, flags
u8, SHA-1[16]); patch p1/p2 after the compressed size is known.

- _write_disc_block: deflate each block; store raw bytes when a block does
  not shrink.
- _write_psar: reserve + patch the index table and p1/p2.
- convert/convert_to_folder/CLI: complevel parameter (default 1).
- Reuse the base PBP's ICON0 and drop generated PIC1 to match popstation.
- Self-test: add a DEFLATE round-trip with index/SHA-1 checks.

app/ps1_convert.py

@@ -24,13 +24,16 @@ blobs that we cannot synthesise:
 Without a base.pbp the resulting EBOOT will not boot on real hardware; the
 converter refuses to run without one.
 
-Compression: only level 0 (uncompressed) is implemented. LZRC (levels 1-9) is a
+Compression: the disc data is split into 0x9300-byte blocks and deflated
-planned fast-follow.
+(raw DEFLATE, no zlib wrapper) at the configured level (default 1). POPS on
+real PSP hardware refuses to boot uncompressed (level-0) PSARs, so level 0
+is kept only for diagnostics; a valid conversion always uses level >= 1.
 """
 
 from __future__ import annotations
 
 import argparse
+import hashlib
 import io
 import os
 import re
@@ -365,6 +368,13 @@ def extract_sfo(base_pbp: Path) -> bytes:
     return read_pbp_section(base_pbp, header, idx)
 
 
+def extract_icon0(base_pbp: Path) -> bytes:
+    """Return the ICON0.PNG section of a base PBP."""
+    header = read_pbp_header(base_pbp)
+    idx = SECTION_OFFSET["icon0"] // 4
+    return read_pbp_section(base_pbp, header, idx)
+
+
 def patch_sfo(base_sfo: bytes, title: str, game_id: str) -> bytes:
     """Patch ``TITLE`` and ``DISC_ID`` in a base PARAM.SFO in place.
 
@@ -605,49 +615,104 @@ def _build_psar_header(disc: "DiscImage", game_id: str, title: str) -> bytes:
     return bytes(block)
 
 
+def _write_disc_block(
+    out: io.BufferedWriter,
+    block: bytes,
+    complevel: int,
+    data_offset: int,
+    indexes: bytearray,
+) -> int:
+    """Compress and write one 0x9300 disc block, appending its index entry.
+
+    Blocks are deflated with raw DEFLATE (the zlib header and Adler-32
+    trailer stripped). If a block does not shrink, the raw bytes are stored
+    instead. The index entry follows pop-fe: offset (u32), length (u16),
+    flags (u8), then the first 16 bytes of the block's SHA-1.
+    """
+    assert len(block) == BLOCK_SIZE
+    compressed = zlib.compress(block, complevel)[2:-4] if complevel > 0 else block
+
+    entry = bytearray(INDEX_ENTRY_SIZE)
+    struct.pack_into("<I", entry, 0, data_offset)
+    if complevel == 0:
+        entry[6] = 0x01  # uncompressed block
+    entry[8:24] = hashlib.sha1(block).digest()[:16]
+
+    if len(compressed) >= BLOCK_SIZE:
+        struct.pack_into("<H", entry, 4, BLOCK_SIZE)
+        out.write(block)
+        data_offset += BLOCK_SIZE
+    else:
+        struct.pack_into("<H", entry, 4, len(compressed))
+        out.write(compressed)
+        data_offset += len(compressed)
+
+    indexes.extend(entry)
+    return data_offset
+
+
 def _write_psar(
     out: io.BufferedWriter,
     disc: "DiscImage",
     game_id: str,
     title: str,
+    complevel: int = 1,
     progress_cb=None,
     cancel_event=None,
 ) -> None:
-    """Write the DATA.PSAR section (uncompressed, single-disc).
+    """Write the DATA.PSAR section (single-disc).
+
+    Disc data is split into 0x9300-byte blocks and deflated (``complevel`` >
+    0). The PSAR header's p1/p2 fields and the index table are patched after
+    the compressed data is written, since the compressed size is not known
+    up front.
 
     All offsets are relative to the start of the PSAR section.
     """
     base = out.tell()
     iso_size = disc.padded_size()
+    block_count = iso_size // BLOCK_SIZE
 
-    # 0x000: PSAR signature; 0x00C: disc size hint.
+    # 0x000: PSAR signature; 0x00C: p1 (disc data end, patched below).
     out.write(PSAR_MAGIC)
     out.write(struct.pack("<I", iso_size + PSAR_ISO_OFFSET))
     _pad_to(out, base + PSAR_GAMEID_OFFSET)
 
+    # 0x400..0x4000: game id / TOC / title (p2 at 0x1220 is patched below).
     out.write(_build_psar_header(disc, game_id, title))
     _pad_to(out, base + PSAR_INDEX_OFFSET)
 
-    # ISO block index: (offset, length) per 0x9300 block. Uncompressed =>
+    # Reserve the index table; filled in once the disc data is compressed.
-    # length is always BLOCK_SIZE and offsets increase linearly.
-    block_count = iso_size // BLOCK_SIZE
-    offset = 0
     for _ in range(block_count):
-        out.write(struct.pack("<II", offset, BLOCK_SIZE))
+        out.write(b"\x00" * INDEX_ENTRY_SIZE)
-        out.write(b"\x00" * (INDEX_ENTRY_SIZE - 8))
-        offset += BLOCK_SIZE
 
     _pad_to(out, base + PSAR_ISO_OFFSET)
 
-    # Disc data, block by block.
+    # Compress + write disc data, one 0x9300 block at a time.
+    indexes = bytearray()
+    data_offset = 0  # relative to PSAR_ISO_OFFSET
     buf = bytearray()
     for sector in disc.iter_raw(progress_cb=progress_cb, cancel_event=cancel_event):
         buf.extend(sector)
-        if len(buf) == BLOCK_SIZE:
+        if len(buf) < BLOCK_SIZE:
-            out.write(buf)
+            continue
-            buf = bytearray()
+        data_offset = _write_disc_block(out, buf, complevel, data_offset, indexes)
+        buf = bytearray()
     if buf:
-        out.write(buf.ljust(BLOCK_SIZE, b"\x00"))
+        data_offset = _write_disc_block(out, buf.ljust(BLOCK_SIZE, b"\x00"), complevel, data_offset, indexes)
+
+    # Align the end of the disc data to 0x10 before STARTDAT (which the
+    # caller appends), then patch p1/p2 and the index table.
+    _pad_to(out, (out.tell() + 0xF) & ~0xF)
+    end_offset = out.tell() - base
+    end = out.tell()
+    out.seek(base + 0x0C)
+    out.write(struct.pack("<I", end_offset))
+    out.seek(base + 0x1220)
+    out.write(struct.pack("<I", end_offset + 0x2D31))
+    out.seek(base + PSAR_INDEX_OFFSET)
+    out.write(bytes(indexes))
+    out.seek(end)
 
 
 class DiscImage:
@@ -734,11 +799,14 @@ def convert(
     out_path: Path,
     progress_cb=None,
     cancel_event=None,
+    complevel: int = 1,
 ) -> Path:
     """Convert a PS1 disc image to ``out_path`` (an ``EBOOT.PBP``).
 
     ``source`` may be a ``.cue`` (multi-track supported) or a single
     ``.iso``/``.img``/``.bin``. ``base_pbp`` must be a PSN PS1 classic EBOOT.
+    ``complevel`` is the DEFLATE level for the disc data (1-9; 0 = raw, which
+    does not boot on real hardware and is only useful for diagnostics).
     """
     if not base_pbp.is_file():
         raise ConversionError(f"base.pbp not found: {base_pbp}")
@@ -761,8 +829,11 @@ def convert(
     # Reuse the base PBP's SFO, patched with our title + game id (the
     # popstation approach), rather than rebuilding it from scratch.
     sfo = patch_sfo(extract_sfo(base_pbp), title, game_id)
-    icon0 = default_icon0_png()
+    # Reuse the base PBP's ICON0 too. The XMB icon parser is picky about PNG
-    pic1 = default_pic1_png()
+    # variants, and the base icon is guaranteed to decode; the other image
+    # sections are left empty exactly like popstation when no assets are
+    # supplied alongside the conversion.
+    icon0 = extract_icon0(base_pbp)
 
     # Sections in header order; empty sections keep the previous offset.
     sections = {
@@ -770,7 +841,7 @@ def convert(
         "icon0": icon0,
         "icon1": b"",
         "pic0": b"",
-        "pic1": pic1,
+        "pic1": b"",
         "snd0": b"",
         "data_psp": data_psp,
         # psar written separately, sized after everything else.
@@ -807,7 +878,8 @@ def convert(
             while fh.tell() < psar_offset:
                 fh.write(b"\x00")
 
-            _write_psar(fh, disc, game_id, title, progress_cb=progress_cb, cancel_event=cancel_event)
+            _write_psar(fh, disc, game_id, title, complevel=complevel,
+                        progress_cb=progress_cb, cancel_event=cancel_event)
 
             # STARTDAT goes at the very end.
             fh.write(startdat)
@@ -829,20 +901,20 @@ def convert_to_folder(
     out_dir: Path,
     progress_cb=None,
     cancel_event=None,
+    complevel: int = 1,
 ) -> Path:
     """Convert a PS1 disc into a game folder ready for /PSP/GAME.
 
     Writes ``<out_dir>/<GAME_ID>/EBOOT.PBP`` — the same single-file output the
-    popstation tool produces (uncompressed/unencrypted level-0 PSAR). No
+    popstation tool produces. No KEYS.BIN or DOCUMENT.DAT is written:
-    KEYS.BIN or DOCUMENT.DAT is written: popstation doesn't emit them and CFW
+    popstation doesn't emit them and CFW (ARK-4 POPS) doesn't need them.
-    (ARK-4 POPS) doesn't need them.
     """
     folder_name = re.sub(r"[^A-Za-z0-9]", "", game_id).upper() or "PS1GAME"
     folder = Path(out_dir) / folder_name
     folder.mkdir(parents=True, exist_ok=True)
 
     convert(source, title, game_id, base_pbp, folder / "EBOOT.PBP",
-            progress_cb=progress_cb, cancel_event=cancel_event)
+            progress_cb=progress_cb, cancel_event=cancel_event, complevel=complevel)
     return folder
 
 
@@ -941,6 +1013,35 @@ def _self_test() -> None:
         assert read_pbp_header(p)[0] == header[0]
         assert read_pbp_section(p, header, 2) == sfo
 
+    # PSAR compression round-trip (level 1): the disc data must survive
+    # raw-DEFLATE and the index table must be readable with the pop-fe
+    # layout (offset u32, length u16, flags u8, SHA-1[16]).
+    with tempfile.TemporaryDirectory() as td:
+        d = Path(td)
+        src = d / "cd.bin"
+        src.write_bytes(bytes(range(256)) * (BLOCK_SIZE // 256 * 3))
+        disc = DiscImage(None, src)
+        buf = io.BytesIO()
+        _write_psar(buf, disc, "SLUS00594", "Round Trip", complevel=1)
+        raw = buf.getvalue()
+        assert raw[:12] == PSAR_MAGIC
+        rebuilt = bytearray()
+        i = 0
+        while True:
+            e = raw[PSAR_INDEX_OFFSET + i * 32: PSAR_INDEX_OFFSET + (i + 1) * 32]
+            off, = struct.unpack("<I", e[0:4])
+            ln, = struct.unpack("<H", e[4:6])
+            if ln == 0:
+                break
+            data = raw[PSAR_ISO_OFFSET + off: PSAR_ISO_OFFSET + off + ln]
+            rebuilt += zlib.decompress(data, wbits=-15) if ln < BLOCK_SIZE else data
+            i += 1
+        assert i == 3, i
+        assert len(rebuilt) == disc.padded_size(), len(rebuilt)
+        assert bytes(rebuilt[: src.stat().st_size]) == src.read_bytes()
+        # first index entry carries the SHA-1 of the first raw block
+        assert raw[PSAR_INDEX_OFFSET + 8: PSAR_INDEX_OFFSET + 24] == hashlib.sha1(bytes(rebuilt[:BLOCK_SIZE])).digest()[:16]
+
     print("ps1_convert self-test OK")
 
 
@@ -952,6 +1053,8 @@ def main(argv: Optional[List[str]] = None) -> int:
     ap.add_argument("--base-pbp", help="path to a PSN PS1 classic EBOOT.PBP (required)")
     ap.add_argument("--out", help="output EBOOT.PBP path (single file)")
     ap.add_argument("--out-dir", help="output folder: writes <dir>/<GAME_ID>/EBOOT.PBP")
+    ap.add_argument("--complevel", type=int, default=1, choices=range(0, 10),
+                    help="DEFLATE compression level 0-9 for the disc data (default 1)")
     ap.add_argument("--test", action="store_true", help="run self-test and exit")
     args = ap.parse_args(argv)
 
@@ -966,13 +1069,13 @@ def main(argv: Optional[List[str]] = None) -> int:
         if args.out_dir:
             folder = convert_to_folder(
                 Path(args.source), args.title, args.game_id,
-                Path(args.base_pbp), Path(args.out_dir),
+                Path(args.base_pbp), Path(args.out_dir), complevel=args.complevel,
             )
             print(f"wrote {folder}")
         else:
             convert(
                 Path(args.source), args.title, args.game_id,
-                Path(args.base_pbp), Path(args.out),
+                Path(args.base_pbp), Path(args.out), complevel=args.complevel,
             )
             print(f"wrote {args.out}")
         return 0