psp-preservation-tool/app/ingest.py

"""Ingest clrmamepro .dat files into the SQLite library database.

Usage::

    python -m app.ingest [--dats data/dats] [--db data/library.sqlite]
"""
import argparse
import sys
from pathlib import Path

from . import config, matcher
from .dat_parser import parse_dat_file
from .db import get_connection, init_db


def _norm_hex(value):
    if not value:
        return None
    return str(value).strip().upper()


def detect_system(dat_name):
    if "UMD Video" in dat_name:
        return "videos"
    if "PlayStation Portable" in dat_name:
        return "games"
    # Redump PS1: "Sony - PlayStation" (assumes no PS2/PS3 DATs are present).
    if "PlayStation" in dat_name:
        return "ps1"
    return "unknown"


def ingest_dat(conn, path):
    header, entries = parse_dat_file(path)
    dat_name = header.get("name") if header else path.stem
    version = header.get("version") if header else None
    description = header.get("description") if header else None
    system = detect_system(dat_name)

    existing = conn.execute("SELECT id FROM dats WHERE name = ?", (dat_name,)).fetchone()
    if existing:
        dat_id = existing["id"]
        # Files that resolved to a release in this DAT must be re-identified
        # against the new catalog (the release ids are about to be recycled).
        conn.execute(
            "UPDATE files SET release_id = NULL WHERE release_id IN (SELECT id FROM releases WHERE dat_id = ?)",
            (dat_id,),
        )
        conn.execute("DELETE FROM roms WHERE release_id IN (SELECT id FROM releases WHERE dat_id = ?)", (dat_id,))
        conn.execute("DELETE FROM releases WHERE dat_id = ?", (dat_id,))
        conn.execute(
            "UPDATE dats SET version=?, description=?, system=?, source_path=?, ingested_at=datetime('now') WHERE id=?",
            (version, description, system, str(path), dat_id),
        )
    else:
        cur = conn.execute(
            "INSERT INTO dats (name, version, description, system, source_path, ingested_at) VALUES (?,?,?,?,?,datetime('now'))",
            (dat_name, version, description, system, str(path)),
        )
        dat_id = cur.lastrowid

    release_count = 0
    rom_count = 0
    for entry in entries:
        cur = conn.execute(
            "INSERT INTO releases (dat_id, name, description, region, serial) VALUES (?,?,?,?,?)",
            (dat_id, entry.get("name"), entry.get("description"), entry.get("region"), entry.get("serial")),
        )
        release_id = cur.lastrowid
        release_count += 1
        for rom in entry.get("rom", []):
            size = rom.get("size")
            conn.execute(
                "INSERT INTO roms (release_id, name, size, crc32, md5, sha1, serial) VALUES (?,?,?,?,?,?,?)",
                (
                    release_id,
                    rom.get("name"),
                    int(size) if size else None,
                    _norm_hex(rom.get("crc")),
                    _norm_hex(rom.get("md5")),
                    _norm_hex(rom.get("sha1")),
                    rom.get("serial"),
                ),
            )
            rom_count += 1

    matcher.clear_cache()
    return dat_name, system, release_count, rom_count


def ensure_catalog(dats_dir=None, conn=None):
    """Ingest any DAT files missing from the DB (idempotent, cheap when current).

    Called once at startup so a fresh/reset database still has the catalog and
    scanning produces matches instead of everything coming back ``unknown``.
    Returns the number of DATs ingested.
    """
    dats_dir = Path(dats_dir) if dats_dir else config.DATS_DIR
    if not dats_dir.is_dir():
        return 0
    files = sorted(p for p in dats_dir.glob("*.dat") if p.is_file())
    if not files:
        return 0

    own = conn is None
    if own:
        conn = get_connection(config.DB_PATH)
    try:
        known = {r["source_path"] for r in conn.execute("SELECT source_path FROM dats")}
        count = 0
        for path in files:
            if str(path) in known:
                continue
            try:
                ingest_dat(conn, path)
            except Exception as exc:  # noqa: BLE001 - a bad DAT shouldn't block startup
                print(f"warning: could not ingest {path.name}: {exc}", file=sys.stderr)
                continue
            count += 1
        conn.commit()
        return count
    finally:
        if own:
            conn.close()


def main(argv=None):
    parser = argparse.ArgumentParser(description="Ingest clrmamepro DAT files into SQLite")
    parser.add_argument("--dats", help="directory containing .dat files (default: data/dats)")
    parser.add_argument("--db", help="SQLite database path (default: data/library.sqlite)")
    args = parser.parse_args(argv)

    dats_dir = Path(args.dats) if args.dats else config.DATS_DIR
    db_path = Path(args.db) if args.db else config.DB_PATH

    conn = init_db(db_path)
    files = sorted(dats_dir.glob("*.dat"))
    if not files:
        print(f"No .dat files found in {dats_dir}", file=sys.stderr)
        return 1

    for path in files:
        dat_name, system, releases, roms = ingest_dat(conn, path)
        print(f"{dat_name} [{system}] -> {releases} releases, {roms} roms")

    conn.commit()
    conn.close()
    print(f"\nLibrary written to {db_path}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())