"""Ingest clrmamepro .dat files into the SQLite library database. Usage:: python -m app.ingest [--dats data/dats] [--db data/library.sqlite] """ import argparse import sys from pathlib import Path from . import config, matcher from .dat_parser import parse_dat_file from .db import get_connection, init_db def _norm_hex(value): if not value: return None return str(value).strip().upper() def detect_system(dat_name): if "UMD Video" in dat_name: return "videos" if "PlayStation Portable" in dat_name: return "games" # Redump PS1: "Sony - PlayStation" (assumes no PS2/PS3 DATs are present). if "PlayStation" in dat_name: return "ps1" return "unknown" def ingest_dat(conn, path): header, entries = parse_dat_file(path) dat_name = header.get("name") if header else path.stem version = header.get("version") if header else None description = header.get("description") if header else None system = detect_system(dat_name) existing = conn.execute("SELECT id FROM dats WHERE name = ?", (dat_name,)).fetchone() if existing: dat_id = existing["id"] # Files that resolved to a release in this DAT must be re-identified # against the new catalog (the release ids are about to be recycled). conn.execute( "UPDATE files SET release_id = NULL WHERE release_id IN (SELECT id FROM releases WHERE dat_id = ?)", (dat_id,), ) conn.execute("DELETE FROM roms WHERE release_id IN (SELECT id FROM releases WHERE dat_id = ?)", (dat_id,)) conn.execute("DELETE FROM releases WHERE dat_id = ?", (dat_id,)) conn.execute( "UPDATE dats SET version=?, description=?, system=?, source_path=?, ingested_at=datetime('now') WHERE id=?", (version, description, system, str(path), dat_id), ) else: cur = conn.execute( "INSERT INTO dats (name, version, description, system, source_path, ingested_at) VALUES (?,?,?,?,?,datetime('now'))", (dat_name, version, description, system, str(path)), ) dat_id = cur.lastrowid release_count = 0 rom_count = 0 for entry in entries: cur = conn.execute( "INSERT INTO releases (dat_id, name, description, region, serial) VALUES (?,?,?,?,?)", (dat_id, entry.get("name"), entry.get("description"), entry.get("region"), entry.get("serial")), ) release_id = cur.lastrowid release_count += 1 for rom in entry.get("rom", []): size = rom.get("size") conn.execute( "INSERT INTO roms (release_id, name, size, crc32, md5, sha1, serial) VALUES (?,?,?,?,?,?,?)", ( release_id, rom.get("name"), int(size) if size else None, _norm_hex(rom.get("crc")), _norm_hex(rom.get("md5")), _norm_hex(rom.get("sha1")), rom.get("serial"), ), ) rom_count += 1 matcher.clear_cache() return dat_name, system, release_count, rom_count def ensure_catalog(dats_dir=None, conn=None): """Ingest any DAT files missing from the DB (idempotent, cheap when current). Called once at startup so a fresh/reset database still has the catalog and scanning produces matches instead of everything coming back ``unknown``. Returns the number of DATs ingested. """ dats_dir = Path(dats_dir) if dats_dir else config.DATS_DIR if not dats_dir.is_dir(): return 0 files = sorted(p for p in dats_dir.glob("*.dat") if p.is_file()) if not files: return 0 own = conn is None if own: conn = get_connection(config.DB_PATH) try: known = {r["source_path"] for r in conn.execute("SELECT source_path FROM dats")} count = 0 for path in files: if str(path) in known: continue try: ingest_dat(conn, path) except Exception as exc: # noqa: BLE001 - a bad DAT shouldn't block startup print(f"warning: could not ingest {path.name}: {exc}", file=sys.stderr) continue count += 1 conn.commit() return count finally: if own: conn.close() def main(argv=None): parser = argparse.ArgumentParser(description="Ingest clrmamepro DAT files into SQLite") parser.add_argument("--dats", help="directory containing .dat files (default: data/dats)") parser.add_argument("--db", help="SQLite database path (default: data/library.sqlite)") args = parser.parse_args(argv) dats_dir = Path(args.dats) if args.dats else config.DATS_DIR db_path = Path(args.db) if args.db else config.DB_PATH conn = init_db(db_path) files = sorted(dats_dir.glob("*.dat")) if not files: print(f"No .dat files found in {dats_dir}", file=sys.stderr) return 1 for path in files: dat_name, system, releases, roms = ingest_dat(conn, path) print(f"{dat_name} [{system}] -> {releases} releases, {roms} roms") conn.commit() conn.close() print(f"\nLibrary written to {db_path}") return 0 if __name__ == "__main__": raise SystemExit(main())