quoije/psp-preservation-tool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Add PS1 games to the Library with covers, metadata, and EBOOT actions

Browse Source
This commit is contained in:
quoije
2026-09-07 16:20:39 -04:00
parent 777d316c0f
commit 4d5c48cb94
4 changed files with 201 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/covers.py
+95 -68
View File
@@ -24,16 +24,31 @@ from .db import get_connection
log = logging.getLogger("covers")
_LIBRETRO_BASE = "https://raw.githubusercontent.com/libretro-thumbnails/Sony_-_PlayStation_Portable/master"
_LIBRETRO_KINDS = ("Named_Boxarts",)
_TREE_URL = "https://api.github.com/repos/libretro-thumbnails/Sony_-_PlayStation_Portable/git/trees/master?recursive=1"
_PLATFORMS = {
"games": {"libretro_repo": "Sony_-_PlayStation_Portable", "igdb_id": 38, "ss_id": "61"},
"ps1": {"libretro_repo": "Sony_-_PlayStation", "igdb_id": 7, "ss_id": "57"},
}
def _platform(system):
"""Platform parameters (libretro repo, IGDB id, ScreenScraper id)."""
return _PLATFORMS.get(system) or _PLATFORMS["games"]
def _libretro_base(system="games"):
return ("https://raw.githubusercontent.com/libretro-thumbnails/"
f"{_platform(system)['libretro_repo']}/master")
def _libretro_tree_url(system="games"):
return ("https://api.github.com/repos/libretro-thumbnails/"
f"{_platform(system)['libretro_repo']}/git/trees/master?recursive=1")
_SS_API = "https://api.screenscraper.fr/api2/jeuInfos.php"
_SS_PSP_ID = "61" # Sony PSP
_IGDB_TOKEN_URL = "https://id.twitch.tv/oauth2/token"
_IGDB_API = "https://api.igdb.com/v4/games"
_IGDB_IMAGE = "https://images.igdb.com/igdb/image/upload/"
_IGDB_PSP_ID = 38 # Sony PSP
_TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie"
_TMDB_IMAGE = "https://image.tmdb.org/t/p/"
_GOOGLE_CSE_API = "https://www.googleapis.com/customsearch/v1"
@@ -52,7 +67,7 @@ _igdb_token_state = {"token": "", "expires": 0.0} # IGDB OAuth2 token cache
_igdb_lock = threading.Lock()
_igdb_last = 0.0 # monotonic clock of the last IGDB API call
_index_state = {"ts": 0.0, "names": {}, "norm": {}}
_index_state = {} # system -> {"ts", "names", "norm"}
_INDEX_TTL = 86400.0 # refresh the libretro file index once a day
_neg_cache = {} # cache key -> time.time() of the last failed fetch
@@ -187,9 +202,9 @@ def ensure_cover_bundles(server_url=""):
return downloaded
def cover_url(name, kind="Named_Boxarts"):
def cover_url(name, kind="Named_Boxarts", system="games"):
"""Public libretro URL of an image for a catalog name."""
return f"{_LIBRETRO_BASE}/{kind}/{urllib.parse.quote(name)}.png"
return f"{_libretro_base(system)}/{kind}/{urllib.parse.quote(name)}.png"
def cover_path(release_id, system):
@@ -261,7 +276,7 @@ def _candidate_names(name):
yield s
def _fetch_png(name, kind, timeout):
def _fetch_png(name, kind, timeout, system="games"):
"""Fetch a real image from libretro, following symlink text bodies.
GitHub serves a repo symlink as a tiny text body containing the target
@@ -269,7 +284,7 @@ def _fetch_png(name, kind, timeout):
"""
target = name
for _ in range(_MAX_HOPS + 1):
data = _http_get(cover_url(target, kind), timeout)
data = _http_get(cover_url(target, kind, system), timeout)
if data is None:
return None
if _is_image(data):
@@ -284,26 +299,29 @@ def _fetch_png(name, kind, timeout):
# --------------------------------------------------------------------------- #
# libretro filename index + fuzzy matching
# --------------------------------------------------------------------------- #
def _index_path():
return config.COVERS_DIR / "libretro-index.json"
def _index_path(system="games"):
if system == "games":
return config.COVERS_DIR / "libretro-index.json"
return config.COVERS_DIR / f"libretro-index-{system}.json"
def _load_index(force=False):
st = _index_state
def _load_index(force=False, system="games"):
st = _index_state.get(system) or {"ts": 0.0, "names": {}, "norm": {}}
_index_state[system] = st
if not force and st["names"] and time.time() - st["ts"] < _INDEX_TTL:
return st["names"]
path = _index_path()
path = _index_path(system)
if not force and path.exists():
try:
# Load the on-disk index so it doubles as a fallback if the network
# is down, but only return it immediately while it's still fresh.
names = json.loads(path.read_text(encoding="utf-8"))
_set_index(names)
_set_index(names, system)
if time.time() - path.stat().st_mtime < _INDEX_TTL:
return names
except Exception:
pass
data = _http_get(_TREE_URL, timeout=30)
data = _http_get(_libretro_tree_url(system), timeout=30)
if data:
try:
tree = json.loads(data).get("tree", [])
@@ -314,7 +332,7 @@ def _load_index(force=False):
prefix = kind + "/"
if p.startswith(prefix) and p.endswith(".png"):
names[kind].append(p[len(prefix):-4])
_set_index(names)
_set_index(names, system)
try:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(names), encoding="utf-8")
@@ -326,9 +344,10 @@ def _load_index(force=False):
return st["names"] or {kind: [] for kind in _LIBRETRO_KINDS}
def _set_index(names):
def _set_index(names, system="games"):
"""Store the raw filename index plus a pre-normalized copy for fast lookup."""
st = _index_state
st = _index_state.get(system) or {"ts": 0.0, "names": {}, "norm": {}}
_index_state[system] = st
st["names"] = names
st["norm"] = {k: [(_norm(n), n) for n in v] for k, v in names.items()}
st["ts"] = time.time()
@@ -344,11 +363,12 @@ def _plain(s):
return re.sub(r"\s*\([^()]*\)", "", s).strip()
def _index_best(name, kind):
def _index_best(name, kind, system="games"):
"""Return the best-matching repo filename for `name`, or None."""
st = _index_state
st = _index_state.get(system) or {"ts": 0.0, "names": {}, "norm": {}}
if not st["norm"]:
_load_index()
_load_index(system=system)
st = _index_state.get(system) or st
pairs = st["norm"].get(kind, [])
db = _norm(name)
if not pairs or len(db) < 5:
@@ -385,24 +405,25 @@ def _index_best(name, kind):
return best
def _fetch_libretro(name, timeout, indexed_only=False):
# 0. Shipped offline bundle first (fast, no network).
data = fetch_libretro_local(name)
if data is not None:
return data
def _fetch_libretro(name, timeout, system="games", indexed_only=False):
# 0. Shipped offline bundle first (fast, no network) — PSP only for now.
if system == "games":
data = fetch_libretro_local(name)
if data is not None:
return data
if not indexed_only:
# 1. Cheap: exact + progressively stripped names (no index needed).
for kind in _LIBRETRO_KINDS:
for cand in _candidate_names(name):
data = _fetch_png(cand, kind, timeout)
data = _fetch_png(cand, kind, timeout, system)
if data is not None:
return data
# 2. Fuzzy/exact via the precomputed index (covers exact names too, so the
# bulk loader skips the wasteful exact pass entirely).
for kind in _LIBRETRO_KINDS:
best = _index_best(name, kind)
best = _index_best(name, kind, system)
if best:
data = _fetch_png(best, kind, timeout)
data = _fetch_png(best, kind, timeout, system)
if data is not None:
return data
return None
@@ -577,7 +598,7 @@ def _screenscraper_box_urls(jeu):
return urls
def _fetch_screenscraper(name, timeout):
def _fetch_screenscraper(name, timeout, system="games"):
creds = sources.load()
devid = creds.get("screenscraper_devid")
devpassword = creds.get("screenscraper_devpassword")
@@ -588,7 +609,7 @@ def _fetch_screenscraper(name, timeout):
"devpassword": devpassword,
"softname": creds.get("screenscraper_softname") or "psp-preserve",
"output": "json",
"systemeid": _SS_PSP_ID,
"systemeid": _platform(system)["ss_id"],
"romtype": "rom",
"romnom": name,
}
@@ -834,19 +855,20 @@ def fetch_cover(name, release_id, system, timeout=10, quick=False):
if data is not None:
log.info("cover %r <- manual", name)
elif system != "videos":
# 2. Games: libretro (full box art) first, then the shipped OpenVGDB
# bundle (offline), then ScreenScraper (full box art, non-quick),
# then Wikipedia (keyless, non-quick), then IGDB last — IGDB only
# stores cropped 3:4 key art.
data = _fetch_libretro(name, t, indexed_only=quick)
# 2. Games/PS1: libretro (full box art) first, then the shipped
# OpenVGDB bundle (PSP only, offline), then ScreenScraper (non-quick),
# then Wikipedia (keyless, non-quick), then IGDB last.
data = _fetch_libretro(name, t, system=system, indexed_only=quick)
if data is not None:
log.info("cover %r <- libretro", name)
else:
data = fetch_openvgdb_local(name)
if data is not None:
log.info("cover %r <- openvgdb", name)
elif not quick and time.time() < deadline:
data = _fetch_screenscraper(name, t)
data = None
if system == "games":
data = fetch_openvgdb_local(name)
if data is not None:
log.info("cover %r <- openvgdb", name)
if data is None and not quick and time.time() < deadline:
data = _fetch_screenscraper(name, t, system=system)
if data is not None:
log.info("cover %r <- screenscraper", name)
if data is None and not quick and time.time() < deadline:
@@ -854,11 +876,12 @@ def fetch_cover(name, release_id, system, timeout=10, quick=False):
if data is not None:
log.info("cover %r <- wikipedia", name)
if data is None and time.time() < deadline:
data = _fetch_igdb_cover(name, t)
data = _fetch_igdb_cover(name, t, system=system)
if data is not None:
# IGDB only has cropped 3:4 key art; composite it into the
# full PSP cover template (header bar + artwork).
data = compose_game_cover(data)
if system == "games":
# IGDB only has cropped 3:4 PSP key art; composite it
# into the full PSP cover template (header + artwork).
data = compose_game_cover(data)
log.info("cover %r <- igdb", name)
if data is None:
@@ -1063,8 +1086,8 @@ def _wiki_file_search(query, limit, timeout):
return out
def _search_libretro(query, limit):
names = _load_index().get("Named_Boxarts", [])
def _search_libretro(query, limit, system="games"):
names = _load_index(system=system).get("Named_Boxarts", [])
db = _norm(query)
results = []
if not db:
@@ -1090,8 +1113,8 @@ def _search_libretro(query, limit):
results.append({
"title": n,
"source": "libretro",
"thumb_url": cover_url(n),
"image_url": cover_url(n),
"thumb_url": cover_url(n, system=system),
"image_url": cover_url(n, system=system),
})
return results
@@ -1141,9 +1164,9 @@ def _search_openvgdb(query, limit):
_META_TTL = 86400.0
def get_game_metadata(title, timeout=10):
"""Return game metadata from IGDB."""
return get_game_metadata_igdb(title, timeout)
def get_game_metadata(title, timeout=10, system="games"):
"""Return game metadata from IGDB (system picks the platform)."""
return get_game_metadata_igdb(title, timeout, system)
# --------------------------------------------------------------------------- #
@@ -1268,28 +1291,30 @@ def _igdb_game_meta(g):
}
def _igdb_search(query, limit=20, timeout=10):
def _igdb_search(query, limit=20, timeout=10, system="games"):
cid, _ = _igdb_creds()
if not cid:
return []
term = plain_title(query) or query
platform_id = _platform(system)["igdb_id"]
q = (
f'search "{term}"; '
"fields name, cover.image_id, first_release_date, summary, genres.name, "
"involved_companies.company.name, involved_companies.developer, involved_companies.publisher, "
"rating, videos.video_id; "
f"where platforms = ({_IGDB_PSP_ID}); "
f"where platforms = ({platform_id}); "
f"limit {max(limit, 1)};"
)
return _igdb_post(q, timeout) or []
def _fetch_igdb_cover(name, timeout):
def _fetch_igdb_cover(name, timeout, system="games"):
cid, _ = _igdb_creds()
if not cid:
return None
term = plain_title(name) or name
q = f'search "{term}"; fields cover.image_id; where platforms = ({_IGDB_PSP_ID}); limit 10;'
platform_id = _platform(system)["igdb_id"]
q = f'search "{term}"; fields cover.image_id; where platforms = ({platform_id}); limit 10;'
games = _igdb_post(q, timeout) or []
tried = set()
for g in games:
@@ -1303,9 +1328,9 @@ def _fetch_igdb_cover(name, timeout):
return None
def search_games_igdb(query, limit=20, timeout=10):
def search_games_igdb(query, limit=20, timeout=10, system="games"):
out = []
for g in _igdb_search(query, limit, timeout):
for g in _igdb_search(query, limit, timeout, system):
cover = (g.get("cover") or {}).get("image_id") or ""
out.append({
"id": g.get("id"),
@@ -1320,8 +1345,8 @@ def search_games_igdb(query, limit=20, timeout=10):
return out[:limit]
def get_game_metadata_igdb(title, timeout=10):
games = _igdb_search(title, limit=10, timeout=timeout)
def get_game_metadata_igdb(title, timeout=10, system="games"):
games = _igdb_search(title, limit=10, timeout=timeout, system=system)
if not games:
return None
games.sort(key=lambda g: -_title_score(title, g.get("name") or ""))
@@ -1545,8 +1570,8 @@ def movie_metadata_from_wikipedia(page_title, timeout=10):
def search_metadata_candidates(system, query, limit=20, timeout=10):
"""Search candidates the user can link an item to.
Games: IGDB (rich metadata) first, then libretro box art and OpenVGDB
(GameFAQs) box art so covers can be scraped even without an IGDB key.
Games/PS1: IGDB (rich metadata) first, then libretro box art (and the
PSP-only OpenVGDB bundle) so covers can be scraped even without an IGDB key.
"""
if system == "videos":
return search_movies(query, limit, timeout)
@@ -1559,13 +1584,15 @@ def search_metadata_candidates(system, query, limit=20, timeout=10):
seen.add(key)
results.append(r)
for r in search_games_igdb(query, limit, timeout):
# Cap IGDB so libretro/OpenVGDB cover picks still fit in the list.
for r in search_games_igdb(query, max(1, min(limit, 6)), timeout, system):
add(r)
for r in _search_libretro(query, limit):
for r in _search_libretro(query, limit, system):
r.setdefault("id", None)
add(r)
for r in _search_openvgdb(query, limit):
add(r)
if system == "games":
for r in _search_openvgdb(query, limit):
add(r)
return results[:limit]
@@ -1596,7 +1623,7 @@ def link_metadata(system, candidate, timeout=10):
# user still gets synopsis/genre/etc. when a key is configured.
title = candidate.get("title") or ""
if title:
meta = get_game_metadata_igdb(title, timeout)
meta = get_game_metadata_igdb(title, timeout, system)
if meta:
return meta
return _empty_meta(title)