UMD videos: add Wikipedia + Google image cover sources to scrape

This commit is contained in:
2026-09-07 16:20:39 -04:00
parent 7d8b449846
commit 625e2f8718
3 changed files with 102 additions and 20 deletions
+96 -20
View File
@@ -36,6 +36,7 @@ _IGDB_IMAGE = "https://images.igdb.com/igdb/image/upload/"
_IGDB_PSP_ID = 38 # Sony PSP _IGDB_PSP_ID = 38 # Sony PSP
_TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie" _TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie"
_TMDB_IMAGE = "https://image.tmdb.org/t/p/" _TMDB_IMAGE = "https://image.tmdb.org/t/p/"
_GOOGLE_CSE_API = "https://www.googleapis.com/customsearch/v1"
_MANUAL_DIR = config.DATA_DIR / "manual_covers" _MANUAL_DIR = config.DATA_DIR / "manual_covers"
_OVGDB_ZIP = config.DATA_DIR / "covers" / "openvgdb-covers.zip" _OVGDB_ZIP = config.DATA_DIR / "covers" / "openvgdb-covers.zip"
_LIBRETRO_ZIP = config.DATA_DIR / "covers" / "libretro-covers.zip" _LIBRETRO_ZIP = config.DATA_DIR / "covers" / "libretro-covers.zip"
@@ -945,6 +946,46 @@ def _search_wikipedia(query, limit=15, timeout=10):
return results return results
def _search_google_images(query, limit=15, timeout=10):
"""Return [{title, source, thumb_url, image_url}] from Google Images via the
Custom Search JSON API. Requires google_api_key + google_cx; keyless -> [].
"""
key = sources.load().get("google_api_key")
cx = sources.load().get("google_cx")
if not key or not cx:
return []
url = _GOOGLE_CSE_API + "?" + urllib.parse.urlencode({
"key": key,
"cx": cx,
"q": query,
"searchType": "image",
"num": min(max(limit, 1), 10),
"imgSize": "xlarge",
"safe": "off",
})
data = _http_get(url, timeout=timeout)
if not data:
return []
try:
items = json.loads(data).get("items") or []
except Exception: # noqa: BLE001
return []
out = []
for it in items:
link = it.get("link") or ""
thumb = ((it.get("image") or {}).get("thumbnailLink")) or link
if not link:
continue
out.append({
"id": None,
"title": it.get("title") or query,
"thumb_url": thumb,
"image_url": link,
"source": "google",
})
return out[:limit]
def _fetch_wikipedia_cover(name, timeout): def _fetch_wikipedia_cover(name, timeout):
"""Return a game cover image from Wikipedia's File: namespace (keyless).""" """Return a game cover image from Wikipedia's File: namespace (keyless)."""
base = plain_title(name) base = plain_title(name)
@@ -1400,11 +1441,19 @@ def get_movie_metadata(title, timeout=10):
# Manual linking (pick the correct movie/game when the auto-match is wrong) # Manual linking (pick the correct movie/game when the auto-match is wrong)
# --------------------------------------------------------------------------- # # --------------------------------------------------------------------------- #
def search_movies(query, limit=20, timeout=10): def search_movies(query, limit=20, timeout=10):
"""Return [{id, title, year, overview, thumb_url, image_url, source}] for a movie search. """Return movie candidates for a UMD video: TMDB first, then Wikipedia
images and Google Images (both keyless-optional). A Wikipedia page-title
TheMovieDB first; keyless Wikipedia page titles as a fallback. fallback is added only when nothing else matched (metadata-only linking).
""" """
out = [] out = []
seen = set()
def add(r):
k = (r.get("source"), _norm(r.get("title") or ""))
if k[1] and k not in seen:
seen.add(k)
out.append(r)
key = sources.load().get("tmdb_api_key") key = sources.load().get("tmdb_api_key")
if key: if key:
url = _TMDB_SEARCH + "?" + urllib.parse.urlencode({"api_key": key, "query": query}) url = _TMDB_SEARCH + "?" + urllib.parse.urlencode({"api_key": key, "query": query})
@@ -1414,9 +1463,10 @@ def search_movies(query, limit=20, timeout=10):
results = json.loads(data).get("results") or [] results = json.loads(data).get("results") or []
except Exception: except Exception:
results = [] results = []
for r in results: # Cap TMDB results so Wikipedia/Google image picks still fit.
for r in results[: max(1, min(limit, 6))]:
poster = r.get("poster_path") poster = r.get("poster_path")
out.append({ add({
"id": r.get("id"), "id": r.get("id"),
"title": r.get("title") or r.get("name") or "", "title": r.get("title") or r.get("name") or "",
"year": (r.get("release_date") or "")[:4], "year": (r.get("release_date") or "")[:4],
@@ -1425,6 +1475,19 @@ def search_movies(query, limit=20, timeout=10):
"image_url": (_TMDB_IMAGE + "w500" + poster) if poster else "", "image_url": (_TMDB_IMAGE + "w500" + poster) if poster else "",
"source": "tmdb", "source": "tmdb",
}) })
# Wikipedia image search (keyless): real poster/cover files.
for r in _search_wikipedia(query, min(limit, 10), timeout):
r["id"] = None
r["movie_title"] = query
add(r)
# Google Images (Custom Search JSON API, when a key + cx are configured).
for r in _search_google_images(query, min(limit, 10), timeout):
r["movie_title"] = query
add(r)
# Last resort: Wikipedia page titles (metadata-only, no image).
if not out: if not out:
for page in _wiki_search(query, limit): for page in _wiki_search(query, limit):
out.append({ out.append({
@@ -1434,6 +1497,7 @@ def search_movies(query, limit=20, timeout=10):
"overview": "", "overview": "",
"thumb_url": "", "thumb_url": "",
"image_url": "", "image_url": "",
"movie_title": page,
"source": "wikipedia", "source": "wikipedia",
}) })
return out[:limit] return out[:limit]
@@ -1505,11 +1569,29 @@ def search_metadata_candidates(system, query, limit=20, timeout=10):
return results[:limit] return results[:limit]
def _empty_meta(title=""):
return {
"title": title,
"release_date": "",
"overview": "",
"genres": [],
"rating": None,
"youtube": "",
}
def link_metadata(system, candidate, timeout=10): def link_metadata(system, candidate, timeout=10):
"""Fetch full metadata for a search candidate the user picked.""" """Fetch full metadata for a search candidate the user picked."""
if candidate.get("source") == "wikipedia": src = candidate.get("source")
return movie_metadata_from_wikipedia(candidate.get("title") or "", timeout) if src == "wikipedia":
if candidate.get("source") in ("libretro", "openvgdb"): title = candidate.get("movie_title") or candidate.get("title") or ""
meta = movie_metadata_from_wikipedia(title, timeout) if title else None
if meta:
return meta
if system == "videos":
return get_movie_metadata(title, timeout) or _empty_meta(title)
return _empty_meta(title)
if src in ("libretro", "openvgdb"):
# Cover repositories have no metadata backend; try IGDB by title so the # Cover repositories have no metadata backend; try IGDB by title so the
# user still gets synopsis/genre/etc. when a key is configured. # user still gets synopsis/genre/etc. when a key is configured.
title = candidate.get("title") or "" title = candidate.get("title") or ""
@@ -1517,19 +1599,13 @@ def link_metadata(system, candidate, timeout=10):
meta = get_game_metadata_igdb(title, timeout) meta = get_game_metadata_igdb(title, timeout)
if meta: if meta:
return meta return meta
return { return _empty_meta(title)
"title": title,
"release_date": "",
"overview": "",
"genres": [],
"rating": None,
"youtube": "",
}
if system == "videos": if system == "videos":
if candidate.get("id") is not None: if src == "tmdb" and candidate.get("id") is not None:
return movie_metadata_by_id(candidate["id"], timeout) return movie_metadata_by_id(candidate["id"], timeout)
return movie_metadata_from_wikipedia(candidate.get("title") or "", timeout) title = candidate.get("movie_title") or candidate.get("title") or ""
if candidate.get("id") is not None and candidate.get("source") == "igdb": return get_movie_metadata(title, timeout) or _empty_meta(title)
if src == "igdb" and candidate.get("id") is not None:
return game_metadata_by_id_igdb(candidate["id"], timeout) return game_metadata_by_id_igdb(candidate["id"], timeout)
return None return _empty_meta(candidate.get("title") or "")
+2
View File
@@ -1118,6 +1118,8 @@ class SettingsPage(QWidget):
form.addRow(QLabel("<b>Videos — metadata &amp; posters</b>")) form.addRow(QLabel("<b>Videos — metadata &amp; posters</b>"))
add_field("TheMovieDB API key:", "tmdb_api_key", password=True) add_field("TheMovieDB API key:", "tmdb_api_key", password=True)
add_field("Google API key:", "google_api_key", password=True)
add_field("Google Search Engine ID (cx):", "google_cx")
hint = QLabel( hint = QLabel(
"Manual UMD covers: drop images into " "Manual UMD covers: drop images into "
+4
View File
@@ -18,6 +18,8 @@ FIELDS = [
"igdb_client_id", "igdb_client_id",
"igdb_client_secret", "igdb_client_secret",
"tmdb_api_key", "tmdb_api_key",
"google_api_key",
"google_cx",
"screenscraper_devid", "screenscraper_devid",
"screenscraper_devpassword", "screenscraper_devpassword",
"screenscraper_ssid", "screenscraper_ssid",
@@ -29,6 +31,8 @@ _ENV_MAP = {
"igdb_client_id": "IGDB_CLIENT_ID", "igdb_client_id": "IGDB_CLIENT_ID",
"igdb_client_secret": "IGDB_CLIENT_SECRET", "igdb_client_secret": "IGDB_CLIENT_SECRET",
"tmdb_api_key": "TMDB_API_KEY", "tmdb_api_key": "TMDB_API_KEY",
"google_api_key": "GOOGLE_API_KEY",
"google_cx": "GOOGLE_CX",
"screenscraper_devid": "SCREENSCRAPER_DEVID", "screenscraper_devid": "SCREENSCRAPER_DEVID",
"screenscraper_devpassword": "SCREENSCRAPER_DEVPASSWORD", "screenscraper_devpassword": "SCREENSCRAPER_DEVPASSWORD",
"screenscraper_ssid": "SCREENSCRAPER_SSID", "screenscraper_ssid": "SCREENSCRAPER_SSID",