UMD videos: add Wikipedia + Google image cover sources to scrape
This commit is contained in:
+96
-20
@@ -36,6 +36,7 @@ _IGDB_IMAGE = "https://images.igdb.com/igdb/image/upload/"
|
|||||||
_IGDB_PSP_ID = 38 # Sony PSP
|
_IGDB_PSP_ID = 38 # Sony PSP
|
||||||
_TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie"
|
_TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie"
|
||||||
_TMDB_IMAGE = "https://image.tmdb.org/t/p/"
|
_TMDB_IMAGE = "https://image.tmdb.org/t/p/"
|
||||||
|
_GOOGLE_CSE_API = "https://www.googleapis.com/customsearch/v1"
|
||||||
_MANUAL_DIR = config.DATA_DIR / "manual_covers"
|
_MANUAL_DIR = config.DATA_DIR / "manual_covers"
|
||||||
_OVGDB_ZIP = config.DATA_DIR / "covers" / "openvgdb-covers.zip"
|
_OVGDB_ZIP = config.DATA_DIR / "covers" / "openvgdb-covers.zip"
|
||||||
_LIBRETRO_ZIP = config.DATA_DIR / "covers" / "libretro-covers.zip"
|
_LIBRETRO_ZIP = config.DATA_DIR / "covers" / "libretro-covers.zip"
|
||||||
@@ -945,6 +946,46 @@ def _search_wikipedia(query, limit=15, timeout=10):
|
|||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def _search_google_images(query, limit=15, timeout=10):
|
||||||
|
"""Return [{title, source, thumb_url, image_url}] from Google Images via the
|
||||||
|
Custom Search JSON API. Requires google_api_key + google_cx; keyless -> [].
|
||||||
|
"""
|
||||||
|
key = sources.load().get("google_api_key")
|
||||||
|
cx = sources.load().get("google_cx")
|
||||||
|
if not key or not cx:
|
||||||
|
return []
|
||||||
|
url = _GOOGLE_CSE_API + "?" + urllib.parse.urlencode({
|
||||||
|
"key": key,
|
||||||
|
"cx": cx,
|
||||||
|
"q": query,
|
||||||
|
"searchType": "image",
|
||||||
|
"num": min(max(limit, 1), 10),
|
||||||
|
"imgSize": "xlarge",
|
||||||
|
"safe": "off",
|
||||||
|
})
|
||||||
|
data = _http_get(url, timeout=timeout)
|
||||||
|
if not data:
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
items = json.loads(data).get("items") or []
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
return []
|
||||||
|
out = []
|
||||||
|
for it in items:
|
||||||
|
link = it.get("link") or ""
|
||||||
|
thumb = ((it.get("image") or {}).get("thumbnailLink")) or link
|
||||||
|
if not link:
|
||||||
|
continue
|
||||||
|
out.append({
|
||||||
|
"id": None,
|
||||||
|
"title": it.get("title") or query,
|
||||||
|
"thumb_url": thumb,
|
||||||
|
"image_url": link,
|
||||||
|
"source": "google",
|
||||||
|
})
|
||||||
|
return out[:limit]
|
||||||
|
|
||||||
|
|
||||||
def _fetch_wikipedia_cover(name, timeout):
|
def _fetch_wikipedia_cover(name, timeout):
|
||||||
"""Return a game cover image from Wikipedia's File: namespace (keyless)."""
|
"""Return a game cover image from Wikipedia's File: namespace (keyless)."""
|
||||||
base = plain_title(name)
|
base = plain_title(name)
|
||||||
@@ -1400,11 +1441,19 @@ def get_movie_metadata(title, timeout=10):
|
|||||||
# Manual linking (pick the correct movie/game when the auto-match is wrong)
|
# Manual linking (pick the correct movie/game when the auto-match is wrong)
|
||||||
# --------------------------------------------------------------------------- #
|
# --------------------------------------------------------------------------- #
|
||||||
def search_movies(query, limit=20, timeout=10):
|
def search_movies(query, limit=20, timeout=10):
|
||||||
"""Return [{id, title, year, overview, thumb_url, image_url, source}] for a movie search.
|
"""Return movie candidates for a UMD video: TMDB first, then Wikipedia
|
||||||
|
images and Google Images (both keyless-optional). A Wikipedia page-title
|
||||||
TheMovieDB first; keyless Wikipedia page titles as a fallback.
|
fallback is added only when nothing else matched (metadata-only linking).
|
||||||
"""
|
"""
|
||||||
out = []
|
out = []
|
||||||
|
seen = set()
|
||||||
|
|
||||||
|
def add(r):
|
||||||
|
k = (r.get("source"), _norm(r.get("title") or ""))
|
||||||
|
if k[1] and k not in seen:
|
||||||
|
seen.add(k)
|
||||||
|
out.append(r)
|
||||||
|
|
||||||
key = sources.load().get("tmdb_api_key")
|
key = sources.load().get("tmdb_api_key")
|
||||||
if key:
|
if key:
|
||||||
url = _TMDB_SEARCH + "?" + urllib.parse.urlencode({"api_key": key, "query": query})
|
url = _TMDB_SEARCH + "?" + urllib.parse.urlencode({"api_key": key, "query": query})
|
||||||
@@ -1414,9 +1463,10 @@ def search_movies(query, limit=20, timeout=10):
|
|||||||
results = json.loads(data).get("results") or []
|
results = json.loads(data).get("results") or []
|
||||||
except Exception:
|
except Exception:
|
||||||
results = []
|
results = []
|
||||||
for r in results:
|
# Cap TMDB results so Wikipedia/Google image picks still fit.
|
||||||
|
for r in results[: max(1, min(limit, 6))]:
|
||||||
poster = r.get("poster_path")
|
poster = r.get("poster_path")
|
||||||
out.append({
|
add({
|
||||||
"id": r.get("id"),
|
"id": r.get("id"),
|
||||||
"title": r.get("title") or r.get("name") or "",
|
"title": r.get("title") or r.get("name") or "",
|
||||||
"year": (r.get("release_date") or "")[:4],
|
"year": (r.get("release_date") or "")[:4],
|
||||||
@@ -1425,6 +1475,19 @@ def search_movies(query, limit=20, timeout=10):
|
|||||||
"image_url": (_TMDB_IMAGE + "w500" + poster) if poster else "",
|
"image_url": (_TMDB_IMAGE + "w500" + poster) if poster else "",
|
||||||
"source": "tmdb",
|
"source": "tmdb",
|
||||||
})
|
})
|
||||||
|
|
||||||
|
# Wikipedia image search (keyless): real poster/cover files.
|
||||||
|
for r in _search_wikipedia(query, min(limit, 10), timeout):
|
||||||
|
r["id"] = None
|
||||||
|
r["movie_title"] = query
|
||||||
|
add(r)
|
||||||
|
|
||||||
|
# Google Images (Custom Search JSON API, when a key + cx are configured).
|
||||||
|
for r in _search_google_images(query, min(limit, 10), timeout):
|
||||||
|
r["movie_title"] = query
|
||||||
|
add(r)
|
||||||
|
|
||||||
|
# Last resort: Wikipedia page titles (metadata-only, no image).
|
||||||
if not out:
|
if not out:
|
||||||
for page in _wiki_search(query, limit):
|
for page in _wiki_search(query, limit):
|
||||||
out.append({
|
out.append({
|
||||||
@@ -1434,6 +1497,7 @@ def search_movies(query, limit=20, timeout=10):
|
|||||||
"overview": "",
|
"overview": "",
|
||||||
"thumb_url": "",
|
"thumb_url": "",
|
||||||
"image_url": "",
|
"image_url": "",
|
||||||
|
"movie_title": page,
|
||||||
"source": "wikipedia",
|
"source": "wikipedia",
|
||||||
})
|
})
|
||||||
return out[:limit]
|
return out[:limit]
|
||||||
@@ -1505,11 +1569,29 @@ def search_metadata_candidates(system, query, limit=20, timeout=10):
|
|||||||
return results[:limit]
|
return results[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _empty_meta(title=""):
|
||||||
|
return {
|
||||||
|
"title": title,
|
||||||
|
"release_date": "",
|
||||||
|
"overview": "",
|
||||||
|
"genres": [],
|
||||||
|
"rating": None,
|
||||||
|
"youtube": "",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def link_metadata(system, candidate, timeout=10):
|
def link_metadata(system, candidate, timeout=10):
|
||||||
"""Fetch full metadata for a search candidate the user picked."""
|
"""Fetch full metadata for a search candidate the user picked."""
|
||||||
if candidate.get("source") == "wikipedia":
|
src = candidate.get("source")
|
||||||
return movie_metadata_from_wikipedia(candidate.get("title") or "", timeout)
|
if src == "wikipedia":
|
||||||
if candidate.get("source") in ("libretro", "openvgdb"):
|
title = candidate.get("movie_title") or candidate.get("title") or ""
|
||||||
|
meta = movie_metadata_from_wikipedia(title, timeout) if title else None
|
||||||
|
if meta:
|
||||||
|
return meta
|
||||||
|
if system == "videos":
|
||||||
|
return get_movie_metadata(title, timeout) or _empty_meta(title)
|
||||||
|
return _empty_meta(title)
|
||||||
|
if src in ("libretro", "openvgdb"):
|
||||||
# Cover repositories have no metadata backend; try IGDB by title so the
|
# Cover repositories have no metadata backend; try IGDB by title so the
|
||||||
# user still gets synopsis/genre/etc. when a key is configured.
|
# user still gets synopsis/genre/etc. when a key is configured.
|
||||||
title = candidate.get("title") or ""
|
title = candidate.get("title") or ""
|
||||||
@@ -1517,19 +1599,13 @@ def link_metadata(system, candidate, timeout=10):
|
|||||||
meta = get_game_metadata_igdb(title, timeout)
|
meta = get_game_metadata_igdb(title, timeout)
|
||||||
if meta:
|
if meta:
|
||||||
return meta
|
return meta
|
||||||
return {
|
return _empty_meta(title)
|
||||||
"title": title,
|
|
||||||
"release_date": "",
|
|
||||||
"overview": "",
|
|
||||||
"genres": [],
|
|
||||||
"rating": None,
|
|
||||||
"youtube": "",
|
|
||||||
}
|
|
||||||
if system == "videos":
|
if system == "videos":
|
||||||
if candidate.get("id") is not None:
|
if src == "tmdb" and candidate.get("id") is not None:
|
||||||
return movie_metadata_by_id(candidate["id"], timeout)
|
return movie_metadata_by_id(candidate["id"], timeout)
|
||||||
return movie_metadata_from_wikipedia(candidate.get("title") or "", timeout)
|
title = candidate.get("movie_title") or candidate.get("title") or ""
|
||||||
if candidate.get("id") is not None and candidate.get("source") == "igdb":
|
return get_movie_metadata(title, timeout) or _empty_meta(title)
|
||||||
|
if src == "igdb" and candidate.get("id") is not None:
|
||||||
return game_metadata_by_id_igdb(candidate["id"], timeout)
|
return game_metadata_by_id_igdb(candidate["id"], timeout)
|
||||||
return None
|
return _empty_meta(candidate.get("title") or "")
|
||||||
|
|
||||||
|
|||||||
@@ -1118,6 +1118,8 @@ class SettingsPage(QWidget):
|
|||||||
|
|
||||||
form.addRow(QLabel("<b>Videos — metadata & posters</b>"))
|
form.addRow(QLabel("<b>Videos — metadata & posters</b>"))
|
||||||
add_field("TheMovieDB API key:", "tmdb_api_key", password=True)
|
add_field("TheMovieDB API key:", "tmdb_api_key", password=True)
|
||||||
|
add_field("Google API key:", "google_api_key", password=True)
|
||||||
|
add_field("Google Search Engine ID (cx):", "google_cx")
|
||||||
|
|
||||||
hint = QLabel(
|
hint = QLabel(
|
||||||
"Manual UMD covers: drop images into "
|
"Manual UMD covers: drop images into "
|
||||||
|
|||||||
@@ -18,6 +18,8 @@ FIELDS = [
|
|||||||
"igdb_client_id",
|
"igdb_client_id",
|
||||||
"igdb_client_secret",
|
"igdb_client_secret",
|
||||||
"tmdb_api_key",
|
"tmdb_api_key",
|
||||||
|
"google_api_key",
|
||||||
|
"google_cx",
|
||||||
"screenscraper_devid",
|
"screenscraper_devid",
|
||||||
"screenscraper_devpassword",
|
"screenscraper_devpassword",
|
||||||
"screenscraper_ssid",
|
"screenscraper_ssid",
|
||||||
@@ -29,6 +31,8 @@ _ENV_MAP = {
|
|||||||
"igdb_client_id": "IGDB_CLIENT_ID",
|
"igdb_client_id": "IGDB_CLIENT_ID",
|
||||||
"igdb_client_secret": "IGDB_CLIENT_SECRET",
|
"igdb_client_secret": "IGDB_CLIENT_SECRET",
|
||||||
"tmdb_api_key": "TMDB_API_KEY",
|
"tmdb_api_key": "TMDB_API_KEY",
|
||||||
|
"google_api_key": "GOOGLE_API_KEY",
|
||||||
|
"google_cx": "GOOGLE_CX",
|
||||||
"screenscraper_devid": "SCREENSCRAPER_DEVID",
|
"screenscraper_devid": "SCREENSCRAPER_DEVID",
|
||||||
"screenscraper_devpassword": "SCREENSCRAPER_DEVPASSWORD",
|
"screenscraper_devpassword": "SCREENSCRAPER_DEVPASSWORD",
|
||||||
"screenscraper_ssid": "SCREENSCRAPER_SSID",
|
"screenscraper_ssid": "SCREENSCRAPER_SSID",
|
||||||
|
|||||||
Reference in New Issue
Block a user