UMD videos: add Wikipedia + Google image cover sources to scrape
This commit is contained in:
+96
-20
@@ -36,6 +36,7 @@ _IGDB_IMAGE = "https://images.igdb.com/igdb/image/upload/"
|
||||
_IGDB_PSP_ID = 38 # Sony PSP
|
||||
_TMDB_SEARCH = "https://api.themoviedb.org/3/search/movie"
|
||||
_TMDB_IMAGE = "https://image.tmdb.org/t/p/"
|
||||
_GOOGLE_CSE_API = "https://www.googleapis.com/customsearch/v1"
|
||||
_MANUAL_DIR = config.DATA_DIR / "manual_covers"
|
||||
_OVGDB_ZIP = config.DATA_DIR / "covers" / "openvgdb-covers.zip"
|
||||
_LIBRETRO_ZIP = config.DATA_DIR / "covers" / "libretro-covers.zip"
|
||||
@@ -945,6 +946,46 @@ def _search_wikipedia(query, limit=15, timeout=10):
|
||||
return results
|
||||
|
||||
|
||||
def _search_google_images(query, limit=15, timeout=10):
|
||||
"""Return [{title, source, thumb_url, image_url}] from Google Images via the
|
||||
Custom Search JSON API. Requires google_api_key + google_cx; keyless -> [].
|
||||
"""
|
||||
key = sources.load().get("google_api_key")
|
||||
cx = sources.load().get("google_cx")
|
||||
if not key or not cx:
|
||||
return []
|
||||
url = _GOOGLE_CSE_API + "?" + urllib.parse.urlencode({
|
||||
"key": key,
|
||||
"cx": cx,
|
||||
"q": query,
|
||||
"searchType": "image",
|
||||
"num": min(max(limit, 1), 10),
|
||||
"imgSize": "xlarge",
|
||||
"safe": "off",
|
||||
})
|
||||
data = _http_get(url, timeout=timeout)
|
||||
if not data:
|
||||
return []
|
||||
try:
|
||||
items = json.loads(data).get("items") or []
|
||||
except Exception: # noqa: BLE001
|
||||
return []
|
||||
out = []
|
||||
for it in items:
|
||||
link = it.get("link") or ""
|
||||
thumb = ((it.get("image") or {}).get("thumbnailLink")) or link
|
||||
if not link:
|
||||
continue
|
||||
out.append({
|
||||
"id": None,
|
||||
"title": it.get("title") or query,
|
||||
"thumb_url": thumb,
|
||||
"image_url": link,
|
||||
"source": "google",
|
||||
})
|
||||
return out[:limit]
|
||||
|
||||
|
||||
def _fetch_wikipedia_cover(name, timeout):
|
||||
"""Return a game cover image from Wikipedia's File: namespace (keyless)."""
|
||||
base = plain_title(name)
|
||||
@@ -1400,11 +1441,19 @@ def get_movie_metadata(title, timeout=10):
|
||||
# Manual linking (pick the correct movie/game when the auto-match is wrong)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def search_movies(query, limit=20, timeout=10):
|
||||
"""Return [{id, title, year, overview, thumb_url, image_url, source}] for a movie search.
|
||||
|
||||
TheMovieDB first; keyless Wikipedia page titles as a fallback.
|
||||
"""Return movie candidates for a UMD video: TMDB first, then Wikipedia
|
||||
images and Google Images (both keyless-optional). A Wikipedia page-title
|
||||
fallback is added only when nothing else matched (metadata-only linking).
|
||||
"""
|
||||
out = []
|
||||
seen = set()
|
||||
|
||||
def add(r):
|
||||
k = (r.get("source"), _norm(r.get("title") or ""))
|
||||
if k[1] and k not in seen:
|
||||
seen.add(k)
|
||||
out.append(r)
|
||||
|
||||
key = sources.load().get("tmdb_api_key")
|
||||
if key:
|
||||
url = _TMDB_SEARCH + "?" + urllib.parse.urlencode({"api_key": key, "query": query})
|
||||
@@ -1414,9 +1463,10 @@ def search_movies(query, limit=20, timeout=10):
|
||||
results = json.loads(data).get("results") or []
|
||||
except Exception:
|
||||
results = []
|
||||
for r in results:
|
||||
# Cap TMDB results so Wikipedia/Google image picks still fit.
|
||||
for r in results[: max(1, min(limit, 6))]:
|
||||
poster = r.get("poster_path")
|
||||
out.append({
|
||||
add({
|
||||
"id": r.get("id"),
|
||||
"title": r.get("title") or r.get("name") or "",
|
||||
"year": (r.get("release_date") or "")[:4],
|
||||
@@ -1425,6 +1475,19 @@ def search_movies(query, limit=20, timeout=10):
|
||||
"image_url": (_TMDB_IMAGE + "w500" + poster) if poster else "",
|
||||
"source": "tmdb",
|
||||
})
|
||||
|
||||
# Wikipedia image search (keyless): real poster/cover files.
|
||||
for r in _search_wikipedia(query, min(limit, 10), timeout):
|
||||
r["id"] = None
|
||||
r["movie_title"] = query
|
||||
add(r)
|
||||
|
||||
# Google Images (Custom Search JSON API, when a key + cx are configured).
|
||||
for r in _search_google_images(query, min(limit, 10), timeout):
|
||||
r["movie_title"] = query
|
||||
add(r)
|
||||
|
||||
# Last resort: Wikipedia page titles (metadata-only, no image).
|
||||
if not out:
|
||||
for page in _wiki_search(query, limit):
|
||||
out.append({
|
||||
@@ -1434,6 +1497,7 @@ def search_movies(query, limit=20, timeout=10):
|
||||
"overview": "",
|
||||
"thumb_url": "",
|
||||
"image_url": "",
|
||||
"movie_title": page,
|
||||
"source": "wikipedia",
|
||||
})
|
||||
return out[:limit]
|
||||
@@ -1505,11 +1569,29 @@ def search_metadata_candidates(system, query, limit=20, timeout=10):
|
||||
return results[:limit]
|
||||
|
||||
|
||||
def _empty_meta(title=""):
|
||||
return {
|
||||
"title": title,
|
||||
"release_date": "",
|
||||
"overview": "",
|
||||
"genres": [],
|
||||
"rating": None,
|
||||
"youtube": "",
|
||||
}
|
||||
|
||||
|
||||
def link_metadata(system, candidate, timeout=10):
|
||||
"""Fetch full metadata for a search candidate the user picked."""
|
||||
if candidate.get("source") == "wikipedia":
|
||||
return movie_metadata_from_wikipedia(candidate.get("title") or "", timeout)
|
||||
if candidate.get("source") in ("libretro", "openvgdb"):
|
||||
src = candidate.get("source")
|
||||
if src == "wikipedia":
|
||||
title = candidate.get("movie_title") or candidate.get("title") or ""
|
||||
meta = movie_metadata_from_wikipedia(title, timeout) if title else None
|
||||
if meta:
|
||||
return meta
|
||||
if system == "videos":
|
||||
return get_movie_metadata(title, timeout) or _empty_meta(title)
|
||||
return _empty_meta(title)
|
||||
if src in ("libretro", "openvgdb"):
|
||||
# Cover repositories have no metadata backend; try IGDB by title so the
|
||||
# user still gets synopsis/genre/etc. when a key is configured.
|
||||
title = candidate.get("title") or ""
|
||||
@@ -1517,19 +1599,13 @@ def link_metadata(system, candidate, timeout=10):
|
||||
meta = get_game_metadata_igdb(title, timeout)
|
||||
if meta:
|
||||
return meta
|
||||
return {
|
||||
"title": title,
|
||||
"release_date": "",
|
||||
"overview": "",
|
||||
"genres": [],
|
||||
"rating": None,
|
||||
"youtube": "",
|
||||
}
|
||||
return _empty_meta(title)
|
||||
if system == "videos":
|
||||
if candidate.get("id") is not None:
|
||||
if src == "tmdb" and candidate.get("id") is not None:
|
||||
return movie_metadata_by_id(candidate["id"], timeout)
|
||||
return movie_metadata_from_wikipedia(candidate.get("title") or "", timeout)
|
||||
if candidate.get("id") is not None and candidate.get("source") == "igdb":
|
||||
title = candidate.get("movie_title") or candidate.get("title") or ""
|
||||
return get_movie_metadata(title, timeout) or _empty_meta(title)
|
||||
if src == "igdb" and candidate.get("id") is not None:
|
||||
return game_metadata_by_id_igdb(candidate["id"], timeout)
|
||||
return None
|
||||
return _empty_meta(candidate.get("title") or "")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user