matcher: token pre-filter for fuzzy name matching (6x faster startup)
The fuzzy fallback ran difflib against every catalog release (~11k) for each unmatched disc, dominating library load (2.7s) and startup scans. Pre-filter candidates to those sharing a significant (>=3 char) word with the disc filename or PARAM.SFO title before the expensive comparison, and cache the pre-computed word sets. Library load: 2.7s -> 0.44s; scan: ~0.5s.
This commit is contained in:
+24
-7
@@ -38,7 +38,11 @@ def clear_cache():
|
|||||||
|
|
||||||
|
|
||||||
def _all_release_candidates(conn):
|
def _all_release_candidates(conn):
|
||||||
"""Catalog releases + supplement entries with pre-normalized names."""
|
"""Catalog releases + supplement entries with pre-normalized names.
|
||||||
|
|
||||||
|
Each entry is ``(normalized_name, significant_word_set, candidate)`` so the
|
||||||
|
fuzzy step can cheaply skip candidates that share no significant word.
|
||||||
|
"""
|
||||||
key = id(conn)
|
key = id(conn)
|
||||||
cands = _releases_cache.get(key)
|
cands = _releases_cache.get(key)
|
||||||
if cands is not None:
|
if cands is not None:
|
||||||
@@ -53,7 +57,11 @@ def _all_release_candidates(conn):
|
|||||||
)
|
)
|
||||||
]
|
]
|
||||||
cands.extend(_supplement_release(e) for e in _load_supplement())
|
cands.extend(_supplement_release(e) for e in _load_supplement())
|
||||||
cands = [(_normalize(c["name"]), c) for c in cands]
|
cands = [
|
||||||
|
(norm, frozenset(w for w in norm.split() if len(w) >= 3), c)
|
||||||
|
for c in cands
|
||||||
|
if (norm := _normalize(c["name"] or ""))
|
||||||
|
]
|
||||||
_releases_cache[key] = cands
|
_releases_cache[key] = cands
|
||||||
return cands
|
return cands
|
||||||
|
|
||||||
@@ -169,12 +177,21 @@ def match(conn, record):
|
|||||||
disk_norm = _normalize(record["name"])
|
disk_norm = _normalize(record["name"])
|
||||||
title_norm = _normalize(record.get("title") or "")
|
title_norm = _normalize(record.get("title") or "")
|
||||||
|
|
||||||
|
# Cheap pre-filter: only run the expensive difflib comparison against
|
||||||
|
# candidates sharing at least one significant (>=3 char) word with the
|
||||||
|
# disc's filename or PARAM.SFO title.
|
||||||
|
target_words = frozenset(w for w in disk_norm.split() if len(w) >= 3) | \
|
||||||
|
frozenset(w for w in title_norm.split() if len(w) >= 3)
|
||||||
|
if not target_words:
|
||||||
|
return "unknown", None
|
||||||
|
|
||||||
scored = []
|
scored = []
|
||||||
for cand_norm, cand in _all_release_candidates(conn):
|
for cand_norm, cand_words, cand in _all_release_candidates(conn):
|
||||||
ratio = max(
|
if not (target_words & cand_words):
|
||||||
SequenceMatcher(None, disk_norm, cand_norm).ratio(),
|
continue
|
||||||
SequenceMatcher(None, title_norm, cand_norm).ratio(),
|
ratio = SequenceMatcher(None, disk_norm, cand_norm).ratio()
|
||||||
)
|
if title_norm:
|
||||||
|
ratio = max(ratio, SequenceMatcher(None, title_norm, cand_norm).ratio())
|
||||||
scored.append((ratio, cand))
|
scored.append((ratio, cand))
|
||||||
best_ratio = max((r for r, _ in scored), default=0.0)
|
best_ratio = max((r for r, _ in scored), default=0.0)
|
||||||
if best_ratio < 0.9:
|
if best_ratio < 0.9:
|
||||||
|
|||||||
Reference in New Issue
Block a user