matcher: token pre-filter for fuzzy name matching (6x faster startup)
The fuzzy fallback ran difflib against every catalog release (~11k) for each unmatched disc, dominating library load (2.7s) and startup scans. Pre-filter candidates to those sharing a significant (>=3 char) word with the disc filename or PARAM.SFO title before the expensive comparison, and cache the pre-computed word sets. Library load: 2.7s -> 0.44s; scan: ~0.5s.
This commit is contained in:
+24
-7
@@ -38,7 +38,11 @@ def clear_cache():
|
||||
|
||||
|
||||
def _all_release_candidates(conn):
|
||||
"""Catalog releases + supplement entries with pre-normalized names."""
|
||||
"""Catalog releases + supplement entries with pre-normalized names.
|
||||
|
||||
Each entry is ``(normalized_name, significant_word_set, candidate)`` so the
|
||||
fuzzy step can cheaply skip candidates that share no significant word.
|
||||
"""
|
||||
key = id(conn)
|
||||
cands = _releases_cache.get(key)
|
||||
if cands is not None:
|
||||
@@ -53,7 +57,11 @@ def _all_release_candidates(conn):
|
||||
)
|
||||
]
|
||||
cands.extend(_supplement_release(e) for e in _load_supplement())
|
||||
cands = [(_normalize(c["name"]), c) for c in cands]
|
||||
cands = [
|
||||
(norm, frozenset(w for w in norm.split() if len(w) >= 3), c)
|
||||
for c in cands
|
||||
if (norm := _normalize(c["name"] or ""))
|
||||
]
|
||||
_releases_cache[key] = cands
|
||||
return cands
|
||||
|
||||
@@ -169,12 +177,21 @@ def match(conn, record):
|
||||
disk_norm = _normalize(record["name"])
|
||||
title_norm = _normalize(record.get("title") or "")
|
||||
|
||||
# Cheap pre-filter: only run the expensive difflib comparison against
|
||||
# candidates sharing at least one significant (>=3 char) word with the
|
||||
# disc's filename or PARAM.SFO title.
|
||||
target_words = frozenset(w for w in disk_norm.split() if len(w) >= 3) | \
|
||||
frozenset(w for w in title_norm.split() if len(w) >= 3)
|
||||
if not target_words:
|
||||
return "unknown", None
|
||||
|
||||
scored = []
|
||||
for cand_norm, cand in _all_release_candidates(conn):
|
||||
ratio = max(
|
||||
SequenceMatcher(None, disk_norm, cand_norm).ratio(),
|
||||
SequenceMatcher(None, title_norm, cand_norm).ratio(),
|
||||
)
|
||||
for cand_norm, cand_words, cand in _all_release_candidates(conn):
|
||||
if not (target_words & cand_words):
|
||||
continue
|
||||
ratio = SequenceMatcher(None, disk_norm, cand_norm).ratio()
|
||||
if title_norm:
|
||||
ratio = max(ratio, SequenceMatcher(None, title_norm, cand_norm).ratio())
|
||||
scored.append((ratio, cand))
|
||||
best_ratio = max((r for r, _ in scored), default=0.0)
|
||||
if best_ratio < 0.9:
|
||||
|
||||
Reference in New Issue
Block a user