matcher: token pre-filter for fuzzy name matching (6x faster startup)

The fuzzy fallback ran difflib against every catalog release (~11k) for each
unmatched disc, dominating library load (2.7s) and startup scans. Pre-filter
candidates to those sharing a significant (>=3 char) word with the disc
filename or PARAM.SFO title before the expensive comparison, and cache the
pre-computed word sets. Library load: 2.7s -> 0.44s; scan: ~0.5s.
This commit is contained in:
2026-09-07 18:23:28 -04:00
parent e179bf2f2e
commit abd201396c
+24 -7
View File
@@ -38,7 +38,11 @@ def clear_cache():
def _all_release_candidates(conn): def _all_release_candidates(conn):
"""Catalog releases + supplement entries with pre-normalized names.""" """Catalog releases + supplement entries with pre-normalized names.
Each entry is ``(normalized_name, significant_word_set, candidate)`` so the
fuzzy step can cheaply skip candidates that share no significant word.
"""
key = id(conn) key = id(conn)
cands = _releases_cache.get(key) cands = _releases_cache.get(key)
if cands is not None: if cands is not None:
@@ -53,7 +57,11 @@ def _all_release_candidates(conn):
) )
] ]
cands.extend(_supplement_release(e) for e in _load_supplement()) cands.extend(_supplement_release(e) for e in _load_supplement())
cands = [(_normalize(c["name"]), c) for c in cands] cands = [
(norm, frozenset(w for w in norm.split() if len(w) >= 3), c)
for c in cands
if (norm := _normalize(c["name"] or ""))
]
_releases_cache[key] = cands _releases_cache[key] = cands
return cands return cands
@@ -169,12 +177,21 @@ def match(conn, record):
disk_norm = _normalize(record["name"]) disk_norm = _normalize(record["name"])
title_norm = _normalize(record.get("title") or "") title_norm = _normalize(record.get("title") or "")
# Cheap pre-filter: only run the expensive difflib comparison against
# candidates sharing at least one significant (>=3 char) word with the
# disc's filename or PARAM.SFO title.
target_words = frozenset(w for w in disk_norm.split() if len(w) >= 3) | \
frozenset(w for w in title_norm.split() if len(w) >= 3)
if not target_words:
return "unknown", None
scored = [] scored = []
for cand_norm, cand in _all_release_candidates(conn): for cand_norm, cand_words, cand in _all_release_candidates(conn):
ratio = max( if not (target_words & cand_words):
SequenceMatcher(None, disk_norm, cand_norm).ratio(), continue
SequenceMatcher(None, title_norm, cand_norm).ratio(), ratio = SequenceMatcher(None, disk_norm, cand_norm).ratio()
) if title_norm:
ratio = max(ratio, SequenceMatcher(None, title_norm, cand_norm).ratio())
scored.append((ratio, cand)) scored.append((ratio, cand))
best_ratio = max((r for r, _ in scored), default=0.0) best_ratio = max((r for r, _ in scored), default=0.0)
if best_ratio < 0.9: if best_ratio < 0.9: