keep break indicators [:-] during normalisation

All punctuation will be converted to '-'. Soft breaks : may be
added by preprocessors. The break signs are only used during
query analysis and are ignored during import token analysis.
This commit is contained in:
Sarah Hoffmann
2025-01-07 21:32:32 +01:00
parent d984100e23
commit 86ad9efa8a
3 changed files with 8 additions and 5 deletions

View File

@@ -133,7 +133,7 @@ class ForwardGeocoder:
"""
assert self.query_analyzer is not None
qwords = [word for phrase in query.source
for word in re.split('[, ]+', phrase.text) if word]
for word in re.split('[-,: ]+', phrase.text) if word]
if not qwords:
return
@@ -146,7 +146,7 @@ class ForwardGeocoder:
distance = 0.0
norm = self.query_analyzer.normalize_text(' '.join((result.display_name,
result.country_code or '')))
words = set((w for w in norm.split(' ') if w))
words = set((w for w in re.split('[-,: ]+', norm) if w))
if not words:
continue
for qword in qwords:

View File

@@ -25,6 +25,8 @@ class ICUTokenAnalysis:
def __init__(self, norm_rules: str, trans_rules: str,
analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule']):
# additional break signs are not relevant during name analysis
norm_rules += ";[[:Space:][-:]]+ > ' ';"
self.normalizer = Transliterator.createFromRules("icu_normalization",
norm_rules)
trans_rules += ";[:Space:]+ > ' '"