keep break indicators [:-] during normalisation

All punctuation will be converted to '-'. Soft breaks : may be
added by preprocessors. The break signs are only used during
query analysis and are ignored during import token analysis.
This commit is contained in:
Sarah Hoffmann
2025-01-07 21:32:32 +01:00
parent d984100e23
commit 86ad9efa8a
3 changed files with 8 additions and 5 deletions

View File

@@ -133,7 +133,7 @@ class ForwardGeocoder:
"""
assert self.query_analyzer is not None
qwords = [word for phrase in query.source
for word in re.split('[, ]+', phrase.text) if word]
for word in re.split('[-,: ]+', phrase.text) if word]
if not qwords:
return
@@ -146,7 +146,7 @@ class ForwardGeocoder:
distance = 0.0
norm = self.query_analyzer.normalize_text(' '.join((result.display_name,
result.country_code or '')))
words = set((w for w in norm.split(' ') if w))
words = set((w for w in re.split('[-,: ]+', norm) if w))
if not words:
continue
for qword in qwords: