forked from hans/Nominatim
keep break indicators [:-] during normalisation
All punctuation will be converted to '-'. Soft breaks : may be added by preprocessors. The break signs are only used during query analysis and are ignored during import token analysis.
This commit is contained in:
@@ -133,7 +133,7 @@ class ForwardGeocoder:
|
||||
"""
|
||||
assert self.query_analyzer is not None
|
||||
qwords = [word for phrase in query.source
|
||||
for word in re.split('[, ]+', phrase.text) if word]
|
||||
for word in re.split('[-,: ]+', phrase.text) if word]
|
||||
if not qwords:
|
||||
return
|
||||
|
||||
@@ -146,7 +146,7 @@ class ForwardGeocoder:
|
||||
distance = 0.0
|
||||
norm = self.query_analyzer.normalize_text(' '.join((result.display_name,
|
||||
result.country_code or '')))
|
||||
words = set((w for w in norm.split(' ') if w))
|
||||
words = set((w for w in re.split('[-,: ]+', norm) if w))
|
||||
if not words:
|
||||
continue
|
||||
for qword in qwords:
|
||||
|
||||
@@ -25,6 +25,8 @@ class ICUTokenAnalysis:
|
||||
|
||||
def __init__(self, norm_rules: str, trans_rules: str,
|
||||
analysis_rules: Mapping[Optional[str], 'TokenAnalyzerRule']):
|
||||
# additional break signs are not relevant during name analysis
|
||||
norm_rules += ";[[:Space:][-:]]+ > ' ';"
|
||||
self.normalizer = Transliterator.createFromRules("icu_normalization",
|
||||
norm_rules)
|
||||
trans_rules += ";[:Space:]+ > ' '"
|
||||
|
||||
Reference in New Issue
Block a user