hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-12 05:44:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

move generation of normalized token form to analyzer

Browse Source 
This gives the analyzer more flexibility in choosing the normalized
form. In particular, an analyzer creating different variants can choose
the variant that will be used as the canonical form.
This commit is contained in:
Sarah Hoffmann
2022-02-15 12:15:18 +01:00
parent 691ec08586
commit 837d44391c
5 changed files with 21 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
nominatim/tokenizer/icu_tokenizer.py
View File

@@ -561,7 +561,8 @@ class LegacyICUNameAnalyzer(AbstractAnalyzer):
for name in names:
analyzer_id = name.get_attr('analyzer')
norm_name = self._normalized(name.name)
analyzer = self.token_analysis.analysis[analyzer_id]
norm_name = analyzer.normalize(name.name)
if analyzer_id is None:
token_id = norm_name
else:
@@ -569,7 +570,7 @@ class LegacyICUNameAnalyzer(AbstractAnalyzer):
full, part = self._cache.names.get(token_id, (None, None))
if full is None:
variants = self.token_analysis.analysis[analyzer_id].get_variants_ascii(norm_name)
variants = analyzer.get_variants_ascii(norm_name)
if not variants:
continue