safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

move generation of normalized token form to analyzer

Browse Source 
This gives the analyzer more flexibility in choosing the normalized
form. In particular, an analyzer creating different variants can choose
the variant that will be used as the canonical form.
This commit is contained in:
Sarah Hoffmann
2022-02-15 12:15:18 +01:00
parent 691ec08586
commit 837d44391c
5 changed files with 21 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
test/python/tokenizer/token_analysis/test_generic.py
View File

@@ -32,8 +32,9 @@ def make_analyser(*variants, variant_only=False):
rules['mode'] = 'variant-only'
config = module.configure(rules, DEFAULT_NORMALIZATION)
trans = Transliterator.createFromRules("test_trans", DEFAULT_TRANSLITERATION)
norm = Transliterator.createFromRules("test_norm", DEFAULT_NORMALIZATION)
return module.create(trans, config)
return module.create(norm, trans, config)
def get_normalized_variants(proc, name):
@@ -45,8 +46,9 @@ def test_no_variants():
rules = { 'analyzer': 'generic' }
config = module.configure(rules, DEFAULT_NORMALIZATION)
trans = Transliterator.createFromRules("test_trans", DEFAULT_TRANSLITERATION)
norm = Transliterator.createFromRules("test_norm", DEFAULT_NORMALIZATION)
proc = module.create(trans, config)
proc = module.create(norm, trans, config)
assert get_normalized_variants(proc, '大德!') == ['dà dé']