forked from hans/Nominatim
97a10ec218
Adds a tagger for names by language so that the analyzer of that language is used. Thus variants are now only applied to names in the specific language and only tag name tags, no longer to reference-like tags.
105 lines
3.9 KiB
Python
105 lines
3.9 KiB
Python
"""
|
|
Tests for import name normalisation and variant generation.
|
|
"""
|
|
from textwrap import dedent
|
|
|
|
import pytest
|
|
|
|
from nominatim.tokenizer.icu_rule_loader import ICURuleLoader
|
|
|
|
from nominatim.errors import UsageError
|
|
|
|
@pytest.fixture
|
|
def cfgfile(def_config, tmp_path):
|
|
project_dir = tmp_path / 'project_dir'
|
|
project_dir.mkdir()
|
|
def_config.project_dir = project_dir
|
|
|
|
def _create_config(*variants, **kwargs):
|
|
content = dedent("""\
|
|
normalization:
|
|
- ":: NFD ()"
|
|
- "'🜳' > ' '"
|
|
- "[[:Nonspacing Mark:] [:Cf:]] >"
|
|
- ":: lower ()"
|
|
- "[[:Punctuation:][:Space:]]+ > ' '"
|
|
- ":: NFC ()"
|
|
transliteration:
|
|
- ":: Latin ()"
|
|
- "'🜵' > ' '"
|
|
""")
|
|
content += "token-analysis:\n - analyzer: generic\n variants:\n - words:\n"
|
|
content += '\n'.join((" - " + s for s in variants)) + '\n'
|
|
for k, v in kwargs:
|
|
content += " {}: {}\n".format(k, v)
|
|
(project_dir / 'icu_tokenizer.yaml').write_text(content)
|
|
|
|
return def_config
|
|
|
|
return _create_config
|
|
|
|
|
|
def get_normalized_variants(proc, name):
|
|
return proc.analysis[None].get_variants_ascii(proc.normalizer.transliterate(name).strip())
|
|
|
|
|
|
def test_variants_empty(cfgfile):
|
|
config = cfgfile('saint -> 🜵', 'street -> st')
|
|
|
|
proc = ICURuleLoader(config).make_token_analysis()
|
|
|
|
assert get_normalized_variants(proc, '🜵') == []
|
|
assert get_normalized_variants(proc, '🜳') == []
|
|
assert get_normalized_variants(proc, 'saint') == ['saint']
|
|
|
|
|
|
VARIANT_TESTS = [
|
|
(('~strasse,~straße -> str', '~weg => weg'), "hallo", {'hallo'}),
|
|
(('weg => wg',), "holzweg", {'holzweg'}),
|
|
(('weg -> wg',), "holzweg", {'holzweg'}),
|
|
(('~weg => weg',), "holzweg", {'holz weg', 'holzweg'}),
|
|
(('~weg -> weg',), "holzweg", {'holz weg', 'holzweg'}),
|
|
(('~weg => w',), "holzweg", {'holz w', 'holzw'}),
|
|
(('~weg -> w',), "holzweg", {'holz weg', 'holzweg', 'holz w', 'holzw'}),
|
|
(('~weg => weg',), "Meier Weg", {'meier weg', 'meierweg'}),
|
|
(('~weg -> weg',), "Meier Weg", {'meier weg', 'meierweg'}),
|
|
(('~weg => w',), "Meier Weg", {'meier w', 'meierw'}),
|
|
(('~weg -> w',), "Meier Weg", {'meier weg', 'meierweg', 'meier w', 'meierw'}),
|
|
(('weg => wg',), "Meier Weg", {'meier wg'}),
|
|
(('weg -> wg',), "Meier Weg", {'meier weg', 'meier wg'}),
|
|
(('~strasse,~straße -> str', '~weg => weg'), "Bauwegstraße",
|
|
{'bauweg straße', 'bauweg str', 'bauwegstraße', 'bauwegstr'}),
|
|
(('am => a', 'bach => b'), "am bach", {'a b'}),
|
|
(('am => a', '~bach => b'), "am bach", {'a b'}),
|
|
(('am -> a', '~bach -> b'), "am bach", {'am bach', 'a bach', 'am b', 'a b'}),
|
|
(('am -> a', '~bach -> b'), "ambach", {'ambach', 'am bach', 'amb', 'am b'}),
|
|
(('saint -> s,st', 'street -> st'), "Saint Johns Street",
|
|
{'saint johns street', 's johns street', 'st johns street',
|
|
'saint johns st', 's johns st', 'st johns st'}),
|
|
(('river$ -> r',), "River Bend Road", {'river bend road'}),
|
|
(('river$ -> r',), "Bent River", {'bent river', 'bent r'}),
|
|
(('^north => n',), "North 2nd Street", {'n 2nd street'}),
|
|
(('^north => n',), "Airport North", {'airport north'}),
|
|
(('am -> a',), "am am am am am am am am", {'am am am am am am am am'}),
|
|
(('am => a',), "am am am am am am am am", {'a a a a a a a a'})
|
|
]
|
|
|
|
@pytest.mark.parametrize("rules,name,variants", VARIANT_TESTS)
|
|
def test_variants(cfgfile, rules, name, variants):
|
|
config = cfgfile(*rules)
|
|
proc = ICURuleLoader(config).make_token_analysis()
|
|
|
|
result = get_normalized_variants(proc, name)
|
|
|
|
assert len(result) == len(set(result))
|
|
assert set(get_normalized_variants(proc, name)) == variants
|
|
|
|
|
|
def test_search_normalized(cfgfile):
|
|
config = cfgfile('~street => s,st', 'master => mstr')
|
|
proc = ICURuleLoader(config).make_token_analysis()
|
|
|
|
assert proc.search.transliterate('Master Street').strip() == 'master street'
|
|
assert proc.search.transliterate('Earnes St').strip() == 'earnes st'
|
|
assert proc.search.transliterate('Nostreet').strip() == 'nostreet'
|