hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-10 12:04:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

limit the number of variants that can be produced

Browse Source
This commit is contained in:
Sarah Hoffmann
2021-07-02 16:42:13 +02:00
parent c32551b4e0
commit 62d5984b1b
2 changed files with 7 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
nominatim/tokenizer/icu_name_processor.py
View File

@@ -105,6 +105,11 @@ class ICUNameProcessor:
partials = [v + done + r partials = [v + done + r
for v, r in itertools.product(partials, repl) for v, r in itertools.product(partials, repl)
if not force_space or r.startswith(' ')] if not force_space or r.startswith(' ')]
if len(partials) > 128:
# If too many variants are produced, they are unlikely
# to be helpful. Only use the original term.
startpos = 0
break
startpos = pos + len(full) startpos = pos + len(full)
if full[-1] == ' ': if full[-1] == ' ':
startpos -= 1 startpos -= 1

2
test/python/test_tokenizer_icu_name_processor.py
View File

@@ -78,6 +78,8 @@ VARIANT_TESTS = [
(('river$ -> r',), "Bent River", {'bent river', 'bent r'}), (('river$ -> r',), "Bent River", {'bent river', 'bent r'}),
(('^north => n',), "North 2nd Street", {'n 2nd street'}), (('^north => n',), "North 2nd Street", {'n 2nd street'}),
(('^north => n',), "Airport North", {'airport north'}), (('^north => n',), "Airport North", {'airport north'}),
(('am -> a',), "am am am am am am am am", {'am am am am am am am am'}),
(('am => a',), "am am am am am am am am", {'a a a a a a a a'})
] ]
@pytest.mark.parametrize("rules,name,variants", VARIANT_TESTS) @pytest.mark.parametrize("rules,name,variants", VARIANT_TESTS)