avoid lookup via partials on frequent words

Drops expensive searches via partials on terms like 'rue de'.

See #2979.
This commit is contained in:
Sarah Hoffmann
2023-07-05 14:07:11 +02:00
parent 3266daa8fd
commit cc45930ef9
2 changed files with 28 additions and 14 deletions

View File

@@ -235,6 +235,8 @@ class SearchBuilder:
yield penalty, sum(t.count for t in rare_names), lookup yield penalty, sum(t.count for t in rare_names), lookup
# To catch remaining results, lookup by name and address # To catch remaining results, lookup by name and address
# We only do this if there is a reasonable number of results expected.
if min(exp_name_count, exp_addr_count) < 10000:
if all(t.is_indexed for t in name_partials): if all(t.is_indexed for t in name_partials):
lookup = [dbf.FieldLookup('name_vector', lookup = [dbf.FieldLookup('name_vector',
[t.token for t in name_partials], 'lookup_all')] [t.token for t in name_partials], 'lookup_all')]

View File

@@ -382,7 +382,7 @@ def test_frequent_partials_in_name_but_not_in_address():
def test_frequent_partials_in_name_and_address(): def test_frequent_partials_in_name_and_address():
searches = make_counted_searches(10000, 1, 10000, 1) searches = make_counted_searches(9999, 1, 9999, 1)
assert len(searches) == 2 assert len(searches) == 2
@@ -393,3 +393,15 @@ def test_frequent_partials_in_name_and_address():
{('name_vector', 'lookup_any'), ('nameaddress_vector', 'restrict')} {('name_vector', 'lookup_any'), ('nameaddress_vector', 'restrict')}
assert set((l.column, l.lookup_type) for l in searches[1].lookups) == \ assert set((l.column, l.lookup_type) for l in searches[1].lookups) == \
{('nameaddress_vector', 'lookup_all'), ('name_vector', 'lookup_all')} {('nameaddress_vector', 'lookup_all'), ('name_vector', 'lookup_all')}
def test_too_frequent_partials_in_name_and_address():
searches = make_counted_searches(10000, 1, 10000, 1)
assert len(searches) == 1
assert all(isinstance(s, dbs.PlaceSearch) for s in searches)
searches.sort(key=lambda s: s.penalty)
assert set((l.column, l.lookup_type) for l in searches[0].lookups) == \
{('name_vector', 'lookup_any'), ('nameaddress_vector', 'restrict')}