From 51847ebfeb204c9fe7e133617304fb702698ae12 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 31 Mar 2025 11:06:21 +0200 Subject: [PATCH 1/3] more agressively reduce expected count for multi-word terms Improves searching of non-latin scripts with forced token spaces. --- src/nominatim_api/search/db_search_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nominatim_api/search/db_search_builder.py b/src/nominatim_api/search/db_search_builder.py index ddfddaa6..fdf9e2f8 100644 --- a/src/nominatim_api/search/db_search_builder.py +++ b/src/nominatim_api/search/db_search_builder.py @@ -208,7 +208,7 @@ class SearchBuilder: addr_partials = [t for r in address for t in self.query.get_partials_list(r)] addr_tokens = list({t.token for t in addr_partials}) - exp_count = min(t.count for t in name_partials.values()) / (2**(len(name_partials) - 1)) + exp_count = min(t.count for t in name_partials.values()) / (3**(len(name_partials) - 1)) if (len(name_partials) > 3 or exp_count < 8000): yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens) From efe65c3e499330fd08422caa3c474720cb8aec78 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 31 Mar 2025 11:12:08 +0200 Subject: [PATCH 2/3] increase allowable address counts --- src/nominatim_api/search/db_search_builder.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nominatim_api/search/db_search_builder.py b/src/nominatim_api/search/db_search_builder.py index fdf9e2f8..1e944958 100644 --- a/src/nominatim_api/search/db_search_builder.py +++ b/src/nominatim_api/search/db_search_builder.py @@ -214,13 +214,13 @@ class SearchBuilder: yield penalty, exp_count, dbf.lookup_by_names(list(name_partials.keys()), addr_tokens) return - addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 30000 + addr_count = min(t.addr_count for t in addr_partials) if addr_partials else 50000 # Partial term to frequent. Try looking up by rare full names first. name_fulls = self.query.get_tokens(name, qmod.TOKEN_WORD) if name_fulls: fulls_count = sum(t.count for t in name_fulls) - if fulls_count < 50000 or addr_count < 30000: + if fulls_count < 50000 or addr_count < 50000: yield penalty, fulls_count / (2**len(addr_tokens)), \ self.get_full_name_ranking(name_fulls, addr_partials, fulls_count > 30000 / max(1, len(addr_tokens))) From f2aa15778f9060b0d90d656f5c59c0fec6953ad4 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 31 Mar 2025 11:15:26 +0200 Subject: [PATCH 3/3] always use lookup when requested Doesn't seem to cause any issues in production. --- src/nominatim_api/search/db_search_builder.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/nominatim_api/search/db_search_builder.py b/src/nominatim_api/search/db_search_builder.py index 1e944958..4dcaf14d 100644 --- a/src/nominatim_api/search/db_search_builder.py +++ b/src/nominatim_api/search/db_search_builder.py @@ -264,16 +264,9 @@ class SearchBuilder: address lookups will use the index, when the occurrences are not too many. """ - # At this point drop unindexed partials from the address. - # This might yield wrong results, nothing we can do about that. if use_lookup: addr_restrict_tokens = [] - addr_lookup_tokens = [] - for t in addr_partials: - if t.addr_count > 20000: - addr_restrict_tokens.append(t.token) - else: - addr_lookup_tokens.append(t.token) + addr_lookup_tokens = [t.token for t in addr_partials] else: addr_restrict_tokens = [t.token for t in addr_partials] addr_lookup_tokens = []