From d0f45155c8560dd17087c023951492c58b933e42 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 13 Jul 2023 16:43:56 +0200 Subject: [PATCH 01/11] fix search for housenumber names The search still included a lookup of housenumbers in children which is wrong. --- nominatim/api/search/db_search_builder.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nominatim/api/search/db_search_builder.py b/nominatim/api/search/db_search_builder.py index 2a3153be..f485de09 100644 --- a/nominatim/api/search/db_search_builder.py +++ b/nominatim/api/search/db_search_builder.py @@ -163,6 +163,7 @@ class SearchBuilder: sdata.lookups = [dbf.FieldLookup('name_vector', [t.token for t in hnrs], 'lookup_any'), dbf.FieldLookup('nameaddress_vector', partial_tokens, 'lookup_all') ] + sdata.housenumbers = dbf.WeightedStrings([], []) yield dbs.PlaceSearch(0.05, sdata, sum(t.count for t in hnrs)) From 8a36ed4f6ff5d16ddfd7fab7dc92b5a409b2cf5b Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 14 Jul 2023 21:03:42 +0200 Subject: [PATCH 02/11] increase threshold for full name searches They still should be preferrred over expensive partial name searches. --- nominatim/api/search/db_search_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nominatim/api/search/db_search_builder.py b/nominatim/api/search/db_search_builder.py index f485de09..794012b0 100644 --- a/nominatim/api/search/db_search_builder.py +++ b/nominatim/api/search/db_search_builder.py @@ -220,7 +220,7 @@ class SearchBuilder: # Partial term to frequent. Try looking up by rare full names first. name_fulls = self.query.get_tokens(name, TokenType.WORD) - rare_names = list(filter(lambda t: t.count < 1000, name_fulls)) + rare_names = list(filter(lambda t: t.count < 10000, name_fulls)) # At this point drop unindexed partials from the address. # This might yield wrong results, nothing we can do about that. if not partials_indexed: From 283db76e458c3159888f01e5837ff8db0bfd9780 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 14 Jul 2023 21:05:13 +0200 Subject: [PATCH 03/11] avoid splitting of first token when a housenumber is present This only covers the case of which is exceedingly rare. --- nominatim/api/search/db_search_builder.py | 2 -- nominatim/api/search/token_assignment.py | 10 ++++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nominatim/api/search/db_search_builder.py b/nominatim/api/search/db_search_builder.py index 794012b0..d18fa964 100644 --- a/nominatim/api/search/db_search_builder.py +++ b/nominatim/api/search/db_search_builder.py @@ -225,9 +225,7 @@ class SearchBuilder: # This might yield wrong results, nothing we can do about that. if not partials_indexed: addr_tokens = [t.token for t in addr_partials if t.is_indexed] - log().var_dump('before', penalty) penalty += 1.2 * sum(t.penalty for t in addr_partials if not t.is_indexed) - log().var_dump('after', penalty) if rare_names: # Any of the full names applies with all of the partials from the address lookup = [dbf.FieldLookup('name_vector', [t.token for t in rare_names], 'lookup_any')] diff --git a/nominatim/api/search/token_assignment.py b/nominatim/api/search/token_assignment.py index 11da2359..f1c2f8e8 100644 --- a/nominatim/api/search/token_assignment.py +++ b/nominatim/api/search/token_assignment.py @@ -309,9 +309,12 @@ class _TokenSequence: first = base.address[0] if (not base.housenumber or first.end >= base.housenumber.start)\ and (not base.qualifier or first.start >= base.qualifier.end): + base_penalty = self.penalty + if base.housenumber and base.housenumber.start > first.start: + base_penalty += 0.25 for i in range(first.start + 1, first.end): name, addr = first.split(i) - penalty = self.penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] + penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] log().comment(f'split first word = name ({i - first.start})') yield dataclasses.replace(base, name=name, penalty=penalty, address=[addr] + base.address[1:]) @@ -321,9 +324,12 @@ class _TokenSequence: last = base.address[-1] if (not base.housenumber or last.start <= base.housenumber.end)\ and (not base.qualifier or last.end <= base.qualifier.start): + base_penalty = self.penalty + if base.housenumber and base.housenumber.start < last.start: + base_penalty += 0.4 for i in range(last.start + 1, last.end): addr, name = last.split(i) - penalty = self.penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] + penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] log().comment(f'split last word = name ({i - last.start})') yield dataclasses.replace(base, name=name, penalty=penalty, address=base.address[:-1] + [addr]) From 8366e4ca8340cd3994c282067804fb047037250f Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Fri, 14 Jul 2023 21:55:07 +0200 Subject: [PATCH 04/11] penalize search with frequent partials Avoid search against frequent partials if we have already looked for the full name equivalents. --- nominatim/api/search/db_search_builder.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nominatim/api/search/db_search_builder.py b/nominatim/api/search/db_search_builder.py index d18fa964..ee06dba5 100644 --- a/nominatim/api/search/db_search_builder.py +++ b/nominatim/api/search/db_search_builder.py @@ -247,8 +247,12 @@ class SearchBuilder: lookup = [dbf.FieldLookup('name_vector', non_rare_names, 'lookup_any')] if addr_tokens: lookup.append(dbf.FieldLookup('nameaddress_vector', addr_tokens, 'lookup_all')) - yield penalty + 0.1 * max(0, 5 - len(name_partials) - len(addr_tokens)),\ - min(exp_name_count, exp_addr_count), lookup + penalty += 0.1 * max(0, 5 - len(name_partials) - len(addr_tokens)) + if len(rare_names) == len(name_fulls): + # if there already was a search for all full tokens, + # avoid this if anything has been found + penalty += 0.25 + yield penalty, min(exp_name_count, exp_addr_count), lookup def get_name_ranking(self, trange: TokenRange) -> dbf.FieldRanking: From 4a00a3c0f5fc3a522fab66808f83ddfe144ccb92 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 15 Jul 2023 10:55:34 +0200 Subject: [PATCH 05/11] penalize name token splitting when phrases are used --- nominatim/api/search/token_assignment.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nominatim/api/search/token_assignment.py b/nominatim/api/search/token_assignment.py index f1c2f8e8..c05c271a 100644 --- a/nominatim/api/search/token_assignment.py +++ b/nominatim/api/search/token_assignment.py @@ -310,7 +310,8 @@ class _TokenSequence: if (not base.housenumber or first.end >= base.housenumber.start)\ and (not base.qualifier or first.start >= base.qualifier.end): base_penalty = self.penalty - if base.housenumber and base.housenumber.start > first.start: + if (base.housenumber and base.housenumber.start > first.start) \ + or len(query.source) > 1: base_penalty += 0.25 for i in range(first.start + 1, first.end): name, addr = first.split(i) @@ -327,6 +328,8 @@ class _TokenSequence: base_penalty = self.penalty if base.housenumber and base.housenumber.start < last.start: base_penalty += 0.4 + if len(query.source) > 1: + base_penalty += 0.25 for i in range(last.start + 1, last.end): addr, name = last.split(i) penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] From 1c189060c2e0970ac61f55c48965b9a8308afc42 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 15 Jul 2023 11:41:31 +0200 Subject: [PATCH 06/11] simplify yield_lookups() function Move creation of field lookups in separate functions to make the code more readable. --- nominatim/api/search/db_search_builder.py | 47 +++++++++-------------- nominatim/api/search/db_search_fields.py | 31 +++++++++++++++ 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/nominatim/api/search/db_search_builder.py b/nominatim/api/search/db_search_builder.py index ee06dba5..72a2dc0b 100644 --- a/nominatim/api/search/db_search_builder.py +++ b/nominatim/api/search/db_search_builder.py @@ -15,7 +15,6 @@ from nominatim.api.search.query import QueryStruct, Token, TokenType, TokenRange from nominatim.api.search.token_assignment import TokenAssignment import nominatim.api.search.db_search_fields as dbf import nominatim.api.search.db_searches as dbs -from nominatim.api.logging import log def wrap_near_search(categories: List[Tuple[str, str]], @@ -188,34 +187,29 @@ class SearchBuilder: be searched for. This takes into account how frequent the terms are and tries to find a lookup that optimizes index use. """ - penalty = 0.0 # extra penalty currently unused - + penalty = 0.0 # extra penalty name_partials = self.query.get_partials_list(name) - exp_name_count = min(t.count for t in name_partials) - addr_partials = [] - for trange in address: - addr_partials.extend(self.query.get_partials_list(trange)) + name_tokens = [t.token for t in name_partials] + + addr_partials = [t for r in address for t in self.query.get_partials_list(r)] addr_tokens = [t.token for t in addr_partials] + partials_indexed = all(t.is_indexed for t in name_partials) \ and all(t.is_indexed for t in addr_partials) + exp_count = min(t.count for t in name_partials) - if (len(name_partials) > 3 or exp_name_count < 1000) and partials_indexed: - # Lookup by name partials, use address partials to restrict results. - lookup = [dbf.FieldLookup('name_vector', - [t.token for t in name_partials], 'lookup_all')] - if addr_tokens: - lookup.append(dbf.FieldLookup('nameaddress_vector', addr_tokens, 'restrict')) - yield penalty, exp_name_count, lookup + if (len(name_partials) > 3 or exp_count < 1000) and partials_indexed: + yield penalty, exp_count, dbf.lookup_by_names(name_tokens, addr_tokens) return - exp_addr_count = min(t.count for t in addr_partials) if addr_partials else exp_name_count - if exp_addr_count < 1000 and partials_indexed: + exp_count = min(exp_count, min(t.count for t in addr_partials)) \ + if addr_partials else exp_count + if exp_count < 1000 and partials_indexed: # Lookup by address partials and restrict results through name terms. # Give this a small penalty because lookups in the address index are # more expensive - yield penalty + exp_addr_count/5000, exp_addr_count,\ - [dbf.FieldLookup('name_vector', [t.token for t in name_partials], 'restrict'), - dbf.FieldLookup('nameaddress_vector', addr_tokens, 'lookup_all')] + yield penalty + exp_count/5000, exp_count,\ + dbf.lookup_by_addr(name_tokens, addr_tokens) return # Partial term to frequent. Try looking up by rare full names first. @@ -228,20 +222,17 @@ class SearchBuilder: penalty += 1.2 * sum(t.penalty for t in addr_partials if not t.is_indexed) if rare_names: # Any of the full names applies with all of the partials from the address - lookup = [dbf.FieldLookup('name_vector', [t.token for t in rare_names], 'lookup_any')] - if addr_tokens: - lookup.append(dbf.FieldLookup('nameaddress_vector', addr_tokens, 'restrict')) - yield penalty, sum(t.count for t in rare_names), lookup + yield penalty, sum(t.count for t in rare_names),\ + dbf.lookup_by_any_name([t.token for t in rare_names], addr_tokens) # To catch remaining results, lookup by name and address # We only do this if there is a reasonable number of results expected. - if min(exp_name_count, exp_addr_count) < 10000: + if exp_count < 10000: if all(t.is_indexed for t in name_partials): - lookup = [dbf.FieldLookup('name_vector', - [t.token for t in name_partials], 'lookup_all')] + lookup = [dbf.FieldLookup('name_vector', name_tokens, 'lookup_all')] else: # we don't have the partials, try with the non-rare names - non_rare_names = [t.token for t in name_fulls if t.count >= 1000] + non_rare_names = [t.token for t in name_fulls if t.count >= 10000] if not non_rare_names: return lookup = [dbf.FieldLookup('name_vector', non_rare_names, 'lookup_any')] @@ -252,7 +243,7 @@ class SearchBuilder: # if there already was a search for all full tokens, # avoid this if anything has been found penalty += 0.25 - yield penalty, min(exp_name_count, exp_addr_count), lookup + yield penalty, exp_count, lookup def get_name_ranking(self, trange: TokenRange) -> dbf.FieldRanking: diff --git a/nominatim/api/search/db_search_fields.py b/nominatim/api/search/db_search_fields.py index 13f1c56e..2b2e3e56 100644 --- a/nominatim/api/search/db_search_fields.py +++ b/nominatim/api/search/db_search_fields.py @@ -211,3 +211,34 @@ class SearchData: self.rankings.append(ranking) else: self.penalty += ranking.default + + +def lookup_by_names(name_tokens: List[int], addr_tokens: List[int]) -> List[FieldLookup]: + """ Create a lookup list where name tokens are looked up via index + and potential address tokens are used to restrict the search further. + """ + lookup = [FieldLookup('name_vector', name_tokens, 'lookup_all')] + if addr_tokens: + lookup.append(FieldLookup('nameaddress_vector', addr_tokens, 'restrict')) + + return lookup + + +def lookup_by_any_name(name_tokens: List[int], addr_tokens: List[int]) -> List[FieldLookup]: + """ Create a lookup list where name tokens are looked up via index + and only one of the name tokens must be present. + Potential address tokens are used to restrict the search further. + """ + lookup = [FieldLookup('name_vector', name_tokens, 'lookup_any')] + if addr_tokens: + lookup.append(FieldLookup('nameaddress_vector', addr_tokens, 'restrict')) + + return lookup + + +def lookup_by_addr(name_tokens: List[int], addr_tokens: List[int]) -> List[FieldLookup]: + """ Create a lookup list where address tokens are looked up via index + and the name tokens are only used to restrict the search further. + """ + return [FieldLookup('name_vector', name_tokens, 'restrict'), + FieldLookup('nameaddress_vector', addr_tokens, 'lookup_all')] From 412bd2ec20575277970beed40e46912d2d7b9f6d Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Sat, 15 Jul 2023 16:31:39 +0200 Subject: [PATCH 07/11] block search queries with too many tokens --- nominatim/api/search/token_assignment.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nominatim/api/search/token_assignment.py b/nominatim/api/search/token_assignment.py index c05c271a..7f75505c 100644 --- a/nominatim/api/search/token_assignment.py +++ b/nominatim/api/search/token_assignment.py @@ -265,6 +265,10 @@ class _TokenSequence: """ base = TokenAssignment.from_ranges(self.seq) + num_addr_tokens = sum(t.end - t.start for t in base.address) + if num_addr_tokens > 50: + return + # Postcode search (postcode-only search is covered in next case) if base.postcode is not None and base.address: if (base.postcode.start == 0 and self.direction != -1)\ From d48ea4f22cd60206613c193dd6aa2dd5e92029bd Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 17 Jul 2023 11:45:35 +0200 Subject: [PATCH 08/11] disallow address searches that start with a postcode These are postcode searches and nothing else. --- nominatim/api/search/token_assignment.py | 30 +++++++++++++++++------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/nominatim/api/search/token_assignment.py b/nominatim/api/search/token_assignment.py index 7f75505c..edddd100 100644 --- a/nominatim/api/search/token_assignment.py +++ b/nominatim/api/search/token_assignment.py @@ -257,6 +257,26 @@ class _TokenSequence: return True + def _get_assignments_postcode(self, base: TokenAssignment, + query_len: int) -> Iterator[TokenAssignment]: + """ Yield possible assignments of Postcode searches with an + address component. + """ + assert base.postcode is not None + + if (base.postcode.start == 0 and self.direction != -1)\ + or (base.postcode.end == query_len and self.direction != 1): + log().comment('postcode search') + #
, should give preference to address search + if base.postcode.start == 0: + penalty = self.penalty + self.direction = -1 # name searches are only possbile backwards + else: + penalty = self.penalty + 0.1 + self.direction = 1 # name searches are only possbile forewards + yield dataclasses.replace(base, penalty=penalty) + + def get_assignments(self, query: qmod.QueryStruct) -> Iterator[TokenAssignment]: """ Yield possible assignments for the current sequence. @@ -271,15 +291,7 @@ class _TokenSequence: # Postcode search (postcode-only search is covered in next case) if base.postcode is not None and base.address: - if (base.postcode.start == 0 and self.direction != -1)\ - or (base.postcode.end == query.num_token_slots() and self.direction != 1): - log().comment('postcode search') - #
, should give preference to address search - if base.postcode.start == 0: - penalty = self.penalty - else: - penalty = self.penalty + 0.1 - yield dataclasses.replace(base, penalty=penalty) + yield from self._get_assignments_postcode(base, query.num_token_slots()) # Postcode or country-only search if not base.address: From 7f9cb4e68d14b0460982bee64af7b2467c2ecd1e Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 17 Jul 2023 15:17:54 +0200 Subject: [PATCH 09/11] split up get_assignment functon in more readable parts --- nominatim/api/search/token_assignment.py | 107 +++++++++++++---------- 1 file changed, 60 insertions(+), 47 deletions(-) diff --git a/nominatim/api/search/token_assignment.py b/nominatim/api/search/token_assignment.py index edddd100..33fb7335 100644 --- a/nominatim/api/search/token_assignment.py +++ b/nominatim/api/search/token_assignment.py @@ -273,10 +273,62 @@ class _TokenSequence: self.direction = -1 # name searches are only possbile backwards else: penalty = self.penalty + 0.1 - self.direction = 1 # name searches are only possbile forewards + self.direction = 1 # name searches are only possbile forwards yield dataclasses.replace(base, penalty=penalty) + def _get_assignments_address_forward(self, base: TokenAssignment, + query: qmod.QueryStruct) -> Iterator[TokenAssignment]: + """ Yield possible assignments of address searches with + left-to-right reading. + """ + first = base.address[0] + + log().comment('first word = name') + yield dataclasses.replace(base, penalty=self.penalty, + name=first, address=base.address[1:]) + + if (not base.housenumber or first.end >= base.housenumber.start)\ + and (not base.qualifier or first.start >= base.qualifier.end): + base_penalty = self.penalty + if (base.housenumber and base.housenumber.start > first.start) \ + or len(query.source) > 1: + base_penalty += 0.25 + for i in range(first.start + 1, first.end): + name, addr = first.split(i) + penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] + log().comment(f'split first word = name ({i - first.start})') + yield dataclasses.replace(base, name=name, penalty=penalty, + address=[addr] + base.address[1:]) + + + def _get_assignments_address_backward(self, base: TokenAssignment, + query: qmod.QueryStruct) -> Iterator[TokenAssignment]: + """ Yield possible assignments of address searches with + right-to-left reading. + """ + last = base.address[-1] + + if self.direction == -1 or len(base.address) > 1: + log().comment('last word = name') + yield dataclasses.replace(base, penalty=self.penalty, + name=last, address=base.address[:-1]) + + if (not base.housenumber or last.start <= base.housenumber.end)\ + and (not base.qualifier or last.end <= base.qualifier.start): + base_penalty = self.penalty + if base.housenumber and base.housenumber.start < last.start: + base_penalty += 0.4 + if len(query.source) > 1: + base_penalty += 0.25 + for i in range(last.start + 1, last.end): + addr, name = last.split(i) + penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] + log().comment(f'split last word = name ({i - last.start})') + yield dataclasses.replace(base, name=name, penalty=penalty, + address=base.address[:-1] + [addr]) + + def get_assignments(self, query: qmod.QueryStruct) -> Iterator[TokenAssignment]: """ Yield possible assignments for the current sequence. @@ -302,58 +354,19 @@ class _TokenSequence: # ,
should give preference to postcode search if base.postcode and base.postcode.start == 0: self.penalty += 0.1 - # Use entire first word as name - if self.direction != -1: - log().comment('first word = name') - yield dataclasses.replace(base, name=base.address[0], - penalty=self.penalty, - address=base.address[1:]) - # Use entire last word as name - if self.direction == -1 or (self.direction == 0 and len(base.address) > 1): - log().comment('last word = name') - yield dataclasses.replace(base, name=base.address[-1], - penalty=self.penalty, - address=base.address[:-1]) + # Right-to-left reading of the address + if self.direction != -1: + yield from self._get_assignments_address_forward(base, query) + + # Left-to-right reading of the address + if self.direction != 1: + yield from self._get_assignments_address_backward(base, query) # variant for special housenumber searches if base.housenumber: yield dataclasses.replace(base, penalty=self.penalty) - # Use beginning of first word as name - if self.direction != -1: - first = base.address[0] - if (not base.housenumber or first.end >= base.housenumber.start)\ - and (not base.qualifier or first.start >= base.qualifier.end): - base_penalty = self.penalty - if (base.housenumber and base.housenumber.start > first.start) \ - or len(query.source) > 1: - base_penalty += 0.25 - for i in range(first.start + 1, first.end): - name, addr = first.split(i) - penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] - log().comment(f'split first word = name ({i - first.start})') - yield dataclasses.replace(base, name=name, penalty=penalty, - address=[addr] + base.address[1:]) - - # Use end of last word as name - if self.direction != 1: - last = base.address[-1] - if (not base.housenumber or last.start <= base.housenumber.end)\ - and (not base.qualifier or last.end <= base.qualifier.start): - base_penalty = self.penalty - if base.housenumber and base.housenumber.start < last.start: - base_penalty += 0.4 - if len(query.source) > 1: - base_penalty += 0.25 - for i in range(last.start + 1, last.end): - addr, name = last.split(i) - penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] - log().comment(f'split last word = name ({i - last.start})') - yield dataclasses.replace(base, name=name, penalty=penalty, - address=base.address[:-1] + [addr]) - - def yield_token_assignments(query: qmod.QueryStruct) -> Iterator[TokenAssignment]: """ Return possible word type assignments to word positions. From 927d2cc824e0437dd2ea6abc4ef47c9b3ed3d0aa Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Mon, 17 Jul 2023 16:25:39 +0200 Subject: [PATCH 10/11] do not split names from typed phrases When phrases are typed, they should only contain exactly one term. --- nominatim/api/search/query.py | 22 ++++++-- nominatim/api/search/token_assignment.py | 69 +++++++++++++++--------- test/python/api/search/test_query.py | 49 +++++++++++++++++ 3 files changed, 112 insertions(+), 28 deletions(-) create mode 100644 test/python/api/search/test_query.py diff --git a/nominatim/api/search/query.py b/nominatim/api/search/query.py index f2b18f87..5d75eb0f 100644 --- a/nominatim/api/search/query.py +++ b/nominatim/api/search/query.py @@ -7,7 +7,7 @@ """ Datastructures for a tokenized query. """ -from typing import List, Tuple, Optional, NamedTuple, Iterator +from typing import List, Tuple, Optional, Iterator from abc import ABC, abstractmethod import dataclasses import enum @@ -107,13 +107,29 @@ class Token(ABC): category objects. """ - -class TokenRange(NamedTuple): +@dataclasses.dataclass +class TokenRange: """ Indexes of query nodes over which a token spans. """ start: int end: int + def __lt__(self, other: 'TokenRange') -> bool: + return self.end <= other.start + + + def __le__(self, other: 'TokenRange') -> bool: + return NotImplemented + + + def __gt__(self, other: 'TokenRange') -> bool: + return self.start >= other.end + + + def __ge__(self, other: 'TokenRange') -> bool: + return NotImplemented + + def replace_start(self, new_start: int) -> 'TokenRange': """ Return a new token range with the new start. """ diff --git a/nominatim/api/search/token_assignment.py b/nominatim/api/search/token_assignment.py index 33fb7335..0ae2cd43 100644 --- a/nominatim/api/search/token_assignment.py +++ b/nominatim/api/search/token_assignment.py @@ -288,18 +288,29 @@ class _TokenSequence: yield dataclasses.replace(base, penalty=self.penalty, name=first, address=base.address[1:]) - if (not base.housenumber or first.end >= base.housenumber.start)\ - and (not base.qualifier or first.start >= base.qualifier.end): - base_penalty = self.penalty - if (base.housenumber and base.housenumber.start > first.start) \ - or len(query.source) > 1: - base_penalty += 0.25 - for i in range(first.start + 1, first.end): - name, addr = first.split(i) - penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] - log().comment(f'split first word = name ({i - first.start})') - yield dataclasses.replace(base, name=name, penalty=penalty, - address=[addr] + base.address[1:]) + # To paraphrase: + # * if another name term comes after the first one and before the + # housenumber + # * a qualifier comes after the name + # * the containing phrase is strictly typed + if (base.housenumber and first.end < base.housenumber.start)\ + or (base.qualifier and base.qualifier > first)\ + or (query.nodes[first.start].ptype != qmod.PhraseType.NONE): + return + + penalty = self.penalty + + # Penalty for: + # * , , , ... + # * queries that are comma-separated + if (base.housenumber and base.housenumber > first) or len(query.source) > 1: + penalty += 0.25 + + for i in range(first.start + 1, first.end): + name, addr = first.split(i) + log().comment(f'split first word = name ({i - first.start})') + yield dataclasses.replace(base, name=name, address=[addr] + base.address[1:], + penalty=penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype]) def _get_assignments_address_backward(self, base: TokenAssignment, @@ -314,19 +325,27 @@ class _TokenSequence: yield dataclasses.replace(base, penalty=self.penalty, name=last, address=base.address[:-1]) - if (not base.housenumber or last.start <= base.housenumber.end)\ - and (not base.qualifier or last.end <= base.qualifier.start): - base_penalty = self.penalty - if base.housenumber and base.housenumber.start < last.start: - base_penalty += 0.4 - if len(query.source) > 1: - base_penalty += 0.25 - for i in range(last.start + 1, last.end): - addr, name = last.split(i) - penalty = base_penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype] - log().comment(f'split last word = name ({i - last.start})') - yield dataclasses.replace(base, name=name, penalty=penalty, - address=base.address[:-1] + [addr]) + # To paraphrase: + # * if another name term comes before the last one and after the + # housenumber + # * a qualifier comes before the name + # * the containing phrase is strictly typed + if (base.housenumber and last.start > base.housenumber.end)\ + or (base.qualifier and base.qualifier < last)\ + or (query.nodes[last.start].ptype != qmod.PhraseType.NONE): + return + + penalty = self.penalty + if base.housenumber and base.housenumber < last: + penalty += 0.4 + if len(query.source) > 1: + penalty += 0.25 + + for i in range(last.start + 1, last.end): + addr, name = last.split(i) + log().comment(f'split last word = name ({i - last.start})') + yield dataclasses.replace(base, name=name, address=base.address[:-1] + [addr], + penalty=penalty + PENALTY_TOKENCHANGE[query.nodes[i].btype]) def get_assignments(self, query: qmod.QueryStruct) -> Iterator[TokenAssignment]: diff --git a/test/python/api/search/test_query.py b/test/python/api/search/test_query.py new file mode 100644 index 00000000..a4b32824 --- /dev/null +++ b/test/python/api/search/test_query.py @@ -0,0 +1,49 @@ + +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Test data types for search queries. +""" +import pytest + +import nominatim.api.search.query as nq + +def test_token_range_equal(): + assert nq.TokenRange(2, 3) == nq.TokenRange(2, 3) + assert not (nq.TokenRange(2, 3) != nq.TokenRange(2, 3)) + + +@pytest.mark.parametrize('lop,rop', [((1, 2), (3, 4)), + ((3, 4), (3, 5)), + ((10, 12), (11, 12))]) +def test_token_range_unequal(lop, rop): + assert not (nq.TokenRange(*lop) == nq.TokenRange(*rop)) + assert nq.TokenRange(*lop) != nq.TokenRange(*rop) + + +def test_token_range_lt(): + assert nq.TokenRange(1, 3) < nq.TokenRange(10, 12) + assert nq.TokenRange(5, 6) < nq.TokenRange(7, 8) + assert nq.TokenRange(1, 4) < nq.TokenRange(4, 5) + assert not(nq.TokenRange(5, 6) < nq.TokenRange(5, 6)) + assert not(nq.TokenRange(10, 11) < nq.TokenRange(4, 5)) + + +def test_token_rankge_gt(): + assert nq.TokenRange(3, 4) > nq.TokenRange(1, 2) + assert nq.TokenRange(100, 200) > nq.TokenRange(10, 11) + assert nq.TokenRange(10, 11) > nq.TokenRange(4, 10) + assert not(nq.TokenRange(5, 6) > nq.TokenRange(5, 6)) + assert not(nq.TokenRange(1, 2) > nq.TokenRange(3, 4)) + assert not(nq.TokenRange(4, 10) > nq.TokenRange(3, 5)) + + +def test_token_range_unimplemented_ops(): + with pytest.raises(TypeError): + nq.TokenRange(1, 3) <= nq.TokenRange(10, 12) + with pytest.raises(TypeError): + nq.TokenRange(1, 3) >= nq.TokenRange(10, 12) From 587698a6f3fa50ec9c2f0fa8cc0475ad5855efff Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Thu, 20 Jul 2023 18:05:54 +0200 Subject: [PATCH 11/11] disallow special housenumber search with a single frequent partial --- nominatim/api/search/db_search_builder.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/nominatim/api/search/db_search_builder.py b/nominatim/api/search/db_search_builder.py index 72a2dc0b..fc444aa2 100644 --- a/nominatim/api/search/db_search_builder.py +++ b/nominatim/api/search/db_search_builder.py @@ -155,13 +155,21 @@ class SearchBuilder: """ Build a simple address search for special entries where the housenumber is the main name token. """ - partial_tokens: List[int] = [] - for trange in address: - partial_tokens.extend(t.token for t in self.query.get_partials_list(trange)) + sdata.lookups = [dbf.FieldLookup('name_vector', [t.token for t in hnrs], 'lookup_any')] + + partials = [t for trange in address + for t in self.query.get_partials_list(trange)] + + if len(partials) != 1 or partials[0].count < 10000: + sdata.lookups.append(dbf.FieldLookup('nameaddress_vector', + [t.token for t in partials], 'lookup_all')) + else: + sdata.lookups.append( + dbf.FieldLookup('nameaddress_vector', + [t.token for t + in self.query.get_tokens(address[0], TokenType.WORD)], + 'lookup_any')) - sdata.lookups = [dbf.FieldLookup('name_vector', [t.token for t in hnrs], 'lookup_any'), - dbf.FieldLookup('nameaddress_vector', partial_tokens, 'lookup_all') - ] sdata.housenumbers = dbf.WeightedStrings([], []) yield dbs.PlaceSearch(0.05, sdata, sum(t.count for t in hnrs))