mirror of
https://github.com/osm-search/Nominatim.git
synced 2026-02-26 11:08:13 +00:00
overhaul the token analysis interface
The functional split betweenthe two functions is now that the first one creates the ID that is used in the word table and the second one creates the variants. There no longer is a requirement that the ID is the normalized version. We might later reintroduce the requirement that a normalized version be available but it doesn't necessarily need to be through the ID. The function that creates the ID now gets the full PlaceName. That way it might take into account attributes that were set by the sanitizers. Finally rename both functions to something more sane.
This commit is contained in:
@@ -11,6 +11,7 @@ token, which produces variants with optional spaces.
|
||||
from typing import Any, List
|
||||
|
||||
from nominatim.tokenizer.token_analysis.generic_mutation import MutationVariantGenerator
|
||||
from nominatim.data.place_name import PlaceName
|
||||
|
||||
### Configuration section
|
||||
|
||||
@@ -31,10 +32,8 @@ class PostcodeTokenAnalysis:
|
||||
""" Special normalization and variant generation for postcodes.
|
||||
|
||||
This analyser must not be used with anything but postcodes as
|
||||
it follows some special rules: `normalize` doesn't necessarily
|
||||
need to return a standard form as per normalization rules. It
|
||||
needs to return the canonical form of the postcode that is also
|
||||
used for output. `get_variants_ascii` then needs to ensure that
|
||||
it follows some special rules: the canonial ID is the form that
|
||||
is used for the output. `compute_variants` then needs to ensure that
|
||||
the generated variants once more follow the standard normalization
|
||||
and transliteration, so that postcodes are correctly recognised by
|
||||
the search algorithm.
|
||||
@@ -46,13 +45,13 @@ class PostcodeTokenAnalysis:
|
||||
self.mutator = MutationVariantGenerator(' ', (' ', ''))
|
||||
|
||||
|
||||
def normalize(self, name: str) -> str:
|
||||
def get_canonical_id(self, name: PlaceName) -> str:
|
||||
""" Return the standard form of the postcode.
|
||||
"""
|
||||
return name.strip().upper()
|
||||
return name.name.strip().upper()
|
||||
|
||||
|
||||
def get_variants_ascii(self, norm_name: str) -> List[str]:
|
||||
def compute_variants(self, norm_name: str) -> List[str]:
|
||||
""" Compute the spelling variants for the given normalized postcode.
|
||||
|
||||
Takes the canonical form of the postcode, normalizes it using the
|
||||
|
||||
Reference in New Issue
Block a user