add new command for cleaning word tokens

Just pulls outdated housenumbers for the moment.
This commit is contained in:
Sarah Hoffmann
2022-01-20 20:05:15 +01:00
parent 86588419fb
commit 344a2bfc1a
4 changed files with 53 additions and 0 deletions

View File

@@ -209,6 +209,13 @@ class AbstractTokenizer(ABC):
"""
@abstractmethod
def update_word_tokens(self) -> None:
""" Do house-keeping on the tokenizers internal data structures.
Remove unused word tokens, resort data etc.
"""
@abstractmethod
def name_analyzer(self) -> AbstractAnalyzer:
""" Create a new analyzer for tokenizing names and queries