hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

ICU: use normalization from config in PHP

Browse Source 
The TERM_NORMALIZATION config option is no longer applicable.
That was already documented but not yet implemented.
This commit is contained in:
Sarah Hoffmann
2021-10-27 10:07:19 +02:00
parent 1722fc537f
commit 37eeccbf4c
2 changed files with 8 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
nominatim/tokenizer/icu_tokenizer.py
View File

@@ -9,7 +9,6 @@ import re
from textwrap import dedent
from nominatim.db.connection import connect
from nominatim.db.properties import set_property, get_property
from nominatim.db.utils import CopyBuffer
from nominatim.db.sql_preprocessor import SQLPreprocessor
from nominatim.indexer.place_info import PlaceInfo
@@ -36,7 +35,6 @@ class LegacyICUTokenizer(AbstractTokenizer):
self.dsn = dsn
self.data_dir = data_dir
self.loader = None
self.term_normalization = None
def init_new_db(self, config, init_db=True):
@@ -47,8 +45,6 @@ class LegacyICUTokenizer(AbstractTokenizer):
"""
self.loader = ICURuleLoader(config)
self.term_normalization = config.TERM_NORMALIZATION
self._install_php(config.lib_dir.php)
self._save_config()
@@ -64,7 +60,6 @@ class LegacyICUTokenizer(AbstractTokenizer):
with connect(self.dsn) as conn:
self.loader.load_config_from_db(conn)
self.term_normalization = get_property(conn, DBCFG_TERM_NORMALIZATION)
def finalize_import(self, config):
@@ -87,13 +82,9 @@ class LegacyICUTokenizer(AbstractTokenizer):
def check_database(self, config):
""" Check that the tokenizer is set up correctly.
"""
# Will throw an error if there is an issue.
self.init_from_project(config)
if self.term_normalization is None:
return "Configuration for tokenizer 'icu' are missing."
return None
def update_statistics(self):
""" Recompute frequencies for all name words.
@@ -141,7 +132,7 @@ class LegacyICUTokenizer(AbstractTokenizer):
php_file.write_text(dedent(f"""\
<?php
@define('CONST_Max_Word_Frequency', 10000000);
@define('CONST_Term_Normalization_Rules', "{self.term_normalization}");
@define('CONST_Term_Normalization_Rules', "{self.loader.normalization_rules}");
@define('CONST_Transliteration', "{self.loader.get_search_rules()}");
require_once('{phpdir}/tokenizer/icu_tokenizer.php');"""))
@@ -152,7 +143,6 @@ class LegacyICUTokenizer(AbstractTokenizer):
"""
with connect(self.dsn) as conn:
self.loader.save_config_to_db(conn)
set_property(conn, DBCFG_TERM_NORMALIZATION, self.term_normalization)
def _init_db_tables(self, config):