hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-11 05:14:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

boilerplate for PHP code of tokenizer

Browse Source 
This adds an installation step for PHP code for the tokenizer. The
PHP code is split in two parts. The updateable code is found in
lib-php. The tokenizer installs an additional script in the
project directory which then includes the code from lib-php and
defines all settings that are static to the database. The website
code then always includes the PHP from the project directory.
This commit is contained in:
Sarah Hoffmann
2021-04-28 10:59:07 +02:00
parent 23fd1d032a
commit 3eb4d88057
11 changed files with 48 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
nominatim/tokenizer/factory.py
View File

@@ -54,8 +54,7 @@ def create_tokenizer(config, init_db=True, module_name=None):
tokenizer_module = _import_tokenizer(module_name)
tokenizer = tokenizer_module.create(config.get_libpq_dsn(), basedir)
if init_db:
tokenizer.init_new_db(config)
tokenizer.init_new_db(config, init_db=init_db)
with connect(config.get_libpq_dsn()) as conn:
properties.set_property(conn, 'tokenizer', module_name)

22
nominatim/tokenizer/legacy_tokenizer.py
View File

@@ -5,6 +5,7 @@ from collections import OrderedDict
import logging
import re
import shutil
from textwrap import dedent
from icu import Transliterator
import psycopg2
@@ -87,7 +88,7 @@ class LegacyTokenizer:
self.normalization = None
def init_new_db(self, config):
def init_new_db(self, config, init_db=True):
""" Set up a new tokenizer for the database.
This copies all necessary data in the project directory to make
@@ -99,13 +100,16 @@ class LegacyTokenizer:
self.normalization = config.TERM_NORMALIZATION
self._install_php(config)
with connect(self.dsn) as conn:
_check_module(module_dir, conn)
self._save_config(conn, config)
conn.commit()
self.update_sql_functions(config)
self._init_db_tables(config)
if init_db:
self.update_sql_functions(config)
self._init_db_tables(config)
def init_from_project(self):
@@ -165,6 +169,18 @@ class LegacyTokenizer:
return LegacyNameAnalyzer(self.dsn, normalizer)
def _install_php(self, config):
""" Install the php script for the tokenizer.
"""
php_file = self.data_dir / "tokenizer.php"
php_file.write_text(dedent("""\
<?php
@define('CONST_Max_Word_Frequency', {0.MAX_WORD_FREQUENCY});
@define('CONST_Term_Normalization_Rules', "{0.TERM_NORMALIZATION}");
require_once('{0.lib_dir.php}/tokenizer/legacy_tokenizer.php');
""".format(config)))
def _init_db_tables(self, config):
""" Set up the word table and fill it with pre-computed word
frequencies.