hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-15 02:47:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

recreate word table when refreshing counts

Browse Source 
The counting touches a large part of the word table, leaving
bloated tables and indexes. Thus recreate the table instead and
swap it in.
This commit is contained in:
Sarah Hoffmann
2024-02-04 16:43:33 +01:00
parent 33c0f249b1
commit 81eed0680c
10 changed files with 130 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
test/python/cli/conftest.py
View File

@@ -38,10 +38,10 @@ class DummyTokenizer:
def finalize_import(self, *args):
self.finalize_import_called = True
def update_statistics(self):
def update_statistics(self, *args):
self.update_statistics_called = True
def update_word_tokens(self):
def update_word_tokens(self, *args):
self.update_word_tokens_called = True

20
test/python/tokenizer/test_icu.py
View File

@@ -7,7 +7,6 @@
"""
Tests for ICU tokenizer.
"""
import shutil
import yaml
import itertools
@@ -32,8 +31,6 @@ def test_config(project_env, tmp_path):
sqldir.mkdir()
(sqldir / 'tokenizer').mkdir()
(sqldir / 'tokenizer' / 'icu_tokenizer.sql').write_text("SELECT 'a'")
shutil.copy(str(project_env.lib_dir.sql / 'tokenizer' / 'icu_tokenizer_tables.sql'),
str(sqldir / 'tokenizer' / 'icu_tokenizer_tables.sql'))
project_env.lib_dir.sql = sqldir
@@ -204,16 +201,14 @@ def test_update_sql_functions(db_prop, temp_db_cursor,
def test_finalize_import(tokenizer_factory, temp_db_conn,
temp_db_cursor, test_config, sql_preprocessor_cfg):
func_file = test_config.lib_dir.sql / 'tokenizer' / 'legacy_tokenizer_indices.sql'
func_file.write_text("""CREATE FUNCTION test() RETURNS TEXT
AS $$ SELECT 'b'::text $$ LANGUAGE SQL""")
tok = tokenizer_factory()
tok.init_new_db(test_config)
assert not temp_db_conn.index_exists('idx_word_word_id')
tok.finalize_import(test_config)
temp_db_cursor.scalar('SELECT test()') == 'b'
assert temp_db_conn.index_exists('idx_word_word_id')
def test_check_database(test_config, tokenizer_factory,
@@ -224,19 +219,20 @@ def test_check_database(test_config, tokenizer_factory,
assert tok.check_database(test_config) is None
def test_update_statistics_reverse_only(word_table, tokenizer_factory):
def test_update_statistics_reverse_only(word_table, tokenizer_factory, test_config):
tok = tokenizer_factory()
tok.update_statistics()
tok.update_statistics(test_config)
def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory):
def test_update_statistics(word_table, table_factory, temp_db_cursor,
tokenizer_factory, test_config):
word_table.add_full_word(1000, 'hello')
table_factory('search_name',
'place_id BIGINT, name_vector INT[]',
[(12, [1000])])
tok = tokenizer_factory()
tok.update_statistics()
tok.update_statistics(test_config)
assert temp_db_cursor.scalar("""SELECT count(*) FROM word
WHERE type = 'W' and

8
test/python/tokenizer/test_legacy.py
View File

@@ -238,19 +238,19 @@ def test_check_database_bad_setup(test_config, tokenizer_factory, monkeypatch,
assert tok.check_database(False) is not None
def test_update_statistics_reverse_only(word_table, tokenizer_factory):
def test_update_statistics_reverse_only(word_table, tokenizer_factory, test_config):
tok = tokenizer_factory()
tok.update_statistics()
tok.update_statistics(test_config)
def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory):
def test_update_statistics(word_table, table_factory, temp_db_cursor, tokenizer_factory, test_config):
word_table.add_full_word(1000, 'hello')
table_factory('search_name',
'place_id BIGINT, name_vector INT[]',
[(12, [1000])])
tok = tokenizer_factory()
tok.update_statistics()
tok.update_statistics(test_config)
assert temp_db_cursor.scalar("""SELECT count(*) FROM word
WHERE word_token like ' %' and