hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-15 02:47:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added --no-replace command for special phrases importation and added corresponding tests

Browse Source
This commit is contained in:
AntoJvlt
2021-05-17 12:40:50 +02:00
parent 06aab389ed
commit 8b8dfc46eb
11 changed files with 76 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
test/python/dummy_tokenizer.py
View File

@@ -54,7 +54,7 @@ class DummyNameAnalyzer:
def add_postcodes_from_db(self):
pass
def update_special_phrases(self, phrases):
def update_special_phrases(self, phrases, should_replace):
self.analyser_cache['special_phrases'] = phrases
def add_country_names(self, code, names):

19
test/python/test_cli.py
View File

@@ -255,18 +255,27 @@ def test_index_command(mock_func_factory, temp_db_cursor, tokenizer_mock,
assert bnd_mock.called == do_bnds
assert rank_mock.called == do_ranks
def test_special_phrases_wiki_command(temp_db, mock_func_factory, tokenizer_mock):
@pytest.mark.parametrize("no_replace", [(True), (False)])
def test_special_phrases_wiki_command(temp_db, mock_func_factory, tokenizer_mock, no_replace):
func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
call_nominatim('special-phrases', '--import-from-wiki')
if no_replace:
call_nominatim('special-phrases', '--import-from-wiki', '--no-replace')
else:
call_nominatim('special-phrases', '--import-from-wiki')
assert func.called == 1
def test_special_phrases_csv_command(temp_db, mock_func_factory, tokenizer_mock):
@pytest.mark.parametrize("no_replace", [(True), (False)])
def test_special_phrases_csv_command(temp_db, mock_func_factory, tokenizer_mock, no_replace):
func = mock_func_factory(nominatim.clicmd.special_phrases.SPImporter, 'import_phrases')
testdata = Path('__file__') / '..' / '..' / 'testdb'
testdata = SRC_DIR / 'test' / 'testdb'
csv_path = str((testdata / 'full_en_phrases_test.csv').resolve())
call_nominatim('special-phrases', '--import-from-csv', csv_path)
if no_replace:
call_nominatim('special-phrases', '--import-from-csv', csv_path, '--no-replace')
else:
call_nominatim('special-phrases', '--import-from-csv', csv_path)
assert func.called == 1

19
test/python/test_tokenizer_legacy.py
View File

@@ -209,7 +209,7 @@ def test_update_special_phrase_empty_table(analyzer, word_table, temp_db_cursor,
("König bei", "amenity", "royal", "near"),
("Könige", "amenity", "royal", "-"),
("strasse", "highway", "primary", "in")
])
], True)
assert temp_db_cursor.row_set("""SELECT word_token, word, class, type, operator
FROM word WHERE class != 'place'""") \
@@ -226,11 +226,24 @@ def test_update_special_phrase_delete_all(analyzer, word_table, temp_db_cursor,
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
analyzer.update_special_phrases([])
analyzer.update_special_phrases([], True)
assert 0 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
def test_update_special_phrases_no_replace(analyzer, word_table, temp_db_cursor,
make_standard_name):
temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator)
VALUES (' foo', 'foo', 'amenity', 'prison', 'in'),
(' bar', 'bar', 'highway', 'road', null)""")
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
analyzer.update_special_phrases([], False)
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
def test_update_special_phrase_modify(analyzer, word_table, temp_db_cursor,
make_standard_name):
temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator)
@@ -243,7 +256,7 @@ def test_update_special_phrase_modify(analyzer, word_table, temp_db_cursor,
('prison', 'amenity', 'prison', 'in'),
('bar', 'highway', 'road', '-'),
('garden', 'leisure', 'garden', 'near')
])
], True)
assert temp_db_cursor.row_set("""SELECT word_token, word, class, type, operator
FROM word WHERE class != 'place'""") \

19
test/python/test_tokenizer_legacy_icu.py
View File

@@ -159,7 +159,7 @@ def test_update_special_phrase_empty_table(analyzer, word_table, temp_db_cursor)
("König bei", "amenity", "royal", "near"),
("Könige", "amenity", "royal", "-"),
("street", "highway", "primary", "in")
])
], True)
assert temp_db_cursor.row_set("""SELECT word_token, word, class, type, operator
FROM word WHERE class != 'place'""") \
@@ -176,11 +176,24 @@ def test_update_special_phrase_delete_all(analyzer, word_table, temp_db_cursor):
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
with analyzer() as a:
a.update_special_phrases([])
a.update_special_phrases([], True)
assert 0 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
def test_update_special_phrases_no_replace(analyzer, word_table, temp_db_cursor,):
temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator)
VALUES (' FOO', 'foo', 'amenity', 'prison', 'in'),
(' BAR', 'bar', 'highway', 'road', null)""")
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
with analyzer() as a:
a.update_special_phrases([], False)
assert 2 == temp_db_cursor.scalar("SELECT count(*) FROM word WHERE class != 'place'""")
def test_update_special_phrase_modify(analyzer, word_table, temp_db_cursor):
temp_db_cursor.execute("""INSERT INTO word (word_token, word, class, type, operator)
VALUES (' FOO', 'foo', 'amenity', 'prison', 'in'),
@@ -193,7 +206,7 @@ def test_update_special_phrase_modify(analyzer, word_table, temp_db_cursor):
('prison', 'amenity', 'prison', 'in'),
('bar', 'highway', 'road', '-'),
('garden', 'leisure', 'garden', 'near')
])
], True)
assert temp_db_cursor.row_set("""SELECT word_token, word, class, type, operator
FROM word WHERE class != 'place'""") \

15
test/python/test_tools_import_special_phrases.py
View File

@@ -185,8 +185,9 @@ def test_remove_non_existent_tables_from_db(sp_importer, default_phrases,
tables_result[0][0] == 'place_classtype_testclasstypetable_to_keep'
)
@pytest.mark.parametrize("should_replace", [(True), (False)])
def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
placex_table, tokenizer_mock):
placex_table, tokenizer_mock, should_replace):
"""
Check that the main import_phrases() method is well executed.
It should create the place_classtype table, the place_id and centroid indexes,
@@ -202,10 +203,10 @@ def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
CREATE TABLE place_classtype_wrongclass_wrongtype();""")
monkeypatch.setattr('nominatim.tools.special_phrases.sp_wiki_loader.SPWikiLoader._get_wiki_content',
mock_get_wiki_content)
mock_get_wiki_content)
tokenizer = tokenizer_mock()
sp_importer.import_phrases(tokenizer)
sp_importer.import_phrases(tokenizer, should_replace)
assert len(tokenizer.analyser_cache['special_phrases']) == 18
@@ -216,7 +217,8 @@ def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
assert check_placeid_and_centroid_indexes(temp_db_conn, class_test, type_test)
assert check_grant_access(temp_db_conn, def_config.DATABASE_WEBUSER, class_test, type_test)
assert check_table_exist(temp_db_conn, 'amenity', 'animal_shelter')
assert not check_table_exist(temp_db_conn, 'wrong_class', 'wrong_type')
if should_replace:
assert not check_table_exist(temp_db_conn, 'wrong_class', 'wrong_type')
#Format (query, should_return_something_bool) use to easily execute all asserts
queries_tests = set()
@@ -237,7 +239,8 @@ def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
WHERE table_schema='public'
AND table_name = 'place_classtype_wrongclass_wrongtype';
"""
queries_tests.add((query_wrong_table, False))
if should_replace:
queries_tests.add((query_wrong_table, False))
with temp_db_conn.cursor() as temp_db_cursor:
for query in queries_tests:
@@ -247,7 +250,7 @@ def test_import_phrases(monkeypatch, temp_db_conn, def_config, sp_importer,
else:
assert not temp_db_cursor.fetchone()
def mock_get_wiki_content(lang):
def mock_get_wiki_content(self, lang):
"""
Mock the _get_wiki_content() method to return
static xml test file content.

1
test/python/test_tools_sp_csv_loader.py
View File

@@ -16,7 +16,6 @@ def test_parse_csv(sp_csv_loader):
phrases = sp_csv_loader.parse_csv()
assert check_phrases_content(phrases)
def test_next(sp_csv_loader):
"""
Test objects returned from the next() method.

2
test/python/test_tools_sp_wiki_loader.py
View File

@@ -47,7 +47,7 @@ def sp_wiki_loader(monkeypatch, def_config):
mock_get_wiki_content)
return loader
def mock_get_wiki_content(lang):
def mock_get_wiki_content(self, lang):
"""
Mock the _get_wiki_content() method to return
static xml test file content.