safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

port code to psycopg3

Browse Source
This commit is contained in:
Sarah Hoffmann
2024-07-05 10:43:10 +02:00
parent 3742fa2929
commit 9659afbade
57 changed files with 800 additions and 1330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
test/python/cli/conftest.py
View File

@@ -25,6 +25,23 @@ class MockParamCapture:
return self.return_value
class AsyncMockParamCapture:
""" Mock that records the parameters with which a function was called
as well as the number of calls.
"""
def __init__(self, retval=0):
self.called = 0
self.return_value = retval
self.last_args = None
self.last_kwargs = None
async def __call__(self, *args, **kwargs):
self.called += 1
self.last_args = args
self.last_kwargs = kwargs
return self.return_value
class DummyTokenizer:
def __init__(self, *args, **kwargs):
self.update_sql_functions_called = False
@@ -69,6 +86,17 @@ def mock_func_factory(monkeypatch):
return get_mock
@pytest.fixture
def async_mock_func_factory(monkeypatch):
def get_mock(module, func):
mock = AsyncMockParamCapture()
mock.func_name = func
monkeypatch.setattr(module, func, mock)
return mock
return get_mock
@pytest.fixture
def cli_tokenizer_mock(monkeypatch):
tok = DummyTokenizer()

54
test/python/cli/test_cli.py
View File

@@ -17,6 +17,7 @@ import pytest
import nominatim_db.indexer.indexer
import nominatim_db.tools.add_osm_data
import nominatim_db.tools.freeze
import nominatim_db.tools.tiger_data
def test_cli_help(cli_call, capsys):
@@ -52,8 +53,8 @@ def test_cli_add_data_object_command(cli_call, mock_func_factory, name, oid):
def test_cli_add_data_tiger_data(cli_call, cli_tokenizer_mock, mock_func_factory):
mock = mock_func_factory(nominatim_db.tools.tiger_data, 'add_tiger_data')
def test_cli_add_data_tiger_data(cli_call, cli_tokenizer_mock, async_mock_func_factory):
mock = async_mock_func_factory(nominatim_db.tools.tiger_data, 'add_tiger_data')
assert cli_call('add-data', '--tiger-data', 'somewhere') == 0
@@ -68,38 +69,6 @@ def test_cli_serve_php(cli_call, mock_func_factory):
assert func.called == 1
def test_cli_serve_starlette_custom_server(cli_call, mock_func_factory):
pytest.importorskip("starlette")
mod = pytest.importorskip("uvicorn")
func = mock_func_factory(mod, "run")
cli_call('serve', '--engine', 'starlette', '--server', 'foobar:4545') == 0
assert func.called == 1
assert func.last_kwargs['host'] == 'foobar'
assert func.last_kwargs['port'] == 4545
def test_cli_serve_starlette_custom_server_bad_port(cli_call, mock_func_factory):
pytest.importorskip("starlette")
mod = pytest.importorskip("uvicorn")
func = mock_func_factory(mod, "run")
cli_call('serve', '--engine', 'starlette', '--server', 'foobar:45:45') == 1
@pytest.mark.parametrize("engine", ['falcon', 'starlette'])
def test_cli_serve_uvicorn_based(cli_call, engine, mock_func_factory):
pytest.importorskip(engine)
mod = pytest.importorskip("uvicorn")
func = mock_func_factory(mod, "run")
cli_call('serve', '--engine', engine) == 0
assert func.called == 1
assert func.last_kwargs['host'] == '127.0.0.1'
assert func.last_kwargs['port'] == 8088
class TestCliWithDb:
@@ -120,16 +89,19 @@ class TestCliWithDb:
@pytest.mark.parametrize("params,do_bnds,do_ranks", [
([], 1, 1),
(['--boundaries-only'], 1, 0),
(['--no-boundaries'], 0, 1),
([], 2, 2),
(['--boundaries-only'], 2, 0),
(['--no-boundaries'], 0, 2),
(['--boundaries-only', '--no-boundaries'], 0, 0)])
def test_index_command(self, mock_func_factory, table_factory,
def test_index_command(self, monkeypatch, async_mock_func_factory, table_factory,
params, do_bnds, do_ranks):
table_factory('import_status', 'indexed bool')
bnd_mock = mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_boundaries')
rank_mock = mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_by_rank')
postcode_mock = mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_postcodes')
bnd_mock = async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_boundaries')
rank_mock = async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_by_rank')
postcode_mock = async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_postcodes')
monkeypatch.setattr(nominatim_db.indexer.indexer.Indexer, 'has_pending',
[False, True].pop)
assert self.call_nominatim('index', *params) == 0

29
test/python/cli/test_cmd_import.py
View File

@@ -34,7 +34,8 @@ class TestCliImportWithDb:
@pytest.mark.parametrize('with_updates', [True, False])
def test_import_full(self, mock_func_factory, with_updates, place_table, property_table):
def test_import_full(self, mock_func_factory, async_mock_func_factory,
with_updates, place_table, property_table):
mocks = [
mock_func_factory(nominatim_db.tools.database_import, 'setup_database_skeleton'),
mock_func_factory(nominatim_db.data.country_info, 'setup_country_tables'),
@@ -42,15 +43,15 @@ class TestCliImportWithDb:
mock_func_factory(nominatim_db.tools.refresh, 'import_wikipedia_articles'),
mock_func_factory(nominatim_db.tools.refresh, 'import_secondary_importance'),
mock_func_factory(nominatim_db.tools.database_import, 'truncate_data_tables'),
mock_func_factory(nominatim_db.tools.database_import, 'load_data'),
async_mock_func_factory(nominatim_db.tools.database_import, 'load_data'),
mock_func_factory(nominatim_db.tools.database_import, 'create_tables'),
mock_func_factory(nominatim_db.tools.database_import, 'create_table_triggers'),
mock_func_factory(nominatim_db.tools.database_import, 'create_partition_tables'),
mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
async_mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim_db.data.country_info, 'create_country_names'),
mock_func_factory(nominatim_db.tools.refresh, 'load_address_levels_from_config'),
mock_func_factory(nominatim_db.tools.postcodes, 'update_postcodes'),
mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim_db.tools.refresh, 'setup_website'),
]
@@ -72,14 +73,14 @@ class TestCliImportWithDb:
assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
def test_import_continue_load_data(self, mock_func_factory):
def test_import_continue_load_data(self, mock_func_factory, async_mock_func_factory):
mocks = [
mock_func_factory(nominatim_db.tools.database_import, 'truncate_data_tables'),
mock_func_factory(nominatim_db.tools.database_import, 'load_data'),
mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
async_mock_func_factory(nominatim_db.tools.database_import, 'load_data'),
async_mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim_db.data.country_info, 'create_country_names'),
mock_func_factory(nominatim_db.tools.postcodes, 'update_postcodes'),
mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim_db.tools.refresh, 'setup_website'),
mock_func_factory(nominatim_db.db.properties, 'set_property')
]
@@ -91,12 +92,12 @@ class TestCliImportWithDb:
assert mock.called == 1, "Mock '{}' not called".format(mock.func_name)
def test_import_continue_indexing(self, mock_func_factory, placex_table,
temp_db_conn):
def test_import_continue_indexing(self, mock_func_factory, async_mock_func_factory,
placex_table, temp_db_conn):
mocks = [
mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
async_mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim_db.data.country_info, 'create_country_names'),
mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim_db.tools.refresh, 'setup_website'),
mock_func_factory(nominatim_db.db.properties, 'set_property')
]
@@ -110,9 +111,9 @@ class TestCliImportWithDb:
assert self.call_nominatim('import', '--continue', 'indexing') == 0
def test_import_continue_postprocess(self, mock_func_factory):
def test_import_continue_postprocess(self, mock_func_factory, async_mock_func_factory):
mocks = [
mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
async_mock_func_factory(nominatim_db.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim_db.data.country_info, 'create_country_names'),
mock_func_factory(nominatim_db.tools.refresh, 'setup_website'),
mock_func_factory(nominatim_db.db.properties, 'set_property')

4
test/python/cli/test_cmd_refresh.py
View File

@@ -45,9 +45,9 @@ class TestRefresh:
assert self.tokenizer_mock.update_word_tokens_called
def test_refresh_postcodes(self, mock_func_factory, place_table):
def test_refresh_postcodes(self, async_mock_func_factory, mock_func_factory, place_table):
func_mock = mock_func_factory(nominatim_db.tools.postcodes, 'update_postcodes')
idx_mock = mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_postcodes')
idx_mock = async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_postcodes')
assert self.call_nominatim('refresh', '--postcodes') == 0
assert func_mock.called == 1

4
test/python/cli/test_cmd_replication.py
View File

@@ -47,8 +47,8 @@ def init_status(temp_db_conn, status_table):
@pytest.fixture
def index_mock(mock_func_factory, tokenizer_mock, init_status):
return mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full')
def index_mock(async_mock_func_factory, tokenizer_mock, init_status):
return async_mock_func_factory(nominatim_db.indexer.indexer.Indexer, 'index_full')
@pytest.fixture