safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

port replication update function to python

Browse Source
This commit is contained in:
Sarah Hoffmann
2021-01-30 15:50:34 +01:00
parent 8f0885f6cb
commit 4cb6dc01f3
13 changed files with 527 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
test/python/conftest.py
View File

@@ -101,7 +101,8 @@ def def_config():
@pytest.fixture
def status_table(temp_db_conn):
""" Create an empty version of the status table.
""" Create an empty version of the status table and
the status logging table.
"""
with temp_db_conn.cursor() as cur:
cur.execute("""CREATE TABLE import_status (
@@ -109,6 +110,14 @@ def status_table(temp_db_conn):
sequence_id integer,
indexed boolean
)""")
cur.execute("""CREATE TABLE import_osmosis_log (
batchend timestamp,
batchseq integer,
batchsize bigint,
starttime timestamp,
endtime timestamp,
event text
)""")
temp_db_conn.commit()

72
test/python/test_cli.py
View File

@@ -1,8 +1,13 @@
"""
Tests for command line interface wrapper.
These tests just check that the various command line parameters route to the
correct functionionality. They use a lot of monkeypatching to avoid executing
the actual functions.
"""
import psycopg2
import pytest
import time
import nominatim.cli
import nominatim.indexer.indexer
@@ -21,9 +26,9 @@ class MockParamCapture:
""" Mock that records the parameters with which a function was called
as well as the number of calls.
"""
def __init__(self):
def __init__(self, retval=0):
self.called = 0
self.return_value = 0
self.return_value = retval
def __call__(self, *args, **kwargs):
self.called += 1
@@ -142,6 +147,69 @@ def test_replication_command(monkeypatch, temp_db, params, func):
assert func_mock.called == 1
def test_replication_update_bad_interval(monkeypatch, temp_db):
monkeypatch.setenv('NOMINATIM_REPLICATION_UPDATE_INTERVAL', 'xx')
with pytest.raises(ValueError):
call_nominatim('replication')
def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db):
monkeypatch.setenv('NOMINATIM_REPLICATION_URL',
'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates')
with pytest.raises(RuntimeError, match='Invalid replication.*'):
call_nominatim('replication')
@pytest.mark.parametrize("state, retval", [
(nominatim.tools.replication.UpdateState.UP_TO_DATE, 0),
(nominatim.tools.replication.UpdateState.NO_CHANGES, 3)
])
def test_replication_update_once_no_index(monkeypatch, temp_db, status_table, state, retval):
func_mock = MockParamCapture(retval=state)
monkeypatch.setattr(nominatim.tools.replication, 'update', func_mock)
assert retval == call_nominatim('replication', '--once', '--no-index')
def test_replication_update_continuous(monkeypatch, status_table):
states = [nominatim.tools.replication.UpdateState.UP_TO_DATE,
nominatim.tools.replication.UpdateState.UP_TO_DATE]
monkeypatch.setattr(nominatim.tools.replication, 'update',
lambda *args, **kwargs: states.pop())
index_mock = MockParamCapture()
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', index_mock)
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', index_mock)
with pytest.raises(IndexError):
call_nominatim('replication')
assert index_mock.called == 4
def test_replication_update_continuous_no_change(monkeypatch, status_table):
states = [nominatim.tools.replication.UpdateState.NO_CHANGES,
nominatim.tools.replication.UpdateState.UP_TO_DATE]
monkeypatch.setattr(nominatim.tools.replication, 'update',
lambda *args, **kwargs: states.pop())
index_mock = MockParamCapture()
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', index_mock)
monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', index_mock)
sleep_mock = MockParamCapture()
monkeypatch.setattr(time, 'sleep', sleep_mock)
with pytest.raises(IndexError):
call_nominatim('replication')
assert index_mock.called == 2
assert sleep_mock.called == 1
assert sleep_mock.last_args[0] == 60
@pytest.mark.parametrize("params", [
('search', '--query', 'new'),
('reverse', '--lat', '0', '--lon', '0'),

60
test/python/test_config.py
View File

@@ -69,6 +69,18 @@ def test_get_libpq_dsn_convert_php(monkeypatch):
assert config.get_libpq_dsn() == 'dbname=gis password=foo host=localhost'
@pytest.mark.parametrize("val,expect", [('foo bar', "'foo bar'"),
("xy'z", "xy\\'z"),
])
def test_get_libpq_dsn_convert_php_special_chars(monkeypatch, val, expect):
config = Configuration(None, DEFCFG_DIR)
monkeypatch.setenv('NOMINATIM_DATABASE_DSN',
'pgsql:dbname=gis;password={}'.format(val))
assert config.get_libpq_dsn() == "dbname=gis password={}".format(expect)
def test_get_libpq_dsn_convert_libpq(monkeypatch):
config = Configuration(None, DEFCFG_DIR)
@@ -93,3 +105,51 @@ def test_get_bool_empty():
assert config.DATABASE_MODULE_PATH == ''
assert config.get_bool('DATABASE_MODULE_PATH') == False
@pytest.mark.parametrize("value,result", [('0', 0), ('1', 1),
('85762513444', 85762513444)])
def test_get_int_success(monkeypatch, value, result):
config = Configuration(None, DEFCFG_DIR)
monkeypatch.setenv('NOMINATIM_FOOBAR', value)
assert config.get_int('FOOBAR') == result
@pytest.mark.parametrize("value", ['1b', 'fg', '0x23'])
def test_get_int_bad_values(monkeypatch, value):
config = Configuration(None, DEFCFG_DIR)
monkeypatch.setenv('NOMINATIM_FOOBAR', value)
with pytest.raises(ValueError):
config.get_int('FOOBAR')
def test_get_int_empty():
config = Configuration(None, DEFCFG_DIR)
assert config.DATABASE_MODULE_PATH == ''
with pytest.raises(ValueError):
config.get_int('DATABASE_MODULE_PATH')
def test_get_import_style_intern(monkeypatch):
config = Configuration(None, DEFCFG_DIR)
monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', 'street')
expected = DEFCFG_DIR / 'import-street.style'
assert config.get_import_style_file() == expected
@pytest.mark.parametrize("value", ['custom', '/foo/bar.stye'])
def test_get_import_style_intern(monkeypatch, value):
config = Configuration(None, DEFCFG_DIR)
monkeypatch.setenv('NOMINATIM_IMPORT_STYLE', value)
assert str(config.get_import_style_file()) == value

27
test/python/test_db_status.py
View File

@@ -84,3 +84,30 @@ def test_get_status_success(status_table, temp_db_conn):
assert nominatim.db.status.get_status(temp_db_conn) == \
(date, 667, False)
@pytest.mark.parametrize("old_state", [True, False])
@pytest.mark.parametrize("new_state", [True, False])
def test_set_indexed(status_table, temp_db_conn, temp_db_cursor, old_state, new_state):
date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
nominatim.db.status.set_status(temp_db_conn, date=date, indexed=old_state)
nominatim.db.status.set_indexed(temp_db_conn, new_state)
assert temp_db_cursor.scalar("SELECT indexed FROM import_status") == new_state
def test_set_indexed_empty_status(status_table, temp_db_conn, temp_db_cursor):
nominatim.db.status.set_indexed(temp_db_conn, True)
assert temp_db_cursor.scalar("SELECT count(*) FROM import_status") == 0
def text_log_status(status_table, temp_db_conn):
date = dt.datetime.fromordinal(1000000).replace(tzinfo=dt.timezone.utc)
start = dt.datetime.now() - dt.timedelta(hours=1)
nominatim.db.status.set_status(temp_db_conn, date=date, seq=56)
nominatim.db.status.log_status(temp_db_conn, start, 'index')
assert temp_db_cursor.scalar("SELECT count(*) FROM import_osmosis_log") == 1
assert temp_db_cursor.scalar("SELECT seq FROM import_osmosis_log") == 56
assert temp_db_cursor.scalar("SELECT date FROM import_osmosis_log") == date

9
test/python/test_tools_exec_utils.py
View File

@@ -99,3 +99,12 @@ def test_run_api_with_extra_env(tmp_project_dir):
extra_env = dict(SCRIPT_FILENAME=str(tmp_project_dir / 'website' / 'test.php'))
assert 0 == exec_utils.run_api_script('badname', tmp_project_dir,
extra_env=extra_env)
### run_osm2pgsql
def test_run_osm2pgsql():
exec_utils.run_osm2pgsql(dict(osm2pgsql='echo', append=False, flatnode_file=None,
dsn='dbname=foobar', threads=1, osm2pgsql_cache=500,
osm2pgsql_style='./my.style',
import_file='foo.bar'))

72
test/python/test_tools_replication.py
View File

@@ -2,6 +2,7 @@
Tests for replication functionality.
"""
import datetime as dt
import time
import pytest
from osmium.replication.server import OsmosisState
@@ -16,6 +17,7 @@ OSM_NODE_DATA = """\
</osm>
"""
### init replication
def test_init_replication_bad_base_url(monkeypatch, status_table, place_row, temp_db_conn, temp_db_cursor):
place_row(osm_type='N', osm_id=100)
@@ -43,20 +45,20 @@ def test_init_replication_success(monkeypatch, status_table, place_row, temp_db_
assert temp_db_cursor.fetchone() == [expected_date, 234, True]
### checking for updates
def test_check_for_updates_empty_status_table(status_table, temp_db_conn):
assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
def test_check_for_updates_seq_not_set(status_table, temp_db_conn):
status.set_status(temp_db_conn, dt.datetime.now().replace(tzinfo=dt.timezone.utc))
status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc))
assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 254
def test_check_for_updates_no_state(monkeypatch, status_table, temp_db_conn):
status.set_status(temp_db_conn,
dt.datetime.now().replace(tzinfo=dt.timezone.utc),
seq=345)
status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=345)
monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
"get_state_info", lambda self: None)
@@ -64,10 +66,10 @@ def test_check_for_updates_no_state(monkeypatch, status_table, temp_db_conn):
assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == 253
@pytest.mark.parametrize("server_sequence,result", [(344, 1), (345, 1), (346, 0)])
@pytest.mark.parametrize("server_sequence,result", [(344, 2), (345, 2), (346, 0)])
def test_check_for_updates_no_new_data(monkeypatch, status_table, temp_db_conn,
server_sequence, result):
date = dt.datetime.now().replace(tzinfo=dt.timezone.utc)
date = dt.datetime.now(dt.timezone.utc)
status.set_status(temp_db_conn, date, seq=345)
monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
@@ -75,3 +77,61 @@ def test_check_for_updates_no_new_data(monkeypatch, status_table, temp_db_conn,
lambda self: OsmosisState(server_sequence, date))
assert nominatim.tools.replication.check_for_updates(temp_db_conn, 'https://test.io') == result
### updating
@pytest.fixture
def update_options(tmpdir):
return dict(base_url='https://test.io',
indexed_only=False,
update_interval=3600,
import_file=tmpdir / 'foo.osm',
max_diff_size=1)
def test_update_empty_status_table(status_table, temp_db_conn):
with pytest.raises(RuntimeError):
nominatim.tools.replication.update(temp_db_conn, {})
def test_update_already_indexed(status_table, temp_db_conn):
status.set_status(temp_db_conn, dt.datetime.now(dt.timezone.utc), seq=34, indexed=False)
assert nominatim.tools.replication.update(temp_db_conn, dict(indexed_only=True)) \
== nominatim.tools.replication.UpdateState.MORE_PENDING
def test_update_no_data_no_sleep(monkeypatch, status_table, temp_db_conn, update_options):
date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(days=1)
status.set_status(temp_db_conn, date, seq=34)
monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
"apply_diffs",
lambda *args, **kwargs: None)
sleeptime = []
monkeypatch.setattr(time, 'sleep', lambda s: sleeptime.append(s))
assert nominatim.tools.replication.update(temp_db_conn, update_options) \
== nominatim.tools.replication.UpdateState.NO_CHANGES
assert not sleeptime
def test_update_no_data_sleep(monkeypatch, status_table, temp_db_conn, update_options):
date = dt.datetime.now(dt.timezone.utc) - dt.timedelta(minutes=30)
status.set_status(temp_db_conn, date, seq=34)
monkeypatch.setattr(nominatim.tools.replication.ReplicationServer,
"apply_diffs",
lambda *args, **kwargs: None)
sleeptime = []
monkeypatch.setattr(time, 'sleep', lambda s: sleeptime.append(s))
assert nominatim.tools.replication.update(temp_db_conn, update_options) \
== nominatim.tools.replication.UpdateState.NO_CHANGES
assert len(sleeptime) == 1
assert sleeptime[0] < 3600
assert sleeptime[0] > 0