introduce custom object for cmdline arguments

Allows to define special functions over the arguments.

Also splits CLI tests in two files as they have become too many.

nominatim/cli.py

@@ -12,6 +12,7 @@ from .config import Configuration
 from .tools.exec_utils import run_legacy_script, run_php_server
 from .errors import UsageError
 from . import clicmd
+from .clicmd.args import NominatimArgs
 
 LOG = logging.getLogger()
 
@@ -62,7 +63,8 @@ class CommandlineParser:
         """ Parse the command line arguments of the program and execute the
             appropriate subcommand.
         """
-        args = self.parser.parse_args(args=kwargs.get('cli_args'))
+        args = NominatimArgs()
+        self.parser.parse_args(args=kwargs.get('cli_args'), namespace=args)
 
         if args.subcommand is None:
             self.parser.print_help()

nominatim/clicmd/args.py

@@ -0,0 +1,22 @@
+"""
+Provides custom functions over command-line arguments.
+"""
+
+
+class NominatimArgs:
+    """ Customized namespace class for the nominatim command line tool
+        to receive the command-line arguments.
+    """
+
+    def osm2pgsql_options(self, default_cache, default_threads):
+        """ Return the standard osm2pgsql options that can be derived
+            from the command line arguments. The resulting dict can be
+            further customized and then used in `run_osm2pgsql()`.
+        """
+        return dict(osm2pgsql=self.config.OSM2PGSQL_BINARY or self.osm2pgsql_path,
+                    osm2pgsql_cache=self.osm2pgsql_cache or default_cache,
+                    osm2pgsql_style=self.config.get_import_style_file(),
+                    threads=self.threads or default_threads,
+                    dsn=self.config.get_libpq_dsn(),
+                    flatnode_file=self.config.FLATNODE_FILE)
+

nominatim/clicmd/replication.py

@@ -17,17 +17,6 @@ LOG = logging.getLogger()
 # Using non-top-level imports to make pyosmium optional for replication only.
 # pylint: disable=E0012,C0415
 
-def _osm2pgsql_options_from_args(args, default_cache, default_threads):
-    """ Set up the standard osm2pgsql from the command line arguments.
-    """
-    return dict(osm2pgsql=args.osm2pgsql_path,
-                osm2pgsql_cache=args.osm2pgsql_cache or default_cache,
-                osm2pgsql_style=args.config.get_import_style_file(),
-                threads=args.threads or default_threads,
-                dsn=args.config.get_libpq_dsn(),
-                flatnode_file=args.config.FLATNODE_FILE)
-
-
 class UpdateReplication:
     """\
     Update the database using an online replication service.
@@ -96,7 +85,7 @@ class UpdateReplication:
         from ..tools import replication
         from ..indexer.indexer import Indexer
 
-        params = _osm2pgsql_options_from_args(args, 2000, 1)
+        params = args.osm2pgsql_options(default_cache=2000, default_threads=1)
         params.update(base_url=args.config.REPLICATION_URL,
                       update_interval=args.config.get_int('REPLICATION_UPDATE_INTERVAL'),
                       import_file=args.project_dir / 'osmosischange.osc',

test/python/mocks.py

@@ -0,0 +1,18 @@
+"""
+Custom mocks for testing.
+"""
+
+
+class MockParamCapture:
+    """ Mock that records the parameters with which a function was called
+        as well as the number of calls.
+    """
+    def __init__(self, retval=0):
+        self.called = 0
+        self.return_value = retval
+
+    def __call__(self, *args, **kwargs):
+        self.called += 1
+        self.last_args = args
+        self.last_kwargs = kwargs
+        return self.return_value

test/python/test_cli.py

@@ -5,11 +5,8 @@ These tests just check that the various command line parameters route to the
 correct functionionality. They use a lot of monkeypatching to avoid executing
 the actual functions.
 """
-import datetime as dt
-import time
 from pathlib import Path
 
-import psycopg2
 import pytest
 
 import nominatim.cli
@@ -21,9 +18,8 @@ import nominatim.tools.admin
 import nominatim.tools.check_database
 import nominatim.tools.freeze
 import nominatim.tools.refresh
-import nominatim.tools.replication
+
-from nominatim.errors import UsageError
+from mocks import MockParamCapture
-from nominatim.db import status
 
 SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()
 
@@ -37,19 +33,6 @@ def call_nominatim(*args):
                                    config_dir=str(SRC_DIR / 'settings'),
                                    cli_args=args)
 
-class MockParamCapture:
-    """ Mock that records the parameters with which a function was called
-        as well as the number of calls.
-    """
-    def __init__(self, retval=0):
-        self.called = 0
-        self.return_value = retval
-
-    def __call__(self, *args, **kwargs):
-        self.called += 1
-        self.last_args = args
-        self.last_kwargs = kwargs
-        return self.return_value
 
 @pytest.fixture
 def mock_run_legacy(monkeypatch):
@@ -186,79 +169,6 @@ def test_refresh_importance_computed_after_wiki_import(mock_func_factory, temp_d
     assert mock_run_legacy.last_args == ('update.php', '--recompute-importance')
 
 
-@pytest.mark.parametrize("params,func", [
-                         (('--init', '--no-update-functions'), 'init_replication'),
-                         (('--check-for-updates',), 'check_for_updates')
-                         ])
-def test_replication_command(mock_func_factory, temp_db, params, func):
-    func_mock = mock_func_factory(nominatim.tools.replication, func)
-
-    assert 0 == call_nominatim('replication', *params)
-    assert func_mock.called == 1
-
-
-def test_replication_update_bad_interval(monkeypatch, temp_db):
-    monkeypatch.setenv('NOMINATIM_REPLICATION_UPDATE_INTERVAL', 'xx')
-
-    assert call_nominatim('replication') == 1
-
-
-def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db):
-    monkeypatch.setenv('NOMINATIM_REPLICATION_URL',
-                       'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates')
-
-    assert call_nominatim('replication') == 1
-
-
-@pytest.mark.parametrize("state", [nominatim.tools.replication.UpdateState.UP_TO_DATE,
-                                   nominatim.tools.replication.UpdateState.NO_CHANGES])
-def test_replication_update_once_no_index(mock_func_factory, temp_db, temp_db_conn,
-                                          status_table, state):
-    status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1)
-    func_mock = mock_func_factory(nominatim.tools.replication, 'update')
-
-    assert 0 == call_nominatim('replication', '--once', '--no-index')
-
-
-def test_replication_update_continuous(monkeypatch, temp_db_conn, status_table):
-    status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1)
-    states = [nominatim.tools.replication.UpdateState.UP_TO_DATE,
-              nominatim.tools.replication.UpdateState.UP_TO_DATE]
-    monkeypatch.setattr(nominatim.tools.replication, 'update',
-                        lambda *args, **kwargs: states.pop())
-
-    index_mock = MockParamCapture()
-    monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', index_mock)
-    monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', index_mock)
-
-    with pytest.raises(IndexError):
-        call_nominatim('replication')
-
-    assert index_mock.called == 4
-
-
-def test_replication_update_continuous_no_change(monkeypatch, temp_db_conn, status_table):
-    status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1)
-    states = [nominatim.tools.replication.UpdateState.NO_CHANGES,
-              nominatim.tools.replication.UpdateState.UP_TO_DATE]
-    monkeypatch.setattr(nominatim.tools.replication, 'update',
-                        lambda *args, **kwargs: states.pop())
-
-    index_mock = MockParamCapture()
-    monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', index_mock)
-    monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', index_mock)
-
-    sleep_mock = MockParamCapture()
-    monkeypatch.setattr(time, 'sleep', sleep_mock)
-
-    with pytest.raises(IndexError):
-        call_nominatim('replication')
-
-    assert index_mock.called == 2
-    assert sleep_mock.called == 1
-    assert sleep_mock.last_args[0] == 60
-
-
 def test_serve_command(mock_func_factory):
     func = mock_func_factory(nominatim.cli, 'run_php_server')
 

test/python/test_cli_replication.py

@@ -0,0 +1,127 @@
+"""
+Tests for replication command of command-line interface wrapper.
+"""
+import datetime as dt
+import time
+from pathlib import Path
+
+import pytest
+
+import nominatim.cli
+import nominatim.indexer.indexer
+import nominatim.tools.replication
+from nominatim.db import status
+
+from mocks import MockParamCapture
+
+SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()
+
+def call_nominatim(*args):
+    return nominatim.cli.nominatim(module_dir='build/module',
+                                   osm2pgsql_path='build/osm2pgsql/osm2pgsql',
+                                   phplib_dir=str(SRC_DIR / 'lib-php'),
+                                   data_dir=str(SRC_DIR / 'data'),
+                                   phpcgi_path='/usr/bin/php-cgi',
+                                   sqllib_dir=str(SRC_DIR / 'lib-sql'),
+                                   config_dir=str(SRC_DIR / 'settings'),
+                                   cli_args=['replication'] + list(args))
+
+@pytest.fixture
+def index_mock(monkeypatch):
+    mock = MockParamCapture()
+    monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', mock)
+    monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', mock)
+
+    return mock
+
+
+@pytest.fixture
+def mock_func_factory(monkeypatch):
+    def get_mock(module, func):
+        mock = MockParamCapture()
+        monkeypatch.setattr(module, func, mock)
+        return mock
+
+    return get_mock
+
+
+@pytest.fixture
+def init_status(temp_db_conn, status_table):
+    status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1)
+    return 1
+
+
+@pytest.fixture
+def update_mock(mock_func_factory, init_status):
+    return mock_func_factory(nominatim.tools.replication, 'update')
+
+@pytest.mark.parametrize("params,func", [
+                         (('--init', '--no-update-functions'), 'init_replication'),
+                         (('--check-for-updates',), 'check_for_updates')
+                         ])
+def test_replication_command(mock_func_factory, temp_db, params, func):
+    func_mock = mock_func_factory(nominatim.tools.replication, func)
+
+    assert 0 == call_nominatim(*params)
+    assert func_mock.called == 1
+
+
+def test_replication_update_bad_interval(monkeypatch, temp_db):
+    monkeypatch.setenv('NOMINATIM_REPLICATION_UPDATE_INTERVAL', 'xx')
+
+    assert call_nominatim() == 1
+
+
+def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db):
+    monkeypatch.setenv('NOMINATIM_REPLICATION_URL',
+                       'https://download.geofabrik.de/europe/ireland-and-northern-ireland-updates')
+
+    assert call_nominatim() == 1
+
+
+def test_replication_update_once_no_index(update_mock):
+    assert 0 == call_nominatim('--once', '--no-index')
+
+    assert str(update_mock.last_args[1]['osm2pgsql']) == 'build/osm2pgsql/osm2pgsql'
+
+
+def test_replication_update_custom_osm2pgsql(monkeypatch, update_mock):
+    monkeypatch.setenv('NOMINATIM_OSM2PGSQL_BINARY', '/secret/osm2pgsql')
+    assert 0 == call_nominatim('--once', '--no-index')
+
+    assert str(update_mock.last_args[1]['osm2pgsql']) == '/secret/osm2pgsql'
+
+
+def test_replication_update_custom_threads(update_mock):
+    assert 0 == call_nominatim('--once', '--no-index', '--threads', '4')
+
+    assert update_mock.last_args[1]['threads'] == 4
+
+
+def test_replication_update_continuous(monkeypatch, init_status, index_mock):
+    states = [nominatim.tools.replication.UpdateState.UP_TO_DATE,
+              nominatim.tools.replication.UpdateState.UP_TO_DATE]
+    monkeypatch.setattr(nominatim.tools.replication, 'update',
+                        lambda *args, **kwargs: states.pop())
+
+    with pytest.raises(IndexError):
+        call_nominatim()
+
+    assert index_mock.called == 4
+
+
+def test_replication_update_continuous_no_change(monkeypatch, init_status, index_mock):
+    states = [nominatim.tools.replication.UpdateState.NO_CHANGES,
+              nominatim.tools.replication.UpdateState.UP_TO_DATE]
+    monkeypatch.setattr(nominatim.tools.replication, 'update',
+                        lambda *args, **kwargs: states.pop())
+
+    sleep_mock = MockParamCapture()
+    monkeypatch.setattr(time, 'sleep', sleep_mock)
+
+    with pytest.raises(IndexError):
+        call_nominatim()
+
+    assert index_mock.called == 2
+    assert sleep_mock.called == 1
+    assert sleep_mock.last_args[0] == 60