convert functon creation to python

The new functions always creates normal and partitioned functions.
Also adds specialised connection and cursor classes for adding
frequently used helper functions.

test/python/test_cli.py

@@ -98,7 +98,6 @@ def test_index_command(monkeypatch, temp_db_cursor, params, do_bnds, do_ranks):
 
 
 @pytest.mark.parametrize("command,params", [
-                         ('functions', ('setup.php',)),
                          ('wiki-data', ('setup.php', '--import-wikipedia-articles')),
                          ('importance', ('update.php', '--recompute-importance')),
                          ('website', ('setup.php', '--setup-website')),
@@ -114,6 +113,7 @@ def test_refresh_legacy_command(mock_run_legacy, command, params):
                          ('postcodes', 'update_postcodes'),
                          ('word-counts', 'recompute_word_counts'),
                          ('address-levels', 'load_address_levels_from_file'),
+                         ('functions', 'create_functions'),
                          ])
 def test_refresh_command(monkeypatch, command, func):
     func_mock = MockParamCapture()
@@ -129,7 +129,6 @@ def test_refresh_importance_computed_after_wiki_import(mock_run_legacy):
     assert mock_run_legacy.called == 2
     assert mock_run_legacy.last_args == ('update.php', '--recompute-importance')
 
-
 @pytest.mark.parametrize("params", [
                          ('search', '--query', 'new'),
                          ('reverse', '--lat', '0', '--lon', '0'),

test/python/test_config.py

@@ -15,6 +15,7 @@ def test_no_project_dir():
 
     assert config.DATABASE_WEBUSER == 'www-data'
 
+
 def test_prefer_project_setting_over_default():
     with tempfile.TemporaryDirectory() as project_dir:
         with open(project_dir + '/.env', 'w') as envfile:
@@ -24,6 +25,7 @@ def test_prefer_project_setting_over_default():
 
         assert config.DATABASE_WEBUSER == 'apache'
 
+
 def test_prefer_os_environ_over_project_setting(monkeypatch):
     with tempfile.TemporaryDirectory() as project_dir:
         with open(project_dir + '/.env', 'w') as envfile:
@@ -35,6 +37,7 @@ def test_prefer_os_environ_over_project_setting(monkeypatch):
 
         assert config.DATABASE_WEBUSER == 'nobody'
 
+
 def test_get_os_env_add_defaults(monkeypatch):
     config = Configuration(None, DEFCFG_DIR)
 
@@ -42,6 +45,7 @@ def test_get_os_env_add_defaults(monkeypatch):
 
     assert config.get_os_env()['NOMINATIM_DATABASE_WEBUSER'] == 'www-data'
 
+
 def test_get_os_env_prefer_os_environ(monkeypatch):
     config = Configuration(None, DEFCFG_DIR)
 
@@ -49,11 +53,13 @@ def test_get_os_env_prefer_os_environ(monkeypatch):
 
     assert config.get_os_env()['NOMINATIM_DATABASE_WEBUSER'] == 'nobody'
 
+
 def test_get_libpq_dsn_convert_default():
     config = Configuration(None, DEFCFG_DIR)
 
     assert config.get_libpq_dsn() == 'dbname=nominatim'
 
+
 def test_get_libpq_dsn_convert_php(monkeypatch):
     config = Configuration(None, DEFCFG_DIR)
 
@@ -62,6 +68,7 @@ def test_get_libpq_dsn_convert_php(monkeypatch):
 
     assert config.get_libpq_dsn() == 'dbname=gis password=foo host=localhost'
 
+
 def test_get_libpq_dsn_convert_libpq(monkeypatch):
     config = Configuration(None, DEFCFG_DIR)
 
@@ -69,3 +76,20 @@ def test_get_libpq_dsn_convert_libpq(monkeypatch):
                        'host=localhost dbname=gis password=foo')
 
     assert config.get_libpq_dsn() == 'host=localhost dbname=gis password=foo'
+
+
+@pytest.mark.parametrize("value,result",
+                         [(x, True) for x in ('1', 'true', 'True', 'yes', 'YES')] +
+                         [(x, False) for x in ('0', 'false', 'no', 'NO', 'x')])
+def test_get_bool(monkeypatch, value, result):
+    config = Configuration(None, DEFCFG_DIR)
+
+    monkeypatch.setenv('NOMINATIM_FOOBAR', value)
+
+    assert config.get_bool('FOOBAR') == result
+
+def test_get_bool_empty():
+    config = Configuration(None, DEFCFG_DIR)
+
+    assert config.DATABASE_MODULE_PATH == ''
+    assert config.get_bool('DATABASE_MODULE_PATH') == False

test/python/test_db_connection.py

@@ -0,0 +1,32 @@
+"""
+Tests for specialised conenction and cursor classes.
+"""
+import pytest
+
+from nominatim.db.connection import connect
+
+@pytest.fixture
+def db(temp_db):
+    conn = connect('dbname=' + temp_db)
+    yield conn
+    conn.close()
+
+
+def test_connection_table_exists(db, temp_db_cursor):
+    assert db.table_exists('foobar') == False
+
+    temp_db_cursor.execute('CREATE TABLE foobar (id INT)')
+
+    assert db.table_exists('foobar') == True
+
+
+def test_cursor_scalar(db, temp_db_cursor):
+    temp_db_cursor.execute('CREATE TABLE dummy (id INT)')
+
+    with db.cursor() as cur:
+        assert cur.scalar('SELECT count(*) FROM dummy') == 0
+
+def test_cursor_scalar_many_rows(db):
+    with db.cursor() as cur:
+        with pytest.raises(ValueError):
+            cur.scalar('SELECT * FROM pg_tables')

test/python/test_tools_refresh_create_functions.py

@@ -0,0 +1,99 @@
+"""
+Tests for creating PL/pgSQL functions for Nominatim.
+"""
+from pathlib import Path
+import pytest
+
+from nominatim.db.connection import connect
+from nominatim.tools.refresh import _get_standard_function_sql, _get_partition_function_sql
+
+SQL_DIR = (Path(__file__) / '..' / '..' / '..' / 'sql').resolve()
+
+@pytest.fixture
+def db(temp_db):
+    conn = connect('dbname=' + temp_db)
+    yield conn
+    conn.close()
+
+@pytest.fixture
+def db_with_tables(db):
+    with db.cursor() as cur:
+        for table in ('place', 'placex', 'location_postcode'):
+            cur.execute('CREATE TABLE {} (place_id BIGINT)'.format(table))
+
+    return db
+
+
+def test_standard_functions_replace_module_default(db, def_config):
+    def_config.project_dir = Path('.')
+    sql = _get_standard_function_sql(db, def_config, SQL_DIR, False, False)
+
+    assert sql
+    assert sql.find('{modulepath}') < 0
+    assert sql.find("'{}'".format(Path('module/nominatim.so').resolve())) >= 0
+
+
+def test_standard_functions_replace_module_custom(monkeypatch, db, def_config):
+    monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', 'custom')
+    sql = _get_standard_function_sql(db, def_config, SQL_DIR, False, False)
+
+    assert sql
+    assert sql.find('{modulepath}') < 0
+    assert sql.find("'custom/nominatim.so'") >= 0
+
+
+@pytest.mark.parametrize("enabled", (True, False))
+def test_standard_functions_enable_diff(db_with_tables, def_config, enabled):
+    def_config.project_dir = Path('.')
+    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, enabled, False)
+
+    assert sql
+    assert (sql.find('%DIFFUPDATES%') < 0) == enabled
+
+
+@pytest.mark.parametrize("enabled", (True, False))
+def test_standard_functions_enable_debug(db_with_tables, def_config, enabled):
+    def_config.project_dir = Path('.')
+    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, enabled)
+
+    assert sql
+    assert (sql.find('--DEBUG') < 0) == enabled
+
+
+@pytest.mark.parametrize("enabled", (True, False))
+def test_standard_functions_enable_limit_reindexing(monkeypatch, db_with_tables, def_config, enabled):
+    def_config.project_dir = Path('.')
+    monkeypatch.setenv('NOMINATIM_LIMIT_REINDEXING', 'yes' if enabled else 'no')
+    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
+
+    assert sql
+    assert (sql.find('--LIMIT INDEXING') < 0) == enabled
+
+
+@pytest.mark.parametrize("enabled", (True, False))
+def test_standard_functions_enable_tiger(monkeypatch, db_with_tables, def_config, enabled):
+    def_config.project_dir = Path('.')
+    monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA', 'yes' if enabled else 'no')
+    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
+
+    assert sql
+    assert (sql.find('%NOTIGERDATA%') >= 0) == enabled
+
+
+@pytest.mark.parametrize("enabled", (True, False))
+def test_standard_functions_enable_aux(monkeypatch, db_with_tables, def_config, enabled):
+    def_config.project_dir = Path('.')
+    monkeypatch.setenv('NOMINATIM_USE_AUX_LOCATION_DATA', 'yes' if enabled else 'no')
+    sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
+
+    assert sql
+    assert (sql.find('%NOAUXDATA%') >= 0) == enabled
+
+
+def test_partition_function(temp_db_cursor, db, def_config):
+    temp_db_cursor.execute("CREATE TABLE country_name (partition SMALLINT)")
+
+    sql = _get_partition_function_sql(db, SQL_DIR)
+
+    assert sql
+    assert sql.find('-partition-') < 0