port database setup function to python

Hide the former PHP functions in a transition command until
they are removed.

lib-php/Shell.php

@@ -48,7 +48,7 @@ class Shell
         return join(' ', $aEscaped);
     }
 
-    public function run()
+    public function run($bExitOnFail = False)
     {
         $sCmd = $this->escapedCmd();
         // $aEnv does not need escaping, proc_open seems to handle it fine
@@ -67,6 +67,11 @@ class Shell
         fclose($aPipes[0]); // no stdin
 
         $iStat = proc_close($hProc);
+
+        if ($iStat != 0 && $bExitOnFail) {
+            exit($iStat);
+        }
+
         return $iStat;
     }
 

lib-php/admin/setup.php

@@ -56,6 +56,15 @@ setupHTTPProxy();
 
 $bDidSomething = false;
 
+$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
+if (isset($aCMDResult['quiet']) && $aCMDResult['quiet']) {
+    $oNominatimCmd->addParams('--quiet');
+}
+if ($aCMDResult['verbose']) {
+    $oNominatimCmd->addParams('--verbose');
+}
+
+
 //*******************************************************
 // Making some sanity check:
 // Check if osm-file is set and points to a valid file
@@ -72,12 +81,12 @@ $oSetup = new SetupFunctions($aCMDResult);
 // go through complete process if 'all' is selected or start selected functions
 if ($aCMDResult['create-db'] || $aCMDResult['all']) {
     $bDidSomething = true;
-    $oSetup->createDB();
+    (clone($oNominatimCmd))->addParams('transition', '--create-db')->run(true);
 }
 
 if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
     $bDidSomething = true;
-    $oSetup->setupDB();
+    (clone($oNominatimCmd))->addParams('transition', '--setup-db')->run(true);
 }
 
 if ($aCMDResult['import-data'] || $aCMDResult['all']) {

lib-php/setup/SetupClass.php

@@ -84,96 +84,6 @@ class SetupFunctions
         }
     }
 
-    public function createDB()
-    {
-        info('Create DB');
-        $oDB = new \Nominatim\DB;
-
-        if ($oDB->checkConnection()) {
-            fail('database already exists ('.getSetting('DATABASE_DSN').')');
-        }
-
-        $oCmd = (new \Nominatim\Shell('createdb'))
-                ->addParams('-E', 'UTF-8')
-                ->addParams('-p', $this->aDSNInfo['port']);
-
-        if (isset($this->aDSNInfo['username'])) {
-            $oCmd->addParams('-U', $this->aDSNInfo['username']);
-        }
-        if (isset($this->aDSNInfo['password'])) {
-            $oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
-        }
-        if (isset($this->aDSNInfo['hostspec'])) {
-            $oCmd->addParams('-h', $this->aDSNInfo['hostspec']);
-        }
-        $oCmd->addParams($this->aDSNInfo['database']);
-
-        $result = $oCmd->run();
-        if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());
-    }
-
-    public function setupDB()
-    {
-        info('Setup DB');
-
-        $fPostgresVersion = $this->db()->getPostgresVersion();
-        echo 'Postgres version found: '.$fPostgresVersion."\n";
-
-        if ($fPostgresVersion < 9.03) {
-            fail('Minimum supported version of Postgresql is 9.3.');
-        }
-
-        $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
-        $this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
-
-        $fPostgisVersion = $this->db()->getPostgisVersion();
-        echo 'Postgis version found: '.$fPostgisVersion."\n";
-
-        if ($fPostgisVersion < 2.2) {
-            echo "Minimum required Postgis version 2.2\n";
-            exit(1);
-        }
-
-        $sPgUser = getSetting('DATABASE_WEBUSER');
-        $i = $this->db()->getOne("select count(*) from pg_user where usename = '$sPgUser'");
-        if ($i == 0) {
-            echo "\nERROR: Web user '".$sPgUser."' does not exist. Create it with:\n";
-            echo "\n          createuser ".$sPgUser."\n\n";
-            exit(1);
-        }
-
-        if (!getSetting('DATABASE_MODULE_PATH')) {
-            // If no custom module path is set then copy the module into the
-            // project directory, but only if it is not the same file already
-            // (aka we are running from the build dir).
-            $sDest = CONST_InstallDir.'/module';
-            if ($sDest != CONST_Default_ModulePath) {
-                if (!file_exists($sDest)) {
-                    mkdir($sDest);
-                }
-                if (!copy(CONST_Default_ModulePath.'/nominatim.so', $sDest.'/nominatim.so')) {
-                    echo "Failed to copy database module to $sDest.";
-                    exit(1);
-                }
-                chmod($sDest.'/nominatim.so', 0755);
-                info("Database module installed at $sDest.");
-            } else {
-                info('Running from build directory. Leaving database module as is.');
-            }
-        } else {
-            info('Using database module from DATABASE_MODULE_PATH ('.getSetting('DATABASE_MODULE_PATH').').');
-        }
-        // Try accessing the C module, so we know early if something is wrong
-        $this->checkModulePresence(); // raises exception on failure
-
-        $this->pgsqlRunScriptFile(CONST_DataDir.'/country_name.sql');
-        $this->pgsqlRunScriptFile(CONST_DataDir.'/country_osm_grid.sql.gz');
-
-        if ($this->bNoPartitions) {
-            $this->pgsqlRunScript('update country_name set partition = 0');
-        }
-    }
-
     public function importData($sOSMFile)
     {
         info('Import data');

nominatim/cli.py

@@ -354,4 +354,6 @@ def nominatim(**kwargs):
     else:
         parser.parser.epilog = 'php-cgi not found. Query commands not available.'
 
+    parser.add_subcommand('transition', clicmd.AdminTransition)
+
     return parser.run(**kwargs)

nominatim/clicmd/__init__.py

@@ -8,3 +8,4 @@ from .index import UpdateIndex
 from .refresh import UpdateRefresh
 from .admin import AdminFuncs
 from .freeze import SetupFreeze
+from .transition import AdminTransition

nominatim/clicmd/refresh.py

@@ -50,13 +50,11 @@ class UpdateRefresh:
 
         if args.postcodes:
             LOG.warning("Update postcodes centroid")
-            with connect(args.config.get_libpq_dsn()) as conn:
+            refresh.update_postcodes(args.config.get_libpq_dsn(), args.sqllib_dir)
-                refresh.update_postcodes(conn, args.sqllib_dir)
 
         if args.word_counts:
             LOG.warning('Recompute frequency of full-word search terms')
-            with connect(args.config.get_libpq_dsn()) as conn:
+            refresh.recompute_word_counts(args.config.get_libpq_dsn(), args.sqllib_dir)
-                refresh.recompute_word_counts(conn, args.sqllib_dir)
 
         if args.address_levels:
             cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)

nominatim/clicmd/transition.py

@@ -0,0 +1,53 @@
+"""
+Implementation of the 'transition' subcommand.
+
+This subcommand provides standins for functions that were available
+through the PHP scripts but are now no longer directly accessible.
+This module will be removed as soon as the transition phase is over.
+"""
+import logging
+
+from ..db.connection import connect
+
+# Do not repeat documentation of subcommand classes.
+# pylint: disable=C0111
+# Using non-top-level imports to avoid eventually unused imports.
+# pylint: disable=E0012,C0415
+
+LOG = logging.getLogger()
+
+class AdminTransition:
+    """\
+    Internal functions for code transition. Do not use.
+    """
+
+    @staticmethod
+    def add_args(parser):
+        group = parser.add_argument_group('Sub-functions')
+        group.add_argument('--create-db', action='store_true',
+                           help='Create nominatim db')
+        group.add_argument('--setup-db', action='store_true',
+                           help='Build a blank nominatim db')
+        group = parser.add_argument_group('Options')
+        group.add_argument('--no-partitions', action='store_true',
+                           help='Do not partition search indices')
+
+    @staticmethod
+    def run(args):
+        from ..tools import database_import
+
+        if args.create_db:
+            LOG.warning('Create DB')
+            database_import.create_db(args.config.get_libpq_dsn())
+
+        if args.setup_db:
+            LOG.warning('Setup DB')
+            mpath = database_import.install_module(args.module_dir, args.project_dir,
+                                                   args.config.DATABASE_MODULE_PATH)
+
+            with connect(args.config.get_libpq_dsn()) as conn:
+                database_import.setup_extensions(conn)
+                database_import.check_module_dir_path(conn, mpath)
+
+            database_import.import_base_data(args.config.get_libpq_dsn(),
+                                             args.data_dir, args.no_partitions)

nominatim/db/connection.py

@@ -81,9 +81,21 @@ class _Connection(psycopg2.extensions.connection):
         """
         version = self.server_version
         if version < 100000:
-            return (version / 10000, (version % 10000) / 100)
+            return (int(version / 10000), (version % 10000) / 100)
+
+        return (int(version / 10000), version % 10000)
+
+
+    def postgis_version_tuple(self):
+        """ Return the postgis version installed in the database as a
+            tuple of (major, minor). Assumes that the PostGIS extension
+            has been installed already.
+        """
+        with self.cursor() as cur:
+            version = cur.scalar('SELECT postgis_lib_version()')
+
+        return tuple((int(x) for x in version.split('.')[:2]))
 
-        return (version / 10000, version % 10000)
 
 def connect(dsn):
     """ Open a connection to the database using the specialised connection
@@ -123,7 +135,7 @@ _PG_CONNECTION_STRINGS = {
     'sslcrl': 'PGSSLCRL',
     'requirepeer': 'PGREQUIREPEER',
     'ssl_min_protocol_version': 'PGSSLMINPROTOCOLVERSION',
-    'ssl_min_protocol_version': 'PGSSLMAXPROTOCOLVERSION',
+    'ssl_max_protocol_version': 'PGSSLMAXPROTOCOLVERSION',
     'gssencmode': 'PGGSSENCMODE',
     'krbsrvname': 'PGKRBSRVNAME',
     'gsslib': 'PGGSSLIB',
@@ -138,7 +150,7 @@ def get_pg_env(dsn, base_env=None):
         If `base_env` is None, then the OS environment is used as a base
         environment.
     """
-    env = base_env if base_env is not None else os.environ
+    env = dict(base_env if base_env is not None else os.environ)
 
     for param, value in psycopg2.extensions.parse_dsn(dsn).items():
         if param in _PG_CONNECTION_STRINGS:

nominatim/db/utils.py

@@ -3,12 +3,24 @@ Helper functions for handling DB accesses.
 """
 import subprocess
 import logging
+import gzip
 
 from .connection import get_pg_env
 from ..errors import UsageError
 
 LOG = logging.getLogger()
 
+def _pipe_to_proc(proc, fdesc):
+    chunk = fdesc.read(2048)
+    while chunk and proc.poll() is None:
+        try:
+            proc.stdin.write(chunk)
+        except BrokenPipeError as exc:
+            raise UsageError("Failed to execute SQL file.") from exc
+        chunk = fdesc.read(2048)
+
+    return len(chunk)
+
 def execute_file(dsn, fname, ignore_errors=False):
     """ Read an SQL file and run its contents against the given database
         using psql.
@@ -21,15 +33,15 @@ def execute_file(dsn, fname, ignore_errors=False):
     if not LOG.isEnabledFor(logging.INFO):
         proc.stdin.write('set client_min_messages to WARNING;'.encode('utf-8'))
 
-    with fname.open('rb') as fdesc:
+    if fname.suffix == '.gz':
-        chunk = fdesc.read(2048)
+        with gzip.open(str(fname), 'rb') as fdesc:
-        while chunk and proc.poll() is None:
+            remain = _pipe_to_proc(proc, fdesc)
-            proc.stdin.write(chunk)
+    else:
-            chunk = fdesc.read(2048)
+        with fname.open('rb') as fdesc:
+            remain = _pipe_to_proc(proc, fdesc)
 
     proc.stdin.close()
 
     ret = proc.wait()
-    print(ret, chunk)
+    if ret != 0 or remain > 0:
-    if ret != 0 or chunk:
         raise UsageError("Failed to execute SQL file.")

nominatim/tools/database_import.py

@@ -0,0 +1,121 @@
+"""
+Functions for setting up and importing a new Nominatim database.
+"""
+import logging
+import subprocess
+import shutil
+
+from ..db.connection import connect, get_pg_env
+from ..db import utils as db_utils
+from ..errors import UsageError
+from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION
+
+LOG = logging.getLogger()
+
+def create_db(dsn, rouser=None):
+    """ Create a new database for the given DSN. Fails when the database
+        already exists or the PostgreSQL version is too old.
+        Uses `createdb` to create the database.
+
+        If 'rouser' is given, then the function also checks that the user
+        with that given name exists.
+
+        Requires superuser rights by the caller.
+    """
+    proc = subprocess.run(['createdb'], env=get_pg_env(dsn), check=False)
+
+    if proc.returncode != 0:
+        raise UsageError('Creating new database failed.')
+
+    with connect(dsn) as conn:
+        postgres_version = conn.server_version_tuple() # pylint: disable=E1101
+        if postgres_version < POSTGRESQL_REQUIRED_VERSION:
+            LOG.fatal('Minimum supported version of Postgresql is %d.%d. '
+                      'Found version %d.%d.',
+                      POSTGRESQL_REQUIRED_VERSION[0], POSTGRESQL_REQUIRED_VERSION[1],
+                      postgres_version[0], postgres_version[1])
+            raise UsageError('PostgreSQL server is too old.')
+
+        if rouser is not None:
+            with conn.cursor() as cur:  # pylint: disable=E1101
+                cnt = cur.scalar('SELECT count(*) FROM pg_user where usename = %s',
+                                 (rouser, ))
+                if cnt == 0:
+                    LOG.fatal("Web user '%s' does not exists. Create it with:\n"
+                              "\n      createuser %s", rouser, rouser)
+                    raise UsageError('Missing read-only user.')
+
+
+
+def setup_extensions(conn):
+    """ Set up all extensions needed for Nominatim. Also checks that the
+        versions of the extensions are sufficient.
+    """
+    with conn.cursor() as cur:
+        cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
+        cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
+    conn.commit()
+
+    postgis_version = conn.postgis_version_tuple()
+    if postgis_version < POSTGIS_REQUIRED_VERSION:
+        LOG.fatal('Minimum supported version of PostGIS is %d.%d. '
+                  'Found version %d.%d.',
+                  POSTGIS_REQUIRED_VERSION[0], POSTGIS_REQUIRED_VERSION[1],
+                  postgis_version[0], postgis_version[1])
+        raise UsageError('PostGIS version is too old.')
+
+
+def install_module(src_dir, project_dir, module_dir):
+    """ Copy the normalization module from src_dir into the project
+        directory under the '/module' directory. If 'module_dir' is set, then
+        use the module from there instead and check that it is accessible
+        for Postgresql.
+
+        The function detects when the installation is run from the
+        build directory. It doesn't touch the module in that case.
+    """
+    if not module_dir:
+        module_dir = project_dir / 'module'
+
+        if not module_dir.exists() or not src_dir.samefile(module_dir):
+
+            if not module_dir.exists():
+                module_dir.mkdir()
+
+            destfile = module_dir / 'nominatim.so'
+            shutil.copy(str(src_dir / 'nominatim.so'), str(destfile))
+            destfile.chmod(0o755)
+
+            LOG.info('Database module installed at %s', str(destfile))
+        else:
+            LOG.info('Running from build directory. Leaving database module as is.')
+    else:
+        LOG.info("Using custom path for database module at '%s'", module_dir)
+
+    return module_dir
+
+
+def check_module_dir_path(conn, path):
+    """ Check that the normalisation module can be found and executed
+        from the given path.
+    """
+    with conn.cursor() as cur:
+        cur.execute("""CREATE FUNCTION nominatim_test_import_func(text)
+                       RETURNS text AS '{}/nominatim.so', 'transliteration'
+                       LANGUAGE c IMMUTABLE STRICT;
+                       DROP FUNCTION nominatim_test_import_func(text)
+                    """.format(path))
+
+
+def import_base_data(dsn, sql_dir, ignore_partitions=False):
+    """ Create and populate the tables with basic static data that provides
+        the background for geocoding.
+    """
+    db_utils.execute_file(dsn, sql_dir / 'country_name.sql')
+    db_utils.execute_file(dsn, sql_dir / 'country_osm_grid.sql.gz')
+
+    if ignore_partitions:
+        with connect(dsn) as conn:
+            with conn.cursor() as cur:  # pylint: disable=E1101
+                cur.execute('UPDATE country_name SET partition = 0')
+            conn.commit()  # pylint: disable=E1101

nominatim/tools/exec_utils.py

@@ -2,13 +2,10 @@
 Helper functions for executing external programs.
 """
 import logging
-import os
 import subprocess
 import urllib.request as urlrequest
 from urllib.parse import urlencode
 
-from psycopg2.extensions import parse_dsn
-
 from ..version import NOMINATIM_VERSION
 from ..db.connection import get_pg_env
 

nominatim/version.py

@@ -3,3 +3,6 @@ Version information for Nominatim.
 """
 
 NOMINATIM_VERSION = "3.6.0"
+
+POSTGRESQL_REQUIRED_VERSION = (9, 3)
+POSTGIS_REQUIRED_VERSION = (2, 2)

test/Makefile

@@ -10,5 +10,8 @@ bdd-no-test-db:
 php:
 	cd php && phpunit ./
 
+python:
+	pytest python
+
 
 .PHONY: bdd php no-test-db

test/python/conftest.py

@@ -105,6 +105,9 @@ def temp_db_cursor(temp_db):
 def def_config():
     return Configuration(None, SRC_DIR.resolve() / 'settings')
 
+@pytest.fixture
+def src_dir():
+    return SRC_DIR.resolve()
 
 @pytest.fixture
 def status_table(temp_db_conn):

test/python/test_cli.py

@@ -6,9 +6,11 @@ correct functionionality. They use a lot of monkeypatching to avoid executing
 the actual functions.
 """
 import datetime as dt
+import time
+from pathlib import Path
+
 import psycopg2
 import pytest
-import time
 
 import nominatim.cli
 import nominatim.clicmd.api
@@ -23,14 +25,16 @@ import nominatim.tools.replication
 from nominatim.errors import UsageError
 from nominatim.db import status
 
+SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()
+
 def call_nominatim(*args):
     return nominatim.cli.nominatim(module_dir='build/module',
                                    osm2pgsql_path='build/osm2pgsql/osm2pgsql',
-                                   phplib_dir='lib-php',
+                                   phplib_dir=str(SRC_DIR / 'lib-php'),
-                                   data_dir='.',
+                                   data_dir=str(SRC_DIR / 'data'),
                                    phpcgi_path='/usr/bin/php-cgi',
-                                   sqllib_dir='lib-sql',
+                                   sqllib_dir=str(SRC_DIR / 'lib-sql'),
-                                   config_dir='settings',
+                                   config_dir=str(SRC_DIR / 'settings'),
                                    cli_args=args)
 
 class MockParamCapture:

test/python/test_db_connection.py

@@ -37,6 +37,17 @@ def test_connection_server_version_tuple(db):
     assert len(ver) == 2
     assert ver[0] > 8
 
+
+def test_connection_postgis_version_tuple(db, temp_db_cursor):
+    temp_db_cursor.execute('CREATE EXTENSION postgis')
+
+    ver = db.postgis_version_tuple()
+
+    assert isinstance(ver, tuple)
+    assert len(ver) == 2
+    assert ver[0] >= 2
+
+
 def test_cursor_scalar(db, temp_db_cursor):
     temp_db_cursor.execute('CREATE TABLE dummy (id INT)')
 

test/python/test_db_utils.py

@@ -26,6 +26,7 @@ def test_execute_file_bad_file(dsn, tmp_path):
     with pytest.raises(FileNotFoundError):
         db_utils.execute_file(dsn, tmp_path / 'test2.sql')
 
+
 def test_execute_file_bad_sql(dsn, tmp_path):
     tmpfile = tmp_path / 'test.sql'
     tmpfile.write_text('CREATE STABLE test (id INT)')

test/python/test_tools_database_import.py

@@ -0,0 +1,96 @@
+"""
+Tests for functions to import a new database.
+"""
+import pytest
+import psycopg2
+import sys
+
+from nominatim.tools import database_import
+from nominatim.errors import UsageError
+
+@pytest.fixture
+def nonexistant_db():
+    dbname = 'test_nominatim_python_unittest'
+
+    conn = psycopg2.connect(database='postgres')
+
+    conn.set_isolation_level(0)
+    with conn.cursor() as cur:
+        cur.execute('DROP DATABASE IF EXISTS {}'.format(dbname))
+
+    yield dbname
+
+    with conn.cursor() as cur:
+        cur.execute('DROP DATABASE IF EXISTS {}'.format(dbname))
+
+
+def test_create_db_success(nonexistant_db):
+    database_import.create_db('dbname=' + nonexistant_db, rouser='www-data')
+
+    conn = psycopg2.connect(database=nonexistant_db)
+    conn.close()
+
+
+def test_create_db_already_exists(temp_db):
+    with pytest.raises(UsageError):
+        database_import.create_db('dbname=' + temp_db)
+
+
+def test_create_db_unsupported_version(nonexistant_db, monkeypatch):
+    monkeypatch.setattr(database_import, 'POSTGRESQL_REQUIRED_VERSION', (100, 4))
+
+    with pytest.raises(UsageError, match='PostgreSQL server is too old.'):
+        database_import.create_db('dbname=' + nonexistant_db)
+
+
+def test_create_db_missing_ro_user(nonexistant_db):
+    with pytest.raises(UsageError, match='Missing read-only user.'):
+        database_import.create_db('dbname=' + nonexistant_db, rouser='sdfwkjkjgdugu2;jgsafkljas;')
+
+
+def test_setup_extensions(temp_db_conn, temp_db_cursor):
+    database_import.setup_extensions(temp_db_conn)
+
+    temp_db_cursor.execute('CREATE TABLE t (h HSTORE, geom GEOMETRY(Geometry, 4326))')
+
+
+def test_setup_extensions_old_postgis(temp_db_conn, monkeypatch):
+    monkeypatch.setattr(database_import, 'POSTGIS_REQUIRED_VERSION', (50, 50))
+
+    with pytest.raises(UsageError, match='PostGIS version is too old.'):
+        database_import.setup_extensions(temp_db_conn)
+
+
+def test_install_module(tmp_path):
+    src_dir = tmp_path / 'source'
+    src_dir.mkdir()
+    (src_dir / 'nominatim.so').write_text('TEST nomiantim.so')
+
+    project_dir = tmp_path / 'project'
+    project_dir.mkdir()
+
+    database_import.install_module(src_dir, project_dir, '')
+
+    outfile = project_dir / 'module' / 'nominatim.so'
+
+    assert outfile.exists()
+    assert outfile.read_text() == 'TEST nomiantim.so'
+    assert outfile.stat().st_mode == 33261
+
+
+def test_import_base_data(src_dir, temp_db, temp_db_cursor):
+    temp_db_cursor.execute('CREATE EXTENSION hstore')
+    temp_db_cursor.execute('CREATE EXTENSION postgis')
+    database_import.import_base_data('dbname=' + temp_db, src_dir / 'data')
+
+    assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0
+
+
+def test_import_base_data_ignore_partitions(src_dir, temp_db, temp_db_cursor):
+    temp_db_cursor.execute('CREATE EXTENSION hstore')
+    temp_db_cursor.execute('CREATE EXTENSION postgis')
+    database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
+                                     ignore_partitions=True)
+
+    assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0
+    assert temp_db_cursor.scalar('SELECT count(*) FROM country_name WHERE partition != 0') == 0