hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-13 22:34:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

port database setup function to python

Browse Source 
Hide the former PHP functions in a transition command until
they are removed.
This commit is contained in:
Sarah Hoffmann
2021-02-23 22:50:23 +01:00
parent b93ec2522e
commit f6e894a53a
18 changed files with 357 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
lib-php/Shell.php
View File

@@ -48,7 +48,7 @@ class Shell
return join(' ', $aEscaped);
}
public function run()
public function run($bExitOnFail = False)
{
$sCmd = $this->escapedCmd();
// $aEnv does not need escaping, proc_open seems to handle it fine
@@ -67,6 +67,11 @@ class Shell
fclose($aPipes[0]); // no stdin
$iStat = proc_close($hProc);
if ($iStat != 0 && $bExitOnFail) {
exit($iStat);
}
return $iStat;
}

13
lib-php/admin/setup.php
View File

@@ -56,6 +56,15 @@ setupHTTPProxy();
$bDidSomething = false;
$oNominatimCmd = new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'));
if (isset($aCMDResult['quiet']) && $aCMDResult['quiet']) {
$oNominatimCmd->addParams('--quiet');
}
if ($aCMDResult['verbose']) {
$oNominatimCmd->addParams('--verbose');
}
//*******************************************************
// Making some sanity check:
// Check if osm-file is set and points to a valid file
@@ -72,12 +81,12 @@ $oSetup = new SetupFunctions($aCMDResult);
// go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createDB();
(clone($oNominatimCmd))->addParams('transition', '--create-db')->run(true);
}
if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->setupDB();
(clone($oNominatimCmd))->addParams('transition', '--setup-db')->run(true);
}
if ($aCMDResult['import-data'] || $aCMDResult['all']) {

90
lib-php/setup/SetupClass.php
View File

@@ -84,96 +84,6 @@ class SetupFunctions
}
}
public function createDB()
{
info('Create DB');
$oDB = new \Nominatim\DB;
if ($oDB->checkConnection()) {
fail('database already exists ('.getSetting('DATABASE_DSN').')');
}
$oCmd = (new \Nominatim\Shell('createdb'))
->addParams('-E', 'UTF-8')
->addParams('-p', $this->aDSNInfo['port']);
if (isset($this->aDSNInfo['username'])) {
$oCmd->addParams('-U', $this->aDSNInfo['username']);
}
if (isset($this->aDSNInfo['password'])) {
$oCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']);
}
if (isset($this->aDSNInfo['hostspec'])) {
$oCmd->addParams('-h', $this->aDSNInfo['hostspec']);
}
$oCmd->addParams($this->aDSNInfo['database']);
$result = $oCmd->run();
if ($result != 0) fail('Error executing external command: '.$oCmd->escapedCmd());
}
public function setupDB()
{
info('Setup DB');
$fPostgresVersion = $this->db()->getPostgresVersion();
echo 'Postgres version found: '.$fPostgresVersion."\n";
if ($fPostgresVersion < 9.03) {
fail('Minimum supported version of Postgresql is 9.3.');
}
$this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS hstore');
$this->pgsqlRunScript('CREATE EXTENSION IF NOT EXISTS postgis');
$fPostgisVersion = $this->db()->getPostgisVersion();
echo 'Postgis version found: '.$fPostgisVersion."\n";
if ($fPostgisVersion < 2.2) {
echo "Minimum required Postgis version 2.2\n";
exit(1);
}
$sPgUser = getSetting('DATABASE_WEBUSER');
$i = $this->db()->getOne("select count(*) from pg_user where usename = '$sPgUser'");
if ($i == 0) {
echo "\nERROR: Web user '".$sPgUser."' does not exist. Create it with:\n";
echo "\n createuser ".$sPgUser."\n\n";
exit(1);
}
if (!getSetting('DATABASE_MODULE_PATH')) {
// If no custom module path is set then copy the module into the
// project directory, but only if it is not the same file already
// (aka we are running from the build dir).
$sDest = CONST_InstallDir.'/module';
if ($sDest != CONST_Default_ModulePath) {
if (!file_exists($sDest)) {
mkdir($sDest);
}
if (!copy(CONST_Default_ModulePath.'/nominatim.so', $sDest.'/nominatim.so')) {
echo "Failed to copy database module to $sDest.";
exit(1);
}
chmod($sDest.'/nominatim.so', 0755);
info("Database module installed at $sDest.");
} else {
info('Running from build directory. Leaving database module as is.');
}
} else {
info('Using database module from DATABASE_MODULE_PATH ('.getSetting('DATABASE_MODULE_PATH').').');
}
// Try accessing the C module, so we know early if something is wrong
$this->checkModulePresence(); // raises exception on failure
$this->pgsqlRunScriptFile(CONST_DataDir.'/country_name.sql');
$this->pgsqlRunScriptFile(CONST_DataDir.'/country_osm_grid.sql.gz');
if ($this->bNoPartitions) {
$this->pgsqlRunScript('update country_name set partition = 0');
}
}
public function importData($sOSMFile)
{
info('Import data');

2
nominatim/cli.py
View File

@@ -354,4 +354,6 @@ def nominatim(**kwargs):
else:
parser.parser.epilog = 'php-cgi not found. Query commands not available.'
parser.add_subcommand('transition', clicmd.AdminTransition)
return parser.run(**kwargs)

1
nominatim/clicmd/__init__.py
View File

@@ -8,3 +8,4 @@ from .index import UpdateIndex
from .refresh import UpdateRefresh
from .admin import AdminFuncs
from .freeze import SetupFreeze
from .transition import AdminTransition

6
nominatim/clicmd/refresh.py
View File

@@ -50,13 +50,11 @@ class UpdateRefresh:
if args.postcodes:
LOG.warning("Update postcodes centroid")
with connect(args.config.get_libpq_dsn()) as conn:
refresh.update_postcodes(conn, args.sqllib_dir)
refresh.update_postcodes(args.config.get_libpq_dsn(), args.sqllib_dir)
if args.word_counts:
LOG.warning('Recompute frequency of full-word search terms')
with connect(args.config.get_libpq_dsn()) as conn:
refresh.recompute_word_counts(conn, args.sqllib_dir)
refresh.recompute_word_counts(args.config.get_libpq_dsn(), args.sqllib_dir)
if args.address_levels:
cfg = Path(args.config.ADDRESS_LEVEL_CONFIG)

53
nominatim/clicmd/transition.py Normal file
View File

@@ -0,0 +1,53 @@
"""
Implementation of the 'transition' subcommand.
This subcommand provides standins for functions that were available
through the PHP scripts but are now no longer directly accessible.
This module will be removed as soon as the transition phase is over.
"""
import logging
from ..db.connection import connect
# Do not repeat documentation of subcommand classes.
# pylint: disable=C0111
# Using non-top-level imports to avoid eventually unused imports.
# pylint: disable=E0012,C0415
LOG = logging.getLogger()
class AdminTransition:
"""\
Internal functions for code transition. Do not use.
"""
@staticmethod
def add_args(parser):
group = parser.add_argument_group('Sub-functions')
group.add_argument('--create-db', action='store_true',
help='Create nominatim db')
group.add_argument('--setup-db', action='store_true',
help='Build a blank nominatim db')
group = parser.add_argument_group('Options')
group.add_argument('--no-partitions', action='store_true',
help='Do not partition search indices')
@staticmethod
def run(args):
from ..tools import database_import
if args.create_db:
LOG.warning('Create DB')
database_import.create_db(args.config.get_libpq_dsn())
if args.setup_db:
LOG.warning('Setup DB')
mpath = database_import.install_module(args.module_dir, args.project_dir,
args.config.DATABASE_MODULE_PATH)
with connect(args.config.get_libpq_dsn()) as conn:
database_import.setup_extensions(conn)
database_import.check_module_dir_path(conn, mpath)
database_import.import_base_data(args.config.get_libpq_dsn(),
args.data_dir, args.no_partitions)

20
nominatim/db/connection.py
View File

@@ -81,9 +81,21 @@ class _Connection(psycopg2.extensions.connection):
"""
version = self.server_version
if version < 100000:
return (version / 10000, (version % 10000) / 100)
return (int(version / 10000), (version % 10000) / 100)
return (int(version / 10000), version % 10000)
def postgis_version_tuple(self):
""" Return the postgis version installed in the database as a
tuple of (major, minor). Assumes that the PostGIS extension
has been installed already.
"""
with self.cursor() as cur:
version = cur.scalar('SELECT postgis_lib_version()')
return tuple((int(x) for x in version.split('.')[:2]))
return (version / 10000, version % 10000)
def connect(dsn):
""" Open a connection to the database using the specialised connection
@@ -123,7 +135,7 @@ _PG_CONNECTION_STRINGS = {
'sslcrl': 'PGSSLCRL',
'requirepeer': 'PGREQUIREPEER',
'ssl_min_protocol_version': 'PGSSLMINPROTOCOLVERSION',
'ssl_min_protocol_version': 'PGSSLMAXPROTOCOLVERSION',
'ssl_max_protocol_version': 'PGSSLMAXPROTOCOLVERSION',
'gssencmode': 'PGGSSENCMODE',
'krbsrvname': 'PGKRBSRVNAME',
'gsslib': 'PGGSSLIB',
@@ -138,7 +150,7 @@ def get_pg_env(dsn, base_env=None):
If `base_env` is None, then the OS environment is used as a base
environment.
"""
env = base_env if base_env is not None else os.environ
env = dict(base_env if base_env is not None else os.environ)
for param, value in psycopg2.extensions.parse_dsn(dsn).items():
if param in _PG_CONNECTION_STRINGS:

26
nominatim/db/utils.py
View File

@@ -3,12 +3,24 @@ Helper functions for handling DB accesses.
"""
import subprocess
import logging
import gzip
from .connection import get_pg_env
from ..errors import UsageError
LOG = logging.getLogger()
def _pipe_to_proc(proc, fdesc):
chunk = fdesc.read(2048)
while chunk and proc.poll() is None:
try:
proc.stdin.write(chunk)
except BrokenPipeError as exc:
raise UsageError("Failed to execute SQL file.") from exc
chunk = fdesc.read(2048)
return len(chunk)
def execute_file(dsn, fname, ignore_errors=False):
""" Read an SQL file and run its contents against the given database
using psql.
@@ -21,15 +33,15 @@ def execute_file(dsn, fname, ignore_errors=False):
if not LOG.isEnabledFor(logging.INFO):
proc.stdin.write('set client_min_messages to WARNING;'.encode('utf-8'))
with fname.open('rb') as fdesc:
chunk = fdesc.read(2048)
while chunk and proc.poll() is None:
proc.stdin.write(chunk)
chunk = fdesc.read(2048)
if fname.suffix == '.gz':
with gzip.open(str(fname), 'rb') as fdesc:
remain = _pipe_to_proc(proc, fdesc)
else:
with fname.open('rb') as fdesc:
remain = _pipe_to_proc(proc, fdesc)
proc.stdin.close()
ret = proc.wait()
print(ret, chunk)
if ret != 0 or chunk:
if ret != 0 or remain > 0:
raise UsageError("Failed to execute SQL file.")

121
nominatim/tools/database_import.py Normal file
View File

@@ -0,0 +1,121 @@
"""
Functions for setting up and importing a new Nominatim database.
"""
import logging
import subprocess
import shutil
from ..db.connection import connect, get_pg_env
from ..db import utils as db_utils
from ..errors import UsageError
from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION
LOG = logging.getLogger()
def create_db(dsn, rouser=None):
""" Create a new database for the given DSN. Fails when the database
already exists or the PostgreSQL version is too old.
Uses `createdb` to create the database.
If 'rouser' is given, then the function also checks that the user
with that given name exists.
Requires superuser rights by the caller.
"""
proc = subprocess.run(['createdb'], env=get_pg_env(dsn), check=False)
if proc.returncode != 0:
raise UsageError('Creating new database failed.')
with connect(dsn) as conn:
postgres_version = conn.server_version_tuple() # pylint: disable=E1101
if postgres_version < POSTGRESQL_REQUIRED_VERSION:
LOG.fatal('Minimum supported version of Postgresql is %d.%d. '
'Found version %d.%d.',
POSTGRESQL_REQUIRED_VERSION[0], POSTGRESQL_REQUIRED_VERSION[1],
postgres_version[0], postgres_version[1])
raise UsageError('PostgreSQL server is too old.')
if rouser is not None:
with conn.cursor() as cur: # pylint: disable=E1101
cnt = cur.scalar('SELECT count(*) FROM pg_user where usename = %s',
(rouser, ))
if cnt == 0:
LOG.fatal("Web user '%s' does not exists. Create it with:\n"
"\n createuser %s", rouser, rouser)
raise UsageError('Missing read-only user.')
def setup_extensions(conn):
""" Set up all extensions needed for Nominatim. Also checks that the
versions of the extensions are sufficient.
"""
with conn.cursor() as cur:
cur.execute('CREATE EXTENSION IF NOT EXISTS hstore')
cur.execute('CREATE EXTENSION IF NOT EXISTS postgis')
conn.commit()
postgis_version = conn.postgis_version_tuple()
if postgis_version < POSTGIS_REQUIRED_VERSION:
LOG.fatal('Minimum supported version of PostGIS is %d.%d. '
'Found version %d.%d.',
POSTGIS_REQUIRED_VERSION[0], POSTGIS_REQUIRED_VERSION[1],
postgis_version[0], postgis_version[1])
raise UsageError('PostGIS version is too old.')
def install_module(src_dir, project_dir, module_dir):
""" Copy the normalization module from src_dir into the project
directory under the '/module' directory. If 'module_dir' is set, then
use the module from there instead and check that it is accessible
for Postgresql.
The function detects when the installation is run from the
build directory. It doesn't touch the module in that case.
"""
if not module_dir:
module_dir = project_dir / 'module'
if not module_dir.exists() or not src_dir.samefile(module_dir):
if not module_dir.exists():
module_dir.mkdir()
destfile = module_dir / 'nominatim.so'
shutil.copy(str(src_dir / 'nominatim.so'), str(destfile))
destfile.chmod(0o755)
LOG.info('Database module installed at %s', str(destfile))
else:
LOG.info('Running from build directory. Leaving database module as is.')
else:
LOG.info("Using custom path for database module at '%s'", module_dir)
return module_dir
def check_module_dir_path(conn, path):
""" Check that the normalisation module can be found and executed
from the given path.
"""
with conn.cursor() as cur:
cur.execute("""CREATE FUNCTION nominatim_test_import_func(text)
RETURNS text AS '{}/nominatim.so', 'transliteration'
LANGUAGE c IMMUTABLE STRICT;
DROP FUNCTION nominatim_test_import_func(text)
""".format(path))
def import_base_data(dsn, sql_dir, ignore_partitions=False):
""" Create and populate the tables with basic static data that provides
the background for geocoding.
"""
db_utils.execute_file(dsn, sql_dir / 'country_name.sql')
db_utils.execute_file(dsn, sql_dir / 'country_osm_grid.sql.gz')
if ignore_partitions:
with connect(dsn) as conn:
with conn.cursor() as cur: # pylint: disable=E1101
cur.execute('UPDATE country_name SET partition = 0')
conn.commit() # pylint: disable=E1101

3
nominatim/tools/exec_utils.py
View File

@@ -2,13 +2,10 @@
Helper functions for executing external programs.
"""
import logging
import os
import subprocess
import urllib.request as urlrequest
from urllib.parse import urlencode
from psycopg2.extensions import parse_dsn
from ..version import NOMINATIM_VERSION
from ..db.connection import get_pg_env

3
nominatim/version.py
View File

@@ -3,3 +3,6 @@ Version information for Nominatim.
"""
NOMINATIM_VERSION = "3.6.0"
POSTGRESQL_REQUIRED_VERSION = (9, 3)
POSTGIS_REQUIRED_VERSION = (2, 2)

3
test/Makefile
View File

@@ -10,5 +10,8 @@ bdd-no-test-db:
php:
cd php && phpunit ./
python:
pytest python
.PHONY: bdd php no-test-db

3
test/python/conftest.py
View File

@@ -105,6 +105,9 @@ def temp_db_cursor(temp_db):
def def_config():
return Configuration(None, SRC_DIR.resolve() / 'settings')
@pytest.fixture
def src_dir():
return SRC_DIR.resolve()
@pytest.fixture
def status_table(temp_db_conn):

14
test/python/test_cli.py
View File

@@ -6,9 +6,11 @@ correct functionionality. They use a lot of monkeypatching to avoid executing
the actual functions.
"""
import datetime as dt
import time
from pathlib import Path
import psycopg2
import pytest
import time
import nominatim.cli
import nominatim.clicmd.api
@@ -23,14 +25,16 @@ import nominatim.tools.replication
from nominatim.errors import UsageError
from nominatim.db import status
SRC_DIR = (Path(__file__) / '..' / '..' / '..').resolve()
def call_nominatim(*args):
return nominatim.cli.nominatim(module_dir='build/module',
osm2pgsql_path='build/osm2pgsql/osm2pgsql',
phplib_dir='lib-php',
data_dir='.',
phplib_dir=str(SRC_DIR / 'lib-php'),
data_dir=str(SRC_DIR / 'data'),
phpcgi_path='/usr/bin/php-cgi',
sqllib_dir='lib-sql',
config_dir='settings',
sqllib_dir=str(SRC_DIR / 'lib-sql'),
config_dir=str(SRC_DIR / 'settings'),
cli_args=args)
class MockParamCapture:

11
test/python/test_db_connection.py
View File

@@ -37,6 +37,17 @@ def test_connection_server_version_tuple(db):
assert len(ver) == 2
assert ver[0] > 8
def test_connection_postgis_version_tuple(db, temp_db_cursor):
temp_db_cursor.execute('CREATE EXTENSION postgis')
ver = db.postgis_version_tuple()
assert isinstance(ver, tuple)
assert len(ver) == 2
assert ver[0] >= 2
def test_cursor_scalar(db, temp_db_cursor):
temp_db_cursor.execute('CREATE TABLE dummy (id INT)')

1
test/python/test_db_utils.py
View File

@@ -26,6 +26,7 @@ def test_execute_file_bad_file(dsn, tmp_path):
with pytest.raises(FileNotFoundError):
db_utils.execute_file(dsn, tmp_path / 'test2.sql')
def test_execute_file_bad_sql(dsn, tmp_path):
tmpfile = tmp_path / 'test.sql'
tmpfile.write_text('CREATE STABLE test (id INT)')

96
test/python/test_tools_database_import.py Normal file
View File

@@ -0,0 +1,96 @@
"""
Tests for functions to import a new database.
"""
import pytest
import psycopg2
import sys
from nominatim.tools import database_import
from nominatim.errors import UsageError
@pytest.fixture
def nonexistant_db():
dbname = 'test_nominatim_python_unittest'
conn = psycopg2.connect(database='postgres')
conn.set_isolation_level(0)
with conn.cursor() as cur:
cur.execute('DROP DATABASE IF EXISTS {}'.format(dbname))
yield dbname
with conn.cursor() as cur:
cur.execute('DROP DATABASE IF EXISTS {}'.format(dbname))
def test_create_db_success(nonexistant_db):
database_import.create_db('dbname=' + nonexistant_db, rouser='www-data')
conn = psycopg2.connect(database=nonexistant_db)
conn.close()
def test_create_db_already_exists(temp_db):
with pytest.raises(UsageError):
database_import.create_db('dbname=' + temp_db)
def test_create_db_unsupported_version(nonexistant_db, monkeypatch):
monkeypatch.setattr(database_import, 'POSTGRESQL_REQUIRED_VERSION', (100, 4))
with pytest.raises(UsageError, match='PostgreSQL server is too old.'):
database_import.create_db('dbname=' + nonexistant_db)
def test_create_db_missing_ro_user(nonexistant_db):
with pytest.raises(UsageError, match='Missing read-only user.'):
database_import.create_db('dbname=' + nonexistant_db, rouser='sdfwkjkjgdugu2;jgsafkljas;')
def test_setup_extensions(temp_db_conn, temp_db_cursor):
database_import.setup_extensions(temp_db_conn)
temp_db_cursor.execute('CREATE TABLE t (h HSTORE, geom GEOMETRY(Geometry, 4326))')
def test_setup_extensions_old_postgis(temp_db_conn, monkeypatch):
monkeypatch.setattr(database_import, 'POSTGIS_REQUIRED_VERSION', (50, 50))
with pytest.raises(UsageError, match='PostGIS version is too old.'):
database_import.setup_extensions(temp_db_conn)
def test_install_module(tmp_path):
src_dir = tmp_path / 'source'
src_dir.mkdir()
(src_dir / 'nominatim.so').write_text('TEST nomiantim.so')
project_dir = tmp_path / 'project'
project_dir.mkdir()
database_import.install_module(src_dir, project_dir, '')
outfile = project_dir / 'module' / 'nominatim.so'
assert outfile.exists()
assert outfile.read_text() == 'TEST nomiantim.so'
assert outfile.stat().st_mode == 33261
def test_import_base_data(src_dir, temp_db, temp_db_cursor):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
database_import.import_base_data('dbname=' + temp_db, src_dir / 'data')
assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0
def test_import_base_data_ignore_partitions(src_dir, temp_db, temp_db_cursor):
temp_db_cursor.execute('CREATE EXTENSION hstore')
temp_db_cursor.execute('CREATE EXTENSION postgis')
database_import.import_base_data('dbname=' + temp_db, src_dir / 'data',
ignore_partitions=True)
assert temp_db_cursor.scalar('SELECT count(*) FROM country_name') > 0
assert temp_db_cursor.scalar('SELECT count(*) FROM country_name WHERE partition != 0') == 0