diff --git a/lib-php/DB.php b/lib-php/DB.php index 0454a0ff..abd23179 100644 --- a/lib-php/DB.php +++ b/lib-php/DB.php @@ -240,16 +240,6 @@ class DB return ($this->getOne($sSQL, array(':tablename' => $sTableName)) == 1); } - /** - * Returns a list of table names in the database - * - * @return array[] - */ - public function getListOfTables() - { - return $this->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"); - } - /** * Deletes a table. Returns true if deleted or didn't exist. * @@ -262,76 +252,6 @@ class DB return $this->exec('DROP TABLE IF EXISTS '.$sTableName.' CASCADE') == 0; } - /** - * Check if an index exists in the database. Optional filtered by tablename - * - * @param string $sTableName - * - * @return boolean - */ - public function indexExists($sIndexName, $sTableName = null) - { - return in_array($sIndexName, $this->getListOfIndices($sTableName)); - } - - /** - * Returns a list of index names in the database, optional filtered by tablename - * - * @param string $sTableName - * - * @return array - */ - public function getListOfIndices($sTableName = null) - { - // table_name | index_name | column_name - // -----------------------+---------------------------------+-------------- - // country_name | idx_country_name_country_code | country_code - // country_osm_grid | idx_country_osm_grid_geometry | geometry - // import_polygon_delete | idx_import_polygon_delete_osmid | osm_id - // import_polygon_delete | idx_import_polygon_delete_osmid | osm_type - // import_polygon_error | idx_import_polygon_error_osmid | osm_id - // import_polygon_error | idx_import_polygon_error_osmid | osm_type - $sSql = <<< END -SELECT - t.relname as table_name, - i.relname as index_name, - a.attname as column_name -FROM - pg_class t, - pg_class i, - pg_index ix, - pg_attribute a -WHERE - t.oid = ix.indrelid - and i.oid = ix.indexrelid - and a.attrelid = t.oid - and a.attnum = ANY(ix.indkey) - and t.relkind = 'r' - and i.relname NOT LIKE 'pg_%' - FILTERS - ORDER BY - t.relname, - i.relname, - a.attname -END; - - $aRows = null; - if ($sTableName) { - $sSql = str_replace('FILTERS', 'and t.relname = :tablename', $sSql); - $aRows = $this->getAll($sSql, array(':tablename' => $sTableName)); - } else { - $sSql = str_replace('FILTERS', '', $sSql); - $aRows = $this->getAll($sSql); - } - - $aIndexNames = array_unique(array_map(function ($aRow) { - return $aRow['index_name']; - }, $aRows)); - sort($aIndexNames); - - return $aIndexNames; - } - /** * Tries to connect to the database but on failure doesn't throw an exception. * diff --git a/lib-php/admin/check_import_finished.php b/lib-php/admin/check_import_finished.php index f189fc9a..d5d011c4 100644 --- a/lib-php/admin/check_import_finished.php +++ b/lib-php/admin/check_import_finished.php @@ -5,197 +5,6 @@ require_once(CONST_LibDir.'/init-cmd.php'); loadSettings(getcwd()); -$term_colors = array( - 'green' => "\033[92m", - 'red' => "\x1B[31m", - 'normal' => "\033[0m" -); - -$print_success = function ($message = 'OK') use ($term_colors) { - echo $term_colors['green'].$message.$term_colors['normal']."\n"; -}; - -$print_fail = function ($message = 'Failed') use ($term_colors) { - echo $term_colors['red'].$message.$term_colors['normal']."\n"; -}; - -$oDB = new Nominatim\DB; - - -function isReverseOnlyInstallation() -{ - global $oDB; - return !$oDB->tableExists('search_name'); -} - -// Check (guess) if the setup.php included --drop -function isNoUpdateInstallation() -{ - global $oDB; - return $oDB->tableExists('placex') && !$oDB->tableExists('planet_osm_rels') ; -} - - -echo 'Checking database got created ... '; -if ($oDB->checkConnection()) { - $print_success(); -} else { - $print_fail(); - echo <<< END - Hints: - * Is the database server started? - * Check the NOMINATIM_DATABASE_DSN variable in your local .env - * Try connecting to the database with the same settings - -END; - exit(1); -} - - -echo 'Checking nominatim.so module installed ... '; -$sStandardWord = $oDB->getOne("SELECT make_standard_name('a')"); -if ($sStandardWord === 'a') { - $print_success(); -} else { - $print_fail(); - echo <<< END - The Postgresql extension nominatim.so was not found in the database. - Hints: - * Check the output of the CMmake/make installation step - * Does nominatim.so exist? - * Does nominatim.so exist on the database server? - * Can nominatim.so be accessed by the database user? - -END; - exit(1); -} - -if (!isNoUpdateInstallation()) { - echo 'Checking place table ... '; - if ($oDB->tableExists('place')) { - $print_success(); - } else { - $print_fail(); - echo <<< END - * The import didn't finish. - Hints: - * Check the output of the utils/setup.php you ran. - Usually the osm2pgsql step failed. Check for errors related to - * the file you imported not containing any places - * harddrive full - * out of memory (RAM) - * osm2pgsql killed by other scripts, for consuming to much memory - - END; - exit(1); - } -} - - -echo 'Checking indexing status ... '; -$iUnindexed = $oDB->getOne('SELECT count(*) FROM placex WHERE indexed_status > 0'); -if ($iUnindexed == 0) { - $print_success(); -} else { - $print_fail(); - echo <<< END - The indexing didn't finish. There is still $iUnindexed places. See the - question 'Can a stopped/killed import process be resumed?' in the - troubleshooting guide. - -END; - exit(1); -} - -echo "Search index creation\n"; -$aExpectedIndices = array( - // sql/indices.src.sql - 'idx_word_word_id', - 'idx_place_addressline_address_place_id', - 'idx_placex_rank_search', - 'idx_placex_rank_address', - 'idx_placex_parent_place_id', - 'idx_placex_geometry_reverse_lookuppolygon', - 'idx_placex_geometry_reverse_placenode', - 'idx_osmline_parent_place_id', - 'idx_osmline_parent_osm_id', - 'idx_postcode_id', - 'idx_postcode_postcode' -); -if (!isReverseOnlyInstallation()) { - $aExpectedIndices = array_merge($aExpectedIndices, array( - // sql/indices_search.src.sql - 'idx_search_name_nameaddress_vector', - 'idx_search_name_name_vector', - 'idx_search_name_centroid' - )); -} -if (!isNoUpdateInstallation()) { - $aExpectedIndices = array_merge($aExpectedIndices, array( - 'idx_placex_pendingsector', - 'idx_location_area_country_place_id', - 'idx_place_osm_unique', - )); -} - -foreach ($aExpectedIndices as $sExpectedIndex) { - echo "Checking index $sExpectedIndex ... "; - if ($oDB->indexExists($sExpectedIndex)) { - $print_success(); - } else { - $print_fail(); - echo <<< END - Hints: - * Run './utils/setup.php --create-search-indices --ignore-errors' to - create missing indices. - -END; - exit(1); - } -} - -echo 'Checking search indices are valid ... '; -$sSQL = <<< END - SELECT relname - FROM pg_class, pg_index - WHERE pg_index.indisvalid = false - AND pg_index.indexrelid = pg_class.oid; -END; -$aInvalid = $oDB->getCol($sSQL); -if (empty($aInvalid)) { - $print_success(); -} else { - $print_fail(); - echo <<< END - At least one index is invalid. That can happen, e.g. when index creation was - disrupted and later restarted. You should delete the affected indices and - run the index stage of setup again. - See the question 'Can a stopped/killed import process be resumed?' in the - troubleshooting guide. - Affected indices: -END; - echo join(', ', $aInvalid) . "\n"; - exit(1); -} - - - -if (getSettingBool('USE_US_TIGER_DATA')) { - echo 'Checking TIGER table exists ... '; - if ($oDB->tableExists('location_property_tiger')) { - $print_success(); - } else { - $print_fail(); - echo <<< END - Table 'location_property_tiger' does not exist. Run the TIGER data - import again. - -END; - exit(1); - } -} - - - - -exit(0); +(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'))) + ->addParams('admin', '--check-database') + ->run(); diff --git a/lib-php/lib.php b/lib-php/lib.php index 6798e749..a1f528fa 100644 --- a/lib-php/lib.php +++ b/lib-php/lib.php @@ -132,24 +132,6 @@ function addQuotes($s) return "'".$s."'"; } -function fwriteConstDef($rFile, $sConstName, $value) -{ - $sEscapedValue; - - if (is_bool($value)) { - $sEscapedValue = $value ? 'true' : 'false'; - } elseif (is_numeric($value)) { - $sEscapedValue = strval($value); - } elseif (!$value) { - $sEscapedValue = 'false'; - } else { - $sEscapedValue = addQuotes(str_replace("'", "\\'", (string)$value)); - } - - fwrite($rFile, "@define('CONST_$sConstName', $sEscapedValue);\n"); -} - - function parseLatLon($sQuery) { $sFound = null; @@ -226,17 +208,6 @@ function parseLatLon($sQuery) return array($sFound, $fQueryLat, $fQueryLon); } -function createPointsAroundCenter($fLon, $fLat, $fRadius) -{ - $iSteps = max(8, min(100, ($fRadius * 40000)^2)); - $fStepSize = (2*pi())/$iSteps; - $aPolyPoints = array(); - for ($f = 0; $f < 2*pi(); $f += $fStepSize) { - $aPolyPoints[] = array('', $fLon+($fRadius*sin($f)), $fLat+($fRadius*cos($f)) ); - } - return $aPolyPoints; -} - function closestHouseNumber($aRow) { $fHouse = $aRow['startnumber'] @@ -256,25 +227,3 @@ function closestHouseNumber($aRow) return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']); } - -function getSearchRankLabel($iRank) -{ - if (!isset($iRank)) return 'unknown'; - if ($iRank < 2) return 'continent'; - if ($iRank < 4) return 'sea'; - if ($iRank < 8) return 'country'; - if ($iRank < 12) return 'state'; - if ($iRank < 16) return 'county'; - if ($iRank == 16) return 'city'; - if ($iRank == 17) return 'town / island'; - if ($iRank == 18) return 'village / hamlet'; - if ($iRank == 20) return 'suburb'; - if ($iRank == 21) return 'postcode area'; - if ($iRank == 22) return 'croft / farm / locality / islet'; - if ($iRank == 23) return 'postcode area'; - if ($iRank == 25) return 'postcode point'; - if ($iRank == 26) return 'street / major landmark'; - if ($iRank == 27) return 'minory street / path'; - if ($iRank == 28) return 'house / building'; - return 'other: ' . $iRank; -} diff --git a/lib-php/setup/SetupClass.php b/lib-php/setup/SetupClass.php index fedbb644..a423e12c 100755 --- a/lib-php/setup/SetupClass.php +++ b/lib-php/setup/SetupClass.php @@ -657,50 +657,7 @@ class SetupFunctions public function drop() { - info('Drop tables only required for updates'); - - // The implementation is potentially a bit dangerous because it uses - // a positive selection of tables to keep, and deletes everything else. - // Including any tables that the unsuspecting user might have manually - // created. USE AT YOUR OWN PERIL. - // tables we want to keep. everything else goes. - $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*', - 'country_osm_grid' - ); - - $aDropTables = array(); - $aHaveTables = $this->db()->getListOfTables(); - - foreach ($aHaveTables as $sTable) { - $bFound = false; - foreach ($aKeepTables as $sKeep) { - if (fnmatch($sKeep, $sTable)) { - $bFound = true; - break; - } - } - if (!$bFound) array_push($aDropTables, $sTable); - } - foreach ($aDropTables as $sDrop) { - $this->dropTable($sDrop); - } - - $this->removeFlatnodeFile(); + (clone($this->oNominatimCmd))->addParams('freeze')->run(); } /** @@ -710,48 +667,7 @@ class SetupFunctions */ public function setupWebsite() { - if (!is_dir(CONST_InstallDir.'/website')) { - info('Creating directory for website scripts at: '.CONST_InstallDir.'/website'); - mkdir(CONST_InstallDir.'/website'); - } - - $aScripts = array( - 'deletable.php', - 'details.php', - 'lookup.php', - 'polygons.php', - 'reverse.php', - 'search.php', - 'status.php' - ); - - foreach ($aScripts as $sScript) { - $rFile = fopen(CONST_InstallDir.'/website/'.$sScript, 'w'); - - fwrite($rFile, "oNominatimCmd))->addParams('refresh', '--website')->run(); } /** diff --git a/lib-php/website/details.php b/lib-php/website/details.php index 91440b54..130dcaf8 100644 --- a/lib-php/website/details.php +++ b/lib-php/website/details.php @@ -53,7 +53,7 @@ if ($sOsmType && $iOsmId > 0) { // Be nice about our error messages for broken geometry - if (!$sPlaceId) { + if (!$sPlaceId && $oDB->tableExists('import_polygon_error')) { $sSQL = 'SELECT '; $sSQL .= ' osm_type, '; $sSQL .= ' osm_id, '; @@ -144,7 +144,6 @@ if (!$aPointDetails) { } $aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber']; -$aPointDetails['rank_search_label'] = getSearchRankLabel($aPointDetails['rank_search']); // only used in HTML format // Get all alternative names (languages, etc) $sSQL = 'SELECT (each(name)).key,(each(name)).value FROM placex '; diff --git a/nominatim/cli.py b/nominatim/cli.py index 8cb73a8e..83ecf67b 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -173,27 +173,6 @@ class SetupAll: return run_legacy_script(*params, nominatim_env=args) -class SetupFreeze: - """\ - Make database read-only. - - About half of data in the Nominatim database is kept only to be able to - keep the data up-to-date with new changes made in OpenStreetMap. This - command drops all this data and only keeps the part needed for geocoding - itself. - - This command has the same effect as the `--no-updates` option for imports. - """ - - @staticmethod - def add_args(parser): - pass # No options - - @staticmethod - def run(args): - return run_legacy_script('setup.php', '--drop', nominatim_env=args) - - class SetupSpecialPhrases: """\ Maintain special phrases. @@ -352,7 +331,7 @@ def nominatim(**kwargs): parser = CommandlineParser('nominatim', nominatim.__doc__) parser.add_subcommand('import', SetupAll) - parser.add_subcommand('freeze', SetupFreeze) + parser.add_subcommand('freeze', clicmd.SetupFreeze) parser.add_subcommand('replication', clicmd.UpdateReplication) parser.add_subcommand('special-phrases', SetupSpecialPhrases) diff --git a/nominatim/clicmd/__init__.py b/nominatim/clicmd/__init__.py index 9a686df2..ae970c82 100644 --- a/nominatim/clicmd/__init__.py +++ b/nominatim/clicmd/__init__.py @@ -7,3 +7,4 @@ from .api import APISearch, APIReverse, APILookup, APIDetails, APIStatus from .index import UpdateIndex from .refresh import UpdateRefresh from .admin import AdminFuncs +from .freeze import SetupFreeze diff --git a/nominatim/clicmd/admin.py b/nominatim/clicmd/admin.py index 8d34f386..e5863575 100644 --- a/nominatim/clicmd/admin.py +++ b/nominatim/clicmd/admin.py @@ -1,6 +1,8 @@ """ Implementation of the 'admin' subcommand. """ +import logging + from ..tools.exec_utils import run_legacy_script from ..db.connection import connect @@ -9,6 +11,8 @@ from ..db.connection import connect # Using non-top-level imports to avoid eventually unused imports. # pylint: disable=E0012,C0415 +LOG = logging.getLogger() + class AdminFuncs: """\ Analyse and maintain the database. @@ -39,14 +43,17 @@ class AdminFuncs: @staticmethod def run(args): - from ..tools import admin if args.warm: AdminFuncs._warm(args) if args.check_database: - run_legacy_script('check_import_finished.php', nominatim_env=args) + LOG.warning('Checking database') + from ..tools import check_database + return check_database.check_database(args.config) if args.analyse_indexing: + LOG.warning('Analysing performance of indexing function') + from ..tools import admin conn = connect(args.config.get_libpq_dsn()) admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id) conn.close() @@ -56,6 +63,7 @@ class AdminFuncs: @staticmethod def _warm(args): + LOG.warning('Warming database caches') params = ['warm.php'] if args.target == 'reverse': params.append('--reverse-only') diff --git a/nominatim/clicmd/freeze.py b/nominatim/clicmd/freeze.py new file mode 100644 index 00000000..8bca04b9 --- /dev/null +++ b/nominatim/clicmd/freeze.py @@ -0,0 +1,37 @@ +""" +Implementation of the 'freeze' subcommand. +""" + +from ..db.connection import connect + +# Do not repeat documentation of subcommand classes. +# pylint: disable=C0111 +# Using non-top-level imports to avoid eventually unused imports. +# pylint: disable=E0012,C0415 + +class SetupFreeze: + """\ + Make database read-only. + + About half of data in the Nominatim database is kept only to be able to + keep the data up-to-date with new changes made in OpenStreetMap. This + command drops all this data and only keeps the part needed for geocoding + itself. + + This command has the same effect as the `--no-updates` option for imports. + """ + + @staticmethod + def add_args(parser): + pass # No options + + @staticmethod + def run(args): + from ..tools import freeze + + conn = connect(args.config.get_libpq_dsn()) + freeze.drop_update_tables(conn) + freeze.drop_flatnode_file(args.config.FLATNODE_FILE) + conn.close() + + return 0 diff --git a/nominatim/clicmd/refresh.py b/nominatim/clicmd/refresh.py index 8e69caca..ffbe628b 100644 --- a/nominatim/clicmd/refresh.py +++ b/nominatim/clicmd/refresh.py @@ -82,7 +82,8 @@ class UpdateRefresh: run_legacy_script('update.php', '--recompute-importance', nominatim_env=args, throw_on_fail=True) if args.website: - run_legacy_script('setup.php', '--setup-website', - nominatim_env=args, throw_on_fail=True) + webdir = args.project_dir / 'website' + LOG.warning('Setting up website directory at %s', webdir) + refresh.setup_website(webdir, args.phplib_dir, args.config) return 0 diff --git a/nominatim/config.py b/nominatim/config.py index 4de2052e..a22f90ab 100644 --- a/nominatim/config.py +++ b/nominatim/config.py @@ -17,7 +17,7 @@ class Configuration: Nominatim uses dotenv to configure the software. Configuration options are resolved in the following order: - * from the OS environment + * from the OS environment (or the dirctionary given in `environ` * from the .env file in the project directory of the installation * from the default installation in the configuration directory @@ -25,7 +25,8 @@ class Configuration: avoid conflicts with other environment variables. """ - def __init__(self, project_dir, config_dir): + def __init__(self, project_dir, config_dir, environ=None): + self.environ = environ or os.environ self.project_dir = project_dir self.config_dir = config_dir self._config = dotenv_values(str((config_dir / 'env.defaults').resolve())) @@ -42,7 +43,7 @@ class Configuration: def __getattr__(self, name): name = 'NOMINATIM_' + name - return os.environ.get(name) or self._config[name] + return self.environ.get(name) or self._config[name] def get_bool(self, name): """ Return the given configuration parameter as a boolean. @@ -100,6 +101,6 @@ class Configuration: merged in. """ env = dict(self._config) - env.update(os.environ) + env.update(self.environ) return env diff --git a/nominatim/db/connection.py b/nominatim/db/connection.py index c7e22c98..b941f46f 100644 --- a/nominatim/db/connection.py +++ b/nominatim/db/connection.py @@ -7,6 +7,8 @@ import psycopg2 import psycopg2.extensions import psycopg2.extras +from ..errors import UsageError + class _Cursor(psycopg2.extras.DictCursor): """ A cursor returning dict-like objects and providing specialised execution functions. @@ -42,14 +44,34 @@ class _Connection(psycopg2.extensions.connection): """ return super().cursor(cursor_factory=cursor_factory, **kwargs) + def table_exists(self, table): """ Check that a table with the given name exists in the database. """ with self.cursor() as cur: num = cur.scalar("""SELECT count(*) FROM pg_tables - WHERE tablename = %s""", (table, )) + WHERE tablename = %s and schemaname = 'public'""", (table, )) return num == 1 + + def index_exists(self, index, table=None): + """ Check that an index with the given name exists in the database. + If table is not None then the index must relate to the given + table. + """ + with self.cursor() as cur: + cur.execute("""SELECT tablename FROM pg_indexes + WHERE indexname = %s and schemaname = 'public'""", (index, )) + if cur.rowcount == 0: + return False + + if table is not None: + row = cur.fetchone() + return row[0] == table + + return True + + def server_version_tuple(self): """ Return the server version as a tuple of (major, minor). Converts correctly for pre-10 and post-10 PostgreSQL versions. @@ -64,4 +86,7 @@ def connect(dsn): """ Open a connection to the database using the specialised connection factory. """ - return psycopg2.connect(dsn, connection_factory=_Connection) + try: + return psycopg2.connect(dsn, connection_factory=_Connection) + except psycopg2.OperationalError as err: + raise UsageError("Cannot connect to database: {}".format(err)) from err diff --git a/nominatim/tools/check_database.py b/nominatim/tools/check_database.py new file mode 100644 index 00000000..7b8da200 --- /dev/null +++ b/nominatim/tools/check_database.py @@ -0,0 +1,269 @@ +""" +Collection of functions that check if the database is complete and functional. +""" +from enum import Enum +from textwrap import dedent + +import psycopg2 + +from ..db.connection import connect +from ..errors import UsageError + +CHECKLIST = [] + +class CheckState(Enum): + """ Possible states of a check. FATAL stops check execution entirely. + """ + OK = 0 + FAIL = 1 + FATAL = 2 + NOT_APPLICABLE = 3 + +def _check(hint=None): + """ Decorator for checks. It adds the function to the list of + checks to execute and adds the code for printing progress messages. + """ + def decorator(func): + title = func.__doc__.split('\n', 1)[0].strip() + def run_check(conn, config): + print(title, end=' ... ') + ret = func(conn, config) + if isinstance(ret, tuple): + ret, params = ret + else: + params = {} + if ret == CheckState.OK: + print('\033[92mOK\033[0m') + elif ret == CheckState.NOT_APPLICABLE: + print('not applicable') + else: + print('\x1B[31mFailed\033[0m') + if hint: + print(dedent(hint.format(**params))) + return ret + + CHECKLIST.append(run_check) + return run_check + + return decorator + +class _BadConnection: # pylint: disable=R0903 + + def __init__(self, msg): + self.msg = msg + + def close(self): + """ Dummy function to provide the implementation. + """ + +def check_database(config): + """ Run a number of checks on the database and return the status. + """ + try: + conn = connect(config.get_libpq_dsn()) + except UsageError as err: + conn = _BadConnection(str(err)) + + overall_result = 0 + for check in CHECKLIST: + ret = check(conn, config) + if ret == CheckState.FATAL: + conn.close() + return 1 + if ret in (CheckState.FATAL, CheckState.FAIL): + overall_result = 1 + + conn.close() + return overall_result + + +def _get_indexes(conn): + indexes = ['idx_word_word_id', + 'idx_place_addressline_address_place_id', + 'idx_placex_rank_search', + 'idx_placex_rank_address', + 'idx_placex_parent_place_id', + 'idx_placex_geometry_reverse_lookuppolygon', + 'idx_placex_geometry_reverse_placenode', + 'idx_osmline_parent_place_id', + 'idx_osmline_parent_osm_id', + 'idx_postcode_id', + 'idx_postcode_postcode' + ] + if conn.table_exists('search_name'): + indexes.extend(('idx_search_name_nameaddress_vector', + 'idx_search_name_name_vector', + 'idx_search_name_centroid')) + if conn.table_exists('place'): + indexes.extend(('idx_placex_pendingsector', + 'idx_location_area_country_place_id', + 'idx_place_osm_unique' + )) + + return indexes + + +### CHECK FUNCTIONS +# +# Functions are exectured in the order they appear here. + +@_check(hint="""\ + {error} + + Hints: + * Is the database server started? + * Check the NOMINATIM_DATABASE_DSN variable in your local .env + * Try connecting to the database with the same settings + + Project directory: {config.project_dir} + Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN} + """) +def check_connection(conn, config): + """ Checking database connection + """ + if isinstance(conn, _BadConnection): + return CheckState.FATAL, dict(error=conn.msg, config=config) + + return CheckState.OK + +@_check(hint="""\ + placex table not found + + Hints: + * Are you connecting to the right database? + * Did the import process finish without errors? + + Project directory: {config.project_dir} + Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN} + """) +def check_placex_table(conn, config): + """ Checking for placex table + """ + if conn.table_exists('placex'): + return CheckState.OK + + return CheckState.FATAL, dict(config=config) + + +@_check(hint="""placex table has no data. Did the import finish sucessfully?""") +def check_placex_size(conn, config): # pylint: disable=W0613 + """ Checking for placex content + """ + with conn.cursor() as cur: + cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x') + + return CheckState.OK if cnt > 0 else CheckState.FATAL + + +@_check(hint="""\ + The Postgresql extension nominatim.so was not correctly loaded. + + Error: {error} + + Hints: + * Check the output of the CMmake/make installation step + * Does nominatim.so exist? + * Does nominatim.so exist on the database server? + * Can nominatim.so be accessed by the database user? + """) +def check_module(conn, config): # pylint: disable=W0613 + """ Checking that nominatim.so module is installed + """ + with conn.cursor() as cur: + try: + out = cur.scalar("SELECT make_standard_name('a')") + except psycopg2.ProgrammingError as err: + return CheckState.FAIL, dict(error=str(err)) + + if out != 'a': + return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()') + + return CheckState.OK + + +@_check(hint="""\ + The indexing didn't finish. {count} entries are not yet indexed. + + To index the remaining entries, run: {index_cmd} + """) +def check_indexing(conn, config): # pylint: disable=W0613 + """ Checking indexing status + """ + with conn.cursor() as cur: + cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0') + + if cnt == 0: + return CheckState.OK + + if conn.index_exists('idx_word_word_id'): + # Likely just an interrupted update. + index_cmd = 'nominatim index' + else: + # Looks like the import process got interrupted. + index_cmd = 'nominatim import --continue indexing' + + return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd) + + +@_check(hint="""\ + The following indexes are missing: + {indexes} + + Rerun the index creation with: nominatim import --continue db-postprocess + """) +def check_database_indexes(conn, config): # pylint: disable=W0613 + """ Checking that database indexes are complete + """ + missing = [] + for index in _get_indexes(conn): + if not conn.index_exists(index): + missing.append(index) + + if missing: + return CheckState.FAIL, dict(indexes='\n '.join(missing)) + + return CheckState.OK + + +@_check(hint="""\ + At least one index is invalid. That can happen, e.g. when index creation was + disrupted and later restarted. You should delete the affected indices + and recreate them. + + Invalid indexes: + {indexes} + """) +def check_database_index_valid(conn, config): # pylint: disable=W0613 + """ Checking that all database indexes are valid + """ + with conn.cursor() as cur: + cur.execute(""" SELECT relname FROM pg_class, pg_index + WHERE pg_index.indisvalid = false + AND pg_index.indexrelid = pg_class.oid""") + + broken = list(cur) + + if broken: + return CheckState.FAIL, dict(indexes='\n '.join(broken)) + + return CheckState.OK + + +@_check(hint="""\ + {error} + Run TIGER import again: nominatim add-data --tiger-data