diff --git a/lib-php/DB.php b/lib-php/DB.php index 0454a0ff..abd23179 100644 --- a/lib-php/DB.php +++ b/lib-php/DB.php @@ -240,16 +240,6 @@ class DB return ($this->getOne($sSQL, array(':tablename' => $sTableName)) == 1); } - /** - * Returns a list of table names in the database - * - * @return array[] - */ - public function getListOfTables() - { - return $this->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"); - } - /** * Deletes a table. Returns true if deleted or didn't exist. * @@ -262,76 +252,6 @@ class DB return $this->exec('DROP TABLE IF EXISTS '.$sTableName.' CASCADE') == 0; } - /** - * Check if an index exists in the database. Optional filtered by tablename - * - * @param string $sTableName - * - * @return boolean - */ - public function indexExists($sIndexName, $sTableName = null) - { - return in_array($sIndexName, $this->getListOfIndices($sTableName)); - } - - /** - * Returns a list of index names in the database, optional filtered by tablename - * - * @param string $sTableName - * - * @return array - */ - public function getListOfIndices($sTableName = null) - { - // table_name | index_name | column_name - // -----------------------+---------------------------------+-------------- - // country_name | idx_country_name_country_code | country_code - // country_osm_grid | idx_country_osm_grid_geometry | geometry - // import_polygon_delete | idx_import_polygon_delete_osmid | osm_id - // import_polygon_delete | idx_import_polygon_delete_osmid | osm_type - // import_polygon_error | idx_import_polygon_error_osmid | osm_id - // import_polygon_error | idx_import_polygon_error_osmid | osm_type - $sSql = <<< END -SELECT - t.relname as table_name, - i.relname as index_name, - a.attname as column_name -FROM - pg_class t, - pg_class i, - pg_index ix, - pg_attribute a -WHERE - t.oid = ix.indrelid - and i.oid = ix.indexrelid - and a.attrelid = t.oid - and a.attnum = ANY(ix.indkey) - and t.relkind = 'r' - and i.relname NOT LIKE 'pg_%' - FILTERS - ORDER BY - t.relname, - i.relname, - a.attname -END; - - $aRows = null; - if ($sTableName) { - $sSql = str_replace('FILTERS', 'and t.relname = :tablename', $sSql); - $aRows = $this->getAll($sSql, array(':tablename' => $sTableName)); - } else { - $sSql = str_replace('FILTERS', '', $sSql); - $aRows = $this->getAll($sSql); - } - - $aIndexNames = array_unique(array_map(function ($aRow) { - return $aRow['index_name']; - }, $aRows)); - sort($aIndexNames); - - return $aIndexNames; - } - /** * Tries to connect to the database but on failure doesn't throw an exception. * diff --git a/lib-php/admin/check_import_finished.php b/lib-php/admin/check_import_finished.php index f189fc9a..d5d011c4 100644 --- a/lib-php/admin/check_import_finished.php +++ b/lib-php/admin/check_import_finished.php @@ -5,197 +5,6 @@ require_once(CONST_LibDir.'/init-cmd.php'); loadSettings(getcwd()); -$term_colors = array( - 'green' => "\033[92m", - 'red' => "\x1B[31m", - 'normal' => "\033[0m" -); - -$print_success = function ($message = 'OK') use ($term_colors) { - echo $term_colors['green'].$message.$term_colors['normal']."\n"; -}; - -$print_fail = function ($message = 'Failed') use ($term_colors) { - echo $term_colors['red'].$message.$term_colors['normal']."\n"; -}; - -$oDB = new Nominatim\DB; - - -function isReverseOnlyInstallation() -{ - global $oDB; - return !$oDB->tableExists('search_name'); -} - -// Check (guess) if the setup.php included --drop -function isNoUpdateInstallation() -{ - global $oDB; - return $oDB->tableExists('placex') && !$oDB->tableExists('planet_osm_rels') ; -} - - -echo 'Checking database got created ... '; -if ($oDB->checkConnection()) { - $print_success(); -} else { - $print_fail(); - echo <<< END - Hints: - * Is the database server started? - * Check the NOMINATIM_DATABASE_DSN variable in your local .env - * Try connecting to the database with the same settings - -END; - exit(1); -} - - -echo 'Checking nominatim.so module installed ... '; -$sStandardWord = $oDB->getOne("SELECT make_standard_name('a')"); -if ($sStandardWord === 'a') { - $print_success(); -} else { - $print_fail(); - echo <<< END - The Postgresql extension nominatim.so was not found in the database. - Hints: - * Check the output of the CMmake/make installation step - * Does nominatim.so exist? - * Does nominatim.so exist on the database server? - * Can nominatim.so be accessed by the database user? - -END; - exit(1); -} - -if (!isNoUpdateInstallation()) { - echo 'Checking place table ... '; - if ($oDB->tableExists('place')) { - $print_success(); - } else { - $print_fail(); - echo <<< END - * The import didn't finish. - Hints: - * Check the output of the utils/setup.php you ran. - Usually the osm2pgsql step failed. Check for errors related to - * the file you imported not containing any places - * harddrive full - * out of memory (RAM) - * osm2pgsql killed by other scripts, for consuming to much memory - - END; - exit(1); - } -} - - -echo 'Checking indexing status ... '; -$iUnindexed = $oDB->getOne('SELECT count(*) FROM placex WHERE indexed_status > 0'); -if ($iUnindexed == 0) { - $print_success(); -} else { - $print_fail(); - echo <<< END - The indexing didn't finish. There is still $iUnindexed places. See the - question 'Can a stopped/killed import process be resumed?' in the - troubleshooting guide. - -END; - exit(1); -} - -echo "Search index creation\n"; -$aExpectedIndices = array( - // sql/indices.src.sql - 'idx_word_word_id', - 'idx_place_addressline_address_place_id', - 'idx_placex_rank_search', - 'idx_placex_rank_address', - 'idx_placex_parent_place_id', - 'idx_placex_geometry_reverse_lookuppolygon', - 'idx_placex_geometry_reverse_placenode', - 'idx_osmline_parent_place_id', - 'idx_osmline_parent_osm_id', - 'idx_postcode_id', - 'idx_postcode_postcode' -); -if (!isReverseOnlyInstallation()) { - $aExpectedIndices = array_merge($aExpectedIndices, array( - // sql/indices_search.src.sql - 'idx_search_name_nameaddress_vector', - 'idx_search_name_name_vector', - 'idx_search_name_centroid' - )); -} -if (!isNoUpdateInstallation()) { - $aExpectedIndices = array_merge($aExpectedIndices, array( - 'idx_placex_pendingsector', - 'idx_location_area_country_place_id', - 'idx_place_osm_unique', - )); -} - -foreach ($aExpectedIndices as $sExpectedIndex) { - echo "Checking index $sExpectedIndex ... "; - if ($oDB->indexExists($sExpectedIndex)) { - $print_success(); - } else { - $print_fail(); - echo <<< END - Hints: - * Run './utils/setup.php --create-search-indices --ignore-errors' to - create missing indices. - -END; - exit(1); - } -} - -echo 'Checking search indices are valid ... '; -$sSQL = <<< END - SELECT relname - FROM pg_class, pg_index - WHERE pg_index.indisvalid = false - AND pg_index.indexrelid = pg_class.oid; -END; -$aInvalid = $oDB->getCol($sSQL); -if (empty($aInvalid)) { - $print_success(); -} else { - $print_fail(); - echo <<< END - At least one index is invalid. That can happen, e.g. when index creation was - disrupted and later restarted. You should delete the affected indices and - run the index stage of setup again. - See the question 'Can a stopped/killed import process be resumed?' in the - troubleshooting guide. - Affected indices: -END; - echo join(', ', $aInvalid) . "\n"; - exit(1); -} - - - -if (getSettingBool('USE_US_TIGER_DATA')) { - echo 'Checking TIGER table exists ... '; - if ($oDB->tableExists('location_property_tiger')) { - $print_success(); - } else { - $print_fail(); - echo <<< END - Table 'location_property_tiger' does not exist. Run the TIGER data - import again. - -END; - exit(1); - } -} - - - - -exit(0); +(new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'))) + ->addParams('admin', '--check-database') + ->run(); diff --git a/lib-php/lib.php b/lib-php/lib.php index 6798e749..a1f528fa 100644 --- a/lib-php/lib.php +++ b/lib-php/lib.php @@ -132,24 +132,6 @@ function addQuotes($s) return "'".$s."'"; } -function fwriteConstDef($rFile, $sConstName, $value) -{ - $sEscapedValue; - - if (is_bool($value)) { - $sEscapedValue = $value ? 'true' : 'false'; - } elseif (is_numeric($value)) { - $sEscapedValue = strval($value); - } elseif (!$value) { - $sEscapedValue = 'false'; - } else { - $sEscapedValue = addQuotes(str_replace("'", "\\'", (string)$value)); - } - - fwrite($rFile, "@define('CONST_$sConstName', $sEscapedValue);\n"); -} - - function parseLatLon($sQuery) { $sFound = null; @@ -226,17 +208,6 @@ function parseLatLon($sQuery) return array($sFound, $fQueryLat, $fQueryLon); } -function createPointsAroundCenter($fLon, $fLat, $fRadius) -{ - $iSteps = max(8, min(100, ($fRadius * 40000)^2)); - $fStepSize = (2*pi())/$iSteps; - $aPolyPoints = array(); - for ($f = 0; $f < 2*pi(); $f += $fStepSize) { - $aPolyPoints[] = array('', $fLon+($fRadius*sin($f)), $fLat+($fRadius*cos($f)) ); - } - return $aPolyPoints; -} - function closestHouseNumber($aRow) { $fHouse = $aRow['startnumber'] @@ -256,25 +227,3 @@ function closestHouseNumber($aRow) return max(min($aRow['endnumber'], $iHn), $aRow['startnumber']); } - -function getSearchRankLabel($iRank) -{ - if (!isset($iRank)) return 'unknown'; - if ($iRank < 2) return 'continent'; - if ($iRank < 4) return 'sea'; - if ($iRank < 8) return 'country'; - if ($iRank < 12) return 'state'; - if ($iRank < 16) return 'county'; - if ($iRank == 16) return 'city'; - if ($iRank == 17) return 'town / island'; - if ($iRank == 18) return 'village / hamlet'; - if ($iRank == 20) return 'suburb'; - if ($iRank == 21) return 'postcode area'; - if ($iRank == 22) return 'croft / farm / locality / islet'; - if ($iRank == 23) return 'postcode area'; - if ($iRank == 25) return 'postcode point'; - if ($iRank == 26) return 'street / major landmark'; - if ($iRank == 27) return 'minory street / path'; - if ($iRank == 28) return 'house / building'; - return 'other: ' . $iRank; -} diff --git a/lib-php/setup/SetupClass.php b/lib-php/setup/SetupClass.php index fedbb644..a423e12c 100755 --- a/lib-php/setup/SetupClass.php +++ b/lib-php/setup/SetupClass.php @@ -657,50 +657,7 @@ class SetupFunctions public function drop() { - info('Drop tables only required for updates'); - - // The implementation is potentially a bit dangerous because it uses - // a positive selection of tables to keep, and deletes everything else. - // Including any tables that the unsuspecting user might have manually - // created. USE AT YOUR OWN PERIL. - // tables we want to keep. everything else goes. - $aKeepTables = array( - '*columns', - 'import_polygon_*', - 'import_status', - 'place_addressline', - 'location_postcode', - 'location_property*', - 'placex', - 'search_name', - 'seq_*', - 'word', - 'query_log', - 'new_query_log', - 'spatial_ref_sys', - 'country_name', - 'place_classtype_*', - 'country_osm_grid' - ); - - $aDropTables = array(); - $aHaveTables = $this->db()->getListOfTables(); - - foreach ($aHaveTables as $sTable) { - $bFound = false; - foreach ($aKeepTables as $sKeep) { - if (fnmatch($sKeep, $sTable)) { - $bFound = true; - break; - } - } - if (!$bFound) array_push($aDropTables, $sTable); - } - foreach ($aDropTables as $sDrop) { - $this->dropTable($sDrop); - } - - $this->removeFlatnodeFile(); + (clone($this->oNominatimCmd))->addParams('freeze')->run(); } /** @@ -710,48 +667,7 @@ class SetupFunctions */ public function setupWebsite() { - if (!is_dir(CONST_InstallDir.'/website')) { - info('Creating directory for website scripts at: '.CONST_InstallDir.'/website'); - mkdir(CONST_InstallDir.'/website'); - } - - $aScripts = array( - 'deletable.php', - 'details.php', - 'lookup.php', - 'polygons.php', - 'reverse.php', - 'search.php', - 'status.php' - ); - - foreach ($aScripts as $sScript) { - $rFile = fopen(CONST_InstallDir.'/website/'.$sScript, 'w'); - - fwrite($rFile, "oNominatimCmd))->addParams('refresh', '--website')->run(); } /** diff --git a/lib-php/website/details.php b/lib-php/website/details.php index 91440b54..130dcaf8 100644 --- a/lib-php/website/details.php +++ b/lib-php/website/details.php @@ -53,7 +53,7 @@ if ($sOsmType && $iOsmId > 0) { // Be nice about our error messages for broken geometry - if (!$sPlaceId) { + if (!$sPlaceId && $oDB->tableExists('import_polygon_error')) { $sSQL = 'SELECT '; $sSQL .= ' osm_type, '; $sSQL .= ' osm_id, '; @@ -144,7 +144,6 @@ if (!$aPointDetails) { } $aPointDetails['localname'] = $aPointDetails['localname']?$aPointDetails['localname']:$aPointDetails['housenumber']; -$aPointDetails['rank_search_label'] = getSearchRankLabel($aPointDetails['rank_search']); // only used in HTML format // Get all alternative names (languages, etc) $sSQL = 'SELECT (each(name)).key,(each(name)).value FROM placex '; diff --git a/nominatim/cli.py b/nominatim/cli.py index 8cb73a8e..83ecf67b 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -173,27 +173,6 @@ class SetupAll: return run_legacy_script(*params, nominatim_env=args) -class SetupFreeze: - """\ - Make database read-only. - - About half of data in the Nominatim database is kept only to be able to - keep the data up-to-date with new changes made in OpenStreetMap. This - command drops all this data and only keeps the part needed for geocoding - itself. - - This command has the same effect as the `--no-updates` option for imports. - """ - - @staticmethod - def add_args(parser): - pass # No options - - @staticmethod - def run(args): - return run_legacy_script('setup.php', '--drop', nominatim_env=args) - - class SetupSpecialPhrases: """\ Maintain special phrases. @@ -352,7 +331,7 @@ def nominatim(**kwargs): parser = CommandlineParser('nominatim', nominatim.__doc__) parser.add_subcommand('import', SetupAll) - parser.add_subcommand('freeze', SetupFreeze) + parser.add_subcommand('freeze', clicmd.SetupFreeze) parser.add_subcommand('replication', clicmd.UpdateReplication) parser.add_subcommand('special-phrases', SetupSpecialPhrases) diff --git a/nominatim/clicmd/__init__.py b/nominatim/clicmd/__init__.py index 9a686df2..ae970c82 100644 --- a/nominatim/clicmd/__init__.py +++ b/nominatim/clicmd/__init__.py @@ -7,3 +7,4 @@ from .api import APISearch, APIReverse, APILookup, APIDetails, APIStatus from .index import UpdateIndex from .refresh import UpdateRefresh from .admin import AdminFuncs +from .freeze import SetupFreeze diff --git a/nominatim/clicmd/admin.py b/nominatim/clicmd/admin.py index 8d34f386..e5863575 100644 --- a/nominatim/clicmd/admin.py +++ b/nominatim/clicmd/admin.py @@ -1,6 +1,8 @@ """ Implementation of the 'admin' subcommand. """ +import logging + from ..tools.exec_utils import run_legacy_script from ..db.connection import connect @@ -9,6 +11,8 @@ from ..db.connection import connect # Using non-top-level imports to avoid eventually unused imports. # pylint: disable=E0012,C0415 +LOG = logging.getLogger() + class AdminFuncs: """\ Analyse and maintain the database. @@ -39,14 +43,17 @@ class AdminFuncs: @staticmethod def run(args): - from ..tools import admin if args.warm: AdminFuncs._warm(args) if args.check_database: - run_legacy_script('check_import_finished.php', nominatim_env=args) + LOG.warning('Checking database') + from ..tools import check_database + return check_database.check_database(args.config) if args.analyse_indexing: + LOG.warning('Analysing performance of indexing function') + from ..tools import admin conn = connect(args.config.get_libpq_dsn()) admin.analyse_indexing(conn, osm_id=args.osm_id, place_id=args.place_id) conn.close() @@ -56,6 +63,7 @@ class AdminFuncs: @staticmethod def _warm(args): + LOG.warning('Warming database caches') params = ['warm.php'] if args.target == 'reverse': params.append('--reverse-only') diff --git a/nominatim/clicmd/freeze.py b/nominatim/clicmd/freeze.py new file mode 100644 index 00000000..8bca04b9 --- /dev/null +++ b/nominatim/clicmd/freeze.py @@ -0,0 +1,37 @@ +""" +Implementation of the 'freeze' subcommand. +""" + +from ..db.connection import connect + +# Do not repeat documentation of subcommand classes. +# pylint: disable=C0111 +# Using non-top-level imports to avoid eventually unused imports. +# pylint: disable=E0012,C0415 + +class SetupFreeze: + """\ + Make database read-only. + + About half of data in the Nominatim database is kept only to be able to + keep the data up-to-date with new changes made in OpenStreetMap. This + command drops all this data and only keeps the part needed for geocoding + itself. + + This command has the same effect as the `--no-updates` option for imports. + """ + + @staticmethod + def add_args(parser): + pass # No options + + @staticmethod + def run(args): + from ..tools import freeze + + conn = connect(args.config.get_libpq_dsn()) + freeze.drop_update_tables(conn) + freeze.drop_flatnode_file(args.config.FLATNODE_FILE) + conn.close() + + return 0 diff --git a/nominatim/clicmd/refresh.py b/nominatim/clicmd/refresh.py index 8e69caca..ffbe628b 100644 --- a/nominatim/clicmd/refresh.py +++ b/nominatim/clicmd/refresh.py @@ -82,7 +82,8 @@ class UpdateRefresh: run_legacy_script('update.php', '--recompute-importance', nominatim_env=args, throw_on_fail=True) if args.website: - run_legacy_script('setup.php', '--setup-website', - nominatim_env=args, throw_on_fail=True) + webdir = args.project_dir / 'website' + LOG.warning('Setting up website directory at %s', webdir) + refresh.setup_website(webdir, args.phplib_dir, args.config) return 0 diff --git a/nominatim/config.py b/nominatim/config.py index 4de2052e..a22f90ab 100644 --- a/nominatim/config.py +++ b/nominatim/config.py @@ -17,7 +17,7 @@ class Configuration: Nominatim uses dotenv to configure the software. Configuration options are resolved in the following order: - * from the OS environment + * from the OS environment (or the dirctionary given in `environ` * from the .env file in the project directory of the installation * from the default installation in the configuration directory @@ -25,7 +25,8 @@ class Configuration: avoid conflicts with other environment variables. """ - def __init__(self, project_dir, config_dir): + def __init__(self, project_dir, config_dir, environ=None): + self.environ = environ or os.environ self.project_dir = project_dir self.config_dir = config_dir self._config = dotenv_values(str((config_dir / 'env.defaults').resolve())) @@ -42,7 +43,7 @@ class Configuration: def __getattr__(self, name): name = 'NOMINATIM_' + name - return os.environ.get(name) or self._config[name] + return self.environ.get(name) or self._config[name] def get_bool(self, name): """ Return the given configuration parameter as a boolean. @@ -100,6 +101,6 @@ class Configuration: merged in. """ env = dict(self._config) - env.update(os.environ) + env.update(self.environ) return env diff --git a/nominatim/db/connection.py b/nominatim/db/connection.py index c7e22c98..b941f46f 100644 --- a/nominatim/db/connection.py +++ b/nominatim/db/connection.py @@ -7,6 +7,8 @@ import psycopg2 import psycopg2.extensions import psycopg2.extras +from ..errors import UsageError + class _Cursor(psycopg2.extras.DictCursor): """ A cursor returning dict-like objects and providing specialised execution functions. @@ -42,14 +44,34 @@ class _Connection(psycopg2.extensions.connection): """ return super().cursor(cursor_factory=cursor_factory, **kwargs) + def table_exists(self, table): """ Check that a table with the given name exists in the database. """ with self.cursor() as cur: num = cur.scalar("""SELECT count(*) FROM pg_tables - WHERE tablename = %s""", (table, )) + WHERE tablename = %s and schemaname = 'public'""", (table, )) return num == 1 + + def index_exists(self, index, table=None): + """ Check that an index with the given name exists in the database. + If table is not None then the index must relate to the given + table. + """ + with self.cursor() as cur: + cur.execute("""SELECT tablename FROM pg_indexes + WHERE indexname = %s and schemaname = 'public'""", (index, )) + if cur.rowcount == 0: + return False + + if table is not None: + row = cur.fetchone() + return row[0] == table + + return True + + def server_version_tuple(self): """ Return the server version as a tuple of (major, minor). Converts correctly for pre-10 and post-10 PostgreSQL versions. @@ -64,4 +86,7 @@ def connect(dsn): """ Open a connection to the database using the specialised connection factory. """ - return psycopg2.connect(dsn, connection_factory=_Connection) + try: + return psycopg2.connect(dsn, connection_factory=_Connection) + except psycopg2.OperationalError as err: + raise UsageError("Cannot connect to database: {}".format(err)) from err diff --git a/nominatim/tools/check_database.py b/nominatim/tools/check_database.py new file mode 100644 index 00000000..7b8da200 --- /dev/null +++ b/nominatim/tools/check_database.py @@ -0,0 +1,269 @@ +""" +Collection of functions that check if the database is complete and functional. +""" +from enum import Enum +from textwrap import dedent + +import psycopg2 + +from ..db.connection import connect +from ..errors import UsageError + +CHECKLIST = [] + +class CheckState(Enum): + """ Possible states of a check. FATAL stops check execution entirely. + """ + OK = 0 + FAIL = 1 + FATAL = 2 + NOT_APPLICABLE = 3 + +def _check(hint=None): + """ Decorator for checks. It adds the function to the list of + checks to execute and adds the code for printing progress messages. + """ + def decorator(func): + title = func.__doc__.split('\n', 1)[0].strip() + def run_check(conn, config): + print(title, end=' ... ') + ret = func(conn, config) + if isinstance(ret, tuple): + ret, params = ret + else: + params = {} + if ret == CheckState.OK: + print('\033[92mOK\033[0m') + elif ret == CheckState.NOT_APPLICABLE: + print('not applicable') + else: + print('\x1B[31mFailed\033[0m') + if hint: + print(dedent(hint.format(**params))) + return ret + + CHECKLIST.append(run_check) + return run_check + + return decorator + +class _BadConnection: # pylint: disable=R0903 + + def __init__(self, msg): + self.msg = msg + + def close(self): + """ Dummy function to provide the implementation. + """ + +def check_database(config): + """ Run a number of checks on the database and return the status. + """ + try: + conn = connect(config.get_libpq_dsn()) + except UsageError as err: + conn = _BadConnection(str(err)) + + overall_result = 0 + for check in CHECKLIST: + ret = check(conn, config) + if ret == CheckState.FATAL: + conn.close() + return 1 + if ret in (CheckState.FATAL, CheckState.FAIL): + overall_result = 1 + + conn.close() + return overall_result + + +def _get_indexes(conn): + indexes = ['idx_word_word_id', + 'idx_place_addressline_address_place_id', + 'idx_placex_rank_search', + 'idx_placex_rank_address', + 'idx_placex_parent_place_id', + 'idx_placex_geometry_reverse_lookuppolygon', + 'idx_placex_geometry_reverse_placenode', + 'idx_osmline_parent_place_id', + 'idx_osmline_parent_osm_id', + 'idx_postcode_id', + 'idx_postcode_postcode' + ] + if conn.table_exists('search_name'): + indexes.extend(('idx_search_name_nameaddress_vector', + 'idx_search_name_name_vector', + 'idx_search_name_centroid')) + if conn.table_exists('place'): + indexes.extend(('idx_placex_pendingsector', + 'idx_location_area_country_place_id', + 'idx_place_osm_unique' + )) + + return indexes + + +### CHECK FUNCTIONS +# +# Functions are exectured in the order they appear here. + +@_check(hint="""\ + {error} + + Hints: + * Is the database server started? + * Check the NOMINATIM_DATABASE_DSN variable in your local .env + * Try connecting to the database with the same settings + + Project directory: {config.project_dir} + Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN} + """) +def check_connection(conn, config): + """ Checking database connection + """ + if isinstance(conn, _BadConnection): + return CheckState.FATAL, dict(error=conn.msg, config=config) + + return CheckState.OK + +@_check(hint="""\ + placex table not found + + Hints: + * Are you connecting to the right database? + * Did the import process finish without errors? + + Project directory: {config.project_dir} + Current setting of NOMINATIM_DATABASE_DSN: {config.DATABASE_DSN} + """) +def check_placex_table(conn, config): + """ Checking for placex table + """ + if conn.table_exists('placex'): + return CheckState.OK + + return CheckState.FATAL, dict(config=config) + + +@_check(hint="""placex table has no data. Did the import finish sucessfully?""") +def check_placex_size(conn, config): # pylint: disable=W0613 + """ Checking for placex content + """ + with conn.cursor() as cur: + cnt = cur.scalar('SELECT count(*) FROM (SELECT * FROM placex LIMIT 100) x') + + return CheckState.OK if cnt > 0 else CheckState.FATAL + + +@_check(hint="""\ + The Postgresql extension nominatim.so was not correctly loaded. + + Error: {error} + + Hints: + * Check the output of the CMmake/make installation step + * Does nominatim.so exist? + * Does nominatim.so exist on the database server? + * Can nominatim.so be accessed by the database user? + """) +def check_module(conn, config): # pylint: disable=W0613 + """ Checking that nominatim.so module is installed + """ + with conn.cursor() as cur: + try: + out = cur.scalar("SELECT make_standard_name('a')") + except psycopg2.ProgrammingError as err: + return CheckState.FAIL, dict(error=str(err)) + + if out != 'a': + return CheckState.FAIL, dict(error='Unexpected result for make_standard_name()') + + return CheckState.OK + + +@_check(hint="""\ + The indexing didn't finish. {count} entries are not yet indexed. + + To index the remaining entries, run: {index_cmd} + """) +def check_indexing(conn, config): # pylint: disable=W0613 + """ Checking indexing status + """ + with conn.cursor() as cur: + cnt = cur.scalar('SELECT count(*) FROM placex WHERE indexed_status > 0') + + if cnt == 0: + return CheckState.OK + + if conn.index_exists('idx_word_word_id'): + # Likely just an interrupted update. + index_cmd = 'nominatim index' + else: + # Looks like the import process got interrupted. + index_cmd = 'nominatim import --continue indexing' + + return CheckState.FAIL, dict(count=cnt, index_cmd=index_cmd) + + +@_check(hint="""\ + The following indexes are missing: + {indexes} + + Rerun the index creation with: nominatim import --continue db-postprocess + """) +def check_database_indexes(conn, config): # pylint: disable=W0613 + """ Checking that database indexes are complete + """ + missing = [] + for index in _get_indexes(conn): + if not conn.index_exists(index): + missing.append(index) + + if missing: + return CheckState.FAIL, dict(indexes='\n '.join(missing)) + + return CheckState.OK + + +@_check(hint="""\ + At least one index is invalid. That can happen, e.g. when index creation was + disrupted and later restarted. You should delete the affected indices + and recreate them. + + Invalid indexes: + {indexes} + """) +def check_database_index_valid(conn, config): # pylint: disable=W0613 + """ Checking that all database indexes are valid + """ + with conn.cursor() as cur: + cur.execute(""" SELECT relname FROM pg_class, pg_index + WHERE pg_index.indisvalid = false + AND pg_index.indexrelid = pg_class.oid""") + + broken = list(cur) + + if broken: + return CheckState.FAIL, dict(indexes='\n '.join(broken)) + + return CheckState.OK + + +@_check(hint="""\ + {error} + Run TIGER import again: nominatim add-data --tiger-data + """) +def check_tiger_table(conn, config): + """ Checking TIGER external data table. + """ + if not config.get_bool('USE_US_TIGER_DATA'): + return CheckState.NOT_APPLICABLE + + if not conn.table_exists('location_property_tiger'): + return CheckState.FAIL, dict(error='TIGER data table not found.') + + with conn.cursor() as cur: + if cur.scalar('SELECT count(*) FROM location_property_tiger') == 0: + return CheckState.FAIL, dict(error='TIGER data table is empty.') + + return CheckState.OK diff --git a/nominatim/tools/freeze.py b/nominatim/tools/freeze.py new file mode 100644 index 00000000..cc1bf97e --- /dev/null +++ b/nominatim/tools/freeze.py @@ -0,0 +1,43 @@ +""" +Functions for removing unnecessary data from the database. +""" +from pathlib import Path + +UPDATE_TABLES = [ + 'address_levels', + 'gb_postcode', + 'import_osmosis_log', + 'import_polygon_%', + 'location_area%', + 'location_road%', + 'place', + 'planet_osm_%', + 'search_name_%', + 'us_postcode', + 'wikipedia_%' +] + +def drop_update_tables(conn): + """ Drop all tables only necessary for updating the database from + OSM replication data. + """ + + where = ' or '.join(["(tablename LIKE '{}')".format(t) for t in UPDATE_TABLES]) + + with conn.cursor() as cur: + cur.execute("SELECT tablename FROM pg_tables WHERE " + where) + tables = [r[0] for r in cur] + + for table in tables: + cur.execute('DROP TABLE IF EXISTS "{}" CASCADE'.format(table)) + + conn.commit() + + +def drop_flatnode_file(fname): + """ Remove the flatnode file if it exists. + """ + if fname: + fpath = Path(fname) + if fpath.exists(): + fpath.unlink() diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index 1fcb1577..f09c0ced 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -2,12 +2,16 @@ Functions for bringing auxiliary data in the database up-to-date. """ import json +import logging import re +from textwrap import dedent from psycopg2.extras import execute_values from ..db.utils import execute_file +LOG = logging.getLogger() + def update_postcodes(conn, sql_dir): """ Recalculate postcode centroids and add, remove and update entries in the location_postcode table. `conn` is an opne connection to the database. @@ -165,3 +169,65 @@ def create_functions(conn, config, sql_dir, cur.execute(sql) conn.commit() + + +WEBSITE_SCRIPTS = ( + 'deletable.php', + 'details.php', + 'lookup.php', + 'polygons.php', + 'reverse.php', + 'search.php', + 'status.php' +) + +# constants needed by PHP scripts: PHP name, config name, type +PHP_CONST_DEFS = ( + ('Database_DSN', 'DATABASE_DSN', str), + ('Default_Language', 'DEFAULT_LANGUAGE', str), + ('Log_DB', 'LOG_DB', bool), + ('Log_File', 'LOG_FILE', str), + ('Max_Word_Frequency', 'MAX_WORD_FREQUENCY', int), + ('NoAccessControl', 'CORS_NOACCESSCONTROL', bool), + ('Places_Max_ID_count', 'LOOKUP_MAX_COUNT', int), + ('PolygonOutput_MaximumTypes', 'POLYGON_OUTPUT_MAX_TYPES', int), + ('Search_BatchMode', 'SEARCH_BATCH_MODE', bool), + ('Search_NameOnlySearchFrequencyThreshold', 'SEARCH_NAME_ONLY_THRESHOLD', str), + ('Term_Normalization_Rules', 'TERM_NORMALIZATION', str), + ('Use_Aux_Location_data', 'USE_AUX_LOCATION_DATA', bool), + ('Use_US_Tiger_Data', 'USE_US_TIGER_DATA', bool), + ('MapIcon_URL', 'MAPICON_URL', str), +) + + +def setup_website(basedir, phplib_dir, config): + """ Create the website script stubs. + """ + if not basedir.exists(): + LOG.info('Creating website directory.') + basedir.mkdir() + + template = dedent("""\ + assertEmpty($oDB->getListOfTables()); $oDB->exec('CREATE TABLE table1 (id integer, city varchar, country varchar)'); - $oDB->exec('CREATE TABLE table2 (id integer, city varchar, country varchar)'); - $this->assertEquals( - array('table1', 'table2'), - $oDB->getListOfTables() - ); - $this->assertTrue($oDB->deleteTable('table2')); - $this->assertTrue($oDB->deleteTable('table99')); - $this->assertEquals( - array('table1'), - $oDB->getListOfTables() - ); $this->assertTrue($oDB->tableExists('table1')); $this->assertFalse($oDB->tableExists('table99')); $this->assertFalse($oDB->tableExists(null)); - - $this->assertEmpty($oDB->getListOfIndices()); - $oDB->exec('CREATE UNIQUE INDEX table1_index ON table1 (id)'); - $this->assertEquals( - array('table1_index'), - $oDB->getListOfIndices() - ); - $this->assertEmpty($oDB->getListOfIndices('table2')); } # select queries diff --git a/test/php/Nominatim/LibTest.php b/test/php/Nominatim/LibTest.php index 6e9038ee..5111b326 100644 --- a/test/php/Nominatim/LibTest.php +++ b/test/php/Nominatim/LibTest.php @@ -15,26 +15,6 @@ class LibTest extends \PHPUnit\Framework\TestCase $this->assertSame("''", addQuotes('')); } - - public function testCreatePointsAroundCenter() - { - // you might say we're creating a circle - $aPoints = createPointsAroundCenter(0, 0, 2); - - $this->assertEquals( - 101, - count($aPoints) - ); - $this->assertEquals( - array( - array('', 0, 2), - array('', 0.12558103905863, 1.9960534568565), - array('', 0.25066646712861, 1.984229402629) - ), - array_splice($aPoints, 0, 3) - ); - } - public function testParseLatLon() { // no coordinates expected @@ -132,12 +112,4 @@ class LibTest extends \PHPUnit\Framework\TestCase // start == end $this->closestHouseNumberEvenOddOther(50, 50, 0.5, array('even' => 50, 'odd' => 50, 'other' => 50)); } - - public function testGetSearchRankLabel() - { - $this->assertEquals('unknown', getSearchRankLabel(null)); - $this->assertEquals('continent', getSearchRankLabel(0)); - $this->assertEquals('continent', getSearchRankLabel(1)); - $this->assertEquals('other: 30', getSearchRankLabel(30)); - } } diff --git a/test/python/conftest.py b/test/python/conftest.py index ecd40d7c..72a56dcf 100644 --- a/test/python/conftest.py +++ b/test/python/conftest.py @@ -36,6 +36,14 @@ class _TestingCursor(psycopg2.extras.DictCursor): return set((tuple(row) for row in self)) + def table_exists(self, table): + """ Check that a table with the given name exists in the database. + """ + num = self.scalar("""SELECT count(*) FROM pg_tables + WHERE tablename = %s""", (table, )) + return num == 1 + + @pytest.fixture def temp_db(monkeypatch): """ Create an empty database for the test. The database name is also diff --git a/test/python/test_cli.py b/test/python/test_cli.py index 0c0a689e..aa6a5c7f 100644 --- a/test/python/test_cli.py +++ b/test/python/test_cli.py @@ -15,6 +15,9 @@ import nominatim.clicmd.api import nominatim.clicmd.refresh import nominatim.clicmd.admin import nominatim.indexer.indexer +import nominatim.tools.admin +import nominatim.tools.check_database +import nominatim.tools.freeze import nominatim.tools.refresh import nominatim.tools.replication from nominatim.errors import UsageError @@ -50,6 +53,14 @@ def mock_run_legacy(monkeypatch): monkeypatch.setattr(nominatim.cli, 'run_legacy_script', mock) return mock +@pytest.fixture +def mock_func_factory(monkeypatch): + def get_mock(module, func): + mock = MockParamCapture() + monkeypatch.setattr(module, func, mock) + return mock + + return get_mock def test_cli_help(capsys): """ Running nominatim tool without arguments prints help. @@ -62,7 +73,6 @@ def test_cli_help(capsys): @pytest.mark.parametrize("command,script", [ (('import', '--continue', 'load-data'), 'setup'), - (('freeze',), 'setup'), (('special-phrases',), 'specialphrases'), (('add-data', '--tiger-data', 'tiger'), 'setup'), (('add-data', '--file', 'foo.osm'), 'update'), @@ -75,26 +85,42 @@ def test_legacy_commands_simple(mock_run_legacy, command, script): assert mock_run_legacy.last_args[0] == script + '.php' +def test_freeze_command(mock_func_factory, temp_db): + mock_drop = mock_func_factory(nominatim.tools.freeze, 'drop_update_tables') + mock_flatnode = mock_func_factory(nominatim.tools.freeze, 'drop_flatnode_file') + + assert 0 == call_nominatim('freeze') + + assert mock_drop.called == 1 + assert mock_flatnode.called == 1 + + @pytest.mark.parametrize("params", [('--warm', ), ('--warm', '--reverse-only'), - ('--warm', '--search-only'), - ('--check-database', )]) -def test_admin_command_legacy(monkeypatch, params): - mock_run_legacy = MockParamCapture() - monkeypatch.setattr(nominatim.clicmd.admin, 'run_legacy_script', mock_run_legacy) + ('--warm', '--search-only')]) +def test_admin_command_legacy(mock_func_factory, params): + mock_run_legacy = mock_func_factory(nominatim.clicmd.admin, 'run_legacy_script') assert 0 == call_nominatim('admin', *params) assert mock_run_legacy.called == 1 + @pytest.mark.parametrize("func, params", [('analyse_indexing', ('--analyse-indexing', ))]) -def test_admin_command_tool(temp_db, monkeypatch, func, params): - mock = MockParamCapture() - monkeypatch.setattr(nominatim.tools.admin, func, mock) +def test_admin_command_tool(temp_db, mock_func_factory, func, params): + mock = mock_func_factory(nominatim.tools.admin, func) assert 0 == call_nominatim('admin', *params) assert mock.called == 1 + +def test_admin_command_check_database(mock_func_factory): + mock = mock_func_factory(nominatim.tools.check_database, 'check_database') + + assert 0 == call_nominatim('admin', '--check-database') + assert mock.called == 1 + + @pytest.mark.parametrize("name,oid", [('file', 'foo.osm'), ('diff', 'foo.osc'), ('node', 12), ('way', 8), ('relation', 32)]) def test_add_data_command(mock_run_legacy, name, oid): @@ -109,12 +135,10 @@ def test_add_data_command(mock_run_legacy, name, oid): (['--boundaries-only'], 1, 0), (['--no-boundaries'], 0, 1), (['--boundaries-only', '--no-boundaries'], 0, 0)]) -def test_index_command(monkeypatch, temp_db_cursor, params, do_bnds, do_ranks): +def test_index_command(mock_func_factory, temp_db_cursor, params, do_bnds, do_ranks): temp_db_cursor.execute("CREATE TABLE import_status (indexed bool)") - bnd_mock = MockParamCapture() - monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_boundaries', bnd_mock) - rank_mock = MockParamCapture() - monkeypatch.setattr(nominatim.indexer.indexer.Indexer, 'index_by_rank', rank_mock) + bnd_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_boundaries') + rank_mock = mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_by_rank') assert 0 == call_nominatim('index', *params) @@ -125,11 +149,9 @@ def test_index_command(monkeypatch, temp_db_cursor, params, do_bnds, do_ranks): @pytest.mark.parametrize("command,params", [ ('wiki-data', ('setup.php', '--import-wikipedia-articles')), ('importance', ('update.php', '--recompute-importance')), - ('website', ('setup.php', '--setup-website')), ]) -def test_refresh_legacy_command(monkeypatch, temp_db, command, params): - mock_run_legacy = MockParamCapture() - monkeypatch.setattr(nominatim.clicmd.refresh, 'run_legacy_script', mock_run_legacy) +def test_refresh_legacy_command(mock_func_factory, temp_db, command, params): + mock_run_legacy = mock_func_factory(nominatim.clicmd.refresh, 'run_legacy_script') assert 0 == call_nominatim('refresh', '--' + command) @@ -142,18 +164,17 @@ def test_refresh_legacy_command(monkeypatch, temp_db, command, params): ('word-counts', 'recompute_word_counts'), ('address-levels', 'load_address_levels_from_file'), ('functions', 'create_functions'), + ('website', 'setup_website'), ]) -def test_refresh_command(monkeypatch, temp_db, command, func): - func_mock = MockParamCapture() - monkeypatch.setattr(nominatim.tools.refresh, func, func_mock) +def test_refresh_command(mock_func_factory, temp_db, command, func): + func_mock = mock_func_factory(nominatim.tools.refresh, func) assert 0 == call_nominatim('refresh', '--' + command) assert func_mock.called == 1 -def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db): - mock_run_legacy = MockParamCapture() - monkeypatch.setattr(nominatim.clicmd.refresh, 'run_legacy_script', mock_run_legacy) +def test_refresh_importance_computed_after_wiki_import(mock_func_factory, temp_db): + mock_run_legacy = mock_func_factory(nominatim.clicmd.refresh, 'run_legacy_script') assert 0 == call_nominatim('refresh', '--importance', '--wiki-data') @@ -165,9 +186,8 @@ def test_refresh_importance_computed_after_wiki_import(monkeypatch, temp_db): (('--init', '--no-update-functions'), 'init_replication'), (('--check-for-updates',), 'check_for_updates') ]) -def test_replication_command(monkeypatch, temp_db, params, func): - func_mock = MockParamCapture() - monkeypatch.setattr(nominatim.tools.replication, func, func_mock) +def test_replication_command(mock_func_factory, temp_db, params, func): + func_mock = mock_func_factory(nominatim.tools.replication, func) assert 0 == call_nominatim('replication', *params) assert func_mock.called == 1 @@ -188,11 +208,10 @@ def test_replication_update_bad_interval_for_geofabrik(monkeypatch, temp_db): @pytest.mark.parametrize("state", [nominatim.tools.replication.UpdateState.UP_TO_DATE, nominatim.tools.replication.UpdateState.NO_CHANGES]) -def test_replication_update_once_no_index(monkeypatch, temp_db, temp_db_conn, +def test_replication_update_once_no_index(mock_func_factory, temp_db, temp_db_conn, status_table, state): status.set_status(temp_db_conn, date=dt.datetime.now(dt.timezone.utc), seq=1) - func_mock = MockParamCapture(retval=state) - monkeypatch.setattr(nominatim.tools.replication, 'update', func_mock) + func_mock = mock_func_factory(nominatim.tools.replication, 'update') assert 0 == call_nominatim('replication', '--once', '--no-index') @@ -236,9 +255,8 @@ def test_replication_update_continuous_no_change(monkeypatch, temp_db_conn, stat assert sleep_mock.last_args[0] == 60 -def test_serve_command(monkeypatch): - func = MockParamCapture() - monkeypatch.setattr(nominatim.cli, 'run_php_server', func) +def test_serve_command(mock_func_factory): + func = mock_func_factory(nominatim.cli, 'run_php_server') call_nominatim('serve') @@ -254,9 +272,8 @@ def test_serve_command(monkeypatch): ('details', '--place_id', '10001'), ('status',) ]) -def test_api_commands_simple(monkeypatch, params): - mock_run_api = MockParamCapture() - monkeypatch.setattr(nominatim.clicmd.api, 'run_api_script', mock_run_api) +def test_api_commands_simple(mock_func_factory, params): + mock_run_api = mock_func_factory(nominatim.clicmd.api, 'run_api_script') assert 0 == call_nominatim(*params) diff --git a/test/python/test_db_connection.py b/test/python/test_db_connection.py index ef1ae741..11ad691a 100644 --- a/test/python/test_db_connection.py +++ b/test/python/test_db_connection.py @@ -20,12 +20,31 @@ def test_connection_table_exists(db, temp_db_cursor): assert db.table_exists('foobar') == True +def test_connection_index_exists(db, temp_db_cursor): + assert db.index_exists('some_index') == False + + temp_db_cursor.execute('CREATE TABLE foobar (id INT)') + temp_db_cursor.execute('CREATE INDEX some_index ON foobar(id)') + + assert db.index_exists('some_index') == True + assert db.index_exists('some_index', table='foobar') == True + assert db.index_exists('some_index', table='bar') == False + + +def test_connection_server_version_tuple(db): + ver = db.server_version_tuple() + + assert isinstance(ver, tuple) + assert len(ver) == 2 + assert ver[0] > 8 + def test_cursor_scalar(db, temp_db_cursor): temp_db_cursor.execute('CREATE TABLE dummy (id INT)') with db.cursor() as cur: assert cur.scalar('SELECT count(*) FROM dummy') == 0 + def test_cursor_scalar_many_rows(db): with db.cursor() as cur: with pytest.raises(RuntimeError): diff --git a/test/python/test_tools_check_database.py b/test/python/test_tools_check_database.py new file mode 100644 index 00000000..0b5c23a6 --- /dev/null +++ b/test/python/test_tools_check_database.py @@ -0,0 +1,76 @@ +""" +Tests for database integrity checks. +""" +import pytest + +from nominatim.tools import check_database as chkdb + +def test_check_database_unknown_db(def_config, monkeypatch): + monkeypatch.setenv('NOMINATIM_DATABASE_DSN', 'pgsql:dbname=fjgkhughwgh2423gsags') + assert 1 == chkdb.check_database(def_config) + + +def test_check_conection_good(temp_db_conn, def_config): + assert chkdb.check_connection(temp_db_conn, def_config) == chkdb.CheckState.OK + + +def test_check_conection_bad(def_config): + badconn = chkdb._BadConnection('Error') + assert chkdb.check_connection(badconn, def_config) == chkdb.CheckState.FATAL + + +def test_check_placex_table_good(temp_db_cursor, temp_db_conn, def_config): + temp_db_cursor.execute('CREATE TABLE placex (place_id int)') + assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.OK + + +def test_check_placex_table_bad(temp_db_conn, def_config): + assert chkdb.check_placex_table(temp_db_conn, def_config) == chkdb.CheckState.FATAL + + +def test_check_placex_table_size_good(temp_db_cursor, temp_db_conn, def_config): + temp_db_cursor.execute('CREATE TABLE placex (place_id int)') + temp_db_cursor.execute('INSERT INTO placex VALUES (1), (2)') + assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.OK + + +def test_check_placex_table_size_bad(temp_db_cursor, temp_db_conn, def_config): + temp_db_cursor.execute('CREATE TABLE placex (place_id int)') + assert chkdb.check_placex_size(temp_db_conn, def_config) == chkdb.CheckState.FATAL + + +def test_check_module_bad(temp_db_conn, def_config): + assert chkdb.check_module(temp_db_conn, def_config) == chkdb.CheckState.FAIL + + +def test_check_indexing_good(temp_db_cursor, temp_db_conn, def_config): + temp_db_cursor.execute('CREATE TABLE placex (place_id int, indexed_status smallint)') + temp_db_cursor.execute('INSERT INTO placex VALUES (1, 0), (2, 0)') + assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.OK + + +def test_check_indexing_bad(temp_db_cursor, temp_db_conn, def_config): + temp_db_cursor.execute('CREATE TABLE placex (place_id int, indexed_status smallint)') + temp_db_cursor.execute('INSERT INTO placex VALUES (1, 0), (2, 2)') + assert chkdb.check_indexing(temp_db_conn, def_config) == chkdb.CheckState.FAIL + + +def test_check_database_indexes_bad(temp_db_conn, def_config): + assert chkdb.check_database_indexes(temp_db_conn, def_config) == chkdb.CheckState.FAIL + + +def test_check_tiger_table_disabled(temp_db_conn, def_config, monkeypatch): + monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA' , 'no') + assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.NOT_APPLICABLE + + +def test_check_tiger_table_enabled(temp_db_cursor, temp_db_conn, def_config, monkeypatch): + monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA' , 'yes') + assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.FAIL + + temp_db_cursor.execute('CREATE TABLE location_property_tiger (place_id int)') + assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.FAIL + + temp_db_cursor.execute('INSERT INTO location_property_tiger VALUES (1), (2)') + assert chkdb.check_tiger_table(temp_db_conn, def_config) == chkdb.CheckState.OK + diff --git a/test/python/test_tools_freeze.py b/test/python/test_tools_freeze.py new file mode 100644 index 00000000..fcdab23a --- /dev/null +++ b/test/python/test_tools_freeze.py @@ -0,0 +1,51 @@ +""" +Tests for freeze functions (removing unused database parts). +""" +import pytest + +from nominatim.tools import freeze + +NOMINATIM_RUNTIME_TABLES = [ + 'country_name', 'country_osm_grid', + 'location_postcode', 'location_property_osmline', 'location_property_tiger', + 'placex', 'place_adressline', + 'search_name', + 'word' +] + +NOMINATIM_DROP_TABLES = [ + 'address_levels', + 'location_area', 'location_area_country', 'location_area_large_100', + 'location_road_1', + 'place', 'planet_osm_nodes', 'planet_osm_rels', 'planet_osm_ways', + 'search_name_111', + 'wikipedia_article', 'wikipedia_redirect' +] + +def test_drop_tables(temp_db_conn, temp_db_cursor): + for table in NOMINATIM_RUNTIME_TABLES + NOMINATIM_DROP_TABLES: + temp_db_cursor.execute('CREATE TABLE {} (id int)'.format(table)) + + freeze.drop_update_tables(temp_db_conn) + + for table in NOMINATIM_RUNTIME_TABLES: + assert temp_db_cursor.table_exists(table) + + for table in NOMINATIM_DROP_TABLES: + assert not temp_db_cursor.table_exists(table) + +def test_drop_flatnode_file_no_file(): + freeze.drop_flatnode_file('') + + +def test_drop_flatnode_file_file_already_gone(tmp_path): + freeze.drop_flatnode_file(str(tmp_path / 'something.store')) + + +def test_drop_flatnode_file_delte(tmp_path): + flatfile = tmp_path / 'flatnode.store' + flatfile.write_text('Some content') + + freeze.drop_flatnode_file(str(flatfile)) + + assert not flatfile.exists() diff --git a/test/python/test_tools_refresh_setup_website.py b/test/python/test_tools_refresh_setup_website.py new file mode 100644 index 00000000..126fc561 --- /dev/null +++ b/test/python/test_tools_refresh_setup_website.py @@ -0,0 +1,70 @@ +""" +Tests for setting up the website scripts. +""" +from pathlib import Path +import subprocess + +import pytest + +from nominatim.tools import refresh + +@pytest.fixture +def envdir(tmpdir): + (tmpdir / 'php').mkdir() + (tmpdir / 'php' / 'website').mkdir() + return tmpdir + + +@pytest.fixture +def test_script(envdir): + def _create_file(code): + outfile = envdir / 'php' / 'website' / 'search.php' + outfile.write_text('