diff --git a/cmake/script.tmpl b/cmake/script.tmpl index 30b8717b..aa25a124 100755 --- a/cmake/script.tmpl +++ b/cmake/script.tmpl @@ -8,5 +8,6 @@ require('@CMAKE_SOURCE_DIR@/lib/dotenv_loader.php'); @define('CONST_DataDir', '@CMAKE_SOURCE_DIR@'); loadDotEnv(); +$_SERVER['NOMINATIM_NOMINATIM_TOOL'] = '@CMAKE_BINARY_DIR@/nominatim'; require_once('@CMAKE_SOURCE_DIR@/lib/admin/@script_source@'); diff --git a/cmake/tool.tmpl b/cmake/tool.tmpl index 40f2b8ea..43646792 100755 --- a/cmake/tool.tmpl +++ b/cmake/tool.tmpl @@ -1,8 +1,11 @@ #!/usr/bin/env python3 import sys +import os sys.path.insert(1, '@CMAKE_SOURCE_DIR@') +os.environ['NOMINATIM_NOMINATIM_TOOL'] = __file__ + from nominatim import cli exit(cli.nominatim(module_dir='@CMAKE_BINARY_DIR@/module', diff --git a/lib/Shell.php b/lib/Shell.php index 59c4473b..72f90735 100644 --- a/lib/Shell.php +++ b/lib/Shell.php @@ -7,7 +7,7 @@ class Shell public function __construct($sBaseCmd, ...$aParams) { if (!$sBaseCmd) { - throw new Exception('Command missing in new() call'); + throw new \Exception('Command missing in new() call'); } $this->baseCmd = $sBaseCmd; $this->aParams = array(); diff --git a/lib/admin/update.php b/lib/admin/update.php index 50f611d7..fe9658b5 100644 --- a/lib/admin/update.php +++ b/lib/admin/update.php @@ -105,25 +105,14 @@ if ($fPostgresVersion >= 11.0) { } -$oIndexCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py')) - ->addParams('--database', $aDSNInfo['database']) - ->addParams('--port', $aDSNInfo['port']) - ->addParams('--threads', $aResult['index-instances']); -if (!$aResult['quiet']) { - $oIndexCmd->addParams('--verbose'); +$oIndexCmd = (new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'))) + ->addParams('index'); +if ($aResult['quiet']) { + $oIndexCmd->addParams('--quiet'); } if ($aResult['verbose']) { $oIndexCmd->addParams('--verbose'); } -if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { - $oIndexCmd->addParams('--host', $aDSNInfo['hostspec']); -} -if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { - $oIndexCmd->addParams('--username', $aDSNInfo['username']); -} -if (isset($aDSNInfo['password']) && $aDSNInfo['password']) { - $oIndexCmd->addEnvPair('PGPASSWORD', $aDSNInfo['password']); -} $sPyosmiumBin = getSetting('PYOSMIUM_BINARY'); $sBaseURL = getSetting('REPLICATION_URL'); @@ -288,15 +277,9 @@ if ($aResult['recompute-word-counts']) { } if ($aResult['index']) { - $oCmd = (clone $oIndexCmd) - ->addParams('--minrank', $aResult['index-rank'], '-b'); - $oCmd->run(); - $oCmd = (clone $oIndexCmd) ->addParams('--minrank', $aResult['index-rank']); $oCmd->run(); - - $oDB->exec('update import_status set indexed = true'); } if ($aResult['update-address-levels']) { @@ -438,15 +421,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { if (!$aResult['no-index']) { $fCMDStartTime = time(); - $oThisIndexCmd = clone($oIndexCmd); - $oThisIndexCmd->addParams('-b'); - echo $oThisIndexCmd->escapedCmd()."\n"; - $iErrorLevel = $oThisIndexCmd->run(); - if ($iErrorLevel) { - echo "Error: $iErrorLevel\n"; - exit($iErrorLevel); - } - $oThisIndexCmd = clone($oIndexCmd); echo $oThisIndexCmd->escapedCmd()."\n"; $iErrorLevel = $oThisIndexCmd->run(); @@ -463,9 +437,6 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) { var_Dump($sSQL); $oDB->exec($sSQL); echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; - - $sSQL = 'update import_status set indexed = true'; - $oDB->exec($sSQL); } else { if ($aResult['import-osmosis-all']) { echo "Error: --no-index cannot be used with continuous imports (--import-osmosis-all).\n"; diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index 77b14a8a..d17fdca7 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -549,26 +549,15 @@ class SetupFunctions { $this->checkModulePresence(); // raises exception on failure - $oBaseCmd = (new \Nominatim\Shell(CONST_DataDir.'/nominatim/nominatim.py')) - ->addParams('--database', $this->aDSNInfo['database']) - ->addParams('--port', $this->aDSNInfo['port']) - ->addParams('--threads', $this->iInstances); + $oBaseCmd = (new \Nominatim\Shell(getSetting('NOMINATIM_TOOL'))) + ->addParams('index'); - if (!$this->bQuiet) { - $oBaseCmd->addParams('-v'); + if ($this->bQuiet) { + $oBaseCmd->addParams('-q'); } if ($this->bVerbose) { $oBaseCmd->addParams('-v'); } - if (isset($this->aDSNInfo['hostspec'])) { - $oBaseCmd->addParams('--host', $this->aDSNInfo['hostspec']); - } - if (isset($this->aDSNInfo['username'])) { - $oBaseCmd->addParams('--user', $this->aDSNInfo['username']); - } - if (isset($this->aDSNInfo['password'])) { - $oBaseCmd->addEnvPair('PGPASSWORD', $this->aDSNInfo['password']); - } info('Index ranks 0 - 4'); $oCmd = (clone $oBaseCmd)->addParams('--maxrank', 4); @@ -581,14 +570,14 @@ class SetupFunctions if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index administrative boundaries'); - $oCmd = (clone $oBaseCmd)->addParams('-b'); + $oCmd = (clone $oBaseCmd)->addParams('--boundaries-only'); $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); } info('Index ranks 5 - 25'); - $oCmd = (clone $oBaseCmd)->addParams('--minrank', 5, '--maxrank', 25); + $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 5, '--maxrank', 25); $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); @@ -597,7 +586,7 @@ class SetupFunctions if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); info('Index ranks 26 - 30'); - $oCmd = (clone $oBaseCmd)->addParams('--minrank', 26); + $oCmd = (clone $oBaseCmd)->addParams('--no-boundaries', '--minrank', 26); $iStatus = $oCmd->run(); if ($iStatus != 0) { fail('error status ' . $iStatus . ' running nominatim!'); diff --git a/nominatim/cli.py b/nominatim/cli.py index acb6839f..65ea90bb 100644 --- a/nominatim/cli.py +++ b/nominatim/cli.py @@ -78,7 +78,7 @@ class CommandlineParser: args.project_dir = Path(args.project_dir) logging.basicConfig(stream=sys.stderr, - format='%(asctime)s %(levelname)s: %(message)s', + format='%(asctime)s: %(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=max(4 - args.verbose, 1) * 10) @@ -328,6 +328,9 @@ class UpdateIndex: if not args.boundaries_only: indexer.index_by_rank(args.minrank, args.maxrank) + if not args.no_boundaries and not args.boundaries_only: + indexer.update_status_table() + return 0 diff --git a/nominatim/indexer/indexer.py b/nominatim/indexer/indexer.py index d86303c4..094d1279 100644 --- a/nominatim/indexer/indexer.py +++ b/nominatim/indexer/indexer.py @@ -124,6 +124,13 @@ class Indexer: else: self.index(RankRunner(maxrank)) + def update_status_table(self): + """ Update the status in the status table to 'indexed'. + """ + with self.conn.cursor() as cur: + cur.execute('UPDATE import_status SET indexed = true') + self.conn.commit() + def index(self, obj, batch=1): """ Index a single rank or table. `obj` describes the SQL to use for indexing. `batch` describes the number of objects that diff --git a/nominatim/indexer/progress.py b/nominatim/indexer/progress.py index 99120673..c9d8816b 100644 --- a/nominatim/indexer/progress.py +++ b/nominatim/indexer/progress.py @@ -26,7 +26,7 @@ class ProgressLogger: self.done_places = 0 self.rank_start_time = datetime.now() self.log_interval = log_interval - self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.INFO) else total + 1 + self.next_info = INITIAL_PROGRESS if LOG.isEnabledFor(logging.WARNING) else total + 1 def add(self, num=1): """ Mark `num` places as processed. Print a log message if the @@ -47,9 +47,9 @@ class ProgressLogger: places_per_sec = self.done_places / done_time eta = (self.total_places - self.done_places) / places_per_sec - LOG.info("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f", - self.done_places, int(done_time), - places_per_sec, self.name, eta) + LOG.warning("Done %d in %d @ %.3f per second - %s ETA (seconds): %.2f", + self.done_places, int(done_time), + places_per_sec, self.name, eta) self.next_info += int(places_per_sec) * self.log_interval diff --git a/nominatim/nominatim.py b/nominatim/nominatim.py deleted file mode 100755 index fdc2bcba..00000000 --- a/nominatim/nominatim.py +++ /dev/null @@ -1,84 +0,0 @@ -#! /usr/bin/env python3 -#----------------------------------------------------------------------------- -# nominatim - [description] -#----------------------------------------------------------------------------- -# -# Indexing tool for the Nominatim database. -# -# Based on C version by Brian Quinion -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -#----------------------------------------------------------------------------- -from argparse import ArgumentParser, RawDescriptionHelpFormatter -import logging -import sys -import getpass - -from indexer.indexer import Indexer - -def nominatim_arg_parser(): - """ Setup the command-line parser for the tool. - """ - parser = ArgumentParser(description="Indexing tool for Nominatim.", - formatter_class=RawDescriptionHelpFormatter) - - parser.add_argument('-d', '--database', - dest='dbname', action='store', default='nominatim', - help='Name of the PostgreSQL database to connect to.') - parser.add_argument('-U', '--username', - dest='user', action='store', - help='PostgreSQL user name.') - parser.add_argument('-W', '--password', - dest='password_prompt', action='store_true', - help='Force password prompt.') - parser.add_argument('-H', '--host', - dest='host', action='store', - help='PostgreSQL server hostname or socket location.') - parser.add_argument('-P', '--port', - dest='port', action='store', - help='PostgreSQL server port') - parser.add_argument('-b', '--boundary-only', - dest='boundary_only', action='store_true', - help='Only index administrative boundaries (ignores min/maxrank).') - parser.add_argument('-r', '--minrank', - dest='minrank', type=int, metavar='RANK', default=0, - help='Minimum/starting rank.') - parser.add_argument('-R', '--maxrank', - dest='maxrank', type=int, metavar='RANK', default=30, - help='Maximum/finishing rank.') - parser.add_argument('-t', '--threads', - dest='threads', type=int, metavar='NUM', default=1, - help='Number of threads to create for indexing.') - parser.add_argument('-v', '--verbose', - dest='loglevel', action='count', default=0, - help='Increase verbosity') - - return parser - -if __name__ == '__main__': - OPTIONS = nominatim_arg_parser().parse_args(sys.argv[1:]) - - logging.basicConfig(stream=sys.stderr, format='%(levelname)s: %(message)s', - level=max(3 - OPTIONS.loglevel, 0) * 10) - - OPTIONS.password = None - if OPTIONS.password_prompt: - PASSWORD = getpass.getpass("Database password: ") - OPTIONS.password = PASSWORD - - if OPTIONS.boundary_only: - Indexer(OPTIONS).index_boundaries() - else: - Indexer(OPTIONS).index_by_rank() diff --git a/test/bdd/steps/nominatim_environment.py b/test/bdd/steps/nominatim_environment.py index 68d7b2f4..0ee92137 100644 --- a/test/bdd/steps/nominatim_environment.py +++ b/test/bdd/steps/nominatim_environment.py @@ -91,6 +91,7 @@ class NominatimEnvironment: self.test_env['NOMINATIM_BINDIR'] = self.src_dir / 'utils' self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.build_dir / 'module' self.test_env['NOMINATIM_OSM2PGSQL_BINARY'] = self.build_dir / 'osm2pgsql' / 'osm2pgsql' + self.test_env['NOMINATIM_NOMINATIM_TOOL'] = self.build_dir / 'nominatim' if self.server_module_path: self.test_env['NOMINATIM_DATABASE_MODULE_PATH'] = self.server_module_path