forked from hans/Nominatim
move setup function to python
There are still back-calls to PHP for some of the sub-steps. These needs some larger refactoring to be moved to Python.
This commit is contained in:
Sarah Hoffmann
140
nominatim/clicmd/setup.py
Normal file
140
nominatim/clicmd/setup.py
Normal file
@@ -0,0 +1,140 @@
|
||||
"""
|
||||
Implementation of the 'import' subcommand.
|
||||
"""
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
|
||||
from ..tools.exec_utils import run_legacy_script
|
||||
from ..db.connection import connect
|
||||
from ..db import status
|
||||
from ..errors import UsageError
|
||||
|
||||
# Do not repeat documentation of subcommand classes.
|
||||
# pylint: disable=C0111
|
||||
# Using non-top-level imports to avoid eventually unused imports.
|
||||
# pylint: disable=E0012,C0415
|
||||
|
||||
LOG = logging.getLogger()
|
||||
|
||||
class SetupAll:
|
||||
"""\
|
||||
Create a new Nominatim database from an OSM file.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
group_name = parser.add_argument_group('Required arguments')
|
||||
group = group_name.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--osm-file', metavar='FILE',
|
||||
help='OSM file to be imported.')
|
||||
group.add_argument('--continue', dest='continue_at',
|
||||
choices=['load-data', 'indexing', 'db-postprocess'],
|
||||
help='Continue an import that was interrupted')
|
||||
group = parser.add_argument_group('Optional arguments')
|
||||
group.add_argument('--osm2pgsql-cache', metavar='SIZE', type=int,
|
||||
help='Size of cache to be used by osm2pgsql (in MB)')
|
||||
group.add_argument('--reverse-only', action='store_true',
|
||||
help='Do not create tables and indexes for searching')
|
||||
group.add_argument('--no-partitions', action='store_true',
|
||||
help="""Do not partition search indices
|
||||
(speeds up import of single country extracts)""")
|
||||
group.add_argument('--no-updates', action='store_true',
|
||||
help="""Do not keep tables that are only needed for
|
||||
updating the database later""")
|
||||
group = parser.add_argument_group('Expert options')
|
||||
group.add_argument('--ignore-errors', action='store_true',
|
||||
help='Continue import even when errors in SQL are present')
|
||||
group.add_argument('--index-noanalyse', action='store_true',
|
||||
help='Do not perform analyse operations during index')
|
||||
|
||||
|
||||
@staticmethod
|
||||
def run(args): # pylint: disable=too-many-statements
|
||||
from ..tools import database_import
|
||||
from ..tools import refresh
|
||||
from ..indexer.indexer import Indexer
|
||||
|
||||
if args.osm_file and not Path(args.osm_file).is_file():
|
||||
LOG.fatal("OSM file '%s' does not exist.", args.osm_file)
|
||||
raise UsageError('Cannot access file.')
|
||||
|
||||
if args.continue_at is None:
|
||||
database_import.setup_database_skeleton(args.config.get_libpq_dsn(),
|
||||
args.data_dir,
|
||||
args.no_partitions,
|
||||
rouser=args.config.DATABASE_WEBUSER)
|
||||
|
||||
LOG.warning('Installing database module')
|
||||
with connect(args.config.get_libpq_dsn()) as conn:
|
||||
database_import.install_module(args.module_dir, args.project_dir,
|
||||
args.config.DATABASE_MODULE_PATH,
|
||||
conn=conn)
|
||||
|
||||
LOG.warning('Importing OSM data file')
|
||||
database_import.import_osm_data(Path(args.osm_file),
|
||||
args.osm2pgsql_options(0, 1),
|
||||
drop=args.no_updates)
|
||||
|
||||
LOG.warning('Create functions (1st pass)')
|
||||
with connect(args.config.get_libpq_dsn()) as conn:
|
||||
refresh.create_functions(conn, args.config, args.sqllib_dir,
|
||||
False, False)
|
||||
|
||||
LOG.warning('Create tables')
|
||||
params = ['setup.php', '--create-tables', '--create-partition-tables']
|
||||
if args.reverse_only:
|
||||
params.append('--reverse-only')
|
||||
run_legacy_script(*params, nominatim_env=args)
|
||||
|
||||
LOG.warning('Create functions (2nd pass)')
|
||||
with connect(args.config.get_libpq_dsn()) as conn:
|
||||
refresh.create_functions(conn, args.config, args.sqllib_dir,
|
||||
False, False)
|
||||
|
||||
LOG.warning('Importing wikipedia importance data')
|
||||
data_path = Path(args.config.WIKIPEDIA_DATA_PATH or args.project_dir)
|
||||
if refresh.import_wikipedia_articles(args.config.get_libpq_dsn(),
|
||||
data_path) > 0:
|
||||
LOG.error('Wikipedia importance dump file not found. '
|
||||
'Will be using default importances.')
|
||||
|
||||
LOG.warning('Initialise tables')
|
||||
with connect(args.config.get_libpq_dsn()) as conn:
|
||||
database_import.truncate_data_tables(conn, args.config.MAX_WORD_FREQUENCY)
|
||||
|
||||
if args.continue_at is None or args.continue_at == 'load-data':
|
||||
LOG.warning('Load data into placex table')
|
||||
database_import.load_data(args.config.get_libpq_dsn(),
|
||||
args.data_dir,
|
||||
args.threads or psutil.cpu_count() or 1)
|
||||
|
||||
LOG.warning('Calculate postcodes')
|
||||
run_legacy_script('setup.php', '--calculate-postcodes', nominatim_env=args)
|
||||
|
||||
if args.continue_at is None or args.continue_at in ('load-data', 'indexing'):
|
||||
LOG.warning('Indexing places')
|
||||
indexer = Indexer(args.config.get_libpq_dsn(),
|
||||
args.threads or psutil.cpu_count() or 1)
|
||||
indexer.index_full(analyse=not args.index_noanalyse)
|
||||
|
||||
LOG.warning('Post-process tables')
|
||||
params = ['setup.php', '--create-search-indices', '--create-country-names']
|
||||
if args.no_updates:
|
||||
params.append('--drop')
|
||||
run_legacy_script(*params, nominatim_env=args)
|
||||
|
||||
webdir = args.project_dir / 'website'
|
||||
LOG.warning('Setup website at %s', webdir)
|
||||
refresh.setup_website(webdir, args.phplib_dir, args.config)
|
||||
|
||||
with connect(args.config.get_libpq_dsn()) as conn:
|
||||
try:
|
||||
dbdate = status.compute_database_date(conn)
|
||||
status.set_status(conn, dbdate)
|
||||
LOG.info('Database is at %s.', dbdate)
|
||||
except Exception as exc: # pylint: disable=broad-except
|
||||
LOG.error('Cannot determine date of database: %s', exc)
|
||||
|
||||
return 0
|
||||
Reference in New Issue
Block a user