hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-09 19:44:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

move table creation to jinja-based preprocessing

Browse Source
This commit is contained in:
Sarah Hoffmann
2021-03-03 22:07:51 +01:00
parent 6cda021d9b
commit eacabb0e96
9 changed files with 142 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
nominatim/clicmd/setup.py
View File

@@ -79,20 +79,22 @@ class SetupAll:
drop=args.no_updates,
ignore_errors=args.ignore_errors)
LOG.warning('Create functions (1st pass)')
with connect(args.config.get_libpq_dsn()) as conn:
LOG.warning('Create functions (1st pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir,
False, False)
LOG.warning('Create tables')
params = ['setup.php', '--create-tables', '--create-partition-tables']
if args.reverse_only:
params.append('--reverse-only')
run_legacy_script(*params, nominatim_env=args,
throw_on_fail=not args.ignore_errors)
LOG.warning('Create functions (2nd pass)')
with connect(args.config.get_libpq_dsn()) as conn:
LOG.warning('Create tables')
database_import.create_tables(conn, args.config, args.sqllib_dir,
reverse_only=args.reverse_only)
refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
LOG.warning('Create functions (2nd pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir,
False, False)
LOG.warning('Create table triggers')
database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
LOG.warning('Create partition tables')
database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
LOG.warning('Create functions (3rd pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir,
False, False)

21
nominatim/clicmd/transition.py
View File

@@ -35,6 +35,10 @@ class AdminTransition:
help='Import a osm file')
group.add_argument('--load-data', action='store_true',
help='Copy data to live tables from import table')
group.add_argument('--create-tables', action='store_true',
help='Create main tables')
group.add_argument('--create-partition-tables', action='store_true',
help='Create required partition tables')
group.add_argument('--index', action='store_true',
help='Index the data')
group = parser.add_argument_group('Options')
@@ -50,10 +54,13 @@ class AdminTransition:
help='Do not perform analyse operations during index')
group.add_argument('--ignore-errors', action='store_true',
help="Ignore certain erros on import.")
group.add_argument('--reverse-only', action='store_true',
help='Do not create search tables and indexes')
@staticmethod
def run(args):
from ..tools import database_import
from ..tools import refresh
if args.create_db:
LOG.warning('Create DB')
@@ -80,6 +87,20 @@ class AdminTransition:
drop=args.drop,
ignore_errors=args.ignore_errors)
if args.create_tables:
LOG.warning('Create Tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_tables(conn, args.config, args.sqllib_dir, args.reverse_only)
refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
refresh.create_functions(conn, args.config, args.sqllib_dir,
enable_diff_updates=False)
database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
if args.create_partition_tables:
LOG.warning('Create Partition Tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
if args.load_data:
LOG.warning('Load data')
with connect(args.config.get_libpq_dsn()) as conn:

8
nominatim/db/sql_preprocessor.py
View File

@@ -45,6 +45,14 @@ class SQLPreprocessor: # pylint: disable=too-few-public-methods
db_info['tables'] = _get_tables(conn)
db_info['reverse_only'] = 'search_name' not in db_info['tables']
db_info['tablespace'] = {}
for subset in ('ADDRESS', 'SEARCH', 'AUX'):
for kind in ('DATA', 'INDEX'):
tspace = getattr(config, 'TABLESPACE_{}_{}'.format(subset, kind))
if tspace:
tspace = 'TABLESPACE "{}"'.format(tspace)
db_info['tablespace']['{}_{}'.format(subset.lower, kind.lower())] = tspace
self.env.globals['config'] = config
self.env.globals['db'] = db_info
self.env.globals['modulepath'] = config.DATABASE_MODULE_PATH or \

27
nominatim/tools/database_import.py
View File

@@ -14,6 +14,7 @@ import psycopg2
from ..db.connection import connect, get_pg_env
from ..db import utils as db_utils
from ..db.async_connection import DBConnection
from ..db.sql_preprocessor import SQLPreprocessor
from .exec_utils import run_osm2pgsql
from ..errors import UsageError
from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION
@@ -178,6 +179,32 @@ def import_osm_data(osm_file, options, drop=False, ignore_errors=False):
Path(options['flatnode_file']).unlink()
def create_tables(conn, config, sqllib_dir, reverse_only=False):
""" Create the set of basic tables.
When `reverse_only` is True, then the main table for searching will
be skipped and only reverse search is possible.
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.env.globals['db']['reverse_only'] = reverse_only
sql.run_sql_file(conn, 'tables.sql')
def create_table_triggers(conn, config, sqllib_dir):
""" Create the triggers for the tables. The trigger functions must already
have been imported with refresh.create_functions().
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'table-triggers.sql')
def create_partition_tables(conn, config, sqllib_dir):
""" Create tables that have explicit partioning.
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'partition-tables.src.sql')
def truncate_data_tables(conn, max_word_frequency=None):
""" Truncate all data tables to prepare for a fresh load.
"""