hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

simplify sql and website creation functions

Browse Source
This commit is contained in:
Sarah Hoffmann
2021-04-19 10:01:17 +02:00
parent 4fa6c0ad53
commit 79d55357e8
9 changed files with 25 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
nominatim/clicmd/refresh.py
View File

@@ -64,7 +64,7 @@ class UpdateRefresh:
if args.functions: if args.functions:
LOG.warning('Create functions') LOG.warning('Create functions')
with connect(args.config.get_libpq_dsn()) as conn: with connect(args.config.get_libpq_dsn()) as conn:
refresh.create_functions(conn, args.config, args.sqllib_dir, refresh.create_functions(conn, args.config,
args.diffs, args.enable_debug_statements) args.diffs, args.enable_debug_statements)
if args.wiki_data: if args.wiki_data:
@@ -85,6 +85,6 @@ class UpdateRefresh:
if args.website: if args.website:
webdir = args.project_dir / 'website' webdir = args.project_dir / 'website'
LOG.warning('Setting up website directory at %s', webdir) LOG.warning('Setting up website directory at %s', webdir)
refresh.setup_website(webdir, args.phplib_dir, args.config) refresh.setup_website(webdir, args.config)
return 0 return 0

3
nominatim/clicmd/replication.py
View File

@@ -55,8 +55,7 @@ class UpdateReplication:
replication.init_replication(conn, base_url=args.config.REPLICATION_URL) replication.init_replication(conn, base_url=args.config.REPLICATION_URL)
if args.update_functions: if args.update_functions:
LOG.warning("Create functions") LOG.warning("Create functions")
refresh.create_functions(conn, args.config, args.sqllib_dir, refresh.create_functions(conn, args.config, True, False)
True, False)
return 0 return 0

18
nominatim/clicmd/setup.py
View File

@@ -81,22 +81,19 @@ class SetupAll:
with connect(args.config.get_libpq_dsn()) as conn: with connect(args.config.get_libpq_dsn()) as conn:
LOG.warning('Create functions (1st pass)') LOG.warning('Create functions (1st pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir, refresh.create_functions(conn, args.config, False, False)
False, False)
LOG.warning('Create tables') LOG.warning('Create tables')
database_import.create_tables(conn, args.config, args.sqllib_dir, database_import.create_tables(conn, args.config,
reverse_only=args.reverse_only) reverse_only=args.reverse_only)
refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG)) refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
LOG.warning('Create functions (2nd pass)') LOG.warning('Create functions (2nd pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir, refresh.create_functions(conn, args.config, False, False)
False, False)
LOG.warning('Create table triggers') LOG.warning('Create table triggers')
database_import.create_table_triggers(conn, args.config, args.sqllib_dir) database_import.create_table_triggers(conn, args.config)
LOG.warning('Create partition tables') LOG.warning('Create partition tables')
database_import.create_partition_tables(conn, args.config, args.sqllib_dir) database_import.create_partition_tables(conn, args.config)
LOG.warning('Create functions (3rd pass)') LOG.warning('Create functions (3rd pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir, refresh.create_functions(conn, args.config, False, False)
False, False)
LOG.warning('Importing wikipedia importance data') LOG.warning('Importing wikipedia importance data')
data_path = Path(args.config.WIKIPEDIA_DATA_PATH or args.project_dir) data_path = Path(args.config.WIKIPEDIA_DATA_PATH or args.project_dir)
@@ -130,14 +127,13 @@ class SetupAll:
LOG.warning('Post-process tables') LOG.warning('Post-process tables')
with connect(args.config.get_libpq_dsn()) as conn: with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_search_indices(conn, args.config, database_import.create_search_indices(conn, args.config,
args.sqllib_dir,
drop=args.no_updates) drop=args.no_updates)
LOG.warning('Create search index for default country names.') LOG.warning('Create search index for default country names.')
database_import.create_country_names(conn, args.config) database_import.create_country_names(conn, args.config)
webdir = args.project_dir / 'website' webdir = args.project_dir / 'website'
LOG.warning('Setup website at %s', webdir) LOG.warning('Setup website at %s', webdir)
refresh.setup_website(webdir, args.phplib_dir, args.config) refresh.setup_website(webdir, args.config)
with connect(args.config.get_libpq_dsn()) as conn: with connect(args.config.get_libpq_dsn()) as conn:
try: try:

8
nominatim/tools/database_import.py
View File

@@ -179,7 +179,7 @@ def import_osm_data(osm_file, options, drop=False, ignore_errors=False):
Path(options['flatnode_file']).unlink() Path(options['flatnode_file']).unlink()
def create_tables(conn, config, sqllib_dir, reverse_only=False): def create_tables(conn, config, reverse_only=False):
""" Create the set of basic tables. """ Create the set of basic tables.
When `reverse_only` is True, then the main table for searching will When `reverse_only` is True, then the main table for searching will
be skipped and only reverse search is possible. be skipped and only reverse search is possible.
@@ -190,7 +190,7 @@ def create_tables(conn, config, sqllib_dir, reverse_only=False):
sql.run_sql_file(conn, 'tables.sql') sql.run_sql_file(conn, 'tables.sql')
def create_table_triggers(conn, config, sqllib_dir): def create_table_triggers(conn, config):
""" Create the triggers for the tables. The trigger functions must already """ Create the triggers for the tables. The trigger functions must already
have been imported with refresh.create_functions(). have been imported with refresh.create_functions().
""" """
@@ -198,7 +198,7 @@ def create_table_triggers(conn, config, sqllib_dir):
sql.run_sql_file(conn, 'table-triggers.sql') sql.run_sql_file(conn, 'table-triggers.sql')
def create_partition_tables(conn, config, sqllib_dir): def create_partition_tables(conn, config):
""" Create tables that have explicit partitioning. """ Create tables that have explicit partitioning.
""" """
sql = SQLPreprocessor(conn, config) sql = SQLPreprocessor(conn, config)
@@ -287,7 +287,7 @@ def load_data(dsn, data_dir, threads):
cur.execute('ANALYSE') cur.execute('ANALYSE')
def create_search_indices(conn, config, sqllib_dir, drop=False): def create_search_indices(conn, config, drop=False):
""" Create tables that have explicit partitioning. """ Create tables that have explicit partitioning.
""" """

2
nominatim/tools/migration.py
View File

@@ -47,7 +47,7 @@ def migrate(config, paths):
if has_run_migration: if has_run_migration:
LOG.warning('Updating SQL functions.') LOG.warning('Updating SQL functions.')
refresh.create_functions(conn, config, paths.sqllib_dir) refresh.create_functions(conn, config)
properties.set_property(conn, 'database_version', properties.set_property(conn, 'database_version',
'{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION)) '{0[0]}.{0[1]}.{0[2]}-{0[3]}'.format(NOMINATIM_VERSION))

9
nominatim/tools/refresh.py
View File

@@ -77,8 +77,7 @@ def load_address_levels_from_file(conn, config_file):
load_address_levels(conn, 'address_levels', json.load(fdesc)) load_address_levels(conn, 'address_levels', json.load(fdesc))
def create_functions(conn, config, sqllib_dir, def create_functions(conn, config, enable_diff_updates=True, enable_debug=False):
enable_diff_updates=True, enable_debug=False):
""" (Re)create the PL/pgSQL functions. """ (Re)create the PL/pgSQL functions.
""" """
sql = SQLPreprocessor(conn, config) sql = SQLPreprocessor(conn, config)
@@ -165,7 +164,7 @@ def recompute_importance(conn):
conn.commit() conn.commit()
def setup_website(basedir, phplib_dir, config): def setup_website(basedir, config):
""" Create the website script stubs. """ Create the website script stubs.
""" """
if not basedir.exists(): if not basedir.exists():
@@ -179,7 +178,7 @@ def setup_website(basedir, phplib_dir, config):
@define('CONST_LibDir', '{0}'); @define('CONST_LibDir', '{0}');
@define('CONST_NominatimVersion', '{1[0]}.{1[1]}.{1[2]}-{1[3]}'); @define('CONST_NominatimVersion', '{1[0]}.{1[1]}.{1[2]}-{1[3]}');
""".format(phplib_dir, NOMINATIM_VERSION)) """.format(config.lib_dir.php, NOMINATIM_VERSION))
for php_name, conf_name, var_type in PHP_CONST_DEFS: for php_name, conf_name, var_type in PHP_CONST_DEFS:
if var_type == bool: if var_type == bool:
@@ -193,7 +192,7 @@ def setup_website(basedir, phplib_dir, config):
template += "@define('CONST_{}', {});\n".format(php_name, varout) template += "@define('CONST_{}', {});\n".format(php_name, varout)
template += "\nrequire_once('{}/website/{{}}');\n".format(phplib_dir) template += "\nrequire_once('{}/website/{{}}');\n".format(config.lib_dir.php)
for script in WEBSITE_SCRIPTS: for script in WEBSITE_SCRIPTS:
(basedir / script).write_text(template.format(script), 'utf-8') (basedir / script).write_text(template.format(script), 'utf-8')

3
test/bdd/steps/nominatim_environment.py
View File

@@ -107,7 +107,8 @@ class NominatimEnvironment:
self.website_dir = tempfile.TemporaryDirectory() self.website_dir = tempfile.TemporaryDirectory()
cfg = Configuration(None, self.src_dir / 'settings', environ=self.test_env) cfg = Configuration(None, self.src_dir / 'settings', environ=self.test_env)
refresh.setup_website(Path(self.website_dir.name) / 'website', self.src_dir / 'lib-php', cfg) cfg.lib_dir.php = self.src_dir / 'lib-php'
refresh.setup_website(Path(self.website_dir.name) / 'website', cfg)
def get_libpq_dsn(self): def get_libpq_dsn(self):
dsn = self.test_env['NOMINATIM_DATABASE_DSN'] dsn = self.test_env['NOMINATIM_DATABASE_DSN']

4
test/python/test_tools_refresh_create_functions.py
View File

@@ -27,7 +27,7 @@ def test_create_functions(temp_db_cursor, conn, def_config, sql_tmp_path):
$$ LANGUAGE plpgsql IMMUTABLE; $$ LANGUAGE plpgsql IMMUTABLE;
""") """)
create_functions(conn, def_config, sql_tmp_path) create_functions(conn, def_config)
assert temp_db_cursor.scalar('SELECT test()') == 43 assert temp_db_cursor.scalar('SELECT test()') == 43
@@ -47,6 +47,6 @@ def test_create_functions_with_template(temp_db_cursor, conn, def_config, sql_tm
$$ LANGUAGE plpgsql IMMUTABLE; $$ LANGUAGE plpgsql IMMUTABLE;
""") """)
create_functions(conn, def_config, sql_tmp_path, enable_debug=dbg) create_functions(conn, def_config, enable_debug=dbg)
assert temp_db_cursor.scalar('SELECT test()') == ret assert temp_db_cursor.scalar('SELECT test()') == ret

3
test/python/test_tools_refresh_setup_website.py
View File

@@ -25,7 +25,8 @@ def test_script(envdir):
def run_website_script(envdir, config): def run_website_script(envdir, config):
refresh.setup_website(envdir, envdir / 'php', config) config.lib_dir.php = envdir / 'php'
refresh.setup_website(envdir, config)
proc = subprocess.run(['/usr/bin/env', 'php', '-Cq', proc = subprocess.run(['/usr/bin/env', 'php', '-Cq',
envdir / 'search.php'], check=False) envdir / 'search.php'], check=False)