hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

move table creation to jinja-based preprocessing

Browse Source
This commit is contained in:
Sarah Hoffmann
2021-03-03 22:07:51 +01:00
parent 6cda021d9b
commit eacabb0e96
9 changed files with 142 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
lib-php/admin/setup.php
View File

@@ -128,14 +128,18 @@ if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
if ($aCMDResult['create-tables'] || $aCMDResult['all']) { if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$oSetup->createTables($aCMDResult['reverse-only']); $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-tables');
$oSetup->createFunctions();
$oSetup->createTableTriggers(); if ($aCMDResult['reverse-only'] ?? false) {
$oCmd->addParams('--reverse-only');
}
run($oCmd);
} }
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$oSetup->createPartitionTables(); run((clone($oNominatimCmd))->addParams('transition', '--create-partition-tables'));
} }
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {

54
lib-php/setup/SetupClass.php
View File

@@ -77,42 +77,6 @@ class SetupFunctions
$this->createSqlFunctions(); $this->createSqlFunctions();
} }
public function createTables($bReverseOnly = false)
{
info('Create Tables');
$sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql');
$sTemplate = $this->replaceSqlPatterns($sTemplate);
$this->pgsqlRunScript($sTemplate, false);
if ($bReverseOnly) {
$this->dropTable('search_name');
}
(clone($this->oNominatimCmd))->addParams('refresh', '--address-levels')->run();
}
public function createTableTriggers()
{
info('Create Tables');
$sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql');
$sTemplate = $this->replaceSqlPatterns($sTemplate);
$this->pgsqlRunScript($sTemplate, false);
}
public function createPartitionTables()
{
info('Create Partition Tables');
$sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql');
$sTemplate = $this->replaceSqlPatterns($sTemplate);
$this->pgsqlRunPartitionScript($sTemplate);
}
public function importTigerData($sTigerPath) public function importTigerData($sTigerPath)
{ {
info('Import Tiger data'); info('Import Tiger data');
@@ -342,24 +306,6 @@ class SetupFunctions
$oCmd->run(!$this->sIgnoreErrors); $oCmd->run(!$this->sIgnoreErrors);
} }
private function pgsqlRunPartitionScript($sTemplate)
{
$sSQL = 'select distinct partition from country_name order by partition';
$aPartitions = $this->db()->getCol($sSQL);
if ($aPartitions[0] != 0) $aPartitions[] = 0;
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach ($aMatches as $aMatch) {
$sResult = '';
foreach ($aPartitions as $sPartitionName) {
$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
}
$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
}
$this->pgsqlRunScript($sTemplate);
}
private function pgsqlRunScriptFile($sFilename) private function pgsqlRunScriptFile($sFilename)
{ {
if (!file_exists($sFilename)) fail('unable to find '.$sFilename); if (!file_exists($sFilename)) fail('unable to find '.$sFilename);

36
lib-sql/partition-tables.src.sql
View File

@@ -7,24 +7,24 @@ CREATE TABLE search_name_blank (
); );
-- start {% for partition in db.partitions %}
CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large) {ts:address-data}; CREATE TABLE location_area_large_{{ partition }} () INHERITS (location_area_large) {{db.tablespace.address_data}};
CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id) {ts:address-index}; CREATE INDEX idx_location_area_large_{{ partition }}_place_id ON location_area_large_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large_-partition- USING GIST (geometry) {ts:address-index}; CREATE INDEX idx_location_area_large_{{ partition }}_geometry ON location_area_large_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
CREATE TABLE search_name_-partition- () INHERITS (search_name_blank) {ts:address-data}; CREATE TABLE search_name_{{ partition }} () INHERITS (search_name_blank) {{db.tablespace.address_data}};
CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id) {ts:address-index}; CREATE INDEX idx_search_name_{{ partition }}_place_id ON search_name_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
CREATE INDEX idx_search_name_-partition-_centroid_street ON search_name_-partition- USING GIST (centroid) {ts:address-index} where address_rank between 26 and 27; CREATE INDEX idx_search_name_{{ partition }}_centroid_street ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 26 and 27;
CREATE INDEX idx_search_name_-partition-_centroid_place ON search_name_-partition- USING GIST (centroid) {ts:address-index} where address_rank between 2 and 25; CREATE INDEX idx_search_name_{{ partition }}_centroid_place ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 2 and 25;
DROP TABLE IF EXISTS location_road_-partition-; DROP TABLE IF EXISTS location_road_{{ partition }};
CREATE TABLE location_road_-partition- ( CREATE TABLE location_road_{{ partition }} (
place_id BIGINT, place_id BIGINT,
partition SMALLINT, partition SMALLINT,
country_code VARCHAR(2), country_code VARCHAR(2),
geometry GEOMETRY(Geometry, 4326) geometry GEOMETRY(Geometry, 4326)
) {ts:address-data}; ) {{db.tablespace.address_data}};
CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry) {ts:address-index}; CREATE INDEX idx_location_road_{{ partition }}_geometry ON location_road_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id) {ts:address-index}; CREATE INDEX idx_location_road_{{ partition }}_place_id ON location_road_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
-- end {% endfor %}

84
lib-sql/tables.sql
View File

@@ -4,7 +4,7 @@ CREATE TABLE import_status (
sequence_id integer, sequence_id integer,
indexed boolean indexed boolean
); );
GRANT SELECT ON import_status TO "{www-user}" ; GRANT SELECT ON import_status TO "{{config.DATABASE_WEBUSER}}" ;
drop table if exists import_osmosis_log; drop table if exists import_osmosis_log;
CREATE TABLE import_osmosis_log ( CREATE TABLE import_osmosis_log (
@@ -30,18 +30,18 @@ CREATE TABLE new_query_log (
secret text secret text
); );
CREATE INDEX idx_new_query_log_starttime ON new_query_log USING BTREE (starttime); CREATE INDEX idx_new_query_log_starttime ON new_query_log USING BTREE (starttime);
GRANT INSERT ON new_query_log TO "{www-user}" ; GRANT INSERT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
GRANT UPDATE ON new_query_log TO "{www-user}" ; GRANT UPDATE ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON new_query_log TO "{www-user}" ; GRANT SELECT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON TABLE country_name TO "{www-user}"; GRANT SELECT ON TABLE country_name TO "{{config.DATABASE_WEBUSER}}";
DROP TABLE IF EXISTS nominatim_properties; DROP TABLE IF EXISTS nominatim_properties;
CREATE TABLE nominatim_properties ( CREATE TABLE nominatim_properties (
property TEXT, property TEXT,
value TEXT value TEXT
); );
GRANT SELECT ON TABLE nominatim_properties TO "{www-user}"; GRANT SELECT ON TABLE nominatim_properties TO "{{config.DATABASE_WEBUSER}}";
drop table IF EXISTS word; drop table IF EXISTS word;
CREATE TABLE word ( CREATE TABLE word (
@@ -53,9 +53,9 @@ CREATE TABLE word (
country_code varchar(2), country_code varchar(2),
search_name_count INTEGER, search_name_count INTEGER,
operator TEXT operator TEXT
) {ts:search-data}; ) {{db.tablespace.search_data}};
CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {ts:search-index}; CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {{db.tablespace.search_index}};
GRANT SELECT ON word TO "{www-user}" ; GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}" ;
DROP SEQUENCE IF EXISTS seq_word; DROP SEQUENCE IF EXISTS seq_word;
CREATE SEQUENCE seq_word start 1; CREATE SEQUENCE seq_word start 1;
@@ -80,8 +80,8 @@ CREATE TABLE location_area_country (
place_id BIGINT, place_id BIGINT,
country_code varchar(2), country_code varchar(2),
geometry GEOMETRY(Geometry, 4326) geometry GEOMETRY(Geometry, 4326)
) {ts:address-data}; ) {{db.tablespace.address_data}};
CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {ts:address-index}; CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {{db.tablespace.address_index}};
drop table IF EXISTS location_property CASCADE; drop table IF EXISTS location_property CASCADE;
@@ -98,7 +98,7 @@ CREATE TABLE location_property_aux () INHERITS (location_property);
CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id); CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id); CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber); CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
GRANT SELECT ON location_property_aux TO "{www-user}"; GRANT SELECT ON location_property_aux TO "{{config.DATABASE_WEBUSER}}";
CREATE TABLE location_property_tiger ( CREATE TABLE location_property_tiger (
place_id BIGINT, place_id BIGINT,
@@ -109,7 +109,7 @@ CREATE TABLE location_property_tiger (
linegeo GEOMETRY, linegeo GEOMETRY,
interpolationtype TEXT, interpolationtype TEXT,
postcode TEXT); postcode TEXT);
GRANT SELECT ON location_property_tiger TO "{www-user}"; GRANT SELECT ON location_property_tiger TO "{{config.DATABASE_WEBUSER}}";
drop table if exists location_property_osmline; drop table if exists location_property_osmline;
CREATE TABLE location_property_osmline ( CREATE TABLE location_property_osmline (
@@ -127,13 +127,14 @@ CREATE TABLE location_property_osmline (
address HSTORE, address HSTORE,
postcode TEXT, postcode TEXT,
country_code VARCHAR(2) country_code VARCHAR(2)
){ts:search-data}; ){{db.tablespace.search_data}};
CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {ts:search-index}; CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {{db.tablespace.search_index}};
CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {ts:address-index}; CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {{db.tablespace.address_index}};
CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {ts:search-index}; CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {{db.tablespace.search_index}};
GRANT SELECT ON location_property_osmline TO "{www-user}"; GRANT SELECT ON location_property_osmline TO "{{config.DATABASE_WEBUSER}}";
drop table IF EXISTS search_name; drop table IF EXISTS search_name;
{% if not db.reverse_only %}
CREATE TABLE search_name ( CREATE TABLE search_name (
place_id BIGINT, place_id BIGINT,
importance FLOAT, importance FLOAT,
@@ -143,8 +144,10 @@ CREATE TABLE search_name (
nameaddress_vector integer[], nameaddress_vector integer[],
country_code varchar(2), country_code varchar(2),
centroid GEOMETRY(Geometry, 4326) centroid GEOMETRY(Geometry, 4326)
) {ts:search-data}; ) {{db.tablespace.search_data}};
CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {ts:search-index}; CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {{db.tablespace.search_index}};
GRANT SELECT ON search_name to "{{config.DATABASE_WEBUSER}}" ;
{% endif %}
drop table IF EXISTS place_addressline; drop table IF EXISTS place_addressline;
CREATE TABLE place_addressline ( CREATE TABLE place_addressline (
@@ -154,8 +157,8 @@ CREATE TABLE place_addressline (
cached_rank_address SMALLINT, cached_rank_address SMALLINT,
fromarea boolean, fromarea boolean,
isaddress boolean isaddress boolean
) {ts:search-data}; ) {{db.tablespace.search_data}};
CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {ts:search-index}; CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {{db.tablespace.search_index}};
drop table if exists placex; drop table if exists placex;
CREATE TABLE placex ( CREATE TABLE placex (
@@ -175,24 +178,23 @@ CREATE TABLE placex (
housenumber TEXT, housenumber TEXT,
postcode TEXT, postcode TEXT,
centroid GEOMETRY(Geometry, 4326) centroid GEOMETRY(Geometry, 4326)
) {ts:search-data}; ) {{db.tablespace.search_data}};
CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {ts:search-index}; CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {{db.tablespace.search_index}};
CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {ts:search-index}; CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {{db.tablespace.search_index}};
CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {ts:address-index} WHERE linked_place_id IS NOT NULL; CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {{db.tablespace.address_index}} WHERE linked_place_id IS NOT NULL;
CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {ts:address-index}; CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}};
CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {ts:search-index}; CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {{db.tablespace.search_index}};
CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {ts:address-index} WHERE osm_type='N' and rank_search < 26; CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {{db.tablespace.address_index}} WHERE osm_type='N' and rank_search < 26;
CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {ts:address-index} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26; CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26;
DROP SEQUENCE IF EXISTS seq_place; DROP SEQUENCE IF EXISTS seq_place;
CREATE SEQUENCE seq_place start 1; CREATE SEQUENCE seq_place start 1;
GRANT SELECT on placex to "{www-user}" ; GRANT SELECT on placex to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON search_name to "{www-user}" ; GRANT SELECT on place_addressline to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT on place_addressline to "{www-user}" ; GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON seq_word to "{www-user}" ; GRANT SELECT ON planet_osm_ways to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON planet_osm_ways to "{www-user}" ; GRANT SELECT ON planet_osm_rels to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON planet_osm_rels to "{www-user}" ; GRANT SELECT on location_area to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT on location_area to "{www-user}" ;
-- Table for synthetic postcodes. -- Table for synthetic postcodes.
DROP TABLE IF EXISTS location_postcode; DROP TABLE IF EXISTS location_postcode;
@@ -207,8 +209,8 @@ CREATE TABLE location_postcode (
postcode TEXT, postcode TEXT,
geometry GEOMETRY(Geometry, 4326) geometry GEOMETRY(Geometry, 4326)
); );
CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {ts:address-index}; CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {{db.tablespace.address_index}};
GRANT SELECT ON location_postcode TO "{www-user}" ; GRANT SELECT ON location_postcode TO "{{config.DATABASE_WEBUSER}}" ;
DROP TABLE IF EXISTS import_polygon_error; DROP TABLE IF EXISTS import_polygon_error;
CREATE TABLE import_polygon_error ( CREATE TABLE import_polygon_error (
@@ -224,7 +226,7 @@ CREATE TABLE import_polygon_error (
newgeometry GEOMETRY(Geometry, 4326) newgeometry GEOMETRY(Geometry, 4326)
); );
CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id); CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);
GRANT SELECT ON import_polygon_error TO "{www-user}"; GRANT SELECT ON import_polygon_error TO "{{config.DATABASE_WEBUSER}}";
DROP TABLE IF EXISTS import_polygon_delete; DROP TABLE IF EXISTS import_polygon_delete;
CREATE TABLE import_polygon_delete ( CREATE TABLE import_polygon_delete (
@@ -234,7 +236,7 @@ CREATE TABLE import_polygon_delete (
type TEXT NOT NULL type TEXT NOT NULL
); );
CREATE INDEX idx_import_polygon_delete_osmid ON import_polygon_delete USING BTREE (osm_type, osm_id); CREATE INDEX idx_import_polygon_delete_osmid ON import_polygon_delete USING BTREE (osm_type, osm_id);
GRANT SELECT ON import_polygon_delete TO "{www-user}"; GRANT SELECT ON import_polygon_delete TO "{{config.DATABASE_WEBUSER}}";
DROP SEQUENCE IF EXISTS file; DROP SEQUENCE IF EXISTS file;
CREATE SEQUENCE file start 1; CREATE SEQUENCE file start 1;

24
nominatim/clicmd/setup.py
View File

@@ -79,20 +79,22 @@ class SetupAll:
drop=args.no_updates, drop=args.no_updates,
ignore_errors=args.ignore_errors) ignore_errors=args.ignore_errors)
LOG.warning('Create functions (1st pass)')
with connect(args.config.get_libpq_dsn()) as conn: with connect(args.config.get_libpq_dsn()) as conn:
LOG.warning('Create functions (1st pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir, refresh.create_functions(conn, args.config, args.sqllib_dir,
False, False) False, False)
LOG.warning('Create tables')
LOG.warning('Create tables') database_import.create_tables(conn, args.config, args.sqllib_dir,
params = ['setup.php', '--create-tables', '--create-partition-tables'] reverse_only=args.reverse_only)
if args.reverse_only: refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
params.append('--reverse-only') LOG.warning('Create functions (2nd pass)')
run_legacy_script(*params, nominatim_env=args, refresh.create_functions(conn, args.config, args.sqllib_dir,
throw_on_fail=not args.ignore_errors) False, False)
LOG.warning('Create table triggers')
LOG.warning('Create functions (2nd pass)') database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
with connect(args.config.get_libpq_dsn()) as conn: LOG.warning('Create partition tables')
database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
LOG.warning('Create functions (3rd pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir, refresh.create_functions(conn, args.config, args.sqllib_dir,
False, False) False, False)

21
nominatim/clicmd/transition.py
View File

@@ -35,6 +35,10 @@ class AdminTransition:
help='Import a osm file') help='Import a osm file')
group.add_argument('--load-data', action='store_true', group.add_argument('--load-data', action='store_true',
help='Copy data to live tables from import table') help='Copy data to live tables from import table')
group.add_argument('--create-tables', action='store_true',
help='Create main tables')
group.add_argument('--create-partition-tables', action='store_true',
help='Create required partition tables')
group.add_argument('--index', action='store_true', group.add_argument('--index', action='store_true',
help='Index the data') help='Index the data')
group = parser.add_argument_group('Options') group = parser.add_argument_group('Options')
@@ -50,10 +54,13 @@ class AdminTransition:
help='Do not perform analyse operations during index') help='Do not perform analyse operations during index')
group.add_argument('--ignore-errors', action='store_true', group.add_argument('--ignore-errors', action='store_true',
help="Ignore certain erros on import.") help="Ignore certain erros on import.")
group.add_argument('--reverse-only', action='store_true',
help='Do not create search tables and indexes')
@staticmethod @staticmethod
def run(args): def run(args):
from ..tools import database_import from ..tools import database_import
from ..tools import refresh
if args.create_db: if args.create_db:
LOG.warning('Create DB') LOG.warning('Create DB')
@@ -80,6 +87,20 @@ class AdminTransition:
drop=args.drop, drop=args.drop,
ignore_errors=args.ignore_errors) ignore_errors=args.ignore_errors)
if args.create_tables:
LOG.warning('Create Tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_tables(conn, args.config, args.sqllib_dir, args.reverse_only)
refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
refresh.create_functions(conn, args.config, args.sqllib_dir,
enable_diff_updates=False)
database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
if args.create_partition_tables:
LOG.warning('Create Partition Tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
if args.load_data: if args.load_data:
LOG.warning('Load data') LOG.warning('Load data')
with connect(args.config.get_libpq_dsn()) as conn: with connect(args.config.get_libpq_dsn()) as conn:

8
nominatim/db/sql_preprocessor.py
View File

@@ -45,6 +45,14 @@ class SQLPreprocessor: # pylint: disable=too-few-public-methods
db_info['tables'] = _get_tables(conn) db_info['tables'] = _get_tables(conn)
db_info['reverse_only'] = 'search_name' not in db_info['tables'] db_info['reverse_only'] = 'search_name' not in db_info['tables']
db_info['tablespace'] = {}
for subset in ('ADDRESS', 'SEARCH', 'AUX'):
for kind in ('DATA', 'INDEX'):
tspace = getattr(config, 'TABLESPACE_{}_{}'.format(subset, kind))
if tspace:
tspace = 'TABLESPACE "{}"'.format(tspace)
db_info['tablespace']['{}_{}'.format(subset.lower, kind.lower())] = tspace
self.env.globals['config'] = config self.env.globals['config'] = config
self.env.globals['db'] = db_info self.env.globals['db'] = db_info
self.env.globals['modulepath'] = config.DATABASE_MODULE_PATH or \ self.env.globals['modulepath'] = config.DATABASE_MODULE_PATH or \

27
nominatim/tools/database_import.py
View File

@@ -14,6 +14,7 @@ import psycopg2
from ..db.connection import connect, get_pg_env from ..db.connection import connect, get_pg_env
from ..db import utils as db_utils from ..db import utils as db_utils
from ..db.async_connection import DBConnection from ..db.async_connection import DBConnection
from ..db.sql_preprocessor import SQLPreprocessor
from .exec_utils import run_osm2pgsql from .exec_utils import run_osm2pgsql
from ..errors import UsageError from ..errors import UsageError
from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION
@@ -178,6 +179,32 @@ def import_osm_data(osm_file, options, drop=False, ignore_errors=False):
Path(options['flatnode_file']).unlink() Path(options['flatnode_file']).unlink()
def create_tables(conn, config, sqllib_dir, reverse_only=False):
""" Create the set of basic tables.
When `reverse_only` is True, then the main table for searching will
be skipped and only reverse search is possible.
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.env.globals['db']['reverse_only'] = reverse_only
sql.run_sql_file(conn, 'tables.sql')
def create_table_triggers(conn, config, sqllib_dir):
""" Create the triggers for the tables. The trigger functions must already
have been imported with refresh.create_functions().
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'table-triggers.sql')
def create_partition_tables(conn, config, sqllib_dir):
""" Create tables that have explicit partioning.
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'partition-tables.src.sql')
def truncate_data_tables(conn, max_word_frequency=None): def truncate_data_tables(conn, max_word_frequency=None):
""" Truncate all data tables to prepare for a fresh load. """ Truncate all data tables to prepare for a fresh load.
""" """

4
test/python/test_cli.py
View File

@@ -92,6 +92,10 @@ def test_import_full(temp_db, mock_func_factory):
mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'), mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'), mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
mock_func_factory(nominatim.tools.database_import, 'load_data'), mock_func_factory(nominatim.tools.database_import, 'load_data'),
mock_func_factory(nominatim.tools.database_import, 'create_tables'),
mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'),
mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'),
mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_file'),
mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim.tools.refresh, 'setup_website'), mock_func_factory(nominatim.tools.refresh, 'setup_website'),
mock_func_factory(nominatim.db.properties, 'set_property') mock_func_factory(nominatim.db.properties, 'set_property')