diff --git a/.github/actions/build-nominatim/action.yml b/.github/actions/build-nominatim/action.yml index 846f31fa..414783d9 100644 --- a/.github/actions/build-nominatim/action.yml +++ b/.github/actions/build-nominatim/action.yml @@ -6,7 +6,7 @@ runs: steps: - name: Install prerequisites run: | - sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil + sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2 shell: bash - name: Download dependencies diff --git a/docs/admin/Installation.md b/docs/admin/Installation.md index 6f9538fe..eadaaff1 100644 --- a/docs/admin/Installation.md +++ b/docs/admin/Installation.md @@ -42,6 +42,7 @@ For running Nominatim: * [Psycopg2](https://www.psycopg.org) (2.7+) * [Python Dotenv](https://github.com/theskumar/python-dotenv) * [psutil](https://github.com/giampaolo/psutil) + * [Jinja2](https://palletsprojects.com/p/jinja/) * [PHP](https://php.net) (7.0 or later) * PHP-pgsql * PHP-intl (bundled with PHP) diff --git a/lib-php/admin/setup.php b/lib-php/admin/setup.php index f8e360bb..f81c0ca8 100644 --- a/lib-php/admin/setup.php +++ b/lib-php/admin/setup.php @@ -123,24 +123,28 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) { if ($aCMDResult['create-functions'] || $aCMDResult['all']) { $bDidSomething = true; - $oSetup->createFunctions(); + $oSetup->createSqlFunctions(); } if ($aCMDResult['create-tables'] || $aCMDResult['all']) { $bDidSomething = true; - $oSetup->createTables($aCMDResult['reverse-only']); - $oSetup->createFunctions(); - $oSetup->createTableTriggers(); + $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-tables'); + + if ($aCMDResult['reverse-only'] ?? false) { + $oCmd->addParams('--reverse-only'); + } + + run($oCmd); } if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { $bDidSomething = true; - $oSetup->createPartitionTables(); + run((clone($oNominatimCmd))->addParams('transition', '--create-partition-tables')); } if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { $bDidSomething = true; - $oSetup->createFunctions(); // also create partition functions + $oSetup->createSqlFunctions(); // also create partition functions } if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { @@ -182,7 +186,14 @@ if ($aCMDResult['drop']) { if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { $bDidSomething = true; - $oSetup->createSearchIndices(); + + $oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-search-indices'); + + if ($aCMDResult['drop'] ?? false) { + $oCmd->addParams('--drop'); + } + + run($oCmd); } if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { diff --git a/lib-php/setup/SetupClass.php b/lib-php/setup/SetupClass.php index 75168619..4b6439a9 100755 --- a/lib-php/setup/SetupClass.php +++ b/lib-php/setup/SetupClass.php @@ -67,52 +67,6 @@ class SetupFunctions } } - public function createFunctions() - { - info('Create Functions'); - - // Try accessing the C module, so we know early if something is wrong - $this->checkModulePresence(); // raises exception on failure - - $this->createSqlFunctions(); - } - - public function createTables($bReverseOnly = false) - { - info('Create Tables'); - - $sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql'); - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunScript($sTemplate, false); - - if ($bReverseOnly) { - $this->dropTable('search_name'); - } - - (clone($this->oNominatimCmd))->addParams('refresh', '--address-levels')->run(); - } - - public function createTableTriggers() - { - info('Create Tables'); - - $sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql'); - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunScript($sTemplate, false); - } - - public function createPartitionTables() - { - info('Create Partition Tables'); - - $sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql'); - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunPartitionScript($sTemplate); - } - public function importTigerData($sTigerPath) { info('Import Tiger data'); @@ -246,31 +200,6 @@ class SetupFunctions $this->db()->exec($sSQL); } - public function createSearchIndices() - { - info('Create Search indices'); - - $sSQL = 'SELECT relname FROM pg_class, pg_index '; - $sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid'; - $aInvalidIndices = $this->db()->getCol($sSQL); - - foreach ($aInvalidIndices as $sIndexName) { - info("Cleaning up invalid index $sIndexName"); - $this->db()->exec("DROP INDEX $sIndexName;"); - } - - $sTemplate = file_get_contents(CONST_SqlDir.'/indices.src.sql'); - if (!$this->bDrop) { - $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_updates.src.sql'); - } - if (!$this->dbReverseOnly()) { - $sTemplate .= file_get_contents(CONST_SqlDir.'/indices_search.src.sql'); - } - $sTemplate = $this->replaceSqlPatterns($sTemplate); - - $this->pgsqlRunScript($sTemplate); - } - public function createCountryNames() { info('Create search index for default country names'); @@ -326,7 +255,7 @@ class SetupFunctions ); } - private function createSqlFunctions() + public function createSqlFunctions() { $oCmd = (clone($this->oNominatimCmd)) ->addParams('refresh', '--functions'); @@ -342,24 +271,6 @@ class SetupFunctions $oCmd->run(!$this->sIgnoreErrors); } - private function pgsqlRunPartitionScript($sTemplate) - { - $sSQL = 'select distinct partition from country_name order by partition'; - $aPartitions = $this->db()->getCol($sSQL); - if ($aPartitions[0] != 0) $aPartitions[] = 0; - - preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); - foreach ($aMatches as $aMatch) { - $sResult = ''; - foreach ($aPartitions as $sPartitionName) { - $sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]); - } - $sTemplate = str_replace($aMatch[0], $sResult, $sTemplate); - } - - $this->pgsqlRunScript($sTemplate); - } - private function pgsqlRunScriptFile($sFilename) { if (!file_exists($sFilename)) fail('unable to find '.$sFilename); @@ -444,44 +355,4 @@ class SetupFunctions return $sSql; } - - /** - * Drop table with the given name if it exists. - * - * @param string $sName Name of table to remove. - * - * @return null - */ - private function dropTable($sName) - { - if ($this->bVerbose) echo "Dropping table $sName\n"; - $this->db()->deleteTable($sName); - } - - /** - * Check if the database is in reverse-only mode. - * - * @return True if there is no search_name table and infrastructure. - */ - private function dbReverseOnly() - { - return !($this->db()->tableExists('search_name')); - } - - /** - * Try accessing the C module, so we know early if something is wrong. - * - * Raises Nominatim\DatabaseError on failure - */ - private function checkModulePresence() - { - $sModulePath = getSetting('DATABASE_MODULE_PATH', CONST_InstallDir.'/module'); - $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; - $sSQL .= $sModulePath . "/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; - $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; - - $oDB = new \Nominatim\DB(); - $oDB->connect(); - $oDB->exec($sSQL, null, 'Database server failed to load '.$sModulePath.'/nominatim.so module'); - } } diff --git a/lib-sql/functions.sql b/lib-sql/functions.sql new file mode 100644 index 00000000..750af9f0 --- /dev/null +++ b/lib-sql/functions.sql @@ -0,0 +1,20 @@ +{% include('functions/utils.sql') %} +{% include('functions/normalization.sql') %} +{% include('functions/ranking.sql') %} +{% include('functions/importance.sql') %} +{% include('functions/address_lookup.sql') %} +{% include('functions/interpolation.sql') %} + +{% if 'place' in db.tables %} + {% include 'functions/place_triggers.sql' %} +{% endif %} + +{% if 'placex' in db.tables %} + {% include 'functions/placex_triggers.sql' %} +{% endif %} + +{% if 'location_postcode' in db.tables %} + {% include 'functions/postcode_triggers.sql' %} +{% endif %} + +{% include('functions/partition-functions.sql') %} diff --git a/lib-sql/functions/address_lookup.sql b/lib-sql/functions/address_lookup.sql index 4d7cc789..f49bc93e 100644 --- a/lib-sql/functions/address_lookup.sql +++ b/lib-sql/functions/address_lookup.sql @@ -121,7 +121,7 @@ BEGIN END IF; --then query tiger data - -- %NOTIGERDATA% IF 0 THEN + {% if config.get_bool('USE_US_TIGER_DATA') %} IF place IS NULL AND in_housenumber >= 0 THEN SELECT parent_place_id as place_id, 'us' as country_code, in_housenumber as housenumber, postcode, @@ -133,9 +133,10 @@ BEGIN WHERE place_id = in_place_id AND in_housenumber between startnumber and endnumber; END IF; - -- %NOTIGERDATA% END IF; + {% endif %} - -- %NOAUXDATA% IF 0 THEN + -- then additional data + {% if config.get_bool('USE_AUX_LOCATION_DATA') %} IF place IS NULL THEN SELECT parent_place_id as place_id, 'us' as country_code, housenumber, postcode, @@ -146,7 +147,7 @@ BEGIN FROM location_property_aux WHERE place_id = in_place_id; END IF; - -- %NOAUXDATA% END IF; + {% endif %} -- postcode table IF place IS NULL THEN diff --git a/lib-sql/functions/normalization.sql b/lib-sql/functions/normalization.sql index 8bb4915b..6fcdf552 100644 --- a/lib-sql/functions/normalization.sql +++ b/lib-sql/functions/normalization.sql @@ -1,12 +1,12 @@ -- Functions for term normalisation and access to the 'word' table. CREATE OR REPLACE FUNCTION transliteration(text) RETURNS text - AS '{modulepath}/nominatim.so', 'transliteration' + AS '{{ modulepath }}/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION gettokenstring(text) RETURNS text - AS '{modulepath}/nominatim.so', 'gettokenstring' + AS '{{ modulepath }}/nominatim.so', 'gettokenstring' LANGUAGE c IMMUTABLE STRICT; diff --git a/lib-sql/partition-functions.src.sql b/lib-sql/functions/partition-functions.sql similarity index 80% rename from lib-sql/partition-functions.src.sql rename to lib-sql/functions/partition-functions.sql index 703c08af..cfa151de 100644 --- a/lib-sql/partition-functions.src.sql +++ b/lib-sql/functions/partition-functions.sql @@ -37,13 +37,13 @@ DECLARE r nearfeaturecentr%rowtype; BEGIN --- start - IF in_partition = -partition- THEN +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN FOR r IN SELECT place_id, keywords, rank_address, rank_search, min(ST_Distance(feature, centroid)) as distance, isguess, postcode, centroid - FROM location_area_large_-partition- + FROM location_area_large_{{ partition }} WHERE geometry && feature AND is_relevant_geometry(ST_Relate(geometry, feature), ST_GeometryType(feature)) AND rank_address < maxrank @@ -56,7 +56,7 @@ BEGIN END LOOP; RETURN; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; END @@ -80,12 +80,12 @@ BEGIN CONTINUE; END IF; --- start - IF in_partition = -partition- THEN +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN SELECT place_id, keywords, rank_address, rank_search, min(ST_Distance(feature, centroid)) as distance, isguess, postcode, centroid INTO r - FROM location_area_large_-partition- + FROM location_area_large_{{ partition }} WHERE geometry && ST_Expand(feature, item.extent) AND rank_address between item.from_rank and item.to_rank AND word_ids_from_name(item.name) && keywords @@ -103,7 +103,7 @@ BEGIN END IF; CONTINUE; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; END LOOP; @@ -120,12 +120,12 @@ BEGIN RETURN TRUE; END IF; --- start - IF in_partition = -partition- THEN - DELETE from location_area_large_-partition- WHERE place_id = in_place_id; +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN + DELETE from location_area_large_{{ partition }} WHERE place_id = in_place_id; RETURN TRUE; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; @@ -150,13 +150,13 @@ BEGIN RETURN TRUE; END IF; --- start - IF in_partition = -partition- THEN - INSERT INTO location_area_large_-partition- (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, postcode, centroid, geometry) +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN + INSERT INTO location_area_large_{{ partition }} (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, postcode, centroid, geometry) values (in_partition, in_place_id, in_country_code, in_keywords, in_rank_search, in_rank_address, in_estimate, postcode, in_centroid, in_geometry); RETURN TRUE; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; RETURN FALSE; @@ -173,9 +173,9 @@ DECLARE parent BIGINT; BEGIN --- start - IF in_partition = -partition- THEN - SELECT place_id FROM search_name_-partition- +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN + SELECT place_id FROM search_name_{{ partition }} INTO parent WHERE name_vector && isin_token AND centroid && ST_Expand(point, 0.015) @@ -183,7 +183,7 @@ BEGIN ORDER BY ST_Distance(centroid, point) ASC limit 1; RETURN parent; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; END @@ -199,18 +199,18 @@ DECLARE parent BIGINT; BEGIN --- start - IF in_partition = -partition- THEN +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN SELECT place_id INTO parent - FROM search_name_-partition- + FROM search_name_{{ partition }} WHERE name_vector && isin_token AND centroid && ST_Expand(point, 0.04) AND address_rank between 16 and 25 ORDER BY ST_Distance(centroid, point) ASC limit 1; RETURN parent; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; END @@ -223,16 +223,16 @@ create or replace function insertSearchName( RETURNS BOOLEAN AS $$ DECLARE BEGIN --- start - IF in_partition = -partition- THEN - DELETE FROM search_name_-partition- values WHERE place_id = in_place_id; +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN + DELETE FROM search_name_{{ partition }} values WHERE place_id = in_place_id; IF in_rank_address > 0 THEN - INSERT INTO search_name_-partition- (place_id, address_rank, name_vector, centroid) + INSERT INTO search_name_{{ partition }} (place_id, address_rank, name_vector, centroid) values (in_place_id, in_rank_address, in_name_vector, in_geometry); END IF; RETURN TRUE; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; RETURN FALSE; @@ -243,12 +243,12 @@ LANGUAGE plpgsql; create or replace function deleteSearchName(in_partition INTEGER, in_place_id BIGINT) RETURNS BOOLEAN AS $$ DECLARE BEGIN --- start - IF in_partition = -partition- THEN - DELETE from search_name_-partition- WHERE place_id = in_place_id; +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN + DELETE from search_name_{{ partition }} WHERE place_id = in_place_id; RETURN TRUE; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; @@ -262,14 +262,14 @@ create or replace function insertLocationRoad( DECLARE BEGIN --- start - IF in_partition = -partition- THEN - DELETE FROM location_road_-partition- where place_id = in_place_id; - INSERT INTO location_road_-partition- (partition, place_id, country_code, geometry) +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN + DELETE FROM location_road_{{ partition }} where place_id = in_place_id; + INSERT INTO location_road_{{ partition }} (partition, place_id, country_code, geometry) values (in_partition, in_place_id, in_country_code, in_geometry); RETURN TRUE; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; RETURN FALSE; @@ -281,12 +281,12 @@ create or replace function deleteRoad(in_partition INTEGER, in_place_id BIGINT) DECLARE BEGIN --- start - IF in_partition = -partition- THEN - DELETE FROM location_road_-partition- where place_id = in_place_id; +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN + DELETE FROM location_road_{{ partition }} where place_id = in_place_id; RETURN TRUE; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; @@ -303,12 +303,12 @@ DECLARE search_diameter FLOAT; BEGIN --- start - IF in_partition = -partition- THEN +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN search_diameter := 0.00005; WHILE search_diameter < 0.1 LOOP FOR r IN - SELECT place_id FROM location_road_-partition- + SELECT place_id FROM location_road_{{ partition }} WHERE ST_DWithin(geometry, point, search_diameter) ORDER BY ST_Distance(geometry, point) ASC limit 1 LOOP @@ -318,7 +318,7 @@ BEGIN END LOOP; RETURN NULL; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; END @@ -345,12 +345,12 @@ BEGIN p2 := ST_LineInterpolatePoint(line,0.5); p3 := ST_LineInterpolatePoint(line,1); --- start - IF in_partition = -partition- THEN +{% for partition in db.partitions %} + IF in_partition = {{ partition }} THEN search_diameter := 0.0005; WHILE search_diameter < 0.01 LOOP FOR r IN - SELECT place_id FROM location_road_-partition- + SELECT place_id FROM location_road_{{ partition }} WHERE ST_DWithin(line, geometry, search_diameter) ORDER BY (ST_distance(geometry, p1)+ ST_distance(geometry, p2)+ @@ -362,7 +362,7 @@ BEGIN END LOOP; RETURN NULL; END IF; --- end +{% endfor %} RAISE EXCEPTION 'Unknown partition %', in_partition; END diff --git a/lib-sql/functions/place_triggers.sql b/lib-sql/functions/place_triggers.sql index eaba12be..53163746 100644 --- a/lib-sql/functions/place_triggers.sql +++ b/lib-sql/functions/place_triggers.sql @@ -12,8 +12,10 @@ DECLARE partition INTEGER; BEGIN - --DEBUG: RAISE WARNING '-----------------------------------------------------------------------------------'; - --DEBUG: RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry); + {% if debug %} + RAISE WARNING '-----------------------------------------------------------------------------------'; + RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry); + {% endif %} -- filter wrong tupels IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN INSERT INTO import_polygon_error (osm_type, osm_id, class, type, name, country_code, updated, errormessage, prevgeometry, newgeometry) @@ -97,8 +99,8 @@ BEGIN DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class; END IF; - --DEBUG: RAISE WARNING 'Existing: %',existing.osm_id; - --DEBUG: RAISE WARNING 'Existing PlaceX: %',existingplacex.place_id; + {% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %} + {% if debug %}RAISE WARNING 'Existing PlaceX: %',existingplacex.place_id;{% endif %} -- Log and discard IF existing.geometry is not null AND st_isvalid(existing.geometry) @@ -122,14 +124,20 @@ BEGIN (existingplacex.type != NEW.type))) THEN + {% if config.get_bool('LIMIT_REINDEXING') %} IF existingplacex.osm_type IS NOT NULL THEN -- sanity check: ignore admin_level changes on places with too many active children -- or we end up reindexing entire countries because somebody accidentally deleted admin_level - --LIMIT INDEXING: SELECT count(*) FROM (SELECT 'a' FROM placex , place_addressline where address_place_id = existingplacex.place_id and placex.place_id = place_addressline.place_id and indexed_status = 0 and place_addressline.isaddress LIMIT 100001) sub INTO i; - --LIMIT INDEXING: IF i > 100000 THEN - --LIMIT INDEXING: RETURN null; - --LIMIT INDEXING: END IF; + SELECT count(*) INTO i FROM + (SELECT 'a' FROM placex, place_addressline + WHERE address_place_id = existingplacex.place_id + and placex.place_id = place_addressline.place_id + and indexed_status = 0 and place_addressline.isaddress LIMIT 100001) sub; + IF i > 100000 THEN + RETURN null; + END IF; END IF; + {% endif %} IF existing.osm_type IS NOT NULL THEN -- pathological case caused by the triggerless copy into place during initial import @@ -144,7 +152,7 @@ BEGIN values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name, NEW.admin_level, NEW.address, NEW.extratags, NEW.geometry); - --DEBUG: RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name; + {% if debug %}RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;{% endif %} RETURN NEW; END IF; @@ -258,7 +266,7 @@ DECLARE has_rank BOOLEAN; BEGIN - --DEBUG: RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type; + {% if debug %}RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;{% endif %} -- deleting large polygons can have a massive effect on the system - require manual intervention to let them through IF st_area(OLD.geometry) > 2 and st_isvalid(OLD.geometry) THEN diff --git a/lib-sql/functions/placex_triggers.sql b/lib-sql/functions/placex_triggers.sql index 6965fe14..086ba930 100644 --- a/lib-sql/functions/placex_triggers.sql +++ b/lib-sql/functions/placex_triggers.sql @@ -20,7 +20,7 @@ DECLARE location RECORD; parent RECORD; BEGIN - --DEBUG: RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id; + {% if debug %}RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;{% endif %} -- Is this object part of an associatedStreet relation? FOR location IN @@ -58,7 +58,7 @@ BEGIN and poi_osm_id = any(x.nodes) LIMIT 1 LOOP - --DEBUG: RAISE WARNING 'Get parent from interpolation: %', parent.parent_place_id; + {% if debug %}RAISE WARNING 'Get parent from interpolation: %', parent.parent_place_id;{% endif %} RETURN parent.parent_place_id; END LOOP; @@ -71,11 +71,11 @@ BEGIN and p.geometry && bbox and w.id = p.osm_id and poi_osm_id = any(w.nodes) LOOP - --DEBUG: RAISE WARNING 'Node is part of way % ', location.osm_id; + {% if debug %}RAISE WARNING 'Node is part of way % ', location.osm_id;{% endif %} -- Way IS a road then we are on it - that must be our road IF location.rank_search < 28 THEN - --DEBUG: RAISE WARNING 'node in way that is a street %',location; + {% if debug %}RAISE WARNING 'node in way that is a street %',location;{% endif %} return location.place_id; END IF; @@ -106,7 +106,7 @@ BEGIN ELSEIF ST_Area(bbox) < 0.005 THEN -- for smaller features get the nearest road SELECT getNearestRoadPlaceId(poi_partition, bbox) INTO parent_place_id; - --DEBUG: RAISE WARNING 'Checked for nearest way (%)', parent_place_id; + {% if debug %}RAISE WARNING 'Checked for nearest way (%)', parent_place_id;{% endif %} ELSE -- for larger features simply find the area with the largest rank that -- contains the bbox, only use addressable features @@ -146,21 +146,21 @@ BEGIN IF bnd.osm_type = 'R' THEN -- see if we have any special relation members SELECT members FROM planet_osm_rels WHERE id = bnd.osm_id INTO relation_members; - --DEBUG: RAISE WARNING 'Got relation members'; + {% if debug %}RAISE WARNING 'Got relation members';{% endif %} -- Search for relation members with role 'lable'. IF relation_members IS NOT NULL THEN FOR rel_member IN SELECT get_rel_node_members(relation_members, ARRAY['label']) as member LOOP - --DEBUG: RAISE WARNING 'Found label member %', rel_member.member; + {% if debug %}RAISE WARNING 'Found label member %', rel_member.member;{% endif %} FOR linked_placex IN SELECT * from placex WHERE osm_type = 'N' and osm_id = rel_member.member and class = 'place' LOOP - --DEBUG: RAISE WARNING 'Linked label member'; + {% if debug %}RAISE WARNING 'Linked label member';{% endif %} RETURN linked_placex; END LOOP; @@ -187,7 +187,7 @@ BEGIN AND placex.rank_search < 26 -- needed to select the right index AND _st_covers(bnd.geometry, placex.geometry) LOOP - --DEBUG: RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id; + {% if debug %}RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id;{% endif %} RETURN linked_placex; END LOOP; END IF; @@ -203,14 +203,14 @@ BEGIN AND _st_covers(bnd.geometry, placex.geometry) ORDER BY make_standard_name(name->'name') = bnd_name desc LOOP - --DEBUG: RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id; + {% if debug %}RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id;{% endif %} RETURN linked_placex; END LOOP; END IF; -- Name searches can be done for ways as well as relations IF bnd_name is not null THEN - --DEBUG: RAISE WARNING 'Looking for nodes with matching names'; + {% if debug %}RAISE WARNING 'Looking for nodes with matching names';{% endif %} FOR linked_placex IN SELECT placex.* from placex WHERE make_standard_name(name->'name') = bnd_name @@ -225,7 +225,7 @@ BEGIN AND placex.rank_search < 26 -- needed to select the right index AND _st_covers(bnd.geometry, placex.geometry) LOOP - --DEBUG: RAISE WARNING 'Found matching place node %', linked_placex.osm_id; + {% if debug %}RAISE WARNING 'Found matching place node %', linked_placex.osm_id;{% endif %} RETURN linked_placex; END LOOP; END IF; @@ -285,10 +285,10 @@ BEGIN address, country) ORDER BY rank_address, distance, isguess desc LOOP - IF NOT %REVERSE-ONLY% THEN + {% if not db.reverse_only %} nameaddress_vector := array_merge(nameaddress_vector, location.keywords::int[]); - END IF; + {% endif %} IF location.place_id is not null THEN location_isaddress := not address_havelevel[location.rank_address]; @@ -362,10 +362,10 @@ BEGIN END IF; -- Add it to the list of search terms - IF NOT %REVERSE-ONLY% THEN + {% if not db.reverse_only %} nameaddress_vector := array_merge(nameaddress_vector, location.keywords::integer[]); - END IF; + {% endif %} INSERT INTO place_addressline (place_id, address_place_id, fromarea, isaddress, distance, cached_rank_address) @@ -388,7 +388,7 @@ DECLARE diameter FLOAT; classtable TEXT; BEGIN - --DEBUG: RAISE WARNING '% % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type; + {% if debug %}RAISE WARNING '% % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;{% endif %} NEW.place_id := nextval('seq_place'); NEW.indexed_status := 1; --STATUS_NEW @@ -440,9 +440,10 @@ BEGIN END IF; - --DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type; + {% if debug %}RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;{% endif %} - RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down +{% if not disable_diff_updates %} + -- The following is not needed until doing diff updates, and slows the main index process down IF NEW.osm_type = 'N' and NEW.rank_search > 28 THEN -- might be part of an interpolation @@ -497,6 +498,8 @@ BEGIN USING NEW.place_id, ST_Centroid(NEW.geometry); END IF; +{% endif %} -- not disable_diff_updates + RETURN NEW; END; @@ -534,7 +537,7 @@ DECLARE BEGIN -- deferred delete IF OLD.indexed_status = 100 THEN - --DEBUG: RAISE WARNING 'placex_update delete % %',NEW.osm_type,NEW.osm_id; + {% if debug %}RAISE WARNING 'placex_update delete % %',NEW.osm_type,NEW.osm_id;{% endif %} delete from placex where place_id = OLD.place_id; RETURN NULL; END IF; @@ -543,13 +546,13 @@ BEGIN RETURN NEW; END IF; - --DEBUG: RAISE WARNING 'placex_update % % (%)',NEW.osm_type,NEW.osm_id,NEW.place_id; + {% if debug %}RAISE WARNING 'placex_update % % (%)',NEW.osm_type,NEW.osm_id,NEW.place_id;{% endif %} NEW.indexed_date = now(); - IF NOT %REVERSE-ONLY% THEN + {% if 'search_name' in db.tables %} DELETE from search_name WHERE place_id = NEW.place_id; - END IF; + {% endif %} result := deleteSearchName(NEW.partition, NEW.place_id); DELETE FROM place_addressline WHERE place_id = NEW.place_id; result := deleteRoad(NEW.partition, NEW.place_id); @@ -562,7 +565,7 @@ BEGIN NEW.address := NEW.address - '_unlisted_place'::TEXT; IF NEW.linked_place_id is not null THEN - --DEBUG: RAISE WARNING 'place already linked to %', NEW.linked_place_id; + {% if debug %}RAISE WARNING 'place already linked to %', NEW.linked_place_id;{% endif %} RETURN NEW; END IF; @@ -578,7 +581,7 @@ BEGIN -- Speed up searches - just use the centroid of the feature -- cheaper but less acurate NEW.centroid := ST_PointOnSurface(NEW.geometry); - --DEBUG: RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(NEW.centroid); + {% if debug %}RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(NEW.centroid);{% endif %} -- recompute the ranks, they might change when linking changes SELECT * INTO NEW.rank_search, NEW.rank_address @@ -658,7 +661,7 @@ BEGIN parent_address_level := 3; END IF; - --DEBUG: RAISE WARNING 'Copy over address tags'; + {% if debug %}RAISE WARNING 'Copy over address tags';{% endif %} -- housenumber is a computed field, so start with an empty value NEW.housenumber := NULL; IF NEW.address is not NULL THEN @@ -707,7 +710,7 @@ BEGIN END IF; NEW.partition := get_partition(NEW.country_code); END IF; - --DEBUG: RAISE WARNING 'Country updated: "%"', NEW.country_code; + {% if debug %}RAISE WARNING 'Country updated: "%"', NEW.country_code;{% endif %} -- waterway ways are linked when they are part of a relation and have the same class/type IF NEW.osm_type = 'R' and NEW.class = 'waterway' THEN @@ -715,21 +718,21 @@ BEGIN LOOP FOR i IN 1..array_upper(relation_members, 1) BY 2 LOOP IF relation_members[i+1] in ('', 'main_stream', 'side_stream') AND substring(relation_members[i],1,1) = 'w' THEN - --DEBUG: RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation_members[i]; + {% if debug %}RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation_members[i];{% endif %} FOR linked_node_id IN SELECT place_id FROM placex WHERE osm_type = 'W' and osm_id = substring(relation_members[i],2,200)::bigint and class = NEW.class and type in ('river', 'stream', 'canal', 'drain', 'ditch') and ( relation_members[i+1] != 'side_stream' or NEW.name->'name' = name->'name') LOOP UPDATE placex SET linked_place_id = NEW.place_id WHERE place_id = linked_node_id; - IF NOT %REVERSE-ONLY% THEN + {% if 'search_name' in db.tables %} DELETE FROM search_name WHERE place_id = linked_node_id; - END IF; + {% endif %} END LOOP; END IF; END LOOP; END LOOP; - --DEBUG: RAISE WARNING 'Waterway processed'; + {% if debug %}RAISE WARNING 'Waterway processed';{% endif %} END IF; NEW.importance := null; @@ -737,13 +740,13 @@ BEGIN FROM compute_importance(NEW.extratags, NEW.country_code, NEW.osm_type, NEW.osm_id) INTO NEW.wikipedia,NEW.importance; ---DEBUG: RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance; +{% if debug %}RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance;{% endif %} -- --------------------------------------------------------------------------- -- For low level elements we inherit from our parent road IF NEW.rank_search > 27 THEN - --DEBUG: RAISE WARNING 'finding street for % %', NEW.osm_type, NEW.osm_id; + {% if debug %}RAISE WARNING 'finding street for % %', NEW.osm_type, NEW.osm_id;{% endif %} NEW.parent_place_id := null; -- if we have a POI and there is no address information, @@ -791,7 +794,7 @@ BEGIN END IF; NEW.country_code := location.country_code; - --DEBUG: RAISE WARNING 'Got parent details from search name'; + {% if debug %}RAISE WARNING 'Got parent details from search name';{% endif %} -- determine postcode IF NEW.address is not null AND NEW.address ? 'postcode' THEN @@ -812,13 +815,14 @@ BEGIN name_vector, NEW.rank_search, NEW.rank_address, upper(trim(NEW.address->'postcode')), NEW.geometry, NEW.centroid); - --DEBUG: RAISE WARNING 'Place added to location table'; + {% if debug %}RAISE WARNING 'Place added to location table';{% endif %} END IF; END IF; - IF not %REVERSE-ONLY% AND (array_length(name_vector, 1) is not NULL - OR inherited_address is not NULL OR NEW.address is not NULL) + {% if not db.reverse_only %} + IF array_length(name_vector, 1) is not NULL + OR inherited_address is not NULL OR NEW.address is not NULL THEN SELECT * INTO name_vector, nameaddress_vector FROM create_poi_search_terms(NEW.place_id, @@ -834,9 +838,10 @@ BEGIN VALUES (NEW.place_id, NEW.rank_search, NEW.rank_address, NEW.importance, NEW.country_code, name_vector, nameaddress_vector, NEW.centroid); - --DEBUG: RAISE WARNING 'Place added to search table'; + {% if debug %}RAISE WARNING 'Place added to search table';{% endif %} END IF; END IF; + {% endif %} RETURN NEW; END IF; @@ -845,10 +850,10 @@ BEGIN -- --------------------------------------------------------------------------- -- Full indexing - --DEBUG: RAISE WARNING 'Using full index mode for % %', NEW.osm_type, NEW.osm_id; + {% if debug %}RAISE WARNING 'Using full index mode for % %', NEW.osm_type, NEW.osm_id;{% endif %} SELECT * INTO location FROM find_linked_place(NEW); IF location.place_id is not null THEN - --DEBUG: RAISE WARNING 'Linked %', location; + {% if debug %}RAISE WARNING 'Linked %', location;{% endif %} -- Use the linked point as the centre point of the geometry, -- but only if it is within the area of the boundary. @@ -857,7 +862,7 @@ BEGIN NEW.centroid := geom; END IF; - --DEBUG: RAISE WARNING 'parent address: % rank address: %', parent_address_level, location.rank_address; + {% if debug %}RAISE WARNING 'parent address: % rank address: %', parent_address_level, location.rank_address;{% endif %} IF location.rank_address > parent_address_level and location.rank_address < 26 THEN @@ -878,9 +883,9 @@ BEGIN UPDATE placex set linked_place_id = NEW.place_id WHERE place_id = location.place_id; -- ensure that those places are not found anymore - IF NOT %REVERSE-ONLY% THEN + {% if 'search_name' in db.tables %} DELETE FROM search_name WHERE place_id = location.place_id; - END IF; + {% endif %} PERFORM deleteLocationArea(NEW.partition, location.place_id, NEW.rank_search); SELECT wikipedia, importance @@ -918,7 +923,7 @@ BEGIN AND NEW.country_code IS NOT NULL AND NEW.osm_type = 'R' THEN PERFORM create_country(NEW.name, lower(NEW.country_code)); - --DEBUG: RAISE WARNING 'Country names updated'; + {% if debug %}RAISE WARNING 'Country names updated';{% endif %} -- Also update the list of country names. Adding an additional sanity -- check here: make sure the country does overlap with the area where @@ -928,7 +933,7 @@ BEGIN WHERE ST_Covers(geometry, NEW.centroid) and country_code = NEW.country_code LIMIT 1 LOOP - --DEBUG: RAISE WARNING 'Updating names for country '%' with: %', NEW.country_code, NEW.name; + {% if debug %}RAISE WARNING 'Updating names for country '%' with: %', NEW.country_code, NEW.name;{% endif %} UPDATE country_name SET name = name || NEW.name WHERE country_code = NEW.country_code; END LOOP; END IF; @@ -960,7 +965,7 @@ BEGIN NEW.address, geom, NEW.country_code) INTO NEW.parent_place_id, NEW.postcode, nameaddress_vector; - --DEBUG: RAISE WARNING 'RETURN insert_addresslines: %, %, %', NEW.parent_place_id, NEW.postcode, nameaddress_vector; + {% if debug %}RAISE WARNING 'RETURN insert_addresslines: %, %, %', NEW.parent_place_id, NEW.postcode, nameaddress_vector;{% endif %} IF NEW.address is not null AND NEW.address ? 'postcode' AND NEW.address->'postcode' not similar to '%(,|;)%' THEN @@ -976,30 +981,30 @@ BEGIN IF NEW.rank_search <= 25 and NEW.rank_address > 0 THEN result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, upper(trim(NEW.address->'postcode')), NEW.geometry, NEW.centroid); - --DEBUG: RAISE WARNING 'added to location (full)'; + {% if debug %}RAISE WARNING 'added to location (full)';{% endif %} END IF; IF NEW.rank_search between 26 and 27 and NEW.class = 'highway' THEN result := insertLocationRoad(NEW.partition, NEW.place_id, NEW.country_code, NEW.geometry); - --DEBUG: RAISE WARNING 'insert into road location table (full)'; + {% if debug %}RAISE WARNING 'insert into road location table (full)';{% endif %} END IF; result := insertSearchName(NEW.partition, NEW.place_id, name_vector, NEW.rank_search, NEW.rank_address, NEW.geometry); - --DEBUG: RAISE WARNING 'added to search name (full)'; + {% if debug %}RAISE WARNING 'added to search name (full)';{% endif %} - IF NOT %REVERSE-ONLY% THEN + {% if not db.reverse_only %} INSERT INTO search_name (place_id, search_rank, address_rank, importance, country_code, name_vector, nameaddress_vector, centroid) VALUES (NEW.place_id, NEW.rank_search, NEW.rank_address, NEW.importance, NEW.country_code, name_vector, nameaddress_vector, NEW.centroid); - END IF; + {% endif %} END IF; - --DEBUG: RAISE WARNING 'place update % % finsihed.', NEW.osm_type, NEW.osm_id; + {% if debug %}RAISE WARNING 'place update % % finsihed.', NEW.osm_type, NEW.osm_id;{% endif %} RETURN NEW; END; @@ -1018,9 +1023,9 @@ BEGIN IF OLD.linked_place_id is null THEN update placex set linked_place_id = null, indexed_status = 2 where linked_place_id = OLD.place_id and indexed_status = 0; - --DEBUG: RAISE WARNING 'placex_delete:01 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:01 % %',OLD.osm_type,OLD.osm_id;{% endif %} update placex set linked_place_id = null where linked_place_id = OLD.place_id; - --DEBUG: RAISE WARNING 'placex_delete:02 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:02 % %',OLD.osm_type,OLD.osm_id;{% endif %} ELSE update placex set indexed_status = 2 where place_id = OLD.linked_place_id and indexed_status = 0; END IF; @@ -1028,44 +1033,44 @@ BEGIN IF OLD.rank_address < 30 THEN -- mark everything linked to this place for re-indexing - --DEBUG: RAISE WARNING 'placex_delete:03 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:03 % %',OLD.osm_type,OLD.osm_id;{% endif %} UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = OLD.place_id and placex.place_id = place_addressline.place_id and indexed_status = 0 and place_addressline.isaddress; - --DEBUG: RAISE WARNING 'placex_delete:04 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:04 % %',OLD.osm_type,OLD.osm_id;{% endif %} DELETE FROM place_addressline where address_place_id = OLD.place_id; - --DEBUG: RAISE WARNING 'placex_delete:05 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:05 % %',OLD.osm_type,OLD.osm_id;{% endif %} b := deleteRoad(OLD.partition, OLD.place_id); - --DEBUG: RAISE WARNING 'placex_delete:06 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:06 % %',OLD.osm_type,OLD.osm_id;{% endif %} update placex set indexed_status = 2 where parent_place_id = OLD.place_id and indexed_status = 0; - --DEBUG: RAISE WARNING 'placex_delete:07 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:07 % %',OLD.osm_type,OLD.osm_id;{% endif %} -- reparenting also for OSM Interpolation Lines (and for Tiger?) update location_property_osmline set indexed_status = 2 where indexed_status = 0 and parent_place_id = OLD.place_id; END IF; - --DEBUG: RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id;{% endif %} IF OLD.rank_address < 26 THEN b := deleteLocationArea(OLD.partition, OLD.place_id, OLD.rank_search); END IF; - --DEBUG: RAISE WARNING 'placex_delete:09 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:09 % %',OLD.osm_type,OLD.osm_id;{% endif %} IF OLD.name is not null THEN - IF NOT %REVERSE-ONLY% THEN + {% if 'search_name' in db.tables %} DELETE from search_name WHERE place_id = OLD.place_id; - END IF; + {% endif %} b := deleteSearchName(OLD.partition, OLD.place_id); END IF; - --DEBUG: RAISE WARNING 'placex_delete:10 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:10 % %',OLD.osm_type,OLD.osm_id;{% endif %} DELETE FROM place_addressline where place_id = OLD.place_id; - --DEBUG: RAISE WARNING 'placex_delete:11 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:11 % %',OLD.osm_type,OLD.osm_id;{% endif %} -- remove from tables for special search classtable := 'place_classtype_' || OLD.class || '_' || OLD.type; @@ -1074,7 +1079,7 @@ BEGIN EXECUTE 'DELETE FROM ' || classtable::regclass || ' WHERE place_id = $1' USING OLD.place_id; END IF; - --DEBUG: RAISE WARNING 'placex_delete:12 % %',OLD.osm_type,OLD.osm_id; + {% if debug %}RAISE WARNING 'placex_delete:12 % %',OLD.osm_type,OLD.osm_id;{% endif %} RETURN OLD; diff --git a/lib-sql/functions/utils.sql b/lib-sql/functions/utils.sql index 18d4211b..4868b828 100644 --- a/lib-sql/functions/utils.sql +++ b/lib-sql/functions/utils.sql @@ -237,7 +237,7 @@ BEGIN IF word_ids is not null THEN parent_place_id := getNearestNamedRoadPlaceId(partition, centroid, word_ids); IF parent_place_id is not null THEN - --DEBUG: RAISE WARNING 'Get parent form addr:street: %', parent_place_id; + {% if debug %}RAISE WARNING 'Get parent form addr:street: %', parent_place_id;{% endif %} RETURN parent_place_id; END IF; END IF; @@ -249,7 +249,7 @@ BEGIN IF word_ids is not null THEN parent_place_id := getNearestNamedPlacePlaceId(partition, centroid, word_ids); IF parent_place_id is not null THEN - --DEBUG: RAISE WARNING 'Get parent form addr:place: %', parent_place_id; + {% if debug %}RAISE WARNING 'Get parent form addr:place: %', parent_place_id;{% endif %} RETURN parent_place_id; END IF; END IF; diff --git a/lib-sql/indices.sql b/lib-sql/indices.sql new file mode 100644 index 00000000..cb77e02b --- /dev/null +++ b/lib-sql/indices.sql @@ -0,0 +1,67 @@ +-- Indices used only during search and update. +-- These indices are created only after the indexing process is done. + +CREATE INDEX {{sql.if_index_not_exists}} idx_word_word_id + ON word USING BTREE (word_id) {{db.tablespace.search_index}}; + +CREATE INDEX {{sql.if_index_not_exists}} idx_place_addressline_address_place_id + ON place_addressline USING BTREE (address_place_id) {{db.tablespace.search_index}}; + +CREATE INDEX {{sql.if_index_not_exists}} idx_placex_rank_search + ON placex USING BTREE (rank_search) {{db.tablespace.search_index}}; + +CREATE INDEX {{sql.if_index_not_exists}} idx_placex_rank_address + ON placex USING BTREE (rank_address) {{db.tablespace.search_index}}; + +CREATE INDEX {{sql.if_index_not_exists}} idx_placex_parent_place_id + ON placex USING BTREE (parent_place_id) {{db.tablespace.search_index}} + WHERE parent_place_id IS NOT NULL; + +CREATE INDEX {{sql.if_index_not_exists}} idx_placex_geometry_reverse_lookupPolygon + ON placex USING gist (geometry) {{db.tablespace.search_index}} + WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon') + AND rank_address between 4 and 25 AND type != 'postcode' + AND name is not null AND indexed_status = 0 AND linked_place_id is null; + +CREATE INDEX {{sql.if_index_not_exists}} idx_placex_geometry_reverse_placeNode + ON placex USING gist (geometry) {{db.tablespace.search_index}} + WHERE osm_type = 'N' AND rank_search between 5 and 25 + AND class = 'place' AND type != 'postcode' + AND name is not null AND indexed_status = 0 AND linked_place_id is null; + +CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_place_id + ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}}; + +CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_osm_id + ON location_property_osmline USING BTREE (osm_id) {{db.tablespace.search_index}}; + +CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_postcode_id + ON location_postcode USING BTREE (place_id) {{db.tablespace.search_index}}; + +CREATE INDEX {{sql.if_index_not_exists}} idx_postcode_postcode + ON location_postcode USING BTREE (postcode) {{db.tablespace.search_index}}; + +-- Indices only needed for updating. + +{% if not drop %} + CREATE INDEX {{sql.if_index_not_exists}} idx_placex_pendingsector + ON placex USING BTREE (rank_address,geometry_sector) {{db.tablespace.address_index}} + WHERE indexed_status > 0; + + CREATE INDEX {{sql.if_index_not_exists}} idx_location_area_country_place_id + ON location_area_country USING BTREE (place_id) {{db.tablespace.address_index}}; + + CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_place_osm_unique + ON place USING btree(osm_id, osm_type, class, type) {{db.tablespace.address_index}}; +{% endif %} + +-- Indices only needed for search. + +{% if 'search_name' in db.tables %} + CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_nameaddress_vector + ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {{db.tablespace.search_index}}; + CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_name_vector + ON search_name USING GIN (name_vector) WITH (fastupdate = off) {{db.tablespace.search_index}}; + CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_centroid + ON search_name USING GIST (centroid) {{db.tablespace.search_index}}; +{% endif %} diff --git a/lib-sql/indices.src.sql b/lib-sql/indices.src.sql deleted file mode 100644 index 7fcd965f..00000000 --- a/lib-sql/indices.src.sql +++ /dev/null @@ -1,30 +0,0 @@ --- Indices used only during search and update. --- These indices are created only after the indexing process is done. - -CREATE INDEX idx_word_word_id on word USING BTREE (word_id) {ts:search-index}; - -CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id) {ts:search-index}; - -DROP INDEX IF EXISTS idx_placex_rank_search; -CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:search-index}; -CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index}; -CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL; - -CREATE INDEX idx_placex_geometry_reverse_lookupPolygon - ON placex USING gist (geometry) {ts:search-index} - WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon') - AND rank_address between 4 and 25 AND type != 'postcode' - AND name is not null AND indexed_status = 0 AND linked_place_id is null; -CREATE INDEX idx_placex_geometry_reverse_placeNode - ON placex USING gist (geometry) {ts:search-index} - WHERE osm_type = 'N' AND rank_search between 5 and 25 - AND class = 'place' AND type != 'postcode' - AND name is not null AND indexed_status = 0 AND linked_place_id is null; - -GRANT SELECT ON table country_osm_grid to "{www-user}"; - -CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index}; -CREATE INDEX idx_osmline_parent_osm_id ON location_property_osmline USING BTREE (osm_id) {ts:search-index}; - -CREATE UNIQUE INDEX idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index}; -CREATE INDEX idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index}; diff --git a/lib-sql/indices_search.src.sql b/lib-sql/indices_search.src.sql deleted file mode 100644 index d1363fc6..00000000 --- a/lib-sql/indices_search.src.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Indices used for /search API. --- These indices are created only after the indexing process is done. - -CREATE INDEX idx_search_name_nameaddress_vector ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {ts:search-index}; -CREATE INDEX idx_search_name_name_vector ON search_name USING GIN (name_vector) WITH (fastupdate = off) {ts:search-index}; -CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid) {ts:search-index}; diff --git a/lib-sql/indices_updates.src.sql b/lib-sql/indices_updates.src.sql deleted file mode 100644 index 6d4c968e..00000000 --- a/lib-sql/indices_updates.src.sql +++ /dev/null @@ -1,9 +0,0 @@ --- Indices used only during search and update. --- These indices are created only after the indexing process is done. - -CREATE INDEX CONCURRENTLY idx_placex_pendingsector ON placex USING BTREE (rank_address,geometry_sector) {ts:address-index} where indexed_status > 0; - -CREATE INDEX CONCURRENTLY idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index}; - -DROP INDEX CONCURRENTLY IF EXISTS place_id_idx; -CREATE UNIQUE INDEX CONCURRENTLY idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index}; diff --git a/lib-sql/partition-tables.src.sql b/lib-sql/partition-tables.src.sql index df2cac70..98ab9874 100644 --- a/lib-sql/partition-tables.src.sql +++ b/lib-sql/partition-tables.src.sql @@ -7,24 +7,24 @@ CREATE TABLE search_name_blank ( ); --- start -CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large) {ts:address-data}; -CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id) {ts:address-index}; -CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large_-partition- USING GIST (geometry) {ts:address-index}; +{% for partition in db.partitions %} + CREATE TABLE location_area_large_{{ partition }} () INHERITS (location_area_large) {{db.tablespace.address_data}}; + CREATE INDEX idx_location_area_large_{{ partition }}_place_id ON location_area_large_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}}; + CREATE INDEX idx_location_area_large_{{ partition }}_geometry ON location_area_large_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}}; -CREATE TABLE search_name_-partition- () INHERITS (search_name_blank) {ts:address-data}; -CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id) {ts:address-index}; -CREATE INDEX idx_search_name_-partition-_centroid_street ON search_name_-partition- USING GIST (centroid) {ts:address-index} where address_rank between 26 and 27; -CREATE INDEX idx_search_name_-partition-_centroid_place ON search_name_-partition- USING GIST (centroid) {ts:address-index} where address_rank between 2 and 25; + CREATE TABLE search_name_{{ partition }} () INHERITS (search_name_blank) {{db.tablespace.address_data}}; + CREATE INDEX idx_search_name_{{ partition }}_place_id ON search_name_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}}; + CREATE INDEX idx_search_name_{{ partition }}_centroid_street ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 26 and 27; + CREATE INDEX idx_search_name_{{ partition }}_centroid_place ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 2 and 25; -DROP TABLE IF EXISTS location_road_-partition-; -CREATE TABLE location_road_-partition- ( - place_id BIGINT, - partition SMALLINT, - country_code VARCHAR(2), - geometry GEOMETRY(Geometry, 4326) - ) {ts:address-data}; -CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry) {ts:address-index}; -CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id) {ts:address-index}; + DROP TABLE IF EXISTS location_road_{{ partition }}; + CREATE TABLE location_road_{{ partition }} ( + place_id BIGINT, + partition SMALLINT, + country_code VARCHAR(2), + geometry GEOMETRY(Geometry, 4326) + ) {{db.tablespace.address_data}}; + CREATE INDEX idx_location_road_{{ partition }}_geometry ON location_road_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}}; + CREATE INDEX idx_location_road_{{ partition }}_place_id ON location_road_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}}; --- end +{% endfor %} diff --git a/lib-sql/tables.sql b/lib-sql/tables.sql index ce11c410..0895c6dd 100644 --- a/lib-sql/tables.sql +++ b/lib-sql/tables.sql @@ -4,7 +4,7 @@ CREATE TABLE import_status ( sequence_id integer, indexed boolean ); -GRANT SELECT ON import_status TO "{www-user}" ; +GRANT SELECT ON import_status TO "{{config.DATABASE_WEBUSER}}" ; drop table if exists import_osmosis_log; CREATE TABLE import_osmosis_log ( @@ -30,18 +30,18 @@ CREATE TABLE new_query_log ( secret text ); CREATE INDEX idx_new_query_log_starttime ON new_query_log USING BTREE (starttime); -GRANT INSERT ON new_query_log TO "{www-user}" ; -GRANT UPDATE ON new_query_log TO "{www-user}" ; -GRANT SELECT ON new_query_log TO "{www-user}" ; +GRANT INSERT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ; +GRANT UPDATE ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ; +GRANT SELECT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ; -GRANT SELECT ON TABLE country_name TO "{www-user}"; +GRANT SELECT ON TABLE country_name TO "{{config.DATABASE_WEBUSER}}"; DROP TABLE IF EXISTS nominatim_properties; CREATE TABLE nominatim_properties ( property TEXT, value TEXT ); -GRANT SELECT ON TABLE nominatim_properties TO "{www-user}"; +GRANT SELECT ON TABLE nominatim_properties TO "{{config.DATABASE_WEBUSER}}"; drop table IF EXISTS word; CREATE TABLE word ( @@ -53,9 +53,9 @@ CREATE TABLE word ( country_code varchar(2), search_name_count INTEGER, operator TEXT - ) {ts:search-data}; -CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {ts:search-index}; -GRANT SELECT ON word TO "{www-user}" ; + ) {{db.tablespace.search_data}}; +CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {{db.tablespace.search_index}}; +GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}" ; DROP SEQUENCE IF EXISTS seq_word; CREATE SEQUENCE seq_word start 1; @@ -80,8 +80,8 @@ CREATE TABLE location_area_country ( place_id BIGINT, country_code varchar(2), geometry GEOMETRY(Geometry, 4326) - ) {ts:address-data}; -CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {ts:address-index}; + ) {{db.tablespace.address_data}}; +CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {{db.tablespace.address_index}}; drop table IF EXISTS location_property CASCADE; @@ -98,7 +98,7 @@ CREATE TABLE location_property_aux () INHERITS (location_property); CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id); CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id); CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber); -GRANT SELECT ON location_property_aux TO "{www-user}"; +GRANT SELECT ON location_property_aux TO "{{config.DATABASE_WEBUSER}}"; CREATE TABLE location_property_tiger ( place_id BIGINT, @@ -109,7 +109,7 @@ CREATE TABLE location_property_tiger ( linegeo GEOMETRY, interpolationtype TEXT, postcode TEXT); -GRANT SELECT ON location_property_tiger TO "{www-user}"; +GRANT SELECT ON location_property_tiger TO "{{config.DATABASE_WEBUSER}}"; drop table if exists location_property_osmline; CREATE TABLE location_property_osmline ( @@ -127,13 +127,14 @@ CREATE TABLE location_property_osmline ( address HSTORE, postcode TEXT, country_code VARCHAR(2) - ){ts:search-data}; -CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {ts:search-index}; -CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {ts:address-index}; -CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {ts:search-index}; -GRANT SELECT ON location_property_osmline TO "{www-user}"; + ){{db.tablespace.search_data}}; +CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {{db.tablespace.search_index}}; +CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {{db.tablespace.address_index}}; +CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {{db.tablespace.search_index}}; +GRANT SELECT ON location_property_osmline TO "{{config.DATABASE_WEBUSER}}"; drop table IF EXISTS search_name; +{% if not db.reverse_only %} CREATE TABLE search_name ( place_id BIGINT, importance FLOAT, @@ -143,8 +144,10 @@ CREATE TABLE search_name ( nameaddress_vector integer[], country_code varchar(2), centroid GEOMETRY(Geometry, 4326) - ) {ts:search-data}; -CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {ts:search-index}; + ) {{db.tablespace.search_data}}; +CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {{db.tablespace.search_index}}; +GRANT SELECT ON search_name to "{{config.DATABASE_WEBUSER}}" ; +{% endif %} drop table IF EXISTS place_addressline; CREATE TABLE place_addressline ( @@ -154,8 +157,8 @@ CREATE TABLE place_addressline ( cached_rank_address SMALLINT, fromarea boolean, isaddress boolean - ) {ts:search-data}; -CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {ts:search-index}; + ) {{db.tablespace.search_data}}; +CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {{db.tablespace.search_index}}; drop table if exists placex; CREATE TABLE placex ( @@ -175,24 +178,23 @@ CREATE TABLE placex ( housenumber TEXT, postcode TEXT, centroid GEOMETRY(Geometry, 4326) - ) {ts:search-data}; -CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {ts:search-index}; -CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {ts:search-index}; -CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {ts:address-index} WHERE linked_place_id IS NOT NULL; -CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {ts:address-index}; -CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {ts:search-index}; -CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {ts:address-index} WHERE osm_type='N' and rank_search < 26; -CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {ts:address-index} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26; + ) {{db.tablespace.search_data}}; +CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {{db.tablespace.search_index}}; +CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {{db.tablespace.search_index}}; +CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {{db.tablespace.address_index}} WHERE linked_place_id IS NOT NULL; +CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}}; +CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {{db.tablespace.search_index}}; +CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {{db.tablespace.address_index}} WHERE osm_type='N' and rank_search < 26; +CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26; DROP SEQUENCE IF EXISTS seq_place; CREATE SEQUENCE seq_place start 1; -GRANT SELECT on placex to "{www-user}" ; -GRANT SELECT ON search_name to "{www-user}" ; -GRANT SELECT on place_addressline to "{www-user}" ; -GRANT SELECT ON seq_word to "{www-user}" ; -GRANT SELECT ON planet_osm_ways to "{www-user}" ; -GRANT SELECT ON planet_osm_rels to "{www-user}" ; -GRANT SELECT on location_area to "{www-user}" ; +GRANT SELECT on placex to "{{config.DATABASE_WEBUSER}}" ; +GRANT SELECT on place_addressline to "{{config.DATABASE_WEBUSER}}" ; +GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}" ; +GRANT SELECT ON planet_osm_ways to "{{config.DATABASE_WEBUSER}}" ; +GRANT SELECT ON planet_osm_rels to "{{config.DATABASE_WEBUSER}}" ; +GRANT SELECT on location_area to "{{config.DATABASE_WEBUSER}}" ; -- Table for synthetic postcodes. DROP TABLE IF EXISTS location_postcode; @@ -207,8 +209,8 @@ CREATE TABLE location_postcode ( postcode TEXT, geometry GEOMETRY(Geometry, 4326) ); -CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {ts:address-index}; -GRANT SELECT ON location_postcode TO "{www-user}" ; +CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {{db.tablespace.address_index}}; +GRANT SELECT ON location_postcode TO "{{config.DATABASE_WEBUSER}}" ; DROP TABLE IF EXISTS import_polygon_error; CREATE TABLE import_polygon_error ( @@ -224,7 +226,7 @@ CREATE TABLE import_polygon_error ( newgeometry GEOMETRY(Geometry, 4326) ); CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id); -GRANT SELECT ON import_polygon_error TO "{www-user}"; +GRANT SELECT ON import_polygon_error TO "{{config.DATABASE_WEBUSER}}"; DROP TABLE IF EXISTS import_polygon_delete; CREATE TABLE import_polygon_delete ( @@ -234,7 +236,7 @@ CREATE TABLE import_polygon_delete ( type TEXT NOT NULL ); CREATE INDEX idx_import_polygon_delete_osmid ON import_polygon_delete USING BTREE (osm_type, osm_id); -GRANT SELECT ON import_polygon_delete TO "{www-user}"; +GRANT SELECT ON import_polygon_delete TO "{{config.DATABASE_WEBUSER}}"; DROP SEQUENCE IF EXISTS file; CREATE SEQUENCE file start 1; @@ -268,3 +270,5 @@ ALTER TABLE ONLY wikipedia_redirect ADD CONSTRAINT wikipedia_redirect_pkey PRIMA -- osm2pgsql does not create indexes on the middle tables for Nominatim -- Add one for lookup of associated street relations. CREATE INDEX planet_osm_rels_parts_associated_idx ON planet_osm_rels USING gin(parts) WHERE tags @> ARRAY['associatedStreet']; + +GRANT SELECT ON table country_osm_grid to "{{config.DATABASE_WEBUSER}}"; diff --git a/nominatim/clicmd/setup.py b/nominatim/clicmd/setup.py index ab57d59b..71980739 100644 --- a/nominatim/clicmd/setup.py +++ b/nominatim/clicmd/setup.py @@ -79,20 +79,22 @@ class SetupAll: drop=args.no_updates, ignore_errors=args.ignore_errors) - LOG.warning('Create functions (1st pass)') with connect(args.config.get_libpq_dsn()) as conn: + LOG.warning('Create functions (1st pass)') refresh.create_functions(conn, args.config, args.sqllib_dir, False, False) - - LOG.warning('Create tables') - params = ['setup.php', '--create-tables', '--create-partition-tables'] - if args.reverse_only: - params.append('--reverse-only') - run_legacy_script(*params, nominatim_env=args, - throw_on_fail=not args.ignore_errors) - - LOG.warning('Create functions (2nd pass)') - with connect(args.config.get_libpq_dsn()) as conn: + LOG.warning('Create tables') + database_import.create_tables(conn, args.config, args.sqllib_dir, + reverse_only=args.reverse_only) + refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG)) + LOG.warning('Create functions (2nd pass)') + refresh.create_functions(conn, args.config, args.sqllib_dir, + False, False) + LOG.warning('Create table triggers') + database_import.create_table_triggers(conn, args.config, args.sqllib_dir) + LOG.warning('Create partition tables') + database_import.create_partition_tables(conn, args.config, args.sqllib_dir) + LOG.warning('Create functions (3rd pass)') refresh.create_functions(conn, args.config, args.sqllib_dir, False, False) @@ -124,10 +126,12 @@ class SetupAll: indexer.index_full(analyse=not args.index_noanalyse) LOG.warning('Post-process tables') - params = ['setup.php', '--create-search-indices', '--create-country-names'] - if args.no_updates: - params.append('--drop') - run_legacy_script(*params, nominatim_env=args, throw_on_fail=not args.ignore_errors) + with connect(args.config.get_libpq_dsn()) as conn: + database_import.create_search_indices(conn, args.config, + args.sqllib_dir, + drop=args.no_updates) + run_legacy_script('setup.php', '--create-country-names', + nominatim_env=args, throw_on_fail=not args.ignore_errors) webdir = args.project_dir / 'website' LOG.warning('Setup website at %s', webdir) diff --git a/nominatim/clicmd/transition.py b/nominatim/clicmd/transition.py index 210eec70..b8db1a38 100644 --- a/nominatim/clicmd/transition.py +++ b/nominatim/clicmd/transition.py @@ -35,8 +35,14 @@ class AdminTransition: help='Import a osm file') group.add_argument('--load-data', action='store_true', help='Copy data to live tables from import table') + group.add_argument('--create-tables', action='store_true', + help='Create main tables') + group.add_argument('--create-partition-tables', action='store_true', + help='Create required partition tables') group.add_argument('--index', action='store_true', help='Index the data') + group.add_argument('--create-search-indices', action='store_true', + help='Create additional indices required for search and update') group = parser.add_argument_group('Options') group.add_argument('--no-partitions', action='store_true', help='Do not partition search indices') @@ -50,10 +56,13 @@ class AdminTransition: help='Do not perform analyse operations during index') group.add_argument('--ignore-errors', action='store_true', help="Ignore certain erros on import.") + group.add_argument('--reverse-only', action='store_true', + help='Do not create search tables and indexes') @staticmethod def run(args): from ..tools import database_import + from ..tools import refresh if args.create_db: LOG.warning('Create DB') @@ -80,6 +89,20 @@ class AdminTransition: drop=args.drop, ignore_errors=args.ignore_errors) + if args.create_tables: + LOG.warning('Create Tables') + with connect(args.config.get_libpq_dsn()) as conn: + database_import.create_tables(conn, args.config, args.sqllib_dir, args.reverse_only) + refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG)) + refresh.create_functions(conn, args.config, args.sqllib_dir, + enable_diff_updates=False) + database_import.create_table_triggers(conn, args.config, args.sqllib_dir) + + if args.create_partition_tables: + LOG.warning('Create Partition Tables') + with connect(args.config.get_libpq_dsn()) as conn: + database_import.create_partition_tables(conn, args.config, args.sqllib_dir) + if args.load_data: LOG.warning('Load data') with connect(args.config.get_libpq_dsn()) as conn: @@ -99,3 +122,8 @@ class AdminTransition: from ..indexer.indexer import Indexer indexer = Indexer(args.config.get_libpq_dsn(), args.threads or 1) indexer.index_full() + + if args.create_search_indices: + LOG.warning('Create Search indices') + with connect(args.config.get_libpq_dsn()) as conn: + database_import.create_search_indices(conn, args.config, args.sqllib_dir, args.drop) diff --git a/nominatim/db/sql_preprocessor.py b/nominatim/db/sql_preprocessor.py new file mode 100644 index 00000000..85244752 --- /dev/null +++ b/nominatim/db/sql_preprocessor.py @@ -0,0 +1,94 @@ +""" +Preprocessing of SQL files. +""" +import jinja2 + + +def _get_partitions(conn): + """ Get the set of partitions currently in use. + """ + with conn.cursor() as cur: + cur.execute('SELECT DISTINCT partition FROM country_name') + partitions = set([0]) + for row in cur: + partitions.add(row[0]) + + return partitions + + +def _get_tables(conn): + """ Return the set of tables currently in use. + Only includes non-partitioned + """ + with conn.cursor() as cur: + cur.execute("SELECT tablename FROM pg_tables WHERE schemaname = 'public'") + + return set((row[0] for row in list(cur))) + + +def _setup_tablespace_sql(config): + """ Returns a dict with tablespace expressions for the different tablespace + kinds depending on whether a tablespace is configured or not. + """ + out = {} + for subset in ('ADDRESS', 'SEARCH', 'AUX'): + for kind in ('DATA', 'INDEX'): + tspace = getattr(config, 'TABLESPACE_{}_{}'.format(subset, kind)) + if tspace: + tspace = 'TABLESPACE "{}"'.format(tspace) + out['{}_{}'.format(subset.lower, kind.lower())] = tspace + + return out + + +def _setup_postgres_sql(conn): + """ Set up a dictionary with various Postgresql/Postgis SQL terms which + are dependent on the database version in use. + """ + out = {} + pg_version = conn.server_version_tuple() + # CREATE INDEX IF NOT EXISTS was introduced in PG9.5. + # Note that you need to ignore failures on older versions when + # unsing this construct. + out['if_index_not_exists'] = ' IF NOT EXISTS ' if pg_version >= (9, 5, 0) else '' + + return out + + +class SQLPreprocessor: # pylint: disable=too-few-public-methods + """ A environment for preprocessing SQL files from the + lib-sql directory. + + The preprocessor provides a number of default filters and variables. + The variables may be overwritten when rendering an SQL file. + + The preprocessing is currently based on the jinja2 templating library + and follows its syntax. + """ + + def __init__(self, conn, config, sqllib_dir): + self.env = jinja2.Environment(autoescape=False, + loader=jinja2.FileSystemLoader(str(sqllib_dir))) + + db_info = {} + db_info['partitions'] = _get_partitions(conn) + db_info['tables'] = _get_tables(conn) + db_info['reverse_only'] = 'search_name' not in db_info['tables'] + db_info['tablespace'] = _setup_tablespace_sql(config) + + self.env.globals['config'] = config + self.env.globals['db'] = db_info + self.env.globals['sql'] = _setup_postgres_sql(conn) + self.env.globals['modulepath'] = config.DATABASE_MODULE_PATH or \ + str((config.project_dir / 'module').resolve()) + + + def run_sql_file(self, conn, name, **kwargs): + """ Execute the given SQL file on the connection. The keyword arguments + may supply additional parameters for preprocessing. + """ + sql = self.env.get_template(name).render(**kwargs) + + with conn.cursor() as cur: + cur.execute(sql) + conn.commit() diff --git a/nominatim/tools/database_import.py b/nominatim/tools/database_import.py index ab8ccdd2..017c74b6 100644 --- a/nominatim/tools/database_import.py +++ b/nominatim/tools/database_import.py @@ -14,6 +14,7 @@ import psycopg2 from ..db.connection import connect, get_pg_env from ..db import utils as db_utils from ..db.async_connection import DBConnection +from ..db.sql_preprocessor import SQLPreprocessor from .exec_utils import run_osm2pgsql from ..errors import UsageError from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION @@ -178,6 +179,32 @@ def import_osm_data(osm_file, options, drop=False, ignore_errors=False): Path(options['flatnode_file']).unlink() +def create_tables(conn, config, sqllib_dir, reverse_only=False): + """ Create the set of basic tables. + When `reverse_only` is True, then the main table for searching will + be skipped and only reverse search is possible. + """ + sql = SQLPreprocessor(conn, config, sqllib_dir) + sql.env.globals['db']['reverse_only'] = reverse_only + + sql.run_sql_file(conn, 'tables.sql') + + +def create_table_triggers(conn, config, sqllib_dir): + """ Create the triggers for the tables. The trigger functions must already + have been imported with refresh.create_functions(). + """ + sql = SQLPreprocessor(conn, config, sqllib_dir) + sql.run_sql_file(conn, 'table-triggers.sql') + + +def create_partition_tables(conn, config, sqllib_dir): + """ Create tables that have explicit partitioning. + """ + sql = SQLPreprocessor(conn, config, sqllib_dir) + sql.run_sql_file(conn, 'partition-tables.src.sql') + + def truncate_data_tables(conn, max_word_frequency=None): """ Truncate all data tables to prepare for a fresh load. """ @@ -258,3 +285,24 @@ def load_data(dsn, data_dir, threads): with connect(dsn) as conn: with conn.cursor() as cur: cur.execute('ANALYSE') + + +def create_search_indices(conn, config, sqllib_dir, drop=False): + """ Create tables that have explicit partitioning. + """ + + # If index creation failed and left an index invalid, they need to be + # cleaned out first, so that the script recreates them. + with conn.cursor() as cur: + cur.execute("""SELECT relname FROM pg_class, pg_index + WHERE pg_index.indisvalid = false + AND pg_index.indexrelid = pg_class.oid""") + bad_indices = [row[0] for row in list(cur)] + for idx in bad_indices: + LOG.info("Drop invalid index %s.", idx) + cur.execute('DROP INDEX "{}"'.format(idx)) + conn.commit() + + sql = SQLPreprocessor(conn, config, sqllib_dir) + + sql.run_sql_file(conn, 'indices.sql', drop=drop) diff --git a/nominatim/tools/refresh.py b/nominatim/tools/refresh.py index 5cfa1ab0..c00a2e10 100644 --- a/nominatim/tools/refresh.py +++ b/nominatim/tools/refresh.py @@ -3,12 +3,12 @@ Functions for bringing auxiliary data in the database up-to-date. """ import json import logging -import re from textwrap import dedent from psycopg2.extras import execute_values from ..db.utils import execute_file +from ..db.sql_preprocessor import SQLPreprocessor from ..version import NOMINATIM_VERSION LOG = logging.getLogger() @@ -76,100 +76,17 @@ def load_address_levels_from_file(conn, config_file): with config_file.open('r') as fdesc: load_address_levels(conn, 'address_levels', json.load(fdesc)) -PLPGSQL_BASE_MODULES = ( - 'utils.sql', - 'normalization.sql', - 'ranking.sql', - 'importance.sql', - 'address_lookup.sql', - 'interpolation.sql' -) -PLPGSQL_TABLE_MODULES = ( - ('place', 'place_triggers.sql'), - ('placex', 'placex_triggers.sql'), - ('location_postcode', 'postcode_triggers.sql') -) - -def _get_standard_function_sql(conn, config, sql_dir, enable_diff_updates, enable_debug): - """ Read all applicable SQLs containing PL/pgSQL functions, replace - placefolders and execute them. - """ - sql_func_dir = sql_dir / 'functions' - sql = '' - - # Get the basic set of functions that is always imported. - for sql_file in PLPGSQL_BASE_MODULES: - with (sql_func_dir / sql_file).open('r') as fdesc: - sql += fdesc.read() - - # Some files require the presence of a certain table - for table, fname in PLPGSQL_TABLE_MODULES: - if conn.table_exists(table): - with (sql_func_dir / fname).open('r') as fdesc: - sql += fdesc.read() - - # Replace placeholders. - sql = sql.replace('{modulepath}', - config.DATABASE_MODULE_PATH or str((config.project_dir / 'module').resolve())) - - if enable_diff_updates: - sql = sql.replace('RETURN NEW; -- %DIFFUPDATES%', '--') - - if enable_debug: - sql = sql.replace('--DEBUG:', '') - - if config.get_bool('LIMIT_REINDEXING'): - sql = sql.replace('--LIMIT INDEXING:', '') - - if not config.get_bool('USE_US_TIGER_DATA'): - sql = sql.replace('-- %NOTIGERDATA% ', '') - - if not config.get_bool('USE_AUX_LOCATION_DATA'): - sql = sql.replace('-- %NOAUXDATA% ', '') - - reverse_only = 'false' if conn.table_exists('search_name') else 'true' - - return sql.replace('%REVERSE-ONLY%', reverse_only) - - -def replace_partition_string(sql, partitions): - """ Replace a partition template with the actual partition code. - """ - for match in re.findall('^-- start(.*?)^-- end', sql, re.M | re.S): - repl = '' - for part in partitions: - repl += match.replace('-partition-', str(part)) - sql = sql.replace(match, repl) - - return sql - -def _get_partition_function_sql(conn, sql_dir): - """ Create functions that work on partition tables. - """ - with conn.cursor() as cur: - cur.execute('SELECT distinct partition FROM country_name') - partitions = set([0]) - for row in cur: - partitions.add(row[0]) - - with (sql_dir / 'partition-functions.src.sql').open('r') as fdesc: - sql = fdesc.read() - - return replace_partition_string(sql, sorted(partitions)) - -def create_functions(conn, config, sql_dir, +def create_functions(conn, config, sqllib_dir, enable_diff_updates=True, enable_debug=False): """ (Re)create the PL/pgSQL functions. """ - sql = _get_standard_function_sql(conn, config, sql_dir, - enable_diff_updates, enable_debug) - sql += _get_partition_function_sql(conn, sql_dir) + sql = SQLPreprocessor(conn, config, sqllib_dir) - with conn.cursor() as cur: - cur.execute(sql) + sql.run_sql_file(conn, 'functions.sql', + disable_diff_update=not enable_diff_updates, + debug=enable_debug) - conn.commit() WEBSITE_SCRIPTS = ( diff --git a/test/python/test_cli.py b/test/python/test_cli.py index 08ea8332..418f4bcf 100644 --- a/test/python/test_cli.py +++ b/test/python/test_cli.py @@ -92,6 +92,11 @@ def test_import_full(temp_db, mock_func_factory): mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'), mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'), mock_func_factory(nominatim.tools.database_import, 'load_data'), + mock_func_factory(nominatim.tools.database_import, 'create_tables'), + mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'), + mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'), + mock_func_factory(nominatim.tools.database_import, 'create_search_indices'), + mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_file'), mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'), mock_func_factory(nominatim.tools.refresh, 'setup_website'), mock_func_factory(nominatim.db.properties, 'set_property') diff --git a/test/python/test_db_sql_preprocessor.py b/test/python/test_db_sql_preprocessor.py new file mode 100644 index 00000000..3c10000f --- /dev/null +++ b/test/python/test_db_sql_preprocessor.py @@ -0,0 +1,51 @@ +""" +Tests for SQL preprocessing. +""" +from pathlib import Path + +import pytest + +from nominatim.db.sql_preprocessor import SQLPreprocessor + +@pytest.fixture +def sql_factory(tmp_path): + def _mk_sql(sql_body): + (tmp_path / 'test.sql').write_text(""" + CREATE OR REPLACE FUNCTION test() RETURNS TEXT + AS $$ + BEGIN + {} + END; + $$ LANGUAGE plpgsql IMMUTABLE;""".format(sql_body)) + return 'test.sql' + + return _mk_sql + + +@pytest.fixture +def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory): + monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.') + table_factory('country_name', 'partition INT', (0, 1, 2)) + return SQLPreprocessor(temp_db_conn, def_config, tmp_path) + +@pytest.mark.parametrize("expr,ret", [ + ("'a'", 'a'), + ("'{{db.partitions|join}}'", '012'), + ("{% if 'country_name' in db.tables %}'yes'{% else %}'no'{% endif %}", "yes"), + ("{% if 'xxx' in db.tables %}'yes'{% else %}'no'{% endif %}", "no"), + ("'{{config.DATABASE_MODULE_PATH}}'", '.') + ]) +def test_load_file_simple(sql_preprocessor, sql_factory, temp_db_conn, temp_db_cursor, expr, ret): + sqlfile = sql_factory("RETURN {};".format(expr)) + + sql_preprocessor.run_sql_file(temp_db_conn, sqlfile) + + assert temp_db_cursor.scalar('SELECT test()') == ret + + +def test_load_file_with_params(sql_preprocessor, sql_factory, temp_db_conn, temp_db_cursor): + sqlfile = sql_factory("RETURN '{{ foo }} {{ bar }}';") + + sql_preprocessor.run_sql_file(temp_db_conn, sqlfile, bar='XX', foo='ZZ') + + assert temp_db_cursor.scalar('SELECT test()') == 'ZZ XX' diff --git a/test/python/test_tools_refresh_create_functions.py b/test/python/test_tools_refresh_create_functions.py index ac2f2211..40d4c81a 100644 --- a/test/python/test_tools_refresh_create_functions.py +++ b/test/python/test_tools_refresh_create_functions.py @@ -1,98 +1,47 @@ """ Tests for creating PL/pgSQL functions for Nominatim. """ -from pathlib import Path import pytest -from nominatim.db.connection import connect -from nominatim.tools.refresh import _get_standard_function_sql, _get_partition_function_sql - -SQL_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-sql').resolve() +from nominatim.tools.refresh import create_functions @pytest.fixture -def db(temp_db): - with connect('dbname=' + temp_db) as conn: - yield conn - -@pytest.fixture -def db_with_tables(db): - with db.cursor() as cur: - for table in ('place', 'placex', 'location_postcode'): - cur.execute('CREATE TABLE {} (place_id BIGINT)'.format(table)) - - return db +def conn(temp_db_conn, table_factory, monkeypatch): + monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.') + table_factory('country_name', 'partition INT', (0, 1, 2)) + return temp_db_conn -def test_standard_functions_replace_module_default(db, def_config): - def_config.project_dir = Path('.') - sql = _get_standard_function_sql(db, def_config, SQL_DIR, False, False) +def test_create_functions(temp_db_cursor, conn, def_config, tmp_path): + sqlfile = tmp_path / 'functions.sql' + sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER + AS $$ + BEGIN + RETURN 43; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """) - assert sql - assert sql.find('{modulepath}') < 0 - assert sql.find("'{}'".format(Path('module/nominatim.so').resolve())) >= 0 + create_functions(conn, def_config, tmp_path) + + assert temp_db_cursor.scalar('SELECT test()') == 43 -def test_standard_functions_replace_module_custom(monkeypatch, db, def_config): - monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', 'custom') - sql = _get_standard_function_sql(db, def_config, SQL_DIR, False, False) +@pytest.mark.parametrize("dbg,ret", ((True, 43), (False, 22))) +def test_create_functions_with_template(temp_db_cursor, conn, def_config, tmp_path, dbg, ret): + sqlfile = tmp_path / 'functions.sql' + sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER + AS $$ + BEGIN + {% if debug %} + RETURN 43; + {% else %} + RETURN 22; + {% endif %} + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """) - assert sql - assert sql.find('{modulepath}') < 0 - assert sql.find("'custom/nominatim.so'") >= 0 + create_functions(conn, def_config, tmp_path, enable_debug=dbg) - -@pytest.mark.parametrize("enabled", (True, False)) -def test_standard_functions_enable_diff(db_with_tables, def_config, enabled): - def_config.project_dir = Path('.') - sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, enabled, False) - - assert sql - assert (sql.find('%DIFFUPDATES%') < 0) == enabled - - -@pytest.mark.parametrize("enabled", (True, False)) -def test_standard_functions_enable_debug(db_with_tables, def_config, enabled): - def_config.project_dir = Path('.') - sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, enabled) - - assert sql - assert (sql.find('--DEBUG') < 0) == enabled - - -@pytest.mark.parametrize("enabled", (True, False)) -def test_standard_functions_enable_limit_reindexing(monkeypatch, db_with_tables, def_config, enabled): - def_config.project_dir = Path('.') - monkeypatch.setenv('NOMINATIM_LIMIT_REINDEXING', 'yes' if enabled else 'no') - sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False) - - assert sql - assert (sql.find('--LIMIT INDEXING') < 0) == enabled - - -@pytest.mark.parametrize("enabled", (True, False)) -def test_standard_functions_enable_tiger(monkeypatch, db_with_tables, def_config, enabled): - def_config.project_dir = Path('.') - monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA', 'yes' if enabled else 'no') - sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False) - - assert sql - assert (sql.find('%NOTIGERDATA%') >= 0) == enabled - - -@pytest.mark.parametrize("enabled", (True, False)) -def test_standard_functions_enable_aux(monkeypatch, db_with_tables, def_config, enabled): - def_config.project_dir = Path('.') - monkeypatch.setenv('NOMINATIM_USE_AUX_LOCATION_DATA', 'yes' if enabled else 'no') - sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False) - - assert sql - assert (sql.find('%NOAUXDATA%') >= 0) == enabled - - -def test_partition_function(temp_db_cursor, db, def_config): - temp_db_cursor.execute("CREATE TABLE country_name (partition SMALLINT)") - - sql = _get_partition_function_sql(db, SQL_DIR) - - assert sql - assert sql.find('-partition-') < 0 + assert temp_db_cursor.scalar('SELECT test()') == ret diff --git a/vagrant/Install-on-Centos-7.sh b/vagrant/Install-on-Centos-7.sh index e0ab7470..24d88926 100755 --- a/vagrant/Install-on-Centos-7.sh +++ b/vagrant/Install-on-Centos-7.sh @@ -42,7 +42,7 @@ python3-pip python3-setuptools python3-devel \ expat-devel zlib-devel - pip3 install --user psycopg2 python-dotenv psutil + pip3 install --user psycopg2 python-dotenv psutil Jinja2 # diff --git a/vagrant/Install-on-Centos-8.sh b/vagrant/Install-on-Centos-8.sh index 0018a452..859c48b9 100755 --- a/vagrant/Install-on-Centos-8.sh +++ b/vagrant/Install-on-Centos-8.sh @@ -35,7 +35,7 @@ python3-pip python3-setuptools python3-devel \ expat-devel zlib-devel - pip3 install --user psycopg2 python-dotenv psutil + pip3 install --user psycopg2 python-dotenv psutil Jinja2 # diff --git a/vagrant/Install-on-Ubuntu-18.sh b/vagrant/Install-on-Ubuntu-18.sh index d8231908..5cbbd583 100755 --- a/vagrant/Install-on-Ubuntu-18.sh +++ b/vagrant/Install-on-Ubuntu-18.sh @@ -30,7 +30,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: postgresql-server-dev-10 postgresql-10-postgis-2.4 \ postgresql-contrib-10 postgresql-10-postgis-scripts \ php php-pgsql php-intl python3-pip \ - python3-psycopg2 python3-psutil git + python3-psycopg2 python3-psutil python3-jinja2 git # The python-dotenv package that comes with Ubuntu 18.04 is too old, so # install the latest version from pip: diff --git a/vagrant/Install-on-Ubuntu-20.sh b/vagrant/Install-on-Ubuntu-20.sh index 6f03bc72..0649c9a6 100755 --- a/vagrant/Install-on-Ubuntu-20.sh +++ b/vagrant/Install-on-Ubuntu-20.sh @@ -33,7 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS: postgresql-server-dev-12 postgresql-12-postgis-3 \ postgresql-contrib-12 postgresql-12-postgis-3-scripts \ php php-pgsql php-intl python3-dotenv \ - python3-psycopg2 python3-psutil git + python3-psycopg2 python3-psutil python3-jinja2 git # # System Configuration