hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-12 13:54:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2197 from lonvia/use-jinja-for-sql-preprocessing

Browse Source 
Use jinja2 for SQL preprocessing
This commit is contained in:
Sarah Hoffmann
2021-03-04 16:36:18 +01:00
committed by GitHub
parent dd301cf5ac 09f4d767e4
commit 9d103503f7
29 changed files with 607 additions and 568 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/build-nominatim/action.yml vendored
View File

@@ -6,7 +6,7 @@ runs:
steps:
- name: Install prerequisites
run: |
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil
sudo apt-get install -y -qq libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libproj-dev python3-psycopg2 python3-pyosmium python3-dotenv python3-psutil python3-jinja2
shell: bash
- name: Download dependencies

1
docs/admin/Installation.md
View File

@@ -42,6 +42,7 @@ For running Nominatim:
* [Psycopg2](https://www.psycopg.org) (2.7+)
* [Python Dotenv](https://github.com/theskumar/python-dotenv)
* [psutil](https://github.com/giampaolo/psutil)
* [Jinja2](https://palletsprojects.com/p/jinja/)
* [PHP](https://php.net) (7.0 or later)
* PHP-pgsql
* PHP-intl (bundled with PHP)

25
lib-php/admin/setup.php
View File

@@ -123,24 +123,28 @@ if ($aCMDResult['import-data'] || $aCMDResult['all']) {
if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createFunctions();
$oSetup->createSqlFunctions();
}
if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createTables($aCMDResult['reverse-only']);
$oSetup->createFunctions();
$oSetup->createTableTriggers();
$oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-tables');
if ($aCMDResult['reverse-only'] ?? false) {
$oCmd->addParams('--reverse-only');
}
run($oCmd);
}
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createPartitionTables();
run((clone($oNominatimCmd))->addParams('transition', '--create-partition-tables'));
}
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createFunctions(); // also create partition functions
$oSetup->createSqlFunctions(); // also create partition functions
}
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
@@ -182,7 +186,14 @@ if ($aCMDResult['drop']) {
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
$bDidSomething = true;
$oSetup->createSearchIndices();
$oCmd = (clone($oNominatimCmd))->addParams('transition', '--create-search-indices');
if ($aCMDResult['drop'] ?? false) {
$oCmd->addParams('--drop');
}
run($oCmd);
}
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {

131
lib-php/setup/SetupClass.php
View File

@@ -67,52 +67,6 @@ class SetupFunctions
}
}
public function createFunctions()
{
info('Create Functions');
// Try accessing the C module, so we know early if something is wrong
$this->checkModulePresence(); // raises exception on failure
$this->createSqlFunctions();
}
public function createTables($bReverseOnly = false)
{
info('Create Tables');
$sTemplate = file_get_contents(CONST_SqlDir.'/tables.sql');
$sTemplate = $this->replaceSqlPatterns($sTemplate);
$this->pgsqlRunScript($sTemplate, false);
if ($bReverseOnly) {
$this->dropTable('search_name');
}
(clone($this->oNominatimCmd))->addParams('refresh', '--address-levels')->run();
}
public function createTableTriggers()
{
info('Create Tables');
$sTemplate = file_get_contents(CONST_SqlDir.'/table-triggers.sql');
$sTemplate = $this->replaceSqlPatterns($sTemplate);
$this->pgsqlRunScript($sTemplate, false);
}
public function createPartitionTables()
{
info('Create Partition Tables');
$sTemplate = file_get_contents(CONST_SqlDir.'/partition-tables.src.sql');
$sTemplate = $this->replaceSqlPatterns($sTemplate);
$this->pgsqlRunPartitionScript($sTemplate);
}
public function importTigerData($sTigerPath)
{
info('Import Tiger data');
@@ -246,31 +200,6 @@ class SetupFunctions
$this->db()->exec($sSQL);
}
public function createSearchIndices()
{
info('Create Search indices');
$sSQL = 'SELECT relname FROM pg_class, pg_index ';
$sSQL .= 'WHERE pg_index.indisvalid = false AND pg_index.indexrelid = pg_class.oid';
$aInvalidIndices = $this->db()->getCol($sSQL);
foreach ($aInvalidIndices as $sIndexName) {
info("Cleaning up invalid index $sIndexName");
$this->db()->exec("DROP INDEX $sIndexName;");
}
$sTemplate = file_get_contents(CONST_SqlDir.'/indices.src.sql');
if (!$this->bDrop) {
$sTemplate .= file_get_contents(CONST_SqlDir.'/indices_updates.src.sql');
}
if (!$this->dbReverseOnly()) {
$sTemplate .= file_get_contents(CONST_SqlDir.'/indices_search.src.sql');
}
$sTemplate = $this->replaceSqlPatterns($sTemplate);
$this->pgsqlRunScript($sTemplate);
}
public function createCountryNames()
{
info('Create search index for default country names');
@@ -326,7 +255,7 @@ class SetupFunctions
);
}
private function createSqlFunctions()
public function createSqlFunctions()
{
$oCmd = (clone($this->oNominatimCmd))
->addParams('refresh', '--functions');
@@ -342,24 +271,6 @@ class SetupFunctions
$oCmd->run(!$this->sIgnoreErrors);
}
private function pgsqlRunPartitionScript($sTemplate)
{
$sSQL = 'select distinct partition from country_name order by partition';
$aPartitions = $this->db()->getCol($sSQL);
if ($aPartitions[0] != 0) $aPartitions[] = 0;
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach ($aMatches as $aMatch) {
$sResult = '';
foreach ($aPartitions as $sPartitionName) {
$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
}
$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
}
$this->pgsqlRunScript($sTemplate);
}
private function pgsqlRunScriptFile($sFilename)
{
if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
@@ -444,44 +355,4 @@ class SetupFunctions
return $sSql;
}
/**
* Drop table with the given name if it exists.
*
* @param string $sName Name of table to remove.
*
* @return null
*/
private function dropTable($sName)
{
if ($this->bVerbose) echo "Dropping table $sName\n";
$this->db()->deleteTable($sName);
}
/**
* Check if the database is in reverse-only mode.
*
* @return True if there is no search_name table and infrastructure.
*/
private function dbReverseOnly()
{
return !($this->db()->tableExists('search_name'));
}
/**
* Try accessing the C module, so we know early if something is wrong.
*
* Raises Nominatim\DatabaseError on failure
*/
private function checkModulePresence()
{
$sModulePath = getSetting('DATABASE_MODULE_PATH', CONST_InstallDir.'/module');
$sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
$sSQL .= $sModulePath . "/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
$sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
$oDB = new \Nominatim\DB();
$oDB->connect();
$oDB->exec($sSQL, null, 'Database server failed to load '.$sModulePath.'/nominatim.so module');
}
}

20
lib-sql/functions.sql Normal file
View File

@@ -0,0 +1,20 @@
{% include('functions/utils.sql') %}
{% include('functions/normalization.sql') %}
{% include('functions/ranking.sql') %}
{% include('functions/importance.sql') %}
{% include('functions/address_lookup.sql') %}
{% include('functions/interpolation.sql') %}
{% if 'place' in db.tables %}
{% include 'functions/place_triggers.sql' %}
{% endif %}
{% if 'placex' in db.tables %}
{% include 'functions/placex_triggers.sql' %}
{% endif %}
{% if 'location_postcode' in db.tables %}
{% include 'functions/postcode_triggers.sql' %}
{% endif %}
{% include('functions/partition-functions.sql') %}

9
lib-sql/functions/address_lookup.sql
View File

@@ -121,7 +121,7 @@ BEGIN
END IF;
--then query tiger data
-- %NOTIGERDATA% IF 0 THEN
{% if config.get_bool('USE_US_TIGER_DATA') %}
IF place IS NULL AND in_housenumber >= 0 THEN
SELECT parent_place_id as place_id, 'us' as country_code,
in_housenumber as housenumber, postcode,
@@ -133,9 +133,10 @@ BEGIN
WHERE place_id = in_place_id
AND in_housenumber between startnumber and endnumber;
END IF;
-- %NOTIGERDATA% END IF;
{% endif %}
-- %NOAUXDATA% IF 0 THEN
-- then additional data
{% if config.get_bool('USE_AUX_LOCATION_DATA') %}
IF place IS NULL THEN
SELECT parent_place_id as place_id, 'us' as country_code,
housenumber, postcode,
@@ -146,7 +147,7 @@ BEGIN
FROM location_property_aux
WHERE place_id = in_place_id;
END IF;
-- %NOAUXDATA% END IF;
{% endif %}
-- postcode table
IF place IS NULL THEN

4
lib-sql/functions/normalization.sql
View File

@@ -1,12 +1,12 @@
-- Functions for term normalisation and access to the 'word' table.
CREATE OR REPLACE FUNCTION transliteration(text) RETURNS text
AS '{modulepath}/nominatim.so', 'transliteration'
AS '{{ modulepath }}/nominatim.so', 'transliteration'
LANGUAGE c IMMUTABLE STRICT;
CREATE OR REPLACE FUNCTION gettokenstring(text) RETURNS text
AS '{modulepath}/nominatim.so', 'gettokenstring'
AS '{{ modulepath }}/nominatim.so', 'gettokenstring'
LANGUAGE c IMMUTABLE STRICT;

100
lib-sql/partition-functions.src.sql → lib-sql/functions/partition-functions.sql
View File

@@ -37,13 +37,13 @@ DECLARE
r nearfeaturecentr%rowtype;
BEGIN
-- start
IF in_partition = -partition- THEN
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
FOR r IN
SELECT place_id, keywords, rank_address, rank_search,
min(ST_Distance(feature, centroid)) as distance,
isguess, postcode, centroid
FROM location_area_large_-partition-
FROM location_area_large_{{ partition }}
WHERE geometry && feature
AND is_relevant_geometry(ST_Relate(geometry, feature), ST_GeometryType(feature))
AND rank_address < maxrank
@@ -56,7 +56,7 @@ BEGIN
END LOOP;
RETURN;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
END
@@ -80,12 +80,12 @@ BEGIN
CONTINUE;
END IF;
-- start
IF in_partition = -partition- THEN
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
SELECT place_id, keywords, rank_address, rank_search,
min(ST_Distance(feature, centroid)) as distance,
isguess, postcode, centroid INTO r
FROM location_area_large_-partition-
FROM location_area_large_{{ partition }}
WHERE geometry && ST_Expand(feature, item.extent)
AND rank_address between item.from_rank and item.to_rank
AND word_ids_from_name(item.name) && keywords
@@ -103,7 +103,7 @@ BEGIN
END IF;
CONTINUE;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
END LOOP;
@@ -120,12 +120,12 @@ BEGIN
RETURN TRUE;
END IF;
-- start
IF in_partition = -partition- THEN
DELETE from location_area_large_-partition- WHERE place_id = in_place_id;
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
DELETE from location_area_large_{{ partition }} WHERE place_id = in_place_id;
RETURN TRUE;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
@@ -150,13 +150,13 @@ BEGIN
RETURN TRUE;
END IF;
-- start
IF in_partition = -partition- THEN
INSERT INTO location_area_large_-partition- (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, postcode, centroid, geometry)
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
INSERT INTO location_area_large_{{ partition }} (partition, place_id, country_code, keywords, rank_search, rank_address, isguess, postcode, centroid, geometry)
values (in_partition, in_place_id, in_country_code, in_keywords, in_rank_search, in_rank_address, in_estimate, postcode, in_centroid, in_geometry);
RETURN TRUE;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
RETURN FALSE;
@@ -173,9 +173,9 @@ DECLARE
parent BIGINT;
BEGIN
-- start
IF in_partition = -partition- THEN
SELECT place_id FROM search_name_-partition-
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
SELECT place_id FROM search_name_{{ partition }}
INTO parent
WHERE name_vector && isin_token
AND centroid && ST_Expand(point, 0.015)
@@ -183,7 +183,7 @@ BEGIN
ORDER BY ST_Distance(centroid, point) ASC limit 1;
RETURN parent;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
END
@@ -199,18 +199,18 @@ DECLARE
parent BIGINT;
BEGIN
-- start
IF in_partition = -partition- THEN
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
SELECT place_id
INTO parent
FROM search_name_-partition-
FROM search_name_{{ partition }}
WHERE name_vector && isin_token
AND centroid && ST_Expand(point, 0.04)
AND address_rank between 16 and 25
ORDER BY ST_Distance(centroid, point) ASC limit 1;
RETURN parent;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
END
@@ -223,16 +223,16 @@ create or replace function insertSearchName(
RETURNS BOOLEAN AS $$
DECLARE
BEGIN
-- start
IF in_partition = -partition- THEN
DELETE FROM search_name_-partition- values WHERE place_id = in_place_id;
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
DELETE FROM search_name_{{ partition }} values WHERE place_id = in_place_id;
IF in_rank_address > 0 THEN
INSERT INTO search_name_-partition- (place_id, address_rank, name_vector, centroid)
INSERT INTO search_name_{{ partition }} (place_id, address_rank, name_vector, centroid)
values (in_place_id, in_rank_address, in_name_vector, in_geometry);
END IF;
RETURN TRUE;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
RETURN FALSE;
@@ -243,12 +243,12 @@ LANGUAGE plpgsql;
create or replace function deleteSearchName(in_partition INTEGER, in_place_id BIGINT) RETURNS BOOLEAN AS $$
DECLARE
BEGIN
-- start
IF in_partition = -partition- THEN
DELETE from search_name_-partition- WHERE place_id = in_place_id;
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
DELETE from search_name_{{ partition }} WHERE place_id = in_place_id;
RETURN TRUE;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
@@ -262,14 +262,14 @@ create or replace function insertLocationRoad(
DECLARE
BEGIN
-- start
IF in_partition = -partition- THEN
DELETE FROM location_road_-partition- where place_id = in_place_id;
INSERT INTO location_road_-partition- (partition, place_id, country_code, geometry)
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
DELETE FROM location_road_{{ partition }} where place_id = in_place_id;
INSERT INTO location_road_{{ partition }} (partition, place_id, country_code, geometry)
values (in_partition, in_place_id, in_country_code, in_geometry);
RETURN TRUE;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
RETURN FALSE;
@@ -281,12 +281,12 @@ create or replace function deleteRoad(in_partition INTEGER, in_place_id BIGINT)
DECLARE
BEGIN
-- start
IF in_partition = -partition- THEN
DELETE FROM location_road_-partition- where place_id = in_place_id;
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
DELETE FROM location_road_{{ partition }} where place_id = in_place_id;
RETURN TRUE;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
@@ -303,12 +303,12 @@ DECLARE
search_diameter FLOAT;
BEGIN
-- start
IF in_partition = -partition- THEN
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
search_diameter := 0.00005;
WHILE search_diameter < 0.1 LOOP
FOR r IN
SELECT place_id FROM location_road_-partition-
SELECT place_id FROM location_road_{{ partition }}
WHERE ST_DWithin(geometry, point, search_diameter)
ORDER BY ST_Distance(geometry, point) ASC limit 1
LOOP
@@ -318,7 +318,7 @@ BEGIN
END LOOP;
RETURN NULL;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
END
@@ -345,12 +345,12 @@ BEGIN
p2 := ST_LineInterpolatePoint(line,0.5);
p3 := ST_LineInterpolatePoint(line,1);
-- start
IF in_partition = -partition- THEN
{% for partition in db.partitions %}
IF in_partition = {{ partition }} THEN
search_diameter := 0.0005;
WHILE search_diameter < 0.01 LOOP
FOR r IN
SELECT place_id FROM location_road_-partition-
SELECT place_id FROM location_road_{{ partition }}
WHERE ST_DWithin(line, geometry, search_diameter)
ORDER BY (ST_distance(geometry, p1)+
ST_distance(geometry, p2)+
@@ -362,7 +362,7 @@ BEGIN
END LOOP;
RETURN NULL;
END IF;
-- end
{% endfor %}
RAISE EXCEPTION 'Unknown partition %', in_partition;
END

28
lib-sql/functions/place_triggers.sql
View File

@@ -12,8 +12,10 @@ DECLARE
partition INTEGER;
BEGIN
--DEBUG: RAISE WARNING '-----------------------------------------------------------------------------------';
--DEBUG: RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
{% if debug %}
RAISE WARNING '-----------------------------------------------------------------------------------';
RAISE WARNING 'place_insert: % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,st_area(NEW.geometry);
{% endif %}
-- filter wrong tupels
IF ST_IsEmpty(NEW.geometry) OR NOT ST_IsValid(NEW.geometry) OR ST_X(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') OR ST_Y(ST_Centroid(NEW.geometry))::text in ('NaN','Infinity','-Infinity') THEN
INSERT INTO import_polygon_error (osm_type, osm_id, class, type, name, country_code, updated, errormessage, prevgeometry, newgeometry)
@@ -97,8 +99,8 @@ BEGIN
DELETE FROM place where osm_type = NEW.osm_type and osm_id = NEW.osm_id and class = NEW.class;
END IF;
--DEBUG: RAISE WARNING 'Existing: %',existing.osm_id;
--DEBUG: RAISE WARNING 'Existing PlaceX: %',existingplacex.place_id;
{% if debug %}RAISE WARNING 'Existing: %',existing.osm_id;{% endif %}
{% if debug %}RAISE WARNING 'Existing PlaceX: %',existingplacex.place_id;{% endif %}
-- Log and discard
IF existing.geometry is not null AND st_isvalid(existing.geometry)
@@ -122,14 +124,20 @@ BEGIN
(existingplacex.type != NEW.type)))
THEN
{% if config.get_bool('LIMIT_REINDEXING') %}
IF existingplacex.osm_type IS NOT NULL THEN
-- sanity check: ignore admin_level changes on places with too many active children
-- or we end up reindexing entire countries because somebody accidentally deleted admin_level
--LIMIT INDEXING: SELECT count(*) FROM (SELECT 'a' FROM placex , place_addressline where address_place_id = existingplacex.place_id and placex.place_id = place_addressline.place_id and indexed_status = 0 and place_addressline.isaddress LIMIT 100001) sub INTO i;
--LIMIT INDEXING: IF i > 100000 THEN
--LIMIT INDEXING: RETURN null;
--LIMIT INDEXING: END IF;
SELECT count(*) INTO i FROM
(SELECT 'a' FROM placex, place_addressline
WHERE address_place_id = existingplacex.place_id
and placex.place_id = place_addressline.place_id
and indexed_status = 0 and place_addressline.isaddress LIMIT 100001) sub;
IF i > 100000 THEN
RETURN null;
END IF;
END IF;
{% endif %}
IF existing.osm_type IS NOT NULL THEN
-- pathological case caused by the triggerless copy into place during initial import
@@ -144,7 +152,7 @@ BEGIN
values (NEW.osm_type, NEW.osm_id, NEW.class, NEW.type, NEW.name,
NEW.admin_level, NEW.address, NEW.extratags, NEW.geometry);
--DEBUG: RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;
{% if debug %}RAISE WARNING 'insert done % % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type,NEW.name;{% endif %}
RETURN NEW;
END IF;
@@ -258,7 +266,7 @@ DECLARE
has_rank BOOLEAN;
BEGIN
--DEBUG: RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;
{% if debug %}RAISE WARNING 'delete: % % % %',OLD.osm_type,OLD.osm_id,OLD.class,OLD.type;{% endif %}
-- deleting large polygons can have a massive effect on the system - require manual intervention to let them through
IF st_area(OLD.geometry) > 2 and st_isvalid(OLD.geometry) THEN

137
lib-sql/functions/placex_triggers.sql
View File

@@ -20,7 +20,7 @@ DECLARE
location RECORD;
parent RECORD;
BEGIN
--DEBUG: RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;
{% if debug %}RAISE WARNING 'finding street for % %', poi_osm_type, poi_osm_id;{% endif %}
-- Is this object part of an associatedStreet relation?
FOR location IN
@@ -58,7 +58,7 @@ BEGIN
and poi_osm_id = any(x.nodes)
LIMIT 1
LOOP
--DEBUG: RAISE WARNING 'Get parent from interpolation: %', parent.parent_place_id;
{% if debug %}RAISE WARNING 'Get parent from interpolation: %', parent.parent_place_id;{% endif %}
RETURN parent.parent_place_id;
END LOOP;
@@ -71,11 +71,11 @@ BEGIN
and p.geometry && bbox
and w.id = p.osm_id and poi_osm_id = any(w.nodes)
LOOP
--DEBUG: RAISE WARNING 'Node is part of way % ', location.osm_id;
{% if debug %}RAISE WARNING 'Node is part of way % ', location.osm_id;{% endif %}
-- Way IS a road then we are on it - that must be our road
IF location.rank_search < 28 THEN
--DEBUG: RAISE WARNING 'node in way that is a street %',location;
{% if debug %}RAISE WARNING 'node in way that is a street %',location;{% endif %}
return location.place_id;
END IF;
@@ -106,7 +106,7 @@ BEGIN
ELSEIF ST_Area(bbox) < 0.005 THEN
-- for smaller features get the nearest road
SELECT getNearestRoadPlaceId(poi_partition, bbox) INTO parent_place_id;
--DEBUG: RAISE WARNING 'Checked for nearest way (%)', parent_place_id;
{% if debug %}RAISE WARNING 'Checked for nearest way (%)', parent_place_id;{% endif %}
ELSE
-- for larger features simply find the area with the largest rank that
-- contains the bbox, only use addressable features
@@ -146,21 +146,21 @@ BEGIN
IF bnd.osm_type = 'R' THEN
-- see if we have any special relation members
SELECT members FROM planet_osm_rels WHERE id = bnd.osm_id INTO relation_members;
--DEBUG: RAISE WARNING 'Got relation members';
{% if debug %}RAISE WARNING 'Got relation members';{% endif %}
-- Search for relation members with role 'lable'.
IF relation_members IS NOT NULL THEN
FOR rel_member IN
SELECT get_rel_node_members(relation_members, ARRAY['label']) as member
LOOP
--DEBUG: RAISE WARNING 'Found label member %', rel_member.member;
{% if debug %}RAISE WARNING 'Found label member %', rel_member.member;{% endif %}
FOR linked_placex IN
SELECT * from placex
WHERE osm_type = 'N' and osm_id = rel_member.member
and class = 'place'
LOOP
--DEBUG: RAISE WARNING 'Linked label member';
{% if debug %}RAISE WARNING 'Linked label member';{% endif %}
RETURN linked_placex;
END LOOP;
@@ -187,7 +187,7 @@ BEGIN
AND placex.rank_search < 26 -- needed to select the right index
AND _st_covers(bnd.geometry, placex.geometry)
LOOP
--DEBUG: RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id;
{% if debug %}RAISE WARNING 'Found type-matching place node %', linked_placex.osm_id;{% endif %}
RETURN linked_placex;
END LOOP;
END IF;
@@ -203,14 +203,14 @@ BEGIN
AND _st_covers(bnd.geometry, placex.geometry)
ORDER BY make_standard_name(name->'name') = bnd_name desc
LOOP
--DEBUG: RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id;
{% if debug %}RAISE WARNING 'Found wikidata-matching place node %', linked_placex.osm_id;{% endif %}
RETURN linked_placex;
END LOOP;
END IF;
-- Name searches can be done for ways as well as relations
IF bnd_name is not null THEN
--DEBUG: RAISE WARNING 'Looking for nodes with matching names';
{% if debug %}RAISE WARNING 'Looking for nodes with matching names';{% endif %}
FOR linked_placex IN
SELECT placex.* from placex
WHERE make_standard_name(name->'name') = bnd_name
@@ -225,7 +225,7 @@ BEGIN
AND placex.rank_search < 26 -- needed to select the right index
AND _st_covers(bnd.geometry, placex.geometry)
LOOP
--DEBUG: RAISE WARNING 'Found matching place node %', linked_placex.osm_id;
{% if debug %}RAISE WARNING 'Found matching place node %', linked_placex.osm_id;{% endif %}
RETURN linked_placex;
END LOOP;
END IF;
@@ -285,10 +285,10 @@ BEGIN
address, country)
ORDER BY rank_address, distance, isguess desc
LOOP
IF NOT %REVERSE-ONLY% THEN
{% if not db.reverse_only %}
nameaddress_vector := array_merge(nameaddress_vector,
location.keywords::int[]);
END IF;
{% endif %}
IF location.place_id is not null THEN
location_isaddress := not address_havelevel[location.rank_address];
@@ -362,10 +362,10 @@ BEGIN
END IF;
-- Add it to the list of search terms
IF NOT %REVERSE-ONLY% THEN
{% if not db.reverse_only %}
nameaddress_vector := array_merge(nameaddress_vector,
location.keywords::integer[]);
END IF;
{% endif %}
INSERT INTO place_addressline (place_id, address_place_id, fromarea,
isaddress, distance, cached_rank_address)
@@ -388,7 +388,7 @@ DECLARE
diameter FLOAT;
classtable TEXT;
BEGIN
--DEBUG: RAISE WARNING '% % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
{% if debug %}RAISE WARNING '% % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;{% endif %}
NEW.place_id := nextval('seq_place');
NEW.indexed_status := 1; --STATUS_NEW
@@ -440,9 +440,10 @@ BEGIN
END IF;
--DEBUG: RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;
{% if debug %}RAISE WARNING 'placex_insert:END: % % % %',NEW.osm_type,NEW.osm_id,NEW.class,NEW.type;{% endif %}
RETURN NEW; -- %DIFFUPDATES% The following is not needed until doing diff updates, and slows the main index process down
{% if not disable_diff_updates %}
-- The following is not needed until doing diff updates, and slows the main index process down
IF NEW.osm_type = 'N' and NEW.rank_search > 28 THEN
-- might be part of an interpolation
@@ -497,6 +498,8 @@ BEGIN
USING NEW.place_id, ST_Centroid(NEW.geometry);
END IF;
{% endif %} -- not disable_diff_updates
RETURN NEW;
END;
@@ -534,7 +537,7 @@ DECLARE
BEGIN
-- deferred delete
IF OLD.indexed_status = 100 THEN
--DEBUG: RAISE WARNING 'placex_update delete % %',NEW.osm_type,NEW.osm_id;
{% if debug %}RAISE WARNING 'placex_update delete % %',NEW.osm_type,NEW.osm_id;{% endif %}
delete from placex where place_id = OLD.place_id;
RETURN NULL;
END IF;
@@ -543,13 +546,13 @@ BEGIN
RETURN NEW;
END IF;
--DEBUG: RAISE WARNING 'placex_update % % (%)',NEW.osm_type,NEW.osm_id,NEW.place_id;
{% if debug %}RAISE WARNING 'placex_update % % (%)',NEW.osm_type,NEW.osm_id,NEW.place_id;{% endif %}
NEW.indexed_date = now();
IF NOT %REVERSE-ONLY% THEN
{% if 'search_name' in db.tables %}
DELETE from search_name WHERE place_id = NEW.place_id;
END IF;
{% endif %}
result := deleteSearchName(NEW.partition, NEW.place_id);
DELETE FROM place_addressline WHERE place_id = NEW.place_id;
result := deleteRoad(NEW.partition, NEW.place_id);
@@ -562,7 +565,7 @@ BEGIN
NEW.address := NEW.address - '_unlisted_place'::TEXT;
IF NEW.linked_place_id is not null THEN
--DEBUG: RAISE WARNING 'place already linked to %', NEW.linked_place_id;
{% if debug %}RAISE WARNING 'place already linked to %', NEW.linked_place_id;{% endif %}
RETURN NEW;
END IF;
@@ -578,7 +581,7 @@ BEGIN
-- Speed up searches - just use the centroid of the feature
-- cheaper but less acurate
NEW.centroid := ST_PointOnSurface(NEW.geometry);
--DEBUG: RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(NEW.centroid);
{% if debug %}RAISE WARNING 'Computing preliminary centroid at %',ST_AsText(NEW.centroid);{% endif %}
-- recompute the ranks, they might change when linking changes
SELECT * INTO NEW.rank_search, NEW.rank_address
@@ -658,7 +661,7 @@ BEGIN
parent_address_level := 3;
END IF;
--DEBUG: RAISE WARNING 'Copy over address tags';
{% if debug %}RAISE WARNING 'Copy over address tags';{% endif %}
-- housenumber is a computed field, so start with an empty value
NEW.housenumber := NULL;
IF NEW.address is not NULL THEN
@@ -707,7 +710,7 @@ BEGIN
END IF;
NEW.partition := get_partition(NEW.country_code);
END IF;
--DEBUG: RAISE WARNING 'Country updated: "%"', NEW.country_code;
{% if debug %}RAISE WARNING 'Country updated: "%"', NEW.country_code;{% endif %}
-- waterway ways are linked when they are part of a relation and have the same class/type
IF NEW.osm_type = 'R' and NEW.class = 'waterway' THEN
@@ -715,21 +718,21 @@ BEGIN
LOOP
FOR i IN 1..array_upper(relation_members, 1) BY 2 LOOP
IF relation_members[i+1] in ('', 'main_stream', 'side_stream') AND substring(relation_members[i],1,1) = 'w' THEN
--DEBUG: RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation_members[i];
{% if debug %}RAISE WARNING 'waterway parent %, child %/%', NEW.osm_id, i, relation_members[i];{% endif %}
FOR linked_node_id IN SELECT place_id FROM placex
WHERE osm_type = 'W' and osm_id = substring(relation_members[i],2,200)::bigint
and class = NEW.class and type in ('river', 'stream', 'canal', 'drain', 'ditch')
and ( relation_members[i+1] != 'side_stream' or NEW.name->'name' = name->'name')
LOOP
UPDATE placex SET linked_place_id = NEW.place_id WHERE place_id = linked_node_id;
IF NOT %REVERSE-ONLY% THEN
{% if 'search_name' in db.tables %}
DELETE FROM search_name WHERE place_id = linked_node_id;
END IF;
{% endif %}
END LOOP;
END IF;
END LOOP;
END LOOP;
--DEBUG: RAISE WARNING 'Waterway processed';
{% if debug %}RAISE WARNING 'Waterway processed';{% endif %}
END IF;
NEW.importance := null;
@@ -737,13 +740,13 @@ BEGIN
FROM compute_importance(NEW.extratags, NEW.country_code, NEW.osm_type, NEW.osm_id)
INTO NEW.wikipedia,NEW.importance;
--DEBUG: RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance;
{% if debug %}RAISE WARNING 'Importance computed from wikipedia: %', NEW.importance;{% endif %}
-- ---------------------------------------------------------------------------
-- For low level elements we inherit from our parent road
IF NEW.rank_search > 27 THEN
--DEBUG: RAISE WARNING 'finding street for % %', NEW.osm_type, NEW.osm_id;
{% if debug %}RAISE WARNING 'finding street for % %', NEW.osm_type, NEW.osm_id;{% endif %}
NEW.parent_place_id := null;
-- if we have a POI and there is no address information,
@@ -791,7 +794,7 @@ BEGIN
END IF;
NEW.country_code := location.country_code;
--DEBUG: RAISE WARNING 'Got parent details from search name';
{% if debug %}RAISE WARNING 'Got parent details from search name';{% endif %}
-- determine postcode
IF NEW.address is not null AND NEW.address ? 'postcode' THEN
@@ -812,13 +815,14 @@ BEGIN
name_vector, NEW.rank_search, NEW.rank_address,
upper(trim(NEW.address->'postcode')), NEW.geometry,
NEW.centroid);
--DEBUG: RAISE WARNING 'Place added to location table';
{% if debug %}RAISE WARNING 'Place added to location table';{% endif %}
END IF;
END IF;
IF not %REVERSE-ONLY% AND (array_length(name_vector, 1) is not NULL
OR inherited_address is not NULL OR NEW.address is not NULL)
{% if not db.reverse_only %}
IF array_length(name_vector, 1) is not NULL
OR inherited_address is not NULL OR NEW.address is not NULL
THEN
SELECT * INTO name_vector, nameaddress_vector
FROM create_poi_search_terms(NEW.place_id,
@@ -834,9 +838,10 @@ BEGIN
VALUES (NEW.place_id, NEW.rank_search, NEW.rank_address,
NEW.importance, NEW.country_code, name_vector,
nameaddress_vector, NEW.centroid);
--DEBUG: RAISE WARNING 'Place added to search table';
{% if debug %}RAISE WARNING 'Place added to search table';{% endif %}
END IF;
END IF;
{% endif %}
RETURN NEW;
END IF;
@@ -845,10 +850,10 @@ BEGIN
-- ---------------------------------------------------------------------------
-- Full indexing
--DEBUG: RAISE WARNING 'Using full index mode for % %', NEW.osm_type, NEW.osm_id;
{% if debug %}RAISE WARNING 'Using full index mode for % %', NEW.osm_type, NEW.osm_id;{% endif %}
SELECT * INTO location FROM find_linked_place(NEW);
IF location.place_id is not null THEN
--DEBUG: RAISE WARNING 'Linked %', location;
{% if debug %}RAISE WARNING 'Linked %', location;{% endif %}
-- Use the linked point as the centre point of the geometry,
-- but only if it is within the area of the boundary.
@@ -857,7 +862,7 @@ BEGIN
NEW.centroid := geom;
END IF;
--DEBUG: RAISE WARNING 'parent address: % rank address: %', parent_address_level, location.rank_address;
{% if debug %}RAISE WARNING 'parent address: % rank address: %', parent_address_level, location.rank_address;{% endif %}
IF location.rank_address > parent_address_level
and location.rank_address < 26
THEN
@@ -878,9 +883,9 @@ BEGIN
UPDATE placex set linked_place_id = NEW.place_id
WHERE place_id = location.place_id;
-- ensure that those places are not found anymore
IF NOT %REVERSE-ONLY% THEN
{% if 'search_name' in db.tables %}
DELETE FROM search_name WHERE place_id = location.place_id;
END IF;
{% endif %}
PERFORM deleteLocationArea(NEW.partition, location.place_id, NEW.rank_search);
SELECT wikipedia, importance
@@ -918,7 +923,7 @@ BEGIN
AND NEW.country_code IS NOT NULL AND NEW.osm_type = 'R'
THEN
PERFORM create_country(NEW.name, lower(NEW.country_code));
--DEBUG: RAISE WARNING 'Country names updated';
{% if debug %}RAISE WARNING 'Country names updated';{% endif %}
-- Also update the list of country names. Adding an additional sanity
-- check here: make sure the country does overlap with the area where
@@ -928,7 +933,7 @@ BEGIN
WHERE ST_Covers(geometry, NEW.centroid) and country_code = NEW.country_code
LIMIT 1
LOOP
--DEBUG: RAISE WARNING 'Updating names for country '%' with: %', NEW.country_code, NEW.name;
{% if debug %}RAISE WARNING 'Updating names for country '%' with: %', NEW.country_code, NEW.name;{% endif %}
UPDATE country_name SET name = name || NEW.name WHERE country_code = NEW.country_code;
END LOOP;
END IF;
@@ -960,7 +965,7 @@ BEGIN
NEW.address, geom, NEW.country_code)
INTO NEW.parent_place_id, NEW.postcode, nameaddress_vector;
--DEBUG: RAISE WARNING 'RETURN insert_addresslines: %, %, %', NEW.parent_place_id, NEW.postcode, nameaddress_vector;
{% if debug %}RAISE WARNING 'RETURN insert_addresslines: %, %, %', NEW.parent_place_id, NEW.postcode, nameaddress_vector;{% endif %}
IF NEW.address is not null AND NEW.address ? 'postcode'
AND NEW.address->'postcode' not similar to '%(,|;)%' THEN
@@ -976,30 +981,30 @@ BEGIN
IF NEW.rank_search <= 25 and NEW.rank_address > 0 THEN
result := add_location(NEW.place_id, NEW.country_code, NEW.partition, name_vector, NEW.rank_search, NEW.rank_address, upper(trim(NEW.address->'postcode')), NEW.geometry, NEW.centroid);
--DEBUG: RAISE WARNING 'added to location (full)';
{% if debug %}RAISE WARNING 'added to location (full)';{% endif %}
END IF;
IF NEW.rank_search between 26 and 27 and NEW.class = 'highway' THEN
result := insertLocationRoad(NEW.partition, NEW.place_id, NEW.country_code, NEW.geometry);
--DEBUG: RAISE WARNING 'insert into road location table (full)';
{% if debug %}RAISE WARNING 'insert into road location table (full)';{% endif %}
END IF;
result := insertSearchName(NEW.partition, NEW.place_id, name_vector,
NEW.rank_search, NEW.rank_address, NEW.geometry);
--DEBUG: RAISE WARNING 'added to search name (full)';
{% if debug %}RAISE WARNING 'added to search name (full)';{% endif %}
IF NOT %REVERSE-ONLY% THEN
{% if not db.reverse_only %}
INSERT INTO search_name (place_id, search_rank, address_rank,
importance, country_code, name_vector,
nameaddress_vector, centroid)
VALUES (NEW.place_id, NEW.rank_search, NEW.rank_address,
NEW.importance, NEW.country_code, name_vector,
nameaddress_vector, NEW.centroid);
END IF;
{% endif %}
END IF;
--DEBUG: RAISE WARNING 'place update % % finsihed.', NEW.osm_type, NEW.osm_id;
{% if debug %}RAISE WARNING 'place update % % finsihed.', NEW.osm_type, NEW.osm_id;{% endif %}
RETURN NEW;
END;
@@ -1018,9 +1023,9 @@ BEGIN
IF OLD.linked_place_id is null THEN
update placex set linked_place_id = null, indexed_status = 2 where linked_place_id = OLD.place_id and indexed_status = 0;
--DEBUG: RAISE WARNING 'placex_delete:01 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:01 % %',OLD.osm_type,OLD.osm_id;{% endif %}
update placex set linked_place_id = null where linked_place_id = OLD.place_id;
--DEBUG: RAISE WARNING 'placex_delete:02 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:02 % %',OLD.osm_type,OLD.osm_id;{% endif %}
ELSE
update placex set indexed_status = 2 where place_id = OLD.linked_place_id and indexed_status = 0;
END IF;
@@ -1028,44 +1033,44 @@ BEGIN
IF OLD.rank_address < 30 THEN
-- mark everything linked to this place for re-indexing
--DEBUG: RAISE WARNING 'placex_delete:03 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:03 % %',OLD.osm_type,OLD.osm_id;{% endif %}
UPDATE placex set indexed_status = 2 from place_addressline where address_place_id = OLD.place_id
and placex.place_id = place_addressline.place_id and indexed_status = 0 and place_addressline.isaddress;
--DEBUG: RAISE WARNING 'placex_delete:04 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:04 % %',OLD.osm_type,OLD.osm_id;{% endif %}
DELETE FROM place_addressline where address_place_id = OLD.place_id;
--DEBUG: RAISE WARNING 'placex_delete:05 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:05 % %',OLD.osm_type,OLD.osm_id;{% endif %}
b := deleteRoad(OLD.partition, OLD.place_id);
--DEBUG: RAISE WARNING 'placex_delete:06 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:06 % %',OLD.osm_type,OLD.osm_id;{% endif %}
update placex set indexed_status = 2 where parent_place_id = OLD.place_id and indexed_status = 0;
--DEBUG: RAISE WARNING 'placex_delete:07 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:07 % %',OLD.osm_type,OLD.osm_id;{% endif %}
-- reparenting also for OSM Interpolation Lines (and for Tiger?)
update location_property_osmline set indexed_status = 2 where indexed_status = 0 and parent_place_id = OLD.place_id;
END IF;
--DEBUG: RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:08 % %',OLD.osm_type,OLD.osm_id;{% endif %}
IF OLD.rank_address < 26 THEN
b := deleteLocationArea(OLD.partition, OLD.place_id, OLD.rank_search);
END IF;
--DEBUG: RAISE WARNING 'placex_delete:09 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:09 % %',OLD.osm_type,OLD.osm_id;{% endif %}
IF OLD.name is not null THEN
IF NOT %REVERSE-ONLY% THEN
{% if 'search_name' in db.tables %}
DELETE from search_name WHERE place_id = OLD.place_id;
END IF;
{% endif %}
b := deleteSearchName(OLD.partition, OLD.place_id);
END IF;
--DEBUG: RAISE WARNING 'placex_delete:10 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:10 % %',OLD.osm_type,OLD.osm_id;{% endif %}
DELETE FROM place_addressline where place_id = OLD.place_id;
--DEBUG: RAISE WARNING 'placex_delete:11 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:11 % %',OLD.osm_type,OLD.osm_id;{% endif %}
-- remove from tables for special search
classtable := 'place_classtype_' || OLD.class || '_' || OLD.type;
@@ -1074,7 +1079,7 @@ BEGIN
EXECUTE 'DELETE FROM ' || classtable::regclass || ' WHERE place_id = $1' USING OLD.place_id;
END IF;
--DEBUG: RAISE WARNING 'placex_delete:12 % %',OLD.osm_type,OLD.osm_id;
{% if debug %}RAISE WARNING 'placex_delete:12 % %',OLD.osm_type,OLD.osm_id;{% endif %}
RETURN OLD;

4
lib-sql/functions/utils.sql
View File

@@ -237,7 +237,7 @@ BEGIN
IF word_ids is not null THEN
parent_place_id := getNearestNamedRoadPlaceId(partition, centroid, word_ids);
IF parent_place_id is not null THEN
--DEBUG: RAISE WARNING 'Get parent form addr:street: %', parent_place_id;
{% if debug %}RAISE WARNING 'Get parent form addr:street: %', parent_place_id;{% endif %}
RETURN parent_place_id;
END IF;
END IF;
@@ -249,7 +249,7 @@ BEGIN
IF word_ids is not null THEN
parent_place_id := getNearestNamedPlacePlaceId(partition, centroid, word_ids);
IF parent_place_id is not null THEN
--DEBUG: RAISE WARNING 'Get parent form addr:place: %', parent_place_id;
{% if debug %}RAISE WARNING 'Get parent form addr:place: %', parent_place_id;{% endif %}
RETURN parent_place_id;
END IF;
END IF;

67
lib-sql/indices.sql Normal file
View File

@@ -0,0 +1,67 @@
-- Indices used only during search and update.
-- These indices are created only after the indexing process is done.
CREATE INDEX {{sql.if_index_not_exists}} idx_word_word_id
ON word USING BTREE (word_id) {{db.tablespace.search_index}};
CREATE INDEX {{sql.if_index_not_exists}} idx_place_addressline_address_place_id
ON place_addressline USING BTREE (address_place_id) {{db.tablespace.search_index}};
CREATE INDEX {{sql.if_index_not_exists}} idx_placex_rank_search
ON placex USING BTREE (rank_search) {{db.tablespace.search_index}};
CREATE INDEX {{sql.if_index_not_exists}} idx_placex_rank_address
ON placex USING BTREE (rank_address) {{db.tablespace.search_index}};
CREATE INDEX {{sql.if_index_not_exists}} idx_placex_parent_place_id
ON placex USING BTREE (parent_place_id) {{db.tablespace.search_index}}
WHERE parent_place_id IS NOT NULL;
CREATE INDEX {{sql.if_index_not_exists}} idx_placex_geometry_reverse_lookupPolygon
ON placex USING gist (geometry) {{db.tablespace.search_index}}
WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
AND rank_address between 4 and 25 AND type != 'postcode'
AND name is not null AND indexed_status = 0 AND linked_place_id is null;
CREATE INDEX {{sql.if_index_not_exists}} idx_placex_geometry_reverse_placeNode
ON placex USING gist (geometry) {{db.tablespace.search_index}}
WHERE osm_type = 'N' AND rank_search between 5 and 25
AND class = 'place' AND type != 'postcode'
AND name is not null AND indexed_status = 0 AND linked_place_id is null;
CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_place_id
ON location_property_osmline USING BTREE (parent_place_id) {{db.tablespace.search_index}};
CREATE INDEX {{sql.if_index_not_exists}} idx_osmline_parent_osm_id
ON location_property_osmline USING BTREE (osm_id) {{db.tablespace.search_index}};
CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_postcode_id
ON location_postcode USING BTREE (place_id) {{db.tablespace.search_index}};
CREATE INDEX {{sql.if_index_not_exists}} idx_postcode_postcode
ON location_postcode USING BTREE (postcode) {{db.tablespace.search_index}};
-- Indices only needed for updating.
{% if not drop %}
CREATE INDEX {{sql.if_index_not_exists}} idx_placex_pendingsector
ON placex USING BTREE (rank_address,geometry_sector) {{db.tablespace.address_index}}
WHERE indexed_status > 0;
CREATE INDEX {{sql.if_index_not_exists}} idx_location_area_country_place_id
ON location_area_country USING BTREE (place_id) {{db.tablespace.address_index}};
CREATE UNIQUE INDEX {{sql.if_index_not_exists}} idx_place_osm_unique
ON place USING btree(osm_id, osm_type, class, type) {{db.tablespace.address_index}};
{% endif %}
-- Indices only needed for search.
{% if 'search_name' in db.tables %}
CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_nameaddress_vector
ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_name_vector
ON search_name USING GIN (name_vector) WITH (fastupdate = off) {{db.tablespace.search_index}};
CREATE INDEX {{sql.if_index_not_exists}} idx_search_name_centroid
ON search_name USING GIST (centroid) {{db.tablespace.search_index}};
{% endif %}

30
lib-sql/indices.src.sql
View File

@@ -1,30 +0,0 @@
-- Indices used only during search and update.
-- These indices are created only after the indexing process is done.
CREATE INDEX idx_word_word_id on word USING BTREE (word_id) {ts:search-index};
CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id) {ts:search-index};
DROP INDEX IF EXISTS idx_placex_rank_search;
CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:search-index};
CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index};
CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL;
CREATE INDEX idx_placex_geometry_reverse_lookupPolygon
ON placex USING gist (geometry) {ts:search-index}
WHERE St_GeometryType(geometry) in ('ST_Polygon', 'ST_MultiPolygon')
AND rank_address between 4 and 25 AND type != 'postcode'
AND name is not null AND indexed_status = 0 AND linked_place_id is null;
CREATE INDEX idx_placex_geometry_reverse_placeNode
ON placex USING gist (geometry) {ts:search-index}
WHERE osm_type = 'N' AND rank_search between 5 and 25
AND class = 'place' AND type != 'postcode'
AND name is not null AND indexed_status = 0 AND linked_place_id is null;
GRANT SELECT ON table country_osm_grid to "{www-user}";
CREATE INDEX idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index};
CREATE INDEX idx_osmline_parent_osm_id ON location_property_osmline USING BTREE (osm_id) {ts:search-index};
CREATE UNIQUE INDEX idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index};
CREATE INDEX idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index};

6
lib-sql/indices_search.src.sql
View File

@@ -1,6 +0,0 @@
-- Indices used for /search API.
-- These indices are created only after the indexing process is done.
CREATE INDEX idx_search_name_nameaddress_vector ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off) {ts:search-index};
CREATE INDEX idx_search_name_name_vector ON search_name USING GIN (name_vector) WITH (fastupdate = off) {ts:search-index};
CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid) {ts:search-index};

9
lib-sql/indices_updates.src.sql
View File

@@ -1,9 +0,0 @@
-- Indices used only during search and update.
-- These indices are created only after the indexing process is done.
CREATE INDEX CONCURRENTLY idx_placex_pendingsector ON placex USING BTREE (rank_address,geometry_sector) {ts:address-index} where indexed_status > 0;
CREATE INDEX CONCURRENTLY idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index};
DROP INDEX CONCURRENTLY IF EXISTS place_id_idx;
CREATE UNIQUE INDEX CONCURRENTLY idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index};

36
lib-sql/partition-tables.src.sql
View File

@@ -7,24 +7,24 @@ CREATE TABLE search_name_blank (
);
-- start
CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large) {ts:address-data};
CREATE INDEX idx_location_area_large_-partition-_place_id ON location_area_large_-partition- USING BTREE (place_id) {ts:address-index};
CREATE INDEX idx_location_area_large_-partition-_geometry ON location_area_large_-partition- USING GIST (geometry) {ts:address-index};
{% for partition in db.partitions %}
CREATE TABLE location_area_large_{{ partition }} () INHERITS (location_area_large) {{db.tablespace.address_data}};
CREATE INDEX idx_location_area_large_{{ partition }}_place_id ON location_area_large_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
CREATE INDEX idx_location_area_large_{{ partition }}_geometry ON location_area_large_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
CREATE TABLE search_name_-partition- () INHERITS (search_name_blank) {ts:address-data};
CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id) {ts:address-index};
CREATE INDEX idx_search_name_-partition-_centroid_street ON search_name_-partition- USING GIST (centroid) {ts:address-index} where address_rank between 26 and 27;
CREATE INDEX idx_search_name_-partition-_centroid_place ON search_name_-partition- USING GIST (centroid) {ts:address-index} where address_rank between 2 and 25;
CREATE TABLE search_name_{{ partition }} () INHERITS (search_name_blank) {{db.tablespace.address_data}};
CREATE INDEX idx_search_name_{{ partition }}_place_id ON search_name_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
CREATE INDEX idx_search_name_{{ partition }}_centroid_street ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 26 and 27;
CREATE INDEX idx_search_name_{{ partition }}_centroid_place ON search_name_{{ partition }} USING GIST (centroid) {{db.tablespace.address_index}} where address_rank between 2 and 25;
DROP TABLE IF EXISTS location_road_-partition-;
CREATE TABLE location_road_-partition- (
place_id BIGINT,
partition SMALLINT,
country_code VARCHAR(2),
geometry GEOMETRY(Geometry, 4326)
) {ts:address-data};
CREATE INDEX idx_location_road_-partition-_geometry ON location_road_-partition- USING GIST (geometry) {ts:address-index};
CREATE INDEX idx_location_road_-partition-_place_id ON location_road_-partition- USING BTREE (place_id) {ts:address-index};
DROP TABLE IF EXISTS location_road_{{ partition }};
CREATE TABLE location_road_{{ partition }} (
place_id BIGINT,
partition SMALLINT,
country_code VARCHAR(2),
geometry GEOMETRY(Geometry, 4326)
) {{db.tablespace.address_data}};
CREATE INDEX idx_location_road_{{ partition }}_geometry ON location_road_{{ partition }} USING GIST (geometry) {{db.tablespace.address_index}};
CREATE INDEX idx_location_road_{{ partition }}_place_id ON location_road_{{ partition }} USING BTREE (place_id) {{db.tablespace.address_index}};
-- end
{% endfor %}

86
lib-sql/tables.sql
View File

@@ -4,7 +4,7 @@ CREATE TABLE import_status (
sequence_id integer,
indexed boolean
);
GRANT SELECT ON import_status TO "{www-user}" ;
GRANT SELECT ON import_status TO "{{config.DATABASE_WEBUSER}}" ;
drop table if exists import_osmosis_log;
CREATE TABLE import_osmosis_log (
@@ -30,18 +30,18 @@ CREATE TABLE new_query_log (
secret text
);
CREATE INDEX idx_new_query_log_starttime ON new_query_log USING BTREE (starttime);
GRANT INSERT ON new_query_log TO "{www-user}" ;
GRANT UPDATE ON new_query_log TO "{www-user}" ;
GRANT SELECT ON new_query_log TO "{www-user}" ;
GRANT INSERT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
GRANT UPDATE ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON new_query_log TO "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON TABLE country_name TO "{www-user}";
GRANT SELECT ON TABLE country_name TO "{{config.DATABASE_WEBUSER}}";
DROP TABLE IF EXISTS nominatim_properties;
CREATE TABLE nominatim_properties (
property TEXT,
value TEXT
);
GRANT SELECT ON TABLE nominatim_properties TO "{www-user}";
GRANT SELECT ON TABLE nominatim_properties TO "{{config.DATABASE_WEBUSER}}";
drop table IF EXISTS word;
CREATE TABLE word (
@@ -53,9 +53,9 @@ CREATE TABLE word (
country_code varchar(2),
search_name_count INTEGER,
operator TEXT
) {ts:search-data};
CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {ts:search-index};
GRANT SELECT ON word TO "{www-user}" ;
) {{db.tablespace.search_data}};
CREATE INDEX idx_word_word_token on word USING BTREE (word_token) {{db.tablespace.search_index}};
GRANT SELECT ON word TO "{{config.DATABASE_WEBUSER}}" ;
DROP SEQUENCE IF EXISTS seq_word;
CREATE SEQUENCE seq_word start 1;
@@ -80,8 +80,8 @@ CREATE TABLE location_area_country (
place_id BIGINT,
country_code varchar(2),
geometry GEOMETRY(Geometry, 4326)
) {ts:address-data};
CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {ts:address-index};
) {{db.tablespace.address_data}};
CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry) {{db.tablespace.address_index}};
drop table IF EXISTS location_property CASCADE;
@@ -98,7 +98,7 @@ CREATE TABLE location_property_aux () INHERITS (location_property);
CREATE INDEX idx_location_property_aux_place_id ON location_property_aux USING BTREE (place_id);
CREATE INDEX idx_location_property_aux_parent_place_id ON location_property_aux USING BTREE (parent_place_id);
CREATE INDEX idx_location_property_aux_housenumber_parent_place_id ON location_property_aux USING BTREE (parent_place_id, housenumber);
GRANT SELECT ON location_property_aux TO "{www-user}";
GRANT SELECT ON location_property_aux TO "{{config.DATABASE_WEBUSER}}";
CREATE TABLE location_property_tiger (
place_id BIGINT,
@@ -109,7 +109,7 @@ CREATE TABLE location_property_tiger (
linegeo GEOMETRY,
interpolationtype TEXT,
postcode TEXT);
GRANT SELECT ON location_property_tiger TO "{www-user}";
GRANT SELECT ON location_property_tiger TO "{{config.DATABASE_WEBUSER}}";
drop table if exists location_property_osmline;
CREATE TABLE location_property_osmline (
@@ -127,13 +127,14 @@ CREATE TABLE location_property_osmline (
address HSTORE,
postcode TEXT,
country_code VARCHAR(2)
){ts:search-data};
CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {ts:search-index};
CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {ts:address-index};
CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {ts:search-index};
GRANT SELECT ON location_property_osmline TO "{www-user}";
){{db.tablespace.search_data}};
CREATE UNIQUE INDEX idx_osmline_place_id ON location_property_osmline USING BTREE (place_id) {{db.tablespace.search_index}};
CREATE INDEX idx_osmline_geometry_sector ON location_property_osmline USING BTREE (geometry_sector) {{db.tablespace.address_index}};
CREATE INDEX idx_osmline_linegeo ON location_property_osmline USING GIST (linegeo) {{db.tablespace.search_index}};
GRANT SELECT ON location_property_osmline TO "{{config.DATABASE_WEBUSER}}";
drop table IF EXISTS search_name;
{% if not db.reverse_only %}
CREATE TABLE search_name (
place_id BIGINT,
importance FLOAT,
@@ -143,8 +144,10 @@ CREATE TABLE search_name (
nameaddress_vector integer[],
country_code varchar(2),
centroid GEOMETRY(Geometry, 4326)
) {ts:search-data};
CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {ts:search-index};
) {{db.tablespace.search_data}};
CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id) {{db.tablespace.search_index}};
GRANT SELECT ON search_name to "{{config.DATABASE_WEBUSER}}" ;
{% endif %}
drop table IF EXISTS place_addressline;
CREATE TABLE place_addressline (
@@ -154,8 +157,8 @@ CREATE TABLE place_addressline (
cached_rank_address SMALLINT,
fromarea boolean,
isaddress boolean
) {ts:search-data};
CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {ts:search-index};
) {{db.tablespace.search_data}};
CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id) {{db.tablespace.search_index}};
drop table if exists placex;
CREATE TABLE placex (
@@ -175,24 +178,23 @@ CREATE TABLE placex (
housenumber TEXT,
postcode TEXT,
centroid GEOMETRY(Geometry, 4326)
) {ts:search-data};
CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {ts:search-index};
CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {ts:search-index};
CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {ts:address-index} WHERE linked_place_id IS NOT NULL;
CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {ts:address-index};
CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {ts:search-index};
CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {ts:address-index} WHERE osm_type='N' and rank_search < 26;
CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {ts:address-index} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26;
) {{db.tablespace.search_data}};
CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id) {{db.tablespace.search_index}};
CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id) {{db.tablespace.search_index}};
CREATE INDEX idx_placex_linked_place_id ON placex USING BTREE (linked_place_id) {{db.tablespace.address_index}} WHERE linked_place_id IS NOT NULL;
CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector) {{db.tablespace.address_index}};
CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry) {{db.tablespace.search_index}};
CREATE INDEX idx_placex_adminname on placex USING BTREE (make_standard_name(name->'name')) {{db.tablespace.address_index}} WHERE osm_type='N' and rank_search < 26;
CREATE INDEX idx_placex_wikidata on placex USING BTREE ((extratags -> 'wikidata')) {{db.tablespace.address_index}} WHERE extratags ? 'wikidata' and class = 'place' and osm_type = 'N' and rank_search < 26;
DROP SEQUENCE IF EXISTS seq_place;
CREATE SEQUENCE seq_place start 1;
GRANT SELECT on placex to "{www-user}" ;
GRANT SELECT ON search_name to "{www-user}" ;
GRANT SELECT on place_addressline to "{www-user}" ;
GRANT SELECT ON seq_word to "{www-user}" ;
GRANT SELECT ON planet_osm_ways to "{www-user}" ;
GRANT SELECT ON planet_osm_rels to "{www-user}" ;
GRANT SELECT on location_area to "{www-user}" ;
GRANT SELECT on placex to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT on place_addressline to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON seq_word to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON planet_osm_ways to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT ON planet_osm_rels to "{{config.DATABASE_WEBUSER}}" ;
GRANT SELECT on location_area to "{{config.DATABASE_WEBUSER}}" ;
-- Table for synthetic postcodes.
DROP TABLE IF EXISTS location_postcode;
@@ -207,8 +209,8 @@ CREATE TABLE location_postcode (
postcode TEXT,
geometry GEOMETRY(Geometry, 4326)
);
CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {ts:address-index};
GRANT SELECT ON location_postcode TO "{www-user}" ;
CREATE INDEX idx_postcode_geometry ON location_postcode USING GIST (geometry) {{db.tablespace.address_index}};
GRANT SELECT ON location_postcode TO "{{config.DATABASE_WEBUSER}}" ;
DROP TABLE IF EXISTS import_polygon_error;
CREATE TABLE import_polygon_error (
@@ -224,7 +226,7 @@ CREATE TABLE import_polygon_error (
newgeometry GEOMETRY(Geometry, 4326)
);
CREATE INDEX idx_import_polygon_error_osmid ON import_polygon_error USING BTREE (osm_type, osm_id);
GRANT SELECT ON import_polygon_error TO "{www-user}";
GRANT SELECT ON import_polygon_error TO "{{config.DATABASE_WEBUSER}}";
DROP TABLE IF EXISTS import_polygon_delete;
CREATE TABLE import_polygon_delete (
@@ -234,7 +236,7 @@ CREATE TABLE import_polygon_delete (
type TEXT NOT NULL
);
CREATE INDEX idx_import_polygon_delete_osmid ON import_polygon_delete USING BTREE (osm_type, osm_id);
GRANT SELECT ON import_polygon_delete TO "{www-user}";
GRANT SELECT ON import_polygon_delete TO "{{config.DATABASE_WEBUSER}}";
DROP SEQUENCE IF EXISTS file;
CREATE SEQUENCE file start 1;
@@ -268,3 +270,5 @@ ALTER TABLE ONLY wikipedia_redirect ADD CONSTRAINT wikipedia_redirect_pkey PRIMA
-- osm2pgsql does not create indexes on the middle tables for Nominatim
-- Add one for lookup of associated street relations.
CREATE INDEX planet_osm_rels_parts_associated_idx ON planet_osm_rels USING gin(parts) WHERE tags @> ARRAY['associatedStreet'];
GRANT SELECT ON table country_osm_grid to "{{config.DATABASE_WEBUSER}}";

34
nominatim/clicmd/setup.py
View File

@@ -79,20 +79,22 @@ class SetupAll:
drop=args.no_updates,
ignore_errors=args.ignore_errors)
LOG.warning('Create functions (1st pass)')
with connect(args.config.get_libpq_dsn()) as conn:
LOG.warning('Create functions (1st pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir,
False, False)
LOG.warning('Create tables')
params = ['setup.php', '--create-tables', '--create-partition-tables']
if args.reverse_only:
params.append('--reverse-only')
run_legacy_script(*params, nominatim_env=args,
throw_on_fail=not args.ignore_errors)
LOG.warning('Create functions (2nd pass)')
with connect(args.config.get_libpq_dsn()) as conn:
LOG.warning('Create tables')
database_import.create_tables(conn, args.config, args.sqllib_dir,
reverse_only=args.reverse_only)
refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
LOG.warning('Create functions (2nd pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir,
False, False)
LOG.warning('Create table triggers')
database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
LOG.warning('Create partition tables')
database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
LOG.warning('Create functions (3rd pass)')
refresh.create_functions(conn, args.config, args.sqllib_dir,
False, False)
@@ -124,10 +126,12 @@ class SetupAll:
indexer.index_full(analyse=not args.index_noanalyse)
LOG.warning('Post-process tables')
params = ['setup.php', '--create-search-indices', '--create-country-names']
if args.no_updates:
params.append('--drop')
run_legacy_script(*params, nominatim_env=args, throw_on_fail=not args.ignore_errors)
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_search_indices(conn, args.config,
args.sqllib_dir,
drop=args.no_updates)
run_legacy_script('setup.php', '--create-country-names',
nominatim_env=args, throw_on_fail=not args.ignore_errors)
webdir = args.project_dir / 'website'
LOG.warning('Setup website at %s', webdir)

28
nominatim/clicmd/transition.py
View File

@@ -35,8 +35,14 @@ class AdminTransition:
help='Import a osm file')
group.add_argument('--load-data', action='store_true',
help='Copy data to live tables from import table')
group.add_argument('--create-tables', action='store_true',
help='Create main tables')
group.add_argument('--create-partition-tables', action='store_true',
help='Create required partition tables')
group.add_argument('--index', action='store_true',
help='Index the data')
group.add_argument('--create-search-indices', action='store_true',
help='Create additional indices required for search and update')
group = parser.add_argument_group('Options')
group.add_argument('--no-partitions', action='store_true',
help='Do not partition search indices')
@@ -50,10 +56,13 @@ class AdminTransition:
help='Do not perform analyse operations during index')
group.add_argument('--ignore-errors', action='store_true',
help="Ignore certain erros on import.")
group.add_argument('--reverse-only', action='store_true',
help='Do not create search tables and indexes')
@staticmethod
def run(args):
from ..tools import database_import
from ..tools import refresh
if args.create_db:
LOG.warning('Create DB')
@@ -80,6 +89,20 @@ class AdminTransition:
drop=args.drop,
ignore_errors=args.ignore_errors)
if args.create_tables:
LOG.warning('Create Tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_tables(conn, args.config, args.sqllib_dir, args.reverse_only)
refresh.load_address_levels_from_file(conn, Path(args.config.ADDRESS_LEVEL_CONFIG))
refresh.create_functions(conn, args.config, args.sqllib_dir,
enable_diff_updates=False)
database_import.create_table_triggers(conn, args.config, args.sqllib_dir)
if args.create_partition_tables:
LOG.warning('Create Partition Tables')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_partition_tables(conn, args.config, args.sqllib_dir)
if args.load_data:
LOG.warning('Load data')
with connect(args.config.get_libpq_dsn()) as conn:
@@ -99,3 +122,8 @@ class AdminTransition:
from ..indexer.indexer import Indexer
indexer = Indexer(args.config.get_libpq_dsn(), args.threads or 1)
indexer.index_full()
if args.create_search_indices:
LOG.warning('Create Search indices')
with connect(args.config.get_libpq_dsn()) as conn:
database_import.create_search_indices(conn, args.config, args.sqllib_dir, args.drop)

94
nominatim/db/sql_preprocessor.py Normal file
View File

@@ -0,0 +1,94 @@
"""
Preprocessing of SQL files.
"""
import jinja2
def _get_partitions(conn):
""" Get the set of partitions currently in use.
"""
with conn.cursor() as cur:
cur.execute('SELECT DISTINCT partition FROM country_name')
partitions = set([0])
for row in cur:
partitions.add(row[0])
return partitions
def _get_tables(conn):
""" Return the set of tables currently in use.
Only includes non-partitioned
"""
with conn.cursor() as cur:
cur.execute("SELECT tablename FROM pg_tables WHERE schemaname = 'public'")
return set((row[0] for row in list(cur)))
def _setup_tablespace_sql(config):
""" Returns a dict with tablespace expressions for the different tablespace
kinds depending on whether a tablespace is configured or not.
"""
out = {}
for subset in ('ADDRESS', 'SEARCH', 'AUX'):
for kind in ('DATA', 'INDEX'):
tspace = getattr(config, 'TABLESPACE_{}_{}'.format(subset, kind))
if tspace:
tspace = 'TABLESPACE "{}"'.format(tspace)
out['{}_{}'.format(subset.lower, kind.lower())] = tspace
return out
def _setup_postgres_sql(conn):
""" Set up a dictionary with various Postgresql/Postgis SQL terms which
are dependent on the database version in use.
"""
out = {}
pg_version = conn.server_version_tuple()
# CREATE INDEX IF NOT EXISTS was introduced in PG9.5.
# Note that you need to ignore failures on older versions when
# unsing this construct.
out['if_index_not_exists'] = ' IF NOT EXISTS ' if pg_version >= (9, 5, 0) else ''
return out
class SQLPreprocessor: # pylint: disable=too-few-public-methods
""" A environment for preprocessing SQL files from the
lib-sql directory.
The preprocessor provides a number of default filters and variables.
The variables may be overwritten when rendering an SQL file.
The preprocessing is currently based on the jinja2 templating library
and follows its syntax.
"""
def __init__(self, conn, config, sqllib_dir):
self.env = jinja2.Environment(autoescape=False,
loader=jinja2.FileSystemLoader(str(sqllib_dir)))
db_info = {}
db_info['partitions'] = _get_partitions(conn)
db_info['tables'] = _get_tables(conn)
db_info['reverse_only'] = 'search_name' not in db_info['tables']
db_info['tablespace'] = _setup_tablespace_sql(config)
self.env.globals['config'] = config
self.env.globals['db'] = db_info
self.env.globals['sql'] = _setup_postgres_sql(conn)
self.env.globals['modulepath'] = config.DATABASE_MODULE_PATH or \
str((config.project_dir / 'module').resolve())
def run_sql_file(self, conn, name, **kwargs):
""" Execute the given SQL file on the connection. The keyword arguments
may supply additional parameters for preprocessing.
"""
sql = self.env.get_template(name).render(**kwargs)
with conn.cursor() as cur:
cur.execute(sql)
conn.commit()

48
nominatim/tools/database_import.py
View File

@@ -14,6 +14,7 @@ import psycopg2
from ..db.connection import connect, get_pg_env
from ..db import utils as db_utils
from ..db.async_connection import DBConnection
from ..db.sql_preprocessor import SQLPreprocessor
from .exec_utils import run_osm2pgsql
from ..errors import UsageError
from ..version import POSTGRESQL_REQUIRED_VERSION, POSTGIS_REQUIRED_VERSION
@@ -178,6 +179,32 @@ def import_osm_data(osm_file, options, drop=False, ignore_errors=False):
Path(options['flatnode_file']).unlink()
def create_tables(conn, config, sqllib_dir, reverse_only=False):
""" Create the set of basic tables.
When `reverse_only` is True, then the main table for searching will
be skipped and only reverse search is possible.
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.env.globals['db']['reverse_only'] = reverse_only
sql.run_sql_file(conn, 'tables.sql')
def create_table_triggers(conn, config, sqllib_dir):
""" Create the triggers for the tables. The trigger functions must already
have been imported with refresh.create_functions().
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'table-triggers.sql')
def create_partition_tables(conn, config, sqllib_dir):
""" Create tables that have explicit partitioning.
"""
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'partition-tables.src.sql')
def truncate_data_tables(conn, max_word_frequency=None):
""" Truncate all data tables to prepare for a fresh load.
"""
@@ -258,3 +285,24 @@ def load_data(dsn, data_dir, threads):
with connect(dsn) as conn:
with conn.cursor() as cur:
cur.execute('ANALYSE')
def create_search_indices(conn, config, sqllib_dir, drop=False):
""" Create tables that have explicit partitioning.
"""
# If index creation failed and left an index invalid, they need to be
# cleaned out first, so that the script recreates them.
with conn.cursor() as cur:
cur.execute("""SELECT relname FROM pg_class, pg_index
WHERE pg_index.indisvalid = false
AND pg_index.indexrelid = pg_class.oid""")
bad_indices = [row[0] for row in list(cur)]
for idx in bad_indices:
LOG.info("Drop invalid index %s.", idx)
cur.execute('DROP INDEX "{}"'.format(idx))
conn.commit()
sql = SQLPreprocessor(conn, config, sqllib_dir)
sql.run_sql_file(conn, 'indices.sql', drop=drop)

95
nominatim/tools/refresh.py
View File

@@ -3,12 +3,12 @@ Functions for bringing auxiliary data in the database up-to-date.
"""
import json
import logging
import re
from textwrap import dedent
from psycopg2.extras import execute_values
from ..db.utils import execute_file
from ..db.sql_preprocessor import SQLPreprocessor
from ..version import NOMINATIM_VERSION
LOG = logging.getLogger()
@@ -76,100 +76,17 @@ def load_address_levels_from_file(conn, config_file):
with config_file.open('r') as fdesc:
load_address_levels(conn, 'address_levels', json.load(fdesc))
PLPGSQL_BASE_MODULES = (
'utils.sql',
'normalization.sql',
'ranking.sql',
'importance.sql',
'address_lookup.sql',
'interpolation.sql'
)
PLPGSQL_TABLE_MODULES = (
('place', 'place_triggers.sql'),
('placex', 'placex_triggers.sql'),
('location_postcode', 'postcode_triggers.sql')
)
def _get_standard_function_sql(conn, config, sql_dir, enable_diff_updates, enable_debug):
""" Read all applicable SQLs containing PL/pgSQL functions, replace
placefolders and execute them.
"""
sql_func_dir = sql_dir / 'functions'
sql = ''
# Get the basic set of functions that is always imported.
for sql_file in PLPGSQL_BASE_MODULES:
with (sql_func_dir / sql_file).open('r') as fdesc:
sql += fdesc.read()
# Some files require the presence of a certain table
for table, fname in PLPGSQL_TABLE_MODULES:
if conn.table_exists(table):
with (sql_func_dir / fname).open('r') as fdesc:
sql += fdesc.read()
# Replace placeholders.
sql = sql.replace('{modulepath}',
config.DATABASE_MODULE_PATH or str((config.project_dir / 'module').resolve()))
if enable_diff_updates:
sql = sql.replace('RETURN NEW; -- %DIFFUPDATES%', '--')
if enable_debug:
sql = sql.replace('--DEBUG:', '')
if config.get_bool('LIMIT_REINDEXING'):
sql = sql.replace('--LIMIT INDEXING:', '')
if not config.get_bool('USE_US_TIGER_DATA'):
sql = sql.replace('-- %NOTIGERDATA% ', '')
if not config.get_bool('USE_AUX_LOCATION_DATA'):
sql = sql.replace('-- %NOAUXDATA% ', '')
reverse_only = 'false' if conn.table_exists('search_name') else 'true'
return sql.replace('%REVERSE-ONLY%', reverse_only)
def replace_partition_string(sql, partitions):
""" Replace a partition template with the actual partition code.
"""
for match in re.findall('^-- start(.*?)^-- end', sql, re.M | re.S):
repl = ''
for part in partitions:
repl += match.replace('-partition-', str(part))
sql = sql.replace(match, repl)
return sql
def _get_partition_function_sql(conn, sql_dir):
""" Create functions that work on partition tables.
"""
with conn.cursor() as cur:
cur.execute('SELECT distinct partition FROM country_name')
partitions = set([0])
for row in cur:
partitions.add(row[0])
with (sql_dir / 'partition-functions.src.sql').open('r') as fdesc:
sql = fdesc.read()
return replace_partition_string(sql, sorted(partitions))
def create_functions(conn, config, sql_dir,
def create_functions(conn, config, sqllib_dir,
enable_diff_updates=True, enable_debug=False):
""" (Re)create the PL/pgSQL functions.
"""
sql = _get_standard_function_sql(conn, config, sql_dir,
enable_diff_updates, enable_debug)
sql += _get_partition_function_sql(conn, sql_dir)
sql = SQLPreprocessor(conn, config, sqllib_dir)
with conn.cursor() as cur:
cur.execute(sql)
sql.run_sql_file(conn, 'functions.sql',
disable_diff_update=not enable_diff_updates,
debug=enable_debug)
conn.commit()
WEBSITE_SCRIPTS = (

5
test/python/test_cli.py
View File

@@ -92,6 +92,11 @@ def test_import_full(temp_db, mock_func_factory):
mock_func_factory(nominatim.tools.refresh, 'import_wikipedia_articles'),
mock_func_factory(nominatim.tools.database_import, 'truncate_data_tables'),
mock_func_factory(nominatim.tools.database_import, 'load_data'),
mock_func_factory(nominatim.tools.database_import, 'create_tables'),
mock_func_factory(nominatim.tools.database_import, 'create_table_triggers'),
mock_func_factory(nominatim.tools.database_import, 'create_partition_tables'),
mock_func_factory(nominatim.tools.database_import, 'create_search_indices'),
mock_func_factory(nominatim.tools.refresh, 'load_address_levels_from_file'),
mock_func_factory(nominatim.indexer.indexer.Indexer, 'index_full'),
mock_func_factory(nominatim.tools.refresh, 'setup_website'),
mock_func_factory(nominatim.db.properties, 'set_property')

51
test/python/test_db_sql_preprocessor.py Normal file
View File

@@ -0,0 +1,51 @@
"""
Tests for SQL preprocessing.
"""
from pathlib import Path
import pytest
from nominatim.db.sql_preprocessor import SQLPreprocessor
@pytest.fixture
def sql_factory(tmp_path):
def _mk_sql(sql_body):
(tmp_path / 'test.sql').write_text("""
CREATE OR REPLACE FUNCTION test() RETURNS TEXT
AS $$
BEGIN
{}
END;
$$ LANGUAGE plpgsql IMMUTABLE;""".format(sql_body))
return 'test.sql'
return _mk_sql
@pytest.fixture
def sql_preprocessor(temp_db_conn, tmp_path, def_config, monkeypatch, table_factory):
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
table_factory('country_name', 'partition INT', (0, 1, 2))
return SQLPreprocessor(temp_db_conn, def_config, tmp_path)
@pytest.mark.parametrize("expr,ret", [
("'a'", 'a'),
("'{{db.partitions|join}}'", '012'),
("{% if 'country_name' in db.tables %}'yes'{% else %}'no'{% endif %}", "yes"),
("{% if 'xxx' in db.tables %}'yes'{% else %}'no'{% endif %}", "no"),
("'{{config.DATABASE_MODULE_PATH}}'", '.')
])
def test_load_file_simple(sql_preprocessor, sql_factory, temp_db_conn, temp_db_cursor, expr, ret):
sqlfile = sql_factory("RETURN {};".format(expr))
sql_preprocessor.run_sql_file(temp_db_conn, sqlfile)
assert temp_db_cursor.scalar('SELECT test()') == ret
def test_load_file_with_params(sql_preprocessor, sql_factory, temp_db_conn, temp_db_cursor):
sqlfile = sql_factory("RETURN '{{ foo }} {{ bar }}';")
sql_preprocessor.run_sql_file(temp_db_conn, sqlfile, bar='XX', foo='ZZ')
assert temp_db_cursor.scalar('SELECT test()') == 'ZZ XX'

117
test/python/test_tools_refresh_create_functions.py
View File

@@ -1,98 +1,47 @@
"""
Tests for creating PL/pgSQL functions for Nominatim.
"""
from pathlib import Path
import pytest
from nominatim.db.connection import connect
from nominatim.tools.refresh import _get_standard_function_sql, _get_partition_function_sql
SQL_DIR = (Path(__file__) / '..' / '..' / '..' / 'lib-sql').resolve()
from nominatim.tools.refresh import create_functions
@pytest.fixture
def db(temp_db):
with connect('dbname=' + temp_db) as conn:
yield conn
@pytest.fixture
def db_with_tables(db):
with db.cursor() as cur:
for table in ('place', 'placex', 'location_postcode'):
cur.execute('CREATE TABLE {} (place_id BIGINT)'.format(table))
return db
def conn(temp_db_conn, table_factory, monkeypatch):
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', '.')
table_factory('country_name', 'partition INT', (0, 1, 2))
return temp_db_conn
def test_standard_functions_replace_module_default(db, def_config):
def_config.project_dir = Path('.')
sql = _get_standard_function_sql(db, def_config, SQL_DIR, False, False)
def test_create_functions(temp_db_cursor, conn, def_config, tmp_path):
sqlfile = tmp_path / 'functions.sql'
sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER
AS $$
BEGIN
RETURN 43;
END;
$$ LANGUAGE plpgsql IMMUTABLE;
""")
assert sql
assert sql.find('{modulepath}') < 0
assert sql.find("'{}'".format(Path('module/nominatim.so').resolve())) >= 0
create_functions(conn, def_config, tmp_path)
assert temp_db_cursor.scalar('SELECT test()') == 43
def test_standard_functions_replace_module_custom(monkeypatch, db, def_config):
monkeypatch.setenv('NOMINATIM_DATABASE_MODULE_PATH', 'custom')
sql = _get_standard_function_sql(db, def_config, SQL_DIR, False, False)
@pytest.mark.parametrize("dbg,ret", ((True, 43), (False, 22)))
def test_create_functions_with_template(temp_db_cursor, conn, def_config, tmp_path, dbg, ret):
sqlfile = tmp_path / 'functions.sql'
sqlfile.write_text("""CREATE OR REPLACE FUNCTION test() RETURNS INTEGER
AS $$
BEGIN
{% if debug %}
RETURN 43;
{% else %}
RETURN 22;
{% endif %}
END;
$$ LANGUAGE plpgsql IMMUTABLE;
""")
assert sql
assert sql.find('{modulepath}') < 0
assert sql.find("'custom/nominatim.so'") >= 0
create_functions(conn, def_config, tmp_path, enable_debug=dbg)
@pytest.mark.parametrize("enabled", (True, False))
def test_standard_functions_enable_diff(db_with_tables, def_config, enabled):
def_config.project_dir = Path('.')
sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, enabled, False)
assert sql
assert (sql.find('%DIFFUPDATES%') < 0) == enabled
@pytest.mark.parametrize("enabled", (True, False))
def test_standard_functions_enable_debug(db_with_tables, def_config, enabled):
def_config.project_dir = Path('.')
sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, enabled)
assert sql
assert (sql.find('--DEBUG') < 0) == enabled
@pytest.mark.parametrize("enabled", (True, False))
def test_standard_functions_enable_limit_reindexing(monkeypatch, db_with_tables, def_config, enabled):
def_config.project_dir = Path('.')
monkeypatch.setenv('NOMINATIM_LIMIT_REINDEXING', 'yes' if enabled else 'no')
sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
assert sql
assert (sql.find('--LIMIT INDEXING') < 0) == enabled
@pytest.mark.parametrize("enabled", (True, False))
def test_standard_functions_enable_tiger(monkeypatch, db_with_tables, def_config, enabled):
def_config.project_dir = Path('.')
monkeypatch.setenv('NOMINATIM_USE_US_TIGER_DATA', 'yes' if enabled else 'no')
sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
assert sql
assert (sql.find('%NOTIGERDATA%') >= 0) == enabled
@pytest.mark.parametrize("enabled", (True, False))
def test_standard_functions_enable_aux(monkeypatch, db_with_tables, def_config, enabled):
def_config.project_dir = Path('.')
monkeypatch.setenv('NOMINATIM_USE_AUX_LOCATION_DATA', 'yes' if enabled else 'no')
sql = _get_standard_function_sql(db_with_tables, def_config, SQL_DIR, False, False)
assert sql
assert (sql.find('%NOAUXDATA%') >= 0) == enabled
def test_partition_function(temp_db_cursor, db, def_config):
temp_db_cursor.execute("CREATE TABLE country_name (partition SMALLINT)")
sql = _get_partition_function_sql(db, SQL_DIR)
assert sql
assert sql.find('-partition-') < 0
assert temp_db_cursor.scalar('SELECT test()') == ret

2
vagrant/Install-on-Centos-7.sh
View File

@@ -42,7 +42,7 @@
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel
pip3 install --user psycopg2 python-dotenv psutil
pip3 install --user psycopg2 python-dotenv psutil Jinja2
#

2
vagrant/Install-on-Centos-8.sh
View File

@@ -35,7 +35,7 @@
python3-pip python3-setuptools python3-devel \
expat-devel zlib-devel
pip3 install --user psycopg2 python-dotenv psutil
pip3 install --user psycopg2 python-dotenv psutil Jinja2
#

2
vagrant/Install-on-Ubuntu-18.sh
View File

@@ -30,7 +30,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-server-dev-10 postgresql-10-postgis-2.4 \
postgresql-contrib-10 postgresql-10-postgis-scripts \
php php-pgsql php-intl python3-pip \
python3-psycopg2 python3-psutil git
python3-psycopg2 python3-psutil python3-jinja2 git
# The python-dotenv package that comes with Ubuntu 18.04 is too old, so
# install the latest version from pip:

2
vagrant/Install-on-Ubuntu-20.sh
View File

@@ -33,7 +33,7 @@ export DEBIAN_FRONTEND=noninteractive #DOCS:
postgresql-server-dev-12 postgresql-12-postgis-3 \
postgresql-contrib-12 postgresql-12-postgis-3-scripts \
php php-pgsql php-intl python3-dotenv \
python3-psycopg2 python3-psutil git
python3-psycopg2 python3-psutil python3-jinja2 git
#
# System Configuration