hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-26 11:08:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

during initial setup defer creation of non-vital until after nominatim indexing,

Browse Source 
analyse DB during nominatim indexing so the query planer uses indexes correctly
This commit is contained in:
Sarah Hoffmann
2012-02-10 21:01:35 +00:00
parent 7239687e5b
commit 8c7bd6beab
4 changed files with 67 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
sql/indices.src.sql Normal file
View File

@@ -0,0 +1,28 @@
CREATE INDEX idx_word_word_id on word USING BTREE (word_id);
CREATE INDEX search_name_name_vector_idx ON search_name USING GIN (name_vector) WITH (fastupdate = off);
CREATE INDEX searchnameplacesearch_search_nameaddress_vector_idx ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off);
CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid);
CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id);
CREATE INDEX idx_place_boundingbox_place_id on place_boundingbox USING BTREE (place_id);
CREATE INDEX idx_place_boundingbox_outline ON place_boundingbox USING GIST (outline);
DROP INDEX IF EXISTS idx_placex_rank_search;
CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search);
CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address);
CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) where indexed_status > 0;
CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) where parent_place_id IS NOT NULL;
CREATE INDEX idx_placex_interpolation ON placex USING BTREE (geometry_sector) where indexed_status > 0 and class='place' and type='houses';
CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_address,osm_type,osm_id);
CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id);
CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid);
CREATE INDEX idx_search_name_country_nameaddress_vector ON search_name_country USING GIN (nameaddress_vector) WITH (fastupdate = off);
-- start
CREATE INDEX idx_search_name_-partition-_nameaddress_vector ON search_name_-partition- USING GIN (nameaddress_vector) WITH (fastupdate = off);
CREATE INDEX idx_location_property_-partition-_centroid ON location_property_-partition- USING GIST (centroid);
-- end

5
sql/partitions.src.sql
View File

@@ -14,14 +14,11 @@ create type nearfeature as (
); );
CREATE TABLE location_area_country () INHERITS (location_area_large); CREATE TABLE location_area_country () INHERITS (location_area_large);
CREATE INDEX idx_location_area_country_place_id ON location_area_country USING BTREE (place_id);
CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry); CREATE INDEX idx_location_area_country_geometry ON location_area_country USING GIST (geometry);
CREATE TABLE search_name_country () INHERITS (search_name_blank); CREATE TABLE search_name_country () INHERITS (search_name_blank);
CREATE INDEX idx_search_name_country_place_id ON search_name_country USING BTREE (place_id); CREATE INDEX idx_search_name_country_place_id ON search_name_country USING BTREE (place_id);
CREATE INDEX idx_search_name_country_centroid ON search_name_country USING GIST (centroid);
CREATE INDEX idx_search_name_country_name_vector ON search_name_country USING GIN (name_vector) WITH (fastupdate = off); CREATE INDEX idx_search_name_country_name_vector ON search_name_country USING GIN (name_vector) WITH (fastupdate = off);
CREATE INDEX idx_search_name_country_nameaddress_vector ON search_name_country USING GIN (nameaddress_vector) WITH (fastupdate = off);
-- start -- start
CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large); CREATE TABLE location_area_large_-partition- () INHERITS (location_area_large);
@@ -32,13 +29,11 @@ CREATE TABLE search_name_-partition- () INHERITS (search_name_blank);
CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id); CREATE INDEX idx_search_name_-partition-_place_id ON search_name_-partition- USING BTREE (place_id);
CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid); CREATE INDEX idx_search_name_-partition-_centroid ON search_name_-partition- USING GIST (centroid);
CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off); CREATE INDEX idx_search_name_-partition-_name_vector ON search_name_-partition- USING GIN (name_vector) WITH (fastupdate = off);
CREATE INDEX idx_search_name_-partition-_nameaddress_vector ON search_name_-partition- USING GIN (nameaddress_vector) WITH (fastupdate = off);
CREATE TABLE location_property_-partition- () INHERITS (location_property); CREATE TABLE location_property_-partition- () INHERITS (location_property);
CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id); CREATE INDEX idx_location_property_-partition-_place_id ON location_property_-partition- USING BTREE (place_id);
CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id); CREATE INDEX idx_location_property_-partition-_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id);
CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber); CREATE INDEX idx_location_property_-partition-_housenumber_parent_place_id ON location_property_-partition- USING BTREE (parent_place_id, housenumber);
--CREATE INDEX idx_location_property_-partition-_centroid ON location_property_-partition- USING GIST (centroid);
CREATE TABLE location_road_-partition- ( CREATE TABLE location_road_-partition- (
partition integer, partition integer,

19
sql/tables.sql
View File

@@ -78,7 +78,6 @@ CREATE TABLE word (
operator TEXT operator TEXT
); );
SELECT AddGeometryColumn('word', 'location', 4326, 'GEOMETRY', 2); SELECT AddGeometryColumn('word', 'location', 4326, 'GEOMETRY', 2);
CREATE INDEX idx_word_word_id on word USING BTREE (word_id);
CREATE INDEX idx_word_word_token on word USING BTREE (word_token); CREATE INDEX idx_word_word_token on word USING BTREE (word_token);
--CREATE INDEX idx_word_trigram ON word USING gin(word_trigram gin_trgm_ops) WITH (fastupdate = off); --CREATE INDEX idx_word_trigram ON word USING gin(word_trigram gin_trgm_ops) WITH (fastupdate = off);
GRANT SELECT ON word TO "www-data" ; GRANT SELECT ON word TO "www-data" ;
@@ -138,9 +137,6 @@ SELECT AddGeometryColumn('search_name_blank', 'centroid', 4326, 'GEOMETRY', 2);
drop table IF EXISTS search_name; drop table IF EXISTS search_name;
CREATE TABLE search_name () INHERITS (search_name_blank); CREATE TABLE search_name () INHERITS (search_name_blank);
CREATE INDEX search_name_name_vector_idx ON search_name USING GIN (name_vector) WITH (fastupdate = off);
CREATE INDEX searchnameplacesearch_search_nameaddress_vector_idx ON search_name USING GIN (nameaddress_vector) WITH (fastupdate = off);
CREATE INDEX idx_search_name_centroid ON search_name USING GIST (centroid);
CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id); CREATE INDEX idx_search_name_place_id ON search_name USING BTREE (place_id);
drop table IF EXISTS place_addressline; drop table IF EXISTS place_addressline;
@@ -153,7 +149,6 @@ CREATE TABLE place_addressline (
cached_rank_address integer cached_rank_address integer
); );
CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id); CREATE INDEX idx_place_addressline_place_id on place_addressline USING BTREE (place_id);
CREATE INDEX idx_place_addressline_address_place_id on place_addressline USING BTREE (address_place_id);
drop table IF EXISTS place_boundingbox CASCADE; drop table IF EXISTS place_boundingbox CASCADE;
CREATE TABLE place_boundingbox ( CREATE TABLE place_boundingbox (
@@ -165,9 +160,7 @@ CREATE TABLE place_boundingbox (
numfeatures integer, numfeatures integer,
area float area float
); );
CREATE INDEX idx_place_boundingbox_place_id on place_boundingbox USING BTREE (place_id);
SELECT AddGeometryColumn('place_boundingbox', 'outline', 4326, 'GEOMETRY', 2); SELECT AddGeometryColumn('place_boundingbox', 'outline', 4326, 'GEOMETRY', 2);
CREATE INDEX idx_place_boundingbox_outline ON place_boundingbox USING GIST (outline);
GRANT SELECT on place_boundingbox to "www-data" ; GRANT SELECT on place_boundingbox to "www-data" ;
GRANT INSERT on place_boundingbox to "www-data" ; GRANT INSERT on place_boundingbox to "www-data" ;
@@ -221,23 +214,13 @@ CREATE TABLE placex (
SELECT AddGeometryColumn('placex', 'geometry', 4326, 'GEOMETRY', 2); SELECT AddGeometryColumn('placex', 'geometry', 4326, 'GEOMETRY', 2);
CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id); CREATE UNIQUE INDEX idx_place_id ON placex USING BTREE (place_id);
CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id); CREATE INDEX idx_placex_osmid ON placex USING BTREE (osm_type, osm_id);
CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search); CREATE INDEX idx_placex_rank_search ON placex USING BTREE (rank_search, geometry_sector);
CREATE INDEX idx_placex_rank_address ON placex USING BTREE (rank_address);
CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry); CREATE INDEX idx_placex_geometry ON placex USING GIST (geometry);
--CREATE INDEX idx_placex_indexed ON placex USING BTREE (indexed); --CREATE INDEX idx_placex_indexed ON placex USING BTREE (indexed);
CREATE INDEX idx_placex_pending ON placex USING BTREE (rank_search) where indexed_status > 0;
CREATE INDEX idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) where indexed_status > 0;
--CREATE INDEX idx_placex_pendingbylatlon ON placex USING BTREE (geometry_index(geometry_sector,indexed,name),rank_search) where geometry_index(geometry_sector,indexed,name) IS NOT NULL; --CREATE INDEX idx_placex_pendingbylatlon ON placex USING BTREE (geometry_index(geometry_sector,indexed,name),rank_search) where geometry_index(geometry_sector,indexed,name) IS NOT NULL;
CREATE INDEX idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) where parent_place_id IS NOT NULL;
CREATE INDEX idx_placex_interpolation ON placex USING BTREE (geometry_sector) where indexed_status > 0 and class='place' and type='houses';
CREATE INDEX idx_placex_sector ON placex USING BTREE (geometry_sector,rank_address,osm_type,osm_id);
CLUSTER placex USING idx_placex_sector;
DROP SEQUENCE seq_place; DROP SEQUENCE seq_place;
CREATE SEQUENCE seq_place start 1; CREATE SEQUENCE seq_place start 1;
GRANT SELECT on placex to "www-data" ; GRANT SELECT on placex to "www-data" ;

41
utils/setup.php
View File

@@ -30,6 +30,7 @@
array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'), array('osmosis-init-date', '', 0, 1, 1, 1, 'string', 'Generate default osmosis configuration'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'), array('index-output', '', 0, 1, 1, 1, 'string', 'File to dump index information to'),
array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'), array('create-website', '', 0, 1, 1, 1, 'realpath', 'Create symlinks to setup web directory'),
); );
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
@@ -107,7 +108,6 @@
$osm2pgsql = CONST_Osm2pgsql_Binary; $osm2pgsql = CONST_Osm2pgsql_Binary;
if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql"); if (!file_exists($osm2pgsql)) fail("please download and build osm2pgsql");
passthru($osm2pgsql.' -lsc -O gazetteer -C 12000 --hstore -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']); passthru($osm2pgsql.' -lsc -O gazetteer -C 12000 --hstore -d '.$aDSNInfo['database'].' '.$aCMDResult['osm-file']);
pgsqlRunScript('ANALYSE');
$oDB =& getDB(); $oDB =& getDB();
$x = $oDB->getRow('select * from place limit 1'); $x = $oDB->getRow('select * from place limit 1');
@@ -241,6 +241,8 @@
echo '.'; echo '.';
} }
echo "\n"; echo "\n";
echo "Reanalysing database...\n";
pgsqlRunScript('ANALYSE');
} }
if ($aCMDResult['create-roads']) if ($aCMDResult['create-roads'])
@@ -344,7 +346,7 @@
if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection)); if (!pg_query($oDB->connection, $sSQL)) fail(pg_last_error($oDB->connection));
} }
if (($aCMDResult['osmosis-init'] || $aCMDResult['all']) && isset($aCMDResult['osmosis-init-date'])) if ($aCMDResult['osmosis-init'] && isset($aCMDResult['osmosis-init-date']))
{ {
$bDidSomething = true; $bDidSomething = true;
@@ -365,7 +367,40 @@
$bDidSomething = true; $bDidSomething = true;
$sOutputFile = ''; $sOutputFile = '';
if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output']; if (isset($aCMDResult['index-output'])) $sOutputFile = ' -F '.$aCMDResult['index-output'];
passthru(CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile); $sBaseCmd = CONST_BasePath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -t '.$iInstances.$sOutputFile;
passthru($sBaseCmd.' -R 4');
pgsqlRunScript('ANALYSE');
passthru($sBaseCmd.' -r 5 -R 25');
pgsqlRunScript('ANALYSE');
passthru($sBaseCmd.' -r 26');
}
if ($aCMDResult['create-search-indices'] || $aCMDResult['all'])
{
echo "Search indices\n";
$bDidSomething = true;
$oDB =& getDB();
$sSQL = 'select partition from country_name order by country_code';
$aPartitions = $oDB->getCol($sSQL);
if (PEAR::isError($aPartitions))
{
fail($aPartitions->getMessage());
}
$aPartitions[] = 0;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach($aMatches as $aMatch)
{
$sResult = '';
foreach($aPartitions as $sPartitionName)
{
$sResult .= str_replace('-partition-', $sPartitionName, $aMatch[1]);
}
$sTemplate = str_replace($aMatch[0], $sResult, $sTemplate);
}
pgsqlRunScript($sTemplate);
} }
if (isset($aCMDResult['create-website'])) if (isset($aCMDResult['create-website']))