From b9171dd10b39fe66a2bd912fe77104f7565fdbe7 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 12 Feb 2020 11:03:20 +0100 Subject: [PATCH 1/2] clean up intermediate tables earlier with --drop When --drop is given, we can remove all node geometry information already after the import with osm2pgsql. Also drop all unnecessary tables before creating the final indices. --- lib/setup/SetupClass.php | 16 ++++++++++++++++ sql/indices.src.sql | 7 ------- sql/indices_updates.src.sql | 9 +++++++++ utils/setup.php | 10 +++++----- 4 files changed, 30 insertions(+), 12 deletions(-) create mode 100644 sql/indices_updates.src.sql diff --git a/lib/setup/SetupClass.php b/lib/setup/SetupClass.php index 100e3847..71da57cf 100755 --- a/lib/setup/SetupClass.php +++ b/lib/setup/SetupClass.php @@ -16,6 +16,7 @@ class SetupFunctions protected $bEnableDiffUpdates; protected $bEnableDebugStatements; protected $bNoPartitions; + protected $bDrop; protected $oDB = null; public function __construct(array $aCMDResult) @@ -74,6 +75,8 @@ class SetupFunctions } else { $this->bEnableDiffUpdates = false; } + + $this->bDrop = $aCMDResult['drop']; } public function createDB() @@ -208,6 +211,11 @@ class SetupFunctions if (!$this->sIgnoreErrors && !$this->oDB->getRow('select * from place limit 1')) { fail('No Data'); } + + if ($this->bDrop) { + $this->dropTable('planet_osm_nodes'); + $this->removeFlatnodeFile(); + } } public function createFunctions() @@ -576,6 +584,9 @@ class SetupFunctions } $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); + if (!$this->bDrop) { + $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_updates.src.sql'); + } if (!$this->dbReverseOnly()) { $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql'); } @@ -655,6 +666,11 @@ class SetupFunctions $this->dropTable($sDrop); } + $this->removeFlatnodeFile(); + } + + private function removeFlatnodeFile() + { if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { if (file_exists(CONST_Osm2pgsql_Flatnode_File)) { if ($this->bVerbose) echo 'Deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; diff --git a/sql/indices.src.sql b/sql/indices.src.sql index a5aae32b..63d4677e 100644 --- a/sql/indices.src.sql +++ b/sql/indices.src.sql @@ -8,7 +8,6 @@ CREATE INDEX CONCURRENTLY idx_place_addressline_address_place_id on place_addres DROP INDEX CONCURRENTLY IF EXISTS idx_placex_rank_search; CREATE INDEX CONCURRENTLY idx_placex_rank_search ON placex USING BTREE (rank_search) {ts:search-index}; CREATE INDEX CONCURRENTLY idx_placex_rank_address ON placex USING BTREE (rank_address) {ts:search-index}; -CREATE INDEX CONCURRENTLY idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) {ts:address-index} where indexed_status > 0; CREATE INDEX CONCURRENTLY idx_placex_parent_place_id ON placex USING BTREE (parent_place_id) {ts:search-index} where parent_place_id IS NOT NULL; CREATE INDEX CONCURRENTLY idx_placex_geometry_reverse_lookupPoint @@ -29,14 +28,8 @@ CREATE INDEX CONCURRENTLY idx_placex_geometry_reverse_placeNode GRANT SELECT ON table country_osm_grid to "{www-user}"; -CREATE INDEX CONCURRENTLY idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index}; - CREATE INDEX CONCURRENTLY idx_osmline_parent_place_id ON location_property_osmline USING BTREE (parent_place_id) {ts:search-index}; CREATE INDEX CONCURRENTLY idx_osmline_parent_osm_id ON location_property_osmline USING BTREE (osm_id) {ts:search-index}; -DROP INDEX CONCURRENTLY IF EXISTS place_id_idx; -CREATE UNIQUE INDEX CONCURRENTLY idx_place_osm_unique on place using btree(osm_id,osm_type,class,type) {ts:address-index}; - - CREATE UNIQUE INDEX CONCURRENTLY idx_postcode_id ON location_postcode USING BTREE (place_id) {ts:search-index}; CREATE INDEX CONCURRENTLY idx_postcode_postcode ON location_postcode USING BTREE (postcode) {ts:search-index}; diff --git a/sql/indices_updates.src.sql b/sql/indices_updates.src.sql new file mode 100644 index 00000000..99f59cdb --- /dev/null +++ b/sql/indices_updates.src.sql @@ -0,0 +1,9 @@ +-- Indices used only during search and update. +-- These indices are created only after the indexing process is done. + +CREATE INDEX CONCURRENTLY idx_placex_pendingsector ON placex USING BTREE (rank_search,geometry_sector) {ts:address-index} where indexed_status > 0; + +CREATE INDEX CONCURRENTLY idx_location_area_country_place_id ON location_area_country USING BTREE (place_id) {ts:address-index}; + +DROP INDEX CONCURRENTLY IF EXISTS place_id_idx; +CREATE UNIQUE INDEX CONCURRENTLY idx_place_osm_unique on place using btree(osm_id,osm_type,class) {ts:address-index}; diff --git a/utils/setup.php b/utils/setup.php index 90df0835..3015f13a 100644 --- a/utils/setup.php +++ b/utils/setup.php @@ -138,6 +138,11 @@ if ($aCMDResult['index'] || $aCMDResult['all']) { $oSetup->index($aCMDResult['index-noanalyse']); } +if ($aCMDResult['drop']) { + $bDidSomething = true; + $oSetup->drop($aCMDResult); +} + if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { $bDidSomething = true; $oSetup->createSearchIndices(); @@ -148,11 +153,6 @@ if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { $oSetup->createCountryNames($aCMDResult); } -if ($aCMDResult['drop']) { - $bDidSomething = true; - $oSetup->drop($aCMDResult); -} - // ****************************************************** // If we did something, repeat the warnings if (!$bDidSomething) { From 6073d948e61e2d62f5c3066db49da0b5dfcfcda5 Mon Sep 17 00:00:00 2001 From: Sarah Hoffmann Date: Wed, 12 Feb 2020 11:29:33 +0100 Subject: [PATCH 2/2] fix duplicate keys in tests The tests suddenly failed because the unique key constraint is more strict and does no longer include the type. --- test/bdd/db/import/search_name.feature | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/bdd/db/import/search_name.feature b/test/bdd/db/import/search_name.feature index cf3ce4dd..af31d46a 100644 --- a/test/bdd/db/import/search_name.feature +++ b/test/bdd/db/import/search_name.feature @@ -29,8 +29,8 @@ Feature: Creation of search terms And the places | osm | class | type | name | geometry | | N1 | place | state | new york | 80 80 | - | N1 | place | city | bonn | 81 81 | - | N1 | place | suburb | smalltown| 80 81 | + | N2 | place | city | bonn | 81 81 | + | N3 | place | suburb | smalltown| 80 81 | And the named places | osm | class | type | addr+city | addr+state | addr+suburb | geometry | | W1 | highway | service | bonn | New York | Smalltown | :w-north | @@ -67,8 +67,8 @@ Feature: Creation of search terms And the places | osm | class | type | name | geometry | | N1 | place | state | new york | 80 80 | - | N1 | place | city | bonn | 81 81 | - | N1 | place | suburb | smalltown| 80 81 | + | N2 | place | city | bonn | 81 81 | + | N3 | place | suburb | smalltown| 80 81 | And the named places | osm | class | type | addr+is_in | geometry | | W1 | highway | service | bonn, New York, Smalltown | :w-north |