hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-12 22:04:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

setup: factor out parameter replacement in SQL scripts

Browse Source 
Put all into a single function and use for all SQL
templates.
This commit is contained in:
Sarah Hoffmann
2020-01-04 23:41:55 +01:00
parent d11ee4c6d9
commit 2059e18e8b
1 changed files with 54 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

190
lib/setup/SetupClass.php
View File

@@ -223,42 +223,12 @@ class SetupFunctions
info('Create Tables'); info('Create Tables');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = $this->replaceSqlPatterns($sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:address-data}',
CONST_Tablespace_Address_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:address-index}',
CONST_Tablespace_Address_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-data}',
CONST_Tablespace_Search_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-index}',
CONST_Tablespace_Search_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
CONST_Tablespace_Aux_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunScript($sTemplate, false); $this->pgsqlRunScript($sTemplate, false);
if ($bReverseOnly) { if ($bReverseOnly) {
$this->pgExec('DROP TABLE search_name'); $this->dropTable('search_name');
} }
$oAlParser = new AddressLevelParser(CONST_Address_Level_Config); $oAlParser = new AddressLevelParser(CONST_Address_Level_Config);
@@ -270,41 +240,7 @@ class SetupFunctions
info('Create Partition Tables'); info('Create Partition Tables');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceSqlPatterns($sTemplate);
'{ts:address-data}',
CONST_Tablespace_Address_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:address-index}',
CONST_Tablespace_Address_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-data}',
CONST_Tablespace_Search_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-index}',
CONST_Tablespace_Search_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
CONST_Tablespace_Aux_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunPartitionScript($sTemplate); $this->pgsqlRunPartitionScript($sTemplate);
} }
@@ -322,8 +258,8 @@ class SetupFunctions
$sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikimedia-importance.sql.gz'; $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikimedia-importance.sql.gz';
if (file_exists($sWikiArticlesFile)) { if (file_exists($sWikiArticlesFile)) {
info('Importing wikipedia articles and redirects'); info('Importing wikipedia articles and redirects');
$this->pgExec('DROP TABLE IF EXISTS wikipedia_article'); $this->dropTable('wikipedia_article');
$this->pgExec('DROP TABLE IF EXISTS wikipedia_redirect'); $this->dropTable('wikipedia_redirect');
$this->pgsqlRunScriptFile($sWikiArticlesFile); $this->pgsqlRunScriptFile($sWikiArticlesFile);
} else { } else {
warn('wikipedia importance dump file not found - places will have default importance'); warn('wikipedia importance dump file not found - places will have default importance');
@@ -334,25 +270,25 @@ class SetupFunctions
{ {
info('Drop old Data'); info('Drop old Data');
$this->pgExec('TRUNCATE word'); $this->oDB->exec('TRUNCATE word');
echo '.'; echo '.';
$this->pgExec('TRUNCATE placex'); $this->oDB->exec('TRUNCATE placex');
echo '.'; echo '.';
$this->pgExec('TRUNCATE location_property_osmline'); $this->oDB->exec('TRUNCATE location_property_osmline');
echo '.'; echo '.';
$this->pgExec('TRUNCATE place_addressline'); $this->oDB->exec('TRUNCATE place_addressline');
echo '.'; echo '.';
$this->pgExec('TRUNCATE location_area'); $this->oDB->exec('TRUNCATE location_area');
echo '.'; echo '.';
if (!$this->dbReverseOnly()) { if (!$this->dbReverseOnly()) {
$this->pgExec('TRUNCATE search_name'); $this->oDB->exec('TRUNCATE search_name');
echo '.'; echo '.';
} }
$this->pgExec('TRUNCATE search_name_blank'); $this->oDB->exec('TRUNCATE search_name_blank');
echo '.'; echo '.';
$this->pgExec('DROP SEQUENCE seq_place'); $this->oDB->exec('DROP SEQUENCE seq_place');
echo '.'; echo '.';
$this->pgExec('CREATE SEQUENCE seq_place start 100000'); $this->oDB->exec('CREATE SEQUENCE seq_place start 100000');
echo '.'; echo '.';
$sSQL = 'select distinct partition from country_name'; $sSQL = 'select distinct partition from country_name';
@@ -360,14 +296,14 @@ class SetupFunctions
if (!$this->bNoPartitions) $aPartitions[] = 0; if (!$this->bNoPartitions) $aPartitions[] = 0;
foreach ($aPartitions as $sPartition) { foreach ($aPartitions as $sPartition) {
$this->pgExec('TRUNCATE location_road_'.$sPartition); $this->oDB->exec('TRUNCATE location_road_'.$sPartition);
echo '.'; echo '.';
} }
// used by getorcreate_word_id to ignore frequent partial words // used by getorcreate_word_id to ignore frequent partial words
$sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
$sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
$this->pgExec($sSQL); $this->oDB->exec($sSQL);
echo ".\n"; echo ".\n";
// pre-create the word list // pre-create the word list
@@ -464,17 +400,8 @@ class SetupFunctions
if (empty($aFilenames)) return; if (empty($aFilenames)) return;
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = $this->replaceSqlPatterns($sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
CONST_Tablespace_Aux_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunScript($sTemplate, false); $this->pgsqlRunScript($sTemplate, false);
$aDBInstances = array(); $aDBInstances = array();
@@ -527,24 +454,15 @@ class SetupFunctions
info('Creating indexes on Tiger data'); info('Creating indexes on Tiger data');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = $this->replaceSqlPatterns($sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:aux-data}',
CONST_Tablespace_Aux_Data,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunScript($sTemplate, false); $this->pgsqlRunScript($sTemplate, false);
} }
public function calculatePostcodes($bCMDResultAll) public function calculatePostcodes($bCMDResultAll)
{ {
info('Calculate Postcodes'); info('Calculate Postcodes');
$this->pgExec('TRUNCATE location_postcode'); $this->oDB->exec('TRUNCATE location_postcode');
$sSQL = 'INSERT INTO location_postcode'; $sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
@@ -555,7 +473,7 @@ class SetupFunctions
$sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'"; $sSQL .= " WHERE address ? 'postcode' AND address->'postcode' NOT SIMILAR TO '%(,|;)%'";
$sSQL .= ' AND geometry IS NOT null'; $sSQL .= ' AND geometry IS NOT null';
$sSQL .= ' GROUP BY country_code, pc'; $sSQL .= ' GROUP BY country_code, pc';
$this->pgExec($sSQL); $this->oDB->exec($sSQL);
// only add postcodes that are not yet available in OSM // only add postcodes that are not yet available in OSM
$sSQL = 'INSERT INTO location_postcode'; $sSQL = 'INSERT INTO location_postcode';
@@ -565,7 +483,7 @@ class SetupFunctions
$sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; $sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'us')"; $sSQL .= " WHERE country_code = 'us')";
$this->pgExec($sSQL); $this->oDB->exec($sSQL);
// add missing postcodes for GB (if available) // add missing postcodes for GB (if available)
$sSQL = 'INSERT INTO location_postcode'; $sSQL = 'INSERT INTO location_postcode';
@@ -574,17 +492,17 @@ class SetupFunctions
$sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'gb')"; $sSQL .= " WHERE country_code = 'gb')";
$this->pgExec($sSQL); $this->oDB->exec($sSQL);
if (!$bCMDResultAll) { if (!$bCMDResultAll) {
$sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
$sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)'; $sSQL .= 'and word NOT IN (SELECT postcode FROM location_postcode)';
$this->pgExec($sSQL); $this->oDB->exec($sSQL);
} }
$sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM '; $sSQL = 'SELECT count(getorcreate_postcode_id(v)) FROM ';
$sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p'; $sSQL .= '(SELECT distinct(postcode) as v FROM location_postcode) p';
$this->pgExec($sSQL); $this->oDB->exec($sSQL);
} }
public function index($bIndexNoanalyse) public function index($bIndexNoanalyse)
@@ -623,7 +541,7 @@ class SetupFunctions
info('Index postcodes'); info('Index postcodes');
$sSQL = 'UPDATE location_postcode SET indexed_status = 0'; $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
$this->pgExec($sSQL); $this->oDB->exec($sSQL);
} }
public function createSearchIndices() public function createSearchIndices()
@@ -634,22 +552,8 @@ class SetupFunctions
if (!$this->dbReverseOnly()) { if (!$this->dbReverseOnly()) {
$sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql'); $sTemplate .= file_get_contents(CONST_BasePath.'/sql/indices_search.src.sql');
} }
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = $this->replaceSqlPatterns($sTemplate);
$sTemplate = $this->replaceTablespace(
'{ts:address-index}',
CONST_Tablespace_Address_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:search-index}',
CONST_Tablespace_Search_Index,
$sTemplate
);
$sTemplate = $this->replaceTablespace(
'{ts:aux-index}',
CONST_Tablespace_Aux_Index,
$sTemplate
);
$this->pgsqlRunScript($sTemplate); $this->pgsqlRunScript($sTemplate);
} }
@@ -721,8 +625,7 @@ class SetupFunctions
if (!$bFound) array_push($aDropTables, $sTable); if (!$bFound) array_push($aDropTables, $sTable);
} }
foreach ($aDropTables as $sDrop) { foreach ($aDropTables as $sDrop) {
if ($this->bVerbose) echo "Dropping table $sDrop\n"; $this->dropTable($sDrop);
$this->oDB->exec("DROP TABLE IF EXISTS $sDrop CASCADE");
} }
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
@@ -845,13 +748,27 @@ class SetupFunctions
} }
} }
private function replaceTablespace($sTemplate, $sTablespace, $sSql) private function replaceSqlPatterns($sSql)
{ {
if ($sTablespace) { $sSql = str_replace('{www-user}', CONST_Database_Web_User, $sSql);
$sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql);
} else { $aPatterns = array(
$sSql = str_replace($sTemplate, '', $sSql); '{ts:address-data}' => CONST_Tablespace_Address_Data,
'{ts:address-index}' => CONST_Tablespace_Address_Index,
'{ts:search-data}' => CONST_Tablespace_Search_Data,
'{ts:search-index}' => CONST_Tablespace_Search_Index,
'{ts:aux-data}' => CONST_Tablespace_Aux_Data,
'{ts:aux-index}' => CONST_Tablespace_Aux_Index,
);
foreach ($aPatterns as $sPattern => $sTablespace) {
if ($sTablespace) {
$sSql = str_replace($sPattern, 'TABLESPACE "'.$sTablespace.'"', $sSql);
} else {
$sSql = str_replace($sPattern, '', $sSql);
}
} }
return $sSql; return $sSql;
} }
@@ -871,17 +788,18 @@ class SetupFunctions
} }
/** /**
* Execute the SQL command on the open database. * Drop table with the given name if it exists.
* *
* @param string $sSQL SQL command to execute. * @param string $sName Name of table to remove.
* *
* @return null * @return null
* *
* @pre connect() must have been called. * @pre connect() must have been called.
*/ */
private function pgExec($sSQL) private function dropTable($sName)
{ {
$this->oDB->exec($sSQL); if ($this->bVerbose) echo "Dropping table $sName\n";
$this->oDB->exec('DROP TABLE IF EXISTS '.$sName.' CASCADE');
} }
/** /**