hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-02-16 15:47:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

splitted createTables and changed formatting to please Travis

Browse Source
This commit is contained in:
ThomasBarris
2018-08-29 22:54:28 +02:00
parent c036480ce2
commit b2f3cfde0b
3 changed files with 64 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
lib/setup_functions.php
View File

@@ -1,6 +1,7 @@
<?php <?php
function checkInFile($sOSMFile) { function checkInFile($sOSMFile)
{
if (!isset($sOSMFile)) { if (!isset($sOSMFile)) {
fail('missing --osm-file for data import'); fail('missing --osm-file for data import');
} }
@@ -14,7 +15,8 @@ function checkInFile($sOSMFile) {
} }
} }
function checkModulePresence() { function checkModulePresence()
{
// Try accessing the C module, so we know early if something is wrong // Try accessing the C module, so we know early if something is wrong
// and can simply error out. // and can simply error out.
$sModulePath = CONST_Database_Module_Path; $sModulePath = CONST_Database_Module_Path;
@@ -36,7 +38,8 @@ function checkModulePresence() {
return $bResult; return $bResult;
} }
function createSetupArgvArray() { function createSetupArgvArray()
{
$aCMDOptions $aCMDOptions
= array( = array(
'Create and setup nominatim search system', 'Create and setup nominatim search system',

12
utils/setup.php
View File

@@ -25,7 +25,7 @@ $bDidSomething = false;
// Check if osm-file is set and points to a valid file // Check if osm-file is set and points to a valid file
if ($aCMDResult['import-data'] || $aCMDResult['all']) { if ($aCMDResult['import-data'] || $aCMDResult['all']) {
// to remain in /lib/setup_functions.php function // to remain in /lib/setup_functions.php function
checkInFile($aCMDResult['osm-file']); checkInFile($aCMDResult['osm-file']);
echo $aCMDResult['osm-file']; echo $aCMDResult['osm-file'];
} }
@@ -37,7 +37,7 @@ if ($aCMDResult['osmosis-init']) {
// ****************************************************** // ******************************************************
// instantiate Setup class // instantiate Setup class
$cSetup = new SetupFunctions ($aCMDResult); $cSetup = new SetupFunctions($aCMDResult);
if ($aCMDResult['create-db'] || $aCMDResult['all']) { if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createDB(); $cSetup -> createDB();
@@ -68,6 +68,7 @@ if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
if ($aCMDResult['create-tables'] || $aCMDResult['all']) { if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createTables(); $cSetup -> createTables();
$cSetup -> recreateFunction();
} }
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
@@ -79,12 +80,12 @@ if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createPartitionFunctions(); $cSetup -> createPartitionFunctions();
} }
/*
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> importWikipediaArticles(); $cSetup -> importWikipediaArticles();
} }
*/
if ($aCMDResult['load-data'] || $aCMDResult['all']) { if ($aCMDResult['load-data'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> loadData($aCMDResult['disable-token-precalc']); $cSetup -> loadData($aCMDResult['disable-token-precalc']);
@@ -130,6 +131,3 @@ if (!$bDidSomething) {
echo "\n"; echo "\n";
info('Setup finished.'); info('Setup finished.');
} }

85
utils/setupClass.php
View File

@@ -1,9 +1,6 @@
<?php <?php
class SetupFunctions class SetupFunctions
{ {
protected $iCacheMemory; // set in constructor protected $iCacheMemory; // set in constructor
protected $iInstances; // set in constructor protected $iInstances; // set in constructor
@@ -17,7 +14,6 @@ class SetupFunctions
protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData
// pgsqlRunPartitionScript, calculatePostcodes, ..if no already set // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set
public function __construct($aCMDResult) { public function __construct($aCMDResult) {
// by default, use all but one processor, but never more than 15. // by default, use all but one processor, but never more than 15.
$this->iInstances = isset($aCMDResult['threads']) $this->iInstances = isset($aCMDResult['threads'])
@@ -50,10 +46,10 @@ class SetupFunctions
$this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates']; $this->bEnableDiffUpdates = $aCMDResult['enable-diff-updates'];
$this->bEnableDebugStatements = $aCMDResult['enable-debug-statements']; $this->bEnableDebugStatements = $aCMDResult['enable-debug-statements'];
$this->bNoPartitions = $aCMDResult['no-partitions']; $this->bNoPartitions = $aCMDResult['no-partitions'];
} }
public function createDB(){ public function createDB()
{
info('Create DB'); info('Create DB');
$sDB = DB::connect(CONST_Database_DSN, false); $sDB = DB::connect(CONST_Database_DSN, false);
if (!PEAR::isError($sDB)) { if (!PEAR::isError($sDB)) {
@@ -78,7 +74,8 @@ class SetupFunctions
if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
} }
public function setupDB() { public function setupDB()
{
info('Setup DB'); info('Setup DB');
$this->oDB =& getDB(); $this->oDB =& getDB();
@@ -155,7 +152,8 @@ class SetupFunctions
$this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false); $this->pgsqlRunScript('CREATE TYPE wikipedia_article_match AS ()', false);
} }
public function importData($sOSMFile) { public function importData($sOSMFile)
{
info('Import data'); info('Import data');
$osm2pgsql = CONST_Osm2pgsql_Binary; $osm2pgsql = CONST_Osm2pgsql_Binary;
@@ -200,13 +198,15 @@ class SetupFunctions
} }
} }
public function createFunctions() { public function createFunctions()
{
info('Create Functions'); info('Create Functions');
$this->createSqlFunctions(); $this->createSqlFunctions();
} }
public function createTables() { public function createTables()
{
info('Create Tables'); info('Create Tables');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
@@ -243,13 +243,17 @@ class SetupFunctions
); );
$this->pgsqlRunScript($sTemplate, false); $this->pgsqlRunScript($sTemplate, false);
}
public function recreateFunction()
{
// re-run the functions // re-run the functions
info('Recreate Functions'); info('Recreate Functions');
$this->createSqlFunctions(); $this->createSqlFunctions();
} }
public function createPartitionTables() { public function createPartitionTables()
{
info('Create Partition Tables'); info('Create Partition Tables');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
@@ -292,14 +296,16 @@ class SetupFunctions
$this->pgsqlRunPartitionScript($sTemplate); $this->pgsqlRunPartitionScript($sTemplate);
} }
public function createPartitionFunctions() { public function createPartitionFunctions()
{
info('Create Partition Functions'); info('Create Partition Functions');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
$this->pgsqlRunPartitionScript($sTemplate); $this->pgsqlRunPartitionScript($sTemplate);
} }
public function importWikipediaArticles() { public function importWikipediaArticles()
{
$sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
$sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
if (file_exists($sWikiArticlesFile)) { if (file_exists($sWikiArticlesFile)) {
@@ -315,9 +321,10 @@ class SetupFunctions
warn('wikipedia redirect dump file not found - some place importance values may be missing'); warn('wikipedia redirect dump file not found - some place importance values may be missing');
} }
echo ' finish wikipedia'; echo ' finish wikipedia';
} }
public function loadData($bDisableTokenPrecalc) { public function loadData($bDisableTokenPrecalc)
{
info('Drop old Data'); info('Drop old Data');
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) $this->oDB =& getDB();
@@ -382,7 +389,6 @@ class SetupFunctions
} }
// last thread for interpolation lines // last thread for interpolation lines
$aDBInstances[$iLoadThreads] =& getDB(true); $aDBInstances[$iLoadThreads] =& getDB(true);
$sSQL = 'insert into location_property_osmline'; $sSQL = 'insert into location_property_osmline';
$sSQL .= ' (osm_id, address, linegeo)'; $sSQL .= ' (osm_id, address, linegeo)';
@@ -426,7 +432,8 @@ class SetupFunctions
} }
} }
public function importTigerData() { public function importTigerData()
{
info('Import Tiger data'); info('Import Tiger data');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
@@ -498,7 +505,8 @@ class SetupFunctions
$this->pgsqlRunScript($sTemplate, false); $this->pgsqlRunScript($sTemplate, false);
} }
public function calculatePostcodes($bCMDResultAll) { public function calculatePostcodes($bCMDResultAll)
{
info('Calculate Postcodes'); info('Calculate Postcodes');
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) $this->oDB =& getDB();
if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) {
@@ -556,9 +564,11 @@ class SetupFunctions
} }
} }
public function index($bIndexNoanalyse) { public function index($bIndexNoanalyse)
{
$sOutputFile = ''; $sOutputFile = '';
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '
.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec'];
} }
@@ -591,9 +601,10 @@ class SetupFunctions
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) $this->oDB =& getDB();
$sSQL = 'UPDATE location_postcode SET indexed_status = 0'; $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
} }
public function createSearchIndices() { public function createSearchIndices()
{
info('Create Search indices'); info('Create Search indices');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
@@ -616,14 +627,16 @@ class SetupFunctions
$this->pgsqlRunScript($sTemplate); $this->pgsqlRunScript($sTemplate);
} }
public function createCountryNames() { public function createCountryNames()
{
info('Create search index for default country names'); info('Create search index for default country names');
$this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')"); $this->pgsqlRunScript("select getorcreate_country(make_standard_name('uk'), 'gb')");
$this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')"); $this->pgsqlRunScript("select getorcreate_country(make_standard_name('united states'), 'us')");
$this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x');
$this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x");
$sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v), country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),'
.'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
if (CONST_Languages) { if (CONST_Languages) {
$sSQL .= 'in '; $sSQL .= 'in ';
$sDelim = '('; $sDelim = '(';
@@ -640,7 +653,8 @@ class SetupFunctions
$this->pgsqlRunScript($sSQL); $this->pgsqlRunScript($sSQL);
} }
public function drop() { public function drop()
{
info('Drop tables only required for updates'); info('Drop tables only required for updates');
// The implementation is potentially a bit dangerous because it uses // The implementation is potentially a bit dangerous because it uses
@@ -693,7 +707,8 @@ class SetupFunctions
} }
} }
private function pgsqlRunDropAndRestore($sDumpFile) { private function pgsqlRunDropAndRestore($sDumpFile)
{
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
$sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
@@ -709,7 +724,8 @@ class SetupFunctions
$iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)"
} }
private function pgsqlRunScript($sScript, $bfatal = true) { private function pgsqlRunScript($sScript, $bfatal = true)
{
runSQLScript( runSQLScript(
$sScript, $sScript,
$bfatal, $bfatal,
@@ -718,7 +734,8 @@ class SetupFunctions
); );
} }
private function createSqlFunctions() { private function createSqlFunctions()
{
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
$sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
if ($this->bEnableDiffUpdates) { if ($this->bEnableDiffUpdates) {
@@ -739,7 +756,8 @@ class SetupFunctions
$this->pgsqlRunScript($sTemplate); $this->pgsqlRunScript($sTemplate);
} }
private function pgsqlRunPartitionScript($sTemplate) { private function pgsqlRunPartitionScript($sTemplate)
{
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) $this->oDB =& getDB();
$sSQL = 'select distinct partition from country_name'; $sSQL = 'select distinct partition from country_name';
@@ -756,9 +774,10 @@ class SetupFunctions
} }
$this->pgsqlRunScript($sTemplate); $this->pgsqlRunScript($sTemplate);
} }
private function pgsqlRunScriptFile($sFilename) { private function pgsqlRunScriptFile($sFilename)
{
if (!file_exists($sFilename)) fail('unable to find '.$sFilename); if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
$sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'];
@@ -812,7 +831,9 @@ class SetupFunctions
proc_close($hGzipProcess); proc_close($hGzipProcess);
} }
} }
private function replaceTablespace($sTemplate, $sTablespace, $sSql) {
private function replaceTablespace($sTemplate, $sTablespace, $sSql)
{
if ($sTablespace) { if ($sTablespace) {
$sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql);
} else { } else {