safety-screen/Nominatim
1
0
Fork 0
forked from hans/Nominatim
Code Pull Requests Activity

more format changes for Mr. Travis

Browse Source
This commit is contained in:
ThomasBarris
2018-08-31 22:01:53 +02:00
parent 42e79bfab9
commit aa6ac5a751
1 changed files with 147 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

398
lib/SetupClass.php
View File

@@ -4,28 +4,29 @@ namespace Nominatim\Setup;
class SetupFunctions class SetupFunctions
{ {
protected $iCacheMemory; // set in constructor protected $iCacheMemory; // set in constructor
protected $iInstances; // set in constructor protected $iInstances; // set in constructor
protected $sModulePath; // set in constructor protected $sModulePath; // set in constructor
protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN); protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN);
protected $sVerbose; // set in constructor protected $sVerbose; // set in constructor
protected $sIgnoreErrors; // set in constructor protected $sIgnoreErrors; // set in constructor
protected $bEnableDiffUpdates; // set in constructor protected $bEnableDiffUpdates; // set in constructor
protected $bEnableDebugStatements; // set in constructor protected $bEnableDebugStatements; // set in constructor
protected $bNoPartitions; // set in constructor protected $bNoPartitions; // set in constructor
protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData
// pgsqlRunPartitionScript, calculatePostcodes, ..if no already set // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set
public function __construct($aCMDResult) public function __construct($aCMDResult)
{ {
// by default, use all but one processor, but never more than 15. // by default, use all but one processor, but never more than 15.
$this->iInstances = isset($aCMDResult['threads']) $this->iInstances = isset($aCMDResult['threads'])
? $aCMDResult['threads'] ? $aCMDResult['threads']
: (min(16, getProcessorCount()) - 1); : (min(16, getProcessorCount()) - 1);
if ($this->iInstances < 1) { if ($this->iInstances < 1) {
$this->iInstances = 1; $this->iInstances = 1;
warn('resetting threads to ' . $this->iInstances); warn('resetting threads to '.$this->iInstances);
} }
// Assume we can steal all the cache memory in the box (unless told otherwise) // Assume we can steal all the cache memory in the box (unless told otherwise)
@@ -40,9 +41,7 @@ class SetupFunctions
// prepares DB for import or update, sets the Data Source Name // prepares DB for import or update, sets the Data Source Name
$this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN); $this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN);
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) { if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
$this->aDSNInfo['port'] = 5432;
}
// setting member variables based on command line options stored in $aCMDResult // setting member variables based on command line options stored in $aCMDResult
$this->sVerbose = $aCMDResult['verbose']; $this->sVerbose = $aCMDResult['verbose'];
@@ -57,10 +56,10 @@ class SetupFunctions
info('Create DB'); info('Create DB');
$sDB = \DB::connect(CONST_Database_DSN, false); $sDB = \DB::connect(CONST_Database_DSN, false);
if (!\PEAR::isError($sDB)) { if (!\PEAR::isError($sDB)) {
fail('database already exists (' . CONST_Database_DSN . ')'); fail('database already exists ('.CONST_Database_DSN.')');
} }
$sCreateDBCmd = 'createdb -E UTF-8 -p ' . $this->aDSNInfo['port'] . ' ' . $this->aDSNInfo['database']; $sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database'];
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username'];
} }
@@ -75,19 +74,16 @@ class SetupFunctions
} }
$result = runWithEnv($sCreateDBCmd, $aProcEnv); $result = runWithEnv($sCreateDBCmd, $aProcEnv);
if ($result != 0) { if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd);
fail('Error executing external command: ' . $sCreateDBCmd);
}
} }
public function setupDB() public function setupDB()
{ {
info('Setup DB'); info('Setup DB');
$this->oDB = &getDB(); $this->oDB =& getDB();
$fPostgresVersion = getPostgresVersion($this->oDB); $fPostgresVersion = getPostgresVersion($this->oDB);
echo 'Postgres version found: ' . $fPostgresVersion . "\n"; echo 'Postgres version found: '.$fPostgresVersion."\n";
if ($fPostgresVersion < 9.1) { if ($fPostgresVersion < 9.1) {
fail('Minimum supported version of Postgresql is 9.1.'); fail('Minimum supported version of Postgresql is 9.1.');
@@ -106,8 +102,9 @@ class SetupFunctions
warn('Postgresql is too old. extratags and namedetails API not available.'); warn('Postgresql is too old. extratags and namedetails API not available.');
} }
$fPostgisVersion = getPostgisVersion($this->oDB); $fPostgisVersion = getPostgisVersion($this->oDB);
echo 'Postgis version found: ' . $fPostgisVersion . "\n"; echo 'Postgis version found: '.$fPostgisVersion."\n";
if ($fPostgisVersion < 2.1) { if ($fPostgisVersion < 2.1) {
// Functions were renamed in 2.1 and throw an annoying deprecation warning // Functions were renamed in 2.1 and throw an annoying deprecation warning
@@ -118,31 +115,32 @@ class SetupFunctions
$this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
} }
$i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '" . CONST_Database_Web_User . "'")); $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'"));
if ($i == 0) { if ($i == 0) {
echo "\nERROR: Web user '" . CONST_Database_Web_User . "' does not exist. Create it with:\n"; echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n";
echo "\n createuser " . CONST_Database_Web_User . "\n\n"; echo "\n createuser ".CONST_Database_Web_User."\n\n";
exit(1); exit(1);
} }
if (!file_exists(CONST_ExtraDataPath . '/country_osm_grid.sql.gz')) { if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) {
echo 'Error: you need to download the country_osm_grid first:'; echo 'Error: you need to download the country_osm_grid first:';
echo "\n wget -O " . CONST_ExtraDataPath . "/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n";
exit(1); exit(1);
} }
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_name.sql'); $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_naturalearthdata.sql'); $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_osm_grid.sql.gz'); $this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_table.sql'); $this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql');
if (file_exists(CONST_BasePath . '/data/gb_postcode_data.sql.gz')) {
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_data.sql.gz'); if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) {
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
} else { } else {
warn('external UK postcode table not found.'); warn('external UK postcode table not found.');
} }
if (CONST_Use_Extra_US_Postcodes) { if (CONST_Use_Extra_US_Postcodes) {
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/us_postcode.sql'); $this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql');
} }
if ($this->bNoPartitions) { if ($this->bNoPartitions) {
@@ -168,29 +166,23 @@ class SetupFunctions
fail("osm2pgsql not found in '$osm2pgsql'"); fail("osm2pgsql not found in '$osm2pgsql'");
} }
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$osm2pgsql .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File; $osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File;
}
if (CONST_Tablespace_Osm2pgsql_Data) {
$osm2pgsql .= ' --tablespace-slim-data ' . CONST_Tablespace_Osm2pgsql_Data;
}
if (CONST_Tablespace_Osm2pgsql_Index) {
$osm2pgsql .= ' --tablespace-slim-index ' . CONST_Tablespace_Osm2pgsql_Index;
}
if (CONST_Tablespace_Place_Data) {
$osm2pgsql .= ' --tablespace-main-data ' . CONST_Tablespace_Place_Data;
}
if (CONST_Tablespace_Place_Index) {
$osm2pgsql .= ' --tablespace-main-index ' . CONST_Tablespace_Place_Index;
} }
if (CONST_Tablespace_Osm2pgsql_Data)
$osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data;
if (CONST_Tablespace_Osm2pgsql_Index)
$osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index;
if (CONST_Tablespace_Place_Data)
$osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data;
if (CONST_Tablespace_Place_Index)
$osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index;
$osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
$osm2pgsql .= ' -C ' . $this->iCacheMemory; $osm2pgsql .= ' -C '.$this->iCacheMemory;
$osm2pgsql .= ' -P ' . $this->aDSNInfo['port']; $osm2pgsql .= ' -P '.$this->aDSNInfo['port'];
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; $osm2pgsql .= ' -U ' . $this->aDSNInfo['username'];
} }
@@ -201,12 +193,9 @@ class SetupFunctions
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
} }
$osm2pgsql .= ' -d ' . $this->aDSNInfo['database'] . ' ' . $sOSMFile; $osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile;
runWithEnv($osm2pgsql, $aProcEnv); runWithEnv($osm2pgsql, $aProcEnv);
if ($this->oDB == null) { if ($this->oDB == null) $this->oDB =& getDB();
$this->oDB = &getDB();
}
if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) {
fail('No Data'); fail('No Data');
} }
@@ -223,7 +212,7 @@ class SetupFunctions
{ {
info('Create Tables'); info('Create Tables');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/tables.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:address-data}', '{ts:address-data}',
@@ -263,7 +252,7 @@ class SetupFunctions
{ {
info('Create Partition Tables'); info('Create Partition Tables');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-tables.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql');
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:address-data}', '{ts:address-data}',
CONST_Tablespace_Address_Data, CONST_Tablespace_Address_Data,
@@ -307,14 +296,14 @@ class SetupFunctions
{ {
info('Create Partition Functions'); info('Create Partition Functions');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-functions.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql');
$this->pgsqlRunPartitionScript($sTemplate); $this->pgsqlRunPartitionScript($sTemplate);
} }
public function importWikipediaArticles() public function importWikipediaArticles()
{ {
$sWikiArticlesFile = CONST_Wikipedia_Data_Path . '/wikipedia_article.sql.bin'; $sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin';
$sWikiRedirectsFile = CONST_Wikipedia_Data_Path . '/wikipedia_redirect.sql.bin'; $sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin';
if (file_exists($sWikiArticlesFile)) { if (file_exists($sWikiArticlesFile)) {
info('Importing wikipedia articles'); info('Importing wikipedia articles');
$this->pgsqlRunDropAndRestore($sWikiArticlesFile); $this->pgsqlRunDropAndRestore($sWikiArticlesFile);
@@ -334,78 +323,40 @@ class SetupFunctions
{ {
info('Drop old Data'); info('Drop old Data');
if ($this->oDB == null) { if ($this->oDB == null) $this->oDB =& getDB();
$this->oDB = &getDB();
}
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection));
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) { if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) { if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) { if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) { if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) { if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) { if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) { if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) { if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) { if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
$sSQL = 'select distinct partition from country_name'; $sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL)); $aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) { if (!$this->bNoPartitions) $aPartitions[] = 0;
$aPartitions[] = 0;
}
foreach ($aPartitions as $sPartition) { foreach ($aPartitions as $sPartition) {
if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_' . $sPartition)) { if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
} }
// used by getorcreate_word_id to ignore frequent partial words // used by getorcreate_word_id to ignore frequent partial words
$sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
$sSQL .= '$$ SELECT ' . CONST_Max_Word_Frequency . ' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; $sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
if (!pg_query($this->oDB->connection, $sSQL)) { if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection)); fail(pg_last_error($this->oDB->connection));
} }
@@ -414,7 +365,7 @@ class SetupFunctions
// pre-create the word list // pre-create the word list
if (!$bDisableTokenPrecalc) { if (!$bDisableTokenPrecalc) {
info('Loading word list'); info('Loading word list');
$this->pgsqlRunScriptFile(CONST_BasePath . '/data/words.sql'); $this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql');
} }
info('Load Data'); info('Load Data');
@@ -422,30 +373,24 @@ class SetupFunctions
$aDBInstances = array(); $aDBInstances = array();
$iLoadThreads = max(1, $this->iInstances - 1); $iLoadThreads = max(1, $this->iInstances - 1);
for ($i = 0; $i < $iLoadThreads; $i++) { for ($i = 0; $i < $iLoadThreads; $i++) {
$aDBInstances[$i] = &getDB(true); $aDBInstances[$i] =& getDB(true);
$sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
$sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; $sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
$sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')";
$sSQL .= ' and ST_IsValid(geometry)'; $sSQL .= ' and ST_IsValid(geometry)';
if ($this->sVerbose) { if ($this->sVerbose) echo "$sSQL\n";
echo "$sSQL\n";
}
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$i]->connection)); fail(pg_last_error($aDBInstances[$i]->connection));
} }
} }
// last thread for interpolation lines // last thread for interpolation lines
$aDBInstances[$iLoadThreads] = &getDB(true); $aDBInstances[$iLoadThreads] =& getDB(true);
$sSQL = 'insert into location_property_osmline'; $sSQL = 'insert into location_property_osmline';
$sSQL .= ' (osm_id, address, linegeo)'; $sSQL .= ' (osm_id, address, linegeo)';
$sSQL .= ' SELECT osm_id, address, geometry from place where '; $sSQL .= ' SELECT osm_id, address, geometry from place where ';
$sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
if ($this->sVerbose) { if ($this->sVerbose) echo "$sSQL\n";
echo "$sSQL\n";
}
if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); fail(pg_last_error($aDBInstances[$iLoadThreads]->connection));
} }
@@ -477,7 +422,7 @@ class SetupFunctions
if ($sDatabaseDate === false) { if ($sDatabaseDate === false) {
warn('could not determine database date.'); warn('could not determine database date.');
} else { } else {
$sSQL = "INSERT INTO import_status (lastimportdate) VALUES('" . $sDatabaseDate . "')"; $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')";
pg_query($this->oDB->connection, $sSQL); pg_query($this->oDB->connection, $sSQL);
echo "Latest data imported from $sDatabaseDate.\n"; echo "Latest data imported from $sDatabaseDate.\n";
} }
@@ -487,7 +432,7 @@ class SetupFunctions
{ {
info('Import Tiger data'); info('Import Tiger data');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_start.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:aux-data}', '{ts:aux-data}',
@@ -503,11 +448,11 @@ class SetupFunctions
$aDBInstances = array(); $aDBInstances = array();
for ($i = 0; $i < $this->iInstances; $i++) { for ($i = 0; $i < $this->iInstances; $i++) {
$aDBInstances[$i] = &getDB(true); $aDBInstances[$i] =& getDB(true);
} }
foreach (glob(CONST_Tiger_Data_Path . '/*.sql') as $sFile) { foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) {
echo $sFile . ': '; echo $sFile.': ';
$hFile = fopen($sFile, 'r'); $hFile = fopen($sFile, 'r');
$sSQL = fgets($hFile, 100000); $sSQL = fgets($hFile, 100000);
$iLines = 0; $iLines = 0;
@@ -516,14 +461,8 @@ class SetupFunctions
if (!pg_connection_busy($aDBInstances[$i]->connection)) { if (!pg_connection_busy($aDBInstances[$i]->connection)) {
while (pg_get_result($aDBInstances[$i]->connection)); while (pg_get_result($aDBInstances[$i]->connection));
$sSQL = fgets($hFile, 100000); $sSQL = fgets($hFile, 100000);
if (!$sSQL) { if (!$sSQL) break 2;
break 2; if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
}
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
$iLines++; $iLines++;
if ($iLines == 1000) { if ($iLines == 1000) {
echo '.'; echo '.';
@@ -539,10 +478,7 @@ class SetupFunctions
while ($bAnyBusy) { while ($bAnyBusy) {
$bAnyBusy = false; $bAnyBusy = false;
for ($i = 0; $i < $this->iInstances; $i++) { for ($i = 0; $i < $this->iInstances; $i++) {
if (pg_connection_busy($aDBInstances[$i]->connection)) { if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true;
$bAnyBusy = true;
}
} }
usleep(10); usleep(10);
} }
@@ -550,7 +486,7 @@ class SetupFunctions
} }
info('Creating indexes on Tiger data'); info('Creating indexes on Tiger data');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_finish.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:aux-data}', '{ts:aux-data}',
@@ -568,15 +504,13 @@ class SetupFunctions
public function calculatePostcodes($bCMDResultAll) public function calculatePostcodes($bCMDResultAll)
{ {
info('Calculate Postcodes'); info('Calculate Postcodes');
if ($this->oDB == null) { if ($this->oDB == null) $this->oDB =& getDB();
$this->oDB = &getDB();
}
if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) {
fail(pg_last_error($this->oDB->connection)); fail(pg_last_error($this->oDB->connection));
} }
$sSQL = 'INSERT INTO location_postcode';
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; $sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
$sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,";
@@ -592,29 +526,24 @@ class SetupFunctions
if (CONST_Use_Extra_US_Postcodes) { if (CONST_Use_Extra_US_Postcodes) {
// only add postcodes that are not yet available in OSM // only add postcodes that are not yet available in OSM
$sSQL = 'INSERT INTO location_postcode'; $sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
$sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)';
$sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; $sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'us')"; $sSQL .= " WHERE country_code = 'us')";
if (!pg_query($this->oDB->connection, $sSQL)) { if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
} }
// add missing postcodes for GB (if available) // add missing postcodes for GB (if available)
$sSQL = 'INSERT INTO location_postcode'; $sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
$sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'gb')"; $sSQL .= " WHERE country_code = 'gb')";
if (!pg_query($this->oDB->connection, $sSQL)) { if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
if (!$bCMDResultAll) { if (!$bCMDResultAll) {
$sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
@@ -634,8 +563,8 @@ class SetupFunctions
public function index($bIndexNoanalyse) public function index($bIndexNoanalyse)
{ {
$sOutputFile = ''; $sOutputFile = '';
$sBaseCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $this->aDSNInfo['database'] . ' -P ' $sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P '
. $this->aDSNInfo['port'] . ' -t ' . $this->iInstances . $sOutputFile; .$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec'];
} }
@@ -648,45 +577,33 @@ class SetupFunctions
} }
info('Index ranks 0 - 4'); info('Index ranks 0 - 4');
$iStatus = runWithEnv($sBaseCmd . ' -R 4', $aProcEnv); $iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv);
if ($iStatus != 0) { if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!'); fail('error status ' . $iStatus . ' running nominatim!');
} }
if (!$bIndexNoanalyse) { if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
$this->pgsqlRunScript('ANALYSE');
}
info('Index ranks 5 - 25'); info('Index ranks 5 - 25');
$iStatus = runWithEnv($sBaseCmd . ' -r 5 -R 25', $aProcEnv); $iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv);
if ($iStatus != 0) { if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!'); fail('error status ' . $iStatus . ' running nominatim!');
} }
if (!$bIndexNoanalyse) { if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE');
$this->pgsqlRunScript('ANALYSE');
}
info('Index ranks 26 - 30'); info('Index ranks 26 - 30');
$iStatus = runWithEnv($sBaseCmd . ' -r 26', $aProcEnv); $iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv);
if ($iStatus != 0) { if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!'); fail('error status ' . $iStatus . ' running nominatim!');
} }
info('Index postcodes'); info('Index postcodes');
if ($this->oDB == null) { if ($this->oDB == null) $this->oDB =& getDB();
$this->oDB = &getDB();
}
$sSQL = 'UPDATE location_postcode SET indexed_status = 0'; $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
if (!pg_query($this->oDB->connection, $sSQL)) { if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection));
fail(pg_last_error($this->oDB->connection));
}
} }
public function createSearchIndices() public function createSearchIndices()
{ {
info('Create Search indices'); info('Create Search indices');
$sTemplate = file_get_contents(CONST_BasePath . '/sql/indices.src.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:address-index}', '{ts:address-index}',
@@ -715,12 +632,12 @@ class SetupFunctions
$this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x');
$this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x");
$sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),'
. 'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; .'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
if (CONST_Languages) { if (CONST_Languages) {
$sSQL .= 'in '; $sSQL .= 'in ';
$sDelim = '('; $sDelim = '(';
foreach (explode(',', CONST_Languages) as $sLang) { foreach (explode(',', CONST_Languages) as $sLang) {
$sSQL .= $sDelim . "'name:$sLang'"; $sSQL .= $sDelim."'name:$sLang'";
$sDelim = ','; $sDelim = ',';
} }
$sSQL .= ')'; $sSQL .= ')';
@@ -742,27 +659,24 @@ class SetupFunctions
// created. USE AT YOUR OWN PERIL. // created. USE AT YOUR OWN PERIL.
// tables we want to keep. everything else goes. // tables we want to keep. everything else goes.
$aKeepTables = array( $aKeepTables = array(
'*columns', '*columns',
'import_polygon_*', 'import_polygon_*',
'import_status', 'import_status',
'place_addressline', 'place_addressline',
'location_postcode', 'location_postcode',
'location_property*', 'location_property*',
'placex', 'placex',
'search_name', 'search_name',
'seq_*', 'seq_*',
'word', 'word',
'query_log', 'query_log',
'new_query_log', 'new_query_log',
'spatial_ref_sys', 'spatial_ref_sys',
'country_name', 'country_name',
'place_classtype_*', 'place_classtype_*'
); );
if ($this->oDB = null) {
$this->oDB = &getDB();
}
if ($this->oDB = null) $this->oDB =& getDB();
$aDropTables = array(); $aDropTables = array();
$aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"));
@@ -774,37 +688,25 @@ class SetupFunctions
break; break;
} }
} }
if (!$bFound) { if (!$bFound) array_push($aDropTables, $sTable);
array_push($aDropTables, $sTable);
}
} }
foreach ($aDropTables as $sDrop) { foreach ($aDropTables as $sDrop) {
if ($this->sVerbose) { if ($this->sVerbose) echo "dropping table $sDrop\n";
echo "dropping table $sDrop\n";
}
@pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE");
// ignore warnings/errors as they might be caused by a table having // ignore warnings/errors as they might be caused by a table having
// been deleted already by CASCADE // been deleted already by CASCADE
} }
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
if ($sVerbose) { if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n";
echo 'deleting ' . CONST_Osm2pgsql_Flatnode_File . "\n";
}
unlink(CONST_Osm2pgsql_Flatnode_File); unlink(CONST_Osm2pgsql_Flatnode_File);
} }
} }
private function pgsqlRunDropAndRestore($sDumpFile) private function pgsqlRunDropAndRestore($sDumpFile)
{ {
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) { if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432;
$this->aDSNInfo['port'] = 5432; $sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile;
}
$sCMD = 'pg_restore -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'] . ' -Fc --clean ' . $sDumpFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; $sCMD .= ' -h ' . $this->aDSNInfo['hostspec'];
} }
@@ -815,7 +717,7 @@ class SetupFunctions
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
} }
$iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)"
} }
private function pgsqlRunScript($sScript, $bfatal = true) private function pgsqlRunScript($sScript, $bfatal = true)
@@ -830,7 +732,7 @@ class SetupFunctions
private function createSqlFunctions() private function createSqlFunctions()
{ {
$sTemplate = file_get_contents(CONST_BasePath . '/sql/functions.sql'); $sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql');
$sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
if ($this->bEnableDiffUpdates) { if ($this->bEnableDiffUpdates) {
$sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
@@ -852,15 +754,11 @@ class SetupFunctions
private function pgsqlRunPartitionScript($sTemplate) private function pgsqlRunPartitionScript($sTemplate)
{ {
if ($this->oDB == null) { if ($this->oDB == null) $this->oDB =& getDB();
$this->oDB = &getDB();
}
$sSQL = 'select distinct partition from country_name'; $sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL)); $aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) { if (!$this->bNoPartitions) $aPartitions[] = 0;
$aPartitions[] = 0;
}
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach ($aMatches as $aMatch) { foreach ($aMatches as $aMatch) {
@@ -876,11 +774,9 @@ class SetupFunctions
private function pgsqlRunScriptFile($sFilename) private function pgsqlRunScriptFile($sFilename)
{ {
if (!file_exists($sFilename)) { if (!file_exists($sFilename)) fail('unable to find '.$sFilename);
fail('unable to find ' . $sFilename);
}
$sCMD = 'psql -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database']; $sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'];
if (!$this->sVerbose) { if (!$this->sVerbose) {
$sCMD .= ' -q'; $sCMD .= ' -q';
} }
@@ -897,32 +793,26 @@ class SetupFunctions
$ahGzipPipes = null; $ahGzipPipes = null;
if (preg_match('/\\.gz$/', $sFilename)) { if (preg_match('/\\.gz$/', $sFilename)) {
$aDescriptors = array( $aDescriptors = array(
0 => array('pipe', 'r'), 0 => array('pipe', 'r'),
1 => array('pipe', 'w'), 1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a'), 2 => array('file', '/dev/null', 'a')
); );
$hGzipProcess = proc_open('zcat ' . $sFilename, $aDescriptors, $ahGzipPipes); $hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes);
if (!is_resource($hGzipProcess)) { if (!is_resource($hGzipProcess)) fail('unable to start zcat');
fail('unable to start zcat');
}
$aReadPipe = $ahGzipPipes[1]; $aReadPipe = $ahGzipPipes[1];
fclose($ahGzipPipes[0]); fclose($ahGzipPipes[0]);
} else { } else {
$sCMD .= ' -f ' . $sFilename; $sCMD .= ' -f '.$sFilename;
$aReadPipe = array('pipe', 'r'); $aReadPipe = array('pipe', 'r');
} }
$aDescriptors = array( $aDescriptors = array(
0 => $aReadPipe, 0 => $aReadPipe,
1 => array('pipe', 'w'), 1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a'), 2 => array('file', '/dev/null', 'a')
); );
$ahPipes = null; $ahPipes = null;
$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
if (!is_resource($hProcess)) { if (!is_resource($hProcess)) fail('unable to start pgsql');
fail('unable to start pgsql');
}
// TODO: error checking // TODO: error checking
while (!feof($ahPipes[1])) { while (!feof($ahPipes[1])) {
echo fread($ahPipes[1], 4096); echo fread($ahPipes[1], 4096);
@@ -941,7 +831,7 @@ class SetupFunctions
private function replaceTablespace($sTemplate, $sTablespace, $sSql) private function replaceTablespace($sTemplate, $sTablespace, $sSql)
{ {
if ($sTablespace) { if ($sTablespace) {
$sSql = str_replace($sTemplate, 'TABLESPACE "' . $sTablespace . '"', $sSql); $sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql);
} else { } else {
$sSql = str_replace($sTemplate, '', $sSql); $sSql = str_replace($sTemplate, '', $sSql);
} }