hans/Nominatim
1
0
Fork 1
mirror of https://github.com/osm-search/Nominatim.git synced 2026-03-09 03:24:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

small fixes on setup.php and a bring update.php to work

Browse Source
This commit is contained in:
ThomasBarris
2018-08-31 21:31:38 +02:00
parent b2f3cfde0b
commit a3b4f80c99
5 changed files with 1297 additions and 1160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@@ -111,7 +111,6 @@ set(CUSTOMFILES
utils/query.php utils/query.php
utils/server_compare.php utils/server_compare.php
utils/setup.php utils/setup.php
utils/setupClass.php
utils/specialphrases.php utils/specialphrases.php
utils/update.php utils/update.php
utils/warm.php utils/warm.php

416
utils/setupClass.php → lib/SetupClass.php Executable file → Normal file
View File

@@ -1,29 +1,31 @@
<?php <?php
namespace Nominatim\Setup;
class SetupFunctions class SetupFunctions
{ {
protected $iCacheMemory; // set in constructor protected $iCacheMemory; // set in constructor
protected $iInstances; // set in constructor protected $iInstances; // set in constructor
protected $sModulePath; // set in constructor protected $sModulePath; // set in constructor
protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN); protected $aDSNInfo; // set in constructor = DB::parseDSN(CONST_Database_DSN);
protected $sVerbose; // set in constructor protected $sVerbose; // set in constructor
protected $sIgnoreErrors; // set in constructor protected $sIgnoreErrors; // set in constructor
protected $bEnableDiffUpdates; // set in constructor protected $bEnableDiffUpdates; // set in constructor
protected $bEnableDebugStatements; // set in constructor protected $bEnableDebugStatements; // set in constructor
protected $bNoPartitions; // set in constructor protected $bNoPartitions; // set in constructor
protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData protected $oDB = null; // set in setupDB (earliest) or later in loadData, importData, drop, createSqlFunctions, importTigerData
// pgsqlRunPartitionScript, calculatePostcodes, ..if no already set // pgsqlRunPartitionScript, calculatePostcodes, ..if no already set
public function __construct($aCMDResult) { public function __construct($aCMDResult)
{
// by default, use all but one processor, but never more than 15. // by default, use all but one processor, but never more than 15.
$this->iInstances = isset($aCMDResult['threads']) $this->iInstances = isset($aCMDResult['threads'])
? $aCMDResult['threads'] ? $aCMDResult['threads']
: (min(16, getProcessorCount()) - 1); : (min(16, getProcessorCount()) - 1);
if ($this->iInstances < 1) { if ($this->iInstances < 1) {
$this->iInstances = 1; $this->iInstances = 1;
warn('resetting threads to '.$this->iInstances); warn('resetting threads to ' . $this->iInstances);
} }
// Assume we can steal all the cache memory in the box (unless told otherwise) // Assume we can steal all the cache memory in the box (unless told otherwise)
@@ -37,8 +39,10 @@ class SetupFunctions
info('module path: ' . $this->sModulePath); info('module path: ' . $this->sModulePath);
// prepares DB for import or update, sets the Data Source Name // prepares DB for import or update, sets the Data Source Name
$this->aDSNInfo = DB::parseDSN(CONST_Database_DSN); $this->aDSNInfo = \DB::parseDSN(CONST_Database_DSN);
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) {
$this->aDSNInfo['port'] = 5432;
}
// setting member variables based on command line options stored in $aCMDResult // setting member variables based on command line options stored in $aCMDResult
$this->sVerbose = $aCMDResult['verbose']; $this->sVerbose = $aCMDResult['verbose'];
@@ -51,12 +55,12 @@ class SetupFunctions
public function createDB() public function createDB()
{ {
info('Create DB'); info('Create DB');
$sDB = DB::connect(CONST_Database_DSN, false); $sDB = \DB::connect(CONST_Database_DSN, false);
if (!PEAR::isError($sDB)) { if (!\PEAR::isError($sDB)) {
fail('database already exists ('.CONST_Database_DSN.')'); fail('database already exists (' . CONST_Database_DSN . ')');
} }
$sCreateDBCmd = 'createdb -E UTF-8 -p '.$this->aDSNInfo['port'].' '.$this->aDSNInfo['database']; $sCreateDBCmd = 'createdb -E UTF-8 -p ' . $this->aDSNInfo['port'] . ' ' . $this->aDSNInfo['database'];
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username']; $sCreateDBCmd .= ' -U ' . $this->aDSNInfo['username'];
} }
@@ -71,16 +75,19 @@ class SetupFunctions
} }
$result = runWithEnv($sCreateDBCmd, $aProcEnv); $result = runWithEnv($sCreateDBCmd, $aProcEnv);
if ($result != 0) fail('Error executing external command: '.$sCreateDBCmd); if ($result != 0) {
fail('Error executing external command: ' . $sCreateDBCmd);
}
} }
public function setupDB() public function setupDB()
{ {
info('Setup DB'); info('Setup DB');
$this->oDB =& getDB(); $this->oDB = &getDB();
$fPostgresVersion = getPostgresVersion($this->oDB); $fPostgresVersion = getPostgresVersion($this->oDB);
echo 'Postgres version found: '.$fPostgresVersion."\n"; echo 'Postgres version found: ' . $fPostgresVersion . "\n";
if ($fPostgresVersion < 9.1) { if ($fPostgresVersion < 9.1) {
fail('Minimum supported version of Postgresql is 9.1.'); fail('Minimum supported version of Postgresql is 9.1.');
@@ -99,9 +106,8 @@ class SetupFunctions
warn('Postgresql is too old. extratags and namedetails API not available.'); warn('Postgresql is too old. extratags and namedetails API not available.');
} }
$fPostgisVersion = getPostgisVersion($this->oDB); $fPostgisVersion = getPostgisVersion($this->oDB);
echo 'Postgis version found: '.$fPostgisVersion."\n"; echo 'Postgis version found: ' . $fPostgisVersion . "\n";
if ($fPostgisVersion < 2.1) { if ($fPostgisVersion < 2.1) {
// Functions were renamed in 2.1 and throw an annoying deprecation warning // Functions were renamed in 2.1 and throw an annoying deprecation warning
@@ -112,32 +118,31 @@ class SetupFunctions
$this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid'); $this->pgsqlRunScript('ALTER FUNCTION ST_Distance_Spheroid(geometry, geometry, spheroid) RENAME TO ST_DistanceSpheroid');
} }
$i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '".CONST_Database_Web_User."'")); $i = chksql($this->oDB->getOne("select count(*) from pg_user where usename = '" . CONST_Database_Web_User . "'"));
if ($i == 0) { if ($i == 0) {
echo "\nERROR: Web user '".CONST_Database_Web_User."' does not exist. Create it with:\n"; echo "\nERROR: Web user '" . CONST_Database_Web_User . "' does not exist. Create it with:\n";
echo "\n createuser ".CONST_Database_Web_User."\n\n"; echo "\n createuser " . CONST_Database_Web_User . "\n\n";
exit(1); exit(1);
} }
if (!file_exists(CONST_ExtraDataPath.'/country_osm_grid.sql.gz')) { if (!file_exists(CONST_ExtraDataPath . '/country_osm_grid.sql.gz')) {
echo 'Error: you need to download the country_osm_grid first:'; echo 'Error: you need to download the country_osm_grid first:';
echo "\n wget -O ".CONST_ExtraDataPath."/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n"; echo "\n wget -O " . CONST_ExtraDataPath . "/country_osm_grid.sql.gz https://www.nominatim.org/data/country_grid.sql.gz\n";
exit(1); exit(1);
} }
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_name.sql'); $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_name.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_naturalearthdata.sql'); $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_naturalearthdata.sql');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/country_osm_grid.sql.gz'); $this->pgsqlRunScriptFile(CONST_BasePath . '/data/country_osm_grid.sql.gz');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_table.sql'); $this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_table.sql');
if (file_exists(CONST_BasePath . '/data/gb_postcode_data.sql.gz')) {
if (file_exists(CONST_BasePath.'/data/gb_postcode_data.sql.gz')) { $this->pgsqlRunScriptFile(CONST_BasePath . '/data/gb_postcode_data.sql.gz');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/gb_postcode_data.sql.gz');
} else { } else {
warn('external UK postcode table not found.'); warn('external UK postcode table not found.');
} }
if (CONST_Use_Extra_US_Postcodes) { if (CONST_Use_Extra_US_Postcodes) {
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/us_postcode.sql'); $this->pgsqlRunScriptFile(CONST_BasePath . '/data/us_postcode.sql');
} }
if ($this->bNoPartitions) { if ($this->bNoPartitions) {
@@ -163,23 +168,29 @@ class SetupFunctions
fail("osm2pgsql not found in '$osm2pgsql'"); fail("osm2pgsql not found in '$osm2pgsql'");
} }
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$osm2pgsql .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; $osm2pgsql .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File;
}
if (CONST_Tablespace_Osm2pgsql_Data) {
$osm2pgsql .= ' --tablespace-slim-data ' . CONST_Tablespace_Osm2pgsql_Data;
}
if (CONST_Tablespace_Osm2pgsql_Index) {
$osm2pgsql .= ' --tablespace-slim-index ' . CONST_Tablespace_Osm2pgsql_Index;
}
if (CONST_Tablespace_Place_Data) {
$osm2pgsql .= ' --tablespace-main-data ' . CONST_Tablespace_Place_Data;
}
if (CONST_Tablespace_Place_Index) {
$osm2pgsql .= ' --tablespace-main-index ' . CONST_Tablespace_Place_Index;
} }
if (CONST_Tablespace_Osm2pgsql_Data)
$osm2pgsql .= ' --tablespace-slim-data '.CONST_Tablespace_Osm2pgsql_Data;
if (CONST_Tablespace_Osm2pgsql_Index)
$osm2pgsql .= ' --tablespace-slim-index '.CONST_Tablespace_Osm2pgsql_Index;
if (CONST_Tablespace_Place_Data)
$osm2pgsql .= ' --tablespace-main-data '.CONST_Tablespace_Place_Data;
if (CONST_Tablespace_Place_Index)
$osm2pgsql .= ' --tablespace-main-index '.CONST_Tablespace_Place_Index;
$osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1'; $osm2pgsql .= ' -lsc -O gazetteer --hstore --number-processes 1';
$osm2pgsql .= ' -C '.$this->iCacheMemory; $osm2pgsql .= ' -C ' . $this->iCacheMemory;
$osm2pgsql .= ' -P '.$this->aDSNInfo['port']; $osm2pgsql .= ' -P ' . $this->aDSNInfo['port'];
if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) { if (isset($this->aDSNInfo['username']) && $this->aDSNInfo['username']) {
$osm2pgsql .= ' -U ' . $this->aDSNInfo['username']; $osm2pgsql .= ' -U ' . $this->aDSNInfo['username'];
} }
@@ -190,9 +201,12 @@ class SetupFunctions
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
} }
$osm2pgsql .= ' -d '.$this->aDSNInfo['database'].' '.$sOSMFile; $osm2pgsql .= ' -d ' . $this->aDSNInfo['database'] . ' ' . $sOSMFile;
runWithEnv($osm2pgsql, $aProcEnv); runWithEnv($osm2pgsql, $aProcEnv);
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) {
$this->oDB = &getDB();
}
if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) { if (!$this->sIgnoreErrors && !chksql($this->oDB->getRow('select * from place limit 1'))) {
fail('No Data'); fail('No Data');
} }
@@ -209,7 +223,7 @@ class SetupFunctions
{ {
info('Create Tables'); info('Create Tables');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tables.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/tables.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:address-data}', '{ts:address-data}',
@@ -245,18 +259,11 @@ class SetupFunctions
$this->pgsqlRunScript($sTemplate, false); $this->pgsqlRunScript($sTemplate, false);
} }
public function recreateFunction()
{
// re-run the functions
info('Recreate Functions');
$this->createSqlFunctions();
}
public function createPartitionTables() public function createPartitionTables()
{ {
info('Create Partition Tables'); info('Create Partition Tables');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-tables.src.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-tables.src.sql');
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:address-data}', '{ts:address-data}',
CONST_Tablespace_Address_Data, CONST_Tablespace_Address_Data,
@@ -300,14 +307,14 @@ class SetupFunctions
{ {
info('Create Partition Functions'); info('Create Partition Functions');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/partition-functions.src.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/partition-functions.src.sql');
$this->pgsqlRunPartitionScript($sTemplate); $this->pgsqlRunPartitionScript($sTemplate);
} }
public function importWikipediaArticles() public function importWikipediaArticles()
{ {
$sWikiArticlesFile = CONST_Wikipedia_Data_Path.'/wikipedia_article.sql.bin'; $sWikiArticlesFile = CONST_Wikipedia_Data_Path . '/wikipedia_article.sql.bin';
$sWikiRedirectsFile = CONST_Wikipedia_Data_Path.'/wikipedia_redirect.sql.bin'; $sWikiRedirectsFile = CONST_Wikipedia_Data_Path . '/wikipedia_redirect.sql.bin';
if (file_exists($sWikiArticlesFile)) { if (file_exists($sWikiArticlesFile)) {
info('Importing wikipedia articles'); info('Importing wikipedia articles');
$this->pgsqlRunDropAndRestore($sWikiArticlesFile); $this->pgsqlRunDropAndRestore($sWikiArticlesFile);
@@ -327,40 +334,78 @@ class SetupFunctions
{ {
info('Drop old Data'); info('Drop old Data');
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) {
$this->oDB = &getDB();
}
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) {
fail(pg_last_error($this->oDB->connection));
}
if (!pg_query($this->oDB->connection, 'TRUNCATE word')) fail(pg_last_error($this->oDB->connection));
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'TRUNCATE placex')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'TRUNCATE location_property_osmline')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'TRUNCATE place_addressline')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'TRUNCATE place_boundingbox')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'TRUNCATE location_area')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'TRUNCATE search_name')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'TRUNCATE search_name_blank')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'DROP SEQUENCE seq_place')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'CREATE SEQUENCE seq_place start 100000')) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
$sSQL = 'select distinct partition from country_name'; $sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL)); $aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) $aPartitions[] = 0; if (!$this->bNoPartitions) {
$aPartitions[] = 0;
}
foreach ($aPartitions as $sPartition) { foreach ($aPartitions as $sPartition) {
if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_'.$sPartition)) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, 'TRUNCATE location_road_' . $sPartition)) {
fail(pg_last_error($this->oDB->connection));
}
echo '.'; echo '.';
} }
// used by getorcreate_word_id to ignore frequent partial words // used by getorcreate_word_id to ignore frequent partial words
$sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS '; $sSQL = 'CREATE OR REPLACE FUNCTION get_maxwordfreq() RETURNS integer AS ';
$sSQL .= '$$ SELECT '.CONST_Max_Word_Frequency.' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE'; $sSQL .= '$$ SELECT ' . CONST_Max_Word_Frequency . ' as maxwordfreq; $$ LANGUAGE SQL IMMUTABLE';
if (!pg_query($this->oDB->connection, $sSQL)) { if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection)); fail(pg_last_error($this->oDB->connection));
} }
@@ -369,7 +414,7 @@ class SetupFunctions
// pre-create the word list // pre-create the word list
if (!$bDisableTokenPrecalc) { if (!$bDisableTokenPrecalc) {
info('Loading word list'); info('Loading word list');
$this->pgsqlRunScriptFile(CONST_BasePath.'/data/words.sql'); $this->pgsqlRunScriptFile(CONST_BasePath . '/data/words.sql');
} }
info('Load Data'); info('Load Data');
@@ -377,24 +422,30 @@ class SetupFunctions
$aDBInstances = array(); $aDBInstances = array();
$iLoadThreads = max(1, $this->iInstances - 1); $iLoadThreads = max(1, $this->iInstances - 1);
for ($i = 0; $i < $iLoadThreads; $i++) { for ($i = 0; $i < $iLoadThreads; $i++) {
$aDBInstances[$i] =& getDB(true); $aDBInstances[$i] = &getDB(true);
$sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i"; $sSQL = "INSERT INTO placex ($sColumns) SELECT $sColumns FROM place WHERE osm_id % $iLoadThreads = $i";
$sSQL .= " and not (class='place' and type='houses' and osm_type='W'"; $sSQL .= " and not (class='place' and type='houses' and osm_type='W'";
$sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')"; $sSQL .= " and ST_GeometryType(geometry) = 'ST_LineString')";
$sSQL .= ' and ST_IsValid(geometry)'; $sSQL .= ' and ST_IsValid(geometry)';
if ($this->sVerbose) echo "$sSQL\n"; if ($this->sVerbose) {
echo "$sSQL\n";
}
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) { if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$i]->connection)); fail(pg_last_error($aDBInstances[$i]->connection));
} }
} }
// last thread for interpolation lines // last thread for interpolation lines
$aDBInstances[$iLoadThreads] =& getDB(true); $aDBInstances[$iLoadThreads] = &getDB(true);
$sSQL = 'insert into location_property_osmline'; $sSQL = 'insert into location_property_osmline';
$sSQL .= ' (osm_id, address, linegeo)'; $sSQL .= ' (osm_id, address, linegeo)';
$sSQL .= ' SELECT osm_id, address, geometry from place where '; $sSQL .= ' SELECT osm_id, address, geometry from place where ';
$sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'"; $sSQL .= "class='place' and type='houses' and osm_type='W' and ST_GeometryType(geometry) = 'ST_LineString'";
if ($this->sVerbose) echo "$sSQL\n"; if ($this->sVerbose) {
echo "$sSQL\n";
}
if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) { if (!pg_send_query($aDBInstances[$iLoadThreads]->connection, $sSQL)) {
fail(pg_last_error($aDBInstances[$iLoadThreads]->connection)); fail(pg_last_error($aDBInstances[$iLoadThreads]->connection));
} }
@@ -426,7 +477,7 @@ class SetupFunctions
if ($sDatabaseDate === false) { if ($sDatabaseDate === false) {
warn('could not determine database date.'); warn('could not determine database date.');
} else { } else {
$sSQL = "INSERT INTO import_status (lastimportdate) VALUES('".$sDatabaseDate."')"; $sSQL = "INSERT INTO import_status (lastimportdate) VALUES('" . $sDatabaseDate . "')";
pg_query($this->oDB->connection, $sSQL); pg_query($this->oDB->connection, $sSQL);
echo "Latest data imported from $sDatabaseDate.\n"; echo "Latest data imported from $sDatabaseDate.\n";
} }
@@ -436,7 +487,7 @@ class SetupFunctions
{ {
info('Import Tiger data'); info('Import Tiger data');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_start.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_start.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:aux-data}', '{ts:aux-data}',
@@ -452,11 +503,11 @@ class SetupFunctions
$aDBInstances = array(); $aDBInstances = array();
for ($i = 0; $i < $this->iInstances; $i++) { for ($i = 0; $i < $this->iInstances; $i++) {
$aDBInstances[$i] =& getDB(true); $aDBInstances[$i] = &getDB(true);
} }
foreach (glob(CONST_Tiger_Data_Path.'/*.sql') as $sFile) { foreach (glob(CONST_Tiger_Data_Path . '/*.sql') as $sFile) {
echo $sFile.': '; echo $sFile . ': ';
$hFile = fopen($sFile, 'r'); $hFile = fopen($sFile, 'r');
$sSQL = fgets($hFile, 100000); $sSQL = fgets($hFile, 100000);
$iLines = 0; $iLines = 0;
@@ -465,8 +516,14 @@ class SetupFunctions
if (!pg_connection_busy($aDBInstances[$i]->connection)) { if (!pg_connection_busy($aDBInstances[$i]->connection)) {
while (pg_get_result($aDBInstances[$i]->connection)); while (pg_get_result($aDBInstances[$i]->connection));
$sSQL = fgets($hFile, 100000); $sSQL = fgets($hFile, 100000);
if (!$sSQL) break 2; if (!$sSQL) {
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); break 2;
}
if (!pg_send_query($aDBInstances[$i]->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
$iLines++; $iLines++;
if ($iLines == 1000) { if ($iLines == 1000) {
echo '.'; echo '.';
@@ -482,7 +539,10 @@ class SetupFunctions
while ($bAnyBusy) { while ($bAnyBusy) {
$bAnyBusy = false; $bAnyBusy = false;
for ($i = 0; $i < $this->iInstances; $i++) { for ($i = 0; $i < $this->iInstances; $i++) {
if (pg_connection_busy($aDBInstances[$i]->connection)) $bAnyBusy = true; if (pg_connection_busy($aDBInstances[$i]->connection)) {
$bAnyBusy = true;
}
} }
usleep(10); usleep(10);
} }
@@ -490,7 +550,7 @@ class SetupFunctions
} }
info('Creating indexes on Tiger data'); info('Creating indexes on Tiger data');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/tiger_import_finish.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/tiger_import_finish.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:aux-data}', '{ts:aux-data}',
@@ -508,13 +568,15 @@ class SetupFunctions
public function calculatePostcodes($bCMDResultAll) public function calculatePostcodes($bCMDResultAll)
{ {
info('Calculate Postcodes'); info('Calculate Postcodes');
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) {
$this->oDB = &getDB();
}
if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) { if (!pg_query($this->oDB->connection, 'TRUNCATE location_postcode')) {
fail(pg_last_error($this->oDB->connection)); fail(pg_last_error($this->oDB->connection));
} }
$sSQL = 'INSERT INTO location_postcode';
$sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, country_code,"; $sSQL .= "SELECT nextval('seq_place'), 1, country_code,";
$sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,"; $sSQL .= " upper(trim (both ' ' from address->'postcode')) as pc,";
@@ -530,24 +592,29 @@ class SetupFunctions
if (CONST_Use_Extra_US_Postcodes) { if (CONST_Use_Extra_US_Postcodes) {
// only add postcodes that are not yet available in OSM // only add postcodes that are not yet available in OSM
$sSQL = 'INSERT INTO location_postcode'; $sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,"; $sSQL .= "SELECT nextval('seq_place'), 1, 'us', postcode,";
$sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)'; $sSQL .= ' ST_SetSRID(ST_Point(x,y),4326)';
$sSQL .= ' FROM us_postcode WHERE postcode NOT IN'; $sSQL .= ' FROM us_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'us')"; $sSQL .= " WHERE country_code = 'us')";
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
} }
// add missing postcodes for GB (if available) // add missing postcodes for GB (if available)
$sSQL = 'INSERT INTO location_postcode'; $sSQL = 'INSERT INTO location_postcode';
$sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) '; $sSQL .= ' (place_id, indexed_status, country_code, postcode, geometry) ';
$sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry"; $sSQL .= "SELECT nextval('seq_place'), 1, 'gb', postcode, geometry";
$sSQL .= ' FROM gb_postcode WHERE postcode NOT IN'; $sSQL .= ' FROM gb_postcode WHERE postcode NOT IN';
$sSQL .= ' (SELECT postcode FROM location_postcode'; $sSQL .= ' (SELECT postcode FROM location_postcode';
$sSQL .= " WHERE country_code = 'gb')"; $sSQL .= " WHERE country_code = 'gb')";
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
if (!$bCMDResultAll) { if (!$bCMDResultAll) {
$sSQL = "DELETE FROM word WHERE class='place' and type='postcode'"; $sSQL = "DELETE FROM word WHERE class='place' and type='postcode'";
@@ -567,8 +634,8 @@ class SetupFunctions
public function index($bIndexNoanalyse) public function index($bIndexNoanalyse)
{ {
$sOutputFile = ''; $sOutputFile = '';
$sBaseCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$this->aDSNInfo['database'].' -P ' $sBaseCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $this->aDSNInfo['database'] . ' -P '
.$this->aDSNInfo['port'].' -t '.$this->iInstances.$sOutputFile; . $this->aDSNInfo['port'] . ' -t ' . $this->iInstances . $sOutputFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec']; $sBaseCmd .= ' -H ' . $this->aDSNInfo['hostspec'];
} }
@@ -581,33 +648,45 @@ class SetupFunctions
} }
info('Index ranks 0 - 4'); info('Index ranks 0 - 4');
$iStatus = runWithEnv($sBaseCmd.' -R 4', $aProcEnv); $iStatus = runWithEnv($sBaseCmd . ' -R 4', $aProcEnv);
if ($iStatus != 0) { if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!'); fail('error status ' . $iStatus . ' running nominatim!');
} }
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); if (!$bIndexNoanalyse) {
$this->pgsqlRunScript('ANALYSE');
}
info('Index ranks 5 - 25'); info('Index ranks 5 - 25');
$iStatus = runWithEnv($sBaseCmd.' -r 5 -R 25', $aProcEnv); $iStatus = runWithEnv($sBaseCmd . ' -r 5 -R 25', $aProcEnv);
if ($iStatus != 0) { if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!'); fail('error status ' . $iStatus . ' running nominatim!');
} }
if (!$bIndexNoanalyse) $this->pgsqlRunScript('ANALYSE'); if (!$bIndexNoanalyse) {
$this->pgsqlRunScript('ANALYSE');
}
info('Index ranks 26 - 30'); info('Index ranks 26 - 30');
$iStatus = runWithEnv($sBaseCmd.' -r 26', $aProcEnv); $iStatus = runWithEnv($sBaseCmd . ' -r 26', $aProcEnv);
if ($iStatus != 0) { if ($iStatus != 0) {
fail('error status ' . $iStatus . ' running nominatim!'); fail('error status ' . $iStatus . ' running nominatim!');
} }
info('Index postcodes'); info('Index postcodes');
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) {
$this->oDB = &getDB();
}
$sSQL = 'UPDATE location_postcode SET indexed_status = 0'; $sSQL = 'UPDATE location_postcode SET indexed_status = 0';
if (!pg_query($this->oDB->connection, $sSQL)) fail(pg_last_error($this->oDB->connection)); if (!pg_query($this->oDB->connection, $sSQL)) {
fail(pg_last_error($this->oDB->connection));
}
} }
public function createSearchIndices() public function createSearchIndices()
{ {
info('Create Search indices'); info('Create Search indices');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/indices.src.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/indices.src.sql');
$sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate); $sTemplate = str_replace('{www-user}', CONST_Database_Web_User, $sTemplate);
$sTemplate = $this->replaceTablespace( $sTemplate = $this->replaceTablespace(
'{ts:address-index}', '{ts:address-index}',
@@ -636,12 +715,12 @@ class SetupFunctions
$this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x'); $this->pgsqlRunScript('select count(*) from (select getorcreate_country(make_standard_name(country_code), country_code) from country_name where country_code is not null) as x');
$this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x"); $this->pgsqlRunScript("select count(*) from (select getorcreate_country(make_standard_name(name->'name'), country_code) from country_name where name ? 'name') as x");
$sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),' $sSQL = 'select count(*) from (select getorcreate_country(make_standard_name(v),'
.'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k '; . 'country_code) from (select country_code, skeys(name) as k, svals(name) as v from country_name) x where k ';
if (CONST_Languages) { if (CONST_Languages) {
$sSQL .= 'in '; $sSQL .= 'in ';
$sDelim = '('; $sDelim = '(';
foreach (explode(',', CONST_Languages) as $sLang) { foreach (explode(',', CONST_Languages) as $sLang) {
$sSQL .= $sDelim."'name:$sLang'"; $sSQL .= $sDelim . "'name:$sLang'";
$sDelim = ','; $sDelim = ',';
} }
$sSQL .= ')'; $sSQL .= ')';
@@ -663,24 +742,27 @@ class SetupFunctions
// created. USE AT YOUR OWN PERIL. // created. USE AT YOUR OWN PERIL.
// tables we want to keep. everything else goes. // tables we want to keep. everything else goes.
$aKeepTables = array( $aKeepTables = array(
'*columns', '*columns',
'import_polygon_*', 'import_polygon_*',
'import_status', 'import_status',
'place_addressline', 'place_addressline',
'location_postcode', 'location_postcode',
'location_property*', 'location_property*',
'placex', 'placex',
'search_name', 'search_name',
'seq_*', 'seq_*',
'word', 'word',
'query_log', 'query_log',
'new_query_log', 'new_query_log',
'spatial_ref_sys', 'spatial_ref_sys',
'country_name', 'country_name',
'place_classtype_*' 'place_classtype_*',
); );
if ($this->oDB = null) {
$this->oDB = &getDB();
}
if ($this->oDB = null) $this->oDB =& getDB();
$aDropTables = array(); $aDropTables = array();
$aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'")); $aHaveTables = chksql($this->oDB->getCol("SELECT tablename FROM pg_tables WHERE schemaname='public'"));
@@ -692,25 +774,37 @@ class SetupFunctions
break; break;
} }
} }
if (!$bFound) array_push($aDropTables, $sTable); if (!$bFound) {
array_push($aDropTables, $sTable);
}
} }
foreach ($aDropTables as $sDrop) { foreach ($aDropTables as $sDrop) {
if ($this->sVerbose) echo "dropping table $sDrop\n"; if ($this->sVerbose) {
echo "dropping table $sDrop\n";
}
@pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE"); @pg_query($this->oDB->connection, "DROP TABLE $sDrop CASCADE");
// ignore warnings/errors as they might be caused by a table having // ignore warnings/errors as they might be caused by a table having
// been deleted already by CASCADE // been deleted already by CASCADE
} }
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
if ($sVerbose) echo 'deleting '.CONST_Osm2pgsql_Flatnode_File."\n"; if ($sVerbose) {
echo 'deleting ' . CONST_Osm2pgsql_Flatnode_File . "\n";
}
unlink(CONST_Osm2pgsql_Flatnode_File); unlink(CONST_Osm2pgsql_Flatnode_File);
} }
} }
private function pgsqlRunDropAndRestore($sDumpFile) private function pgsqlRunDropAndRestore($sDumpFile)
{ {
if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) $this->aDSNInfo['port'] = 5432; if (!isset($this->aDSNInfo['port']) || !$this->aDSNInfo['port']) {
$sCMD = 'pg_restore -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database'].' -Fc --clean '.$sDumpFile; $this->aDSNInfo['port'] = 5432;
}
$sCMD = 'pg_restore -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'] . ' -Fc --clean ' . $sDumpFile;
if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) { if (isset($this->aDSNInfo['hostspec']) && $this->aDSNInfo['hostspec']) {
$sCMD .= ' -h ' . $this->aDSNInfo['hostspec']; $sCMD .= ' -h ' . $this->aDSNInfo['hostspec'];
} }
@@ -721,7 +815,7 @@ class SetupFunctions
if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) { if (isset($this->aDSNInfo['password']) && $this->aDSNInfo['password']) {
$aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV); $aProcEnv = array_merge(array('PGPASSWORD' => $this->aDSNInfo['password']), $_ENV);
} }
$iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)" $iReturn = runWithEnv($sCMD, $aProcEnv); // /lib/cmd.php "function runWithEnv($sCmd, $aEnv)"
} }
private function pgsqlRunScript($sScript, $bfatal = true) private function pgsqlRunScript($sScript, $bfatal = true)
@@ -736,7 +830,7 @@ class SetupFunctions
private function createSqlFunctions() private function createSqlFunctions()
{ {
$sTemplate = file_get_contents(CONST_BasePath.'/sql/functions.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/functions.sql');
$sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate); $sTemplate = str_replace('{modulepath}', $this->sModulePath, $sTemplate);
if ($this->bEnableDiffUpdates) { if ($this->bEnableDiffUpdates) {
$sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate); $sTemplate = str_replace('RETURN NEW; -- %DIFFUPDATES%', '--', $sTemplate);
@@ -758,11 +852,15 @@ class SetupFunctions
private function pgsqlRunPartitionScript($sTemplate) private function pgsqlRunPartitionScript($sTemplate)
{ {
if ($this->oDB == null) $this->oDB =& getDB(); if ($this->oDB == null) {
$this->oDB = &getDB();
}
$sSQL = 'select distinct partition from country_name'; $sSQL = 'select distinct partition from country_name';
$aPartitions = chksql($this->oDB->getCol($sSQL)); $aPartitions = chksql($this->oDB->getCol($sSQL));
if (!$this->bNoPartitions) $aPartitions[] = 0; if (!$this->bNoPartitions) {
$aPartitions[] = 0;
}
preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER); preg_match_all('#^-- start(.*?)^-- end#ms', $sTemplate, $aMatches, PREG_SET_ORDER);
foreach ($aMatches as $aMatch) { foreach ($aMatches as $aMatch) {
@@ -778,9 +876,11 @@ class SetupFunctions
private function pgsqlRunScriptFile($sFilename) private function pgsqlRunScriptFile($sFilename)
{ {
if (!file_exists($sFilename)) fail('unable to find '.$sFilename); if (!file_exists($sFilename)) {
fail('unable to find ' . $sFilename);
}
$sCMD = 'psql -p '.$this->aDSNInfo['port'].' -d '.$this->aDSNInfo['database']; $sCMD = 'psql -p ' . $this->aDSNInfo['port'] . ' -d ' . $this->aDSNInfo['database'];
if (!$this->sVerbose) { if (!$this->sVerbose) {
$sCMD .= ' -q'; $sCMD .= ' -q';
} }
@@ -797,26 +897,32 @@ class SetupFunctions
$ahGzipPipes = null; $ahGzipPipes = null;
if (preg_match('/\\.gz$/', $sFilename)) { if (preg_match('/\\.gz$/', $sFilename)) {
$aDescriptors = array( $aDescriptors = array(
0 => array('pipe', 'r'), 0 => array('pipe', 'r'),
1 => array('pipe', 'w'), 1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a') 2 => array('file', '/dev/null', 'a'),
); );
$hGzipProcess = proc_open('zcat '.$sFilename, $aDescriptors, $ahGzipPipes); $hGzipProcess = proc_open('zcat ' . $sFilename, $aDescriptors, $ahGzipPipes);
if (!is_resource($hGzipProcess)) fail('unable to start zcat'); if (!is_resource($hGzipProcess)) {
fail('unable to start zcat');
}
$aReadPipe = $ahGzipPipes[1]; $aReadPipe = $ahGzipPipes[1];
fclose($ahGzipPipes[0]); fclose($ahGzipPipes[0]);
} else { } else {
$sCMD .= ' -f '.$sFilename; $sCMD .= ' -f ' . $sFilename;
$aReadPipe = array('pipe', 'r'); $aReadPipe = array('pipe', 'r');
} }
$aDescriptors = array( $aDescriptors = array(
0 => $aReadPipe, 0 => $aReadPipe,
1 => array('pipe', 'w'), 1 => array('pipe', 'w'),
2 => array('file', '/dev/null', 'a') 2 => array('file', '/dev/null', 'a'),
); );
$ahPipes = null; $ahPipes = null;
$hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv); $hProcess = proc_open($sCMD, $aDescriptors, $ahPipes, null, $aProcEnv);
if (!is_resource($hProcess)) fail('unable to start pgsql'); if (!is_resource($hProcess)) {
fail('unable to start pgsql');
}
// TODO: error checking // TODO: error checking
while (!feof($ahPipes[1])) { while (!feof($ahPipes[1])) {
echo fread($ahPipes[1], 4096); echo fread($ahPipes[1], 4096);
@@ -835,7 +941,7 @@ class SetupFunctions
private function replaceTablespace($sTemplate, $sTablespace, $sSql) private function replaceTablespace($sTemplate, $sTablespace, $sSql)
{ {
if ($sTablespace) { if ($sTablespace) {
$sSql = str_replace($sTemplate, 'TABLESPACE "'.$sTablespace.'"', $sSql); $sSql = str_replace($sTemplate, 'TABLESPACE "' . $sTablespace . '"', $sSql);
} else { } else {
$sSql = str_replace($sTemplate, '', $sSql); $sSql = str_replace($sTemplate, '', $sSql);
} }

114
lib/setup_functions.php
View File

@@ -11,7 +11,7 @@ function checkInFile($sOSMFile)
} }
if (!is_readable($sOSMFile)) { if (!is_readable($sOSMFile)) {
fail('osm-file "'.$aCMDResult['osm-file'].'" not readable'); fail('osm-file "' . $aCMDResult['osm-file'] . '" not readable');
} }
} }
@@ -21,60 +21,106 @@ function checkModulePresence()
// and can simply error out. // and can simply error out.
$sModulePath = CONST_Database_Module_Path; $sModulePath = CONST_Database_Module_Path;
$sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '"; $sSQL = "CREATE FUNCTION nominatim_test_import_func(text) RETURNS text AS '";
$sSQL .= $sModulePath."/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT"; $sSQL .= $sModulePath . "/nominatim.so', 'transliteration' LANGUAGE c IMMUTABLE STRICT";
$sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);'; $sSQL .= ';DROP FUNCTION nominatim_test_import_func(text);';
$oDB =& getDB(); $oDB = &getDB();
$oResult = $oDB->query($sSQL); $oResult = $oDB->query($sSQL);
$bResult = true; $bResult = true;
if (PEAR::isError($oResult)) { if (PEAR::isError($oResult)) {
echo "\nERROR: Failed to load nominatim module. Reason:\n"; echo "\nERROR: Failed to load nominatim module. Reason:\n";
echo $oResult->userinfo."\n\n"; echo $oResult->userinfo . "\n\n";
$bResult = false; $bResult = false;
} }
return $bResult; return $bResult;
} }
// (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help)
// create and array
function createSetupArgvArray() function createSetupArgvArray()
{ {
$aCMDOptions $aCMDOptions
= array( = array(
'Create and setup nominatim search system', 'Create and setup nominatim search system',
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'), array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'), array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'), array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'), array('osm-file', '', 0, 1, 1, 1, 'realpath', 'File to import'),
array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'), array('threads', '', 0, 1, 1, 1, 'int', 'Number of threads (where possible)'),
array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'), array('all', '', 0, 1, 0, 0, 'bool', 'Do the complete process'),
array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'), array('create-db', '', 0, 1, 0, 0, 'bool', 'Create nominatim db'),
array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'), array('setup-db', '', 0, 1, 0, 0, 'bool', 'Build a blank nominatim db'),
array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'), array('import-data', '', 0, 1, 0, 0, 'bool', 'Import a osm file'),
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'), array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'), array('create-functions', '', 0, 1, 0, 0, 'bool', 'Create functions'),
array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'), array('enable-diff-updates', '', 0, 1, 0, 0, 'bool', 'Turn on the code required to make diff updates work'),
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'), array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'), array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'), array('create-tables', '', 0, 1, 0, 0, 'bool', 'Create main tables'),
array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'), array('create-partition-tables', '', 0, 1, 0, 0, 'bool', 'Create required partition tables'),
array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'), array('create-partition-functions', '', 0, 1, 0, 0, 'bool', 'Create required partition triggers'),
array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'), array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'), array('import-wikipedia-articles', '', 0, 1, 0, 0, 'bool', 'Import wikipedia article dump'),
array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'), array('load-data', '', 0, 1, 0, 0, 'bool', 'Copy data to live tables from import table'),
array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'), array('disable-token-precalc', '', 0, 1, 0, 0, 'bool', 'Disable name precalculation (EXPERT)'),
array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'), array('import-tiger-data', '', 0, 1, 0, 0, 'bool', 'Import tiger data (not included in \'all\')'),
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'), array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Calculate postcode centroids'),
array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'), array('osmosis-init', '', 0, 1, 0, 0, 'bool', 'Generate default osmosis configuration'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'), array('index', '', 0, 1, 0, 0, 'bool', 'Index the data'),
array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'), array('index-noanalyse', '', 0, 1, 0, 0, 'bool', 'Do not perform analyse operations during index (EXPERT)'),
array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'), array('create-search-indices', '', 0, 1, 0, 0, 'bool', 'Create additional indices required for search and update'),
array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'), array('create-country-names', '', 0, 1, 0, 0, 'bool', 'Create default list of searchable country names'),
array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'), array('drop', '', 0, 1, 0, 0, 'bool', 'Drop tables needed for updates, making the database readonly (EXPERIMENTAL)'),
);
return $aCMDOptions;
}
function createUpdateArgvArray()
{
$aCMDOptions
= array(
'Import / update / index osm data',
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
array('create-functions', '', 0, 1, 1, 1, 'bool', 'Create functions'),
array('enable-diff-updates', '', 0, 1, 1, 1, 'bool', 'Turn on the code required to make diff updates work'),
array('ignore-errors', '', 0, 1, 0, 0, 'bool', 'Continue import even when errors in SQL are present (EXPERT)'),
array('enable-debug-statements', '', 0, 1, 0, 0, 'bool', 'Include debug warning statements in pgsql commands'),
array('no-partitions', '', 0, 1, 0, 0, 'bool', 'Do not partition search indices (speeds up import of single country extracts)'),
); );
return $aCMDOptions; return $aCMDOptions;
} }

54
utils/setup.php
View File

@@ -1,23 +1,22 @@
#!@PHP_BIN@ -Cq #!@PHP_BIN@ -Cq
<?php <?php
require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
require_once(CONST_BasePath.'/lib/init-cmd.php'); require_once CONST_BasePath . '/lib/init-cmd.php';
include_once(CONST_InstallPath.'/utils/setupClass.php'); require_once CONST_BasePath . '/lib/SetupClass.php';
// ->indirect via init-cmd.php -> /lib/cmd.php for runWithEnv, getCmdOpt // ->indirect via init-cmd.php->/lib/cmd.php for runWithEnv, getCmdOpt
// ->indirect via init-cmd.php -> /lib/init.php -> db.php for &getDB() // ->indirect via init-cmd.php->/lib/init.php->db.php for &getDB()
include_once(CONST_BasePath.'/lib/setup_functions.php'); require_once CONST_BasePath . '/lib/setup_functions.php';
include_once(CONST_BasePath.'/lib/setup_functions.php');
ini_set('memory_limit', '800M'); ini_set('memory_limit', '800M');
use Nominatim\Setup\SetupFunctions as SetupFunctions;
$aCMDOptions = createSetupArgvArray(); $aCMDOptions = createSetupArgvArray();
// $aCMDOptions passed to getCmdOpt by reference // $aCMDOptions passed to getCmdOpt by reference
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aCMDResult, true, true);
$bDidSomething = false; $bDidSomething = false;
//******************************************************* //*******************************************************
@@ -38,16 +37,17 @@ if ($aCMDResult['osmosis-init']) {
// ****************************************************** // ******************************************************
// instantiate Setup class // instantiate Setup class
$cSetup = new SetupFunctions($aCMDResult); $cSetup = new SetupFunctions($aCMDResult);
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup -> createDB();
}
// ******************************************************* // *******************************************************
// go through complete process if 'all' is selected or start selected functions // go through complete process if 'all' is selected or start selected functions
if ($aCMDResult['create-db'] || $aCMDResult['all']) {
$bDidSomething = true;
$cSetup->createDB();
}
if ($aCMDResult['setup-db'] || $aCMDResult['all']) { if ($aCMDResult['setup-db'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> setupDB(); $cSetup->setupDB();
} }
// Try accessing the C module, so we know early if something is wrong // Try accessing the C module, so we know early if something is wrong
@@ -57,68 +57,68 @@ if (!checkModulePresence()) {
if ($aCMDResult['import-data'] || $aCMDResult['all']) { if ($aCMDResult['import-data'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> importData($aCMDResult['osm-file']); $cSetup->importData($aCMDResult['osm-file']);
} }
if ($aCMDResult['create-functions'] || $aCMDResult['all']) { if ($aCMDResult['create-functions'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createFunctions(); $cSetup->createFunctions();
} }
if ($aCMDResult['create-tables'] || $aCMDResult['all']) { if ($aCMDResult['create-tables'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createTables(); $cSetup->createTables();
$cSetup -> recreateFunction(); $cSetup->createFunctions();
} }
if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) { if ($aCMDResult['create-partition-tables'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createPartitionTables(); $cSetup->createPartitionTables();
} }
if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) { if ($aCMDResult['create-partition-functions'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createPartitionFunctions(); $cSetup->createPartitionFunctions();
} }
if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) { if ($aCMDResult['import-wikipedia-articles'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> importWikipediaArticles(); $cSetup->importWikipediaArticles();
} }
if ($aCMDResult['load-data'] || $aCMDResult['all']) { if ($aCMDResult['load-data'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> loadData($aCMDResult['disable-token-precalc']); $cSetup->loadData($aCMDResult['disable-token-precalc']);
} }
if ($aCMDResult['import-tiger-data']) { if ($aCMDResult['import-tiger-data']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> importTigerData(); $cSetup->importTigerData();
} }
if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) { if ($aCMDResult['calculate-postcodes'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> calculatePostcodes($aCMDResult['all']); $cSetup->calculatePostcodes($aCMDResult['all']);
} }
if ($aCMDResult['index'] || $aCMDResult['all']) { if ($aCMDResult['index'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> index($aCMDResult['index-noanalyse']); $cSetup->index($aCMDResult['index-noanalyse']);
} }
if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) { if ($aCMDResult['create-search-indices'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createSearchIndices(); $cSetup->createSearchIndices();
} }
if ($aCMDResult['create-country-names'] || $aCMDResult['all']) { if ($aCMDResult['create-country-names'] || $aCMDResult['all']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> createCountryNames($aCMDResult); $cSetup->createCountryNames($aCMDResult);
} }
if ($aCMDResult['drop']) { if ($aCMDResult['drop']) {
$bDidSomething = true; $bDidSomething = true;
$cSetup -> drop($aCMDResult); $cSetup->drop($aCMDResult);
} }
// ****************************************************** // ******************************************************

190
utils/update.php
View File

@@ -1,66 +1,42 @@
#!@PHP_BIN@ -Cq #!@PHP_BIN@ -Cq
<?php <?php
require_once(dirname(dirname(__FILE__)).'/settings/settings.php'); require_once dirname(dirname(__FILE__)) . '/settings/settings.php';
require_once(CONST_BasePath.'/lib/init-cmd.php'); require_once CONST_BasePath . '/lib/init-cmd.php';
include_once(CONST_BasePath.'/lib/setup_functions.php'); require_once CONST_BasePath . '/lib/setup_functions.php';
require_once CONST_BasePath . '/lib/SetupClass.php';
ini_set('memory_limit', '800M'); ini_set('memory_limit', '800M');
use Nominatim\Setup\SetupFunctions as SetupFunctions;
# (long-opt, short-opt, min-occurs, max-occurs, num-arguments, num-arguments, type, help) $aCMDOptions = createUpdateArgvArray();
$aCMDOptions
= array(
'Import / update / index osm data',
array('help', 'h', 0, 1, 0, 0, false, 'Show Help'),
array('quiet', 'q', 0, 1, 0, 0, 'bool', 'Quiet output'),
array('verbose', 'v', 0, 1, 0, 0, 'bool', 'Verbose output'),
array('init-updates', '', 0, 1, 0, 0, 'bool', 'Set up database for updating'),
array('check-for-updates', '', 0, 1, 0, 0, 'bool', 'Check if new updates are available'),
array('no-update-functions', '', 0, 1, 0, 0, 'bool', 'Do not update trigger functions to support differential updates (assuming the diff update logic is already present)'),
array('import-osmosis', '', 0, 1, 0, 0, 'bool', 'Import updates once'),
array('import-osmosis-all', '', 0, 1, 0, 0, 'bool', 'Import updates forever'),
array('no-index', '', 0, 1, 0, 0, 'bool', 'Do not index the new data'),
array('calculate-postcodes', '', 0, 1, 0, 0, 'bool', 'Update postcode centroid table'),
array('import-file', '', 0, 1, 1, 1, 'realpath', 'Re-import data from an OSM file'),
array('import-diff', '', 0, 1, 1, 1, 'realpath', 'Import a diff (osc) file from local file system'),
array('osm2pgsql-cache', '', 0, 1, 1, 1, 'int', 'Cache size used by osm2pgsql'),
array('import-node', '', 0, 1, 1, 1, 'int', 'Re-import node'),
array('import-way', '', 0, 1, 1, 1, 'int', 'Re-import way'),
array('import-relation', '', 0, 1, 1, 1, 'int', 'Re-import relation'),
array('import-from-main-api', '', 0, 1, 0, 0, 'bool', 'Use OSM API instead of Overpass to download objects'),
array('index', '', 0, 1, 0, 0, 'bool', 'Index'),
array('index-rank', '', 0, 1, 1, 1, 'int', 'Rank to start indexing from'),
array('index-instances', '', 0, 1, 1, 1, 'int', 'Number of indexing instances (threads)'),
array('deduplicate', '', 0, 1, 0, 0, 'bool', 'Deduplicate tokens'),
array('recompute-word-counts', '', 0, 1, 0, 0, 'bool', 'Compute frequency of full-word search terms'),
array('no-npi', '', 0, 1, 0, 0, 'bool', '(obsolete)'),
);
getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true); getCmdOpt($_SERVER['argv'], $aCMDOptions, $aResult, true, true);
if (!isset($aResult['index-instances'])) $aResult['index-instances'] = 1; if (!isset($aResult['index-instances'])) {
if (!isset($aResult['index-rank'])) $aResult['index-rank'] = 0; $aResult['index-instances'] = 1;
}
if (!isset($aResult['index-rank'])) {
$aResult['index-rank'] = 0;
}
date_default_timezone_set('Etc/UTC'); date_default_timezone_set('Etc/UTC');
$oDB =& getDB(); $oDB = &getDB();
$aDSNInfo = DB::parseDSN(CONST_Database_DSN); $aDSNInfo = DB::parseDSN(CONST_Database_DSN);
if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) $aDSNInfo['port'] = 5432; if (!isset($aDSNInfo['port']) || !$aDSNInfo['port']) {
$aDSNInfo['port'] = 5432;
}
// cache memory to be used by osm2pgsql, should not be more than the available memory // cache memory to be used by osm2pgsql, should not be more than the available memory
$iCacheMemory = (isset($aResult['osm2pgsql-cache'])?$aResult['osm2pgsql-cache']:2000); $iCacheMemory = (isset($aResult['osm2pgsql-cache']) ? $aResult['osm2pgsql-cache'] : 2000);
if ($iCacheMemory + 500 > getTotalMemoryMB()) { if ($iCacheMemory + 500 > getTotalMemoryMB()) {
$iCacheMemory = getCacheMemoryMB(); $iCacheMemory = getCacheMemoryMB();
echo "WARNING: resetting cache memory to $iCacheMemory\n"; echo "WARNING: resetting cache memory to $iCacheMemory\n";
} }
$sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary.' -klas --number-processes 1 -C '.$iCacheMemory.' -O gazetteer -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port']; $sOsm2pgsqlCmd = CONST_Osm2pgsql_Binary . ' -klas --number-processes 1 -C ' . $iCacheMemory . ' -O gazetteer -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'];
if (isset($aDSNInfo['username']) && $aDSNInfo['username']) { if (isset($aDSNInfo['username']) && $aDSNInfo['username']) {
$sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username']; $sOsm2pgsqlCmd .= ' -U ' . $aDSNInfo['username'];
} }
@@ -73,12 +49,12 @@ if (isset($aDSNInfo['password']) && $aDSNInfo['password']) {
} }
if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) { if (!is_null(CONST_Osm2pgsql_Flatnode_File) && CONST_Osm2pgsql_Flatnode_File) {
$sOsm2pgsqlCmd .= ' --flat-nodes '.CONST_Osm2pgsql_Flatnode_File; $sOsm2pgsqlCmd .= ' --flat-nodes ' . CONST_Osm2pgsql_Flatnode_File;
} }
if ($aResult['init-updates']) { if ($aResult['init-updates']) {
// sanity check that the replication URL is correct // sanity check that the replication URL is correct
$sBaseState = file_get_contents(CONST_Replication_Url.'/state.txt'); $sBaseState = file_get_contents(CONST_Replication_Url . '/state.txt');
if ($sBaseState === false) { if ($sBaseState === false) {
echo "\nCannot find state.txt file at the configured replication URL.\n"; echo "\nCannot find state.txt file at the configured replication URL.\n";
echo "Does the URL point to a directory containing OSM update data?\n\n"; echo "Does the URL point to a directory containing OSM update data?\n\n";
@@ -92,7 +68,7 @@ if ($aResult['init-updates']) {
fail('CONST_Pyosmium_Binary not configured'); fail('CONST_Pyosmium_Binary not configured');
} }
$aOutput = 0; $aOutput = 0;
$sCmd = CONST_Pyosmium_Binary.' --help'; $sCmd = CONST_Pyosmium_Binary . ' --help';
exec($sCmd, $aOutput, $iRet); exec($sCmd, $aOutput, $iRet);
if ($iRet != 0) { if ($iRet != 0) {
echo "Cannot execute pyosmium-get-changes.\n"; echo "Cannot execute pyosmium-get-changes.\n";
@@ -100,19 +76,23 @@ if ($aResult['init-updates']) {
echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n"; echo "and have set up CONST_Pyosmium_Binary to point to pyosmium-get-changes.\n";
fail('pyosmium-get-changes not found or not usable'); fail('pyosmium-get-changes not found or not usable');
} }
if (!$aResult['no-update-functions']) { if (!$aResult['no-update-functions']) {
createFunctions($aCMDResult); // instatiate setupClass to use the function therein
// instantiate Setup class
$cSetup = new SetupFunctions($aResult);
$cSetup->createFunctions();
} }
$sDatabaseDate = getDatabaseDate($oDB); $sDatabaseDate = getDatabaseDate($oDB);
if ($sDatabaseDate === false) { if ($sDatabaseDate === false) {
fail('Cannot determine date of database.'); fail('Cannot determine date of database.');
} }
$sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3*60*60)); $sWindBack = strftime('%Y-%m-%dT%H:%M:%SZ', strtotime($sDatabaseDate) - (3 * 60 * 60));
// get the appropriate state id // get the appropriate state id
$aOutput = 0; $aOutput = 0;
$sCmd = CONST_Pyosmium_Binary.' -D '.$sWindBack.' --server '.CONST_Replication_Url; $sCmd = CONST_Pyosmium_Binary . ' -D ' . $sWindBack . ' --server ' . CONST_Replication_Url;
exec($sCmd, $aOutput, $iRet); exec($sCmd, $aOutput, $iRet);
if ($iRet != 0 || $aOutput[0] == 'None') { if ($iRet != 0 || $aOutput[0] == 'None') {
fail('Error running pyosmium tools'); fail('Error running pyosmium tools');
@@ -120,7 +100,7 @@ if ($aResult['init-updates']) {
pg_query($oDB->connection, 'TRUNCATE import_status'); pg_query($oDB->connection, 'TRUNCATE import_status');
$sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('"; $sSQL = "INSERT INTO import_status (lastimportdate, sequence_id, indexed) VALUES('";
$sSQL .= $sDatabaseDate."',".$aOutput[0].', true)'; $sSQL .= $sDatabaseDate . "'," . $aOutput[0] . ', true)';
if (!pg_query($oDB->connection, $sSQL)) { if (!pg_query($oDB->connection, $sSQL)) {
fail('Could not enter sequence into database.'); fail('Could not enter sequence into database.');
} }
@@ -135,7 +115,7 @@ if ($aResult['check-for-updates']) {
fail('Updates not set up. Please run ./utils/update.php --init-updates.'); fail('Updates not set up. Please run ./utils/update.php --init-updates.');
} }
system(CONST_BasePath.'/utils/check_server_for_updates.py '.CONST_Replication_Url.' '.$aLastState['sequence_id'], $iRet); system(CONST_BasePath . '/utils/check_server_for_updates.py ' . CONST_Replication_Url . ' ' . $aLastState['sequence_id'], $iRet);
exit($iRet); exit($iRet);
} }
@@ -148,8 +128,8 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
} }
// Import the file // Import the file
$sCMD = $sOsm2pgsqlCmd.' '.$sNextFile; $sCMD = $sOsm2pgsqlCmd . ' ' . $sNextFile;
echo $sCMD."\n"; echo $sCMD . "\n";
$iErrorLevel = runWithEnv($sCMD, $aProcEnv); $iErrorLevel = runWithEnv($sCMD, $aProcEnv);
if ($iErrorLevel) { if ($iErrorLevel) {
@@ -161,35 +141,35 @@ if (isset($aResult['import-diff']) || isset($aResult['import-file'])) {
if ($aResult['calculate-postcodes']) { if ($aResult['calculate-postcodes']) {
info('Update postcodes centroids'); info('Update postcodes centroids');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/update-postcodes.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/update-postcodes.sql');
runSQLScript($sTemplate, true, true); runSQLScript($sTemplate, true, true);
} }
$sTemporaryFile = CONST_BasePath.'/data/osmosischange.osc'; $sTemporaryFile = CONST_BasePath . '/data/osmosischange.osc';
$bHaveDiff = false; $bHaveDiff = false;
$bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api']; $bUseOSMApi = isset($aResult['import-from-main-api']) && $aResult['import-from-main-api'];
$sContentURL = ''; $sContentURL = '';
if (isset($aResult['import-node']) && $aResult['import-node']) { if (isset($aResult['import-node']) && $aResult['import-node']) {
if ($bUseOSMApi) { if ($bUseOSMApi) {
$sContentURL = 'https://www.openstreetmap.org/api/0.6/node/'.$aResult['import-node']; $sContentURL = 'https://www.openstreetmap.org/api/0.6/node/' . $aResult['import-node'];
} else { } else {
$sContentURL = 'https://overpass-api.de/api/interpreter?data=node('.$aResult['import-node'].');out%20meta;'; $sContentURL = 'https://overpass-api.de/api/interpreter?data=node(' . $aResult['import-node'] . ');out%20meta;';
} }
} }
if (isset($aResult['import-way']) && $aResult['import-way']) { if (isset($aResult['import-way']) && $aResult['import-way']) {
if ($bUseOSMApi) { if ($bUseOSMApi) {
$sContentURL = 'https://www.openstreetmap.org/api/0.6/way/'.$aResult['import-way'].'/full'; $sContentURL = 'https://www.openstreetmap.org/api/0.6/way/' . $aResult['import-way'] . '/full';
} else { } else {
$sContentURL = 'https://overpass-api.de/api/interpreter?data=(way('.$aResult['import-way'].');node(w););out%20meta;'; $sContentURL = 'https://overpass-api.de/api/interpreter?data=(way(' . $aResult['import-way'] . ');node(w););out%20meta;';
} }
} }
if (isset($aResult['import-relation']) && $aResult['import-relation']) { if (isset($aResult['import-relation']) && $aResult['import-relation']) {
if ($bUseOSMApi) { if ($bUseOSMApi) {
$sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/'.$aResult['import-relation'].'/full'; $sContentURLsModifyXMLstr = 'https://www.openstreetmap.org/api/0.6/relation/' . $aResult['import-relation'] . '/full';
} else { } else {
$sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel('.$aResult['import-relation'].');way(r);node(w));node(r));out%20meta;'; $sContentURL = 'https://overpass-api.de/api/interpreter?data=((rel(' . $aResult['import-relation'] . ');way(r);node(w));node(r));out%20meta;';
} }
} }
@@ -200,8 +180,8 @@ if ($sContentURL) {
if ($bHaveDiff) { if ($bHaveDiff) {
// import generated change file // import generated change file
$sCMD = $sOsm2pgsqlCmd.' '.$sTemporaryFile; $sCMD = $sOsm2pgsqlCmd . ' ' . $sTemporaryFile;
echo $sCMD."\n"; echo $sCMD . "\n";
$iErrorLevel = runWithEnv($sCMD, $aProcEnv); $iErrorLevel = runWithEnv($sCMD, $aProcEnv);
if ($iErrorLevel) { if ($iErrorLevel) {
fail("osm2pgsql exited with error level $iErrorLevel\n"); fail("osm2pgsql exited with error level $iErrorLevel\n");
@@ -209,7 +189,7 @@ if ($bHaveDiff) {
} }
if ($aResult['deduplicate']) { if ($aResult['deduplicate']) {
$oDB =& getDB(); $oDB = &getDB();
if (getPostgresVersion($oDB) < 9.3) { if (getPostgresVersion($oDB) < 9.3) {
fail('ERROR: deduplicate is only currently supported in postgresql 9.3'); fail('ERROR: deduplicate is only currently supported in postgresql 9.3');
@@ -221,7 +201,7 @@ if ($aResult['deduplicate']) {
// we don't care about empty search_name_* partitions, they can't contain mentions of duplicates // we don't care about empty search_name_* partitions, they can't contain mentions of duplicates
foreach ($aPartitions as $i => $sPartition) { foreach ($aPartitions as $i => $sPartition) {
$sSQL = 'select count(*) from search_name_'.$sPartition; $sSQL = 'select count(*) from search_name_' . $sPartition;
$nEntries = chksql($oDB->getOne($sSQL)); $nEntries = chksql($oDB->getOne($sSQL));
if ($nEntries == 0) { if ($nEntries == 0) {
unset($aPartitions[$i]); unset($aPartitions[$i]);
@@ -233,11 +213,14 @@ if ($aResult['deduplicate']) {
$sSQL .= ' group by word_token having count(*) > 1 order by word_token'; $sSQL .= ' group by word_token having count(*) > 1 order by word_token';
$aDuplicateTokens = chksql($oDB->getAll($sSQL)); $aDuplicateTokens = chksql($oDB->getAll($sSQL));
foreach ($aDuplicateTokens as $aToken) { foreach ($aDuplicateTokens as $aToken) {
if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') continue; if (trim($aToken['word_token']) == '' || trim($aToken['word_token']) == '-') {
echo 'Deduping '.$aToken['word_token']."\n"; continue;
}
echo 'Deduping ' . $aToken['word_token'] . "\n";
$sSQL = 'select word_id,'; $sSQL = 'select word_id,';
$sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num'; $sSQL .= ' (select count(*) from search_name where nameaddress_vector @> ARRAY[word_id]) as num';
$sSQL .= " from word where word_token = '".$aToken['word_token']; $sSQL .= " from word where word_token = '" . $aToken['word_token'];
$sSQL .= "' and class is null and type is null and country_code is null order by num desc"; $sSQL .= "' and class is null and type is null and country_code is null order by num desc";
$aTokenSet = chksql($oDB->getAll($sSQL)); $aTokenSet = chksql($oDB->getAll($sSQL));
@@ -246,34 +229,34 @@ if ($aResult['deduplicate']) {
foreach ($aTokenSet as $aRemove) { foreach ($aTokenSet as $aRemove) {
$sSQL = 'update search_name set'; $sSQL = 'update search_name set';
$sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.'),'; $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . '),';
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));
$sSQL = 'update search_name set'; $sSQL = 'update search_name set';
$sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,'.$aRemove['word_id'].','.$iKeepID.')'; $sSQL .= ' nameaddress_vector = array_replace(nameaddress_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where nameaddress_vector @> ARRAY['.$aRemove['word_id'].']'; $sSQL .= ' where nameaddress_vector @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));
$sSQL = 'update location_area_country set'; $sSQL = 'update location_area_country set';
$sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));
foreach ($aPartitions as $sPartition) { foreach ($aPartitions as $sPartition) {
$sSQL = 'update search_name_'.$sPartition.' set'; $sSQL = 'update search_name_' . $sPartition . ' set';
$sSQL .= ' name_vector = array_replace(name_vector,'.$aRemove['word_id'].','.$iKeepID.')'; $sSQL .= ' name_vector = array_replace(name_vector,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where name_vector @> ARRAY['.$aRemove['word_id'].']'; $sSQL .= ' where name_vector @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));
$sSQL = 'update location_area_country set'; $sSQL = 'update location_area_country set';
$sSQL .= ' keywords = array_replace(keywords,'.$aRemove['word_id'].','.$iKeepID.')'; $sSQL .= ' keywords = array_replace(keywords,' . $aRemove['word_id'] . ',' . $iKeepID . ')';
$sSQL .= ' where keywords @> ARRAY['.$aRemove['word_id'].']'; $sSQL .= ' where keywords @> ARRAY[' . $aRemove['word_id'] . ']';
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));
} }
$sSQL = 'delete from word where word_id = '.$aRemove['word_id']; $sSQL = 'delete from word where word_id = ' . $aRemove['word_id'];
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));
} }
} }
@@ -281,12 +264,12 @@ if ($aResult['deduplicate']) {
if ($aResult['recompute-word-counts']) { if ($aResult['recompute-word-counts']) {
info('Recompute frequency of full-word search terms'); info('Recompute frequency of full-word search terms');
$sTemplate = file_get_contents(CONST_BasePath.'/sql/words_from_search_name.sql'); $sTemplate = file_get_contents(CONST_BasePath . '/sql/words_from_search_name.sql');
runSQLScript($sTemplate, true, true); runSQLScript($sTemplate, true, true);
} }
if ($aResult['index']) { if ($aResult['index']) {
$sCmd = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances'].' -r '.$aResult['index-rank']; $sCmd = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'] . ' -r ' . $aResult['index-rank'];
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
$sCmd .= ' -H ' . $aDSNInfo['hostspec']; $sCmd .= ' -H ' . $aDSNInfo['hostspec'];
} }
@@ -301,13 +284,13 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
// //
if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) { if (strpos(CONST_Replication_Url, 'download.geofabrik.de') !== false && CONST_Replication_Update_Interval < 86400) {
fail('Error: Update interval too low for download.geofabrik.de. ' . fail('Error: Update interval too low for download.geofabrik.de. ' .
"Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n"); "Please check install documentation (http://nominatim.org/release-docs/latest/Import-and-Update#setting-up-the-update-process)\n");
} }
$sImportFile = CONST_InstallPath.'/osmosischange.osc'; $sImportFile = CONST_InstallPath . '/osmosischange.osc';
$sCMDDownload = CONST_Pyosmium_Binary.' --server '.CONST_Replication_Url.' -o '.$sImportFile.' -s '.CONST_Replication_Max_Diff_size; $sCMDDownload = CONST_Pyosmium_Binary . ' --server ' . CONST_Replication_Url . ' -o ' . $sImportFile . ' -s ' . CONST_Replication_Max_Diff_size;
$sCMDImport = $sOsm2pgsqlCmd.' '.$sImportFile; $sCMDImport = $sOsm2pgsqlCmd . ' ' . $sImportFile;
$sCMDIndex = CONST_InstallPath.'/nominatim/nominatim -i -d '.$aDSNInfo['database'].' -P '.$aDSNInfo['port'].' -t '.$aResult['index-instances']; $sCMDIndex = CONST_InstallPath . '/nominatim/nominatim -i -d ' . $aDSNInfo['database'] . ' -P ' . $aDSNInfo['port'] . ' -t ' . $aResult['index-instances'];
if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) { if (isset($aDSNInfo['hostspec']) && $aDSNInfo['hostspec']) {
$sCMDIndex .= ' -H ' . $aDSNInfo['hostspec']; $sCMDIndex .= ' -H ' . $aDSNInfo['hostspec'];
} }
@@ -324,7 +307,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
exit(1); exit(1);
} }
echo 'Currently at sequence '.$aLastState['sequence_id'].' ('.$aLastState['lastimportdate'].') - '.$aLastState['indexed']." indexed\n"; echo 'Currently at sequence ' . $aLastState['sequence_id'] . ' (' . $aLastState['lastimportdate'] . ') - ' . $aLastState['indexed'] . " indexed\n";
$sBatchEnd = $aLastState['lastimportdate']; $sBatchEnd = $aLastState['lastimportdate'];
$iEndSequence = $aLastState['sequence_id']; $iEndSequence = $aLastState['sequence_id'];
@@ -347,23 +330,23 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
if (file_exists($sImportFile)) { if (file_exists($sImportFile)) {
unlink($sImportFile); unlink($sImportFile);
} }
exec($sCMDDownload.' -I '.$iNextSeq, $aOutput, $iResult); exec($sCMDDownload . ' -I ' . $iNextSeq, $aOutput, $iResult);
if ($iResult == 3) { if ($iResult == 3) {
echo 'No new updates. Sleeping for '.CONST_Replication_Recheck_Interval." sec.\n"; echo 'No new updates. Sleeping for ' . CONST_Replication_Recheck_Interval . " sec.\n";
sleep(CONST_Replication_Recheck_Interval); sleep(CONST_Replication_Recheck_Interval);
} elseif ($iResult != 0) { } elseif ($iResult != 0) {
echo 'ERROR: updates failed.'; echo 'ERROR: updates failed.';
exit($iResult); exit($iResult);
} else { } else {
$iEndSequence = (int)$aOutput[0]; $iEndSequence = (int) $aOutput[0];
} }
} while ($iResult); } while ($iResult);
// get the newest object from the diff file // get the newest object from the diff file
$sBatchEnd = 0; $sBatchEnd = 0;
$iRet = 0; $iRet = 0;
exec(CONST_BasePath.'/utils/osm_file_date.py '.$sImportFile, $sBatchEnd, $iRet); exec(CONST_BasePath . '/utils/osm_file_date.py ' . $sImportFile, $sBatchEnd, $iRet);
if ($iRet == 5) { if ($iRet == 5) {
echo "Diff file is empty. skipping import.\n"; echo "Diff file is empty. skipping import.\n";
if (!$aResult['import-osmosis-all']) { if (!$aResult['import-osmosis-all']) {
@@ -379,7 +362,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
// Import the file // Import the file
$fCMDStartTime = time(); $fCMDStartTime = time();
echo $sCMDImport."\n"; echo $sCMDImport . "\n";
unset($sJunk); unset($sJunk);
$iErrorLevel = runWithEnv($sCMDImport, $aProcEnv); $iErrorLevel = runWithEnv($sCMDImport, $aProcEnv);
if ($iErrorLevel) { if ($iErrorLevel) {
@@ -392,8 +375,8 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
$sSQL = 'INSERT INTO import_osmosis_log'; $sSQL = 'INSERT INTO import_osmosis_log';
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
$sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','";
$sSQL .= date('Y-m-d H:i:s')."','import')"; $sSQL .= date('Y-m-d H:i:s') . "','import')";
var_Dump($sSQL); var_Dump($sSQL);
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));
@@ -401,7 +384,7 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
$sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence"; $sSQL = "UPDATE import_status SET lastimportdate = '$sBatchEnd', indexed=false, sequence_id = $iEndSequence";
var_Dump($sSQL); var_Dump($sSQL);
chksql($oDB->query($sSQL)); chksql($oDB->query($sSQL));
echo date('Y-m-d H:i:s')." Completed download step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; echo date('Y-m-d H:i:s') . " Completed download step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n";
} }
// Index file // Index file
@@ -419,18 +402,21 @@ if ($aResult['import-osmosis'] || $aResult['import-osmosis-all']) {
$sSQL = 'INSERT INTO import_osmosis_log'; $sSQL = 'INSERT INTO import_osmosis_log';
$sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)'; $sSQL .= '(batchend, batchseq, batchsize, starttime, endtime, event)';
$sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'"; $sSQL .= " values ('$sBatchEnd',$iEndSequence,$iFileSize,'";
$sSQL .= date('Y-m-d H:i:s', $fCMDStartTime)."','"; $sSQL .= date('Y-m-d H:i:s', $fCMDStartTime) . "','";
$sSQL .= date('Y-m-d H:i:s')."','index')"; $sSQL .= date('Y-m-d H:i:s') . "','index')";
var_Dump($sSQL); var_Dump($sSQL);
$oDB->query($sSQL); $oDB->query($sSQL);
echo date('Y-m-d H:i:s')." Completed index step for $sBatchEnd in ".round((time()-$fCMDStartTime)/60, 2)." minutes\n"; echo date('Y-m-d H:i:s') . " Completed index step for $sBatchEnd in " . round((time() - $fCMDStartTime) / 60, 2) . " minutes\n";
$sSQL = 'update import_status set indexed = true'; $sSQL = 'update import_status set indexed = true';
$oDB->query($sSQL); $oDB->query($sSQL);
} }
$fDuration = time() - $fStartTime; $fDuration = time() - $fStartTime;
echo date('Y-m-d H:i:s')." Completed all for $sBatchEnd in ".round($fDuration/60, 2)." minutes\n"; echo date('Y-m-d H:i:s') . " Completed all for $sBatchEnd in " . round($fDuration / 60, 2) . " minutes\n";
if (!$aResult['import-osmosis-all']) exit(0); if (!$aResult['import-osmosis-all']) {
exit(0);
}
} }
} }